/** * Property-based tests for the agent loop's inbox/turn scheduling (the * property-testing Agent Note). Deterministic by construction: schedules are driven * through the `agent/status` settle signal (no wall-clock sleeps), so a flake * is a finding, not timing noise. * * Invariants: every sent message appears exactly once in the log (none lost); * turn numbers strictly increase; status transitions follow * idle→running→idle, while teardown is a registry lifecycle. */ import { describe, expect, it } from 'vitest' import { Context } from 'cordis' import LlmService from '@deepseek-ai/dsh-llm' import { createUserMessage, LlmAdapter } from '@deepseek-ai/dsh-llm' import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm' import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' import SystemPrompt from '@deepseek-ai/dsh-system-prompt' import ToolRegistry from '@deepseek-ai/dsh-tools' import AgentRegistry, { type Agent } from '@deepseek-ai/dsh-agent' import AgentLoop from '@deepseek-ai/dsh-agent-loop' import fc from 'fast-check' /** A never-exhausting adapter: every model call returns the same short reply. */ class EchoAdapter extends LlmAdapter { async * stream(options: GenerateOptions): AsyncIterable { if (options.signal?.aborted) throw new Error('aborted') const text = 'ok' yield { type: 'block-start', index: 0, blockType: 'text' } yield { type: 'text-delta', index: 0, text } yield { type: 'block-end', index: 0, block: { type: 'text', text } } yield { type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } } yield { type: 'finish', reason: { kind: 'stop' } } } } async function harness() { const ctx = new Context() await ctx.plugin(LlmService) await ctx.plugin(SessionStore) await ctx.plugin(SystemPrompt) await ctx.plugin(ToolRegistry) await ctx.plugin(AgentRegistry) await ctx.plugin(AgentLoop, { agents: [] }) ctx.llm.registerAdapter(['mock'], new EchoAdapter()) return ctx } /** Resolve on the agent's next transition to idle (event-based, not polled). */ function nextIdle(ctx: Context, agent: Agent): Promise { return new Promise((resolve) => { const dispose = ctx.on('agent/status', (subject, status) => { if (subject === agent && status === 'idle') { dispose() resolve() } }) }) } /** Record every status transition for the legal-machine assertion. Returns * the seen list plus a disposer for the listener (per the registry convention). */ function recordStatus(ctx: Context, agent: Agent): { seen: string[]; dispose: () => void } { const seen: string[] = [] const dispose = ctx.on('agent/status', (subject, status) => { if (subject === agent) seen.push(status) }) return { seen, dispose } } function userMessageTexts(agent: Agent): string[] { return agent.session.events .filter(e => e.type === 'user/message') .map(e => (e.data as { content: { type: string; text?: string }[] }).content.map(b => b.text ?? '').join('')) } function turnNumbers(agent: Agent): number[] { return agent.session.events .filter(e => e.type === 'turn/start') .map(e => e.data.turn) } function turnEndNumbers(agent: Agent): number[] { return agent.session.events .filter(e => e.type === 'turn/end') .map(e => (e.data as { turn: number }).turn) } function userMessageCountsByTurn(agent: Agent): number[] { const counts: number[] = [] for (const event of agent.session.events) { if (event.type === 'turn/start') counts.push(0) if (event.type === 'user/message') counts[counts.length - 1]! += 1 } return counts } /** Assert a status trace is a legal run: idle/running alternating, ending idle. */ function assertLegalStatusTrace(trace: string[]): void { for (let i = 1; i < trace.length; i++) { expect(trace[i]).not.toBe(trace[i - 1]) // no repeats (setStatus dedups) } for (const s of trace) expect(['idle', 'running']).toContain(s) } describe('agent loop scheduling properties', () => { it('a synchronous burst gives every message its own strictly increasing turn', async () => { await fc.assert(fc.asyncProperty( fc.array(fc.string({ minLength: 1 }), { minLength: 1, maxLength: 6 }), async (texts) => { const ctx = await harness() try { const agent = ctx.agentLoop.create(SessionId('a'), { provider: 'mock', model: 'mock' }) const { seen: trace } = recordStatus(ctx, agent) const idle = nextIdle(ctx, agent) // Send all in one synchronous tick: they queue before the loop wakes. for (const text of texts) agent.followup(createUserMessage({ content: [{ type: 'text', text }], source: { kind: 'user' } })) await idle // No message lost: every send appears as a user/message, in order. expect(userMessageTexts(agent)).toEqual(texts) // This failure-free fixture maps every item to an independent turn. expect(turnNumbers(agent)).toEqual(texts.map((_, i) => i + 1)) expect(turnEndNumbers(agent)).toEqual(texts.map((_, i) => i + 1)) expect(userMessageCountsByTurn(agent)).toEqual(texts.map(() => 1)) expect(trace).toEqual(['running', 'idle']) assertLegalStatusTrace(trace) } finally { await ctx.fiber.dispose() } }, ), { numRuns: 25, timeout: 2000 }) }) it('sequential sends each get their own turn with increasing numbers', async () => { await fc.assert(fc.asyncProperty( fc.array(fc.string({ minLength: 1 }), { minLength: 1, maxLength: 5 }), async (texts) => { const ctx = await harness() try { const agent = ctx.agentLoop.create(SessionId('a'), { provider: 'mock', model: 'mock' }) for (const text of texts) { const idle = nextIdle(ctx, agent) agent.followup(createUserMessage({ content: [{ type: 'text', text }], source: { kind: 'user' } })) await idle } // Each send was drained at a separate turn start: N turns, 1..N. expect(turnNumbers(agent)).toEqual(texts.map((_, i) => i + 1)) expect(userMessageTexts(agent)).toEqual(texts) } finally { await ctx.fiber.dispose() } }, ), { numRuns: 20, timeout: 2000 }) }) it('mixed settled and same-tick sends preserve one turn per message', async () => { // Each step optionally waits for idle before the next send; that scheduling // choice must not change the ordinary message-to-turn mapping. const stepArb = fc.record({ text: fc.string({ minLength: 1 }), settle: fc.boolean() }) await fc.assert(fc.asyncProperty( fc.array(stepArb, { minLength: 1, maxLength: 6 }), async (steps) => { const ctx = await harness() try { const agent = ctx.agentLoop.create(SessionId('a'), { provider: 'mock', model: 'mock' }) // Capture before each send; the last waiter covers the final turn, and // awaiting an already-settled earlier waiter is harmless. let lastIdle: Promise | undefined for (const step of steps) { const idle = nextIdle(ctx, agent) lastIdle = idle agent.followup(createUserMessage({ content: [{ type: 'text', text: step.text }], source: { kind: 'user' } })) if (step.settle) await idle } await lastIdle // No message is lost or reordered, regardless of driver timing. expect(userMessageTexts(agent)).toEqual(steps.map(s => s.text)) // Every item forms one FIFO-ordered turn containing only that message. const turns = turnNumbers(agent) expect(turns).toEqual(steps.map((_, i) => i + 1)) expect(turnEndNumbers(agent)).toEqual(turns) expect(userMessageCountsByTurn(agent)).toEqual(steps.map(() => 1)) } finally { await ctx.fiber.dispose() } }, ), { numRuns: 25, timeout: 3000 }) }) })