Files
deepseek-harness/packages/agent-loop/tests/review-fixes.spec.ts
T
Tianyi Cui d2fb352f3e Enable maximum-strict TypeScript across our packages
tsconfig.base.json adds noUncheckedIndexedAccess,
exactOptionalPropertyTypes, noImplicitOverride,
noFallthroughCasesInSwitch, noUnusedLocals, and noUnusedParameters on
top of strict. Vendored packages opt out of the new flags locally
(their tsconfigs are ours to regenerate; their source is not), keeping
upstream-sync friendliness.

Our code fixed accordingly: index accesses acknowledge undefined
(assembler flush cursors, lastTurnNumber); optional properties are
omitted instead of set-to-undefined (GenerateResult.usage,
ToolDefinition.strict, GenerateOptions.system/tools, error payloads
via an errorData helper); Session.onAppend is explicitly
`(…) => void | undefined`; tests and examples updated for unused
parameters and indexed access.
2026-06-11 14:02:47 +08:00

549 lines
21 KiB
TypeScript

import { describe, expect, it } from 'vitest'
import { Context } from 'cordis'
import LlmService, { ContentBlock, MessageSource, StreamChunk } from '@deepseek-ai/dsh-llm'
import SessionStore, { Session, SessionEvent, TurnEndReason } from '@deepseek-ai/dsh-session'
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
import ToolRegistry, { defineTool } from '@deepseek-ai/dsh-tools'
import AgentRegistry from '@deepseek-ai/dsh-agent'
import AgentLoop, { LoopAgent } from '@deepseek-ai/dsh-agent-loop'
import { MockAdapter, textResponse, toolCallResponse } from './mock-adapter.ts'
/**
* Regression tests for the findings of the first architecture review
* (Codex + sub-agent, post phase-1). Each describe block names the finding.
*/
async function harness(adapter: MockAdapter) {
const ctx = new Context()
await ctx.plugin(LlmService)
await ctx.plugin(SessionStore)
await ctx.plugin(SystemPrompt)
await ctx.plugin(ToolRegistry)
await ctx.plugin(AgentRegistry)
await ctx.plugin(AgentLoop, { agents: [] })
ctx.llm.registerAdapter(['mock'], adapter)
return ctx
}
function waitForIdle(ctx: Context, agent: LoopAgent): Promise<void> {
return new Promise((resolve) => {
const dispose = ctx.on('agent/status', (subject, status) => {
if (subject === agent && status === 'idle') {
dispose()
resolve()
}
})
})
}
function send(agent: LoopAgent, text: string) {
agent.send([{ type: 'text', text }])
}
describe('HIGH: session log records what agent/step-result actually produced', () => {
it('a step-result rewrite is what the log, derived history, and tool dispatch all see', async () => {
const adapter = new MockAdapter([textResponse('original'), textResponse('done')])
const ctx = await harness(adapter)
const executed: string[] = []
ctx.tools.register(defineTool({
name: 'injected-tool',
description: '',
parameters: {},
async execute() {
executed.push('injected-tool')
return [{ type: 'text', text: 'ran' }]
},
}))
const agent = ctx.agentLoop.create('a1', { model: 'mock' })
// Plugin rewrites the message: replaces the text AND adds a tool call.
let rewritten = false
ctx.on('agent/step-result', async (_agent, _turn, _step, _message, next) => {
if (rewritten) return next()
rewritten = true
return {
role: 'assistant' as const,
content: [
{ type: 'text' as const, text: 'rewritten' },
{ type: 'tool-call' as const, id: 'c-injected', name: 'injected-tool', arguments: '{}' },
],
}
})
send(agent, 'go')
await waitForIdle(ctx, agent)
// the injected tool call was dispatched…
expect(executed).toEqual(['injected-tool'])
// …and the session log recorded the REWRITTEN message, not the original
const recorded = agent.session.events.find(e => e.type === 'assistant/message')!
expect(JSON.stringify(recorded.data)).toContain('rewritten')
expect(JSON.stringify(recorded.data)).not.toContain('original')
// tool/call + tool/result correlate with the injected call id
const callEvent = agent.session.events.find(e => e.type === 'tool/call')!
if (callEvent.type !== 'tool/call') throw new Error('wrong event type')
expect(callEvent.data.callId).toBe('c-injected')
// derived history shows the rewritten message (replay-correct)
const derived = agent.session.deriveMessages()
expect(JSON.stringify(derived)).toContain('rewritten')
expect(JSON.stringify(derived)).not.toContain('original')
})
})
describe('HIGH: abort during tool execution ends the turn', () => {
it('abort() inside a tool prevents both remaining tools and the next model step', async () => {
const adapter = new MockAdapter([
// model asks for two tool calls in one step
[
{ type: 'block-start', index: 0, blockType: 'tool-call' },
{ type: 'block-end', index: 0, block: { type: 'tool-call', id: 'c1', name: 'aborter', arguments: '{}' } },
{ type: 'block-start', index: 1, blockType: 'tool-call' },
{ type: 'block-end', index: 1, block: { type: 'tool-call', id: 'c2', name: 'second', arguments: '{}' } },
{ type: 'finish', reason: { kind: 'tool-calls' } },
] satisfies StreamChunk[],
textResponse('should never be requested'),
])
const ctx = await harness(adapter)
const executed: string[] = []
const agent = ctx.agentLoop.create('a1', { model: 'mock' })
ctx.tools.register(defineTool({
name: 'aborter',
description: '',
parameters: {},
async execute() {
executed.push('aborter')
agent.abort('user interrupt')
return [{ type: 'text', text: 'done' }]
},
}))
ctx.tools.register(defineTool({
name: 'second',
description: '',
parameters: {},
async execute() {
executed.push('second')
return [{ type: 'text', text: 'done' }]
},
}))
const reasons: TurnEndReason[] = []
ctx.on('agent/turn-end', (_agent, _turn, reason) => void reasons.push(reason))
send(agent, 'go')
await waitForIdle(ctx, agent)
expect(executed).toEqual(['aborter']) // second tool never ran
expect(adapter.requests).toHaveLength(1) // no follow-up model call
expect(reasons).toEqual([{ kind: 'aborted', reason: 'user interrupt' }])
})
})
describe('HIGH: steering from late extension points is never stranded', () => {
it('steer() from an agent/step-end listener reaches the next request (/goal pattern)', async () => {
const adapter = new MockAdapter([
toolCallResponse('c1', 'echo', { text: 'x' }),
textResponse('after steering'),
])
const ctx = await harness(adapter)
ctx.tools.register(defineTool({
name: 'echo',
description: '',
parameters: { text: { type: 'string' } },
async execute(args) {
return [{ type: 'text', text: String(args.text) }]
},
}))
const agent = ctx.agentLoop.create('a1', { model: 'mock' })
let steeredOnce = false
ctx.on('agent/step-end', () => {
if (steeredOnce) return
steeredOnce = true
agent.steer([{ type: 'text', text: 'goal reminder from step-end' }])
})
send(agent, 'go')
await waitForIdle(ctx, agent)
expect(adapter.requests).toHaveLength(2)
expect(JSON.stringify(adapter.requests[1]!.messages)).toContain('goal reminder from step-end')
})
it('steer() from an agent/turn-continuation listener overrides a stop decision', async () => {
const adapter = new MockAdapter([
textResponse('no tools, would stop here'),
textResponse('continued because of steering'),
])
const ctx = await harness(adapter)
const agent = ctx.agentLoop.create('a1', { model: 'mock' })
let steeredOnce = false
ctx.on('agent/turn-continuation', async (_agent, _turn, _decision, next) => {
if (!steeredOnce) {
steeredOnce = true
agent.steer([{ type: 'text', text: 'one more thing' }])
}
return next()
})
send(agent, 'go')
await waitForIdle(ctx, agent)
// the default decision was false (no tools), but steering forced step 2
expect(adapter.requests).toHaveLength(2)
expect(JSON.stringify(adapter.requests[1]!.messages)).toContain('one more thing')
})
it('steer() from an agent/turn-end listener becomes a queued message for the next turn', async () => {
const adapter = new MockAdapter([textResponse('turn 1'), textResponse('turn 2')])
const ctx = await harness(adapter)
const agent = ctx.agentLoop.create('a1', { model: 'mock' })
let steeredOnce = false
ctx.on('agent/turn-end', () => {
if (steeredOnce) return
steeredOnce = true
agent.steer([{ type: 'text', text: 'too late for this turn' }])
})
const turns: number[] = []
ctx.on('agent/turn-start', (_agent, turn) => void turns.push(turn))
send(agent, 'go')
await waitForIdle(ctx, agent)
// the loop chains directly into turn 2 (status never returns to idle in
// between), so the first idle transition means both turns are complete
expect(turns).toEqual([1, 2])
expect(adapter.requests).toHaveLength(2)
expect(JSON.stringify(adapter.requests[1]!.messages)).toContain('too late for this turn')
})
it('steering queued during an aborted step is re-delivered, not silently consumed', async () => {
const adapter = new MockAdapter(['hang', textResponse('recovered')])
const ctx = await harness(adapter)
const agent = ctx.agentLoop.create('a1', { model: 'mock' })
send(agent, 'go')
await new Promise(r => setTimeout(r, 30))
agent.steer([{ type: 'text', text: 'redirect' }])
agent.abort('user interrupt')
await waitForIdle(ctx, agent)
// a new turn ran with the steering content delivered as a message
expect(adapter.requests).toHaveLength(2)
expect(JSON.stringify(adapter.requests[1]!.messages)).toContain('redirect')
})
})
describe('HIGH: plugin exceptions are contained', () => {
it('a throwing agent/turn-continuation listener ends the turn with an error, loop survives', async () => {
const adapter = new MockAdapter([textResponse('one'), textResponse('two')])
const ctx = await harness(adapter)
const agent = ctx.agentLoop.create('a1', { model: 'mock' })
let threwOnce = false
ctx.on('agent/turn-continuation', async (): Promise<boolean> => {
if (!threwOnce) {
threwOnce = true
throw new Error('broken continuation plugin')
}
return false
})
const errors: Error[] = []
ctx.on('agent/error', (_agent, _turn, _step, error) => void errors.push(error))
send(agent, 'first')
await waitForIdle(ctx, agent)
expect(errors.map(e => e.message)).toEqual(['broken continuation plugin'])
// the loop is still alive: a second send works normally
send(agent, 'second')
await waitForIdle(ctx, agent)
expect(adapter.requests).toHaveLength(2)
expect(agent.status).toBe('idle')
})
it('a rejecting session/flush listener is reported but does not kill the agent', async () => {
const adapter = new MockAdapter([textResponse('one'), textResponse('two')])
const ctx = await harness(adapter)
const agent = ctx.agentLoop.create('a1', { model: 'mock' })
let rejectedOnce = false
ctx.on('session/flush', async () => {
if (!rejectedOnce) {
rejectedOnce = true
throw new Error('disk full')
}
})
const errors: Error[] = []
ctx.on('agent/error', (_agent, _turn, _step, error) => void errors.push(error))
send(agent, 'first')
await waitForIdle(ctx, agent)
expect(errors.map(e => e.message)).toEqual(['disk full'])
send(agent, 'second')
await waitForIdle(ctx, agent)
expect(adapter.requests).toHaveLength(2)
})
})
describe('MEDIUM: disposed status is part of the agent/status contract', () => {
it('disposing the fiber emits agent/status(disposed) and ends the turn with reason disposed', async () => {
const adapter = new MockAdapter(['hang'])
const ctx = await harness(adapter)
let agent!: LoopAgent
const fiber = await ctx.plugin(Object.assign((inner: Context) => {
agent = inner.agentLoop.create('scoped', { model: 'mock' })
}, { inject: ['agentLoop'] }))
const statuses: string[] = []
const reasons: TurnEndReason[] = []
ctx.on('agent/status', (_agent, status) => void statuses.push(status))
ctx.on('agent/turn-end', (_agent, _turn, reason) => void reasons.push(reason))
send(agent, 'go')
await new Promise(r => setTimeout(r, 30))
await fiber.dispose()
await agent.done
expect(statuses).toEqual(['running', 'disposed'])
expect(reasons).toEqual([{ kind: 'disposed' }])
})
it('a throwing agent/status listener cannot break disposal or leak the registry entry', async () => {
const adapter = new MockAdapter(['hang'])
const ctx = await harness(adapter)
let agent!: LoopAgent
const fiber = await ctx.plugin(Object.assign((inner: Context) => {
agent = inner.agentLoop.create('scoped', { model: 'mock' })
}, { inject: ['agentLoop'] }))
ctx.on('agent/status', (_agent, status) => {
if (status === 'disposed') throw new Error('broken status listener')
})
send(agent, 'go')
await new Promise(r => setTimeout(r, 30))
await fiber.dispose()
await agent.done // must not hang
expect(agent.status).toBe('disposed')
expect(ctx.agents.get('scoped')).toBeUndefined() // unregistered despite the throw
})
})
describe('MEDIUM: misc registry and config fixes', () => {
it('duplicate adapter registration is rejected', async () => {
const ctx = new Context()
await ctx.plugin(LlmService)
const adapter = new MockAdapter([])
ctx.llm.registerAdapter(['m1'], adapter)
expect(() => ctx.llm.registerAdapter(['m1'], new MockAdapter([])))
.toThrow('already registered')
// the original registration survives the failed attempt
expect(ctx.llm.models()).toEqual(['m1'])
})
it('an agent without a model fails the step with a clear error (not NO_ADAPTER for "default")', async () => {
const adapter = new MockAdapter([textResponse('never')])
const ctx = await harness(adapter)
const agent = ctx.agentLoop.create('a1', {}) // no model
const errors: Error[] = []
ctx.on('agent/error', (_agent, _turn, _step, error) => void errors.push(error))
send(agent, 'go')
await waitForIdle(ctx, agent)
expect(errors).toHaveLength(1)
expect(errors[0]!.message).toContain('has no model')
expect(errors[0]!.message).toContain('agent/request')
})
it('the agent/request waterfall can supply the model for a model-less agent', async () => {
const adapter = new MockAdapter([textResponse('routed')])
const ctx = await harness(adapter)
const agent = ctx.agentLoop.create('a1', {}) // no model — router plugin decides
ctx.on('agent/request', async (_agent, _turn, _step, options, next) => {
options.model = 'mock'
return next()
})
send(agent, 'go')
await waitForIdle(ctx, agent)
expect(adapter.requests).toHaveLength(1)
expect(agent.session.deriveMessages().at(-1)?.content).toEqual([{ type: 'text', text: 'routed' }])
})
it('agent/queued carries the resolved source; agent/steering carries its source', async () => {
const adapter = new MockAdapter([toolCallResponse('c1', 'noop', {}), textResponse('done')])
const ctx = await harness(adapter)
const agent = ctx.agentLoop.create('a1', { model: 'mock' })
ctx.tools.register(defineTool({
name: 'noop',
description: '',
parameters: {},
async execute() {
agent.steer([{ type: 'text', text: 's' }], { source: { kind: 'plugin', plugin: 'goal' } })
return []
},
}))
const queuedSources: { source: MessageSource; steering: boolean }[] = []
const steeringSources: MessageSource[] = []
ctx.on('agent/queued', (_agent, _content, info) => void queuedSources.push(info))
ctx.on('agent/steering', (_agent, _turn, _content, source) => void steeringSources.push(source))
send(agent, 'go') // no explicit source → default {kind:'user'} must be visible
await waitForIdle(ctx, agent)
expect(queuedSources[0]).toEqual({ source: { kind: 'user' }, steering: false })
expect(queuedSources[1]).toEqual({ source: { kind: 'plugin', plugin: 'goal' }, steering: true })
expect(steeringSources).toEqual([{ kind: 'plugin', plugin: 'goal' }])
})
})
describe('MEDIUM: turn numbering continues across seeded (forked) sessions', () => {
it('a forked agent continues turn numbers after the seed log', async () => {
const first = new MockAdapter([textResponse('turn one')])
const ctx = await harness(first)
const agent = ctx.agentLoop.create('a1', { model: 'mock' })
send(agent, 'first')
await waitForIdle(ctx, agent)
// fork: seed a second context's agent with the first session's log
const second = new MockAdapter([textResponse('turn two')])
const ctx2 = new Context()
await ctx2.plugin(LlmService)
await ctx2.plugin(SessionStore)
await ctx2.plugin(SystemPrompt)
await ctx2.plugin(ToolRegistry)
await ctx2.plugin(AgentRegistry)
await ctx2.plugin(AgentLoop, { agents: [] })
ctx2.llm.registerAdapter(['mock'], second)
const seeded = ctx2.sessions.create('forked', [...agent.session.events])
const forked = new LoopAgent(ctx2, 'forked-agent', { model: 'mock' }, seeded)
ctx2.effect(() => forked.start())
const turns: number[] = []
ctx2.on('agent/turn-start', (_agent, turn) => void turns.push(turn))
forked.send([{ type: 'text', text: 'continue' }])
await new Promise<void>((resolve) => {
ctx2.on('agent/status', (subject, status) => {
if (subject === forked && status === 'idle') resolve()
})
})
expect(turns).toEqual([2])
})
})
describe('LOW: BlockAssembler and streamBlocks edge cases', () => {
it('ignores deltas arriving after block-end for the same index (malformed stream)', async () => {
const { BlockAssembler } = await import('@deepseek-ai/dsh-llm')
const assembler = new BlockAssembler()
assembler.push({ type: 'block-start', index: 0, blockType: 'text' })
assembler.push({ type: 'text-delta', index: 0, text: 'good' })
assembler.push({ type: 'block-end', index: 0, block: { type: 'text', text: 'good' } })
assembler.push({ type: 'text-delta', index: 0, text: ' straggler' })
expect(assembler.blocks()).toEqual([{ type: 'text', text: 'good' }])
})
it('assembles tool-call blocks from deltas without block-end', async () => {
const { BlockAssembler } = await import('@deepseek-ai/dsh-llm')
const assembler = new BlockAssembler()
assembler.push({ type: 'tool-call-delta', index: 0, id: 'c9', name: 'echo', argumentsDelta: '{"a"' })
assembler.push({ type: 'tool-call-delta', index: 0, id: 'c9', argumentsDelta: ':1}' })
expect(assembler.blocks()).toEqual([
{ type: 'tool-call', id: 'c9', name: 'echo', arguments: '{"a":1}' },
])
})
it('streamBlocks flushes delta-only blocks at end of stream (matches generate())', async () => {
const ctx = new Context()
await ctx.plugin(LlmService)
const deltaOnly: StreamChunk[] = [
{ type: 'text-delta', index: 0, text: 'no ' },
{ type: 'text-delta', index: 0, text: 'block-end' },
{ type: 'finish', reason: { kind: 'stop' } },
]
ctx.llm.registerAdapter(['m'], new MockAdapter([deltaOnly, deltaOnly]))
const blocks: ContentBlock[] = []
for await (const block of ctx.llm.streamBlocks({ model: 'm', messages: [] })) blocks.push(block)
expect(blocks).toEqual([{ type: 'text', text: 'no block-end' }])
const generated = await ctx.llm.generate({ model: 'm', messages: [] })
expect(generated.message.content).toEqual(blocks)
})
it('streamBlocks preserves stream order when an open block precedes a closed one', async () => {
const ctx = new Context()
await ctx.plugin(LlmService)
// index 0 never gets block-end (delta-only); index 1 closes mid-stream.
const interleaved: StreamChunk[] = [
{ type: 'text-delta', index: 0, text: 'first, open' },
{ type: 'block-start', index: 1, blockType: 'text' },
{ type: 'text-delta', index: 1, text: 'second, closed' },
{ type: 'block-end', index: 1, block: { type: 'text', text: 'second, closed' } },
{ type: 'finish', reason: { kind: 'stop' } },
]
ctx.llm.registerAdapter(['m'], new MockAdapter([interleaved, interleaved]))
const blocks: ContentBlock[] = []
for await (const block of ctx.llm.streamBlocks({ model: 'm', messages: [] })) blocks.push(block)
expect(blocks).toEqual([
{ type: 'text', text: 'first, open' },
{ type: 'text', text: 'second, closed' },
])
// identical to generate()'s assembled order
const generated = await ctx.llm.generate({ model: 'm', messages: [] })
expect(generated.message.content).toEqual(blocks)
})
it('streamBlocks yields closed blocks incrementally once preceding blocks close', async () => {
const ctx = new Context()
await ctx.plugin(LlmService)
const script: StreamChunk[] = [
{ type: 'block-start', index: 0, blockType: 'text' },
{ type: 'text-delta', index: 0, text: 'a' },
{ type: 'block-end', index: 0, block: { type: 'text', text: 'a' } },
{ type: 'block-start', index: 1, blockType: 'text' },
{ type: 'text-delta', index: 1, text: 'b' },
{ type: 'block-end', index: 1, block: { type: 'text', text: 'b' } },
{ type: 'finish', reason: { kind: 'stop' } },
]
ctx.llm.registerAdapter(['m'], new MockAdapter([script]))
const blocks: ContentBlock[] = []
for await (const block of ctx.llm.streamBlocks({ model: 'm', messages: [] })) blocks.push(block)
expect(blocks).toEqual([{ type: 'text', text: 'a' }, { type: 'text', text: 'b' }])
})
})
describe('LOW: discriminated SessionEvent narrows without casts', () => {
it('narrows event.data from event.type', () => {
const session = new Session('s')
const appended: SessionEvent = session.append('tool/call', {
turn: 1, step: 1, callId: 'c1', name: 'echo', arguments: '{}',
})
// compile-time: this switch narrows; runtime: values flow through
switch (appended.type) {
case 'tool/call': {
expect(appended.data.callId).toBe('c1')
expect(appended.data.name).toBe('echo')
break
}
default: throw new Error('wrong narrow')
}
})
})