395 lines
18 KiB
TypeScript
395 lines
18 KiB
TypeScript
import { describe, expect, it } from 'vitest'
|
|
import { Context } from 'cordis'
|
|
import { CallId } from '@deepseek-ai/dsh-llm'
|
|
import { SessionId, type SessionEvent } from '@deepseek-ai/dsh-session'
|
|
import { defineContentToolFixture } from '@deepseek-ai/dsh-tools'
|
|
import type { Agent } from '@deepseek-ai/dsh-agent'
|
|
import AgentLoop from '@deepseek-ai/dsh-agent-loop'
|
|
import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit'
|
|
import * as RepeatToolGuard from '@deepseek-ai/dsh-repeat-tool-guard'
|
|
import type { Config } from '@deepseek-ai/dsh-repeat-tool-guard'
|
|
import { MockAdapter, textResponse, toolCallResponse } from '../../../core/agent-loop/tests/mock-adapter.ts'
|
|
|
|
const testToolSignal = new AbortController().signal
|
|
|
|
/**
|
|
* Behavior suite for the repeat-tool-call guard: chain semantics (identical /
|
|
* different-tracked / untracked-transparent / per-agent / resets), threshold
|
|
* escalation incl. the `thresholds[0]` gentle-text rule, canonicalization,
|
|
* fold-onto-downstream-decision, and fail-loud config validation — all driven
|
|
* through a real agent loop against a scripted mock adapter (no network).
|
|
*/
|
|
|
|
/** Boot the core spine + the guard; the caller registers adapters and extra listeners. */
|
|
async function harness(config: Config = {}): Promise<Context> {
|
|
const ctx = new Context()
|
|
await mountAgentLoopTestDependencies(ctx)
|
|
await ctx.plugin(AgentLoop, { agents: [] })
|
|
await ctx.plugin(RepeatToolGuard, config)
|
|
ctx.tools.register(defineContentToolFixture({ name: 'probe', description: 'p', parameters: {}, async execute() { return [{ type: 'text', text: 'ok' }] } }))
|
|
ctx.tools.register(defineContentToolFixture({ name: 'other', description: 'o', parameters: {}, async execute() { return [{ type: 'text', text: 'ok' }] } }))
|
|
return ctx
|
|
}
|
|
|
|
function waitForIdle(ctx: Context, agent: Agent): Promise<void> {
|
|
return new Promise((resolve) => { const d = ctx.on('agent/status', (s, st) => { if (s === agent && st === 'idle') { d(); resolve() } }) })
|
|
}
|
|
|
|
/** Every injected-context user message in the agent's log, flattened to joined text + source for terse assertions. */
|
|
function reminders(agent: Agent): { text: string; source: unknown }[] {
|
|
return [...agent.session.events]
|
|
.filter((e): e is SessionEvent<'user/message'> => e.type === 'user/message' && e.data.source.kind !== 'user')
|
|
.map(e => ({
|
|
text: e.data.content.map(block => block.type === 'text' ? block.text : '').join('|'),
|
|
source: e.data.source,
|
|
}))
|
|
}
|
|
|
|
const GUARD_SOURCE = { kind: 'plugin', plugin: 'repeat-tool-guard' }
|
|
|
|
describe('threshold escalation', () => {
|
|
it('reminds gently at the first default threshold (3) and in detail at the second (5)', async () => {
|
|
const ctx = await harness()
|
|
const adapter = new MockAdapter([
|
|
...Array.from({ length: 5 }, (_, i) => toolCallResponse(`c${i}`, 'probe', { q: 'same' })),
|
|
textResponse('done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
const found = reminders(agent)
|
|
expect(found).toHaveLength(2)
|
|
expect(found[0]!.text).toContain('repeating the exact same tool call')
|
|
expect(found[0]!.source).toEqual(GUARD_SOURCE)
|
|
expect(found[1]!.text).toContain('consecutive_calls: 5')
|
|
expect(found[1]!.text).toContain('- tool: probe')
|
|
expect(found[1]!.text).toContain('{"q":"same"}')
|
|
expect(found[1]!.source).toEqual(GUARD_SOURCE)
|
|
})
|
|
|
|
it('keys the gentle text to thresholds[0], not the literal 3', async () => {
|
|
const ctx = await harness({ thresholds: [4, 2] }) // unsorted on purpose: normalized ascending
|
|
const adapter = new MockAdapter([
|
|
...Array.from({ length: 4 }, (_, i) => toolCallResponse(`c${i}`, 'probe', {})),
|
|
textResponse('done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
const found = reminders(agent)
|
|
expect(found).toHaveLength(2)
|
|
expect(found[0]!.text).toContain('repeating the exact same tool call') // gentle at 2
|
|
expect(found[1]!.text).toContain('consecutive_calls: 4') // detailed at 4
|
|
})
|
|
})
|
|
|
|
describe('chain semantics', () => {
|
|
it('caps the detailed reminder arguments at argumentsPreviewChars (detection still keys on the full string)', async () => {
|
|
const ctx = await harness({ thresholds: [2, 3], argumentsPreviewChars: 24 })
|
|
const bigPayload = 'x'.repeat(400)
|
|
const adapter = new MockAdapter([
|
|
toolCallResponse('c1', 'probe', { body: bigPayload }),
|
|
toolCallResponse('c2', 'probe', { body: bigPayload }),
|
|
toolCallResponse('c3', 'probe', { body: bigPayload }),
|
|
textResponse('done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
const found = reminders(agent)
|
|
expect(found).toHaveLength(2) // gentle at 2, detailed at 3 — full-key matching survived the cap
|
|
const detailed = found[1]!.text
|
|
expect(detailed).toContain('- arguments: {"body":"xxxxxxxxxxxxxx') // 24-char head
|
|
expect(detailed).toContain('… (+387 more chars)')
|
|
expect(detailed).not.toContain(bigPayload)
|
|
})
|
|
|
|
it('a different tracked call resets the chain', async () => {
|
|
const ctx = await harness()
|
|
const adapter = new MockAdapter([
|
|
toolCallResponse('c1', 'probe', { q: 1 }),
|
|
toolCallResponse('c2', 'probe', { q: 1 }),
|
|
toolCallResponse('c3', 'other', {}), // tracked, different → reset
|
|
toolCallResponse('c4', 'probe', { q: 1 }),
|
|
toolCallResponse('c5', 'probe', { q: 1 }),
|
|
toolCallResponse('c6', 'probe', { q: 1 }), // 3rd consecutive AFTER the reset
|
|
textResponse('done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
expect(reminders(agent)).toHaveLength(1)
|
|
})
|
|
|
|
it('excluded calls are transparent: they neither count nor reset', async () => {
|
|
const ctx = await harness({ exclude: ['other'] })
|
|
const adapter = new MockAdapter([
|
|
toolCallResponse('c1', 'probe', { q: 1 }),
|
|
toolCallResponse('c2', 'other', {}), // excluded → invisible to the chain
|
|
toolCallResponse('c3', 'probe', { q: 1 }),
|
|
toolCallResponse('c4', 'other', {}),
|
|
toolCallResponse('c5', 'probe', { q: 1 }), // 3rd consecutive probe
|
|
textResponse('done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
const found = reminders(agent)
|
|
expect(found).toHaveLength(1)
|
|
expect(found[0]!.text).toContain('repeating the exact same tool call')
|
|
})
|
|
|
|
it('include patterns track only matching tools (wildcard star)', async () => {
|
|
const ctx = await harness({ include: ['pro*'] })
|
|
const adapter = new MockAdapter([
|
|
toolCallResponse('c1', 'other', {}),
|
|
toolCallResponse('c2', 'other', {}),
|
|
toolCallResponse('c3', 'other', {}), // 3 identical, but untracked
|
|
toolCallResponse('c4', 'probe', {}),
|
|
toolCallResponse('c5', 'probe', {}),
|
|
toolCallResponse('c6', 'probe', {}), // 3 identical, tracked
|
|
textResponse('done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
const found = reminders(agent)
|
|
expect(found).toHaveLength(1)
|
|
expect(found[0]!.text).toContain('repeating the exact same tool call')
|
|
})
|
|
|
|
it('escapes regex metacharacters in patterns (a dot matches only a literal dot)', async () => {
|
|
const ctx = await harness({ exclude: ['pr.be'] }) // would match 'probe' as a regex; must not as a wildcard
|
|
const adapter = new MockAdapter([
|
|
...Array.from({ length: 3 }, (_, i) => toolCallResponse(`c${i}`, 'probe', {})),
|
|
textResponse('done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
expect(reminders(agent)).toHaveLength(1) // probe was NOT excluded
|
|
})
|
|
|
|
it('canonicalization ignores property order, deeply', async () => {
|
|
const ctx = await harness()
|
|
const adapter = new MockAdapter([
|
|
toolCallResponse('c1', 'probe', { a: 1, nested: { x: [1, 2], y: null } }),
|
|
toolCallResponse('c2', 'probe', { nested: { y: null, x: [1, 2] }, a: 1 }),
|
|
toolCallResponse('c3', 'probe', { a: 1, nested: { x: [1, 2], y: null } }),
|
|
textResponse('done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
expect(reminders(agent)).toHaveLength(1) // all three canonicalize identically
|
|
})
|
|
|
|
it('keys chains per agent: one agent repeating never trips another', async () => {
|
|
const ctx = await harness()
|
|
ctx.llm.registerAdapter(['mock-a'], new MockAdapter([
|
|
toolCallResponse('a1', 'probe', { q: 1 }),
|
|
toolCallResponse('a2', 'probe', { q: 1 }),
|
|
textResponse('done'),
|
|
]))
|
|
ctx.llm.registerAdapter(['mock-b'], new MockAdapter([
|
|
toolCallResponse('b1', 'probe', { q: 1 }),
|
|
toolCallResponse('b2', 'probe', { q: 1 }),
|
|
toolCallResponse('b3', 'probe', { q: 1 }),
|
|
textResponse('done'),
|
|
]))
|
|
const agentA = ctx.agentLoop.create(SessionId('a'), { provider: 'mock-a', model: 'model-a' })
|
|
const agentB = ctx.agentLoop.create(SessionId('b'), { provider: 'mock-b', model: 'model-b' })
|
|
agentA.followup([{ type: 'text', text: 'go' }])
|
|
agentB.followup([{ type: 'text', text: 'go' }])
|
|
await Promise.all([waitForIdle(ctx, agentA), waitForIdle(ctx, agentB)])
|
|
|
|
expect(reminders(agentA)).toHaveLength(0) // 2 repeats < 3, despite B's 3 in the same registry
|
|
expect(reminders(agentB)).toHaveLength(1)
|
|
})
|
|
|
|
it('a new user prompt resets the chain', async () => {
|
|
const ctx = await harness()
|
|
const adapter = new MockAdapter([
|
|
toolCallResponse('c1', 'probe', { q: 1 }),
|
|
toolCallResponse('c2', 'probe', { q: 1 }),
|
|
textResponse('turn one done'),
|
|
toolCallResponse('c3', 'probe', { q: 1 }), // without the reset this would be the 3rd
|
|
textResponse('turn two done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
agent.followup([{ type: 'text', text: 'again' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
expect(reminders(agent)).toHaveLength(0)
|
|
})
|
|
|
|
it('drops an agent chain on disposal', async () => {
|
|
const ctx = await harness({ thresholds: [2] })
|
|
ctx.llm.registerAdapter(['mock'], new MockAdapter([
|
|
toolCallResponse('c1', 'probe', { q: 1 }),
|
|
textResponse('done'),
|
|
toolCallResponse('c2', 'probe', { q: 1 }), // same id, fresh agent: count 1, not 2
|
|
textResponse('done'),
|
|
]))
|
|
// Loop agents are torn down by disposing the scope that created them
|
|
// (the loop.spec pattern): a child plugin fiber owns `first`.
|
|
let first!: Agent
|
|
const fiber = await ctx.plugin(Object.assign((inner: Context) => {
|
|
first = inner.agentLoop.create(SessionId('reused'), { provider: 'mock', model: 'mock' })
|
|
}, { inject: ['agentLoop'] }))
|
|
first.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, first)
|
|
await fiber.dispose()
|
|
await first.whenIdle()
|
|
|
|
const second = ctx.agentLoop.create(SessionId('reused'), { provider: 'mock', model: 'mock' })
|
|
second.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, second)
|
|
|
|
expect(reminders(second)).toHaveLength(0)
|
|
})
|
|
|
|
it('counts denied calls: hammering a denied tool still draws the reminder', async () => {
|
|
const ctx = await harness({ thresholds: [2] })
|
|
ctx.on('tools/pre-execute', async () => ({ kind: 'deny' as const, reason: 'sealed' }))
|
|
const adapter = new MockAdapter([
|
|
toolCallResponse('c1', 'probe', { q: 1 }),
|
|
toolCallResponse('c2', 'probe', { q: 1 }),
|
|
textResponse('done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
expect(reminders(agent)).toHaveLength(1)
|
|
})
|
|
|
|
it('ignores direct executes with no agent (they neither crash nor advance any chain)', async () => {
|
|
const ctx = await harness({ thresholds: [2] })
|
|
const direct = await ctx.tools.execute({ signal: testToolSignal, callId: CallId('d1'), name: 'probe', arguments: { q: 1 } })
|
|
expect(direct.isError).toBe(false)
|
|
|
|
ctx.llm.registerAdapter(['mock'], new MockAdapter([
|
|
toolCallResponse('c1', 'probe', { q: 1 }), // if the direct call had counted, this would be #2
|
|
textResponse('done'),
|
|
]))
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
expect(reminders(agent)).toHaveLength(0)
|
|
})
|
|
})
|
|
|
|
describe('fold onto the downstream decision', () => {
|
|
it('folds the reminder onto a downstream block and keeps its feedback', async () => {
|
|
const ctx = await harness({ thresholds: [2] })
|
|
ctx.on('tools/post-execute', async () => ({
|
|
kind: 'block' as const,
|
|
feedback: [{ type: 'text' as const, text: 'nope' }],
|
|
additionalContexts: [{ content: [{ type: 'text' as const, text: 'downstream-ctx' }], source: { kind: 'plugin' as const, plugin: 'test' } }],
|
|
}))
|
|
const adapter = new MockAdapter([
|
|
toolCallResponse('c1', 'probe', { q: 1 }),
|
|
toolCallResponse('c2', 'probe', { q: 1 }),
|
|
textResponse('done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
const found = reminders(agent)
|
|
expect(found).toHaveLength(3)
|
|
// Only the repeated call adds guard context; downstream provenance survives.
|
|
expect(found[0]!.text).toBe('downstream-ctx')
|
|
expect(found[0]!.source).toEqual({ kind: 'plugin', plugin: 'test' })
|
|
expect(found[1]!.text).toContain('repeating the exact same tool call')
|
|
expect(found[1]!.source).toEqual(GUARD_SOURCE)
|
|
expect(found[2]).toEqual({ text: 'downstream-ctx', source: { kind: 'plugin', plugin: 'test' } })
|
|
// The block's feedback reached the tool result unchanged.
|
|
const results = [...agent.session.events].filter((e): e is SessionEvent<'tool/result'> => e.type === 'tool/result')
|
|
expect(results.every(r => r.data.isError)).toBe(true)
|
|
expect(results[1]!.data.content).toEqual([{ type: 'text', text: 'nope' }])
|
|
})
|
|
|
|
it('preserves a downstream canonical value replacement while folding', async () => {
|
|
const ctx = await harness({ thresholds: [2] })
|
|
ctx.on('tools/post-execute', async () => ({
|
|
kind: 'accept' as const,
|
|
value: [{ type: 'text' as const, text: 'replaced' }],
|
|
}))
|
|
const adapter = new MockAdapter([
|
|
toolCallResponse('c1', 'probe', { q: 1 }),
|
|
toolCallResponse('c2', 'probe', { q: 1 }),
|
|
textResponse('done'),
|
|
])
|
|
ctx.llm.registerAdapter(['mock'], adapter)
|
|
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
|
|
agent.followup([{ type: 'text', text: 'go' }])
|
|
await waitForIdle(ctx, agent)
|
|
|
|
const found = reminders(agent)
|
|
expect(found).toHaveLength(1)
|
|
expect(found[0]!.text).toContain('repeating the exact same tool call')
|
|
const results = [...agent.session.events].filter((e): e is SessionEvent<'tool/result'> => e.type === 'tool/result')
|
|
expect(results[1]!.data.content).toEqual([{ type: 'text', text: 'replaced' }])
|
|
})
|
|
})
|
|
|
|
describe('config validation fails loud', () => {
|
|
async function spine(): Promise<Context> {
|
|
const ctx = new Context()
|
|
await mountAgentLoopTestDependencies(ctx)
|
|
await ctx.plugin(AgentLoop, { agents: [] })
|
|
return ctx
|
|
}
|
|
|
|
it('rejects an empty thresholds list', async () => {
|
|
const ctx = await spine()
|
|
await expect(ctx.plugin(RepeatToolGuard, { thresholds: [] })).rejects.toThrow(/must not be empty/)
|
|
})
|
|
|
|
it('rejects a threshold below 2', async () => {
|
|
const ctx = await spine()
|
|
await expect(ctx.plugin(RepeatToolGuard, { thresholds: [1, 3] })).rejects.toThrow(/integer >= 2/)
|
|
})
|
|
|
|
it('rejects a non-integer threshold', async () => {
|
|
const ctx = await spine()
|
|
await expect(ctx.plugin(RepeatToolGuard, { thresholds: [2.5] })).rejects.toThrow(/integer >= 2/)
|
|
})
|
|
|
|
it('rejects duplicate thresholds', async () => {
|
|
const ctx = await spine()
|
|
await expect(ctx.plugin(RepeatToolGuard, { thresholds: [3, 3] })).rejects.toThrow(/duplicates/)
|
|
})
|
|
|
|
it('rejects a non-positive or fractional argumentsPreviewChars', async () => {
|
|
const ctx = await spine()
|
|
await expect(ctx.plugin(RepeatToolGuard, { argumentsPreviewChars: 0 })).rejects.toThrow(/argumentsPreviewChars/)
|
|
const ctx2 = await spine()
|
|
await expect(ctx2.plugin(RepeatToolGuard, { argumentsPreviewChars: 12.5 })).rejects.toThrow(/argumentsPreviewChars/)
|
|
})
|
|
})
|