From edaaa50fe375cf1a058367289d0bd553880af293 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:03:31 +0800 Subject: [PATCH] test(acp): snapshot cancelled queued tool calls --- examples/acp-agent/tests/acp.snapshot.ts | 1 + .../snapshots/cancel-tool-calls/input.json | 11 +++++++++ .../cancel-tool-calls/replay.override.json | 15 ++++++++++++ .../snapshots/cancel-tool-calls/session.jsonl | 20 ++++++++++++++++ .../cancel-tool-calls/stdout.golden.jsonl | 7 ++++++ packages/support/acp-snapshot/src/harness.ts | 24 ++++++++----------- .../tests/fixtures/fake-acp-agent.ts | 20 +++++++++++++++- .../acp-snapshot/tests/harness.spec.ts | 10 ++++++++ 8 files changed, 93 insertions(+), 15 deletions(-) create mode 100644 examples/acp-agent/tests/snapshots/cancel-tool-calls/input.json create mode 100644 examples/acp-agent/tests/snapshots/cancel-tool-calls/replay.override.json create mode 100644 examples/acp-agent/tests/snapshots/cancel-tool-calls/session.jsonl create mode 100644 examples/acp-agent/tests/snapshots/cancel-tool-calls/stdout.golden.jsonl diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 124117afef..d60d8b9724 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -103,6 +103,7 @@ const SCENARIOS: Scenario[] = [ configPath: WORKSPACE_CONTEXT_CONFIG, }, { name: 'cancel', hasModelTurn: true, recorded: false, overridden: true }, + { name: 'cancel-tool-calls', hasModelTurn: true, recorded: false, overridden: true }, { name: 'subagent-spawn', hasModelTurn: true, recorded: true }, { name: 'subagent-multi', hasModelTurn: true, recorded: true }, { name: 'subagent-fork', hasModelTurn: true, recorded: true }, diff --git a/examples/acp-agent/tests/snapshots/cancel-tool-calls/input.json b/examples/acp-agent/tests/snapshots/cancel-tool-calls/input.json new file mode 100644 index 0000000000..3610e1f436 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/cancel-tool-calls/input.json @@ -0,0 +1,11 @@ +{ + "steps": [ + { "op": "initialize" }, + { "op": "newSession" }, + { + "op": "promptAndCancel", + "text": "Run two shell commands: wait for cancellation, then write skipped.txt.", + "afterUpdate": "tool_call" + } + ] +} diff --git a/examples/acp-agent/tests/snapshots/cancel-tool-calls/replay.override.json b/examples/acp-agent/tests/snapshots/cancel-tool-calls/replay.override.json new file mode 100644 index 0000000000..c0aa7730d7 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/cancel-tool-calls/replay.override.json @@ -0,0 +1,15 @@ +[ + { + "kind": "chunks", + "chunks": [ + { "type": "block-start", "index": 0, "blockType": "tool-call" }, + { "type": "tool-call-delta", "index": 0, "id": "call_wait", "name": "bash", "argumentsDelta": "{\"command\":\"node -e \\\"setInterval(() => {}, 1000)\\\"\",\"description\":\"Wait until cancellation\"}" }, + { "type": "block-end", "index": 0, "block": { "type": "tool-call", "id": "call_wait", "name": "bash", "arguments": "{\"command\":\"node -e \\\"setInterval(() => {}, 1000)\\\"\",\"description\":\"Wait until cancellation\"}" } }, + { "type": "block-start", "index": 1, "blockType": "tool-call" }, + { "type": "tool-call-delta", "index": 1, "id": "call_skipped", "name": "bash", "argumentsDelta": "{\"command\":\"printf skipped > skipped.txt\",\"description\":\"Write skipped marker\"}" }, + { "type": "block-end", "index": 1, "block": { "type": "tool-call", "id": "call_skipped", "name": "bash", "arguments": "{\"command\":\"printf skipped > skipped.txt\",\"description\":\"Write skipped marker\"}" } }, + { "type": "usage", "usage": { "inputTokens": 10, "outputTokens": 10 } }, + { "type": "finish", "reason": { "kind": "tool-calls" } } + ] + } +] diff --git a/examples/acp-agent/tests/snapshots/cancel-tool-calls/session.jsonl b/examples/acp-agent/tests/snapshots/cancel-tool-calls/session.jsonl new file mode 100644 index 0000000000..e6d18515a6 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/cancel-tool-calls/session.jsonl @@ -0,0 +1,20 @@ +{"type":"session","version":0,"id":"{{sessionId}}","createdAt":0,"cwd":"{{cwd}}"} +{"type":"turn/start","seq":0,"time":1784437195072,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}} +{"type":"user/message","seq":1,"time":1784437195072,"data":{"content":[{"type":"text","text":"Run two shell commands: wait for cancellation, then write skipped.txt."}],"source":{"kind":"user"}},"surfaceOp":"append"} +{"type":"step/start","seq":2,"time":1784437195076,"data":{"turn":1,"step":1}} +{"type":"request/header","seq":3,"time":1784437195076,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} +{"type":"assistant/chunk","seq":4,"time":1784437195077,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}} +{"type":"assistant/chunk","seq":5,"time":1784437195077,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"call_wait","name":"bash","argumentsDelta":"{\"command\":\"node -e \\\"setInterval(() => {}, 1000)\\\"\",\"description\":\"Wait until cancellation\"}"}}} +{"type":"assistant/chunk","seq":6,"time":1784437195077,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_wait","name":"bash","arguments":"{\"command\":\"node -e \\\"setInterval(() => {}, 1000)\\\"\",\"description\":\"Wait until cancellation\"}"}}}} +{"type":"assistant/chunk","seq":7,"time":1784437195077,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"tool-call"}}} +{"type":"assistant/chunk","seq":8,"time":1784437195077,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_skipped","name":"bash","argumentsDelta":"{\"command\":\"printf skipped > skipped.txt\",\"description\":\"Write skipped marker\"}"}}} +{"type":"assistant/chunk","seq":9,"time":1784437195077,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"tool-call","id":"call_skipped","name":"bash","arguments":"{\"command\":\"printf skipped > skipped.txt\",\"description\":\"Write skipped marker\"}"}}}} +{"type":"assistant/chunk","seq":10,"time":1784437195077,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":10,"outputTokens":10}}}} +{"type":"assistant/chunk","seq":11,"time":1784437195077,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} +{"type":"assistant/message","seq":12,"time":1784437195078,"data":{"turn":1,"step":1,"content":[{"type":"tool-call","id":"call_wait","name":"bash","arguments":"{\"command\":\"node -e \\\"setInterval(() => {}, 1000)\\\"\",\"description\":\"Wait until cancellation\"}"},{"type":"tool-call","id":"call_skipped","name":"bash","arguments":"{\"command\":\"printf skipped > skipped.txt\",\"description\":\"Write skipped marker\"}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":10,"outputTokens":10}},"sourceEventSeqs":[4,5,6,7,8,9,10,11],"surfaceOp":"append"} +{"type":"tool/call","seq":13,"time":1784437195078,"data":{"turn":1,"step":1,"callId":"call_wait","name":"bash","arguments":"{\"command\":\"node -e \\\"setInterval(() => {}, 1000)\\\"\",\"description\":\"Wait until cancellation\"}"}} +{"type":"tool/result","seq":14,"time":1784437195089,"data":{"turn":1,"step":1,"callId":"call_wait","content":[{"type":"text","text":"Error: command aborted"}],"isError":true},"sourceEventSeqs":[13],"surfaceOp":"append"} +{"type":"tool/call","seq":15,"time":1784437195089,"data":{"turn":1,"step":1,"callId":"call_skipped","name":"bash","arguments":"{\"command\":\"printf skipped > skipped.txt\",\"description\":\"Write skipped marker\"}"}} +{"type":"tool/result","seq":16,"time":1784437195089,"data":{"turn":1,"step":1,"callId":"call_skipped","content":[{"type":"text","text":"Error: tool call skipped because the step was aborted before execution"}],"isError":true,"error":{"name":"AbortError","code":"ABORTED"}},"sourceEventSeqs":[15],"surfaceOp":"append"} +{"type":"step/end","seq":17,"time":1784437195090,"data":{"turn":1,"step":1}} +{"type":"turn/end","seq":18,"time":1784437195090,"data":{"turn":1,"reason":{"kind":"aborted","reason":"session/cancel"}}} diff --git a/examples/acp-agent/tests/snapshots/cancel-tool-calls/stdout.golden.jsonl b/examples/acp-agent/tests/snapshots/cancel-tool-calls/stdout.golden.jsonl new file mode 100644 index 0000000000..11178b9bc7 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/cancel-tool-calls/stdout.golden.jsonl @@ -0,0 +1,7 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"loadSession":true,"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}","configOptions":[{"id":"model","name":"Model","description":"Sets this session's provider and model.","category":"model","type":"select","currentValue":"[\"deepseek\",\"deepseek-v4-flash\"]","options":[{"value":"[\"deepseek\",\"deepseek-v4-flash\"]","name":"deepseek-v4-flash"},{"value":"[\"deepseek\",\"deepseek-v4-pro\"]","name":"deepseek-v4-pro"}]},{"id":"permission","name":"Permissions","description":"The session permission preset: each choice bundles a sandbox mode and an approval policy.","category":"mode","type":"select","currentValue":"danger-full-access","options":[{"value":"workspace-write","name":"workspace-write","description":"Write inside the workspace and permitted temporary directories; wider retries require approval."},{"value":"danger-full-access","name":"danger-full-access","description":"Full file access without approval prompts."}]}]}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_wait","title":"node -e \"setInterval(() => {}, 1000)\"","kind":"execute","status":"in_progress","rawInput":"node -e \"setInterval(() => {}, 1000)\"","content":[{"type":"content","content":{"type":"text","text":"Wait until cancellation"}}]}}} +{"jsonrpc":"2.0","id":3,"result":{"stopReason":"cancelled"}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_wait","status":"failed","content":[{"type":"content","content":{"type":"text","text":"```console\nError: command aborted\n```"}}]}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_skipped","title":"printf skipped > skipped.txt","kind":"execute","status":"in_progress","rawInput":"printf skipped > skipped.txt","content":[{"type":"content","content":{"type":"text","text":"Write skipped marker"}}]}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_skipped","status":"failed","content":[{"type":"content","content":{"type":"text","text":"```console\nError: tool call skipped because the step was aborted before execution\n```"}}]}}} diff --git a/packages/support/acp-snapshot/src/harness.ts b/packages/support/acp-snapshot/src/harness.ts index e3efca8da9..ca00056d93 100644 --- a/packages/support/acp-snapshot/src/harness.ts +++ b/packages/support/acp-snapshot/src/harness.ts @@ -37,11 +37,10 @@ export type { AgentUnderTest } from './launcher.ts' * (random) session id into a `{{sessionId}}` variable that later steps * reference, since a committed file cannot know the id in advance. * - * `promptAndCancel` sends a prompt WITHOUT awaiting its response, waits until - * the client observes the first streamed `agent_message_chunk` (so the emitted - * frames deterministically precede the cancellation), then cancels the turn — - * the only way to exercise a cancel deterministically (a plain `prompt` step - * awaits the response, which a cancel/hang scenario would block on forever). + * `promptAndCancel` starts a prompt without awaiting completion, waits until + * the client observes the selected update (`agent_message_chunk` by default), + * then cancels and awaits completion. This keeps update/cancel order + * deterministic for fixtures that a plain `prompt` cannot drive. */ export type InputStep = | { op: 'initialize'; terminalOutput?: boolean } @@ -49,7 +48,7 @@ export type InputStep = | { op: 'newSessionExpectError'; additionalDirectories?: string[] } | { op: 'prompt'; text: string } | { op: 'promptExpectError'; text: string } - | { op: 'promptAndCancel'; text: string } + | { op: 'promptAndCancel'; text: string; afterUpdate?: 'agent_message_chunk' | 'tool_call' } | { op: 'cancel' } | { op: 'setConfigOption'; configId: string; value: string } | { op: 'setConfigOptionExpectError'; configId: string; value: string } @@ -342,15 +341,12 @@ async function runStep( case 'promptAndCancel': { const sessionId = getSessionId() if (sessionId === undefined) throw new Error('snapshot-harness: promptAndCancel before newSession') - // Dispatch the prompt WITHOUT awaiting (a hang fixture never resolves on - // its own). To pin frame order deterministically, wait until the client - // has OBSERVED the hang's streamed agent_message_chunk before cancelling — - // so those update frames always precede the cancelled prompt response in - // the transcript (without this, the late chunk and the response race). - // Then cancel and await the prompt, which the bridge settles as - // `cancelled` once the abort propagates. + // Dispatch without awaiting because the fixture does not settle on its + // own. Waiting for the selected update pins it before cancellation and + // the cancelled prompt response in the transcript. const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] }) - await waitForUpdate(u => u.sessionUpdate === 'agent_message_chunk') + const afterUpdate = step.afterUpdate ?? 'agent_message_chunk' + await waitForUpdate(u => u.sessionUpdate === afterUpdate) await client.cancel({ sessionId }) await promptDone return diff --git a/packages/support/acp-snapshot/tests/fixtures/fake-acp-agent.ts b/packages/support/acp-snapshot/tests/fixtures/fake-acp-agent.ts index 5dd5524ed0..dd8ca9ce92 100644 --- a/packages/support/acp-snapshot/tests/fixtures/fake-acp-agent.ts +++ b/packages/support/acp-snapshot/tests/fixtures/fake-acp-agent.ts @@ -33,6 +33,8 @@ interface Behavior { rejectExtraDirs?: boolean /** How `session/prompt` settles: a clean response, a JSON-RPC error, or a hang until `session/cancel`. */ prompt?: 'respond' | 'error' | 'hang-until-cancel' + /** Emit a tool call instead of a message chunk before parking a cancellable prompt. */ + cancelAtToolCall?: boolean /** Before responding to a prompt, send a `session/request_permission` request and echo its outcome as a chunk. */ permissionProbe?: boolean /** Echo the `DSH_SNAPSHOT_*` env the harness set as a chunk (spec-side env-plumbing assertions). */ @@ -126,7 +128,23 @@ async function handlePrompt(id: number | string): Promise { params: { sessionId, update: { sessionUpdate: 'agent_thought_chunk', content: { type: 'text', text: 'mulling' } } }, }) } - chunk('thinking about it') + if (behavior.cancelAtToolCall === true) { + send({ + method: 'session/update', + params: { + sessionId, + update: { + sessionUpdate: 'tool_call', + toolCallId: 'call_fake_1', + title: 'fake tool', + kind: 'execute', + status: 'in_progress', + }, + }, + }) + } else { + chunk('thinking about it') + } if (behavior.echoEnv === true) { chunk(`env:${JSON.stringify({ mode: process.env.DSH_SNAPSHOT, diff --git a/packages/support/acp-snapshot/tests/harness.spec.ts b/packages/support/acp-snapshot/tests/harness.spec.ts index 2b9d6e032f..be03b27c78 100644 --- a/packages/support/acp-snapshot/tests/harness.spec.ts +++ b/packages/support/acp-snapshot/tests/harness.spec.ts @@ -334,6 +334,16 @@ describe('runScenario', () => { expect(result.rawStdout.indexOf('thinking about it')).toBeLessThan(result.rawStdout.indexOf('cancelled')) }) + it('promptAndCancel can wait for a tool call before cancelling', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ prompt: 'hang-until-cancel', cancelAtToolCall: true }) + const result = await runScenario( + { steps: [...boot, { op: 'promptAndCancel', text: 'hang', afterUpdate: 'tool_call' }] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + expect(result.rawStdout).toContain('"sessionUpdate":"tool_call"') + expect(result.rawStdout.indexOf('"sessionUpdate":"tool_call"')).toBeLessThan(result.rawStdout.indexOf('cancelled')) + }) + it('promptExpectError swallows a model-error response as the expected outcome', { timeout: 20_000 }, async () => { const { fixtureFile } = await scenario({ prompt: 'error' }) const result = await runScenario(