From ed39ff096fe6424853bc41ce16cc945dba321119 Mon Sep 17 00:00:00 2001 From: Huanqi Cao Date: Sun, 2 Aug 2026 20:07:26 +0800 Subject: [PATCH] test(fixtures): pin the TOOL_ABORTED message and keep the pwsh fixture replayable; restore trimmed testing.md clauses --- docs/testing.i18n.yaml | 4 ++-- docs/testing.md | 2 +- docs/testing.zh.md | 2 +- examples/acp-agent/tests/acp.snapshot.ts | 2 +- .../acp-agent/tests/snapshots/cancel-tool-calls/session.jsonl | 2 +- .../acp-agent/tests/snapshots/pwsh-tool-turn/session.jsonl | 2 +- scripts/doc-budgets.manifest.json | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/testing.i18n.yaml b/docs/testing.i18n.yaml index ba8b076346..3d7569c620 100644 --- a/docs/testing.i18n.yaml +++ b/docs/testing.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/testing.md -testing.md: e441b4f467b031aecb595b86664ec8d7aeddf2c7 -testing.zh.md: f787cf0c3131acc8f4ddaf4da2f6205f5a0f0a48 +testing.md: d460d9ba76eab84b79e8c66454325d03d7c5908e +testing.zh.md: 821e49b6593b924e754d5ed0d588dd1b5b566051 diff --git a/docs/testing.md b/docs/testing.md index e441b4f467..d460d9ba76 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -46,4 +46,4 @@ An e2e assertion re-runs the command or re-reads the file externally; a keyword ## When a snapshot test is required -Every non-trivial model-, protocol-, or human-visible change adds or updates a keyless scenario in the same PR through a runnable example's owning snapshot suite. Package tests, e2e assertions, mock-only compositions, and PR rationale do not replace the assembled transcript. ACP automation scenarios use `examples//tests/snapshots/`, a scenario table over the [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) suite factory; `examples/headless-agent` owns the `stream-json` snapshot and replay fixtures. The `pwsh-tool-turn` ACP scenario boots real `pwsh` and skips where it is absent. Completed interactive-terminal journeys use JSONL-driven scenarios under `apps/cli/tests/snapshots/`; transient presentation uses the package-local semantic matrix, with a PTY case when terminal teardown changes. Browser-rendered web GUI journeys use `apps/web/tests/snapshots/`. New capability seams, lifecycle shapes, or transcript surfaces name every coverage tier at plan time and verify the harness expresses it before implementation. +Every non-trivial model-, protocol-, or human-visible change adds or updates a keyless scenario in the same PR through a runnable example's owning snapshot suite. Package tests, e2e assertions, mock/test-only compositions, and PR rationale do not replace the assembled transcript; extend the harness when needed. ACP automation scenarios use `examples//tests/snapshots/`, a scenario table over the [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) suite factory (`examples/acp-agent` is primary); `examples/headless-agent` owns the `stream-json` snapshot and replay fixtures. The `pwsh-tool-turn` ACP scenario boots real `pwsh` and skips where it is absent. Completed interactive-terminal journeys use JSONL-driven scenarios under `apps/cli/tests/snapshots/`; transient presentation uses the package-local semantic matrix, with a PTY case when input, Loader selection, or terminal teardown changes. Browser-rendered web GUI journeys use `apps/web/tests/snapshots/`. New capability seams, lifecycle shapes, or transcript surfaces name every coverage tier at plan time and verify the harness expresses it before implementation. diff --git a/docs/testing.zh.md b/docs/testing.zh.md index f787cf0c31..821e49b659 100644 --- a/docs/testing.zh.md +++ b/docs/testing.zh.md @@ -46,4 +46,4 @@ e2e 断言应重新运行命令或从外部重新读取文件；对 agent 自身 ## 何时需要快照测试 -每项非平凡的模型可见、协议可见或人类可见变更，都必须在同一 PR 中，通过可运行示例所属的快照套件添加或更新无密钥场景。包测试、e2e 断言、mock 与仅测试组合、PR 理由都不能取代组装后的 transcript。ACP 自动化场景使用 `examples//tests/snapshots/`，即基于 [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) 套件工厂的场景表；`examples/headless-agent` 拥有 `stream-json` 快照与回放 fixture。`pwsh-tool-turn` ACP 场景启动真实 `pwsh`，在无 `pwsh` 的主机上跳过。已完成的交互式终端旅程使用 `apps/cli/tests/snapshots/` 下由 JSONL 驱动的场景；瞬态呈现使用包内语义矩阵，终端清理发生变化时还要添加 PTY 用例。新的能力 seam、生命周期形态或 transcript 呈现接口在计划阶段就要列出每个覆盖层级，并在实现前验证 harness 能够表达它们。 +每项非平凡的模型可见、协议可见或人类可见变更，都必须在同一 PR 中，通过可运行示例所属的快照套件添加或更新无密钥场景。包测试、e2e 断言、mock 与仅测试组合、PR 理由都不能取代组装后的 transcript；必要时应扩展 harness。ACP 自动化场景使用 `examples//tests/snapshots/`，即基于 [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) 套件工厂的场景表（`examples/acp-agent` 为主套件）；`examples/headless-agent` 拥有 `stream-json` 快照与回放 fixture。`pwsh-tool-turn` ACP 场景启动真实 `pwsh`，在无 `pwsh` 的主机上跳过。已完成的交互式终端旅程使用 `apps/cli/tests/snapshots/` 下由 JSONL 驱动的场景；瞬态呈现使用包内语义矩阵，输入、Loader 选择或终端清理发生变化时还要添加 PTY 用例。新的能力 seam、生命周期形态或 transcript 呈现接口在计划阶段就要列出每个覆盖层级，并在实现前验证 harness 能够表达它们。 diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 7e36e854d6..824780ae0f 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -436,7 +436,7 @@ const SCENARIOS: Scenario[] = [ ] // Hosts without a `pwsh` binary skip the pwsh-tool-turn run (its fixtures -// stay guarded); the probe follows the executor's own resolution. +// stay guarded); a bare `pwsh` probe keeps this suite dependency-light. const hasPwsh = spawnSync('pwsh', ['-NoLogo', '-NoProfile', '-NonInteractive', '-Command', '$true'], { encoding: 'utf8' }).status === 0 defineAcpSnapshotSuite({ diff --git a/examples/acp-agent/tests/snapshots/cancel-tool-calls/session.jsonl b/examples/acp-agent/tests/snapshots/cancel-tool-calls/session.jsonl index 5fe449ae48..a26655947d 100644 --- a/examples/acp-agent/tests/snapshots/cancel-tool-calls/session.jsonl +++ b/examples/acp-agent/tests/snapshots/cancel-tool-calls/session.jsonl @@ -16,7 +16,7 @@ {"type":"assistant/chunk","seq":14,"time":1785487611319,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} {"type":"assistant/message","seq":15,"time":1785487611319,"data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"call_wait","name":"bash","arguments":"{\"command\":\"node -e \\\"require('node:fs').writeFileSync('started.txt', 'started'); setInterval(() => {}, 1000)\\\"\",\"description\":\"Wait until cancellation\"}"},{"type":"tool-call","id":"call_skipped","name":"bash","arguments":"{\"command\":\"printf skipped > skipped.txt\",\"description\":\"Write skipped marker\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"bd630f41-b45d-4183-a785-1ff6e7049b62"},"usage":{"inputTokens":10,"outputTokens":10}},"sourceEventSeqs":[7,8,9,10,11,12,13,14],"surfaceOp":"append"} {"type":"tool/call","seq":16,"time":1785487611319,"data":{"turn":1,"step":1,"callId":"call_wait","name":"bash","arguments":"{\"command\":\"node -e \\\"require('node:fs').writeFileSync('started.txt', 'started'); setInterval(() => {}, 1000)\\\"\",\"description\":\"Wait until cancellation\"}"}} -{"type":"tool/result","seq":17,"time":1785487611378,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_wait"},"content":[{"type":"tool-result","toolCallId":"call_wait","content":[{"type":"text","text":"Error: command aborted"}],"isError":true}],"role":"user","id":"252903b2-b4e1-4a33-81d8-d5befefcb27e"}},"sourceEventSeqs":[16],"surfaceOp":"append"} +{"type":"tool/result","seq":17,"time":1785487611378,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_wait"},"content":[{"type":"tool-result","toolCallId":"call_wait","content":[{"type":"text","text":"Error: tool call aborted"}],"isError":true}],"role":"user","id":"252903b2-b4e1-4a33-81d8-d5befefcb27e"}},"sourceEventSeqs":[16],"surfaceOp":"append"} {"type":"tool/call","seq":18,"time":1785487611378,"data":{"turn":1,"step":1,"callId":"call_skipped","name":"bash","arguments":"{\"command\":\"printf skipped > skipped.txt\",\"description\":\"Write skipped marker\"}"}} {"type":"tool/result","seq":19,"time":1785487611378,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_skipped"},"content":[{"type":"tool-result","toolCallId":"call_skipped","content":[{"type":"text","text":"Error: tool call aborted before dispatch"}],"isError":true}],"role":"user","id":"282c5c8c-7296-4545-8014-e6393b351436"},"error":{"name":"AbortError","code":"ABORTED_BEFORE_DISPATCH"}},"sourceEventSeqs":[18],"surfaceOp":"append"} {"type":"step/end","seq":20,"time":1785487611378,"data":{"turn":1,"step":1}} diff --git a/examples/acp-agent/tests/snapshots/pwsh-tool-turn/session.jsonl b/examples/acp-agent/tests/snapshots/pwsh-tool-turn/session.jsonl index 061d3261a3..333948541c 100644 --- a/examples/acp-agent/tests/snapshots/pwsh-tool-turn/session.jsonl +++ b/examples/acp-agent/tests/snapshots/pwsh-tool-turn/session.jsonl @@ -15,7 +15,7 @@ {"type":"assistant/chunk","seq":65,"time":1785655507602,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} {"type":"assistant/message","seq":66,"time":1785655507604,"data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"reasoning","text":"The user wants me to run a PowerShell command that outputs \"PWSH_OK\" and then reply with \"DONE\"."},{"type":"tool-call","id":"call_00_oEhmLGLNsvlumiE0WkXD0511","name":"pwsh","arguments":"{\"command\": \"Write-Output PWSH_OK\", \"description\": \"Output PWSH_OK string\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-pro"},"id":"4d81cc10-507e-49a8-96f4-aa3ae5ad2eb5"},"usage":{"inputTokens":816,"outputTokens":96,"cacheReadTokens":0,"reasoningTokens":25}},"sourceEventSeqs":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65],"surfaceOp":"append"} {"type":"tool/call","seq":67,"time":1785655507605,"data":{"turn":1,"step":1,"callId":"call_00_oEhmLGLNsvlumiE0WkXD0511","name":"pwsh","arguments":"{\"command\": \"Write-Output PWSH_OK\", \"description\": \"Output PWSH_OK string\"}"}} -{"type":"tool/result","seq":68,"time":1785655507994,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_00_oEhmLGLNsvlumiE0WkXD0511"},"content":[{"type":"tool-result","toolCallId":"call_00_oEhmLGLNsvlumiE0WkXD0511","content":[{"type":"text","text":"PWSH_OK\r\n"}],"isError":false}],"role":"user","id":"84a96d55-5bd5-46fc-ab0d-918882c504e2"}},"sourceEventSeqs":[67],"surfaceOp":"append"} +{"type":"tool/result","seq":68,"time":1785655507994,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_00_oEhmLGLNsvlumiE0WkXD0511"},"content":[{"type":"tool-result","toolCallId":"call_00_oEhmLGLNsvlumiE0WkXD0511","content":[{"type":"text","text":"PWSH_OK\n"}],"isError":false}],"role":"user","id":"84a96d55-5bd5-46fc-ab0d-918882c504e2"}},"sourceEventSeqs":[67],"surfaceOp":"append"} {"type":"step/end","seq":69,"time":1785655507994,"data":{"turn":1,"step":1}} {"type":"step/start","seq":70,"time":1785655508000,"data":{"turn":1,"step":2}} {"type":"assistant/chunk","seq":71,"time":1785655508608,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} diff --git a/scripts/doc-budgets.manifest.json b/scripts/doc-budgets.manifest.json index 5c88ab3fc2..4387a894f1 100644 --- a/scripts/doc-budgets.manifest.json +++ b/scripts/doc-budgets.manifest.json @@ -4,7 +4,7 @@ "docs/architecture.md": 2160, "docs/cordis-primer.md": 600, "docs/defensive-patterns.md": 550, - "docs/testing.md": 1100, + "docs/testing.md": 1120, "examples/AGENTS.md": 310, "packages/AGENTS.md": 675, "packages/README.md": 920