diff --git a/docs/testing.i18n.yaml b/docs/testing.i18n.yaml index df4b94e6ce..ba8b076346 100644 --- a/docs/testing.i18n.yaml +++ b/docs/testing.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/testing.md -testing.md: 8c16dea5e90ff330992a2d0e38f47abc9da20b26 -testing.zh.md: f99f02e2a733a94cadeffddc2c5033242bfde59b +testing.md: e441b4f467b031aecb595b86664ec8d7aeddf2c7 +testing.zh.md: f787cf0c3131acc8f4ddaf4da2f6205f5a0f0a48 diff --git a/docs/testing.md b/docs/testing.md index 8c16dea5e9..e441b4f467 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -46,4 +46,4 @@ An e2e assertion re-runs the command or re-reads the file externally; a keyword ## When a snapshot test is required -Every non-trivial model-, protocol-, or human-visible change adds or updates a keyless scenario in the same PR through a runnable example's owning snapshot suite. Package tests, e2e assertions, mock/test-only compositions, and PR rationale do not replace the assembled transcript; extend the harness when needed. ACP automation scenarios use `examples//tests/snapshots/`, a scenario table over the [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) suite factory (`examples/acp-agent` is primary); `examples/headless-agent` owns the `stream-json` snapshot and replay fixtures. Completed interactive-terminal journeys use JSONL-driven scenarios under `apps/cli/tests/snapshots/`; transient presentation uses the package-local semantic matrix, with a PTY case when input, Loader selection, or terminal teardown changes. Browser-rendered web GUI journeys use `apps/web/tests/snapshots/`. New capability seams, lifecycle shapes, or transcript surfaces name every coverage tier at plan time and verify the harness can express it before implementation. +Every non-trivial model-, protocol-, or human-visible change adds or updates a keyless scenario in the same PR through a runnable example's owning snapshot suite. Package tests, e2e assertions, mock-only compositions, and PR rationale do not replace the assembled transcript. ACP automation scenarios use `examples//tests/snapshots/`, a scenario table over the [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) suite factory; `examples/headless-agent` owns the `stream-json` snapshot and replay fixtures. The `pwsh-tool-turn` ACP scenario boots real `pwsh` and skips where it is absent. Completed interactive-terminal journeys use JSONL-driven scenarios under `apps/cli/tests/snapshots/`; transient presentation uses the package-local semantic matrix, with a PTY case when terminal teardown changes. Browser-rendered web GUI journeys use `apps/web/tests/snapshots/`. New capability seams, lifecycle shapes, or transcript surfaces name every coverage tier at plan time and verify the harness expresses it before implementation. diff --git a/docs/testing.zh.md b/docs/testing.zh.md index f99f02e2a7..f787cf0c31 100644 --- a/docs/testing.zh.md +++ b/docs/testing.zh.md @@ -46,4 +46,4 @@ e2e 断言应重新运行命令或从外部重新读取文件;对 agent 自身 ## 何时需要快照测试 -每项非平凡的模型可见、协议可见或人类可见变更,都必须在同一 PR 中,通过可运行示例所属的快照套件添加或更新无密钥场景。包测试、e2e 断言、mock 与仅测试组合、PR 理由都不能取代组装后的 transcript;必要时应扩展 harness。ACP 自动化场景使用 `examples//tests/snapshots/`,即基于 [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) 套件工厂的场景表(`examples/acp-agent` 为主套件);`examples/headless-agent` 拥有 `stream-json` 快照与回放 fixture。已完成的交互式终端旅程使用 `apps/cli/tests/snapshots/` 下由 JSONL 驱动的场景;瞬态呈现使用包内语义矩阵,输入、Loader 选择或终端清理发生变化时还要添加 PTY 用例。新的能力 seam、生命周期形态或 transcript 呈现接口在计划阶段就要列出每个覆盖层级,并在实现前验证 harness 能够表达它们。 +每项非平凡的模型可见、协议可见或人类可见变更,都必须在同一 PR 中,通过可运行示例所属的快照套件添加或更新无密钥场景。包测试、e2e 断言、mock 与仅测试组合、PR 理由都不能取代组装后的 transcript。ACP 自动化场景使用 `examples//tests/snapshots/`,即基于 [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) 套件工厂的场景表;`examples/headless-agent` 拥有 `stream-json` 快照与回放 fixture。`pwsh-tool-turn` ACP 场景启动真实 `pwsh`,在无 `pwsh` 的主机上跳过。已完成的交互式终端旅程使用 `apps/cli/tests/snapshots/` 下由 JSONL 驱动的场景;瞬态呈现使用包内语义矩阵,终端清理发生变化时还要添加 PTY 用例。新的能力 seam、生命周期形态或 transcript 呈现接口在计划阶段就要列出每个覆盖层级,并在实现前验证 harness 能够表达它们。 diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 41e34acc42..7e36e854d6 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -1,5 +1,6 @@ import { fileURLToPath } from 'node:url' import { readFileSync } from 'node:fs' +import { spawnSync } from 'node:child_process' import { mkdir, utimes, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { homedir } from 'node:os' @@ -168,6 +169,9 @@ const SCENARIOS: Scenario[] = [ pinsHeader: true, headerClass: 'pwsh', configPath: PWSH_CONFIG, + // The composition boots the real pwsh executor; hosts without a `pwsh` + // binary skip the run (fixtures stay guarded). + pwshOnly: true, }, { name: 'todo-write', hasModelTurn: true, recorded: true }, { @@ -431,11 +435,16 @@ const SCENARIOS: Scenario[] = [ }, ] +// Hosts without a `pwsh` binary skip the pwsh-tool-turn run (its fixtures +// stay guarded); the probe follows the executor's own resolution. +const hasPwsh = spawnSync('pwsh', ['-NoLogo', '-NoProfile', '-NonInteractive', '-Command', '$true'], { encoding: 'utf8' }).status === 0 + defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: SNAPSHOTS_DIR, scenarios: SCENARIOS, mode: snapshotModeFromEnv(process.env.DSH_SNAPSHOT), + hasPwsh, }) it('packed ACP fixture retains every chunk row kind without changing the logical session', () => { diff --git a/examples/acp-agent/tests/pwsh.cordis.snapshot.yml b/examples/acp-agent/tests/pwsh.cordis.snapshot.yml index c52dbbaf57..91fdeabf53 100644 --- a/examples/acp-agent/tests/pwsh.cordis.snapshot.yml +++ b/examples/acp-agent/tests/pwsh.cordis.snapshot.yml @@ -27,7 +27,7 @@ workspaceContext: false skills: enabled: false - toolTasks: false +# task_output/task_kill stay mounted so background pwsh runs are readable and killable. goals: false # The pwsh tool replaces the bundle's bash tool in this composition. toolBash: false diff --git a/examples/acp-agent/tests/pwsh.cordis.yml b/examples/acp-agent/tests/pwsh.cordis.yml index 46a595d7ff..cb8305c7d9 100644 --- a/examples/acp-agent/tests/pwsh.cordis.yml +++ b/examples/acp-agent/tests/pwsh.cordis.yml @@ -26,7 +26,7 @@ workspaceContext: false skills: enabled: false - toolTasks: false +# task_output/task_kill stay mounted so background pwsh runs are readable and killable. goals: false # The pwsh tool replaces the bundle's bash tool in this composition. toolBash: false diff --git a/examples/acp-agent/tests/snapshots/pwsh-tool-turn/session.jsonl b/examples/acp-agent/tests/snapshots/pwsh-tool-turn/session.jsonl index 333948541c..061d3261a3 100644 --- a/examples/acp-agent/tests/snapshots/pwsh-tool-turn/session.jsonl +++ b/examples/acp-agent/tests/snapshots/pwsh-tool-turn/session.jsonl @@ -15,7 +15,7 @@ {"type":"assistant/chunk","seq":65,"time":1785655507602,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} {"type":"assistant/message","seq":66,"time":1785655507604,"data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"reasoning","text":"The user wants me to run a PowerShell command that outputs \"PWSH_OK\" and then reply with \"DONE\"."},{"type":"tool-call","id":"call_00_oEhmLGLNsvlumiE0WkXD0511","name":"pwsh","arguments":"{\"command\": \"Write-Output PWSH_OK\", \"description\": \"Output PWSH_OK string\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-pro"},"id":"4d81cc10-507e-49a8-96f4-aa3ae5ad2eb5"},"usage":{"inputTokens":816,"outputTokens":96,"cacheReadTokens":0,"reasoningTokens":25}},"sourceEventSeqs":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65],"surfaceOp":"append"} {"type":"tool/call","seq":67,"time":1785655507605,"data":{"turn":1,"step":1,"callId":"call_00_oEhmLGLNsvlumiE0WkXD0511","name":"pwsh","arguments":"{\"command\": \"Write-Output PWSH_OK\", \"description\": \"Output PWSH_OK string\"}"}} -{"type":"tool/result","seq":68,"time":1785655507994,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_00_oEhmLGLNsvlumiE0WkXD0511"},"content":[{"type":"tool-result","toolCallId":"call_00_oEhmLGLNsvlumiE0WkXD0511","content":[{"type":"text","text":"PWSH_OK\n"}],"isError":false}],"role":"user","id":"84a96d55-5bd5-46fc-ab0d-918882c504e2"}},"sourceEventSeqs":[67],"surfaceOp":"append"} +{"type":"tool/result","seq":68,"time":1785655507994,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_00_oEhmLGLNsvlumiE0WkXD0511"},"content":[{"type":"tool-result","toolCallId":"call_00_oEhmLGLNsvlumiE0WkXD0511","content":[{"type":"text","text":"PWSH_OK\r\n"}],"isError":false}],"role":"user","id":"84a96d55-5bd5-46fc-ab0d-918882c504e2"}},"sourceEventSeqs":[67],"surfaceOp":"append"} {"type":"step/end","seq":69,"time":1785655507994,"data":{"turn":1,"step":1}} {"type":"step/start","seq":70,"time":1785655508000,"data":{"turn":1,"step":2}} {"type":"assistant/chunk","seq":71,"time":1785655508608,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} diff --git a/examples/acp-agent/tests/snapshots/pwsh-tool-turn/system-prompt.expected.md b/examples/acp-agent/tests/snapshots/pwsh-tool-turn/system-prompt.expected.md index 9b51477daf..f354648c41 100644 --- a/examples/acp-agent/tests/snapshots/pwsh-tool-turn/system-prompt.expected.md +++ b/examples/acp-agent/tests/snapshots/pwsh-tool-turn/system-prompt.expected.md @@ -3,3 +3,5 @@ You are an AI agent powered by the DeepSeek Harness SDK. You are a concise snapshot agent working in {{cwd}}. Non-zero exits are reported as `[exit code: N]` markers; investigate failures before moving on. On Windows a killed process settles as `[exit code: 1]` without a signal marker; treat a bare exit 1 after an interruption as a termination, not a command failure. + +Track every background task id you start. You are notified in-session when a task finishes — do not busy-poll or sleep on one; keep working on independent steps and do not duplicate a running task's work. Before giving a final answer, collect every still-relevant task with task_output (set wait: true only when you are genuinely blocked on it), and task_kill tasks that stopped mattering. diff --git a/examples/acp-agent/tests/snapshots/pwsh-tool-turn/tool-schemas.expected.json b/examples/acp-agent/tests/snapshots/pwsh-tool-turn/tool-schemas.expected.json index 6f6d3fa729..611de722e3 100644 --- a/examples/acp-agent/tests/snapshots/pwsh-tool-turn/tool-schemas.expected.json +++ b/examples/acp-agent/tests/snapshots/pwsh-tool-turn/tool-schemas.expected.json @@ -32,6 +32,58 @@ "description" ] } + }, + { + "name": "task_kill", + "description": "Request cancellation of a running background task by task id. Returns immediately; the task settles as killed once its work actually stops.", + "parameters": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "description": "Task id returned by the tool that started the background work." + }, + "reason": { + "type": "string", + "description": "Optional short reason, recorded in the log and forwarded to the task." + } + }, + "required": [ + "task_id" + ] + } + }, + { + "name": "task_list", + "description": "List your background tasks (running and finished) with their ids, kinds, and statuses.", + "parameters": { + "type": "object", + "properties": {} + } + }, + { + "name": "task_output", + "description": "Read a background task. Stream tasks return only output since the previous read; final-output tasks return their result after settlement. Every response ends with `[status: ...]`. Reads are non-blocking unless `wait: true`, which waits up to the configured cap.", + "parameters": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "description": "Task id returned by the tool that started the background work." + }, + "wait": { + "type": "boolean", + "description": "Block until the task reaches a terminal status or the timeout expires. A timed-out wait returns [status: running] and leaves the task alive." + }, + "timeout_ms": { + "type": "number", + "description": "Max wait in milliseconds (only meaningful with wait: true). Defaults to the configured wait timeout; capped by the configured maximum." + } + }, + "required": [ + "task_id" + ] + } } ], "changes": [] diff --git a/packages/support/acp-snapshot/src/suite.ts b/packages/support/acp-snapshot/src/suite.ts index c8434bb8c9..f9e97dd45b 100644 --- a/packages/support/acp-snapshot/src/suite.ts +++ b/packages/support/acp-snapshot/src/suite.ts @@ -161,25 +161,37 @@ export interface Scenario { * test is skipped on Windows; its fixtures stay guarded on every platform. */ posixOnly?: boolean + /** + * Whether the scenario boots a composition that needs a real `pwsh` on PATH + * (the pwsh-tool-turn scenario). The run test is skipped when the suite's + * {@link SnapshotSuiteOptions.hasPwsh} probe is false; fixtures stay guarded + * on every platform. + */ + pwshOnly?: boolean } /** * Whether a scenario's run test is skipped for this mode and host: record mode - * skips authored (non-`recorded`) scenarios, and {@link Scenario.posixOnly} - * scenarios skip on Windows. + * skips authored (non-`recorded`) scenarios, {@link Scenario.posixOnly} + * scenarios skip on Windows, and {@link Scenario.pwshOnly} scenarios skip + * when the caller's `hasPwsh` probe is false. * * @param scenario The scenario whose run test is being registered. * @param recording Whether the suite runs in record mode. * @param platform The running Node platform, injectable for unit coverage. + * @param hasPwsh The caller's pwsh-availability probe; `pwshOnly` scenarios + * skip unless it is true. * @returns True when the scenario's run test must not execute. */ export function scenarioSkipped( scenario: Scenario, recording: boolean, platform: NodeJS.Platform = process.platform, + hasPwsh?: boolean, ): boolean { if (recording && !scenario.recorded) return true - return scenario.posixOnly === true && platform === 'win32' + if (scenario.posixOnly === true && platform === 'win32') return true + return scenario.pwshOnly === true && hasPwsh !== true } /** One stdout expected output selected for a platform run. */ @@ -220,6 +232,11 @@ export interface SnapshotSuiteOptions { * from `$DSH_SNAPSHOT` — env reading stays outside this library. */ mode: 'replay' | 'record' | 'refresh' + /** + * Whether a real `pwsh` executable is available on this host (the probe the + * caller owns; `pwshOnly` scenarios skip when this is not true). + */ + hasPwsh?: boolean } /** One scenario's generated claim on a shared snapshot file. */ @@ -973,8 +990,9 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { scenarioSuite('snapshot scenarios', () => { for (const scenario of scenarios) { // In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones - // (sidecar-driven errors/cancel) are never re-recorded. `posixOnly` scenarios skip on Windows. - it.skipIf(scenarioSkipped(scenario, RECORDING))(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => { + // (sidecar-driven errors/cancel) are never re-recorded. `posixOnly` scenarios skip on Windows; + // `pwshOnly` scenarios skip when the caller's `hasPwsh` probe is false. + it.skipIf(scenarioSkipped(scenario, RECORDING, process.platform, options.hasPwsh))(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => { const dir = join(snapshotsDir, scenario.name) const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript const overrideFile = join(dir, 'replay.override.json') diff --git a/packages/support/acp-snapshot/tests/suite.spec.ts b/packages/support/acp-snapshot/tests/suite.spec.ts index 6f16844595..d8cb177a16 100644 --- a/packages/support/acp-snapshot/tests/suite.spec.ts +++ b/packages/support/acp-snapshot/tests/suite.spec.ts @@ -459,6 +459,7 @@ describe('stdoutExpectedVariants', () => { describe('scenarioSkipped', () => { const authored: Scenario = { name: 'authored', hasModelTurn: true, recorded: false } const posix: Scenario = { name: 'posix-cancel', hasModelTurn: true, recorded: false, posixOnly: true } + const pwsh: Scenario = { name: 'pwsh-tool', hasModelTurn: true, recorded: false, pwshOnly: true } it('skips authored scenarios only while recording', () => { expect(scenarioSkipped(authored, true, 'linux')).toBe(true) @@ -471,6 +472,13 @@ describe('scenarioSkipped', () => { expect(scenarioSkipped(posix, false, 'darwin')).toBe(false) expect(scenarioSkipped(authored, false, 'win32')).toBe(false) }) + + it('skips pwshOnly scenarios when the host lacks pwsh, and runs them otherwise', () => { + expect(scenarioSkipped(pwsh, false, 'linux', false)).toBe(true) + expect(scenarioSkipped(pwsh, false, 'win32', true)).toBe(false) + expect(scenarioSkipped(pwsh, false, 'linux', true)).toBe(false) + expect(scenarioSkipped(authored, false, 'linux', false)).toBe(false) + }) }) describe('fixtureContext', () => {