fix(acp-snapshot): skip pwshOnly scenarios without pwsh and mount task tools in the pwsh composition
This commit is contained in:
Huanqi Cao
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write docs/testing.md
|
||||
testing.md: 8c16dea5e90ff330992a2d0e38f47abc9da20b26
|
||||
testing.zh.md: f99f02e2a733a94cadeffddc2c5033242bfde59b
|
||||
testing.md: e441b4f467b031aecb595b86664ec8d7aeddf2c7
|
||||
testing.zh.md: f787cf0c3131acc8f4ddaf4da2f6205f5a0f0a48
|
||||
+1
-1
@@ -46,4 +46,4 @@ An e2e assertion re-runs the command or re-reads the file externally; a keyword
|
||||
|
||||
## When a snapshot test is required
|
||||
|
||||
Every non-trivial model-, protocol-, or human-visible change adds or updates a keyless scenario in the same PR through a runnable example's owning snapshot suite. Package tests, e2e assertions, mock/test-only compositions, and PR rationale do not replace the assembled transcript; extend the harness when needed. ACP automation scenarios use `examples/<name>/tests/snapshots/`, a scenario table over the [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) suite factory (`examples/acp-agent` is primary); `examples/headless-agent` owns the `stream-json` snapshot and replay fixtures. Completed interactive-terminal journeys use JSONL-driven scenarios under `apps/cli/tests/snapshots/`; transient presentation uses the package-local semantic matrix, with a PTY case when input, Loader selection, or terminal teardown changes. Browser-rendered web GUI journeys use `apps/web/tests/snapshots/`. New capability seams, lifecycle shapes, or transcript surfaces name every coverage tier at plan time and verify the harness can express it before implementation.
|
||||
Every non-trivial model-, protocol-, or human-visible change adds or updates a keyless scenario in the same PR through a runnable example's owning snapshot suite. Package tests, e2e assertions, mock-only compositions, and PR rationale do not replace the assembled transcript. ACP automation scenarios use `examples/<name>/tests/snapshots/`, a scenario table over the [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) suite factory; `examples/headless-agent` owns the `stream-json` snapshot and replay fixtures. The `pwsh-tool-turn` ACP scenario boots real `pwsh` and skips where it is absent. Completed interactive-terminal journeys use JSONL-driven scenarios under `apps/cli/tests/snapshots/`; transient presentation uses the package-local semantic matrix, with a PTY case when terminal teardown changes. Browser-rendered web GUI journeys use `apps/web/tests/snapshots/`. New capability seams, lifecycle shapes, or transcript surfaces name every coverage tier at plan time and verify the harness expresses it before implementation.
|
||||
+1
-1
@@ -46,4 +46,4 @@ e2e 断言应重新运行命令或从外部重新读取文件；对 agent 自身
|
||||
|
||||
## 何时需要快照测试
|
||||
|
||||
每项非平凡的模型可见、协议可见或人类可见变更，都必须在同一 PR 中，通过可运行示例所属的快照套件添加或更新无密钥场景。包测试、e2e 断言、mock 与仅测试组合、PR 理由都不能取代组装后的 transcript；必要时应扩展 harness。ACP 自动化场景使用 `examples/<name>/tests/snapshots/`，即基于 [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) 套件工厂的场景表（`examples/acp-agent` 为主套件）；`examples/headless-agent` 拥有 `stream-json` 快照与回放 fixture。已完成的交互式终端旅程使用 `apps/cli/tests/snapshots/` 下由 JSONL 驱动的场景；瞬态呈现使用包内语义矩阵，输入、Loader 选择或终端清理发生变化时还要添加 PTY 用例。新的能力 seam、生命周期形态或 transcript 呈现接口在计划阶段就要列出每个覆盖层级，并在实现前验证 harness 能够表达它们。
|
||||
每项非平凡的模型可见、协议可见或人类可见变更，都必须在同一 PR 中，通过可运行示例所属的快照套件添加或更新无密钥场景。包测试、e2e 断言、mock 与仅测试组合、PR 理由都不能取代组装后的 transcript。ACP 自动化场景使用 `examples/<name>/tests/snapshots/`，即基于 [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) 套件工厂的场景表；`examples/headless-agent` 拥有 `stream-json` 快照与回放 fixture。`pwsh-tool-turn` ACP 场景启动真实 `pwsh`，在无 `pwsh` 的主机上跳过。已完成的交互式终端旅程使用 `apps/cli/tests/snapshots/` 下由 JSONL 驱动的场景；瞬态呈现使用包内语义矩阵，终端清理发生变化时还要添加 PTY 用例。新的能力 seam、生命周期形态或 transcript 呈现接口在计划阶段就要列出每个覆盖层级，并在实现前验证 harness 能够表达它们。
|
||||
@@ -1,5 +1,6 @@
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { mkdir, utimes, writeFile } from 'node:fs/promises'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { homedir } from 'node:os'
|
||||
@@ -168,6 +169,9 @@ const SCENARIOS: Scenario[] = [
|
||||
pinsHeader: true,
|
||||
headerClass: 'pwsh',
|
||||
configPath: PWSH_CONFIG,
|
||||
// The composition boots the real pwsh executor; hosts without a `pwsh`
|
||||
// binary skip the run (fixtures stay guarded).
|
||||
pwshOnly: true,
|
||||
},
|
||||
{ name: 'todo-write', hasModelTurn: true, recorded: true },
|
||||
{
|
||||
@@ -431,11 +435,16 @@ const SCENARIOS: Scenario[] = [
|
||||
},
|
||||
]
|
||||
|
||||
// Hosts without a `pwsh` binary skip the pwsh-tool-turn run (its fixtures
|
||||
// stay guarded); the probe follows the executor's own resolution.
|
||||
const hasPwsh = spawnSync('pwsh', ['-NoLogo', '-NoProfile', '-NonInteractive', '-Command', '$true'], { encoding: 'utf8' }).status === 0
|
||||
|
||||
defineAcpSnapshotSuite({
|
||||
agent: AGENT,
|
||||
snapshotsDir: SNAPSHOTS_DIR,
|
||||
scenarios: SCENARIOS,
|
||||
mode: snapshotModeFromEnv(process.env.DSH_SNAPSHOT),
|
||||
hasPwsh,
|
||||
})
|
||||
|
||||
it('packed ACP fixture retains every chunk row kind without changing the logical session', () => {
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
workspaceContext: false
|
||||
skills:
|
||||
enabled: false
|
||||
toolTasks: false
|
||||
# task_output/task_kill stay mounted so background pwsh runs are readable and killable.
|
||||
goals: false
|
||||
# The pwsh tool replaces the bundle's bash tool in this composition.
|
||||
toolBash: false
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
workspaceContext: false
|
||||
skills:
|
||||
enabled: false
|
||||
toolTasks: false
|
||||
# task_output/task_kill stay mounted so background pwsh runs are readable and killable.
|
||||
goals: false
|
||||
# The pwsh tool replaces the bundle's bash tool in this composition.
|
||||
toolBash: false
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
{"type":"assistant/chunk","seq":65,"time":1785655507602,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}
|
||||
{"type":"assistant/message","seq":66,"time":1785655507604,"data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"reasoning","text":"The user wants me to run a PowerShell command that outputs \"PWSH_OK\" and then reply with \"DONE\"."},{"type":"tool-call","id":"call_00_oEhmLGLNsvlumiE0WkXD0511","name":"pwsh","arguments":"{\"command\": \"Write-Output PWSH_OK\", \"description\": \"Output PWSH_OK string\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-pro"},"id":"4d81cc10-507e-49a8-96f4-aa3ae5ad2eb5"},"usage":{"inputTokens":816,"outputTokens":96,"cacheReadTokens":0,"reasoningTokens":25}},"sourceEventSeqs":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65],"surfaceOp":"append"}
|
||||
{"type":"tool/call","seq":67,"time":1785655507605,"data":{"turn":1,"step":1,"callId":"call_00_oEhmLGLNsvlumiE0WkXD0511","name":"pwsh","arguments":"{\"command\": \"Write-Output PWSH_OK\", \"description\": \"Output PWSH_OK string\"}"}}
|
||||
{"type":"tool/result","seq":68,"time":1785655507994,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_00_oEhmLGLNsvlumiE0WkXD0511"},"content":[{"type":"tool-result","toolCallId":"call_00_oEhmLGLNsvlumiE0WkXD0511","content":[{"type":"text","text":"PWSH_OK\n"}],"isError":false}],"role":"user","id":"84a96d55-5bd5-46fc-ab0d-918882c504e2"}},"sourceEventSeqs":[67],"surfaceOp":"append"}
|
||||
{"type":"tool/result","seq":68,"time":1785655507994,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_00_oEhmLGLNsvlumiE0WkXD0511"},"content":[{"type":"tool-result","toolCallId":"call_00_oEhmLGLNsvlumiE0WkXD0511","content":[{"type":"text","text":"PWSH_OK\r\n"}],"isError":false}],"role":"user","id":"84a96d55-5bd5-46fc-ab0d-918882c504e2"}},"sourceEventSeqs":[67],"surfaceOp":"append"}
|
||||
{"type":"step/end","seq":69,"time":1785655507994,"data":{"turn":1,"step":1}}
|
||||
{"type":"step/start","seq":70,"time":1785655508000,"data":{"turn":1,"step":2}}
|
||||
{"type":"assistant/chunk","seq":71,"time":1785655508608,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}}
|
||||
|
||||
@@ -3,3 +3,5 @@ You are an AI agent powered by the DeepSeek Harness SDK.
|
||||
You are a concise snapshot agent working in {{cwd}}.
|
||||
|
||||
Non-zero exits are reported as `[exit code: N]` markers; investigate failures before moving on. On Windows a killed process settles as `[exit code: 1]` without a signal marker; treat a bare exit 1 after an interruption as a termination, not a command failure.
|
||||
|
||||
Track every background task id you start. You are notified in-session when a task finishes — do not busy-poll or sleep on one; keep working on independent steps and do not duplicate a running task's work. Before giving a final answer, collect every still-relevant task with task_output (set wait: true only when you are genuinely blocked on it), and task_kill tasks that stopped mattering.
|
||||
@@ -32,6 +32,58 @@
|
||||
"description"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "task_kill",
|
||||
"description": "Request cancellation of a running background task by task id. Returns immediately; the task settles as killed once its work actually stops.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"task_id": {
|
||||
"type": "string",
|
||||
"description": "Task id returned by the tool that started the background work."
|
||||
},
|
||||
"reason": {
|
||||
"type": "string",
|
||||
"description": "Optional short reason, recorded in the log and forwarded to the task."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"task_id"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "task_list",
|
||||
"description": "List your background tasks (running and finished) with their ids, kinds, and statuses.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "task_output",
|
||||
"description": "Read a background task. Stream tasks return only output since the previous read; final-output tasks return their result after settlement. Every response ends with `[status: ...]`. Reads are non-blocking unless `wait: true`, which waits up to the configured cap.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"task_id": {
|
||||
"type": "string",
|
||||
"description": "Task id returned by the tool that started the background work."
|
||||
},
|
||||
"wait": {
|
||||
"type": "boolean",
|
||||
"description": "Block until the task reaches a terminal status or the timeout expires. A timed-out wait returns [status: running] and leaves the task alive."
|
||||
},
|
||||
"timeout_ms": {
|
||||
"type": "number",
|
||||
"description": "Max wait in milliseconds (only meaningful with wait: true). Defaults to the configured wait timeout; capped by the configured maximum."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"task_id"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"changes": []
|
||||
|
||||
@@ -161,25 +161,37 @@ export interface Scenario {
|
||||
* test is skipped on Windows; its fixtures stay guarded on every platform.
|
||||
*/
|
||||
posixOnly?: boolean
|
||||
/**
|
||||
* Whether the scenario boots a composition that needs a real `pwsh` on PATH
|
||||
* (the pwsh-tool-turn scenario). The run test is skipped when the suite's
|
||||
* {@link SnapshotSuiteOptions.hasPwsh} probe is false; fixtures stay guarded
|
||||
* on every platform.
|
||||
*/
|
||||
pwshOnly?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a scenario's run test is skipped for this mode and host: record mode
|
||||
* skips authored (non-`recorded`) scenarios, and {@link Scenario.posixOnly}
|
||||
* scenarios skip on Windows.
|
||||
* skips authored (non-`recorded`) scenarios, {@link Scenario.posixOnly}
|
||||
* scenarios skip on Windows, and {@link Scenario.pwshOnly} scenarios skip
|
||||
* when the caller's `hasPwsh` probe is false.
|
||||
*
|
||||
* @param scenario The scenario whose run test is being registered.
|
||||
* @param recording Whether the suite runs in record mode.
|
||||
* @param platform The running Node platform, injectable for unit coverage.
|
||||
* @param hasPwsh The caller's pwsh-availability probe; `pwshOnly` scenarios
|
||||
* skip unless it is true.
|
||||
* @returns True when the scenario's run test must not execute.
|
||||
*/
|
||||
export function scenarioSkipped(
|
||||
scenario: Scenario,
|
||||
recording: boolean,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
hasPwsh?: boolean,
|
||||
): boolean {
|
||||
if (recording && !scenario.recorded) return true
|
||||
return scenario.posixOnly === true && platform === 'win32'
|
||||
if (scenario.posixOnly === true && platform === 'win32') return true
|
||||
return scenario.pwshOnly === true && hasPwsh !== true
|
||||
}
|
||||
|
||||
/** One stdout expected output selected for a platform run. */
|
||||
@@ -220,6 +232,11 @@ export interface SnapshotSuiteOptions {
|
||||
* from `$DSH_SNAPSHOT` — env reading stays outside this library.
|
||||
*/
|
||||
mode: 'replay' | 'record' | 'refresh'
|
||||
/**
|
||||
* Whether a real `pwsh` executable is available on this host (the probe the
|
||||
* caller owns; `pwshOnly` scenarios skip when this is not true).
|
||||
*/
|
||||
hasPwsh?: boolean
|
||||
}
|
||||
|
||||
/** One scenario's generated claim on a shared snapshot file. */
|
||||
@@ -973,8 +990,9 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
scenarioSuite('snapshot scenarios', () => {
|
||||
for (const scenario of scenarios) {
|
||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
|
||||
// (sidecar-driven errors/cancel) are never re-recorded. `posixOnly` scenarios skip on Windows.
|
||||
it.skipIf(scenarioSkipped(scenario, RECORDING))(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => {
|
||||
// (sidecar-driven errors/cancel) are never re-recorded. `posixOnly` scenarios skip on Windows;
|
||||
// `pwshOnly` scenarios skip when the caller's `hasPwsh` probe is false.
|
||||
it.skipIf(scenarioSkipped(scenario, RECORDING, process.platform, options.hasPwsh))(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => {
|
||||
const dir = join(snapshotsDir, scenario.name)
|
||||
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
||||
const overrideFile = join(dir, 'replay.override.json')
|
||||
|
||||
@@ -459,6 +459,7 @@ describe('stdoutExpectedVariants', () => {
|
||||
describe('scenarioSkipped', () => {
|
||||
const authored: Scenario = { name: 'authored', hasModelTurn: true, recorded: false }
|
||||
const posix: Scenario = { name: 'posix-cancel', hasModelTurn: true, recorded: false, posixOnly: true }
|
||||
const pwsh: Scenario = { name: 'pwsh-tool', hasModelTurn: true, recorded: false, pwshOnly: true }
|
||||
|
||||
it('skips authored scenarios only while recording', () => {
|
||||
expect(scenarioSkipped(authored, true, 'linux')).toBe(true)
|
||||
@@ -471,6 +472,13 @@ describe('scenarioSkipped', () => {
|
||||
expect(scenarioSkipped(posix, false, 'darwin')).toBe(false)
|
||||
expect(scenarioSkipped(authored, false, 'win32')).toBe(false)
|
||||
})
|
||||
|
||||
it('skips pwshOnly scenarios when the host lacks pwsh, and runs them otherwise', () => {
|
||||
expect(scenarioSkipped(pwsh, false, 'linux', false)).toBe(true)
|
||||
expect(scenarioSkipped(pwsh, false, 'win32', true)).toBe(false)
|
||||
expect(scenarioSkipped(pwsh, false, 'linux', true)).toBe(false)
|
||||
expect(scenarioSkipped(authored, false, 'linux', false)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('fixtureContext', () => {
|
||||
|
||||
Reference in New Issue
Block a user