Merge branch 'master' into all-checks-passed-gate
This commit is contained in:
Tianyi Cui
GitHub
@@ -810,6 +810,7 @@ Abstract service classes — a deployment loads a concrete implementation packag
|
||||
|
||||
Imported as libraries by other packages; a `cordis.yml` cannot load them.
|
||||
|
||||
- `@deepseek-ai/dsh-acp-snapshot` ([`packages/support/acp-snapshot/src/index.ts`](../packages/support/acp-snapshot/src/index.ts))
|
||||
- `@deepseek-ai/dsh-app-boot` ([`packages/ui/app-boot/src/index.ts`](../packages/ui/app-boot/src/index.ts))
|
||||
- `@deepseek-ai/dsh-brand` ([`packages/util/brand/src/index.ts`](../packages/util/brand/src/index.ts))
|
||||
- `@deepseek-ai/dsh-hook-protocol` ([`packages/hooks/hook-protocol/src/index.ts`](../packages/hooks/hook-protocol/src/index.ts))
|
||||
|
||||
@@ -68,6 +68,7 @@ flowchart TD
|
||||
pkg_session_persistence_sqlite["session-persistence-sqlite"]
|
||||
end
|
||||
subgraph group_support["packages/support"]
|
||||
pkg_acp_snapshot["acp-snapshot"]
|
||||
pkg_invariants["invariants"]
|
||||
pkg_llm_replay["llm-replay"]
|
||||
pkg_subagent_mock["subagent-mock"]
|
||||
@@ -207,6 +208,7 @@ flowchart TD
|
||||
| Package | Group | Depends on |
|
||||
| --- | --- | --- |
|
||||
| [`brand`](../packages/util/brand) | `util` | — |
|
||||
| [`acp-snapshot`](../packages/support/acp-snapshot) | `support` | — |
|
||||
| [`app-boot`](../packages/ui/app-boot) | `ui` | — |
|
||||
| [`llm`](../packages/llm/llm) | `llm` | [`brand`](../packages/util/brand) |
|
||||
| [`bash`](../packages/bash/bash) | `bash` | [`brand`](../packages/util/brand) |
|
||||
|
||||
@@ -165,6 +165,7 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand;
|
||||
| [Hook snapshot matrix — end-to-end goldens for both bridges](implemented/testing/2026-07-04-hook-snapshot-matrix.md) | 2026-07-04 |
|
||||
| [Single-source the acp-agent replay config](implemented/testing/2026-07-04-single-source-acp-replay-config.md) | 2026-07-04 |
|
||||
| [Pin request-header content in one snapshot scenario](implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md) | 2026-07-06 |
|
||||
| [Extract the ACP snapshot suite into a support package](implemented/testing/2026-07-08-shared-acp-snapshot-package.md) | 2026-07-08 |
|
||||
|
||||
## Rejected
|
||||
|
||||
|
||||
@@ -32,4 +32,4 @@ Reviewers lose one artifact name that made the expected persisted log visually s
|
||||
|
||||
## Implementation note
|
||||
|
||||
The comparison normalizes BOTH sides, but each against its OWN volatile values, not a shared context. A raw harvested `session.jsonl` bakes in the recording run's session id, cwd, and timestamps; the replay run produces fresh ones. `normalizeSessionLog` scrubs cwd by exact string match, so normalizing the fixture against the *replay* run's cwd would leave the recorded cwd in the header unscrubbed and the compare would fail. The harness therefore derives the fixture's normalize context from its OWN header line (`{ type:'session', id, cwd }`) — `fixtureContext()` in `acp.snapshot.ts` — so both sides scrub to the same `{{sessionId}}`/`{{cwd}}` tokens. An authored fixture copied from the old golden already carries the normalized header (`id:'{{sessionId}}'`, `cwd:'{{cwd}}'`), which yields those tokens as the volatile values and scrubs idempotently. The session-log side uses a plain normalized-string `toEqual`, NOT `toMatchFileSnapshot`, so a run never overwrites the fixture.
|
||||
The comparison normalizes BOTH sides, but each against its OWN volatile values, not a shared context. A raw harvested `session.jsonl` bakes in the recording run's session id, cwd, and timestamps; the replay run produces fresh ones. `normalizeSessionLog` scrubs cwd by exact string match, so normalizing the fixture against the *replay* run's cwd would leave the recorded cwd in the header unscrubbed and the compare would fail. The harness therefore derives the fixture's normalize context from its OWN header line (`{ type:'session', id, cwd }`) — `fixtureContext()` in `dsh-acp-snapshot`'s suite module — so both sides scrub to the same `{{sessionId}}`/`{{cwd}}` tokens. An authored fixture copied from the old golden already carries the normalized header (`id:'{{sessionId}}'`, `cwd:'{{cwd}}'`), which yields those tokens as the volatile values and scrubs idempotently. The session-log side uses a plain normalized-string `toEqual`, NOT `toMatchFileSnapshot`, so a run never overwrites the fixture.
|
||||
+1
-1
@@ -8,7 +8,7 @@ Every model-driving ACP snapshot fixture (`session.jsonl`) embedded the full com
|
||||
|
||||
## Decision
|
||||
|
||||
Exactly one scenario — `text-turn`, flagged `pinsHeader` in `acp.snapshot.ts` — commits and compares the full request-header content. Every other fixture stores and compares that content as stable tokens via the pure normalizer `scrubRequestHeaders` in `snapshot-normalize.ts`: a `request/header` event's `header.system` becomes `"{{system}}"` and `header.tools` becomes `"{{tools}}"`; a `request/header-delta` keeps its structural facts — the system delta's `keepStart`/`keepEnd` line positions with one `{{system}}` token per inserted line, the tools delta's added/removed/changed tool names — and tokenizes only the bulk (prompt text, schema bodies), so two different deltas still compare different. The scrub is composed in front of `normalizeSessionLog` on BOTH sides of a non-pinning scenario's log compare and applied to the harvested logs record mode writes, so a re-record cannot smuggle the content back. Absent fields stay absent — WHETHER a header carried a prompt or tools is behavior and stays visible — and `config`/`reason` stay verbatim: a model swap churns every fixture by design (it invalidates the recorded responses), while a prompt or schema edit churns none of them (replay derives model behavior exclusively from `assistant/chunk` events and never reads header content — see `dsh-llm-replay`).
|
||||
Exactly one scenario — `text-turn`, flagged `pinsHeader` in the `acp.snapshot.ts` scenario table — commits and compares the full request-header content; the pin mechanics live in [`dsh-acp-snapshot`](../../../../packages/support/acp-snapshot/README.md), whose suite factory enforces one pin per consuming suite. Every other fixture stores and compares that content as stable tokens via the pure normalizer `scrubRequestHeaders` in that package's `normalize.ts`: a `request/header` event's `header.system` becomes `"{{system}}"` and `header.tools` becomes `"{{tools}}"`; a `request/header-delta` keeps its structural facts — the system delta's `keepStart`/`keepEnd` line positions with one `{{system}}` token per inserted line, the tools delta's added/removed/changed tool names — and tokenizes only the bulk (prompt text, schema bodies), so two different deltas still compare different. The scrub is composed in front of `normalizeSessionLog` on BOTH sides of a non-pinning scenario's log compare and applied to the harvested logs record mode writes, so a re-record cannot smuggle the content back. Absent fields stay absent — WHETHER a header carried a prompt or tools is behavior and stays visible — and `config`/`reason` stay verbatim: a model swap churns every fixture by design (it invalidates the recorded responses), while a prompt or schema edit churns none of them (replay derives model behavior exclusively from `assistant/chunk` events and never reads header content — see `dsh-llm-replay`).
|
||||
|
||||
A system-prompt or tool-schema change therefore lands as exactly one committed-fixture diff — the pinned `text-turn` header line — updated by hand or by re-recording that one scenario (`pnpm run test:snapshot:record` with `-t text-turn`).
|
||||
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# RFC: Extract the ACP snapshot suite into a support package
|
||||
|
||||
Status: implemented
|
||||
|
||||
## Problem
|
||||
|
||||
The ACP snapshot tier ([snapshot RFC](2026-06-19-acp-snapshot-tests.md)) was built from three modules living inside one example's test directory: `snapshot-harness.ts` (boot the real bin subprocess, drive it over ACP JSON-RPC, harvest the persisted logs), `snapshot-normalize.ts` (the pure golden normalizers), and the ~150-line scenario body plus fixture guards in `acp.snapshot.ts` (record/replay modes, the stdout-golden and log compares, the pinned-header uniformity guard, the orphan/required-file/single-pin meta-tests).
|
||||
|
||||
A second ACP example wanting snapshot coverage — the sandbox/approval composition is the immediate consumer — could only copy those modules, forking exactly the logic that must not drift: record write-back, header scrubbing, child-session harvest ordering. The spawn/client glue was already triplicated across `acp.e2e.ts`, `hooks.e2e.ts`, and the harness (`TODO(acp-test-harness)`). Location also decided test rigor: the per-file 100% coverage gate measures `packages/*/*/src` only, so none of this machinery was measured — the same gap that had moved `dsh-llm-replay` out of `examples/` into [packages/support](../../../../packages/support/README.md). And the harness's ACP client hardcoded `requestPermission → cancelled`, so an approval round-trip — the headline behavior of the sandbox composition — could not be expressed at the snapshot tier at all.
|
||||
|
||||
## Decision
|
||||
|
||||
The machinery lives in [`packages/support/acp-snapshot`](../../../../packages/support/acp-snapshot/README.md) (`@deepseek-ai/dsh-acp-snapshot`); an example's `*.snapshot.ts` is its scenario table, its agent paths, and one factory call, over its own `snapshots/` fixtures and `cordis.snapshot.yml` overlay ([single-source replay config](2026-07-04-single-source-acp-replay-config.md)). Reading `DSH_SNAPSHOT` stays at that edge — the library takes a resolved `mode`.
|
||||
|
||||
**`src/harness.ts`** — `runScenario` and the input-script/result types, parameterized by an `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath`; absolute paths the consuming suite resolves from its own `import.meta.url`). The client's `session/request_permission` handler consumes an optional `InputScript.permissionAnswers` FIFO queue, each entry selecting by option **kind** (ids are agent-issued randoms a committed script cannot know; kinds are the ACP-stable vocabulary, mapped to the offered `optionId` at answer time); an absent or exhausted queue answers `cancelled`, and a kind the request never offered rejects the run — the agent itself is answered `cancelled`, so the scenario bug fails the harness rather than being absorbed as an agent-side denial. This is what lets an approval suite drive allow/reject round-trips deterministically from `input.json`.
|
||||
|
||||
**`src/normalize.ts`** — the pure normalizers, hook-free by policy: when a future event carries a new volatile field (an approval duration, say), the shared normalizer learns it in the same change, keeping one home for what "normalized" means rather than per-suite scrub extensions.
|
||||
|
||||
**`src/suite.ts`** — the `Scenario` type and `defineAcpSnapshotSuite(options)`, registering the per-scenario compares, record-mode fixture write-back, the header pin with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin, non-pinning fixtures are `scrubRequestHeaders` fixed points). The pinned-header contract ([pinned-header RFC](2026-07-06-pin-request-header-content-in-one-scenario.md)) is per-suite: each suite flags exactly one `pinsHeader` scenario (the factory throws on zero, a meta-test rejects more than one; WHICH scenario pins is the table's reviewable choice), and the uniformity guard compares only that suite's sessions. The pure helpers (`childFixturePaths`, `fixtureContext`, `normalizedHeaders`, `headerDeltaCount`) are exported for direct unit coverage.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- **Copy the modules into each example** — the fork this RFC exists to prevent: the record/guard logic is exactly the code that must stay byte-identical across suites, and examples are outside the coverage gate, so each copy is also unmeasured.
|
||||
- **A shared module directory under `examples/`** — keeps the code outside the coverage gate and forces relative imports across example boundaries, against the package-name import convention; `examples/` leaves stay thin by design.
|
||||
- **A `/testing` subpath export of `dsh-acp-agent`** — couples test infrastructure into a product package's surface and dependency set; `packages/support/` exists precisely for real-but-lower-compatibility dev/test packages, with `dsh-llm-replay` as the precedent this package completes.
|
||||
- **Export raw test-body functions instead of a suite factory** — each example would re-own the `describe`/`it` skeleton (~80 lines of registration boilerplate per suite) for no flexibility gain; the factory keeps consumers to a scenario table plus one call, and the exported pure helpers preserve unit-testability inside the factory design.
|
||||
- **An injectable ACP `Client` factory instead of declarative `permissionAnswers`** — maximally flexible, but it leaks SDK client construction to every consumer and reopens per-example drift in exactly the layer being unified; a declarative queue keeps `input.json` the single scripting surface and stays golden-normalizable.
|
||||
- **Generalize beyond ACP (a transport-agnostic snapshot harness)** — no second transport exists; the harness is ACP-shaped end to end (SDK client, JSON-RPC frames, `session/update` waiters), and a speculative abstraction would be a seam split ahead of any consumer.
|
||||
|
||||
## Testing
|
||||
|
||||
Extraction parity was proven mechanically: after the move, `pnpm run test:snapshot` matched the base commit's result with zero byte changes under `examples/acp-agent/tests/snapshots/`. The package's `src/` holds per-file 100% statements/branches/functions/lines under the gating unit run, driven through the REAL spawn path by a scripted fake ACP bin (`tests/fixtures/fake-acp-agent.ts`, behavior scripted per scenario via a `behavior.json` beside the fixture): `harness.spec.ts` covers every step op, both expect-error arms, the permission queue (selection, fallback, impossible-click), env forwarding, workspace seeding, and the harvest ordering/noise/fallback branches; `suite.spec.ts` runs the factory for real at collection time — a replay suite over committed synthetic fixtures and a record suite over a temp copy (write-back never touches the committed tree; `ACP_SNAPSHOT_SPEC_BOOTSTRAP=1` re-bootstraps it) — plus direct cases for the pure helpers. Two structurally unreachable guards carry reasoned `v8 ignore` comments. The fake bin substitutes the `session/new` cwd, not `process.cwd()`, into scripted logs, matching what the real bin's header carries (darwin realpaths `/var/folders/…` to `/private/var/folders/…`).
|
||||
|
||||
## Consequences
|
||||
|
||||
A new example gets the whole snapshot tier from a scenario table plus fixtures — the sandbox branch merges master down and adds its own suite (own pin scenario, own overlay, fixtures via `test:snapshot:record`, approvals via `permissionAnswers`). The costs: `suite.ts` imports vitest, so the package is importable only inside a vitest run — a shape no other package has, stated in its README; each suite pins its own ~8 KB header fixture (a genuinely distinct composition deserves its own pin; an identical one would be caught by that suite's uniformity guard); and the e2e launcher duplication remains (`TODO(acp-test-harness)`) — the harness is the extraction target when that migration lands.
|
||||
+1
-1
@@ -30,4 +30,4 @@ An e2e assertion re-runs the command or re-reads the file externally; a keyword
|
||||
|
||||
## When a snapshot test is required
|
||||
|
||||
Any change affecting the editor-facing transcript or end-to-end agent UX — the ACP bridge, the loop's observable output, tool presentation — adds or updates a scenario under `examples/acp-agent/tests/snapshots/` (or states in the PR why none applies). New capability seams, lifecycle shapes, or transcript surfaces name their coverage at every tier at plan time and verify the harness can express it — a harness gap is scheduled work, not a mid-build surprise.
|
||||
Any change affecting the editor-facing transcript or end-to-end agent UX — the ACP bridge, the loop's observable output, tool presentation — adds or updates a scenario in the owning example's snapshot suite (`examples/<name>/tests/snapshots/`, a scenario table over the [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) suite factory; `examples/acp-agent` is the primary suite), or states in the PR why none applies. New capability seams, lifecycle shapes, or transcript surfaces name their coverage at every tier at plan time and verify the harness can express it — a harness gap is scheduled work, not a mid-build surprise.
|
||||
@@ -57,8 +57,8 @@ interface Spawned {
|
||||
}
|
||||
|
||||
// TODO(acp-test-harness): this subprocess/client boot glue is duplicated with
|
||||
// hooks.e2e.ts and partly with snapshot-harness.ts. Extract one shared ACP test
|
||||
// launcher before the TSX/env/permission-stub details drift again.
|
||||
// hooks.e2e.ts and partly with dsh-acp-snapshot's harness. Migrate both e2e
|
||||
// files onto that launcher before the TSX/env/permission-stub details drift.
|
||||
function spawnAcpAgent(cwd: string, env: NodeJS.ProcessEnv = process.env): Spawned {
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
|
||||
@@ -1,86 +1,24 @@
|
||||
import { readFile, readdir, writeFile } from 'node:fs/promises'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { type HarvestedLog, type InputScript, runScenario } from './snapshot-harness.ts'
|
||||
import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from './snapshot-normalize.ts'
|
||||
import { defineAcpSnapshotSuite, type Scenario } from '@deepseek-ai/dsh-acp-snapshot'
|
||||
|
||||
/**
|
||||
* ACP snapshot tests (REPLAY by default, keyless). Each scenario under
|
||||
* `snapshots/<name>/` ships an `input.json` (the client stdin script) and a
|
||||
* `session.jsonl` fixture; replay boots the real acp-agent subprocess, drives
|
||||
* it, and diffs the normalized stdout transcript against the committed
|
||||
* `stdout.golden.jsonl`. For model scenarios it ALSO checks the re-persisted
|
||||
* session log — against the `session.jsonl` fixture itself, not a separate
|
||||
* golden: the fixture doubles as the replay source (recorded scenarios) and the
|
||||
* expected produced log (both sides normalized before comparing).
|
||||
*
|
||||
* Request-header content (the composed system prompt + tool schemas riding on
|
||||
* `request/header` events) is pinned by exactly ONE scenario — the one with
|
||||
* `pinsHeader` — and scrubbed to `{{system}}`/`{{tools}}` tokens in every
|
||||
* other fixture and compare, so a prompt or tool-schema edit churns one
|
||||
* committed line instead of every fixture. A per-run uniformity guard keeps
|
||||
* the single pin sound: every live header must equal the pinned one, and no
|
||||
* header-delta may appear outside the pinning scenario (see the
|
||||
* pinned-header RFC,
|
||||
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
|
||||
*
|
||||
* `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the
|
||||
* `session.jsonl` fixtures against the real API and refreshes the stdout golden
|
||||
* in one pass.
|
||||
* The acp-agent example's snapshot suite: the scenario table for
|
||||
* `dsh-acp-snapshot`'s suite factory, which owns every compare/guard mechanic
|
||||
* (golden + re-persisted-log diffs, record write-back, the pinned-header
|
||||
* uniformity guard, the fixture guards). Fixtures live under `snapshots/<name>/`;
|
||||
* `pnpm run test:snapshot:record` re-records the `recorded` scenarios against
|
||||
* the real API. See the package README (packages/support/acp-snapshot) and the
|
||||
* snapshot RFC, docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
|
||||
*/
|
||||
|
||||
const SNAPSHOTS_DIR = join(dirname(fileURLToPath(import.meta.url)), 'snapshots')
|
||||
const RECORDING = process.env.DSH_SNAPSHOT === 'record'
|
||||
|
||||
/** A snapshot scenario and how its fixtures are produced. */
|
||||
interface Scenario {
|
||||
name: string
|
||||
/** Whether the scenario drives at least one model turn (so a JSONL golden applies). */
|
||||
hasModelTurn: boolean
|
||||
/**
|
||||
* Whether the run persists a comparable session log to diff against the
|
||||
* `session.jsonl` fixture. Defaults to {@link hasModelTurn} (a model turn
|
||||
* always produces a log worth comparing). Set it independently for a scenario
|
||||
* that produces a non-trivial log WITHOUT a model turn — e.g. a prompt blocked
|
||||
* by a `UserPromptSubmit` hook, which opens a `rejected` turn carrying `hook/*`
|
||||
* events but never calls the model.
|
||||
*/
|
||||
comparesLog?: boolean
|
||||
/**
|
||||
* Whether `test:snapshot:record` regenerates this scenario's `session.jsonl`
|
||||
* from the LIVE API. `recorded` scenarios are model-driven and reproducible;
|
||||
* `authored` scenarios (a hand-written `replay.override.json` sidecar drives
|
||||
* replay — e.g. a provider error or a cancel, which the live API can't be
|
||||
* coaxed into deterministically — or a deterministic hook scenario whose
|
||||
* derived empty script needs no sidecar) are NEVER re-recorded.
|
||||
*/
|
||||
recorded: boolean
|
||||
/**
|
||||
* How many SUBAGENT child sessions this scenario records beyond the top-level
|
||||
* one (0 for a single-session scenario). Each child rides in a sibling fixture
|
||||
* `session.<n>.jsonl` (1-based); replay forwards them to `dsh-llm-replay` so
|
||||
* each child session replays from its own script, and record mode writes the
|
||||
* harvested child logs back to those files. Defaults to 0.
|
||||
*/
|
||||
childSessions?: number
|
||||
/**
|
||||
* Whether THIS scenario's fixtures keep the full request-header content (the
|
||||
* composed system prompt and tool schema list on `request/header` /
|
||||
* `request/header-delta` events) and compare it verbatim. Exactly one
|
||||
* scenario pins it; every other scenario stores and compares that content as
|
||||
* `{{system}}`/`{{tools}}` tokens ({@link scrubRequestHeaders}), so a system
|
||||
* prompt or tool-schema change shows up as ONE committed-fixture diff, not
|
||||
* one per scenario. One pin suffices because header composition is
|
||||
* suite-uniform (parent, spawn child, and fork child all compose the same
|
||||
* prompt-modulo-cwd and the same tools) — and that premise is ASSERTED, not
|
||||
* assumed: every non-pinning run's live headers must equal the pinned
|
||||
* fixture's (normalized), so a session-dependent header (say, a restricted
|
||||
* subagent toolset) fails loud until it gets its own pinning scenario.
|
||||
* Defaults to false.
|
||||
*/
|
||||
pinsHeader?: boolean
|
||||
// The dsh-acp-agent bin (the demo:acp entry), this example's cordis.yml, and
|
||||
// the repo-root tsconfig (four levels up from examples/acp-agent/tests) — all
|
||||
// ABSOLUTE: the subprocess cwd is a temp dir outside the repo.
|
||||
const AGENT = {
|
||||
binScript: fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url)),
|
||||
configPath: fileURLToPath(new URL('../cordis.yml', import.meta.url)),
|
||||
tsconfigPath: fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)),
|
||||
}
|
||||
|
||||
const SCENARIOS: Scenario[] = [
|
||||
@@ -148,238 +86,9 @@ const SCENARIOS: Scenario[] = [
|
||||
{ name: 'hook-codex-stop-continue', hasModelTurn: true, recorded: true },
|
||||
]
|
||||
|
||||
/** The single header-pinning scenario. Guarded here (and by a meta-test) so the pin cannot silently vanish. */
|
||||
const pinningScenario = SCENARIOS.find(s => s.pinsHeader === true)
|
||||
if (pinningScenario === undefined) throw new Error('acp.snapshot: no scenario pins the request-header content')
|
||||
|
||||
/** The sibling child-fixture paths for a scenario (`session.1.jsonl` …). */
|
||||
function childFixturePaths(dir: string, childSessions: number): string[] {
|
||||
return Array.from({ length: childSessions }, (_, i) => join(dir, `session.${i + 1}.jsonl`))
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the {@link NormalizeContext} for a `session.jsonl` fixture from its own
|
||||
* header line (`{ type: 'session', id, cwd }`). A committed fixture carries the
|
||||
* session id and cwd of the run that harvested it — different from the live
|
||||
* replay run — so normalizing it against the live run's ctx would leave those
|
||||
* recorded values unscrubbed. Reading them from the header scrubs the fixture's
|
||||
* own id/cwd to the same `{{sessionId}}`/`{{cwd}}` tokens the replay output gets.
|
||||
* An authored fixture whose header is already normalized (`id:'{{sessionId}}'`,
|
||||
* `cwd:'{{cwd}}'`) yields those tokens as the volatile values, so scrubbing them
|
||||
* is an idempotent no-op. A header with no `cwd` falls back to a sentinel that
|
||||
* cannot occur in a log (NOT `''`, which `String.split` would match on every
|
||||
* character boundary and corrupt the output).
|
||||
*/
|
||||
function fixtureContext(fixture: string): NormalizeContext {
|
||||
const firstLine = fixture.split('\n').find(line => line.trim().length > 0) ?? '{}'
|
||||
const header = JSON.parse(firstLine) as { id?: unknown; cwd?: unknown }
|
||||
return {
|
||||
sessionIds: typeof header.id === 'string' ? [header.id] : [],
|
||||
cwd: typeof header.cwd === 'string' ? header.cwd : '\0no-cwd\0',
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The `data.header` payload of every `request/header` event in a session
|
||||
* JSONL, in log order, with the log's volatile values scrubbed first
|
||||
* ({@link normalizeSessionLog}) so headers harvested from different runs —
|
||||
* each embedding its own temp cwd in the composed prompt — compare on equal
|
||||
* footing.
|
||||
*/
|
||||
function normalizedHeaders(rawLog: string, ctx: NormalizeContext): unknown[] {
|
||||
return normalizeSessionLog(rawLog, ctx)
|
||||
.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.map(line => JSON.parse(line) as { type?: unknown; data?: { header?: unknown } })
|
||||
.filter(record => record.type === 'request/header')
|
||||
.map(record => record.data?.header)
|
||||
}
|
||||
|
||||
/** Count the `request/header-delta` events in a session JSONL. */
|
||||
function headerDeltaCount(rawLog: string): number {
|
||||
return rawLog.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.filter(line => (JSON.parse(line) as { type?: unknown }).type === 'request/header-delta')
|
||||
.length
|
||||
}
|
||||
|
||||
for (const scenario of SCENARIOS) {
|
||||
describe(`snapshot: ${scenario.name}`, () => {
|
||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the
|
||||
// `authored` ones (sidecar-driven errors/cancel) are never re-recorded.
|
||||
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
|
||||
const dir = join(SNAPSHOTS_DIR, scenario.name)
|
||||
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
||||
const overrideFile = join(dir, 'replay.override.json')
|
||||
const workspaceDir = join(dir, 'workspace')
|
||||
const childSessions = scenario.childSessions ?? 0
|
||||
const result = await runScenario(input, {
|
||||
mode: RECORDING ? 'record' : 'replay',
|
||||
fixtureFile: join(dir, 'session.jsonl'),
|
||||
...existsSync(overrideFile) ? { overrideFile } : {},
|
||||
// In REPLAY, forward the recorded child fixtures so each subagent session
|
||||
// replays from its own script. In RECORD they are harvested, not read.
|
||||
...!RECORDING && childSessions > 0 ? { childFiles: childFixturePaths(dir, childSessions) } : {},
|
||||
...existsSync(workspaceDir) ? { workspaceDir } : {},
|
||||
})
|
||||
|
||||
// Scrub every volatile id the run produced: the ACP server-issued session
|
||||
// id plus every harvested log's recorded id (a subagent child id never
|
||||
// surfaces over ACP, but it appears in the child's own log header). The
|
||||
// normalizer's UUID catch-all covers any we don't enumerate.
|
||||
const ctx: NormalizeContext = {
|
||||
sessionIds: [
|
||||
...result.sessionId !== undefined ? [result.sessionId] : [],
|
||||
...result.sessionLogs.map(l => l.id),
|
||||
],
|
||||
cwd: result.cwd,
|
||||
}
|
||||
|
||||
// RECORD mode (recorded model scenarios only): persist the freshly-harvested
|
||||
// logs back to their fixtures — the primary to session.jsonl, each child to
|
||||
// session.<n>.jsonl in harvest order. `--update` refreshes the Vitest
|
||||
// goldens but NOT these fixtures, so write them here. A non-pinning
|
||||
// scenario's fixtures are written header-scrubbed, so a re-record can
|
||||
// never smuggle the full prompt/schema content back into every fixture.
|
||||
const scrub = scenario.pinsHeader === true
|
||||
? (log: string): string => log
|
||||
: scrubRequestHeaders
|
||||
if (RECORDING && scenario.recorded && scenario.hasModelTurn) {
|
||||
expect(result.sessionLogs.length, 'record produced no session log to harvest').toBeGreaterThan(0)
|
||||
expect(result.sessionLogs.length, `expected ${childSessions + 1} session logs (parent + children)`)
|
||||
.toBe(childSessions + 1)
|
||||
await writeFile(join(dir, 'session.jsonl'), scrub((result.sessionLogs[0] as HarvestedLog).content))
|
||||
for (let i = 1; i < result.sessionLogs.length; i++) {
|
||||
await writeFile(join(dir, `session.${i}.jsonl`), scrub((result.sessionLogs[i] as HarvestedLog).content))
|
||||
}
|
||||
}
|
||||
|
||||
await expect(normalizeStdout(result.rawStdout, ctx))
|
||||
.toMatchFileSnapshot(join(dir, 'stdout.golden.jsonl'))
|
||||
|
||||
// A model turn always produces a log worth comparing; a hook scenario can
|
||||
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
|
||||
const comparesLog = scenario.comparesLog ?? scenario.hasModelTurn
|
||||
if (comparesLog) {
|
||||
// The harvested logs (primary-first) must match their committed fixtures
|
||||
// 1:1. Each side passes through normalizeSessionLog, scrubbed against ITS
|
||||
// OWN volatile values — the live run's via `ctx`, the committed fixture's
|
||||
// via its own header (a committed file cannot share the live run's ids).
|
||||
// Unless this scenario pins the header, both sides ALSO pass through
|
||||
// scrubRequestHeaders: the live log carries the real prompt/schemas, the
|
||||
// fixture carries the `{{system}}`/`{{tools}}` tokens, and the scrub is
|
||||
// idempotent — so the compare checks the header's presence, position,
|
||||
// reason, and config, but not its bulk content (pinned once, in the
|
||||
// `pinsHeader` scenario).
|
||||
expect(result.sessionLogs.length, 'this scenario must persist a session log').toBe(childSessions + 1)
|
||||
const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)]
|
||||
for (let i = 0; i < fixtureFiles.length; i++) {
|
||||
const harvested = scrub((result.sessionLogs[i] as HarvestedLog).content)
|
||||
const fixture = scrub(await readFile(join(dir, fixtureFiles[i] as string), 'utf8'))
|
||||
expect(normalizeSessionLog(harvested, ctx), `${fixtureFiles[i]} mismatch`)
|
||||
.toEqual(normalizeSessionLog(fixture, fixtureContext(fixture)))
|
||||
}
|
||||
}
|
||||
|
||||
// Header-uniformity guard: the single pin is sound only while every
|
||||
// session in the suite composes the SAME header and keeps it for the
|
||||
// whole run. Assert both halves live. (1) Every request/header the run
|
||||
// produced (parent, spawn child, fork child, initial or resume) must
|
||||
// equal the pinned fixture's header after each side is normalized
|
||||
// against its own volatile values. (2) No request/header-delta may
|
||||
// appear at all — a mid-run header change diverges from the pin by
|
||||
// construction, and its content would be invisible under the scrub. If
|
||||
// either fails, either the header changed (update the pin: re-record or
|
||||
// hand-edit the pinning scenario's fixture) or composition became
|
||||
// session-dependent by design (give the divergent shape its own
|
||||
// pinning scenario).
|
||||
if (scenario.pinsHeader !== true) {
|
||||
const pinnedFixture = await readFile(join(SNAPSHOTS_DIR, pinningScenario.name, 'session.jsonl'), 'utf8')
|
||||
const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture))
|
||||
expect(pinned.length, `the pinning fixture (${pinningScenario.name}) must carry exactly one request/header`)
|
||||
.toBe(1)
|
||||
for (const log of result.sessionLogs) {
|
||||
expect(headerDeltaCount(log.content), `session ${log.id}: a request/header-delta in a non-pinning scenario`)
|
||||
.toBe(0)
|
||||
const headers = normalizedHeaders(log.content, ctx)
|
||||
for (const [k, header] of headers.entries()) {
|
||||
expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`)
|
||||
.toEqual(pinned[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
describe('snapshot fixtures', () => {
|
||||
it('every scenario directory is registered (no orphans)', async () => {
|
||||
// toMatchFileSnapshot does not prune orphaned golden/fixture files, so a
|
||||
// renamed/removed scenario could leave a stale dir that nothing exercises.
|
||||
// Fail loud on any snapshots/<dir> not present in SCENARIOS.
|
||||
const entries = await readdir(SNAPSHOTS_DIR, { withFileTypes: true })
|
||||
const onDisk = entries.filter(e => e.isDirectory()).map(e => e.name).sort()
|
||||
const registered = SCENARIOS.map(s => s.name).sort()
|
||||
expect(onDisk).toEqual(registered)
|
||||
})
|
||||
|
||||
it('every registered scenario has its required fixture files', async () => {
|
||||
// Every scenario has an input script and an stdout golden. EVERY scenario
|
||||
// also needs `session.jsonl`: the harness boots `llm-replay` with that path
|
||||
// as the replay source for ALL scenarios (acp.snapshot.ts passes
|
||||
// `fixtureFile: <dir>/session.jsonl` unconditionally), and `loadReplayScript`
|
||||
// throws "fixture not found" when it is absent and no override replaces it.
|
||||
// A no-model scenario ships a header-only `session.jsonl` (it derives to an
|
||||
// empty script — no model call is made); a model scenario's fixture also
|
||||
// doubles as the expected-log artifact the run is diffed against. An authored
|
||||
// (non-`recorded`) model scenario additionally ships a `replay.override.json`
|
||||
// sidecar for the throw/hang cases a derived script cannot express.
|
||||
for (const { name, hasModelTurn, recorded, childSessions } of SCENARIOS) {
|
||||
const dir = join(SNAPSHOTS_DIR, name)
|
||||
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
|
||||
expect(existsSync(join(dir, 'stdout.golden.jsonl')), `${name}/stdout.golden.jsonl`).toBe(true)
|
||||
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
|
||||
if (hasModelTurn && !recorded) {
|
||||
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json`).toBe(true)
|
||||
}
|
||||
// A nested-agent scenario ships one child fixture per recorded subagent
|
||||
// session (`session.1.jsonl` …), the replay source for that child session.
|
||||
for (const childFixture of childFixturePaths(dir, childSessions ?? 0)) {
|
||||
expect(existsSync(childFixture), childFixture).toBe(true)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('exactly one scenario pins the request-header content', () => {
|
||||
// Zero pins would drop the prompt/schema surface from the suite entirely;
|
||||
// two would split it. The single pin is the design (pinned-header RFC).
|
||||
expect(SCENARIOS.filter(s => s.pinsHeader === true).map(s => s.name)).toEqual(['text-turn'])
|
||||
})
|
||||
|
||||
it('committed fixtures carry request-header content ONLY in the pinning scenario', async () => {
|
||||
// The whole point of the pin: a system-prompt or tool-schema change must
|
||||
// churn exactly one committed line. A non-pinning fixture that carries the
|
||||
// full header (a hand-recorded file, or a header line hand-edited out of
|
||||
// its canonical JSON form) silently reopens the suite-wide churn, so fail
|
||||
// loud here: every non-pinning session*.jsonl must be a fixed point of
|
||||
// scrubRequestHeaders (apply the scrub to fix a violation), and the
|
||||
// pinning scenario's fixtures must NOT be (their content IS the pin).
|
||||
for (const scenario of SCENARIOS) {
|
||||
const dir = join(SNAPSHOTS_DIR, scenario.name)
|
||||
const files = [
|
||||
'session.jsonl',
|
||||
...Array.from({ length: scenario.childSessions ?? 0 }, (_, i) => `session.${i + 1}.jsonl`),
|
||||
]
|
||||
for (const file of files) {
|
||||
const fixture = await readFile(join(dir, file), 'utf8')
|
||||
if (scenario.pinsHeader === true) {
|
||||
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} must PIN the full header content`)
|
||||
.not.toEqual(fixture)
|
||||
} else {
|
||||
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} carries unscrubbed header content`)
|
||||
.toEqual(fixture)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
defineAcpSnapshotSuite({
|
||||
agent: AGENT,
|
||||
snapshotsDir: join(dirname(fileURLToPath(import.meta.url)), 'snapshots'),
|
||||
scenarios: SCENARIOS,
|
||||
mode: process.env.DSH_SNAPSHOT === 'record' ? 'record' : 'replay',
|
||||
})
|
||||
@@ -9,7 +9,7 @@
|
||||
"examples/echo-agent/tests/**/*.e2e.ts",
|
||||
"examples/coding-agent/tests/**/*.e2e.ts",
|
||||
"examples/acp-agent/tests/**/*.e2e.ts",
|
||||
"examples/acp-agent/tests/**/*.snapshot.ts"
|
||||
"examples/*/tests/**/*.snapshot.ts"
|
||||
],
|
||||
"project": ["scripts/**/*.ts", "examples/**/*.ts"]
|
||||
},
|
||||
@@ -21,6 +21,11 @@
|
||||
"project": ["src/**/*.ts"],
|
||||
"ignoreDependencies": ["cordis"]
|
||||
},
|
||||
"packages/support/acp-snapshot": {
|
||||
"entry": ["tests/**/*.spec.ts", "tests/fixtures/fake-acp-agent.ts"],
|
||||
"project": ["src/**/*.ts", "tests/**/*.ts"],
|
||||
"ignoreDependencies": ["cordis"]
|
||||
},
|
||||
"packages/core/agent-loop": {
|
||||
"entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"],
|
||||
"project": ["src/**/*.ts", "tests/**/*.ts"]
|
||||
|
||||
@@ -4,8 +4,9 @@ Packages that exist to serve development, testing, and the examples rather than
|
||||
|
||||
| Package | Role | ctx key |
|
||||
|---|---|---|
|
||||
| `acp-snapshot/` | ACP snapshot suite kit: subprocess scenario harness + golden normalizers + the `defineAcpSnapshotSuite` factory | (library — imported by example `*.snapshot.ts` suites) |
|
||||
| `invariants/` | Dev-mode event-contract invariants + session-log freeze | (listens on `session/*`, `agent/*`) |
|
||||
| `llm-replay/` | Record/replay adapter: short-circuits `llm/stream` from a recorded session JSONL (keyless snapshot tests) | (listens on `llm/stream`) |
|
||||
| `subagent-mock/` | Scripted `SubagentProvider` for deterministic seam/tool tests | (registers on `ctx.subagents`) |
|
||||
|
||||
`invariants` runs only in dev mode (contract checks, not runtime behavior). `llm-replay` backs the demos and the snapshot test tier under the per-file coverage gate. `subagent-mock` exercises the real `ctx.subagents` load path without a model or child agent. A package graduates OUT of `support/` into a product group only when it gains documented product consumers.
|
||||
`invariants` runs only in dev mode (contract checks, not runtime behavior). `llm-replay` backs the demos and the snapshot test tier under the per-file coverage gate. `acp-snapshot` carries the snapshot tier's harness/normalizer/suite machinery so every example's suite is a scenario table over one shared, gate-covered implementation. `subagent-mock` exercises the real `ctx.subagents` load path without a model or child agent. A package graduates OUT of `support/` into a product group only when it gains documented product consumers.
|
||||
@@ -0,0 +1,36 @@
|
||||
# `@deepseek-ai/dsh-acp-snapshot`
|
||||
|
||||
The ACP snapshot suite kit: the shared machinery behind the keyless snapshot tier (`pnpm run test:snapshot`, [testing policy](../../../docs/testing.md)). An example gets a full snapshot suite from a scenario table plus a fixtures directory; every compare/guard mechanic lives here, under the per-file coverage gate, instead of being copied per example.
|
||||
|
||||
Three layers, importable separately:
|
||||
|
||||
- **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the golden + purity check, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths; the subprocess cwd is a temp dir outside the repo).
|
||||
- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), and the composable `scrubRequestHeaders` (header bulk → `{{system}}`/`{{tools}}`, structure kept — [pinned-header RFC](../../../docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)).
|
||||
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario golden + re-persisted-log compares, record-mode fixture write-back, the per-suite header pin with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin, non-pinning fixtures header-scrubbed). Must be called at vitest collection time.
|
||||
|
||||
A consuming `*.snapshot.ts` is the scenario table plus one factory call:
|
||||
|
||||
```ts
|
||||
import { dirname, join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { defineAcpSnapshotSuite, type Scenario } from '@deepseek-ai/dsh-acp-snapshot'
|
||||
|
||||
const SCENARIOS: Scenario[] = [
|
||||
{ name: 'text-turn', hasModelTurn: true, recorded: true, pinsHeader: true },
|
||||
]
|
||||
|
||||
defineAcpSnapshotSuite({
|
||||
agent: { // absolute paths, resolved from the suite's own location
|
||||
binScript: fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url)),
|
||||
configPath: fileURLToPath(new URL('../cordis.yml', import.meta.url)),
|
||||
tsconfigPath: fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)),
|
||||
},
|
||||
snapshotsDir: join(dirname(fileURLToPath(import.meta.url)), 'snapshots'),
|
||||
scenarios: SCENARIOS, // exactly one entry sets pinsHeader
|
||||
mode: process.env.DSH_SNAPSHOT === 'record' ? 'record' : 'replay',
|
||||
})
|
||||
```
|
||||
|
||||
The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config RFC](../../../docs/rfc/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. Fixture roles, record/replay semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md).
|
||||
|
||||
Constraints: `suite.ts` imports vitest, so the package is importable only inside a vitest run (the harness and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the harness speaks the SDK's `ClientSideConnection`. Permission round-trips are scriptable: `InputScript.permissionAnswers` is a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) the client maps to the agent-issued `optionId` at answer time; an absent or exhausted queue answers `cancelled`, and a kind the request never offered rejects the run (the agent is answered `cancelled`, so a tolerant agent cannot absorb the scenario bug).
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"name": "@deepseek-ai/dsh-acp-snapshot",
|
||||
"description": "ACP snapshot suite kit: real-subprocess scenario harness, golden normalizers, and the suite factory behind the keyless snapshot tier",
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "lib/index.js",
|
||||
"types": "lib/types/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./lib/types/index.d.ts",
|
||||
"default": "./lib/index.js"
|
||||
},
|
||||
"./src/*": "./src/*",
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"files": [
|
||||
"lib/index.js",
|
||||
"lib/types/**/*.d.ts",
|
||||
"lib/types/**/*.d.ts.map",
|
||||
"src"
|
||||
],
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"@agentclientprotocol/sdk": "0.25.1",
|
||||
"tsx": "^4.22.4",
|
||||
"vitest": "^4.1.8"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"cordis": "^4.0.0-rc.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"cordis": "^4.0.0-rc.6"
|
||||
}
|
||||
}
|
||||
+113
-29
@@ -1,16 +1,19 @@
|
||||
/**
|
||||
* Shared harness for the ACP snapshot tests. A plain module (NOT a *.spec.ts /
|
||||
* *.snapshot.ts) so importing it never re-registers another file's tests.
|
||||
* Shared subprocess harness for ACP snapshot suites. A library module driven by
|
||||
* the suite factory in ./suite.ts (and directly by harness-level specs); each
|
||||
* example's `*.snapshot.ts` names its own agent-under-test paths.
|
||||
*
|
||||
* It boots the REAL examples/acp-agent subprocess via the cordis Loader (so the
|
||||
* It boots the REAL agent bin subprocess via the cordis Loader (so the
|
||||
* export-shape bug class stays guarded — see docs/postmortem/0001), drives it
|
||||
* over real ACP JSON-RPC stdio with a deterministic input script, tees raw
|
||||
* stdout (for the golden + a purity check) into an SDK `ClientSideConnection`,
|
||||
* and — in record mode — harvests the persisted session JSONL after a graceful
|
||||
* shutdown flush. Two pure normalizers turn the captured stdout frames and the
|
||||
* session-log events into stable, snapshot-able text.
|
||||
* shutdown flush. The pure normalizers in ./normalize.ts turn the captured
|
||||
* stdout frames and the session-log events into stable, snapshot-able text.
|
||||
*
|
||||
* See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/harness
|
||||
*/
|
||||
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
|
||||
@@ -31,19 +34,36 @@ import {
|
||||
type SessionNotification,
|
||||
} from '@agentclientprotocol/sdk'
|
||||
|
||||
// The dsh-acp-agent bin (the demo:acp entry) and this example's cordis.yml.
|
||||
// The bin resolves its config-path arg from CWD and, under DSH_SNAPSHOT=replay,
|
||||
// swaps it for the sibling cordis.snapshot.yml. The child's cwd is a temp dir
|
||||
// OUTSIDE the repo, so pass the example config's ABSOLUTE path.
|
||||
const binScript = fileURLToPath(new URL('../../../packages/ui/acp-agent/src/bin.ts', import.meta.url))
|
||||
const configPath = fileURLToPath(new URL('../cordis.yml', import.meta.url))
|
||||
// Resolve tsx's ESM loader to an ABSOLUTE path once: the child runs with its
|
||||
// cwd in a temp dir OUTSIDE the repo, where a bare `--import tsx` would not
|
||||
// resolve from node_modules. import.meta.resolve gives this package's tsx
|
||||
// regardless of the child cwd.
|
||||
const tsxLoader = fileURLToPath(import.meta.resolve('tsx'))
|
||||
// The repo-root tsconfig: dev/test run UNBUILT and the `@deepseek-ai/dsh-*`
|
||||
// imports resolve through its `paths` map. The child's cwd is a temp dir
|
||||
// OUTSIDE the repo, so tsx's upward search would miss it — point tsx at the
|
||||
// repo tsconfig explicitly (same fix the e2e harness uses). Repo root is four
|
||||
// levels up from this file (examples/acp-agent/tests).
|
||||
const repoTsconfig = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
|
||||
|
||||
/**
|
||||
* The agent composition a scenario runs against: which bin to boot and which
|
||||
* leaf config it loads. All paths are ABSOLUTE — the subprocess cwd is a temp
|
||||
* dir outside the repo, so relative resolution would miss; a suite resolves
|
||||
* them from its own `import.meta.url`.
|
||||
*/
|
||||
export interface AgentUnderTest {
|
||||
/** The agent bin entry (e.g. `packages/ui/acp-agent/src/bin.ts`), run unbuilt via tsx. */
|
||||
binScript: string
|
||||
/**
|
||||
* The example's live `cordis.yml`. Under `DSH_SNAPSHOT=replay` the bin swaps
|
||||
* it for the sibling `cordis.snapshot.yml` (the keyless replay overlay), so
|
||||
* one path serves both modes.
|
||||
*/
|
||||
configPath: string
|
||||
/**
|
||||
* The repo-root tsconfig whose `paths` map resolves the unbuilt workspace
|
||||
* imports. Passed to the child as `TSX_TSCONFIG_PATH`: tsx finds a tsconfig
|
||||
* by searching UP from the child's cwd — a temp dir outside the repo — so
|
||||
* without the explicit pin the dsh-* imports fail before the bin writes a
|
||||
* byte.
|
||||
*/
|
||||
tsconfigPath: string
|
||||
}
|
||||
|
||||
/**
|
||||
* One step of a scenario's deterministic input script (`input.json`). The
|
||||
@@ -57,7 +77,7 @@ const repoTsconfig = fileURLToPath(new URL('../../../tsconfig.json', import.meta
|
||||
* the only way to exercise a cancel deterministically (a plain `prompt` step
|
||||
* awaits the response, which a cancel/hang scenario would block on forever).
|
||||
*/
|
||||
type InputStep =
|
||||
export type InputStep =
|
||||
| { op: 'initialize'; terminalOutput?: boolean }
|
||||
| { op: 'newSession' }
|
||||
| { op: 'newSessionExpectError'; additionalDirectories?: string[] }
|
||||
@@ -69,6 +89,25 @@ type InputStep =
|
||||
/** A scenario's `input.json`: an ordered list of input steps. */
|
||||
export interface InputScript {
|
||||
steps: InputStep[]
|
||||
/**
|
||||
* Ordered answers for the agent's `session/request_permission` round-trips,
|
||||
* consumed FIFO — the Nth request gets the Nth answer. Each answer selects
|
||||
* by option KIND: option ids are agent-issued randoms a committed script
|
||||
* cannot know, while kinds are the ACP-stable vocabulary, so the client maps
|
||||
* kind → the offered `optionId` at answer time. A request beyond the queue
|
||||
* (or with no queue at all) is answered `cancelled` — the stub behavior a
|
||||
* scenario without approvals relies on. A scripted kind the request does
|
||||
* not offer REJECTS the run: the scenario scripted an impossible click,
|
||||
* and {@link runScenario} throws once the in-flight step settles (the
|
||||
* agent itself just sees `cancelled`, so it cannot absorb the bug).
|
||||
*/
|
||||
permissionAnswers?: PermissionAnswer[]
|
||||
}
|
||||
|
||||
/** One scripted answer to a permission request: which offered option kind to select. */
|
||||
export interface PermissionAnswer {
|
||||
/** The `PermissionOption.kind` to select (`allow_once`, `reject_always`, …). */
|
||||
kind: 'allow_once' | 'allow_always' | 'reject_once' | 'reject_always'
|
||||
}
|
||||
|
||||
/** One harvested session log plus the identifying facts off its header line. */
|
||||
@@ -102,7 +141,10 @@ export interface RunResult {
|
||||
sessionLogs: HarvestedLog[]
|
||||
}
|
||||
|
||||
interface RunOptions {
|
||||
/** How to run one scenario: the agent to boot, the mode, and the fixture wiring. */
|
||||
export interface RunOptions {
|
||||
/** The agent composition to boot. */
|
||||
agent: AgentUnderTest
|
||||
/** `replay` (default, keyless) or `record` (real API, harvests the log). */
|
||||
mode: 'replay' | 'record'
|
||||
/** The recorded session JSONL fixture path (replay reads it; record writes near it). */
|
||||
@@ -130,6 +172,10 @@ interface RunOptions {
|
||||
* Run a scenario end-to-end against a freshly-spawned subprocess. Owns the
|
||||
* child and its temp dirs; always tears them down. Returns the captured stdout
|
||||
* and (record mode) the harvested session-log path.
|
||||
*
|
||||
* @param input The scenario's input script (steps + optional permission answers).
|
||||
* @param opts The agent to boot, the mode, and the fixture wiring.
|
||||
* @returns The captured stdout/stderr, session id, temp cwd, and harvested logs.
|
||||
*/
|
||||
export async function runScenario(input: InputScript, opts: RunOptions): Promise<RunResult> {
|
||||
const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
|
||||
@@ -151,7 +197,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
}
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
TSX_TSCONFIG_PATH: repoTsconfig,
|
||||
TSX_TSCONFIG_PATH: opts.agent.tsconfigPath,
|
||||
DSH_SNAPSHOT: opts.mode,
|
||||
DSH_SNAPSHOT_FILE: opts.fixtureFile,
|
||||
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
|
||||
@@ -163,7 +209,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
|
||||
child = spawn(
|
||||
process.execPath,
|
||||
['--import', tsxLoader, binScript, configPath],
|
||||
['--import', tsxLoader, opts.agent.binScript, opts.agent.configPath],
|
||||
{ cwd, env, stdio: ['pipe', 'pipe', 'pipe'] },
|
||||
)
|
||||
|
||||
@@ -193,25 +239,59 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
const waitForUpdate = (match: (u: SessionNotification['update']) => boolean): Promise<void> =>
|
||||
new Promise<void>(resolve => updateWaiters.push({ match, resolve }))
|
||||
|
||||
// Permission answers are consumed FIFO across the whole run; exhaustion
|
||||
// falls back to `cancelled` so approval-free scenarios keep the plain stub.
|
||||
const permissionQueue = [...input.permissionAnswers ?? []]
|
||||
// A scenario bug detected inside a client callback (a scripted permission
|
||||
// kind the agent never offered). It cannot fail the run from in there: a
|
||||
// callback throw only becomes a JSON-RPC error RESPONSE to the agent, and
|
||||
// a tolerant agent treats that as a denial and carries on — the run (or
|
||||
// worse, a record) would absorb the impossible click silently. So the
|
||||
// callback answers `cancelled` (a well-defined path for the agent),
|
||||
// captures the error here, and the step loop fails the run on it.
|
||||
let scriptError: Error | undefined
|
||||
const makeClient = (_agent: AcpAgent): Client => ({
|
||||
sessionUpdate(params: SessionNotification): Promise<void> {
|
||||
for (let i = updateWaiters.length - 1; i >= 0; i--) {
|
||||
const waiter = updateWaiters[i]
|
||||
if (waiter !== undefined && waiter.match(params.update)) {
|
||||
// The index is always in-bounds (i only decreases; splice removes at
|
||||
// i, so lower entries stay valid); the guard satisfies
|
||||
// noUncheckedIndexedAccess.
|
||||
/* v8 ignore next 1 -- unreachable in-bounds guard, see above */
|
||||
if (waiter === undefined) continue
|
||||
if (waiter.match(params.update)) {
|
||||
updateWaiters.splice(i, 1)
|
||||
waiter.resolve()
|
||||
}
|
||||
}
|
||||
return Promise.resolve()
|
||||
},
|
||||
requestPermission(_params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
|
||||
return Promise.resolve({ outcome: { outcome: 'cancelled' } })
|
||||
requestPermission(params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
|
||||
const answer = permissionQueue.shift()
|
||||
if (answer === undefined) return Promise.resolve({ outcome: { outcome: 'cancelled' } })
|
||||
const option = params.options.find(o => o.kind === answer.kind)
|
||||
if (option === undefined) {
|
||||
// The scenario scripted a click the agent never offered — a scenario
|
||||
// bug. Captured (last one wins; same bug class either way) and
|
||||
// answered `cancelled`; the step loop rejects the run on it.
|
||||
scriptError = new Error(
|
||||
`snapshot-harness: scripted permission answer ${answer.kind} not among `
|
||||
+ `the offered options [${params.options.map(o => o.kind).join(', ')}]`,
|
||||
)
|
||||
return Promise.resolve({ outcome: { outcome: 'cancelled' } })
|
||||
}
|
||||
return Promise.resolve({ outcome: { outcome: 'selected', optionId: option.optionId } })
|
||||
},
|
||||
})
|
||||
const client = new ClientSideConnection(makeClient, stream)
|
||||
|
||||
for (const step of input.steps) {
|
||||
await runStep(client, step, cwd, waitForUpdate, () => sessionId, (id) => { sessionId = id })
|
||||
// A permission exchange happens while a step's request is in flight, so
|
||||
// by the time the step settles any script bug it exposed is captured —
|
||||
// fail the run HERE, as a harness error, rather than hoping the agent's
|
||||
// reaction to the answer perturbs the transcript.
|
||||
if (scriptError !== undefined) throw scriptError
|
||||
}
|
||||
// Done driving: close stdin so the server disposes gracefully (flushing
|
||||
// persistence) and exits. Then await exit so the harvested log is complete.
|
||||
@@ -302,9 +382,9 @@ async function runStep(
|
||||
// its own). To pin frame order deterministically, wait until the client
|
||||
// has OBSERVED the hang's streamed agent_message_chunk before cancelling —
|
||||
// so those update frames always precede the cancelled prompt response in
|
||||
// the transcript (without this, the late chunk and the response race; see
|
||||
// the Codex review of commit 5). Then cancel and await the prompt, which
|
||||
// the bridge settles as `cancelled` once the abort propagates.
|
||||
// the transcript (without this, the late chunk and the response race).
|
||||
// Then cancel and await the prompt, which the bridge settles as
|
||||
// `cancelled` once the abort propagates.
|
||||
const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
||||
await waitForUpdate(u => u.sessionUpdate === 'agent_message_chunk')
|
||||
await client.cancel({ sessionId })
|
||||
@@ -324,6 +404,10 @@ async function runStep(
|
||||
|
||||
/** Resolve once the child process exits (any code/signal). */
|
||||
function waitForExit(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
// Race guard: both call sites run within one synchronous frame of
|
||||
// stdin.end()/kill(), so the exit event cannot have been delivered yet;
|
||||
// kept for any future caller that awaits in between.
|
||||
/* v8 ignore next 1 -- unreachable race guard, see above */
|
||||
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve()
|
||||
return new Promise<void>(resolve => child.once('exit', () => { resolve() }))
|
||||
}
|
||||
@@ -335,8 +419,8 @@ function waitForExit(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
*
|
||||
* The JSONL backend lays sessions out as `<root>/<cwd-bucket>/<encoded-id>.jsonl`
|
||||
* (one bucket per cwd), so a parent and its same-cwd in-process child land in
|
||||
* the SAME bucket — collecting all files across all buckets catches both (the
|
||||
* old first-match short-circuit silently dropped the child). Returns `[]` if no
|
||||
* the SAME bucket — collecting all files across all buckets catches both (a
|
||||
* first-match short-circuit would silently drop the child). Returns `[]` if no
|
||||
* log was produced (a no-session scenario).
|
||||
*/
|
||||
async function harvestSessionLogs(root: string): Promise<HarvestedLog[]> {
|
||||
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* ACP snapshot suite kit — the shared machinery behind the keyless snapshot
|
||||
* tier (`pnpm run test:snapshot`). Three layers, composable per example:
|
||||
* the subprocess scenario harness ({@link runScenario}), the pure golden
|
||||
* normalizers ({@link normalizeStdout} / {@link normalizeSessionLog} /
|
||||
* {@link scrubRequestHeaders}), and the suite factory
|
||||
* ({@link defineAcpSnapshotSuite}) that registers a scenario table as a full
|
||||
* describe/it tree. An example's `*.snapshot.ts` supplies only its
|
||||
* {@link AgentUnderTest} paths, its snapshots directory, and its
|
||||
* {@link Scenario} table.
|
||||
*
|
||||
* NOTE: ./suite.ts imports vitest, so this package is importable only inside a
|
||||
* vitest run — a support-tier constraint stated in the README.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-acp-snapshot
|
||||
*/
|
||||
|
||||
export {
|
||||
runScenario,
|
||||
type AgentUnderTest,
|
||||
type HarvestedLog,
|
||||
type InputScript,
|
||||
type InputStep,
|
||||
type PermissionAnswer,
|
||||
type RunOptions,
|
||||
type RunResult,
|
||||
} from './harness.ts'
|
||||
export {
|
||||
normalizeSessionLog,
|
||||
normalizeStdout,
|
||||
scrubRequestHeaders,
|
||||
type NormalizeContext,
|
||||
} from './normalize.ts'
|
||||
export {
|
||||
defineAcpSnapshotSuite,
|
||||
type Scenario,
|
||||
type SnapshotSuiteOptions,
|
||||
} from './suite.ts'
|
||||
+18
-4
@@ -15,12 +15,15 @@
|
||||
* A separate, composable normalizer — {@link scrubRequestHeaders} — replaces
|
||||
* the bulky request-header CONTENT (the composed system prompt and the tool
|
||||
* schema list) with `{{system}}`/`{{tools}}` tokens. It is deliberately NOT
|
||||
* folded into {@link normalizeSessionLog}: the one header-pinning scenario
|
||||
* compares that content verbatim, every other scenario composes the scrub in
|
||||
* (the `pinsHeader` flag in acp.snapshot.ts; see the pinned-header RFC,
|
||||
* folded into {@link normalizeSessionLog}: each suite's one header-pinning
|
||||
* scenario compares that content verbatim, every other scenario composes the
|
||||
* scrub in (the `pinsHeader` flag on the scenario table, consumed by the suite
|
||||
* factory in ./suite.ts; see the pinned-header RFC,
|
||||
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
|
||||
*
|
||||
* See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/normalize
|
||||
*/
|
||||
|
||||
const SESSION_ID = '{{sessionId}}'
|
||||
@@ -69,6 +72,10 @@ function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
|
||||
* (1, 2, 3, …) and all volatile strings scrubbed. Throws if any non-empty line
|
||||
* is not valid JSON — that doubles as the stdout-purity check (no logger leaked
|
||||
* onto the protocol).
|
||||
*
|
||||
* @param rawStdout The captured stdout bytes, decoded utf8.
|
||||
* @param ctx The run's volatile values to scrub.
|
||||
* @returns The normalized NDJSON transcript, one frame per line.
|
||||
*/
|
||||
export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): string {
|
||||
const lines = rawStdout.split('\n').filter(line => line.trim().length > 0)
|
||||
@@ -97,6 +104,10 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin
|
||||
* zeroed/scrubbed, all volatile strings scrubbed, and `seq` is LEFT INTACT
|
||||
* (deterministic by contract). Output is JSONL in the same shape as the input —
|
||||
* one compact record per line.
|
||||
*
|
||||
* @param rawLog The raw session `.jsonl` content.
|
||||
* @param ctx The run's volatile values to scrub.
|
||||
* @returns The normalized JSONL log, one record per line.
|
||||
*/
|
||||
export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): string {
|
||||
const lines = rawLog.split('\n').filter(line => line.trim().length > 0)
|
||||
@@ -140,7 +151,10 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri
|
||||
* Only lines with something to scrub are re-serialized; every other line
|
||||
* passes through byte-for-byte, so the transform is idempotent and applying
|
||||
* it to an already-scrubbed fixture is a no-op — the on-disk-fixtures guard
|
||||
* in acp.snapshot.ts relies on exactly that.
|
||||
* in ./suite.ts relies on exactly that.
|
||||
*
|
||||
* @param rawLog The raw session `.jsonl` content.
|
||||
* @returns The JSONL with header content tokenized, other lines byte-identical.
|
||||
*/
|
||||
export function scrubRequestHeaders(rawLog: string): string {
|
||||
const lines = rawLog.split('\n')
|
||||
@@ -0,0 +1,373 @@
|
||||
/**
|
||||
* The ACP snapshot suite factory (REPLAY by default, keyless). A suite is a
|
||||
* scenario table plus a snapshots directory: each scenario under
|
||||
* `<snapshotsDir>/<name>/` ships an `input.json` (the client stdin script) and
|
||||
* a `session.jsonl` fixture; replay boots the real agent subprocess
|
||||
* (./harness.ts), drives it, and diffs the normalized stdout transcript
|
||||
* against the committed `stdout.golden.jsonl`. For model scenarios it ALSO
|
||||
* checks the re-persisted session log — against the `session.jsonl` fixture
|
||||
* itself, not a separate golden: the fixture doubles as the replay source
|
||||
* (recorded scenarios) and the expected produced log (both sides normalized
|
||||
* before comparing).
|
||||
*
|
||||
* Request-header content (the composed system prompt + tool schemas riding on
|
||||
* `request/header` events) is pinned by exactly ONE scenario per suite — the
|
||||
* one with `pinsHeader` — and scrubbed to `{{system}}`/`{{tools}}` tokens in
|
||||
* every other fixture and compare, so a prompt or tool-schema edit churns one
|
||||
* committed line instead of every fixture. A per-run uniformity guard keeps
|
||||
* the single pin sound: every live header must equal the pinned one, and no
|
||||
* header-delta may appear outside the pinning scenario (see the
|
||||
* pinned-header RFC,
|
||||
* docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md).
|
||||
*
|
||||
* `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the
|
||||
* `session.jsonl` fixtures against the real API and refreshes the stdout golden
|
||||
* in one pass; the caller resolves that env into {@link SnapshotSuiteOptions}
|
||||
* (env reading stays at the suite edge, not in this library).
|
||||
*
|
||||
* @module @deepseek-ai/dsh-acp-snapshot/suite
|
||||
*/
|
||||
|
||||
import { readFile, readdir, writeFile } from 'node:fs/promises'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { type AgentUnderTest, type HarvestedLog, type InputScript, runScenario } from './harness.ts'
|
||||
import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from './normalize.ts'
|
||||
|
||||
/** A snapshot scenario and how its fixtures are produced. */
|
||||
export interface Scenario {
|
||||
name: string
|
||||
/** Whether the scenario drives at least one model turn (so a JSONL golden applies). */
|
||||
hasModelTurn: boolean
|
||||
/**
|
||||
* Whether the run persists a comparable session log to diff against the
|
||||
* `session.jsonl` fixture. Defaults to {@link hasModelTurn} (a model turn
|
||||
* always produces a log worth comparing). Set it independently for a scenario
|
||||
* that produces a non-trivial log WITHOUT a model turn — e.g. a prompt blocked
|
||||
* by a `UserPromptSubmit` hook, which opens a `rejected` turn carrying `hook/*`
|
||||
* events but never calls the model.
|
||||
*/
|
||||
comparesLog?: boolean
|
||||
/**
|
||||
* Whether `test:snapshot:record` regenerates this scenario's `session.jsonl`
|
||||
* from the LIVE API. `recorded` scenarios are model-driven and reproducible;
|
||||
* `authored` scenarios (a hand-written `replay.override.json` sidecar drives
|
||||
* replay — e.g. a provider error or a cancel, which the live API can't be
|
||||
* coaxed into deterministically — or a deterministic hook scenario whose
|
||||
* derived empty script needs no sidecar) are NEVER re-recorded.
|
||||
*/
|
||||
recorded: boolean
|
||||
/**
|
||||
* How many SUBAGENT child sessions this scenario records beyond the top-level
|
||||
* one (0 for a single-session scenario). Each child rides in a sibling fixture
|
||||
* `session.<n>.jsonl` (1-based); replay forwards them to `dsh-llm-replay` so
|
||||
* each child session replays from its own script, and record mode writes the
|
||||
* harvested child logs back to those files. Defaults to 0.
|
||||
*/
|
||||
childSessions?: number
|
||||
/**
|
||||
* Whether THIS scenario's fixtures keep the full request-header content (the
|
||||
* composed system prompt and tool schema list on `request/header` /
|
||||
* `request/header-delta` events) and compare it verbatim. Exactly one
|
||||
* scenario per suite pins it; every other scenario stores and compares that
|
||||
* content as `{{system}}`/`{{tools}}` tokens ({@link scrubRequestHeaders}),
|
||||
* so a system prompt or tool-schema change shows up as ONE committed-fixture
|
||||
* diff, not one per scenario. One pin suffices because header composition is
|
||||
* suite-uniform (parent, spawn child, and fork child all compose the same
|
||||
* prompt-modulo-cwd and the same tools) — and that premise is ASSERTED, not
|
||||
* assumed: every non-pinning run's live headers must equal the pinned
|
||||
* fixture's (normalized), so a session-dependent header (say, a restricted
|
||||
* subagent toolset) fails loud until it gets its own pinning scenario.
|
||||
* Defaults to false.
|
||||
*/
|
||||
pinsHeader?: boolean
|
||||
}
|
||||
|
||||
/** One suite's inputs: the agent to boot, where its fixtures live, and its scenario table. */
|
||||
export interface SnapshotSuiteOptions {
|
||||
/** The agent composition every scenario boots. */
|
||||
agent: AgentUnderTest
|
||||
/** Absolute path of the suite's `snapshots/` directory (one subdir per scenario). */
|
||||
snapshotsDir: string
|
||||
/** The scenario table; exactly one entry must set `pinsHeader`. */
|
||||
scenarios: Scenario[]
|
||||
/**
|
||||
* `replay` (keyless, the default tier) or `record` (live API; re-records the
|
||||
* `recorded` scenarios' fixtures and refreshes the vitest goldens under
|
||||
* `--update`). The caller derives this from `$DSH_SNAPSHOT` — env reading
|
||||
* stays outside this library.
|
||||
*/
|
||||
mode: 'replay' | 'record'
|
||||
}
|
||||
|
||||
/**
|
||||
* The sibling child-fixture paths for a scenario (`session.1.jsonl` …).
|
||||
*
|
||||
* @param dir The scenario's snapshots directory (`<snapshotsDir>/<name>`).
|
||||
* @param childSessions How many subagent child sessions the scenario records.
|
||||
* @returns One path per child, 1-based, in fixture order.
|
||||
*/
|
||||
export function childFixturePaths(dir: string, childSessions: number): string[] {
|
||||
return Array.from({ length: childSessions }, (_, i) => join(dir, `session.${i + 1}.jsonl`))
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the {@link NormalizeContext} for a `session.jsonl` fixture from its own
|
||||
* header line (`{ type: 'session', id, cwd }`). A committed fixture carries the
|
||||
* session id and cwd of the run that harvested it — different from the live
|
||||
* replay run — so normalizing it against the live run's ctx would leave those
|
||||
* recorded values unscrubbed. Reading them from the header scrubs the fixture's
|
||||
* own id/cwd to the same `{{sessionId}}`/`{{cwd}}` tokens the replay output gets.
|
||||
* An authored fixture whose header is already normalized (`id:'{{sessionId}}'`,
|
||||
* `cwd:'{{cwd}}'`) yields those tokens as the volatile values, so scrubbing them
|
||||
* is an idempotent no-op. A header with no `cwd` falls back to a sentinel that
|
||||
* cannot occur in a log (NOT `''`, which `String.split` would match on every
|
||||
* character boundary and corrupt the output).
|
||||
*
|
||||
* @param fixture The committed `session.jsonl` content.
|
||||
* @returns The fixture's own volatile values, ready for {@link normalizeSessionLog}.
|
||||
*/
|
||||
export function fixtureContext(fixture: string): NormalizeContext {
|
||||
const firstLine = fixture.split('\n').find(line => line.trim().length > 0) ?? '{}'
|
||||
const header = JSON.parse(firstLine) as { id?: unknown; cwd?: unknown }
|
||||
return {
|
||||
sessionIds: typeof header.id === 'string' ? [header.id] : [],
|
||||
cwd: typeof header.cwd === 'string' ? header.cwd : '\0no-cwd\0',
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The `data.header` payload of every `request/header` event in a session
|
||||
* JSONL, in log order, with the log's volatile values scrubbed first
|
||||
* ({@link normalizeSessionLog}) so headers harvested from different runs —
|
||||
* each embedding its own temp cwd in the composed prompt — compare on equal
|
||||
* footing.
|
||||
*
|
||||
* @param rawLog The session `.jsonl` content to extract headers from.
|
||||
* @param ctx The volatile values of the run that produced it.
|
||||
* @returns The normalized `data.header` payloads, in log order.
|
||||
*/
|
||||
export function normalizedHeaders(rawLog: string, ctx: NormalizeContext): unknown[] {
|
||||
return normalizeSessionLog(rawLog, ctx)
|
||||
.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.map(line => JSON.parse(line) as { type?: unknown; data?: { header?: unknown } })
|
||||
.filter(record => record.type === 'request/header')
|
||||
.map(record => record.data?.header)
|
||||
}
|
||||
|
||||
/**
|
||||
* Count the `request/header-delta` events in a session JSONL.
|
||||
*
|
||||
* @param rawLog The session `.jsonl` content.
|
||||
* @returns How many `request/header-delta` events the log carries.
|
||||
*/
|
||||
export function headerDeltaCount(rawLog: string): number {
|
||||
return rawLog.split('\n')
|
||||
.filter(line => line.trim().length > 0)
|
||||
.filter(line => (JSON.parse(line) as { type?: unknown }).type === 'request/header-delta')
|
||||
.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the suite: one `describe` per scenario (the golden/log compares and
|
||||
* the header-uniformity guard) plus the fixture guard block (no orphan
|
||||
* scenario dirs, required files present, exactly one pin, non-pinning fixtures
|
||||
* header-scrubbed). Must run at vitest collection time — it calls
|
||||
* `describe`/`it`. Throws immediately if no scenario pins the header (the
|
||||
* uniformity guard would have nothing to compare against).
|
||||
*
|
||||
* @param options The agent, snapshots directory, scenario table, and mode.
|
||||
*/
|
||||
export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
||||
const { agent, snapshotsDir, scenarios, mode } = options
|
||||
const RECORDING = mode === 'record'
|
||||
|
||||
/** The suite's single header-pinning scenario. Guarded here (and by a meta-test) so the pin cannot silently vanish. */
|
||||
const pinningScenario = scenarios.find(s => s.pinsHeader === true)
|
||||
if (pinningScenario === undefined) throw new Error('acp-snapshot: no scenario pins the request-header content')
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
describe(`snapshot: ${scenario.name}`, () => {
|
||||
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the
|
||||
// `authored` ones (sidecar-driven errors/cancel) are never re-recorded.
|
||||
it.skipIf(RECORDING && !scenario.recorded)('matches the goldens', async () => {
|
||||
const dir = join(snapshotsDir, scenario.name)
|
||||
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
||||
const overrideFile = join(dir, 'replay.override.json')
|
||||
const workspaceDir = join(dir, 'workspace')
|
||||
const childSessions = scenario.childSessions ?? 0
|
||||
const result = await runScenario(input, {
|
||||
agent,
|
||||
mode,
|
||||
fixtureFile: join(dir, 'session.jsonl'),
|
||||
...existsSync(overrideFile) ? { overrideFile } : {},
|
||||
// In REPLAY, forward the recorded child fixtures so each subagent session
|
||||
// replays from its own script. In RECORD they are harvested, not read.
|
||||
...!RECORDING && childSessions > 0 ? { childFiles: childFixturePaths(dir, childSessions) } : {},
|
||||
...existsSync(workspaceDir) ? { workspaceDir } : {},
|
||||
})
|
||||
|
||||
// Scrub every volatile id the run produced: the ACP server-issued session
|
||||
// id plus every harvested log's recorded id (a subagent child id never
|
||||
// surfaces over ACP, but it appears in the child's own log header). The
|
||||
// normalizer's UUID catch-all covers any we don't enumerate.
|
||||
const ctx: NormalizeContext = {
|
||||
sessionIds: [
|
||||
...result.sessionId !== undefined ? [result.sessionId] : [],
|
||||
...result.sessionLogs.map(l => l.id),
|
||||
],
|
||||
cwd: result.cwd,
|
||||
}
|
||||
|
||||
// RECORD mode (recorded model scenarios only): persist the freshly-harvested
|
||||
// logs back to their fixtures — the primary to session.jsonl, each child to
|
||||
// session.<n>.jsonl in harvest order. `--update` refreshes the Vitest
|
||||
// goldens but NOT these fixtures, so write them here. A non-pinning
|
||||
// scenario's fixtures are written header-scrubbed, so a re-record can
|
||||
// never smuggle the full prompt/schema content back into every fixture.
|
||||
const scrub = scenario.pinsHeader === true
|
||||
? (log: string): string => log
|
||||
: scrubRequestHeaders
|
||||
if (RECORDING && scenario.recorded && scenario.hasModelTurn) {
|
||||
expect(result.sessionLogs.length, 'record produced no session log to harvest').toBeGreaterThan(0)
|
||||
expect(result.sessionLogs.length, `expected ${childSessions + 1} session logs (parent + children)`)
|
||||
.toBe(childSessions + 1)
|
||||
await writeFile(join(dir, 'session.jsonl'), scrub((result.sessionLogs[0] as HarvestedLog).content))
|
||||
for (let i = 1; i < result.sessionLogs.length; i++) {
|
||||
await writeFile(join(dir, `session.${i}.jsonl`), scrub((result.sessionLogs[i] as HarvestedLog).content))
|
||||
}
|
||||
}
|
||||
|
||||
await expect(normalizeStdout(result.rawStdout, ctx))
|
||||
.toMatchFileSnapshot(join(dir, 'stdout.golden.jsonl'))
|
||||
|
||||
// A model turn always produces a log worth comparing; a hook scenario can
|
||||
// produce one without a model turn (a `rejected` turn carrying `hook/*`).
|
||||
const comparesLog = scenario.comparesLog ?? scenario.hasModelTurn
|
||||
if (comparesLog) {
|
||||
// The harvested logs (primary-first) must match their committed fixtures
|
||||
// 1:1. Each side passes through normalizeSessionLog, scrubbed against ITS
|
||||
// OWN volatile values — the live run's via `ctx`, the committed fixture's
|
||||
// via its own header (a committed file cannot share the live run's ids).
|
||||
// Unless this scenario pins the header, both sides ALSO pass through
|
||||
// scrubRequestHeaders: the live log carries the real prompt/schemas, the
|
||||
// fixture carries the `{{system}}`/`{{tools}}` tokens, and the scrub is
|
||||
// idempotent — so the compare checks the header's presence, position,
|
||||
// reason, and config, but not its bulk content (pinned once, in the
|
||||
// `pinsHeader` scenario).
|
||||
expect(result.sessionLogs.length, 'this scenario must persist a session log').toBe(childSessions + 1)
|
||||
const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)]
|
||||
for (let i = 0; i < fixtureFiles.length; i++) {
|
||||
const harvested = scrub((result.sessionLogs[i] as HarvestedLog).content)
|
||||
const fixture = scrub(await readFile(join(dir, fixtureFiles[i] as string), 'utf8'))
|
||||
expect(normalizeSessionLog(harvested, ctx), `${fixtureFiles[i]} mismatch`)
|
||||
.toEqual(normalizeSessionLog(fixture, fixtureContext(fixture)))
|
||||
}
|
||||
}
|
||||
|
||||
// Header-uniformity guard: the single pin is sound only while every
|
||||
// session in the suite composes the SAME header and keeps it for the
|
||||
// whole run. Assert both halves live. (1) Every request/header the run
|
||||
// produced (parent, spawn child, fork child, initial or resume) must
|
||||
// equal the pinned fixture's header after each side is normalized
|
||||
// against its own volatile values. (2) No request/header-delta may
|
||||
// appear at all — a mid-run header change diverges from the pin by
|
||||
// construction, and its content would be invisible under the scrub. If
|
||||
// either fails, either the header changed (update the pin: re-record or
|
||||
// hand-edit the pinning scenario's fixture) or composition became
|
||||
// session-dependent by design (give the divergent shape its own
|
||||
// pinning scenario).
|
||||
if (scenario.pinsHeader !== true) {
|
||||
const pinnedFixture = await readFile(join(snapshotsDir, pinningScenario.name, 'session.jsonl'), 'utf8')
|
||||
const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture))
|
||||
expect(pinned.length, `the pinning fixture (${pinningScenario.name}) must carry exactly one request/header`)
|
||||
.toBe(1)
|
||||
for (const log of result.sessionLogs) {
|
||||
expect(headerDeltaCount(log.content), `session ${log.id}: a request/header-delta in a non-pinning scenario`)
|
||||
.toBe(0)
|
||||
const headers = normalizedHeaders(log.content, ctx)
|
||||
for (const [k, header] of headers.entries()) {
|
||||
expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`)
|
||||
.toEqual(pinned[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
describe('snapshot fixtures', () => {
|
||||
it('every scenario directory is registered (no orphans)', async () => {
|
||||
// toMatchFileSnapshot does not prune orphaned golden/fixture files, so a
|
||||
// renamed/removed scenario could leave a stale dir that nothing exercises.
|
||||
// Fail loud on any snapshots/<dir> not present in the scenario table.
|
||||
const entries = await readdir(snapshotsDir, { withFileTypes: true })
|
||||
const onDisk = entries.filter(e => e.isDirectory()).map(e => e.name).sort()
|
||||
const registered = scenarios.map(s => s.name).sort()
|
||||
expect(onDisk).toEqual(registered)
|
||||
})
|
||||
|
||||
it('every registered scenario has its required fixture files', () => {
|
||||
// Every scenario has an input script and an stdout golden. EVERY scenario
|
||||
// also needs `session.jsonl`: the suite boots `llm-replay` with that path
|
||||
// as the replay source for ALL scenarios (the factory passes
|
||||
// `fixtureFile: <dir>/session.jsonl` unconditionally), and `loadReplayScript`
|
||||
// throws "fixture not found" when it is absent and no override replaces it.
|
||||
// A no-model scenario ships a header-only `session.jsonl` (it derives to an
|
||||
// empty script — no model call is made); a model scenario's fixture also
|
||||
// doubles as the expected-log artifact the run is diffed against. An authored
|
||||
// (non-`recorded`) model scenario additionally ships a `replay.override.json`
|
||||
// sidecar for the throw/hang cases a derived script cannot express.
|
||||
for (const { name, hasModelTurn, recorded, childSessions } of scenarios) {
|
||||
const dir = join(snapshotsDir, name)
|
||||
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
|
||||
expect(existsSync(join(dir, 'stdout.golden.jsonl')), `${name}/stdout.golden.jsonl`).toBe(true)
|
||||
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
|
||||
if (hasModelTurn && !recorded) {
|
||||
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json`).toBe(true)
|
||||
}
|
||||
// A nested-agent scenario ships one child fixture per recorded subagent
|
||||
// session (`session.1.jsonl` …), the replay source for that child session.
|
||||
for (const childFixture of childFixturePaths(dir, childSessions ?? 0)) {
|
||||
expect(existsSync(childFixture), childFixture).toBe(true)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('exactly one scenario pins the request-header content', () => {
|
||||
// Zero pins would drop the prompt/schema surface from the suite entirely;
|
||||
// two would split it. One pin per suite is the design (pinned-header RFC);
|
||||
// WHICH scenario pins is the scenario table's reviewable choice.
|
||||
expect(scenarios.filter(s => s.pinsHeader === true).map(s => s.name)).toEqual([pinningScenario.name])
|
||||
})
|
||||
|
||||
it('committed fixtures carry request-header content ONLY in the pinning scenario', async () => {
|
||||
// The whole point of the pin: a system-prompt or tool-schema change must
|
||||
// churn exactly one committed line. A non-pinning fixture that carries the
|
||||
// full header (a hand-recorded file, or a header line hand-edited out of
|
||||
// its canonical JSON form) silently reopens the suite-wide churn, so fail
|
||||
// loud here: every non-pinning session*.jsonl must be a fixed point of
|
||||
// scrubRequestHeaders (apply the scrub to fix a violation), and the
|
||||
// pinning scenario's fixtures must NOT be (their content IS the pin).
|
||||
for (const scenario of scenarios) {
|
||||
const dir = join(snapshotsDir, scenario.name)
|
||||
const files = [
|
||||
'session.jsonl',
|
||||
...Array.from({ length: scenario.childSessions ?? 0 }, (_, i) => `session.${i + 1}.jsonl`),
|
||||
]
|
||||
for (const file of files) {
|
||||
const fixture = await readFile(join(dir, file), 'utf8')
|
||||
if (scenario.pinsHeader === true) {
|
||||
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} must PIN the full header content`)
|
||||
.not.toEqual(fixture)
|
||||
} else {
|
||||
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} carries unscrubbed header content`)
|
||||
.toEqual(fixture)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
/**
|
||||
* Scripted fake ACP agent bin for `dsh-acp-snapshot`'s unit specs. Speaks
|
||||
* newline-delimited JSON-RPC on stdio like the real `dsh-acp-agent` bin, but
|
||||
* every behavior — how prompts settle, whether session/new rejects, which
|
||||
* session logs get persisted, what filesystem noise to leave — comes from a
|
||||
* `behavior.json` sitting NEXT to the `$DSH_SNAPSHOT_FILE` fixture, so a spec
|
||||
* scripts a whole subprocess run from data. The specs launch it through the
|
||||
* REAL `runScenario` spawn path (tsx loader, temp cwd, env plumbing), so the
|
||||
* harness plumbing is exercised for real; only the agent behind the protocol
|
||||
* is scripted.
|
||||
*
|
||||
* The specs (not the golden tier) own this bin: it asserts nothing, echoes
|
||||
* observable facts into `session/update` text chunks (env probe, permission
|
||||
* outcome, seeded-workspace listing) for the spec to read off `rawStdout`, and
|
||||
* exits 0 on stdin EOF after writing the scripted logs — mirroring the real
|
||||
* bin's dispose-flush-exit shape.
|
||||
*/
|
||||
|
||||
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { readdirSync } from 'node:fs'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { createInterface } from 'node:readline'
|
||||
|
||||
/** One scripted session log: a file path under the sessions root plus its JSONL lines. */
|
||||
interface ScriptedLog {
|
||||
/** Path relative to `$DSH_SNAPSHOT_SESSIONS_ROOT`, e.g. `bucket/a.jsonl` (an empty dir segment is invalid). */
|
||||
file: string
|
||||
/**
|
||||
* The JSONL records. String templates `{{CWD}}` and `{{SID}}` are replaced
|
||||
* with the run's real cwd and the ACP session id this bin issued, so a
|
||||
* written log carries genuine volatile values for the normalizers to scrub.
|
||||
*/
|
||||
lines: unknown[]
|
||||
}
|
||||
|
||||
/** The whole scripted behavior for one run. Every field defaults to the least surprising choice. */
|
||||
interface Behavior {
|
||||
/** Reject every `session/new` (exercises the expect-error step without extra dirs). */
|
||||
rejectNewSession?: boolean
|
||||
/** Reject `session/new` only when `additionalDirectories` is non-empty (the real bridge's rule). */
|
||||
rejectExtraDirs?: boolean
|
||||
/** How `session/prompt` settles: a clean response, a JSON-RPC error, or a hang until `session/cancel`. */
|
||||
prompt?: 'respond' | 'error' | 'hang-until-cancel'
|
||||
/** Before responding to a prompt, send a `session/request_permission` request and echo its outcome as a chunk. */
|
||||
permissionProbe?: boolean
|
||||
/** Echo the `DSH_SNAPSHOT_*` env the harness set as a chunk (spec-side env-plumbing assertions). */
|
||||
echoEnv?: boolean
|
||||
/** Echo the sorted cwd listing as a chunk (spec-side workspace-seeding assertions). */
|
||||
echoWorkspace?: boolean
|
||||
/** Write a line to stderr on boot (spec-side stderr-capture assertions). */
|
||||
stderrNote?: string
|
||||
/** Session logs to persist on stdin EOF. */
|
||||
logs?: ScriptedLog[]
|
||||
/** Leave a stray FILE directly under the sessions root (harvest must skip it). */
|
||||
strayRootFile?: boolean
|
||||
/** Leave a stray non-`.jsonl` file inside a bucket (harvest must skip it). */
|
||||
strayBucketFile?: boolean
|
||||
/** Delete the sessions root entirely (harvest must yield no logs). */
|
||||
deleteSessionsRoot?: boolean
|
||||
}
|
||||
|
||||
const sessionsRoot = process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? ''
|
||||
const fixtureFile = process.env.DSH_SNAPSHOT_FILE ?? ''
|
||||
const behavior: Behavior = fixtureFile === ''
|
||||
? {}
|
||||
: JSON.parse(readFileSync(join(dirname(fixtureFile), 'behavior.json'), 'utf8')) as Behavior
|
||||
|
||||
if (behavior.stderrNote !== undefined) process.stderr.write(`${behavior.stderrNote}\n`)
|
||||
|
||||
let nextOutboundId = 1000
|
||||
let sessionId = ''
|
||||
/**
|
||||
* The cwd the client passed to `session/new` — used verbatim for `{{CWD}}`
|
||||
* substitution, mirroring the real bin (whose persisted header carries the
|
||||
* session cwd as given, NOT `process.cwd()`, which the OS realpaths — on
|
||||
* macOS `/var/folders/…` vs `/private/var/folders/…`).
|
||||
*/
|
||||
let sessionCwd = ''
|
||||
/** The parked prompt request id while `hang-until-cancel` waits for the cancel notification. */
|
||||
let parkedPromptId: number | string | null = null
|
||||
/** Resolvers for permission-probe responses, keyed by outbound request id. */
|
||||
const pendingPermission = new Map<number, (outcome: unknown) => void>()
|
||||
|
||||
function send(frame: Record<string, unknown>): void {
|
||||
process.stdout.write(`${JSON.stringify({ jsonrpc: '2.0', ...frame })}\n`)
|
||||
}
|
||||
|
||||
function respond(id: number | string, result: unknown): void {
|
||||
send({ id, result })
|
||||
}
|
||||
|
||||
function respondError(id: number | string, message: string): void {
|
||||
send({ id, error: { code: -32603, message } })
|
||||
}
|
||||
|
||||
function chunk(text: string): void {
|
||||
send({
|
||||
method: 'session/update',
|
||||
params: { sessionId, update: { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text } } },
|
||||
})
|
||||
}
|
||||
|
||||
/** Substitute the `{{CWD}}`/`{{SID}}` templates through a scripted log record. */
|
||||
function instantiate(value: unknown): unknown {
|
||||
if (typeof value === 'string') return value.split('{{CWD}}').join(sessionCwd).split('{{SID}}').join(sessionId)
|
||||
if (Array.isArray(value)) return value.map(instantiate)
|
||||
if (value !== null && typeof value === 'object') {
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(value)) out[k] = instantiate(v)
|
||||
return out
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
async function handlePrompt(id: number | string): Promise<void> {
|
||||
if ((behavior.prompt ?? 'respond') === 'hang-until-cancel') {
|
||||
// A thought chunk BEFORE any message chunk: a promptAndCancel waiter
|
||||
// watches for agent_message_chunk, so this exercises its non-matching
|
||||
// update path while the waiter is armed.
|
||||
send({
|
||||
method: 'session/update',
|
||||
params: { sessionId, update: { sessionUpdate: 'agent_thought_chunk', content: { type: 'text', text: 'mulling' } } },
|
||||
})
|
||||
}
|
||||
chunk('thinking about it')
|
||||
if (behavior.echoEnv === true) {
|
||||
chunk(`env:${JSON.stringify({
|
||||
mode: process.env.DSH_SNAPSHOT,
|
||||
override: process.env.DSH_SNAPSHOT_OVERRIDE ?? null,
|
||||
childFiles: process.env.DSH_SNAPSHOT_CHILD_FILES ?? null,
|
||||
})}`)
|
||||
}
|
||||
if (behavior.echoWorkspace === true) {
|
||||
chunk(`workspace:${readdirSync(process.cwd()).sort().join(',')}`)
|
||||
}
|
||||
if (behavior.permissionProbe === true) {
|
||||
const requestId = nextOutboundId++
|
||||
const outcome = await new Promise<unknown>((resolve) => {
|
||||
pendingPermission.set(requestId, resolve)
|
||||
send({
|
||||
id: requestId,
|
||||
method: 'session/request_permission',
|
||||
params: {
|
||||
sessionId,
|
||||
toolCall: { toolCallId: 'call_fake_1', title: 'fake tool', kind: 'execute', status: 'pending' },
|
||||
options: [
|
||||
{ optionId: 'opt-allow', name: 'Allow once', kind: 'allow_once' },
|
||||
{ optionId: 'opt-reject', name: 'Reject once', kind: 'reject_once' },
|
||||
],
|
||||
},
|
||||
})
|
||||
})
|
||||
chunk(`permission:${JSON.stringify(outcome)}`)
|
||||
}
|
||||
switch (behavior.prompt ?? 'respond') {
|
||||
case 'respond':
|
||||
respond(id, { stopReason: 'end_turn' })
|
||||
return
|
||||
case 'error':
|
||||
respondError(id, 'model exploded')
|
||||
return
|
||||
case 'hang-until-cancel':
|
||||
parkedPromptId = id
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
function handleFrame(frame: Record<string, unknown>): void {
|
||||
const id = frame.id as number | string | undefined
|
||||
const method = frame.method as string | undefined
|
||||
const params = (frame.params ?? {}) as Record<string, unknown>
|
||||
// A response to one of OUR outbound requests (the permission probe).
|
||||
if (method === undefined && id !== undefined && typeof id === 'number' && pendingPermission.has(id)) {
|
||||
const resolve = pendingPermission.get(id) as (outcome: unknown) => void
|
||||
pendingPermission.delete(id)
|
||||
resolve((frame.result as { outcome?: unknown } | undefined)?.outcome ?? null)
|
||||
return
|
||||
}
|
||||
switch (method) {
|
||||
case 'initialize':
|
||||
respond(id as number | string, { protocolVersion: 1, agentCapabilities: { loadSession: false } })
|
||||
return
|
||||
case 'session/new': {
|
||||
const extra = params.additionalDirectories as unknown[] | undefined
|
||||
if (behavior.rejectNewSession === true || (behavior.rejectExtraDirs === true && extra !== undefined && extra.length > 0)) {
|
||||
respondError(id as number | string, 'unsupported workspace scope')
|
||||
return
|
||||
}
|
||||
sessionId = randomUUID()
|
||||
sessionCwd = typeof params.cwd === 'string' ? params.cwd : process.cwd()
|
||||
respond(id as number | string, { sessionId })
|
||||
return
|
||||
}
|
||||
case 'session/prompt':
|
||||
void handlePrompt(id as number | string)
|
||||
return
|
||||
case 'session/cancel':
|
||||
if (parkedPromptId !== null) {
|
||||
const parked = parkedPromptId
|
||||
parkedPromptId = null
|
||||
respond(parked, { stopReason: 'cancelled' })
|
||||
}
|
||||
return
|
||||
default:
|
||||
// Unknown method: a notification is ignored; a request gets an error so
|
||||
// the SDK never waits forever on a frame this fake doesn't model.
|
||||
if (id !== undefined) respondError(id, `unhandled method ${String(method)}`)
|
||||
}
|
||||
}
|
||||
|
||||
function flushLogsAndExit(): void {
|
||||
for (const log of behavior.logs ?? []) {
|
||||
const target = join(sessionsRoot, log.file)
|
||||
mkdirSync(dirname(target), { recursive: true })
|
||||
writeFileSync(target, log.lines.map(l => JSON.stringify(instantiate(l))).join('\n') + '\n')
|
||||
}
|
||||
if (behavior.strayRootFile === true) writeFileSync(join(sessionsRoot, 'stray.txt'), 'not a bucket\n')
|
||||
if (behavior.strayBucketFile === true) {
|
||||
mkdirSync(join(sessionsRoot, 'bucket-noise'), { recursive: true })
|
||||
writeFileSync(join(sessionsRoot, 'bucket-noise', 'notes.txt'), 'not a session log\n')
|
||||
}
|
||||
if (behavior.deleteSessionsRoot === true) rmSync(sessionsRoot, { recursive: true, force: true })
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
const rl = createInterface({ input: process.stdin })
|
||||
rl.on('line', (line) => {
|
||||
if (line.trim().length === 0) return
|
||||
handleFrame(JSON.parse(line) as Record<string, unknown>)
|
||||
})
|
||||
rl.on('close', () => { flushLogsAndExit() })
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"prompt": "respond",
|
||||
"logs": [
|
||||
{ "file": "b/parent.jsonl", "lines": [
|
||||
{ "type": "session", "id": "{{SID}}", "createdAt": 700, "cwd": "{{CWD}}" },
|
||||
{ "type": "request/header", "seq": 0, "time": 3, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } }
|
||||
]},
|
||||
{ "file": "b/child.jsonl", "lines": [
|
||||
{ "type": "session", "id": "abababab-cdcd-4efe-8ada-badabadabada", "createdAt": 800, "cwd": "{{CWD}}", "parentSession": "{{SID}}" },
|
||||
{ "type": "request/header", "seq": 0, "time": 2, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } }
|
||||
]}
|
||||
]
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "prompt", "text": "rec child" }] }
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
{"type":"session","id":"abababab-cdcd-4efe-8ada-badabadabada","createdAt":800,"cwd":"/var/folders/2g/b32ct0qn1d728l_v6tdkjytr0000gn/T/acp-snap-cwd-KBQJbW","parentSession":"f6fa7fcf-dd9c-4b39-8815-b25ddcebfd88"}
|
||||
{"type":"request/header","seq":0,"time":2,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
{"type":"session","id":"f6fa7fcf-dd9c-4b39-8815-b25ddcebfd88","createdAt":700,"cwd":"/var/folders/2g/b32ct0qn1d728l_v6tdkjytr0000gn/T/acp-snap-cwd-KBQJbW"}
|
||||
{"type":"request/header","seq":0,"time":3,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}}
|
||||
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"prompt": "respond",
|
||||
"logs": [{
|
||||
"file": "b/main.jsonl",
|
||||
"lines": [
|
||||
{ "type": "session", "id": "{{SID}}", "createdAt": 600, "cwd": "{{CWD}}" },
|
||||
{ "type": "request/header", "seq": 0, "time": 4, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } }
|
||||
]
|
||||
}]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "prompt", "text": "rec pin" }] }
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
{"type":"session","id":"ccdc749f-56f3-4267-9750-598b5c60b7b2","createdAt":600,"cwd":"/var/folders/2g/b32ct0qn1d728l_v6tdkjytr0000gn/T/acp-snap-cwd-nOQ4Gy"}
|
||||
{"type":"request/header","seq":0,"time":4,"data":{"header":{"config":{"model":"fake"},"system":"SYS PROMPT","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}}
|
||||
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{}
|
||||
@@ -0,0 +1 @@
|
||||
{ "steps": [{ "op": "initialize" }] }
|
||||
+1
@@ -0,0 +1 @@
|
||||
[{ "kind": "hang" }]
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"type":"session","id":"{{sessionId}}","createdAt":0,"cwd":"{{cwd}}"}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"prompt": "error",
|
||||
"logs": [{
|
||||
"file": "b/main.jsonl",
|
||||
"lines": [
|
||||
{ "type": "session", "id": "{{SID}}", "createdAt": 500, "cwd": "{{CWD}}" },
|
||||
{ "type": "turn/end", "seq": 1, "time": 9, "data": { "error": "model exploded" } }
|
||||
]
|
||||
}]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "promptExpectError", "text": "boom" }] }
|
||||
+1
@@ -0,0 +1 @@
|
||||
[{ "kind": "throw", "chunks": [], "message": "model exploded", "code": "PROVIDER" }]
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
{"type":"session","id":"44444444-3333-4222-8111-000000000000","createdAt":17,"cwd":"/rec/authored-cwd"}
|
||||
{"type":"turn/end","seq":1,"time":17,"data":{"error":"model exploded"}}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}}
|
||||
{"jsonrpc":"2.0","id":3,"error":{"code":-32603,"message":"model exploded"}}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"prompt": "error",
|
||||
"logs": [{
|
||||
"file": "b/main.jsonl",
|
||||
"lines": [
|
||||
{ "type": "session", "id": "{{SID}}", "createdAt": 400, "cwd": "{{CWD}}" },
|
||||
{ "type": "hook/result", "seq": 1, "time": 8, "data": { "decision": "block", "durationMs": 37 } }
|
||||
]
|
||||
}]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "promptExpectError", "text": "blocked" }] }
|
||||
@@ -0,0 +1,2 @@
|
||||
{"type":"session","id":"99999999-8888-4777-8666-555555555555","createdAt":13,"cwd":"/rec/blocked-cwd"}
|
||||
{"type":"hook/result","seq":1,"time":13,"data":{"decision":"block","durationMs":99}}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}}
|
||||
{"jsonrpc":"2.0","id":3,"error":{"code":-32603,"message":"model exploded"}}
|
||||
@@ -0,0 +1 @@
|
||||
{}
|
||||
@@ -0,0 +1 @@
|
||||
{ "steps": [{ "op": "initialize" }] }
|
||||
@@ -0,0 +1 @@
|
||||
{"type":"session","id":"{{sessionId}}","createdAt":0,"cwd":"{{cwd}}"}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"prompt": "respond",
|
||||
"logs": [{
|
||||
"file": "b/main.jsonl",
|
||||
"lines": [
|
||||
{ "type": "session", "id": "{{SID}}", "createdAt": 100, "cwd": "{{CWD}}" },
|
||||
{ "type": "request/header", "seq": 0, "time": 100, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } },
|
||||
{ "type": "turn/start", "seq": 1, "time": 100, "data": { "turn": 1 } }
|
||||
]
|
||||
}]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "prompt", "text": "pin" }] }
|
||||
@@ -0,0 +1,3 @@
|
||||
{"type":"session","id":"12121212-3434-4545-8686-787878787878","createdAt":7,"cwd":"/rec/pin-cwd"}
|
||||
{"type":"request/header","seq":0,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"SYS PROMPT","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}}
|
||||
{"type":"turn/start","seq":1,"time":7,"data":{"turn":1}}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}}
|
||||
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"prompt": "respond",
|
||||
"echoWorkspace": true,
|
||||
"logs": [
|
||||
{ "file": "b/parent.jsonl", "lines": [
|
||||
{ "type": "session", "id": "{{SID}}", "createdAt": 200, "cwd": "{{CWD}}" },
|
||||
{ "type": "request/header", "seq": 0, "time": 5, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } },
|
||||
{ "type": "assistant/chunk", "seq": 1, "time": 5, "data": { "turn": 1, "step": 1, "chunk": { "type": "text-delta", "index": 0, "text": "hi" } } }
|
||||
]},
|
||||
{ "file": "b/child.jsonl", "lines": [
|
||||
{ "type": "session", "id": "eeeeeeee-1111-4222-8333-444444444444", "createdAt": 300, "cwd": "{{CWD}}", "parentSession": "{{SID}}" },
|
||||
{ "type": "request/header", "seq": 0, "time": 6, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } }
|
||||
]}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "prompt", "text": "plain" }] }
|
||||
@@ -0,0 +1,2 @@
|
||||
{"type":"session","id":"eeeeeeee-1111-4222-8333-444444444444","createdAt":12,"cwd":"/rec/plain-cwd","parentSession":"56565656-7878-4989-8a9a-9b9b9b9b9b9b"}
|
||||
{"type":"request/header","seq":0,"time":12,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}
|
||||
@@ -0,0 +1,3 @@
|
||||
{"type":"session","id":"56565656-7878-4989-8a9a-9b9b9b9b9b9b","createdAt":11,"cwd":"/rec/plain-cwd"}
|
||||
{"type":"request/header","seq":0,"time":11,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}
|
||||
{"type":"assistant/chunk","seq":1,"time":11,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":0,"text":"hi"}}}
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"workspace:seed.txt"}}}}
|
||||
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
|
||||
+1
@@ -0,0 +1 @@
|
||||
seeded
|
||||
@@ -0,0 +1,272 @@
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { delimiter, join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { afterAll, describe, expect, it } from 'vitest'
|
||||
import { runScenario, type AgentUnderTest, type InputStep } from '../src/harness.ts'
|
||||
|
||||
/**
|
||||
* Unit tests for the subprocess harness, driven through the REAL spawn path
|
||||
* (tsx loader, temp cwd, env plumbing) against the scripted fake ACP bin in
|
||||
* ./fixtures/fake-acp-agent.ts. Each case writes a `behavior.json` next to a
|
||||
* throwaway fixture path; the fake bin echoes observable facts (env, seeded
|
||||
* workspace, permission outcomes) into `agent_message_chunk` text, so the
|
||||
* assertions read plain `rawStdout`.
|
||||
*/
|
||||
|
||||
const AGENT: AgentUnderTest = {
|
||||
binScript: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)),
|
||||
// The fake bin ignores its config argv; any real path documents the shape.
|
||||
configPath: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)),
|
||||
tsconfigPath: fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url)),
|
||||
}
|
||||
|
||||
/** Temp scenario dirs to drop after the suite. */
|
||||
const tempDirs: string[] = []
|
||||
afterAll(async () => {
|
||||
for (const dir of tempDirs) await rm(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
/** Write a behavior.json into a fresh temp dir; return the sibling fixture path the harness points the bin at. */
|
||||
async function scenario(behavior: object): Promise<{ dir: string; fixtureFile: string }> {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'acp-snap-spec-'))
|
||||
tempDirs.push(dir)
|
||||
await writeFile(join(dir, 'behavior.json'), JSON.stringify(behavior))
|
||||
return { dir, fixtureFile: join(dir, 'session.jsonl') }
|
||||
}
|
||||
|
||||
const boot: InputStep[] = [{ op: 'initialize' }, { op: 'newSession' }]
|
||||
|
||||
describe('runScenario', () => {
|
||||
it('drives a full turn: initialize (terminal caps), session, prompt, permission stub, harvest', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({
|
||||
permissionProbe: true,
|
||||
logs: [{
|
||||
file: 'bucket/main.jsonl',
|
||||
lines: [
|
||||
{ type: 'session', id: '{{SID}}', createdAt: 42, cwd: '{{CWD}}' },
|
||||
{ type: 'turn/start', seq: 1, time: 9, data: { turn: 1 } },
|
||||
],
|
||||
}],
|
||||
})
|
||||
const result = await runScenario(
|
||||
{ steps: [{ op: 'initialize', terminalOutput: true }, { op: 'newSession' }, { op: 'prompt', text: 'go' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
expect(result.sessionId).toBeDefined()
|
||||
// The harness's client answers a permission request with `cancelled`; the
|
||||
// fake bin echoes the outcome it received back as a chunk.
|
||||
expect(result.rawStdout).toContain('permission:{\\"outcome\\":\\"cancelled\\"}')
|
||||
expect(result.sessionLogs).toHaveLength(1)
|
||||
expect(result.sessionLogs[0]?.id).toBe(result.sessionId)
|
||||
expect(result.sessionLogs[0]?.createdAt).toBe(42)
|
||||
expect(result.sessionLogs[0]?.content).toContain('turn/start')
|
||||
// The harvested log embeds the run's REAL temp cwd (template-substituted).
|
||||
expect(result.sessionLogs[0]?.content).toContain(result.cwd)
|
||||
})
|
||||
|
||||
it('forwards override/child fixture paths into the child env and captures stderr', { timeout: 20_000 }, async () => {
|
||||
const { dir, fixtureFile } = await scenario({ echoEnv: true, stderrNote: 'fake bin booted' })
|
||||
const childFiles = [join(dir, 'session.1.jsonl'), join(dir, 'session.2.jsonl')]
|
||||
const result = await runScenario(
|
||||
{ steps: [...boot, { op: 'prompt', text: 'env?' }] },
|
||||
{
|
||||
agent: AGENT,
|
||||
mode: 'replay',
|
||||
fixtureFile,
|
||||
overrideFile: join(dir, 'replay.override.json'),
|
||||
childFiles,
|
||||
// A workspaceDir that does not exist is skipped, not an error.
|
||||
workspaceDir: join(dir, 'no-such-workspace'),
|
||||
},
|
||||
)
|
||||
expect(result.stderr).toContain('fake bin booted')
|
||||
expect(result.rawStdout).toContain('replay.override.json')
|
||||
// Child paths ride one env var, joined with the platform delimiter.
|
||||
expect(result.rawStdout).toContain(JSON.stringify(childFiles.join(delimiter)).slice(1, -1))
|
||||
})
|
||||
|
||||
it('seeds the workspace dir into the temp cwd before the run', { timeout: 20_000 }, async () => {
|
||||
const { dir, fixtureFile } = await scenario({ echoWorkspace: true })
|
||||
const workspaceDir = join(dir, 'workspace')
|
||||
await writeFile(join(dir, 'behavior.json'), JSON.stringify({ echoWorkspace: true }))
|
||||
const { mkdir } = await import('node:fs/promises')
|
||||
await mkdir(workspaceDir, { recursive: true })
|
||||
await writeFile(join(workspaceDir, 'seeded.txt'), 'hello')
|
||||
const result = await runScenario(
|
||||
{ steps: [...boot, { op: 'prompt', text: 'ls' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile, workspaceDir },
|
||||
)
|
||||
expect(result.rawStdout).toContain('workspace:seeded.txt')
|
||||
})
|
||||
|
||||
it('promptAndCancel waits for the streamed chunk, cancels, and settles the prompt', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ prompt: 'hang-until-cancel' })
|
||||
const result = await runScenario(
|
||||
{ steps: [...boot, { op: 'promptAndCancel', text: 'hang' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
expect(result.rawStdout).toContain('"stopReason":"cancelled"')
|
||||
// The streamed chunk deterministically precedes the cancelled response.
|
||||
expect(result.rawStdout.indexOf('thinking about it')).toBeLessThan(result.rawStdout.indexOf('cancelled'))
|
||||
})
|
||||
|
||||
it('promptExpectError swallows a model-error response as the expected outcome', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ prompt: 'error' })
|
||||
const result = await runScenario(
|
||||
{ steps: [...boot, { op: 'promptExpectError', text: 'boom' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
expect(result.rawStdout).toContain('model exploded')
|
||||
})
|
||||
|
||||
it('promptExpectError throws when the prompt unexpectedly succeeds (and teardown kills the live child)', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ prompt: 'respond' })
|
||||
await expect(runScenario(
|
||||
{ steps: [...boot, { op: 'promptExpectError', text: 'fine' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)).rejects.toThrow(/expected the prompt to fail/)
|
||||
})
|
||||
|
||||
it('newSessionExpectError swallows the rejection, with and without extra dirs', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ rejectExtraDirs: true })
|
||||
const result = await runScenario(
|
||||
{ steps: [{ op: 'initialize' }, { op: 'newSessionExpectError', additionalDirectories: ['/elsewhere'] }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
// No session was created, so no id and no logs.
|
||||
expect(result.sessionId).toBeUndefined()
|
||||
expect(result.sessionLogs).toHaveLength(0)
|
||||
|
||||
const rejectAll = await scenario({ rejectNewSession: true })
|
||||
const second = await runScenario(
|
||||
{ steps: [{ op: 'initialize' }, { op: 'newSessionExpectError' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile: rejectAll.fixtureFile },
|
||||
)
|
||||
expect(second.rawStdout).toContain('unsupported workspace scope')
|
||||
})
|
||||
|
||||
it('newSessionExpectError throws when session/new unexpectedly succeeds', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({})
|
||||
await expect(runScenario(
|
||||
{ steps: [{ op: 'initialize' }, { op: 'newSessionExpectError' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)).rejects.toThrow(/expected session\/new to be rejected/)
|
||||
})
|
||||
|
||||
it('a plain cancel step is forwarded (and ignored by an idle agent)', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({})
|
||||
const result = await runScenario(
|
||||
{ steps: [...boot, { op: 'cancel' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
expect(result.sessionId).toBeDefined()
|
||||
})
|
||||
|
||||
it.each([
|
||||
[{ op: 'prompt', text: 'x' }, /prompt before newSession/],
|
||||
[{ op: 'promptExpectError', text: 'x' }, /promptExpectError before newSession/],
|
||||
[{ op: 'promptAndCancel', text: 'x' }, /promptAndCancel before newSession/],
|
||||
[{ op: 'cancel' }, /cancel before newSession/],
|
||||
] as [InputStep, RegExp][])('rejects %j before newSession', { timeout: 20_000 }, async (step, message) => {
|
||||
const { fixtureFile } = await scenario({})
|
||||
await expect(runScenario(
|
||||
{ steps: [{ op: 'initialize' }, step] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)).rejects.toThrow(message)
|
||||
})
|
||||
|
||||
it('rejects an unknown input op', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({})
|
||||
const bogus = { op: 'reticulate' } as unknown as InputStep
|
||||
await expect(runScenario(
|
||||
{ steps: [bogus] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)).rejects.toThrow(/unknown input op/)
|
||||
})
|
||||
|
||||
it('harvests all logs primary-first, children by createdAt then id, skipping filesystem noise', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({
|
||||
strayRootFile: true,
|
||||
strayBucketFile: true,
|
||||
logs: [
|
||||
// File names chosen so readdir feeds the sort children-first AND
|
||||
// parent-in-the-middle: the comparator then sees a parent on both
|
||||
// sides of a pair, plus the same-createdAt (localeCompare) tiebreak.
|
||||
{ file: 'b1/aa-child-c.jsonl', lines: [{ type: 'session', id: 'cccccccc-0000-4000-8000-000000000000', createdAt: 500, parentSession: '{{SID}}' }] },
|
||||
{ file: 'b1/bb-parent.jsonl', lines: [{ type: 'session', id: '{{SID}}', createdAt: 900 }] },
|
||||
{ file: 'b1/cc-child-a.jsonl', lines: [{ type: 'session', id: 'aaaaaaaa-0000-4000-8000-000000000000', createdAt: 500, parentSession: '{{SID}}' }] },
|
||||
// Missing id/createdAt fall back to ''/0; earliest child by createdAt.
|
||||
{ file: 'b2/orphan-fields.jsonl', lines: [{ type: 'session', parentSession: '{{SID}}' }] },
|
||||
],
|
||||
})
|
||||
const result = await runScenario(
|
||||
{ steps: [...boot, { op: 'prompt', text: 'go' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
expect(result.sessionLogs.map(l => [l.id, l.createdAt])).toEqual([
|
||||
[result.sessionId, 900],
|
||||
['', 0],
|
||||
['aaaaaaaa-0000-4000-8000-000000000000', 500],
|
||||
['cccccccc-0000-4000-8000-000000000000', 500],
|
||||
])
|
||||
expect(result.sessionLogs[1]?.parentSession).toBe(result.sessionId)
|
||||
})
|
||||
|
||||
it('treats an empty log file as a header-less primary with default fields', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ logs: [{ file: 'b/empty.jsonl', lines: [] }] })
|
||||
const result = await runScenario(
|
||||
{ steps: boot },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
expect(result.sessionLogs.map(l => [l.id, l.createdAt, l.parentSession])).toEqual([['', 0, undefined]])
|
||||
})
|
||||
|
||||
it('yields no logs when the sessions root vanished', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ deleteSessionsRoot: true })
|
||||
const result = await runScenario(
|
||||
{ steps: boot },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
expect(result.sessionLogs).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('answers permission requests from the scripted queue by option kind, falling back to cancelled', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ permissionProbe: true })
|
||||
// Two prompts → two permission round-trips; one scripted answer, so the
|
||||
// second request exercises the exhausted-queue fallback.
|
||||
const result = await runScenario(
|
||||
{
|
||||
steps: [...boot, { op: 'prompt', text: 'one' }, { op: 'prompt', text: 'two' }],
|
||||
permissionAnswers: [{ kind: 'allow_once' }],
|
||||
},
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
const first = result.rawStdout.indexOf('permission:{\\"outcome\\":\\"selected\\",\\"optionId\\":\\"opt-allow\\"}')
|
||||
const second = result.rawStdout.indexOf('permission:{\\"outcome\\":\\"cancelled\\"}')
|
||||
expect(first).toBeGreaterThanOrEqual(0)
|
||||
expect(second).toBeGreaterThan(first)
|
||||
})
|
||||
|
||||
it('selects a non-first offered option by kind', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ permissionProbe: true })
|
||||
const result = await runScenario(
|
||||
{ steps: [...boot, { op: 'prompt', text: 'deny it' }], permissionAnswers: [{ kind: 'reject_once' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)
|
||||
expect(result.rawStdout).toContain('permission:{\\"outcome\\":\\"selected\\",\\"optionId\\":\\"opt-reject\\"}')
|
||||
})
|
||||
|
||||
it('rejects the run on a scripted permission kind the agent never offered', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({ permissionProbe: true })
|
||||
// The fake bin offers allow_once/reject_once; scripting allow_always is a
|
||||
// scenario bug. The agent is answered `cancelled` (it must not be able to
|
||||
// absorb the bug as an error-means-denial), and the RUN fails: a callback
|
||||
// throw would only reach the agent as a JSON-RPC error response, letting
|
||||
// a tolerant agent carry on and the scenario pass — or record.
|
||||
await expect(runScenario(
|
||||
{ steps: [...boot, { op: 'prompt', text: 'impossible click' }], permissionAnswers: [{ kind: 'allow_always' }] },
|
||||
{ agent: AGENT, mode: 'replay', fixtureFile },
|
||||
)).rejects.toThrow(/allow_always not among the offered options \[allow_once, reject_once\]/)
|
||||
})
|
||||
})
|
||||
+47
-2
@@ -1,9 +1,9 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from '../tests/snapshot-normalize.ts'
|
||||
import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from '../src/normalize.ts'
|
||||
|
||||
/**
|
||||
* Unit tests for the pure snapshot normalizers. Live as a *.spec.ts (runs in
|
||||
* the default unit gate) and import the harness-side normalizers directly.
|
||||
* the default unit gate) and import the normalizers directly.
|
||||
*/
|
||||
|
||||
const ctx: NormalizeContext = {
|
||||
@@ -107,6 +107,17 @@ describe('normalizeSessionLog', () => {
|
||||
const out = normalizeSessionLog(`${header({})}\n${ev}\n`, ctx)
|
||||
expect(out).toContain('"durationMs":88')
|
||||
})
|
||||
|
||||
it('tolerates records missing the volatile fields it would zero', () => {
|
||||
const bareHeader = JSON.stringify({ type: 'session', id: 's' })
|
||||
const timeless = JSON.stringify({ type: 'note', seq: 1 })
|
||||
const bareHook = JSON.stringify({ type: 'hook/result', seq: 2, time: 5, data: { decision: 'allow' } })
|
||||
const nullDataHook = JSON.stringify({ type: 'hook/result', seq: 3, time: 6, data: null })
|
||||
const out = normalizeSessionLog(`${bareHeader}\n${timeless}\n${bareHook}\n${nullDataHook}\n`, ctx)
|
||||
expect(out).toContain('"type":"note","seq":1')
|
||||
expect(out).toContain('"decision":"allow"')
|
||||
expect(out).not.toContain('durationMs')
|
||||
})
|
||||
})
|
||||
|
||||
describe('scrubRequestHeaders', () => {
|
||||
@@ -135,6 +146,40 @@ describe('scrubRequestHeaders', () => {
|
||||
expect(out).not.toContain('{{tools}}')
|
||||
})
|
||||
|
||||
it('scrubs a header carrying only one of system/tools, leaving the other absent', () => {
|
||||
const systemOnly = scrubRequestHeaders(`${headerLine}\n${headerEvent({ system: 'secret prompt' })}\n`)
|
||||
expect(systemOnly).toContain('"system":"{{system}}"')
|
||||
expect(systemOnly).not.toContain('{{tools}}')
|
||||
const toolsOnly = scrubRequestHeaders(`${headerLine}\n${headerEvent({ tools: [{ name: 't' }] })}\n`)
|
||||
expect(toolsOnly).toContain('"tools":"{{tools}}"')
|
||||
expect(toolsOnly).not.toContain('{{system}}')
|
||||
})
|
||||
|
||||
it('leaves a delta with no scrubbable payload byte-identical (config-only, or non-array shapes)', () => {
|
||||
const configOnly = JSON.stringify({ type: 'request/header-delta', seq: 8, time: 9, data: { config: { model: 'm2' } } })
|
||||
const oddShapes = JSON.stringify({ type: 'request/header-delta', seq: 9, time: 9, data: { system: { insert: 'not-an-array' }, tools: null } })
|
||||
const headerless = JSON.stringify({ type: 'request/header', seq: 10, time: 9, data: { reason: 'initial' } })
|
||||
const nullData = JSON.stringify({ type: 'request/header', seq: 11, time: 9, data: null })
|
||||
const raw = `${headerLine}\n${configOnly}\n${oddShapes}\n${headerless}\n${nullData}\n`
|
||||
expect(scrubRequestHeaders(raw)).toBe(raw)
|
||||
})
|
||||
|
||||
it('scrubs a one-sided tools delta and passes non-object schema entries through', () => {
|
||||
const addedOnly = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { tools: { added: [null, 'weird', { name: 'x', description: 'D' }] } },
|
||||
})
|
||||
const out = scrubRequestHeaders(`${headerLine}\n${addedOnly}\n`)
|
||||
// Non-object entries survive untouched; the object entry keeps only name.
|
||||
expect(out).toContain('"added":[null,"weird",{"name":"x","description":"{{tools}}"}]')
|
||||
const changedOnly = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
data: { tools: { changed: [{ name: 'y', parameters: {} }] } },
|
||||
})
|
||||
expect(scrubRequestHeaders(`${headerLine}\n${changedOnly}\n`))
|
||||
.toContain('"changed":[{"name":"y","parameters":"{{tools}}"}]')
|
||||
})
|
||||
|
||||
it('scrubs a header-delta system payload but keeps its line positions and arity', () => {
|
||||
const delta = JSON.stringify({
|
||||
type: 'request/header-delta', seq: 8, time: 9,
|
||||
@@ -0,0 +1,145 @@
|
||||
import { cpSync, mkdtempSync } from 'node:fs'
|
||||
import { rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { afterAll, describe, expect, it } from 'vitest'
|
||||
import { defineAcpSnapshotSuite, type Scenario } from '../src/index.ts'
|
||||
import { childFixturePaths, fixtureContext, headerDeltaCount, normalizedHeaders } from '../src/suite.ts'
|
||||
|
||||
/**
|
||||
* Unit tests for the suite factory, by running it: two synthetic suites over
|
||||
* the scripted fake ACP bin (./fixtures/fake-acp-agent.ts) register REAL
|
||||
* describe/it trees at collection time, so every factory path — golden and log
|
||||
* compares, the per-suite header pin and its uniformity guard, record-mode
|
||||
* fixture write-back, skip semantics, and the fixture guard block — executes
|
||||
* as an ordinary green test. The pure helpers get direct cases below.
|
||||
*
|
||||
* The replay suite runs against the committed fixtures in ./fixtures/suite.
|
||||
* The record suite runs against a TEMP COPY of ./fixtures/record-suite
|
||||
* (record mode writes session fixtures back into its snapshots dir; a run must
|
||||
* never touch the committed tree). To re-bootstrap the record tree's goldens
|
||||
* after changing the fake bin's output, run this spec once with
|
||||
* `ACP_SNAPSHOT_SPEC_BOOTSTRAP=1` (points the record suite at the committed
|
||||
* tree so vitest creates/updates the goldens and the write-back lands there),
|
||||
* then commit the result.
|
||||
*/
|
||||
|
||||
const AGENT = {
|
||||
binScript: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)),
|
||||
configPath: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)),
|
||||
tsconfigPath: fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url)),
|
||||
}
|
||||
|
||||
const REPLAY_DIR = fileURLToPath(new URL('./fixtures/suite', import.meta.url))
|
||||
const RECORD_SRC = fileURLToPath(new URL('./fixtures/record-suite', import.meta.url))
|
||||
|
||||
const REPLAY_SCENARIOS: Scenario[] = [
|
||||
{ name: 'pin-turn', hasModelTurn: true, recorded: true, pinsHeader: true },
|
||||
{ name: 'plain-turn', hasModelTurn: true, recorded: true, childSessions: 1 },
|
||||
{ name: 'no-model', hasModelTurn: false, recorded: false },
|
||||
{ name: 'blocked-log', hasModelTurn: false, comparesLog: true, recorded: false },
|
||||
{ name: 'authored-error', hasModelTurn: true, recorded: false },
|
||||
]
|
||||
|
||||
const RECORD_SCENARIOS: Scenario[] = [
|
||||
{ name: 'rec-pin', hasModelTurn: true, recorded: true, pinsHeader: true },
|
||||
{ name: 'rec-child', hasModelTurn: true, recorded: true, childSessions: 1 },
|
||||
// recorded:false in record mode → registered but skipped (never re-recorded).
|
||||
{ name: 'rec-skip', hasModelTurn: true, recorded: false },
|
||||
]
|
||||
|
||||
// Record mode mutates its snapshots dir, so run it on a throwaway copy —
|
||||
// except under the documented bootstrap knob, which regenerates the committed
|
||||
// fixtures/goldens in place.
|
||||
const BOOTSTRAP = process.env.ACP_SNAPSHOT_SPEC_BOOTSTRAP === '1'
|
||||
const recordDir = BOOTSTRAP ? RECORD_SRC : mkdtempSync(join(tmpdir(), 'acp-snap-record-suite-'))
|
||||
if (!BOOTSTRAP) cpSync(RECORD_SRC, recordDir, { recursive: true })
|
||||
afterAll(async () => {
|
||||
if (!BOOTSTRAP) await rm(recordDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('defineAcpSnapshotSuite: replay mode', () => {
|
||||
defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: REPLAY_DIR, scenarios: REPLAY_SCENARIOS, mode: 'replay' })
|
||||
})
|
||||
|
||||
// The record suite's tests run in registration order: rec-pin re-records the
|
||||
// pinned fixture FIRST, so rec-child's uniformity guard reads the fresh pin.
|
||||
describe('defineAcpSnapshotSuite: record mode', () => {
|
||||
defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: recordDir, scenarios: RECORD_SCENARIOS, mode: 'record' })
|
||||
})
|
||||
|
||||
describe('defineAcpSnapshotSuite: registration contract', () => {
|
||||
it('throws when no scenario pins the request-header content', () => {
|
||||
expect(() => {
|
||||
defineAcpSnapshotSuite({
|
||||
agent: AGENT,
|
||||
snapshotsDir: REPLAY_DIR,
|
||||
scenarios: [{ name: 'pinless', hasModelTurn: true, recorded: true }],
|
||||
mode: 'replay',
|
||||
})
|
||||
}).toThrow(/no scenario pins/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('childFixturePaths', () => {
|
||||
it('yields one sibling path per child, 1-based', () => {
|
||||
expect(childFixturePaths('/snap/s', 2)).toEqual(['/snap/s/session.1.jsonl', '/snap/s/session.2.jsonl'])
|
||||
})
|
||||
|
||||
it('yields nothing for a single-session scenario', () => {
|
||||
expect(childFixturePaths('/snap/s', 0)).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('fixtureContext', () => {
|
||||
it('reads the fixture header id and cwd', () => {
|
||||
const ctx = fixtureContext('{"type":"session","id":"abc","cwd":"/rec"}\n{"type":"turn/start"}\n')
|
||||
expect(ctx).toEqual({ sessionIds: ['abc'], cwd: '/rec' })
|
||||
})
|
||||
|
||||
it('yields no session ids for a header without a string id', () => {
|
||||
expect(fixtureContext('{"type":"session","cwd":"/rec"}\n').sessionIds).toEqual([])
|
||||
})
|
||||
|
||||
it('falls back to an impossible sentinel cwd (never the empty string)', () => {
|
||||
const ctx = fixtureContext('{"type":"session","id":"abc"}\n')
|
||||
expect(ctx.cwd).toBe('\0no-cwd\0')
|
||||
expect(ctx.cwd).not.toBe('')
|
||||
})
|
||||
|
||||
it('treats an empty fixture as an empty header', () => {
|
||||
expect(fixtureContext('')).toEqual({ sessionIds: [], cwd: '\0no-cwd\0' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizedHeaders', () => {
|
||||
const header = (system: string): string => JSON.stringify({
|
||||
type: 'request/header', seq: 0, time: 9, data: { header: { config: { model: 'm' }, system }, reason: 'initial' },
|
||||
})
|
||||
|
||||
it('extracts every request/header payload in log order, normalized', () => {
|
||||
const id = '11111111-2222-4333-8444-555555555555'
|
||||
const log = `${JSON.stringify({ type: 'session', id, createdAt: 5, cwd: '/w' })}\n${header('one')}\n`
|
||||
+ `${JSON.stringify({ type: 'turn/start', seq: 1, time: 9, data: { turn: 1 } })}\n${header('two')}\n`
|
||||
const headers = normalizedHeaders(log, { sessionIds: [id], cwd: '/w' })
|
||||
expect(headers).toEqual([
|
||||
{ config: { model: 'm' }, system: 'one' },
|
||||
{ config: { model: 'm' }, system: 'two' },
|
||||
])
|
||||
})
|
||||
|
||||
it('yields nothing for a log without header events', () => {
|
||||
const log = `${JSON.stringify({ type: 'session', id: 'a', createdAt: 5 })}\n`
|
||||
expect(normalizedHeaders(log, { sessionIds: [], cwd: '/w' })).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('headerDeltaCount', () => {
|
||||
it('counts request/header-delta events, ignoring blanks and other lines', () => {
|
||||
const delta = JSON.stringify({ type: 'request/header-delta', seq: 2, time: 9, data: {} })
|
||||
const other = JSON.stringify({ type: 'request/header', seq: 0, time: 9, data: {} })
|
||||
expect(headerDeltaCount(`${other}\n\n${delta}\n${delta}\n`)).toBe(2)
|
||||
expect(headerDeltaCount(`${other}\n`)).toBe(0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"extends": "../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "src",
|
||||
"outDir": "lib/types"
|
||||
},
|
||||
"include": [
|
||||
"src"
|
||||
],
|
||||
"references": []
|
||||
}
|
||||
Generated
+68
@@ -748,6 +748,22 @@ importers:
|
||||
specifier: ^4.0.0-rc.6
|
||||
version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4)
|
||||
|
||||
packages/support/acp-snapshot:
|
||||
dependencies:
|
||||
'@agentclientprotocol/sdk':
|
||||
specifier: 0.25.1
|
||||
version: 0.25.1(zod@4.4.3)
|
||||
tsx:
|
||||
specifier: ^4.22.4
|
||||
version: 4.22.4
|
||||
vitest:
|
||||
specifier: ^4.1.8
|
||||
version: 4.1.8(@types/node@25.9.3)(@vitest/coverage-v8@4.1.8)(jsdom@29.1.1)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0))
|
||||
devDependencies:
|
||||
cordis:
|
||||
specifier: ^4.0.0-rc.6
|
||||
version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4)
|
||||
|
||||
packages/support/invariants:
|
||||
devDependencies:
|
||||
'@deepseek-ai/dsh-agent':
|
||||
@@ -5241,6 +5257,14 @@ snapshots:
|
||||
optionalDependencies:
|
||||
vite: 8.0.16(@types/node@22.20.0)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)
|
||||
|
||||
'@vitest/mocker@4.1.8(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0))':
|
||||
dependencies:
|
||||
'@vitest/spy': 4.1.8
|
||||
estree-walker: 3.0.3
|
||||
magic-string: 0.30.21
|
||||
optionalDependencies:
|
||||
vite: 8.0.16(@types/node@25.9.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)
|
||||
|
||||
'@vitest/pretty-format@4.1.8':
|
||||
dependencies:
|
||||
tinyrainbow: 3.1.0
|
||||
@@ -6891,6 +6915,21 @@ snapshots:
|
||||
tsx: 4.22.4
|
||||
yaml: 2.9.0
|
||||
|
||||
vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0):
|
||||
dependencies:
|
||||
lightningcss: 1.32.0
|
||||
picomatch: 4.0.4
|
||||
postcss: 8.5.15
|
||||
rolldown: 1.0.3
|
||||
tinyglobby: 0.2.17
|
||||
optionalDependencies:
|
||||
'@types/node': 25.9.3
|
||||
esbuild: 0.28.1
|
||||
fsevents: 2.3.3
|
||||
jiti: 2.7.0
|
||||
tsx: 4.22.4
|
||||
yaml: 2.9.0
|
||||
|
||||
vitest@4.1.8(@types/node@22.20.0)(@vitest/coverage-v8@4.1.8)(jsdom@29.1.1)(vite@8.0.16(@types/node@22.20.0)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)):
|
||||
dependencies:
|
||||
'@vitest/expect': 4.1.8
|
||||
@@ -6920,6 +6959,35 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- msw
|
||||
|
||||
vitest@4.1.8(@types/node@25.9.3)(@vitest/coverage-v8@4.1.8)(jsdom@29.1.1)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)):
|
||||
dependencies:
|
||||
'@vitest/expect': 4.1.8
|
||||
'@vitest/mocker': 4.1.8(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0))
|
||||
'@vitest/pretty-format': 4.1.8
|
||||
'@vitest/runner': 4.1.8
|
||||
'@vitest/snapshot': 4.1.8
|
||||
'@vitest/spy': 4.1.8
|
||||
'@vitest/utils': 4.1.8
|
||||
es-module-lexer: 2.1.0
|
||||
expect-type: 1.3.0
|
||||
magic-string: 0.30.21
|
||||
obug: 2.1.3
|
||||
pathe: 2.0.3
|
||||
picomatch: 4.0.4
|
||||
std-env: 4.1.0
|
||||
tinybench: 2.9.0
|
||||
tinyexec: 1.2.4
|
||||
tinyglobby: 0.2.17
|
||||
tinyrainbow: 3.1.0
|
||||
vite: 8.0.16(@types/node@25.9.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)
|
||||
why-is-node-running: 2.3.0
|
||||
optionalDependencies:
|
||||
'@types/node': 25.9.3
|
||||
'@vitest/coverage-v8': 4.1.8(vitest@4.1.8)
|
||||
jsdom: 29.1.1
|
||||
transitivePeerDependencies:
|
||||
- msw
|
||||
|
||||
w3c-xmlserializer@5.0.0:
|
||||
dependencies:
|
||||
xml-name-validator: 5.0.0
|
||||
|
||||
@@ -44,6 +44,7 @@
|
||||
{ "path": "./packages/ui/app-boot" },
|
||||
{ "path": "./packages/ui/stdio-agent" },
|
||||
{ "path": "./packages/support/llm-replay" },
|
||||
{ "path": "./packages/support/acp-snapshot" },
|
||||
{ "path": "./packages/subagent/subagent" },
|
||||
{ "path": "./packages/support/subagent-mock" },
|
||||
{ "path": "./packages/subagent/tool-subagent" },
|
||||
|
||||
@@ -55,6 +55,7 @@
|
||||
{ "path": "./packages/ui/app-boot" },
|
||||
{ "path": "./packages/ui/stdio-agent" },
|
||||
{ "path": "./packages/support/llm-replay" },
|
||||
{ "path": "./packages/support/acp-snapshot" },
|
||||
{ "path": "./packages/subagent/subagent" },
|
||||
{ "path": "./packages/support/subagent-mock" },
|
||||
{ "path": "./packages/subagent/tool-subagent" },
|
||||
|
||||
Reference in New Issue
Block a user