feat(acp-example): snapshot harness, normalizers, wiring, and handshake scenario
Adds the snapshot-test harness and the keyless replay pipeline end-to-end. - snapshot-harness.ts: boots the real acp-agent subprocess via the cordis Loader (preserving TSX_TSCONFIG_PATH so unbuilt dsh-* imports resolve from a temp cwd), tees raw stdout into an SDK ClientSideConnection, interprets a per-scenario input.json DSL (initialize / newSession capturing the random sessionId / prompt / cancel), closes stdin to trigger graceful shutdown, and harvests the persisted session.jsonl. Failure-safe: a finally block SIGKILLs a live child, awaits its exit, and removes both temp dirs even on a thrown step or harvest. Raw bytes are buffered and decoded once (no multibyte split). - snapshot-normalize.ts (+ spec): two pure normalizers (stdout frames + session JSONL) scrub cwd, session ids / UUIDs, and JSON-RPC ids, and zero time / createdAt — but keep `seq` (deterministic by contract). normalizeStdout throws on a non-JSON line (the stdout-purity check). - start.ts: selects cordis.snapshot.yml (replay, providerless) or cordis.snapshot-record.yml (record, real adapter) from DSH_SNAPSHOT, skips .env in replay, and disposes the ctx on stdin end so persistence flushes before exit (harvest-after-flush, not on the prompt response). - acp.snapshot.ts: asserts the normalized stdout golden (and, for model scenarios, the re-persisted JSONL golden) via toMatchFileSnapshot; record mode writes the harvested log back to the scenario fixture; an orphan-fixture guard fails on an unregistered scenario dir. - handshake scenario: initialize + session/new (no model call; a header-only session.jsonl, since session/new persists no events). - vitest.snapshot.config.ts, test:snapshot / test:snapshot:record scripts, a pre-push snapshot job, and the knip entry. Incorporates Codex review: record-fixture writeback, failure-safe teardown, seq-not-scrubbed, harvest-after-flush. Per docs/rfc/implemented/2026-06-19.
This commit is contained in:
Tianyi Cui
@@ -0,0 +1,43 @@
|
||||
# Snapshot-test RECORD config: a real run whose persisted session JSONL is
|
||||
# harvested into a scenario fixture. Identical to cordis.yml (real llm-deepseek
|
||||
# adapter + JSONL persistence) — recording must exercise the REAL model so the
|
||||
# recorded log is a genuine product of the system. Needs DEEPSEEK_API_KEY.
|
||||
#
|
||||
# It is a separate file (rather than reusing cordis.yml) only so the snapshot
|
||||
# harness selects it explicitly via $DSH_SNAPSHOT=record and so its persistence
|
||||
# root can be pointed at the harness's harvest directory by the same env the
|
||||
# replay path uses. The graceful-shutdown path in start.ts flushes persistence
|
||||
# before exit so the harvested log is complete.
|
||||
|
||||
- id: timer
|
||||
name: '@cordisjs/plugin-timer'
|
||||
|
||||
# Shared provider/tool core, INCLUDING the real llm-deepseek adapter.
|
||||
- id: base
|
||||
name: '@cordisjs/plugin-include'
|
||||
config:
|
||||
path: '../base.yml'
|
||||
|
||||
- id: agent-loop
|
||||
name: '@deepseek-ai/dsh-agent-loop'
|
||||
config:
|
||||
agents: []
|
||||
|
||||
- id: session-persistence
|
||||
name: '@deepseek-ai/dsh-session-persistence-jsonl'
|
||||
config:
|
||||
root: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT
|
||||
|
||||
- id: acp
|
||||
name: '@deepseek-ai/dsh-acp'
|
||||
config:
|
||||
model: deepseek-v4-flash
|
||||
systemPrompt: |
|
||||
You are a coding assistant driven over the Agent Client Protocol.
|
||||
|
||||
Your only tools are bash (plus bash_output/bash_kill for background
|
||||
tasks). Do ALL file operations through bash: read with cat/sed/head,
|
||||
search with grep, write with heredocs (cat <<'EOF' > file), edit with
|
||||
sed or a rewrite. Each bash call runs in a fresh shell — pass workdir
|
||||
instead of cd. Check the [exit code: N] marker; verify your work. Keep
|
||||
answers brief and factual.
|
||||
@@ -0,0 +1,69 @@
|
||||
# Snapshot-test REPLAY config: the acp-agent plugin tree with the model replaced
|
||||
# by llm-replay (serves a recorded session JSONL — no API key, no network).
|
||||
#
|
||||
# This does NOT include ../base.yml: base.yml always loads
|
||||
# @deepseek-ai/dsh-llm-deepseek, whose apply() throws without DEEPSEEK_API_KEY,
|
||||
# so a keyless replay run would die at boot. We inline the providerless core
|
||||
# instead and install llm-replay where the adapter would be.
|
||||
#
|
||||
# stdout is reserved for the ACP JSON-RPC protocol — no stdout logger (see
|
||||
# cordis.yml). The replay fixture path comes from $DSH_SNAPSHOT_FILE (and an
|
||||
# optional $DSH_SNAPSHOT_OVERRIDE sidecar), set by the snapshot harness.
|
||||
|
||||
- id: timer
|
||||
name: '@cordisjs/plugin-timer'
|
||||
|
||||
# Providerless core (everything base.yml has EXCEPT llm-deepseek).
|
||||
- id: llm
|
||||
name: '@deepseek-ai/dsh-llm'
|
||||
|
||||
- id: sessions
|
||||
name: '@deepseek-ai/dsh-session'
|
||||
|
||||
- id: system-prompt
|
||||
name: '@deepseek-ai/dsh-system-prompt'
|
||||
|
||||
- id: tools
|
||||
name: '@deepseek-ai/dsh-tools'
|
||||
|
||||
- id: agents
|
||||
name: '@deepseek-ai/dsh-agent'
|
||||
|
||||
- id: invariants
|
||||
name: '@deepseek-ai/dsh-invariants'
|
||||
|
||||
- id: bash
|
||||
name: '@deepseek-ai/dsh-bash-local'
|
||||
config:
|
||||
timeoutMs: 60000
|
||||
|
||||
- id: tool-bash
|
||||
name: '@deepseek-ai/dsh-tool-bash'
|
||||
|
||||
# The replay adapter: short-circuits llm/stream with the recorded log's chunks.
|
||||
- id: llm-replay
|
||||
name: './src/llm-replay.ts'
|
||||
|
||||
- id: agent-loop
|
||||
name: '@deepseek-ai/dsh-agent-loop'
|
||||
config:
|
||||
agents: []
|
||||
|
||||
- id: session-persistence
|
||||
name: '@deepseek-ai/dsh-session-persistence-jsonl'
|
||||
config:
|
||||
root: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT
|
||||
|
||||
- id: acp
|
||||
name: '@deepseek-ai/dsh-acp'
|
||||
config:
|
||||
model: deepseek-v4-flash
|
||||
systemPrompt: |
|
||||
You are a coding assistant driven over the Agent Client Protocol.
|
||||
|
||||
Your only tools are bash (plus bash_output/bash_kill for background
|
||||
tasks). Do ALL file operations through bash: read with cat/sed/head,
|
||||
search with grep, write with heredocs (cat <<'EOF' > file), edit with
|
||||
sed or a rewrite. Each bash call runs in a fresh shell — pass workdir
|
||||
instead of cd. Check the [exit code: N] marker; verify your work. Keep
|
||||
answers brief and factual.
|
||||
@@ -2,20 +2,36 @@ import { pathToFileURL } from 'node:url'
|
||||
import { Context } from 'cordis'
|
||||
import Loader from '@cordisjs/plugin-loader'
|
||||
|
||||
// Snapshot-test modes (set by the snapshot harness via env):
|
||||
// DSH_SNAPSHOT=replay — load cordis.snapshot.yml (providerless; llm-replay
|
||||
// serves a recorded session log). Skip .env so a stray
|
||||
// key can never trigger a live model call.
|
||||
// DSH_SNAPSHOT=record — load cordis.snapshot-record.yml (the real adapter +
|
||||
// persistence) so a real run can be harvested.
|
||||
// Absent — the normal demo (cordis.yml), driven by a real editor.
|
||||
const snapshotMode = process.env.DSH_SNAPSHOT
|
||||
const configPath = snapshotMode === 'replay' ? './cordis.snapshot.yml'
|
||||
: snapshotMode === 'record' ? './cordis.snapshot-record.yml'
|
||||
: './cordis.yml'
|
||||
|
||||
// Load DEEPSEEK_API_KEY / DEEPSEEK_BASE_URL from a gitignored repo-root .env
|
||||
// (Node native). Absent file is fine — the environment may already carry them.
|
||||
// In REPLAY mode we deliberately skip this: replay must never reach the network,
|
||||
// so we don't want a present .env to enable a live call.
|
||||
//
|
||||
// IMPORTANT: this server speaks ACP JSON-RPC on stdout. Do NOT add any
|
||||
// stdout logging here or in cordis.yml — it would corrupt the protocol frames.
|
||||
// A present-but-unreadable/malformed .env is a real misconfiguration: surface
|
||||
// it on STDERR (never stdout) rather than silently running with the wrong env.
|
||||
try {
|
||||
process.loadEnvFile(new URL('../../.env', import.meta.url).pathname)
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException | null)?.code !== 'ENOENT') {
|
||||
process.stderr.write(`acp-agent: failed to load .env: ${String(error)}\n`)
|
||||
if (snapshotMode !== 'replay') {
|
||||
try {
|
||||
process.loadEnvFile(new URL('../../.env', import.meta.url).pathname)
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException | null)?.code !== 'ENOENT') {
|
||||
process.stderr.write(`acp-agent: failed to load .env: ${String(error)}\n`)
|
||||
}
|
||||
// ENOENT (no .env) is fine — rely on the ambient environment.
|
||||
}
|
||||
// ENOENT (no .env) is fine — rely on the ambient environment.
|
||||
}
|
||||
|
||||
const ctx = new Context()
|
||||
@@ -25,6 +41,18 @@ await ctx.plugin(Loader)
|
||||
await ctx.loader.create({
|
||||
name: '@cordisjs/plugin-include',
|
||||
config: {
|
||||
path: './cordis.yml',
|
||||
path: configPath,
|
||||
},
|
||||
})
|
||||
|
||||
// Graceful shutdown for snapshot RECORD runs: when the client closes our stdin
|
||||
// (it is done driving the session), dispose the whole context. Disposal awaits
|
||||
// the agent-loop teardown and the persistence backend's final `session/flush`,
|
||||
// so the recorded `.jsonl` is fully written before the process exits and the
|
||||
// harness harvests it. (In a normal editor session stdin stays open for the
|
||||
// connection's lifetime; the editor kills the process, so this never fires.)
|
||||
if (snapshotMode !== undefined) {
|
||||
process.stdin.on('end', () => {
|
||||
void ctx.fiber.dispose().then(() => { process.exit(0) })
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { readFile, readdir, writeFile } from 'node:fs/promises'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { type InputScript, runScenario } from './snapshot-harness.ts'
|
||||
import { type NormalizeContext, normalizeSessionLog, normalizeStdout } from './snapshot-normalize.ts'
|
||||
|
||||
/**
|
||||
* ACP snapshot tests (REPLAY by default, keyless). Each scenario under
|
||||
* `snapshots/<name>/` ships an `input.json` (the client stdin script) and a
|
||||
* recorded `session.jsonl` fixture; replay boots the real acp-agent subprocess,
|
||||
* drives it, and diffs the normalized stdout transcript (and, for model
|
||||
* scenarios, the re-persisted session log) against committed goldens.
|
||||
*
|
||||
* `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the
|
||||
* fixtures against the real API and refreshes the goldens in one pass.
|
||||
*/
|
||||
|
||||
const SNAPSHOTS_DIR = join(dirname(fileURLToPath(import.meta.url)), 'snapshots')
|
||||
const RECORDING = process.env.DSH_SNAPSHOT === 'record'
|
||||
|
||||
/** A scenario and whether it makes any model call (→ has a behavioral JSONL golden). */
|
||||
interface Scenario {
|
||||
name: string
|
||||
/** Whether the scenario drives at least one model turn (so a JSONL golden applies). */
|
||||
hasModelTurn: boolean
|
||||
}
|
||||
|
||||
const SCENARIOS: Scenario[] = [
|
||||
{ name: 'handshake', hasModelTurn: false },
|
||||
]
|
||||
|
||||
for (const scenario of SCENARIOS) {
|
||||
describe(`snapshot: ${scenario.name}`, () => {
|
||||
it('matches the stdout transcript golden', async () => {
|
||||
const dir = join(SNAPSHOTS_DIR, scenario.name)
|
||||
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
||||
const overrideFile = join(dir, 'replay.override.json')
|
||||
const result = await runScenario(input, {
|
||||
mode: RECORDING ? 'record' : 'replay',
|
||||
fixtureFile: join(dir, 'session.jsonl'),
|
||||
...existsSync(overrideFile) ? { overrideFile } : {},
|
||||
})
|
||||
|
||||
const ctx: NormalizeContext = {
|
||||
sessionIds: result.sessionId !== undefined ? [result.sessionId] : [],
|
||||
cwd: result.cwd,
|
||||
}
|
||||
|
||||
// RECORD mode: persist the freshly-harvested log back to the scenario's
|
||||
// session.jsonl fixture (a model scenario must produce one). `--update`
|
||||
// refreshes the Vitest goldens but NOT this fixture, so write it here.
|
||||
if (RECORDING && scenario.hasModelTurn) {
|
||||
expect(result.sessionLog, 'record produced no session log to harvest').toBeDefined()
|
||||
await writeFile(join(dir, 'session.jsonl'), result.sessionLog as string)
|
||||
}
|
||||
|
||||
await expect(normalizeStdout(result.rawStdout, ctx))
|
||||
.toMatchFileSnapshot(join(dir, 'stdout.golden.txt'))
|
||||
|
||||
if (scenario.hasModelTurn) {
|
||||
expect(result.sessionLog, 'a model scenario must persist a session log').toBeDefined()
|
||||
await expect(normalizeSessionLog(result.sessionLog as string, ctx))
|
||||
.toMatchFileSnapshot(join(dir, 'session.golden.txt'))
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
describe('snapshot fixtures', () => {
|
||||
it('every scenario directory is registered (no orphans)', async () => {
|
||||
// toMatchFileSnapshot does not prune orphaned golden/fixture files, so a
|
||||
// renamed/removed scenario could leave a stale dir that nothing exercises.
|
||||
// Fail loud on any snapshots/<dir> not present in SCENARIOS.
|
||||
const entries = await readdir(SNAPSHOTS_DIR, { withFileTypes: true })
|
||||
const onDisk = entries.filter(e => e.isDirectory()).map(e => e.name).sort()
|
||||
const registered = SCENARIOS.map(s => s.name).sort()
|
||||
expect(onDisk).toEqual(registered)
|
||||
})
|
||||
|
||||
it('every registered scenario has its required fixture files', async () => {
|
||||
for (const { name } of SCENARIOS) {
|
||||
const dir = join(SNAPSHOTS_DIR, name)
|
||||
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
|
||||
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
|
||||
expect(existsSync(join(dir, 'stdout.golden.txt')), `${name}/stdout.golden.txt`).toBe(true)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,231 @@
|
||||
/**
|
||||
* Shared harness for the ACP snapshot tests. A plain module (NOT a *.spec.ts /
|
||||
* *.snapshot.ts) so importing it never re-registers another file's tests.
|
||||
*
|
||||
* It boots the REAL examples/acp-agent subprocess via the cordis Loader (so the
|
||||
* export-shape bug class stays guarded — see docs/postmortem/0001), drives it
|
||||
* over real ACP JSON-RPC stdio with a deterministic input script, tees raw
|
||||
* stdout (for the golden + a purity check) into an SDK `ClientSideConnection`,
|
||||
* and — in record mode — harvests the persisted session JSONL after a graceful
|
||||
* shutdown flush. Two pure normalizers turn the captured stdout frames and the
|
||||
* session-log events into stable, snapshot-able text.
|
||||
*
|
||||
* See docs/rfc/implemented/2026-06-19-acp-snapshot-tests.md.
|
||||
*/
|
||||
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
|
||||
import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { Readable, Writable } from 'node:stream'
|
||||
import {
|
||||
ClientSideConnection,
|
||||
ndJsonStream,
|
||||
PROTOCOL_VERSION,
|
||||
type Agent as AcpAgent,
|
||||
type Client,
|
||||
type RequestPermissionRequest,
|
||||
type RequestPermissionResponse,
|
||||
type SessionNotification,
|
||||
} from '@agentclientprotocol/sdk'
|
||||
|
||||
const startScript = fileURLToPath(new URL('../start.ts', import.meta.url))
|
||||
const tsxLoader = fileURLToPath(import.meta.resolve('tsx'))
|
||||
// The repo-root tsconfig: dev/test run UNBUILT and the `@deepseek-ai/dsh-*`
|
||||
// imports resolve through its `paths` map. The child's cwd is a temp dir
|
||||
// OUTSIDE the repo, so tsx's upward search would miss it — point tsx at the
|
||||
// repo tsconfig explicitly (same fix the e2e harness uses). Repo root is four
|
||||
// levels up from this file (examples/acp-agent/tests).
|
||||
const repoTsconfig = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url))
|
||||
|
||||
/**
|
||||
* One step of a scenario's deterministic input script (`input.json`). The
|
||||
* harness interprets these in order. `newSession` captures the server-issued
|
||||
* (random) session id into a `{{sessionId}}` variable that later steps
|
||||
* reference, since a committed file cannot know the id in advance.
|
||||
*/
|
||||
type InputStep =
|
||||
| { op: 'initialize'; terminalOutput?: boolean }
|
||||
| { op: 'newSession' }
|
||||
| { op: 'prompt'; text: string }
|
||||
| { op: 'cancel' }
|
||||
|
||||
/** A scenario's `input.json`: an ordered list of input steps. */
|
||||
export interface InputScript {
|
||||
steps: InputStep[]
|
||||
}
|
||||
|
||||
/** The result of running a scenario: raw stdout + the harvested session log. */
|
||||
export interface RunResult {
|
||||
/** Raw stdout bytes (decoded utf8), every newline-delimited JSON-RPC frame. */
|
||||
rawStdout: string
|
||||
/** stderr (for diagnostics on failure). */
|
||||
stderr: string
|
||||
/** The session id the server issued (undefined if no session was created). */
|
||||
sessionId?: string
|
||||
/** The temp cwd the session ran in (the bash workspace). */
|
||||
cwd: string
|
||||
/** The persisted session log's content, if one was produced. */
|
||||
sessionLog?: string
|
||||
}
|
||||
|
||||
interface RunOptions {
|
||||
/** `replay` (default, keyless) or `record` (real API, harvests the log). */
|
||||
mode: 'replay' | 'record'
|
||||
/** The recorded session JSONL fixture path (replay reads it; record writes near it). */
|
||||
fixtureFile: string
|
||||
/** Optional sidecar override path (replay). */
|
||||
overrideFile?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a scenario end-to-end against a freshly-spawned subprocess. Owns the
|
||||
* child and its temp dirs; always tears them down. Returns the captured stdout
|
||||
* and (record mode) the harvested session-log path.
|
||||
*/
|
||||
export async function runScenario(input: InputScript, opts: RunOptions): Promise<RunResult> {
|
||||
const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
|
||||
const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
TSX_TSCONFIG_PATH: repoTsconfig,
|
||||
DSH_SNAPSHOT: opts.mode,
|
||||
DSH_SNAPSHOT_FILE: opts.fixtureFile,
|
||||
DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
|
||||
...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {},
|
||||
}
|
||||
|
||||
const child: ChildProcessWithoutNullStreams = spawn(
|
||||
process.execPath,
|
||||
['--import', tsxLoader, startScript],
|
||||
{ cwd, env, stdio: ['pipe', 'pipe', 'pipe'] },
|
||||
)
|
||||
|
||||
const rawBuffers: Buffer[] = []
|
||||
const stderrChunks: string[] = []
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stderr.on('data', (c: string) => stderrChunks.push(c))
|
||||
|
||||
// Tee raw stdout: accumulate the bytes for the golden + purity check, and ALSO
|
||||
// feed the same bytes to the SDK client through a passthrough. Buffer the raw
|
||||
// bytes (not per-chunk utf8 strings) and decode once at the end, so a
|
||||
// multibyte sequence split across two 'data' events can't corrupt the golden.
|
||||
const passthrough = new Readable({ read() {} })
|
||||
child.stdout.on('data', (buf: Buffer) => {
|
||||
rawBuffers.push(buf)
|
||||
passthrough.push(buf)
|
||||
})
|
||||
child.stdout.on('end', () => passthrough.push(null))
|
||||
|
||||
const stream = ndJsonStream(
|
||||
Writable.toWeb(child.stdin) as WritableStream<Uint8Array>,
|
||||
Readable.toWeb(passthrough) as ReadableStream<Uint8Array>,
|
||||
)
|
||||
const makeClient = (_agent: AcpAgent): Client => ({
|
||||
sessionUpdate(_params: SessionNotification): Promise<void> {
|
||||
return Promise.resolve()
|
||||
},
|
||||
requestPermission(_params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
|
||||
return Promise.resolve({ outcome: { outcome: 'cancelled' } })
|
||||
},
|
||||
})
|
||||
const client = new ClientSideConnection(makeClient, stream)
|
||||
|
||||
let sessionId: string | undefined
|
||||
let sessionLog: string | undefined
|
||||
try {
|
||||
for (const step of input.steps) {
|
||||
await runStep(client, step, cwd, () => sessionId, (id) => { sessionId = id })
|
||||
}
|
||||
// Done driving: close stdin so the server disposes gracefully (flushing
|
||||
// persistence) and exits. Then await exit so the harvested log is complete.
|
||||
child.stdin.end()
|
||||
await waitForExit(child)
|
||||
// Harvest the persisted log (if any) while the temp dirs still exist.
|
||||
const sessionLogPath = await findSessionLog(sessionsRoot)
|
||||
if (sessionLogPath !== undefined) sessionLog = await readFile(sessionLogPath, 'utf8')
|
||||
} finally {
|
||||
// Failure-safe teardown: kill a still-running child and drop the temp dirs
|
||||
// even if a step/harvest threw, so a flaky run never leaks a process or dir.
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
child.kill('SIGKILL')
|
||||
await waitForExit(child)
|
||||
}
|
||||
await rm(cwd, { recursive: true, force: true })
|
||||
await rm(sessionsRoot, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
return {
|
||||
rawStdout: Buffer.concat(rawBuffers).toString('utf8'),
|
||||
stderr: stderrChunks.join(''),
|
||||
cwd,
|
||||
...sessionId !== undefined ? { sessionId } : {},
|
||||
...sessionLog !== undefined ? { sessionLog } : {},
|
||||
}
|
||||
}
|
||||
|
||||
/** Drive one input step over the client connection. */
|
||||
async function runStep(
|
||||
client: ClientSideConnection,
|
||||
step: InputStep,
|
||||
cwd: string,
|
||||
getSessionId: () => string | undefined,
|
||||
setSessionId: (id: string) => void,
|
||||
): Promise<void> {
|
||||
switch (step.op) {
|
||||
case 'initialize':
|
||||
await client.initialize({
|
||||
protocolVersion: PROTOCOL_VERSION,
|
||||
clientCapabilities: step.terminalOutput === true ? { _meta: { terminal_output: true } } : {},
|
||||
})
|
||||
return
|
||||
case 'newSession': {
|
||||
const { sessionId } = await client.newSession({ cwd, mcpServers: [] })
|
||||
setSessionId(sessionId)
|
||||
return
|
||||
}
|
||||
case 'prompt': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: prompt before newSession')
|
||||
await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
|
||||
return
|
||||
}
|
||||
case 'cancel': {
|
||||
const sessionId = getSessionId()
|
||||
if (sessionId === undefined) throw new Error('snapshot-harness: cancel before newSession')
|
||||
await client.cancel({ sessionId })
|
||||
return
|
||||
}
|
||||
default:
|
||||
throw new Error(`snapshot-harness: unknown input op ${JSON.stringify(step)}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve once the child process exits (any code/signal). */
|
||||
function waitForExit(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve()
|
||||
return new Promise<void>(resolve => child.once('exit', () => { resolve() }))
|
||||
}
|
||||
|
||||
/** Find the single produced `.jsonl` session log under a sessions root, if any. */
|
||||
async function findSessionLog(root: string): Promise<string | undefined> {
|
||||
let cwdDirs: string[]
|
||||
try {
|
||||
cwdDirs = await readdir(root)
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
for (const dir of cwdDirs) {
|
||||
const sub = join(root, dir)
|
||||
let files: string[]
|
||||
try {
|
||||
files = await readdir(sub)
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
const jsonl = files.find(f => f.endsWith('.jsonl'))
|
||||
if (jsonl !== undefined) return join(sub, jsonl)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { type NormalizeContext, normalizeSessionLog, normalizeStdout } from '../tests/snapshot-normalize.ts'
|
||||
|
||||
/**
|
||||
* Unit tests for the pure snapshot normalizers. Live as a *.spec.ts (runs in
|
||||
* the default unit gate) and import the harness-side normalizers directly.
|
||||
*/
|
||||
|
||||
const ctx: NormalizeContext = {
|
||||
sessionIds: ['11111111-2222-3333-4444-555555555555'],
|
||||
cwd: '/tmp/acp-snap-cwd-abc123',
|
||||
}
|
||||
|
||||
describe('normalizeStdout', () => {
|
||||
it('rewrites JSON-RPC ids to a stable first-seen sequence', () => {
|
||||
const raw = [
|
||||
JSON.stringify({ jsonrpc: '2.0', id: 42, method: 'initialize' }),
|
||||
JSON.stringify({ jsonrpc: '2.0', id: 42, result: {} }),
|
||||
JSON.stringify({ jsonrpc: '2.0', id: 99, method: 'session/new' }),
|
||||
].join('\n')
|
||||
const out = normalizeStdout(raw, ctx)
|
||||
expect(out).toContain('"id": 1')
|
||||
expect(out).toContain('"id": 2')
|
||||
expect(out).not.toContain('42')
|
||||
expect(out).not.toContain('99')
|
||||
})
|
||||
|
||||
it('scrubs the cwd and session id anywhere they appear', () => {
|
||||
const raw = JSON.stringify({
|
||||
jsonrpc: '2.0', method: 'session/update',
|
||||
params: { sessionId: ctx.sessionIds[0], cwd: ctx.cwd, note: `at ${ctx.cwd}/x` },
|
||||
})
|
||||
const out = normalizeStdout(raw, ctx)
|
||||
expect(out).toContain('{{sessionId}}')
|
||||
expect(out).toContain('{{cwd}}')
|
||||
expect(out).not.toContain(ctx.cwd)
|
||||
expect(out).not.toContain(ctx.sessionIds[0] as string)
|
||||
})
|
||||
|
||||
it('scrubs a stray UUID not in the known list', () => {
|
||||
const raw = JSON.stringify({ jsonrpc: '2.0', method: 'x', params: { id: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee' } })
|
||||
expect(normalizeStdout(raw, ctx)).toContain('{{sessionId}}')
|
||||
})
|
||||
|
||||
it('leaves notification frames without an id untouched in id-space', () => {
|
||||
const raw = JSON.stringify({ jsonrpc: '2.0', method: 'session/update', params: {} })
|
||||
const out = normalizeStdout(raw, ctx)
|
||||
expect(out).not.toContain('"id"')
|
||||
})
|
||||
|
||||
it('throws on a non-JSON stdout line (the purity check)', () => {
|
||||
const raw = `${JSON.stringify({ jsonrpc: '2.0', id: 1 })}\noops a log leaked\n`
|
||||
expect(() => normalizeStdout(raw, ctx)).toThrow()
|
||||
})
|
||||
|
||||
it('ignores blank lines', () => {
|
||||
const raw = `\n${JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'm' })}\n\n`
|
||||
expect(() => normalizeStdout(raw, ctx)).not.toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizeSessionLog', () => {
|
||||
const header = (over: object) => JSON.stringify({ type: 'session', version: 1, id: 's', createdAt: 123, ...over })
|
||||
const event = (over: object) => JSON.stringify({ type: 'turn/start', seq: 1, time: 999, data: { turn: 1 }, ...over })
|
||||
|
||||
it('zeroes the header createdAt', () => {
|
||||
const out = normalizeSessionLog(`${header({})}\n`, ctx)
|
||||
expect(out).toContain('"createdAt": 0')
|
||||
expect(out).not.toContain('123')
|
||||
})
|
||||
|
||||
it('zeroes each event time but keeps seq', () => {
|
||||
const out = normalizeSessionLog(`${header({})}\n${event({ seq: 7, time: 999 })}\n`, ctx)
|
||||
expect(out).toContain('"time": 0')
|
||||
expect(out).toContain('"seq": 7') // seq is deterministic — NOT scrubbed
|
||||
expect(out).not.toContain('999')
|
||||
})
|
||||
|
||||
it('scrubs cwd and session id deep inside event data', () => {
|
||||
const ev = JSON.stringify({
|
||||
type: 'tool/result', seq: 2, time: 5,
|
||||
data: { content: [{ type: 'text', text: `wrote ${ctx.cwd}/proof.txt` }] },
|
||||
})
|
||||
const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx)
|
||||
expect(out).toContain('{{cwd}}')
|
||||
expect(out).not.toContain(ctx.cwd)
|
||||
})
|
||||
|
||||
it('scrubs the session id in the header', () => {
|
||||
const out = normalizeSessionLog(`${header({ id: ctx.sessionIds[0] })}\n`, ctx)
|
||||
expect(out).toContain('{{sessionId}}')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Pure normalizers for the ACP snapshot goldens. They replace the
|
||||
* non-deterministic values in the two captured surfaces — the stdout JSON-RPC
|
||||
* transcript and the persisted session JSONL — with stable tokens, so a golden
|
||||
* compare reflects behavior, not run-to-run noise. Kept dependency-free and
|
||||
* side-effect-free so they unit-test trivially.
|
||||
*
|
||||
* Scrubbed: `randomUUID()` session ids → `{{sessionId}}`; the temp `mkdtemp`
|
||||
* cwd → `{{cwd}}` (it appears in terminal-card `_meta` and the log header);
|
||||
* JSON-RPC request `id` → a stable per-transcript sequence; the log's per-event
|
||||
* `time` (epoch ms) and header `createdAt` → 0. NOT scrubbed: the log's `seq`
|
||||
* (deterministic — `seq = log.length`, part of the event-log contract).
|
||||
*
|
||||
* See docs/rfc/implemented/2026-06-19-acp-snapshot-tests.md.
|
||||
*/
|
||||
|
||||
const SESSION_ID = '{{sessionId}}'
|
||||
const CWD = '{{cwd}}'
|
||||
|
||||
/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */
|
||||
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
|
||||
|
||||
/** Inputs the normalizers need to recognize a run's volatile values. */
|
||||
export interface NormalizeContext {
|
||||
/** The session id(s) the run issued — replaced with `{{sessionId}}`. */
|
||||
sessionIds: string[]
|
||||
/** The temp cwd the run used — replaced with `{{cwd}}`. */
|
||||
cwd: string
|
||||
}
|
||||
|
||||
/** Replace cwd, session ids, and any stray UUID with stable tokens in a string. */
|
||||
function scrubString(value: string, ctx: NormalizeContext): string {
|
||||
let out = value
|
||||
// cwd first (longest, most specific), then explicit session ids, then any
|
||||
// residual UUID (covers ids that appear in places we didn't enumerate).
|
||||
out = out.split(ctx.cwd).join(CWD)
|
||||
for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID)
|
||||
out = out.replace(UUID_RE, SESSION_ID)
|
||||
return out
|
||||
}
|
||||
|
||||
/** Recursively scrub a parsed JSON value (strings replaced; structure kept). */
|
||||
function scrubValue(value: unknown, ctx: NormalizeContext): unknown {
|
||||
if (typeof value === 'string') return scrubString(value, ctx)
|
||||
if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx))
|
||||
if (value !== null && typeof value === 'object') {
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx)
|
||||
return out
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a
|
||||
* stable, line-diffable golden: one pretty-printed frame per block, with the
|
||||
* JSON-RPC `id` rewritten to a per-transcript sequence (1, 2, 3, …) and all
|
||||
* volatile strings scrubbed. Throws if any non-empty line is not valid JSON —
|
||||
* that doubles as the stdout-purity check (no logger leaked onto the protocol).
|
||||
*/
|
||||
export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): string {
|
||||
const lines = rawStdout.split('\n').filter(line => line.trim().length > 0)
|
||||
// Map each distinct JSON-RPC id (request/response correlate by id) to a stable
|
||||
// sequence number, in first-seen order, so id churn doesn't perturb the golden.
|
||||
const idSeq = new Map<string, number>()
|
||||
const stableId = (id: unknown): number => {
|
||||
const key = JSON.stringify(id)
|
||||
let n = idSeq.get(key)
|
||||
if (n === undefined) { n = idSeq.size + 1; idSeq.set(key, n) }
|
||||
return n
|
||||
}
|
||||
const frames = lines.map((line) => {
|
||||
const frame = JSON.parse(line) as Record<string, unknown>
|
||||
if ('id' in frame && frame.id !== undefined && frame.id !== null) {
|
||||
frame.id = stableId(frame.id)
|
||||
}
|
||||
return scrubValue(frame, ctx) as Record<string, unknown>
|
||||
})
|
||||
return frames.map(f => JSON.stringify(f, null, 2)).join('\n') + '\n'
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a session JSONL log into a stable golden: the header line's
|
||||
* volatile fields (`createdAt`, `id`, `cwd`) and every event's `time` are
|
||||
* zeroed/scrubbed, all volatile strings scrubbed, and `seq` is LEFT INTACT
|
||||
* (deterministic by contract). One pretty-printed record per block.
|
||||
*/
|
||||
export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): string {
|
||||
const lines = rawLog.split('\n').filter(line => line.trim().length > 0)
|
||||
const records = lines.map((line) => {
|
||||
const record = JSON.parse(line) as Record<string, unknown>
|
||||
// Header line: { type: 'session', createdAt, id, cwd, … }.
|
||||
if (record.type === 'session') {
|
||||
if ('createdAt' in record) record.createdAt = 0
|
||||
} else if ('time' in record) {
|
||||
// Event line: zero the epoch-ms timestamp; keep seq (deterministic).
|
||||
record.time = 0
|
||||
}
|
||||
return scrubValue(record, ctx) as Record<string, unknown>
|
||||
})
|
||||
return records.map(r => JSON.stringify(r, null, 2)).join('\n') + '\n'
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"steps": [
|
||||
{ "op": "initialize" },
|
||||
{ "op": "newSession" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"type":"session","version":1,"id":"00000000-0000-0000-0000-000000000000","createdAt":0}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"result": {
|
||||
"protocolVersion": 1,
|
||||
"agentInfo": {
|
||||
"name": "deepseek-harness-acp",
|
||||
"version": "0.0.1"
|
||||
},
|
||||
"agentCapabilities": {
|
||||
"loadSession": true,
|
||||
"promptCapabilities": {
|
||||
"image": false,
|
||||
"audio": false,
|
||||
"embeddedContext": false
|
||||
}
|
||||
},
|
||||
"authMethods": []
|
||||
}
|
||||
}
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"result": {
|
||||
"sessionId": "{{sessionId}}"
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,8 @@
|
||||
"examples/echo-agent/src/*.ts",
|
||||
"examples/coding-agent/src/*.ts",
|
||||
"examples/acp-agent/src/*.ts",
|
||||
"examples/acp-agent/tests/**/*.e2e.ts"
|
||||
"examples/acp-agent/tests/**/*.e2e.ts",
|
||||
"examples/acp-agent/tests/**/*.snapshot.ts"
|
||||
],
|
||||
"project": ["scripts/**/*.ts", "examples/**/*.ts"]
|
||||
},
|
||||
|
||||
@@ -25,6 +25,9 @@ pre-push:
|
||||
- name: test
|
||||
run: pnpm run test
|
||||
|
||||
- name: snapshot
|
||||
run: pnpm run test:snapshot
|
||||
|
||||
- name: hygiene
|
||||
run: pnpm run hygiene
|
||||
|
||||
|
||||
@@ -19,6 +19,8 @@
|
||||
"test": "vitest run",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"test:e2e": "vitest run --config vitest.e2e.config.ts",
|
||||
"test:snapshot": "vitest run --config vitest.snapshot.config.ts",
|
||||
"test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update",
|
||||
"knip": "knip",
|
||||
"publint": "tsx scripts/publint-all.ts",
|
||||
"doc-typecheck": "tsx scripts/doc-typecheck.ts",
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import tsconfigPaths from 'vite-tsconfig-paths'
|
||||
import { defineConfig } from 'vitest/config'
|
||||
|
||||
// Snapshot tests: `pnpm run test:snapshot`, file pattern *.snapshot.ts.
|
||||
// REPLAY by default — they boot the real acp-agent subprocess against a
|
||||
// recorded session JSONL fixture (no API key, no network) and diff the
|
||||
// normalized stdout transcript + re-persisted log against committed goldens.
|
||||
// `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the
|
||||
// fixtures against the real API and refreshes the goldens.
|
||||
//
|
||||
// Replay loads no .env; record reads DEEPSEEK_API_KEY from the env or a
|
||||
// gitignored repo-root .env (loaded here, mirroring the e2e config), so a
|
||||
// contributor with a key only in .env can still record.
|
||||
try {
|
||||
process.loadEnvFile(new URL('.env', import.meta.url).pathname)
|
||||
} catch {
|
||||
// No .env — fine; replay needs no key and record reads it from the env.
|
||||
}
|
||||
|
||||
export default defineConfig({
|
||||
// Same resolution note as vitest.config.ts: bare workspace names resolve
|
||||
// through the root tsconfig paths map; the native option cannot do this.
|
||||
plugins: [tsconfigPaths({ projects: ['./tsconfig.test.json'] })],
|
||||
test: {
|
||||
include: ['examples/*/tests/**/*.snapshot.ts'],
|
||||
// Each test boots a subprocess; give it room, and run files one at a time
|
||||
// (a record run hits the live API, and replay subprocess boot is heavy).
|
||||
testTimeout: 120_000,
|
||||
hookTimeout: 30_000,
|
||||
fileParallelism: false,
|
||||
},
|
||||
})
|
||||
Reference in New Issue
Block a user