diff --git a/examples/acp-agent/cordis.snapshot-record.yml b/examples/acp-agent/cordis.snapshot-record.yml new file mode 100644 index 0000000000..7a4a431f0a --- /dev/null +++ b/examples/acp-agent/cordis.snapshot-record.yml @@ -0,0 +1,43 @@ +# Snapshot-test RECORD config: a real run whose persisted session JSONL is +# harvested into a scenario fixture. Identical to cordis.yml (real llm-deepseek +# adapter + JSONL persistence) — recording must exercise the REAL model so the +# recorded log is a genuine product of the system. Needs DEEPSEEK_API_KEY. +# +# It is a separate file (rather than reusing cordis.yml) only so the snapshot +# harness selects it explicitly via $DSH_SNAPSHOT=record and so its persistence +# root can be pointed at the harness's harvest directory by the same env the +# replay path uses. The graceful-shutdown path in start.ts flushes persistence +# before exit so the harvested log is complete. + +- id: timer + name: '@cordisjs/plugin-timer' + +# Shared provider/tool core, INCLUDING the real llm-deepseek adapter. +- id: base + name: '@cordisjs/plugin-include' + config: + path: '../base.yml' + +- id: agent-loop + name: '@deepseek-ai/dsh-agent-loop' + config: + agents: [] + +- id: session-persistence + name: '@deepseek-ai/dsh-session-persistence-jsonl' + config: + root: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT + +- id: acp + name: '@deepseek-ai/dsh-acp' + config: + model: deepseek-v4-flash + systemPrompt: | + You are a coding assistant driven over the Agent Client Protocol. + + Your only tools are bash (plus bash_output/bash_kill for background + tasks). Do ALL file operations through bash: read with cat/sed/head, + search with grep, write with heredocs (cat <<'EOF' > file), edit with + sed or a rewrite. Each bash call runs in a fresh shell — pass workdir + instead of cd. Check the [exit code: N] marker; verify your work. Keep + answers brief and factual. diff --git a/examples/acp-agent/cordis.snapshot.yml b/examples/acp-agent/cordis.snapshot.yml new file mode 100644 index 0000000000..ed5690db18 --- /dev/null +++ b/examples/acp-agent/cordis.snapshot.yml @@ -0,0 +1,69 @@ +# Snapshot-test REPLAY config: the acp-agent plugin tree with the model replaced +# by llm-replay (serves a recorded session JSONL — no API key, no network). +# +# This does NOT include ../base.yml: base.yml always loads +# @deepseek-ai/dsh-llm-deepseek, whose apply() throws without DEEPSEEK_API_KEY, +# so a keyless replay run would die at boot. We inline the providerless core +# instead and install llm-replay where the adapter would be. +# +# stdout is reserved for the ACP JSON-RPC protocol — no stdout logger (see +# cordis.yml). The replay fixture path comes from $DSH_SNAPSHOT_FILE (and an +# optional $DSH_SNAPSHOT_OVERRIDE sidecar), set by the snapshot harness. + +- id: timer + name: '@cordisjs/plugin-timer' + +# Providerless core (everything base.yml has EXCEPT llm-deepseek). +- id: llm + name: '@deepseek-ai/dsh-llm' + +- id: sessions + name: '@deepseek-ai/dsh-session' + +- id: system-prompt + name: '@deepseek-ai/dsh-system-prompt' + +- id: tools + name: '@deepseek-ai/dsh-tools' + +- id: agents + name: '@deepseek-ai/dsh-agent' + +- id: invariants + name: '@deepseek-ai/dsh-invariants' + +- id: bash + name: '@deepseek-ai/dsh-bash-local' + config: + timeoutMs: 60000 + +- id: tool-bash + name: '@deepseek-ai/dsh-tool-bash' + +# The replay adapter: short-circuits llm/stream with the recorded log's chunks. +- id: llm-replay + name: './src/llm-replay.ts' + +- id: agent-loop + name: '@deepseek-ai/dsh-agent-loop' + config: + agents: [] + +- id: session-persistence + name: '@deepseek-ai/dsh-session-persistence-jsonl' + config: + root: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT + +- id: acp + name: '@deepseek-ai/dsh-acp' + config: + model: deepseek-v4-flash + systemPrompt: | + You are a coding assistant driven over the Agent Client Protocol. + + Your only tools are bash (plus bash_output/bash_kill for background + tasks). Do ALL file operations through bash: read with cat/sed/head, + search with grep, write with heredocs (cat <<'EOF' > file), edit with + sed or a rewrite. Each bash call runs in a fresh shell — pass workdir + instead of cd. Check the [exit code: N] marker; verify your work. Keep + answers brief and factual. diff --git a/examples/acp-agent/start.ts b/examples/acp-agent/start.ts index 1c97c0c8c0..01bcc32093 100644 --- a/examples/acp-agent/start.ts +++ b/examples/acp-agent/start.ts @@ -2,20 +2,36 @@ import { pathToFileURL } from 'node:url' import { Context } from 'cordis' import Loader from '@cordisjs/plugin-loader' +// Snapshot-test modes (set by the snapshot harness via env): +// DSH_SNAPSHOT=replay — load cordis.snapshot.yml (providerless; llm-replay +// serves a recorded session log). Skip .env so a stray +// key can never trigger a live model call. +// DSH_SNAPSHOT=record — load cordis.snapshot-record.yml (the real adapter + +// persistence) so a real run can be harvested. +// Absent — the normal demo (cordis.yml), driven by a real editor. +const snapshotMode = process.env.DSH_SNAPSHOT +const configPath = snapshotMode === 'replay' ? './cordis.snapshot.yml' + : snapshotMode === 'record' ? './cordis.snapshot-record.yml' + : './cordis.yml' + // Load DEEPSEEK_API_KEY / DEEPSEEK_BASE_URL from a gitignored repo-root .env // (Node native). Absent file is fine — the environment may already carry them. +// In REPLAY mode we deliberately skip this: replay must never reach the network, +// so we don't want a present .env to enable a live call. // // IMPORTANT: this server speaks ACP JSON-RPC on stdout. Do NOT add any // stdout logging here or in cordis.yml — it would corrupt the protocol frames. // A present-but-unreadable/malformed .env is a real misconfiguration: surface // it on STDERR (never stdout) rather than silently running with the wrong env. -try { - process.loadEnvFile(new URL('../../.env', import.meta.url).pathname) -} catch (error) { - if ((error as NodeJS.ErrnoException | null)?.code !== 'ENOENT') { - process.stderr.write(`acp-agent: failed to load .env: ${String(error)}\n`) +if (snapshotMode !== 'replay') { + try { + process.loadEnvFile(new URL('../../.env', import.meta.url).pathname) + } catch (error) { + if ((error as NodeJS.ErrnoException | null)?.code !== 'ENOENT') { + process.stderr.write(`acp-agent: failed to load .env: ${String(error)}\n`) + } + // ENOENT (no .env) is fine — rely on the ambient environment. } - // ENOENT (no .env) is fine — rely on the ambient environment. } const ctx = new Context() @@ -25,6 +41,18 @@ await ctx.plugin(Loader) await ctx.loader.create({ name: '@cordisjs/plugin-include', config: { - path: './cordis.yml', + path: configPath, }, }) + +// Graceful shutdown for snapshot RECORD runs: when the client closes our stdin +// (it is done driving the session), dispose the whole context. Disposal awaits +// the agent-loop teardown and the persistence backend's final `session/flush`, +// so the recorded `.jsonl` is fully written before the process exits and the +// harness harvests it. (In a normal editor session stdin stays open for the +// connection's lifetime; the editor kills the process, so this never fires.) +if (snapshotMode !== undefined) { + process.stdin.on('end', () => { + void ctx.fiber.dispose().then(() => { process.exit(0) }) + }) +} diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts new file mode 100644 index 0000000000..a2422d4395 --- /dev/null +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -0,0 +1,90 @@ +import { readFile, readdir, writeFile } from 'node:fs/promises' +import { existsSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { dirname, join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { type InputScript, runScenario } from './snapshot-harness.ts' +import { type NormalizeContext, normalizeSessionLog, normalizeStdout } from './snapshot-normalize.ts' + +/** + * ACP snapshot tests (REPLAY by default, keyless). Each scenario under + * `snapshots//` ships an `input.json` (the client stdin script) and a + * recorded `session.jsonl` fixture; replay boots the real acp-agent subprocess, + * drives it, and diffs the normalized stdout transcript (and, for model + * scenarios, the re-persisted session log) against committed goldens. + * + * `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the + * fixtures against the real API and refreshes the goldens in one pass. + */ + +const SNAPSHOTS_DIR = join(dirname(fileURLToPath(import.meta.url)), 'snapshots') +const RECORDING = process.env.DSH_SNAPSHOT === 'record' + +/** A scenario and whether it makes any model call (→ has a behavioral JSONL golden). */ +interface Scenario { + name: string + /** Whether the scenario drives at least one model turn (so a JSONL golden applies). */ + hasModelTurn: boolean +} + +const SCENARIOS: Scenario[] = [ + { name: 'handshake', hasModelTurn: false }, +] + +for (const scenario of SCENARIOS) { + describe(`snapshot: ${scenario.name}`, () => { + it('matches the stdout transcript golden', async () => { + const dir = join(SNAPSHOTS_DIR, scenario.name) + const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript + const overrideFile = join(dir, 'replay.override.json') + const result = await runScenario(input, { + mode: RECORDING ? 'record' : 'replay', + fixtureFile: join(dir, 'session.jsonl'), + ...existsSync(overrideFile) ? { overrideFile } : {}, + }) + + const ctx: NormalizeContext = { + sessionIds: result.sessionId !== undefined ? [result.sessionId] : [], + cwd: result.cwd, + } + + // RECORD mode: persist the freshly-harvested log back to the scenario's + // session.jsonl fixture (a model scenario must produce one). `--update` + // refreshes the Vitest goldens but NOT this fixture, so write it here. + if (RECORDING && scenario.hasModelTurn) { + expect(result.sessionLog, 'record produced no session log to harvest').toBeDefined() + await writeFile(join(dir, 'session.jsonl'), result.sessionLog as string) + } + + await expect(normalizeStdout(result.rawStdout, ctx)) + .toMatchFileSnapshot(join(dir, 'stdout.golden.txt')) + + if (scenario.hasModelTurn) { + expect(result.sessionLog, 'a model scenario must persist a session log').toBeDefined() + await expect(normalizeSessionLog(result.sessionLog as string, ctx)) + .toMatchFileSnapshot(join(dir, 'session.golden.txt')) + } + }) + }) +} + +describe('snapshot fixtures', () => { + it('every scenario directory is registered (no orphans)', async () => { + // toMatchFileSnapshot does not prune orphaned golden/fixture files, so a + // renamed/removed scenario could leave a stale dir that nothing exercises. + // Fail loud on any snapshots/ not present in SCENARIOS. + const entries = await readdir(SNAPSHOTS_DIR, { withFileTypes: true }) + const onDisk = entries.filter(e => e.isDirectory()).map(e => e.name).sort() + const registered = SCENARIOS.map(s => s.name).sort() + expect(onDisk).toEqual(registered) + }) + + it('every registered scenario has its required fixture files', async () => { + for (const { name } of SCENARIOS) { + const dir = join(SNAPSHOTS_DIR, name) + expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true) + expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true) + expect(existsSync(join(dir, 'stdout.golden.txt')), `${name}/stdout.golden.txt`).toBe(true) + } + }) +}) diff --git a/examples/acp-agent/tests/snapshot-harness.ts b/examples/acp-agent/tests/snapshot-harness.ts new file mode 100644 index 0000000000..7fb7eec21d --- /dev/null +++ b/examples/acp-agent/tests/snapshot-harness.ts @@ -0,0 +1,231 @@ +/** + * Shared harness for the ACP snapshot tests. A plain module (NOT a *.spec.ts / + * *.snapshot.ts) so importing it never re-registers another file's tests. + * + * It boots the REAL examples/acp-agent subprocess via the cordis Loader (so the + * export-shape bug class stays guarded — see docs/postmortem/0001), drives it + * over real ACP JSON-RPC stdio with a deterministic input script, tees raw + * stdout (for the golden + a purity check) into an SDK `ClientSideConnection`, + * and — in record mode — harvests the persisted session JSONL after a graceful + * shutdown flush. Two pure normalizers turn the captured stdout frames and the + * session-log events into stable, snapshot-able text. + * + * See docs/rfc/implemented/2026-06-19-acp-snapshot-tests.md. + */ + +import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' +import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { Readable, Writable } from 'node:stream' +import { + ClientSideConnection, + ndJsonStream, + PROTOCOL_VERSION, + type Agent as AcpAgent, + type Client, + type RequestPermissionRequest, + type RequestPermissionResponse, + type SessionNotification, +} from '@agentclientprotocol/sdk' + +const startScript = fileURLToPath(new URL('../start.ts', import.meta.url)) +const tsxLoader = fileURLToPath(import.meta.resolve('tsx')) +// The repo-root tsconfig: dev/test run UNBUILT and the `@deepseek-ai/dsh-*` +// imports resolve through its `paths` map. The child's cwd is a temp dir +// OUTSIDE the repo, so tsx's upward search would miss it — point tsx at the +// repo tsconfig explicitly (same fix the e2e harness uses). Repo root is four +// levels up from this file (examples/acp-agent/tests). +const repoTsconfig = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)) + +/** + * One step of a scenario's deterministic input script (`input.json`). The + * harness interprets these in order. `newSession` captures the server-issued + * (random) session id into a `{{sessionId}}` variable that later steps + * reference, since a committed file cannot know the id in advance. + */ +type InputStep = + | { op: 'initialize'; terminalOutput?: boolean } + | { op: 'newSession' } + | { op: 'prompt'; text: string } + | { op: 'cancel' } + +/** A scenario's `input.json`: an ordered list of input steps. */ +export interface InputScript { + steps: InputStep[] +} + +/** The result of running a scenario: raw stdout + the harvested session log. */ +export interface RunResult { + /** Raw stdout bytes (decoded utf8), every newline-delimited JSON-RPC frame. */ + rawStdout: string + /** stderr (for diagnostics on failure). */ + stderr: string + /** The session id the server issued (undefined if no session was created). */ + sessionId?: string + /** The temp cwd the session ran in (the bash workspace). */ + cwd: string + /** The persisted session log's content, if one was produced. */ + sessionLog?: string +} + +interface RunOptions { + /** `replay` (default, keyless) or `record` (real API, harvests the log). */ + mode: 'replay' | 'record' + /** The recorded session JSONL fixture path (replay reads it; record writes near it). */ + fixtureFile: string + /** Optional sidecar override path (replay). */ + overrideFile?: string +} + +/** + * Run a scenario end-to-end against a freshly-spawned subprocess. Owns the + * child and its temp dirs; always tears them down. Returns the captured stdout + * and (record mode) the harvested session-log path. + */ +export async function runScenario(input: InputScript, opts: RunOptions): Promise { + const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-')) + const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-')) + const env: NodeJS.ProcessEnv = { + ...process.env, + TSX_TSCONFIG_PATH: repoTsconfig, + DSH_SNAPSHOT: opts.mode, + DSH_SNAPSHOT_FILE: opts.fixtureFile, + DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot, + ...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {}, + } + + const child: ChildProcessWithoutNullStreams = spawn( + process.execPath, + ['--import', tsxLoader, startScript], + { cwd, env, stdio: ['pipe', 'pipe', 'pipe'] }, + ) + + const rawBuffers: Buffer[] = [] + const stderrChunks: string[] = [] + child.stderr.setEncoding('utf8') + child.stderr.on('data', (c: string) => stderrChunks.push(c)) + + // Tee raw stdout: accumulate the bytes for the golden + purity check, and ALSO + // feed the same bytes to the SDK client through a passthrough. Buffer the raw + // bytes (not per-chunk utf8 strings) and decode once at the end, so a + // multibyte sequence split across two 'data' events can't corrupt the golden. + const passthrough = new Readable({ read() {} }) + child.stdout.on('data', (buf: Buffer) => { + rawBuffers.push(buf) + passthrough.push(buf) + }) + child.stdout.on('end', () => passthrough.push(null)) + + const stream = ndJsonStream( + Writable.toWeb(child.stdin) as WritableStream, + Readable.toWeb(passthrough) as ReadableStream, + ) + const makeClient = (_agent: AcpAgent): Client => ({ + sessionUpdate(_params: SessionNotification): Promise { + return Promise.resolve() + }, + requestPermission(_params: RequestPermissionRequest): Promise { + return Promise.resolve({ outcome: { outcome: 'cancelled' } }) + }, + }) + const client = new ClientSideConnection(makeClient, stream) + + let sessionId: string | undefined + let sessionLog: string | undefined + try { + for (const step of input.steps) { + await runStep(client, step, cwd, () => sessionId, (id) => { sessionId = id }) + } + // Done driving: close stdin so the server disposes gracefully (flushing + // persistence) and exits. Then await exit so the harvested log is complete. + child.stdin.end() + await waitForExit(child) + // Harvest the persisted log (if any) while the temp dirs still exist. + const sessionLogPath = await findSessionLog(sessionsRoot) + if (sessionLogPath !== undefined) sessionLog = await readFile(sessionLogPath, 'utf8') + } finally { + // Failure-safe teardown: kill a still-running child and drop the temp dirs + // even if a step/harvest threw, so a flaky run never leaks a process or dir. + if (child.exitCode === null && child.signalCode === null) { + child.kill('SIGKILL') + await waitForExit(child) + } + await rm(cwd, { recursive: true, force: true }) + await rm(sessionsRoot, { recursive: true, force: true }) + } + + return { + rawStdout: Buffer.concat(rawBuffers).toString('utf8'), + stderr: stderrChunks.join(''), + cwd, + ...sessionId !== undefined ? { sessionId } : {}, + ...sessionLog !== undefined ? { sessionLog } : {}, + } +} + +/** Drive one input step over the client connection. */ +async function runStep( + client: ClientSideConnection, + step: InputStep, + cwd: string, + getSessionId: () => string | undefined, + setSessionId: (id: string) => void, +): Promise { + switch (step.op) { + case 'initialize': + await client.initialize({ + protocolVersion: PROTOCOL_VERSION, + clientCapabilities: step.terminalOutput === true ? { _meta: { terminal_output: true } } : {}, + }) + return + case 'newSession': { + const { sessionId } = await client.newSession({ cwd, mcpServers: [] }) + setSessionId(sessionId) + return + } + case 'prompt': { + const sessionId = getSessionId() + if (sessionId === undefined) throw new Error('snapshot-harness: prompt before newSession') + await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] }) + return + } + case 'cancel': { + const sessionId = getSessionId() + if (sessionId === undefined) throw new Error('snapshot-harness: cancel before newSession') + await client.cancel({ sessionId }) + return + } + default: + throw new Error(`snapshot-harness: unknown input op ${JSON.stringify(step)}`) + } +} + +/** Resolve once the child process exits (any code/signal). */ +function waitForExit(child: ChildProcessWithoutNullStreams): Promise { + if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve() + return new Promise(resolve => child.once('exit', () => { resolve() })) +} + +/** Find the single produced `.jsonl` session log under a sessions root, if any. */ +async function findSessionLog(root: string): Promise { + let cwdDirs: string[] + try { + cwdDirs = await readdir(root) + } catch { + return undefined + } + for (const dir of cwdDirs) { + const sub = join(root, dir) + let files: string[] + try { + files = await readdir(sub) + } catch { + continue + } + const jsonl = files.find(f => f.endsWith('.jsonl')) + if (jsonl !== undefined) return join(sub, jsonl) + } + return undefined +} diff --git a/examples/acp-agent/tests/snapshot-normalize.spec.ts b/examples/acp-agent/tests/snapshot-normalize.spec.ts new file mode 100644 index 0000000000..339cb8493f --- /dev/null +++ b/examples/acp-agent/tests/snapshot-normalize.spec.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest' +import { type NormalizeContext, normalizeSessionLog, normalizeStdout } from '../tests/snapshot-normalize.ts' + +/** + * Unit tests for the pure snapshot normalizers. Live as a *.spec.ts (runs in + * the default unit gate) and import the harness-side normalizers directly. + */ + +const ctx: NormalizeContext = { + sessionIds: ['11111111-2222-3333-4444-555555555555'], + cwd: '/tmp/acp-snap-cwd-abc123', +} + +describe('normalizeStdout', () => { + it('rewrites JSON-RPC ids to a stable first-seen sequence', () => { + const raw = [ + JSON.stringify({ jsonrpc: '2.0', id: 42, method: 'initialize' }), + JSON.stringify({ jsonrpc: '2.0', id: 42, result: {} }), + JSON.stringify({ jsonrpc: '2.0', id: 99, method: 'session/new' }), + ].join('\n') + const out = normalizeStdout(raw, ctx) + expect(out).toContain('"id": 1') + expect(out).toContain('"id": 2') + expect(out).not.toContain('42') + expect(out).not.toContain('99') + }) + + it('scrubs the cwd and session id anywhere they appear', () => { + const raw = JSON.stringify({ + jsonrpc: '2.0', method: 'session/update', + params: { sessionId: ctx.sessionIds[0], cwd: ctx.cwd, note: `at ${ctx.cwd}/x` }, + }) + const out = normalizeStdout(raw, ctx) + expect(out).toContain('{{sessionId}}') + expect(out).toContain('{{cwd}}') + expect(out).not.toContain(ctx.cwd) + expect(out).not.toContain(ctx.sessionIds[0] as string) + }) + + it('scrubs a stray UUID not in the known list', () => { + const raw = JSON.stringify({ jsonrpc: '2.0', method: 'x', params: { id: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee' } }) + expect(normalizeStdout(raw, ctx)).toContain('{{sessionId}}') + }) + + it('leaves notification frames without an id untouched in id-space', () => { + const raw = JSON.stringify({ jsonrpc: '2.0', method: 'session/update', params: {} }) + const out = normalizeStdout(raw, ctx) + expect(out).not.toContain('"id"') + }) + + it('throws on a non-JSON stdout line (the purity check)', () => { + const raw = `${JSON.stringify({ jsonrpc: '2.0', id: 1 })}\noops a log leaked\n` + expect(() => normalizeStdout(raw, ctx)).toThrow() + }) + + it('ignores blank lines', () => { + const raw = `\n${JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'm' })}\n\n` + expect(() => normalizeStdout(raw, ctx)).not.toThrow() + }) +}) + +describe('normalizeSessionLog', () => { + const header = (over: object) => JSON.stringify({ type: 'session', version: 1, id: 's', createdAt: 123, ...over }) + const event = (over: object) => JSON.stringify({ type: 'turn/start', seq: 1, time: 999, data: { turn: 1 }, ...over }) + + it('zeroes the header createdAt', () => { + const out = normalizeSessionLog(`${header({})}\n`, ctx) + expect(out).toContain('"createdAt": 0') + expect(out).not.toContain('123') + }) + + it('zeroes each event time but keeps seq', () => { + const out = normalizeSessionLog(`${header({})}\n${event({ seq: 7, time: 999 })}\n`, ctx) + expect(out).toContain('"time": 0') + expect(out).toContain('"seq": 7') // seq is deterministic — NOT scrubbed + expect(out).not.toContain('999') + }) + + it('scrubs cwd and session id deep inside event data', () => { + const ev = JSON.stringify({ + type: 'tool/result', seq: 2, time: 5, + data: { content: [{ type: 'text', text: `wrote ${ctx.cwd}/proof.txt` }] }, + }) + const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx) + expect(out).toContain('{{cwd}}') + expect(out).not.toContain(ctx.cwd) + }) + + it('scrubs the session id in the header', () => { + const out = normalizeSessionLog(`${header({ id: ctx.sessionIds[0] })}\n`, ctx) + expect(out).toContain('{{sessionId}}') + }) +}) diff --git a/examples/acp-agent/tests/snapshot-normalize.ts b/examples/acp-agent/tests/snapshot-normalize.ts new file mode 100644 index 0000000000..c729c429b8 --- /dev/null +++ b/examples/acp-agent/tests/snapshot-normalize.ts @@ -0,0 +1,102 @@ +/** + * Pure normalizers for the ACP snapshot goldens. They replace the + * non-deterministic values in the two captured surfaces — the stdout JSON-RPC + * transcript and the persisted session JSONL — with stable tokens, so a golden + * compare reflects behavior, not run-to-run noise. Kept dependency-free and + * side-effect-free so they unit-test trivially. + * + * Scrubbed: `randomUUID()` session ids → `{{sessionId}}`; the temp `mkdtemp` + * cwd → `{{cwd}}` (it appears in terminal-card `_meta` and the log header); + * JSON-RPC request `id` → a stable per-transcript sequence; the log's per-event + * `time` (epoch ms) and header `createdAt` → 0. NOT scrubbed: the log's `seq` + * (deterministic — `seq = log.length`, part of the event-log contract). + * + * See docs/rfc/implemented/2026-06-19-acp-snapshot-tests.md. + */ + +const SESSION_ID = '{{sessionId}}' +const CWD = '{{cwd}}' + +/** A UUID v4 string, the shape `randomUUID()` produces for session ids. */ +const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi + +/** Inputs the normalizers need to recognize a run's volatile values. */ +export interface NormalizeContext { + /** The session id(s) the run issued — replaced with `{{sessionId}}`. */ + sessionIds: string[] + /** The temp cwd the run used — replaced with `{{cwd}}`. */ + cwd: string +} + +/** Replace cwd, session ids, and any stray UUID with stable tokens in a string. */ +function scrubString(value: string, ctx: NormalizeContext): string { + let out = value + // cwd first (longest, most specific), then explicit session ids, then any + // residual UUID (covers ids that appear in places we didn't enumerate). + out = out.split(ctx.cwd).join(CWD) + for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID) + out = out.replace(UUID_RE, SESSION_ID) + return out +} + +/** Recursively scrub a parsed JSON value (strings replaced; structure kept). */ +function scrubValue(value: unknown, ctx: NormalizeContext): unknown { + if (typeof value === 'string') return scrubString(value, ctx) + if (Array.isArray(value)) return value.map(v => scrubValue(v, ctx)) + if (value !== null && typeof value === 'object') { + const out: Record = {} + for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx) + return out + } + return value +} + +/** + * Normalize a raw stdout transcript (newline-delimited JSON-RPC frames) into a + * stable, line-diffable golden: one pretty-printed frame per block, with the + * JSON-RPC `id` rewritten to a per-transcript sequence (1, 2, 3, …) and all + * volatile strings scrubbed. Throws if any non-empty line is not valid JSON — + * that doubles as the stdout-purity check (no logger leaked onto the protocol). + */ +export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): string { + const lines = rawStdout.split('\n').filter(line => line.trim().length > 0) + // Map each distinct JSON-RPC id (request/response correlate by id) to a stable + // sequence number, in first-seen order, so id churn doesn't perturb the golden. + const idSeq = new Map() + const stableId = (id: unknown): number => { + const key = JSON.stringify(id) + let n = idSeq.get(key) + if (n === undefined) { n = idSeq.size + 1; idSeq.set(key, n) } + return n + } + const frames = lines.map((line) => { + const frame = JSON.parse(line) as Record + if ('id' in frame && frame.id !== undefined && frame.id !== null) { + frame.id = stableId(frame.id) + } + return scrubValue(frame, ctx) as Record + }) + return frames.map(f => JSON.stringify(f, null, 2)).join('\n') + '\n' +} + +/** + * Normalize a session JSONL log into a stable golden: the header line's + * volatile fields (`createdAt`, `id`, `cwd`) and every event's `time` are + * zeroed/scrubbed, all volatile strings scrubbed, and `seq` is LEFT INTACT + * (deterministic by contract). One pretty-printed record per block. + */ +export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): string { + const lines = rawLog.split('\n').filter(line => line.trim().length > 0) + const records = lines.map((line) => { + const record = JSON.parse(line) as Record + // Header line: { type: 'session', createdAt, id, cwd, … }. + if (record.type === 'session') { + if ('createdAt' in record) record.createdAt = 0 + } else if ('time' in record) { + // Event line: zero the epoch-ms timestamp; keep seq (deterministic). + record.time = 0 + } + return scrubValue(record, ctx) as Record + }) + return records.map(r => JSON.stringify(r, null, 2)).join('\n') + '\n' +} diff --git a/examples/acp-agent/tests/snapshots/handshake/input.json b/examples/acp-agent/tests/snapshots/handshake/input.json new file mode 100644 index 0000000000..e1e84be919 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/handshake/input.json @@ -0,0 +1,6 @@ +{ + "steps": [ + { "op": "initialize" }, + { "op": "newSession" } + ] +} diff --git a/examples/acp-agent/tests/snapshots/handshake/session.jsonl b/examples/acp-agent/tests/snapshots/handshake/session.jsonl new file mode 100644 index 0000000000..ab44090be6 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/handshake/session.jsonl @@ -0,0 +1 @@ +{"type":"session","version":1,"id":"00000000-0000-0000-0000-000000000000","createdAt":0} diff --git a/examples/acp-agent/tests/snapshots/handshake/stdout.golden.txt b/examples/acp-agent/tests/snapshots/handshake/stdout.golden.txt new file mode 100644 index 0000000000..ec34bb7f4a --- /dev/null +++ b/examples/acp-agent/tests/snapshots/handshake/stdout.golden.txt @@ -0,0 +1,27 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "protocolVersion": 1, + "agentInfo": { + "name": "deepseek-harness-acp", + "version": "0.0.1" + }, + "agentCapabilities": { + "loadSession": true, + "promptCapabilities": { + "image": false, + "audio": false, + "embeddedContext": false + } + }, + "authMethods": [] + } +} +{ + "jsonrpc": "2.0", + "id": 2, + "result": { + "sessionId": "{{sessionId}}" + } +} diff --git a/knip.json b/knip.json index 0693f71c02..cb836485f1 100644 --- a/knip.json +++ b/knip.json @@ -8,7 +8,8 @@ "examples/echo-agent/src/*.ts", "examples/coding-agent/src/*.ts", "examples/acp-agent/src/*.ts", - "examples/acp-agent/tests/**/*.e2e.ts" + "examples/acp-agent/tests/**/*.e2e.ts", + "examples/acp-agent/tests/**/*.snapshot.ts" ], "project": ["scripts/**/*.ts", "examples/**/*.ts"] }, diff --git a/lefthook.yml b/lefthook.yml index 24600e4985..2a255424fa 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -25,6 +25,9 @@ pre-push: - name: test run: pnpm run test + - name: snapshot + run: pnpm run test:snapshot + - name: hygiene run: pnpm run hygiene diff --git a/package.json b/package.json index fc8bce92c4..b892288a0e 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,8 @@ "test": "vitest run", "test:coverage": "vitest run --coverage", "test:e2e": "vitest run --config vitest.e2e.config.ts", + "test:snapshot": "vitest run --config vitest.snapshot.config.ts", + "test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update", "knip": "knip", "publint": "tsx scripts/publint-all.ts", "doc-typecheck": "tsx scripts/doc-typecheck.ts", diff --git a/vitest.snapshot.config.ts b/vitest.snapshot.config.ts new file mode 100644 index 0000000000..1fb7ee48a4 --- /dev/null +++ b/vitest.snapshot.config.ts @@ -0,0 +1,32 @@ +import tsconfigPaths from 'vite-tsconfig-paths' +import { defineConfig } from 'vitest/config' + +// Snapshot tests: `pnpm run test:snapshot`, file pattern *.snapshot.ts. +// REPLAY by default — they boot the real acp-agent subprocess against a +// recorded session JSONL fixture (no API key, no network) and diff the +// normalized stdout transcript + re-persisted log against committed goldens. +// `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the +// fixtures against the real API and refreshes the goldens. +// +// Replay loads no .env; record reads DEEPSEEK_API_KEY from the env or a +// gitignored repo-root .env (loaded here, mirroring the e2e config), so a +// contributor with a key only in .env can still record. +try { + process.loadEnvFile(new URL('.env', import.meta.url).pathname) +} catch { + // No .env — fine; replay needs no key and record reads it from the env. +} + +export default defineConfig({ + // Same resolution note as vitest.config.ts: bare workspace names resolve + // through the root tsconfig paths map; the native option cannot do this. + plugins: [tsconfigPaths({ projects: ['./tsconfig.test.json'] })], + test: { + include: ['examples/*/tests/**/*.snapshot.ts'], + // Each test boots a subprocess; give it room, and run files one at a time + // (a record run hits the live API, and replay subprocess boot is heavy). + testTimeout: 120_000, + hookTimeout: 30_000, + fileParallelism: false, + }, +})