Merge branch 'codex/simp-shared-acp-test-launcher' into codex/simp-trim-hook-snapshot-noise
This commit is contained in:
@@ -4,7 +4,7 @@ The ACP snapshot suite kit: the shared machinery behind the keyless snapshot tie
|
||||
|
||||
Four layers, importable separately:
|
||||
|
||||
- **`launchAcpTestAgent` (launcher)** — boots an unbuilt ACP agent from a temp cwd, pins tsx to the repo tsconfig, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through its startup lifecycle, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit before resolving or propagating a child error, so callers can remove owned paths after either outcome. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy.
|
||||
- **`launchAcpTestAgent` (launcher)** — boots an unbuilt ACP agent from a temp cwd, pins tsx to the repo tsconfig, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through its startup lifecycle, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit, inherited stdio closure, and ACP parser exhaustion before resolving or propagating a child error, so captures are complete and callers can remove owned paths after either outcome. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy.
|
||||
- **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the golden + purity check, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths; the subprocess cwd is a temp dir outside the repo).
|
||||
- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), `scrubSystemPrompts` (prompt text → `{{system}}` in every JSONL), and `scrubRequestHeaders` (the remaining header bulk → `{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header RFC](../../../docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)).
|
||||
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario golden + re-persisted-log compares, record/refresh fixture write-back, the per-header-class pin (`system-prompt.golden.md` plus the JSONL's full tool schemas) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt-scrubbed, non-pinning fixtures fully header-scrubbed). Each scenario directory's `session.jsonl` plus contiguous `session.<n>.jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time.
|
||||
|
||||
@@ -65,7 +65,7 @@ export interface LaunchedAcpTestAgent {
|
||||
stderr(): string
|
||||
/** Resolve when a future session update matches the predicate. */
|
||||
waitForUpdate(match: (update: SessionNotification['update']) => boolean): Promise<SessionNotification['update']>
|
||||
/** Gracefully close stdin, or send a signal, and wait for process exit. */
|
||||
/** Gracefully close stdin, or send a signal, then wait for process exit, inherited stdio closure, and ACP parser drain. */
|
||||
close(signal?: NodeJS.Signals): Promise<void>
|
||||
}
|
||||
|
||||
@@ -132,7 +132,6 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
})
|
||||
child.stdout.on('end', () => {
|
||||
passthrough.push(null)
|
||||
closeUpdateStream()
|
||||
})
|
||||
const stream = ndJsonStream(
|
||||
Writable.toWeb(child.stdin) as WritableStream<Uint8Array>,
|
||||
@@ -163,6 +162,17 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
?? (() => Promise.resolve({ outcome: { outcome: 'cancelled' } })),
|
||||
})
|
||||
const client = new ClientSideConnection(makeClient, stream)
|
||||
// `exit` only reports the parent process's status. Descendants may retain
|
||||
// inherited stdout/stderr handles and buffered ACP frames may still be
|
||||
// crossing the SDK parser. Node's `close` follows stdio closure; the SDK's
|
||||
// `closed` follows parser exhaustion. Capture both eagerly so a caller that
|
||||
// invokes close after process exit still joins the complete drain boundary.
|
||||
const stdioClosed = new Promise<void>(resolve => child.once('close', () => { resolve() }))
|
||||
const drained = Promise.all([stdioClosed, client.closed]).then(() => undefined)
|
||||
// A caller may await a pending update without calling close(). Make natural
|
||||
// stream exhaustion terminal for those waiters too, but only after the
|
||||
// parser has dispatched every buffered frame.
|
||||
void client.closed.then(closeUpdateStream)
|
||||
|
||||
return {
|
||||
child,
|
||||
@@ -183,6 +193,7 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
throw error
|
||||
}
|
||||
if (!isRunning(child)) {
|
||||
await drained
|
||||
closeUpdateStream()
|
||||
return
|
||||
}
|
||||
@@ -194,6 +205,7 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
childFailure,
|
||||
])
|
||||
if (failure === undefined) {
|
||||
await drained
|
||||
closeUpdateStream()
|
||||
return
|
||||
}
|
||||
@@ -204,6 +216,7 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
// callers may safely remove cwd/session resources after close rejects.
|
||||
child.kill('SIGKILL')
|
||||
await exited
|
||||
await drained
|
||||
closeUpdateStream()
|
||||
throw failure
|
||||
},
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { readdirSync } from 'node:fs'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { createInterface } from 'node:readline'
|
||||
@@ -50,6 +51,8 @@ interface Behavior {
|
||||
echoWorkspace?: boolean
|
||||
/** Write a line to stderr on boot (spec-side stderr-capture assertions). */
|
||||
stderrNote?: string
|
||||
/** Let a short-lived descendant retain stdio and emit one final ACP update plus stderr line after this parent exits. */
|
||||
lateInheritedOutput?: boolean
|
||||
/** Session logs to persist on stdin EOF. */
|
||||
logs?: ScriptedLog[]
|
||||
/** Leave a stray FILE directly under the sessions root (harvest must skip it). */
|
||||
@@ -256,6 +259,24 @@ function flushLogsAndExit(): void {
|
||||
writeFileSync(join(sessionsRoot, 'bucket-noise', 'notes.txt'), 'not a session log\n')
|
||||
}
|
||||
if (behavior.deleteSessionsRoot === true) rmSync(sessionsRoot, { recursive: true, force: true })
|
||||
if (behavior.lateInheritedOutput === true) {
|
||||
const frame = JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
method: 'session/update',
|
||||
params: {
|
||||
sessionId,
|
||||
update: {
|
||||
sessionUpdate: 'agent_message_chunk',
|
||||
content: { type: 'text', text: 'late inherited stdout' },
|
||||
},
|
||||
},
|
||||
})
|
||||
const code = [
|
||||
`setTimeout(() => process.stdout.write(${JSON.stringify(`${frame}\n`)}), 50)`,
|
||||
`setTimeout(() => process.stderr.write(${JSON.stringify('late inherited stderr\n')}), 75)`,
|
||||
].join(';')
|
||||
spawn(process.execPath, ['-e', code], { stdio: ['ignore', 1, 2] }).unref()
|
||||
}
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
|
||||
@@ -91,6 +91,27 @@ describe('runScenario', () => {
|
||||
expect(exited).toBe(true)
|
||||
})
|
||||
|
||||
it('waits for inherited stdio and buffered ACP parsing after the parent exits', { timeout: 20_000 }, async () => {
|
||||
const { dir, fixtureFile } = await scenario({ lateInheritedOutput: true })
|
||||
const launched = launchAcpTestAgent({
|
||||
agent: AGENT,
|
||||
cwd: dir,
|
||||
env: { DSH_SNAPSHOT_FILE: fixtureFile },
|
||||
})
|
||||
await launched.client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
|
||||
await launched.client.newSession({ cwd: dir, mcpServers: [] })
|
||||
const lateUpdate = launched.waitForUpdate(update =>
|
||||
update.sessionUpdate === 'agent_message_chunk'
|
||||
&& update.content.type === 'text'
|
||||
&& update.content.text === 'late inherited stdout')
|
||||
|
||||
await launched.close()
|
||||
|
||||
await expect(lateUpdate).resolves.toMatchObject({ sessionUpdate: 'agent_message_chunk' })
|
||||
expect(launched.rawStdout()).toContain('late inherited stdout')
|
||||
expect(launched.stderr()).toContain('late inherited stderr')
|
||||
})
|
||||
|
||||
it('drives a full turn: initialize (terminal caps), session, prompt, permission stub, harvest', { timeout: 20_000 }, async () => {
|
||||
const { fixtureFile } = await scenario({
|
||||
permissionProbe: true,
|
||||
|
||||
Reference in New Issue
Block a user