fix: make ACP test teardown failure-safe
This commit is contained in:
@@ -30,10 +30,16 @@ let spawned: LaunchedAcpTestAgent | undefined
|
||||
let workdir: string | undefined
|
||||
|
||||
afterEach(async () => {
|
||||
await spawned?.close('SIGKILL')
|
||||
spawned = undefined
|
||||
if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
|
||||
workdir = undefined
|
||||
try {
|
||||
await spawned?.close('SIGKILL')
|
||||
} finally {
|
||||
spawned = undefined
|
||||
try {
|
||||
if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
|
||||
} finally {
|
||||
workdir = undefined
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
describe('acp-agent over real stdio (no key required)', () => {
|
||||
|
||||
@@ -38,10 +38,16 @@ let spawned: LaunchedAcpTestAgent | undefined
|
||||
let workdir: string | undefined
|
||||
|
||||
afterEach(async () => {
|
||||
await spawned?.close('SIGKILL')
|
||||
spawned = undefined
|
||||
if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
|
||||
workdir = undefined
|
||||
try {
|
||||
await spawned?.close('SIGKILL')
|
||||
} finally {
|
||||
spawned = undefined
|
||||
try {
|
||||
if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
|
||||
} finally {
|
||||
workdir = undefined
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
describe.skipIf(!process.env.DEEPSEEK_API_KEY)('acp-agent e2e: a PreToolUse hook blocks bash (real model)', () => {
|
||||
|
||||
@@ -81,10 +81,16 @@ let spawned: Spawned | undefined
|
||||
let workdir: string | undefined
|
||||
|
||||
afterEach(async () => {
|
||||
await spawned?.close('SIGKILL')
|
||||
spawned = undefined
|
||||
if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
|
||||
workdir = undefined
|
||||
try {
|
||||
await spawned?.close('SIGKILL')
|
||||
} finally {
|
||||
spawned = undefined
|
||||
try {
|
||||
if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
|
||||
} finally {
|
||||
workdir = undefined
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
describe('sandbox-acp-agent keyless smoke (real cordis.yml via the Loader)', () => {
|
||||
|
||||
@@ -4,7 +4,7 @@ The ACP snapshot suite kit: the shared machinery behind the keyless snapshot tie
|
||||
|
||||
Four layers, importable separately:
|
||||
|
||||
- **`launchAcpTestAgent` (launcher)** — boots an unbuilt ACP agent from a temp cwd, pins tsx to the repo tsconfig, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through its startup lifecycle, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy.
|
||||
- **`launchAcpTestAgent` (launcher)** — boots an unbuilt ACP agent from a temp cwd, pins tsx to the repo tsconfig, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through its startup lifecycle, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit before resolving or propagating a child error, so callers can remove owned paths after either outcome. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy.
|
||||
- **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the golden + purity check, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths; the subprocess cwd is a temp dir outside the repo).
|
||||
- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), `scrubSystemPrompts` (prompt text → `{{system}}` in every JSONL), and `scrubRequestHeaders` (the remaining header bulk → `{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header RFC](../../../docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)).
|
||||
- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario golden + re-persisted-log compares, record/refresh fixture write-back, the per-header-class pin (`system-prompt.golden.md` plus the JSONL's full tool schemas) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt-scrubbed, non-pinning fixtures fully header-scrubbed). Each scenario directory's `session.jsonl` plus contiguous `session.<n>.jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time.
|
||||
|
||||
@@ -163,7 +163,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
let launched: LaunchedAcpTestAgent | undefined
|
||||
let sessionId: string | undefined
|
||||
let sessionLogs: HarvestedLog[] = []
|
||||
try {
|
||||
const outcome = await (async (): Promise<RunResult> => {
|
||||
// Seed the workspace if the scenario ships one (a file the agent reads/edits).
|
||||
// Copied into the temp cwd so the agent's bash tools see it; the goldens
|
||||
// normalize the cwd, so the seeded paths stay stable across runs.
|
||||
@@ -231,22 +231,36 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise
|
||||
// Harvest EVERY persisted log (parent + any subagent children) while the
|
||||
// temp dirs still exist, ordered primary-first.
|
||||
sessionLogs = await harvestSessionLogs(sessionsRoot)
|
||||
} finally {
|
||||
// Failure-safe teardown: kill a still-running child and drop the temp dirs
|
||||
// even if seeding/spawn/a step/harvest threw, so a flaky run never leaks a
|
||||
// process or dir. `launched` is undefined only if launch itself threw.
|
||||
await launched?.close('SIGKILL')
|
||||
await rm(cwd, { recursive: true, force: true })
|
||||
await rm(sessionsRoot, { recursive: true, force: true })
|
||||
}
|
||||
return {
|
||||
rawStdout: launched.rawStdout(),
|
||||
stderr: launched.stderr(),
|
||||
cwd,
|
||||
...sessionId !== undefined ? { sessionId } : {},
|
||||
sessionLogs,
|
||||
}
|
||||
})().then(
|
||||
value => ({ status: 'fulfilled', value } as const),
|
||||
(error: unknown) => ({ status: 'rejected', error } as const),
|
||||
)
|
||||
|
||||
return {
|
||||
rawStdout: launched.rawStdout(),
|
||||
stderr: launched.stderr(),
|
||||
cwd,
|
||||
...sessionId !== undefined ? { sessionId } : {},
|
||||
sessionLogs,
|
||||
// Failure-safe teardown: wait for a still-running child, then attempt BOTH
|
||||
// directory removals even when an earlier cleanup rejects. The main outcome
|
||||
// wins over teardown noise so a step/harvest failure is never replaced; on a
|
||||
// successful run, the first cleanup failure remains visible to the caller.
|
||||
const cleanupResults: PromiseSettledResult<unknown>[] = []
|
||||
const cleanup = async (action: () => Promise<unknown>): Promise<void> => {
|
||||
cleanupResults.push(...await Promise.allSettled([action()]))
|
||||
}
|
||||
/* v8 ignore next 1 -- launch itself can only throw on a defensive synchronous spawn API failure */
|
||||
await cleanup(() => launched?.close('SIGKILL') ?? Promise.resolve())
|
||||
await cleanup(() => rm(cwd, { recursive: true, force: true }))
|
||||
await cleanup(() => rm(sessionsRoot, { recursive: true, force: true }))
|
||||
|
||||
if (outcome.status === 'rejected') throw outcome.error
|
||||
const cleanupFailure = cleanupResults.find((result): result is PromiseRejectedResult => result.status === 'rejected')
|
||||
/* v8 ignore next 1 -- defensive OS cleanup failure after an otherwise successful real subprocess run */
|
||||
if (cleanupFailure !== undefined) throw cleanupFailure.reason
|
||||
return outcome.value
|
||||
}
|
||||
|
||||
/** Drive one input step over the client connection. */
|
||||
|
||||
@@ -95,7 +95,10 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
// A spawn-level failure is an asynchronous `error` event. Observe it in the
|
||||
// same tick as spawn so a missing cwd or OS rejection cannot crash the test
|
||||
// runner, then make startup and shutdown surface the original error.
|
||||
const childFailure = new Promise<Error>(resolve => child.once('error', resolve))
|
||||
// Keep observing after the first error: a fallback kill attempted during
|
||||
// shutdown may itself report another process error, which must not become an
|
||||
// unhandled EventEmitter error after the promise has already settled.
|
||||
const childFailure = new Promise<Error>(resolve => child.on('error', resolve))
|
||||
const spawned = Promise.race([
|
||||
new Promise<void>(resolve => child.once('spawn', resolve)),
|
||||
childFailure.then((error): never => { throw error }),
|
||||
@@ -163,14 +166,23 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
waitForUpdate: match => new Promise((resolve, reject) => updateWaiters.push({ match, resolve, reject })),
|
||||
async close(signal?: NodeJS.Signals): Promise<void> {
|
||||
await spawned
|
||||
if (child.exitCode !== null || child.signalCode !== null) return
|
||||
if (!isRunning(child)) return
|
||||
const exited = waitForExit(child)
|
||||
if (signal === undefined) child.stdin.end()
|
||||
else child.kill(signal)
|
||||
const failure = await Promise.race([
|
||||
waitForExit(child).then((): undefined => undefined),
|
||||
exited.then((): undefined => undefined),
|
||||
childFailure,
|
||||
])
|
||||
if (failure !== undefined) throw failure
|
||||
if (failure === undefined) return
|
||||
|
||||
// An `error` after spawn is not an exit edge: in particular, a failed
|
||||
// signal can leave the subprocess live. Force termination, await the
|
||||
// already-observed exit edge, and only then propagate the child error so
|
||||
// callers may safely remove cwd/session resources after close rejects.
|
||||
child.kill('SIGKILL')
|
||||
await exited
|
||||
throw failure
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -179,3 +191,8 @@ export function launchAcpTestAgent(options: AcpTestLaunchOptions): LaunchedAcpTe
|
||||
function waitForExit(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
return new Promise<void>(resolve => child.once('exit', () => { resolve() }))
|
||||
}
|
||||
|
||||
/** Whether the child still lacks either OS termination marker. */
|
||||
function isRunning(child: ChildProcessWithoutNullStreams): boolean {
|
||||
return child.exitCode === null && child.signalCode === null
|
||||
}
|
||||
@@ -80,10 +80,12 @@ describe('runScenario', () => {
|
||||
const minimal = launchAcpTestAgent({ agent: AGENT, cwd: dir })
|
||||
await minimal.client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
|
||||
const childFailure = new Error('child process failed')
|
||||
const exited = new Promise<void>(resolve => minimal.child.once('exit', () => { resolve() }))
|
||||
let exited = false
|
||||
minimal.child.once('exit', () => { exited = true })
|
||||
minimal.child.emit('error', childFailure)
|
||||
await expect(minimal.close('SIGKILL')).rejects.toBe(childFailure)
|
||||
await exited
|
||||
await expect(minimal.close('SIGTERM')).rejects.toBe(childFailure)
|
||||
// close rejects only after the fallback SIGKILL has produced an exit edge.
|
||||
expect(exited).toBe(true)
|
||||
})
|
||||
|
||||
it('drives a full turn: initialize (terminal caps), session, prompt, permission stub, harvest', { timeout: 20_000 }, async () => {
|
||||
|
||||
Reference in New Issue
Block a user