1296 lines
59 KiB
TypeScript
1296 lines
59 KiB
TypeScript
/**
|
|
* Keyless-by-default ACP snapshot suite factory. Each scenario drives the real
|
|
* subprocess and compares normalized stdout; comparable session fixtures are
|
|
* both replay input and expected output. Record mode refreshes reproducible
|
|
* model scenarios from the live API, while refresh mode replays committed
|
|
* scripts and rewrites derived artifacts without a key.
|
|
* Replay scenarios run concurrently because each subprocess owns unique temp
|
|
* cwd and persistence roots and reads only committed fixtures. Record and
|
|
* refresh stay serial while writing.
|
|
*
|
|
* Exactly one scenario per header-composition class pins the tokenized header
|
|
* sequence. Its prompt and tool-schema sequences live in independent
|
|
* sidecars, each of which may be shared with another class pin when the bytes
|
|
* are identical. Every live header is checked against the composed pin, so
|
|
* session-dependent composition must declare a separate class instead of
|
|
* escaping coverage.
|
|
* @module @deepseek-ai/dsh-acp-snapshot/suite
|
|
*/
|
|
|
|
import { readFile, readdir, rm, writeFile } from 'node:fs/promises'
|
|
import { existsSync } from 'node:fs'
|
|
import { join } from 'node:path'
|
|
import { describe, expect, it } from 'vitest'
|
|
import { type AgentUnderTest, type HarvestedLog, type InputScript, runScenario } from './harness.ts'
|
|
import {
|
|
type CwdPathMode,
|
|
type NormalizeContext,
|
|
extractSnapshotSpillPaths,
|
|
normalizeSessionLog,
|
|
normalizeStdout,
|
|
scrubRequestHeaders,
|
|
scrubSystemPrompts,
|
|
scrubToolSchemas,
|
|
tokenizeSessionFixtureCwd,
|
|
} from './normalize.ts'
|
|
|
|
/** The readable system-prompt snapshot beside its owning header pin. */
|
|
const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.expected.md'
|
|
|
|
/** The structured tool-schema snapshot beside its owning header pin. */
|
|
const TOOL_SCHEMAS_SNAPSHOT = 'tool-schemas.expected.json'
|
|
|
|
/** The optional full Windows-native stdout transcript. */
|
|
const WINDOWS_STDOUT_SNAPSHOT = 'stdout.expected.windows.jsonl'
|
|
|
|
/** Stable session-log token standing in for the sidecar's initial schemas. */
|
|
const TOOLS_TOKEN = '{{tools}}'
|
|
|
|
const PACKED_CHUNK_ROW_TYPES = new Set(['text-chunks', 'reasoning-chunks', 'tool-call-chunks'])
|
|
|
|
/** A snapshot scenario and how its fixtures are produced. */
|
|
export interface Scenario {
|
|
name: string
|
|
/** Deployment environment for this scenario's subprocess. */
|
|
env?: NodeJS.ProcessEnv
|
|
/** Whether the scenario drives at least one model turn (so a JSONL expected output applies). */
|
|
hasModelTurn: boolean
|
|
/**
|
|
* Whether the run persists a comparable session log to diff against the
|
|
* `session.jsonl` fixture. Defaults to {@link hasModelTurn} (a model turn
|
|
* always produces a log worth comparing). Set it independently for a scenario
|
|
* that produces a non-trivial durable log without calling the model.
|
|
*/
|
|
comparesLog?: boolean
|
|
/**
|
|
* Whether `test:snapshot:record` regenerates this scenario's `session.jsonl`
|
|
* from the LIVE API. `recorded` scenarios are model-driven and reproducible;
|
|
* `authored` scenarios (fixtures hand-written or hand-harvested — e.g. a
|
|
* provider error or a cancel the live API can't be coaxed into
|
|
* deterministically, a deterministic hook scenario, or a scripted repetition
|
|
* a live model won't reproduce) are NEVER re-recorded.
|
|
*/
|
|
recorded: boolean
|
|
/**
|
|
* Whether replay is driven by a hand-written `replay.override.json` sidecar
|
|
* (a `ReplayOverrideDoc` that replaces or patches the script derived from
|
|
* `session.jsonl`) — the throw/hang cases chunks cannot express. The fixture
|
|
* guard requires the sidecar exactly when this is set: the harness forwards
|
|
* the file purely on existence, so an unregistered stray sidecar would
|
|
* silently alter the derived script. The guard fails loud on either
|
|
* mismatch. Defaults to false (replay derives from the fixture's
|
|
* `assistant/chunk` events).
|
|
*/
|
|
overridden?: boolean
|
|
/**
|
|
* Whether this scenario is its header class's sole tokenized request-header
|
|
* pin. Prompt and tool-schema sidecars are selected independently, while
|
|
* every classmate is checked for equality with the reconstructed header.
|
|
*/
|
|
pinsHeader?: boolean
|
|
/**
|
|
* Header-pinning scenario whose `system-prompt.expected.md` this pin reuses.
|
|
* Defaults to this scenario. The source must own its prompt sidecar and
|
|
* declare the same {@link expectedHeaderChanges}; meaningless off a pin.
|
|
*/
|
|
systemPromptSource?: string
|
|
/**
|
|
* Header-pinning scenario whose `tool-schemas.expected.json` this pin reuses.
|
|
* Defaults to this scenario. The source must own its schema sidecar and
|
|
* declare the same {@link expectedHeaderChanges}; meaningless off a pin.
|
|
*/
|
|
toolSchemasSource?: string
|
|
/**
|
|
* How many changed `request/header` snapshots this PINNING scenario's primary
|
|
* fixture legitimately carries (default 0). Their full prompt text is kept in
|
|
* the readable Markdown pin; any other count fails. Meaningless off the pin.
|
|
*/
|
|
expectedHeaderChanges?: number
|
|
/**
|
|
* Which header-composition class this scenario belongs to. Scenarios that
|
|
* boot the same config compose the same header; each class has exactly one
|
|
* {@link pinsHeader} scenario, and the uniformity guard compares every
|
|
* other member against ITS class's pin. Defaults to `'default'`; a
|
|
* scenario booting an alternate config ({@link configPath}) whose tool
|
|
* list or prompt sections differ by construction carries its own class.
|
|
*/
|
|
headerClass?: string
|
|
/**
|
|
* Alternate LIVE config path (absolute) this scenario boots instead of
|
|
* {@link AgentUnderTest.configPath} — an overlay composing a different
|
|
* tree (its basename must still end in `cordis.yml` so the bin's replay
|
|
* swap finds the sibling `*cordis.snapshot.yml`). A scenario whose
|
|
* overlay changes the composed header also needs its own
|
|
* {@link headerClass}.
|
|
*/
|
|
configPath?: string
|
|
/**
|
|
* Parent directory for the generated session cwd. Defaults to the platform
|
|
* temp directory; set this when temp is itself part of the behavior under
|
|
* test and the scenario needs an independent project location.
|
|
*/
|
|
workspaceParent?: string
|
|
/**
|
|
* Optional final workspace preparation after the committed fixture is
|
|
* copied. Reserve this for paths that Git cannot represent portably; normal
|
|
* scenario files belong under the scenario's `workspace/` directory.
|
|
*/
|
|
prepareWorkspace?: (cwd: string) => void | Promise<void>
|
|
/**
|
|
* Whether Windows additionally compares stdout with native separators against
|
|
* `stdout.expected.windows.jsonl`. The shared canonical stdout expected output is still
|
|
* compared on every platform, and the fixture guard requires this sidecar
|
|
* exactly when the option is set.
|
|
*/
|
|
pinsNativeWindowsStdout?: boolean
|
|
/**
|
|
* Whether the scenario requires a non-Windows host, such as for POSIX process
|
|
* semantics or generated paths Windows cannot represent. The scenario's run
|
|
* test is skipped on Windows; its fixtures stay guarded on every platform.
|
|
*/
|
|
posixOnly?: boolean
|
|
}
|
|
|
|
/**
|
|
* Whether a scenario's run test is skipped for this mode and host: record mode
|
|
* skips authored (non-`recorded`) scenarios, and {@link Scenario.posixOnly}
|
|
* scenarios skip on Windows.
|
|
*
|
|
* @param scenario The scenario whose run test is being registered.
|
|
* @param recording Whether the suite runs in record mode.
|
|
* @param platform The running Node platform, injectable for unit coverage.
|
|
* @returns True when the scenario's run test must not execute.
|
|
*/
|
|
export function scenarioSkipped(
|
|
scenario: Scenario,
|
|
recording: boolean,
|
|
platform: NodeJS.Platform = process.platform,
|
|
): boolean {
|
|
if (recording && !scenario.recorded) return true
|
|
return scenario.posixOnly === true && platform === 'win32'
|
|
}
|
|
|
|
/** One stdout expected output selected for a platform run. */
|
|
interface StdoutExpectedVariant {
|
|
file: string
|
|
cwdPathMode: CwdPathMode
|
|
}
|
|
|
|
/**
|
|
* Select the shared stdout expected output plus any platform-native assertion declared by a scenario.
|
|
*
|
|
* @param scenario The scenario whose stdout contract is being selected.
|
|
* @param platform The running Node platform, injectable for unit coverage.
|
|
* @returns The ordered expected-output variants: shared canonical first, then optional Windows native.
|
|
*/
|
|
export function stdoutExpectedVariants(
|
|
scenario: Scenario,
|
|
platform: NodeJS.Platform = process.platform,
|
|
): StdoutExpectedVariant[] {
|
|
const canonical: StdoutExpectedVariant = { file: 'stdout.expected.jsonl', cwdPathMode: 'canonical' }
|
|
if (platform !== 'win32' || scenario.pinsNativeWindowsStdout !== true) return [canonical]
|
|
return [canonical, { file: WINDOWS_STDOUT_SNAPSHOT, cwdPathMode: 'native' }]
|
|
}
|
|
|
|
/** One suite's inputs: the agent to boot, where its fixtures live, and its scenario table. */
|
|
export interface SnapshotSuiteOptions {
|
|
/** The agent composition every scenario boots. */
|
|
agent: AgentUnderTest
|
|
/** Absolute path of the suite's `snapshots/` directory (one subdir per scenario). */
|
|
snapshotsDir: string
|
|
/** The scenario table; exactly one entry per header class must set `pinsHeader`. */
|
|
scenarios: Scenario[]
|
|
/**
|
|
* `replay` (keyless, the default tier), `record` (live API; re-records the
|
|
* `recorded` scenarios' fixtures and refreshes the Vitest expected outputs under
|
|
* `--update`), or `refresh` (keyless replay that rewrites stdout expected outputs and
|
|
* comparable session fixtures from the replay run). The caller derives this
|
|
* from `$DSH_SNAPSHOT` — env reading stays outside this library.
|
|
*/
|
|
mode: 'replay' | 'record' | 'refresh'
|
|
}
|
|
|
|
/** One scenario's generated claim on a shared snapshot file. */
|
|
export interface SharedSnapshotClaim {
|
|
/** Scenario that first generated the snapshot in this suite run. */
|
|
scenario: string
|
|
/** Complete generated file content. */
|
|
content: string
|
|
}
|
|
|
|
/** One committed snapshot file and its complete content. */
|
|
export interface NamedSnapshotContent {
|
|
/** Diagnostic path of the committed file. */
|
|
path: string
|
|
/** Complete committed file content. */
|
|
content: string
|
|
}
|
|
|
|
/**
|
|
* Record one scenario's generated content for a shared snapshot source.
|
|
* A later claimant must generate identical bytes; otherwise record/refresh
|
|
* would make the final file depend on scenario order.
|
|
*
|
|
* @param claims Claims already made in this suite run, keyed by source path.
|
|
* @param source The shared snapshot path being claimed.
|
|
* @param scenario The scenario generating the content.
|
|
* @param content The complete content the scenario generated.
|
|
* @returns Nothing.
|
|
*/
|
|
export function claimSharedSnapshot(
|
|
claims: Map<string, SharedSnapshotClaim>,
|
|
source: string,
|
|
scenario: string,
|
|
content: string,
|
|
): void {
|
|
const previous = claims.get(source)
|
|
if (previous !== undefined && previous.content !== content) {
|
|
throw new Error(
|
|
`acp-snapshot: shared snapshot ${source} diverged between ${previous.scenario} and ${scenario}`,
|
|
)
|
|
}
|
|
if (previous === undefined) claims.set(source, { scenario, content })
|
|
}
|
|
|
|
/**
|
|
* Reject byte-identical committed snapshots stored under different paths.
|
|
*
|
|
* @param kind Human-readable snapshot kind for the diagnostic.
|
|
* @param snapshots The committed files to compare.
|
|
* @returns Nothing.
|
|
*/
|
|
export function assertUniqueSnapshotContents(
|
|
kind: string,
|
|
snapshots: readonly NamedSnapshotContent[],
|
|
): void {
|
|
const firstPathByContent = new Map<string, string>()
|
|
for (const snapshot of snapshots) {
|
|
const firstPath = firstPathByContent.get(snapshot.content)
|
|
if (firstPath !== undefined) {
|
|
throw new Error(
|
|
`acp-snapshot: identical ${kind} snapshots appear in ${firstPath} and ${snapshot.path}; reuse one source`,
|
|
)
|
|
}
|
|
firstPathByContent.set(snapshot.content, snapshot.path)
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Validate and order a scenario directory's session-fixture filenames.
|
|
*
|
|
* The primary fixture is always `session.jsonl`; child sessions are discovered
|
|
* from contiguous `session.1.jsonl` … filenames. The directory is the source of
|
|
* truth, so scenario tables do not duplicate a child count that can drift from
|
|
* the files. A session-like JSONL with any other suffix fails loud.
|
|
*
|
|
* @param names File names in one scenario directory.
|
|
* @returns The primary and child fixture names in replay/harvest order.
|
|
*/
|
|
export function sessionFixtureNames(names: readonly string[]): string[] {
|
|
if (!names.includes('session.jsonl')) throw new Error('missing session.jsonl')
|
|
const children: { name: string; index: number }[] = []
|
|
for (const name of names) {
|
|
if (name === 'session.jsonl') continue
|
|
if (!name.startsWith('session.') || !name.endsWith('.jsonl')) continue
|
|
const match = /^session\.([1-9]\d*)\.jsonl$/.exec(name)
|
|
if (match === null) throw new Error(`invalid child session fixture name: ${name}`)
|
|
children.push({ name, index: Number(match[1]) })
|
|
}
|
|
children.sort((a, b) => a.index - b.index)
|
|
for (const [offset, child] of children.entries()) {
|
|
const expected = offset + 1
|
|
if (child.index !== expected) {
|
|
throw new Error(`child session fixtures must be contiguous: expected session.${expected}.jsonl, found ${child.name}`)
|
|
}
|
|
}
|
|
return ['session.jsonl', ...children.map(child => child.name)]
|
|
}
|
|
|
|
/** Read one scenario directory's validated session-fixture inventory. */
|
|
async function sessionFixtures(dir: string): Promise<string[]> {
|
|
const entries = await readdir(dir, { withFileTypes: true })
|
|
return sessionFixtureNames(entries.filter(entry => entry.isFile()).map(entry => entry.name))
|
|
}
|
|
|
|
/**
|
|
* Derive normalization values from a fixture's own session header. Recorded ids and cwd differ
|
|
* from the live replay run; the non-empty sentinel for missing cwd avoids accidental empty-
|
|
* string replacement.
|
|
*
|
|
* @param fixture The committed `session.jsonl` content.
|
|
* @returns The fixture's own volatile values, ready for {@link normalizeSessionLog}.
|
|
*/
|
|
export function fixtureContext(fixture: string): NormalizeContext {
|
|
const firstLine = fixture.split('\n').find(line => line.trim().length > 0) ?? '{}'
|
|
const header = JSON.parse(firstLine) as { id?: unknown; cwd?: unknown }
|
|
return {
|
|
sessionIds: typeof header.id === 'string' ? [header.id] : [],
|
|
cwd: typeof header.cwd === 'string' ? header.cwd : '\0no-cwd\0',
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The `data.header` payload of every `request/header` event in a session
|
|
* JSONL, in log order, with the log's volatile values scrubbed first
|
|
* ({@link normalizeSessionLog}) so headers harvested from different runs —
|
|
* each embedding its own generated cwd in the composed prompt — compare on equal
|
|
* footing.
|
|
*
|
|
* @param rawLog The session `.jsonl` content to extract headers from.
|
|
* @param ctx The volatile values of the run that produced it.
|
|
* @returns The normalized `data.header` payloads, in log order.
|
|
*/
|
|
export function normalizedHeaders(rawLog: string, ctx: NormalizeContext): unknown[] {
|
|
return normalizeSessionLog(rawLog, ctx)
|
|
.split('\n')
|
|
.filter(line => line.trim().length > 0)
|
|
.map(line => JSON.parse(line) as { type?: unknown; data?: { header?: unknown } })
|
|
.filter(record => record.type === 'request/header')
|
|
.map(record => record.data?.header)
|
|
}
|
|
|
|
/**
|
|
* The normalized string-valued system prompts carried by request headers in a
|
|
* session JSONL, in log order. Headers without a string prompt are omitted so
|
|
* callers can assert one prompt per header explicitly.
|
|
*
|
|
* @param rawLog The session `.jsonl` content to inspect.
|
|
* @param ctx The volatile values of the run that produced it.
|
|
* @returns The normalized system prompts, in header order.
|
|
*/
|
|
export function normalizedSystemPrompts(rawLog: string, ctx: NormalizeContext): string[] {
|
|
return normalizedHeaders(rawLog, ctx).flatMap((header) => {
|
|
if (header === null || typeof header !== 'object') return []
|
|
const system = (header as { system?: unknown }).system
|
|
return typeof system === 'string' ? [system] : []
|
|
})
|
|
}
|
|
|
|
/**
|
|
* The normalized tool-schema arrays carried by request headers in a session
|
|
* JSONL, in log order. Headers without an array-valued tools field are omitted
|
|
* so callers can assert one schema set per header explicitly.
|
|
*
|
|
* @param rawLog The session `.jsonl` content to inspect.
|
|
* @param ctx The volatile values of the run that produced it.
|
|
* @returns The normalized initial tool-schema arrays, in header order.
|
|
*/
|
|
export function normalizedToolSchemas(rawLog: string, ctx: NormalizeContext): unknown[][] {
|
|
return normalizedHeaders(rawLog, ctx).flatMap((header) => {
|
|
if (header === null || typeof header !== 'object') return []
|
|
const tools = (header as { tools?: unknown }).tools
|
|
return Array.isArray(tools) ? [tools] : []
|
|
})
|
|
}
|
|
|
|
/** The structured contents of a tool-schema sidecar. */
|
|
export interface ToolSchemasSnapshot {
|
|
/** The complete tool schemas from the pinned request header. */
|
|
initial: unknown[]
|
|
/** Complete tool schemas from subsequent changed-header snapshots. */
|
|
changes: unknown[][]
|
|
}
|
|
|
|
/**
|
|
* Render the full tool-schema sequence as canonical, readable JSON.
|
|
*
|
|
* @param initial The pinned request header's complete tool schemas.
|
|
* @param changes Complete tool schemas from later changed headers.
|
|
* @returns A pretty-printed JSON snapshot ending in one newline.
|
|
*/
|
|
export function formatToolSchemasSnapshot(initial: readonly unknown[], changes: readonly unknown[][] = []): string {
|
|
return `${JSON.stringify({ initial, changes }, null, 2)}\n`
|
|
}
|
|
|
|
/**
|
|
* Parse and validate the stable top-level shape of a tool-schema sidecar.
|
|
*
|
|
* @param snapshot The JSON sidecar text.
|
|
* @returns Its initial and changed-header schema sets.
|
|
*/
|
|
export function parseToolSchemasSnapshot(snapshot: string): ToolSchemasSnapshot {
|
|
const parsed = JSON.parse(snapshot) as unknown
|
|
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
throw new Error('acp-snapshot: tool-schema snapshot must be an object')
|
|
}
|
|
const { initial, changes } = parsed as { initial?: unknown; changes?: unknown }
|
|
if (!Array.isArray(initial) || !Array.isArray(changes) || !changes.every(Array.isArray)) {
|
|
throw new Error('acp-snapshot: tool-schema snapshot must carry array-valued initial and changes fields')
|
|
}
|
|
return { initial, changes }
|
|
}
|
|
|
|
/**
|
|
* Restore one sidecar schema set into a tokenized pinned header.
|
|
*
|
|
* @param header The parsed request header carrying `tools: "{{tools}}"`.
|
|
* @param schemas The complete schemas for this full header snapshot.
|
|
* @returns A copy of the header with its complete schemas restored.
|
|
*/
|
|
export function restorePinnedToolSchemas(header: unknown, schemas: readonly unknown[]): unknown {
|
|
if (header === null || typeof header !== 'object' || Array.isArray(header)) {
|
|
throw new Error('acp-snapshot: pinned request header must be an object')
|
|
}
|
|
if ((header as { tools?: unknown }).tools !== TOOLS_TOKEN) {
|
|
throw new Error(`acp-snapshot: pinned request header tools must equal ${TOOLS_TOKEN}`)
|
|
}
|
|
return { ...header, tools: schemas }
|
|
}
|
|
|
|
/**
|
|
* Render a normalized prompt as a repository-friendly Markdown snapshot.
|
|
* Prompt text is unchanged except that a missing terminal newline is added so
|
|
* the committed file follows the repository newline contract.
|
|
*
|
|
* @param prompt The normalized system prompt.
|
|
* @param changes Full normalized prompts from later changed-header snapshots.
|
|
* @returns Markdown snapshot text ending in a newline.
|
|
*/
|
|
export function formatSystemPromptSnapshot(
|
|
prompt: string,
|
|
changes: readonly string[] = [],
|
|
): string {
|
|
let snapshot = prompt.endsWith('\n') ? prompt : `${prompt}\n`
|
|
for (const [index, change] of changes.entries()) {
|
|
snapshot += `\n<!-- request/header change ${index + 1} -->\n\n`
|
|
snapshot += change.endsWith('\n') ? change : `${change}\n`
|
|
}
|
|
return snapshot
|
|
}
|
|
|
|
/** Return the initial-prompt portion of a possibly multi-header snapshot. */
|
|
function initialSystemPromptSnapshot(snapshot: string): string {
|
|
const marker = snapshot.indexOf('\n<!-- request/header change ')
|
|
return marker < 0 ? snapshot : snapshot.slice(0, marker)
|
|
}
|
|
|
|
/**
|
|
* Count changed `request/header` snapshots in a session JSONL.
|
|
*
|
|
* @param rawLog The session `.jsonl` content.
|
|
* @returns How many headers carry reason `change`.
|
|
*/
|
|
export function headerChangeCount(rawLog: string): number {
|
|
return rawLog.split('\n')
|
|
.filter(line => line.trim().length > 0)
|
|
.filter((line) => {
|
|
const record = JSON.parse(line) as { type?: unknown; data?: { reason?: unknown } }
|
|
return record.type === 'request/header' && record.data?.reason === 'change'
|
|
})
|
|
.length
|
|
}
|
|
|
|
/** A literal string replacement used to carry an existing fixture's volatile value into a refreshed log. */
|
|
export interface FixtureReplacement {
|
|
/** The fresh replay-run value to replace. */
|
|
from: string
|
|
/** The existing fixture value to keep. */
|
|
to: string
|
|
}
|
|
|
|
function parseJsonlRecords(text: string): Record<string, unknown>[] {
|
|
return text.split('\n')
|
|
.filter(line => line.trim().length > 0)
|
|
.map(line => JSON.parse(line) as Record<string, unknown>)
|
|
}
|
|
|
|
/** One packed row's member times, or `undefined` for an ordinary record. */
|
|
function packedTimes(record: Record<string, unknown>): number[] | undefined {
|
|
if (!PACKED_CHUNK_ROW_TYPES.has(record.type as string)) return undefined
|
|
const row = record as unknown as { time0: number; data: { dt: number[] } }
|
|
const times = [row.time0]
|
|
for (const gap of row.data.dt) times.push((times[times.length - 1] as number) + gap)
|
|
return times
|
|
}
|
|
|
|
/** Expand packed timing envelopes so refresh alignment follows logical events, not physical lines. */
|
|
function logicalRecords(records: Record<string, unknown>[]): Record<string, unknown>[] {
|
|
return records.flatMap((record) => {
|
|
const times = packedTimes(record)
|
|
return times === undefined ? [record] : times.map(time => ({ type: 'assistant/chunk', time }))
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Find tool calls whose structured result reports `UNKNOWN_TOOL`.
|
|
*
|
|
* Snapshot refresh must not turn a missing registration into accepted behavior;
|
|
* intentional unknown-tool behavior belongs in a focused unit or e2e test.
|
|
*
|
|
* @param rawLog The session JSONL to inspect.
|
|
* @returns The failing call ids in log order, using a diagnostic placeholder when absent.
|
|
*/
|
|
export function unknownToolCallIds(rawLog: string): string[] {
|
|
return parseJsonlRecords(rawLog).flatMap((record) => {
|
|
if (record.type !== 'tool/result') return []
|
|
const data = record.data
|
|
if (data === null || typeof data !== 'object') return []
|
|
const { message, error } = data as { message?: unknown; error?: unknown }
|
|
if (error === null || typeof error !== 'object') return []
|
|
if ((error as { code?: unknown }).code !== 'UNKNOWN_TOOL') return []
|
|
const source = typeof message === 'object' && message !== null
|
|
? (message as { source?: unknown }).source
|
|
: undefined
|
|
const callId = typeof source === 'object' && source !== null
|
|
? (source as { callId?: unknown }).callId
|
|
: undefined
|
|
return [typeof callId === 'string' ? callId : '<missing callId>']
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Build the cross-log id/cwd/spill-path replacements used by refresh write-back.
|
|
*
|
|
* @param logs The freshly harvested logs, in fixture order.
|
|
* @param fixtures The existing fixture contents, in matching order.
|
|
* @returns Literal replacements from fresh volatile values to the fixture's old values.
|
|
*/
|
|
export function refreshFixtureReplacements(logs: HarvestedLog[], fixtures: string[]): FixtureReplacement[] {
|
|
const replacements: FixtureReplacement[] = []
|
|
for (let i = 0; i < logs.length; i++) {
|
|
const fresh = parseJsonlRecords((logs[i] as HarvestedLog).content)[0]
|
|
const existing = parseJsonlRecords(fixtures[i] ?? '')[0]
|
|
for (const field of ['id', 'cwd'] as const) {
|
|
const from = fresh?.[field]
|
|
const to = existing?.[field]
|
|
if (typeof from === 'string' && typeof to === 'string' && from.length > 0 && from !== to) {
|
|
replacements.push({ from, to })
|
|
}
|
|
}
|
|
// Stabilize snapshot spill paths: match by filename suffix so the raw
|
|
// fixture does not churn on every refresh from a different session run.
|
|
const freshSpills = extractSnapshotSpillPaths((logs[i] as HarvestedLog).content)
|
|
const existingSpills = extractSnapshotSpillPaths(fixtures[i] ?? '')
|
|
for (const [name, existingPath] of existingSpills) {
|
|
const freshPath = freshSpills.get(name)
|
|
if (freshPath !== undefined && freshPath !== existingPath) {
|
|
replacements.push({ from: freshPath, to: existingPath })
|
|
}
|
|
}
|
|
}
|
|
return replacements
|
|
}
|
|
|
|
function preserveFixtureVolatiles(record: Record<string, unknown>, existing: Record<string, unknown> | undefined): void {
|
|
if (existing === undefined || existing.type !== record.type) return
|
|
if (record.type === 'session') {
|
|
for (const field of ['id', 'createdAt', 'cwd', 'parentSession'] as const) {
|
|
if (field in record && field in existing) record[field] = existing[field]
|
|
}
|
|
return
|
|
}
|
|
if ('time' in record && 'time' in existing) record.time = existing.time
|
|
if (record.type !== 'hook/result') return
|
|
const data = record.data
|
|
const existingData = existing.data
|
|
if (
|
|
data !== null && typeof data === 'object'
|
|
&& existingData !== null && typeof existingData === 'object'
|
|
&& 'durationMs' in data && 'durationMs' in existingData
|
|
) {
|
|
(data as Record<string, unknown>).durationMs = (existingData as Record<string, unknown>).durationMs
|
|
}
|
|
}
|
|
|
|
/** Carry logical member times into a fresh packed row while leaving its fragment arrays untouched. */
|
|
function preservePackedMemberTimes(
|
|
record: Record<string, unknown>,
|
|
existingMembers: Record<string, unknown>[],
|
|
): void {
|
|
if (!PACKED_CHUNK_ROW_TYPES.has(record.type as string)) return
|
|
const row = record as unknown as { time0: number; data: { dt: number[] } }
|
|
const firstTime = existingMembers[0]?.time
|
|
if (!Number.isSafeInteger(firstTime)) return
|
|
row.time0 = firstTime as number
|
|
if (existingMembers.length !== row.data.dt.length + 1) return
|
|
const times = existingMembers.map(member => Number.isSafeInteger(member.time) ? member.time as number : undefined)
|
|
if (times.some(time => time === undefined)) return
|
|
const memberTimes = times as number[]
|
|
const gaps = memberTimes.slice(1).map((time, index) => time - (memberTimes[index] as number))
|
|
if (gaps.some(gap => !Number.isSafeInteger(gap))) return
|
|
row.data.dt = gaps
|
|
}
|
|
|
|
/** Whether a parsed JSON value is a non-array object. */
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
}
|
|
|
|
/**
|
|
* Reuse existing leaves whose normalized values equal the fresh values.
|
|
* Objects merge by key; arrays merge only when their positions still align.
|
|
*/
|
|
function preserveNormalizedVolatiles(
|
|
fresh: unknown,
|
|
existing: unknown,
|
|
normalizedFresh: unknown,
|
|
normalizedExisting: unknown,
|
|
stringMappings: ReadonlyMap<string, string>,
|
|
): unknown {
|
|
if (
|
|
Array.isArray(fresh)
|
|
&& Array.isArray(existing)
|
|
&& Array.isArray(normalizedFresh)
|
|
&& Array.isArray(normalizedExisting)
|
|
) {
|
|
if (
|
|
fresh.length !== existing.length
|
|
|| fresh.length !== normalizedFresh.length
|
|
|| fresh.length !== normalizedExisting.length
|
|
) return fresh
|
|
return fresh.map((value, index) => preserveNormalizedVolatiles(
|
|
value,
|
|
existing[index],
|
|
normalizedFresh[index],
|
|
normalizedExisting[index],
|
|
stringMappings,
|
|
))
|
|
}
|
|
if (
|
|
isRecord(fresh)
|
|
&& isRecord(existing)
|
|
&& isRecord(normalizedFresh)
|
|
&& isRecord(normalizedExisting)
|
|
) {
|
|
return Object.fromEntries(Object.entries(fresh).map(([key, value]) => [
|
|
key,
|
|
Object.hasOwn(existing, key)
|
|
&& Object.hasOwn(normalizedFresh, key)
|
|
&& Object.hasOwn(normalizedExisting, key)
|
|
? preserveNormalizedVolatiles(
|
|
value,
|
|
existing[key],
|
|
normalizedFresh[key],
|
|
normalizedExisting[key],
|
|
stringMappings,
|
|
)
|
|
: value,
|
|
]))
|
|
}
|
|
if (
|
|
typeof fresh === 'string'
|
|
&& typeof existing === 'string'
|
|
&& typeof normalizedFresh === 'string'
|
|
&& normalizedFresh === normalizedExisting
|
|
) {
|
|
return stringMappings.get(JSON.stringify([normalizedFresh, fresh])) === existing
|
|
? existing
|
|
: fresh
|
|
}
|
|
return Object.is(normalizedFresh, normalizedExisting) ? existing : fresh
|
|
}
|
|
|
|
/** Normalize one aligned record with the same contract used by fixture comparison. */
|
|
function normalizedRefreshRecord(
|
|
record: Record<string, unknown>,
|
|
context: NormalizeContext,
|
|
): Record<string, unknown> {
|
|
return JSON.parse(normalizeSessionLog(`${JSON.stringify(record)}\n`, context)) as Record<string, unknown>
|
|
}
|
|
|
|
/**
|
|
* Add normalized-equivalent string replacements to a bijection.
|
|
* Structural differences are fresh-owned and therefore contribute no mapping.
|
|
*/
|
|
function collectNormalizedStringMappings(
|
|
fresh: unknown,
|
|
existing: unknown,
|
|
normalizedFresh: unknown,
|
|
normalizedExisting: unknown,
|
|
forward: Map<string, string>,
|
|
reverse: Map<string, string>,
|
|
): boolean {
|
|
if (
|
|
Array.isArray(fresh)
|
|
&& Array.isArray(existing)
|
|
&& Array.isArray(normalizedFresh)
|
|
&& Array.isArray(normalizedExisting)
|
|
) {
|
|
if (
|
|
fresh.length !== existing.length
|
|
|| fresh.length !== normalizedFresh.length
|
|
|| fresh.length !== normalizedExisting.length
|
|
) return true
|
|
return fresh.every((value, index) => collectNormalizedStringMappings(
|
|
value,
|
|
existing[index],
|
|
normalizedFresh[index],
|
|
normalizedExisting[index],
|
|
forward,
|
|
reverse,
|
|
))
|
|
}
|
|
if (
|
|
isRecord(fresh)
|
|
&& isRecord(existing)
|
|
&& isRecord(normalizedFresh)
|
|
&& isRecord(normalizedExisting)
|
|
) {
|
|
return Object.entries(fresh).every(([key, value]) =>
|
|
!Object.hasOwn(existing, key)
|
|
|| !Object.hasOwn(normalizedFresh, key)
|
|
|| !Object.hasOwn(normalizedExisting, key)
|
|
|| collectNormalizedStringMappings(
|
|
value,
|
|
existing[key],
|
|
normalizedFresh[key],
|
|
normalizedExisting[key],
|
|
forward,
|
|
reverse,
|
|
))
|
|
}
|
|
if (
|
|
typeof fresh !== 'string'
|
|
|| typeof existing !== 'string'
|
|
|| typeof normalizedFresh !== 'string'
|
|
|| normalizedFresh !== normalizedExisting
|
|
|| fresh === existing
|
|
) return true
|
|
const freshKey = JSON.stringify([normalizedFresh, fresh])
|
|
const existingKey = JSON.stringify([normalizedFresh, existing])
|
|
const mappedExisting = forward.get(freshKey)
|
|
const mappedFresh = reverse.get(existingKey)
|
|
if (
|
|
mappedExisting !== undefined && mappedExisting !== existing
|
|
|| mappedFresh !== undefined && mappedFresh !== fresh
|
|
) return false
|
|
forward.set(freshKey, existing)
|
|
reverse.set(existingKey, fresh)
|
|
return true
|
|
}
|
|
|
|
/**
|
|
* Build a log-wide bijection for normalized-equivalent strings.
|
|
* Any unexplained record mismatch or conflicting replacement disables reuse.
|
|
*/
|
|
function normalizedStringMappings(
|
|
records: Record<string, unknown>[],
|
|
freshRecords: Record<string, unknown>[],
|
|
existingRecords: Record<string, unknown>[],
|
|
freshContext: NormalizeContext,
|
|
existingContext: NormalizeContext,
|
|
): Map<string, string> | undefined {
|
|
const forward = new Map<string, string>()
|
|
const reverse = new Map<string, string>()
|
|
let existingIndex = 0
|
|
for (let recordIndex = 0; recordIndex < records.length; recordIndex++) {
|
|
const record = records[recordIndex] as Record<string, unknown>
|
|
const existingRecord = existingRecords[existingIndex]
|
|
const memberCount = packedTimes(record)?.length ?? 1
|
|
if (record.type === 'session/title' && existingRecord?.type !== 'session/title') continue
|
|
if (memberCount > 1) {
|
|
const existingMembers = existingRecords.slice(existingIndex, existingIndex + memberCount)
|
|
if (
|
|
existingMembers.length !== memberCount
|
|
|| existingMembers.some(member => member.type !== 'assistant/chunk')
|
|
) return undefined
|
|
} else {
|
|
if (existingRecord === undefined || existingRecord.type !== record.type) return undefined
|
|
if (!collectNormalizedStringMappings(
|
|
record,
|
|
existingRecord,
|
|
normalizedRefreshRecord(freshRecords[recordIndex] as Record<string, unknown>, freshContext),
|
|
normalizedRefreshRecord(existingRecord, existingContext),
|
|
forward,
|
|
reverse,
|
|
)) return undefined
|
|
}
|
|
existingIndex += memberCount
|
|
}
|
|
return existingIndex === existingRecords.length ? forward : undefined
|
|
}
|
|
|
|
/**
|
|
* Rewrite a fresh replay-produced log so repeated refreshes do not churn
|
|
* volatile fixture fields. Meaningful event payloads come from `fresh`; the
|
|
* existing fixture lends normalized-equivalent values, including ids, paths,
|
|
* creation/event times, spill locators, and hook durations, only when the
|
|
* complete record layout aligns and volatile strings form a consistent
|
|
* bijection. Ambiguous layouts or mappings keep fresh strings. Packed timing
|
|
* envelopes expand for alignment, so packing does not shift later records;
|
|
* fresh semantic values and fragment arrays remain authoritative.
|
|
*
|
|
* @param fresh The newly harvested session JSONL.
|
|
* @param existing The committed fixture JSONL being refreshed.
|
|
* @param replacements Cross-log literal replacements from {@link refreshFixtureReplacements}.
|
|
* @param freshContext The harvested run's ids, cwd, and every cwd alias.
|
|
* @returns The stabilized JSONL content to write back.
|
|
*/
|
|
export function stabilizeRefreshLog(
|
|
fresh: string,
|
|
existing: string,
|
|
replacements: FixtureReplacement[],
|
|
freshContext: NormalizeContext,
|
|
): string {
|
|
const freshRecords = parseJsonlRecords(fresh)
|
|
let stable = fresh
|
|
for (const { from, to } of replacements) stable = stable.split(from).join(to)
|
|
const existingRecords = logicalRecords(parseJsonlRecords(existing))
|
|
const records = parseJsonlRecords(stable)
|
|
const existingContext = fixtureContext(existing)
|
|
const stringMappings = normalizedStringMappings(
|
|
records,
|
|
freshRecords,
|
|
existingRecords,
|
|
freshContext,
|
|
existingContext,
|
|
)
|
|
let existingIndex = 0
|
|
let previousEventTime: unknown
|
|
for (let i = 0; i < records.length; i++) {
|
|
let record = records[i] as Record<string, unknown>
|
|
const existingRecord = existingRecords[existingIndex]
|
|
const memberCount = packedTimes(record)?.length ?? 1
|
|
const insertedTitle = record.type === 'session/title' && existingRecord?.type !== 'session/title'
|
|
if (insertedTitle) {
|
|
/* v8 ignore next -- a title is turn-enclosed, so a preceding event time exists in every valid fixture. */
|
|
if (typeof previousEventTime !== 'number') throw new Error('acp-snapshot: inserted title has no preceding event time')
|
|
record.time = previousEventTime
|
|
} else {
|
|
if (
|
|
stringMappings !== undefined
|
|
&& memberCount === 1
|
|
&& existingRecord !== undefined
|
|
&& existingRecord.type === record.type
|
|
) {
|
|
record = preserveNormalizedVolatiles(
|
|
record,
|
|
existingRecord,
|
|
normalizedRefreshRecord(freshRecords[i] as Record<string, unknown>, freshContext),
|
|
normalizedRefreshRecord(existingRecord, existingContext),
|
|
stringMappings,
|
|
) as Record<string, unknown>
|
|
records[i] = record
|
|
}
|
|
preservePackedMemberTimes(record, existingRecords.slice(existingIndex, existingIndex + memberCount))
|
|
preserveFixtureVolatiles(record, existingRecord)
|
|
existingIndex += memberCount
|
|
}
|
|
if (typeof record.time === 'number') previousEventTime = record.time
|
|
}
|
|
return records.map(record => JSON.stringify(record)).join('\n') + '\n'
|
|
}
|
|
|
|
/**
|
|
* Register the suite: one test per scenario (the expected-output and log comparisons and
|
|
* the header-uniformity guard) plus the fixture guard block (no orphan
|
|
* scenario dirs, required files present, exactly one pin per header class,
|
|
* shared sidecars unique and well-formed, every JSONL prompt-scrubbed,
|
|
* non-pinning fixtures fully header-scrubbed). Must
|
|
* run at vitest collection time — it calls `describe`/`it`. Throws
|
|
* immediately if any header class lacks a pinning scenario or carries two
|
|
* (the uniformity guard needs exactly one comparison anchor per class).
|
|
*
|
|
* @param options The agent, snapshots directory, scenario table, and mode.
|
|
*/
|
|
export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void {
|
|
const { agent, snapshotsDir, scenarios, mode } = options
|
|
const RECORDING = mode === 'record'
|
|
const REFRESHING = mode === 'refresh'
|
|
const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay'
|
|
const scenarioSuite = mode === 'replay' ? describe.concurrent : describe
|
|
|
|
/** The class a scenario's header composition belongs to (see {@link Scenario.headerClass}). */
|
|
const classOf = (scenario: Scenario): string => scenario.headerClass ?? 'default'
|
|
|
|
const scenariosByName = new Map<string, Scenario>()
|
|
for (const scenario of scenarios) {
|
|
if (scenariosByName.has(scenario.name)) {
|
|
throw new Error(`acp-snapshot: duplicate scenario name "${scenario.name}"`)
|
|
}
|
|
scenariosByName.set(scenario.name, scenario)
|
|
for (const field of ['systemPromptSource', 'toolSchemasSource'] as const) {
|
|
if (scenario[field] !== undefined && scenario.pinsHeader !== true) {
|
|
throw new Error(`acp-snapshot: ${scenario.name}.${field} is only valid on a header-pinning scenario`)
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Each header class's single pinning scenario. Guarded here (and by meta-tests) so a pin cannot silently vanish or split. */
|
|
const pinningByClass = new Map<string, Scenario>()
|
|
for (const scenario of scenarios) {
|
|
if (scenario.pinsHeader !== true) continue
|
|
const cls = classOf(scenario)
|
|
const existing = pinningByClass.get(cls)
|
|
if (existing) throw new Error(`acp-snapshot: header class "${cls}" pinned by both ${existing.name} and ${scenario.name}`)
|
|
pinningByClass.set(cls, scenario)
|
|
}
|
|
for (const scenario of scenarios) {
|
|
if (!pinningByClass.has(classOf(scenario))) {
|
|
throw new Error(`acp-snapshot: no scenario pins the request-header content of class "${classOf(scenario)}" (needed by ${scenario.name})`)
|
|
}
|
|
}
|
|
|
|
const sourceFor = (
|
|
pinningScenario: Scenario,
|
|
field: 'systemPromptSource' | 'toolSchemasSource',
|
|
label: string,
|
|
): Scenario => {
|
|
const sourceName = pinningScenario[field] ?? pinningScenario.name
|
|
const source = scenariosByName.get(sourceName)
|
|
if (source === undefined) {
|
|
throw new Error(`acp-snapshot: ${pinningScenario.name} names unknown ${label} source "${sourceName}"`)
|
|
}
|
|
if (source.pinsHeader !== true) {
|
|
throw new Error(`acp-snapshot: ${pinningScenario.name} names non-pinning ${label} source "${sourceName}"`)
|
|
}
|
|
if (source[field] !== undefined && source[field] !== source.name) {
|
|
throw new Error(`acp-snapshot: ${pinningScenario.name} names ${label} source "${sourceName}", which does not own its sidecar`)
|
|
}
|
|
const expectedChanges = pinningScenario.expectedHeaderChanges ?? 0
|
|
const sourceChanges = source.expectedHeaderChanges ?? 0
|
|
if (sourceChanges !== expectedChanges) {
|
|
throw new Error(
|
|
`acp-snapshot: ${pinningScenario.name} and ${sourceName} declare different header-change counts for shared ${label}`,
|
|
)
|
|
}
|
|
return source
|
|
}
|
|
|
|
const promptSourceByClass = new Map<string, Scenario>()
|
|
const schemaSourceByClass = new Map<string, Scenario>()
|
|
for (const [cls, pinningScenario] of pinningByClass) {
|
|
promptSourceByClass.set(cls, sourceFor(pinningScenario, 'systemPromptSource', 'system-prompt snapshot'))
|
|
schemaSourceByClass.set(cls, sourceFor(pinningScenario, 'toolSchemasSource', 'tool-schema snapshot'))
|
|
}
|
|
const promptOwners = new Set([...promptSourceByClass.values()].map(source => source.name))
|
|
const schemaOwners = new Set([...schemaSourceByClass.values()].map(source => source.name))
|
|
const promptClaims = new Map<string, SharedSnapshotClaim>()
|
|
const schemaClaims = new Map<string, SharedSnapshotClaim>()
|
|
|
|
scenarioSuite('snapshot scenarios', () => {
|
|
for (const scenario of scenarios) {
|
|
// In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones
|
|
// (sidecar-driven errors/cancel) are never re-recorded. `posixOnly` scenarios skip on Windows.
|
|
it.skipIf(scenarioSkipped(scenario, RECORDING))(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => {
|
|
const dir = join(snapshotsDir, scenario.name)
|
|
const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as InputScript
|
|
const overrideFile = join(dir, 'replay.override.json')
|
|
const workspaceDir = join(dir, 'workspace')
|
|
// Replay/refresh need the committed inventory up front because those
|
|
// files drive the model scripts. Record mode creates that inventory
|
|
// from the harvested live logs, so it must also work for a brand-new
|
|
// scenario with no session.jsonl yet.
|
|
let fixtureFiles = RECORDING ? [] : await sessionFixtures(dir)
|
|
const childFixtureFiles = fixtureFiles.slice(1)
|
|
const comparesLog = scenario.comparesLog ?? scenario.hasModelTurn
|
|
const result = await runScenario(input, {
|
|
agent,
|
|
mode: childMode,
|
|
fixtureFile: join(dir, 'session.jsonl'),
|
|
...scenario.env !== undefined ? { env: scenario.env } : {},
|
|
...existsSync(overrideFile) ? { overrideFile } : {},
|
|
// In REPLAY, forward the recorded child fixtures so each subagent session
|
|
// replays from its own script. In RECORD they are harvested, not read.
|
|
...!RECORDING && childFixtureFiles.length > 0 ? { childFiles: childFixtureFiles.map(file => join(dir, file)) } : {},
|
|
...existsSync(workspaceDir) ? { workspaceDir } : {},
|
|
...scenario.prepareWorkspace !== undefined ? { prepareWorkspace: scenario.prepareWorkspace } : {},
|
|
...scenario.workspaceParent !== undefined ? { workspaceParent: scenario.workspaceParent } : {},
|
|
// A scenario booting an overlay tree passes its own live config; the
|
|
// bin's replay swap derives the sibling `*cordis.snapshot.yml` from it.
|
|
...scenario.configPath !== undefined ? { configPath: scenario.configPath } : {},
|
|
})
|
|
|
|
for (const log of result.sessionLogs) {
|
|
expect(unknownToolCallIds(log.content), `session ${log.id}: snapshot scenarios must not accept UNKNOWN_TOOL`)
|
|
.toEqual([])
|
|
}
|
|
|
|
// Scrub every volatile id the run produced: the ACP server-issued session id plus every
|
|
// harvested log's recorded id (a subagent child id never surfaces over ACP, but it
|
|
// appears in the child's own log header).
|
|
const ctx: NormalizeContext = {
|
|
sessionIds: [
|
|
...result.sessionId !== undefined ? [result.sessionId] : [],
|
|
...result.sessionLogs.map(l => l.id),
|
|
],
|
|
cwd: result.cwd,
|
|
cwdAliases: result.cwdAliases,
|
|
}
|
|
|
|
// Record writes live model fixtures; keyless refresh writes every comparable replayed
|
|
// fixture. Pinning JSONL keeps prefixes but moves prompts and schemas into sidecars.
|
|
const scrub = scenario.pinsHeader === true
|
|
? (log: string): string => scrubToolSchemas(scrubSystemPrompts(log))
|
|
: scrubRequestHeaders
|
|
const portableFixture = scenario.workspaceParent === undefined
|
|
? tokenizeSessionFixtureCwd
|
|
: (log: string): string => log
|
|
const existingFixtures = REFRESHING
|
|
? await Promise.all(fixtureFiles.map(file => readFile(join(dir, file), 'utf8')))
|
|
: []
|
|
const replacements = REFRESHING ? refreshFixtureReplacements(result.sessionLogs, existingFixtures) : []
|
|
const writesSessionFixtures = (RECORDING && scenario.recorded && scenario.hasModelTurn)
|
|
|| (REFRESHING && comparesLog)
|
|
if (writesSessionFixtures) {
|
|
expect(result.sessionLogs.length, `${mode} produced no session log to harvest`).toBeGreaterThan(0)
|
|
if (REFRESHING) {
|
|
expect(result.sessionLogs.length, `expected ${fixtureFiles.length} session logs (parent + children)`)
|
|
.toBe(fixtureFiles.length)
|
|
}
|
|
const outputFixtureFiles = [
|
|
'session.jsonl',
|
|
...Array.from({ length: result.sessionLogs.length - 1 }, (_, i) => `session.${i + 1}.jsonl`),
|
|
]
|
|
const primary = (result.sessionLogs[0] as HarvestedLog).content
|
|
await writeFile(join(dir, outputFixtureFiles[0] as string), scrub(portableFixture(
|
|
REFRESHING ? stabilizeRefreshLog(primary, existingFixtures[0] as string, replacements, ctx) : primary,
|
|
)))
|
|
for (let i = 1; i < result.sessionLogs.length; i++) {
|
|
const child = (result.sessionLogs[i] as HarvestedLog).content
|
|
await writeFile(join(dir, outputFixtureFiles[i] as string), scrub(portableFixture(
|
|
REFRESHING ? stabilizeRefreshLog(child, existingFixtures[i] as string, replacements, ctx) : child,
|
|
)))
|
|
}
|
|
if (RECORDING) {
|
|
const outputNames = new Set(outputFixtureFiles)
|
|
const entries = await readdir(dir, { withFileTypes: true })
|
|
await Promise.all(entries
|
|
.filter(entry => entry.isFile()
|
|
// Only valid numbered children are record-owned stale output.
|
|
// Malformed session-like names stay for the inventory guard to
|
|
// reject instead of being silently deleted during mutation.
|
|
&& /^session\.[1-9]\d*\.jsonl$/.test(entry.name)
|
|
&& !outputNames.has(entry.name))
|
|
.map(entry => rm(join(dir, entry.name))))
|
|
fixtureFiles = outputFixtureFiles
|
|
}
|
|
if (scenario.pinsHeader === true) {
|
|
const primary = result.sessionLogs[0] as HarvestedLog
|
|
const prompts = normalizedSystemPrompts(primary.content, ctx)
|
|
expect(prompts.length, `${mode} produced no system prompt to snapshot`).toBeGreaterThan(0)
|
|
const promptSnapshot = formatSystemPromptSnapshot(prompts[0] as string, prompts.slice(1))
|
|
/* v8 ignore next -- registration guarantees every scenario class has resolved sources. */
|
|
const promptSource = promptSourceByClass.get(classOf(scenario)) ?? scenario
|
|
const promptPath = join(snapshotsDir, promptSource.name, SYSTEM_PROMPT_SNAPSHOT)
|
|
claimSharedSnapshot(promptClaims, promptPath, scenario.name, promptSnapshot)
|
|
await writeFile(promptPath, promptSnapshot)
|
|
|
|
const schemaSets = normalizedToolSchemas(primary.content, ctx)
|
|
expect(schemaSets.length, `${mode} produced no tool schemas to snapshot`).toBeGreaterThan(0)
|
|
expect(schemaSets.length, `${mode} produced a tool-schema sequence that differs from its prompt sequence`)
|
|
.toBe(prompts.length)
|
|
const toolSchemasSnapshot = formatToolSchemasSnapshot(
|
|
schemaSets[0] as unknown[],
|
|
schemaSets.slice(1),
|
|
)
|
|
/* v8 ignore next -- registration guarantees every scenario class has resolved sources. */
|
|
const schemaSource = schemaSourceByClass.get(classOf(scenario)) ?? scenario
|
|
const schemaPath = join(snapshotsDir, schemaSource.name, TOOL_SCHEMAS_SNAPSHOT)
|
|
claimSharedSnapshot(schemaClaims, schemaPath, scenario.name, toolSchemasSnapshot)
|
|
await writeFile(schemaPath, toolSchemasSnapshot)
|
|
}
|
|
}
|
|
|
|
for (const expected of stdoutExpectedVariants(scenario)) {
|
|
const stdout = normalizeStdout(result.rawStdout, ctx, { cwdPathMode: expected.cwdPathMode })
|
|
if (REFRESHING) {
|
|
await writeFile(join(dir, expected.file), stdout)
|
|
}
|
|
await expect(stdout, `${expected.file} mismatch`).toMatchFileSnapshot(join(dir, expected.file))
|
|
}
|
|
|
|
// A model turn always produces a log worth comparing; an explicitly
|
|
// authored non-model scenario may opt in independently.
|
|
if (comparesLog) {
|
|
// The harvested logs (primary-first) must match their committed fixtures 1:1.
|
|
expect(result.sessionLogs.length, 'this scenario must persist one log per session fixture').toBe(fixtureFiles.length)
|
|
for (let i = 0; i < fixtureFiles.length; i++) {
|
|
const harvested = scrub((result.sessionLogs[i] as HarvestedLog).content)
|
|
const fixture = scrub(await readFile(join(dir, fixtureFiles[i] as string), 'utf8'))
|
|
expect(normalizeSessionLog(harvested, ctx), `${fixtureFiles[i]} mismatch`)
|
|
.toEqual(normalizeSessionLog(fixture, fixtureContext(fixture)))
|
|
}
|
|
}
|
|
|
|
// Every live full header must equal its class pin reconstructed from
|
|
// tokenized JSONL plus readable prompt and structured schema sidecars.
|
|
/* v8 ignore next -- construction guarantees the pin exists; a miss would fail the one-header assertion loudly. */
|
|
const pinningScenario = pinningByClass.get(classOf(scenario)) ?? scenario
|
|
/* v8 ignore next -- registration guarantees every scenario class has resolved sources. */
|
|
const promptSource = promptSourceByClass.get(classOf(scenario)) ?? pinningScenario
|
|
/* v8 ignore next -- registration guarantees every scenario class has resolved sources. */
|
|
const schemaSource = schemaSourceByClass.get(classOf(scenario)) ?? pinningScenario
|
|
const pinningDir = join(snapshotsDir, pinningScenario.name)
|
|
const pinnedFixture = await readFile(join(pinningDir, 'session.jsonl'), 'utf8')
|
|
const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture))
|
|
const promptSnapshot = await readFile(
|
|
join(snapshotsDir, promptSource.name, SYSTEM_PROMPT_SNAPSHOT),
|
|
'utf8',
|
|
)
|
|
const initialPromptSnapshot = initialSystemPromptSnapshot(promptSnapshot)
|
|
expect(pinned.length, `the pinning fixture (${pinningScenario.name}) has an unexpected request/header count`)
|
|
.toBe(1 + (pinningScenario.expectedHeaderChanges ?? 0))
|
|
const toolSchemasSnapshot = await readFile(
|
|
join(snapshotsDir, schemaSource.name, TOOL_SCHEMAS_SNAPSHOT),
|
|
'utf8',
|
|
)
|
|
const toolSchemas = parseToolSchemasSnapshot(toolSchemasSnapshot)
|
|
const pinnedSchemaSets = [toolSchemas.initial, ...toolSchemas.changes]
|
|
expect(pinnedSchemaSets.length, `the schema source (${schemaSource.name}) has an unexpected tool-schema count`)
|
|
.toBe(pinned.length)
|
|
const pinnedHeaders = pinned.map((header, index) => restorePinnedToolSchemas(
|
|
header,
|
|
pinnedSchemaSets[index] as unknown[],
|
|
))
|
|
for (const [logIndex, log] of result.sessionLogs.entries()) {
|
|
const expectedChanges = scenario.pinsHeader === true && logIndex === 0
|
|
? scenario.expectedHeaderChanges ?? 0
|
|
: 0
|
|
expect(headerChangeCount(log.content), `session ${log.id}: changed request/header count`)
|
|
.toBe(expectedChanges)
|
|
const headers = normalizedHeaders(scrubSystemPrompts(log.content), ctx)
|
|
const prompts = normalizedSystemPrompts(log.content, ctx)
|
|
const schemaSets = normalizedToolSchemas(log.content, ctx)
|
|
expect(prompts.length, `session ${log.id}: every request/header must carry a string system prompt`)
|
|
.toBe(headers.length)
|
|
expect(schemaSets.length, `session ${log.id}: every request/header must carry an array-valued tools field`)
|
|
.toBe(headers.length)
|
|
for (const [k, header] of headers.entries()) {
|
|
const expected = expectedChanges > 0 ? pinnedHeaders[k] : pinnedHeaders[0]
|
|
expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`)
|
|
.toEqual(expected)
|
|
if (expectedChanges === 0) {
|
|
expect(formatSystemPromptSnapshot(prompts[k] as string), `session ${log.id}: initial system prompt #${k + 1} diverged from ${promptSource.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
|
|
.toEqual(initialPromptSnapshot)
|
|
}
|
|
}
|
|
if (scenario.pinsHeader === true && logIndex === 0) {
|
|
expect(formatSystemPromptSnapshot(
|
|
prompts[0] as string,
|
|
prompts.slice(1),
|
|
), `session ${log.id}: changed system prompts diverged from ${promptSource.name}/${SYSTEM_PROMPT_SNAPSHOT}`)
|
|
.toEqual(promptSnapshot)
|
|
expect(formatToolSchemasSnapshot(
|
|
schemaSets[0] as unknown[],
|
|
schemaSets.slice(1),
|
|
), `session ${log.id}: changed tool schemas diverged from ${schemaSource.name}/${TOOL_SCHEMAS_SNAPSHOT}`)
|
|
.toEqual(toolSchemasSnapshot)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
})
|
|
|
|
describe('snapshot fixtures', () => {
|
|
it('every scenario directory is registered (no orphans)', async () => {
|
|
// toMatchFileSnapshot does not prune orphaned expected-output or fixture files, so a
|
|
// renamed/removed scenario could leave a stale dir that nothing exercises.
|
|
// Fail loud on any snapshots/<dir> not present in the scenario table.
|
|
const entries = await readdir(snapshotsDir, { withFileTypes: true })
|
|
const onDisk = entries.filter(e => e.isDirectory()).map(e => e.name).sort()
|
|
const registered = scenarios.map(s => s.name).sort()
|
|
expect(onDisk).toEqual(registered)
|
|
})
|
|
|
|
it('every registered scenario has its required fixture files', async () => {
|
|
// Every scenario needs input, stdout, a primary session fixture, and matching optional sidecars.
|
|
for (const { name, overridden, pinsNativeWindowsStdout } of scenarios) {
|
|
const dir = join(snapshotsDir, name)
|
|
expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true)
|
|
expect(existsSync(join(dir, 'stdout.expected.jsonl')), `${name}/stdout.expected.jsonl`).toBe(true)
|
|
expect(
|
|
existsSync(join(dir, WINDOWS_STDOUT_SNAPSHOT)),
|
|
`${name}/${WINDOWS_STDOUT_SNAPSHOT} presence must match \`pinsNativeWindowsStdout\``,
|
|
).toBe(pinsNativeWindowsStdout === true)
|
|
expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true)
|
|
expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json presence must match \`overridden\``)
|
|
.toBe(overridden === true)
|
|
expect(existsSync(join(dir, SYSTEM_PROMPT_SNAPSHOT)), `${name}/${SYSTEM_PROMPT_SNAPSHOT} presence must match snapshot-source ownership`)
|
|
.toBe(promptOwners.has(name))
|
|
expect(existsSync(join(dir, TOOL_SCHEMAS_SNAPSHOT)), `${name}/${TOOL_SCHEMAS_SNAPSHOT} presence must match snapshot-source ownership`)
|
|
.toBe(schemaOwners.has(name))
|
|
await expect(sessionFixtures(dir), `${name}: session fixture inventory`).resolves.toBeDefined()
|
|
}
|
|
})
|
|
|
|
it('exactly one scenario pins the request-header content of each header class', () => {
|
|
// Zero pins would drop a class's structural header surface from the suite entirely; two
|
|
// would split it.
|
|
const pins = new Map<string, string[]>()
|
|
for (const scenario of scenarios.filter(s => s.pinsHeader === true)) {
|
|
const cls = classOf(scenario)
|
|
pins.set(cls, [...pins.get(cls) ?? [], scenario.name])
|
|
}
|
|
expect(Object.fromEntries([...pins].map(([cls, names]) => [cls, names.length]))).toEqual(
|
|
Object.fromEntries([...pinningByClass.keys()].map(cls => [cls, 1])))
|
|
for (const scenario of scenarios) {
|
|
expect(pinningByClass.has(classOf(scenario)), `class "${classOf(scenario)}" (scenario ${scenario.name}) has a pin`).toBe(true)
|
|
}
|
|
})
|
|
|
|
it('every pinning fixture composes one tokenized header sequence with its referenced sidecars', async () => {
|
|
// Assert the committed pin directly because a class containing only its
|
|
// pinning scenario has no non-pinning live run to catch undeclared changes.
|
|
for (const scenario of pinningByClass.values()) {
|
|
/* v8 ignore next -- registration guarantees every pin has resolved sources. */
|
|
const promptSource = promptSourceByClass.get(classOf(scenario)) ?? scenario
|
|
/* v8 ignore next -- registration guarantees every pin has resolved sources. */
|
|
const schemaSource = schemaSourceByClass.get(classOf(scenario)) ?? scenario
|
|
const fixture = await readFile(join(snapshotsDir, scenario.name, 'session.jsonl'), 'utf8')
|
|
const headers = normalizedHeaders(fixture, fixtureContext(fixture))
|
|
const promptSnapshot = await readFile(
|
|
join(snapshotsDir, promptSource.name, SYSTEM_PROMPT_SNAPSHOT),
|
|
'utf8',
|
|
)
|
|
expect(headers.length, `${scenario.name}: unexpected request/header count`)
|
|
.toBe(1 + (scenario.expectedHeaderChanges ?? 0))
|
|
const toolSchemasSnapshot = await readFile(
|
|
join(snapshotsDir, schemaSource.name, TOOL_SCHEMAS_SNAPSHOT),
|
|
'utf8',
|
|
)
|
|
const toolSchemas = parseToolSchemasSnapshot(toolSchemasSnapshot)
|
|
const schemaSets = [toolSchemas.initial, ...toolSchemas.changes]
|
|
expect(schemaSets.length, `${schemaSource.name}: tool-schema sequence must match ${scenario.name}'s header sequence`)
|
|
.toBe(headers.length)
|
|
for (const [index, header] of headers.entries()) {
|
|
expect(() => restorePinnedToolSchemas(header, schemaSets[index] as unknown[]), `${scenario.name}: tools must use the sidecar token`)
|
|
.not.toThrow()
|
|
}
|
|
expect(promptSnapshot.length, `${promptSource.name}/${SYSTEM_PROMPT_SNAPSHOT} must not be empty`).toBeGreaterThan(0)
|
|
expect(promptSnapshot.endsWith('\n'), `${promptSource.name}/${SYSTEM_PROMPT_SNAPSHOT} must end in a newline`).toBe(true)
|
|
expect(toolSchemasSnapshot, `${schemaSource.name}/${TOOL_SCHEMAS_SNAPSHOT} must use canonical JSON formatting`)
|
|
.toBe(formatToolSchemasSnapshot(toolSchemas.initial, toolSchemas.changes))
|
|
expect(headerChangeCount(fixture), `${scenario.name}: a pinning fixture must carry exactly its declared changed headers`)
|
|
.toBe(scenario.expectedHeaderChanges ?? 0)
|
|
}
|
|
})
|
|
|
|
it('stores each distinct prompt and tool-schema snapshot once', async () => {
|
|
const prompts = await Promise.all([...promptOwners].map(async (owner): Promise<NamedSnapshotContent> => ({
|
|
path: `${owner}/${SYSTEM_PROMPT_SNAPSHOT}`,
|
|
content: await readFile(join(snapshotsDir, owner, SYSTEM_PROMPT_SNAPSHOT), 'utf8'),
|
|
})))
|
|
const schemas = await Promise.all([...schemaOwners].map(async (owner): Promise<NamedSnapshotContent> => ({
|
|
path: `${owner}/${TOOL_SCHEMAS_SNAPSHOT}`,
|
|
content: await readFile(join(snapshotsDir, owner, TOOL_SCHEMAS_SNAPSHOT), 'utf8'),
|
|
})))
|
|
assertUniqueSnapshotContents('system-prompt', prompts)
|
|
assertUniqueSnapshotContents('tool-schema', schemas)
|
|
})
|
|
|
|
it('every committed JSONL has valid tool results and canonical header storage', async () => {
|
|
// Prompts and schemas always leave JSONL. Header pins retain prefixes;
|
|
// every other fixture tokenizes those too. Fixed-point checks make both
|
|
// storage rules fail loud.
|
|
for (const scenario of scenarios) {
|
|
const dir = join(snapshotsDir, scenario.name)
|
|
const files = await sessionFixtures(dir)
|
|
for (const file of files) {
|
|
const fixture = await readFile(join(dir, file), 'utf8')
|
|
expect(unknownToolCallIds(fixture), `${scenario.name}/${file} contains UNKNOWN_TOOL`)
|
|
.toEqual([])
|
|
expect(scrubSystemPrompts(fixture), `${scenario.name}/${file} carries an unscrubbed system prompt`)
|
|
.toEqual(fixture)
|
|
expect(scrubToolSchemas(fixture), `${scenario.name}/${file} carries unscrubbed tool schemas`)
|
|
.toEqual(fixture)
|
|
if (scenario.pinsHeader !== true) {
|
|
expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} carries unscrubbed header content`)
|
|
.toEqual(fixture)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
})
|
|
}
|