diff --git a/docs/config-catalog.md b/docs/config-catalog.md index feb8aa9d86..f4302984a8 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -857,7 +857,7 @@ export interface ReplayModelConfig { Depends on: [`RetryPolicyConfig`](../packages/llm/llm/src/index.ts) -Source: [`packages/support/llm-replay/src/index.ts:710`](../packages/support/llm-replay/src/index.ts) +Source: [`packages/support/llm-replay/src/index.ts:731`](../packages/support/llm-replay/src/index.ts) ## `@deepseek-ai/dsh-llm-retry` diff --git a/docs/module-graph.md b/docs/module-graph.md index 14a6b71dc1..d963273363 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -422,9 +422,6 @@ flowchart TD pkg_session_persistence --> pkg_brand pkg_session_persistence --> pkg_invariants pkg_session_persistence --> pkg_session - pkg_llm_replay --> pkg_invariants - pkg_llm_replay --> pkg_llm - pkg_llm_replay --> pkg_session pkg_app_boot --> pkg_environment pkg_app_boot --> pkg_invariants pkg_app_boot --> pkg_paths @@ -500,6 +497,10 @@ flowchart TD pkg_session_title --> pkg_llm pkg_session_title --> pkg_session pkg_session_title --> pkg_session_projection + pkg_llm_replay --> pkg_compact + pkg_llm_replay --> pkg_invariants + pkg_llm_replay --> pkg_llm + pkg_llm_replay --> pkg_session pkg_commands --> pkg_agent pkg_commands --> pkg_brand pkg_commands --> pkg_invariants @@ -1219,7 +1220,6 @@ flowchart TD | [`web-search-perplexity`](../packages/web/web-search-perplexity) | `web` | [`environment`](../packages/util/environment), [`invariants`](../packages/support/invariants), [`web`](../packages/web/web) | | [`spill`](../packages/spill/spill) | `spill` | [`brand`](../packages/util/brand), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | | [`session-persistence`](../packages/session-persistence/session-persistence) | `session-persistence` | [`brand`](../packages/util/brand), [`invariants`](../packages/support/invariants), [`session`](../packages/core/session) | -| [`llm-replay`](../packages/support/llm-replay) | `support` | [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | | [`app-boot`](../packages/ui/app-boot) | `ui` | [`environment`](../packages/util/environment), [`invariants`](../packages/support/invariants), [`paths`](../packages/util/paths), [`system-prompt`](../packages/core/system-prompt) | | [`code-runtime-worker`](../packages/code-runtime/code-runtime-worker) | `code-runtime` | [`code-runtime`](../packages/code-runtime/code-runtime), [`invariants`](../packages/support/invariants), [`session`](../packages/core/session), [`timeout`](../packages/util/timeout) | | [`lsp-local`](../packages/lsp/lsp-local) | `lsp` | [`brand`](../packages/util/brand), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`lsp`](../packages/lsp/lsp), [`subprocess`](../packages/subprocess/subprocess), [`timeout`](../packages/util/timeout) | @@ -1239,6 +1239,7 @@ flowchart TD | [`session-persistence-jsonl`](../packages/session-persistence/session-persistence-jsonl) | `session-persistence` | [`invariants`](../packages/support/invariants), [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | | [`session-persistence-sqlite`](../packages/session-persistence/session-persistence-sqlite) | `session-persistence` | [`invariants`](../packages/support/invariants), [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | | [`session-title`](../packages/session-title/session-title) | `session-title` | [`brand`](../packages/util/brand), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session-projection/session-projection) | +| [`llm-replay`](../packages/support/llm-replay) | `support` | [`compact`](../packages/compact/compact), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | | [`commands`](../packages/ui/commands) | `ui` | [`agent`](../packages/core/agent), [`brand`](../packages/util/brand), [`invariants`](../packages/support/invariants), [`scope`](../packages/core/scope), [`session`](../packages/core/session) | | [`user-approval`](../packages/ui/user-approval) | `ui` | [`agent`](../packages/core/agent), [`brand`](../packages/util/brand), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt) | | [`user-interaction`](../packages/ui/user-interaction) | `ui` | [`agent`](../packages/core/agent), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm) | diff --git a/examples/headless-agent/compaction.cordis.snapshot.yml b/examples/headless-agent/compaction.cordis.snapshot.yml new file mode 100644 index 0000000000..42fc5306ac --- /dev/null +++ b/examples/headless-agent/compaction.cordis.snapshot.yml @@ -0,0 +1,25 @@ +# Keyless context-overflow composition for the assembled compaction snapshot. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - id: llm-deepseek + name: '@deepseek-ai/dsh-llm-deepseek' + disabled: true + - id: compact-basic + name: '@deepseek-ai/dsh-compact-basic' + config: + thresholdRatio: 0.99 + retainTokens: 20 + maxTokens: 32 + compactionRetries: 1 + - insert: + - id: llm-replay + name: '@deepseek-ai/dsh-llm-replay' + config: + providers: + - id: deepseek-official + models: + - id: deepseek-v4-flash + contextWindow: 128000 diff --git a/examples/headless-agent/tests/compaction.e2e.ts b/examples/headless-agent/tests/compaction.e2e.ts index 07f239a73e..6fe6f4055b 100644 --- a/examples/headless-agent/tests/compaction.e2e.ts +++ b/examples/headless-agent/tests/compaction.e2e.ts @@ -11,8 +11,8 @@ import { SessionId } from '@deepseek-ai/dsh-session' * Key-gated smoke for mid-session compaction. It verifies the compact event * pair, replacement of older surface nodes, and a final answer after compaction. */ -// FIXME(compaction-snapshot): this is the only full compaction coverage because -// replay cannot serve the summarizer's unlogged model call. +// The keyless headless snapshot pins deterministic overflow recovery; this test +// remains the independent live-provider smoke for organic pressure and summary quality. let workdir: string | undefined let ctx: Context | undefined diff --git a/examples/headless-agent/tests/headless.snapshot.ts b/examples/headless-agent/tests/headless.snapshot.ts index 121bbf75ed..f9cb46111a 100644 --- a/examples/headless-agent/tests/headless.snapshot.ts +++ b/examples/headless-agent/tests/headless.snapshot.ts @@ -29,6 +29,10 @@ const goalScenarioDir = join(snapshotsDir, 'goal-tools') const goalConfigPath = fileURLToPath(new URL('../goal.cordis.snapshot.yml', import.meta.url)) const retryScenarioDir = join(snapshotsDir, 'provider-retry') const retryConfigPath = fileURLToPath(new URL('../retry.cordis.snapshot.yml', import.meta.url)) +const compactionScenarioDir = join(snapshotsDir, 'compaction-recovery') +const compactionSessionFixture = join(compactionScenarioDir, 'session.jsonl') +const compactionStreamExpected = join(compactionScenarioDir, 'stream-json.expected.jsonl') +const compactionConfigPath = fileURLToPath(new URL('../compaction.cordis.snapshot.yml', import.meta.url)) const credentialsScenarioDir = join(snapshotsDir, 'missing-credential') const credentialsConfigPath = fileURLToPath(new URL('../credentials.cordis.snapshot.yml', import.meta.url)) // Same keyless composition as the missing-credential scenario: the endpoint is @@ -227,6 +231,75 @@ describe('headless stream-json snapshots', () => { expect(normalized).toBe(await readFile(streamExpected, 'utf8')) }, LOADER_SMOKE_TEST_TIMEOUT_MS) + it('recovers from context overflow through an assembled compaction', async () => { + const prompt = await scenarioPrompt(compactionScenarioDir, 'compaction-recovery') + let expectedSession = await readFile(compactionSessionFixture, 'utf8') + let runCwd = '' + const result = await runLoaderSmoke({ + label: 'compaction recovery headless stream-json snapshot', + tempDirPrefix: 'headless-snapshot-compaction-recovery-', + binScript, + configPath: compactionConfigPath, + binArgs: ['--config', compactionConfigPath, '--output-format', 'stream-json', prompt], + tsconfigPath, + env: { + DSH_SNAPSHOT: 'replay', + DSH_SNAPSHOT_FILE: compactionSessionFixture, + NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '), + }, + prepare: (cwd) => { runCwd = cwd }, + inspect: async (cwd) => { + const logs = await persistedLogs(cwd) + expect(logs).toHaveLength(1) + const actual = logs[0] + if (actual === undefined) throw new Error('compaction snapshot did not persist its session') + const records = parseJsonl(actual.content) + const types = records.map(record => record.type) + expect(types.filter(type => type === 'compact/start')).toHaveLength(1) + expect(types.filter(type => type === 'compact/summary')).toHaveLength(1) + expect(types.filter(type => type === 'compact/end')).toHaveLength(1) + const start = types.indexOf('compact/start') + const summary = types.indexOf('compact/summary') + const replacement = records.findIndex((record) => { + if (record.type !== 'user/message') return false + const surfaceOp = record.surfaceOp as JsonObject | undefined + return surfaceOp?.op === 'replace' + }) + const end = types.indexOf('compact/end') + expect(start).toBeLessThan(summary) + expect(summary).toBeLessThan(replacement) + expect(replacement).toBeLessThan(end) + const summaryRecord = records[summary] + const summaryData = summaryRecord?.data as JsonObject | undefined + expect(summaryData?.shadowedSeqs).toEqual(expect.arrayContaining([expect.any(Number)])) + const final = [...records].reverse().find(record => record.type === 'assistant/message') + expect(JSON.stringify(final)).toContain('COMPACTION RECOVERED') + + const actualContext = contextFromLogs([actual.content]) + if (refreshing) { + const harvested: HarvestedLog = { + id: String(actual.header.id), + createdAt: Number(actual.header.createdAt), + content: actual.content, + } + const replacements = refreshFixtureReplacements([harvested], [expectedSession]) + expectedSession = tokenizeSessionFixtureCwd( + stabilizeRefreshLog(actual.content, expectedSession, replacements, actualContext), + ) + await writeFile(compactionSessionFixture, expectedSession) + } + const expectedContext = contextFromLogs([expectedSession]) + expect(scrubRequestHeaders(normalizeSessionLog(actual.content, actualContext))) + .toBe(scrubRequestHeaders(normalizeSessionLog(expectedSession, expectedContext))) + }, + }) + + expect(result.stderr).toBe('') + const normalized = normalizeHeadlessStream(result.stdout, runCwd) + if (refreshing) await writeFile(compactionStreamExpected, normalized) + expect(normalized).toBe(await readFile(compactionStreamExpected, 'utf8')) + }, LOADER_SMOKE_TEST_TIMEOUT_MS) + it('logs actionable missing-credential guidance through the one-shot app', async () => { const streamExpected = join(credentialsScenarioDir, 'stream-json.expected.jsonl') let runCwd = '' diff --git a/examples/headless-agent/tests/snapshots/compaction-recovery/input.json b/examples/headless-agent/tests/snapshots/compaction-recovery/input.json new file mode 100644 index 0000000000..3ccad96b83 --- /dev/null +++ b/examples/headless-agent/tests/snapshots/compaction-recovery/input.json @@ -0,0 +1,8 @@ +{ + "steps": [ + { + "op": "prompt", + "text": "Establish a durable compaction premise before continuing. Record every part of this historical evidence: the snapshot uses keyless replay; persistence uses JSONL; the assembled headless application loads its real Cordis composition; model-visible inputs remain logged; tool calls and results remain paired and ordered; context overflow retains the original failure while recovery is attempted; compaction opens with compact/start and closes with compact/end; a successful auxiliary summary records compact/summary provenance; the replacement surface shadows only an older balanced range; the checkpoint remains smaller than the history it replaces; the newest tool result remains verbatim; the retried request sees that checkpoint; the final response proves the same turn continued; deterministic snapshot evidence stays separate from the live-provider smoke; and no external API key is needed. Emit one alpha marker through bash, then finish the task after any required recovery with the exact words COMPACTION RECOVERED." + } + ] +} diff --git a/examples/headless-agent/tests/snapshots/compaction-recovery/session.jsonl b/examples/headless-agent/tests/snapshots/compaction-recovery/session.jsonl new file mode 100644 index 0000000000..855e66ac14 --- /dev/null +++ b/examples/headless-agent/tests/snapshots/compaction-recovery/session.jsonl @@ -0,0 +1,32 @@ +{"type":"session","version":0,"id":"{{sessionId}}","createdAt":0,"cwd":"{{cwd}}","delegationDepth":0} +{"type":"agent/inbox/spliced","seq":0,"time":1786123401613,"data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Establish a durable compaction premise before continuing. Record every part of this historical evidence: the snapshot uses keyless replay; persistence uses JSONL; the assembled headless application loads its real Cordis composition; model-visible inputs remain logged; tool calls and results remain paired and ordered; context overflow retains the original failure while recovery is attempted; compaction opens with compact/start and closes with compact/end; a successful auxiliary summary records compact/summary provenance; the replacement surface shadows only an older balanced range; the checkpoint remains smaller than the history it replaces; the newest tool result remains verbatim; the retried request sees that checkpoint; the final response proves the same turn continued; deterministic snapshot evidence stays separate from the live-provider smoke; and no external API key is needed. Emit one alpha marker through bash, then finish the task after any required recovery with the exact words COMPACTION RECOVERED."}],"source":{"kind":"user"},"role":"user","id":"6335ca4a-a577-47dd-8219-aa81f39cdbc0"}]}} +{"type":"turn/start","seq":1,"time":1786123401614,"data":{"turn":1}} +{"type":"agent/inbox/spliced","seq":2,"time":1786123401614,"data":{"target":"next-turn","start":0,"removedCount":1,"inserted":[]}} +{"type":"step/start","seq":3,"time":1786123401667,"data":{"turn":1,"step":1}} +{"type":"user/message","seq":4,"time":1786123401667,"data":{"content":[{"type":"text","text":"Establish a durable compaction premise before continuing. Record every part of this historical evidence: the snapshot uses keyless replay; persistence uses JSONL; the assembled headless application loads its real Cordis composition; model-visible inputs remain logged; tool calls and results remain paired and ordered; context overflow retains the original failure while recovery is attempted; compaction opens with compact/start and closes with compact/end; a successful auxiliary summary records compact/summary provenance; the replacement surface shadows only an older balanced range; the checkpoint remains smaller than the history it replaces; the newest tool result remains verbatim; the retried request sees that checkpoint; the final response proves the same turn continued; deterministic snapshot evidence stays separate from the live-provider smoke; and no external API key is needed. Emit one alpha marker through bash, then finish the task after any required recovery with the exact words COMPACTION RECOVERED."}],"source":{"kind":"user"},"role":"user","id":"6335ca4a-a577-47dd-8219-aa81f39cdbc0"},"surfaceOp":"append"} +{"type":"session/title","seq":5,"time":1786123401667,"data":{"title":"Establish a durable compaction premise","messageSeqs":[4],"source":{"kind":"fallback"}}} +{"type":"request/header","seq":6,"time":1786123401668,"data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash"},"system":"You are an AI agent powered by the DeepSeek Harness SDK.\n\nYou are headless-agent, a coding assistant powered by the deepseek-v4-flash model.\n\nVerify your work by running the code or tests. Keep answers brief and\nfactual.\n\n\nUse the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files.\n\nUse the write tool to create files or completely replace file contents. Existing files are overwritten, so read an existing file first (the default fs-policy requires it) and prefer edit for targeted changes.\n\nUse the edit tool for targeted changes to existing UTF-8 text files. It replaces literal old_string with new_string; by default old_string must appear exactly once. If old_string appears multiple times, provide a more specific old_string or set replace_all to true. Read the file first (the default fs-policy requires it), unless you just created or edited it in this session.\n\nCheck the [exit code: N] marker on every bash result; investigate failures before moving on.\n\nTrack every background task id you start. You are notified in-session when a task finishes — do not busy-poll or sleep on one; keep working on independent steps and do not duplicate a running task's work. Before giving a final answer, collect every still-relevant task with task_output (set wait: true only when you are genuinely blocked on it), and task_kill tasks that stopped mattering.\n\nUse the workflow tool ONLY when the user explicitly asks for a workflow or for large multi-agent orchestration: you write a JavaScript script (the tool description documents the exact format) that fans work out across many subagents with phases and structured results. For one or two delegations, prefer plain subagent calls.\n\nUse the ralph tool ONLY when the direct human explicitly asks for a Ralph loop or fresh-agent iterative execution. Each Ralph round starts a fresh child with no conversation seed and uses the shared workspace as durable memory. Completion and blockers are worker reports, not independent evaluation. Use same-session goal tools for ordinary long-running objectives, and plain subagents or workflows for bounded delegation and fan-out.","tools":[{"name":"bash","description":"Execute a bash command (`bash -c`) and return its stdout/stderr. Each call runs in a fresh shell: no state (cwd, variables, functions) persists between calls — pass `workdir` instead of using `cd`. Non-zero exits are reported as `[exit code: N]`. Current harness environment facts are exposed through managed `$DSH_*` variables; inspect them when needed. Commands may run under a file sandbox; a blocked file operation is reported as `[sandbox: file access denied under mode]` — a policy denial, not a bug in the command; do not retry another way. Long output is truncated to its tail; the full output is saved to a file whose path is reported when available. Set `run_in_background: true` for long-running commands: the call returns a task id immediately; read its output with `task_output` and stop it with `task_kill`.","parameters":{"type":"object","properties":{"command":{"type":"string","description":"The bash command to execute."},"description":{"type":"string","description":"Clear, concise description of what this command does in active voice, 5-10 words (shown in the UI). Examples: \"ls\" → \"List files in current directory\"; \"git status\" → \"Show working tree status\"; \"npm install\" → \"Install package dependencies\"."},"timeoutMs":{"type":"number","description":"Timeout in milliseconds. The executor applies its configured default and cap, and kills the command on expiry."},"workdir":{"type":"string","description":"Working directory for this command. Defaults to the session workspace; a relative path is resolved against it."},"run_in_background":{"type":"boolean","description":"Run in the background and return a task id immediately (collect with task_output, stop with task_kill). No timeout applies."}},"required":["command","description"]}},{"name":"edit","description":"Edit an existing UTF-8 text file by replacing literal text.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to edit, resolved by the filesystem backend."},"old_string":{"type":"string","description":"Literal text to replace. Must match exactly."},"new_string":{"type":"string","description":"Literal replacement text. Use an empty string to delete the match."},"replace_all":{"type":"boolean","description":"Replace all matches. Defaults to false; when false, old_string must appear exactly once."}},"required":["file_path","old_string","new_string"]}},{"name":"ralph","description":"Run a foreground fresh-agent Ralph loop toward one immutable objective. Use only when the direct human explicitly asks for Ralph or fresh-agent iteration. Each round opens a new child with no parent conversation or prior child session; the shared workspace is long-term memory, and only a bounded structured report crosses rounds. The call returns when a worker reports completion or a concrete blocker, or at the round limit. Ordinary long-running same-session work belongs to goal tools.","parameters":{"type":"object","properties":{"objective":{"type":"string","description":"The immutable completion objective for every fresh Ralph round."},"maxRounds":{"type":"number","description":"Optional positive safe-integer round cap, bounded by the deployment ceiling."}},"required":["objective"]}},{"name":"read","description":"Read a UTF-8 text file and return line-numbered content.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to read, resolved by the filesystem backend."},"offset":{"type":"number","description":"1-based first line to return. Defaults to 1."},"limit":{"type":"number","description":"Maximum number of lines to return. Defaults to 2000."}},"required":["file_path"]}},{"name":"send_message","description":"Send a message to a background subagent by its subagent id, continuing the same conversation. It becomes the subagent's next turn: if it is still working, the message waits until its current turn finishes, so it cannot redirect work already underway. This call returns no answer from the subagent — only confirmation that the message was delivered — so use it to give it more work. A failure means the message was NOT delivered.","parameters":{"type":"object","properties":{"subagent_id":{"type":"string","description":"The subagent id returned when the background subagent was started."},"message":{"type":"string","description":"The message to deliver to the subagent."}},"required":["subagent_id","message"]}},{"name":"skill","description":"Load the full instructions for an available skill. Call this with the exact skill name from the session skill catalog before acting on a task that names or clearly matches that skill.","parameters":{"type":"object","properties":{"name":{"type":"string","description":"The exact skill name from the available skills list."}},"required":["name"]}},{"name":"subagent","description":"Delegate a self-contained task to a subagent (a separate agent that works in its own context) and return its final result. Use this to offload focused, independent work — research, a scoped implementation, an analysis — so it does not consume this conversation's context. The subagent runs to completion and you receive only its final answer, not its intermediate steps. Give it a complete, standalone prompt: it does not see this conversation. Set `run_in_background: true` to start a background subagent that keeps its conversation: you receive only its subagent id, never its result, and it works on its own. Use this for work whose result you do not need returned by this call; `send_message` sends it more work.","parameters":{"type":"object","properties":{"description":{"type":"string","description":"A short (3-5 word) description of the delegated task, for display."},"prompt":{"type":"string","description":"The complete, self-contained task for the subagent. It does not share this conversation's context, so include everything it needs."},"run_in_background":{"type":"boolean","description":"Run as a background subagent that keeps its conversation and return only its subagent id. This call never returns its result; send it more work with send_message."}},"required":["description","prompt"]}},{"name":"subagent_fork","description":"Delegate a task to a subagent that inherits this conversation: a child agent seeded with all completed turns so far (it does not see the current in-flight turn), returning only its final result. Use this when the subtask builds on this conversation's context — a follow-up analysis, a review, a continuation — without consuming this conversation's context for the work itself. You receive only its final answer, not its intermediate steps. Set `run_in_background: true` to start a background subagent that keeps its conversation: you receive only its subagent id, never its result, and it works on its own. Use this for work whose result you do not need returned by this call; `send_message` sends it more work.","parameters":{"type":"object","properties":{"description":{"type":"string","description":"A short (3-5 word) description of the delegated task, for display."},"prompt":{"type":"string","description":"The task for the subagent. It already sees this conversation's completed turns, so build on them freely and state only what is new."},"run_in_background":{"type":"boolean","description":"Run as a background subagent that keeps its conversation and return only its subagent id. This call never returns its result; send it more work with send_message."}},"required":["description","prompt"]}},{"name":"task_kill","description":"Request cancellation of a running background task by task id. Returns immediately; the task settles as killed once its work actually stops.","parameters":{"type":"object","properties":{"task_id":{"type":"string","description":"Task id returned by the tool that started the background work."},"reason":{"type":"string","description":"Optional short reason, recorded in the log and forwarded to the task."}},"required":["task_id"]}},{"name":"task_list","description":"List your background tasks (running and finished) with their ids, kinds, and statuses.","parameters":{"type":"object","properties":{}}},{"name":"task_output","description":"Read a background task. Stream tasks return only output since the previous read; final-output tasks return their result after settlement. Every response ends with `[status: ...]`. Reads are non-blocking unless `wait: true`, which waits up to the configured cap.","parameters":{"type":"object","properties":{"task_id":{"type":"string","description":"Task id returned by the tool that started the background work."},"wait":{"type":"boolean","description":"Block until the task reaches a terminal status or the timeout expires. A timed-out wait returns [status: running] and leaves the task alive."},"timeout_ms":{"type":"number","description":"Max wait in milliseconds (only meaningful with wait: true). Defaults to the configured wait timeout; capped by the configured maximum."}},"required":["task_id"]}},{"name":"todo_write","description":"Record and update a structured task list for the current work. Send the ENTIRE list every call — it REPLACES the previous list (there are no partial updates, no per-item edits). Use it to plan multi-step work and show progress: add one todo per concrete step before you start. Mark every todo being actively worked on `in_progress` — several at once when work genuinely runs in parallel (e.g. concurrent subagents or background commands), one for sequential work; while work remains, at least one task should be `in_progress`. Mark a todo `completed` the moment it is done (do not batch completions), and allow no `in_progress` item only once all work is complete. Skip the list for trivial single-step tasks. Statuses: `pending` (not started), `in_progress` (being worked on now), `completed` (finished).","parameters":{"type":"object","properties":{"todos":{"type":"array","description":"The COMPLETE task list, replacing any previous list.","items":{"type":"object","additionalProperties":false,"properties":{"content":{"type":"string","description":"What the task is — a short imperative line."},"status":{"type":"string","description":"pending (not started) | in_progress (now) | completed (done).","enum":["pending","in_progress","completed"]}},"required":["content","status"]}}},"required":["todos"]}},{"name":"workflow","description":"Run a JavaScript workflow script that orchestrates subagents at scale. Use this for work that fans out across many independent pieces — an audit over many files, a migration, multi-angle research, adversarial verification of findings — where you write the orchestration as a script instead of delegating turn by turn.\n\nThe workflow's identity rides the `meta` parameter as JSON: required `name` (short kebab-case) and `description` strings, optional `whenToUse` string and `phases` array (`{title, detail?, provider?, model?}`). The `script` parameter is the plain JavaScript body ONLY (NOT TypeScript, and NO `export const meta` statement — meta is a parameter, not code), running with top-level await; end with `return ` — the value must be JSON-serializable and is this tool's result.\n\nScript-body hooks:\n- `agent(prompt, opts?): Promise` — run one subagent to completion. Without `opts.schema` it resolves to the child's final text; with `opts.schema` (an object-rooted JSON Schema using ONLY type/properties/required/additionalProperties/items/enum/const/oneOf — no pattern/format/numeric bounds) it resolves to the validated object. Resolves `null` when the child fails (filter with `.filter(Boolean)`). Other opts: `label` (display), `phase` (progress group), and independent `provider`/`model` LLM target overrides (either may be provided alone). Anything else (`effort`/`isolation`/`agentType`) is rejected loudly.\n- `pipeline(items, ...stages): Promise` — run each item through the stages independently with NO barrier between stages (prefer this for multi-stage work). Each stage receives `(prev, item, index)`. An ordinary stage throw drops that ITEM to `null` and skips its remaining stages.\n- `parallel(thunks): Promise` — run zero-argument functions concurrently and await ALL of them (a barrier; use only when a stage genuinely needs every prior result together). A throwing thunk resolves to `null`.\n- `phase(title)` — start a progress phase; `log(message)` — narrate progress; `args` — the tool call's `args` input, verbatim.\n\nMisused hooks (bad arguments, unknown options, unsupported schemas, tripped caps) throw errors that ALWAYS kill the script — they never dissolve into a per-item `null`.\n\nConstraints: concurrency and total-agent caps apply; no filesystem, network, timers, or Node.js APIs are provided — the agents do the work, the script only coordinates them. The run executes in the foreground: this call returns when the whole script finishes.","parameters":{"type":"object","properties":{"script":{"type":"string","description":"The plain-JS workflow script body (top-level await allowed; NO `export const meta` statement; end with `return `)."},"meta":{"type":"object","description":"The workflow identity block (plain JSON — never code).","additionalProperties":true,"properties":{"name":{"type":"string","description":"Short kebab-case workflow name."},"description":{"type":"string","description":"One-line description of what the workflow does."},"whenToUse":{"type":"string","description":"Optional guidance on when this workflow applies."},"phases":{"type":"array","description":"Optional phase declarations matched by phase() calls.","items":{"type":"object","additionalProperties":true,"properties":{"title":{"type":"string","description":"The phase title phase() calls match by exact string."},"detail":{"type":"string","description":"Optional one-line description of the phase."},"provider":{"type":"string","description":"Optional provider override this phase is expected to use."},"model":{"type":"string","description":"Optional model override this phase is expected to use."}},"required":["title"]}}},"required":["name","description"]},"args":{"type":"object","description":"Optional JSON input exposed to the script as the `args` global (wrap a bare list as a field, e.g. {\"files\": [...]}).","additionalProperties":true}},"required":["script","meta"]}},{"name":"write","description":"Create or fully replace a UTF-8 text file.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to write, resolved by the filesystem backend."},"content":{"type":"string","description":"Full UTF-8 text content to write."}},"required":["file_path","content"]}}]},"reason":"initial"}} +{"type":"request/context","seq":7,"time":1786123401669,"data":{"provider":"deepseek-official","model":"deepseek-v4-flash","contextWindow":128000}} +{"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}} +{"type":"assistant/chunk","seq":9,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"call_compaction_marker","name":"bash","argumentsDelta":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}}} +{"type":"assistant/chunk","seq":10,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_compaction_marker","name":"bash","arguments":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}}}} +{"type":"assistant/chunk","seq":11,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":24,"outputTokens":6}}}} +{"type":"assistant/chunk","seq":12,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} +{"type":"assistant/message","seq":13,"time":1786123401680,"data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"call_compaction_marker","name":"bash","arguments":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"e54f73a8-572a-40ee-b908-8a8a27b83bf8"},"usage":{"inputTokens":24,"outputTokens":6}},"sourceEventSeqs":[8,9,10,11,12],"surfaceOp":"append"} +{"type":"tool/call","seq":14,"time":1786123401680,"data":{"turn":1,"step":1,"callId":"call_compaction_marker","name":"bash","arguments":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}} +{"type":"tool/result","seq":15,"time":1786123401700,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_compaction_marker"},"content":[{"type":"tool-result","toolCallId":"call_compaction_marker","content":[{"type":"text","text":"alpha\n"}],"isError":false}],"role":"user","id":"b4a6504e-f39d-40b0-b51a-b11fbd60b135"}},"sourceEventSeqs":[14],"surfaceOp":"append"} +{"type":"step/end","seq":16,"time":1786123401700,"data":{"turn":1,"step":1}} +{"type":"step/start","seq":17,"time":1786123401710,"data":{"turn":1,"step":2}} +{"type":"assistant/chunk","seq":18,"time":1786123401715,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"error","failure":{"message":"snapshot request exceeded the model context window","code":"CONTEXT_WINDOW_EXCEEDED"}}}}} +{"type":"compact/start","seq":19,"time":1786123401715,"data":{"turn":1}} +{"type":"compact/summary","seq":20,"time":1786123401725,"data":{"summary":[{"type":"text","text":"The request established a durable compaction premise."}],"rawOutput":[{"type":"text","text":"The request established a durable compaction premise."}],"shadowedRange":{"start":4,"end":4},"shadowedSeqs":[4],"shadowedTokenCount":264,"provider":"deepseek-official","model":"deepseek-v4-flash","maxTokens":32,"usage":{"inputTokens":20,"outputTokens":4}}} +{"type":"user/message","seq":21,"time":1786123401725,"data":{"content":[{"type":"text","text":"This is an automatically generated checkpoint condensing an earlier span of the conversation to free up context. Treat the captured context as established background and build on it without restating it. Continue the task directly from the messages that follow, without acknowledging this checkpoint.\n\n"},{"type":"text","text":"The request established a durable compaction premise."},{"type":"text","text":""}],"source":{"kind":"plugin","plugin":"compact"},"role":"user","id":"6d2afb13-a37b-48d6-9ea5-fc8734127377"},"sourceEventSeqs":[19,20,4],"surfaceOp":{"op":"replace","start":4,"end":4}} +{"type":"compact/end","seq":22,"time":1786123401725,"data":{"turn":1}} +{"type":"assistant/chunk","seq":23,"time":1786123401730,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}} +{"type":"assistant/chunk","seq":24,"time":1786123401730,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":0,"text":"COMPACTION RECOVERED"}}} +{"type":"assistant/chunk","seq":25,"time":1786123401730,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"COMPACTION RECOVERED"}}}} +{"type":"assistant/chunk","seq":26,"time":1786123401730,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":20,"outputTokens":4}}}} +{"type":"assistant/chunk","seq":27,"time":1786123401730,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} +{"type":"assistant/message","seq":28,"time":1786123401730,"data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"text","text":"COMPACTION RECOVERED"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"346742d0-50e3-4594-b53c-f26c7da82c56"},"usage":{"inputTokens":20,"outputTokens":4}},"sourceEventSeqs":[23,24,25,26,27],"surfaceOp":"append"} +{"type":"step/end","seq":29,"time":1786123401730,"data":{"turn":1,"step":2}} +{"type":"turn/end","seq":30,"time":1786123401730,"data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/examples/headless-agent/tests/snapshots/compaction-recovery/stream-json.expected.jsonl b/examples/headless-agent/tests/snapshots/compaction-recovery/stream-json.expected.jsonl new file mode 100644 index 0000000000..4d798adc37 --- /dev/null +++ b/examples/headless-agent/tests/snapshots/compaction-recovery/stream-json.expected.jsonl @@ -0,0 +1,32 @@ +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"agent/inbox/spliced","seq":0,"time":0,"data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Establish a durable compaction premise before continuing. Record every part of this historical evidence: the snapshot uses keyless replay; persistence uses JSONL; the assembled headless application loads its real Cordis composition; model-visible inputs remain logged; tool calls and results remain paired and ordered; context overflow retains the original failure while recovery is attempted; compaction opens with compact/start and closes with compact/end; a successful auxiliary summary records compact/summary provenance; the replacement surface shadows only an older balanced range; the checkpoint remains smaller than the history it replaces; the newest tool result remains verbatim; the retried request sees that checkpoint; the final response proves the same turn continued; deterministic snapshot evidence stays separate from the live-provider smoke; and no external API key is needed. Emit one alpha marker through bash, then finish the task after any required recovery with the exact words COMPACTION RECOVERED."}],"source":{"kind":"user"},"role":"user","id":"{{sessionId}}"}]}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/start","seq":1,"time":0,"data":{"turn":1}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"agent/inbox/spliced","seq":2,"time":0,"data":{"target":"next-turn","start":0,"removedCount":1,"inserted":[]}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":3,"time":0,"data":{"turn":1,"step":1}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"user/message","seq":4,"time":0,"data":{"content":[{"type":"text","text":"Establish a durable compaction premise before continuing. Record every part of this historical evidence: the snapshot uses keyless replay; persistence uses JSONL; the assembled headless application loads its real Cordis composition; model-visible inputs remain logged; tool calls and results remain paired and ordered; context overflow retains the original failure while recovery is attempted; compaction opens with compact/start and closes with compact/end; a successful auxiliary summary records compact/summary provenance; the replacement surface shadows only an older balanced range; the checkpoint remains smaller than the history it replaces; the newest tool result remains verbatim; the retried request sees that checkpoint; the final response proves the same turn continued; deterministic snapshot evidence stays separate from the live-provider smoke; and no external API key is needed. Emit one alpha marker through bash, then finish the task after any required recovery with the exact words COMPACTION RECOVERED."}],"source":{"kind":"user"},"role":"user","id":"{{sessionId}}"},"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"session/title","seq":5,"time":0,"data":{"title":"Establish a durable compaction premise","messageSeqs":[4],"source":{"kind":"fallback"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"request/header","seq":6,"time":0,"data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"request/context","seq":7,"time":0,"data":{"provider":"deepseek-official","model":"deepseek-v4-flash","contextWindow":128000}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":9,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"call_compaction_marker","name":"bash","argumentsDelta":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":10,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_compaction_marker","name":"bash","arguments":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":11,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":24,"outputTokens":6}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":12,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":13,"time":0,"data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"call_compaction_marker","name":"bash","arguments":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"{{sessionId}}"},"usage":{"inputTokens":24,"outputTokens":6}},"sourceEventSeqs":[8,9,10,11,12],"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/call","seq":14,"time":0,"data":{"turn":1,"step":1,"callId":"call_compaction_marker","name":"bash","arguments":"{\"command\":\"printf 'alpha\\n'\",\"description\":\"Emit compaction premise marker\"}"}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/result","seq":15,"time":0,"data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"call_compaction_marker"},"content":[{"type":"tool-result","toolCallId":"call_compaction_marker","content":[{"type":"text","text":"alpha\n"}],"isError":false}],"role":"user","id":"{{sessionId}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":16,"time":0,"data":{"turn":1,"step":1}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":17,"time":0,"data":{"turn":1,"step":2}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":18,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"error","failure":{"message":"snapshot request exceeded the model context window","code":"CONTEXT_WINDOW_EXCEEDED"}}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"compact/start","seq":19,"time":0,"data":{"turn":1}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"compact/summary","seq":20,"time":0,"data":{"summary":[{"type":"text","text":"The request established a durable compaction premise."}],"rawOutput":[{"type":"text","text":"The request established a durable compaction premise."}],"shadowedRange":{"start":4,"end":4},"shadowedSeqs":[4],"shadowedTokenCount":264,"provider":"deepseek-official","model":"deepseek-v4-flash","maxTokens":32,"usage":{"inputTokens":20,"outputTokens":4}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"user/message","seq":21,"time":0,"data":{"content":[{"type":"text","text":"This is an automatically generated checkpoint condensing an earlier span of the conversation to free up context. Treat the captured context as established background and build on it without restating it. Continue the task directly from the messages that follow, without acknowledging this checkpoint.\n\n"},{"type":"text","text":"The request established a durable compaction premise."},{"type":"text","text":""}],"source":{"kind":"plugin","plugin":"compact"},"role":"user","id":"{{sessionId}}"},"sourceEventSeqs":[19,20,4],"surfaceOp":{"op":"replace","start":4,"end":4}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"compact/end","seq":22,"time":0,"data":{"turn":1}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":23,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":24,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":0,"text":"COMPACTION RECOVERED"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":25,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"COMPACTION RECOVERED"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":26,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":20,"outputTokens":4}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":27,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":28,"time":0,"data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"text","text":"COMPACTION RECOVERED"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash"},"id":"{{sessionId}}"},"usage":{"inputTokens":20,"outputTokens":4}},"sourceEventSeqs":[23,24,25,26,27],"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":29,"time":0,"data":{"turn":1,"step":2}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/end","seq":30,"time":0,"data":{"turn":1,"reason":{"kind":"completed"}}}} +{"type":"result","sessionId":"{{sessionId}}","output":"COMPACTION RECOVERED","usage":{"inputTokens":44,"outputTokens":10}} diff --git a/packages/support/llm-replay/README.i18n.yaml b/packages/support/llm-replay/README.i18n.yaml index a4729b2e69..3f3e349a84 100644 --- a/packages/support/llm-replay/README.i18n.yaml +++ b/packages/support/llm-replay/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/support/llm-replay/README.md -README.md: ee062d0c2804905f33f1ff476d12bb6dd57666e5 -README.zh.md: ab3420d9500a6ca77f04a2ad96095f8883aeb874 +README.md: 46d391970f320708914d11f0868cbbc5361ae196 +README.zh.md: a67b078a1396968dc3ddecb0e616a832c4faaf3a diff --git a/packages/support/llm-replay/README.md b/packages/support/llm-replay/README.md index ee062d0c28..46d391970f 100644 --- a/packages/support/llm-replay/README.md +++ b/packages/support/llm-replay/README.md @@ -8,7 +8,9 @@ Its consumers are the ACP and headless `stream-json` snapshot suites plus the We ## How the fixture works -The fixture IS the persisted session log (`/session.jsonl`). Its `assistant/chunk` events carry every `StreamChunk`, so grouping them by `(turn, step)` reconstructs each `stream()` call's chunk sequence (one model call per loop step). Recording is therefore "run the real agent once and harvest the `.jsonl`", done by the snapshot harness — this plugin does not record. A fixture may carry its `request/header` content tokenized to `{{system}}`/`{{tools}}` (the harness pins that content in one scenario and scrubs the rest); replay is indifferent — derivation reads only `assistant/chunk` events and the line-0 session header. +The fixture IS the persisted session log (`/session.jsonl`). Its `assistant/chunk` events carry every `StreamChunk`, so grouping them by `(turn, step)` reconstructs each agent-loop `stream()` call's chunk sequence. A successful compaction summarizer is logged differently: when `compact/summary` carries its complete `rawOutput`, replay reconstructs a canonical successful stream at that event's position using one `block-start`/`block-end` pair per block, the recorded usage when present, and a terminal `stop`. Exact provider delta partitioning is not part of the durable compaction result. A summary without `rawOutput` does not imply an LLM call because template and remote summarizers may produce it without the local adapter. + +Recording is therefore "run the real agent once and harvest the `.jsonl`", done by the snapshot harness — this plugin does not record. A fixture may carry its `request/header` content tokenized to `{{system}}`/`{{tools}}` (the harness pins that content in one scenario and scrubs the rest); replay is indifferent — derivation reads only `assistant/chunk` and `compact/summary` events plus the line-0 session header. Two failure modes are not reconstructable from `assistant/chunk` alone — a pure throw before any chunk (e.g. an HTTP 401, where the log holds only a `turn/end {error}` and no chunks) and a cancel/hang (timing, not chunk content). A scenario that needs those supplies an optional sidecar (`/replay.override.json`) that either replaces the derived script (a bare `ReplayEntry[]`) or augments it (`{ patches: [{ at, entry }] }`: keep every JSONL-derived call and swap the named 0-based call indexes; `at` equal to the derived length appends the retry attempt after an injected transient throw). Patch indexes must be unique. The override document, each patch and entry, and every chunk discriminant are validated when the file loads. A `hang` entry may name `readyFile`; replay writes that empty marker after its prefix chunks reach the loop and before it waits for cancellation, so an external driver can cancel deterministically without observing a presentation update. @@ -57,7 +59,7 @@ Replay keys every call by its calling session id (`GenerateOptions.sessionId`, s - `installLlmReplay(ctx, config)` — install the configured replay adapter or catch-all `llm/stream` listener; returns a `ReplayHandle` (`dispose()` for HMR safety plus `assertConsumed()`, the teardown check that every recorded script bound to a live session and every bound cursor drained — turning a scenario that silently drove fewer model calls than recorded into a crisp diagnostic). Use this in tests to drive replay without the Loader or env vars. - `loadSessionScripts(config)` — resolve the ordered `SessionScript[]` (primary + children) for a scenario, ready to bind to live sessions in first-call order. - `loadReplayScript(config)` — resolve the `ReplayEntry[]` for the primary session only (validated sidecar replacement/patches if present, else derived from the JSONL; fail-loud if the fixture is missing). -- `deriveReplayScript(events)` / `parseSessionLog(text)` / `parseSessionHeader(text)` / `resolveScriptedEntry(entry, messages)` — the pure helpers that turn a recorded session log into a script, read its header `id`/`createdAt`, and resolve `{{fromRequest:...}}` placeholders against one live request. A derived group must end in a `finish` chunk; a group without one is the fingerprint of a thrown `stream()` and must instead be expressed via an override sidecar. +- `deriveReplayScript(events)` / `parseSessionLog(text)` / `parseSessionHeader(text)` / `resolveScriptedEntry(entry, messages)` — the pure helpers that turn ordinary loop chunks and complete compaction outputs in a recorded session log into a script, read its header `id`/`createdAt`, and resolve `{{fromRequest:...}}` placeholders against one live request. A derived assistant group must end in a `finish` chunk; a group without one is the fingerprint of a thrown `stream()` and must instead be expressed via an override sidecar. - Types `ReplayEntry` / `ReplayOverrideDoc` / `ReplayOverridePatch` / `SessionScript` / `ReplayConfig` / `ReplayProviderConfig` / `ReplayModelConfig` / `ReplayHandle` / `Config`. ## Plugin export shape @@ -74,5 +76,5 @@ None; this package neither assembles nor sends a provider request. ## Known Limitations and Deferred Work -- **First-call-order script binding assumes sequential delegation** — a cut that runs sibling subagents concurrently (or a compaction summarize call landing mid-run) would bind live sessions to recorded scripts non-deterministically; a stronger keying is deferred until such a scenario exists (`XXX(concurrent-subagents)`). -- **Only chunk-producing calls are derivable** — a pure pre-chunk throw or a cancel/hang scenario needs the `replay.override.json` sidecar. Replacement and patch forms affect only the primary session; child scripts still derive from their logs. +- **First-call-order script binding assumes sequential delegation** — a cut that runs sibling subagents concurrently would bind live sessions to recorded scripts non-deterministically; a stronger keying is deferred until such a scenario exists (`XXX(concurrent-subagents)`). +- **Only ordinary loop chunks and completed compaction outputs are derivable** — a pure pre-chunk throw or a cancel/hang scenario needs the `replay.override.json` sidecar. Replacement and patch forms affect only the primary session; child scripts still derive from their logs. diff --git a/packages/support/llm-replay/README.zh.md b/packages/support/llm-replay/README.zh.md index ab3420d950..a67b078a13 100644 --- a/packages/support/llm-replay/README.zh.md +++ b/packages/support/llm-replay/README.zh.md @@ -8,7 +8,9 @@ ## fixture 的工作方式 -fixture 就是持久化的会话日志(`/session.jsonl`)。其 `assistant/chunk` 事件包含每个 `StreamChunk`,因此按 `(turn, step)` 分组即可重建每次 `stream()` 调用的分片序列(每个循环步骤调用一次模型)。因此,录制就是「运行一次真实 agent 并收集 `.jsonl`」,由快照 harness 完成;该插件本身不录制。fixture 的 `request/header` 内容可能被标记化为 `{{system}}`/`{{tools}}`(harness 会在一个场景中固定该内容,并清除其余场景中的内容);回放不受影响,因为派生过程只读取 `assistant/chunk` 事件和第 0 行的会话 header。 +fixture 就是持久化的会话日志(`/session.jsonl`)。其 `assistant/chunk` 事件包含每个 `StreamChunk`,因此按 `(turn, step)` 分组即可重建每次 agent-loop `stream()` 调用的分片序列。压缩(compaction)摘要器成功时,日志记录方式有所不同:当 `compact/summary` 携带完整的 `rawOutput` 时,回放会在该事件的位置重建一条规范成功流,其中每个块各使用一对 `block-start`/`block-end`,带上已记录的 usage(如有),并以 `stop` 终止。提供方增量的精确切分不属于持久压缩结果。不带 `rawOutput` 的摘要并不意味着发生了 LLM 调用,因为模板摘要器和远程摘要器可能不经本地适配器生成该摘要。 + +因此,录制就是「运行一次真实 agent 并收集 `.jsonl`」,由快照 harness 完成;该插件本身不录制。fixture 的 `request/header` 内容可能被标记化为 `{{system}}`/`{{tools}}`(harness 会在一个场景中固定该内容,并清除其余场景中的内容);回放不受影响,因为派生过程只读取 `assistant/chunk` 和 `compact/summary` 事件以及第 0 行的会话 header。 有两种失败模式无法仅根据 `assistant/chunk` 重建:在产生任何分片前直接抛出异常(例如 HTTP 401,此时日志只有 `turn/end {error}` 而没有分片),以及取消或挂起(差异在时序,而非分片内容)。需要这些行为的场景可提供伴随文件(`/replay.override.json`):它可以替换派生脚本(裸 `ReplayEntry[]`),也可以增补派生脚本(`{ patches: [{ at, entry }] }`:保留所有从 JSONL 派生的调用,只替换指定的从 0 开始计数的调用索引;当 `at` 等于派生长度时,则在注入瞬态异常后的重试位置追加一次调用)。补丁索引不得重复。文件加载时会校验覆写文档、每个补丁和条目,以及每个分片的判别标签。`hang` 条目可以指定 `readyFile`;当前缀分片到达循环后、开始等待取消前,回放会写入这个空标记,使外部驱动程序无需观察展示层更新即可确定性地取消。 @@ -57,7 +59,7 @@ fixture 就是持久化的会话日志(`/session.jsonl`)。其 `as - `installLlmReplay(ctx, config)`:安装已配置回放适配器或 catch-all `llm/stream` 监听器;返回 `ReplayHandle`(包含用于保证 HMR(热模块替换)安全的 `dispose()`,以及清理阶段执行的 `assertConsumed()` 检查;后者确保每个已记录脚本都绑定到实时会话,且每个已绑定游标都已耗尽,从而将场景静默驱动的模型调用少于记录数转换为明确诊断)。在测试中使用它,可以不通过 Loader 或 env var 驱动回放。 - `loadSessionScripts(config)`:解析场景中有序的 `SessionScript[]`(主会话 + 子会话),准备按首次调用顺序绑定到实时会话。 - `loadReplayScript(config)`:只解析主会话的 `ReplayEntry[]`(如果伴随文件存在,则使用经校验的替换或补丁;否则从 JSONL 派生;fixture 缺失时明确报错)。 -- `deriveReplayScript(events)` / `parseSessionLog(text)` / `parseSessionHeader(text)` / `resolveScriptedEntry(entry, messages)`:将已记录会话日志转换为脚本、读取其 header `id`/`createdAt`、并针对单次实时请求解析 `{{fromRequest:...}}` 占位符的纯辅助工具。派生分组必须以 `finish` 分片结束;没有该分片的分组是 `stream()` 抛出异常的指纹,必须改用 override 伴随文件表达。 +- `deriveReplayScript(events)` / `parseSessionLog(text)` / `parseSessionHeader(text)` / `resolveScriptedEntry(entry, messages)`:将已记录会话日志中的普通 loop 分片和完整压缩输出转换为脚本、读取其 header `id`/`createdAt`、并针对单次实时请求解析 `{{fromRequest:...}}` 占位符的纯辅助工具。派生的 assistant 分组必须以 `finish` 分片结束;没有该分片的分组是 `stream()` 抛出异常的指纹,必须改用 override 伴随文件表达。 - 类型 `ReplayEntry` / `ReplayOverrideDoc` / `ReplayOverridePatch` / `SessionScript` / `ReplayConfig` / `ReplayProviderConfig` / `ReplayModelConfig` / `ReplayHandle` / `Config`。 ## 插件导出形态 @@ -74,5 +76,5 @@ fixture 就是持久化的会话日志(`/session.jsonl`)。其 `as ## 已知限制与暂缓事项 -- **首次调用顺序脚本绑定假设串行委托**:并发运行同级 subagent 的 cut(或运行中发生的上下文压缩(context compaction)摘要调用)会非确定性地将实时会话绑定到已记录脚本;在这种场景出现前暂不实现更强的键控(`XXX(concurrent-subagents)`)。 -- **只有会产生分片的调用才能派生**:在产生分片前直接抛出异常或取消/挂起的场景需要 `replay.override.json` 伴随文件。替换和补丁两种形式都只影响主会话;子会话脚本仍从各自日志派生。 +- **首次调用顺序脚本绑定假设串行委托**:并发运行同级 subagent 的 cut 会非确定性地将实时会话绑定到已记录脚本;在这种场景出现前暂不实现更强的键控(`XXX(concurrent-subagents)`)。 +- **只有普通 loop 分片和已完成的压缩输出才能派生**:在产生分片前直接抛出异常或取消/挂起的场景需要 `replay.override.json` 伴随文件。替换和补丁两种形式都只影响主会话;子会话脚本仍从各自日志派生。 diff --git a/packages/support/llm-replay/package.json b/packages/support/llm-replay/package.json index 708e84b57a..af5e2929f3 100644 --- a/packages/support/llm-replay/package.json +++ b/packages/support/llm-replay/package.json @@ -25,12 +25,14 @@ ], "license": "BSD-3-Clause", "peerDependencies": { + "@deepseek-ai/dsh-compact": "^0.0.1", "@deepseek-ai/dsh-invariants": "^0.0.1", "@deepseek-ai/dsh-llm": "^0.0.1", "@deepseek-ai/dsh-session": "^0.0.1", "cordis": "^4.0.0-rc.7" }, "devDependencies": { + "@deepseek-ai/dsh-compact": "workspace:^", "@deepseek-ai/dsh-invariants": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", "@deepseek-ai/dsh-session": "workspace:^", diff --git a/packages/support/llm-replay/src/index.ts b/packages/support/llm-replay/src/index.ts index ae62843492..8733a4296c 100644 --- a/packages/support/llm-replay/src/index.ts +++ b/packages/support/llm-replay/src/index.ts @@ -1,14 +1,16 @@ /** * Keyless snapshot-test LLM replay. It derives one model-call script per - * recorded session from `assistant/chunk` events and binds fresh live sessions - * to parent/child scripts by first-call order. Throw and hang cases require an - * explicit override because a session log cannot reconstruct them alone. + * recorded session from `assistant/chunk` events and durable compaction + * summaries, then binds fresh live sessions to parent/child scripts by + * first-call order. Throw and hang cases require an explicit override because + * a session log cannot reconstruct them alone. * @module @deepseek-ai/dsh-llm-replay */ import { existsSync, readFileSync, writeFileSync } from 'node:fs' import { delimiter as pathDelimiter } from 'node:path' import type { Context } from 'cordis' +import type {} from '@deepseek-ai/dsh-compact' import { decodeStorageRecord } from '@deepseek-ai/dsh-session' import type { SessionEvent } from '@deepseek-ai/dsh-session' import type { @@ -24,8 +26,9 @@ import { LlmAdapter, LlmError, assertNever, resolveRetryPolicy } from '@deepseek /** * One recorded model call. `throw` may replay prefix chunks before failing; - * `hang` models cancellation. Only ordinary chunk entries derive from JSONL; - * the other variants come from an override sidecar. + * `hang` models cancellation. Chunk entries derive from ordinary model streams + * and complete compaction outputs in JSONL; the other variants come from an + * override sidecar. */ export type ReplayEntry = | { kind: 'chunks'; chunks: StreamChunk[] } @@ -174,10 +177,12 @@ export function parseSessionHeader(text: string): { id: string; createdAt: numbe * Reconstruct the per-`stream()` replay script from a recorded session log. * * Splits `assistant/chunk` events at every `finish`, using turn and step changes - * to detect an unterminated prior call. A missing terminator means the live - * stream threw, so derivation rejects and the scenario must provide an explicit - * override. Multiple calls may share one turn and step when the loop retries. - * @param events - the recorded session's events; only `assistant/chunk` is consulted. + * to detect an unterminated prior call. A complete `compact/summary.rawOutput` + * becomes a canonical successful stream at the summary's log position. A + * missing assistant terminator means the live stream threw, so derivation + * rejects and the scenario must provide an explicit override. Multiple calls + * may share one turn and step when the loop retries. + * @param events - the recorded session's events. * @returns one `chunks` entry per recorded model call, in call order. */ export function deriveReplayScript(events: SessionEvent[]): ReplayEntry[] { @@ -195,6 +200,22 @@ export function deriveReplayScript(events: SessionEvent[]): ReplayEntry[] { script.push({ kind: 'chunks', chunks }) } for (const event of events) { + if (event.type === 'compact/summary') { + close(currentKey, current) + currentKey = undefined + current = [] + if (event.data.rawOutput !== undefined) { + const chunks: StreamChunk[] = [] + for (const [index, block] of event.data.rawOutput.entries()) { + chunks.push({ type: 'block-start', index, blockType: block.type }) + chunks.push({ type: 'block-end', index, block }) + } + if (event.data.usage !== undefined) chunks.push({ type: 'usage', usage: event.data.usage }) + chunks.push({ type: 'finish', reason: { kind: 'stop' } }) + script.push({ kind: 'chunks', chunks }) + } + continue + } if (event.type !== 'assistant/chunk') continue const { turn, step, chunk } = event.data const key = `${turn}/${step}` diff --git a/packages/support/llm-replay/tests/llm-replay.spec.ts b/packages/support/llm-replay/tests/llm-replay.spec.ts index 0b7d87ad13..9483a4c4f6 100644 --- a/packages/support/llm-replay/tests/llm-replay.spec.ts +++ b/packages/support/llm-replay/tests/llm-replay.spec.ts @@ -178,6 +178,93 @@ describe('deriveReplayScript', () => { expect(deriveReplayScript(events)).toEqual([{ kind: 'chunks', chunks: errChunks }]) }) + it('inserts compact/summary output between the calls surrounding it', () => { + const overflow: StreamChunk[] = [ + { type: 'finish', reason: { kind: 'error', failure: { message: 'too large', code: 'CONTEXT_WINDOW_EXCEEDED' } } }, + ] + const block = { type: 'text' as const, text: 'durable checkpoint' } + const rawOutput = [block] + const usage = { inputTokens: 9, outputTokens: 2 } + const summaryChunks: StreamChunk[] = [ + { type: 'block-start', index: 0, blockType: 'text' }, + { type: 'block-end', index: 0, block }, + { type: 'usage', usage }, + { type: 'finish', reason: { kind: 'stop' } }, + ] + let seq = 1 + const events: SessionEvent[] = [ + ...overflow.map(chunk => chunkEvent(seq++, 1, 2, chunk)), + { type: 'compact/start', seq: seq++, time: 0, data: { turn: 1 } }, + { + type: 'compact/summary', + seq: seq++, + time: 0, + data: { + summary: rawOutput, + rawOutput, + shadowedRange: { start: 1, end: 1 }, + shadowedSeqs: [1], + shadowedTokenCount: 20, + provider: 'mock', + model: 'mock', + usage, + }, + }, + ...TEXT_CHUNKS.map(chunk => chunkEvent(seq++, 1, 2, chunk)), + ] + + expect(deriveReplayScript(events)).toEqual([ + { kind: 'chunks', chunks: overflow }, + { kind: 'chunks', chunks: summaryChunks }, + { kind: 'chunks', chunks: TEXT_CHUNKS }, + ]) + }) + + it('does not infer an LLM call from compact/summary without raw output', () => { + const event: SessionEvent<'compact/summary'> = { + type: 'compact/summary', + seq: 1, + time: 0, + data: { + summary: [{ type: 'text', text: 'template result' }], + shadowedRange: { start: 1, end: 1 }, + shadowedSeqs: [1], + shadowedTokenCount: 20, + provider: 'template', + model: 'template', + }, + } + + expect(deriveReplayScript([event])).toEqual([]) + }) + + it('derives a compact/summary stream when usage is unavailable', () => { + const block = { type: 'text' as const, text: 'summary without usage' } + const event: SessionEvent<'compact/summary'> = { + type: 'compact/summary', + seq: 1, + time: 0, + data: { + summary: [block], + rawOutput: [block], + shadowedRange: { start: 1, end: 1 }, + shadowedSeqs: [1], + shadowedTokenCount: 20, + provider: 'mock', + model: 'mock', + }, + } + + expect(deriveReplayScript([event])).toEqual([{ + kind: 'chunks', + chunks: [ + { type: 'block-start', index: 0, blockType: 'text' }, + { type: 'block-end', index: 0, block }, + { type: 'finish', reason: { kind: 'stop' } }, + ], + }]) + }) + it('throws on a group that lacks a terminal finish chunk (a thrown stream)', () => { // A thrown stream(): prefix chunks logged, then turn/end (error reason), NO finish. const events: SessionEvent[] = [ diff --git a/packages/support/llm-replay/tsconfig.json b/packages/support/llm-replay/tsconfig.json index 673ee51547..b8dc74e792 100644 --- a/packages/support/llm-replay/tsconfig.json +++ b/packages/support/llm-replay/tsconfig.json @@ -14,6 +14,9 @@ { "path": "../../../vendor/cordis" }, + { + "path": "../../compact/compact" + }, { "path": "../../llm/llm" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c3a3d5ecca..078f775ecf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5959,6 +5959,9 @@ importers: packages/support/llm-replay: devDependencies: + '@deepseek-ai/dsh-compact': + specifier: workspace:^ + version: link:../../compact/compact '@deepseek-ai/dsh-invariants': specifier: workspace:^ version: link:../invariants