diff --git a/docs/module-graph.md b/docs/module-graph.md index a4e729484d..d2537028f6 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -9,6 +9,7 @@ Inter-package dependencies among the `@deepseek-ai/dsh-*` harness packages, deri flowchart TD subgraph group_util["packages/util"] pkg_brand["brand"] + pkg_retention["retention"] pkg_timeout["timeout"] end subgraph group_llm["packages/llm"] @@ -208,6 +209,7 @@ flowchart TD | Package | Group | Depends on | | --- | --- | --- | | [`brand`](../packages/util/brand) | `util` | — | +| [`retention`](../packages/util/retention) | `util` | — | | [`timeout`](../packages/util/timeout) | `util` | — | | [`app-boot`](../packages/ui/app-boot) | `ui` | — | | [`llm`](../packages/llm/llm) | `llm` | [`brand`](../packages/util/brand) | diff --git a/docs/rfc/INDEX.md b/docs/rfc/INDEX.md index a7e0506cad..1ce999d071 100644 --- a/docs/rfc/INDEX.md +++ b/docs/rfc/INDEX.md @@ -26,7 +26,6 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand; |---|---| | [Runtime schemas for the event vocabulary (Zod vs the merge-extensible-map pattern)](proposed/architecture/2026-06-16-typed-event-schemas.md) | 2026-06-16 | | [Extract a generic long-running tool runtime](proposed/architecture/2026-06-20-generic-long-running-tool-runtime.md) | 2026-06-20 | -| [Tool result retention library](proposed/architecture/2026-07-06-tool-result-retention-library.md) | 2026-07-06 | ### Process @@ -120,6 +119,7 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand; | [Prompt variables and tool-guidance ownership](implemented/architecture/2026-07-05-prompt-variables-and-tool-guidance-ownership.md) | 2026-07-05 | | [Subagent provider-lifecycle events — `subagent/provider-added` / `subagent/provider-removed`](implemented/architecture/2026-07-05-subagent-provider-lifecycle-events.md) | 2026-07-05 | | [A shared timeout/deadline primitive, with hard-kill left to each capability](implemented/architecture/2026-07-06-timeout-deadline-library.md) | 2026-07-06 | +| [Tool result retention library](implemented/architecture/2026-07-06-tool-result-retention-library.md) | 2026-07-06 | ### Process diff --git a/docs/rfc/proposed/architecture/2026-07-06-tool-result-retention-library.md b/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md similarity index 75% rename from docs/rfc/proposed/architecture/2026-07-06-tool-result-retention-library.md rename to docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md index b8f16f6bb1..7ea954b0a2 100644 --- a/docs/rfc/proposed/architecture/2026-07-06-tool-result-retention-library.md +++ b/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md @@ -1,6 +1,6 @@ # RFC: Tool result retention library -Status: proposed +Status: implemented ## Problem @@ -8,9 +8,9 @@ Several model-facing tools already bound the amount of context they return, but The shared abstraction the tools need is **retention**, not generic collection. A caller feeds items or text chunks into a bounded object, receives a per-push decision about whether the upstream can stop, and later receives the retained content plus exact or partial omission metadata. Tool-specific code still owns business semantics: file grouping, line numbering, exit codes, provider error states, and model-facing prose. The common library owns only the mechanical question "what did we keep, what did we omit, and may the caller stop reading now?" -## Proposal +## Decision -Add a small, dependency-light retention library under `packages/util/retention` (package name `@deepseek-ai/dsh-retention`). It exports pure item and text retainers plus notice helpers. It is not a Cordis service and registers no plugin; tool packages import it directly when they need bounded model-facing output. +`@deepseek-ai/dsh-retention` lives under `packages/util/` (peer to `dsh-brand` and `dsh-timeout`) and owns bounded model-facing output. It is a library of pure classes and functions, **not** a Cordis service or plugin: it takes no `ctx`, registers nothing, holds no cross-call state, and emits no events. Tool packages import it directly when they need bounded output. The library has two independent retainers: @@ -116,7 +116,7 @@ type TextRetentionStrategy = `grep` uses `ItemRetainer` with `{ kind: 'head', maxItems: grepMaxMatches, stop: 'stopWhenFull' }` before grouping. The backend parses a ripgrep match record, maps the path, applies per-line preview truncation, then pushes a flat match. After `finish()`, the backend groups retained matches by file and sorts the returned subset. Grouping is not part of the retainer because the cap is total matches, not files; per-match preview truncation and `incomplete` are also separate from result-level retention. -`bash` uses `TextRetainer` with `tail` or `headTail` and reads to process completion. It does not stop when full: stopping the read would lose the real tail and can create pipe backpressure. The bash executor still owns spill files, exit status, signal, timeout, and background-task behavior; the retention helper only replaces ad hoc in-memory head/tail accounting where that behavior is desired. Long-running task ownership remains orthogonal to the [generic long-running tool runtime](2026-06-20-generic-long-running-tool-runtime.md) proposal. +`bash` uses `TextRetainer` with `tail` or `headTail` and reads to process completion. It does not stop when full: stopping the read would lose the real tail and can create pipe backpressure. The bash executor still owns spill files, exit status, signal, timeout, and background-task behavior; the retention helper only replaces ad hoc in-memory head/tail accounting where that behavior is desired. Long-running task ownership remains orthogonal to the [generic long-running tool runtime](../../proposed/architecture/2026-06-20-generic-long-running-tool-runtime.md) proposal. `web_fetch` can use `TextRetainer` with `head` when the provider exposes a stream, or keep provider-owned body caps when the provider must read and decode internally. Either way, the fetch result's `truncated` remains a provider/tool fact, and the library only supplies retained text and omission metadata. @@ -147,6 +147,16 @@ The formatter hook is deliberately small: a tool turns a `RetentionNotice` into `truncated` means the retainer omitted otherwise-available content because of a budget. It does not mean the upstream was incomplete. Tools keep separate fields for permission failures, skipped binary files, provider partial failures, unreadable candidates, invalid UTF-8, and any other "could not inspect" condition. +## Consequences + +**What shipped.** `@deepseek-ai/dsh-retention` exports `ItemRetainer`, `TextRetainer`, the result types (`RetainedItems`, `RetainedText`), the strategy types (`ItemRetentionStrategy`, `TextRetentionStrategy`, `StopMode`), `Omitted`, `PushDecision`, `RetentionNotice`, and the neutral notice helpers `describeOmitted` / `formatRetentionNotice` — with no dependency on Cordis or any tool package. Unit tests cover item-head early stop with a probe item, item-head read-to-end with exact omission counts, text-head early stop, text-tail retention with exact omission counts, head-tail byte retention, zero budgets, UTF-8 boundary handling (2-, 3-, and 4-byte codepoints and invalid lead bytes at each cut), and the difference between `{ kind: 'atLeast', count: 1 }` and exact omission. + +**What is documented but not yet migrated.** `glob`, `grep`, `bash`, `web_fetch`, and `web_search` have their mappings documented in the [package README](../../../../packages/util/retention/README.md) — each stating whether it may stop upstream early — but no tool has been migrated onto the library in this change; migration is deliberately separate follow-up work. `glob` / `grep` do not yet exist as tools, so the `shouldStop` early-stop path has no in-repo caller until they land. `read` is documented as intentionally out of scope: its `read-render` line-window contract (`offset`/`limit`, `totalLines`, offset-range errors, per-line preview truncation, a byte cap over the selected window) is not generic retention, and one `Omitted` count cannot represent both sides of a line window. + +**Boundaries the library holds.** `truncated` means the retainer omitted otherwise-available content because of a budget; it never means the upstream was incomplete. Tool-specific states — `incomplete`, permission failures, provider partial failures, binary skips, bash spill-path recovery, invalid UTF-8 — stay in tool-domain fields, outside the retainer. When a future change migrates a tool, that package's README and tests must prove the model-facing result text is unchanged except for deliberate notice wording. + +**Tradeoffs accepted.** The v1 surface deliberately supports only item `head` retention and text `head` / `tail` / `headTail`; windows, grouped budgets, and sort-aware caps wait until a second consumer proves the need (the generic-collector alternative is why). Text retention counts bytes for process/body safety, leaving character- and line-level preview budgets as separate tool-owned concerns. `glob` / `grep` cannot report an exact omitted count once they stop the upstream at the first overflow item, so `Omitted.atLeast` exists and `describeOmitted` prints no number for it — formatters never claim "omitted 1" when the true count may be far larger. + ## Alternatives considered **Post-hoc `truncate(text)` only.** Rejected: it matches Codex's history/tool-output truncation use case but fails the `glob` / `grep` resource model. The tool must stop ripgrep once the probe result proves truncation; collecting all output and trimming afterward defeats the point and can exceed the command runner's in-memory output cap. @@ -158,18 +168,3 @@ The formatter hook is deliberately small: a tool turns a `RetentionNotice` into **Make truncation part of `ToolExecutionResult`.** Rejected: the tool registry would have to understand tool-specific recovery guidance, grouping, line numbering, exit status, and provider semantics. Retention is a library used before a tool returns `ContentBlock[]`; the model-facing result remains tool-owned. **Expose limits in every model-facing tool schema.** Rejected as the default: Claude Code's grep exposes `head_limit` / `offset`, but this harness keeps routine budgets as deployment config unless the model genuinely needs pagination control. A future read-like continuation field can be added per tool; it does not belong in the shared retention primitive. - -## Acceptance criteria - -- A new `@deepseek-ai/dsh-retention` utility package exports `ItemRetainer`, `TextRetainer`, `RetainedItems`, `RetainedText`, the strategy types, `Omitted`, `PushDecision`, and neutral notice helpers without depending on Cordis or any tool package. -- Unit tests cover item-head early stop with a probe item, item-head read-to-end with exact omission counts, text-head early stop, text-tail retention with exact omission counts, head-tail byte retention, zero budgets, UTF-8 boundary handling, and the difference between `{ kind: 'atLeast', count: 1 }` and exact omission. -- `glob`, `grep`, `bash`, `web_fetch`, and `web_search` have documented mappings to the library before any broad migration begins; each mapping states whether it may stop upstream early. `read` is documented as intentionally out of scope for v1. -- Existing tool-specific states such as `incomplete`, provider failures, binary skips, and bash spill-path recovery remain outside the retention library. -- If the first implementation migrates an existing tool, that package's README and tests prove the model-facing result text is unchanged except for deliberate notice wording. - -## Risks - -- **Over-generalizing the v1 surface.** A generic callback-heavy collector would be harder to reason about than the duplicated code it replaces. The v1 surface deliberately supports only item `head` retention and text `head` / `tail` / `headTail`; windows, grouped budgets, and sort-aware caps can wait until a second consumer proves it needs them. -- **Conflating truncation with incomplete execution.** The library name may invite callers to stuff permission or provider partial failures into `truncated`. Tests and README examples must keep the rule explicit: retention budgets omit available content; incomplete inspection is a tool-domain state. -- **Byte-vs-character confusion.** Text retainers count bytes for process/body safety, while some model-facing previews care about characters or lines. The v1 API should make byte retention explicit and leave character-level preview helpers as separate functions. -- **False precision after early stop.** `glob` and `grep` cannot report exact omitted counts when they stop the upstream at the first overflow item. The `Omitted.atLeast` variant exists so formatters do not claim "omitted 1" when the true count may be much larger. diff --git a/knip.json b/knip.json index 2e3e101ae1..586528db74 100644 --- a/knip.json +++ b/knip.json @@ -26,6 +26,11 @@ "project": ["src/**/*.ts", "tests/**/*.ts"], "ignoreDependencies": ["cordis"] }, + "packages/util/retention": { + "entry": ["tests/**/*.spec.ts"], + "project": ["src/**/*.ts", "tests/**/*.ts"], + "ignoreDependencies": ["cordis"] + }, "packages/llm/llm-deepseek": { "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], "project": ["src/**/*.ts", "tests/**/*.ts"] diff --git a/packages/README.md b/packages/README.md index f07e0d5a36..1f7cb9121b 100644 --- a/packages/README.md +++ b/packages/README.md @@ -20,7 +20,7 @@ Packages are grouped by modular role at `packages///`. The group dir | [`session-persistence/`](session-persistence/README.md) | Persistence capability family: the seam + JSONL/SQLite backends | Product — stable surface | | [`ui/`](ui/README.md) | Editor/client integration surfaces (the ACP bridge) + the app packages | Product — stable surface | | [`support/`](support/README.md) | Dev/test/example infrastructure (invariants, replay adapter, subagent mock) | Support — lower compatibility expectations | -| [`util/`](util/README.md) | Low-level zero-dependency utilities shared across groups (the `Branded` primitive) | Support — small, stable, harness-dep-free | +| [`util/`](util/README.md) | Low-level zero-dependency primitives shared across groups (branding, timeout, retention) | Support — small, stable, harness-dep-free | The split is the point: a package's group says whether it is part of the product API or support/test/example infrastructure, so release and removal decisions do not treat every package as an equal public contract. New packages join an existing group; adding a new top-level group is a deliberate act (extend the group READMEs and this table). diff --git a/packages/util/README.md b/packages/util/README.md index 45afe7b0a9..6477523861 100644 --- a/packages/util/README.md +++ b/packages/util/README.md @@ -6,7 +6,10 @@ Zero-dependency primitives shared across the other groups. A package lands here |---|---| | `brand/` | The type-only `Branded` nominal-typing primitive (no runtime code, no harness deps) | | `timeout/` | The timing/classification half of a timeout — `clampTimeout`/`deadline`/`timeoutOf`/`TimeoutReason` (pure functions, no harness deps); termination stays in each capability | +| `retention/` | Bounded model-facing output — `ItemRetainer`/`TextRetainer` + neutral notice helpers (pure, no harness deps); business semantics stay in each tool | `dsh-brand` is the canonical case: it owns ONLY the `Branded` helper, so a capability package can brand the ids it owns (`dsh-bash`'s `BashTaskId`/`OwnerToken`, `dsh-session`'s `SessionId`, …) by depending on `dsh-brand` alone, without pulling in an unrelated package just to reach `Branded`. `dsh-timeout` follows the same shape for the timeout family: `dsh-bash` and `dsh-web-fetch-local` each fuse a caller's cancellation with a deadline and later classify "timed out" vs "cancelled" by depending on `dsh-timeout` alone. It deliberately owns only the timing/classification half — the *termination* (SIGKILL a process group, tear down a fetch socket) stays in each capability, because no shared layer can own every capability's kill (see [the timeout-library RFC](../../docs/rfc/implemented/architecture/2026-07-06-timeout-deadline-library.md)). + +`dsh-retention` is the same split for bounded tool output: a tool (`glob`/`grep`/`bash`/`web_fetch`/`web_search`) feeds items or text into a retainer and gets back "what we kept, what we omitted, may you stop reading" — while grouping, exit codes, provider errors, and recovery prose stay tool-owned. It deliberately owns only the retention mechanic; `truncated` is a budget fact, never an "incomplete inspection" state (see [the retention-library RFC](../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md)). diff --git a/packages/util/retention/README.md b/packages/util/retention/README.md new file mode 100644 index 0000000000..50bac13828 --- /dev/null +++ b/packages/util/retention/README.md @@ -0,0 +1,83 @@ +# dsh-retention + +A dependency-light **retention** library: bounded model-facing output for tools that must cap how much context they return. A caller feeds items or text chunks into a bounded object, gets a per-push decision about whether the upstream may stop, and later gets the retained content plus exact or partial omission metadata. + +The library owns **only** the mechanical question *"what did we keep, what did we omit, and may the caller stop reading now?"*. Tool-specific code keeps its business semantics: file grouping, line numbering, exit codes, provider error states, per-line preview truncation, spill files, and the model-facing prose. This is the boundary the [RFC](../../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md) draws. + +It is a **library, not a service or plugin**: no `ctx`, registers nothing, emits no events. The only state is per-retainer (one accumulation), never cross-call. Tool packages import it directly. + +## Surface + +```ts +import { + ItemRetainer, TextRetainer, + describeOmitted, formatRetentionNotice, +} from '@deepseek-ai/dsh-retention' +import type { + Omitted, PushDecision, RetainedItems, RetainedText, + ItemRetentionStrategy, TextRetentionStrategy, StopMode, RetentionNotice, +} from '@deepseek-ai/dsh-retention' +``` + +| Export | Role | +|---|---| +| `ItemRetainer` | Bounds ordered logical units (paths, grep matches, sources). `head` only in v1. `push()` → `PushDecision`; `finish()` → `RetainedItems`. | +| `TextRetainer` | Bounds a byte-oriented text stream. `head` / `tail` / `headTail`, UTF-8 boundaries preserved at `finish()`. `push()` → `PushDecision`; `finish()` → `RetainedText`. | +| `describeOmitted(omitted, unit)` | Standardized, false-precision-safe omission clause (`exact` prints a count; `atLeast`/`unknown` do not). | +| `formatRetentionNotice(notice, recovery)` | Joins the standardized omission clause with the tool's own recovery guidance. | +| `Omitted` | `none` / `exact` / `atLeast` / `unknown` — how much was omitted, and whether the count is a lower bound. | +| `PushDecision` | `{ kept, truncated, shouldStop }` — the per-push control-flow result. | + +## The two resource modes + +The two retainers are separate names, not one generic collector, because they differ in **resource model** — and that difference is the whole point of the `shouldStop` field. + +- **`ItemRetainer` can stop the upstream early.** With `stop: 'stopWhenFull'`, the first over-cap unit is a *probe*: it is not retained, sets `truncated`, and returns `shouldStop: true`. A discovery tool uses that to kill ripgrep / cancel a stream the moment truncation is proven, instead of collecting everything and trimming afterward. Because it stopped before the true total was known, `omitted` is `{ kind: 'atLeast', count: 1 }` — a lower bound, never a false-precise exact count. +- **`TextRetainer` tail/headTail must read to the end.** A true tail is unknowable until the stream closes, and draining avoids pipe backpressure on a child process, so `tail` and `headTail` never set `shouldStop` and report an `exact` omitted byte count. Only `head` + `stopWhenFull` can stop a text stream early. + +`shouldStop` is **advisory**: the retainer cannot reach the upstream. The tool owns the actual stop — abort the HTTP body, break the scan, kill the process group. + +## `truncated` is a budget fact, never "incomplete" + +`truncated` means *the retainer omitted otherwise-available content because of a budget*. It does **not** mean the upstream was incomplete. Permission failures, skipped binary files, provider partial failures, unreadable candidates, and invalid UTF-8 stay in tool-domain fields — never folded into `truncated`. Conflating the two is the bug this library's naming most invites; keep them separate. + +## Bytes, not characters + +Text caps and `omittedBytes` count **bytes**, for process/body safety (a child's pipe and an HTTP body are byte streams). A chunk that straddles a codepoint is handled: `finish()` trims a partial codepoint at each cut so the returned text never introduces a replacement char at the boundary, and the two sides are decoded separately so a codepoint is never reconstructed across the omitted middle. Character- or line-level preview budgets are a separate, tool-owned concern. + +## Tool mappings + +Every current retention consumer maps to the library below; each row states whether it may stop its upstream early. A broad migration is out of scope for the library's first landing — these are the intended shapes. + +| Tool | Retainer & strategy | Stops upstream early? | Notes | +|---|---|---|---| +| `glob` | `ItemRetainer`, `head` + `stopWhenFull` | **Yes** — the `(maxItems+1)`th path is the probe; `shouldStop` kills ripgrep. | Path mapping, skipped candidates, `incomplete` stay outside. `omitted` is `atLeast`. | +| `grep` | `ItemRetainer`, `head` + `stopWhenFull` | **Yes** — cap is total matches; stop on the probe match. | Per-match preview truncation, then push a flat match; group + sort the retained subset *after* `finish()`. | +| `bash` | `TextRetainer`, `tail` or `headTail`, reads to completion | No — stopping would lose the true tail and risk pipe backpressure. | Executor still owns spill files, exit status, signal, timeout, background tasks. | +| `web_fetch` | `TextRetainer`, `head` (streaming provider) | Optional — a streaming body can stop; a decode-internally provider keeps its own cap. | The fetch result's `truncated` remains a provider/tool fact. | +| `web_search` | `ItemRetainer`, `head` | Post-hoc today (providers return arrays); a streaming provider can use `stopWhenFull`. | Standardizes the "sources capped" notice. | + +`read` is **intentionally out of scope for v1.** Its `read-render` helper owns a file-specific pagination contract — `offset`/`limit`, line numbers, `totalLines`, offset-out-of-range errors, per-line preview truncation, a byte cap over the selected window — which is a line-window renderer, not generic retention. A single `Omitted` count cannot represent both sides of a line window. + +## Usage shape + +```ts ignore-check +// glob: stop ripgrep the moment truncation is proven. +const retainer = new ItemRetainer({ kind: 'head', maxItems: globMaxResults, stop: 'stopWhenFull' }) +for await (const entry of candidates) { + const { shouldStop } = retainer.push(entry) + if (shouldStop) { killRipgrep(); break } // the tool owns the actual stop +} +const { items, truncated, omitted } = retainer.finish() + +// bash: keep a head + tail, read to process exit. +const out = new TextRetainer({ kind: 'headTail', headBytes: headCap, tailBytes: tailCap }) +child.stdout.on('data', (chunk: Buffer) => { out.push(chunk) }) // shouldStop ignored: must drain +const { text, omittedBytes } = out.finish() + +// A footer: the library standardizes the omission clause; the tool owns recovery words. +const footer = formatRetentionNotice( + { scope: 'grep', strategy: 'head', unit: 'items', limit: grepMaxMatches, kept: items.length, omitted }, + ({ kept }) => `Results capped at ${kept}. Narrow the pattern, path, or include to see more.`, +) +``` diff --git a/packages/util/retention/package.json b/packages/util/retention/package.json new file mode 100644 index 0000000000..2926144ace --- /dev/null +++ b/packages/util/retention/package.json @@ -0,0 +1,30 @@ +{ + "name": "@deepseek-ai/dsh-retention", + "description": "Zero-dependency bounded-retention primitive: ItemRetainer/TextRetainer + neutral notice helpers (what did we keep, what did we omit, may the caller stop reading)", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "cordis": "^4.0.0-rc.6" + }, + "devDependencies": { + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/util/retention/src/index.ts b/packages/util/retention/src/index.ts new file mode 100644 index 0000000000..bf754e2a8b --- /dev/null +++ b/packages/util/retention/src/index.ts @@ -0,0 +1,474 @@ +/** + * A dependency-light **retention** library: bounded model-facing output for + * tools that must cap how much context they return. A caller feeds items or + * text chunks into a bounded object, gets a per-push {@link PushDecision} about + * whether the upstream may stop, and later gets the retained content plus exact + * or partial omission metadata ({@link RetainedItems} / {@link RetainedText}). + * + * The library owns ONLY the mechanical question "what did we keep, what did we + * omit, and may the caller stop reading now?". Tool-specific code still owns + * business semantics: file grouping, line numbering, exit codes, provider error + * states, per-line preview truncation, spill files, and the model-facing prose. + * In particular {@link RetainedText.truncated}/{@link RetainedItems.truncated} + * means "the retainer omitted otherwise-available content because of a budget" — + * NOT "the upstream was incomplete". Permission failures, skipped binaries, + * provider partial failures, and unreadable candidates stay in tool-domain + * fields, never folded into `truncated`. + * + * This is deliberately a library, not a cordis service or plugin: it takes no + * `ctx`, registers nothing, and emits no events. The two retainers are the only + * stateful pieces and their state is per-instance (one accumulation), never + * cross-call. Tool packages import it directly when they need bounded output. + * + * The two retainers differ in resource model, which is why they are two names + * rather than one generic collector: + * - {@link ItemRetainer} bounds ordered logical units (paths, grep matches, + * search sources). `head` retention only in v1. With `stopWhenFull` it can ask + * the caller to stop the upstream after the first over-cap probe item. + * - {@link TextRetainer} bounds byte-oriented text streams (bash stdout/stderr, + * web bodies). `head` / `tail` / `headTail`, preserving UTF-8 boundaries at + * {@link TextRetainer.finish}. Only `head` can stop early; `tail`/`headTail` + * must read to the end to know the true suffix and exact omission. + * + * @module @deepseek-ai/dsh-retention + */ + +/** + * How much content the retainer omitted. + * + * `atLeast` is the early-stop shape: an {@link ItemRetainer}/{@link TextRetainer} + * with `stopWhenFull` sees the first unit/chunk past the cap, asks the caller to + * stop the upstream, and therefore knows only a LOWER bound — reporting an exact + * count there would be false precision when the true total may be much larger. + * `exact` is the read-to-end shape (`tail`, `headTail`, or `head` with + * `readToEnd`), where every unit/byte was observed. `unknown` is reserved for a + * caller that omits without a count; the retainers themselves never return it. + */ +export type Omitted = + | { kind: 'none' } + | { kind: 'exact'; count: number } + | { kind: 'atLeast'; count: number } + | { kind: 'unknown' } + +/** + * The caller receives this after each `push()`. + * + * `shouldStop` is ADVISORY, not automatic: the tool owns how to stop its upstream + * source — aborting an HTTP body, breaking a file scan, killing ripgrep. The + * retainer cannot reach the upstream; it only reports that keeping more would + * exceed the budget. A `readToEnd` / `tail` / `headTail` retainer never sets it + * (those must drain to the end). + */ +export interface PushDecision { + /** Was this whole unit / all of this chunk's bytes retained (nothing dropped)? */ + kept: boolean + /** Cumulative: has the retainer omitted anything due to the budget yet? */ + truncated: boolean + /** Advisory: keeping more would exceed the budget — the tool may stop its upstream. */ + shouldStop: boolean +} + +/** + * Final result for ordered logical units. + * + * `seen` means units OBSERVED by the retainer, not necessarily the total in the + * upstream source; with an early stop, the true total is intentionally unknown + * (hence {@link Omitted.atLeast}). `kept` is `items.length`, surfaced explicitly + * so a notice formatter need not re-count. + */ +export interface RetainedItems { + items: T[] + truncated: boolean + seen: number + kept: number + omitted: Omitted +} + +/** + * Final result for text streams. + * + * The returned `text` is safe to hand to a formatter: the retainer adds no + * tool-specific headers, exit markers, XML tags, or recovery instructions, and + * `omittedBytes` counts BYTES (not characters or lines) — text retention is + * byte-oriented for process/body safety. UTF-8 boundaries at each cut are + * preserved, so `text` never carries a replacement char introduced by the cut + * itself. + */ +export interface RetainedText { + text: string + truncated: boolean + omittedBytes: Omitted +} + +/** + * Whether a retainer asks the caller to stop the upstream once keeping more + * would exceed the budget (`stopWhenFull`), or must keep accepting input even + * after the retained output is full (`readToEnd`) — usually to preserve a true + * tail, count exact omission, or drain an upstream process to avoid pipe + * backpressure. Names avoid implementation phrases like "overflow". + */ +export type StopMode = 'stopWhenFull' | 'readToEnd' + +/** Item retention strategy. Only `head` in v1; windows/grouped budgets wait for a second consumer. */ +export type ItemRetentionStrategy = { + /** Keep the first `maxItems` units. Use for `glob`, `grep`, and web sources. */ + kind: 'head' + maxItems: number + stop: StopMode +} + +/** Text retention strategy: keep a prefix, a suffix, or both, counted in bytes. */ +export type TextRetentionStrategy = + | { + /** Keep the first `maxBytes` bytes. May stop an upstream body early. */ + kind: 'head' + maxBytes: number + stop: StopMode + } + | { + /** Keep the final `maxBytes` bytes. Requires reading to the end. */ + kind: 'tail' + maxBytes: number + } + | { + /** Keep a stable prefix and suffix, omitting the middle. Requires reading to the end. */ + kind: 'headTail' + headBytes: number + tailBytes: number + } + +/** + * A neutral, tool-agnostic description of one retention outcome — the input to + * {@link formatRetentionNotice}. It carries the mechanical facts (strategy, + * unit, limit, kept count, {@link Omitted}); the tool supplies the recovery + * words, because only the tool knows the recovery action ("narrow the pattern", + * "fetch a more specific URL", "read the spill file"). + */ +export interface RetentionNotice { + /** Tool/scope label, e.g. `grep`, `web_fetch`, `bash stdout`. */ + scope: string + strategy: 'head' | 'tail' | 'headTail' + unit: 'items' | 'bytes' | 'chars' | 'lines' + limit: number | { head: number; tail: number } + kept: number + omitted: Omitted +} + +/** Assert a budget field is a non-negative integer (the retainer request contract). */ +function assertBudget(value: number, name: string): void { + if (!Number.isInteger(value) || value < 0) { + throw new Error(`${name} must be a non-negative integer`) + } +} + +/** + * Bounds an ordered stream of logical units, keeping the first `maxItems` + * ({@link ItemRetentionStrategy} `head`). `push()` reports, per unit, whether it + * was kept and — under `stopWhenFull` — whether the caller should stop the + * upstream now that the first over-cap probe unit has been seen. + * + * Grouping, sorting, path mapping, per-unit preview truncation, and any + * `incomplete` state stay OUTSIDE the retainer: it counts and keeps, nothing + * more. The caller pushes already-shaped units and, after {@link finish}, + * groups/sorts the retained subset itself. + */ +export class ItemRetainer { + private readonly maxItems: number + private readonly stop: StopMode + private readonly items: T[] = [] + private seen = 0 + private omittedCount = 0 + + /** @param strategy Head strategy: `maxItems` (non-negative integer) and the {@link StopMode}. */ + constructor(strategy: ItemRetentionStrategy) { + assertBudget(strategy.maxItems, 'maxItems') + this.maxItems = strategy.maxItems + this.stop = strategy.stop + } + + /** + * Offer one unit. Kept when the retainer is below `maxItems`; otherwise dropped + * and counted as omitted. Under `stopWhenFull` the first dropped unit is the + * probe: `shouldStop` is `true` so the caller can kill ripgrep / cancel the + * stream, and the final {@link Omitted} stays `atLeast` (the true total is + * unknown). Under `readToEnd` the caller keeps pushing so omission is `exact`. + * + * @param item The already-shaped logical unit (path, flat match, source). + * @returns The per-push {@link PushDecision}. + */ + push(item: T): PushDecision { + this.seen++ + if (this.items.length < this.maxItems) { + // Reached only below the cap, before any omission (items only grow, the + // cap is fixed), so nothing has been dropped yet: truncated is always false. + this.items.push(item) + return { kept: true, truncated: false, shouldStop: false } + } + this.omittedCount++ + return { + kept: false, + truncated: true, + // Only ask to stop when the caller opted into it; readToEnd must keep + // draining to reach an exact omission count. + shouldStop: this.stop === 'stopWhenFull', + } + } + + /** + * Finalize and report what was kept and omitted. `omitted` is `atLeast` under + * `stopWhenFull` (a lower bound — the caller was asked to stop before the true + * total was known) and `exact` under `readToEnd`. + * + * @returns The {@link RetainedItems} snapshot (safe to group/sort downstream). + */ + finish(): RetainedItems { + const truncated = this.omittedCount > 0 + return { + items: this.items, + truncated, + seen: this.seen, + kept: this.items.length, + omitted: truncated + ? { kind: this.stop === 'stopWhenFull' ? 'atLeast' : 'exact', count: this.omittedCount } + : { kind: 'none' }, + } + } +} + +const encoder = new TextEncoder() +const decoder = new TextDecoder() // utf-8, non-fatal: internal malformed bytes → U+FFFD + +/** + * Drop a trailing incomplete UTF-8 sequence so a prefix cut never emits a + * replacement char at the boundary. Walks back over continuation bytes + * (`10xxxxxx`) to the lead byte; if fewer bytes follow it than the lead byte's + * length declares, the sequence is incomplete and is trimmed. A complete tail, + * or a run too long/short to be a valid lead, is returned untouched (any + * genuinely malformed interior is left for the decoder to replace). + */ +function trimTrailingPartialUtf8(bytes: Uint8Array): Uint8Array { + let i = bytes.length - 1 + // Continuation bytes are 0b10xxxxxx; scan back at most 3 (max sequence is 4). + // Indices are bounds-checked by the loop guard, so the reads are in range (a + // cast, not `!`, per the repo's no-non-null-assertion rule). + while (i >= 0 && ((bytes[i] as number) & 0xc0) === 0x80 && bytes.length - i <= 3) i-- + if (i < 0) return bytes + const lead = bytes[i] as number + const expected = lead < 0x80 ? 1 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : lead < 0xf8 ? 4 : 0 + // expected 0 → not a lead byte (stray continuation / invalid): leave it. + if (expected === 0) return bytes + return bytes.length - i < expected ? bytes.subarray(0, i) : bytes +} + +/** + * Drop leading continuation bytes (`10xxxxxx`) so a suffix cut starts on a + * lead/ASCII byte instead of mid-codepoint. + */ +function trimLeadingContinuationUtf8(bytes: Uint8Array): Uint8Array { + let i = 0 + // i < length guards the read; cast rather than `!` (no-non-null-assertion). + while (i < bytes.length && ((bytes[i] as number) & 0xc0) === 0x80) i++ + return bytes.subarray(i) +} + +/** + * Bounds a byte-oriented text stream, keeping a prefix, a suffix, or both + * ({@link TextRetentionStrategy}). All three strategies share one prefix/suffix + * accumulator: `head` is prefix-only, `tail` is suffix-only, `headTail` is both. + * Only `head` with `stopWhenFull` sets `shouldStop`; `tail`/`headTail` must read + * to the end to know the true suffix and the exact omitted byte count. + * + * Bytes, not characters: caps and `omittedBytes` are byte counts for process/ + * body safety. Chunks that straddle a codepoint are handled — {@link finish} + * trims a partial codepoint at each cut so the returned text never introduces a + * replacement char at the boundary. The retainer holds at most + * `prefixCap + tailBytes + one chunk` in memory (old suffix chunks are dropped + * as they slide out), so a large stream does not accumulate unbounded. + */ +export class TextRetainer { + private readonly prefixCap: number + private readonly suffixCap: number + private readonly allowStop: boolean + private readonly prefixChunks: Uint8Array[] = [] + private prefixHeld = 0 + private readonly suffixChunks: Uint8Array[] = [] + private suffixHeld = 0 + private total = 0 + + /** @param strategy One of the {@link TextRetentionStrategy} shapes; byte budgets must be non-negative integers. */ + constructor(strategy: TextRetentionStrategy) { + switch (strategy.kind) { + case 'head': + assertBudget(strategy.maxBytes, 'maxBytes') + this.prefixCap = strategy.maxBytes + this.suffixCap = 0 + this.allowStop = strategy.stop === 'stopWhenFull' + break + case 'tail': + assertBudget(strategy.maxBytes, 'maxBytes') + this.prefixCap = 0 + this.suffixCap = strategy.maxBytes + this.allowStop = false + break + case 'headTail': + assertBudget(strategy.headBytes, 'headBytes') + assertBudget(strategy.tailBytes, 'tailBytes') + this.prefixCap = strategy.headBytes + this.suffixCap = strategy.tailBytes + this.allowStop = false + break + } + } + + /** + * Offer one chunk (a `Uint8Array`, or a `string` encoded as UTF-8). Prefix + * bytes fill up to the prefix cap then stop; suffix bytes roll so only the + * last `suffixCap` bytes are retained. `kept` is `true` only when no byte of + * this chunk was dropped. Under `head` + `stopWhenFull`, `shouldStop` turns + * `true` on the chunk that first drops a byte (the caller may then abort the + * body); other strategies never set it. + * + * @param chunk The next bytes of the stream (`Uint8Array` or UTF-8 `string`). + * @returns The per-push {@link PushDecision}. + */ + push(chunk: Uint8Array | string): PushDecision { + const bytes = typeof chunk === 'string' ? encoder.encode(chunk) : chunk + const before = this.total + this.total += bytes.length + + // Prefix: take only up to the cap; the rest of this chunk is "not prefixed". + const room = this.prefixCap - this.prefixHeld + const take = Math.max(0, Math.min(room, bytes.length)) + if (take > 0) { + this.prefixChunks.push(bytes.subarray(0, take)) + this.prefixHeld += take + } + + // Suffix: append the whole chunk, then drop whole leading chunks that have + // fully slid out of the last `suffixCap` bytes (bounded memory). + if (this.suffixCap > 0) { + this.suffixChunks.push(bytes) + this.suffixHeld += bytes.length + let head = this.suffixChunks[0] + while (head !== undefined && this.suffixHeld - head.length >= this.suffixCap) { + this.suffixChunks.shift() + this.suffixHeld -= head.length + head = this.suffixChunks[0] + } + } + + // Dropped = bytes that no side can keep. Compute cumulative omission the + // SAME way finish() does (via omittedAt), so push and finish never disagree; + // per-push we only need whether THIS chunk pushed the total past what the + // two caps hold, and — for head+stopWhenFull — whether to stop. + const droppedThisChunk = this.omittedAt(this.total) > this.omittedAt(before) + return { + kept: !droppedThisChunk, + truncated: this.omittedAt(this.total) > 0, + shouldStop: this.allowStop && droppedThisChunk, + } + } + + /** Bytes omitted once `total` bytes have been seen: `total − keptPrefix − keptSuffix`. */ + private omittedAt(total: number): number { + const prefixLen = Math.min(total, this.prefixCap) + const suffixLen = Math.min(total - prefixLen, this.suffixCap) + return total - prefixLen - suffixLen + } + + /** + * Finalize: decode the retained prefix and suffix (each trimmed to a UTF-8 + * boundary at its cut) and report the exact or lower-bound omitted byte count. + * `head` + `stopWhenFull` yields `atLeast` (a lower bound — the caller was + * asked to stop before the true size was known); every other case reads to the + * end and yields `exact`. + * + * @returns The {@link RetainedText} snapshot (safe to hand to a formatter). + */ + finish(): RetainedText { + const prefixLen = Math.min(this.total, this.prefixCap) + const suffixLen = Math.min(this.total - prefixLen, this.suffixCap) + const omitted = this.omittedAt(this.total) + const truncated = omitted > 0 + + // A cut exists at the prefix end only if content followed it (moved to the + // suffix or omitted); likewise the suffix start is a cut only if content + // preceded it. When the whole stream fits in one side, pass bytes through + // untrimmed so valid output is never altered. + let prefix = concat(this.prefixChunks) + if (suffixLen > 0 || omitted > 0) prefix = trimTrailingPartialUtf8(prefix) + + const suffixBuf = concat(this.suffixChunks) + let suffix = suffixBuf.subarray(this.suffixHeld - suffixLen) + if (prefixLen > 0 || omitted > 0) suffix = trimLeadingContinuationUtf8(suffix) + + return { + // Decode the two sides separately so a codepoint is never reconstructed + // across the omitted middle. + text: decoder.decode(prefix) + decoder.decode(suffix), + truncated, + omittedBytes: truncated + ? { kind: this.allowStop ? 'atLeast' : 'exact', count: omitted } + : { kind: 'none' }, + } + } +} + +/** Concatenate chunks into one contiguous buffer (their exact total length). */ +function concat(chunks: readonly Uint8Array[]): Uint8Array { + let length = 0 + for (const chunk of chunks) length += chunk.length + const out = new Uint8Array(length) + let offset = 0 + for (const chunk of chunks) { + out.set(chunk, offset) + offset += chunk.length + } + return out +} + +/** + * Standardized, false-precision-safe wording for one {@link Omitted} value — + * the "may standardize omission wording" half the library owns. `exact` prints + * the count (`Omitted 3 items`); `atLeast`/`unknown` print NO count, because an + * early stop knows only that more was dropped, not how much (claiming "omitted + * 1" when the true total may be huge is the false-precision trap the `atLeast` + * variant exists to avoid). `none` is the empty string. + * + * @param omitted The omission metadata from a retainer result. + * @param unit The noun for the omitted quantity (`items`, `bytes`, `chars`, `lines`). + * @returns A neutral clause (no trailing space), or `''` when nothing was omitted. + */ +export function describeOmitted(omitted: Omitted, unit: RetentionNotice['unit']): string { + switch (omitted.kind) { + case 'none': + return '' + case 'exact': + return `Omitted ${omitted.count} ${unit}.` + case 'atLeast': + case 'unknown': + return `More ${unit} were omitted.` + } +} + +/** + * Turn a {@link RetentionNotice} into a one-line footer: the library-owned + * standardized omission clause ({@link describeOmitted}) followed by the tool's + * own recovery guidance. The library never owns recovery words — only the tool + * knows the action ("narrow the pattern", "fetch a more specific URL", "read the + * spill file") — so `recovery` supplies them and receives the full notice to + * phrase from (`kept`, `limit`, `omitted`, …). Either half may be empty; the two + * are joined with a single space. + * + * @param notice The neutral retention outcome. + * @param recovery Tool-supplied guidance builder; receives the notice, returns a sentence (or `''`). + * @returns The combined footer line. + */ +export function formatRetentionNotice( + notice: RetentionNotice, + recovery: (notice: RetentionNotice) => string, +): string { + return [describeOmitted(notice.omitted, notice.unit), recovery(notice)] + .filter(part => part.length > 0) + .join(' ') +} diff --git a/packages/util/retention/tests/retention.spec.ts b/packages/util/retention/tests/retention.spec.ts new file mode 100644 index 0000000000..b3401cf50a --- /dev/null +++ b/packages/util/retention/tests/retention.spec.ts @@ -0,0 +1,348 @@ +import { describe, expect, it } from 'vitest' +import { + describeOmitted, + formatRetentionNotice, + ItemRetainer, + type Omitted, + type RetentionNotice, + TextRetainer, +} from '@deepseek-ai/dsh-retention' + +/** Decode a RetainedText via a round-trip helper for readable UTF-8 assertions. */ +const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s) + +describe('ItemRetainer — head, stopWhenFull (glob/grep early stop)', () => { + it('keeps the first maxItems and asks to stop on the probe item', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 2, stop: 'stopWhenFull' }) + expect(r.push('a')).toEqual({ kept: true, truncated: false, shouldStop: false }) + expect(r.push('b')).toEqual({ kept: true, truncated: false, shouldStop: false }) + // The (maxItems + 1)th valid item is the probe: not retained, sets truncated, + // and shouldStop tells the caller to kill the upstream. + expect(r.push('c')).toEqual({ kept: false, truncated: true, shouldStop: true }) + + const result = r.finish() + expect(result.items).toEqual(['a', 'b']) + expect(result.kept).toBe(2) + expect(result.seen).toBe(3) + expect(result.truncated).toBe(true) + // Early stop knows only a lower bound, never an exact total. + expect(result.omitted).toEqual({ kind: 'atLeast', count: 1 }) + }) + + it('reports none when everything fits', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 3, stop: 'stopWhenFull' }) + r.push(1) + r.push(2) + const result = r.finish() + expect(result.items).toEqual([1, 2]) + expect(result.truncated).toBe(false) + expect(result.omitted).toEqual({ kind: 'none' }) + }) +}) + +describe('ItemRetainer — head, readToEnd (exact omission)', () => { + it('keeps draining past the cap and reports an exact omitted count', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 1, stop: 'readToEnd' }) + expect(r.push('a')).toEqual({ kept: true, truncated: false, shouldStop: false }) + // readToEnd never asks to stop — the caller must keep pushing to count exactly. + expect(r.push('b')).toEqual({ kept: false, truncated: true, shouldStop: false }) + expect(r.push('c')).toEqual({ kept: false, truncated: true, shouldStop: false }) + + const result = r.finish() + expect(result.items).toEqual(['a']) + expect(result.seen).toBe(3) + expect(result.omitted).toEqual({ kind: 'exact', count: 2 }) + }) +}) + +describe('ItemRetainer — zero budget', () => { + it('keeps nothing; first item is the probe under stopWhenFull', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 0, stop: 'stopWhenFull' }) + expect(r.push('a')).toEqual({ kept: false, truncated: true, shouldStop: true }) + const result = r.finish() + expect(result.items).toEqual([]) + expect(result.kept).toBe(0) + expect(result.omitted).toEqual({ kind: 'atLeast', count: 1 }) + }) + + it('rejects a non-integer / negative maxItems', () => { + expect(() => new ItemRetainer({ kind: 'head', maxItems: -1, stop: 'readToEnd' })) + .toThrow(/maxItems must be a non-negative integer/) + expect(() => new ItemRetainer({ kind: 'head', maxItems: 1.5, stop: 'readToEnd' })) + .toThrow(/maxItems must be a non-negative integer/) + }) +}) + +describe('TextRetainer — head, stopWhenFull (early body stop)', () => { + it('keeps the prefix and asks to stop on the overflowing chunk', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 5, stop: 'stopWhenFull' }) + expect(r.push('abc')).toEqual({ kept: true, truncated: false, shouldStop: false }) + // 'de' fills the cap exactly (5 bytes) — still fully kept. + expect(r.push('de')).toEqual({ kept: true, truncated: false, shouldStop: false }) + // 'fgh' is wholly dropped: kept:false, and stopWhenFull → shouldStop. + expect(r.push('fgh')).toEqual({ kept: false, truncated: true, shouldStop: true }) + + const result = r.finish() + expect(result.text).toBe('abcde') + expect(result.truncated).toBe(true) + // Early stop: a lower bound, not an exact size. + expect(result.omittedBytes).toEqual({ kind: 'atLeast', count: 3 }) + }) + + it('flags a partially-dropped chunk as not fully kept', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 4, stop: 'stopWhenFull' }) + r.push('ab') + // 'cde' straddles the cap: 'c','d' fit, 'e' drops → kept:false, shouldStop. + expect(r.push('cde')).toEqual({ kept: false, truncated: true, shouldStop: true }) + expect(r.finish().text).toBe('abcd') + }) +}) + +describe('TextRetainer — head, readToEnd (exact omission)', () => { + it('keeps the prefix, drains the rest, and counts exactly', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + r.push('abc') + expect(r.push('defg')).toEqual({ kept: false, truncated: true, shouldStop: false }) + const result = r.finish() + expect(result.text).toBe('abc') + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 4 }) + }) +}) + +describe('TextRetainer — tail (exact omission, reads to end)', () => { + it('keeps the final maxBytes and reports exact omission', () => { + const r = new TextRetainer({ kind: 'tail', maxBytes: 4 }) + // tail never asks to stop — it must read to the end to know the true suffix. + expect(r.push('hello')).toEqual({ kept: false, truncated: true, shouldStop: false }) + r.push('world') + const result = r.finish() + expect(result.text).toBe('orld') // last 4 bytes of 'helloworld' + expect(result.truncated).toBe(true) + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 6 }) + }) + + it('keeps everything when the stream is under the cap', () => { + const r = new TextRetainer({ kind: 'tail', maxBytes: 100 }) + r.push('short') + const result = r.finish() + expect(result.text).toBe('short') + expect(result.truncated).toBe(false) + expect(result.omittedBytes).toEqual({ kind: 'none' }) + }) + + it('drops old chunks as they slide out of the tail window', () => { + const r = new TextRetainer({ kind: 'tail', maxBytes: 3 }) + for (const c of ['11', '22', '33', '44']) r.push(c) + // Only the final 3 bytes survive; earlier whole chunks are dropped. + expect(r.finish().text).toBe('344') + }) +}) + +describe('TextRetainer — headTail (prefix + suffix, omit the middle)', () => { + it('keeps a stable head and tail, omitting the middle exactly', () => { + const r = new TextRetainer({ kind: 'headTail', headBytes: 3, tailBytes: 3 }) + r.push('abcdefghij') // 10 bytes: head 'abc', tail 'hij', middle 'defg' omitted + const result = r.finish() + expect(result.text).toBe('abchij') + expect(result.truncated).toBe(true) + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 4 }) + }) + + it('does not double-count when head+tail cover the whole stream', () => { + const r = new TextRetainer({ kind: 'headTail', headBytes: 3, tailBytes: 3 }) + r.push('abcdef') // exactly head(3) + tail(3), nothing omitted + const result = r.finish() + expect(result.text).toBe('abcdef') + expect(result.truncated).toBe(false) + expect(result.omittedBytes).toEqual({ kind: 'none' }) + }) +}) + +describe('TextRetainer — zero budgets', () => { + it('head maxBytes 0 keeps nothing and stops on first byte (stopWhenFull)', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 0, stop: 'stopWhenFull' }) + expect(r.push('x')).toEqual({ kept: false, truncated: true, shouldStop: true }) + const result = r.finish() + expect(result.text).toBe('') + expect(result.omittedBytes).toEqual({ kind: 'atLeast', count: 1 }) + }) + + it('an empty stream omits nothing', () => { + const r = new TextRetainer({ kind: 'headTail', headBytes: 2, tailBytes: 2 }) + const result = r.finish() + expect(result.text).toBe('') + expect(result.truncated).toBe(false) + expect(result.omittedBytes).toEqual({ kind: 'none' }) + }) + + it('rejects non-integer / negative byte budgets', () => { + expect(() => new TextRetainer({ kind: 'head', maxBytes: -1, stop: 'readToEnd' })) + .toThrow(/maxBytes must be a non-negative integer/) + expect(() => new TextRetainer({ kind: 'tail', maxBytes: 2.5 })) + .toThrow(/maxBytes must be a non-negative integer/) + expect(() => new TextRetainer({ kind: 'headTail', headBytes: -1, tailBytes: 2 })) + .toThrow(/headBytes must be a non-negative integer/) + expect(() => new TextRetainer({ kind: 'headTail', headBytes: 2, tailBytes: 1.1 })) + .toThrow(/tailBytes must be a non-negative integer/) + }) +}) + +describe('TextRetainer — UTF-8 boundary handling', () => { + it('trims a partial codepoint at the head cut instead of emitting U+FFFD', () => { + // '€' is 3 bytes (E2 82 AC). A 2-byte head cap keeps 'a' (61) + the first + // byte of '€' (E2); that partial lead byte must be trimmed, not decoded to + // a replacement char. + const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + r.push('a€b') // bytes: 61 E2 82 AC 62 + const result = r.finish() + expect(result.text).toBe('a') // partial '€' dropped, no U+FFFD + expect(result.text).not.toContain('�') + // Omission counts BYTES not kept by retention: 5 total − 2 prefix = 3. + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 3 }) + }) + + it('trims a leading partial codepoint at the tail cut', () => { + // Tail cap 2 over 'a€b' (5 bytes) keeps AC 62 — AC is a continuation byte + // (the middle of '€'); the leading continuation byte is dropped so the tail + // begins on a boundary. + const r = new TextRetainer({ kind: 'tail', maxBytes: 2 }) + r.push('a€b') + const result = r.finish() + expect(result.text).toBe('b') // partial '€' at the front dropped + expect(result.text).not.toContain('�') + }) + + it('preserves a whole multibyte codepoint that fits exactly', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + r.push('€x') // '€' is exactly 3 bytes + expect(r.finish().text).toBe('€') + }) + + it('does not reconstruct a codepoint across the omitted middle', () => { + // headBytes ends mid-'€' and tailBytes starts mid-another '€'; neither cut + // may glue a valid codepoint across the gap. + const r = new TextRetainer({ kind: 'headTail', headBytes: 2, tailBytes: 2 }) + r.push('€€€') // 9 bytes + const result = r.finish() + expect(result.text).not.toContain('�') + expect(result.truncated).toBe(true) + }) + + it('accepts a raw Uint8Array chunk', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + r.push(utf8('xy')) + r.push(utf8('z')) + expect(r.finish().text).toBe('xy') + }) + + it('trims a partial 2-byte codepoint at the head cut', () => { + // 'é' is 2 bytes (C3 A9). A 2-byte head cap over 'aé' keeps 'a' (61) + the + // lead byte of 'é' (C3) — an incomplete 2-byte sequence to trim. + const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + r.push('aé') // bytes: 61 C3 A9 + const result = r.finish() + expect(result.text).toBe('a') + expect(result.text).not.toContain('�') + }) + + it('trims a partial 4-byte codepoint (emoji) at the head cut', () => { + // '😀' is 4 bytes (F0 9F 98 80). A 3-byte head cap keeps 'a' + the first two + // bytes of the emoji — an incomplete 4-byte sequence that must be trimmed. + const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + r.push('a😀') // bytes: 61 F0 9F 98 80 + const result = r.finish() + expect(result.text).toBe('a') + expect(result.text).not.toContain('�') + }) + + it('keeps a whole 4-byte codepoint that fits exactly', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 4, stop: 'readToEnd' }) + r.push('😀x') + expect(r.finish().text).toBe('😀') + }) + + it('leaves a head cut ending on a stray continuation run untouched', () => { + // A cut whose trailing bytes are ALL continuation bytes with no lead in + // reach is not a trimmable incomplete sequence — the trimmer bails (no lead + // byte found) and leaves them for the non-fatal decoder to replace. + const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + // 0x80 0x80 are bare continuation bytes; 'z' follows so the head keeps just + // the two continuation bytes and the cut lands right after them. + r.push(new Uint8Array([0x80, 0x80, 0x7a])) + const result = r.finish() + // The trimmer did not throw and did not eat the bytes as a partial sequence; + // only the trailing 'z' is omitted by the 2-byte cap. + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 1 }) + }) + + it('leaves a head cut ending on an invalid lead byte untouched', () => { + // 0xF8 is not a valid UTF-8 lead byte (only 0x00–0xF7 lead). The trimmer + // recognizes it as "not a lead" (expected length 0) and leaves the byte in + // place rather than trimming a phantom partial sequence. + const r = new TextRetainer({ kind: 'head', maxBytes: 1, stop: 'readToEnd' }) + r.push(new Uint8Array([0xf8, 0x61])) // 0xF8 kept, 'a' dropped by the 1-byte cap + const result = r.finish() + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 1 }) + }) +}) + +describe('describeOmitted — false precision safety', () => { + it('prints an exact count for exact omission', () => { + expect(describeOmitted({ kind: 'exact', count: 3 }, 'items')).toBe('Omitted 3 items.') + expect(describeOmitted({ kind: 'exact', count: 12 }, 'bytes')).toBe('Omitted 12 bytes.') + }) + + it('prints NO count for atLeast (early stop) and unknown', () => { + // The whole point of atLeast: never claim "omitted 1" when the true count is + // unknown. Both atLeast and unknown collapse to a countless clause. + expect(describeOmitted({ kind: 'atLeast', count: 1 }, 'items')).toBe('More items were omitted.') + expect(describeOmitted({ kind: 'unknown' }, 'lines')).toBe('More lines were omitted.') + }) + + it('returns empty string when nothing was omitted', () => { + expect(describeOmitted({ kind: 'none' }, 'chars')).toBe('') + }) +}) + +describe('formatRetentionNotice', () => { + const notice = (omitted: Omitted): RetentionNotice => ({ + scope: 'grep', + strategy: 'head', + unit: 'items', + limit: 100, + kept: 100, + omitted, + }) + + it('joins the standardized omission clause with the tool recovery guidance', () => { + const out = formatRetentionNotice( + notice({ kind: 'atLeast', count: 1 }), + ({ kept }) => `Results capped at ${kept}. Narrow the pattern, path, or include to see more.`, + ) + expect(out).toBe('More items were omitted. Results capped at 100. Narrow the pattern, path, or include to see more.') + }) + + it('omits the empty half when nothing was omitted', () => { + const out = formatRetentionNotice(notice({ kind: 'none' }), () => 'Recovery text.') + expect(out).toBe('Recovery text.') + }) + + it('omits the empty half when the tool supplies no recovery text', () => { + const out = formatRetentionNotice(notice({ kind: 'exact', count: 2 }), () => '') + expect(out).toBe('Omitted 2 items.') + }) + + it('passes the full notice to the recovery builder (limit as a head/tail pair)', () => { + const headTail: RetentionNotice = { + scope: 'bash stdout', + strategy: 'headTail', + unit: 'bytes', + limit: { head: 2_000, tail: 2_000 }, + kept: 4_000, + omitted: { kind: 'exact', count: 500 }, + } + const out = formatRetentionNotice(headTail, n => + typeof n.limit === 'object' ? `Kept ${n.limit.head}B head + ${n.limit.tail}B tail.` : '') + expect(out).toBe('Omitted 500 bytes. Kept 2000B head + 2000B tail.') + }) +}) diff --git a/packages/util/retention/tsconfig.json b/packages/util/retention/tsconfig.json new file mode 100644 index 0000000000..749cb0208e --- /dev/null +++ b/packages/util/retention/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": [ + "src" + ], + "references": [] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 54050bd0f4..3bf60ac60a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -963,6 +963,12 @@ importers: specifier: ^4.0.0-rc.6 version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/util/retention: + devDependencies: + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/util/timeout: devDependencies: cordis: diff --git a/tsconfig.build.json b/tsconfig.build.json index ebf8ffef14..8f1a869d91 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -12,6 +12,7 @@ { "path": "./vendor/logger-console" }, { "path": "./packages/util/brand" }, { "path": "./packages/util/timeout" }, + { "path": "./packages/util/retention" }, { "path": "./packages/llm/llm" }, { "path": "./packages/core/session" }, { "path": "./packages/session-persistence/session-persistence" }, diff --git a/tsconfig.json b/tsconfig.json index 49cce594dd..4a4cc1668f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -23,6 +23,7 @@ { "path": "./vendor/logger-console" }, { "path": "./packages/util/brand" }, { "path": "./packages/util/timeout" }, + { "path": "./packages/util/retention" }, { "path": "./packages/llm/llm" }, { "path": "./packages/core/session" }, { "path": "./packages/session-persistence/session-persistence" },