From a4a9900be1c6f597a92a5955fe2768ca40314756 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 13:53:51 +0800 Subject: [PATCH] simplify retention omitted metadata --- ...026-07-06-tool-result-retention-library.md | 49 +++----- .../2026-07-08-tool-output-spill-files.md | 6 +- packages/util/README.md | 2 +- packages/util/retention/README.md | 49 ++++---- packages/util/retention/package.json | 2 +- packages/util/retention/src/index.ts | 104 ++++------------ .../util/retention/tests/retention.spec.ts | 111 ++++++++---------- 7 files changed, 119 insertions(+), 204 deletions(-) diff --git a/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md b/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md index 7ea954b0a2..344b50753a 100644 --- a/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md +++ b/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md @@ -4,9 +4,9 @@ Status: implemented ## Problem -Several model-facing tools already bound the amount of context they return, but each one owns a different local mechanism and vocabulary: bash keeps a tail plus spill files, web search caps source lists, web fetch caps body content, and `glob` / `grep` discovery needs `cap + 1` early stop while reading ripgrep output. A single post-hoc `truncate(text)` helper cannot cover those cases: by the time `grep` or `glob` has collected every result, the expensive traversal has already happened and the process may have emitted more output than the harness intended to buffer. +Several model-facing tools already bound the amount of context they return, but each one owns a different local mechanism and vocabulary: bash keeps a tail plus spill files, web search caps source lists, web fetch caps body content, and `glob` / `grep` discovery needs an inline first page while keeping exact omission metadata for the full result set. A single `truncate(text)` helper cannot cover those cases: item tools need item counts and grouping outside the primitive, while text tools need byte budgets and UTF-8-safe head/tail cuts. -The shared abstraction the tools need is **retention**, not generic collection. A caller feeds items or text chunks into a bounded object, receives a per-push decision about whether the upstream can stop, and later receives the retained content plus exact or partial omission metadata. Tool-specific code still owns business semantics: file grouping, line numbering, exit codes, provider error states, and model-facing prose. The common library owns only the mechanical question "what did we keep, what did we omit, and may the caller stop reading now?" +The shared abstraction the tools need is **retention**, not generic collection. A caller feeds items or text chunks into a bounded object and later receives the retained content plus exact omission metadata. Tool-specific code still owns business semantics: file grouping, line numbering, exit codes, provider error states, spill files, and model-facing prose. The common library owns only the mechanical question "what did we keep, and what did we omit?" ## Decision @@ -17,38 +17,27 @@ The library has two independent retainers: - `ItemRetainer` handles ordered logical units such as paths, grep matches, or search sources. It supports `head` retention only in v1. - `TextRetainer` handles byte-oriented text streams such as bash stdout/stderr or web response bodies. It supports `head`, `tail`, and `headTail` retention while preserving UTF-8 boundaries at `finish()`. -Both retainers return a `PushDecision` after each `push()`. `shouldStop` is the critical control-flow field: `glob` / `grep` use it to kill ripgrep once the probe item proves truncation, while bash ignores it because tail/head-tail retention must read to process exit to know the true suffix and to avoid pipe backpressure. +Both retainers return a small `PushDecision` after each `push()` so callers can tell whether that unit/chunk was fully retained and whether the accumulated result is now truncated. Omission counts are exact because callers keep feeding every observed item/chunk. ```ts ignore-check /** * How much content the retainer omitted. * - * `atLeast` is the early-stop shape: `glob` / `grep` see the first item past the cap, - * stop the upstream process, and know only that at least one item was omitted. + * `unknown` is reserved for callers that omit without a count; the retainers + * themselves return `none` or `exact`. */ type Omitted = | { kind: 'none' } | { kind: 'exact'; count: number } - | { kind: 'atLeast'; count: number } | { kind: 'unknown' } -/** - * The caller receives this after each `push()`. - * - * `shouldStop` is advisory, not automatic: the tool owns how to stop its upstream - * source, such as aborting an HTTP body, breaking a file scan, or killing ripgrep. - */ interface PushDecision { kept: boolean truncated: boolean - shouldStop: boolean } /** * Final result for ordered logical units. - * - * `seen` means units observed by the retainer, not necessarily total units in the - * upstream source; with early stop, total is intentionally unknown. */ interface RetainedItems { items: T[] @@ -73,25 +62,21 @@ interface RetainedText { ### Strategies -The strategy names are caller-facing and avoid implementation phrases such as "overflow". `stopWhenFull` means the retainer should ask the caller to stop once keeping more would exceed the budget. `readToEnd` means the retainer must keep accepting input even after the retained output is full, usually to preserve a true tail, count exact omission, or drain an upstream process. +Item retention supports a head window. Text retention supports head, tail, and headTail byte windows. ```ts ignore-check -type StopMode = 'stopWhenFull' | 'readToEnd' - type ItemRetentionStrategy = | { /** Keep the first `maxItems` units. Use for `glob`, `grep`, and web sources. */ kind: 'head' maxItems: number - stop: StopMode } type TextRetentionStrategy = | { - /** Keep the first `maxBytes` bytes. May stop an upstream body early. */ + /** Keep the first `maxBytes` bytes. */ kind: 'head' maxBytes: number - stop: StopMode } | { /** Keep the final `maxBytes` bytes. Requires reading to the end. */ @@ -112,15 +97,15 @@ type TextRetentionStrategy = `FsGlobEntry` and `FlatGrepMatch` below are the intended discovery-tool item shapes, not existing retention-library exports. `FsGlobEntry` is one backend-derived path, and `FlatGrepMatch` is one ungrouped grep match before the backend groups retained matches by file. -`glob` uses `ItemRetainer` with `{ kind: 'head', maxItems: globMaxResults, stop: 'stopWhenFull' }` inside the backend or executor that is consuming traversal output. The `(maxItems + 1)`th valid path is the probe item: it is not retained, it sets `truncated: true`, and `shouldStop: true` tells the caller to stop ripgrep, cancel a remote stream, or stop whatever upstream is producing candidates. `omitted` is `{ kind: 'atLeast', count: 1 }` because the traversal stopped before the full count was known. Path mapping, skipped candidates, and `incomplete` stay outside the retainer. +`glob` uses `ItemRetainer` with `{ kind: 'head', maxItems: globMaxResults }` after collecting the full sorted path list. The tool keeps the retained first page inline and may save the full list through the spill seam. Path mapping, skipped candidates, and `incomplete` stay outside the retainer. -`grep` uses `ItemRetainer` with `{ kind: 'head', maxItems: grepMaxMatches, stop: 'stopWhenFull' }` before grouping. The backend parses a ripgrep match record, maps the path, applies per-line preview truncation, then pushes a flat match. After `finish()`, the backend groups retained matches by file and sorts the returned subset. Grouping is not part of the retainer because the cap is total matches, not files; per-match preview truncation and `incomplete` are also separate from result-level retention. +`grep` uses `ItemRetainer` with `{ kind: 'head', maxItems: grepMaxMatches }` before grouping. The executor parses ripgrep output, maps paths, applies per-line preview truncation, and pushes flat matches. After `finish()`, the tool groups retained matches by file and can save the full match list through the spill seam when the inline result is capped. Grouping is not part of the retainer because the cap is total matches, not files; per-match preview truncation and `incomplete` are also separate from result-level retention. -`bash` uses `TextRetainer` with `tail` or `headTail` and reads to process completion. It does not stop when full: stopping the read would lose the real tail and can create pipe backpressure. The bash executor still owns spill files, exit status, signal, timeout, and background-task behavior; the retention helper only replaces ad hoc in-memory head/tail accounting where that behavior is desired. Long-running task ownership remains orthogonal to the [generic long-running tool runtime](../../proposed/architecture/2026-06-20-generic-long-running-tool-runtime.md) proposal. +`bash` can use `TextRetainer` with `tail` or `headTail` and reads to process completion. The bash executor still owns spill files, exit status, signal, timeout, and background-task behavior; the retention helper only replaces ad hoc in-memory head/tail accounting where that behavior is desired. Long-running task ownership remains orthogonal to the [generic long-running tool runtime](../../proposed/architecture/2026-06-20-generic-long-running-tool-runtime.md) proposal. -`web_fetch` can use `TextRetainer` with `head` when the provider exposes a stream, or keep provider-owned body caps when the provider must read and decode internally. Either way, the fetch result's `truncated` remains a provider/tool fact, and the library only supplies retained text and omission metadata. +`web_fetch` can use `TextRetainer` with `head` or `headTail`, or keep provider-owned body caps when the provider must read and decode internally. Either way, the fetch result's `truncated` remains a provider/tool fact, and the library only supplies retained text and omission metadata. -`web_search` can use `ItemRetainer` with `head`. Current providers often return an array, so this is post-hoc but still standardizes notices; a streaming provider can use the same strategy with `stopWhenFull`. +`web_search` can use `ItemRetainer` with `head`. Current providers often return an array, so this is post-hoc but still standardizes notices. ### Notices @@ -149,19 +134,19 @@ The formatter hook is deliberately small: a tool turns a `RetentionNotice` into ## Consequences -**What shipped.** `@deepseek-ai/dsh-retention` exports `ItemRetainer`, `TextRetainer`, the result types (`RetainedItems`, `RetainedText`), the strategy types (`ItemRetentionStrategy`, `TextRetentionStrategy`, `StopMode`), `Omitted`, `PushDecision`, `RetentionNotice`, and the neutral notice helpers `describeOmitted` / `formatRetentionNotice` — with no dependency on Cordis or any tool package. Unit tests cover item-head early stop with a probe item, item-head read-to-end with exact omission counts, text-head early stop, text-tail retention with exact omission counts, head-tail byte retention, zero budgets, UTF-8 boundary handling (2-, 3-, and 4-byte codepoints and invalid lead bytes at each cut), and the difference between `{ kind: 'atLeast', count: 1 }` and exact omission. +**What shipped.** `@deepseek-ai/dsh-retention` exports `ItemRetainer`, `TextRetainer`, the result types (`RetainedItems`, `RetainedText`), the strategy types (`ItemRetentionStrategy`, `TextRetentionStrategy`), `Omitted`, `PushDecision`, `RetentionNotice`, and the neutral notice helpers `describeOmitted` / `formatRetentionNotice` — with no dependency on Cordis or any tool package. Unit tests cover item-head retention with exact omission counts, text-head retention, text-tail retention, head-tail byte retention, zero budgets, UTF-8 boundary handling (2-, 3-, and 4-byte codepoints and invalid lead bytes at each cut), and unknown omission wording. -**What is documented but not yet migrated.** `glob`, `grep`, `bash`, `web_fetch`, and `web_search` have their mappings documented in the [package README](../../../../packages/util/retention/README.md) — each stating whether it may stop upstream early — but no tool has been migrated onto the library in this change; migration is deliberately separate follow-up work. `glob` / `grep` do not yet exist as tools, so the `shouldStop` early-stop path has no in-repo caller until they land. `read` is documented as intentionally out of scope: its `read-render` line-window contract (`offset`/`limit`, `totalLines`, offset-range errors, per-line preview truncation, a byte cap over the selected window) is not generic retention, and one `Omitted` count cannot represent both sides of a line window. +**What is documented but not yet migrated.** `glob`, `grep`, `bash`, `web_fetch`, and `web_search` have their mappings documented in the [package README](../../../../packages/util/retention/README.md), but not every tool has been migrated onto the library in this change; migration is deliberately separate follow-up work. `read` is documented as intentionally out of scope: its `read-render` line-window contract (`offset`/`limit`, `totalLines`, offset-range errors, per-line preview truncation, a byte cap over the selected window) is not generic retention, and one `Omitted` count cannot represent both sides of a line window. **Boundaries the library holds.** `truncated` means the retainer omitted otherwise-available content because of a budget; it never means the upstream was incomplete. Tool-specific states — `incomplete`, permission failures, provider partial failures, binary skips, bash spill-path recovery, invalid UTF-8 — stay in tool-domain fields, outside the retainer. When a future change migrates a tool, that package's README and tests must prove the model-facing result text is unchanged except for deliberate notice wording. -**Tradeoffs accepted.** The v1 surface deliberately supports only item `head` retention and text `head` / `tail` / `headTail`; windows, grouped budgets, and sort-aware caps wait until a second consumer proves the need (the generic-collector alternative is why). Text retention counts bytes for process/body safety, leaving character- and line-level preview budgets as separate tool-owned concerns. `glob` / `grep` cannot report an exact omitted count once they stop the upstream at the first overflow item, so `Omitted.atLeast` exists and `describeOmitted` prints no number for it — formatters never claim "omitted 1" when the true count may be far larger. +**Tradeoffs accepted.** The v1 surface deliberately supports only item `head` retention and text `head` / `tail` / `headTail`; windows, grouped budgets, sort-aware caps, and upstream-stop control wait until a second consumer proves the need. Text retention counts bytes for process/body safety, leaving character- and line-level preview budgets as separate tool-owned concerns. ## Alternatives considered -**Post-hoc `truncate(text)` only.** Rejected: it matches Codex's history/tool-output truncation use case but fails the `glob` / `grep` resource model. The tool must stop ripgrep once the probe result proves truncation; collecting all output and trimming afterward defeats the point and can exceed the command runner's in-memory output cap. +**Post-hoc `truncate(text)` only.** Rejected: it matches Codex's history/tool-output truncation use case but loses item counts, grouping boundaries, UTF-8-safe byte windows, and exact omission metadata. -**One generic `Collector` with pluggable callbacks.** Rejected for v1: it hides the two important resource modes. Logical item retention can ask the caller to stop after a probe item; text tail/head-tail retention usually must read to the end. Separate `ItemRetainer` and `TextRetainer` names make that difference explicit while keeping the API small. +**One generic `Collector` with pluggable callbacks.** Rejected for v1: it hides the two important resource modes. Logical item retention counts items; text retention counts bytes and preserves UTF-8 boundaries. Separate `ItemRetainer` and `TextRetainer` names make that difference explicit while keeping the API small. **Put `read` windowing behind `ItemRetainer`.** Rejected for v1: `read` is the only current window consumer, and its semantics are file pagination rather than generic retention. A single `Omitted` count cannot represent both sides of a line window, and `read` also carries `totalLines`, offset-range errors, per-line preview truncation, and a byte cap over selected output. Keeping `read-render` tool-owned avoids growing the shared library around one special case. diff --git a/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md b/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md index a7d5aa4627..f261ebf84e 100644 --- a/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md +++ b/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md @@ -128,7 +128,7 @@ This separation is important. `web-fetch-local` still owns resource caps (`maxRe Retention is separate from spill storage: -- `@deepseek-ai/dsh-retention` owns preview mechanics (`TextRetainer`, `ItemRetainer`, omitted metadata, early-stop decisions). +- `@deepseek-ai/dsh-retention` owns preview mechanics (`TextRetainer`, `ItemRetainer`, and omitted metadata). - `@deepseek-ai/dsh-spill` owns saving final text to a session-scoped path. - `@deepseek-ai/dsh-spill-policy` applies the default final-result policy in the tool pipeline, composing the two. @@ -136,7 +136,7 @@ The final-result policy cannot replace tool-owned early spill. Some useful conte - `bash` final output is already a tail plus a temp spill path; the complete stdout/stderr streams live in executor files. - `subagent` final output is the child final answer, not the child rollout. -- Future `grep`/`glob` may early-stop and never collect full results. +- Future tools may produce runtime artifacts that are never represented by their final `ToolExecutionResult.content`. Those cases can consume `ctx.spillFiles` directly in later work. They are not part of the first showcase. @@ -188,4 +188,4 @@ The policy can become too large if it starts owning tool-specific semantics. It **Let `web-fetch-local` fetch without caps and rely on spill-policy.** Rejected: spill-policy runs after the final tool result exists and cannot protect network, memory, or decoding resources. Provider resource caps stay mandatory. -**Merge retention into spill.** Rejected: retention and spill have different responsibilities. `TextRetainer`/`ItemRetainer` decide what preview is kept and whether upstream may stop; spill storage only saves the final text the policy asks it to save. +**Merge retention into spill.** Rejected: retention and spill have different responsibilities. `TextRetainer`/`ItemRetainer` decide what preview is kept and what was omitted; spill storage only saves the final text the policy asks it to save. diff --git a/packages/util/README.md b/packages/util/README.md index 6477523861..dcfb019207 100644 --- a/packages/util/README.md +++ b/packages/util/README.md @@ -12,4 +12,4 @@ Zero-dependency primitives shared across the other groups. A package lands here `dsh-timeout` follows the same shape for the timeout family: `dsh-bash` and `dsh-web-fetch-local` each fuse a caller's cancellation with a deadline and later classify "timed out" vs "cancelled" by depending on `dsh-timeout` alone. It deliberately owns only the timing/classification half — the *termination* (SIGKILL a process group, tear down a fetch socket) stays in each capability, because no shared layer can own every capability's kill (see [the timeout-library RFC](../../docs/rfc/implemented/architecture/2026-07-06-timeout-deadline-library.md)). -`dsh-retention` is the same split for bounded tool output: a tool (`glob`/`grep`/`bash`/`web_fetch`/`web_search`) feeds items or text into a retainer and gets back "what we kept, what we omitted, may you stop reading" — while grouping, exit codes, provider errors, and recovery prose stay tool-owned. It deliberately owns only the retention mechanic; `truncated` is a budget fact, never an "incomplete inspection" state (see [the retention-library RFC](../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md)). +`dsh-retention` is the same split for bounded tool output: a tool (`glob`/`grep`/`bash`/`web_fetch`/`web_search`) feeds items or text into a retainer and gets back what it kept and exactly what it omitted — while grouping, exit codes, provider errors, and recovery prose stay tool-owned. It deliberately owns only the retention mechanic; `truncated` is a budget fact, never an "incomplete inspection" state (see [the retention-library RFC](../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md)). diff --git a/packages/util/retention/README.md b/packages/util/retention/README.md index 50bac13828..7256bd3596 100644 --- a/packages/util/retention/README.md +++ b/packages/util/retention/README.md @@ -1,8 +1,8 @@ # dsh-retention -A dependency-light **retention** library: bounded model-facing output for tools that must cap how much context they return. A caller feeds items or text chunks into a bounded object, gets a per-push decision about whether the upstream may stop, and later gets the retained content plus exact or partial omission metadata. +A dependency-light **retention** library: bounded model-facing output for tools that must cap how much context they return. A caller feeds items or text chunks into a bounded object, then gets the retained content plus exact omission metadata. -The library owns **only** the mechanical question *"what did we keep, what did we omit, and may the caller stop reading now?"*. Tool-specific code keeps its business semantics: file grouping, line numbering, exit codes, provider error states, per-line preview truncation, spill files, and the model-facing prose. This is the boundary the [RFC](../../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md) draws. +The library owns **only** the mechanical question *"what did we keep, and what did we omit?"*. Tool-specific code keeps its business semantics: file grouping, line numbering, exit codes, provider error states, per-line preview truncation, spill files, and the model-facing prose. This is the boundary the [RFC](../../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md) draws. It is a **library, not a service or plugin**: no `ctx`, registers nothing, emits no events. The only state is per-retainer (one accumulation), never cross-call. Tool packages import it directly. @@ -15,7 +15,7 @@ import { } from '@deepseek-ai/dsh-retention' import type { Omitted, PushDecision, RetainedItems, RetainedText, - ItemRetentionStrategy, TextRetentionStrategy, StopMode, RetentionNotice, + ItemRetentionStrategy, TextRetentionStrategy, RetentionNotice, } from '@deepseek-ai/dsh-retention' ``` @@ -23,19 +23,17 @@ import type { |---|---| | `ItemRetainer` | Bounds ordered logical units (paths, grep matches, sources). `head` only in v1. `push()` → `PushDecision`; `finish()` → `RetainedItems`. | | `TextRetainer` | Bounds a byte-oriented text stream. `head` / `tail` / `headTail`, UTF-8 boundaries preserved at `finish()`. `push()` → `PushDecision`; `finish()` → `RetainedText`. | -| `describeOmitted(omitted, unit)` | Standardized, false-precision-safe omission clause (`exact` prints a count; `atLeast`/`unknown` do not). | +| `describeOmitted(omitted, unit)` | Standardized omission clause (`exact` prints a count; `unknown` does not). | | `formatRetentionNotice(notice, recovery)` | Joins the standardized omission clause with the tool's own recovery guidance. | -| `Omitted` | `none` / `exact` / `atLeast` / `unknown` — how much was omitted, and whether the count is a lower bound. | -| `PushDecision` | `{ kept, truncated, shouldStop }` — the per-push control-flow result. | +| `Omitted` | `none` / `exact` / `unknown` — how much was omitted. | +| `PushDecision` | `{ kept, truncated }` — the per-push retention result. | -## The two resource modes +## Resource Modes -The two retainers are separate names, not one generic collector, because they differ in **resource model** — and that difference is the whole point of the `shouldStop` field. +The two retainers are separate names, not one generic collector, because they differ in **resource model**. -- **`ItemRetainer` can stop the upstream early.** With `stop: 'stopWhenFull'`, the first over-cap unit is a *probe*: it is not retained, sets `truncated`, and returns `shouldStop: true`. A discovery tool uses that to kill ripgrep / cancel a stream the moment truncation is proven, instead of collecting everything and trimming afterward. Because it stopped before the true total was known, `omitted` is `{ kind: 'atLeast', count: 1 }` — a lower bound, never a false-precise exact count. -- **`TextRetainer` tail/headTail must read to the end.** A true tail is unknowable until the stream closes, and draining avoids pipe backpressure on a child process, so `tail` and `headTail` never set `shouldStop` and report an `exact` omitted byte count. Only `head` + `stopWhenFull` can stop a text stream early. - -`shouldStop` is **advisory**: the retainer cannot reach the upstream. The tool owns the actual stop — abort the HTTP body, break the scan, kill the process group. +- **`ItemRetainer` bounds ordered logical units.** A search tool can collect a full result set for spill-file recovery while retaining only the first `maxItems` for the model-facing preview. The omission count is exact because the caller keeps feeding every observed item. +- **`TextRetainer` bounds byte-oriented text.** `head`, `tail`, and `headTail` preserve UTF-8 boundaries at `finish()`; `headTail` is the shape `dsh-spill-policy` uses to build a bounded preview around a spill-file notice. ## `truncated` is a budget fact, never "incomplete" @@ -47,32 +45,33 @@ Text caps and `omittedBytes` count **bytes**, for process/body safety (a child's ## Tool mappings -Every current retention consumer maps to the library below; each row states whether it may stop its upstream early. A broad migration is out of scope for the library's first landing — these are the intended shapes. +Every current retention consumer maps to the library below. A broad migration is out of scope for the library's first landing — these are the intended shapes. -| Tool | Retainer & strategy | Stops upstream early? | Notes | -|---|---|---|---| -| `glob` | `ItemRetainer`, `head` + `stopWhenFull` | **Yes** — the `(maxItems+1)`th path is the probe; `shouldStop` kills ripgrep. | Path mapping, skipped candidates, `incomplete` stay outside. `omitted` is `atLeast`. | -| `grep` | `ItemRetainer`, `head` + `stopWhenFull` | **Yes** — cap is total matches; stop on the probe match. | Per-match preview truncation, then push a flat match; group + sort the retained subset *after* `finish()`. | -| `bash` | `TextRetainer`, `tail` or `headTail`, reads to completion | No — stopping would lose the true tail and risk pipe backpressure. | Executor still owns spill files, exit status, signal, timeout, background tasks. | -| `web_fetch` | `TextRetainer`, `head` (streaming provider) | Optional — a streaming body can stop; a decode-internally provider keeps its own cap. | The fetch result's `truncated` remains a provider/tool fact. | -| `web_search` | `ItemRetainer`, `head` | Post-hoc today (providers return arrays); a streaming provider can use `stopWhenFull`. | Standardizes the "sources capped" notice. | +| Tool | Retainer & strategy | Notes | +|---|---|---| +| `glob` | `ItemRetainer`, `head` | Collect the full sorted path list for a spill file while retaining the first page inline. Path mapping, skipped candidates, and `incomplete` stay outside. | +| `grep` | `ItemRetainer`, `head` | Collect matches for a spill file while retaining the first page inline. Per-match preview truncation, grouping, sorting, and `incomplete` stay outside. | +| `bash` | `TextRetainer`, `tail` or `headTail` | Executor still owns spill files, exit status, signal, timeout, and background tasks. | +| `web_fetch` | `TextRetainer`, `head` or `headTail` | Provider/resource caps stay provider facts; the retainer supplies only retained text and omission metadata. | +| `web_search` | `ItemRetainer`, `head` | Standardizes the "sources capped" notice when providers return more sources than the model-facing result should include. | `read` is **intentionally out of scope for v1.** Its `read-render` helper owns a file-specific pagination contract — `offset`/`limit`, line numbers, `totalLines`, offset-out-of-range errors, per-line preview truncation, a byte cap over the selected window — which is a line-window renderer, not generic retention. A single `Omitted` count cannot represent both sides of a line window. ## Usage shape ```ts ignore-check -// glob: stop ripgrep the moment truncation is proven. -const retainer = new ItemRetainer({ kind: 'head', maxItems: globMaxResults, stop: 'stopWhenFull' }) +// glob: keep the first page inline while still collecting the full list for spill. +const retainer = new ItemRetainer({ kind: 'head', maxItems: globMaxResults }) +const allEntries: FsGlobEntry[] = [] for await (const entry of candidates) { - const { shouldStop } = retainer.push(entry) - if (shouldStop) { killRipgrep(); break } // the tool owns the actual stop + allEntries.push(entry) + retainer.push(entry) } const { items, truncated, omitted } = retainer.finish() // bash: keep a head + tail, read to process exit. const out = new TextRetainer({ kind: 'headTail', headBytes: headCap, tailBytes: tailCap }) -child.stdout.on('data', (chunk: Buffer) => { out.push(chunk) }) // shouldStop ignored: must drain +child.stdout.on('data', (chunk: Buffer) => { out.push(chunk) }) const { text, omittedBytes } = out.finish() // A footer: the library standardizes the omission clause; the tool owns recovery words. diff --git a/packages/util/retention/package.json b/packages/util/retention/package.json index 2926144ace..db8bab3342 100644 --- a/packages/util/retention/package.json +++ b/packages/util/retention/package.json @@ -1,6 +1,6 @@ { "name": "@deepseek-ai/dsh-retention", - "description": "Zero-dependency bounded-retention primitive: ItemRetainer/TextRetainer + neutral notice helpers (what did we keep, what did we omit, may the caller stop reading)", + "description": "Zero-dependency bounded-retention primitive: ItemRetainer/TextRetainer + neutral notice helpers (what did we keep, what did we omit)", "version": "0.0.1", "private": true, "type": "module", diff --git a/packages/util/retention/src/index.ts b/packages/util/retention/src/index.ts index 8c3b924a16..07547a7d93 100644 --- a/packages/util/retention/src/index.ts +++ b/packages/util/retention/src/index.ts @@ -1,12 +1,11 @@ /** * A dependency-light **retention** library: bounded model-facing output for * tools that must cap how much context they return. A caller feeds items or - * text chunks into a bounded object, gets a per-push {@link PushDecision} about - * whether the upstream may stop, and later gets the retained content plus exact - * or partial omission metadata ({@link RetainedItems} / {@link RetainedText}). + * text chunks into a bounded object, then gets the retained content plus exact + * omission metadata ({@link RetainedItems} / {@link RetainedText}). * * The library owns ONLY the mechanical question "what did we keep, what did we - * omit, and may the caller stop reading now?". Tool-specific code still owns + * omit?". Tool-specific code still owns * business semantics: file grouping, line numbering, exit codes, provider error * states, per-line preview truncation, spill files, and the model-facing prose. * In particular {@link RetainedText.truncated}/{@link RetainedItems.truncated} @@ -23,12 +22,10 @@ * The two retainers differ in resource model, which is why they are two names * rather than one generic collector: * - {@link ItemRetainer} bounds ordered logical units (paths, grep matches, - * search sources). `head` retention only in v1. With `stopWhenFull` it can ask - * the caller to stop the upstream after the first over-cap probe item. + * search sources). `head` retention only in v1. * - {@link TextRetainer} bounds byte-oriented text streams (bash stdout/stderr, * web bodies). `head` / `tail` / `headTail`, preserving UTF-8 boundaries at - * {@link TextRetainer.finish}. Only `head` can stop early; `tail`/`headTail` - * must read to the end to know the true suffix and exact omission. + * {@link TextRetainer.finish}. * * @module @deepseek-ai/dsh-retention */ @@ -36,45 +33,31 @@ /** * How much content the retainer omitted. * - * `atLeast` is the early-stop shape: an {@link ItemRetainer}/{@link TextRetainer} - * with `stopWhenFull` sees the first unit/chunk past the cap, asks the caller to - * stop the upstream, and therefore knows only a LOWER bound — reporting an exact - * count there would be false precision when the true total may be much larger. - * `exact` is the read-to-end shape (`tail`, `headTail`, or `head` with - * `readToEnd`), where every unit/byte was observed. `unknown` is reserved for a - * caller that omits without a count; the retainers themselves never return it. + * `exact` is the normal retainer shape: every unit/byte was observed, so the + * omitted count is precise. `unknown` is reserved for a caller that omits + * without a count; the retainers themselves never return it. */ export type Omitted = | { kind: 'none' } | { kind: 'exact'; count: number } - | { kind: 'atLeast'; count: number } | { kind: 'unknown' } /** * The caller receives this after each `push()`. - * - * `shouldStop` is ADVISORY, not automatic: the tool owns how to stop its upstream - * source — aborting an HTTP body, breaking a file scan, killing ripgrep. The - * retainer cannot reach the upstream; it only reports that keeping more would - * exceed the budget. A `readToEnd` / `tail` / `headTail` retainer never sets it - * (those must drain to the end). */ export interface PushDecision { /** Was this whole unit / all of this chunk's bytes retained (nothing dropped)? */ kept: boolean /** Cumulative: has the retainer omitted anything due to the budget yet? */ truncated: boolean - /** Advisory: keeping more would exceed the budget — the tool may stop its upstream. */ - shouldStop: boolean } /** * Final result for ordered logical units. * * `seen` means units OBSERVED by the retainer, not necessarily the total in the - * upstream source; with an early stop, the true total is intentionally unknown - * (hence {@link Omitted.atLeast}). `kept` is `items.length`, surfaced explicitly - * so a notice formatter need not re-count. + * upstream source. `kept` is `items.length`, surfaced explicitly so a notice + * formatter need not re-count. */ export interface RetainedItems { items: T[] @@ -100,30 +83,19 @@ export interface RetainedText { omittedBytes: Omitted } -/** - * Whether a retainer asks the caller to stop the upstream once keeping more - * would exceed the budget (`stopWhenFull`), or must keep accepting input even - * after the retained output is full (`readToEnd`) — usually to preserve a true - * tail, count exact omission, or drain an upstream process to avoid pipe - * backpressure. Names avoid implementation phrases like "overflow". - */ -export type StopMode = 'stopWhenFull' | 'readToEnd' - /** Item retention strategy. Only `head` in v1; windows/grouped budgets wait for a second consumer. */ export type ItemRetentionStrategy = { /** Keep the first `maxItems` units. Use for `glob`, `grep`, and web sources. */ kind: 'head' maxItems: number - stop: StopMode } /** Text retention strategy: keep a prefix, a suffix, or both, counted in bytes. */ export type TextRetentionStrategy = | { - /** Keep the first `maxBytes` bytes. May stop an upstream body early. */ + /** Keep the first `maxBytes` bytes. */ kind: 'head' maxBytes: number - stop: StopMode } | { /** Keep the final `maxBytes` bytes. Requires reading to the end. */ @@ -164,8 +136,7 @@ function assertBudget(value: number, name: string): void { /** * Bounds an ordered stream of logical units, keeping the first `maxItems` * ({@link ItemRetentionStrategy} `head`). `push()` reports, per unit, whether it - * was kept and — under `stopWhenFull` — whether the caller should stop the - * upstream now that the first over-cap probe unit has been seen. + * was kept and whether the retained result is now truncated. * * Grouping, sorting, path mapping, per-unit preview truncation, and any * `incomplete` state stay OUTSIDE the retainer: it counts and keeps, nothing @@ -174,24 +145,20 @@ function assertBudget(value: number, name: string): void { */ export class ItemRetainer { private readonly maxItems: number - private readonly stop: StopMode private readonly items: T[] = [] private seen = 0 private omittedCount = 0 - /** @param strategy Head strategy: `maxItems` (non-negative integer) and the {@link StopMode}. */ + /** @param strategy Head strategy: `maxItems` (non-negative integer). */ constructor(strategy: ItemRetentionStrategy) { assertBudget(strategy.maxItems, 'maxItems') this.maxItems = strategy.maxItems - this.stop = strategy.stop } /** * Offer one unit. Kept when the retainer is below `maxItems`; otherwise dropped - * and counted as omitted. Under `stopWhenFull` the first dropped unit is the - * probe: `shouldStop` is `true` so the caller can kill ripgrep / cancel the - * stream, and the final {@link Omitted} stays `atLeast` (the true total is - * unknown). Under `readToEnd` the caller keeps pushing so omission is `exact`. + * and counted as omitted. Callers keep pushing all observed units, so the final + * {@link Omitted} count is exact. * * @param item The already-shaped logical unit (path, flat match, source). * @returns The per-push {@link PushDecision}. @@ -202,22 +169,17 @@ export class ItemRetainer { // Reached only below the cap, before any omission (items only grow, the // cap is fixed), so nothing has been dropped yet: truncated is always false. this.items.push(item) - return { kept: true, truncated: false, shouldStop: false } + return { kept: true, truncated: false } } this.omittedCount++ return { kept: false, truncated: true, - // Only ask to stop when the caller opted into it; readToEnd must keep - // draining to reach an exact omission count. - shouldStop: this.stop === 'stopWhenFull', } } /** - * Finalize and report what was kept and omitted. `omitted` is `atLeast` under - * `stopWhenFull` (a lower bound — the caller was asked to stop before the true - * total was known) and `exact` under `readToEnd`. + * Finalize and report what was kept and omitted. * * @returns The {@link RetainedItems} snapshot (safe to group/sort downstream). */ @@ -229,7 +191,7 @@ export class ItemRetainer { seen: this.seen, kept: this.items.length, omitted: truncated - ? { kind: this.stop === 'stopWhenFull' ? 'atLeast' : 'exact', count: this.omittedCount } + ? { kind: 'exact', count: this.omittedCount } : { kind: 'none' }, } } @@ -275,8 +237,6 @@ function trimLeadingContinuationUtf8(bytes: Uint8Array): Uint8Array { * Bounds a byte-oriented text stream, keeping a prefix, a suffix, or both * ({@link TextRetentionStrategy}). All three strategies share one prefix/suffix * accumulator: `head` is prefix-only, `tail` is suffix-only, `headTail` is both. - * Only `head` with `stopWhenFull` sets `shouldStop`; `tail`/`headTail` must read - * to the end to know the true suffix and the exact omitted byte count. * * Bytes, not characters: caps and `omittedBytes` are byte counts for process/ * body safety. Chunks that straddle a codepoint are handled — {@link finish} @@ -288,7 +248,6 @@ function trimLeadingContinuationUtf8(bytes: Uint8Array): Uint8Array { export class TextRetainer { private readonly prefixCap: number private readonly suffixCap: number - private readonly allowStop: boolean private readonly prefixChunks: Uint8Array[] = [] private prefixHeld = 0 private readonly suffixChunks: Uint8Array[] = [] @@ -302,20 +261,17 @@ export class TextRetainer { assertBudget(strategy.maxBytes, 'maxBytes') this.prefixCap = strategy.maxBytes this.suffixCap = 0 - this.allowStop = strategy.stop === 'stopWhenFull' break case 'tail': assertBudget(strategy.maxBytes, 'maxBytes') this.prefixCap = 0 this.suffixCap = strategy.maxBytes - this.allowStop = false break case 'headTail': assertBudget(strategy.headBytes, 'headBytes') assertBudget(strategy.tailBytes, 'tailBytes') this.prefixCap = strategy.headBytes this.suffixCap = strategy.tailBytes - this.allowStop = false break } } @@ -324,9 +280,7 @@ export class TextRetainer { * Offer one chunk (a `Uint8Array`, or a `string` encoded as UTF-8). Prefix * bytes fill up to the prefix cap then stop; suffix bytes roll so only the * last `suffixCap` bytes are retained. `kept` is `true` only when no byte of - * this chunk was dropped. Under `head` + `stopWhenFull`, `shouldStop` turns - * `true` on the chunk that first drops a byte (the caller may then abort the - * body); other strategies never set it. + * this chunk was dropped. * * @param chunk The next bytes of the stream (`Uint8Array` or UTF-8 `string`). * @returns The per-push {@link PushDecision}. @@ -373,12 +327,11 @@ export class TextRetainer { // Dropped = bytes that no side can keep. Compute cumulative omission the // SAME way finish() does (via omittedAt), so push and finish never disagree; // per-push we only need whether THIS chunk pushed the total past what the - // two caps hold, and — for head+stopWhenFull — whether to stop. + // two caps hold. const droppedThisChunk = this.omittedAt(this.total) > this.omittedAt(before) return { kept: !droppedThisChunk, truncated: this.omittedAt(this.total) > 0, - shouldStop: this.allowStop && droppedThisChunk, } } @@ -391,10 +344,7 @@ export class TextRetainer { /** * Finalize: decode the retained prefix and suffix (each trimmed to a UTF-8 - * boundary at its cut) and report the exact or lower-bound omitted byte count. - * `head` + `stopWhenFull` yields `atLeast` (a lower bound — the caller was - * asked to stop before the true size was known); every other case reads to the - * end and yields `exact`. + * boundary at its cut) and report the exact omitted byte count. * * @returns The {@link RetainedText} snapshot (safe to hand to a formatter). */ @@ -423,8 +373,7 @@ export class TextRetainer { // Report omission against the bytes ACTUALLY returned, not the pre-trim // budget: a boundary trim drops partial-codepoint bytes too, so an exact // count derived from the budget alone would overstate the retained text (and - // any "Omitted N bytes" notice built from it would be a lie). total_seen − - // retained stays a valid lower bound under `atLeast` (true total ≥ seen). + // any "Omitted N bytes" notice built from it would be a lie). const omitted = this.total - keptPrefix.length - keptSuffix.length const truncated = omitted > 0 @@ -432,7 +381,7 @@ export class TextRetainer { text, truncated, omittedBytes: truncated - ? { kind: this.allowStop ? 'atLeast' : 'exact', count: omitted } + ? { kind: 'exact', count: omitted } : { kind: 'none' }, } } @@ -454,10 +403,8 @@ function concat(chunks: readonly Uint8Array[]): Uint8Array { /** * Standardized, false-precision-safe wording for one {@link Omitted} value — * the "may standardize omission wording" half the library owns. `exact` prints - * the count (`Omitted 3 items`); `atLeast`/`unknown` print NO count, because an - * early stop knows only that more was dropped, not how much (claiming "omitted - * 1" when the true total may be huge is the false-precision trap the `atLeast` - * variant exists to avoid). `none` is the empty string. + * the count (`Omitted 3 items`); `unknown` prints NO count because the caller + * did not provide one. `none` is the empty string. * * @param omitted The omission metadata from a retainer result. * @param unit The noun for the omitted quantity (`items`, `bytes`, `chars`, `lines`). @@ -469,7 +416,6 @@ export function describeOmitted(omitted: Omitted, unit: RetentionNotice['unit']) return '' case 'exact': return `Omitted ${omitted.count} ${unit}.` - case 'atLeast': case 'unknown': return `More ${unit} were omitted.` } diff --git a/packages/util/retention/tests/retention.spec.ts b/packages/util/retention/tests/retention.spec.ts index ec424595cb..8fac7d8575 100644 --- a/packages/util/retention/tests/retention.spec.ts +++ b/packages/util/retention/tests/retention.spec.ts @@ -11,26 +11,23 @@ import { /** Decode a RetainedText via a round-trip helper for readable UTF-8 assertions. */ const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s) -describe('ItemRetainer — head, stopWhenFull (glob/grep early stop)', () => { - it('keeps the first maxItems and asks to stop on the probe item', () => { - const r = new ItemRetainer({ kind: 'head', maxItems: 2, stop: 'stopWhenFull' }) - expect(r.push('a')).toEqual({ kept: true, truncated: false, shouldStop: false }) - expect(r.push('b')).toEqual({ kept: true, truncated: false, shouldStop: false }) - // The (maxItems + 1)th valid item is the probe: not retained, sets truncated, - // and shouldStop tells the caller to kill the upstream. - expect(r.push('c')).toEqual({ kept: false, truncated: true, shouldStop: true }) +describe('ItemRetainer — head retention', () => { + it('keeps the first maxItems while callers keep draining for an exact omitted count', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 2 }) + expect(r.push('a')).toEqual({ kept: true, truncated: false }) + expect(r.push('b')).toEqual({ kept: true, truncated: false }) + expect(r.push('c')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.items).toEqual(['a', 'b']) expect(result.kept).toBe(2) expect(result.seen).toBe(3) expect(result.truncated).toBe(true) - // Early stop knows only a lower bound, never an exact total. - expect(result.omitted).toEqual({ kind: 'atLeast', count: 1 }) + expect(result.omitted).toEqual({ kind: 'exact', count: 1 }) }) it('reports none when everything fits', () => { - const r = new ItemRetainer({ kind: 'head', maxItems: 3, stop: 'stopWhenFull' }) + const r = new ItemRetainer({ kind: 'head', maxItems: 3 }) r.push(1) r.push(2) const result = r.finish() @@ -38,15 +35,11 @@ describe('ItemRetainer — head, stopWhenFull (glob/grep early stop)', () => { expect(result.truncated).toBe(false) expect(result.omitted).toEqual({ kind: 'none' }) }) -}) - -describe('ItemRetainer — head, readToEnd (exact omission)', () => { it('keeps draining past the cap and reports an exact omitted count', () => { - const r = new ItemRetainer({ kind: 'head', maxItems: 1, stop: 'readToEnd' }) - expect(r.push('a')).toEqual({ kept: true, truncated: false, shouldStop: false }) - // readToEnd never asks to stop — the caller must keep pushing to count exactly. - expect(r.push('b')).toEqual({ kept: false, truncated: true, shouldStop: false }) - expect(r.push('c')).toEqual({ kept: false, truncated: true, shouldStop: false }) + const r = new ItemRetainer({ kind: 'head', maxItems: 1 }) + expect(r.push('a')).toEqual({ kept: true, truncated: false }) + expect(r.push('b')).toEqual({ kept: false, truncated: true }) + expect(r.push('c')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.items).toEqual(['a']) @@ -56,53 +49,49 @@ describe('ItemRetainer — head, readToEnd (exact omission)', () => { }) describe('ItemRetainer — zero budget', () => { - it('keeps nothing; first item is the probe under stopWhenFull', () => { - const r = new ItemRetainer({ kind: 'head', maxItems: 0, stop: 'stopWhenFull' }) - expect(r.push('a')).toEqual({ kept: false, truncated: true, shouldStop: true }) + it('keeps nothing and counts every pushed item as omitted', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 0 }) + expect(r.push('a')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.items).toEqual([]) expect(result.kept).toBe(0) - expect(result.omitted).toEqual({ kind: 'atLeast', count: 1 }) + expect(result.omitted).toEqual({ kind: 'exact', count: 1 }) }) it('rejects a non-integer / negative maxItems', () => { - expect(() => new ItemRetainer({ kind: 'head', maxItems: -1, stop: 'readToEnd' })) + expect(() => new ItemRetainer({ kind: 'head', maxItems: -1 })) .toThrow(/maxItems must be a non-negative integer/) - expect(() => new ItemRetainer({ kind: 'head', maxItems: 1.5, stop: 'readToEnd' })) + expect(() => new ItemRetainer({ kind: 'head', maxItems: 1.5 })) .toThrow(/maxItems must be a non-negative integer/) }) }) -describe('TextRetainer — head, stopWhenFull (early body stop)', () => { - it('keeps the prefix and asks to stop on the overflowing chunk', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 5, stop: 'stopWhenFull' }) - expect(r.push('abc')).toEqual({ kept: true, truncated: false, shouldStop: false }) +describe('TextRetainer — head (exact omission, reads to end)', () => { + it('keeps the prefix and counts omitted bytes exactly', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 5 }) + expect(r.push('abc')).toEqual({ kept: true, truncated: false }) // 'de' fills the cap exactly (5 bytes) — still fully kept. - expect(r.push('de')).toEqual({ kept: true, truncated: false, shouldStop: false }) - // 'fgh' is wholly dropped: kept:false, and stopWhenFull → shouldStop. - expect(r.push('fgh')).toEqual({ kept: false, truncated: true, shouldStop: true }) + expect(r.push('de')).toEqual({ kept: true, truncated: false }) + expect(r.push('fgh')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.text).toBe('abcde') expect(result.truncated).toBe(true) - // Early stop: a lower bound, not an exact size. - expect(result.omittedBytes).toEqual({ kind: 'atLeast', count: 3 }) + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 3 }) }) it('flags a partially-dropped chunk as not fully kept', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 4, stop: 'stopWhenFull' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 4 }) r.push('ab') - // 'cde' straddles the cap: 'c','d' fit, 'e' drops → kept:false, shouldStop. - expect(r.push('cde')).toEqual({ kept: false, truncated: true, shouldStop: true }) + // 'cde' straddles the cap: 'c','d' fit, 'e' drops → kept:false. + expect(r.push('cde')).toEqual({ kept: false, truncated: true }) expect(r.finish().text).toBe('abcd') }) -}) -describe('TextRetainer — head, readToEnd (exact omission)', () => { - it('keeps the prefix, drains the rest, and counts exactly', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + it('keeps draining past the cap', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 3 }) r.push('abc') - expect(r.push('defg')).toEqual({ kept: false, truncated: true, shouldStop: false }) + expect(r.push('defg')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.text).toBe('abc') expect(result.omittedBytes).toEqual({ kind: 'exact', count: 4 }) @@ -112,8 +101,7 @@ describe('TextRetainer — head, readToEnd (exact omission)', () => { describe('TextRetainer — tail (exact omission, reads to end)', () => { it('keeps the final maxBytes and reports exact omission', () => { const r = new TextRetainer({ kind: 'tail', maxBytes: 4 }) - // tail never asks to stop — it must read to the end to know the true suffix. - expect(r.push('hello')).toEqual({ kept: false, truncated: true, shouldStop: false }) + expect(r.push('hello')).toEqual({ kept: false, truncated: true }) r.push('world') const result = r.finish() expect(result.text).toBe('orld') // last 4 bytes of 'helloworld' @@ -185,12 +173,12 @@ describe('TextRetainer — headTail (prefix + suffix, omit the middle)', () => { }) describe('TextRetainer — zero budgets', () => { - it('head maxBytes 0 keeps nothing and stops on first byte (stopWhenFull)', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 0, stop: 'stopWhenFull' }) - expect(r.push('x')).toEqual({ kept: false, truncated: true, shouldStop: true }) + it('head maxBytes 0 keeps nothing and counts every byte exactly', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 0 }) + expect(r.push('x')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.text).toBe('') - expect(result.omittedBytes).toEqual({ kind: 'atLeast', count: 1 }) + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 1 }) }) it('an empty stream omits nothing', () => { @@ -202,7 +190,7 @@ describe('TextRetainer — zero budgets', () => { }) it('rejects non-integer / negative byte budgets', () => { - expect(() => new TextRetainer({ kind: 'head', maxBytes: -1, stop: 'readToEnd' })) + expect(() => new TextRetainer({ kind: 'head', maxBytes: -1 })) .toThrow(/maxBytes must be a non-negative integer/) expect(() => new TextRetainer({ kind: 'tail', maxBytes: 2.5 })) .toThrow(/maxBytes must be a non-negative integer/) @@ -218,7 +206,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { // '€' is 3 bytes (E2 82 AC). A 2-byte head cap keeps 'a' (61) + the first // byte of '€' (E2); that partial lead byte must be trimmed, not decoded to // a replacement char. - const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 2 }) r.push('a€b') // bytes: 61 E2 82 AC 62 const result = r.finish() expect(result.text).toBe('a') // partial '€' dropped, no U+FFFD @@ -256,7 +244,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { }) it('preserves a whole multibyte codepoint that fits exactly', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 3 }) r.push('€x') // '€' is exactly 3 bytes expect(r.finish().text).toBe('€') }) @@ -272,7 +260,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { }) it('accepts a raw Uint8Array chunk', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 2 }) r.push(utf8('xy')) r.push(utf8('z')) expect(r.finish().text).toBe('xy') @@ -281,7 +269,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { it('trims a partial 2-byte codepoint at the head cut', () => { // 'é' is 2 bytes (C3 A9). A 2-byte head cap over 'aé' keeps 'a' (61) + the // lead byte of 'é' (C3) — an incomplete 2-byte sequence to trim. - const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 2 }) r.push('aé') // bytes: 61 C3 A9 const result = r.finish() expect(result.text).toBe('a') @@ -291,7 +279,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { it('trims a partial 4-byte codepoint (emoji) at the head cut', () => { // '😀' is 4 bytes (F0 9F 98 80). A 3-byte head cap keeps 'a' + the first two // bytes of the emoji — an incomplete 4-byte sequence that must be trimmed. - const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 3 }) r.push('a😀') // bytes: 61 F0 9F 98 80 const result = r.finish() expect(result.text).toBe('a') @@ -299,7 +287,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { }) it('keeps a whole 4-byte codepoint that fits exactly', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 4, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 4 }) r.push('😀x') expect(r.finish().text).toBe('😀') }) @@ -308,7 +296,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { // A cut whose trailing bytes are ALL continuation bytes with no lead in // reach is not a trimmable incomplete sequence — the trimmer bails (no lead // byte found) and leaves them for the non-fatal decoder to replace. - const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 2 }) // 0x80 0x80 are bare continuation bytes; 'z' follows so the head keeps just // the two continuation bytes and the cut lands right after them. r.push(new Uint8Array([0x80, 0x80, 0x7a])) @@ -322,7 +310,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { // 0xF8 is not a valid UTF-8 lead byte (only 0x00–0xF7 lead). The trimmer // recognizes it as "not a lead" (expected length 0) and leaves the byte in // place rather than trimming a phantom partial sequence. - const r = new TextRetainer({ kind: 'head', maxBytes: 1, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 1 }) r.push(new Uint8Array([0xf8, 0x61])) // 0xF8 kept, 'a' dropped by the 1-byte cap const result = r.finish() expect(result.omittedBytes).toEqual({ kind: 'exact', count: 1 }) @@ -335,10 +323,7 @@ describe('describeOmitted — false precision safety', () => { expect(describeOmitted({ kind: 'exact', count: 12 }, 'bytes')).toBe('Omitted 12 bytes.') }) - it('prints NO count for atLeast (early stop) and unknown', () => { - // The whole point of atLeast: never claim "omitted 1" when the true count is - // unknown. Both atLeast and unknown collapse to a countless clause. - expect(describeOmitted({ kind: 'atLeast', count: 1 }, 'items')).toBe('More items were omitted.') + it('prints NO count for unknown omission', () => { expect(describeOmitted({ kind: 'unknown' }, 'lines')).toBe('More lines were omitted.') }) @@ -359,10 +344,10 @@ describe('formatRetentionNotice', () => { it('joins the standardized omission clause with the tool recovery guidance', () => { const out = formatRetentionNotice( - notice({ kind: 'atLeast', count: 1 }), + notice({ kind: 'exact', count: 25 }), ({ kept }) => `Results capped at ${kept}. Narrow the pattern, path, or include to see more.`, ) - expect(out).toBe('More items were omitted. Results capped at 100. Narrow the pattern, path, or include to see more.') + expect(out).toBe('Omitted 25 items. Results capped at 100. Narrow the pattern, path, or include to see more.') }) it('omits the empty half when nothing was omitted', () => {