From b58e33268afa9771e96bca1a565555d519d97743 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 25 Jul 2026 10:18:16 +0800 Subject: [PATCH 01/11] feat(llm): add per-provider retry policies --- ...2026-06-21-bounded-llm-request-recovery.md | 34 +- ...26-07-24-provider-retry-policies.i18n.yaml | 6 + .../2026-07-24-provider-retry-policies.md | 65 +++ .../2026-07-24-provider-retry-policies.zh.md | 65 +++ docs/architecture.i18n.yaml | 4 +- docs/architecture.md | 2 +- docs/architecture.zh.md | 2 +- docs/config-catalog.md | 46 +- docs/cordis-catalog/events.md | 2 +- docs/cordis-catalog/services.md | 11 +- docs/core-data-structures/llm-streaming.md | 12 +- docs/event-producer-consumer.md | 2 +- docs/persistence-catalog.md | 12 +- .../tests/compact-loop-repro.spec.ts | 20 +- .../cordis/tool-cordis/src/api-catalog.ts | 20 + packages/core/agent-loop/README.md | 2 +- packages/examples/acp-demo/README.md | 1 - packages/examples/acp-demo/src/index.ts | 3 - packages/examples/agent-spine-demo/README.md | 8 +- .../examples/agent-spine-demo/package.json | 2 +- .../examples/agent-spine-demo/src/index.ts | 16 +- .../agent-spine-demo/tests/agent-core.spec.ts | 34 +- packages/examples/cli-demo/README.md | 1 - packages/examples/cli-demo/src/index.ts | 3 - packages/examples/cli-demo/tests/cli.spec.ts | 19 +- packages/llm/README.md | 4 +- packages/llm/llm-deepseek/README.md | 10 +- packages/llm/llm-deepseek/src/adapter.ts | 12 +- packages/llm/llm-deepseek/src/index.ts | 7 +- .../llm/llm-deepseek/tests/adapter.spec.ts | 32 ++ packages/llm/llm-pi-ai/README.md | 9 +- packages/llm/llm-pi-ai/src/adapter.ts | 13 +- packages/llm/llm-pi-ai/src/config.ts | 13 +- packages/llm/llm-pi-ai/src/index.ts | 5 +- packages/llm/llm-pi-ai/tests/adapter.spec.ts | 38 +- packages/llm/llm-retry/README.md | 39 +- packages/llm/llm-retry/package.json | 2 +- packages/llm/llm-retry/src/history.ts | 38 ++ packages/llm/llm-retry/src/index.ts | 230 +++++----- packages/llm/llm-retry/src/invariant.ts | 62 ++- .../llm/llm-retry/tests/invariant.spec.ts | 179 +++++++- .../tests/loader-composition.spec.ts | 22 +- .../llm/llm-retry/tests/persistence.spec.ts | 7 +- packages/llm/llm-retry/tests/retry.spec.ts | 405 ++++++++++++++++-- packages/llm/llm-retry/tsdown.config.ts | 25 ++ packages/llm/llm/README.md | 7 +- packages/llm/llm/package.json | 5 + packages/llm/llm/src/index.ts | 47 +- packages/llm/llm/src/retry-policy.ts | 184 ++++++++ packages/llm/llm/tests/retry-policy.spec.ts | 84 ++++ packages/llm/llm/tests/service.spec.ts | 22 + packages/llm/llm/tsconfig.json | 3 + packages/ui/acp/src/index.ts | 3 +- packages/ui/acp/tests/stream-update.spec.ts | 17 + packages/ui/tui/src/index.ts | 3 +- .../snapshots/retry-cancelled.expected.txt | 2 +- packages/ui/tui/tests/tui.snapshot.ts | 5 +- packages/ui/tui/tests/tui.spec.ts | 6 + pnpm-lock.yaml | 7 + scripts/gen-cordis-catalog.ts | 1 + 60 files changed, 1606 insertions(+), 334 deletions(-) create mode 100644 .agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml create mode 100644 .agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md create mode 100644 .agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md create mode 100644 packages/llm/llm-retry/src/history.ts create mode 100644 packages/llm/llm-retry/tsdown.config.ts create mode 100644 packages/llm/llm/src/retry-policy.ts create mode 100644 packages/llm/llm/tests/retry-policy.spec.ts diff --git a/.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md b/.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md index 28de5eb97c..1e0ef18f9c 100644 --- a/.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md +++ b/.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md @@ -2,6 +2,8 @@ Status: implemented +The [per-provider request retry policy](../feature/2026-07-24-provider-retry-policies.md) extends this foundation with exact-provider configuration and an explicit unbounded mode. This note continues to own structured failure facts, the closed-step recovery boundary, normal mode's transient defaults, visible single attempts, and durable retry status. + ## Problem `dsh-llm` can report provider failures either by throwing during adapter dispatch or iteration or by ending with `finish { kind: 'error' | 'aborted' }`. The final adapter boundary tags thrown failures so `dsh-agent-loop` can distinguish them from middleware and result-processing defects, and the loop normalizes both delivery forms into `agent/request-error` after closing the failed step. The default decision is `fail`; `dsh-compact-basic` is the only shipped recovery listener, and it retries a canonical context-window overflow only after compaction proves that the durable surface shrank. @@ -14,7 +16,7 @@ The prior boundary left three narrower gaps. - Retry ownership differs by adapter. The hand-written DeepSeek adapter makes one attempt, while pi-ai profiles can enable opaque library retries. Combining hidden transport retries with an `agent/request-error` listener would multiply attempts and omit intermediate failures from the session log. - A recovered failure has no durable status fact. The failed step and chunks remain reconstructable, but an observer cannot tell whether the agent is deliberately backing off, for how long, or why. A long silent wait looks like a stalled loop. -The goal is bounded recovery from transient failures of the same explicit provider/model request. Provider or model failover, response splicing, and semantic output repair are different problems and have no current consumer. +The default policy provides bounded recovery from transient failures of the same explicit provider/model request. Provider or model failover, response splicing, and semantic output repair are different problems and have no current consumer. ## Decision @@ -48,31 +50,19 @@ The initial shared transient-code set is intentionally small: the adapters' exis `@deepseek-ai/dsh-llm-retry` is a function plugin that listens to `agent/request-error`. It introduces no service or new loop branch; the agent-loop package changes only the data carried through its existing failed-step recovery control flow. -The `agent/request-error` seam carries the current `LlmFailure` and an immutable list of prior failures that led to another request attempt in this consecutive recovery sequence. `dsh-llm-retry` counts only prior failures whose codes are in its configured transient set, while `dsh-compact-basic` counts only prior context-overflow failures. A successful model request clears the history. Alternating transient and context-overflow failures therefore consume their owning policy budgets independently; the maximum request count is one plus the sum of the finite budgets of the loaded recovery policies. +The `agent/request-error` seam carries the current `LlmFailure` and an immutable list of prior failures that led to another request attempt in this consecutive recovery sequence. Normal `dsh-llm-retry` policy counts retry records scheduled by the same exact-provider policy, while `dsh-compact-basic` counts prior context-overflow failures. A successful model request clears the history. Alternating transient and context-overflow failures therefore consume their owning finite budgets independently. -The plugin resolves and validates this deployment configuration at load: - -```ts ignore-check -interface Config { - maxTransientRetries?: number - initialDelayMs?: number - maxDelayMs?: number - jitterRatio?: number - retryableCodes?: string[] -} -``` - -The defaults are two transient retries, a 500 millisecond initial delay, a 10 second delay cap, 10 percent jitter, and the four transient codes above. The count and delay bounds match the conservative edge of the inspected implementations: [OpenCode uses two request retries with 500 ms/10 s bounds](https://github.com/anomalyco/opencode/blob/9976269ab1accfc9f9dc98a4a688c516934de422/%70ackages/llm/src/route/executor.ts#L36-L39), [Pi separates three agent-level retries from provider retries and defaults provider retries to zero](https://github.com/earendil-works/pi/blob/3da591ab74ab9ab407e72ed882600b2c851fae21/%70ackages/coding-agent/docs/settings.md#L139-L147), and [Codex uses finite request/stream budgets plus a five-minute idle timeout](https://github.com/openai/codex/blob/0fb559f0f6e231a88ac02ea002d3ecd248e2b515/codex-rs/model-provider-info/src/lib.rs#L25-L33). Ten percent follows [Codex's bounded jitter](https://github.com/openai/codex/blob/0fb559f0f6e231a88ac02ea002d3ecd248e2b515/codex-rs/codex-client/src/retry.rs#L40-L47). Two retries mean at most three provider requests when no other recovery policy applies. `maxTransientRetries` is a non-negative integer, delays are positive finite numbers with `initialDelayMs <= maxDelayMs`, `jitterRatio` is in `[0, 1]`, and codes are non-empty and unique. These are Cordis config fields rather than hidden constants so deployments can choose different cost and latency budgets. +The [provider-policy decision](../feature/2026-07-24-provider-retry-policies.md) owns the current configuration shape. Provider adapters register their nested `retryPolicy`; omission uses normal defaults: two transient retries, a 500 millisecond initial delay, a 10 second delay cap, 10 percent jitter, and the four transient codes above. The count and delay bounds match the conservative edge of the inspected implementations: [OpenCode uses two request retries with 500 ms/10 s bounds](https://github.com/anomalyco/opencode/blob/9976269ab1accfc9f9dc98a4a688c516934de422/%70ackages/llm/src/route/executor.ts#L36-L39), [Pi separates three agent-level retries from provider retries and defaults provider retries to zero](https://github.com/earendil-works/pi/blob/3da591ab74ab9ab407e72ed882600b2c851fae21/%70ackages/coding-agent/docs/settings.md#L139-L147), and [Codex uses finite request/stream budgets plus a five-minute idle timeout](https://github.com/openai/codex/blob/0fb559f0f6e231a88ac02ea002d3ecd248e2b515/codex-rs/model-provider-info/src/lib.rs#L25-L33). Ten percent follows [Codex's bounded jitter](https://github.com/openai/codex/blob/0fb559f0f6e231a88ac02ea002d3ecd248e2b515/codex-rs/codex-client/src/retry.rs#L40-L47). For an eligible failure with budget remaining, the one-based transient retry count uses bounded exponential backoff. A valid `providerRetryAfterMs` replaces exponential backoff only when it does not exceed `maxDelayMs`; a longer provider delay causes delegation instead of an earlier retry that violates the provider instruction. Local backoff multiplies by an injected random factor in `[1 - jitterRatio, 1 + jitterRatio]` and clamps the final value to `maxDelayMs`; provider delay is not jittered. The plugin owns a lifetime `AbortController` and tracks every active backoff callback. Each wait fuses the waterfall's turn signal with that lifetime signal. Effect cleanup first unregisters the listener, then aborts and awaits the active callbacks; a captured callback whose lifetime signal aborts returns `fail` and can neither retry nor enter the rest of its captured waterfall after disposal. This makes HMR disposal quiescent even though Cordis has already captured the listener. -Before sleeping, `dsh-llm-retry` appends one non-surface `llm/retry` session event containing the turn, failed step, one-based transient retry number, configured maximum, scheduled delay, and `LlmFailure`. The plugin owns the `SessionEventMap` augmentation; `dsh-session` remains generic persistence and does not absorb the optional policy's vocabulary. The event says what was scheduled, not that the next request completed; cancellation during the delay is subsequently visible on `turn/end`. The event ships only with a production renderer and replay/snapshot coverage, because its purpose is operational state rather than trace collection. +Before sleeping, `dsh-llm-retry` appends one non-surface `llm/retry` session event containing the turn, failed step, provider, policy mode, provider-policy retry number, mode-specific finite maximum when present, scheduled delay, and `LlmFailure`. The plugin owns the `SessionEventMap` augmentation; `dsh-session` remains generic persistence and does not absorb the optional policy's vocabulary. The event says what was scheduled, not that the next request completed; cancellation during the delay is subsequently visible on `turn/end`. The event ships only with a production renderer and replay/snapshot coverage, because its purpose is operational state rather than trace collection. The listener calls `next()` for a non-transient code, an exhausted policy budget, or an over-cap provider delay. This preserves composition with context-overflow recovery and later policy plugins. It returns `{ action: 'retry' }` only after the delay completes under both signals; turn cancellation and plugin disposal return `fail`, after which the loop's cancellation/disposal checks remain authoritative. -The agent-spine demo bundle loads the plugin so the shared stdio/TUI, one-shot CLI, and ACP example compositions use the same bounded policy. Library consumers retain explicit plugin composition: omitting the plugin leaves `agent/request-error` at its current fail default. +The agent-spine demo bundle loads the plugin so the shared stdio/TUI, one-shot CLI, and ACP example compositions use the same provider-routed policy. Library consumers retain explicit plugin composition: omitting the plugin leaves `agent/request-error` at its current fail default. ### Make one layer own visible attempts @@ -99,7 +89,7 @@ If recovery is exhausted, the final failure is stored once on `turn/end.reason` - Automatic provider or model failover. Requests already select one explicit provider and model, and the provider registry deliberately has one adapter owner per provider. - Retrying or continuing after a successful terminal finish, or splicing chunks from two attempts into one assistant message. - Repairing malformed tool arguments, refusals, content filters, or other semantic model output. -- Unbounded retries, unattended retry-until-cancelled behavior, circuit breakers, shared provider health, or cross-agent retry budgets. +- Circuit breakers, shared provider health, or cross-agent retry budgets. - Changing `llm/stream` into a response lifecycle or adding convenience generation APIs without a production consumer. ## Alternatives considered @@ -108,7 +98,7 @@ If recovery is exhausted, the final failure is stored once on `turn/end.reason` - **Add response start, interrupted, discarded, failed, and committed events to `dsh-llm`** — rejected because the agent log already separates raw chunks, successful messages, and numbered attempts. A second state machine would duplicate ownership without enabling the bounded same-route retry. - **Add logical routes, capability matrices, and failover selection** — rejected because current requests already name provider and model explicitly, one adapter owns each provider, and no current consumer requires automatic fallback or can prove semantic compatibility. - **Put `retryable` or `failover` on `LlmFailure`** — rejected because adapters report facts while deployment policy decides action. The same 429 may be retried in an interactive bundle and rejected in a cost-capped batch. -- **Retry forever while the caller remains active** — rejected because it gives one request unbounded cost and latency. Visible status makes bounded waiting understandable; it does not make an unlimited budget safe. +- **Retry forever while the caller remains active** — the [per-provider policy](../feature/2026-07-24-provider-retry-policies.md) supersedes this rejection for explicit `always` entries while retaining bounded normal mode as the default. - **Log retry status only through the process logger** — rejected because process logs do not reconstruct session behavior and cannot drive replayed UI state. - **Keep only flat codes** — rejected because retry delay and provider request id are structured provider facts, and HTTP status is necessary for diagnosis when different wire failures share one stable code. @@ -119,7 +109,7 @@ If recovery is exhausted, the final failure is stored once on `turn/end.reason` - DeepSeek and pi-ai adapter tests cover representative 400, 401/403, 429, 5xx, connection, malformed/truncated stream, timeout, abort, retry-after seconds/date, request-id, and unknown-SDK-error paths without recovery policy parsing message text. - Pi-ai pins the SDK option to zero retries and performs one observed wire attempt for a retryable provider response; separate tests make removing either boundary fail. - `agent/request-error` carries current failure facts plus immutable prior-retried failure facts; a success clears that history, and alternating transient/context-overflow integration tests prove the two policies consume only their own finite budgets. -- `dsh-llm-retry` validates every config field at Loader startup, delegates all ineligible paths with `next()`, and makes at most `maxTransientRetries + 1` provider requests when no other policy applies. +- Each provider adapter validates its nested retry policy at Loader startup, and `ctx.llm` captures it with the route; normal mode delegates ineligible paths and makes at most `maxRetries + 1` provider requests when no other policy applies. - HMR-during-backoff tests prove disposal unregisters the listener, aborts and awaits its captured callbacks, emits no retry decision after disposal, and leaves no timer or promise alive. - Pure unit tests cover transient-code selection, exponential backoff and jitter bounds, valid and over-cap `Retry-After`, exhausted budgets, deterministic timer/random seams, and abort during backoff. - Real agent-loop tests cover failure before chunks, partial chunks then failure, thrown and in-band failures, retry to success in a new step, exhaustion to structured `turn/end.reason`, and composition with `dsh-compact-basic` context-overflow recovery. @@ -130,12 +120,12 @@ If recovery is exhausted, the final failure is stored once on `turn/end.reason` ## Consequences -- Every transient recovery attempt is visible as a closed step plus `llm/retry`, and the bounded policy prevents hidden SDK retries from multiplying cost. A retry can still duplicate provider billing even when no chunk arrived; the finite attempt budget limits but cannot remove that risk. +- Every retry attempt is visible as a closed step plus `llm/retry`, and adapter-level single-attempt behavior prevents hidden SDK retries from multiplying policy decisions. A retry can still duplicate provider billing even when no chunk arrived; normal mode limits that risk, while explicit always mode accepts it until cancellation or success. - Provider SDKs may hide status or retry headers. Those adapters retain the stable facts they expose and otherwise use a coarse code rather than letting recovery policy parse fragile text. - Durable retry events expand the session protocol and UI state machine. Shipping the event and its consumer together prevents an unused telemetry vocabulary, but later schema changes still require persistence and replay work. - Clearing a failed step's live chunks can visibly retract output. That is preferable to presenting discarded text or partial tool JSON as committed history, and snapshots pin the transition. - Adapter-local idle enforcement stops stalled transports without counting consumer think time. Contract tests at each transport boundary guard against SDK drift. -- Multiple recovery plugins add their finite budgets. Their classifiers remain disjoint here; an overlapping classifier would be registration-order policy and must be documented and tested by the plugins that introduce it. +- Multiple normal recovery plugins add their finite budgets. Always mode delegates first and then supplies an unbounded fallback; overlapping classifiers remain registration-order policy and must be documented and tested by the plugins that introduce them. ## Related diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml new file mode 100644 index 0000000000..ee221716ce --- /dev/null +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write +2026-07-24-provider-retry-policies.md: 0ff304715aa5e1bfaf296e3f53e1da28d8b4042d +2026-07-24-provider-retry-policies.zh.md: 3b1d8f12a5fb37424e8b964f7bbb2be278263056 diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md new file mode 100644 index 0000000000..0ff304715a --- /dev/null +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md @@ -0,0 +1,65 @@ +# Agent Note: Per-provider request retry policies + +Status: implemented + +English | [中文](2026-07-24-provider-retry-policies.zh.md) + +## Problem + +One process may route model requests to providers with different reliability and cost constraints. A single transient classifier and finite retry budget cannot express a deployment that wants bounded recovery for most providers but requires one provider to keep retrying every model-request failure until the request succeeds or the caller cancels it. + +Provider policy must follow the request that actually failed, including a route selected by `agent/request`, rather than the agent's initial options. Unbounded policy also cannot store JavaScript `Infinity` in the durable session event, and neither provider error text nor discarded partial output may enter the next model request. + +## Decision + +Each concrete adapter accepts an optional `retryPolicy` inside its provider configuration. The adapter validates and resolves the policy, and `ctx.llm` captures it when that exact provider route registers. `@deepseek-ai/dsh-llm-retry` reads the registered policy for the provider whose step failed. A provider without `retryPolicy` uses the normal defaults. + +```yaml +providers: + - provider: deepseek + retryPolicy: + mode: normal + maxRetries: 2 + retryableCodes: [RATE_LIMIT, SERVER, TIMEOUT, TRANSPORT] + backoff: + initialDelayMs: 500 + maxDelayMs: 10000 + jitterRatio: 0.1 + - provider: internal + retryPolicy: + mode: always + backoff: + initialDelayMs: 1000 + maxDelayMs: 30000 + jitterRatio: 0.2 +``` + +The listener selects the policy from the durable `request/header` in force when the failed step closed, excluding later recovery mutations. Normal mode retains the bounded transient behavior: it retries configured codes up to `maxRetries`, counts retries scheduled by the same provider policy in the current consecutive failure sequence, and otherwise delegates. + +Always mode asks downstream recovery first so a specialized policy such as context-overflow compaction can make progress. A downstream retry wins. A downstream failure decision or thrown recovery error falls back to an unbounded retry of the same provider request; the thrown error is logged. Success, turn cancellation, and plugin disposal are the only termination paths. + +Both modes use exponential local delays from `initialDelayMs` to `maxDelayMs`. `jitterRatio` multiplies each target by a uniform sample in `[1 - jitterRatio, 1 + jitterRatio]`, then applies the cap. A positive provider `Retry-After` within the cap remains exact and unjittered. An over-cap provider delay makes normal mode delegate; always mode retains its guarantee by using the configured local backoff. + +Each scheduled retry appends a non-surface `llm/retry` event with the failed provider, policy mode, provider-policy retry number, delay, and failure facts. Normal events carry finite `maxRetries`; always events omit it, and UIs render the limit as `∞`. The event and failed `assistant/chunk` records do not contribute surface messages, so the next request contains the same derived context as the failed request unless another recovery policy deliberately changes the surface. + +## Alternatives considered + +**One global `always` switch** — rejected because it cannot isolate the unbounded cost and latency risk to the provider that needs it and can silently apply after runtime rerouting. + +**A separate exact-provider list on `dsh-llm-retry`** — rejected because it duplicates provider route names outside their owning adapter configuration and lets provider registration drift from recovery policy. + +**A very large finite retry count** — rejected because it eventually violates the requested keep-retrying contract and serializes an arbitrary operational limit as if it were meaningful. + +**Provider-SDK retries** — rejected because hidden attempts multiply agent-level budgets, cannot use the closed-step durability boundary, and may splice or discard streamed output without a reconstructable retry record. + +**Put the error into model context** — rejected because a transport or provider diagnostic is operational state, not conversation content. It can expose sensitive provider details and changes the retried request instead of repeating the failed request. + +## Verification + +Adapter tests validate nested policies at provider load and prove registration captures configured and default policies. Unit and real-Loader composition tests select policies from the failed request's provider, exercise always mode beyond the normal budget, pin jitter and delay caps, prove downstream recovery ordering, prove cancellation interrupts stalled downstream recovery, and prove cancellation and disposal stop active backoff waits. Request-level coverage compares the complete messages of failed and retried attempts and rejects both provider error text and discarded partial output. JSONL and SQLite tests round-trip an always event without `Infinity`; invariant tests bind its provider to the request header and its retry number to the active provider policy; ACP and TUI tests render finite and infinite limits. + +## Consequences + +Normal mode remains a finite default, while an explicit always policy can spend unbounded requests and time on permanent authentication, quota, invalid-request, protocol, or context failures. Operators must pair always mode with a cancellable caller and provider-specific cost controls. Retry state stays observable and durable without becoming model-visible, and exact-provider selection keeps one provider's exceptional policy from changing another provider's recovery behavior. + +This decision extends the closed-step recovery, single visible adapter attempt, structured failure, and durable status design in [bounded recovery for transient LLM request failures](../architecture/2026-06-21-bounded-llm-request-recovery.md). diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md new file mode 100644 index 0000000000..3b1d8f12a5 --- /dev/null +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md @@ -0,0 +1,65 @@ +# Agent Note: 逐提供方请求重试策略 + +Status: implemented + +[English](2026-07-24-provider-retry-policies.md) | 中文 + +## 问题 + +同一进程可能把模型请求路由到可靠性和成本约束各不相同的提供方。单一的瞬态错误分类器与有限重试预算无法表达这种部署需求：大多数提供方只需有界恢复，但其中一个提供方必须持续重试每次模型请求失败，直到请求成功或调用方取消。 + +提供方策略必须跟随实际失败的请求，包括 `agent/request` 选择的路由，而不能跟随 agent（智能体）的初始选项。无界策略也不能把 JavaScript `Infinity` 存入持久会话事件；提供方错误文本与丢弃的部分输出都不得进入下一次模型请求。 + +## 决策 + +每个具体适配器都在其提供方配置中接受可选的 `retryPolicy`。适配器负责校验并解析策略，`ctx.llm` 则在该精确提供方路由注册时捕获策略。`@deepseek-ai/dsh-llm-retry` 读取失败步骤对应提供方的已注册策略。未配置 `retryPolicy` 的提供方使用 normal 默认值。 + +```yaml +providers: + - provider: deepseek + retryPolicy: + mode: normal + maxRetries: 2 + retryableCodes: [RATE_LIMIT, SERVER, TIMEOUT, TRANSPORT] + backoff: + initialDelayMs: 500 + maxDelayMs: 10000 + jitterRatio: 0.1 + - provider: internal + retryPolicy: + mode: always + backoff: + initialDelayMs: 1000 + maxDelayMs: 30000 + jitterRatio: 0.2 +``` + +监听器根据失败步骤关闭时生效的持久 `request/header` 选择策略，后续恢复产生的改动不参与选择。normal 模式保留有界瞬态错误处理行为：它重试配置的错误代码，次数不超过 `maxRetries`；在当前连续失败序列中，同一提供方策略安排的重试都计入次数；其他情况委托后续处理。 + +always 模式先请求下游恢复，使上下文溢出压缩（compaction）之类的专用策略有机会取得进展。下游若决定重试，则以该决定为准。下游若决定失败或恢复过程抛出错误，则回退为无界重试同一提供方请求；抛出的错误会写入日志。成功、轮次取消和插件 dispose（资源释放）是仅有的终止路径。 + +两种模式的本地延迟都按指数增长，从 `initialDelayMs` 增至 `maxDelayMs`。`jitterRatio` 用 `[1 - jitterRatio, 1 + jitterRatio]` 区间内的均匀随机样本乘以每次目标值，再应用上限。提供方给出的正数 `Retry-After` 若未超过上限，则保持精确且不加抖动。若提供方延迟超过上限，normal 模式会委托后续处理；always 模式则改用配置的本地退避，以维持无限重试保证。 + +每次安排重试都会追加一条不进入表层的 `llm/retry` 事件，其中包含失败的提供方、策略模式、提供方策略内的重试编号、延迟和失败事实。normal 事件包含有限的 `maxRetries`；always 事件省略该字段，UI 将上限渲染为 `∞`。该事件与失败的 `assistant/chunk` 记录都不会生成表层消息，因此除非其他恢复策略有意改变表层，否则下一次请求包含的派生上下文与失败请求相同。 + +## 曾考虑的替代方案 + +**单一全局 `always` 开关**：不予采纳，因为它无法把无界成本与延迟风险限制在确有需要的提供方，还可能在运行时重新路由后悄然生效。 + +**在 `dsh-llm-retry` 上维护单独的精确提供方列表**：不予采纳，因为它会在所属适配器配置之外重复提供方路由名称，并让提供方注册与恢复策略发生偏差。 + +**设置很大的有限重试次数**：不予采纳，因为它最终仍会违反持续重试的契约，并把任意选取的运维上限序列化成看似有意义的数值。 + +**使用提供方 SDK 重试**：不予采纳，因为隐藏尝试会叠加 agent 层预算，无法利用已关闭 step 的持久性边界，还可能在没有可重建重试记录的情况下拼接或丢弃流式输出。 + +**把错误放入模型上下文**：不予采纳，因为传输或提供方诊断信息属于运维状态，而非对话内容。它可能暴露敏感的提供方细节，并会改变重试请求，无法重复原本失败的请求。 + +## 验证 + +适配器测试会在提供方加载时校验嵌套策略，并证明注册流程会捕获已配置策略和默认策略。单元测试与真实 Loader 组合测试根据失败请求的提供方选择策略、验证 always 模式可越过 normal 预算、固定抖动和延迟上限、证明下游恢复顺序、证明取消会中断停滞的下游恢复，并证明取消与 dispose 会停止正在进行的退避等待。请求级覆盖会比较失败尝试与重试尝试的完整消息，并排除提供方错误文本和丢弃的部分输出。JSONL 与 SQLite 测试会往返读写不含 `Infinity` 的 always 事件；不变式测试会将事件中的提供方绑定到请求头，并将重试编号绑定到活跃的提供方策略；ACP 与 TUI 测试会分别渲染有限和无限上限。 + +## 后果 + +normal 模式仍是有限的默认策略；显式的 always 策略可能在永久性的身份验证、配额、无效请求、协议或上下文错误上耗费无限次请求和无限时间。运维方必须为 always 模式配备可取消的调用方和针对提供方的成本控制。重试状态保持可观察且持久，但不会对模型可见；精确提供方选择也能避免某个提供方的例外策略改变其他提供方的恢复行为。 + +本决策扩展了[瞬态 LLM（大语言模型）请求失败的有界恢复](../architecture/2026-06-21-bounded-llm-request-recovery.md)中确定的已关闭 step 恢复、单次可见适配器尝试、结构化失败与持久状态设计。 diff --git a/docs/architecture.i18n.yaml b/docs/architecture.i18n.yaml index 466a96dd30..afa1d20efd 100644 --- a/docs/architecture.i18n.yaml +++ b/docs/architecture.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -architecture.md: 5a0ff63413a0c2a59d042f935d341dd39234f669 -architecture.zh.md: e1fb143982ad968fe6be2a5f6154722a602a297b +architecture.md: 3347e529f7e2c7b4c376d01132851d883c90182e +architecture.zh.md: f3d3bf8b19bace2124b89c3b6ee7e4587445b9fe diff --git a/docs/architecture.md b/docs/architecture.md index 5a0ff63413..3347e529f7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -119,7 +119,7 @@ Each step assembles ordered prompt sections, tool schemas, and variables; unknow Tool-time context—including async `inject()` and post-tool `additionalContexts`—settles after results. Steering drains before `agent/post-step`, which sees durable output, results, context, and steering. Leftovers queue. Terminal `agent/turn-stop` remains authoritative through close/flush; later steering is discarded while queued prompts remain. -Pruning precedes summaries; overflow retries require durable progress. Bounded transient retries compose on `agent/request-error`; cancellation wins ([compaction](../.agents/notes/implemented/architecture/2026-07-10-after-call-compaction-pressure-and-overflow-recovery.md), [retry](../.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md)). +Pruning precedes summaries; overflow retries require durable progress. Adapters register nested `retryPolicy`; normal bounds transient failures, while always retries until success or cancellation ([compaction](../.agents/notes/implemented/architecture/2026-07-10-after-call-compaction-pressure-and-overflow-recovery.md), [retry foundation](../.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md), [provider policy](../.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md)). ### Failure Boundaries diff --git a/docs/architecture.zh.md b/docs/architecture.zh.md index e1fb143982..f3d3bf8b19 100644 --- a/docs/architecture.zh.md +++ b/docs/architecture.zh.md @@ -119,7 +119,7 @@ forever: 工具执行阶段的上下文，包括异步 `inject()` 和工具执行后的 `additionalContexts`，会在结果产生后稳定。steering（中途引导）会在 `agent/post-step` 前排空；该事件会观察持久输出、结果、上下文和 steering。余留内容进入队列。终止型 `agent/turn-stop` 在关闭和刷写期间始终具有最终决定权；后续 steering 会被丢弃，排队提示词仍予保留。 -裁剪先于摘要；溢出重试必须取得持久进展。有界的瞬态重试在 `agent/request-error` 上组合；取消优先（[压缩](../.agents/notes/implemented/architecture/2026-07-10-after-call-compaction-pressure-and-overflow-recovery.md)、[重试](../.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md)）。 +裁剪先于摘要；溢出重试必须取得持久进展。适配器会注册嵌套的 `retryPolicy`；normal 限制瞬态错误重试次数，always 则持续重试，直至成功或取消（[压缩](../.agents/notes/implemented/architecture/2026-07-10-after-call-compaction-pressure-and-overflow-recovery.md)、[重试基础](../.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md)、[提供方策略](../.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md)）。 ### 失败边界 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 49a0a1510f..2036a34307 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -76,8 +76,6 @@ export interface Config { toolTasks?: NonNullable /** Persisted same-session goals; owner defaults enable them, or false disables the stack and command. */ goals?: agentCore.GoalConfig | false - /** Bounded transient model-request retry policy forwarded through agent-core. */ - llmRetry?: NonNullable } ``` @@ -127,8 +125,9 @@ Source: [`packages/core/agent-loop/src/index.ts:360`](../packages/core/agent-loo * `dshHome` to bash environment and local skill discovery, `sessionTitle` to * the fallback title service, `skills` to the * skill registry/local provider/tool consumer, `workspaceContext` to the - * workspace-context loader, `llmRetry` to the bounded request-recovery policy, - * and `toolBash`/`toolTasks` to the model-facing tool plugins this bundle owns. + * workspace-context loader, and `toolBash`/`toolTasks` to the model-facing tool + * plugins this bundle owns. Provider adapters own their `retryPolicy`; this + * bundle always mounts its executor. * `goals` opts into and configures the persisted goal domain plus its model tool * and same-session driver; `invariants` configures global and package-filtered * relational checks. Owner schemas supply defaults for optional input; @@ -164,8 +163,6 @@ export interface Config { invariants?: InvariantConfig /** Opt-in persisted same-session goal stack; set false or omit to leave it unmounted. */ goals?: GoalConfig | false - /** Bounded transient model-request retry policy. */ - llmRetry?: llmRetry.Config } /** Skill bundle config forwarded to the registry, local provider, and model-facing consumer. */ @@ -189,9 +186,9 @@ export interface GoalConfig { } ``` -Depends on: [`AgentLoopConfig`](#deepseek-aidsh-agent-loop) · [`GoalDomainConfig`](#deepseek-aidsh-goal) · [`InvariantConfig`](#deepseek-aidsh-invariants) · [`llmRetry`](../packages/llm/llm-retry/src/index.ts) · [`SessionTitleConfig`](#deepseek-aidsh-session-title) · [`SkillLocal`](../packages/skill/skill-local/src/index.ts) · [`SkillRegistryConfig`](#deepseek-aidsh-skill) · [`SystemPromptConfig`](#deepseek-aidsh-system-prompt) · [`toolBash`](../packages/bash/tool-bash/src/index.ts) · [`toolGoal`](../packages/goal/tool-goal/src/index.ts) · [`ToolsConfig`](#deepseek-aidsh-tools) · [`toolSkill`](../packages/skill/tool-skill/src/index.ts) · [`toolTasks`](../packages/tasks/tool-tasks/src/index.ts) · [`workspaceContext`](../packages/context/workspace-context/src/index.ts) +Depends on: [`AgentLoopConfig`](#deepseek-aidsh-agent-loop) · [`GoalDomainConfig`](#deepseek-aidsh-goal) · [`InvariantConfig`](#deepseek-aidsh-invariants) · [`SessionTitleConfig`](#deepseek-aidsh-session-title) · [`SkillLocal`](../packages/skill/skill-local/src/index.ts) · [`SkillRegistryConfig`](#deepseek-aidsh-skill) · [`SystemPromptConfig`](#deepseek-aidsh-system-prompt) · [`toolBash`](../packages/bash/tool-bash/src/index.ts) · [`toolGoal`](../packages/goal/tool-goal/src/index.ts) · [`ToolsConfig`](#deepseek-aidsh-tools) · [`toolSkill`](../packages/skill/tool-skill/src/index.ts) · [`toolTasks`](../packages/tasks/tool-tasks/src/index.ts) · [`workspaceContext`](../packages/context/workspace-context/src/index.ts) -Source: [`packages/examples/agent-spine-demo/src/index.ts:87`](../packages/examples/agent-spine-demo/src/index.ts) +Source: [`packages/examples/agent-spine-demo/src/index.ts:88`](../packages/examples/agent-spine-demo/src/index.ts) ## `@deepseek-ai/dsh-bash-local` @@ -265,8 +262,6 @@ export interface Config { toolBash?: NonNullable /** Generic background-task control-tool config forwarded through agent-spine-demo. */ toolTasks?: NonNullable - /** Bounded transient model-request retry policy forwarded through agent-spine-demo. */ - llmRetry?: NonNullable /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ workspaceContext: agentCore.Config['workspaceContext'] } @@ -538,6 +533,8 @@ export interface Config { models?: DeepSeekCatalogModel[] /** Maximum provider idle time while one stream read is outstanding (default five minutes). */ streamIdleTimeoutMs?: number + /** Provider-owned model-request retry policy; omission uses normal defaults. */ + retryPolicy?: RetryPolicyConfig } /** One optional model entry advertised by the hand-written adapter. */ @@ -553,7 +550,9 @@ export interface DeepSeekCatalogModel { } ``` -Source: [`packages/llm/llm-deepseek/src/index.ts:34`](../packages/llm/llm-deepseek/src/index.ts) +Depends on: [`RetryPolicyConfig`](../packages/llm/llm/src/index.ts) + +Source: [`packages/llm/llm-deepseek/src/index.ts:35`](../packages/llm/llm-deepseek/src/index.ts) ## `@deepseek-ai/dsh-llm-pi-ai` @@ -590,12 +589,14 @@ export interface PiAiProviderProfile { websocketConnectTimeoutMs?: number /** Maximum provider idle time while one stream read is outstanding. */ streamIdleTimeoutMs?: number + /** Provider-owned model-request retry policy; omission uses normal defaults. */ + retryPolicy?: RetryPolicyConfig } ``` -Depends on: `CacheRetention` (`@earendil-works/pi-ai`) · `ThinkingBudgets` (`@earendil-works/pi-ai`) · `ThinkingLevel` (`@earendil-works/pi-ai`) · `Transport` (`@earendil-works/pi-ai`) +Depends on: `CacheRetention` (`@earendil-works/pi-ai`) · [`RetryPolicyConfig`](../packages/llm/llm/src/index.ts) · `ThinkingBudgets` (`@earendil-works/pi-ai`) · `ThinkingLevel` (`@earendil-works/pi-ai`) · `Transport` (`@earendil-works/pi-ai`) -Source: [`packages/llm/llm-pi-ai/src/config.ts:48`](../packages/llm/llm-pi-ai/src/config.ts) +Source: [`packages/llm/llm-pi-ai/src/config.ts:54`](../packages/llm/llm-pi-ai/src/config.ts) ## `@deepseek-ai/dsh-llm-replay` @@ -645,25 +646,14 @@ Source: [`packages/support/llm-replay/src/index.ts:387`](../packages/support/llm ## `@deepseek-ai/dsh-llm-retry` -Requires: `agents` +Requires: `agents` · `llm` ```ts config-catalog -/** Deployment-owned limits and classification for transient request recovery. */ -export interface Config { - /** Maximum transient retries after the first request (default 2). */ - maxTransientRetries?: number - /** Initial local exponential-backoff delay in milliseconds (default 500). */ - initialDelayMs?: number - /** Maximum accepted or locally scheduled delay in milliseconds (default 10000). */ - maxDelayMs?: number - /** Symmetric random multiplier range around one (default 0.1). */ - jitterRatio?: number - /** Stable failure codes eligible for this policy. */ - retryableCodes?: string[] -} +/** This policy executor has no config; providers own `retryPolicy`. */ +export type Config = Readonly> ``` -Source: [`packages/llm/llm-retry/src/index.ts:39`](../packages/llm/llm-retry/src/index.ts) +Source: [`packages/llm/llm-retry/src/index.ts:43`](../packages/llm/llm-retry/src/index.ts) ## `@deepseek-ai/dsh-lsp-local` diff --git a/docs/cordis-catalog/events.md b/docs/cordis-catalog/events.md index 6becfd9434..f301ff6528 100644 --- a/docs/cordis-catalog/events.md +++ b/docs/cordis-catalog/events.md @@ -547,7 +547,7 @@ Waterfall around every streaming model call (retry, replay, routing). Bound to t Types: [GenerateOptions](../core-data-structures/core.md) · [LlmService](../core-data-structures/llm-streaming.md) · [StreamChunk](../core-data-structures/llm-streaming.md) -Source: [`packages/llm/llm/src/index.ts:52`](../../packages/llm/llm/src/index.ts) +Source: [`packages/llm/llm/src/index.ts:55`](../../packages/llm/llm/src/index.ts) ## `session/*` diff --git a/docs/cordis-catalog/services.md b/docs/cordis-catalog/services.md index 5858799e7c..99c89b7ff3 100644 --- a/docs/cordis-catalog/services.md +++ b/docs/cordis-catalog/services.md @@ -653,6 +653,13 @@ registerAdapter(providers: string[], adapter: LlmAdapter): () => void */ listProviders(): LlmProviderInfo[] +/** + * Resolve the retry policy captured when one provider route was registered. + * @param provider - registered provider route to inspect. + * @returns the provider-owned policy, with normal defaults already resolved. + */ +providerRetryPolicy(provider: string): ResolvedRetryPolicy + /** * Discover models advertised by one registered provider. Catalog membership * is advisory and never changes routing or request validation. @@ -686,9 +693,9 @@ async resolveModelContext( provider: string, model: string, ): Promise ``` -Types: [GenerateOptions](../core-data-structures/core.md) · [LlmAdapter](../core-data-structures/llm-streaming.md) · [LlmModelContext](../core-data-structures/core.md) · [LlmModelInfo](../core-data-structures/core.md) · [LlmProviderInfo](../core-data-structures/core.md) · [StreamChunk](../core-data-structures/llm-streaming.md) +Types: [GenerateOptions](../core-data-structures/core.md) · [LlmAdapter](../core-data-structures/llm-streaming.md) · [LlmModelContext](../core-data-structures/core.md) · [LlmModelInfo](../core-data-structures/core.md) · [LlmProviderInfo](../core-data-structures/core.md) · [ResolvedRetryPolicy](../core-data-structures/llm-streaming.md) · [StreamChunk](../core-data-structures/llm-streaming.md) -Source: [`packages/llm/llm/src/index.ts:159`](../../packages/llm/llm/src/index.ts) +Source: [`packages/llm/llm/src/index.ts:171`](../../packages/llm/llm/src/index.ts) ## `ctx.permission` — `PermissionService` diff --git a/docs/core-data-structures/llm-streaming.md b/docs/core-data-structures/llm-streaming.md index 257ce90cda..820454495a 100644 --- a/docs/core-data-structures/llm-streaming.md +++ b/docs/core-data-structures/llm-streaming.md @@ -66,6 +66,10 @@ Every adapter MUST obey these, and every consumer may rely on them: This contract is pinned down by two deliberately independent implementations: `dsh-llm-deepseek` (hand-rolled fetch/SSE) and `dsh-llm-pi-ai` (a generic multi-provider adapter through `@earendil-works/pi-ai`). The library-backed adapter exercises the finish-chunk error path, while transport-boundary tests prove each idle watchdog stops its actual request. +## `ResolvedRetryPolicy` + +Provider configuration resolves before route registration into an immutable discriminated union. Normal mode carries `mode: 'normal'`, finite `maxRetries`, `retryableCodes`, and required `initialDelayMs`, `maxDelayMs`, and `jitterRatio`; always mode carries `mode: 'always'` and the same required backoff fields without a finite maximum. `LlmService.providerRetryPolicy(provider)` returns the captured value and supplies normal defaults when the adapter omits one. The [generated config catalog](../config-catalog.md) owns the optional input shapes. + ## `AppIdentity` — app attribution The static public application identity every adapter sends to providers ([`packages/llm/llm/src/attribution.ts`](../../packages/llm/llm/src/attribution.ts)). `attributionHeaders(identity?)` maps it to the standard `User-Agent` header only; OpenRouter-specific app attribution headers are intentionally not supported by this contract. The default `APP_IDENTITY` sources its version from the package manifest; every field is a public product fact - no secrets, paths, session ids, or per-user identifiers, and nothing per-request may influence the values. Rationale: [Mandatory `User-Agent` attribution](../../.agents/notes/implemented/architecture/2026-06-21-mandatory-app-attribution-headers.md). @@ -154,7 +158,7 @@ declare class BlockAssembler { ## The seam -`LlmAdapter` is the provider seam: subclass, implement `stream()`, and register one adapter instance with `ctx.llm.registerAdapter(providers, adapter)`. `GenerateOptions.provider` selects the registered adapter; `GenerateOptions.model` is passed to that adapter and need not be registered at lifecycle start. Duplicate provider routes fail atomically. Optional `providerInfo()` and asynchronous `listModels()` methods feed `LlmService.listProviders()` / `listModels()` with detached selector metadata. That catalog is advisory rather than a request whitelist: the adapter remains authoritative and may accept unlisted model ids. The separate `resolveModelContext()` query exposes correctness-sensitive capacity for an exact route without making catalog membership authoritative; absence means unknown metadata, not invalid routing. Adapter lookup happens at the terminal continuation of the `llm/stream` waterfall, so a listener may short-circuit the call or route a mutable one-shot request before lookup. The `block-start` / `block-end` `index` correlation and the assembler together mean an adapter only has to emit well-formed chunks — block reassembly is not each adapter's problem. The consumer surface (`ctx.llm.stream()`) and the `llm/stream` waterfall are described in [architecture.md § Content blocks and streaming](../architecture.md#content-blocks-and-streaming-dsh-llm). +`LlmAdapter` is the provider seam: subclass, implement `stream()`, and register one adapter instance with `ctx.llm.registerAdapter(providers, adapter)`. `GenerateOptions.provider` selects the registered adapter; `GenerateOptions.model` is passed to that adapter and need not be registered at lifecycle start. Duplicate provider routes fail atomically. Optional `providerRetryPolicy()` is captured per route with normal defaults, while `providerInfo()` and asynchronous `listModels()` feed `LlmService.listProviders()` / `listModels()` with detached selector metadata. That catalog is advisory rather than a request whitelist: the adapter remains authoritative and may accept unlisted model ids. The separate `resolveModelContext()` query exposes correctness-sensitive capacity for an exact route without making catalog membership authoritative; absence means unknown metadata, not invalid routing. Adapter lookup happens at the terminal continuation of the `llm/stream` waterfall, so a listener may short-circuit the call or route a mutable one-shot request before lookup. The `block-start` / `block-end` `index` correlation and the assembler together mean an adapter only has to emit well-formed chunks — block reassembly is not each adapter's problem. The consumer surface (`ctx.llm.stream()`) and the `llm/stream` waterfall are described in [architecture.md § Content blocks and streaming](../architecture.md#content-blocks-and-streaming-dsh-llm). ```ts public-api /** @@ -170,6 +174,12 @@ declare abstract class LlmAdapter { * @returns detached display metadata whose id must equal `provider`. */ providerInfo(provider: string): LlmProviderInfo; + /** + * Return the provider-owned retry policy captured with this route. + * @param _provider - a route passed to `registerAdapter()` for this instance. + * @returns a resolved policy, or `undefined` to use the normal defaults. + */ + providerRetryPolicy(_provider: string): ResolvedRetryPolicy | undefined; /** * List models this adapter can currently advertise for one owned provider. * The result is advisory: an adapter may accept unlisted model ids, and diff --git a/docs/event-producer-consumer.md b/docs/event-producer-consumer.md index 179ad0dcee..9db87fd0a3 100644 --- a/docs/event-producer-consumer.md +++ b/docs/event-producer-consumer.md @@ -30,7 +30,7 @@ This matrix shows which packages dispatch each harness-owned event and which pac | `fs/observed` | `emit` | [`packages/fs/fs/src/index.ts:71`](../packages/fs/fs/src/index.ts) | [`tool-fs`](../packages/fs/tool-fs) (`emit`) | [`fs-policy`](../packages/fs/fs-policy) | | `fs/write-intent` | `waterfall` | [`packages/fs/fs/src/index.ts:54`](../packages/fs/fs/src/index.ts) | [`tool-fs`](../packages/fs/tool-fs) (`waterfall`) | [`fs-policy`](../packages/fs/fs-policy) | | `goal/changed` | `emit` | [`packages/goal/goal/src/types.ts:167`](../packages/goal/goal/src/types.ts) | [`goal`](../packages/goal/goal) (`emit`) | [`goal-session`](../packages/goal/goal-session) | -| `llm/stream` | `waterfall` | [`packages/llm/llm/src/index.ts:52`](../packages/llm/llm/src/index.ts) | [`llm`](../packages/llm/llm) (`waterfall`) | [`agent-loop`](../packages/core/agent-loop), [`llm`](../packages/llm/llm), [`llm-replay`](../packages/support/llm-replay), [`session-checkpoint-policy`](../packages/session-persistence/session-checkpoint-policy), [`session-title`](../packages/session-title/session-title) | +| `llm/stream` | `waterfall` | [`packages/llm/llm/src/index.ts:55`](../packages/llm/llm/src/index.ts) | [`llm`](../packages/llm/llm) (`waterfall`) | [`agent-loop`](../packages/core/agent-loop), [`llm`](../packages/llm/llm), [`llm-replay`](../packages/support/llm-replay), [`session-checkpoint-policy`](../packages/session-persistence/session-checkpoint-policy), [`session-title`](../packages/session-title/session-title) | | `session/created` | `emit` | [`packages/core/session/src/index.ts:79`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`compact`](../packages/compact/compact), [`goal`](../packages/goal/goal), [`hook-protocol`](../packages/hooks/hook-protocol), [`jsonrpc`](../packages/ui/jsonrpc), [`llm-retry`](../packages/llm/llm-retry), `runtime`, [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence), [`user-approval`](../packages/ui/user-approval) | | `session/disposed` | `emit` | [`packages/core/session/src/index.ts:89`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`agent-loop`](../packages/core/agent-loop), `runtime`, [`session-persistence`](../packages/session-persistence/session-persistence), [`session-title`](../packages/session-title/session-title) | | `session/event` | `emit` | [`packages/core/session/src/index.ts:101`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`acp`](../packages/ui/acp), [`cli-demo`](../packages/examples/cli-demo), [`compact`](../packages/compact/compact), [`goal`](../packages/goal/goal), [`goal-session`](../packages/goal/goal-session), [`hook-protocol`](../packages/hooks/hook-protocol), [`jsonrpc`](../packages/ui/jsonrpc), `runtime`, [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence), [`session-title`](../packages/session-title/session-title), [`token-meter`](../packages/llm/token-meter), [`tui`](../packages/ui/tui), [`user-approval`](../packages/ui/user-approval), [`workspace-context`](../packages/context/workspace-context) | diff --git a/docs/persistence-catalog.md b/docs/persistence-catalog.md index 8c1d815bc0..0dce9a770c 100644 --- a/docs/persistence-catalog.md +++ b/docs/persistence-catalog.md @@ -299,14 +299,24 @@ Source: [`packages/hooks/hook-protocol/src/types.ts:31`](../packages/hooks/hook- #### `llm/retry` — log-only ```ts persistence-catalog -/** Durable, non-surface record of one transient retry scheduled after a closed failed step. */ +/** Durable, non-surface record of one provider-routed retry scheduled after a closed failed step. */ 'llm/retry': { turn: number step: number + provider: string + mode: 'normal' retry: number maxRetries: number delayMs: number failure: LlmFailure +} | { + turn: number + step: number + provider: string + mode: 'always' + retry: number + delayMs: number + failure: LlmFailure } ``` diff --git a/packages/compact/compact-basic/tests/compact-loop-repro.spec.ts b/packages/compact/compact-basic/tests/compact-loop-repro.spec.ts index f9f61e4096..86b0fdc6fb 100644 --- a/packages/compact/compact-basic/tests/compact-loop-repro.spec.ts +++ b/packages/compact/compact-basic/tests/compact-loop-repro.spec.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from 'vitest' import { Context } from 'cordis' import { toolPairingBalancedAfter, toolPairingBalancedBefore } from '@deepseek-ai/dsh-compact' -import { CONTEXT_WINDOW_EXCEEDED_CODE, LlmError } from '@deepseek-ai/dsh-llm' -import type { ContentBlock, GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm' +import { CONTEXT_WINDOW_EXCEEDED_CODE, LlmError, resolveRetryPolicy } from '@deepseek-ai/dsh-llm' +import type { ContentBlock, GenerateOptions, ResolvedRetryPolicy, StreamChunk } from '@deepseek-ai/dsh-llm' import { CallId, LlmAdapter } from '@deepseek-ai/dsh-llm' import { defineContentToolFixture } from '@deepseek-ai/dsh-tools' import type { Agent } from '@deepseek-ai/dsh-agent' @@ -68,6 +68,11 @@ class StepwiseToolAdapter extends LlmAdapter { class OverflowRecoveryAdapter extends LlmAdapter { readonly conversationRequests: GenerateOptions[] = [] readonly summaryRequests: GenerateOptions[] = [] + private readonly retryPolicy = resolveRetryPolicy({ + mode: 'normal', + maxRetries: 1, + backoff: { initialDelayMs: 1, maxDelayMs: 1, jitterRatio: 0 }, + }, 'compaction test provider retryPolicy') constructor( private readonly delivery: 'thrown' | 'in-band', @@ -80,6 +85,10 @@ class OverflowRecoveryAdapter extends LlmAdapter { return Promise.resolve({ contextWindow: 128 }) } + override providerRetryPolicy(_provider: string): ResolvedRetryPolicy { + return this.retryPolicy + } + override async * stream(options: GenerateOptions): AsyncIterable { // The cache-reusing summarizer replays the conversation prefix and marks // its call only by the compaction instruction in the trailing user message. @@ -340,12 +349,7 @@ describe('context-overflow recovery across the real loop and compact-basic', () const adapter = new OverflowRecoveryAdapter('thrown', true) await mountAgentLoopTestDependencies(ctx) await mountInvariants(ctx) - await ctx.plugin(LlmRetry, { - maxTransientRetries: 1, - initialDelayMs: 1, - maxDelayMs: 1, - jitterRatio: 0, - }) + await ctx.plugin(LlmRetry) await ctx.plugin(AgentLoop, { agents: [] }) await ctx.plugin(TokenMeterService) ctx.llm.registerAdapter(['mock'], adapter) diff --git a/packages/cordis/tool-cordis/src/api-catalog.ts b/packages/cordis/tool-cordis/src/api-catalog.ts index 6939ec9927..d25338d99b 100644 --- a/packages/cordis/tool-cordis/src/api-catalog.ts +++ b/packages/cordis/tool-cordis/src/api-catalog.ts @@ -336,6 +336,10 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ signature: 'listProviders(): LlmProviderInfo[]', jsDoc: '/**\n * Describe provider routes with a registered adapter.\n * @returns detached provider metadata in registration order.\n */', }, + { + signature: 'providerRetryPolicy(provider: string): ResolvedRetryPolicy', + jsDoc: '/**\n * Resolve the retry policy captured when one provider route was registered.\n * @param provider - registered provider route to inspect.\n * @returns the provider-owned policy, with normal defaults already resolved.\n */', + }, { signature: 'async listModels(provider: string): Promise', jsDoc: '/**\n * Discover models advertised by one registered provider. Catalog membership\n * is advisory and never changes routing or request validation.\n * @param provider - registered provider route to inspect.\n * @returns detached model metadata in adapter-preferred order.\n */', @@ -1693,6 +1697,22 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'ReasoningBlock', declaration: 'export interface ReasoningBlock {\n type: \'reasoning\';\n text: string;\n}', }, + { + name: 'ResolvedAlwaysRetryPolicy', + declaration: 'export interface ResolvedAlwaysRetryPolicy extends ResolvedRetryBackoff {\n readonly mode: \'always\';\n}', + }, + { + name: 'ResolvedNormalRetryPolicy', + declaration: 'export interface ResolvedNormalRetryPolicy extends ResolvedRetryBackoff {\n readonly mode: \'normal\';\n readonly maxRetries: number;\n readonly retryableCodes: readonly string[];\n}', + }, + { + name: 'ResolvedRetryBackoff', + declaration: 'export interface ResolvedRetryBackoff {\n readonly initialDelayMs: number;\n readonly maxDelayMs: number;\n readonly jitterRatio: number;\n}', + }, + { + name: 'ResolvedRetryPolicy', + declaration: 'export type ResolvedRetryPolicy = ResolvedNormalRetryPolicy | ResolvedAlwaysRetryPolicy;', + }, { name: 'ResumeAgentOptions', declaration: 'export interface ResumeAgentOptions {\n readonly resumeSessionId: SessionId;\n readonly agentOptions?: AgentOptions;\n readonly signal?: AbortSignal;\n readonly setup?: (agentCtx: Context) => Promise | void;\n}', diff --git a/packages/core/agent-loop/README.md b/packages/core/agent-loop/README.md index 79fbbd7824..3314dfd7c1 100644 --- a/packages/core/agent-loop/README.md +++ b/packages/core/agent-loop/README.md @@ -69,7 +69,7 @@ Within a step, exclusive calls form barriers; parallel-safe calls use a bounded Everything that goes beyond "call the model, run the tools, repeat" belongs to plugins listening on the event taxonomy: - Hooks and policy: the relevant `agent/*` checkpoints plus the guarded `tools/pre-execute` → `tools/execute` → `tools/post-execute` → definition-owned `finalizeContent` → `tools/result` pipeline; exact event signatures and modes live in the [generated event catalog](../../../docs/cordis-catalog/events.md) - Compaction: pressure on `agent/post-step`; canonical context overflow on `agent/request-error` -- Transient model recovery: `dsh-llm-retry` on `agent/request-error`, with finite code-specific budgets and non-surface `llm/retry` status events +- Model-request recovery: `dsh-llm-retry` on `agent/request-error`, with exact-provider normal or unbounded policies and non-surface `llm/retry` status events - Sandbox, permission, plan mode: `tools/pre-execute` for extensible deny/ask, `tools.guard()` for monotonic owner policy, `tools/post-execute` for result decisions, and `tools/result` for final observation - Sub-agents: implemented outside the loop as `ctx.subagents` providers; in-process providers use `ctx.agents.create()` and owned `AgentHandle` teardown, while generic [`ctx.tasks`](../../tasks/tasks/) plus [`dsh-tool-subagent`](../../subagent/tool-subagent/) own background collection. - Persistence: `session/event` + `session/flush` diff --git a/packages/examples/acp-demo/README.md b/packages/examples/acp-demo/README.md index 93d89d6557..ef46d95f98 100644 --- a/packages/examples/acp-demo/README.md +++ b/packages/examples/acp-demo/README.md @@ -42,7 +42,6 @@ The app owns this cluster through one ordered Cordis effect. Teardown drains the | `toolBash` | owner defaults | model-facing bash config routed through `dsh-agent-spine-demo`, including bash's producer-local `enableRunInBackground` | | `toolTasks` | owner defaults | generic `task_output` wait bounds routed through `dsh-agent-spine-demo` | | `goals` | owner defaults | persisted goal-domain and model-tool config; `false` removes the goal stack and `/goal` producer | -| `llmRetry` | owner defaults | bounded transient model-request retry policy routed through `dsh-agent-spine-demo` | | `persistenceRoot` | `./.sessions` | the JSONL backend's root directory and the parent of the derived `session-query.db` index | | `packChunks` | `false` | write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`) | | `persistenceCompression` | `'zstd'` | JSONL artifact encoding (`'zstd'` or raw `'none'`) | diff --git a/packages/examples/acp-demo/src/index.ts b/packages/examples/acp-demo/src/index.ts index 1a05a14e3e..3772e2ce1c 100644 --- a/packages/examples/acp-demo/src/index.ts +++ b/packages/examples/acp-demo/src/index.ts @@ -76,8 +76,6 @@ export interface Config { toolTasks?: NonNullable /** Persisted same-session goals; owner defaults enable them, or false disables the stack and command. */ goals?: agentCore.GoalConfig | false - /** Bounded transient model-request retry policy forwarded through agent-core. */ - llmRetry?: NonNullable } // Each front door owns a complete, directly readable config schema; extracting @@ -104,7 +102,6 @@ export const Config: z = z.object({ toolBash: agentCore.ToolBashConfigSchema, toolTasks: z.union([z.const(false), agentCore.ToolTasksConfigSchema]), goals: z.union([z.const(false), agentCore.GoalConfigSchema]), - llmRetry: agentCore.LlmRetryConfigSchema, }) /* jscpd:ignore-end */ diff --git a/packages/examples/agent-spine-demo/README.md b/packages/examples/agent-spine-demo/README.md index 57d5922535..af8b0af0f2 100644 --- a/packages/examples/agent-spine-demo/README.md +++ b/packages/examples/agent-spine-demo/README.md @@ -21,7 +21,7 @@ Read this package for the whole plugin tree and its composition order. @deepseek-ai/dsh-goal optional persisted same-session goal domain @deepseek-ai/dsh-tool-goal optional model-facing goal controls @deepseek-ai/dsh-goal-session optional same-session goal-round driver -@deepseek-ai/dsh-llm-retry bounded transient request retry policy +@deepseek-ai/dsh-llm-retry provider-routed request retry policy @deepseek-ai/dsh-tasks generic background-task registry @deepseek-ai/dsh-invariants configurable invariant registry service @deepseek-ai/dsh-session/invariant @@ -53,11 +53,11 @@ This is the [interface/implementation/consumer seam](../../../.agents/notes/impl ```ts import type { Config } from '@deepseek-ai/dsh-agent-spine-demo' -// { agents?, maxParallelToolCalls?, persona?, toolOrder?, tools?, dshHome?, sessionTitle?, skills?, workspaceContext, toolBash?, toolTasks?, goals?, invariants?, llmRetry? } +// { agents?, maxParallelToolCalls?, persona?, toolOrder?, tools?, dshHome?, sessionTitle?, skills?, workspaceContext, toolBash?, toolTasks?, goals?, invariants? } // workspaceContext requires { maxBytes } or false; the other owner schemas supply defaults. ``` -The bundle FORWARDS each field to the child that owns it: `agents` and `maxParallelToolCalls` to `agent-loop` (`agents` defaults to `[]`; the cap defaults there), so each app supplies its own pre-created agents — TUI and headless apps pre-create `main`, while the ACP app creates agents on demand at `session/new`; `llmRetry` to the bounded retry policy; `persona` and `toolOrder` to `dsh-system-prompt`; `tools` to the tool registry for its presentation mode; `sessionTitle` to the fallback title service; `skills.registry`, `skills.local`, and `skills.tool` to the skill registry, local provider, and model-facing consumer; the required `workspaceContext` choice to `dsh-workspace-context` (`{ maxBytes }` enables loading and `false` disables it); `invariants` to the invariant service; and `toolBash`/`toolTasks` to the two model-facing tool plugins the bundle owns. Omitted `sessionTitle` uses the explicit example policy of 5 words, 40 fallback bytes, and 80 accepted-title bytes. A `goals` object opts into the persisted domain, model tools, and same-session driver while forwarding `goals.domain` and `goals.tool` to their owners; omission or `false` leaves the stack absent so headless callers retain one-turn settlement. Set `skills.enabled: false` to omit both the local provider and model-facing skill tool, and set `toolTasks: false` to retain the task service for foreground producers without exposing `task_output` / `task_list` / `task_kill`. It resolves `dshHome` once through [`@deepseek-ai/dsh-paths`](../../util/paths/README.md) and forwards that absolute value to tool-bash's managed environment and enabled local skill discovery. An absent top-level `dshHome` adopts `skills.local.dshHome`; supplying both with different resolved paths fails loudly. `toolBash.enableRunInBackground` controls only the bash producer; independently loaded producers keep their own config. Workspace instructions register before the skill catalog so their session-prefix message renders first. App packages use `pickSpineConfig()` to copy only these bundle-owned fields. +The bundle FORWARDS each field to the child that owns it: `agents` and `maxParallelToolCalls` to `agent-loop` (`agents` defaults to `[]`; the cap defaults there), so each app supplies its own pre-created agents — TUI and headless apps pre-create `main`, while the ACP app creates agents on demand at `session/new`; `persona` and `toolOrder` to `dsh-system-prompt`; `tools` to the tool registry for its presentation mode; `sessionTitle` to the fallback title service; `skills.registry`, `skills.local`, and `skills.tool` to the skill registry, local provider, and model-facing consumer; the required `workspaceContext` choice to `dsh-workspace-context` (`{ maxBytes }` enables loading and `false` disables it); `invariants` to the invariant service; and `toolBash`/`toolTasks` to the two model-facing tool plugins the bundle owns. It always mounts `dsh-llm-retry`, while each leaf adapter owns its nested `retryPolicy`. Omitted `sessionTitle` uses the explicit example policy of 5 words, 40 fallback bytes, and 80 accepted-title bytes. A `goals` object opts into the persisted domain, model tools, and same-session driver while forwarding `goals.domain` and `goals.tool` to their owners; omission or `false` leaves the stack absent so headless callers retain one-turn settlement. Set `skills.enabled: false` to omit both the local provider and model-facing skill tool, and set `toolTasks: false` to retain the task service for foreground producers without exposing `task_output` / `task_list` / `task_kill`. It resolves `dshHome` once through [`@deepseek-ai/dsh-paths`](../../util/paths/README.md) and forwards that absolute value to tool-bash's managed environment and enabled local skill discovery. An absent top-level `dshHome` adopts `skills.local.dshHome`; supplying both with different resolved paths fails loudly. `toolBash.enableRunInBackground` controls only the bash producer; independently loaded producers keep their own config. Workspace instructions register before the skill catalog so their session-prefix message renders first. App packages use `pickSpineConfig()` to copy only these bundle-owned fields. For example, `{ invariants: { enabled: true, package_allowlist: ['^@deepseek-ai/dsh-'], package_blocklist: ['agent-loop$'] } }` keeps the package-owned companions mounted but suppresses the blocked owner. Blocklist matches override allowlist matches; see [`dsh-invariants`](../../support/invariants/README.md) for regex and lifecycle rules. @@ -65,7 +65,7 @@ For example, `{ invariants: { enabled: true, package_allowlist: ['^@deepseek-ai/ A YAML include can deduplicate config but cannot own a bin or provide front-door defaults. App packages make stdout-safe ACP wiring the default, though a leaf can still add an unsafe logger. Bundle children register services in the root isolate-keyed store, so injected leaf siblings see them without load-order coupling. -The bounded retry policy may repeat a transiently failed request in a new numbered step. Retry status and failed partial chunks stay outside model history, each provider attempt can still incur billing, front doors derive usage across every logged step, and the reconstructed request preserves the prior prefix for provider cache reuse. +The retry policy may repeat a failed request in a new numbered step. Retry status, provider errors, and failed partial chunks stay outside model history; each provider attempt can still incur billing, always mode has no attempt limit, front doors derive usage across every logged step, and the reconstructed request preserves the prior prefix for provider cache reuse. ## Model Experience diff --git a/packages/examples/agent-spine-demo/package.json b/packages/examples/agent-spine-demo/package.json index bf69e27787..11a8a890c9 100644 --- a/packages/examples/agent-spine-demo/package.json +++ b/packages/examples/agent-spine-demo/package.json @@ -1,6 +1,6 @@ { "name": "@deepseek-ai/dsh-agent-spine-demo", - "description": "The default executor-less/UI-less agent spine with fallback session titles, bounded retry, and optional persisted goals", + "description": "The default executor-less/UI-less agent spine with fallback session titles, provider-routed retry, and optional persisted goals", "version": "0.0.1", "private": true, "type": "module", diff --git a/packages/examples/agent-spine-demo/src/index.ts b/packages/examples/agent-spine-demo/src/index.ts index c43ee2ab8d..d2a65844ea 100644 --- a/packages/examples/agent-spine-demo/src/index.ts +++ b/packages/examples/agent-spine-demo/src/index.ts @@ -74,8 +74,9 @@ export interface GoalConfig { * `dshHome` to bash environment and local skill discovery, `sessionTitle` to * the fallback title service, `skills` to the * skill registry/local provider/tool consumer, `workspaceContext` to the - * workspace-context loader, `llmRetry` to the bounded request-recovery policy, - * and `toolBash`/`toolTasks` to the model-facing tool plugins this bundle owns. + * workspace-context loader, and `toolBash`/`toolTasks` to the model-facing tool + * plugins this bundle owns. Provider adapters own their `retryPolicy`; this + * bundle always mounts its executor. * `goals` opts into and configures the persisted goal domain plus its model tool * and same-session driver; `invariants` configures global and package-filtered * relational checks. Owner schemas supply defaults for optional input; @@ -111,8 +112,6 @@ export interface Config { invariants?: InvariantConfig /** Opt-in persisted same-session goal stack; set false or omit to leave it unmounted. */ goals?: GoalConfig | false - /** Bounded transient model-request retry policy. */ - llmRetry?: llmRetry.Config } /** The skill config schema exported for app packages that forward `skills`. */ @@ -139,9 +138,6 @@ export const GoalConfigSchema: z = z.object({ tool: toolGoal.Config, }) -/** The bounded LLM retry schema exported for app packages that forward `llmRetry`. */ -export const LlmRetryConfigSchema: z = llmRetry.Config - /** Intersect the owners' schemas so validation + defaulting stay identical. */ export const Config = z.intersect([ AgentLoop.Config, @@ -156,8 +152,7 @@ export const Config = z.intersect([ toolTasks: z.union([z.const(false), ToolTasksConfigSchema]), invariants: InvariantService.Config, goals: z.union([z.const(false), GoalConfigSchema]), - llmRetry: LlmRetryConfigSchema, - }) as unknown as z>, + }) as unknown as z>, ]) as unknown as z /** @@ -179,7 +174,6 @@ export function pickSpineConfig(config: Omit): Omit { this.requests += 1 @@ -219,15 +237,7 @@ describe('dsh-agent-spine-demo bundle', () => { it('loads and configures bounded request recovery for every bundled front door', async () => { const adapter = new TransientOnceAdapter() - const ctx = await mount({ - workspaceContext: false, - llmRetry: { - maxTransientRetries: 1, - initialDelayMs: 1, - maxDelayMs: 1, - jitterRatio: 0, - }, - }) + const ctx = await mount({ workspaceContext: false }) ctx.llm.registerAdapter(['mock'], adapter) const handle = await ctx.agents.create({ sessionId: SessionId('bundled-retry-session'), @@ -242,7 +252,7 @@ describe('dsh-agent-spine-demo bundle', () => { const retryEvents = handle.agent.session.events.filter(event => event.type === 'llm/retry') expect(retryEvents).toHaveLength(1) expect(retryEvents[0]?.data.retry).toBe(1) - expect(retryEvents[0]?.data.maxRetries).toBe(1) + expect(retryEvents[0]?.data).toMatchObject({ provider: 'mock', mode: 'normal', maxRetries: 1 }) expect(handle.agent.session.events.find(event => event.type === 'session/title')?.data.title).toBe('recover') expect(messageText(handle.agent.session.deriveMessages().at(-1))).toBe('recovered by bundled policy') await handle.dispose() @@ -523,7 +533,6 @@ describe('dsh-agent-spine-demo bundle', () => { toolBash: { enableRunInBackground: false }, toolTasks: false as const, invariants: { enabled: false }, - llmRetry: { maxTransientRetries: 1, jitterRatio: 0 }, } expect(agentCore.pickSpineConfig(appConfig)).toEqual({ @@ -537,7 +546,6 @@ describe('dsh-agent-spine-demo bundle', () => { toolBash: appConfig.toolBash, toolTasks: appConfig.toolTasks, invariants: appConfig.invariants, - llmRetry: appConfig.llmRetry, }) expect(agentCore.pickSpineConfig({ workspaceContext: false })).toEqual({ workspaceContext: false }) }) diff --git a/packages/examples/cli-demo/README.md b/packages/examples/cli-demo/README.md index 8a931cc147..a0b049ee68 100644 --- a/packages/examples/cli-demo/README.md +++ b/packages/examples/cli-demo/README.md @@ -19,7 +19,6 @@ The package mounts no console logger, interactive UI, user-interaction service, | `skills` | owner defaults | skill registry, local provider, and model-facing skill tool | | `toolBash` | owner defaults | model-facing bash config, including this producer's background opt-in | | `toolTasks` | owner defaults | generic `task_output` wait bounds | -| `llmRetry` | owner defaults | bounded transient model-request retry policy | | `persistenceRoot` | `./.sessions` | JSONL session root | | `persistenceCompression` | `'zstd'` | JSONL artifact encoding (`'zstd'` or raw `'none'`) | | `workspaceContext` | required | workspace-instruction byte budget, or `false` to disable loading | diff --git a/packages/examples/cli-demo/src/index.ts b/packages/examples/cli-demo/src/index.ts index f7d543c3fe..bfec7c9a4f 100644 --- a/packages/examples/cli-demo/src/index.ts +++ b/packages/examples/cli-demo/src/index.ts @@ -50,8 +50,6 @@ export interface Config { toolBash?: NonNullable /** Generic background-task control-tool config forwarded through agent-spine-demo. */ toolTasks?: NonNullable - /** Bounded transient model-request retry policy forwarded through agent-spine-demo. */ - llmRetry?: NonNullable /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ workspaceContext: agentCore.Config['workspaceContext'] } @@ -74,7 +72,6 @@ export const Config: z = z.object({ tools: ToolRegistry.Config, toolBash: agentCore.ToolBashConfigSchema, toolTasks: z.union([z.const(false), agentCore.ToolTasksConfigSchema]), - llmRetry: agentCore.LlmRetryConfigSchema, workspaceContext: z.union([z.const(false), workspaceContext.Config]).required(), }) /* jscpd:ignore-end */ diff --git a/packages/examples/cli-demo/tests/cli.spec.ts b/packages/examples/cli-demo/tests/cli.spec.ts index 65eafff1cc..f467033128 100644 --- a/packages/examples/cli-demo/tests/cli.spec.ts +++ b/packages/examples/cli-demo/tests/cli.spec.ts @@ -3,7 +3,15 @@ import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' import { Context } from 'cordis' import type { Agent } from '@deepseek-ai/dsh-agent' -import { CallId, LlmAdapter, type GenerateOptions, type StreamChunk, type TokenUsage } from '@deepseek-ai/dsh-llm' +import { + CallId, + LlmAdapter, + resolveRetryPolicy, + type GenerateOptions, + type ResolvedRetryPolicy, + type StreamChunk, + type TokenUsage, +} from '@deepseek-ai/dsh-llm' import { SessionId, type SessionEvent, type TurnEndReason } from '@deepseek-ai/dsh-session' import { afterEach, describe, expect, it } from 'vitest' import * as cliDemo from '../src/index.ts' @@ -20,11 +28,19 @@ type ScriptEntry = readonly StreamChunk[] | 'hang' class ScriptedAdapter extends LlmAdapter { readonly requests: GenerateOptions[] = [] private cursor = 0 + private readonly retryPolicy = resolveRetryPolicy({ + mode: 'normal', + backoff: { initialDelayMs: 1, maxDelayMs: 1, jitterRatio: 0 }, + }, 'cli test provider retryPolicy') constructor(private readonly script: readonly ScriptEntry[]) { super() } + override providerRetryPolicy(_provider: string): ResolvedRetryPolicy { + return this.retryPolicy + } + async * stream(options: GenerateOptions): AsyncIterable { this.requests.push(options) const entry = this.script[this.cursor++] @@ -107,7 +123,6 @@ async function harness(script: readonly ScriptEntry[]): Promise { persistenceRoot: root, skills: { local: { dshHome: join(skillHome, '.dsh'), agentsHome: join(skillHome, '.agents') } }, workspaceContext: false, - llmRetry: { initialDelayMs: 1, maxDelayMs: 1, jitterRatio: 0 }, }) await new Promise(resolve => setTimeout(resolve, 80)) ctx.llm.registerAdapter(['mock'], new ScriptedAdapter(script)) diff --git a/packages/llm/README.md b/packages/llm/README.md index 0c937c17dc..6585de4b9a 100644 --- a/packages/llm/README.md +++ b/packages/llm/README.md @@ -6,8 +6,8 @@ The LLM seam and its provider adapters. The interface package (`llm`) owns the a |---|---|---| | `llm/` | Abstract LLM service + content-block vocabulary + chunk assembler | `ctx.llm` | | `token-meter/` | Replay-aware request and surface token measurement | `ctx.tokenMeter` | -| `llm-retry/` | Bounded transient request retry policy | (listens to `agent/request-error`) | +| `llm-retry/` | Exact-provider normal or unbounded request retry policy | (listens to `agent/request-error`) | | `llm-deepseek/` | DeepSeek API adapter (hand-rolled fetch/SSE) | (registers on `ctx.llm`) | | `llm-pi-ai/` | Multi-provider adapter via `@earendil-works/pi-ai` | (registers on `ctx.llm`) | -The interface lives at `llm/llm/`; adapters, retry policy, and the reusable token meter are flat siblings under the group. Requests route by `provider`, while `model` is passed through to the selected adapter. The route-owning adapter optionally resolves exact provider/model context capacity; the token meter remains model-agnostic. A new provider adapter registers one or more provider routes on `ctx.llm` without touching the interface or consumers. See [twin LLM adapters](../../.agents/notes/implemented/architecture/2026-06-13-twin-llm-adapters.md) for the two shipping implementations, the [replay token meter Agent Note](../../.agents/notes/implemented/architecture/2026-07-15-replay-token-meter-service.md) for measurement ownership, and the [routed model context Agent Note](../../.agents/notes/implemented/architecture/2026-07-20-routed-model-context-and-compaction-policy.md) for capacity and compaction-policy ownership. +The interface lives at `llm/llm/`; adapters, retry policy, and the reusable token meter are flat siblings under the group. Requests route by `provider`, while `model` is passed through to the selected adapter. The route-owning adapter supplies retry policy and optionally resolves exact provider/model context capacity; the retry executor and token meter remain provider-agnostic. A new provider adapter registers one or more provider routes on `ctx.llm` without touching the consumers. See [twin LLM adapters](../../.agents/notes/implemented/architecture/2026-06-13-twin-llm-adapters.md) for the two shipping implementations, the [replay token meter Agent Note](../../.agents/notes/implemented/architecture/2026-07-15-replay-token-meter-service.md) for measurement ownership, and the [routed model context Agent Note](../../.agents/notes/implemented/architecture/2026-07-20-routed-model-context-and-compaction-policy.md) for capacity and compaction-policy ownership. diff --git a/packages/llm/llm-deepseek/README.md b/packages/llm/llm-deepseek/README.md index 5a9a1bfdbc..177312c1ef 100644 --- a/packages/llm/llm-deepseek/README.md +++ b/packages/llm/llm-deepseek/README.md @@ -17,6 +17,12 @@ The package root exposes the Cordis plugin contract and `DeepSeekAdapter`; wire thinking: enabled # optional; provider default is enabled reasoningEffort: high # optional; high | max — omitted ⇒ not sent streamIdleTimeoutMs: 300000 # optional; positive finite Node timer delay; five-minute default + retryPolicy: # optional; omission uses bounded normal defaults + mode: always # normal | always + backoff: + initialDelayMs: 500 + maxDelayMs: 10000 + jitterRatio: 0.1 defaultContextWindow: 256000 # optional positive-integer fallback for models without an exact value models: # optional; defaults to V4 Flash and V4 Pro - id: deepseek-v4-flash @@ -26,7 +32,7 @@ The package root exposes the Cordis plugin contract and `DeepSeekAdapter`; wire contextWindow: 64000 ``` -The plugin registers the single provider route `deepseek`. A request selects it with `provider: deepseek`; its `model` is passed through as the wire `model` string, so changing DeepSeek models does not require lifecycle-time registration. Omitting `models` advertises `deepseek-v4-flash` and `deepseek-v4-pro`, each with a 128,000-token context window; an explicit list replaces those defaults, while `models: []` advertises none. Catalog entries are exposed through `ctx.llm.listModels('deepseek')` for clients such as ACP editors, but remain advisory: unlisted model ids still pass through unchanged. An omitted entry name defaults to its id. +The plugin registers the single provider route `deepseek` together with its resolved `retryPolicy`. A request selects it with `provider: deepseek`; its `model` is passed through as the wire `model` string, so changing DeepSeek models does not require lifecycle-time registration. Omitting `models` advertises `deepseek-v4-flash` and `deepseek-v4-pro`, each with a 128,000-token context window; an explicit list replaces those defaults, while `models: []` advertises none. Catalog entries are exposed through `ctx.llm.listModels('deepseek')` for clients such as ACP editors, but remain advisory: unlisted model ids still pass through unchanged. An omitted entry name defaults to its id. `contextWindow` is optional per configured model and is not exposed through the advisory catalog. `ctx.llm.resolveModelContext('deepseek', model)` returns an exact model value first, then `defaultContextWindow` for an entry without capacity or an unlisted pass-through id. When neither value exists it returns `undefined` without invalidating routing. Pressure-sensitive plugins therefore get deployment-owned capacity without treating the model selector as authoritative. Registering another adapter for `deepseek` throws `LlmError('DUPLICATE_ADAPTER')`. @@ -34,7 +40,7 @@ The plugin registers the single provider route `deepseek`. A request selects it `thinking`/`reasoningEffort` are adapter-level request defaults serialized as the official top-level `thinking: {type}` / `reasoning_effort` wire fields. They live in adapter config (not `GenerateOptions`) to keep the core vocabulary provider-neutral. A request with `GenerateOptions.purpose: 'session-title'` forces thinking disabled and omits `reasoning_effort`, reserving its bounded output for visible title text without changing conversation or compaction defaults. -`streamIdleTimeoutMs` bounds each outstanding provider read, including the initial `fetch`, without counting time the consumer spends between chunks. One stable abort signal reaches the request and body reader for the whole call; expiry stops the transport and throws `LlmError('TIMEOUT')`, while an earlier caller abort throws `LlmError('ABORTED')`. The adapter makes exactly one provider request per `stream()` call; agent-level retry is a separate plugin policy. +`streamIdleTimeoutMs` bounds each outstanding provider read, including the initial `fetch`, without counting time the consumer spends between chunks. One stable abort signal reaches the request and body reader for the whole call; expiry stops the transport and throws `LlmError('TIMEOUT')`, while an earlier caller abort throws `LlmError('ABORTED')`. The adapter makes exactly one provider request per `stream()` call; it registers the configured policy as provider metadata, and `dsh-llm-retry` separately executes it at durable agent-step boundaries. ## App attribution diff --git a/packages/llm/llm-deepseek/src/adapter.ts b/packages/llm/llm-deepseek/src/adapter.ts index 64faa3b725..4a270fecd1 100644 --- a/packages/llm/llm-deepseek/src/adapter.ts +++ b/packages/llm/llm-deepseek/src/adapter.ts @@ -5,12 +5,14 @@ * @module dsh-llm-deepseek/adapter */ -import { attributionHeaders, CONTEXT_WINDOW_EXCEEDED_CODE, isContextWindowExceededError, isQuotaExceededError, LlmAdapter, LlmError, ProviderRequestId, QUOTA_EXCEEDED_CODE } from '@deepseek-ai/dsh-llm' +import { attributionHeaders, CONTEXT_WINDOW_EXCEEDED_CODE, isContextWindowExceededError, isQuotaExceededError, LlmAdapter, LlmError, ProviderRequestId, QUOTA_EXCEEDED_CODE, resolveRetryPolicy } from '@deepseek-ai/dsh-llm' import type { GenerateOptions, LlmModelContext, LlmModelInfo, LlmProviderInfo, + ResolvedRetryPolicy, + RetryPolicyConfig, StreamChunk, } from '@deepseek-ai/dsh-llm' import { idleWatchdog, MAX_TIMER_DELAY_MS, timeoutOf } from '@deepseek-ai/dsh-timeout' @@ -46,6 +48,8 @@ export interface DeepSeekAdapterOptions { models?: readonly DeepSeekCatalogModel[] /** Maximum provider idle time while one stream read is outstanding. */ streamIdleTimeoutMs?: number + /** Provider-owned model-request retry policy; omission uses normal defaults. */ + retryPolicy?: RetryPolicyConfig } /** Default maximum idle interval while an adapter stream read is outstanding. */ @@ -95,6 +99,7 @@ export function httpErrorCode(status: number, error?: WireError['error']): strin */ export class DeepSeekAdapter extends LlmAdapter { private readonly streamIdleTimeoutMs: number + private readonly retryPolicy: ResolvedRetryPolicy constructor(private readonly options: DeepSeekAdapterOptions) { super() @@ -110,12 +115,17 @@ export class DeepSeekAdapter extends LlmAdapter { `llm-deepseek: streamIdleTimeoutMs must be a positive finite number no greater than ${MAX_TIMER_DELAY_MS}`, ) } + this.retryPolicy = resolveRetryPolicy(options.retryPolicy, 'llm-deepseek: retryPolicy') } override providerInfo(provider: string): LlmProviderInfo { return { id: provider, name: 'DeepSeek' } } + override providerRetryPolicy(_provider: string): ResolvedRetryPolicy { + return this.retryPolicy + } + override listModels(provider: string): Promise { return Promise.resolve((this.options.models ?? []).map(model => ({ provider, diff --git a/packages/llm/llm-deepseek/src/index.ts b/packages/llm/llm-deepseek/src/index.ts index 66828fc954..8172dd5420 100644 --- a/packages/llm/llm-deepseek/src/index.ts +++ b/packages/llm/llm-deepseek/src/index.ts @@ -7,7 +7,8 @@ import type { Context } from 'cordis' import z from 'schemastery' -import type {} from '@deepseek-ai/dsh-llm' +import { RetryPolicySchema } from '@deepseek-ai/dsh-llm' +import type { RetryPolicyConfig } from '@deepseek-ai/dsh-llm' import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' import { DEFAULT_STREAM_IDLE_TIMEOUT_MS, DeepSeekAdapter } from './adapter.ts' import type { DeepSeekCatalogModel } from './adapter.ts' @@ -46,6 +47,8 @@ export interface Config { models?: DeepSeekCatalogModel[] /** Maximum provider idle time while one stream read is outstanding (default five minutes). */ streamIdleTimeoutMs?: number + /** Provider-owned model-request retry policy; omission uses normal defaults. */ + retryPolicy?: RetryPolicyConfig } const catalogModel: z = z.object({ @@ -63,6 +66,7 @@ export const Config: z = z.object({ defaultContextWindow: z.number().step(1).min(1), models: z.array(catalogModel).default(DEFAULT_MODELS), streamIdleTimeoutMs: z.number().min(Number.MIN_VALUE).max(MAX_TIMER_DELAY_MS).default(DEFAULT_STREAM_IDLE_TIMEOUT_MS), + retryPolicy: RetryPolicySchema, }) /** Public API default; the internal endpoint comes from $DEEPSEEK_BASE_URL. */ @@ -111,5 +115,6 @@ export function apply(ctx: Context, config: Config): void { : { defaultContextWindow: config.defaultContextWindow }, models: resolveModels(config.models), streamIdleTimeoutMs: config.streamIdleTimeoutMs ?? DEFAULT_STREAM_IDLE_TIMEOUT_MS, + ...config.retryPolicy === undefined ? {} : { retryPolicy: config.retryPolicy }, })) } diff --git a/packages/llm/llm-deepseek/tests/adapter.spec.ts b/packages/llm/llm-deepseek/tests/adapter.spec.ts index 145017ea3d..6203c7db67 100644 --- a/packages/llm/llm-deepseek/tests/adapter.spec.ts +++ b/packages/llm/llm-deepseek/tests/adapter.spec.ts @@ -522,6 +522,26 @@ describe('plugin registration and config', () => { expect(ctx.llm.listProviders()).toEqual([]) }) + it('registers retryPolicy from the provider config', async () => { + const ctx = new Context() + await ctx.plugin(LlmService) + await ctx.plugin(LlmDeepSeek, { + apiKey: 'k', + baseURL: 'http://127.0.0.1:1', + retryPolicy: { + mode: 'always', + backoff: { initialDelayMs: 25, maxDelayMs: 100, jitterRatio: 0.2 }, + }, + }) + + expect(ctx.llm.providerRetryPolicy('deepseek')).toEqual({ + mode: 'always', + initialDelayMs: 25, + maxDelayMs: 100, + jitterRatio: 0.2, + }) + }) + it('owns the deepseek provider and advertises the default models', async () => { const ctx = new Context() await ctx.plugin(LlmService) @@ -731,4 +751,16 @@ describe('plugin registration and config', () => { streamIdleTimeoutMs: MAX_TIMER_DELAY_MS + 1, })).rejects.toThrow(/streamIdleTimeoutMs/) }) + + it('rejects invalid nested retryPolicy before registering the provider', async () => { + const ctx = new Context() + await ctx.plugin(LlmService) + + await expect(ctx.plugin(LlmDeepSeek, { + apiKey: 'k', + baseURL: 'http://127.0.0.1:1', + retryPolicy: { mode: 'normal', maxRetries: -1 }, + })).rejects.toThrow(/retryPolicy/) + expect(ctx.llm.listProviders()).toEqual([]) + }) }) diff --git a/packages/llm/llm-pi-ai/README.md b/packages/llm/llm-pi-ai/README.md index 8a6736f112..31d890c0f1 100644 --- a/packages/llm/llm-pi-ai/README.md +++ b/packages/llm/llm-pi-ai/README.md @@ -17,6 +17,13 @@ Configure credentials and deployment-specific transport settings per provider. O apiKey: !!js process.env.OPENAI_API_KEY baseURL: https://proxy.example.com:8443 reasoning: high + retryPolicy: + mode: normal + maxRetries: 3 + backoff: + initialDelayMs: 500 + maxDelayMs: 10000 + jitterRatio: 0.1 - provider: anthropic apiKey: !!js process.env.ANTHROPIC_API_KEY streamIdleTimeoutMs: 300000 @@ -30,7 +37,7 @@ Each provider name must exist in pi-ai's installed catalog and may appear only o The adapter exposes each configured provider's installed pi-ai models through `ctx.llm.listModels(provider)`. This is provider-neutral selector metadata derived from `getModels(provider)`; request-time resolution still performs the authoritative catalog lookup, so discovery does not create a second model registry. `ctx.llm.resolveModelContext(provider, model)` performs the same exact descriptor lookup and returns its context window, keeping capacity metadata on the route-owning adapter rather than a consuming plugin. -Supported profile fields are `provider`, `apiKey`, `baseURL`, `headers`, `reasoning`, `thinkingBudgets`, `cacheRetention`, `transport`, `timeoutMs`, `websocketConnectTimeoutMs`, and `streamIdleTimeoutMs`. The stream-idle interval is a positive finite Node timer delay, defaults to five minutes, and covers only an outstanding provider read, not consumer think time. Harness app attribution wins a conflicting configured header name. +Supported profile fields are `provider`, `apiKey`, `baseURL`, `headers`, `reasoning`, `thinkingBudgets`, `cacheRetention`, `transport`, `timeoutMs`, `websocketConnectTimeoutMs`, `streamIdleTimeoutMs`, and `retryPolicy`. Each profile's optional retry policy is captured with that provider route; omission uses bounded normal defaults. The stream-idle interval is a positive finite Node timer delay, defaults to five minutes, and covers only an outstanding provider read, not consumer think time. Harness app attribution wins a conflicting configured header name. The adapter forces pi-ai's SDK `maxRetries` to zero so one `stream()` call makes one provider request. The removed profile fields `maxRetries` and `maxRetryDelayMs` fail load instead of silently multiplying or hiding the separately composed agent-level retry budget. Idle expiry aborts the SDK's stable request signal and surfaces `TIMEOUT`; an earlier caller abort remains `ABORTED`. diff --git a/packages/llm/llm-pi-ai/src/adapter.ts b/packages/llm/llm-pi-ai/src/adapter.ts index ed0fb9fae4..59f8fce354 100644 --- a/packages/llm/llm-pi-ai/src/adapter.ts +++ b/packages/llm/llm-pi-ai/src/adapter.ts @@ -13,7 +13,7 @@ import type { SimpleStreamOptions, } from '@earendil-works/pi-ai' import { attributionHeaders, LlmAdapter, LlmError } from '@deepseek-ai/dsh-llm' -import type { GenerateOptions, LlmModelContext, LlmModelInfo, StreamChunk } from '@deepseek-ai/dsh-llm' +import type { GenerateOptions, LlmModelContext, LlmModelInfo, ResolvedRetryPolicy, StreamChunk } from '@deepseek-ai/dsh-llm' import { idleWatchdog, timeoutOf } from '@deepseek-ai/dsh-timeout' import { resolveProfiles } from './config.ts' import type { PiAiProviderProfile, ResolvedPiAiProviderProfile } from './config.ts' @@ -30,7 +30,10 @@ export interface PiAiAdapterOptions { * Resolve a catalog model dynamically and apply only the configured endpoint * override, preserving the catalog's API/capability/compatibility metadata. */ -function resolveModel(profile: PiAiProviderProfile, modelId: string): Model { +function resolveModel( + profile: Omit, + modelId: string, +): Model { const model = getBuiltinModels(profile.provider as BuiltinProvider).find(candidate => candidate.id === modelId) as Model | undefined if (model === undefined) { throw new LlmError(`pi-ai provider "${profile.provider}" has no catalog model "${modelId}"`, 'UNKNOWN_MODEL') @@ -39,7 +42,7 @@ function resolveModel(profile: PiAiProviderProfile, modelId: string): Model } /** Copy profile stream knobs into pi-ai's common option vocabulary. */ -function profileOptions(profile: PiAiProviderProfile): SimpleStreamOptions { +function profileOptions(profile: Omit): SimpleStreamOptions { return { ...profile.apiKey === undefined ? {} : { apiKey: profile.apiKey }, ...profile.reasoning === undefined ? {} : { reasoning: profile.reasoning }, @@ -75,6 +78,10 @@ export class PiAiAdapter extends LlmAdapter { this.profiles = new Map(resolveProfiles(options.profiles).map(profile => [profile.provider, profile])) } + override providerRetryPolicy(provider: string): ResolvedRetryPolicy | undefined { + return this.profiles.get(provider)?.retryPolicy + } + override listModels(provider: string): Promise { const profile = this.profiles.get(provider) if (profile === undefined) { diff --git a/packages/llm/llm-pi-ai/src/config.ts b/packages/llm/llm-pi-ai/src/config.ts index 199463aaf6..5c6916c38e 100644 --- a/packages/llm/llm-pi-ai/src/config.ts +++ b/packages/llm/llm-pi-ai/src/config.ts @@ -8,6 +8,8 @@ import { getBuiltinProviders } from '@earendil-works/pi-ai/providers/all' import type { CacheRetention, ThinkingBudgets, ThinkingLevel, Transport } from '@earendil-works/pi-ai' import z from 'schemastery' import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' +import { resolveRetryPolicy, RetryPolicySchema } from '@deepseek-ai/dsh-llm' +import type { ResolvedRetryPolicy, RetryPolicyConfig } from '@deepseek-ai/dsh-llm' /** Default maximum idle interval while an adapter stream read is outstanding. */ export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 300_000 @@ -36,12 +38,16 @@ export interface PiAiProviderProfile { websocketConnectTimeoutMs?: number /** Maximum provider idle time while one stream read is outstanding. */ streamIdleTimeoutMs?: number + /** Provider-owned model-request retry policy; omission uses normal defaults. */ + retryPolicy?: RetryPolicyConfig } /** Validated profile with every adapter-owned default resolved. */ -export interface ResolvedPiAiProviderProfile extends PiAiProviderProfile { +export interface ResolvedPiAiProviderProfile extends Omit { /** Positive finite provider-idle interval after defaulting. */ streamIdleTimeoutMs: number + /** Immutable retry policy captured with this provider route. */ + retryPolicy: ResolvedRetryPolicy } /** Plugin configuration: the non-empty provider profiles this instance owns. */ @@ -69,6 +75,7 @@ const profile = z.object({ timeoutMs: z.natural(), websocketConnectTimeoutMs: z.natural(), streamIdleTimeoutMs: z.number().min(Number.MIN_VALUE).max(MAX_TIMER_DELAY_MS).default(DEFAULT_STREAM_IDLE_TIMEOUT_MS), + retryPolicy: RetryPolicySchema, }) /** Runtime schema for {@link Config}. */ @@ -115,6 +122,10 @@ export function resolveProfiles(profiles: readonly PiAiProviderProfile[]): Resol return { ...source, streamIdleTimeoutMs, + retryPolicy: resolveRetryPolicy( + source.retryPolicy, + `llm-pi-ai: provider "${source.provider}" retryPolicy`, + ), ...source.headers === undefined ? {} : { headers: { ...source.headers } }, ...source.thinkingBudgets === undefined ? {} : { thinkingBudgets: { ...source.thinkingBudgets } }, } diff --git a/packages/llm/llm-pi-ai/src/index.ts b/packages/llm/llm-pi-ai/src/index.ts index ab08f21b81..da104cb22d 100644 --- a/packages/llm/llm-pi-ai/src/index.ts +++ b/packages/llm/llm-pi-ai/src/index.ts @@ -10,6 +10,9 @@ * providers: * - provider: openai * apiKey: !!js process.env.OPENAI_API_KEY + * retryPolicy: + * mode: normal + * maxRetries: 2 * - provider: anthropic * apiKey: !!js process.env.ANTHROPIC_API_KEY * - provider: openrouter @@ -36,6 +39,6 @@ export const inject = ['llm'] /** Register one generic pi-ai adapter for all configured provider routes. */ export function apply(ctx: Context, config: Config): void { const profiles = resolveProfiles(config.providers) - const adapter = new PiAiAdapter({ profiles }) + const adapter = new PiAiAdapter({ profiles: config.providers }) ctx.llm.registerAdapter(profiles.map(entry => entry.provider), adapter) } diff --git a/packages/llm/llm-pi-ai/tests/adapter.spec.ts b/packages/llm/llm-pi-ai/tests/adapter.spec.ts index f37b07f624..7b059e2f78 100644 --- a/packages/llm/llm-pi-ai/tests/adapter.spec.ts +++ b/packages/llm/llm-pi-ai/tests/adapter.spec.ts @@ -303,12 +303,31 @@ describe('provider profile lifecycle', () => { const ctx = new Context() await ctx.plugin(LlmService) const fiber = await ctx.plugin(LlmPiAi, { - providers: [{ provider: 'openai' }, { provider: 'anthropic' }], + providers: [ + { + provider: 'openai', + retryPolicy: { + mode: 'always', + backoff: { initialDelayMs: 25, maxDelayMs: 100, jitterRatio: 0.2 }, + }, + }, + { provider: 'anthropic' }, + ], }) expect(ctx.llm.listProviders()).toEqual([ { id: 'openai', name: 'openai' }, { id: 'anthropic', name: 'anthropic' }, ]) + expect(ctx.llm.providerRetryPolicy('openai')).toEqual({ + mode: 'always', + initialDelayMs: 25, + maxDelayMs: 100, + jitterRatio: 0.2, + }) + expect(ctx.llm.providerRetryPolicy('anthropic')).toMatchObject({ + mode: 'normal', + maxRetries: 2, + }) await fiber.dispose() expect(ctx.llm.listProviders()).toEqual([]) }) @@ -373,6 +392,23 @@ describe('provider profile lifecycle', () => { } }) + it('rejects invalid nested retryPolicy at the provider-profile boundary', async () => { + expect(() => resolveProfiles([{ + provider: 'openai', + retryPolicy: { mode: 'always', backoff: { jitterRatio: -1 } }, + }])).toThrow(/retryPolicy\.backoff\.jitterRatio/) + + const ctx = new Context() + await ctx.plugin(LlmService) + await expect(ctx.plugin(LlmPiAi, { + providers: [{ + provider: 'openai', + retryPolicy: { mode: 'normal', maxRetries: -1 }, + }], + })).rejects.toThrow(/retryPolicy/) + expect(ctx.llm.listProviders()).toEqual([]) + }) + it('constructs the adapter directly and rejects routes it does not own', async () => { const adapter = new PiAiAdapter({ profiles: [{ provider: 'openai' }] }) await expect(adapter.listModels('anthropic')).rejects.toMatchObject({ code: 'NO_ADAPTER' }) diff --git a/packages/llm/llm-retry/README.md b/packages/llm/llm-retry/README.md index 699e7e3dad..711666b8a0 100644 --- a/packages/llm/llm-retry/README.md +++ b/packages/llm/llm-retry/README.md @@ -1,41 +1,50 @@ # `@deepseek-ai/dsh-llm-retry` -Function plugin that retries selected transient model-request failures on the agent loop's closed-step recovery seam. It does not wrap `ctx.llm.stream()`: every adapter call remains one provider attempt, and every retry opens a fresh numbered step. +Function plugin that applies exact-provider retry policy on the agent loop's closed-step recovery seam. It does not wrap `ctx.llm.stream()`: every adapter call remains one provider attempt, and every retry opens a fresh numbered step. -The default policy permits two retries for `RATE_LIMIT`, `SERVER`, `TIMEOUT`, and `TRANSPORT`, using bounded exponential backoff from 500 ms to 10 seconds with 10 percent jitter. Delay bounds must fit Node's supported timer range. A valid `providerRetryAfterMs` replaces local backoff when it is within the configured cap; an over-cap instruction delegates to the next recovery policy instead. +Each provider adapter owns an optional nested `retryPolicy`, captured when its route registers on `ctx.llm`. Omission uses normal mode: two retries for `RATE_LIMIT`, `SERVER`, `TIMEOUT`, and `TRANSPORT`. A normal policy can change its finite budget, eligible codes, and backoff. Always mode asks downstream recovery first, then retries every model-request failure without an attempt limit; success, cancellation, or plugin disposal stops it. -Before waiting, the plugin appends a non-surface `llm/retry` event with the failure and scheduled delay. Cancellation and plugin disposal abort the wait; disposal drains the plugin's active backoffs, and a callback captured before disposal fails closed if invoked afterward. +Both modes use bounded exponential backoff with symmetric jitter. A valid `providerRetryAfterMs` at or below `maxDelayMs` replaces local backoff without jitter. An over-cap provider delay makes normal mode delegate, while always mode uses its configured local backoff so it cannot terminate on that instruction. -The separately published `./invariant` companion checks that every retry record names the current open turn and its latest closed step, has a unique step record and increasing retry number, and carries a positive bounded retry budget and non-negative bounded timer delay. Full jitter may schedule zero milliseconds at its lower boundary. +Before waiting, the plugin appends a non-surface `llm/retry` event with the provider, mode, failure, and scheduled delay. Normal events include the finite maximum; always events omit it, and UIs render `∞`. Cancellation and plugin disposal abort the wait; disposal drains active backoffs, and a callback captured before disposal fails closed. + +The separately published `./invariant` companion checks that every retry record names the current open turn and latest closed step, matches the failed request's durable provider, has a unique step record and correct provider-policy retry number, and carries a valid mode-specific budget and bounded timer delay. Full jitter may schedule zero milliseconds at its lower boundary. ```yaml -- name: '@deepseek-ai/dsh-llm-retry' +- name: '@deepseek-ai/dsh-llm-deepseek' config: - maxTransientRetries: 2 - initialDelayMs: 500 - maxDelayMs: 10000 - jitterRatio: 0.1 - retryableCodes: [RATE_LIMIT, SERVER, TIMEOUT, TRANSPORT] + apiKey: !!js process.env.DEEPSEEK_API_KEY + retryPolicy: + mode: always + backoff: + initialDelayMs: 1000 + maxDelayMs: 30000 + jitterRatio: 0.2 + +- name: '@deepseek-ai/dsh-llm-retry' ``` +The executor has no policy config. Multi-provider adapters such as `dsh-llm-pi-ai` place `retryPolicy` inside each provider profile, avoiding a second provider-name list. + ## Model Experience -### Transient request recovery +### Model-request recovery #### What the model sees -No retry event, delay, or failure prose is model-visible. After a retry, the next numbered step reconstructs the same explicit provider/model request from durable session history; failed chunks never enter derived messages. +No retry event, delay, provider error, or failed partial output is model-visible. The next numbered step reconstructs the same explicit provider/model request from durable surface history unless a downstream recovery policy deliberately changes that surface. #### Token effect -Each retry is a new provider request and may repeat input-token billing. The finite budget caps attempts; `llm/retry` itself contributes no tokens. +Each retry is a new provider request and may repeat input-token billing. Normal mode has a finite budget; always mode can consume unbounded requests until success or cancellation. `llm/retry` itself contributes no tokens. #### KV Cache effect -The reconstructed request preserves the prior prefix and is eligible for provider cache reuse under that provider's rules. The non-surface status event does not change cache identity. +The reconstructed request preserves the prior prefix and is eligible for provider cache reuse under that provider's rules. The non-surface retry event does not change cache identity. ## Known Limitations and Deferred Work - **Agent steps are the only retry boundary** — direct `ctx.llm.stream()` consumers remain single-attempt because a raw stream cannot separate already-emitted chunks durably. -- **Finite plugin budgets add** — this policy counts only configured transient codes; context-overflow compaction counts only its own code. A future policy with overlapping codes must document and test registration-order behavior. +- **Always mode retries permanent failures** — authentication, quota, invalid-request, protocol, and unrecoverable context errors continue until success, cancellation, or disposal; deployments own provider-specific cost and latency controls. +- **Recovery policies compose by waterfall order** — always mode accepts a downstream retry before applying its fallback. A later policy that never settles also prevents the fallback from running. - **`llm/retry` records scheduling, not completion** — later step and turn events establish success, exhaustion, or cancellation. diff --git a/packages/llm/llm-retry/package.json b/packages/llm/llm-retry/package.json index 6d6c27636c..9ee3b5f896 100644 --- a/packages/llm/llm-retry/package.json +++ b/packages/llm/llm-retry/package.json @@ -1,6 +1,6 @@ { "name": "@deepseek-ai/dsh-llm-retry", - "description": "Bounded transient LLM request retry policy for the DeepSeek Harness", + "description": "Provider-routed LLM request retry policy for the DeepSeek Harness", "version": "0.0.1", "private": true, "type": "module", diff --git a/packages/llm/llm-retry/src/history.ts b/packages/llm/llm-retry/src/history.ts new file mode 100644 index 0000000000..2b86a77e03 --- /dev/null +++ b/packages/llm/llm-retry/src/history.ts @@ -0,0 +1,38 @@ +/** Durable request-route lookup for one closed model step. @module @deepseek-ai/dsh-llm-retry/history */ + +import type { SessionEvent } from '@deepseek-ai/dsh-session' + +/** + * Find the provider in force when one step closed, excluding later recovery mutations. + * A preceding retry is also a route marker because every provider change + * requires a newer full request-header snapshot. + * @param events - session events containing the closed step. + * @param turn - turn that owns the failed step. + * @param step - failed step whose provider is required. + * @returns the provider from the request header in force at that step boundary. + */ +export function providerForClosedStep( + events: readonly SessionEvent[], + turn: number, + step: number, +): string | undefined { + const stepEndIndex = events.findLastIndex(event => + event.type === 'step/end' + && event.data.turn === turn + && event.data.step === step, + ) + if (stepEndIndex < 0) return undefined + for (let index = stepEndIndex; index >= 0; index -= 1) { + // The loop bounds prove this indexed read exists. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const event = events[index]! + if (event.type === 'request/header') return event.data.header.config.provider + if (event.type === 'llm/retry' + && event.data.turn === turn + && event.data.step < step) { + return event.data.provider + } + if (event.type === 'turn/start' || event.type === 'turn/end') return undefined + } + return undefined +} diff --git a/packages/llm/llm-retry/src/index.ts b/packages/llm/llm-retry/src/index.ts index 4edf22d6f2..04f1b0996a 100644 --- a/packages/llm/llm-retry/src/index.ts +++ b/packages/llm/llm-retry/src/index.ts @@ -1,5 +1,5 @@ /** - * Bounded transient model-request retry policy on the agent loop's closed-step + * Provider-routed model-request retry policy on the agent loop's closed-step * recovery seam. Each scheduled retry is durable before its cancellable wait. * * @module @deepseek-ai/dsh-llm-retry @@ -8,103 +8,50 @@ import type { Context } from 'cordis' import z from 'schemastery' import type { Agent, RequestError, RequestErrorDecision } from '@deepseek-ai/dsh-agent' -import type { LlmFailure } from '@deepseek-ai/dsh-llm' -import type {} from '@deepseek-ai/dsh-session' -import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' +import type { LlmFailure, ResolvedRetryPolicy } from '@deepseek-ai/dsh-llm' +import type { SessionEvent } from '@deepseek-ai/dsh-session' +import { providerForClosedStep } from './history.ts' declare module '@deepseek-ai/dsh-session' { interface SessionEventMap { - /** Durable, non-surface record of one transient retry scheduled after a closed failed step. */ + /** Durable, non-surface record of one provider-routed retry scheduled after a closed failed step. */ 'llm/retry': { turn: number step: number + provider: string + mode: 'normal' retry: number maxRetries: number delayMs: number failure: LlmFailure + } | { + turn: number + step: number + provider: string + mode: 'always' + retry: number + delayMs: number + failure: LlmFailure } } } export const name = 'llm-retry' -export const inject = ['agents'] +export const inject = ['agents', 'llm'] -const DEFAULT_MAX_TRANSIENT_RETRIES = 2 -const DEFAULT_INITIAL_DELAY_MS = 500 -const DEFAULT_MAX_DELAY_MS = 10_000 -const DEFAULT_JITTER_RATIO = 0.1 -const DEFAULT_RETRYABLE_CODES = Object.freeze(['RATE_LIMIT', 'SERVER', 'TIMEOUT', 'TRANSPORT']) - -/** Deployment-owned limits and classification for transient request recovery. */ -export interface Config { - /** Maximum transient retries after the first request (default 2). */ - maxTransientRetries?: number - /** Initial local exponential-backoff delay in milliseconds (default 500). */ - initialDelayMs?: number - /** Maximum accepted or locally scheduled delay in milliseconds (default 10000). */ - maxDelayMs?: number - /** Symmetric random multiplier range around one (default 0.1). */ - jitterRatio?: number - /** Stable failure codes eligible for this policy. */ - retryableCodes?: string[] -} +/** This policy executor has no config; providers own `retryPolicy`. */ +export type Config = Readonly> /** Runtime schema for {@link Config}. */ -export const Config: z = z.object({ - maxTransientRetries: z.number().step(1).min(0).default(DEFAULT_MAX_TRANSIENT_RETRIES), - initialDelayMs: z.number().max(MAX_TIMER_DELAY_MS).default(DEFAULT_INITIAL_DELAY_MS), - maxDelayMs: z.number().max(MAX_TIMER_DELAY_MS).default(DEFAULT_MAX_DELAY_MS), - jitterRatio: z.number().min(0).max(1).default(DEFAULT_JITTER_RATIO), - retryableCodes: z.array(z.string()).default([...DEFAULT_RETRYABLE_CODES]), -}) +export const Config: z = z.object({}) -interface ResolvedConfig { - readonly maxTransientRetries: number - readonly initialDelayMs: number - readonly maxDelayMs: number - readonly jitterRatio: number - readonly retryableCodes: ReadonlySet -} - -function resolveConfig(config: Config): ResolvedConfig { - const maxTransientRetries = config.maxTransientRetries ?? DEFAULT_MAX_TRANSIENT_RETRIES - const initialDelayMs = config.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS - const maxDelayMs = config.maxDelayMs ?? DEFAULT_MAX_DELAY_MS - const jitterRatio = config.jitterRatio ?? DEFAULT_JITTER_RATIO - const codes = config.retryableCodes ?? [...DEFAULT_RETRYABLE_CODES] - - if (!Number.isInteger(maxTransientRetries) || maxTransientRetries < 0) { - throw new Error('llm-retry: maxTransientRetries must be a non-negative integer') +function validateConfig(config: Config): void { + const [key] = Object.keys(config) + if (key === undefined) return + if (key === 'retryPolicy') { + throw new Error('llm-retry: retryPolicy belongs under each provider configuration') } - if (!Number.isFinite(initialDelayMs) || initialDelayMs <= 0 || initialDelayMs > MAX_TIMER_DELAY_MS) { - throw new Error(`llm-retry: initialDelayMs must be a positive finite number no greater than ${MAX_TIMER_DELAY_MS}`) - } - if (!Number.isFinite(maxDelayMs) || maxDelayMs <= 0 || maxDelayMs > MAX_TIMER_DELAY_MS) { - throw new Error(`llm-retry: maxDelayMs must be a positive finite number no greater than ${MAX_TIMER_DELAY_MS}`) - } - if (initialDelayMs > maxDelayMs) { - throw new Error('llm-retry: initialDelayMs must be less than or equal to maxDelayMs') - } - if (!Number.isFinite(jitterRatio) || jitterRatio < 0 || jitterRatio > 1) { - throw new Error('llm-retry: jitterRatio must be between 0 and 1') - } - if (codes.length === 0) { - throw new Error('llm-retry: retryableCodes must not be empty') - } - if (codes.some(code => code.length === 0)) { - throw new Error('llm-retry: retryableCodes must contain only non-empty strings') - } - if (new Set(codes).size !== codes.length) { - throw new Error('llm-retry: retryableCodes must not contain duplicates') - } - - return Object.freeze({ - maxTransientRetries, - initialDelayMs, - maxDelayMs, - jitterRatio, - retryableCodes: new Set(codes), - }) + throw new Error(`llm-retry: unknown key "${key}"`) } /** Non-serializable seams used to make timing policy deterministic in tests. */ @@ -113,7 +60,38 @@ export interface RetryInternals { random?: () => number } -function localDelay(config: ResolvedConfig, retry: number, random: () => number): number { +type DownstreamOutcome = + | { readonly type: 'decision'; readonly decision: RequestErrorDecision } + | { readonly type: 'error'; readonly error: unknown } + | { readonly type: 'aborted' } + +function downstreamUntilAbort( + next: () => Promise, + signal: AbortSignal, +): Promise { + if (signal.aborted) return Promise.resolve({ type: 'aborted' }) + return new Promise((resolve) => { + const finish = (outcome: DownstreamOutcome): void => { + signal.removeEventListener('abort', onAbort) + resolve(outcome) + } + const onAbort = (): void => { finish({ type: 'aborted' }) } + signal.addEventListener('abort', onAbort, { once: true }) + let downstream: Promise + try { + downstream = next() + } catch (error: unknown) { + finish({ type: 'error', error }) + return + } + void downstream.then( + (decision) => { finish({ type: 'decision', decision }) }, + (error: unknown) => { finish({ type: 'error', error }) }, + ) + }) +} + +function localDelay(config: ResolvedRetryPolicy, retry: number, random: () => number): number { const exponent = Math.min(retry - 1, 1024) const exponential = Math.min(config.initialDelayMs * 2 ** exponent, config.maxDelayMs) const jitter = 1 - config.jitterRatio + 2 * config.jitterRatio * random() @@ -136,13 +114,13 @@ function cancellableDelay(delayMs: number, signal: AbortSignal): Promise>() @@ -152,25 +130,40 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna turn: number, step: number, failure: LlmFailure, + provider: string, + policy: ResolvedRetryPolicy, retry: number, delayMs: number, signal: AbortSignal, ): Promise { const fusedSignal = AbortSignal.any([signal, lifetime.signal]) if (fusedSignal.aborted) return { action: 'fail' } - agent.session.append('llm/retry', { - turn, - step, - retry, - maxRetries: resolved.maxTransientRetries, - delayMs, - failure, - }) + const eventData = policy.mode === 'normal' + ? { + turn, + step, + provider, + mode: policy.mode, + retry, + maxRetries: policy.maxRetries, + delayMs, + failure, + } + : { + turn, + step, + provider, + mode: policy.mode, + retry, + delayMs, + failure, + } + agent.session.append('llm/retry', eventData) if (!await cancellableDelay(delayMs, fusedSignal)) return { action: 'fail' } return { action: 'retry' } } - const disposeListener = ctx.on('agent/request-error', ( + const disposeListener = ctx.on('agent/request-error', async ( agent: Agent, turn: number, step: number, @@ -184,22 +177,61 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna // removed. Lifetime cancellation must prevent that stale callback from // entering a downstream policy after disposal. if (lifetime.signal.aborted) return Promise.resolve({ action: 'fail' }) - if (!resolved.retryableCodes.has(failure.code)) return next() - const priorTransientFailures = priorFailures.filter(item => resolved.retryableCodes.has(item.code)).length - if (priorTransientFailures >= resolved.maxTransientRetries) return next() + // Bind policy to the header in force when this step closed. Downstream + // recovery may append later state before an always fallback runs. + const provider = providerForClosedStep(agent.session.events, turn, step) + /* v8 ignore next 3 -- agent-loop closes only steps whose request header was recorded */ + if (provider === undefined) { + throw new Error(`llm-retry: no request provider for closed turn ${turn}/step ${step}`) + } + const policy = ctx.llm.providerRetryPolicy(provider) - const retry = priorTransientFailures + 1 + if (policy.mode === 'always') { + const downstream = await downstreamUntilAbort( + next, + AbortSignal.any([signal, lifetime.signal]), + ) + if (downstream.type === 'aborted') return { action: 'fail' } + if (downstream.type === 'error') { + ctx.logger.warn( + `llm-retry: provider "${provider}" always policy ignored a downstream recovery failure: %o`, + downstream.error, + ) + } + if (downstream.type === 'decision' && downstream.decision.action === 'retry') { + return downstream.decision + } + } else if (!policy.retryableCodes.includes(failure.code)) { + return next() + } + + const firstPriorStep = step - priorFailures.length + const priorPolicyRetry = agent.session.events.findLast((event): event is SessionEvent<'llm/retry'> => + event.type === 'llm/retry' + && event.data.turn === turn + && event.data.step >= firstPriorStep + && event.data.step < step + && event.data.provider === provider + && event.data.mode === policy.mode, + ) + const previousRetry = priorPolicyRetry?.data.retry ?? 0 + if (policy.mode === 'normal' && previousRetry >= policy.maxRetries) return next() + const retry = previousRetry + 1 let delayMs: number if (failure.providerRetryAfterMs !== undefined && Number.isFinite(failure.providerRetryAfterMs) && failure.providerRetryAfterMs > 0) { - if (failure.providerRetryAfterMs > resolved.maxDelayMs) return next() - delayMs = failure.providerRetryAfterMs + if (failure.providerRetryAfterMs > policy.maxDelayMs) { + if (policy.mode === 'normal') return next() + delayMs = localDelay(policy, retry, random) + } else { + delayMs = failure.providerRetryAfterMs + } } else { - delayMs = localDelay(resolved, retry, random) + delayMs = localDelay(policy, retry, random) } - const tracked = backoff(agent, turn, step, failure, retry, delayMs, signal) + const tracked = backoff(agent, turn, step, failure, provider, policy, retry, delayMs, signal) .finally(() => active.delete(tracked)) active.add(tracked) return tracked diff --git a/packages/llm/llm-retry/src/invariant.ts b/packages/llm/llm-retry/src/invariant.ts index 784f459606..0cabbd6365 100644 --- a/packages/llm/llm-retry/src/invariant.ts +++ b/packages/llm/llm-retry/src/invariant.ts @@ -4,6 +4,7 @@ import type { Context } from 'cordis' import type { Session, SessionEvent } from '@deepseek-ai/dsh-session' import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' import type { InvariantFailure, InvariantInstaller } from '@deepseek-ai/dsh-invariants' +import { providerForClosedStep } from './history.ts' import type {} from './index.ts' const PACKAGE_NAME = '@deepseek-ai/dsh-llm-retry' @@ -19,34 +20,46 @@ function validateRetry( event: SessionEvent<'llm/retry'>, fail: InvariantFailure, ): void { - const { turn, step, retry, maxRetries, delayMs } = event.data + const { turn, step, provider, mode, retry, delayMs } = event.data if (!Number.isSafeInteger(retry) || retry < 1) { fail('llm/retry retry must be a positive safe integer') } - if (!Number.isSafeInteger(maxRetries) || maxRetries < 1 || retry > maxRetries) { - fail(`llm/retry retry ${retry} must not exceed a positive safe maxRetries ${maxRetries}`) + if (typeof provider !== 'string' || provider.length === 0) { + fail('llm/retry provider must be non-empty string') } - if (!(delayMs >= 0 && delayMs <= MAX_TIMER_DELAY_MS)) { - fail(`llm/retry delayMs must be within 0..${MAX_TIMER_DELAY_MS}`) - } - - const currentTurnEvents: SessionEvent[] = [] - let openTurn: number | undefined - for (const prior of history.slice().reverse()) { - if (prior.type === 'turn/end') fail('llm/retry must be appended inside an open turn') - if (prior.type === 'turn/start') { - openTurn = prior.data.turn + switch (mode) { + case 'normal': { + const { maxRetries } = event.data + if (!Number.isSafeInteger(maxRetries) || maxRetries < 1 || retry > maxRetries) { + fail(`llm/retry retry ${retry} must not exceed a positive safe maxRetries ${maxRetries}`) + } break } - currentTurnEvents.push(prior) + case 'always': + if ('maxRetries' in event.data) fail('llm/retry always mode must omit maxRetries') + break + default: + fail(`llm/retry mode must be normal or always, got ${String(mode)}`) } - if (openTurn === undefined) fail('llm/retry must be appended inside an open turn') + if (typeof delayMs !== 'number' || !Number.isFinite(delayMs) + || delayMs < 0 || delayMs > MAX_TIMER_DELAY_MS) { + fail(`llm/retry delayMs must be a finite number within 0..${MAX_TIMER_DELAY_MS}`) + } + + const turnStartIndex = history.findLastIndex(prior => + prior.type === 'turn/start' || prior.type === 'turn/end') + const turnBoundary = history[turnStartIndex] + if (turnBoundary?.type !== 'turn/start') { + fail('llm/retry must be appended inside an open turn') + } + const openTurn = turnBoundary.data.turn if (turn !== openTurn) { fail(`llm/retry names turn ${turn}, but the open turn is ${openTurn}`) } + const currentTurnEvents = history.slice(turnStartIndex + 1) let closedStep: number | undefined - for (const prior of currentTurnEvents) { + for (const prior of currentTurnEvents.slice().reverse()) { if (prior.type === 'step/start') { fail(`llm/retry must follow step/end, but step ${prior.data.step} is still open`) } @@ -58,15 +71,26 @@ function validateRetry( if (closedStep === undefined || step !== closedStep) { fail(`llm/retry names step ${step}, but the latest closed step is ${String(closedStep)}`) } + const routedProvider = providerForClosedStep(history, turn, step) + if (routedProvider !== provider) { + fail(`llm/retry provider ${provider} does not match the failed request provider ${String(routedProvider)}`) + } const priorRetries = currentTurnEvents .filter((prior): prior is SessionEvent<'llm/retry'> => prior.type === 'llm/retry') if (priorRetries.some(prior => prior.data.step === step)) { fail(`llm/retry duplicates the retry record for turn ${turn}/step ${step}`) } - const priorRetry = priorRetries[0] - if (priorRetry !== undefined && retry <= priorRetry.data.retry) { - fail(`llm/retry retry ${retry} must increase after retry ${priorRetry.data.retry}`) + const lastSuccessIndex = currentTurnEvents.findLastIndex(prior => prior.type === 'assistant/message') + const priorPolicyRetry = currentTurnEvents.findLast((prior, index): prior is SessionEvent<'llm/retry'> => ( + index > lastSuccessIndex + && prior.type === 'llm/retry' + && prior.data.provider === provider + && prior.data.mode === mode + )) + const expectedRetry = (priorPolicyRetry?.data.retry ?? 0) + 1 + if (retry !== expectedRetry) { + fail(`llm/retry retry ${retry} must equal provider policy retry ${expectedRetry}`) } } diff --git a/packages/llm/llm-retry/tests/invariant.spec.ts b/packages/llm/llm-retry/tests/invariant.spec.ts index 7f9bc6b061..c9c5d2c564 100644 --- a/packages/llm/llm-retry/tests/invariant.spec.ts +++ b/packages/llm/llm-retry/tests/invariant.spec.ts @@ -4,6 +4,7 @@ import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' import InvariantService from '@deepseek-ai/dsh-invariants' import * as RetryInvariant from '@deepseek-ai/dsh-llm-retry/invariant' +import { providerForClosedStep } from '../src/history.ts' async function setup(): Promise { const ctx = new Context() @@ -17,33 +18,116 @@ function closeStep(ctx: Context, id: string, turn = 1, step = 1) { const session = ctx.sessions.create(SessionId(id)) session.append('turn/start', { turn, trigger: { kind: 'message', source: { kind: 'user' } } }) session.append('step/start', { turn, step }) + session.append('request/header', { + header: { config: { provider: 'mock', model: 'mock' } }, + reason: 'initial', + }) session.append('step/end', { turn, step }) return session } const failure = { message: 'provider busy', code: 'RATE_LIMIT', status: 429 } +const normal = { provider: 'mock', mode: 'normal' as const } describe('llm-retry invariants', () => { + it('has no provider without the requested closed step', () => { + expect(providerForClosedStep([], 1, 1)).toBeUndefined() + expect(providerForClosedStep([{ + type: 'step/end', + data: { turn: 1, step: 1 }, + }] as never, 1, 1)).toBeUndefined() + }) + + it('does not inherit a provider across a turn boundary', () => { + expect(providerForClosedStep([ + { type: 'turn/start', data: { turn: 1 } }, + { + type: 'request/header', + data: { header: { config: { provider: 'prior' } } }, + }, + { type: 'turn/end', data: { turn: 1 } }, + { type: 'turn/start', data: { turn: 2 } }, + { type: 'step/end', data: { turn: 2, step: 1 } }, + ] as never, 2, 1)).toBeUndefined() + }) + it('accepts increasing retry records for successive closed steps and ignores unrelated events', async () => { const ctx = await setup() const session = closeStep(ctx, 'retry-invariant-valid') expect(() => { session.append('llm/retry', { - turn: 1, step: 1, retry: 1, maxRetries: 2, delayMs: 500, failure, + turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 500, failure, }) session.append('step/start', { turn: 1, step: 2 }) session.append('step/end', { turn: 1, step: 2 }) session.append('llm/retry', { - turn: 1, step: 2, retry: 2, maxRetries: 2, delayMs: 1_000, failure, + turn: 1, step: 2, ...normal, retry: 2, maxRetries: 2, delayMs: 1_000, failure, }) const zeroDelay = closeStep(ctx, 'retry-invariant-zero-delay') zeroDelay.append('llm/retry', { - turn: 1, step: 1, retry: 1, maxRetries: 1, delayMs: 0, failure, + turn: 1, step: 1, ...normal, retry: 1, maxRetries: 1, delayMs: 0, failure, }) }).not.toThrow() expect(() => { ctx.emit('tools/change') }).not.toThrow() }) + it('accepts unbounded always records without serializing an infinite maximum', async () => { + const ctx = await setup() + const session = closeStep(ctx, 'retry-invariant-always') + expect(() => { + session.append('llm/retry', { + turn: 1, + step: 1, + provider: 'mock', + mode: 'always', + retry: 1, + delayMs: 500, + failure, + }) + }).not.toThrow() + expect(() => { + session.append('llm/retry', { + turn: 1, + step: 1, + provider: 'mock', + mode: 'always', + retry: 1, + maxRetries: 2, + delayMs: 500, + failure, + } as never) + }).toThrow(/always mode must omit maxRetries/) + }) + + it('rejects empty providers and unknown modes from hostile durable input', async () => { + const ctx = await setup() + const emptyProvider = closeStep(ctx, 'retry-invariant-empty-provider') + expect(() => { + emptyProvider.append('llm/retry', { + turn: 1, + step: 1, + provider: '', + mode: 'always', + retry: 1, + delayMs: 1, + failure, + }) + }).toThrow(/provider must be non-empty/) + + const unknownMode = closeStep(ctx, 'retry-invariant-unknown-mode') + expect(() => { + unknownMode.append('llm/retry', { + turn: 1, + step: 1, + provider: 'mock', + mode: 'sometimes', + retry: 1, + delayMs: 1, + failure, + } as never) + }).toThrow(/mode must be normal or always/) + }) + it.each([ [{ retry: 0, maxRetries: 2, delayMs: 1 }, /positive safe integer/], [{ retry: 1.5, maxRetries: 2, delayMs: 1 }, /positive safe integer/], @@ -56,7 +140,7 @@ describe('llm-retry invariants', () => { const ctx = await setup() const session = closeStep(ctx, `retry-invariant-bounds-${data.retry}-${data.maxRetries}-${data.delayMs}`) expect(() => { - session.append('llm/retry', { turn: 1, step: 1, ...data, failure }) + session.append('llm/retry', { turn: 1, step: 1, ...normal, ...data, failure }) }).toThrow(message) }) @@ -65,14 +149,14 @@ describe('llm-retry invariants', () => { const absent = ctx.sessions.create(SessionId('retry-invariant-no-turn')) expect(() => { absent.append('llm/retry', { - turn: 1, step: 1, retry: 1, maxRetries: 2, delayMs: 1, failure, + turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, }) }).toThrow(/inside an open turn/) const wrongTurn = closeStep(ctx, 'retry-invariant-wrong-turn') expect(() => { wrongTurn.append('llm/retry', { - turn: 2, step: 1, retry: 1, maxRetries: 2, delayMs: 1, failure, + turn: 2, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, }) }).toThrow(/open turn is 1/) @@ -81,7 +165,7 @@ describe('llm-retry invariants', () => { openStep.append('step/start', { turn: 1, step: 1 }) expect(() => { openStep.append('llm/retry', { - turn: 1, step: 1, retry: 1, maxRetries: 2, delayMs: 1, failure, + turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, }) }).toThrow(/step 1 is still open/) @@ -89,14 +173,14 @@ describe('llm-retry invariants', () => { noStep.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) expect(() => { noStep.append('llm/retry', { - turn: 1, step: 1, retry: 1, maxRetries: 2, delayMs: 1, failure, + turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, }) }).toThrow(/latest closed step is undefined/) const wrongStep = closeStep(ctx, 'retry-invariant-wrong-step') expect(() => { wrongStep.append('llm/retry', { - turn: 1, step: 2, retry: 1, maxRetries: 2, delayMs: 1, failure, + turn: 1, step: 2, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, }) }).toThrow(/latest closed step is 1/) @@ -104,34 +188,97 @@ describe('llm-retry invariants', () => { closedTurn.append('turn/end', { turn: 1, reason: { kind: 'aborted' } }) expect(() => { closedTurn.append('llm/retry', { - turn: 1, step: 1, retry: 1, maxRetries: 2, delayMs: 1, failure, + turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, }) }).toThrow(/inside an open turn/) }) + it('binds the policy provider to the failed step rather than a later header', async () => { + const ctx = await setup() + const session = closeStep(ctx, 'retry-invariant-provider') + session.append('request/header', { + header: { config: { provider: 'other', model: 'mock' } }, + reason: 'change', + }) + expect(() => { + session.append('llm/retry', { + turn: 1, + step: 1, + provider: 'mock', + mode: 'always', + retry: 1, + delayMs: 1, + failure, + }) + }).not.toThrow() + + const mismatch = closeStep(ctx, 'retry-invariant-provider-mismatch') + expect(() => { + mismatch.append('llm/retry', { + turn: 1, + step: 1, + provider: 'other', + mode: 'always', + retry: 1, + delayMs: 1, + failure, + }) + }).toThrow(/does not match the failed request provider mock/) + }) + + it('rejects a current-turn retry without a current-turn provider route', async () => { + const ctx = await setup() + const session = closeStep(ctx, 'retry-invariant-prior-route') + session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + session.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 2, step: 1 }) + session.append('step/end', { turn: 2, step: 1 }) + expect(() => { + session.append('llm/retry', { + turn: 2, + step: 1, + provider: 'mock', + mode: 'always', + retry: 1, + delayMs: 1, + failure, + }) + }).toThrow(/does not match the failed request provider undefined/) + }) + + it('rejects non-numeric durable delays', async () => { + const ctx = await setup() + const session = closeStep(ctx, 'retry-invariant-delay-type') + expect(() => { + session.append('llm/retry', { + turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: '1', failure, + } as never) + }).toThrow(/delayMs must be a finite number/) + }) + it('rejects duplicate and non-increasing retry records', async () => { const ctx = await setup() const duplicate = closeStep(ctx, 'retry-invariant-duplicate') duplicate.append('llm/retry', { - turn: 1, step: 1, retry: 1, maxRetries: 3, delayMs: 1, failure, + turn: 1, step: 1, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure, }) expect(() => { duplicate.append('llm/retry', { - turn: 1, step: 1, retry: 2, maxRetries: 3, delayMs: 1, failure, + turn: 1, step: 1, ...normal, retry: 2, maxRetries: 3, delayMs: 1, failure, }) }).toThrow(/duplicates the retry record/) const nonIncreasing = closeStep(ctx, 'retry-invariant-non-increasing') nonIncreasing.append('llm/retry', { - turn: 1, step: 1, retry: 1, maxRetries: 3, delayMs: 1, failure, + turn: 1, step: 1, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure, }) nonIncreasing.append('step/start', { turn: 1, step: 2 }) nonIncreasing.append('step/end', { turn: 1, step: 2 }) expect(() => { nonIncreasing.append('llm/retry', { - turn: 1, step: 2, retry: 1, maxRetries: 3, delayMs: 1, failure, + turn: 1, step: 2, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure, }) - }).toThrow(/must increase/) + }).toThrow(/must equal provider policy retry 2/) }) it('validates existing histories on late registration', async () => { @@ -140,7 +287,7 @@ describe('llm-retry invariants', () => { const session = ctx.sessions.create(SessionId('retry-invariant-late')) session.append('step/end', { turn: 1, step: 1 }) session.append('llm/retry', { - turn: 1, step: 1, retry: 1, maxRetries: 2, delayMs: 1, failure, + turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, }) await ctx.plugin(InvariantService) await expect(ctx.plugin(RetryInvariant)).rejects.toThrow(/inside an open turn/) diff --git a/packages/llm/llm-retry/tests/loader-composition.spec.ts b/packages/llm/llm-retry/tests/loader-composition.spec.ts index 1aafc91d92..3ecc5985fe 100644 --- a/packages/llm/llm-retry/tests/loader-composition.spec.ts +++ b/packages/llm/llm-retry/tests/loader-composition.spec.ts @@ -9,8 +9,8 @@ import Include from '@cordisjs/plugin-include' import AgentRegistry from '@deepseek-ai/dsh-agent' import type { Agent } from '@deepseek-ai/dsh-agent' import AgentLoop from '@deepseek-ai/dsh-agent-loop' -import LlmService, { LlmAdapter, LlmError } from '@deepseek-ai/dsh-llm' -import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm' +import LlmService, { LlmAdapter, LlmError, resolveRetryPolicy } from '@deepseek-ai/dsh-llm' +import type { GenerateOptions, ResolvedRetryPolicy, StreamChunk } from '@deepseek-ai/dsh-llm' import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' import SystemPrompt from '@deepseek-ai/dsh-system-prompt' import ToolRegistry from '@deepseek-ai/dsh-tools' @@ -21,6 +21,16 @@ let context: Context | undefined class TransientOnceAdapter extends LlmAdapter { requests = 0 + private readonly retryPolicy = resolveRetryPolicy({ + mode: 'normal', + maxRetries: 1, + retryableCodes: ['RATE_LIMIT', 'SERVER'], + backoff: { initialDelayMs: 1, maxDelayMs: 1, jitterRatio: 0 }, + }, 'loader test provider retryPolicy') + + override providerRetryPolicy(_provider: string): ResolvedRetryPolicy { + return this.retryPolicy + } async * stream(_options: GenerateOptions): AsyncIterable { this.requests += 1 @@ -87,7 +97,7 @@ describe('real Loader composition', () => { // Real-Loader composition resolves workspace packages through tsx at test // time; first resolution after the host/client program split is slow enough // to trip the default 5s budget on cold caches. - it('loads the flat policy and records recovery through the shipping loop', { timeout: 60_000 }, async () => { + it('loads provider-supplied policy and records recovery through the shipping loop', { timeout: 60_000 }, async () => { const loaded = await loadYaml([ "- name: '@deepseek-ai/dsh-llm'", "- name: '@deepseek-ai/dsh-session'", @@ -95,12 +105,6 @@ describe('real Loader composition', () => { "- name: '@deepseek-ai/dsh-tools'", "- name: '@deepseek-ai/dsh-agent'", "- name: '@deepseek-ai/dsh-llm-retry'", - ' config:', - ' maxTransientRetries: 1', - ' initialDelayMs: 1', - ' maxDelayMs: 1', - ' jitterRatio: 0', - ' retryableCodes: [RATE_LIMIT, SERVER]', "- name: '@deepseek-ai/dsh-agent-loop'", ]) diff --git a/packages/llm/llm-retry/tests/persistence.spec.ts b/packages/llm/llm-retry/tests/persistence.spec.ts index 1668c36d73..41c70b257d 100644 --- a/packages/llm/llm-retry/tests/persistence.spec.ts +++ b/packages/llm/llm-retry/tests/persistence.spec.ts @@ -34,12 +34,17 @@ describe.each(['jsonl', 'sqlite'] as const)('%s retry-event persistence', (kind) const session = ctx.sessions.create(SessionId(`retry-${kind}`)) session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) session.append('step/start', { turn: 1, step: 1 }) + session.append('request/header', { + header: { config: { provider: 'mock', model: 'mock' } }, + reason: 'initial', + }) session.append('step/end', { turn: 1, step: 1 }) const event = session.append('llm/retry', { turn: 1, step: 1, + provider: 'mock', + mode: 'always', retry: 1, - maxRetries: 2, delayMs: 750, failure: { message: 'provider busy', code: 'RATE_LIMIT', status: 429 }, }) diff --git a/packages/llm/llm-retry/tests/retry.spec.ts b/packages/llm/llm-retry/tests/retry.spec.ts index 4dc1c06bd6..238385baf5 100644 --- a/packages/llm/llm-retry/tests/retry.spec.ts +++ b/packages/llm/llm-retry/tests/retry.spec.ts @@ -1,8 +1,16 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { Context } from 'cordis' import type { Fiber } from 'cordis' -import LlmService, { CallId, LlmAdapter, LlmError } from '@deepseek-ai/dsh-llm' -import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm' +import LlmService, { CallId, LlmAdapter, LlmError, resolveRetryPolicy } from '@deepseek-ai/dsh-llm' +import type { + AlwaysRetryPolicyConfig, + BackoffConfig, + GenerateOptions, + NormalRetryPolicyConfig, + ResolvedRetryPolicy, + RetryPolicyConfig, + StreamChunk, +} from '@deepseek-ai/dsh-llm' import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' import type { SessionEvent } from '@deepseek-ai/dsh-session' import SystemPrompt from '@deepseek-ai/dsh-system-prompt' @@ -10,13 +18,13 @@ import ToolRegistry, { defineContentToolFixture } from '@deepseek-ai/dsh-tools' import AgentRegistry from '@deepseek-ai/dsh-agent' import type { Agent, RequestErrorDecision } from '@deepseek-ai/dsh-agent' import AgentLoop from '@deepseek-ai/dsh-agent-loop' -import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' import * as retry from '../src/index.ts' type ScriptEntry = Error | Iterable | AsyncIterable class ScriptedAdapter extends LlmAdapter { readonly requests: GenerateOptions[] = [] + private retryPolicies: Readonly> = {} constructor(private readonly entries: ScriptEntry[]) { super() @@ -29,6 +37,21 @@ class ScriptedAdapter extends LlmAdapter { if (entry instanceof Error) throw entry yield* entry } + + configureRetryPolicies( + policies: Readonly>, + ): void { + this.retryPolicies = Object.fromEntries(Object.entries(policies).map(([provider, policy]) => [ + provider, + policy === undefined + ? undefined + : resolveRetryPolicy(policy, `retry test provider "${provider}" retryPolicy`), + ])) + } + + override providerRetryPolicy(provider: string): ResolvedRetryPolicy | undefined { + return this.retryPolicies[provider] + } } async function* partialToolFailure(error: Error): AsyncGenerator { @@ -52,8 +75,8 @@ function textResponse(text: string): StreamChunk[] { } async function harness( - adapter: LlmAdapter, - config: retry.Config = {}, + adapter: ScriptedAdapter, + policies: Readonly> = { mock: normalConfig() }, beforeRetry?: (ctx: Context) => void, internals: retry.RetryInternals = {}, ): Promise<{ ctx: Context; retryFiber: Fiber }> { @@ -64,20 +87,44 @@ async function harness( await ctx.plugin(ToolRegistry) await ctx.plugin(AgentRegistry) beforeRetry?.(ctx) - const resolvedConfig = Object.assign({ - maxTransientRetries: 2, - initialDelayMs: 500, - maxDelayMs: 10_000, - jitterRatio: 0, - }, config) + adapter.configureRetryPolicies(policies) const retryFiber = await ctx.plugin(Object.assign((inner: Context) => { - retry.apply(inner, resolvedConfig, internals) + retry.apply(inner, {}, internals) }, { inject: retry.inject })) await ctx.plugin(AgentLoop, { agents: [] }) - ctx.llm.registerAdapter(['mock'], adapter) + ctx.llm.registerAdapter(['mock', 'other'], adapter) return { ctx, retryFiber } } +function normalConfig( + overrides: Partial> = {}, +): NormalRetryPolicyConfig { + const { backoff, ...policy } = overrides + return { + mode: 'normal', + maxRetries: 2, + ...policy, + backoff: { + initialDelayMs: 500, + maxDelayMs: 10_000, + jitterRatio: 0, + ...backoff, + }, + } +} + +function alwaysConfig(backoff: BackoffConfig = {}): AlwaysRetryPolicyConfig { + return { + mode: 'always', + backoff: { + initialDelayMs: 500, + maxDelayMs: 10_000, + jitterRatio: 0, + ...backoff, + }, + } +} + function waitForIdle(ctx: Context, agent: Agent): Promise { return new Promise((resolve) => { const dispose = ctx.on('agent/status', (subject, status) => { @@ -108,14 +155,14 @@ afterEach(async () => { context = undefined }) -describe('bounded transient retry policy', () => { +describe('provider-routed retry policy', () => { it('records the scheduled delay before opening a fresh request attempt', async () => { vi.useFakeTimers() const adapter = new ScriptedAdapter([ new LlmError('busy', 'RATE_LIMIT', { status: 429 }), textResponse('done'), ]) - ;({ ctx: context } = await harness(adapter)) + ;({ ctx: context } = await harness(adapter, {}, undefined, { random: () => 0.5 })) const agent = context.agentLoop.create(SessionId('retry-success'), { provider: 'mock', model: 'mock', @@ -135,6 +182,8 @@ describe('bounded transient retry policy', () => { expect(event.data).toEqual({ turn: 1, step: 1, + provider: 'mock', + mode: 'normal', retry: 1, maxRetries: 2, delayMs: 500, @@ -207,7 +256,9 @@ describe('bounded transient retry policy', () => { new LlmError('busy two', 'SERVER'), new LlmError('busy three', 'SERVER'), ]) - ;({ ctx: context } = await harness(adapter, { jitterRatio: 0.1 }, undefined, { + ;({ ctx: context } = await harness(adapter, { mock: normalConfig({ + backoff: { jitterRatio: 0.1 }, + }) }, undefined, { random: () => samples.shift() ?? 0.5, })) const agent = context.agentLoop.create(SessionId('retry-exhausted'), { provider: 'mock', model: 'mock' }) @@ -238,11 +289,9 @@ describe('bounded transient retry policy', () => { new LlmError('busy', 'SERVER'), textResponse('done'), ]) - ;({ ctx: context } = await harness(adapter, { - initialDelayMs: 1, - maxDelayMs: 1, - jitterRatio: 1, - }, undefined, { random: () => 0 })) + ;({ ctx: context } = await harness(adapter, { mock: normalConfig({ + backoff: { initialDelayMs: 1, maxDelayMs: 1, jitterRatio: 1 }, + }) }, undefined, { random: () => 0 })) const agent = context.agentLoop.create(SessionId('retry-zero-delay'), { provider: 'mock', model: 'mock' }) const scheduled = waitForRetry(context, agent, 1) @@ -261,7 +310,9 @@ describe('bounded transient retry policy', () => { new LlmError('wait', 'RATE_LIMIT', { providerRetryAfterMs: 2_000 }), textResponse('done'), ]) - ;({ ctx: context } = await harness(accepted, { jitterRatio: 1 })) + ;({ ctx: context } = await harness(accepted, { mock: normalConfig({ + backoff: { jitterRatio: 1 }, + }) })) const acceptedAgent = context.agentLoop.create(SessionId('retry-after-accepted'), { provider: 'mock', model: 'mock' }) const scheduled = waitForRetry(context, acceptedAgent, 1) acceptedAgent.send([{ type: 'text', text: 'go' }]) @@ -284,6 +335,32 @@ describe('bounded transient retry policy', () => { expect(rejectedAgent.session.events.some(event => event.type === 'llm/retry')).toBe(false) }) + it('uses local jittered backoff when always mode receives an over-cap Retry-After', async () => { + vi.useFakeTimers() + const adapter = new ScriptedAdapter([ + new LlmError('wait too long', 'AUTH', { providerRetryAfterMs: 10 }), + textResponse('done'), + ]) + ;({ ctx: context } = await harness(adapter, { mock: alwaysConfig({ + initialDelayMs: 2, + maxDelayMs: 4, + jitterRatio: 0.5, + }) }, undefined, { random: () => 1 })) + const agent = context.agentLoop.create(SessionId('retry-always-over-cap'), { + provider: 'mock', + model: 'mock', + }) + const scheduled = waitForRetry(context, agent, 1) + + agent.send([{ type: 'text', text: 'go' }]) + expect((await scheduled).data.delayMs).toBe(3) + const idle = waitForIdle(context, agent) + await vi.advanceTimersByTimeAsync(3) + await idle + + expect(adapter.requests).toHaveLength(2) + }) + it('delegates non-transient failures without scheduling a timer', async () => { vi.useFakeTimers() const adapter = new ScriptedAdapter([new LlmError('bad key', 'AUTH')]) @@ -297,13 +374,203 @@ describe('bounded transient retry policy', () => { expect(vi.getTimerCount()).toBe(0) }) + it('selects policy by the failed request provider', async () => { + vi.useFakeTimers() + const adapter = new ScriptedAdapter([ + new LlmError('mock auth failed', 'AUTH'), + new LlmError('other auth failed', 'AUTH'), + textResponse('other recovered'), + ]) + ;({ ctx: context } = await harness(adapter, { + other: alwaysConfig({ initialDelayMs: 1, maxDelayMs: 1, jitterRatio: 0 }), + })) + + const normalAgent = context.agentLoop.create(SessionId('retry-provider-normal'), { + provider: 'mock', + model: 'mock', + }) + const normalIdle = waitForIdle(context, normalAgent) + normalAgent.send([{ type: 'text', text: 'normal' }]) + await normalIdle + expect(normalAgent.session.events.some(event => event.type === 'llm/retry')).toBe(false) + + const alwaysAgent = context.agentLoop.create(SessionId('retry-provider-always'), { + provider: 'other', + model: 'mock', + }) + const scheduled = waitForRetry(context, alwaysAgent, 1) + alwaysAgent.send([{ type: 'text', text: 'always' }]) + expect((await scheduled).data).toMatchObject({ + provider: 'other', + mode: 'always', + retry: 1, + delayMs: 1, + }) + const alwaysIdle = waitForIdle(context, alwaysAgent) + await vi.advanceTimersByTimeAsync(1) + await alwaysIdle + + expect(adapter.requests.map(request => request.provider)).toEqual(['mock', 'other', 'other']) + }) + + it('selects an always policy from the provider chosen by agent/request', async () => { + vi.useFakeTimers() + const adapter = new ScriptedAdapter([ + new LlmError('rerouted auth failed', 'AUTH'), + textResponse('rerouted recovery'), + ]) + ;({ ctx: context } = await harness(adapter, { + other: alwaysConfig({ initialDelayMs: 1, maxDelayMs: 1, jitterRatio: 0 }), + }, (ctx) => { + ctx.on('agent/request', async (_agent, _turn, _step, config) => ({ + ...config, + provider: 'other', + })) + })) + const agent = context.agentLoop.create(SessionId('retry-provider-rerouted'), { + provider: 'mock', + model: 'mock', + }) + const scheduled = waitForRetry(context, agent, 1) + + agent.send([{ type: 'text', text: 'reroute' }]) + expect((await scheduled).data).toMatchObject({ provider: 'other', mode: 'always' }) + const idle = waitForIdle(context, agent) + await vi.advanceTimersByTimeAsync(1) + await idle + + expect(adapter.requests.map(request => request.provider)).toEqual(['other', 'other']) + }) + + it('keeps always mode unbounded while preserving cancellable jittered backoff', async () => { + vi.useFakeTimers() + const adapter = new ScriptedAdapter([ + new LlmError('auth one', 'AUTH'), + new LlmError('auth two', 'AUTH'), + new LlmError('auth three', 'AUTH'), + new LlmError('auth four', 'AUTH'), + textResponse('eventually recovered'), + ]) + ;({ ctx: context } = await harness(adapter, { mock: alwaysConfig({ + initialDelayMs: 1, + maxDelayMs: 4, + jitterRatio: 0.1, + }) }, undefined, { random: () => 1 })) + const agent = context.agentLoop.create(SessionId('retry-always-unbounded'), { + provider: 'mock', + model: 'mock', + }) + const idle = waitForIdle(context, agent) + + agent.send([{ type: 'text', text: 'keep trying' }]) + await vi.runAllTimersAsync() + await idle + + const events = agent.session.events.filter(event => event.type === 'llm/retry') + expect(adapter.requests).toHaveLength(5) + expect(events.map(event => ({ + provider: event.data.provider, + mode: event.data.mode, + retry: event.data.retry, + delayMs: event.data.delayMs, + hasMax: 'maxRetries' in event.data, + }))).toEqual([ + { provider: 'mock', mode: 'always', retry: 1, delayMs: 1.1, hasMax: false }, + { provider: 'mock', mode: 'always', retry: 2, delayMs: 2.2, hasMax: false }, + { provider: 'mock', mode: 'always', retry: 3, delayMs: 4, hasMax: false }, + { provider: 'mock', mode: 'always', retry: 4, delayMs: 4, hasMax: false }, + ]) + }) + + it('keeps failed error text and partial output out of every retried model context', async () => { + vi.useFakeTimers() + const diagnostic = 'private provider diagnostic must not enter context' + const adapter = new ScriptedAdapter([ + partialToolFailure(new LlmError(diagnostic, 'AUTH')), + textResponse('recovered without leaked context'), + ]) + ;({ ctx: context } = await harness(adapter, { mock: alwaysConfig({ + initialDelayMs: 1, + maxDelayMs: 1, + }) })) + const agent = context.agentLoop.create(SessionId('retry-always-context-isolation'), { + provider: 'mock', + model: 'mock', + }) + const scheduled = waitForRetry(context, agent, 1) + + agent.send([{ type: 'text', text: 'safe input' }]) + await scheduled + const idle = waitForIdle(context, agent) + await vi.advanceTimersByTimeAsync(1) + await idle + + expect(adapter.requests).toHaveLength(2) + expect(adapter.requests[1]?.messages).toEqual(adapter.requests[0]?.messages) + const retriedContext = JSON.stringify(adapter.requests[1]?.messages) + expect(retriedContext).not.toContain(diagnostic) + expect(retriedContext).not.toContain('discarded partial output') + expect(agent.session.events.some(event => + event.type === 'llm/retry' && event.data.failure.message === diagnostic, + )).toBe(true) + }) + + it('lets downstream specialized recovery run before always fallback', async () => { + const adapter = new ScriptedAdapter([ + new LlmError('requires specialized recovery', 'AUTH'), + textResponse('specialized recovery won'), + ]) + ;({ ctx: context } = await harness(adapter, { mock: alwaysConfig() })) + context.on('agent/request-error', async () => ({ action: 'retry' })) + const agent = context.agentLoop.create(SessionId('retry-always-composition'), { + provider: 'mock', + model: 'mock', + }) + const idle = waitForIdle(context, agent) + + agent.send([{ type: 'text', text: 'recover' }]) + await idle + + expect(adapter.requests).toHaveLength(2) + expect(agent.session.events.some(event => event.type === 'llm/retry')).toBe(false) + }) + + it.each([ + ['synchronously', () => { throw new Error('downstream recovery failed') }], + ['asynchronously', async () => { throw new Error('downstream recovery failed') }], + ])('falls back to always retry when downstream recovery throws %s', async (_kind, failDownstream) => { + vi.useFakeTimers() + const adapter = new ScriptedAdapter([ + new LlmError('requires fallback', 'AUTH'), + textResponse('always recovered'), + ]) + ;({ ctx: context } = await harness(adapter, { mock: alwaysConfig({ + initialDelayMs: 1, + maxDelayMs: 1, + }) })) + context.on('agent/request-error', failDownstream) + const agent = context.agentLoop.create(SessionId('retry-always-downstream-error'), { + provider: 'mock', + model: 'mock', + }) + const scheduled = waitForRetry(context, agent, 1) + + agent.send([{ type: 'text', text: 'recover' }]) + await scheduled + const idle = waitForIdle(context, agent) + await vi.advanceTimersByTimeAsync(1) + await idle + + expect(adapter.requests).toHaveLength(2) + }) + it('aborts and drains a captured backoff before plugin disposal completes', async () => { vi.useFakeTimers() const adapter = new ScriptedAdapter([ new LlmError('temporary', 'TRANSPORT'), textResponse('must not run'), ]) - const mounted = await harness(adapter) + const mounted = await harness(adapter, { mock: alwaysConfig() }) context = mounted.ctx const agent = context.agentLoop.create(SessionId('retry-hmr'), { provider: 'mock', model: 'mock' }) const scheduled = waitForRetry(context, agent, 1) @@ -322,7 +589,7 @@ describe('bounded transient retry policy', () => { it('does not make plugin disposal wait for a delegated recovery policy', async () => { const adapter = new ScriptedAdapter([new LlmError('bad key', 'AUTH')]) - const mounted = await harness(adapter) + const mounted = await harness(adapter, { mock: alwaysConfig() }) context = mounted.ctx const downstream = Promise.withResolvers() const entered = Promise.withResolvers() @@ -353,6 +620,61 @@ describe('bounded transient retry policy', () => { expect(adapter.requests).toHaveLength(1) }) + it('lets turn cancellation interrupt a delegated recovery policy', async () => { + const adapter = new ScriptedAdapter([new LlmError('bad key', 'AUTH')]) + const mounted = await harness(adapter, { mock: alwaysConfig() }) + context = mounted.ctx + const downstream = Promise.withResolvers() + const entered = Promise.withResolvers() + context.on('agent/request-error', () => { + entered.resolve(undefined) + return downstream.promise + }) + const agent = context.agentLoop.create(SessionId('retry-delegated-cancel'), { + provider: 'mock', + model: 'mock', + }) + const idle = waitForIdle(context, agent) + agent.send([{ type: 'text', text: 'go' }]) + await entered.promise + + agent.cancel({ kind: 'user' }) + await idle + downstream.resolve({ action: 'fail' }) + + expect(adapter.requests).toHaveLength(1) + expect(agent.session.events.at(-1)).toMatchObject({ + type: 'turn/end', + data: { reason: { kind: 'aborted' } }, + }) + }) + + it('handles synchronous cancellation while entering delegated recovery', async () => { + const adapter = new ScriptedAdapter([new LlmError('bad key', 'AUTH')]) + const mounted = await harness(adapter, { mock: alwaysConfig() }) + context = mounted.ctx + const downstream = Promise.withResolvers() + context.on('agent/request-error', (agent) => { + agent.cancel({ kind: 'user' }) + return downstream.promise + }) + const agent = context.agentLoop.create(SessionId('retry-delegated-sync-cancel'), { + provider: 'mock', + model: 'mock', + }) + const idle = waitForIdle(context, agent) + + agent.send([{ type: 'text', text: 'go' }]) + await idle + downstream.resolve({ action: 'fail' }) + + expect(adapter.requests).toHaveLength(1) + expect(agent.session.events.at(-1)).toMatchObject({ + type: 'turn/end', + data: { reason: { kind: 'aborted' } }, + }) + }) + it('fails a captured callback after disposal without entering downstream policy', async () => { const adapter = new ScriptedAdapter([new LlmError('bad key', 'AUTH')]) const captured = Promise.withResolvers() @@ -391,10 +713,10 @@ describe('bounded transient retry policy', () => { it('lets turn cancellation win during backoff without opening another step', async () => { vi.useFakeTimers() const adapter = new ScriptedAdapter([ - new LlmError('temporary', 'TIMEOUT'), + new LlmError('permanent', 'AUTH'), textResponse('must not run'), ]) - ;({ ctx: context } = await harness(adapter)) + ;({ ctx: context } = await harness(adapter, { mock: alwaysConfig() })) const agent = context.agentLoop.create(SessionId('retry-cancel'), { provider: 'mock', model: 'mock' }) const scheduled = waitForRetry(context, agent, 1) agent.send([{ type: 'text', text: 'go' }]) @@ -411,13 +733,16 @@ describe('bounded transient retry policy', () => { expect(vi.getTimerCount()).toBe(0) }) - it('lets an earlier recovery listener cancel before retry policy runs', async () => { + it.each([ + ['normal', normalConfig()], + ['always', alwaysConfig()], + ])('lets an earlier recovery listener cancel before %s retry policy runs', async (_mode, policy) => { vi.useFakeTimers() const adapter = new ScriptedAdapter([ new LlmError('temporary', 'SERVER'), textResponse('must not run'), ]) - ;({ ctx: context } = await harness(adapter, {}, (ctx) => { + ;({ ctx: context } = await harness(adapter, { mock: policy }, (ctx) => { ctx.on('agent/request-error', async (agent, _turn, _step, _error, _failure, _history, _signal, next) => { agent.cancel({ kind: 'user' }) return next() @@ -458,19 +783,15 @@ describe('bounded transient retry policy', () => { expect(vi.getTimerCount()).toBe(0) }) - it.each([ - [{ maxTransientRetries: -1 }, /maxTransientRetries/], - [{ maxTransientRetries: 1.5 }, /maxTransientRetries/], - [{ initialDelayMs: 0 }, /initialDelayMs/], - [{ maxDelayMs: Number.POSITIVE_INFINITY }, /maxDelayMs/], - [{ initialDelayMs: MAX_TIMER_DELAY_MS + 1 }, /initialDelayMs/], - [{ maxDelayMs: MAX_TIMER_DELAY_MS + 1 }, /maxDelayMs/], - [{ initialDelayMs: 20, maxDelayMs: 10 }, /less than or equal/], - [{ jitterRatio: 1.1 }, /jitterRatio/], - [{ retryableCodes: [] }, /must not be empty/], - [{ retryableCodes: ['SERVER', 'SERVER'] }, /duplicates/], - [{ retryableCodes: [''] }, /non-empty strings/], - ] as const)('fails direct composition for invalid config %#', (config, message) => { - expect(() => { retry.apply(new Context(), config as retry.Config) }).toThrow(message) + it('rejects retry policy configured on the executor instead of a provider', () => { + expect(() => { + retry.apply(new Context(), { retryPolicy: { mode: 'always' } }) + }).toThrow(/retryPolicy belongs under each provider/) + }) + + it('rejects unknown executor config', () => { + expect(() => { + retry.apply(new Context(), { retryPolciy: {} }) + }).toThrow(/unknown key "retryPolciy"/) }) }) diff --git a/packages/llm/llm-retry/tsdown.config.ts b/packages/llm/llm-retry/tsdown.config.ts new file mode 100644 index 0000000000..ab8dc26ee8 --- /dev/null +++ b/packages/llm/llm-retry/tsdown.config.ts @@ -0,0 +1,25 @@ +import { defineConfig } from 'tsdown' + +/** Build the package root and invariant companion as independent bundles. */ +export default defineConfig([ + { + entry: ['lib/types/index.js'], + outDir: 'lib', + format: ['esm'], + platform: 'node', + target: 'es2024', + fixedExtension: false, + dts: false, + clean: false, + }, + { + entry: ['lib/types/invariant.js'], + outDir: 'lib', + format: ['esm'], + platform: 'node', + target: 'es2024', + fixedExtension: false, + dts: false, + clean: false, + }, +]) diff --git a/packages/llm/llm/README.md b/packages/llm/llm/README.md index beac6d5e8e..74dafa3e48 100644 --- a/packages/llm/llm/README.md +++ b/packages/llm/llm/README.md @@ -10,13 +10,14 @@ An adapter registry plus a single streaming call surface, interceptable via a wa - `ctx.llm.registerAdapter(providers: string[], adapter: LlmAdapter): () => void` Register one adapter instance for the given provider routes. Registration is all-or-nothing, and is disposed with the calling fiber. - `ctx.llm.listProviders(): LlmProviderInfo[]` Describe registered provider routes in registration order. +- `ctx.llm.providerRetryPolicy(provider: string): ResolvedRetryPolicy` Return the provider-owned retry policy captured during registration, with normal defaults resolved. - `ctx.llm.listModels(provider: string): Promise` Discover the models one registered provider currently advertises. - `ctx.llm.resolveModelContext(provider: string, model: string): Promise` Resolve authoritative context capacity for one exact route from its owning adapter. - `ctx.llm.stream(options: GenerateOptions): AsyncIterable` Stream one model call as raw chunks (token-level deltas). Consumers assemble the chunks into blocks/messages with `BlockAssembler`. `LlmService` preserves errors from final adapter selection, synchronous dispatch, iterator construction, and iteration, and binds their provenance to the exact stream handle returned for that model call. `isLlmAdapterFailure(stream, value)` reports only errors from that call's final adapter boundary; `llmFailureOf(stream, value)` returns the adjacent immutable `LlmFailure`. Nested model calls, `llm/stream` middleware, and downstream consumer failures remain unclassified for the outer call. Classification never replaces or mutates the adapter's original coded `Error`. -Provider and model metadata is a discovery surface, not a routing whitelist. `registerAdapter()` still owns provider exclusivity, while an adapter may accept model ids absent from `listModels()`; consumers must not reject a request because its model is unlisted. Returned metadata is detached and invalid or duplicate adapter entries fail with `INVALID_ADAPTER` or `INVALID_CATALOG`. +Provider and model metadata is a discovery surface, not a routing whitelist. `registerAdapter()` still owns provider exclusivity and captures the adapter's retry policy for each route, while an adapter may accept model ids absent from `listModels()`; consumers must not reject a request because its model is unlisted. Returned selector metadata is detached and invalid or duplicate adapter entries fail with `INVALID_ADAPTER` or `INVALID_CATALOG`. Context capacity is a separate correctness query, not a catalog decoration or global LLM setting. `resolveModelContext()` asks the adapter that owns the exact provider/model route; an adapter can describe an unlisted dynamic model, and `undefined` means only that capacity is unavailable. Invalid returned capacity fails with `INVALID_MODEL_CONTEXT`. @@ -28,7 +29,7 @@ Context capacity is a separate correctness query, not a catalog decoration or gl ### Extension points -- Subclass `LlmAdapter` and call `ctx.llm.registerAdapter(providers, adapter)` to add one or more provider routes. `GenerateOptions.provider` selects the adapter; `GenerateOptions.model` is adapter-owned and may be resolved dynamically. Override `providerInfo()` and asynchronous `listModels()` to expose selector metadata, and `resolveModelContext()` when exact capacity is known; the defaults use the route id as its name, advertise no models, and return no capacity. +- Subclass `LlmAdapter` and call `ctx.llm.registerAdapter(providers, adapter)` to add one or more provider routes. `GenerateOptions.provider` selects the adapter; `GenerateOptions.model` is adapter-owned and may be resolved dynamically. Override `providerRetryPolicy()` to supply provider-owned recovery configuration, `providerInfo()` and asynchronous `listModels()` to expose selector metadata, and `resolveModelContext()` when exact capacity is known; the defaults use bounded normal retry policy, use the route id as its name, advertise no models, and return no capacity. - Wrap `llm/stream` via `ctx.on()` waterfall listeners for caching, logging, or routing. A wrapper that retries after emitting a chunk has no durable attempt boundary; shipped agent retry policy therefore uses `agent/request-error` instead. ### Content-block vocabulary (`types.ts`) @@ -69,7 +70,7 @@ Pass-through; the registry preserves the assembled request prefix, while the sel ## Known Limitations and Deferred Work -- **No default retry/caching/rate-limit policy ships in this service** — `llm/stream` remains a single-attempt call-wrapper seam; the agent loop separately offers proven model-request failures to `agent/request-error`, whose default preserves the original failure. `@deepseek-ai/dsh-llm-retry` is an optional policy plugin loaded by the shared example spine. +- **No retry execution, caching, or rate limiting ships in this service** — provider registration stores retry policy, but `llm/stream` remains a single-attempt call-wrapper seam. The agent loop separately offers proven model-request failures to `agent/request-error`, whose default preserves the original failure; `@deepseek-ai/dsh-llm-retry` is the optional executor loaded by the shared example spine. - **`GenerateOptions` sampling is `temperature`/`maxTokens`/`stop` only** — no `tool_choice`, `top_p`, or penalty fields; the vocabulary grows when a producer lands ([dropped inert knobs](../../../.agents/notes/implemented/simplification/2026-07-04-drop-inert-request-knobs.md)). - **Producer-gated variants stay out until produced** — `prefill`, per-tool `strict`, block `cache` hints, and the `agent` message-source variant were pruned as producerless ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-04-prune-producerless-vocabulary-variants.md)). - **`BlockAssembler` handles core block kinds only** — a plugin-added block type whose stream is never closed by `block-end` makes `blocks()` throw. diff --git a/packages/llm/llm/package.json b/packages/llm/llm/package.json index cc13bc183e..d5b0298e50 100644 --- a/packages/llm/llm/package.json +++ b/packages/llm/llm/package.json @@ -38,11 +38,16 @@ "peerDependencies": { "@deepseek-ai/dsh-brand": "^0.0.1", "@deepseek-ai/dsh-invariants": "^0.0.1", + "@deepseek-ai/dsh-timeout": "^0.0.1", "cordis": "^4.0.0-rc.7" }, + "dependencies": { + "schemastery": "^3.18.0" + }, "devDependencies": { "@deepseek-ai/dsh-brand": "workspace:^", "@deepseek-ai/dsh-invariants": "workspace:^", + "@deepseek-ai/dsh-timeout": "workspace:^", "cordis": "^4.0.0-rc.7" } } diff --git a/packages/llm/llm/src/index.ts b/packages/llm/llm/src/index.ts index 6765833e8c..773b9a5ca1 100644 --- a/packages/llm/llm/src/index.ts +++ b/packages/llm/llm/src/index.ts @@ -16,6 +16,8 @@ import type { Message, StreamChunk, } from './types.ts' +import { resolveRetryPolicy } from './retry-policy.ts' +import type { ResolvedRetryPolicy } from './retry-policy.ts' import type { ProviderRequestId } from './brand.ts' import { deepFreeze } from './call-config.ts' import { HarnessError } from './error.ts' @@ -27,6 +29,7 @@ export * from './brand.ts' export * from './never.ts' export * from './error.ts' export * from './types.ts' +export * from './retry-policy.ts' export { BlockAssembler } from './assembler.ts' export { callConfigEquals, deepFreeze, isAgentLoopRequest, markAgentLoopRequest } from './call-config.ts' export type { LlmCallConfig } from './call-config.ts' @@ -119,6 +122,15 @@ export abstract class LlmAdapter { return { id: provider, name: provider } } + /** + * Return the provider-owned retry policy captured with this route. + * @param _provider - a route passed to `registerAdapter()` for this instance. + * @returns a resolved policy, or `undefined` to use the normal defaults. + */ + providerRetryPolicy(_provider: string): ResolvedRetryPolicy | undefined { + return undefined + } + /** * List models this adapter can currently advertise for one owned provider. * The result is advisory: an adapter may accept unlisted model ids, and @@ -157,7 +169,11 @@ export abstract class LlmAdapter { * surface, interceptable via the `llm/stream` waterfall. */ export class LlmService extends Service { - private adapters = new Map() + private adapters = new Map() constructor(ctx: Context) { super(ctx, 'llm') @@ -175,7 +191,11 @@ export class LlmService extends Service { const dispose = this.ctx.effect(function* (this: LlmService) { if (providers.length === 0) throw new LlmError('an adapter must register at least one provider', 'INVALID_ADAPTER') const unique = new Set() - const registrations: { adapter: LlmAdapter; provider: LlmProviderInfo }[] = [] + const registrations: { + adapter: LlmAdapter + provider: LlmProviderInfo + retryPolicy: ResolvedRetryPolicy + }[] = [] for (const provider of providers) { if (provider.length === 0) throw new LlmError('adapter provider names must be non-empty', 'INVALID_ADAPTER') if (unique.has(provider) || this.adapters.has(provider)) { @@ -186,7 +206,13 @@ export class LlmService extends Service { throw new LlmError(`adapter metadata for provider "${provider}" must preserve its id and have a non-empty name`, 'INVALID_ADAPTER') } unique.add(provider) - registrations.push({ adapter, provider: { id: info.id, name: info.name } }) + const retryPolicy = adapter.providerRetryPolicy(provider) + ?? resolveRetryPolicy(undefined, `llm: provider "${provider}" retryPolicy`) + registrations.push({ + adapter, + provider: { id: info.id, name: info.name }, + retryPolicy, + }) } for (const registration of registrations) this.adapters.set(registration.provider.id, registration) yield () => { @@ -206,6 +232,15 @@ export class LlmService extends Service { return [...this.adapters.values()].map(({ provider }) => ({ ...provider })) } + /** + * Resolve the retry policy captured when one provider route was registered. + * @param provider - registered provider route to inspect. + * @returns the provider-owned policy, with normal defaults already resolved. + */ + providerRetryPolicy(provider: string): ResolvedRetryPolicy { + return this.registration(provider).retryPolicy + } + /** * Discover models advertised by one registered provider. Catalog membership * is advisory and never changes routing or request validation. @@ -262,7 +297,11 @@ export class LlmService extends Service { return { contextWindow: context.contextWindow } } - private registration(provider: string): { adapter: LlmAdapter; provider: LlmProviderInfo } { + private registration(provider: string): { + adapter: LlmAdapter + provider: LlmProviderInfo + retryPolicy: ResolvedRetryPolicy + } { const registration = this.adapters.get(provider) if (!registration) throw new LlmError(`no adapter registered for provider "${provider}"`, 'NO_ADAPTER') return registration diff --git a/packages/llm/llm/src/retry-policy.ts b/packages/llm/llm/src/retry-policy.ts new file mode 100644 index 0000000000..a1932d50a2 --- /dev/null +++ b/packages/llm/llm/src/retry-policy.ts @@ -0,0 +1,184 @@ +/** + * Provider-owned request-retry policy configuration and resolution. + * + * Adapters expose one resolved policy per registered provider route; the + * optional dsh-llm-retry plugin executes it on the agent's failed-step seam. + * + * @module @deepseek-ai/dsh-llm/retry-policy + */ + +import z from 'schemastery' +import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' + +const DEFAULT_MAX_RETRIES = 2 +const DEFAULT_INITIAL_DELAY_MS = 500 +const DEFAULT_MAX_DELAY_MS = 10_000 +const DEFAULT_JITTER_RATIO = 0.1 +const DEFAULT_RETRYABLE_CODES = Object.freeze(['RATE_LIMIT', 'SERVER', 'TIMEOUT', 'TRANSPORT']) + +/** Bounded exponential backoff with symmetric jitter around each local delay. */ +export interface BackoffConfig { + /** Initial local exponential-backoff delay in milliseconds (default 500). */ + initialDelayMs?: number + /** Maximum locally scheduled or accepted provider delay in milliseconds (default 10000). */ + maxDelayMs?: number + /** Symmetric random multiplier range around one (default 0.1). */ + jitterRatio?: number +} + +/** Current bounded transient retry behavior for one provider route. */ +export interface NormalRetryPolicyConfig { + /** Retry only configured transient failure codes. */ + mode: 'normal' + /** Maximum eligible retries after the first request (default 2). */ + maxRetries?: number + /** Stable failure codes eligible for this policy. */ + retryableCodes?: string[] + /** Local exponential-backoff and jitter configuration. */ + backoff?: BackoffConfig +} + +/** Unbounded retry behavior for every model-request failure on one provider route. */ +export interface AlwaysRetryPolicyConfig { + /** Retry every model-request failure until success, cancellation, or disposal. */ + mode: 'always' + /** Local exponential-backoff and jitter configuration. */ + backoff?: BackoffConfig +} + +/** Provider-owned model-request retry policy configuration. */ +export type RetryPolicyConfig = NormalRetryPolicyConfig | AlwaysRetryPolicyConfig + +/** Fully resolved backoff shared by both retry modes. */ +export interface ResolvedRetryBackoff { + readonly initialDelayMs: number + readonly maxDelayMs: number + readonly jitterRatio: number +} + +/** Fully resolved bounded transient retry policy. */ +export interface ResolvedNormalRetryPolicy extends ResolvedRetryBackoff { + readonly mode: 'normal' + readonly maxRetries: number + readonly retryableCodes: readonly string[] +} + +/** Fully resolved unbounded retry policy. */ +export interface ResolvedAlwaysRetryPolicy extends ResolvedRetryBackoff { + readonly mode: 'always' +} + +/** Immutable provider policy captured when its adapter route is registered. */ +export type ResolvedRetryPolicy = ResolvedNormalRetryPolicy | ResolvedAlwaysRetryPolicy + +const backoffSchema: z = z.object({ + initialDelayMs: z.number().max(MAX_TIMER_DELAY_MS).default(DEFAULT_INITIAL_DELAY_MS), + maxDelayMs: z.number().max(MAX_TIMER_DELAY_MS).default(DEFAULT_MAX_DELAY_MS), + jitterRatio: z.number().min(0).max(1).default(DEFAULT_JITTER_RATIO), +}) + +const normalPolicySchema: z = z.object({ + mode: z.const('normal').required(), + maxRetries: z.number().step(1).min(0).max(Number.MAX_SAFE_INTEGER).default(DEFAULT_MAX_RETRIES), + retryableCodes: z.array(z.string()).default([...DEFAULT_RETRYABLE_CODES]), + backoff: backoffSchema, +}) + +const alwaysPolicySchema: z = z.object({ + mode: z.const('always').required(), + backoff: backoffSchema, +}) + +/** Cordis schema embedded by each concrete provider configuration. */ +export const RetryPolicySchema: z = z.union([ + normalPolicySchema, + alwaysPolicySchema, +]) + +const NORMAL_POLICY_KEYS: ReadonlySet = new Set([ + 'mode', 'maxRetries', 'retryableCodes', 'backoff', +]) +const ALWAYS_POLICY_KEYS: ReadonlySet = new Set(['mode', 'backoff']) +const BACKOFF_KEYS: ReadonlySet = new Set(['initialDelayMs', 'maxDelayMs', 'jitterRatio']) + +function validateKeys(value: object, allowed: ReadonlySet, path: string): void { + for (const key of Object.keys(value)) { + if (!allowed.has(key)) throw new Error(`${path}: unknown key "${key}"`) + } +} + +function resolveBackoff(config: BackoffConfig | undefined, path: string): ResolvedRetryBackoff { + if (config !== undefined) validateKeys(config, BACKOFF_KEYS, path) + const initialDelayMs = config?.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS + const maxDelayMs = config?.maxDelayMs ?? DEFAULT_MAX_DELAY_MS + const jitterRatio = config?.jitterRatio ?? DEFAULT_JITTER_RATIO + + if (!Number.isFinite(initialDelayMs) || initialDelayMs <= 0 || initialDelayMs > MAX_TIMER_DELAY_MS) { + throw new Error(`${path}.initialDelayMs must be a positive finite number no greater than ${MAX_TIMER_DELAY_MS}`) + } + if (!Number.isFinite(maxDelayMs) || maxDelayMs <= 0 || maxDelayMs > MAX_TIMER_DELAY_MS) { + throw new Error(`${path}.maxDelayMs must be a positive finite number no greater than ${MAX_TIMER_DELAY_MS}`) + } + if (initialDelayMs > maxDelayMs) { + throw new Error(`${path}.initialDelayMs must be less than or equal to maxDelayMs`) + } + if (!Number.isFinite(jitterRatio) || jitterRatio < 0 || jitterRatio > 1) { + throw new Error(`${path}.jitterRatio must be between 0 and 1`) + } + + return Object.freeze({ initialDelayMs, maxDelayMs, jitterRatio }) +} + +/** + * Validate, default, and detach one provider-owned retry policy. + * @param config - optional provider configuration; omission selects normal defaults. + * @param path - diagnostic path naming the provider config that owns the value. + * @returns an immutable policy safe to capture in provider registration state. + */ +export function resolveRetryPolicy( + config: RetryPolicyConfig | undefined, + path: string, +): ResolvedRetryPolicy { + if (config === undefined) { + return Object.freeze({ + mode: 'normal', + maxRetries: DEFAULT_MAX_RETRIES, + retryableCodes: DEFAULT_RETRYABLE_CODES, + ...resolveBackoff(undefined, `${path}.backoff`), + }) + } + + switch (config.mode) { + case 'normal': { + validateKeys(config, NORMAL_POLICY_KEYS, path) + const maxRetries = config.maxRetries ?? DEFAULT_MAX_RETRIES + const retryableCodes = config.retryableCodes ?? [...DEFAULT_RETRYABLE_CODES] + if (!Number.isSafeInteger(maxRetries) || maxRetries < 0) { + throw new Error(`${path}.maxRetries must be a non-negative safe integer`) + } + if (retryableCodes.length === 0) { + throw new Error(`${path}.retryableCodes must not be empty`) + } + if (retryableCodes.some(code => code.length === 0)) { + throw new Error(`${path}.retryableCodes must contain only non-empty strings`) + } + if (new Set(retryableCodes).size !== retryableCodes.length) { + throw new Error(`${path}.retryableCodes must not contain duplicates`) + } + return Object.freeze({ + mode: 'normal', + maxRetries, + retryableCodes: Object.freeze([...retryableCodes]), + ...resolveBackoff(config.backoff, `${path}.backoff`), + }) + } + case 'always': + validateKeys(config, ALWAYS_POLICY_KEYS, path) + return Object.freeze({ + mode: 'always', + ...resolveBackoff(config.backoff, `${path}.backoff`), + }) + default: + throw new Error(`${path}.mode must be "normal" or "always"`) + } +} diff --git a/packages/llm/llm/tests/retry-policy.spec.ts b/packages/llm/llm/tests/retry-policy.spec.ts new file mode 100644 index 0000000000..d262533787 --- /dev/null +++ b/packages/llm/llm/tests/retry-policy.spec.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest' +import { + resolveRetryPolicy, + RetryPolicySchema, +} from '@deepseek-ai/dsh-llm' +import type { RetryPolicyConfig } from '@deepseek-ai/dsh-llm' +import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' + +describe('provider retry policy', () => { + it('resolves immutable normal defaults', () => { + const policy = resolveRetryPolicy(undefined, 'provider.retryPolicy') + + expect(policy).toEqual({ + mode: 'normal', + maxRetries: 2, + retryableCodes: ['RATE_LIMIT', 'SERVER', 'TIMEOUT', 'TRANSPORT'], + initialDelayMs: 500, + maxDelayMs: 10_000, + jitterRatio: 0.1, + }) + expect(Object.isFrozen(policy)).toBe(true) + if (policy.mode !== 'normal') throw new Error('expected normal policy') + expect(Object.isFrozen(policy.retryableCodes)).toBe(true) + }) + + it('resolves and detaches a configured normal policy', () => { + const retryableCodes = ['BUSY'] + const config: RetryPolicyConfig = { + mode: 'normal', + maxRetries: 4, + retryableCodes, + backoff: { + initialDelayMs: 25, + maxDelayMs: 100, + jitterRatio: 0, + }, + } + + const policy = resolveRetryPolicy(config, 'provider.retryPolicy') + retryableCodes.push('LATE') + + expect(policy).toEqual({ + mode: 'normal', + maxRetries: 4, + retryableCodes: ['BUSY'], + initialDelayMs: 25, + maxDelayMs: 100, + jitterRatio: 0, + }) + }) + + it('resolves always mode with default backoff', () => { + expect(resolveRetryPolicy({ mode: 'always' }, 'provider.retryPolicy')).toEqual({ + mode: 'always', + initialDelayMs: 500, + maxDelayMs: 10_000, + jitterRatio: 0.1, + }) + expect(RetryPolicySchema).toBeDefined() + }) + + it.each([ + [{ mode: 'normal', maxRetries: -1 }, /maxRetries/], + [{ mode: 'normal', maxRetries: 1.5 }, /maxRetries/], + [{ mode: 'normal', maxRetries: Number.MAX_SAFE_INTEGER + 1 }, /maxRetries/], + [{ mode: 'always', backoff: { initialDelayMs: 0 } }, /initialDelayMs/], + [{ mode: 'normal', backoff: { maxDelayMs: Number.POSITIVE_INFINITY } }, /maxDelayMs/], + [{ mode: 'normal', backoff: { initialDelayMs: MAX_TIMER_DELAY_MS + 1 } }, /initialDelayMs/], + [{ mode: 'always', backoff: { maxDelayMs: MAX_TIMER_DELAY_MS + 1 } }, /maxDelayMs/], + [{ mode: 'normal', backoff: { initialDelayMs: 20, maxDelayMs: 10 } }, /less than or equal/], + [{ mode: 'always', backoff: { jitterRatio: 1.1 } }, /jitterRatio/], + [{ mode: 'normal', retryableCodes: [] }, /must not be empty/], + [{ mode: 'normal', retryableCodes: ['SERVER', 'SERVER'] }, /duplicates/], + [{ mode: 'normal', retryableCodes: [''] }, /non-empty strings/], + [{ mode: 'normal', maxRetires: 1 }, /unknown key "maxRetires"/], + [{ mode: 'always', maxRetries: 1 }, /unknown key "maxRetries"/], + [{ mode: 'always', backoff: { initialDelay: 1 } }, /unknown key "initialDelay"/], + [{ mode: 'sometimes' }, /mode must be "normal" or "always"/], + ] as const)('rejects invalid policy %#', (config, message) => { + expect(() => { + resolveRetryPolicy(config as unknown as RetryPolicyConfig, 'provider.retryPolicy') + }).toThrow(message) + }) +}) diff --git a/packages/llm/llm/tests/service.spec.ts b/packages/llm/llm/tests/service.spec.ts index 90be1ffcb0..52c51852ba 100644 --- a/packages/llm/llm/tests/service.spec.ts +++ b/packages/llm/llm/tests/service.spec.ts @@ -11,6 +11,7 @@ import LlmService, { LlmError, llmFailureOf, ProviderRequestId, + resolveRetryPolicy, StreamChunk, } from '@deepseek-ai/dsh-llm' import type { LlmModelContext, LlmModelInfo, LlmProviderInfo } from '@deepseek-ai/dsh-llm' @@ -159,6 +160,27 @@ describe('LlmService', () => { expect(chunks).toEqual(SCRIPT) }) + it('captures provider-owned retry policy at registration and defaults omission', async () => { + const configured = resolveRetryPolicy({ mode: 'always' }, 'test retryPolicy') + const adapter = new class extends ScriptedAdapter { + override providerRetryPolicy(provider: string) { + return provider === 'configured' ? configured : undefined + } + }(SCRIPT) + const ctx = new Context() + await ctx.plugin(LlmService) + ctx.llm.registerAdapter(['configured', 'defaulted'], adapter) + + expect(ctx.llm.providerRetryPolicy('configured')).toBe(configured) + expect(ctx.llm.providerRetryPolicy('defaulted')).toMatchObject({ + mode: 'normal', + maxRetries: 2, + }) + expect(() => ctx.llm.providerRetryPolicy('missing')).toThrow( + expect.objectContaining({ code: 'NO_ADAPTER' }), + ) + }) + it('throws NO_ADAPTER for unregistered providers', async () => { const ctx = new Context() await ctx.plugin(LlmService) diff --git a/packages/llm/llm/tsconfig.json b/packages/llm/llm/tsconfig.json index 5bc7a9fcf5..fa4adda095 100644 --- a/packages/llm/llm/tsconfig.json +++ b/packages/llm/llm/tsconfig.json @@ -19,6 +19,9 @@ }, { "path": "../../support/invariants" + }, + { + "path": "../../util/timeout" } ] } diff --git a/packages/ui/acp/src/index.ts b/packages/ui/acp/src/index.ts index 8a731a1bf6..d1e78a0e35 100644 --- a/packages/ui/acp/src/index.ts +++ b/packages/ui/acp/src/index.ts @@ -1366,8 +1366,9 @@ export function streamSessionEventUpdate( return } case 'llm/retry': { + const retryLimit = event.data.mode === 'always' ? '∞' : String(event.data.maxRetries) const text = '\n\n[Previous model attempt discarded; retrying ' - + `${event.data.retry}/${event.data.maxRetries} in ${event.data.delayMs}ms: ` + + `${event.data.retry}/${retryLimit} in ${event.data.delayMs}ms: ` + `${event.data.failure.message}]\n\n` notify({ sessionId, update: { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text } } }) return diff --git a/packages/ui/acp/tests/stream-update.spec.ts b/packages/ui/acp/tests/stream-update.spec.ts index 2585968b7e..08993f7ce1 100644 --- a/packages/ui/acp/tests/stream-update.spec.ts +++ b/packages/ui/acp/tests/stream-update.spec.ts @@ -103,6 +103,8 @@ describe('streamSessionEventUpdate', () => { expect(updatesFor(evt('llm/retry', { turn: 1, step: 1, + provider: 'mock', + mode: 'normal', retry: 1, maxRetries: 2, delayMs: 500, @@ -114,6 +116,21 @@ describe('streamSessionEventUpdate', () => { text: '\n\n[Previous model attempt discarded; retrying 1/2 in 500ms: backend busy]\n\n', }, }]) + expect(updatesFor(evt('llm/retry', { + turn: 1, + step: 2, + provider: 'mock', + mode: 'always', + retry: 7, + delayMs: 1_000, + failure: { message: 'still unavailable', code: 'AUTH' }, + }))).toEqual([{ + sessionUpdate: 'agent_message_chunk', + content: { + type: 'text', + text: '\n\n[Previous model attempt discarded; retrying 7/∞ in 1000ms: still unavailable]\n\n', + }, + }]) expect(updatesFor(evt('turn/end', { turn: 1, reason: { kind: 'error', step: 2, failure: { message: 'still busy', code: 'SERVER' } }, diff --git a/packages/ui/tui/src/index.ts b/packages/ui/tui/src/index.ts index 2ade1937e3..bfe199e13b 100644 --- a/packages/ui/tui/src/index.ts +++ b/packages/ui/tui/src/index.ts @@ -2298,8 +2298,9 @@ export function createTuiChat( } case 'llm/retry': { clearStreaming() + const retryLimit = event.data.mode === 'always' ? '∞' : String(event.data.maxRetries) appendNotice( - `Retrying model request (${event.data.retry}/${event.data.maxRetries}) in ${event.data.delayMs}ms: ${event.data.failure.message}`, + `Retrying model request (${event.data.retry}/${retryLimit}) in ${event.data.delayMs}ms: ${event.data.failure.message}`, 'warning', ) break diff --git a/packages/ui/tui/tests/snapshots/retry-cancelled.expected.txt b/packages/ui/tui/tests/snapshots/retry-cancelled.expected.txt index accef4fffc..39862c8260 100644 --- a/packages/ui/tui/tests/snapshots/retry-cancelled.expected.txt +++ b/packages/ui/tui/tests/snapshots/retry-cancelled.expected.txt @@ -21,7 +21,7 @@ buffer 7| "▌ " style 0-0 fg=bright-blue 8| -9| " Retrying model request (1/2) in 1000ms: temporary transport failure " +9| " Retrying model request (1/∞) in 1000ms: temporary transport failure " style 1-67 fg=yellow 10| 11| " Turn cancelled. " diff --git a/packages/ui/tui/tests/tui.snapshot.ts b/packages/ui/tui/tests/tui.snapshot.ts index 16d6794002..1aec66d631 100644 --- a/packages/ui/tui/tests/tui.snapshot.ts +++ b/packages/ui/tui/tests/tui.snapshot.ts @@ -269,6 +269,8 @@ describe('TUI terminal-state snapshots', () => { harness.session.append('llm/retry', { turn: 1, step: 1, + provider: 'mock', + mode: 'normal', retry: 1, maxRetries: 2, delayMs: 500, @@ -297,8 +299,9 @@ describe('TUI terminal-state snapshots', () => { harness.session.append('llm/retry', { turn: 1, step: 1, + provider: 'mock', + mode: 'always', retry: 1, - maxRetries: 2, delayMs: 1_000, failure: { message: 'temporary transport failure', code: 'TRANSPORT' }, }) diff --git a/packages/ui/tui/tests/tui.spec.ts b/packages/ui/tui/tests/tui.spec.ts index 73be9abe76..6af58fe97e 100644 --- a/packages/ui/tui/tests/tui.spec.ts +++ b/packages/ui/tui/tests/tui.spec.ts @@ -1290,6 +1290,8 @@ describe('pi-tui chat lifecycle and transcript', () => { result.session.append('llm/retry', { turn: 1, step: 1, + provider: 'mock', + mode: 'normal', retry: 1, maxRetries: 2, delayMs: 500, @@ -1322,6 +1324,8 @@ describe('pi-tui chat lifecycle and transcript', () => { result.session.append('llm/retry', { turn: 1, step: 1, + provider: 'mock', + mode: 'normal', retry: 1, maxRetries: 2, delayMs: 500, @@ -1330,6 +1334,8 @@ describe('pi-tui chat lifecycle and transcript', () => { result.session.append('llm/retry', { turn: 1, step: 2, + provider: 'mock', + mode: 'normal', retry: 2, maxRetries: 2, delayMs: 1_000, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index edd75466c5..7b19523d33 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2217,6 +2217,10 @@ importers: version: 4.0.0-rc.7(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.5) packages/llm/llm: + dependencies: + schemastery: + specifier: ^3.18.0 + version: 3.18.0 devDependencies: '@deepseek-ai/dsh-brand': specifier: workspace:^ @@ -2224,6 +2228,9 @@ importers: '@deepseek-ai/dsh-invariants': specifier: workspace:^ version: link:../../support/invariants + '@deepseek-ai/dsh-timeout': + specifier: workspace:^ + version: link:../../util/timeout cordis: specifier: ^4.0.0-rc.7 version: 4.0.0-rc.7(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.5) diff --git a/scripts/gen-cordis-catalog.ts b/scripts/gen-cordis-catalog.ts index d87fba3d08..3394801d3d 100644 --- a/scripts/gen-cordis-catalog.ts +++ b/scripts/gen-cordis-catalog.ts @@ -41,6 +41,7 @@ export const LINK_MAP: Record = { LlmFailure: 'llm-streaming.md', LlmModelInfo: 'core.md', LlmProviderInfo: 'core.md', + ResolvedRetryPolicy: 'llm-streaming.md', Message: 'core.md', MessageSource: 'core.md', PromptDecision: 'core.md', From 4c6618c26da6ff0fe08b1881be31d19ceebe20c4 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 25 Jul 2026 10:36:03 +0800 Subject: [PATCH 02/11] test(llm): adapt retry coverage to followup API --- packages/llm/llm-retry/tests/retry.spec.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/llm/llm-retry/tests/retry.spec.ts b/packages/llm/llm-retry/tests/retry.spec.ts index fc43bb774f..1c83d3bacc 100644 --- a/packages/llm/llm-retry/tests/retry.spec.ts +++ b/packages/llm/llm-retry/tests/retry.spec.ts @@ -352,7 +352,7 @@ describe('provider-routed retry policy', () => { }) const scheduled = waitForRetry(context, agent, 1) - agent.send([{ type: 'text', text: 'go' }]) + agent.followup([{ type: 'text', text: 'go' }]) expect((await scheduled).data.delayMs).toBe(3) const idle = waitForIdle(context, agent) await vi.advanceTimersByTimeAsync(3) @@ -390,7 +390,7 @@ describe('provider-routed retry policy', () => { model: 'mock', }) const normalIdle = waitForIdle(context, normalAgent) - normalAgent.send([{ type: 'text', text: 'normal' }]) + normalAgent.followup([{ type: 'text', text: 'normal' }]) await normalIdle expect(normalAgent.session.events.some(event => event.type === 'llm/retry')).toBe(false) @@ -399,7 +399,7 @@ describe('provider-routed retry policy', () => { model: 'mock', }) const scheduled = waitForRetry(context, alwaysAgent, 1) - alwaysAgent.send([{ type: 'text', text: 'always' }]) + alwaysAgent.followup([{ type: 'text', text: 'always' }]) expect((await scheduled).data).toMatchObject({ provider: 'other', mode: 'always', @@ -433,7 +433,7 @@ describe('provider-routed retry policy', () => { }) const scheduled = waitForRetry(context, agent, 1) - agent.send([{ type: 'text', text: 'reroute' }]) + agent.followup([{ type: 'text', text: 'reroute' }]) expect((await scheduled).data).toMatchObject({ provider: 'other', mode: 'always' }) const idle = waitForIdle(context, agent) await vi.advanceTimersByTimeAsync(1) @@ -462,7 +462,7 @@ describe('provider-routed retry policy', () => { }) const idle = waitForIdle(context, agent) - agent.send([{ type: 'text', text: 'keep trying' }]) + agent.followup([{ type: 'text', text: 'keep trying' }]) await vi.runAllTimersAsync() await idle @@ -499,7 +499,7 @@ describe('provider-routed retry policy', () => { }) const scheduled = waitForRetry(context, agent, 1) - agent.send([{ type: 'text', text: 'safe input' }]) + agent.followup([{ type: 'text', text: 'safe input' }]) await scheduled const idle = waitForIdle(context, agent) await vi.advanceTimersByTimeAsync(1) @@ -528,7 +528,7 @@ describe('provider-routed retry policy', () => { }) const idle = waitForIdle(context, agent) - agent.send([{ type: 'text', text: 'recover' }]) + agent.followup([{ type: 'text', text: 'recover' }]) await idle expect(adapter.requests).toHaveLength(2) @@ -555,7 +555,7 @@ describe('provider-routed retry policy', () => { }) const scheduled = waitForRetry(context, agent, 1) - agent.send([{ type: 'text', text: 'recover' }]) + agent.followup([{ type: 'text', text: 'recover' }]) await scheduled const idle = waitForIdle(context, agent) await vi.advanceTimersByTimeAsync(1) @@ -635,7 +635,7 @@ describe('provider-routed retry policy', () => { model: 'mock', }) const idle = waitForIdle(context, agent) - agent.send([{ type: 'text', text: 'go' }]) + agent.followup([{ type: 'text', text: 'go' }]) await entered.promise agent.cancel({ kind: 'user' }) @@ -664,7 +664,7 @@ describe('provider-routed retry policy', () => { }) const idle = waitForIdle(context, agent) - agent.send([{ type: 'text', text: 'go' }]) + agent.followup([{ type: 'text', text: 'go' }]) await idle downstream.resolve({ action: 'fail' }) From d07e7dd317c14bacf8af244057f328fa4c5c86dd Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 25 Jul 2026 10:38:42 +0800 Subject: [PATCH 03/11] docs: refresh module dependency graph --- docs/module-graph.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/module-graph.md b/docs/module-graph.md index 212fd723a7..d172cad2af 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -232,6 +232,7 @@ flowchart TD pkg_host_webserver --> pkg_invariants pkg_llm --> pkg_brand pkg_llm --> pkg_invariants + pkg_llm --> pkg_timeout pkg_client_hmr --> pkg_client_modules pkg_client_hmr --> pkg_invariants pkg_client_ui_conversation --> pkg_client_runtime @@ -805,7 +806,7 @@ flowchart TD | [`host-apiproxy`](../packages/host/apiproxy) | `host` | [`invariants`](../packages/support/invariants) | | [`host-runtime`](../packages/host/runtime) | `host` | [`invariants`](../packages/support/invariants) | | [`host-webserver`](../packages/host/webserver) | `host` | [`invariants`](../packages/support/invariants) | -| [`llm`](../packages/llm/llm) | `llm` | [`brand`](../packages/util/brand), [`invariants`](../packages/support/invariants) | +| [`llm`](../packages/llm/llm) | `llm` | [`brand`](../packages/util/brand), [`invariants`](../packages/support/invariants), [`timeout`](../packages/util/timeout) | | [`client-hmr`](../packages/client/hmr) | `client` | [`client-modules`](../packages/client/modules), [`invariants`](../packages/support/invariants) | | [`client-ui-conversation`](../packages/client/ui-conversation) | `client` | [`client-runtime`](../packages/client/runtime), [`client-ui-primitives`](../packages/client/ui-primitives), [`client-ui-slots`](../packages/client/ui-slots), [`invariants`](../packages/support/invariants) | | [`client-ui-layout`](../packages/client/ui-layout) | `client` | [`client-runtime`](../packages/client/runtime), [`client-ui-slots`](../packages/client/ui-slots), [`invariants`](../packages/support/invariants) | From c1f148ce7f4c1e5e96f50a0f13985ed971a1456c Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 25 Jul 2026 10:49:48 +0800 Subject: [PATCH 04/11] fix(llm): preserve retry routes across turns --- packages/llm/llm-retry/src/history.ts | 6 +-- .../llm/llm-retry/tests/invariant.spec.ts | 8 ++-- packages/llm/llm-retry/tests/retry.spec.ts | 39 +++++++++++++++++++ .../sandbox-local/tests/packed-install.e2e.ts | 1 + 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/packages/llm/llm-retry/src/history.ts b/packages/llm/llm-retry/src/history.ts index 2b86a77e03..d31fe3f201 100644 --- a/packages/llm/llm-retry/src/history.ts +++ b/packages/llm/llm-retry/src/history.ts @@ -4,8 +4,9 @@ import type { SessionEvent } from '@deepseek-ai/dsh-session' /** * Find the provider in force when one step closed, excluding later recovery mutations. - * A preceding retry is also a route marker because every provider change - * requires a newer full request-header snapshot. + * Request headers remain effective across turn boundaries until a newer full + * snapshot changes them. A preceding retry in the same turn is also a route + * marker because every provider change requires a newer full snapshot. * @param events - session events containing the closed step. * @param turn - turn that owns the failed step. * @param step - failed step whose provider is required. @@ -32,7 +33,6 @@ export function providerForClosedStep( && event.data.step < step) { return event.data.provider } - if (event.type === 'turn/start' || event.type === 'turn/end') return undefined } return undefined } diff --git a/packages/llm/llm-retry/tests/invariant.spec.ts b/packages/llm/llm-retry/tests/invariant.spec.ts index c9c5d2c564..aa4030a886 100644 --- a/packages/llm/llm-retry/tests/invariant.spec.ts +++ b/packages/llm/llm-retry/tests/invariant.spec.ts @@ -38,7 +38,7 @@ describe('llm-retry invariants', () => { }] as never, 1, 1)).toBeUndefined() }) - it('does not inherit a provider across a turn boundary', () => { + it('inherits the latest provider across a turn boundary when the header is unchanged', () => { expect(providerForClosedStep([ { type: 'turn/start', data: { turn: 1 } }, { @@ -48,7 +48,7 @@ describe('llm-retry invariants', () => { { type: 'turn/end', data: { turn: 1 } }, { type: 'turn/start', data: { turn: 2 } }, { type: 'step/end', data: { turn: 2, step: 1 } }, - ] as never, 2, 1)).toBeUndefined() + ] as never, 2, 1)).toBe('prior') }) it('accepts increasing retry records for successive closed steps and ignores unrelated events', async () => { @@ -226,7 +226,7 @@ describe('llm-retry invariants', () => { }).toThrow(/does not match the failed request provider mock/) }) - it('rejects a current-turn retry without a current-turn provider route', async () => { + it('accepts a current-turn retry under an unchanged prior provider route', async () => { const ctx = await setup() const session = closeStep(ctx, 'retry-invariant-prior-route') session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) @@ -243,7 +243,7 @@ describe('llm-retry invariants', () => { delayMs: 1, failure, }) - }).toThrow(/does not match the failed request provider undefined/) + }).not.toThrow() }) it('rejects non-numeric durable delays', async () => { diff --git a/packages/llm/llm-retry/tests/retry.spec.ts b/packages/llm/llm-retry/tests/retry.spec.ts index 1c83d3bacc..56f785f38c 100644 --- a/packages/llm/llm-retry/tests/retry.spec.ts +++ b/packages/llm/llm-retry/tests/retry.spec.ts @@ -207,6 +207,45 @@ describe('provider-routed retry policy', () => { }) }) + it('retries a later turn under its unchanged provider header', async () => { + vi.useFakeTimers() + const adapter = new ScriptedAdapter([ + textResponse('first turn'), + new LlmError('busy on second turn', 'RATE_LIMIT'), + textResponse('second turn recovered'), + ]) + ;({ ctx: context } = await harness(adapter, { mock: normalConfig({ + backoff: { initialDelayMs: 1, maxDelayMs: 1 }, + }) })) + const agent = context.agentLoop.create(SessionId('retry-later-turn'), { + provider: 'mock', + model: 'mock', + }) + + const firstIdle = waitForIdle(context, agent) + agent.followup([{ type: 'text', text: 'first' }]) + await firstIdle + expect(agent.session.events.filter(event => event.type === 'request/header')).toHaveLength(1) + + const scheduled = waitForRetry(context, agent, 1) + agent.followup([{ type: 'text', text: 'second' }]) + expect((await scheduled).data).toMatchObject({ + turn: 2, + step: 1, + provider: 'mock', + }) + const secondIdle = waitForIdle(context, agent) + await vi.advanceTimersByTimeAsync(1) + await secondIdle + + expect(adapter.requests).toHaveLength(3) + expect(agent.session.events.filter(event => event.type === 'request/header')).toHaveLength(1) + expect(agent.session.deriveMessages().at(-1)).toMatchObject({ + role: 'assistant', + content: [{ type: 'text', text: 'second turn recovered' }], + }) + }) + it('leaves partial failed chunks on their step without committing a message or tool side effect', async () => { vi.useFakeTimers() const adapter = new ScriptedAdapter([ diff --git a/packages/sandbox/sandbox-local/tests/packed-install.e2e.ts b/packages/sandbox/sandbox-local/tests/packed-install.e2e.ts index 9fcfe23de8..a032e1add7 100644 --- a/packages/sandbox/sandbox-local/tests/packed-install.e2e.ts +++ b/packages/sandbox/sandbox-local/tests/packed-install.e2e.ts @@ -26,6 +26,7 @@ const WORKSPACE_CLOSURE = [ 'packages/sandbox/sandbox', 'packages/llm/llm', 'packages/util/brand', + 'packages/util/timeout', 'packages/support/invariants', ] From a623ed518398ae9471aea6481ccb286ec7e24d45 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 25 Jul 2026 10:54:30 +0800 Subject: [PATCH 05/11] test(tui): cover unbounded retry status --- packages/ui/tui/tests/tui.spec.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/ui/tui/tests/tui.spec.ts b/packages/ui/tui/tests/tui.spec.ts index d1aac9b10c..2dd3ed2098 100644 --- a/packages/ui/tui/tests/tui.spec.ts +++ b/packages/ui/tui/tests/tui.spec.ts @@ -1345,10 +1345,20 @@ describe('pi-tui chat lifecycle and transcript', () => { delayMs: 1_000, failure: { message: 'failed before chunks', code: 'SERVER', status: 503 }, }) + result.session.append('llm/retry', { + turn: 1, + step: 3, + provider: 'mock', + mode: 'always', + retry: 1, + delayMs: 2_000, + failure: { message: 'retry without limit', code: 'AUTH', status: 401 }, + }) await tick() expect(result.terminal.output).toContain('Retrying model request (1/2) in 500ms: rate limited') expect(result.terminal.output).toContain('Retrying model request (2/2) in 1000ms: failed before chunks') + expect(result.terminal.output).toContain('Retrying model request (1/∞) in 2000ms: retry without limit') await dispose(result) }) From 015ba14bae3bcd817b88636d375add27ee31d6cf Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 25 Jul 2026 13:30:34 +0800 Subject: [PATCH 06/11] fix(llm): preserve serving retry policy --- ...26-07-24-provider-retry-policies.i18n.yaml | 4 +- .../2026-07-24-provider-retry-policies.md | 8 +- .../2026-07-24-provider-retry-policies.zh.md | 8 +- docs/config-catalog.md | 4 +- docs/cordis-catalog/events.md | 13 ++- .../llm-streaming.i18n.yaml | 4 +- docs/core-data-structures/llm-streaming.md | 4 +- docs/core-data-structures/llm-streaming.zh.md | 4 +- docs/event-producer-consumer.md | 8 +- packages/compact/compact-basic/src/index.ts | 1 + .../compact-basic/tests/compact-basic.spec.ts | 2 +- .../cordis/tool-cordis/src/api-catalog.ts | 4 +- packages/core/agent-loop/README.md | 2 +- packages/core/agent-loop/src/loop.ts | 23 ++-- .../agent-loop/tests/request-recovery.spec.ts | 44 ++++++-- packages/core/agent/src/types.ts | 6 +- packages/core/scope/tests/invariant.spec.ts | 2 +- packages/llm/llm-retry/README.md | 2 +- packages/llm/llm-retry/src/index.ts | 15 +-- packages/llm/llm-retry/tests/retry.spec.ts | 106 ++++++++++++++++-- packages/llm/llm/README.md | 2 +- packages/llm/llm/src/adapter-failure.ts | 29 ++++- packages/llm/llm/src/index.ts | 8 +- packages/llm/llm/tests/service.spec.ts | 47 ++++++++ packages/plan/plan-mode/src/index.ts | 1 + .../plan/plan-mode/tests/integration.spec.ts | 4 +- .../plan/plan-mode/tests/plan-mode.spec.ts | 5 +- 27 files changed, 278 insertions(+), 82 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml index ee221716ce..000b11d7f0 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-24-provider-retry-policies.md: 0ff304715aa5e1bfaf296e3f53e1da28d8b4042d -2026-07-24-provider-retry-policies.zh.md: 3b1d8f12a5fb37424e8b964f7bbb2be278263056 +2026-07-24-provider-retry-policies.md: 0b9456eb1563bfcbfa06d93403fd68124ed1f707 +2026-07-24-provider-retry-policies.zh.md: d3ef90ec739eec7300b7d0bb526cb5de958ed93f diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md index 0ff304715a..0b9456eb15 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md @@ -12,7 +12,7 @@ Provider policy must follow the request that actually failed, including a route ## Decision -Each concrete adapter accepts an optional `retryPolicy` inside its provider configuration. The adapter validates and resolves the policy, and `ctx.llm` captures it when that exact provider route registers. `@deepseek-ai/dsh-llm-retry` reads the registered policy for the provider whose step failed. A provider without `retryPolicy` uses the normal defaults. +Each concrete adapter accepts an optional `retryPolicy` inside its provider configuration. The adapter validates and resolves the policy, and `ctx.llm` captures it when that exact provider route registers. When a call enters its final adapter boundary, `ctx.llm` binds the serving registration's immutable policy to that call; the agent loop passes it to closed-step recovery even if the route is disposed or replaced while the request is in flight. `@deepseek-ai/dsh-llm-retry` combines that call-local policy with the failed step's durable provider identity. A call that never reaches a final adapter has no serving policy and delegates. A provider without `retryPolicy` uses the normal defaults. ```yaml providers: @@ -34,7 +34,7 @@ providers: jitterRatio: 0.2 ``` -The listener selects the policy from the durable `request/header` in force when the failed step closed, excluding later recovery mutations. Normal mode retains the bounded transient behavior: it retries configured codes up to `maxRetries`, counts retries scheduled by the same provider policy in the current consecutive failure sequence, and otherwise delegates. +The listener reads the provider from the durable `request/header` in force when the failed step closed, excluding later recovery mutations, but never re-resolves policy from the mutable provider registry. Normal mode retains the bounded transient behavior: it retries configured codes up to `maxRetries`, counts retries scheduled by the same provider policy in the current consecutive failure sequence, and otherwise delegates. Always mode asks downstream recovery first so a specialized policy such as context-overflow compaction can make progress. A downstream retry wins. A downstream failure decision or thrown recovery error falls back to an unbounded retry of the same provider request; the thrown error is logged. Success, turn cancellation, and plugin disposal are the only termination paths. @@ -56,10 +56,10 @@ Each scheduled retry appends a non-surface `llm/retry` event with the failed pro ## Verification -Adapter tests validate nested policies at provider load and prove registration captures configured and default policies. Unit and real-Loader composition tests select policies from the failed request's provider, exercise always mode beyond the normal budget, pin jitter and delay caps, prove downstream recovery ordering, prove cancellation interrupts stalled downstream recovery, and prove cancellation and disposal stop active backoff waits. Request-level coverage compares the complete messages of failed and retried attempts and rejects both provider error text and discarded partial output. JSONL and SQLite tests round-trip an always event without `Infinity`; invariant tests bind its provider to the request header and its retry number to the active provider policy; ACP and TUI tests render finite and infinite limits. +Adapter tests validate nested policies at provider load, prove registration captures configured and default policies, and retain the serving policy across in-flight route replacement. Unit and real-Loader composition tests select policies from the failed request's serving registration, exercise always mode beyond the normal budget, pin jitter and delay caps, prove downstream recovery ordering, prove cancellation interrupts stalled downstream recovery, and prove cancellation and disposal stop active backoff waits. Request-level coverage compares the complete messages of failed and retried attempts and rejects both provider error text and discarded partial output. JSONL and SQLite tests round-trip an always event without `Infinity`; invariant tests bind its provider to the request header and its retry number to the active provider policy; ACP and TUI tests render finite and infinite limits. ## Consequences -Normal mode remains a finite default, while an explicit always policy can spend unbounded requests and time on permanent authentication, quota, invalid-request, protocol, or context failures. Operators must pair always mode with a cancellable caller and provider-specific cost controls. Retry state stays observable and durable without becoming model-visible, and exact-provider selection keeps one provider's exceptional policy from changing another provider's recovery behavior. +Normal mode remains a finite default, while an explicit always policy can spend unbounded requests and time on permanent authentication, quota, invalid-request, protocol, or context failures. Operators must pair always mode with a cancellable caller and provider-specific cost controls. Retry state stays observable and durable without becoming model-visible, and serving-registration capture prevents adapter lifecycle changes from retroactively changing an in-flight request's recovery contract. This decision extends the closed-step recovery, single visible adapter attempt, structured failure, and durable status design in [bounded recovery for transient LLM request failures](../architecture/2026-06-21-bounded-llm-request-recovery.md). diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md index 3b1d8f12a5..d3ef90ec73 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md @@ -12,7 +12,7 @@ Status: implemented ## 决策 -每个具体适配器都在其提供方配置中接受可选的 `retryPolicy`。适配器负责校验并解析策略，`ctx.llm` 则在该精确提供方路由注册时捕获策略。`@deepseek-ai/dsh-llm-retry` 读取失败步骤对应提供方的已注册策略。未配置 `retryPolicy` 的提供方使用 normal 默认值。 +每个具体适配器都在其提供方配置中接受可选的 `retryPolicy`。适配器负责校验并解析策略，`ctx.llm` 则在该精确提供方路由注册时捕获策略。当调用进入最终适配器边界时，`ctx.llm` 会把实际提供服务的注册项所持不可变策略绑定到该调用；即使路由在请求进行期间被 dispose 或替换，agent loop 仍会把该策略传给已关闭步骤恢复。`@deepseek-ai/dsh-llm-retry` 会把绑定到该调用的策略与失败步骤的持久提供方标识结合起来。未到达最终适配器的调用没有实际提供服务的策略，因而会委托后续处理。未配置 `retryPolicy` 的提供方使用 normal 默认值。 ```yaml providers: @@ -34,7 +34,7 @@ providers: jitterRatio: 0.2 ``` -监听器根据失败步骤关闭时生效的持久 `request/header` 选择策略，后续恢复产生的改动不参与选择。normal 模式保留有界瞬态错误处理行为：它重试配置的错误代码，次数不超过 `maxRetries`；在当前连续失败序列中，同一提供方策略安排的重试都计入次数；其他情况委托后续处理。 +监听器从失败步骤关闭时生效的持久 `request/header` 读取提供方，后续恢复产生的改动不参与选择，但绝不会从可变的提供方注册表重新解析策略。normal 模式保留有界瞬态错误处理行为：它重试配置的错误代码，次数不超过 `maxRetries`；在当前连续失败序列中，同一提供方策略安排的重试都计入次数；其他情况委托后续处理。 always 模式先请求下游恢复，使上下文溢出压缩（compaction）之类的专用策略有机会取得进展。下游若决定重试，则以该决定为准。下游若决定失败或恢复过程抛出错误，则回退为无界重试同一提供方请求；抛出的错误会写入日志。成功、轮次取消和插件 dispose（资源释放）是仅有的终止路径。 @@ -56,10 +56,10 @@ always 模式先请求下游恢复，使上下文溢出压缩（compaction）之 ## 验证 -适配器测试会在提供方加载时校验嵌套策略，并证明注册流程会捕获已配置策略和默认策略。单元测试与真实 Loader 组合测试根据失败请求的提供方选择策略、验证 always 模式可越过 normal 预算、固定抖动和延迟上限、证明下游恢复顺序、证明取消会中断停滞的下游恢复，并证明取消与 dispose 会停止正在进行的退避等待。请求级覆盖会比较失败尝试与重试尝试的完整消息，并排除提供方错误文本和丢弃的部分输出。JSONL 与 SQLite 测试会往返读写不含 `Infinity` 的 always 事件；不变式测试会将事件中的提供方绑定到请求头，并将重试编号绑定到活跃的提供方策略；ACP 与 TUI 测试会分别渲染有限和无限上限。 +适配器测试会在提供方加载时校验嵌套策略，证明注册流程会捕获已配置策略和默认策略，并证明请求进行期间替换路由后仍会保留实际提供服务的策略。单元测试与真实 Loader 组合测试根据失败请求实际使用的注册项选择策略、验证 always 模式可越过 normal 预算、固定抖动和延迟上限、证明下游恢复顺序、证明取消会中断停滞的下游恢复，并证明取消与 dispose 会停止正在进行的退避等待。请求级覆盖会比较失败尝试与重试尝试的完整消息，并排除提供方错误文本和丢弃的部分输出。JSONL 与 SQLite 测试会往返读写不含 `Infinity` 的 always 事件；不变式测试会将事件中的提供方绑定到请求头，并将重试编号绑定到活跃的提供方策略；ACP 与 TUI 测试会分别渲染有限和无限上限。 ## 后果 -normal 模式仍是有限的默认策略；显式的 always 策略可能在永久性的身份验证、配额、无效请求、协议或上下文错误上耗费无限次请求和无限时间。运维方必须为 always 模式配备可取消的调用方和针对提供方的成本控制。重试状态保持可观察且持久，但不会对模型可见；精确提供方选择也能避免某个提供方的例外策略改变其他提供方的恢复行为。 +normal 模式仍是有限的默认策略；显式的 always 策略可能在永久性的身份验证、配额、无效请求、协议或上下文错误上耗费无限次请求和无限时间。运维方必须为 always 模式配备可取消的调用方和针对提供方的成本控制。重试状态保持可观察且持久，但不会对模型可见；捕获实际提供服务的注册项，也能防止适配器生命周期变化反过来改变进行中请求的恢复契约。 本决策扩展了[瞬态 LLM（大语言模型）请求失败的有界恢复](../architecture/2026-06-21-bounded-llm-request-recovery.md)中确定的已关闭 step 恢复、单次可见适配器尝试、结构化失败与持久状态设计。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 2036a34307..9219ee0f10 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -646,11 +646,11 @@ Source: [`packages/support/llm-replay/src/index.ts:387`](../packages/support/llm ## `@deepseek-ai/dsh-llm-retry` -Requires: `agents` · `llm` +Requires: `agents` ```ts config-catalog /** This policy executor has no config; providers own `retryPolicy`. */ -export type Config = Readonly> +export type Config = Readonly> ``` Source: [`packages/llm/llm-retry/src/index.ts:43`](../packages/llm/llm-retry/src/index.ts) diff --git a/docs/cordis-catalog/events.md b/docs/cordis-catalog/events.md index b6a15c19b6..941c92eacc 100644 --- a/docs/cordis-catalog/events.md +++ b/docs/cordis-catalog/events.md @@ -96,7 +96,7 @@ A step or turn errored. The loop reports a failure here (plus the logger) even w Types: [Agent](../core-data-structures/core.md) · [Scoped](../core-data-structures/scope.md) -Source: [`packages/core/agent/src/types.ts:498`](../../packages/core/agent/src/types.ts) +Source: [`packages/core/agent/src/types.ts:499`](../../packages/core/agent/src/types.ts) ### `agent/inbox/dequeue` — emit @@ -281,16 +281,17 @@ Recover a model-request failure after its failed step has closed. `retry` opens * @param error - the original model-request failure. * @param failure - serializable facts normalized at the final adapter boundary. * @param priorFailures - immutable failures that already authorized another request in this consecutive sequence. + * @param retryPolicy - immutable policy of the adapter registration that served the failed request, or `undefined` if no final adapter served it. * @param signal - the turn abort signal. * Scope-filtered dispatch (`@deepseek-ai/dsh-scope`): agent-scoped listeners receive only that agent. * @mode waterfall */ -'agent/request-error'(this: Scoped, agent: Agent, turn: number, step: number, error: RequestError, failure: LlmFailure, priorFailures: readonly LlmFailure[], signal: AbortSignal, next: () => Promise): Promise +'agent/request-error'(this: Scoped, agent: Agent, turn: number, step: number, error: RequestError, failure: LlmFailure, priorFailures: readonly LlmFailure[], retryPolicy: ResolvedRetryPolicy | undefined, signal: AbortSignal, next: () => Promise): Promise ``` -Types: [Agent](../core-data-structures/core.md) · [LlmFailure](../core-data-structures/llm-streaming.md) · [RequestError](../core-data-structures/core.md) · [RequestErrorDecision](../core-data-structures/core.md) · [Scoped](../core-data-structures/scope.md) +Types: [Agent](../core-data-structures/core.md) · [LlmFailure](../core-data-structures/llm-streaming.md) · [RequestError](../core-data-structures/core.md) · [RequestErrorDecision](../core-data-structures/core.md) · [ResolvedRetryPolicy](../core-data-structures/llm-streaming.md) · [Scoped](../core-data-structures/scope.md) -Source: [`packages/core/agent/src/types.ts:463`](../../packages/core/agent/src/types.ts) +Source: [`packages/core/agent/src/types.ts:464`](../../packages/core/agent/src/types.ts) ### `agent/session-prefix` — waterfall @@ -403,7 +404,7 @@ Override whether the turn continues. The default continues after tool calls or s Types: [Agent](../core-data-structures/core.md) · [ContinuationDecision](../core-data-structures/core.md) · [Scoped](../core-data-structures/scope.md) -Source: [`packages/core/agent/src/types.ts:474`](../../packages/core/agent/src/types.ts) +Source: [`packages/core/agent/src/types.ts:475`](../../packages/core/agent/src/types.ts) ### `agent/turn-stop` — serial @@ -425,7 +426,7 @@ Monotonic terminal-stop checkpoint after continuation and steering are folded; a Types: [Agent](../core-data-structures/core.md) · [ContinuationStop](../core-data-structures/core.md) · [Scoped](../core-data-structures/scope.md) -Source: [`packages/core/agent/src/types.ts:485`](../../packages/core/agent/src/types.ts) +Source: [`packages/core/agent/src/types.ts:486`](../../packages/core/agent/src/types.ts) ## `agent-loop/*` diff --git a/docs/core-data-structures/llm-streaming.i18n.yaml b/docs/core-data-structures/llm-streaming.i18n.yaml index 0d7edf9dfe..d7bab0aa3e 100644 --- a/docs/core-data-structures/llm-streaming.i18n.yaml +++ b/docs/core-data-structures/llm-streaming.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -llm-streaming.md: 2917d9796b32956f11ed5703f1f34c7510d0526c -llm-streaming.zh.md: 1bf7b3c226b5878deff3252396d96c2e97bf2982 +llm-streaming.md: 2d185e694f272114d4efa4f6be9020ef7c9a950f +llm-streaming.zh.md: 9734c643724e19fc39aff2ee94bd8de61935e16c diff --git a/docs/core-data-structures/llm-streaming.md b/docs/core-data-structures/llm-streaming.md index 2917d9796b..2d185e694f 100644 --- a/docs/core-data-structures/llm-streaming.md +++ b/docs/core-data-structures/llm-streaming.md @@ -59,7 +59,7 @@ Every adapter MUST obey these, and every consumer may rely on them: - **`usage` before `finish`, nothing after `finish`.** Defer both to the provider's end-of-stream marker so a trailing usage-only chunk can't violate the ordering. - **Tool-call `arguments` stay raw JSON strings end-to-end.** Partial fragments stream via `argumentsDelta`; a provider that hands back parsed objects re-stringifies at `block-end`. -- **Two sanctioned error paths, one fact shape.** A failure may either THROW from `stream()` (transport/protocol errors) **or** end the stream with `finish {kind:'error'|'aborted', failure}` (provider in-band errors, for adapters that can't throw mid-stream). `LlmError.failure` carries the same `LlmFailure`. The final adapter boundary preserves the exact thrown `Error` object and associates immutable facts with that call; the agent loop closes the failed step and offers the error, facts, and immutable prior-retried facts to `agent/request-error`. Absent recovery the structured failure becomes the turn error, and no normal assistant message or tool side effect is committed for that attempt. +- **Two sanctioned error paths, one fact shape.** A failure may either THROW from `stream()` (transport/protocol errors) **or** end the stream with `finish {kind:'error'|'aborted', failure}` (provider in-band errors, for adapters that can't throw mid-stream). `LlmError.failure` carries the same `LlmFailure`. The final adapter boundary preserves the exact thrown `Error` object and associates immutable facts plus the serving registration's immutable retry policy with that call; the agent loop closes the failed step and offers the error, facts, immutable prior-retried facts, and serving policy to `agent/request-error`. Absent recovery the structured failure becomes the turn error, and no normal assistant message or tool side effect is committed for that attempt. - **One adapter call is one provider attempt.** Adapters disable library retries. Agent-level recovery opens another durable numbered step; direct `ctx.llm.stream()` callers remain single-attempt. - **Provider stalls are bounded at the transport.** Both shipping remote adapters expose positive finite `streamIdleTimeoutMs` with a five-minute default. The watchdog arms only while iterator `next()` is outstanding, uses one stable signal for the whole request, maps its own expiry to `TIMEOUT`, and keeps an earlier caller abort as `ABORTED`. - **Context overflow has one canonical code.** Both DeepSeek adapters classify explicit provider detail through `isContextWindowExceededError()` and surface `CONTEXT_WINDOW_EXCEEDED`, whether the failure arrives as a thrown HTTP `LlmError` or an in-band finish error. Consumers route on the code, never provider text. @@ -70,7 +70,7 @@ This contract is pinned down by two deliberately independent implementations: `d ## `ResolvedRetryPolicy` -Provider configuration resolves before route registration into an immutable discriminated union. Normal mode carries `mode: 'normal'`, finite `maxRetries`, `retryableCodes`, and required `initialDelayMs`, `maxDelayMs`, and `jitterRatio`; always mode carries `mode: 'always'` and the same required backoff fields without a finite maximum. `LlmService.providerRetryPolicy(provider)` returns the captured value and supplies normal defaults when the adapter omits one. The [generated config catalog](../config-catalog.md) owns the optional input shapes. +Provider configuration resolves before route registration into an immutable discriminated union. Normal mode carries `mode: 'normal'`, finite `maxRetries`, `retryableCodes`, and required `initialDelayMs`, `maxDelayMs`, and `jitterRatio`; always mode carries `mode: 'always'` and the same required backoff fields without a finite maximum. `LlmService.providerRetryPolicy(provider)` returns the currently registered value and supplies normal defaults when the adapter omits one; `llmRetryPolicyOf(stream)` returns the exact serving registration's captured value after that call enters its final adapter boundary, so later route disposal or replacement cannot change an in-flight failure's recovery policy. The [generated config catalog](../config-catalog.md) owns the optional input shapes. ## `AppIdentity` — app attribution diff --git a/docs/core-data-structures/llm-streaming.zh.md b/docs/core-data-structures/llm-streaming.zh.md index 1bf7b3c226..9734c64372 100644 --- a/docs/core-data-structures/llm-streaming.zh.md +++ b/docs/core-data-structures/llm-streaming.zh.md @@ -59,7 +59,7 @@ interface LlmFailure { - **`usage` 在 `finish` 之前，`finish` 之后不再有任何分片。** 将两者都推迟到提供方的流结束标记，这样尾部的 usage-only 分片就不会违反顺序。 - **工具调用的 `arguments` 全程保持原始 JSON 字符串。** 部分片段通过 `argumentsDelta` 流式传输；如果提供方返回的是已解析的对象，适配器在 `block-end` 时重新序列化为字符串。 -- **两条受支持的错误路径，一种事实形状。** 失败可以从 `stream()` 抛出（传输/协议错误），**或者**以 `finish {kind:'error'|'aborted', failure}` 结束流（无法在流中途抛异常的适配器用它表示提供方带内错误）。`LlmError.failure` 携带同一个 `LlmFailure`。最终适配器边界保留被抛出的确切 `Error` 对象，并将不可变事实关联到该调用；agent loop（智能体循环）关闭失败的步骤，再把错误、事实与不可变的先前已重试事实提供给 `agent/request-error`。若未恢复，结构化失败会成为轮次错误，并且该次尝试不会提交正常 assistant 消息或工具副作用。 +- **两条受支持的错误路径，一种事实形状。** 失败可以从 `stream()` 抛出（传输/协议错误），**或者**以 `finish {kind:'error'|'aborted', failure}` 结束流（无法在流中途抛异常的适配器用它表示提供方带内错误）。`LlmError.failure` 携带同一个 `LlmFailure`。最终适配器边界保留被抛出的确切 `Error` 对象，并将不可变事实以及实际提供服务的注册项所持不可变重试策略关联到该调用；agent loop（智能体循环）关闭失败的步骤，再把错误、事实、不可变的先前已重试事实与实际提供服务的策略提供给 `agent/request-error`。若未恢复，结构化失败会成为轮次错误，并且该次尝试不会提交正常 assistant 消息或工具副作用。 - **一次适配器调用就是一次提供方尝试。** 适配器禁用库重试。agent 层恢复会打开另一个持久、带编号的步骤；直接调用 `ctx.llm.stream()` 的调用方仍然只尝试一次。 - **提供方停顿在传输层受到时限约束。** 两个已交付的远程适配器都暴露正数且有限的 `streamIdleTimeoutMs`，默认五分钟。watchdog 只在 iterator `next()` 尚未完成时启动，整个请求使用同一个稳定 signal，把自身到期映射为 `TIMEOUT`，并把更早发生的调用方中止保留为 `ABORTED`。 - **上下文溢出只有一个规范 code。** 两个 DeepSeek 适配器都通过 `isContextWindowExceededError()` 对提供方的显式细节分类并暴露 `CONTEXT_WINDOW_EXCEEDED`，无论失败以抛出的 HTTP `LlmError` 还是带内 finish error 到达。消费方按 code 路由，绝不依赖提供方文本。 @@ -70,7 +70,7 @@ interface LlmFailure { ## `ResolvedRetryPolicy` -提供方配置会在路由注册前解析为不可变的可辨识联合类型。normal 模式包含 `mode: 'normal'`、有限的 `maxRetries`、`retryableCodes`，以及必填的 `initialDelayMs`、`maxDelayMs` 和 `jitterRatio`；always 模式包含 `mode: 'always'` 和相同的必填退避字段，但不含有限上限。`LlmService.providerRetryPolicy(provider)` 返回捕获的值；适配器未提供策略时，该方法会补上 normal 默认值。可选输入形状由[生成的配置目录](../config-catalog.md)定义。 +提供方配置会在路由注册前解析为不可变的可辨识联合类型。normal 模式包含 `mode: 'normal'`、有限的 `maxRetries`、`retryableCodes`，以及必填的 `initialDelayMs`、`maxDelayMs` 和 `jitterRatio`；always 模式包含 `mode: 'always'` 和相同的必填退避字段，但不含有限上限。`LlmService.providerRetryPolicy(provider)` 返回当前已注册的值；适配器未提供策略时，该方法会补上 normal 默认值。调用进入最终适配器边界后，`llmRetryPolicyOf(stream)` 返回实际提供服务的确切注册项所捕获的值，因此后续路由 dispose 或替换无法改变请求进行期间发生的失败所用恢复策略。可选输入形状由[生成的配置目录](../config-catalog.md)定义。 ## `AppIdentity`：应用归属 diff --git a/docs/event-producer-consumer.md b/docs/event-producer-consumer.md index 08dcc19d54..3a2802588b 100644 --- a/docs/event-producer-consumer.md +++ b/docs/event-producer-consumer.md @@ -11,7 +11,7 @@ This matrix shows which packages dispatch each harness-owned event and which pac | `agent/cancel-requested` | `emit` | [`packages/core/agent/src/types.ts:350`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`goal-session`](../packages/goal/goal-session) | | `agent/created` | `emit` | [`packages/core/agent/src/types.ts:285`](../packages/core/agent/src/types.ts) | [`agent`](../packages/core/agent) (`events.dispatch`) | [`goal-session`](../packages/goal/goal-session), [`tui`](../packages/ui/tui) | | `agent/disposed` | `emit` | [`packages/core/agent/src/types.ts:294`](../packages/core/agent/src/types.ts) | [`agent`](../packages/core/agent) (`events.dispatch`) | [`agent-loop`](../packages/core/agent-loop), [`goal-session`](../packages/goal/goal-session), [`tui`](../packages/ui/tui) | -| `agent/error` | `emit` | [`packages/core/agent/src/types.ts:498`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`goal-session`](../packages/goal/goal-session), `runtime`, [`tui`](../packages/ui/tui) | +| `agent/error` | `emit` | [`packages/core/agent/src/types.ts:499`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`goal-session`](../packages/goal/goal-session), `runtime`, [`tui`](../packages/ui/tui) | | `agent/inbox/dequeue` | `emit` | [`packages/core/agent/src/types.ts:326`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`agent`](../packages/core/agent) | | `agent/inbox/discard` | `emit` | [`packages/core/agent/src/types.ts:340`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`agent`](../packages/core/agent) | | `agent/inbox/enqueue` | `emit` | [`packages/core/agent/src/types.ts:316`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`agent`](../packages/core/agent), [`goal-session`](../packages/goal/goal-session), [`tui`](../packages/ui/tui) | @@ -19,13 +19,13 @@ This matrix shows which packages dispatch each harness-owned event and which pac | `agent/pre-step` | `serial` | [`packages/core/agent/src/types.ts:379`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`serial`) | [`time-context`](../packages/context/time-context), [`user-approval`](../packages/ui/user-approval) | | `agent/prompt-submit` | `waterfall` | [`packages/core/agent/src/types.ts:395`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`waterfall`) | [`acp`](../packages/ui/acp), [`goal-session`](../packages/goal/goal-session), [`hooks-claude`](../packages/hooks/hooks-claude), [`hooks-codex`](../packages/hooks/hooks-codex), [`plan-mode`](../packages/plan/plan-mode), [`repeat-tool-guard`](../packages/guard/repeat-tool-guard) | | `agent/request` | `waterfall` | [`packages/core/agent/src/types.ts:409`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`waterfall`) | [`agent`](../packages/core/agent) | -| `agent/request-error` | `waterfall` | [`packages/core/agent/src/types.ts:463`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`waterfall`) | [`compact-basic`](../packages/compact/compact-basic), [`llm-retry`](../packages/llm/llm-retry), [`plan-mode`](../packages/plan/plan-mode) | +| `agent/request-error` | `waterfall` | [`packages/core/agent/src/types.ts:464`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`waterfall`) | [`compact-basic`](../packages/compact/compact-basic), [`llm-retry`](../packages/llm/llm-retry), [`plan-mode`](../packages/plan/plan-mode) | | `agent/session-prefix` | `waterfall` | [`packages/core/agent/src/types.ts:424`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`waterfall`) | [`tool-skill`](../packages/skill/tool-skill), [`workspace-context`](../packages/context/workspace-context) | | `agent/session-start` | `emit` | [`packages/core/agent/src/types.ts:363`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`goal`](../packages/goal/goal), [`goal-session`](../packages/goal/goal-session), [`hooks-claude`](../packages/hooks/hooks-claude), [`hooks-codex`](../packages/hooks/hooks-codex) | | `agent/status` | `emit` | [`packages/core/agent/src/types.ts:303`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`emit`) | [`agent`](../packages/core/agent), [`goal-session`](../packages/goal/goal-session), `runtime`, [`tui`](../packages/ui/tui) | | `agent/step-result` | `waterfall` | [`packages/core/agent/src/types.ts:436`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`waterfall`) | - | -| `agent/turn-continuation` | `waterfall` | [`packages/core/agent/src/types.ts:474`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`waterfall`) | [`hooks-claude`](../packages/hooks/hooks-claude), [`hooks-codex`](../packages/hooks/hooks-codex), [`plan-mode`](../packages/plan/plan-mode) | -| `agent/turn-stop` | `serial` | [`packages/core/agent/src/types.ts:485`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`serial`) | [`subagent-inprocess`](../packages/subagent/subagent-inprocess), [`tool-goal`](../packages/goal/tool-goal) | +| `agent/turn-continuation` | `waterfall` | [`packages/core/agent/src/types.ts:475`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`waterfall`) | [`hooks-claude`](../packages/hooks/hooks-claude), [`hooks-codex`](../packages/hooks/hooks-codex), [`plan-mode`](../packages/plan/plan-mode) | +| `agent/turn-stop` | `serial` | [`packages/core/agent/src/types.ts:486`](../packages/core/agent/src/types.ts) | [`agent-loop`](../packages/core/agent-loop) (`serial`) | [`subagent-inprocess`](../packages/subagent/subagent-inprocess), [`tool-goal`](../packages/goal/tool-goal) | | `approval/request` | `waterfall` | [`packages/ui/user-approval/src/index.ts:30`](../packages/ui/user-approval/src/index.ts) | [`user-approval`](../packages/ui/user-approval) (`waterfall`) | [`acp`](../packages/ui/acp) | | `commands/change` | `emit` | [`packages/ui/commands/src/index.ts:103`](../packages/ui/commands/src/index.ts) | [`commands`](../packages/ui/commands) (`events.dispatch`) | [`acp`](../packages/ui/acp), [`tui`](../packages/ui/tui) | | `fs/edit-intent` | `waterfall` | [`packages/fs/fs/src/index.ts:62`](../packages/fs/fs/src/index.ts) | [`tool-fs`](../packages/fs/tool-fs) (`waterfall`) | [`fs-policy`](../packages/fs/fs-policy) | diff --git a/packages/compact/compact-basic/src/index.ts b/packages/compact/compact-basic/src/index.ts index 10c64d10ca..0ae0e02f7b 100644 --- a/packages/compact/compact-basic/src/index.ts +++ b/packages/compact/compact-basic/src/index.ts @@ -160,6 +160,7 @@ export class BasicCompactService extends CompactService { _error, failure, priorFailures, + _retryPolicy, signal, next, ) => { diff --git a/packages/compact/compact-basic/tests/compact-basic.spec.ts b/packages/compact/compact-basic/tests/compact-basic.spec.ts index 1db86d38e4..711dc3ee95 100644 --- a/packages/compact/compact-basic/tests/compact-basic.spec.ts +++ b/packages/compact/compact-basic/tests/compact-basic.spec.ts @@ -1277,7 +1277,7 @@ describe('automatic listener and loader composition', () => { const failure: LlmFailure = { message: error.message, code: error.code ?? 'UNKNOWN' } const priorFailures = Object.freeze(Array.from({ length: retryAttempt }, () => failure)) return agentEvents(ctx, owner).waterfall( - 'agent/request-error', 1, 1, error, failure, priorFailures, signal, next, + 'agent/request-error', 1, 1, error, failure, priorFailures, undefined, signal, next, ) } diff --git a/packages/cordis/tool-cordis/src/api-catalog.ts b/packages/cordis/tool-cordis/src/api-catalog.ts index 8284aa022c..acbf9c74ef 100644 --- a/packages/cordis/tool-cordis/src/api-catalog.ts +++ b/packages/cordis/tool-cordis/src/api-catalog.ts @@ -941,8 +941,8 @@ export const EVENT_API: readonly EventApiEntry[] = [ { name: 'agent/request-error', mode: 'waterfall', - signature: '\'agent/request-error\'(this: Scoped, agent: Agent, turn: number, step: number, error: RequestError, failure: LlmFailure, priorFailures: readonly LlmFailure[], signal: AbortSignal, next: () => Promise): Promise', - jsDoc: '/**\n * Recover a model-request failure after its failed step has closed. `retry`\n * opens a new numbered step; `fail` preserves the original request error.\n * Call `next()` to delegate to the next recovery listener or the default.\n * @param agent - the agent whose request failed.\n * @param turn - the open turn number.\n * @param step - the failed step number.\n * @param error - the original model-request failure.\n * @param failure - serializable facts normalized at the final adapter boundary.\n * @param priorFailures - immutable failures that already authorized another request in this consecutive sequence.\n * @param signal - the turn abort signal.\n * Scope-filtered dispatch (`@deepseek-ai/dsh-scope`): agent-scoped listeners receive only that agent.\n * @mode waterfall\n */', + signature: '\'agent/request-error\'(this: Scoped, agent: Agent, turn: number, step: number, error: RequestError, failure: LlmFailure, priorFailures: readonly LlmFailure[], retryPolicy: ResolvedRetryPolicy | undefined, signal: AbortSignal, next: () => Promise): Promise', + jsDoc: '/**\n * Recover a model-request failure after its failed step has closed. `retry`\n * opens a new numbered step; `fail` preserves the original request error.\n * Call `next()` to delegate to the next recovery listener or the default.\n * @param agent - the agent whose request failed.\n * @param turn - the open turn number.\n * @param step - the failed step number.\n * @param error - the original model-request failure.\n * @param failure - serializable facts normalized at the final adapter boundary.\n * @param priorFailures - immutable failures that already authorized another request in this consecutive sequence.\n * @param retryPolicy - immutable policy of the adapter registration that served the failed request, or `undefined` if no final adapter served it.\n * @param signal - the turn abort signal.\n * Scope-filtered dispatch (`@deepseek-ai/dsh-scope`): agent-scoped listeners receive only that agent.\n * @mode waterfall\n */', summary: 'Recover a model-request failure after its failed step has closed.', }, { diff --git a/packages/core/agent-loop/README.md b/packages/core/agent-loop/README.md index f937067588..2a9a2468f3 100644 --- a/packages/core/agent-loop/README.md +++ b/packages/core/agent-loop/README.md @@ -60,7 +60,7 @@ The driver owns one agent for its lifetime and runs inside `ctx.agents.withIniti Every provider call that reaches a successful finish appends exactly one `assistant/message` completion anchor, including content-less calls and `max-tokens` finishes. A successful `agent/step-result` stores its transformed content; a rejected result records empty content before the original failure continues. The anchor retains exact chunk provenance (`[]` for a stream with no chunks) and usage when available, while empty content stays out of derived message history. -Plugin failure ends the current turn, not the loop. Only final adapter dispatch/iteration failures and terminal in-band error or aborted finishes enter `agent/request-error`; middleware, result processing, tools, and `agent/post-step` remain ordinary turn failures. Recovery receives the exact live error, immutable provider facts, and immutable prior failures after the failed step closes. A retry rebuilds from the durable log in a new numbered step, success clears the consecutive history, and exhaustion records the structured failure once on `turn/end`. AgentLoop privately owns one cancellation holder whose explicit signal spans prompt policy, assembly, every step, model and tool work, recovery, continuation, and terminal stop; it retires the holder immediately before publishing `turn/end`, while the driver may remain `running` through the durability flush. An effective `cancel()` emits the typed runtime-only `user | parent` cause before clearing pending work and cooperatively aborting the holder; notification failures cannot veto cancellation, work queued by a notification observer is cleared, work queued by a later abort observer belongs to the next turn, and idle cancellation emits nothing. Durable `turn/end` remains coarse `aborted`; undispatched model tool calls receive synthetic `tool/call` and `ABORTED_BEFORE_DISPATCH` result pairs. Disposal wins terminal classification, and work that ignores the signal must settle before quiescence. The [explicit-cancellation decision](../../../.agents/notes/implemented/architecture/2026-07-16-explicit-turn-cancellation.md) owns the lifecycle and race contract. Terminal continuation stops remain authoritative through turn close and durability flush. +Plugin failure ends the current turn, not the loop. Only final adapter dispatch/iteration failures and terminal in-band error or aborted finishes enter `agent/request-error`; middleware, result processing, tools, and `agent/post-step` remain ordinary turn failures. Recovery receives the exact live error, immutable provider facts, immutable prior failures, and the immutable retry policy of the adapter registration that served the request after the failed step closes; the policy is absent if no final adapter served it. A retry rebuilds from the durable log in a new numbered step, success clears the consecutive history, and exhaustion records the structured failure once on `turn/end`. AgentLoop privately owns one cancellation holder whose explicit signal spans prompt policy, assembly, every step, model and tool work, recovery, continuation, and terminal stop; it retires the holder immediately before publishing `turn/end`, while the driver may remain `running` through the durability flush. An effective `cancel()` emits the typed runtime-only `user | parent` cause before clearing pending work and cooperatively aborting the holder; notification failures cannot veto cancellation, work queued by a notification observer is cleared, work queued by a later abort observer belongs to the next turn, and idle cancellation emits nothing. Durable `turn/end` remains coarse `aborted`; undispatched model tool calls receive synthetic `tool/call` and `ABORTED_BEFORE_DISPATCH` result pairs. Disposal wins terminal classification, and work that ignores the signal must settle before quiescence. The [explicit-cancellation decision](../../../.agents/notes/implemented/architecture/2026-07-16-explicit-turn-cancellation.md) owns the lifecycle and race contract. Terminal continuation stops remain authoritative through turn close and durability flush. Within a step, exclusive calls form barriers; parallel-safe calls use a bounded rolling pool and are reclassified before start. Only dispatch/body overlaps. Policy, durable results, and result context remain model-ordered. Abort stops new calls, drains started results, then drains accepted batch context before the turn closes through the normal abort path. diff --git a/packages/core/agent-loop/src/loop.ts b/packages/core/agent-loop/src/loop.ts index 4324deca8d..fcf4dfa846 100644 --- a/packages/core/agent-loop/src/loop.ts +++ b/packages/core/agent-loop/src/loop.ts @@ -7,9 +7,9 @@ import { randomUUID } from 'node:crypto' import type { Context } from 'cordis' -import type { ContentBlock, FinishReason, GenerateOptions, LlmCallConfig, LlmFailure, Message } from '@deepseek-ai/dsh-llm' +import type { ContentBlock, FinishReason, GenerateOptions, LlmCallConfig, LlmFailure, Message, ResolvedRetryPolicy } from '@deepseek-ai/dsh-llm' import { isDeepStrictEqual } from 'node:util' -import { BlockAssembler, HarnessError, LlmError, assertNever, deepFreeze, errorChain, llmFailureOf, markAgentLoopRequest } from '@deepseek-ai/dsh-llm' +import { BlockAssembler, HarnessError, LlmError, assertNever, deepFreeze, errorChain, llmFailureOf, llmRetryPolicyOf, markAgentLoopRequest } from '@deepseek-ai/dsh-llm' import { agentEvents, agentInterruptReasonOf, assembleContextFor, AgentMessageId } from '@deepseek-ai/dsh-agent' import type { AgentEventDispatch, ContinuationDecision, HookContext, PromptDecision, RequestError, RequestErrorDecision } from '@deepseek-ai/dsh-agent' import { canonicalHeader } from '@deepseek-ai/dsh-session' @@ -33,6 +33,7 @@ class TerminalModelRequestFailure extends Error { constructor( readonly requestError: RequestError, readonly failure: LlmFailure, + readonly retryPolicy: ResolvedRetryPolicy | undefined, ) { super(failure.message, { cause: requestError }) this.name = 'TerminalModelRequestFailure' @@ -442,14 +443,18 @@ async function runTurn( let stepOutcome: | { hadToolCalls: boolean; finish: FinishReason } - | { requestError: RequestError; failure: LlmFailure } + | { requestError: RequestError; failure: LlmFailure; retryPolicy: ResolvedRetryPolicy | undefined } | { error: RequestError } try { stepOutcome = await runStep( ctx, events, handle, turn, step, assembly, fullSystemPrompt, boundaryMessages, transmission, signal) } catch (error: unknown) { if (error instanceof TerminalModelRequestFailure) { - stepOutcome = { requestError: error.requestError, failure: error.failure } + stepOutcome = { + requestError: error.requestError, + failure: error.failure, + retryPolicy: error.retryPolicy, + } } else { stepOutcome = { error: toError(error) } } @@ -470,7 +475,7 @@ async function runTurn( try { recoveryDecision = await events.waterfall( 'agent/request-error', turn, step, stepOutcome.requestError, - stepOutcome.failure, requestFailureHistory, signal, + stepOutcome.failure, requestFailureHistory, stepOutcome.retryPolicy, signal, () => Promise.resolve(defaultDecision), ) } catch (recoveryError: unknown) { @@ -714,14 +719,18 @@ async function runStep( } } catch (error: unknown) { const failure = llmFailureOf(stream, error) - if (failure !== undefined && error instanceof Error) throw new TerminalModelRequestFailure(error, failure) + if (failure !== undefined && error instanceof Error) { + throw new TerminalModelRequestFailure(error, failure, llmRetryPolicyOf(stream)) + } throw error } interruptionCheckpoint(signal) // Normalize failure finish chunks into the same path as thrown stream errors. const stepError = finishError(assembler.finish) - if (stepError) throw new TerminalModelRequestFailure(stepError.error, stepError.failure) + if (stepError) { + throw new TerminalModelRequestFailure(stepError.error, stepError.failure, llmRetryPolicyOf(stream)) + } const recordAssistantMessage = ( assembledContent: ContentBlock[], diff --git a/packages/core/agent-loop/tests/request-recovery.spec.ts b/packages/core/agent-loop/tests/request-recovery.spec.ts index 1fc44e3431..494c2f95a9 100644 --- a/packages/core/agent-loop/tests/request-recovery.spec.ts +++ b/packages/core/agent-loop/tests/request-recovery.spec.ts @@ -269,10 +269,11 @@ describe('agent post-step and request-error lifecycle', () => { const ctx = await harness(adapter) const agent = ctx.agentLoop.create(SessionId(`recover-${_style}`), { provider: 'mock', model: 'mock' }) const attempts: number[] = [] - ctx.on('agent/request-error', async (subject, turn, step, error, facts, history) => { + ctx.on('agent/request-error', async (subject, turn, step, error, facts, history, retryPolicy) => { expect(subject).toBe(agent) expect({ turn, step, code: error.code }).toEqual({ turn: 1, step: 1, code: CONTEXT_WINDOW_EXCEEDED_CODE }) expect(facts.code).toBe(CONTEXT_WINDOW_EXCEEDED_CODE) + expect(retryPolicy).toMatchObject({ mode: 'normal', maxRetries: 2 }) attempts.push(history.length) subject.session.append('user/message', { content: [{ type: 'text', text: 'RECOVERY SURFACE MUTATION' }], @@ -301,7 +302,9 @@ describe('agent post-step and request-error lifecycle', () => { const agent = ctx.agentLoop.create(SessionId(`stream-plugin-${_name.replaceAll(' ', '-')}`), { provider: 'mock', model: 'mock' }) let recoveries = 0 install(ctx) - ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => { + ctx.on('agent/request-error', async ( + _agent, _turn, _step, _error, _failure, _history, _retryPolicy, _signal, next, + ) => { recoveries += 1 return next() }) @@ -332,7 +335,9 @@ describe('agent post-step and request-error lifecycle', () => { }) const agent = ctx.agentLoop.create(SessionId('nested-stream-not-recoverable'), { provider: 'mock', model: 'mock' }) let recoveries = 0 - ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => { + ctx.on('agent/request-error', async ( + _agent, _turn, _step, _error, _failure, _history, _retryPolicy, _signal, next, + ) => { recoveries += 1 return next() }) @@ -365,7 +370,9 @@ describe('agent post-step and request-error lifecycle', () => { } const agent = ctx.agentLoop.create(SessionId(`${boundary}-not-recoverable`), { provider: 'mock', model: 'mock' }) let recoveries = 0 - ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => { + ctx.on('agent/request-error', async ( + _agent, _turn, _step, _error, _failure, _history, _retryPolicy, _signal, next, + ) => { recoveries += 1 return next() }) @@ -393,7 +400,9 @@ describe('agent post-step and request-error lifecycle', () => { } const agent = ctx.agentLoop.create(SessionId(`${failure}-not-recoverable`), { provider: 'mock', model: 'mock' }) let recoveries = 0 - ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => { + ctx.on('agent/request-error', async ( + _agent, _turn, _step, _error, _failure, _history, _retryPolicy, _signal, next, + ) => { recoveries += 1 return next() }) @@ -412,7 +421,9 @@ describe('agent post-step and request-error lifecycle', () => { const ctx = await harness(makeAdapter(original)) const agent = ctx.agentLoop.create(SessionId(`identity-${_name.replaceAll(' ', '-')}`), { provider: 'mock', model: 'mock' }) let seen: Error | undefined - ctx.on('agent/request-error', async (_agent, _turn, _step, error, _failure, _history, _signal, next) => { + ctx.on('agent/request-error', async ( + _agent, _turn, _step, error, _failure, _history, _retryPolicy, _signal, next, + ) => { seen = error return next() }) @@ -431,7 +442,9 @@ describe('agent post-step and request-error lifecycle', () => { const agent = ctx.agentLoop.create(SessionId('hostile-message-recovery'), { provider: 'mock', model: 'mock' }) let seenError: Error | undefined let seenFailure: LlmFailure | undefined - ctx.on('agent/request-error', async (_agent, _turn, _step, error, failure, _history, _signal, next) => { + ctx.on('agent/request-error', async ( + _agent, _turn, _step, error, failure, _history, _retryPolicy, _signal, next, + ) => { seenError = error seenFailure = failure return next() @@ -462,7 +475,7 @@ describe('agent post-step and request-error lifecycle', () => { let seenFailure: LlmFailure | undefined let seenHistory: readonly LlmFailure[] | undefined ctx.on('agent/request-error', async ( - _agent, _turn, _step, error, failure, history, _signal, next, + _agent, _turn, _step, error, failure, history, _retryPolicy, _signal, next, ) => { seenError = error seenFailure = failure @@ -506,13 +519,18 @@ describe('agent post-step and request-error lifecycle', () => { const ctx = scenario === 'iterator' ? await harness(new IteratorConstructionFailureAdapter()) : await harness() const agent = ctx.agentLoop.create(SessionId(`request-boundary-${scenario}`), { provider: 'mock', model: 'mock' }) let seen = '' - ctx.on('agent/request-error', async (_agent, _turn, _step, error, _failure, _history, _signal, next) => { + let sawServingPolicy = false + ctx.on('agent/request-error', async ( + _agent, _turn, _step, error, _failure, _history, retryPolicy, _signal, next, + ) => { seen = error.code ?? '' + sawServingPolicy = retryPolicy !== undefined return next() }) send(agent) await waitForIdle(ctx, agent) expect(seen).toBe(scenario === 'iterator' ? 'ITERATOR_CONSTRUCTION' : 'NO_ADAPTER') + expect(sawServingPolicy).toBe(scenario === 'iterator') } }) @@ -522,7 +540,7 @@ describe('agent post-step and request-error lifecycle', () => { const cappedAgent = cappedCtx.agentLoop.create(SessionId('retry-cap'), { provider: 'mock', model: 'mock' }) const cappedHistories: string[][] = [] cappedCtx.on('agent/request-error', async ( - _agent, _turn, _step, _error, _failure, history, _signal, next, + _agent, _turn, _step, _error, _failure, history, _retryPolicy, _signal, next, ) => { const codes = history.map(entry => entry.code) cappedHistories.push(codes) @@ -547,7 +565,7 @@ describe('agent post-step and request-error lifecycle', () => { const resetAgent = resetCtx.agentLoop.create(SessionId('retry-reset'), { provider: 'mock', model: 'mock' }) const resetHistories: { step: number; codes: string[] }[] = [] resetCtx.on('agent/request-error', async ( - _agent, _turn, step, _error, _failure, history, _signal, next, + _agent, _turn, step, _error, _failure, history, _retryPolicy, _signal, next, ) => { resetHistories.push({ step, codes: history.map(entry => entry.code) }) return resetHistories.length === 1 ? { action: 'retry' } : next() @@ -578,7 +596,9 @@ describe('agent post-step and request-error lifecycle', () => { const agent = ctx.agentLoop.create(SessionId(`${action}-recovery`), { provider: 'mock', model: 'mock' }) let entered!: () => void const recoveryEntered = new Promise((resolve) => { entered = resolve }) - ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, signal) => { + ctx.on('agent/request-error', async ( + _agent, _turn, _step, _error, _failure, _history, _retryPolicy, signal, + ) => { entered() await new Promise((resolve) => { signal.addEventListener('abort', () => { resolve() }, { once: true }) diff --git a/packages/core/agent/src/types.ts b/packages/core/agent/src/types.ts index 0f442e3f88..0efe140b22 100644 --- a/packages/core/agent/src/types.ts +++ b/packages/core/agent/src/types.ts @@ -8,7 +8,7 @@ import type { Context } from 'cordis' import type { Branded } from '@deepseek-ai/dsh-brand' import type { Scoped } from '@deepseek-ai/dsh-scope' -import type { ContentBlock, LlmCallConfig, LlmFailure, Message, MessageSource } from '@deepseek-ai/dsh-llm' +import type { ContentBlock, LlmCallConfig, LlmFailure, Message, MessageSource, ResolvedRetryPolicy } from '@deepseek-ai/dsh-llm' import type { JsonValue, Session, SessionId } from '@deepseek-ai/dsh-session' import type {} from '@deepseek-ai/dsh-system-prompt' declare module '@deepseek-ai/dsh-system-prompt' { @@ -456,11 +456,13 @@ declare module 'cordis' { * @param error - the original model-request failure. * @param failure - serializable facts normalized at the final adapter boundary. * @param priorFailures - immutable failures that already authorized another request in this consecutive sequence. + * @param retryPolicy - immutable policy of the adapter registration that served + * the failed request, or `undefined` if no final adapter served it. * @param signal - the turn abort signal. * Scope-filtered dispatch (`@deepseek-ai/dsh-scope`): agent-scoped listeners receive only that agent. * @mode waterfall */ - 'agent/request-error'(this: Scoped, agent: Agent, turn: number, step: number, error: RequestError, failure: LlmFailure, priorFailures: readonly LlmFailure[], signal: AbortSignal, next: () => Promise): Promise + 'agent/request-error'(this: Scoped, agent: Agent, turn: number, step: number, error: RequestError, failure: LlmFailure, priorFailures: readonly LlmFailure[], retryPolicy: ResolvedRetryPolicy | undefined, signal: AbortSignal, next: () => Promise): Promise /** * Override whether the turn continues. The default continues after tool * calls or steering and stops otherwise; a continue reason becomes steering. diff --git a/packages/core/scope/tests/invariant.spec.ts b/packages/core/scope/tests/invariant.spec.ts index ca0841165b..c54753c301 100644 --- a/packages/core/scope/tests/invariant.spec.ts +++ b/packages/core/scope/tests/invariant.spec.ts @@ -51,7 +51,7 @@ describe('scoped-dispatch invariants', () => { 'agent/post-step': [agent, 1, 1, signal], 'agent/prompt-submit': [agent, [], { kind: 'user' }, signal, () => Promise.resolve({ kind: 'allow' })], 'agent/request': [agent, 1, 1, config, signal, () => Promise.resolve(config)], - 'agent/request-error': [agent, 1, 1, new Error('request failed'), { message: 'request failed', code: 'UNKNOWN' }, [], signal, () => Promise.resolve({ action: 'fail' })], + 'agent/request-error': [agent, 1, 1, new Error('request failed'), { message: 'request failed', code: 'UNKNOWN' }, [], undefined, signal, () => Promise.resolve({ action: 'fail' })], 'agent/session-prefix': [agent, [], signal, () => Promise.resolve([])], 'agent/step-result': [agent, 1, 1, message, signal, () => Promise.resolve(message)], 'agent/turn-continuation': [agent, 1, { action: 'stop' }, signal, () => Promise.resolve({ action: 'stop' })], diff --git a/packages/llm/llm-retry/README.md b/packages/llm/llm-retry/README.md index 711666b8a0..72d7ac6831 100644 --- a/packages/llm/llm-retry/README.md +++ b/packages/llm/llm-retry/README.md @@ -2,7 +2,7 @@ Function plugin that applies exact-provider retry policy on the agent loop's closed-step recovery seam. It does not wrap `ctx.llm.stream()`: every adapter call remains one provider attempt, and every retry opens a fresh numbered step. -Each provider adapter owns an optional nested `retryPolicy`, captured when its route registers on `ctx.llm`. Omission uses normal mode: two retries for `RATE_LIMIT`, `SERVER`, `TIMEOUT`, and `TRANSPORT`. A normal policy can change its finite budget, eligible codes, and backoff. Always mode asks downstream recovery first, then retries every model-request failure without an attempt limit; success, cancellation, or plugin disposal stops it. +Each provider adapter owns an optional nested `retryPolicy`, captured when its route registers on `ctx.llm` and carried with each call that reaches that registration's final adapter boundary. An in-flight failure retains that serving policy if the route is later disposed or replaced; a failure before any final adapter is selected has no provider policy and delegates. Omission uses normal mode: two retries for `RATE_LIMIT`, `SERVER`, `TIMEOUT`, and `TRANSPORT`. A normal policy can change its finite budget, eligible codes, and backoff. Always mode asks downstream recovery first, then retries every model-request failure without an attempt limit; success, cancellation, or plugin disposal stops it. Both modes use bounded exponential backoff with symmetric jitter. A valid `providerRetryAfterMs` at or below `maxDelayMs` replaces local backoff without jitter. An over-cap provider delay makes normal mode delegate, while always mode uses its configured local backoff so it cannot terminate on that instruction. diff --git a/packages/llm/llm-retry/src/index.ts b/packages/llm/llm-retry/src/index.ts index 04f1b0996a..7e9c2d56e6 100644 --- a/packages/llm/llm-retry/src/index.ts +++ b/packages/llm/llm-retry/src/index.ts @@ -37,13 +37,13 @@ declare module '@deepseek-ai/dsh-session' { } export const name = 'llm-retry' -export const inject = ['agents', 'llm'] +export const inject = ['agents'] /** This policy executor has no config; providers own `retryPolicy`. */ -export type Config = Readonly> +export type Config = Readonly> /** Runtime schema for {@link Config}. */ -export const Config: z = z.object({}) +export const Config = z.object({}) as unknown as z function validateConfig(config: Config): void { const [key] = Object.keys(config) @@ -170,6 +170,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna _error: RequestError, failure: LlmFailure, priorFailures: readonly LlmFailure[], + policy: ResolvedRetryPolicy | undefined, signal: AbortSignal, next: () => Promise, ) => { @@ -177,15 +178,15 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna // removed. Lifetime cancellation must prevent that stale callback from // entering a downstream policy after disposal. if (lifetime.signal.aborted) return Promise.resolve({ action: 'fail' }) - // Bind policy to the header in force when this step closed. Downstream - // recovery may append later state before an always fallback runs. + if (policy === undefined) return next() + // The call-local policy belongs to the registration that served this + // failure. Recover only the durable provider identity from the header; + // downstream recovery may append later state before an always fallback. const provider = providerForClosedStep(agent.session.events, turn, step) /* v8 ignore next 3 -- agent-loop closes only steps whose request header was recorded */ if (provider === undefined) { throw new Error(`llm-retry: no request provider for closed turn ${turn}/step ${step}`) } - const policy = ctx.llm.providerRetryPolicy(provider) - if (policy.mode === 'always') { const downstream = await downstreamUntilAbort( next, diff --git a/packages/llm/llm-retry/tests/retry.spec.ts b/packages/llm/llm-retry/tests/retry.spec.ts index 56f785f38c..232f4b4459 100644 --- a/packages/llm/llm-retry/tests/retry.spec.ts +++ b/packages/llm/llm-retry/tests/retry.spec.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it, vi } from 'vitest' +import { afterEach, describe, expect, expectTypeOf, it, vi } from 'vitest' import { Context } from 'cordis' import type { Fiber } from 'cordis' import LlmService, { CallId, LlmAdapter, LlmError, resolveRetryPolicy } from '@deepseek-ai/dsh-llm' @@ -79,7 +79,7 @@ async function harness( policies: Readonly> = { mock: normalConfig() }, beforeRetry?: (ctx: Context) => void, internals: retry.RetryInternals = {}, -): Promise<{ ctx: Context; retryFiber: Fiber }> { +): Promise<{ ctx: Context; retryFiber: Fiber; disposeAdapter: () => void }> { const ctx = new Context() await ctx.plugin(LlmService) await ctx.plugin(SessionStore) @@ -92,8 +92,8 @@ async function harness( retry.apply(inner, {}, internals) }, { inject: retry.inject })) await ctx.plugin(AgentLoop, { agents: [] }) - ctx.llm.registerAdapter(['mock', 'other'], adapter) - return { ctx, retryFiber } + const disposeAdapter = ctx.llm.registerAdapter(['mock', 'other'], adapter) + return { ctx, retryFiber, disposeAdapter } } function normalConfig( @@ -413,6 +413,28 @@ describe('provider-routed retry policy', () => { expect(vi.getTimerCount()).toBe(0) }) + it('delegates when no final adapter served the failed request', async () => { + const adapter = new ScriptedAdapter([textResponse('must not run')]) + const mounted = await harness(adapter, { mock: alwaysConfig() }) + context = mounted.ctx + mounted.disposeAdapter() + const agent = context.agentLoop.create(SessionId('retry-no-serving-policy'), { + provider: 'mock', + model: 'mock', + }) + const idle = waitForIdle(context, agent) + + agent.followup([{ type: 'text', text: 'missing route' }]) + await idle + + expect(adapter.requests).toHaveLength(0) + expect(agent.session.events.some(event => event.type === 'llm/retry')).toBe(false) + expect(agent.session.events.at(-1)).toMatchObject({ + type: 'turn/end', + data: { reason: { kind: 'error', failure: { code: 'NO_ADAPTER' } } }, + }) + }) + it('selects policy by the failed request provider', async () => { vi.useFakeTimers() const adapter = new ScriptedAdapter([ @@ -481,6 +503,64 @@ describe('provider-routed retry policy', () => { expect(adapter.requests.map(request => request.provider)).toEqual(['other', 'other']) }) + it.each(['thrown', 'in-band'] as const)( + 'uses the serving registration policy when an in-flight route is replaced after a %s failure', + async (failureKind) => { + vi.useFakeTimers() + const entered = Promise.withResolvers() + const release = Promise.withResolvers() + const oldAdapter = new ScriptedAdapter([(async function * (): AsyncGenerator { + entered.resolve(undefined) + await release.promise + if (failureKind === 'thrown') { + throw new LlmError('old route auth failed', 'AUTH') + } + yield { + type: 'finish', + reason: { + kind: 'error', + failure: { message: 'old route auth failed', code: 'AUTH' }, + }, + } + })()]) + const mounted = await harness(oldAdapter, { mock: alwaysConfig({ + initialDelayMs: 1, + maxDelayMs: 1, + }) }) + context = mounted.ctx + const agent = context.agentLoop.create(SessionId('retry-serving-registration'), { + provider: 'mock', + model: 'mock', + }) + const scheduled = waitForRetry(context, agent, 1) + agent.followup([{ type: 'text', text: 'replace while in flight' }]) + await entered.promise + + mounted.disposeAdapter() + const replacement = new ScriptedAdapter([textResponse('replacement recovered')]) + replacement.configureRetryPolicies({ mock: normalConfig({ maxRetries: 0 }) }) + context.llm.registerAdapter(['mock'], replacement) + release.resolve(undefined) + + expect((await scheduled).data).toMatchObject({ + provider: 'mock', + mode: 'always', + retry: 1, + delayMs: 1, + }) + const idle = waitForIdle(context, agent) + await vi.advanceTimersByTimeAsync(1) + await idle + + expect(oldAdapter.requests).toHaveLength(1) + expect(replacement.requests).toHaveLength(1) + expect(agent.session.deriveMessages().at(-1)).toMatchObject({ + role: 'assistant', + content: [{ type: 'text', text: 'replacement recovered' }], + }) + }, + ) + it('keeps always mode unbounded while preserving cancellable jittered backoff', async () => { vi.useFakeTimers() const adapter = new ScriptedAdapter([ @@ -719,7 +799,9 @@ describe('provider-routed retry policy', () => { const captured = Promise.withResolvers() let invokeCaptured: (() => Promise) | undefined const mounted = await harness(adapter, {}, (ctx) => { - ctx.on('agent/request-error', (_agent, _turn, _step, _error, _failure, _history, _signal, next) => { + ctx.on('agent/request-error', ( + _agent, _turn, _step, _error, _failure, _history, _retryPolicy, _signal, next, + ) => { return new Promise((resolve) => { invokeCaptured = async () => { resolve(await next()) } captured.resolve(undefined) @@ -728,7 +810,9 @@ describe('provider-routed retry policy', () => { }) context = mounted.ctx let downstreamCalls = 0 - context.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, next) => { + context.on('agent/request-error', async ( + _agent, _turn, _step, _error, _failure, _history, _retryPolicy, _signal, next, + ) => { downstreamCalls += 1 return next() }) @@ -782,7 +866,9 @@ describe('provider-routed retry policy', () => { textResponse('must not run'), ]) ;({ ctx: context } = await harness(adapter, { mock: policy }, (ctx) => { - ctx.on('agent/request-error', async (agent, _turn, _step, _error, _failure, _history, _signal, next) => { + ctx.on('agent/request-error', async ( + agent, _turn, _step, _error, _failure, _history, _retryPolicy, _signal, next, + ) => { agent.cancel({ kind: 'user' }) return next() }) @@ -823,14 +909,16 @@ describe('provider-routed retry policy', () => { }) it('rejects retry policy configured on the executor instead of a provider', () => { + expectTypeOf<{}>().toExtend() + expectTypeOf<{ retryPolicy: { mode: 'always' } }>().not.toExtend() expect(() => { - retry.apply(new Context(), { retryPolicy: { mode: 'always' } }) + retry.apply(new Context(), { retryPolicy: { mode: 'always' } } as unknown as retry.Config) }).toThrow(/retryPolicy belongs under each provider/) }) it('rejects unknown executor config', () => { expect(() => { - retry.apply(new Context(), { retryPolciy: {} }) + retry.apply(new Context(), { retryPolciy: {} } as unknown as retry.Config) }).toThrow(/unknown key "retryPolciy"/) }) }) diff --git a/packages/llm/llm/README.md b/packages/llm/llm/README.md index 74dafa3e48..6a855f6004 100644 --- a/packages/llm/llm/README.md +++ b/packages/llm/llm/README.md @@ -15,7 +15,7 @@ An adapter registry plus a single streaming call surface, interceptable via a wa - `ctx.llm.resolveModelContext(provider: string, model: string): Promise` Resolve authoritative context capacity for one exact route from its owning adapter. - `ctx.llm.stream(options: GenerateOptions): AsyncIterable` Stream one model call as raw chunks (token-level deltas). Consumers assemble the chunks into blocks/messages with `BlockAssembler`. -`LlmService` preserves errors from final adapter selection, synchronous dispatch, iterator construction, and iteration, and binds their provenance to the exact stream handle returned for that model call. `isLlmAdapterFailure(stream, value)` reports only errors from that call's final adapter boundary; `llmFailureOf(stream, value)` returns the adjacent immutable `LlmFailure`. Nested model calls, `llm/stream` middleware, and downstream consumer failures remain unclassified for the outer call. Classification never replaces or mutates the adapter's original coded `Error`. +`LlmService` preserves errors from final adapter selection, synchronous dispatch, iterator construction, and iteration, and binds their provenance to the exact stream handle returned for that model call. `isLlmAdapterFailure(stream, value)` reports only errors from that call's final adapter boundary; `llmFailureOf(stream, value)` returns the adjacent immutable `LlmFailure`; `llmRetryPolicyOf(stream)` returns the immutable policy of the exact registration selected at that boundary, even if the route is later disposed or replaced. A call that never reaches a final adapter has no serving policy. Nested model calls, `llm/stream` middleware, and downstream consumer failures remain unclassified for the outer call. Classification never replaces or mutates the adapter's original coded `Error`. Provider and model metadata is a discovery surface, not a routing whitelist. `registerAdapter()` still owns provider exclusivity and captures the adapter's retry policy for each route, while an adapter may accept model ids absent from `listModels()`; consumers must not reject a request because its model is unlisted. Returned selector metadata is detached and invalid or duplicate adapter entries fail with `INVALID_ADAPTER` or `INVALID_CATALOG`. diff --git a/packages/llm/llm/src/adapter-failure.ts b/packages/llm/llm/src/adapter-failure.ts index 390282327d..87e783dcea 100644 --- a/packages/llm/llm/src/adapter-failure.ts +++ b/packages/llm/llm/src/adapter-failure.ts @@ -6,9 +6,15 @@ import { HarnessError } from './error.ts' import type { LlmFailure, StreamChunk } from './types.ts' +import type { ResolvedRetryPolicy } from './retry-policy.ts' -/** Errors and normalized facts proven to originate in one model call's final adapter boundary. */ -export type AdapterFailureScope = WeakMap +/** Call-local facts captured when one model call enters its final adapter boundary. */ +export interface AdapterFailureScope { + /** Errors and normalized facts proven to originate in this call's final adapter boundary. */ + readonly failures: WeakMap + /** Immutable policy of the exact adapter registration selected for this call. */ + retryPolicy?: ResolvedRetryPolicy +} /** Call-local failure scopes keyed by the exact stream handle returned to a consumer. */ const adapterFailureScopes = new WeakMap, AdapterFailureScope>() @@ -52,7 +58,7 @@ export function markLlmAdapterFailure( message: errorMessage(error), code: harnessErrorCode(error), }) - failures.set(error, failure) + failures.failures.set(error, failure) return error } @@ -124,7 +130,7 @@ export function isLlmAdapterFailure( value: unknown, ): value is Error & { code?: string } { const failures = adapterFailureScopes.get(stream) - return value instanceof Error && failures !== undefined && failures.has(value) + return value instanceof Error && failures !== undefined && failures.failures.has(value) } /** @@ -139,5 +145,18 @@ export function llmFailureOf( value: unknown, ): LlmFailure | undefined { const failures = adapterFailureScopes.get(stream) - return value instanceof Error ? failures?.get(value) : undefined + return value instanceof Error ? failures?.failures.get(value) : undefined +} + +/** + * Read the retry policy of the exact registration selected at this call's + * final adapter boundary. The policy remains available after that registration + * is disposed or replaced; absence means no final adapter served the call. + * @param stream - the exact stream returned by the model call. + * @returns the immutable serving-registration policy, or `undefined`. + */ +export function llmRetryPolicyOf( + stream: AsyncIterable, +): ResolvedRetryPolicy | undefined { + return adapterFailureScopes.get(stream)?.retryPolicy } diff --git a/packages/llm/llm/src/index.ts b/packages/llm/llm/src/index.ts index 773b9a5ca1..6b714a7074 100644 --- a/packages/llm/llm/src/index.ts +++ b/packages/llm/llm/src/index.ts @@ -33,7 +33,7 @@ export * from './retry-policy.ts' export { BlockAssembler } from './assembler.ts' export { callConfigEquals, deepFreeze, isAgentLoopRequest, markAgentLoopRequest } from './call-config.ts' export type { LlmCallConfig } from './call-config.ts' -export { isLlmAdapterFailure, llmFailureOf } from './adapter-failure.ts' +export { isLlmAdapterFailure, llmFailureOf, llmRetryPolicyOf } from './adapter-failure.ts' declare module 'cordis' { interface Context { @@ -337,7 +337,9 @@ export class LlmService extends Service { ): AsyncGenerator { let iterator: AsyncIterator try { - const adapter = this.registration(options.provider).adapter + const registration = this.registration(options.provider) + failures.retryPolicy = registration.retryPolicy + const adapter = registration.adapter const stream = adapter.stream(this.forAdapter(options, adapter)) iterator = stream[Symbol.asyncIterator]() } catch (error: unknown) { @@ -386,7 +388,7 @@ export class LlmService extends Service { * @returns the chunk stream, possibly wrapped by `llm/stream` listeners. */ stream(options: GenerateOptions): AsyncIterable { - const failures: AdapterFailureScope = new WeakMap() + const failures: AdapterFailureScope = { failures: new WeakMap() } const stream = this.ctx.waterfall(this, 'llm/stream', options, () => this.adapterStream(options, failures)) return bindAdapterFailureScope(stream, failures) } diff --git a/packages/llm/llm/tests/service.spec.ts b/packages/llm/llm/tests/service.spec.ts index 52c51852ba..5631191b22 100644 --- a/packages/llm/llm/tests/service.spec.ts +++ b/packages/llm/llm/tests/service.spec.ts @@ -10,6 +10,7 @@ import LlmService, { LlmAdapter, LlmError, llmFailureOf, + llmRetryPolicyOf, ProviderRequestId, resolveRetryPolicy, StreamChunk, @@ -181,6 +182,51 @@ describe('LlmService', () => { ) }) + it('keeps the serving registration policy on an in-flight call after route replacement', async () => { + const oldPolicy = resolveRetryPolicy({ mode: 'always' }, 'old retryPolicy') + const newPolicy = resolveRetryPolicy({ mode: 'normal', maxRetries: 0 }, 'new retryPolicy') + const entered = Promise.withResolvers() + const release = Promise.withResolvers() + const failure = new LlmError('old route failed', 'AUTH') + const oldAdapter = new class extends LlmAdapter { + override providerRetryPolicy(): typeof oldPolicy { + return oldPolicy + } + + async * stream(_options: GenerateOptions): AsyncIterable { + entered.resolve(undefined) + await release.promise + throw failure + } + }() + const newAdapter = new class extends ScriptedAdapter { + override providerRetryPolicy(): typeof newPolicy { + return newPolicy + } + }(SCRIPT) + const ctx = new Context() + await ctx.plugin(LlmService) + const disposeOld = ctx.llm.registerAdapter(['route'], oldAdapter) + const stream = ctx.llm.stream({ provider: 'route', model: 'model', messages: [] }) + const outcome = (async (): Promise => { + try { + for await (const _chunk of stream) { /* drain */ } + } catch (error: unknown) { + return error + } + return undefined + })() + await entered.promise + + disposeOld() + ctx.llm.registerAdapter(['route'], newAdapter) + release.resolve(undefined) + + expect(await outcome).toBe(failure) + expect(llmRetryPolicyOf(stream)).toBe(oldPolicy) + expect(ctx.llm.providerRetryPolicy('route')).toBe(newPolicy) + }) + it('throws NO_ADAPTER for unregistered providers', async () => { const ctx = new Context() await ctx.plugin(LlmService) @@ -195,6 +241,7 @@ describe('LlmService', () => { expect((caught as LlmError).code).toBe('NO_ADAPTER') expect((caught as LlmError).message).toContain('no adapter registered') expect(isLlmAdapterFailure(stream, caught)).toBe(true) + expect(llmRetryPolicyOf(stream)).toBeUndefined() }) it.each(['done', 'value'] as const)('tags a throwing IteratorResult.%s getter without replacing its Error', async (field) => { diff --git a/packages/plan/plan-mode/src/index.ts b/packages/plan/plan-mode/src/index.ts index 26ad904d30..6ba7a719d7 100644 --- a/packages/plan/plan-mode/src/index.ts +++ b/packages/plan/plan-mode/src/index.ts @@ -184,6 +184,7 @@ export class PlanModeService extends Service { _error, _failure, _priorFailures, + _retryPolicy, _signal, next, ) => { diff --git a/packages/plan/plan-mode/tests/integration.spec.ts b/packages/plan/plan-mode/tests/integration.spec.ts index e195ed54b2..1500554ad7 100644 --- a/packages/plan/plan-mode/tests/integration.spec.ts +++ b/packages/plan/plan-mode/tests/integration.spec.ts @@ -138,7 +138,9 @@ describe('plan mode through the agent loop', () => { const agent = ctx.agentLoop.create(SessionId('it-plan-retry-flip'), { provider: 'mock', model: 'mock' }) const recoveryEntered = Promise.withResolvers() const releaseRecovery = Promise.withResolvers() - ctx.on('agent/request-error', async (subject, _turn, _step, _error, _failure, _history, _signal, next) => { + ctx.on('agent/request-error', async ( + subject, _turn, _step, _error, _failure, _history, _retryPolicy, _signal, next, + ) => { if (subject !== agent) return next() recoveryEntered.resolve(true) await releaseRecovery.promise diff --git a/packages/plan/plan-mode/tests/plan-mode.spec.ts b/packages/plan/plan-mode/tests/plan-mode.spec.ts index a7f7743497..786d1425c2 100644 --- a/packages/plan/plan-mode/tests/plan-mode.spec.ts +++ b/packages/plan/plan-mode/tests/plan-mode.spec.ts @@ -83,6 +83,7 @@ function recoveryBoundary( new Error('request failed'), { message: 'request failed', code: 'SERVER' }, [], + undefined, new AbortController().signal, () => Promise.resolve(decision), ) @@ -945,7 +946,9 @@ describe('HMR disposal', () => { const agent = await agentWithSession(ctx, 'disposed-in-flight-recovery') const recoveryEntered = Promise.withResolvers() const releaseRecovery = Promise.withResolvers() - ctx.on('agent/request-error', async (_agent, _turn, _step, _error, _failure, _history, _signal, _next) => { + ctx.on('agent/request-error', async ( + _agent, _turn, _step, _error, _failure, _history, _retryPolicy, _signal, _next, + ) => { recoveryEntered.resolve(true) await releaseRecovery.promise return { action: 'retry' } From 38ce422f71d2d27ae49d107f23009705f16fd93a Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 25 Jul 2026 14:23:55 +0800 Subject: [PATCH 07/11] fix(llm): drain delegated retry recovery --- ...26-07-24-provider-retry-policies.i18n.yaml | 4 +- .../2026-07-24-provider-retry-policies.md | 4 +- .../2026-07-24-provider-retry-policies.zh.md | 4 +- packages/examples/acp-demo/README.md | 1 - packages/llm/llm-retry/README.md | 6 +- packages/llm/llm-retry/src/index.ts | 79 ++++++++++--------- packages/llm/llm-retry/tests/retry.spec.ts | 60 ++++++++++---- 7 files changed, 94 insertions(+), 64 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml index 000b11d7f0..6d41bf0ab7 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-24-provider-retry-policies.md: 0b9456eb1563bfcbfa06d93403fd68124ed1f707 -2026-07-24-provider-retry-policies.zh.md: d3ef90ec739eec7300b7d0bb526cb5de958ed93f +2026-07-24-provider-retry-policies.md: 3799dbee9658c883f4084ac9c792d4552cc8e2fb +2026-07-24-provider-retry-policies.zh.md: 8024ea707343686d2d1b1351cd7c81ecb8767b85 diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md index 0b9456eb15..3799dbee96 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md @@ -36,7 +36,7 @@ providers: The listener reads the provider from the durable `request/header` in force when the failed step closed, excluding later recovery mutations, but never re-resolves policy from the mutable provider registry. Normal mode retains the bounded transient behavior: it retries configured codes up to `maxRetries`, counts retries scheduled by the same provider policy in the current consecutive failure sequence, and otherwise delegates. -Always mode asks downstream recovery first so a specialized policy such as context-overflow compaction can make progress. A downstream retry wins. A downstream failure decision or thrown recovery error falls back to an unbounded retry of the same provider request; the thrown error is logged. Success, turn cancellation, and plugin disposal are the only termination paths. +Always mode asks downstream recovery first so a specialized policy such as context-overflow compaction can make progress. A downstream retry wins. A downstream failure decision or thrown recovery error falls back to an unbounded retry of the same provider request; the thrown error is logged. The retry listener owns and drains delegated recovery before cancellation or plugin disposal can finish, then applies the abort instead of a late downstream decision. Success, turn cancellation, and plugin disposal are the only termination paths. Both modes use exponential local delays from `initialDelayMs` to `maxDelayMs`. `jitterRatio` multiplies each target by a uniform sample in `[1 - jitterRatio, 1 + jitterRatio]`, then applies the cap. A positive provider `Retry-After` within the cap remains exact and unjittered. An over-cap provider delay makes normal mode delegate; always mode retains its guarantee by using the configured local backoff. @@ -56,7 +56,7 @@ Each scheduled retry appends a non-surface `llm/retry` event with the failed pro ## Verification -Adapter tests validate nested policies at provider load, prove registration captures configured and default policies, and retain the serving policy across in-flight route replacement. Unit and real-Loader composition tests select policies from the failed request's serving registration, exercise always mode beyond the normal budget, pin jitter and delay caps, prove downstream recovery ordering, prove cancellation interrupts stalled downstream recovery, and prove cancellation and disposal stop active backoff waits. Request-level coverage compares the complete messages of failed and retried attempts and rejects both provider error text and discarded partial output. JSONL and SQLite tests round-trip an always event without `Infinity`; invariant tests bind its provider to the request header and its retry number to the active provider policy; ACP and TUI tests render finite and infinite limits. +Adapter tests validate nested policies at provider load, prove registration captures configured and default policies, and retain the serving policy across in-flight route replacement. Unit and real-Loader composition tests select policies from the failed request's serving registration, exercise always mode beyond the normal budget, pin jitter and delay caps, prove downstream recovery ordering, prove cancellation and disposal drain delegated recovery before reaching quiescence, and prove both abort active backoff waits. Request-level coverage compares the complete messages of failed and retried attempts and rejects both provider error text and discarded partial output. JSONL and SQLite tests round-trip an always event without `Infinity`; invariant tests bind its provider to the request header and its retry number to the active provider policy; TUI tests render finite and infinite limits. ## Consequences diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md index d3ef90ec73..8024ea7073 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md @@ -36,7 +36,7 @@ providers: 监听器从失败步骤关闭时生效的持久 `request/header` 读取提供方，后续恢复产生的改动不参与选择，但绝不会从可变的提供方注册表重新解析策略。normal 模式保留有界瞬态错误处理行为：它重试配置的错误代码，次数不超过 `maxRetries`；在当前连续失败序列中，同一提供方策略安排的重试都计入次数；其他情况委托后续处理。 -always 模式先请求下游恢复，使上下文溢出压缩（compaction）之类的专用策略有机会取得进展。下游若决定重试，则以该决定为准。下游若决定失败或恢复过程抛出错误，则回退为无界重试同一提供方请求；抛出的错误会写入日志。成功、轮次取消和插件 dispose（资源释放）是仅有的终止路径。 +always 模式先请求下游恢复，使上下文溢出压缩（compaction）之类的专用策略有机会取得进展。下游若决定重试，则以该决定为准。下游若决定失败或恢复过程抛出错误，则回退为无界重试同一提供方请求；抛出的错误会写入日志。重试监听器会持有并排空已委托的恢复，轮次取消或插件 dispose（资源释放）只能在其结束后完成；随后监听器会应用取消，而不会采用迟到的下游决定。成功、轮次取消和插件 dispose 是仅有的终止路径。 两种模式的本地延迟都按指数增长，从 `initialDelayMs` 增至 `maxDelayMs`。`jitterRatio` 用 `[1 - jitterRatio, 1 + jitterRatio]` 区间内的均匀随机样本乘以每次目标值，再应用上限。提供方给出的正数 `Retry-After` 若未超过上限，则保持精确且不加抖动。若提供方延迟超过上限，normal 模式会委托后续处理；always 模式则改用配置的本地退避，以维持无限重试保证。 @@ -56,7 +56,7 @@ always 模式先请求下游恢复，使上下文溢出压缩（compaction）之 ## 验证 -适配器测试会在提供方加载时校验嵌套策略，证明注册流程会捕获已配置策略和默认策略，并证明请求进行期间替换路由后仍会保留实际提供服务的策略。单元测试与真实 Loader 组合测试根据失败请求实际使用的注册项选择策略、验证 always 模式可越过 normal 预算、固定抖动和延迟上限、证明下游恢复顺序、证明取消会中断停滞的下游恢复，并证明取消与 dispose 会停止正在进行的退避等待。请求级覆盖会比较失败尝试与重试尝试的完整消息，并排除提供方错误文本和丢弃的部分输出。JSONL 与 SQLite 测试会往返读写不含 `Infinity` 的 always 事件；不变式测试会将事件中的提供方绑定到请求头，并将重试编号绑定到活跃的提供方策略；ACP 与 TUI 测试会分别渲染有限和无限上限。 +适配器测试会在提供方加载时校验嵌套策略，证明注册流程会捕获已配置策略和默认策略，并证明请求进行期间替换路由后仍会保留实际提供服务的策略。单元测试与真实 Loader 组合测试根据失败请求实际使用的注册项选择策略、验证 always 模式可越过 normal 预算、固定抖动和延迟上限、证明下游恢复顺序、证明取消与 dispose 会在达到静止状态前排空已委托的恢复，并证明二者都会停止正在进行的退避等待。请求级覆盖会比较失败尝试与重试尝试的完整消息，并排除提供方错误文本和丢弃的部分输出。JSONL 与 SQLite 测试会往返读写不含 `Infinity` 的 always 事件；不变式测试会将事件中的提供方绑定到请求头，并将重试编号绑定到活跃的提供方策略；TUI 测试会渲染有限和无限上限。 ## 后果 diff --git a/packages/examples/acp-demo/README.md b/packages/examples/acp-demo/README.md index ca101fb675..c5803e4f1b 100644 --- a/packages/examples/acp-demo/README.md +++ b/packages/examples/acp-demo/README.md @@ -33,7 +33,6 @@ The app does not install commands, user interaction, session navigation, configu | `toolBash` | owner defaults | Model-facing bash tool config. | | `toolTasks` | owner defaults | Generic background-task control config, or `false`. | | `goals` | owner defaults | Persisted same-session goal domain and model tools, or `false`. | -| `llmRetry` | owner defaults | Provider-owned normal or unbounded model-request retry policy. | The shipped [`examples/acp-agent/cordis.yml`](../../../examples/acp-agent/cordis.yml) adds the DeepSeek adapter, sandboxed bash and filesystem providers, one-shot approval policy, compaction, subagents, workflows, hooks, and model-facing tools. Snapshot overlays replace only nondeterministic providers or policy values. diff --git a/packages/llm/llm-retry/README.md b/packages/llm/llm-retry/README.md index 72d7ac6831..452b395d4d 100644 --- a/packages/llm/llm-retry/README.md +++ b/packages/llm/llm-retry/README.md @@ -2,11 +2,11 @@ Function plugin that applies exact-provider retry policy on the agent loop's closed-step recovery seam. It does not wrap `ctx.llm.stream()`: every adapter call remains one provider attempt, and every retry opens a fresh numbered step. -Each provider adapter owns an optional nested `retryPolicy`, captured when its route registers on `ctx.llm` and carried with each call that reaches that registration's final adapter boundary. An in-flight failure retains that serving policy if the route is later disposed or replaced; a failure before any final adapter is selected has no provider policy and delegates. Omission uses normal mode: two retries for `RATE_LIMIT`, `SERVER`, `TIMEOUT`, and `TRANSPORT`. A normal policy can change its finite budget, eligible codes, and backoff. Always mode asks downstream recovery first, then retries every model-request failure without an attempt limit; success, cancellation, or plugin disposal stops it. +Each provider adapter owns an optional nested `retryPolicy`, captured when its route registers on `ctx.llm` and carried with each call that reaches that registration's final adapter boundary. An in-flight failure retains that serving policy if the route is later disposed or replaced; a failure before any final adapter is selected has no provider policy and delegates. Omission uses normal mode: two retries for `RATE_LIMIT`, `SERVER`, `TIMEOUT`, and `TRANSPORT`. A normal policy can change its finite budget, eligible codes, and backoff. Always mode asks downstream recovery first, then retries every model-request failure without an attempt limit; success, cancellation, or plugin disposal stops it after active delegated recovery reaches quiescence. Both modes use bounded exponential backoff with symmetric jitter. A valid `providerRetryAfterMs` at or below `maxDelayMs` replaces local backoff without jitter. An over-cap provider delay makes normal mode delegate, while always mode uses its configured local backoff so it cannot terminate on that instruction. -Before waiting, the plugin appends a non-surface `llm/retry` event with the provider, mode, failure, and scheduled delay. Normal events include the finite maximum; always events omit it, and UIs render `∞`. Cancellation and plugin disposal abort the wait; disposal drains active backoffs, and a callback captured before disposal fails closed. +Before waiting, the plugin appends a non-surface `llm/retry` event with the provider, mode, failure, and scheduled delay. Normal events include the finite maximum; always events omit it, and UIs render `∞`. Cancellation and plugin disposal abort active backoff, drain active delegated recovery before applying the abort, and make a callback captured before disposal fail closed. The separately published `./invariant` companion checks that every retry record names the current open turn and latest closed step, matches the failed request's durable provider, has a unique step record and correct provider-policy retry number, and carries a valid mode-specific budget and bounded timer delay. Full jitter may schedule zero milliseconds at its lower boundary. @@ -46,5 +46,5 @@ The reconstructed request preserves the prior prefix and is eligible for provide - **Agent steps are the only retry boundary** — direct `ctx.llm.stream()` consumers remain single-attempt because a raw stream cannot separate already-emitted chunks durably. - **Always mode retries permanent failures** — authentication, quota, invalid-request, protocol, and unrecoverable context errors continue until success, cancellation, or disposal; deployments own provider-specific cost and latency controls. -- **Recovery policies compose by waterfall order** — always mode accepts a downstream retry before applying its fallback. A later policy that never settles also prevents the fallback from running. +- **Recovery policies compose by waterfall order** — always mode accepts a downstream retry before applying its fallback. A later policy that ignores cancellation and never settles also prevents fallback, turn quiescence, and plugin disposal from completing. - **`llm/retry` records scheduling, not completion** — later step and turn events establish success, exhaustion, or cancellation. diff --git a/packages/llm/llm-retry/src/index.ts b/packages/llm/llm-retry/src/index.ts index 7e9c2d56e6..7de5d4f06b 100644 --- a/packages/llm/llm-retry/src/index.ts +++ b/packages/llm/llm-retry/src/index.ts @@ -63,32 +63,15 @@ export interface RetryInternals { type DownstreamOutcome = | { readonly type: 'decision'; readonly decision: RequestErrorDecision } | { readonly type: 'error'; readonly error: unknown } - | { readonly type: 'aborted' } -function downstreamUntilAbort( +async function settleDownstream( next: () => Promise, - signal: AbortSignal, ): Promise { - if (signal.aborted) return Promise.resolve({ type: 'aborted' }) - return new Promise((resolve) => { - const finish = (outcome: DownstreamOutcome): void => { - signal.removeEventListener('abort', onAbort) - resolve(outcome) - } - const onAbort = (): void => { finish({ type: 'aborted' }) } - signal.addEventListener('abort', onAbort, { once: true }) - let downstream: Promise - try { - downstream = next() - } catch (error: unknown) { - finish({ type: 'error', error }) - return - } - void downstream.then( - (decision) => { finish({ type: 'decision', decision }) }, - (error: unknown) => { finish({ type: 'error', error }) }, - ) - }) + try { + return { type: 'decision', decision: await next() } + } catch (error: unknown) { + return { type: 'error', error } + } } function localDelay(config: ResolvedRetryPolicy, retry: number, random: () => number): number { @@ -125,6 +108,12 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna const lifetime = new AbortController() const active = new Set>() + function track(operation: Promise): Promise { + const tracked = operation.finally(() => active.delete(tracked)) + active.add(tracked) + return tracked + } + async function backoff( agent: Agent, turn: number, @@ -163,7 +152,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna return { action: 'retry' } } - const disposeListener = ctx.on('agent/request-error', async ( + async function recover( agent: Agent, turn: number, step: number, @@ -173,11 +162,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna policy: ResolvedRetryPolicy | undefined, signal: AbortSignal, next: () => Promise, - ) => { - // A waterfall may have captured this callback before its registration was - // removed. Lifetime cancellation must prevent that stale callback from - // entering a downstream policy after disposal. - if (lifetime.signal.aborted) return Promise.resolve({ action: 'fail' }) + ): Promise { if (policy === undefined) return next() // The call-local policy belongs to the registration that served this // failure. Recover only the durable provider identity from the header; @@ -188,11 +173,12 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna throw new Error(`llm-retry: no request provider for closed turn ${turn}/step ${step}`) } if (policy.mode === 'always') { - const downstream = await downstreamUntilAbort( - next, - AbortSignal.any([signal, lifetime.signal]), - ) - if (downstream.type === 'aborted') return { action: 'fail' } + if (signal.aborted || lifetime.signal.aborted) return { action: 'fail' } + const fusedSignal = AbortSignal.any([signal, lifetime.signal]) + // The loop and plugin lifetime stay open until delegated recovery settles. + // An abort then wins before the decision or fallback can mutate later state. + const downstream = await settleDownstream(next) + if (fusedSignal.aborted) return { action: 'fail' } if (downstream.type === 'error') { ctx.logger.warn( `llm-retry: provider "${provider}" always policy ignored a downstream recovery failure: %o`, @@ -232,15 +218,30 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna delayMs = localDelay(policy, retry, random) } - const tracked = backoff(agent, turn, step, failure, provider, policy, retry, delayMs, signal) - .finally(() => active.delete(tracked)) - active.add(tracked) - return tracked + return backoff(agent, turn, step, failure, provider, policy, retry, delayMs, signal) + } + + const disposeListener = ctx.on('agent/request-error', ( + agent: Agent, + turn: number, + step: number, + error: RequestError, + failure: LlmFailure, + priorFailures: readonly LlmFailure[], + policy: ResolvedRetryPolicy | undefined, + signal: AbortSignal, + next: () => Promise, + ) => { + // A waterfall may have captured this callback before its registration was + // removed. Lifetime cancellation must prevent that stale callback from + // entering a downstream policy after disposal. + if (lifetime.signal.aborted) return Promise.resolve({ action: 'fail' }) + return track(recover(agent, turn, step, error, failure, priorFailures, policy, signal, next)) }) ctx.effect(() => async () => { disposeListener() lifetime.abort(new Error('llm-retry plugin disposed')) await Promise.allSettled([...active]) - }, 'llm-retry: abort and drain backoffs') + }, 'llm-retry: abort and drain active recovery') } diff --git a/packages/llm/llm-retry/tests/retry.spec.ts b/packages/llm/llm-retry/tests/retry.spec.ts index 232f4b4459..2533eb045f 100644 --- a/packages/llm/llm-retry/tests/retry.spec.ts +++ b/packages/llm/llm-retry/tests/retry.spec.ts @@ -706,61 +706,80 @@ describe('provider-routed retry policy', () => { expect(vi.getTimerCount()).toBe(0) }) - it('does not make plugin disposal wait for a delegated recovery policy', async () => { + it('drains delegated recovery before completing plugin disposal', async () => { const adapter = new ScriptedAdapter([new LlmError('bad key', 'AUTH')]) const mounted = await harness(adapter, { mock: alwaysConfig() }) context = mounted.ctx - const downstream = Promise.withResolvers() + const release = Promise.withResolvers() const entered = Promise.withResolvers() - context.on('agent/request-error', () => { + const order: string[] = [] + context.on('agent/request-error', async () => { entered.resolve(undefined) - return downstream.promise + await release.promise + order.push('downstream') + return { action: 'retry' } }) const agent = context.agentLoop.create(SessionId('retry-delegated-disposal'), { provider: 'mock', model: 'mock', }) - const idle = waitForIdle(context, agent) + const idle = waitForIdle(context, agent).then(() => { order.push('idle') }) agent.followup([{ type: 'text', text: 'go' }]) await entered.promise - const disposing = mounted.retryFiber.dispose() + const disposing = mounted.retryFiber.dispose().then(() => { order.push('disposed') }) let timer: ReturnType | undefined const outcome = await Promise.race([ disposing.then(() => 'disposed' as const), new Promise<'blocked'>((resolve) => { timer = setTimeout(() => { resolve('blocked') }, 100) }), ]) if (timer !== undefined) clearTimeout(timer) - downstream.resolve({ action: 'fail' }) + expect(outcome).toBe('blocked') + + release.resolve(undefined) await disposing await idle - expect(outcome).toBe('disposed') + expect(order[0]).toBe('downstream') + expect(order).toEqual(expect.arrayContaining(['disposed', 'idle'])) expect(adapter.requests).toHaveLength(1) + expect(agent.session.events.some(event => event.type === 'llm/retry')).toBe(false) }) - it('lets turn cancellation interrupt a delegated recovery policy', async () => { + it('drains delegated recovery before turn cancellation reaches idle', async () => { const adapter = new ScriptedAdapter([new LlmError('bad key', 'AUTH')]) const mounted = await harness(adapter, { mock: alwaysConfig() }) context = mounted.ctx const downstream = Promise.withResolvers() const entered = Promise.withResolvers() - context.on('agent/request-error', () => { + const order: string[] = [] + context.on('agent/request-error', async () => { entered.resolve(undefined) - return downstream.promise + const decision = await downstream.promise + order.push('downstream') + return decision }) const agent = context.agentLoop.create(SessionId('retry-delegated-cancel'), { provider: 'mock', model: 'mock', }) - const idle = waitForIdle(context, agent) + const idle = waitForIdle(context, agent).then(() => { order.push('idle') }) agent.followup([{ type: 'text', text: 'go' }]) await entered.promise agent.cancel({ kind: 'user' }) - await idle - downstream.resolve({ action: 'fail' }) + let timer: ReturnType | undefined + const outcome = await Promise.race([ + idle.then(() => 'idle' as const), + new Promise<'blocked'>((resolve) => { timer = setTimeout(() => { resolve('blocked') }, 100) }), + ]) + if (timer !== undefined) clearTimeout(timer) + expect(outcome).toBe('blocked') + downstream.resolve({ action: 'retry' }) + await idle + + expect(order).toEqual(['downstream', 'idle']) expect(adapter.requests).toHaveLength(1) expect(agent.session.events.at(-1)).toMatchObject({ type: 'turn/end', @@ -773,8 +792,10 @@ describe('provider-routed retry policy', () => { const mounted = await harness(adapter, { mock: alwaysConfig() }) context = mounted.ctx const downstream = Promise.withResolvers() + const entered = Promise.withResolvers() context.on('agent/request-error', (agent) => { agent.cancel({ kind: 'user' }) + entered.resolve(undefined) return downstream.promise }) const agent = context.agentLoop.create(SessionId('retry-delegated-sync-cancel'), { @@ -784,8 +805,17 @@ describe('provider-routed retry policy', () => { const idle = waitForIdle(context, agent) agent.followup([{ type: 'text', text: 'go' }]) + await entered.promise + let timer: ReturnType | undefined + const outcome = await Promise.race([ + idle.then(() => 'idle' as const), + new Promise<'blocked'>((resolve) => { timer = setTimeout(() => { resolve('blocked') }, 100) }), + ]) + if (timer !== undefined) clearTimeout(timer) + expect(outcome).toBe('blocked') + + downstream.resolve({ action: 'retry' }) await idle - downstream.resolve({ action: 'fail' }) expect(adapter.requests).toHaveLength(1) expect(agent.session.events.at(-1)).toMatchObject({ From efc725b7e4b44a7fa2198db8fbfea90b896ada61 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 25 Jul 2026 15:38:58 +0800 Subject: [PATCH 08/11] fix(llm): isolate retry policy histories --- ...2026-06-21-bounded-llm-request-recovery.md | 4 +- ...26-07-24-provider-retry-policies.i18n.yaml | 4 +- .../2026-07-24-provider-retry-policies.md | 6 +- .../2026-07-24-provider-retry-policies.zh.md | 6 +- docs/config-catalog.md | 10 +- docs/persistence-catalog.md | 4 +- .../headless-agent/retry.cordis.snapshot.yml | 12 ++ .../tests/fixtures/retry-snapshot-backend.mjs | 53 +++++++++ .../headless-agent/tests/headless.snapshot.ts | 42 +++++++ .../tests/snapshots/provider-retry/input.json | 8 ++ .../provider-retry/stream-json.expected.jsonl | 17 +++ packages/examples/acp-demo/src/index.ts | 4 + .../examples/acp-demo/tests/acp-agent.spec.ts | 10 ++ .../examples/acp-demo/tests/built-bin.e2e.ts | 14 +++ .../examples/agent-spine-demo/src/index.ts | 5 + .../agent-spine-demo/tests/agent-core.spec.ts | 8 ++ packages/examples/cli-demo/src/index.ts | 4 + .../examples/cli-demo/tests/built-bin.e2e.ts | 33 ++++++ packages/examples/cli-demo/tests/cli.spec.ts | 10 ++ packages/examples/tui-demo/src/index.ts | 4 + .../examples/tui-demo/tests/tui-agent.spec.ts | 12 +- packages/llm/llm-retry/README.md | 4 +- packages/llm/llm-retry/src/index.ts | 11 +- packages/llm/llm-retry/src/invariant.ts | 26 ++++- packages/llm/llm-retry/src/policy-key.ts | 100 +++++++++++++++++ .../llm/llm-retry/tests/invariant.spec.ts | 100 ++++++++++++++++- .../llm/llm-retry/tests/persistence.spec.ts | 1 + .../llm/llm-retry/tests/policy-key.spec.ts | 71 ++++++++++++ packages/llm/llm-retry/tests/retry.spec.ts | 105 ++++++++++++++++++ packages/ui/tui/tests/tui.snapshot.ts | 2 + packages/ui/tui/tests/tui.spec.ts | 4 + 31 files changed, 666 insertions(+), 28 deletions(-) create mode 100644 examples/headless-agent/retry.cordis.snapshot.yml create mode 100644 examples/headless-agent/tests/fixtures/retry-snapshot-backend.mjs create mode 100644 examples/headless-agent/tests/snapshots/provider-retry/input.json create mode 100644 examples/headless-agent/tests/snapshots/provider-retry/stream-json.expected.jsonl create mode 100644 packages/llm/llm-retry/src/policy-key.ts create mode 100644 packages/llm/llm-retry/tests/policy-key.spec.ts diff --git a/.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md b/.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md index 1e0ef18f9c..268ade3cfc 100644 --- a/.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md +++ b/.agents/notes/implemented/architecture/2026-06-21-bounded-llm-request-recovery.md @@ -56,9 +56,9 @@ The [provider-policy decision](../feature/2026-07-24-provider-retry-policies.md) For an eligible failure with budget remaining, the one-based transient retry count uses bounded exponential backoff. A valid `providerRetryAfterMs` replaces exponential backoff only when it does not exceed `maxDelayMs`; a longer provider delay causes delegation instead of an earlier retry that violates the provider instruction. Local backoff multiplies by an injected random factor in `[1 - jitterRatio, 1 + jitterRatio]` and clamps the final value to `maxDelayMs`; provider delay is not jittered. -The plugin owns a lifetime `AbortController` and tracks every active backoff callback. Each wait fuses the waterfall's turn signal with that lifetime signal. Effect cleanup first unregisters the listener, then aborts and awaits the active callbacks; a captured callback whose lifetime signal aborts returns `fail` and can neither retry nor enter the rest of its captured waterfall after disposal. This makes HMR disposal quiescent even though Cordis has already captured the listener. +The plugin owns a lifetime `AbortController` and tracks every active recovery callback, including delegated waterfall work and backoff. Effect cleanup first unregisters the listener, then aborts and awaits the active callbacks; abort wins over a late delegated retry decision, and a captured callback can neither retry nor enter the rest of its waterfall after disposal. This makes HMR disposal quiescent even though Cordis has already captured the listener. -Before sleeping, `dsh-llm-retry` appends one non-surface `llm/retry` session event containing the turn, failed step, provider, policy mode, provider-policy retry number, mode-specific finite maximum when present, scheduled delay, and `LlmFailure`. The plugin owns the `SessionEventMap` augmentation; `dsh-session` remains generic persistence and does not absorb the optional policy's vocabulary. The event says what was scheduled, not that the next request completed; cancellation during the delay is subsequently visible on `turn/end`. The event ships only with a production renderer and replay/snapshot coverage, because its purpose is operational state rather than trace collection. +Before sleeping, `dsh-llm-retry` appends one non-surface `llm/retry` session event containing the turn, failed step, provider, policy mode, complete resolved-policy key, provider-policy retry number, mode-specific finite maximum when present, scheduled delay, and `LlmFailure`. The key sorts the code set and separates retry histories when a provider route is replaced by a behaviorally different same-mode policy. The plugin owns the `SessionEventMap` augmentation; `dsh-session` remains generic persistence and does not absorb the optional policy's vocabulary. The event says what was scheduled, not that the next request completed; cancellation during the delay is subsequently visible on `turn/end`. The event ships only with a production renderer and replay/snapshot coverage, because its purpose is operational state rather than trace collection. The listener calls `next()` for a non-transient code, an exhausted policy budget, or an over-cap provider delay. This preserves composition with context-overflow recovery and later policy plugins. It returns `{ action: 'retry' }` only after the delay completes under both signals; turn cancellation and plugin disposal return `fail`, after which the loop's cancellation/disposal checks remain authoritative. diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml index 6d41bf0ab7..792625b86a 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-24-provider-retry-policies.md: 3799dbee9658c883f4084ac9c792d4552cc8e2fb -2026-07-24-provider-retry-policies.zh.md: 8024ea707343686d2d1b1351cd7c81ecb8767b85 +2026-07-24-provider-retry-policies.md: 06d21d10d31cc277e998ff6006bb01f10fe0b6b9 +2026-07-24-provider-retry-policies.zh.md: d8351bd94d444b116483c81dc70bf0f6768dc37d diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md index 3799dbee96..06d21d10d3 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md @@ -34,13 +34,13 @@ providers: jitterRatio: 0.2 ``` -The listener reads the provider from the durable `request/header` in force when the failed step closed, excluding later recovery mutations, but never re-resolves policy from the mutable provider registry. Normal mode retains the bounded transient behavior: it retries configured codes up to `maxRetries`, counts retries scheduled by the same provider policy in the current consecutive failure sequence, and otherwise delegates. +The listener reads the provider from the durable `request/header` in force when the failed step closed, excluding later recovery mutations, but never re-resolves policy from the mutable provider registry. It derives a canonical key from every field of the resolved serving policy, sorting `retryableCodes` because eligibility uses set membership, and continues retry history only for the same provider and key. Replacing a route with different limits, code membership, or backoff therefore starts a new count and initial delay even when the mode is unchanged. Normal mode retains the bounded transient behavior: it retries configured codes up to `maxRetries` and otherwise delegates. Always mode asks downstream recovery first so a specialized policy such as context-overflow compaction can make progress. A downstream retry wins. A downstream failure decision or thrown recovery error falls back to an unbounded retry of the same provider request; the thrown error is logged. The retry listener owns and drains delegated recovery before cancellation or plugin disposal can finish, then applies the abort instead of a late downstream decision. Success, turn cancellation, and plugin disposal are the only termination paths. Both modes use exponential local delays from `initialDelayMs` to `maxDelayMs`. `jitterRatio` multiplies each target by a uniform sample in `[1 - jitterRatio, 1 + jitterRatio]`, then applies the cap. A positive provider `Retry-After` within the cap remains exact and unjittered. An over-cap provider delay makes normal mode delegate; always mode retains its guarantee by using the configured local backoff. -Each scheduled retry appends a non-surface `llm/retry` event with the failed provider, policy mode, provider-policy retry number, delay, and failure facts. Normal events carry finite `maxRetries`; always events omit it, and UIs render the limit as `∞`. The event and failed `assistant/chunk` records do not contribute surface messages, so the next request contains the same derived context as the failed request unless another recovery policy deliberately changes the surface. +Each scheduled retry appends a non-surface `llm/retry` event with the failed provider, policy mode, canonical resolved-policy key, provider-policy retry number, delay, and failure facts. Normal events carry finite `maxRetries`; always events omit it, and UIs render the limit as `∞`. The event and failed `assistant/chunk` records do not contribute surface messages, so the next request contains the same derived context as the failed request unless another recovery policy deliberately changes the surface. ## Alternatives considered @@ -56,7 +56,7 @@ Each scheduled retry appends a non-surface `llm/retry` event with the failed pro ## Verification -Adapter tests validate nested policies at provider load, prove registration captures configured and default policies, and retain the serving policy across in-flight route replacement. Unit and real-Loader composition tests select policies from the failed request's serving registration, exercise always mode beyond the normal budget, pin jitter and delay caps, prove downstream recovery ordering, prove cancellation and disposal drain delegated recovery before reaching quiescence, and prove both abort active backoff waits. Request-level coverage compares the complete messages of failed and retried attempts and rejects both provider error text and discarded partial output. JSONL and SQLite tests round-trip an always event without `Infinity`; invariant tests bind its provider to the request header and its retry number to the active provider policy; TUI tests render finite and infinite limits. +Adapter tests validate nested policies at provider load, prove registration captures configured and default policies, and retain the serving policy across in-flight route replacement. Unit and plugin-validation tests select policies from the failed request's serving registration, reject top-level `llmRetry` at the spine, CLI, TUI, and ACP schemas, separate different same-mode policies while preserving histories across reordered code sets, exercise always mode beyond the normal budget, pin jitter and delay caps, prove downstream recovery ordering, prove cancellation and disposal drain delegated recovery before reaching quiescence, and prove both abort active backoff waits. Published Loader fixtures reject the invalid app-level key in CLI and ACP and the invalid bundle-level key when loading the spine directly. Request-level coverage compares the complete messages of failed and retried attempts and rejects both provider error text and discarded partial output. A keyless headless `stream-json` snapshot runs failure, retry, and success through the assembled app, pins the complete `llm/retry` record, and rejects any model-message change between attempts. JSONL and SQLite tests round-trip an always event without `Infinity`; invariant tests validate the canonical policy tuple, bind its provider to the request header, bind its failure code and delay to the encoded policy, and bind its retry number to the active provider policy key; TUI tests render finite and infinite limits. ## Consequences diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md index 8024ea7073..d8351bd94d 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md @@ -34,13 +34,13 @@ providers: jitterRatio: 0.2 ``` -监听器从失败步骤关闭时生效的持久 `request/header` 读取提供方，后续恢复产生的改动不参与选择，但绝不会从可变的提供方注册表重新解析策略。normal 模式保留有界瞬态错误处理行为：它重试配置的错误代码，次数不超过 `maxRetries`；在当前连续失败序列中，同一提供方策略安排的重试都计入次数；其他情况委托后续处理。 +监听器从失败步骤关闭时生效的持久 `request/header` 读取提供方，后续恢复产生的改动不参与选择，但绝不会从可变的提供方注册表重新解析策略。它会根据已解析实际服务策略的所有字段生成规范键；由于错误资格按集合成员判断，生成时会对 `retryableCodes` 排序。重试历史只会对同一提供方和同一规范键延续。因此，即使模式未变，只要路由替换后的次数上限、错误代码成员或退避不同，重试计数与初始延迟都会重新开始。normal 模式保留有界瞬态错误处理行为：它重试配置的错误代码，次数不超过 `maxRetries`；其他情况委托后续处理。 always 模式先请求下游恢复，使上下文溢出压缩（compaction）之类的专用策略有机会取得进展。下游若决定重试，则以该决定为准。下游若决定失败或恢复过程抛出错误，则回退为无界重试同一提供方请求；抛出的错误会写入日志。重试监听器会持有并排空已委托的恢复，轮次取消或插件 dispose（资源释放）只能在其结束后完成；随后监听器会应用取消，而不会采用迟到的下游决定。成功、轮次取消和插件 dispose 是仅有的终止路径。 两种模式的本地延迟都按指数增长，从 `initialDelayMs` 增至 `maxDelayMs`。`jitterRatio` 用 `[1 - jitterRatio, 1 + jitterRatio]` 区间内的均匀随机样本乘以每次目标值，再应用上限。提供方给出的正数 `Retry-After` 若未超过上限，则保持精确且不加抖动。若提供方延迟超过上限，normal 模式会委托后续处理；always 模式则改用配置的本地退避，以维持无限重试保证。 -每次安排重试都会追加一条不进入表层的 `llm/retry` 事件，其中包含失败的提供方、策略模式、提供方策略内的重试编号、延迟和失败事实。normal 事件包含有限的 `maxRetries`；always 事件省略该字段，UI 将上限渲染为 `∞`。该事件与失败的 `assistant/chunk` 记录都不会生成表层消息，因此除非其他恢复策略有意改变表层，否则下一次请求包含的派生上下文与失败请求相同。 +每次安排重试都会追加一条不进入表层的 `llm/retry` 事件，其中包含失败的提供方、策略模式、已解析策略的规范键、提供方策略内的重试编号、延迟和失败事实。normal 事件包含有限的 `maxRetries`；always 事件省略该字段，UI 将上限渲染为 `∞`。该事件与失败的 `assistant/chunk` 记录都不会生成表层消息，因此除非其他恢复策略有意改变表层，否则下一次请求包含的派生上下文与失败请求相同。 ## 曾考虑的替代方案 @@ -56,7 +56,7 @@ always 模式先请求下游恢复，使上下文溢出压缩（compaction）之 ## 验证 -适配器测试会在提供方加载时校验嵌套策略，证明注册流程会捕获已配置策略和默认策略，并证明请求进行期间替换路由后仍会保留实际提供服务的策略。单元测试与真实 Loader 组合测试根据失败请求实际使用的注册项选择策略、验证 always 模式可越过 normal 预算、固定抖动和延迟上限、证明下游恢复顺序、证明取消与 dispose 会在达到静止状态前排空已委托的恢复，并证明二者都会停止正在进行的退避等待。请求级覆盖会比较失败尝试与重试尝试的完整消息，并排除提供方错误文本和丢弃的部分输出。JSONL 与 SQLite 测试会往返读写不含 `Infinity` 的 always 事件；不变式测试会将事件中的提供方绑定到请求头，并将重试编号绑定到活跃的提供方策略；TUI 测试会渲染有限和无限上限。 +适配器测试会在提供方加载时校验嵌套策略，证明注册流程会捕获已配置策略和默认策略，并证明请求进行期间替换路由后仍会保留实际提供服务的策略。单元测试与插件校验测试根据失败请求实际使用的注册项选择策略、在主干、CLI、TUI 与 ACP schema 拒绝顶层 `llmRetry`、分离模式相同但策略不同的替换路由历史，同时在错误代码集合仅顺序不同时延续历史、验证 always 模式可越过 normal 预算、固定抖动和延迟上限、证明下游恢复顺序、证明取消与 dispose 会先排空已委托的恢复再达到完全停稳，并证明二者都会停止正在进行的退避等待。发布版 Loader fixture（测试前置数据）会在 CLI 与 ACP 中拒绝无效的应用级配置键，并在直接加载主干时拒绝无效的 bundle 级配置键。请求级覆盖会比较失败尝试与重试尝试的完整消息，并排除提供方错误文本和丢弃的部分输出。一个无密钥 headless `stream-json` 快照会通过组装后的应用执行失败、重试与成功流程，固定完整的 `llm/retry` 记录，并拒绝各次尝试之间出现任何模型消息变化。JSONL 与 SQLite 测试会往返读写不含 `Infinity` 的 always 事件；不变式测试会校验规范策略元组、将事件中的提供方绑定到请求头、将失败代码与延迟绑定到编码后的策略，并将重试编号绑定到活跃的提供方策略键；TUI 测试会渲染有限和无限上限。 ## 后果 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index bc2028abac..1aeee0570e 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -73,6 +73,8 @@ export interface Config { toolTasks?: NonNullable /** Persisted same-session goals; owner defaults enable them, or false disables the stack and tools. */ goals?: agentCore.GoalConfig | false + /** Invalid at app level; configure `retryPolicy` under each provider. */ + llmRetry?: never } ``` @@ -160,6 +162,8 @@ export interface Config { invariants?: InvariantConfig /** Opt-in persisted same-session goal stack; set false or omit to leave it unmounted. */ goals?: GoalConfig | false + /** Invalid at bundle level; configure `retryPolicy` under each provider. */ + llmRetry?: never } /** Skill bundle config forwarded to the registry, local provider, and model-facing consumer. */ @@ -261,6 +265,8 @@ export interface Config { toolTasks?: NonNullable /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ workspaceContext: agentCore.Config['workspaceContext'] + /** Invalid at app level; configure `retryPolicy` under each provider. */ + llmRetry?: never } ``` @@ -696,7 +702,7 @@ Requires: `agents` export type Config = Readonly> ``` -Source: [`packages/llm/llm-retry/src/index.ts:43`](../packages/llm/llm-retry/src/index.ts) +Source: [`packages/llm/llm-retry/src/index.ts:46`](../packages/llm/llm-retry/src/index.ts) ## `@deepseek-ai/dsh-lsp-local` @@ -1794,6 +1800,8 @@ export interface Config { resumeSessionId?: string /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ workspaceContext: agentCore.Config['workspaceContext'] + /** Invalid at app level; configure `retryPolicy` under each provider. */ + llmRetry?: never } ``` diff --git a/docs/persistence-catalog.md b/docs/persistence-catalog.md index 6f74a5ee36..8cd8c78110 100644 --- a/docs/persistence-catalog.md +++ b/docs/persistence-catalog.md @@ -277,6 +277,7 @@ Source: [`packages/hooks/hook-protocol/src/types.ts:31`](../packages/hooks/hook- step: number provider: string mode: 'normal' + policyKey: string retry: number maxRetries: number delayMs: number @@ -286,13 +287,14 @@ Source: [`packages/hooks/hook-protocol/src/types.ts:31`](../packages/hooks/hook- step: number provider: string mode: 'always' + policyKey: string retry: number delayMs: number failure: LlmFailure } ``` -Source: [`packages/llm/llm-retry/src/index.ts:18`](../packages/llm/llm-retry/src/index.ts) +Source: [`packages/llm/llm-retry/src/index.ts:19`](../packages/llm/llm-retry/src/index.ts) ### `permission/*` diff --git a/examples/headless-agent/retry.cordis.snapshot.yml b/examples/headless-agent/retry.cordis.snapshot.yml new file mode 100644 index 0000000000..b8fc79d17f --- /dev/null +++ b/examples/headless-agent/retry.cordis.snapshot.yml @@ -0,0 +1,12 @@ +# Keyless provider-retry composition for the headless stream-json snapshot. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - id: llm-deepseek + name: '@deepseek-ai/dsh-llm-deepseek' + disabled: true + - insert: + - id: retry-snapshot-backend + name: './tests/fixtures/retry-snapshot-backend.mjs' diff --git a/examples/headless-agent/tests/fixtures/retry-snapshot-backend.mjs b/examples/headless-agent/tests/fixtures/retry-snapshot-backend.mjs new file mode 100644 index 0000000000..28dc4f5742 --- /dev/null +++ b/examples/headless-agent/tests/fixtures/retry-snapshot-backend.mjs @@ -0,0 +1,53 @@ +/** Deterministic provider adapter for the headless retry-policy snapshot. */ + +import { + LlmAdapter, + LlmError, + resolveRetryPolicy, +} from '@deepseek-ai/dsh-llm' + +class RetrySnapshotAdapter extends LlmAdapter { + requests = 0 + firstMessages + policy = resolveRetryPolicy({ + mode: 'normal', + maxRetries: 1, + retryableCodes: ['RATE_LIMIT'], + backoff: { initialDelayMs: 1, maxDelayMs: 1, jitterRatio: 0 }, + }, 'retry-snapshot-backend.retryPolicy') + + providerRetryPolicy() { + return this.policy + } + + async * stream(options) { + const messages = JSON.stringify(options.messages) + this.requests++ + if (this.requests === 1) { + this.firstMessages = messages + throw new LlmError('snapshot transient failure', 'RATE_LIMIT', { status: 429 }) + } + if (this.requests === 2 && messages !== this.firstMessages) { + throw new Error('retry snapshot changed the model-visible messages') + } + const text = 'RETRY_OK' + yield { type: 'block-start', index: 0, blockType: 'text' } + yield { type: 'text-delta', index: 0, text } + yield { type: 'block-end', index: 0, block: { type: 'text', text } } + yield { type: 'usage', usage: { inputTokens: 4, outputTokens: 2 } } + yield { type: 'finish', reason: { kind: 'stop' } } + } +} + +/** Cordis plugin name. */ +export const name = 'retry-snapshot-backend' +/** Required LLM registry service. */ +export const inject = ['llm'] + +/** + * Register the deterministic provider adapter. + * @param {import('cordis').Context} ctx - plugin context carrying the LLM service. + */ +export function apply(ctx) { + ctx.llm.registerAdapter(['deepseek'], new RetrySnapshotAdapter()) +} diff --git a/examples/headless-agent/tests/headless.snapshot.ts b/examples/headless-agent/tests/headless.snapshot.ts index 6852fed20b..61405d98ef 100644 --- a/examples/headless-agent/tests/headless.snapshot.ts +++ b/examples/headless-agent/tests/headless.snapshot.ts @@ -24,6 +24,8 @@ const ptyStreamExpected = join(ptyScenarioDir, 'stream-json.expected.jsonl') const ptyConfigPath = fileURLToPath(new URL('../pty.cordis.snapshot.yml', import.meta.url)) const goalScenarioDir = join(snapshotsDir, 'goal-tools') const goalConfigPath = fileURLToPath(new URL('../goal.cordis.snapshot.yml', import.meta.url)) +const retryScenarioDir = join(snapshotsDir, 'provider-retry') +const retryConfigPath = fileURLToPath(new URL('../retry.cordis.snapshot.yml', import.meta.url)) const ralphScenarioDir = join(snapshotsDir, 'ralph-loop') const ralphConfigPath = fileURLToPath(new URL('../ralph.cordis.snapshot.yml', import.meta.url)) const binScript = fileURLToPath(new URL('../../../packages/examples/cli-demo/src/bin.ts', import.meta.url)) @@ -124,6 +126,46 @@ async function persistedLogs(cwd: string): Promise { } describe('headless stream-json snapshots', () => { + it('retries a transient provider failure through the one-shot app', async () => { + const prompt = await scenarioPrompt(retryScenarioDir, 'provider-retry') + const streamExpected = join(retryScenarioDir, 'stream-json.expected.jsonl') + let runCwd = '' + const result = await runLoaderSmoke({ + label: 'provider retry headless stream-json snapshot', + tempDirPrefix: 'headless-snapshot-provider-retry-', + binScript, + configPath: retryConfigPath, + binArgs: ['--config', retryConfigPath, '--output-format', 'stream-json', prompt], + tsconfigPath, + env: { + DSH_SNAPSHOT: 'replay', + NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '), + }, + prepare: (cwd) => { runCwd = cwd }, + inspect: async (cwd) => { + const logs = await persistedLogs(cwd) + expect(logs).toHaveLength(1) + const records = parseJsonl(logs[0]?.content ?? '') + const retries = records.filter(record => record.type === 'llm/retry') + expect(retries).toHaveLength(1) + expect(retries[0]?.data).toMatchObject({ + provider: 'deepseek', + mode: 'normal', + policyKey: '["normal",1,["RATE_LIMIT"],1,1,0]', + retry: 1, + maxRetries: 1, + delayMs: 1, + failure: { message: 'snapshot transient failure', code: 'RATE_LIMIT', status: 429 }, + }) + }, + }) + + expect(result.stderr).toBe('') + const normalized = normalizeHeadlessStream(result.stdout, runCwd) + if (refreshing) await writeFile(streamExpected, normalized) + expect(normalized).toBe(await readFile(streamExpected, 'utf8')) + }, LOADER_SMOKE_TEST_TIMEOUT_MS) + it('replays the advanced toolchain through the one-shot app', async () => { const prompt = await scenarioPrompt(advancedScenarioDir, 'advanced-toolchain') const fixtureFiles = [ diff --git a/examples/headless-agent/tests/snapshots/provider-retry/input.json b/examples/headless-agent/tests/snapshots/provider-retry/input.json new file mode 100644 index 0000000000..2dc62032a6 --- /dev/null +++ b/examples/headless-agent/tests/snapshots/provider-retry/input.json @@ -0,0 +1,8 @@ +{ + "steps": [ + { + "op": "prompt", + "text": "retry the transient provider failure" + } + ] +} diff --git a/examples/headless-agent/tests/snapshots/provider-retry/stream-json.expected.jsonl b/examples/headless-agent/tests/snapshots/provider-retry/stream-json.expected.jsonl new file mode 100644 index 0000000000..a686a03ec3 --- /dev/null +++ b/examples/headless-agent/tests/snapshots/provider-retry/stream-json.expected.jsonl @@ -0,0 +1,17 @@ +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/start","seq":0,"time":0,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"user/message","seq":1,"time":0,"data":{"content":[{"type":"text","text":"retry the transient provider failure"}],"source":{"kind":"user"}},"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"session/title","seq":2,"time":0,"data":{"title":"retry the transient provider failure","messageSeqs":[1],"source":{"kind":"fallback"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":3,"time":0,"data":{"turn":1,"step":1}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"request/header","seq":4,"time":0,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":5,"time":0,"data":{"turn":1,"step":1}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"llm/retry","seq":6,"time":0,"data":{"turn":1,"step":1,"provider":"deepseek","mode":"normal","policyKey":"[\"normal\",1,[\"RATE_LIMIT\"],1,1,0]","retry":1,"maxRetries":1,"delayMs":1,"failure":{"message":"snapshot transient failure","code":"RATE_LIMIT","status":429}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":7,"time":0,"data":{"turn":1,"step":2}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":9,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":0,"text":"RETRY_OK"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":10,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"RETRY_OK"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":11,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":4,"outputTokens":2}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":12,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":13,"time":0,"data":{"turn":1,"step":2,"content":[{"type":"text","text":"RETRY_OK"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":4,"outputTokens":2}},"sourceEventSeqs":[8,9,10,11,12],"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":14,"time":0,"data":{"turn":1,"step":2}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/end","seq":15,"time":0,"data":{"turn":1,"reason":{"kind":"completed"}}}} +{"type":"result","success":true,"sessionId":"{{sessionId}}","turn":1,"result":"RETRY_OK","reason":{"kind":"completed"},"usage":{"inputTokens":4,"outputTokens":2}} diff --git a/packages/examples/acp-demo/src/index.ts b/packages/examples/acp-demo/src/index.ts index d7161c589f..c4d52d0053 100644 --- a/packages/examples/acp-demo/src/index.ts +++ b/packages/examples/acp-demo/src/index.ts @@ -67,6 +67,8 @@ export interface Config { toolTasks?: NonNullable /** Persisted same-session goals; owner defaults enable them, or false disables the stack and tools. */ goals?: agentCore.GoalConfig | false + /** Invalid at app level; configure `retryPolicy` under each provider. */ + llmRetry?: never } // Each front door owns a complete, directly readable config schema; extracting @@ -92,6 +94,8 @@ export const Config: z = z.object({ toolBash: agentCore.ToolBashConfigSchema, toolTasks: z.union([z.const(false), agentCore.ToolTasksConfigSchema]), goals: z.union([z.const(false), agentCore.GoalConfigSchema]), + // Provider retryPolicy makes a top-level llmRetry invalid. + llmRetry: z.never(), }) /* jscpd:ignore-end */ diff --git a/packages/examples/acp-demo/tests/acp-agent.spec.ts b/packages/examples/acp-demo/tests/acp-agent.spec.ts index e932c7fb17..7c32da8cfa 100644 --- a/packages/examples/acp-demo/tests/acp-agent.spec.ts +++ b/packages/examples/acp-demo/tests/acp-agent.spec.ts @@ -76,6 +76,16 @@ async function withIsolatedSkillHomes(run: () => Promise): Promise { } describe('dsh-acp-demo composition', () => { + it('rejects app-level llmRetry config through plugin validation', async () => { + const ctx = new Context() + await expect(ctx.plugin(acpAgent, { + provider: 'mock', + model: 'mock', + workspaceContext: false, + llmRetry: { maxTransientRetries: 2 }, + } as never)).rejects.toThrow(/llmRetry/) + }) + it('brings up the spine + persistence + the ACP bridge', async () => { const ctx = await mount({ provider: 'mock', diff --git a/packages/examples/acp-demo/tests/built-bin.e2e.ts b/packages/examples/acp-demo/tests/built-bin.e2e.ts index 02e82ac5ac..dd7139815f 100644 --- a/packages/examples/acp-demo/tests/built-bin.e2e.ts +++ b/packages/examples/acp-demo/tests/built-bin.e2e.ts @@ -207,6 +207,20 @@ describe.skipIf(!existsSync(acpBin))('dsh-acp-demo BUILT bin (node lib/bin.js, n expect(code).not.toBe(0) expect(stderr).toContain('config file not found') }, 30_000) + + it('rejects legacy app-level llmRetry through the published Loader path', async () => { + consumer = await makeConsumer() + const configPath = join(consumer, 'cordis.yml') + const config = await readFile(configPath, 'utf8') + await writeFile(configPath, config.replace( + ' workspaceContext: false', + ' workspaceContext: false\n llmRetry:\n maxTransientRetries: 2', + )) + + const { code, stderr } = await runBinExpectingExit('./cordis.yml', consumer) + expect(code).not.toBe(0) + expect(stderr).toContain('llmRetry') + }, 30_000) }) /** Spawn the built acp bin against `configArg` and resolve with its exit code + stderr. */ diff --git a/packages/examples/agent-spine-demo/src/index.ts b/packages/examples/agent-spine-demo/src/index.ts index d2a65844ea..397966813f 100644 --- a/packages/examples/agent-spine-demo/src/index.ts +++ b/packages/examples/agent-spine-demo/src/index.ts @@ -112,6 +112,8 @@ export interface Config { invariants?: InvariantConfig /** Opt-in persisted same-session goal stack; set false or omit to leave it unmounted. */ goals?: GoalConfig | false + /** Invalid at bundle level; configure `retryPolicy` under each provider. */ + llmRetry?: never } /** The skill config schema exported for app packages that forward `skills`. */ @@ -152,6 +154,9 @@ export const Config = z.intersect([ toolTasks: z.union([z.const(false), ToolTasksConfigSchema]), invariants: InvariantService.Config, goals: z.union([z.const(false), GoalConfigSchema]), + // Schemastery preserves unknown object properties. A top-level llmRetry is + // known-but-impossible because provider retryPolicy owns this configuration. + llmRetry: z.never(), }) as unknown as z>, ]) as unknown as z diff --git a/packages/examples/agent-spine-demo/tests/agent-core.spec.ts b/packages/examples/agent-spine-demo/tests/agent-core.spec.ts index 2051aee4b8..33e8ddc6cd 100644 --- a/packages/examples/agent-spine-demo/tests/agent-core.spec.ts +++ b/packages/examples/agent-spine-demo/tests/agent-core.spec.ts @@ -594,6 +594,14 @@ describe('dsh-agent-spine-demo bundle', () => { expect(agentCore.name).toBe('agent-spine-demo') }) + it('rejects bundle-level llmRetry config through plugin validation', async () => { + const ctx = new Context() + await expect(ctx.plugin(agentCore, { + workspaceContext: false, + llmRetry: { maxTransientRetries: 2 }, + } as never)).rejects.toThrow(/llmRetry/) + }) + it('has the namespace-plugin export shape (no stray default) so the Loader keeps name/Config/apply', () => { // A default export would make `unwrapExports` collapse this inject-less namespace and silently // drop `name`/`Config`. Apps import the bundle directly, so this is its Loader-shape guard. diff --git a/packages/examples/cli-demo/src/index.ts b/packages/examples/cli-demo/src/index.ts index bfec7c9a4f..e9bbb0a237 100644 --- a/packages/examples/cli-demo/src/index.ts +++ b/packages/examples/cli-demo/src/index.ts @@ -52,6 +52,8 @@ export interface Config { toolTasks?: NonNullable /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ workspaceContext: agentCore.Config['workspaceContext'] + /** Invalid at app level; configure `retryPolicy` under each provider. */ + llmRetry?: never } // Each front door keeps a complete Loader schema so its deployment contract is @@ -73,6 +75,8 @@ export const Config: z = z.object({ toolBash: agentCore.ToolBashConfigSchema, toolTasks: z.union([z.const(false), agentCore.ToolTasksConfigSchema]), workspaceContext: z.union([z.const(false), workspaceContext.Config]).required(), + // Provider retryPolicy makes a top-level llmRetry invalid. + llmRetry: z.never(), }) /* jscpd:ignore-end */ diff --git a/packages/examples/cli-demo/tests/built-bin.e2e.ts b/packages/examples/cli-demo/tests/built-bin.e2e.ts index 5c3a6ad62e..ddc07a93a1 100644 --- a/packages/examples/cli-demo/tests/built-bin.e2e.ts +++ b/packages/examples/cli-demo/tests/built-bin.e2e.ts @@ -192,6 +192,39 @@ describe.skipIf(!existsSync(cliBin))('dsh-cli-demo BUILT bin', () => { } }, 30_000) + it('rejects legacy app-level llmRetry through the published Loader path', async () => { + consumer = await makeConsumer() + const configPath = join(consumer, 'cordis.yml') + const config = await readFile(configPath, 'utf8') + await writeFile(configPath, config.replace( + ' workspaceContext: false', + ' workspaceContext: false\n llmRetry:\n maxTransientRetries: 2', + )) + + const result = await runBuiltBin(consumer, ['--config', './cordis.yml', 'task']) + expect(result.code).not.toBe(0) + expect(result.stdout).toBe('') + expect(result.stderr).toContain('llmRetry') + }, 30_000) + + it('rejects legacy bundle-level llmRetry when the published spine is loaded directly', async () => { + consumer = await makeConsumer() + await writeFile(join(consumer, 'cordis.yml'), [ + '- id: spine', + " name: '@deepseek-ai/dsh-agent-spine-demo'", + ' config:', + ' workspaceContext: false', + ' llmRetry:', + ' maxTransientRetries: 2', + '', + ].join('\n')) + + const result = await runBuiltBin(consumer, ['--config', './cordis.yml', 'task']) + expect(result.code).not.toBe(0) + expect(result.stdout).toBe('') + expect(result.stderr).toContain('llmRetry') + }, 30_000) + describe.skipIf(process.platform === 'win32')('POSIX signal delivery', () => { it.each([ ['SIGINT', 130], diff --git a/packages/examples/cli-demo/tests/cli.spec.ts b/packages/examples/cli-demo/tests/cli.spec.ts index bf681af2f8..786662caba 100644 --- a/packages/examples/cli-demo/tests/cli.spec.ts +++ b/packages/examples/cli-demo/tests/cli.spec.ts @@ -173,6 +173,16 @@ afterEach(async () => { }) describe('parseCliArgs', () => { + it('rejects app-level llmRetry config through plugin validation', async () => { + const ctx = new Context() + await expect(ctx.plugin(cliDemo, { + provider: 'mock', + model: 'mock', + workspaceContext: false, + llmRetry: { maxTransientRetries: 2 }, + } as never)).rejects.toThrow(/llmRetry/) + }) + it('parses defaults, explicit options, spaces, and an option-like task after --', () => { expect(parseCliArgs(['task with spaces'])).toEqual({ kind: 'run', configPath: './cordis.yml', outputFormat: 'text', task: 'task with spaces', diff --git a/packages/examples/tui-demo/src/index.ts b/packages/examples/tui-demo/src/index.ts index 29f985c8e7..cd38ab3cd2 100644 --- a/packages/examples/tui-demo/src/index.ts +++ b/packages/examples/tui-demo/src/index.ts @@ -82,6 +82,8 @@ export interface Config { resumeSessionId?: string /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ workspaceContext: agentCore.Config['workspaceContext'] + /** Invalid at app level; configure `retryPolicy` under each provider. */ + llmRetry?: never } export const Config: z = z.object({ @@ -106,6 +108,8 @@ export const Config: z = z.object({ goals: z.union([z.const(false), agentCore.GoalConfigSchema]), resumeSessionId: z.string(), workspaceContext: z.union([z.const(false), workspaceContext.Config]).required(), + // Provider retryPolicy makes a top-level llmRetry invalid. + llmRetry: z.never(), }) /* jscpd:ignore-end */ diff --git a/packages/examples/tui-demo/tests/tui-agent.spec.ts b/packages/examples/tui-demo/tests/tui-agent.spec.ts index cdab42289b..37c8609733 100644 --- a/packages/examples/tui-demo/tests/tui-agent.spec.ts +++ b/packages/examples/tui-demo/tests/tui-agent.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { join } from 'node:path' -import type { Context } from 'cordis' +import { Context } from 'cordis' import Loader from '@cordisjs/plugin-loader' import { TOOL_ORDER_REST } from '@deepseek-ai/dsh-system-prompt' import * as tuiAgent from '../src/index.ts' @@ -21,6 +21,16 @@ function recordingContext(): { readonly ctx: Context; readonly calls: PluginCall } describe('dsh-tui-demo app', () => { + it('rejects app-level llmRetry config through plugin validation', async () => { + const ctx = new Context() + await expect(ctx.plugin(tuiAgent, { + provider: 'mock', + model: 'mock', + workspaceContext: false, + llmRetry: { maxTransientRetries: 2 }, + } as never)).rejects.toThrow(/llmRetry/) + }) + it('composes the TUI cluster around one fresh exact session identity', () => { const { ctx, calls } = recordingContext() tuiAgent.composeTuiApp(ctx, { diff --git a/packages/llm/llm-retry/README.md b/packages/llm/llm-retry/README.md index 452b395d4d..e8d2357063 100644 --- a/packages/llm/llm-retry/README.md +++ b/packages/llm/llm-retry/README.md @@ -6,9 +6,9 @@ Each provider adapter owns an optional nested `retryPolicy`, captured when its r Both modes use bounded exponential backoff with symmetric jitter. A valid `providerRetryAfterMs` at or below `maxDelayMs` replaces local backoff without jitter. An over-cap provider delay makes normal mode delegate, while always mode uses its configured local backoff so it cannot terminate on that instruction. -Before waiting, the plugin appends a non-surface `llm/retry` event with the provider, mode, failure, and scheduled delay. Normal events include the finite maximum; always events omit it, and UIs render `∞`. Cancellation and plugin disposal abort active backoff, drain active delegated recovery before applying the abort, and make a callback captured before disposal fail closed. +Before waiting, the plugin appends a non-surface `llm/retry` event with the provider, mode, canonical resolved-policy key, failure, and scheduled delay. The key includes every behavior-affecting field and sorts normal-mode codes because eligibility uses set membership. Retry numbers continue only across events with the same provider and complete policy key, so a route replacement with different limits, code membership, or backoff starts its own history. Normal events include the finite maximum; always events omit it, and UIs render `∞`. Cancellation and plugin disposal abort active backoff, drain active delegated recovery before applying the abort, and make a callback captured before disposal fail closed. -The separately published `./invariant` companion checks that every retry record names the current open turn and latest closed step, matches the failed request's durable provider, has a unique step record and correct provider-policy retry number, and carries a valid mode-specific budget and bounded timer delay. Full jitter may schedule zero milliseconds at its lower boundary. +The separately published `./invariant` companion checks that every retry record names the current open turn and latest closed step, matches the failed request's durable provider, carries a producer-canonical policy key consistent with its mode and finite budget, binds normal failures and every scheduled delay to that policy, has a unique step record and correct provider-policy retry number, and carries a bounded timer delay. Full jitter may schedule zero milliseconds at its lower boundary. ```yaml - name: '@deepseek-ai/dsh-llm-deepseek' diff --git a/packages/llm/llm-retry/src/index.ts b/packages/llm/llm-retry/src/index.ts index 7de5d4f06b..f759697b9e 100644 --- a/packages/llm/llm-retry/src/index.ts +++ b/packages/llm/llm-retry/src/index.ts @@ -11,6 +11,7 @@ import type { Agent, RequestError, RequestErrorDecision } from '@deepseek-ai/dsh import type { LlmFailure, ResolvedRetryPolicy } from '@deepseek-ai/dsh-llm' import type { SessionEvent } from '@deepseek-ai/dsh-session' import { providerForClosedStep } from './history.ts' +import { retryPolicyKey } from './policy-key.ts' declare module '@deepseek-ai/dsh-session' { interface SessionEventMap { @@ -20,6 +21,7 @@ declare module '@deepseek-ai/dsh-session' { step: number provider: string mode: 'normal' + policyKey: string retry: number maxRetries: number delayMs: number @@ -29,6 +31,7 @@ declare module '@deepseek-ai/dsh-session' { step: number provider: string mode: 'always' + policyKey: string retry: number delayMs: number failure: LlmFailure @@ -121,6 +124,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna failure: LlmFailure, provider: string, policy: ResolvedRetryPolicy, + policyKey: string, retry: number, delayMs: number, signal: AbortSignal, @@ -133,6 +137,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna step, provider, mode: policy.mode, + policyKey, retry, maxRetries: policy.maxRetries, delayMs, @@ -143,6 +148,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna step, provider, mode: policy.mode, + policyKey, retry, delayMs, failure, @@ -192,6 +198,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna return next() } + const policyKey = retryPolicyKey(policy) const firstPriorStep = step - priorFailures.length const priorPolicyRetry = agent.session.events.findLast((event): event is SessionEvent<'llm/retry'> => event.type === 'llm/retry' @@ -199,7 +206,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna && event.data.step >= firstPriorStep && event.data.step < step && event.data.provider === provider - && event.data.mode === policy.mode, + && event.data.policyKey === policyKey, ) const previousRetry = priorPolicyRetry?.data.retry ?? 0 if (policy.mode === 'normal' && previousRetry >= policy.maxRetries) return next() @@ -218,7 +225,7 @@ export function apply(ctx: Context, config: Config = {}, internals: RetryInterna delayMs = localDelay(policy, retry, random) } - return backoff(agent, turn, step, failure, provider, policy, retry, delayMs, signal) + return backoff(agent, turn, step, failure, provider, policy, policyKey, retry, delayMs, signal) } const disposeListener = ctx.on('agent/request-error', ( diff --git a/packages/llm/llm-retry/src/invariant.ts b/packages/llm/llm-retry/src/invariant.ts index 0cabbd6365..ddf5e8b975 100644 --- a/packages/llm/llm-retry/src/invariant.ts +++ b/packages/llm/llm-retry/src/invariant.ts @@ -2,9 +2,9 @@ import type { Context } from 'cordis' import type { Session, SessionEvent } from '@deepseek-ai/dsh-session' -import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' import type { InvariantFailure, InvariantInstaller } from '@deepseek-ai/dsh-invariants' import { providerForClosedStep } from './history.ts' +import { parseRetryPolicyKey } from './policy-key.ts' import type {} from './index.ts' const PACKAGE_NAME = '@deepseek-ai/dsh-llm-retry' @@ -20,30 +20,46 @@ function validateRetry( event: SessionEvent<'llm/retry'>, fail: InvariantFailure, ): void { - const { turn, step, provider, mode, retry, delayMs } = event.data + const { turn, step, provider, mode, policyKey, retry, delayMs } = event.data if (!Number.isSafeInteger(retry) || retry < 1) { fail('llm/retry retry must be a positive safe integer') } if (typeof provider !== 'string' || provider.length === 0) { fail('llm/retry provider must be non-empty string') } + const keyedPolicy = parseRetryPolicyKey(policyKey) + if (keyedPolicy === undefined) { + fail('llm/retry policyKey must encode a canonical resolved policy') + } switch (mode) { case 'normal': { const { maxRetries } = event.data if (!Number.isSafeInteger(maxRetries) || maxRetries < 1 || retry > maxRetries) { fail(`llm/retry retry ${retry} must not exceed a positive safe maxRetries ${maxRetries}`) } + if (keyedPolicy.mode !== 'normal') { + fail(`llm/retry mode normal must match policyKey mode ${keyedPolicy.mode}`) + } + if (keyedPolicy.maxRetries !== maxRetries) { + fail(`llm/retry maxRetries ${maxRetries} must match policyKey`) + } + if (!keyedPolicy.retryableCodes.includes(event.data.failure.code)) { + fail(`llm/retry failure code ${event.data.failure.code} must be eligible under policyKey`) + } break } case 'always': + if (keyedPolicy.mode !== 'always') { + fail(`llm/retry mode always must match policyKey mode ${keyedPolicy.mode}`) + } if ('maxRetries' in event.data) fail('llm/retry always mode must omit maxRetries') break default: fail(`llm/retry mode must be normal or always, got ${String(mode)}`) } if (typeof delayMs !== 'number' || !Number.isFinite(delayMs) - || delayMs < 0 || delayMs > MAX_TIMER_DELAY_MS) { - fail(`llm/retry delayMs must be a finite number within 0..${MAX_TIMER_DELAY_MS}`) + || delayMs < 0 || delayMs > keyedPolicy.maxDelayMs) { + fail(`llm/retry delayMs must be a finite number within policyKey range 0..${keyedPolicy.maxDelayMs}`) } const turnStartIndex = history.findLastIndex(prior => @@ -86,7 +102,7 @@ function validateRetry( index > lastSuccessIndex && prior.type === 'llm/retry' && prior.data.provider === provider - && prior.data.mode === mode + && prior.data.policyKey === policyKey )) const expectedRetry = (priorPolicyRetry?.data.retry ?? 0) + 1 if (retry !== expectedRetry) { diff --git a/packages/llm/llm-retry/src/policy-key.ts b/packages/llm/llm-retry/src/policy-key.ts new file mode 100644 index 0000000000..2103d87920 --- /dev/null +++ b/packages/llm/llm-retry/src/policy-key.ts @@ -0,0 +1,100 @@ +/** Canonical durable identity for resolved retry policies. @module @deepseek-ai/dsh-llm-retry/policy-key */ + +import type { ResolvedRetryBackoff, ResolvedRetryPolicy } from '@deepseek-ai/dsh-llm' +import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' + +function parseBackoff( + tuple: readonly unknown[], + offset: number, +): ResolvedRetryBackoff | undefined { + const initialDelayMs = tuple[offset] + const maxDelayMs = tuple[offset + 1] + const jitterRatio = tuple[offset + 2] + if (typeof initialDelayMs !== 'number' || !Number.isFinite(initialDelayMs) + || initialDelayMs <= 0 || initialDelayMs > MAX_TIMER_DELAY_MS + || typeof maxDelayMs !== 'number' || !Number.isFinite(maxDelayMs) + || maxDelayMs <= 0 || maxDelayMs > MAX_TIMER_DELAY_MS + || initialDelayMs > maxDelayMs + || typeof jitterRatio !== 'number' || !Number.isFinite(jitterRatio) + || jitterRatio < 0 || jitterRatio > 1) { + return undefined + } + return { initialDelayMs, maxDelayMs, jitterRatio } +} + +/** + * Derive the canonical durable key for one fully resolved provider policy. + * Retryable-code order is normalized because eligibility uses set membership. + * @param policy - immutable policy captured from the serving registration. + * @returns canonical JSON tuple containing every behavior-affecting field. + */ +export function retryPolicyKey(policy: ResolvedRetryPolicy): string { + if (policy.mode === 'always') { + return JSON.stringify([ + policy.mode, + policy.initialDelayMs, + policy.maxDelayMs, + policy.jitterRatio, + ]) + } + return JSON.stringify([ + policy.mode, + policy.maxRetries, + [...policy.retryableCodes].sort(), + policy.initialDelayMs, + policy.maxDelayMs, + policy.jitterRatio, + ]) +} + +/** + * Parse a producer-canonical policy key from durable input. + * @param value - untrusted persisted event field. + * @returns the resolved policy encoded by the key, or `undefined` for any non-canonical value. + */ +export function parseRetryPolicyKey(value: unknown): ResolvedRetryPolicy | undefined { + if (typeof value !== 'string' || value.length === 0) return undefined + let tuple: unknown + try { + tuple = JSON.parse(value) as unknown + } catch (_invalidPolicyKeyJson) { + return undefined + } + if (!Array.isArray(tuple)) return undefined + const items = tuple as readonly unknown[] + const mode = items[0] + let policy: ResolvedRetryPolicy + switch (mode) { + case 'always': { + if (items.length !== 4) return undefined + const backoff = parseBackoff(items, 1) + if (backoff === undefined) return undefined + policy = Object.freeze({ mode, ...backoff }) + break + } + case 'normal': { + if (items.length !== 6) return undefined + const maxRetries = items[1] + const retryableCodes = items[2] + const backoff = parseBackoff(items, 3) + if (!Number.isSafeInteger(maxRetries) || (maxRetries as number) < 0 + || !Array.isArray(retryableCodes) || retryableCodes.length === 0 + || (retryableCodes as readonly unknown[]) + .some(code => typeof code !== 'string' || code.length === 0) + || new Set(retryableCodes).size !== retryableCodes.length + || backoff === undefined) { + return undefined + } + policy = Object.freeze({ + mode, + maxRetries: maxRetries as number, + retryableCodes: Object.freeze(retryableCodes as string[]), + ...backoff, + }) + break + } + default: + return undefined + } + return retryPolicyKey(policy) === value ? policy : undefined +} diff --git a/packages/llm/llm-retry/tests/invariant.spec.ts b/packages/llm/llm-retry/tests/invariant.spec.ts index aa4030a886..a7fd36993a 100644 --- a/packages/llm/llm-retry/tests/invariant.spec.ts +++ b/packages/llm/llm-retry/tests/invariant.spec.ts @@ -27,7 +27,10 @@ function closeStep(ctx: Context, id: string, turn = 1, step = 1) { } const failure = { message: 'provider busy', code: 'RATE_LIMIT', status: 429 } -const normal = { provider: 'mock', mode: 'normal' as const } +const normalPolicyKey = (maxRetries: number): string => + `["normal",${maxRetries},["RATE_LIMIT"],1,10000,0]` +const alwaysPolicyKey = '["always",1,10000,0]' +const normal = { provider: 'mock', mode: 'normal' as const, policyKey: normalPolicyKey(2) } describe('llm-retry invariants', () => { it('has no provider without the requested closed step', () => { @@ -65,7 +68,8 @@ describe('llm-retry invariants', () => { }) const zeroDelay = closeStep(ctx, 'retry-invariant-zero-delay') zeroDelay.append('llm/retry', { - turn: 1, step: 1, ...normal, retry: 1, maxRetries: 1, delayMs: 0, failure, + turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(1), + retry: 1, maxRetries: 1, delayMs: 0, failure, }) }).not.toThrow() expect(() => { ctx.emit('tools/change') }).not.toThrow() @@ -80,6 +84,7 @@ describe('llm-retry invariants', () => { step: 1, provider: 'mock', mode: 'always', + policyKey: alwaysPolicyKey, retry: 1, delayMs: 500, failure, @@ -91,6 +96,7 @@ describe('llm-retry invariants', () => { step: 1, provider: 'mock', mode: 'always', + policyKey: alwaysPolicyKey, retry: 1, maxRetries: 2, delayMs: 500, @@ -99,6 +105,65 @@ describe('llm-retry invariants', () => { }).toThrow(/always mode must omit maxRetries/) }) + it('binds event mode and finite budget to the canonical policy key', async () => { + const ctx = await setup() + const normalModeMismatch = closeStep(ctx, 'retry-invariant-normal-mode-key') + expect(() => { + normalModeMismatch.append('llm/retry', { + turn: 1, step: 1, ...normal, policyKey: alwaysPolicyKey, + retry: 1, maxRetries: 2, delayMs: 1, failure, + }) + }).toThrow(/mode normal must match policyKey mode always/) + + const alwaysModeMismatch = closeStep(ctx, 'retry-invariant-always-mode-key') + expect(() => { + alwaysModeMismatch.append('llm/retry', { + turn: 1, + step: 1, + provider: 'mock', + mode: 'always', + policyKey: normalPolicyKey(2), + retry: 1, + delayMs: 1, + failure, + }) + }).toThrow(/mode always must match policyKey mode normal/) + + const budgetMismatch = closeStep(ctx, 'retry-invariant-budget-key') + expect(() => { + budgetMismatch.append('llm/retry', { + turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3), + retry: 1, maxRetries: 2, delayMs: 1, failure, + }) + }).toThrow(/maxRetries 2 must match policyKey/) + }) + + it('binds the failure code and scheduled delay to the canonical policy key', async () => { + const ctx = await setup() + const ineligibleFailure = closeStep(ctx, 'retry-invariant-failure-code-key') + expect(() => { + ineligibleFailure.append('llm/retry', { + turn: 1, step: 1, ...normal, + retry: 1, maxRetries: 2, delayMs: 1, + failure: { message: 'authentication failed', code: 'AUTH', status: 401 }, + }) + }).toThrow(/failure code AUTH must be eligible under policyKey/) + + const overPolicyDelay = closeStep(ctx, 'retry-invariant-delay-key') + expect(() => { + overPolicyDelay.append('llm/retry', { + turn: 1, + step: 1, + provider: 'mock', + mode: 'always', + policyKey: '["always",1,1,0]', + retry: 1, + delayMs: 2, + failure, + }) + }).toThrow(/within policyKey range 0\.\.1/) + }) + it('rejects empty providers and unknown modes from hostile durable input', async () => { const ctx = await setup() const emptyProvider = closeStep(ctx, 'retry-invariant-empty-provider') @@ -108,6 +173,7 @@ describe('llm-retry invariants', () => { step: 1, provider: '', mode: 'always', + policyKey: alwaysPolicyKey, retry: 1, delayMs: 1, failure, @@ -121,11 +187,26 @@ describe('llm-retry invariants', () => { step: 1, provider: 'mock', mode: 'sometimes', + policyKey: alwaysPolicyKey, retry: 1, delayMs: 1, failure, } as never) }).toThrow(/mode must be normal or always/) + + const emptyPolicyKey = closeStep(ctx, 'retry-invariant-empty-policy-key') + expect(() => { + emptyPolicyKey.append('llm/retry', { + turn: 1, + step: 1, + provider: 'mock', + mode: 'always', + policyKey: '', + retry: 1, + delayMs: 1, + failure, + }) + }).toThrow(/policyKey must encode a canonical resolved policy/) }) it.each([ @@ -206,6 +287,7 @@ describe('llm-retry invariants', () => { step: 1, provider: 'mock', mode: 'always', + policyKey: alwaysPolicyKey, retry: 1, delayMs: 1, failure, @@ -219,6 +301,7 @@ describe('llm-retry invariants', () => { step: 1, provider: 'other', mode: 'always', + policyKey: alwaysPolicyKey, retry: 1, delayMs: 1, failure, @@ -239,6 +322,7 @@ describe('llm-retry invariants', () => { step: 1, provider: 'mock', mode: 'always', + policyKey: alwaysPolicyKey, retry: 1, delayMs: 1, failure, @@ -260,23 +344,27 @@ describe('llm-retry invariants', () => { const ctx = await setup() const duplicate = closeStep(ctx, 'retry-invariant-duplicate') duplicate.append('llm/retry', { - turn: 1, step: 1, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure, + turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3), + retry: 1, maxRetries: 3, delayMs: 1, failure, }) expect(() => { duplicate.append('llm/retry', { - turn: 1, step: 1, ...normal, retry: 2, maxRetries: 3, delayMs: 1, failure, + turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3), + retry: 2, maxRetries: 3, delayMs: 1, failure, }) }).toThrow(/duplicates the retry record/) const nonIncreasing = closeStep(ctx, 'retry-invariant-non-increasing') nonIncreasing.append('llm/retry', { - turn: 1, step: 1, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure, + turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3), + retry: 1, maxRetries: 3, delayMs: 1, failure, }) nonIncreasing.append('step/start', { turn: 1, step: 2 }) nonIncreasing.append('step/end', { turn: 1, step: 2 }) expect(() => { nonIncreasing.append('llm/retry', { - turn: 1, step: 2, ...normal, retry: 1, maxRetries: 3, delayMs: 1, failure, + turn: 1, step: 2, ...normal, policyKey: normalPolicyKey(3), + retry: 1, maxRetries: 3, delayMs: 1, failure, }) }).toThrow(/must equal provider policy retry 2/) }) diff --git a/packages/llm/llm-retry/tests/persistence.spec.ts b/packages/llm/llm-retry/tests/persistence.spec.ts index 41c70b257d..d99d255e7b 100644 --- a/packages/llm/llm-retry/tests/persistence.spec.ts +++ b/packages/llm/llm-retry/tests/persistence.spec.ts @@ -44,6 +44,7 @@ describe.each(['jsonl', 'sqlite'] as const)('%s retry-event persistence', (kind) step: 1, provider: 'mock', mode: 'always', + policyKey: '["always",500,10000,0.1]', retry: 1, delayMs: 750, failure: { message: 'provider busy', code: 'RATE_LIMIT', status: 429 }, diff --git a/packages/llm/llm-retry/tests/policy-key.spec.ts b/packages/llm/llm-retry/tests/policy-key.spec.ts new file mode 100644 index 0000000000..99c7da7802 --- /dev/null +++ b/packages/llm/llm-retry/tests/policy-key.spec.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from 'vitest' +import { resolveRetryPolicy } from '@deepseek-ai/dsh-llm' +import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' +import { parseRetryPolicyKey, retryPolicyKey } from '../src/policy-key.ts' + +describe('retry policy durable key', () => { + it('includes every policy field while normalizing code-set order', () => { + const first = resolveRetryPolicy({ + mode: 'normal', + maxRetries: 4, + retryableCodes: ['SERVER', 'RATE_LIMIT'], + backoff: { initialDelayMs: 3, maxDelayMs: 9, jitterRatio: 0.25 }, + }, 'first') + const reordered = resolveRetryPolicy({ + mode: 'normal', + maxRetries: 4, + retryableCodes: ['RATE_LIMIT', 'SERVER'], + backoff: { initialDelayMs: 3, maxDelayMs: 9, jitterRatio: 0.25 }, + }, 'reordered') + const key = retryPolicyKey(first) + + expect(key).toBe('["normal",4,["RATE_LIMIT","SERVER"],3,9,0.25]') + expect(retryPolicyKey(reordered)).toBe(key) + expect(parseRetryPolicyKey(key)).toEqual(reordered) + }) + + it('round-trips always mode', () => { + const policy = resolveRetryPolicy({ + mode: 'always', + backoff: { initialDelayMs: 2, maxDelayMs: 8, jitterRatio: 1 }, + }, 'always') + const key = retryPolicyKey(policy) + + expect(key).toBe('["always",2,8,1]') + expect(parseRetryPolicyKey(key)).toEqual(policy) + }) + + it.each([ + undefined, + '', + '{', + '{}', + '["sometimes",1,2,0]', + '["always",1,2]', + '["always","1",2,0]', + '["always",1e400,2,0]', + '["always",0,2,0]', + `["always",${MAX_TIMER_DELAY_MS + 1},${MAX_TIMER_DELAY_MS + 1},0]`, + '["always",1,"2",0]', + '["always",1,1e400,0]', + '["always",1,0,0]', + `["always",1,${MAX_TIMER_DELAY_MS + 1},0]`, + '["always",2,1,0]', + '["always",1,2,"0"]', + '["always",1,2,1e400]', + '["always",1,2,-0.1]', + '["always",1,2,1.1]', + '["normal",2,["SERVER"],1,2]', + '["normal","2",["SERVER"],1,2,0]', + '["normal",-1,["SERVER"],1,2,0]', + '["normal",2,"SERVER",1,2,0]', + '["normal",2,[],1,2,0]', + '["normal",2,[1],1,2,0]', + '["normal",2,[""],1,2,0]', + '["normal",2,["SERVER","SERVER"],1,2,0]', + '["normal",2,["SERVER"],2,1,0]', + '["normal",2,["SERVER","RATE_LIMIT"],1,2,0]', + ])('rejects non-canonical durable input %#', (value) => { + expect(parseRetryPolicyKey(value)).toBeUndefined() + }) +}) diff --git a/packages/llm/llm-retry/tests/retry.spec.ts b/packages/llm/llm-retry/tests/retry.spec.ts index 2533eb045f..7677cbac20 100644 --- a/packages/llm/llm-retry/tests/retry.spec.ts +++ b/packages/llm/llm-retry/tests/retry.spec.ts @@ -184,6 +184,7 @@ describe('provider-routed retry policy', () => { step: 1, provider: 'mock', mode: 'normal', + policyKey: '["normal",2,["RATE_LIMIT","SERVER","TIMEOUT","TRANSPORT"],500,10000,0.1]', retry: 1, maxRetries: 2, delayMs: 500, @@ -561,6 +562,110 @@ describe('provider-routed retry policy', () => { }, ) + it('starts a new retry history when a same-mode route replacement changes policy', async () => { + vi.useFakeTimers() + const oldAdapter = new ScriptedAdapter([ + new LlmError('old route failed', 'AUTH'), + ]) + const mounted = await harness(oldAdapter, { mock: alwaysConfig({ + initialDelayMs: 1, + maxDelayMs: 1, + }) }) + context = mounted.ctx + const agent = context.agentLoop.create(SessionId('retry-policy-replacement'), { + provider: 'mock', + model: 'mock', + }) + const first = waitForRetry(context, agent, 1) + + agent.followup([{ type: 'text', text: 'replace policy between attempts' }]) + expect((await first).data).toMatchObject({ + mode: 'always', + retry: 1, + delayMs: 1, + }) + + mounted.disposeAdapter() + const replacement = new ScriptedAdapter([ + new LlmError('replacement failed', 'AUTH'), + textResponse('replacement recovered'), + ]) + replacement.configureRetryPolicies({ mock: alwaysConfig({ + initialDelayMs: 3, + maxDelayMs: 9, + }) }) + context.llm.registerAdapter(['mock'], replacement) + + const second = waitForRetry(context, agent, 1) + await vi.advanceTimersByTimeAsync(1) + expect((await second).data).toMatchObject({ + mode: 'always', + retry: 1, + delayMs: 3, + }) + + const idle = waitForIdle(context, agent) + await vi.advanceTimersByTimeAsync(3) + await idle + + expect(oldAdapter.requests).toHaveLength(1) + expect(replacement.requests).toHaveLength(2) + expect(agent.session.events.filter(event => event.type === 'llm/retry').map(event => ({ + policyKey: event.data.policyKey, + retry: event.data.retry, + }))).toEqual([ + { policyKey: '["always",1,1,0]', retry: 1 }, + { policyKey: '["always",3,9,0]', retry: 1 }, + ]) + }) + + it('continues retry history when a replacement only reorders retryable codes', async () => { + vi.useFakeTimers() + const oldAdapter = new ScriptedAdapter([ + new LlmError('old route failed', 'SERVER'), + ]) + const mounted = await harness(oldAdapter, { mock: normalConfig({ + maxRetries: 2, + retryableCodes: ['SERVER', 'RATE_LIMIT'], + backoff: { initialDelayMs: 1, maxDelayMs: 4 }, + }) }) + context = mounted.ctx + const agent = context.agentLoop.create(SessionId('retry-policy-code-order'), { + provider: 'mock', + model: 'mock', + }) + const first = waitForRetry(context, agent, 1) + + agent.followup([{ type: 'text', text: 'replace equivalent policy between attempts' }]) + const firstEvent = await first + expect(firstEvent.data.delayMs).toBe(1) + + mounted.disposeAdapter() + const replacement = new ScriptedAdapter([ + new LlmError('replacement failed', 'SERVER'), + textResponse('replacement recovered'), + ]) + replacement.configureRetryPolicies({ mock: normalConfig({ + maxRetries: 2, + retryableCodes: ['RATE_LIMIT', 'SERVER'], + backoff: { initialDelayMs: 1, maxDelayMs: 4 }, + }) }) + context.llm.registerAdapter(['mock'], replacement) + + const second = waitForRetry(context, agent, 2) + await vi.advanceTimersByTimeAsync(1) + const secondEvent = await second + expect(secondEvent.data).toMatchObject({ retry: 2, delayMs: 2 }) + expect(secondEvent.data.policyKey).toBe(firstEvent.data.policyKey) + + const idle = waitForIdle(context, agent) + await vi.advanceTimersByTimeAsync(2) + await idle + + expect(oldAdapter.requests).toHaveLength(1) + expect(replacement.requests).toHaveLength(2) + }) + it('keeps always mode unbounded while preserving cancellable jittered backoff', async () => { vi.useFakeTimers() const adapter = new ScriptedAdapter([ diff --git a/packages/ui/tui/tests/tui.snapshot.ts b/packages/ui/tui/tests/tui.snapshot.ts index 693c2852c1..44ac1a35fb 100644 --- a/packages/ui/tui/tests/tui.snapshot.ts +++ b/packages/ui/tui/tests/tui.snapshot.ts @@ -271,6 +271,7 @@ describe('TUI terminal-state snapshots', () => { step: 1, provider: 'mock', mode: 'normal', + policyKey: '["normal",2,["RATE_LIMIT"],1,10000,0]', retry: 1, maxRetries: 2, delayMs: 500, @@ -301,6 +302,7 @@ describe('TUI terminal-state snapshots', () => { step: 1, provider: 'mock', mode: 'always', + policyKey: '["always",1,10000,0]', retry: 1, delayMs: 1_000, failure: { message: 'temporary transport failure', code: 'TRANSPORT' }, diff --git a/packages/ui/tui/tests/tui.spec.ts b/packages/ui/tui/tests/tui.spec.ts index 2dd3ed2098..916ec90580 100644 --- a/packages/ui/tui/tests/tui.spec.ts +++ b/packages/ui/tui/tests/tui.spec.ts @@ -1296,6 +1296,7 @@ describe('pi-tui chat lifecycle and transcript', () => { step: 1, provider: 'mock', mode: 'normal', + policyKey: '["normal",2,["RATE_LIMIT"],1,10000,0]', retry: 1, maxRetries: 2, delayMs: 500, @@ -1330,6 +1331,7 @@ describe('pi-tui chat lifecycle and transcript', () => { step: 1, provider: 'mock', mode: 'normal', + policyKey: '["normal",2,["RATE_LIMIT"],1,10000,0]', retry: 1, maxRetries: 2, delayMs: 500, @@ -1340,6 +1342,7 @@ describe('pi-tui chat lifecycle and transcript', () => { step: 2, provider: 'mock', mode: 'normal', + policyKey: '["normal",2,["RATE_LIMIT"],1,10000,0]', retry: 2, maxRetries: 2, delayMs: 1_000, @@ -1350,6 +1353,7 @@ describe('pi-tui chat lifecycle and transcript', () => { step: 3, provider: 'mock', mode: 'always', + policyKey: '["always",1,10000,0]', retry: 1, delayMs: 2_000, failure: { message: 'retry without limit', code: 'AUTH', status: 401 }, From fcd355bf45cb13cd500dd56aec6918ee627f7060 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 25 Jul 2026 16:28:31 +0800 Subject: [PATCH 09/11] fix(examples): remove legacy retry rejection --- ...26-07-24-provider-retry-policies.i18n.yaml | 4 +-- .../2026-07-24-provider-retry-policies.md | 2 +- .../2026-07-24-provider-retry-policies.zh.md | 2 +- docs/config-catalog.md | 8 ----- packages/examples/acp-demo/src/index.ts | 4 --- .../examples/acp-demo/tests/acp-agent.spec.ts | 10 ------ .../examples/acp-demo/tests/built-bin.e2e.ts | 13 -------- .../examples/agent-spine-demo/src/index.ts | 5 --- .../agent-spine-demo/tests/agent-core.spec.ts | 8 ----- packages/examples/cli-demo/src/index.ts | 4 --- .../examples/cli-demo/tests/built-bin.e2e.ts | 33 ------------------- packages/examples/cli-demo/tests/cli.spec.ts | 10 ------ packages/examples/tui-demo/src/index.ts | 4 --- .../examples/tui-demo/tests/tui-agent.spec.ts | 10 ------ 14 files changed, 4 insertions(+), 113 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml index fdc0fadce3..14b24b70b4 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-24-provider-retry-policies.md: cd39717c03f2a7714b2f5e93404efb0d2d17a6da -2026-07-24-provider-retry-policies.zh.md: 1768f67dfb98757d964589693a108f946924749b +2026-07-24-provider-retry-policies.md: ed6b401c792f85eec6ca1c52495733222ea998a1 +2026-07-24-provider-retry-policies.zh.md: 2a4853ceec71d0ce78a1225f455575035f0bca13 diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md index cd39717c03..ed6b401c79 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md @@ -56,7 +56,7 @@ Each scheduled retry appends a non-surface `llm/retry` event with the failed pro ## Verification -Adapter tests validate nested policies at provider load, prove registration captures configured and default policies, and retain the serving policy across in-flight route replacement. Unit and plugin-validation tests select policies from the failed request's serving registration, reject top-level `llmRetry` at the spine, CLI, TUI, and ACP schemas, separate different same-mode policies while preserving histories across reordered code sets, exercise always mode beyond the normal budget, pin jitter and delay caps, prove downstream recovery ordering, prove cancellation and disposal drain delegated recovery before reaching quiescence, and prove both abort active backoff waits. Published Loader fixtures reject the invalid app-level key in CLI and ACP and the invalid bundle-level key when loading the spine directly. Request-level coverage compares the complete messages of failed and retried attempts and rejects both provider error text and discarded partial output. A keyless headless `stream-json` snapshot runs failure, retry, and success through the assembled app, pins the complete `llm/retry` record, and rejects any model-message change between attempts. JSONL and SQLite tests round-trip an always event without `Infinity`; invariant tests validate the canonical policy tuple, bind its provider to the request header, bind its failure code and delay to the encoded policy, and bind its retry number to the active provider policy key; TUI tests render finite and infinite limits. +Adapter tests validate nested policies at provider load, prove registration captures configured and default policies, and retain the serving policy across in-flight route replacement. Unit tests select policies from the failed request's serving registration, separate different same-mode policies while preserving histories across reordered code sets, exercise always mode beyond the normal budget, pin jitter and delay caps, prove downstream recovery ordering, prove cancellation and disposal drain delegated recovery before reaching quiescence, and prove both abort active backoff waits. Request-level coverage compares the complete messages of failed and retried attempts and rejects both provider error text and discarded partial output. A keyless headless `stream-json` snapshot runs failure, retry, and success through the assembled app, pins the complete `llm/retry` record, and rejects any model-message change between attempts. JSONL and SQLite tests round-trip an always event without `Infinity`; invariant tests validate the canonical policy tuple, bind its provider to the request header, bind its failure code and delay to the encoded policy, and bind its retry number to the active provider policy key; TUI tests render finite and infinite limits. ## Consequences diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md index 1768f67dfb..2a4853ceec 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md @@ -56,7 +56,7 @@ always 模式先请求下游恢复，使上下文溢出压缩（compaction）之 ## 验证 -适配器测试会在提供方加载时校验嵌套策略，证明注册流程会捕获已配置策略和默认策略，并证明请求进行期间替换路由后仍会保留实际提供服务的策略。单元测试与插件校验测试根据失败请求实际使用的注册项选择策略、在主干、CLI、TUI 与 ACP schema 拒绝顶层 `llmRetry`、分离模式相同但策略不同的替换路由历史，同时在错误代码集合仅顺序不同时延续历史、验证 always 模式可越过 normal 预算、固定抖动和延迟上限、证明下游恢复顺序、证明取消与 dispose 会先排空已委托的恢复再达到完全停稳，并证明二者都会停止正在进行的退避等待。发布版 Loader fixture（测试前置数据）会在 CLI 与 ACP 中拒绝无效的应用级配置键，并在直接加载主干时拒绝无效的 bundle 级配置键。请求级覆盖会比较失败尝试与重试尝试的完整消息，并排除提供方错误文本和丢弃的部分输出。一个无密钥 headless `stream-json` 快照会通过组装后的应用执行失败、重试与成功流程，固定完整的 `llm/retry` 记录，并拒绝各次尝试之间出现任何模型消息变化。JSONL 与 SQLite 测试会往返读写不含 `Infinity` 的 always 事件；不变式测试会校验规范策略元组、将事件中的提供方绑定到请求头、将失败代码与延迟绑定到编码后的策略，并将重试编号绑定到活跃的提供方策略键；TUI 测试会渲染有限和无限上限。 +适配器测试会在提供方加载时校验嵌套策略，证明注册流程会捕获已配置策略和默认策略，并证明请求进行期间替换路由后仍会保留实际提供服务的策略。单元测试根据失败请求实际使用的注册项选择策略、分离模式相同但策略不同的替换路由历史，同时在错误代码集合仅顺序不同时延续历史、验证 always 模式可越过 normal 预算、固定抖动和延迟上限、证明下游恢复顺序、证明取消与 dispose 会先排空已委托的恢复再达到完全停稳，并证明二者都会停止正在进行的退避等待。请求级覆盖会比较失败尝试与重试尝试的完整消息，并排除提供方错误文本和丢弃的部分输出。一个无密钥 headless `stream-json` 快照会通过组装后的应用执行失败、重试与成功流程，固定完整的 `llm/retry` 记录，并拒绝各次尝试之间出现任何模型消息变化。JSONL 与 SQLite 测试会往返读写不含 `Infinity` 的 always 事件；不变式测试会校验规范策略元组、将事件中的提供方绑定到请求头、将失败代码与延迟绑定到编码后的策略，并将重试编号绑定到活跃的提供方策略键；TUI 测试会渲染有限和无限上限。 ## 后果 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index ed704ff08b..9fa6280870 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -73,8 +73,6 @@ export interface Config { toolTasks?: NonNullable /** Persisted same-session goals; owner defaults enable them, or false disables the stack and tools. */ goals?: agentCore.GoalConfig | false - /** Invalid at app level; configure `retryPolicy` under each provider. */ - llmRetry?: never } ``` @@ -162,8 +160,6 @@ export interface Config { invariants?: InvariantConfig /** Opt-in persisted same-session goal stack; set false or omit to leave it unmounted. */ goals?: GoalConfig | false - /** Invalid at bundle level; configure `retryPolicy` under each provider. */ - llmRetry?: never } /** Skill bundle config forwarded to the registry, local provider, and model-facing consumer. */ @@ -265,8 +261,6 @@ export interface Config { toolTasks?: NonNullable /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ workspaceContext: agentCore.Config['workspaceContext'] - /** Invalid at app level; configure `retryPolicy` under each provider. */ - llmRetry?: never } ``` @@ -1804,8 +1798,6 @@ export interface Config { resumeSessionId?: string /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ workspaceContext: agentCore.Config['workspaceContext'] - /** Invalid at app level; configure `retryPolicy` under each provider. */ - llmRetry?: never } ``` diff --git a/packages/examples/acp-demo/src/index.ts b/packages/examples/acp-demo/src/index.ts index c4d52d0053..d7161c589f 100644 --- a/packages/examples/acp-demo/src/index.ts +++ b/packages/examples/acp-demo/src/index.ts @@ -67,8 +67,6 @@ export interface Config { toolTasks?: NonNullable /** Persisted same-session goals; owner defaults enable them, or false disables the stack and tools. */ goals?: agentCore.GoalConfig | false - /** Invalid at app level; configure `retryPolicy` under each provider. */ - llmRetry?: never } // Each front door owns a complete, directly readable config schema; extracting @@ -94,8 +92,6 @@ export const Config: z = z.object({ toolBash: agentCore.ToolBashConfigSchema, toolTasks: z.union([z.const(false), agentCore.ToolTasksConfigSchema]), goals: z.union([z.const(false), agentCore.GoalConfigSchema]), - // Provider retryPolicy makes a top-level llmRetry invalid. - llmRetry: z.never(), }) /* jscpd:ignore-end */ diff --git a/packages/examples/acp-demo/tests/acp-agent.spec.ts b/packages/examples/acp-demo/tests/acp-agent.spec.ts index 7c32da8cfa..e932c7fb17 100644 --- a/packages/examples/acp-demo/tests/acp-agent.spec.ts +++ b/packages/examples/acp-demo/tests/acp-agent.spec.ts @@ -76,16 +76,6 @@ async function withIsolatedSkillHomes(run: () => Promise): Promise { } describe('dsh-acp-demo composition', () => { - it('rejects app-level llmRetry config through plugin validation', async () => { - const ctx = new Context() - await expect(ctx.plugin(acpAgent, { - provider: 'mock', - model: 'mock', - workspaceContext: false, - llmRetry: { maxTransientRetries: 2 }, - } as never)).rejects.toThrow(/llmRetry/) - }) - it('brings up the spine + persistence + the ACP bridge', async () => { const ctx = await mount({ provider: 'mock', diff --git a/packages/examples/acp-demo/tests/built-bin.e2e.ts b/packages/examples/acp-demo/tests/built-bin.e2e.ts index dd7139815f..870a43cf88 100644 --- a/packages/examples/acp-demo/tests/built-bin.e2e.ts +++ b/packages/examples/acp-demo/tests/built-bin.e2e.ts @@ -208,19 +208,6 @@ describe.skipIf(!existsSync(acpBin))('dsh-acp-demo BUILT bin (node lib/bin.js, n expect(stderr).toContain('config file not found') }, 30_000) - it('rejects legacy app-level llmRetry through the published Loader path', async () => { - consumer = await makeConsumer() - const configPath = join(consumer, 'cordis.yml') - const config = await readFile(configPath, 'utf8') - await writeFile(configPath, config.replace( - ' workspaceContext: false', - ' workspaceContext: false\n llmRetry:\n maxTransientRetries: 2', - )) - - const { code, stderr } = await runBinExpectingExit('./cordis.yml', consumer) - expect(code).not.toBe(0) - expect(stderr).toContain('llmRetry') - }, 30_000) }) /** Spawn the built acp bin against `configArg` and resolve with its exit code + stderr. */ diff --git a/packages/examples/agent-spine-demo/src/index.ts b/packages/examples/agent-spine-demo/src/index.ts index 397966813f..d2a65844ea 100644 --- a/packages/examples/agent-spine-demo/src/index.ts +++ b/packages/examples/agent-spine-demo/src/index.ts @@ -112,8 +112,6 @@ export interface Config { invariants?: InvariantConfig /** Opt-in persisted same-session goal stack; set false or omit to leave it unmounted. */ goals?: GoalConfig | false - /** Invalid at bundle level; configure `retryPolicy` under each provider. */ - llmRetry?: never } /** The skill config schema exported for app packages that forward `skills`. */ @@ -154,9 +152,6 @@ export const Config = z.intersect([ toolTasks: z.union([z.const(false), ToolTasksConfigSchema]), invariants: InvariantService.Config, goals: z.union([z.const(false), GoalConfigSchema]), - // Schemastery preserves unknown object properties. A top-level llmRetry is - // known-but-impossible because provider retryPolicy owns this configuration. - llmRetry: z.never(), }) as unknown as z>, ]) as unknown as z diff --git a/packages/examples/agent-spine-demo/tests/agent-core.spec.ts b/packages/examples/agent-spine-demo/tests/agent-core.spec.ts index 33e8ddc6cd..2051aee4b8 100644 --- a/packages/examples/agent-spine-demo/tests/agent-core.spec.ts +++ b/packages/examples/agent-spine-demo/tests/agent-core.spec.ts @@ -594,14 +594,6 @@ describe('dsh-agent-spine-demo bundle', () => { expect(agentCore.name).toBe('agent-spine-demo') }) - it('rejects bundle-level llmRetry config through plugin validation', async () => { - const ctx = new Context() - await expect(ctx.plugin(agentCore, { - workspaceContext: false, - llmRetry: { maxTransientRetries: 2 }, - } as never)).rejects.toThrow(/llmRetry/) - }) - it('has the namespace-plugin export shape (no stray default) so the Loader keeps name/Config/apply', () => { // A default export would make `unwrapExports` collapse this inject-less namespace and silently // drop `name`/`Config`. Apps import the bundle directly, so this is its Loader-shape guard. diff --git a/packages/examples/cli-demo/src/index.ts b/packages/examples/cli-demo/src/index.ts index e9bbb0a237..bfec7c9a4f 100644 --- a/packages/examples/cli-demo/src/index.ts +++ b/packages/examples/cli-demo/src/index.ts @@ -52,8 +52,6 @@ export interface Config { toolTasks?: NonNullable /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ workspaceContext: agentCore.Config['workspaceContext'] - /** Invalid at app level; configure `retryPolicy` under each provider. */ - llmRetry?: never } // Each front door keeps a complete Loader schema so its deployment contract is @@ -75,8 +73,6 @@ export const Config: z = z.object({ toolBash: agentCore.ToolBashConfigSchema, toolTasks: z.union([z.const(false), agentCore.ToolTasksConfigSchema]), workspaceContext: z.union([z.const(false), workspaceContext.Config]).required(), - // Provider retryPolicy makes a top-level llmRetry invalid. - llmRetry: z.never(), }) /* jscpd:ignore-end */ diff --git a/packages/examples/cli-demo/tests/built-bin.e2e.ts b/packages/examples/cli-demo/tests/built-bin.e2e.ts index ddc07a93a1..5c3a6ad62e 100644 --- a/packages/examples/cli-demo/tests/built-bin.e2e.ts +++ b/packages/examples/cli-demo/tests/built-bin.e2e.ts @@ -192,39 +192,6 @@ describe.skipIf(!existsSync(cliBin))('dsh-cli-demo BUILT bin', () => { } }, 30_000) - it('rejects legacy app-level llmRetry through the published Loader path', async () => { - consumer = await makeConsumer() - const configPath = join(consumer, 'cordis.yml') - const config = await readFile(configPath, 'utf8') - await writeFile(configPath, config.replace( - ' workspaceContext: false', - ' workspaceContext: false\n llmRetry:\n maxTransientRetries: 2', - )) - - const result = await runBuiltBin(consumer, ['--config', './cordis.yml', 'task']) - expect(result.code).not.toBe(0) - expect(result.stdout).toBe('') - expect(result.stderr).toContain('llmRetry') - }, 30_000) - - it('rejects legacy bundle-level llmRetry when the published spine is loaded directly', async () => { - consumer = await makeConsumer() - await writeFile(join(consumer, 'cordis.yml'), [ - '- id: spine', - " name: '@deepseek-ai/dsh-agent-spine-demo'", - ' config:', - ' workspaceContext: false', - ' llmRetry:', - ' maxTransientRetries: 2', - '', - ].join('\n')) - - const result = await runBuiltBin(consumer, ['--config', './cordis.yml', 'task']) - expect(result.code).not.toBe(0) - expect(result.stdout).toBe('') - expect(result.stderr).toContain('llmRetry') - }, 30_000) - describe.skipIf(process.platform === 'win32')('POSIX signal delivery', () => { it.each([ ['SIGINT', 130], diff --git a/packages/examples/cli-demo/tests/cli.spec.ts b/packages/examples/cli-demo/tests/cli.spec.ts index 786662caba..bf681af2f8 100644 --- a/packages/examples/cli-demo/tests/cli.spec.ts +++ b/packages/examples/cli-demo/tests/cli.spec.ts @@ -173,16 +173,6 @@ afterEach(async () => { }) describe('parseCliArgs', () => { - it('rejects app-level llmRetry config through plugin validation', async () => { - const ctx = new Context() - await expect(ctx.plugin(cliDemo, { - provider: 'mock', - model: 'mock', - workspaceContext: false, - llmRetry: { maxTransientRetries: 2 }, - } as never)).rejects.toThrow(/llmRetry/) - }) - it('parses defaults, explicit options, spaces, and an option-like task after --', () => { expect(parseCliArgs(['task with spaces'])).toEqual({ kind: 'run', configPath: './cordis.yml', outputFormat: 'text', task: 'task with spaces', diff --git a/packages/examples/tui-demo/src/index.ts b/packages/examples/tui-demo/src/index.ts index cd38ab3cd2..29f985c8e7 100644 --- a/packages/examples/tui-demo/src/index.ts +++ b/packages/examples/tui-demo/src/index.ts @@ -82,8 +82,6 @@ export interface Config { resumeSessionId?: string /** Controls automatic AGENTS.md/CLAUDE.md loading; configure a byte budget or set `false`. */ workspaceContext: agentCore.Config['workspaceContext'] - /** Invalid at app level; configure `retryPolicy` under each provider. */ - llmRetry?: never } export const Config: z = z.object({ @@ -108,8 +106,6 @@ export const Config: z = z.object({ goals: z.union([z.const(false), agentCore.GoalConfigSchema]), resumeSessionId: z.string(), workspaceContext: z.union([z.const(false), workspaceContext.Config]).required(), - // Provider retryPolicy makes a top-level llmRetry invalid. - llmRetry: z.never(), }) /* jscpd:ignore-end */ diff --git a/packages/examples/tui-demo/tests/tui-agent.spec.ts b/packages/examples/tui-demo/tests/tui-agent.spec.ts index 37c8609733..19d5802e71 100644 --- a/packages/examples/tui-demo/tests/tui-agent.spec.ts +++ b/packages/examples/tui-demo/tests/tui-agent.spec.ts @@ -21,16 +21,6 @@ function recordingContext(): { readonly ctx: Context; readonly calls: PluginCall } describe('dsh-tui-demo app', () => { - it('rejects app-level llmRetry config through plugin validation', async () => { - const ctx = new Context() - await expect(ctx.plugin(tuiAgent, { - provider: 'mock', - model: 'mock', - workspaceContext: false, - llmRetry: { maxTransientRetries: 2 }, - } as never)).rejects.toThrow(/llmRetry/) - }) - it('composes the TUI cluster around one fresh exact session identity', () => { const { ctx, calls } = recordingContext() tuiAgent.composeTuiApp(ctx, { From 19a1aec89e82c8c37c130fc01c2ee77b0760e1fa Mon Sep 17 00:00:00 2001 From: Turtle Date: Sun, 26 Jul 2026 01:16:41 +0800 Subject: [PATCH 10/11] refactor(llm-retry): simplify policy history validation --- ...26-07-24-provider-retry-policies.i18n.yaml | 4 +- .../2026-07-24-provider-retry-policies.md | 2 +- .../2026-07-24-provider-retry-policies.zh.md | 2 +- docs/config-catalog.md | 2 +- docs/persistence-catalog.md | 2 +- packages/llm/llm-retry/README.md | 2 +- packages/llm/llm-retry/src/index.ts | 14 +- packages/llm/llm-retry/src/invariant.ts | 25 +- packages/llm/llm-retry/src/policy-key.ts | 100 ----- .../llm/llm-retry/tests/invariant.spec.ts | 398 ++++-------------- .../llm/llm-retry/tests/policy-key.spec.ts | 71 ---- packages/llm/llm-retry/tests/retry.spec.ts | 227 +++------- 12 files changed, 186 insertions(+), 663 deletions(-) delete mode 100644 packages/llm/llm-retry/src/policy-key.ts delete mode 100644 packages/llm/llm-retry/tests/policy-key.spec.ts diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml index 14b24b70b4..7e865d0d6d 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-24-provider-retry-policies.md: ed6b401c792f85eec6ca1c52495733222ea998a1 -2026-07-24-provider-retry-policies.zh.md: 2a4853ceec71d0ce78a1225f455575035f0bca13 +2026-07-24-provider-retry-policies.md: 1831ce6b96178d11e7c9927ceccbe07ea578cd2c +2026-07-24-provider-retry-policies.zh.md: 788f1f1963861e1b46ff0d8798e53e01bd9892b4 diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md index ed6b401c79..1831ce6b96 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.md @@ -56,7 +56,7 @@ Each scheduled retry appends a non-surface `llm/retry` event with the failed pro ## Verification -Adapter tests validate nested policies at provider load, prove registration captures configured and default policies, and retain the serving policy across in-flight route replacement. Unit tests select policies from the failed request's serving registration, separate different same-mode policies while preserving histories across reordered code sets, exercise always mode beyond the normal budget, pin jitter and delay caps, prove downstream recovery ordering, prove cancellation and disposal drain delegated recovery before reaching quiescence, and prove both abort active backoff waits. Request-level coverage compares the complete messages of failed and retried attempts and rejects both provider error text and discarded partial output. A keyless headless `stream-json` snapshot runs failure, retry, and success through the assembled app, pins the complete `llm/retry` record, and rejects any model-message change between attempts. JSONL and SQLite tests round-trip an always event without `Infinity`; invariant tests validate the canonical policy tuple, bind its provider to the request header, bind its failure code and delay to the encoded policy, and bind its retry number to the active provider policy key; TUI tests render finite and infinite limits. +Adapter tests validate nested policies at provider load, prove registration captures configured and default policies, and retain the serving policy across in-flight route replacement. Unit tests select policies from the failed request's serving registration, separate provider and changed-policy histories, exercise always mode beyond the normal budget, pin jitter and delay caps, prove downstream recovery ordering, prove cancellation and disposal drain delegated recovery before reaching quiescence, and prove both abort active backoff waits. Request-level coverage compares the complete messages of failed and retried attempts and rejects both provider error text and discarded partial output. A keyless headless `stream-json` snapshot runs failure, retry, and success through the assembled app, pins the complete `llm/retry` record, and rejects any model-message change between attempts. JSONL and SQLite tests round-trip an always event without `Infinity`; invariant tests bind provider identity to the request header, validate failure and mode-specific timer bounds, and bind retry numbers to provider-policy keys; TUI tests render finite and infinite limits. ## Consequences diff --git a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md index 2a4853ceec..788f1f1963 100644 --- a/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md +++ b/.agents/notes/implemented/feature/2026-07-24-provider-retry-policies.zh.md @@ -56,7 +56,7 @@ always 模式先请求下游恢复，使上下文溢出压缩（compaction）之 ## 验证 -适配器测试会在提供方加载时校验嵌套策略，证明注册流程会捕获已配置策略和默认策略，并证明请求进行期间替换路由后仍会保留实际提供服务的策略。单元测试根据失败请求实际使用的注册项选择策略、分离模式相同但策略不同的替换路由历史，同时在错误代码集合仅顺序不同时延续历史、验证 always 模式可越过 normal 预算、固定抖动和延迟上限、证明下游恢复顺序、证明取消与 dispose 会先排空已委托的恢复再达到完全停稳，并证明二者都会停止正在进行的退避等待。请求级覆盖会比较失败尝试与重试尝试的完整消息，并排除提供方错误文本和丢弃的部分输出。一个无密钥 headless `stream-json` 快照会通过组装后的应用执行失败、重试与成功流程，固定完整的 `llm/retry` 记录，并拒绝各次尝试之间出现任何模型消息变化。JSONL 与 SQLite 测试会往返读写不含 `Infinity` 的 always 事件；不变式测试会校验规范策略元组、将事件中的提供方绑定到请求头、将失败代码与延迟绑定到编码后的策略，并将重试编号绑定到活跃的提供方策略键；TUI 测试会渲染有限和无限上限。 +适配器测试会在提供方加载时校验嵌套策略，证明注册流程会捕获已配置策略和默认策略，并证明请求进行期间替换路由后仍会保留实际提供服务的策略。单元测试根据失败请求实际使用的注册项选择策略、分离不同提供方和策略变更后的重试历史、验证 always 模式可越过 normal 预算、固定抖动和延迟上限、证明下游恢复顺序、证明取消与 dispose 会先排空已委托的恢复再达到完全停稳，并证明二者都会停止正在进行的退避等待。请求级覆盖会比较失败尝试与重试尝试的完整消息，并排除提供方错误文本和丢弃的部分输出。一个无密钥 headless `stream-json` 快照会通过组装后的应用执行失败、重试与成功流程，固定完整的 `llm/retry` 记录，并拒绝各次尝试之间出现任何模型消息变化。JSONL 与 SQLite 测试会往返读写不含 `Infinity` 的 always 事件；不变式测试会将提供方标识绑定到请求头、校验失败事实和各模式的计时器边界，并将重试编号绑定到提供方策略键；TUI 测试会渲染有限和无限上限。 ## 后果 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 9fa6280870..e9dfe554be 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -700,7 +700,7 @@ Requires: `agents` export type Config = Readonly> ``` -Source: [`packages/llm/llm-retry/src/index.ts:46`](../packages/llm/llm-retry/src/index.ts) +Source: [`packages/llm/llm-retry/src/index.ts:45`](../packages/llm/llm-retry/src/index.ts) ## `@deepseek-ai/dsh-lsp-local` diff --git a/docs/persistence-catalog.md b/docs/persistence-catalog.md index 8cd8c78110..c183a78abf 100644 --- a/docs/persistence-catalog.md +++ b/docs/persistence-catalog.md @@ -294,7 +294,7 @@ Source: [`packages/hooks/hook-protocol/src/types.ts:31`](../packages/hooks/hook- } ``` -Source: [`packages/llm/llm-retry/src/index.ts:19`](../packages/llm/llm-retry/src/index.ts) +Source: [`packages/llm/llm-retry/src/index.ts:18`](../packages/llm/llm-retry/src/index.ts) ### `permission/*` diff --git a/packages/llm/llm-retry/README.md b/packages/llm/llm-retry/README.md index 2becaa7645..f6e5e9e524 100644 --- a/packages/llm/llm-retry/README.md +++ b/packages/llm/llm-retry/README.md @@ -8,7 +8,7 @@ Both modes use bounded exponential backoff with symmetric jitter. A valid `provi Before waiting, the plugin appends a non-surface `llm/retry` event with the provider, mode, canonical resolved-policy key, failure, and scheduled delay. The key includes every behavior-affecting field and sorts normal-mode codes because eligibility uses set membership. Retry numbers continue only across events with the same provider and complete policy key, so a route replacement with different limits, code membership, or backoff starts its own history. Normal events include the finite maximum; always events omit it, and UIs render `∞`. Cancellation and plugin disposal abort active backoff, drain active delegated recovery before applying the abort, and make a callback captured before disposal fail closed. -The separately published `./invariant` companion checks that every retry record names the current open turn and latest closed step, matches the failed request's durable provider, carries a producer-canonical policy key consistent with its mode and finite budget, binds normal failures and every scheduled delay to that policy, has a unique step record and correct provider-policy retry number, and carries a bounded timer delay. Full jitter may schedule zero milliseconds at its lower boundary. +The separately published `./invariant` companion checks that every retry record names the current open turn and latest closed step, matches the failed request's durable provider, carries non-empty provider and policy identities, has mode-specific bounds, a unique step record, the correct provider-policy retry number, and a bounded timer delay. Full jitter may schedule zero milliseconds at its lower boundary. ```yaml - name: '@deepseek-ai/dsh-llm-deepseek' diff --git a/packages/llm/llm-retry/src/index.ts b/packages/llm/llm-retry/src/index.ts index f759697b9e..03eeee25cf 100644 --- a/packages/llm/llm-retry/src/index.ts +++ b/packages/llm/llm-retry/src/index.ts @@ -11,7 +11,6 @@ import type { Agent, RequestError, RequestErrorDecision } from '@deepseek-ai/dsh import type { LlmFailure, ResolvedRetryPolicy } from '@deepseek-ai/dsh-llm' import type { SessionEvent } from '@deepseek-ai/dsh-session' import { providerForClosedStep } from './history.ts' -import { retryPolicyKey } from './policy-key.ts' declare module '@deepseek-ai/dsh-session' { interface SessionEventMap { @@ -84,6 +83,19 @@ function localDelay(config: ResolvedRetryPolicy, retry: number, random: () => nu return Math.min(exponential * jitter, config.maxDelayMs) } +function retryPolicyKey(policy: ResolvedRetryPolicy): string { + return policy.mode === 'always' + ? JSON.stringify([policy.mode, policy.initialDelayMs, policy.maxDelayMs, policy.jitterRatio]) + : JSON.stringify([ + policy.mode, + policy.maxRetries, + [...policy.retryableCodes].sort(), + policy.initialDelayMs, + policy.maxDelayMs, + policy.jitterRatio, + ]) +} + function cancellableDelay(delayMs: number, signal: AbortSignal): Promise { if (signal.aborted) return Promise.resolve(false) return new Promise((resolve) => { diff --git a/packages/llm/llm-retry/src/invariant.ts b/packages/llm/llm-retry/src/invariant.ts index 41fb5155b5..9ea66d9793 100644 --- a/packages/llm/llm-retry/src/invariant.ts +++ b/packages/llm/llm-retry/src/invariant.ts @@ -3,9 +3,9 @@ import type { Context } from 'cordis' import type { Session, SessionEvent } from '@deepseek-ai/dsh-session' import type { LlmFailure } from '@deepseek-ai/dsh-llm' +import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' import type { InvariantFailure, InvariantInstaller } from '@deepseek-ai/dsh-invariants' import { providerForClosedStep } from './history.ts' -import { parseRetryPolicyKey } from './policy-key.ts' import type {} from './index.ts' const PACKAGE_NAME = '@deepseek-ai/dsh-llm-retry' @@ -54,11 +54,10 @@ function validateRetry( fail('llm/retry retry must be a positive safe integer') } if (typeof provider !== 'string' || provider.length === 0) { - fail('llm/retry provider must be non-empty string') + fail('llm/retry provider must be a non-empty string') } - const keyedPolicy = parseRetryPolicyKey(policyKey) - if (keyedPolicy === undefined) { - fail('llm/retry policyKey must encode a canonical resolved policy') + if (typeof policyKey !== 'string' || policyKey.length === 0) { + fail('llm/retry policyKey must be a non-empty string') } switch (mode) { case 'normal': { @@ -66,29 +65,17 @@ function validateRetry( if (!Number.isSafeInteger(maxRetries) || maxRetries < 1 || retry > maxRetries) { fail(`llm/retry retry ${retry} must not exceed a positive safe maxRetries ${maxRetries}`) } - if (keyedPolicy.mode !== 'normal') { - fail(`llm/retry mode normal must match policyKey mode ${keyedPolicy.mode}`) - } - if (keyedPolicy.maxRetries !== maxRetries) { - fail(`llm/retry maxRetries ${maxRetries} must match policyKey`) - } - if (!keyedPolicy.retryableCodes.includes(failure.code)) { - fail(`llm/retry failure code ${failure.code} must be eligible under policyKey`) - } break } case 'always': - if (keyedPolicy.mode !== 'always') { - fail(`llm/retry mode always must match policyKey mode ${keyedPolicy.mode}`) - } if ('maxRetries' in event.data) fail('llm/retry always mode must omit maxRetries') break default: fail(`llm/retry mode must be normal or always, got ${String(mode)}`) } if (typeof delayMs !== 'number' || !Number.isFinite(delayMs) - || delayMs < 0 || delayMs > keyedPolicy.maxDelayMs) { - fail(`llm/retry delayMs must be a finite number within policyKey range 0..${keyedPolicy.maxDelayMs}`) + || delayMs < 0 || delayMs > MAX_TIMER_DELAY_MS) { + fail(`llm/retry delayMs must be a finite number within 0..${MAX_TIMER_DELAY_MS}`) } const turnStartIndex = history.findLastIndex(prior => diff --git a/packages/llm/llm-retry/src/policy-key.ts b/packages/llm/llm-retry/src/policy-key.ts deleted file mode 100644 index 2103d87920..0000000000 --- a/packages/llm/llm-retry/src/policy-key.ts +++ /dev/null @@ -1,100 +0,0 @@ -/** Canonical durable identity for resolved retry policies. @module @deepseek-ai/dsh-llm-retry/policy-key */ - -import type { ResolvedRetryBackoff, ResolvedRetryPolicy } from '@deepseek-ai/dsh-llm' -import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' - -function parseBackoff( - tuple: readonly unknown[], - offset: number, -): ResolvedRetryBackoff | undefined { - const initialDelayMs = tuple[offset] - const maxDelayMs = tuple[offset + 1] - const jitterRatio = tuple[offset + 2] - if (typeof initialDelayMs !== 'number' || !Number.isFinite(initialDelayMs) - || initialDelayMs <= 0 || initialDelayMs > MAX_TIMER_DELAY_MS - || typeof maxDelayMs !== 'number' || !Number.isFinite(maxDelayMs) - || maxDelayMs <= 0 || maxDelayMs > MAX_TIMER_DELAY_MS - || initialDelayMs > maxDelayMs - || typeof jitterRatio !== 'number' || !Number.isFinite(jitterRatio) - || jitterRatio < 0 || jitterRatio > 1) { - return undefined - } - return { initialDelayMs, maxDelayMs, jitterRatio } -} - -/** - * Derive the canonical durable key for one fully resolved provider policy. - * Retryable-code order is normalized because eligibility uses set membership. - * @param policy - immutable policy captured from the serving registration. - * @returns canonical JSON tuple containing every behavior-affecting field. - */ -export function retryPolicyKey(policy: ResolvedRetryPolicy): string { - if (policy.mode === 'always') { - return JSON.stringify([ - policy.mode, - policy.initialDelayMs, - policy.maxDelayMs, - policy.jitterRatio, - ]) - } - return JSON.stringify([ - policy.mode, - policy.maxRetries, - [...policy.retryableCodes].sort(), - policy.initialDelayMs, - policy.maxDelayMs, - policy.jitterRatio, - ]) -} - -/** - * Parse a producer-canonical policy key from durable input. - * @param value - untrusted persisted event field. - * @returns the resolved policy encoded by the key, or `undefined` for any non-canonical value. - */ -export function parseRetryPolicyKey(value: unknown): ResolvedRetryPolicy | undefined { - if (typeof value !== 'string' || value.length === 0) return undefined - let tuple: unknown - try { - tuple = JSON.parse(value) as unknown - } catch (_invalidPolicyKeyJson) { - return undefined - } - if (!Array.isArray(tuple)) return undefined - const items = tuple as readonly unknown[] - const mode = items[0] - let policy: ResolvedRetryPolicy - switch (mode) { - case 'always': { - if (items.length !== 4) return undefined - const backoff = parseBackoff(items, 1) - if (backoff === undefined) return undefined - policy = Object.freeze({ mode, ...backoff }) - break - } - case 'normal': { - if (items.length !== 6) return undefined - const maxRetries = items[1] - const retryableCodes = items[2] - const backoff = parseBackoff(items, 3) - if (!Number.isSafeInteger(maxRetries) || (maxRetries as number) < 0 - || !Array.isArray(retryableCodes) || retryableCodes.length === 0 - || (retryableCodes as readonly unknown[]) - .some(code => typeof code !== 'string' || code.length === 0) - || new Set(retryableCodes).size !== retryableCodes.length - || backoff === undefined) { - return undefined - } - policy = Object.freeze({ - mode, - maxRetries: maxRetries as number, - retryableCodes: Object.freeze(retryableCodes as string[]), - ...backoff, - }) - break - } - default: - return undefined - } - return retryPolicyKey(policy) === value ? policy : undefined -} diff --git a/packages/llm/llm-retry/tests/invariant.spec.ts b/packages/llm/llm-retry/tests/invariant.spec.ts index 056c0c1721..b9ec18c9f7 100644 --- a/packages/llm/llm-retry/tests/invariant.spec.ts +++ b/packages/llm/llm-retry/tests/invariant.spec.ts @@ -28,13 +28,26 @@ function closeStep(ctx: Context, id: string, turn = 1, step = 1) { } const failure = { message: 'provider busy', code: 'RATE_LIMIT', status: 429 } -const normalPolicyKey = (maxRetries: number): string => - `["normal",${maxRetries},["RATE_LIMIT"],1,10000,0]` -const alwaysPolicyKey = '["always",1,10000,0]' -const normal = { provider: 'mock', mode: 'normal' as const, policyKey: normalPolicyKey(2) } +const normal = { + provider: 'mock', + mode: 'normal' as const, + policyKey: 'normal-policy', + retry: 1, + maxRetries: 2, + delayMs: 1, + failure, +} +const always = { + provider: 'mock', + mode: 'always' as const, + policyKey: 'always-policy', + retry: 1, + delayMs: 1, + failure, +} describe('llm-retry invariants', () => { - it('has no provider without the requested closed step', () => { + it('has no provider without the requested closed step or a route marker', () => { expect(providerForClosedStep([], 1, 1)).toBeUndefined() expect(providerForClosedStep([{ type: 'step/end', @@ -42,82 +55,31 @@ describe('llm-retry invariants', () => { }] as never, 1, 1)).toBeUndefined() }) - it('inherits the latest provider across a turn boundary when the header is unchanged', () => { - expect(providerForClosedStep([ - { type: 'turn/start', data: { turn: 1 } }, - { - type: 'request/header', - data: { header: { config: { provider: 'prior' } } }, - }, - { type: 'turn/end', data: { turn: 1 } }, - { type: 'turn/start', data: { turn: 2 } }, - { type: 'step/end', data: { turn: 2, step: 1 } }, - ] as never, 2, 1)).toBe('prior') - }) - - it('accepts increasing retry records for successive closed steps and ignores unrelated events', async () => { + it('accepts bounded and unbounded records after successive closed steps', async () => { const ctx = await setup() const session = closeStep(ctx, 'retry-invariant-valid') + expect(() => { - session.append('llm/retry', { - turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 500, failure, - }) + session.append('llm/retry', { turn: 1, step: 1, ...normal }) session.append('step/start', { turn: 1, step: 2 }) session.append('step/end', { turn: 1, step: 2 }) session.append('llm/retry', { - turn: 1, step: 2, ...normal, retry: 2, maxRetries: 2, delayMs: 1_000, failure, - }) - const zeroDelay = closeStep(ctx, 'retry-invariant-zero-delay') - zeroDelay.append('llm/retry', { - turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(1), - retry: 1, maxRetries: 1, delayMs: 0, failure, + turn: 1, step: 2, ...normal, retry: 2, delayMs: 0, }) + const unbounded = closeStep(ctx, 'retry-invariant-always') + unbounded.append('llm/retry', { turn: 1, step: 1, ...always }) }).not.toThrow() expect(() => { ctx.emit('tools/change') }).not.toThrow() }) - it('accepts unbounded always records without serializing an infinite maximum', async () => { - const ctx = await setup() - const session = closeStep(ctx, 'retry-invariant-always') - expect(() => { - session.append('llm/retry', { - turn: 1, - step: 1, - provider: 'mock', - mode: 'always', - policyKey: alwaysPolicyKey, - retry: 1, - delayMs: 500, - failure, - }) - }).not.toThrow() - expect(() => { - session.append('llm/retry', { - turn: 1, - step: 1, - provider: 'mock', - mode: 'always', - policyKey: alwaysPolicyKey, - retry: 1, - maxRetries: 2, - delayMs: 500, - failure, - } as never) - }).toThrow(/always mode must omit maxRetries/) - }) - - it('validates complete durable failures before either retry mode uses them', async () => { + it('validates the complete durable failure payload', async () => { const ctx = await setup() const complete = closeStep(ctx, 'retry-invariant-complete-failure') expect(() => { complete.append('llm/retry', { turn: 1, step: 1, - provider: 'mock', - mode: 'always', - policyKey: alwaysPolicyKey, - retry: 1, - delayMs: 1, + ...always, failure: { message: 'provider busy', code: 'RATE_LIMIT', @@ -128,16 +90,8 @@ describe('llm-retry invariants', () => { }) }).not.toThrow() - const normalNull = closeStep(ctx, 'retry-invariant-normal-null-failure') - expect(() => { - normalNull.append('llm/retry', { - turn: 1, step: 1, ...normal, - retry: 1, maxRetries: 2, delayMs: 1, failure: null, - } as never) - }).toThrow(/failure must be an object/) - const invalidFailures: readonly [string, unknown, RegExp][] = [ - ['always-null', null, /failure must be an object/], + ['null', null, /failure must be an object/], ['message-type', { message: 1, code: 'RATE_LIMIT' }, /failure\.message/], ['message-empty', { message: '', code: 'RATE_LIMIT' }, /failure\.message/], ['code-type', { message: 'failed', code: 1 }, /failure\.code/], @@ -159,294 +113,124 @@ describe('llm-retry invariants', () => { ['request-id-empty', { message: 'failed', code: 'RATE_LIMIT', requestId: '' }, /failure\.requestId/], ] for (const [name, invalidFailure, message] of invalidFailures) { - const session = closeStep(ctx, `retry-invariant-${name}`) + const session = closeStep(ctx, `retry-invariant-failure-${name}`) expect(() => { session.append('llm/retry', { - turn: 1, - step: 1, - provider: 'mock', - mode: 'always', - policyKey: alwaysPolicyKey, - retry: 1, - delayMs: 1, - failure: invalidFailure, + turn: 1, step: 1, ...always, failure: invalidFailure, } as never) }).toThrow(message) } }) - it('binds event mode and finite budget to the canonical policy key', async () => { - const ctx = await setup() - const normalModeMismatch = closeStep(ctx, 'retry-invariant-normal-mode-key') - expect(() => { - normalModeMismatch.append('llm/retry', { - turn: 1, step: 1, ...normal, policyKey: alwaysPolicyKey, - retry: 1, maxRetries: 2, delayMs: 1, failure, - }) - }).toThrow(/mode normal must match policyKey mode always/) - - const alwaysModeMismatch = closeStep(ctx, 'retry-invariant-always-mode-key') - expect(() => { - alwaysModeMismatch.append('llm/retry', { - turn: 1, - step: 1, - provider: 'mock', - mode: 'always', - policyKey: normalPolicyKey(2), - retry: 1, - delayMs: 1, - failure, - }) - }).toThrow(/mode always must match policyKey mode normal/) - - const budgetMismatch = closeStep(ctx, 'retry-invariant-budget-key') - expect(() => { - budgetMismatch.append('llm/retry', { - turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3), - retry: 1, maxRetries: 2, delayMs: 1, failure, - }) - }).toThrow(/maxRetries 2 must match policyKey/) - }) - - it('binds the failure code and scheduled delay to the canonical policy key', async () => { - const ctx = await setup() - const ineligibleFailure = closeStep(ctx, 'retry-invariant-failure-code-key') - expect(() => { - ineligibleFailure.append('llm/retry', { - turn: 1, step: 1, ...normal, - retry: 1, maxRetries: 2, delayMs: 1, - failure: { message: 'authentication failed', code: 'AUTH', status: 401 }, - }) - }).toThrow(/failure code AUTH must be eligible under policyKey/) - - const overPolicyDelay = closeStep(ctx, 'retry-invariant-delay-key') - expect(() => { - overPolicyDelay.append('llm/retry', { - turn: 1, - step: 1, - provider: 'mock', - mode: 'always', - policyKey: '["always",1,1,0]', - retry: 1, - delayMs: 2, - failure, - }) - }).toThrow(/within policyKey range 0\.\.1/) - }) - - it('rejects empty providers and unknown modes from hostile durable input', async () => { - const ctx = await setup() - const emptyProvider = closeStep(ctx, 'retry-invariant-empty-provider') - expect(() => { - emptyProvider.append('llm/retry', { - turn: 1, - step: 1, - provider: '', - mode: 'always', - policyKey: alwaysPolicyKey, - retry: 1, - delayMs: 1, - failure, - }) - }).toThrow(/provider must be non-empty/) - - const unknownMode = closeStep(ctx, 'retry-invariant-unknown-mode') - expect(() => { - unknownMode.append('llm/retry', { - turn: 1, - step: 1, - provider: 'mock', - mode: 'sometimes', - policyKey: alwaysPolicyKey, - retry: 1, - delayMs: 1, - failure, - } as never) - }).toThrow(/mode must be normal or always/) - - const emptyPolicyKey = closeStep(ctx, 'retry-invariant-empty-policy-key') - expect(() => { - emptyPolicyKey.append('llm/retry', { - turn: 1, - step: 1, - provider: 'mock', - mode: 'always', - policyKey: '', - retry: 1, - delayMs: 1, - failure, - }) - }).toThrow(/policyKey must encode a canonical resolved policy/) - }) - it.each([ - [{ retry: 0, maxRetries: 2, delayMs: 1 }, /positive safe integer/], - [{ retry: 1.5, maxRetries: 2, delayMs: 1 }, /positive safe integer/], - [{ retry: 1, maxRetries: 0, delayMs: 1 }, /positive safe maxRetries/], - [{ retry: 1, maxRetries: 1.5, delayMs: 1 }, /positive safe maxRetries/], - [{ retry: 3, maxRetries: 2, delayMs: 1 }, /must not exceed/], - [{ retry: 1, maxRetries: 2, delayMs: -1 }, /delayMs/], - [{ retry: 1, maxRetries: 2, delayMs: MAX_TIMER_DELAY_MS + 1 }, /delayMs/], - ])('rejects invalid retry bounds %#', async (data, message) => { + ['retry-zero', { ...normal, retry: 0 }, /positive safe integer/], + ['retry-fraction', { ...normal, retry: 1.5 }, /positive safe integer/], + ['max-zero', { ...normal, maxRetries: 0 }, /positive safe maxRetries/], + ['max-fraction', { ...normal, maxRetries: 1.5 }, /positive safe maxRetries/], + ['over-budget', { ...normal, retry: 3 }, /must not exceed/], + ['always-maximum', { ...always, maxRetries: 2 }, /always mode must omit maxRetries/], + ['unknown-mode', { ...always, mode: 'sometimes' }, /mode must be normal or always/], + ['empty-provider', { ...always, provider: '' }, /provider must be a non-empty string/], + ['empty-policy-key', { ...always, policyKey: '' }, /policyKey must be a non-empty string/], + ['delay-negative', { ...normal, delayMs: -1 }, /delayMs/], + ['delay-overflow', { ...normal, delayMs: MAX_TIMER_DELAY_MS + 1 }, /delayMs/], + ['delay-type', { ...normal, delayMs: '1' }, /delayMs/], + ])('rejects invalid retry data: %s', async (name, data, message) => { const ctx = await setup() - const session = closeStep(ctx, `retry-invariant-bounds-${data.retry}-${data.maxRetries}-${data.delayMs}`) + const session = closeStep(ctx, `retry-invariant-${name}`) expect(() => { - session.append('llm/retry', { turn: 1, step: 1, ...normal, ...data, failure }) + session.append('llm/retry', { turn: 1, step: 1, ...data } as never) }).toThrow(message) }) - it('rejects retry records outside the matching closed-step boundary', async () => { + it('rejects records outside the latest closed step of an open turn', async () => { const ctx = await setup() const absent = ctx.sessions.create(SessionId('retry-invariant-no-turn')) expect(() => { - absent.append('llm/retry', { - turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, - }) + absent.append('llm/retry', { turn: 1, step: 1, ...normal }) }).toThrow(/inside an open turn/) const wrongTurn = closeStep(ctx, 'retry-invariant-wrong-turn') expect(() => { - wrongTurn.append('llm/retry', { - turn: 2, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, - }) + wrongTurn.append('llm/retry', { turn: 2, step: 1, ...normal }) }).toThrow(/open turn is 1/) const openStep = ctx.sessions.create(SessionId('retry-invariant-open-step')) openStep.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) openStep.append('step/start', { turn: 1, step: 1 }) expect(() => { - openStep.append('llm/retry', { - turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, - }) + openStep.append('llm/retry', { turn: 1, step: 1, ...normal }) }).toThrow(/step 1 is still open/) const noStep = ctx.sessions.create(SessionId('retry-invariant-no-step')) noStep.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) expect(() => { - noStep.append('llm/retry', { - turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, - }) + noStep.append('llm/retry', { turn: 1, step: 1, ...normal }) }).toThrow(/latest closed step is undefined/) const wrongStep = closeStep(ctx, 'retry-invariant-wrong-step') expect(() => { - wrongStep.append('llm/retry', { - turn: 1, step: 2, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, - }) + wrongStep.append('llm/retry', { turn: 1, step: 2, ...normal }) }).toThrow(/latest closed step is 1/) const closedTurn = closeStep(ctx, 'retry-invariant-closed-turn') closedTurn.append('turn/end', { turn: 1, reason: { kind: 'aborted' } }) expect(() => { - closedTurn.append('llm/retry', { - turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, - }) + closedTurn.append('llm/retry', { turn: 1, step: 1, ...normal }) }).toThrow(/inside an open turn/) }) - it('binds the policy provider to the failed step rather than a later header', async () => { + it('rejects a second retry record for the same step', async () => { + const ctx = await setup() + const session = closeStep(ctx, 'retry-invariant-duplicate') + session.append('llm/retry', { turn: 1, step: 1, ...normal }) + + expect(() => { + session.append('llm/retry', { turn: 1, step: 1, ...normal, retry: 2 }) + }).toThrow(/duplicates the retry record/) + }) + + it('binds retry numbering to the provider policy and resets it after success', async () => { + const ctx = await setup() + const mismatch = closeStep(ctx, 'retry-invariant-numbering') + mismatch.append('llm/retry', { turn: 1, step: 1, ...normal }) + mismatch.append('step/start', { turn: 1, step: 2 }) + mismatch.append('step/end', { turn: 1, step: 2 }) + expect(() => { + mismatch.append('llm/retry', { turn: 1, step: 2, ...normal, retry: 1 }) + }).toThrow(/must equal provider policy retry 2/) + + const reset = closeStep(ctx, 'retry-invariant-reset') + reset.append('llm/retry', { turn: 1, step: 1, ...normal }) + reset.append('step/start', { turn: 1, step: 2 }) + reset.append('assistant/message', { + turn: 1, + step: 2, + content: [{ type: 'text', text: 'success' }], + provenance: { provider: 'mock', model: 'mock' }, + }, { surfaceOp: 'append' }) + reset.append('step/end', { turn: 1, step: 2 }) + reset.append('step/start', { turn: 1, step: 3 }) + reset.append('step/end', { turn: 1, step: 3 }) + expect(() => { + reset.append('llm/retry', { turn: 1, step: 3, ...normal }) + }).not.toThrow() + }) + + it('rejects a provider that does not match the failed request route', async () => { const ctx = await setup() const session = closeStep(ctx, 'retry-invariant-provider') - session.append('request/header', { - header: { config: { provider: 'other', model: 'mock' } }, - reason: 'change', - }) expect(() => { - session.append('llm/retry', { - turn: 1, - step: 1, - provider: 'mock', - mode: 'always', - policyKey: alwaysPolicyKey, - retry: 1, - delayMs: 1, - failure, - }) - }).not.toThrow() - - const mismatch = closeStep(ctx, 'retry-invariant-provider-mismatch') - expect(() => { - mismatch.append('llm/retry', { - turn: 1, - step: 1, - provider: 'other', - mode: 'always', - policyKey: alwaysPolicyKey, - retry: 1, - delayMs: 1, - failure, - }) + session.append('llm/retry', { turn: 1, step: 1, ...always, provider: 'other' }) }).toThrow(/does not match the failed request provider mock/) }) - it('accepts a current-turn retry under an unchanged prior provider route', async () => { - const ctx = await setup() - const session = closeStep(ctx, 'retry-invariant-prior-route') - session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) - session.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } }) - session.append('step/start', { turn: 2, step: 1 }) - session.append('step/end', { turn: 2, step: 1 }) - expect(() => { - session.append('llm/retry', { - turn: 2, - step: 1, - provider: 'mock', - mode: 'always', - policyKey: alwaysPolicyKey, - retry: 1, - delayMs: 1, - failure, - }) - }).not.toThrow() - }) - - it('rejects non-numeric durable delays', async () => { - const ctx = await setup() - const session = closeStep(ctx, 'retry-invariant-delay-type') - expect(() => { - session.append('llm/retry', { - turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: '1', failure, - } as never) - }).toThrow(/delayMs must be a finite number/) - }) - - it('rejects duplicate and non-increasing retry records', async () => { - const ctx = await setup() - const duplicate = closeStep(ctx, 'retry-invariant-duplicate') - duplicate.append('llm/retry', { - turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3), - retry: 1, maxRetries: 3, delayMs: 1, failure, - }) - expect(() => { - duplicate.append('llm/retry', { - turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3), - retry: 2, maxRetries: 3, delayMs: 1, failure, - }) - }).toThrow(/duplicates the retry record/) - - const nonIncreasing = closeStep(ctx, 'retry-invariant-non-increasing') - nonIncreasing.append('llm/retry', { - turn: 1, step: 1, ...normal, policyKey: normalPolicyKey(3), - retry: 1, maxRetries: 3, delayMs: 1, failure, - }) - nonIncreasing.append('step/start', { turn: 1, step: 2 }) - nonIncreasing.append('step/end', { turn: 1, step: 2 }) - expect(() => { - nonIncreasing.append('llm/retry', { - turn: 1, step: 2, ...normal, policyKey: normalPolicyKey(3), - retry: 1, maxRetries: 3, delayMs: 1, failure, - }) - }).toThrow(/must equal provider policy retry 2/) - }) - it('validates existing histories on late registration', async () => { const ctx = new Context() await ctx.plugin(SessionStore) const session = ctx.sessions.create(SessionId('retry-invariant-late')) session.append('step/end', { turn: 1, step: 1 }) - session.append('llm/retry', { - turn: 1, step: 1, ...normal, retry: 1, maxRetries: 2, delayMs: 1, failure, - }) + session.append('llm/retry', { turn: 1, step: 1, ...normal }) await ctx.plugin(InvariantService) await expect(ctx.plugin(RetryInvariant)).rejects.toThrow(/inside an open turn/) }) diff --git a/packages/llm/llm-retry/tests/policy-key.spec.ts b/packages/llm/llm-retry/tests/policy-key.spec.ts deleted file mode 100644 index 99c7da7802..0000000000 --- a/packages/llm/llm-retry/tests/policy-key.spec.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { resolveRetryPolicy } from '@deepseek-ai/dsh-llm' -import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' -import { parseRetryPolicyKey, retryPolicyKey } from '../src/policy-key.ts' - -describe('retry policy durable key', () => { - it('includes every policy field while normalizing code-set order', () => { - const first = resolveRetryPolicy({ - mode: 'normal', - maxRetries: 4, - retryableCodes: ['SERVER', 'RATE_LIMIT'], - backoff: { initialDelayMs: 3, maxDelayMs: 9, jitterRatio: 0.25 }, - }, 'first') - const reordered = resolveRetryPolicy({ - mode: 'normal', - maxRetries: 4, - retryableCodes: ['RATE_LIMIT', 'SERVER'], - backoff: { initialDelayMs: 3, maxDelayMs: 9, jitterRatio: 0.25 }, - }, 'reordered') - const key = retryPolicyKey(first) - - expect(key).toBe('["normal",4,["RATE_LIMIT","SERVER"],3,9,0.25]') - expect(retryPolicyKey(reordered)).toBe(key) - expect(parseRetryPolicyKey(key)).toEqual(reordered) - }) - - it('round-trips always mode', () => { - const policy = resolveRetryPolicy({ - mode: 'always', - backoff: { initialDelayMs: 2, maxDelayMs: 8, jitterRatio: 1 }, - }, 'always') - const key = retryPolicyKey(policy) - - expect(key).toBe('["always",2,8,1]') - expect(parseRetryPolicyKey(key)).toEqual(policy) - }) - - it.each([ - undefined, - '', - '{', - '{}', - '["sometimes",1,2,0]', - '["always",1,2]', - '["always","1",2,0]', - '["always",1e400,2,0]', - '["always",0,2,0]', - `["always",${MAX_TIMER_DELAY_MS + 1},${MAX_TIMER_DELAY_MS + 1},0]`, - '["always",1,"2",0]', - '["always",1,1e400,0]', - '["always",1,0,0]', - `["always",1,${MAX_TIMER_DELAY_MS + 1},0]`, - '["always",2,1,0]', - '["always",1,2,"0"]', - '["always",1,2,1e400]', - '["always",1,2,-0.1]', - '["always",1,2,1.1]', - '["normal",2,["SERVER"],1,2]', - '["normal","2",["SERVER"],1,2,0]', - '["normal",-1,["SERVER"],1,2,0]', - '["normal",2,"SERVER",1,2,0]', - '["normal",2,[],1,2,0]', - '["normal",2,[1],1,2,0]', - '["normal",2,[""],1,2,0]', - '["normal",2,["SERVER","SERVER"],1,2,0]', - '["normal",2,["SERVER"],2,1,0]', - '["normal",2,["SERVER","RATE_LIMIT"],1,2,0]', - ])('rejects non-canonical durable input %#', (value) => { - expect(parseRetryPolicyKey(value)).toBeUndefined() - }) -}) diff --git a/packages/llm/llm-retry/tests/retry.spec.ts b/packages/llm/llm-retry/tests/retry.spec.ts index 303c94efa4..7ed040f06a 100644 --- a/packages/llm/llm-retry/tests/retry.spec.ts +++ b/packages/llm/llm-retry/tests/retry.spec.ts @@ -181,19 +181,14 @@ describe('provider-routed retry policy', () => { new LlmError('busy', 'RATE_LIMIT', { status: 429 }), textResponse('done'), ]) - ;({ ctx: context } = await harness(adapter, {}, undefined, { random: () => 0.5 })) + ;({ ctx: context } = await harness(adapter, { + mock: normalConfig({ retryableCodes: ['SERVER', 'RATE_LIMIT'] }), + }, undefined, { random: () => 0.5 })) const agent = context.agentLoop.create(SessionId('retry-success'), { provider: 'mock', model: 'mock', }) - const scheduled = new Promise>((resolve) => { - const dispose = context?.on('session/event', (session, event) => { - if (session === agent.session && event.type === 'llm/retry') { - dispose?.() - resolve(event) - } - }) - }) + const scheduled = waitForRetry(context, agent, 1) agent.followup([{ type: 'text', text: 'go' }]) const event = await scheduled @@ -203,7 +198,7 @@ describe('provider-routed retry policy', () => { step: 1, provider: 'mock', mode: 'normal', - policyKey: '["normal",2,["EMPTY_RESPONSE","RATE_LIMIT","SERVER","TIMEOUT","TRANSPORT"],500,10000,0.1]', + policyKey: '["normal",2,["RATE_LIMIT","SERVER"],500,10000,0]', retry: 1, maxRetries: 2, delayMs: 500, @@ -227,45 +222,6 @@ describe('provider-routed retry policy', () => { }) }) - it('retries a later turn under its unchanged provider header', async () => { - vi.useFakeTimers() - const adapter = new ScriptedAdapter([ - textResponse('first turn'), - new LlmError('busy on second turn', 'RATE_LIMIT'), - textResponse('second turn recovered'), - ]) - ;({ ctx: context } = await harness(adapter, { mock: normalConfig({ - backoff: { initialDelayMs: 1, maxDelayMs: 1 }, - }) })) - const agent = context.agentLoop.create(SessionId('retry-later-turn'), { - provider: 'mock', - model: 'mock', - }) - - const firstIdle = waitForIdle(context, agent) - agent.followup([{ type: 'text', text: 'first' }]) - await firstIdle - expect(agent.session.events.filter(event => event.type === 'request/header')).toHaveLength(1) - - const scheduled = waitForRetry(context, agent, 1) - agent.followup([{ type: 'text', text: 'second' }]) - expect((await scheduled).data).toMatchObject({ - turn: 2, - step: 1, - provider: 'mock', - }) - const secondIdle = waitForIdle(context, agent) - await vi.advanceTimersByTimeAsync(1) - await secondIdle - - expect(adapter.requests).toHaveLength(3) - expect(agent.session.events.filter(event => event.type === 'request/header')).toHaveLength(1) - expect(agent.session.deriveMessages().at(-1)).toMatchObject({ - role: 'assistant', - content: [{ type: 'text', text: 'second turn recovered' }], - }) - }) - it('retries an EMPTY_RESPONSE error finish under the default retryable codes', async () => { vi.useFakeTimers() const adapter = new ScriptedAdapter([ @@ -556,8 +512,50 @@ describe('provider-routed retry policy', () => { expect(adapter.requests.map(request => request.provider)).toEqual(['other', 'other']) }) + it('keeps finite retry budgets scoped to the failed provider', async () => { + vi.useFakeTimers() + const adapter = new ScriptedAdapter([ + new LlmError('mock failed', 'SERVER'), + new LlmError('other failed', 'SERVER'), + textResponse('other recovered'), + ]) + ;({ ctx: context } = await harness(adapter, { + mock: normalConfig({ + maxRetries: 1, + backoff: { initialDelayMs: 1, maxDelayMs: 1 }, + }), + other: normalConfig({ + maxRetries: 1, + backoff: { initialDelayMs: 1, maxDelayMs: 1 }, + }), + }, (ctx) => { + ctx.on('agent/request', async (_agent, _turn, step, config) => ({ + ...config, + provider: step === 1 ? 'mock' : 'other', + })) + })) + const agent = context.agentLoop.create(SessionId('retry-provider-budgets'), { + provider: 'mock', + model: 'mock', + }) + const idle = waitForIdle(context, agent) + + agent.followup([{ type: 'text', text: 'switch provider after failure' }]) + await vi.runAllTimersAsync() + await idle + + expect(adapter.requests.map(request => request.provider)).toEqual(['mock', 'other', 'other']) + expect(agent.session.events.filter(event => event.type === 'llm/retry').map(event => ({ + provider: event.data.provider, + retry: event.data.retry, + }))).toEqual([ + { provider: 'mock', retry: 1 }, + { provider: 'other', retry: 1 }, + ]) + }) + it.each(['thrown', 'in-band'] as const)( - 'uses the serving registration policy when an in-flight route is replaced after a %s failure', + 'uses the serving registration policy and resets changed-policy history after a %s failure', async (failureKind) => { vi.useFakeTimers() const entered = Promise.withResolvers() @@ -590,23 +588,40 @@ describe('provider-routed retry policy', () => { await entered.promise mounted.disposeAdapter() - const replacement = new ScriptedAdapter([textResponse('replacement recovered')]) - replacement.configureRetryPolicies({ mock: normalConfig({ maxRetries: 0 }) }) + const replacement = new ScriptedAdapter([ + new LlmError('replacement failed', 'AUTH'), + textResponse('replacement recovered'), + ]) + replacement.configureRetryPolicies({ mock: alwaysConfig({ + initialDelayMs: 3, + maxDelayMs: 3, + }) }) context.llm.registerAdapter(['mock'], replacement) release.resolve(undefined) - expect((await scheduled).data).toMatchObject({ + const firstEvent = await scheduled + expect(firstEvent.data).toMatchObject({ provider: 'mock', mode: 'always', retry: 1, delayMs: 1, }) + const replacementScheduled = waitForRetry(context, agent, 1) const idle = waitForIdle(context, agent) await vi.advanceTimersByTimeAsync(1) + const replacementEvent = await replacementScheduled + expect(replacementEvent.data).toMatchObject({ + provider: 'mock', + mode: 'always', + retry: 1, + delayMs: 3, + }) + expect(replacementEvent.data.policyKey).not.toBe(firstEvent.data.policyKey) + await vi.advanceTimersByTimeAsync(3) await idle expect(oldAdapter.requests).toHaveLength(1) - expect(replacement.requests).toHaveLength(1) + expect(replacement.requests).toHaveLength(2) expect(agent.session.deriveMessages().at(-1)).toMatchObject({ role: 'assistant', content: [{ type: 'text', text: 'replacement recovered' }], @@ -614,110 +629,6 @@ describe('provider-routed retry policy', () => { }, ) - it('starts a new retry history when a same-mode route replacement changes policy', async () => { - vi.useFakeTimers() - const oldAdapter = new ScriptedAdapter([ - new LlmError('old route failed', 'AUTH'), - ]) - const mounted = await harness(oldAdapter, { mock: alwaysConfig({ - initialDelayMs: 1, - maxDelayMs: 1, - }) }) - context = mounted.ctx - const agent = context.agentLoop.create(SessionId('retry-policy-replacement'), { - provider: 'mock', - model: 'mock', - }) - const first = waitForRetry(context, agent, 1) - - agent.followup([{ type: 'text', text: 'replace policy between attempts' }]) - expect((await first).data).toMatchObject({ - mode: 'always', - retry: 1, - delayMs: 1, - }) - - mounted.disposeAdapter() - const replacement = new ScriptedAdapter([ - new LlmError('replacement failed', 'AUTH'), - textResponse('replacement recovered'), - ]) - replacement.configureRetryPolicies({ mock: alwaysConfig({ - initialDelayMs: 3, - maxDelayMs: 9, - }) }) - context.llm.registerAdapter(['mock'], replacement) - - const second = waitForRetry(context, agent, 1) - await vi.advanceTimersByTimeAsync(1) - expect((await second).data).toMatchObject({ - mode: 'always', - retry: 1, - delayMs: 3, - }) - - const idle = waitForIdle(context, agent) - await vi.advanceTimersByTimeAsync(3) - await idle - - expect(oldAdapter.requests).toHaveLength(1) - expect(replacement.requests).toHaveLength(2) - expect(agent.session.events.filter(event => event.type === 'llm/retry').map(event => ({ - policyKey: event.data.policyKey, - retry: event.data.retry, - }))).toEqual([ - { policyKey: '["always",1,1,0]', retry: 1 }, - { policyKey: '["always",3,9,0]', retry: 1 }, - ]) - }) - - it('continues retry history when a replacement only reorders retryable codes', async () => { - vi.useFakeTimers() - const oldAdapter = new ScriptedAdapter([ - new LlmError('old route failed', 'SERVER'), - ]) - const mounted = await harness(oldAdapter, { mock: normalConfig({ - maxRetries: 2, - retryableCodes: ['SERVER', 'RATE_LIMIT'], - backoff: { initialDelayMs: 1, maxDelayMs: 4 }, - }) }) - context = mounted.ctx - const agent = context.agentLoop.create(SessionId('retry-policy-code-order'), { - provider: 'mock', - model: 'mock', - }) - const first = waitForRetry(context, agent, 1) - - agent.followup([{ type: 'text', text: 'replace equivalent policy between attempts' }]) - const firstEvent = await first - expect(firstEvent.data.delayMs).toBe(1) - - mounted.disposeAdapter() - const replacement = new ScriptedAdapter([ - new LlmError('replacement failed', 'SERVER'), - textResponse('replacement recovered'), - ]) - replacement.configureRetryPolicies({ mock: normalConfig({ - maxRetries: 2, - retryableCodes: ['RATE_LIMIT', 'SERVER'], - backoff: { initialDelayMs: 1, maxDelayMs: 4 }, - }) }) - context.llm.registerAdapter(['mock'], replacement) - - const second = waitForRetry(context, agent, 2) - await vi.advanceTimersByTimeAsync(1) - const secondEvent = await second - expect(secondEvent.data).toMatchObject({ retry: 2, delayMs: 2 }) - expect(secondEvent.data.policyKey).toBe(firstEvent.data.policyKey) - - const idle = waitForIdle(context, agent) - await vi.advanceTimersByTimeAsync(2) - await idle - - expect(oldAdapter.requests).toHaveLength(1) - expect(replacement.requests).toHaveLength(2) - }) - it('keeps always mode unbounded while preserving cancellable jittered backoff', async () => { vi.useFakeTimers() const adapter = new ScriptedAdapter([ From 892577ee526545a40ebeb612f753fac2758fad8d Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:38:06 +0800 Subject: [PATCH 11/11] test(llm-retry): cover incomplete retry predecessors --- .../llm/llm-retry/tests/invariant.spec.ts | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/packages/llm/llm-retry/tests/invariant.spec.ts b/packages/llm/llm-retry/tests/invariant.spec.ts index a0d0bdba35..978a9fc9a8 100644 --- a/packages/llm/llm-retry/tests/invariant.spec.ts +++ b/packages/llm/llm-retry/tests/invariant.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { Context } from 'cordis' -import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' +import SessionStore, { SessionId, type Session } from '@deepseek-ai/dsh-session' import { ProviderRequestId } from '@deepseek-ai/dsh-llm' import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' import InvariantService from '@deepseek-ai/dsh-invariants' @@ -27,6 +27,17 @@ function closeStep(ctx: Context, id: string, turn = 1, step = 1) { return session } +function appendRetryTurn(session: Session, turn: number) { + session.append('turn/start', { turn, trigger: { kind: 'retry' } }) + session.append('step/start', { turn, step: 1 }) + session.append('request/header', { + header: { config: { provider: 'mock', model: 'mock' } }, + reason: 'initial', + }) + session.append('step/end', { turn, step: 1 }) + session.append('llm/retry', { turn, step: 1, ...normal }) +} + const failure = { message: 'provider busy', code: 'RATE_LIMIT', status: 429 } const normal = { provider: 'mock', @@ -225,6 +236,36 @@ describe('llm-retry invariants', () => { }).not.toThrow() }) + it('starts a fresh retry chain after incomplete predecessor boundaries', async () => { + const ctx = new Context() + await ctx.plugin(SessionStore) + + const missingEnd = ctx.sessions.create(SessionId('retry-invariant-missing-end')) + missingEnd.append('user/message', { + content: [{ type: 'text', text: 'idle context' }], + source: { kind: 'user' }, + }, { surfaceOp: 'append' }) + appendRetryTurn(missingEnd, 2) + + const nonFailureEnd = ctx.sessions.create(SessionId('retry-invariant-non-failure-end')) + nonFailureEnd.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + nonFailureEnd.append('user/message', { + content: [{ type: 'text', text: 'idle context' }], + source: { kind: 'user' }, + }, { surfaceOp: 'append' }) + appendRetryTurn(nonFailureEnd, 2) + + const missingStart = ctx.sessions.create(SessionId('retry-invariant-missing-start')) + missingStart.append('turn/end', { + turn: 1, + reason: { kind: 'error', step: 1, failure }, + }) + appendRetryTurn(missingStart, 2) + + await ctx.plugin(InvariantService) + await expect(ctx.plugin(RetryInvariant)).resolves.toBeDefined() + }) + it('rejects a provider that does not match the failed request route', async () => { const ctx = await setup() const session = closeStep(ctx, 'retry-invariant-provider')