From a4091daa3d7bf3f9f9a958969ae45878e57e5d83 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 25 Jun 2026 13:59:34 +0800 Subject: [PATCH 01/16] docs: propose web capability seam --- docs/rfc/README.md | 1 + .../2026-06-24-web-capability-seam.md | 385 ++++++++++++++++++ 2 files changed, 386 insertions(+) create mode 100644 docs/rfc/proposed/architecture/2026-06-24-web-capability-seam.md diff --git a/docs/rfc/README.md b/docs/rfc/README.md index a731aa8ff9..73a2675caa 100644 --- a/docs/rfc/README.md +++ b/docs/rfc/README.md @@ -59,6 +59,7 @@ Do NOT write one for a mechanical or local choice (a variable name, a one-file r |---|---| | [Runtime schemas for the event vocabulary (Zod vs the merge-extensible-map pattern)](proposed/architecture/2026-06-16-typed-event-schemas.md) | 2026-06-16 | | [Extract a generic long-running tool runtime](proposed/architecture/2026-06-20-generic-long-running-tool-runtime.md) | 2026-06-20 | +| [Web capability seam - provider registry and model-facing web tools](proposed/architecture/2026-06-24-web-capability-seam.md) | 2026-06-24 | ### Process diff --git a/docs/rfc/proposed/architecture/2026-06-24-web-capability-seam.md b/docs/rfc/proposed/architecture/2026-06-24-web-capability-seam.md new file mode 100644 index 0000000000..e97e8380da --- /dev/null +++ b/docs/rfc/proposed/architecture/2026-06-24-web-capability-seam.md @@ -0,0 +1,385 @@ +# RFC: Web capability seam - stable tools over multiple providers + +Status: proposed + +## Problem + +The harness needs model-facing web tools without binding the model contract to one vendor's API shape. Search is the immediate pressure point: the first version should support at least Exa search and Perplexity search — two deliberately different provider shapes (Exa returns a flat `results[]` of `{title, url, highlights, publishedDate}`; Perplexity returns a generated answer plus citations), which is what proves the normalized seam does not just mirror one vendor. Fetch is a separate capability: an anonymous public HTTP(S) fetch backend has transport, security, redirect, decoding, and size-limit concerns that are not the same as provider-backed search. + +The model-facing surface should stay stable while backends change. A search provider swap should not change how the model asks for a query, and a fetch implementation swap should not change how the model asks for a URL. Conversely, a provider package should not expose its own model-facing tool schema just because it has extra provider-specific knobs. + +Putting search and fetch directly in `dsh-tool-web` would make the model-facing tool own provider selection, backend request mapping, transport policy, result normalization, prompt guidance, presentation, and schema registration at once. Letting each provider register its own tool has the opposite problem: tool availability, names, descriptions, and parameters would depend on whichever provider packages happen to load, and provider-specific fields would leak into the model contract. + +There is also a provider-selection question. Existing `tool-bash` and `tool-fs` can rely on Cordis `inject` because there is one backend service key. Web has two independent capabilities (`search` and `fetch`) and potentially multiple providers per capability. `inject: ['web']` proves the seam exists; it does not prove a usable search or fetch provider exists, and it does not define which provider should win when several are registered. + +## Proposal + +Introduce web access as a first-class capability seam following [the capability-seam RFC](../../implemented/architecture/2026-06-13-capability-seams.md): + +1. `@deepseek-ai/dsh-web` (`packages/web/web`) owns `ctx.web`, provider registration, provider selection, shared request/result vocabulary, and web-specific errors. +2. Provider packages implement concrete backends and register capabilities with `ctx.web`, for example `@deepseek-ai/dsh-web-search-exa`, `@deepseek-ai/dsh-web-search-perplexity`, and `@deepseek-ai/dsh-web-fetch-local`. +3. `@deepseek-ai/dsh-tool-web` (`packages/web/tool-web`) owns the model-facing `web_search` and `web_fetch` tool schemas, prompt sections, argument validation, result formatting, and tool-owned presentation over `ctx.web`. + +Providers do not register tools. Providers register capabilities. `dsh-tool-web` is the only owner of model-facing names, descriptions, prompt guidance, JSON schemas, and presentation. + +Search and fetch are separate capabilities and separate model-facing tools, but they are deliberately one seam. `ctx.web` is a single web-access middle layer between provider packages on one side and the tool consumer on the other: one service to inject, one provider-selection policy owner, one abort/error vocabulary, one place a product configures "how this harness reaches the web." The two halves do not share a request schema and have no shared business logic — search normalizes provider-backed discovery into a portable result with optional answer text and citeable sources, while fetch retrieves a concrete public HTTP(S) URL and returns a status code plus bounded decoded content — but they are parallel registries on one capability surface, not two surfaces. The cost is a `WebService` whose registry/status/exec methods come in `Search`/`Fetch` pairs; that parallelism is intentional, not a missed extraction. Splitting into `dsh-search` and `dsh-fetch` is the rejected alternative below. + +`dsh-tool-web` should register model-facing web tools when the product has enabled those tools and the `ctx.web` seam is present. Backend availability is an execution-time concern, not a schema-registration concern: + +- Register `web_search` when web search is enabled for the product/app. +- Register `web_fetch` when web fetch is enabled for the product/app. +- Do not unregister a tool merely because its selected provider is missing, misconfigured, missing credentials, ambiguous, or temporarily unavailable. +- Resolve the provider at execution time, and return a structured `WebError` when the selected capability cannot run. + +This keeps the model schema stable without making plugin load order, credential state, or HMR timing part of the model-facing contract. If web search is enabled but no usable search provider exists, `web_search` remains visible and execution fails with a structured `WebError` such as `WEB_PROVIDER_UNAVAILABLE` or `WEB_PROVIDER_CONFIGURED_UNAVAILABLE`. If a provider appears after `dsh-tool-web`, the next execution can use it without changing the schema. If a provider disappears mid-call, execution fails with a structured `WebError` instead of silently choosing another provider or falling through to `UNKNOWN_TOOL`. + +The first version's provider-change signal is intentionally small. `web/providers-change` has no payload, carries no capability graph, and does not expose provider metadata. It means only "the provider registry changed; observers may recompute status from `ctx.web`." `searchStatus()` and `fetchStatus()` remain derived, not stored, and they are diagnostics plus execution-resolution inputs rather than tool-schema visibility switches. + +## Package topology + +The three-package interface/implementation/consumer split follows bash and filesystem, but the *interface* package is closer to the LLM seam. `LlmService` (`packages/llm/llm/src/index.ts`) is a name-keyed provider registry: `registerAdapter(models, adapter)` stores adapters in a `Map`, returns a disposer, throws `DUPLICATE_ADAPTER` on duplicate keys, and throws `NO_ADAPTER` at resolution time. `ctx.web` follows that registry shape, but has two capability kinds and one small selection-status layer so diagnostics and execution can explain why a search or fetch capability can or cannot run. + +The dependency direction mirrors bash and filesystem: + +```text +@deepseek-ai/dsh-tool-web --depends on--> @deepseek-ai/dsh-web <--depends on-- @deepseek-ai/dsh-web-search-exa + consumer interface implementation + <--depends on-- @deepseek-ai/dsh-web-search-perplexity + implementation + <--depends on-- @deepseek-ai/dsh-web-fetch-local + implementation +``` + +At runtime, provider packages register capabilities with `ctx.web`; `tool-web` reads capability status and registers stable tools with `ctx.tools`: + +```mermaid +flowchart LR + exa["@deepseek-ai/dsh-web-search-exa"] -->|registerSearchProvider| web["@deepseek-ai/dsh-web / ctx.web"] + perplexity["@deepseek-ai/dsh-web-search-perplexity"] -->|registerSearchProvider| web + fetchLocal["@deepseek-ai/dsh-web-fetch-local"] -->|registerFetchProvider| web + toolWeb["@deepseek-ai/dsh-tool-web"] -->|searchStatus/fetchStatus| web + toolWeb -->|ctx.tools.register| webSearch["tool: web_search"] + toolWeb -->|ctx.tools.register| webFetch["tool: web_fetch"] +``` + +`@deepseek-ai/dsh-web` depends only on Cordis and low-level harness support. It declares `ctx.web`, provider interfaces, request/result types, status types, and error codes. It does not import tool, agent, session, LLM, or provider packages. + +Provider packages depend on `@deepseek-ai/dsh-web` and Cordis. They own credentials, endpoint config, provider-specific request mapping, provider-specific response parsing, and provider-specific error translation into `WebError`. They issue network requests with the platform-native `fetch` (Node 24), mirroring `@deepseek-ai/dsh-llm-deepseek`'s adapter, NOT a cordis HTTP-client service (`ctx.http`/`@cordisjs/plugin-http`) — even where a Perplexity provider's request is shaped like an OpenAI-compatible chat completion, that wire shape is a provider-private detail and does not make the provider depend on `ctx.llm`. + +`@deepseek-ai/dsh-tool-web` depends on `@deepseek-ai/dsh-web`, `@deepseek-ai/dsh-tools`, `@deepseek-ai/dsh-system-prompt`, and Cordis. It never imports concrete provider packages. + +## `ctx.web` contract + +`ctx.web` is a provider registry plus a provider-selecting execution surface. The registry half should stay close to `LlmService`: a `Map` per capability kind, `registerSearchProvider` / `registerFetchProvider` methods that return disposers, duplicate ids that throw `WebError`, and execution-time resolution that throws when the selected provider is absent or unusable. The exact TypeScript signatures belong to the implementation PR, but the seam should expose this shape: + +```ts +interface WebSearchProvider { + readonly id: string + status(): WebProviderStatus + search(request: WebSearchRequest, exec?: WebExecContext): Promise +} + +interface WebFetchProvider { + readonly id: string + status(): WebProviderStatus + fetch(request: WebFetchRequest, exec?: WebExecContext): Promise +} + +interface WebService { + registerSearchProvider(provider: WebSearchProvider): () => void + registerFetchProvider(provider: WebFetchProvider): () => void + + searchStatus(): WebCapabilityStatus + fetchStatus(): WebCapabilityStatus + + search(request: WebSearchRequest, exec?: WebExecContext): Promise + fetch(request: WebFetchRequest, exec?: WebExecContext): Promise +} + +interface WebExecContext { + readonly signal?: AbortSignal +} +``` + +`WebExecContext` is execution control, not business input. The first version should carry only `signal` so `tool-web` can propagate turn cancellation, tool timeout, and agent disposal into provider network requests, SSE readers, and expensive decoding. It should not pass `ToolExecution` through the seam, because that would make `dsh-web` depend on `dsh-tools`. + +`@deepseek-ai/dsh-web` should also declare a Cordis event named `web/providers-change`. Provider ids are stable strings and unique within their capability kind. Registering a duplicate search provider id or duplicate fetch provider id should fail rather than silently replace the old provider. Provider registration returns a disposer, emits `web/providers-change` after successful registration, and emits it again when the provider is disposed. The registry should follow the existing `ctx.tools.register()` / `ctx.systemPrompt.section()` pattern: wrap the mutation in `ctx.effect()`, install the rollback disposer before emitting `web/providers-change`, and let a throwing registration-time change listener roll back the just-added provider instead of leaking it into the registry. + +## Provider status and selection + +Provider status and capability selection are separate concepts, but both stay minimal. A provider reports only whether that concrete implementation is usable by cheap local checks such as credential presence or parseable endpoint config. A provider `status()` must not make network calls. The service reports whether the capability has a selected usable provider, or why execution would fail. + +`LlmService` has no status type at all: availability is expressed as registry membership plus a resolution-time throw. `ctx.web` needs a small status answer because product apps, diagnostics, tests, and execution can report precise provider-selection failures without probing individual providers from the tool layer. Status must be derived from the configured provider id, registered providers, and each provider's cheap local `status()` on each call; it must not be stored as mutable service state. + +`WebCapabilityStatus` stays intentionally small: `available` plus a `reason` discriminant, and the selected `providerId` on the available branch so diagnostics can report which provider won. It does NOT carry the per-reason payload (the unavailable provider id, the ambiguous candidate set, the underlying provider-unavailable reason). That branchable detail lives in the structured `WebError` thrown at execution time, which is the surface callers route on; duplicating it into the status union would give the same fact two homes that can disagree. `searchStatus()` / `fetchStatus()` answer "can this capability run, and if not, in which broad category does it fail" — enough for startup diagnostics and the execution-resolution decision — and the thrown error answers "exactly which provider/ids/reason." + +`WebProviderStatus` is an input to selection, not a health system. `tool-web` reads only the aggregated `searchStatus()` / `fetchStatus()`, never each provider's `status()` directly, so selection policy has one owner. + +```ts +type WebProviderStatus = + | { readonly available: true } + | { readonly available: false; readonly reason: 'missing-credential' | 'misconfigured' } + +type WebCapabilityStatus = + | { readonly available: true; readonly providerId: string } + | { readonly available: false; readonly reason: 'none' | 'configured-missing' | 'configured-unavailable' | 'ambiguous' } +``` + +Selection must not depend on registration order. Cordis load order, config ordering, and HMR timing are not product semantics. + +| Situation | Status / behavior | +|---|---| +| A configured provider id is registered and `status().available === true` | `available: true` for that provider | +| A configured provider id is not registered | `configured-missing`; execution fails with `WEB_PROVIDER_CONFIGURED_MISSING` | +| A configured provider id is registered but unavailable | `configured-unavailable`; execution fails with `WEB_PROVIDER_CONFIGURED_UNAVAILABLE` | +| No provider id is configured and exactly one provider for that kind is registered and available | `available: true` for that single provider | +| No provider id is configured and no provider for that kind is registered | `none`; execution fails with `WEB_PROVIDER_UNAVAILABLE` | +| No provider id is configured and multiple usable providers for that kind are registered | `ambiguous`; execution fails with `WEB_PROVIDER_AMBIGUOUS` rather than choosing by registration order | +| No provider id is configured and providers exist but none are usable | `none`; execution fails with `WEB_PROVIDER_UNAVAILABLE` | + +The "single provider auto-selects" rule is for tests, demos, and simple deployments. Product configs should set explicit provider ids: + +```yaml +- id: web + name: '@deepseek-ai/dsh-web' + config: + searchProvider: exa + fetchProvider: local-http + +- id: web-search-exa + name: '@deepseek-ai/dsh-web-search-exa' + +- id: web-search-perplexity + name: '@deepseek-ai/dsh-web-search-perplexity' + +- id: web-fetch-local + name: '@deepseek-ai/dsh-web-fetch-local' + +- id: tool-web + name: '@deepseek-ai/dsh-tool-web' +``` + +Operational overrides such as environment variables may exist, but they must feed the same explicit selection path. For example, `DSH_WEB_SEARCH_PROVIDER=perplexity` is equivalent to config `searchProvider: perplexity`; it is not a hidden priority chain inside `dsh-tool-web`. + +`ctx.web.search()` and `ctx.web.fetch()` resolve the provider at execution time using the same rules as the status query. If the selected capability is unavailable, they throw `WebError` with a structured code such as `WEB_PROVIDER_UNAVAILABLE`, `WEB_PROVIDER_CONFIGURED_MISSING`, `WEB_PROVIDER_CONFIGURED_UNAVAILABLE`, or `WEB_PROVIDER_AMBIGUOUS`. If no provider is explicitly configured and no usable provider exists, the status and execution error are both the generic `none` / `WEB_PROVIDER_UNAVAILABLE` case; the first version should not add a diagnostic summary of every unavailable provider. + +## Search request and result schema + +The first `web_search` model-facing tool should be small. The only model-facing argument is: + +- `query`: required string. + +`max_results` is NOT exposed to the model in the first version. It is a `dsh-tool-web`-layer decision: the tool sets the result bound — a default of `8` (aligning with OpenCode's Exa default), as an exported constant mirroring `dsh-tool-fs`'s `READ_LIMIT` / `GREP_LIMIT` — and passes it to the seam as `maxResults` on the `WebSearchRequest`. Keeping it off the model schema means the model just asks a question and the product controls how much context comes back; the field can be promoted to a model-facing argument later without breaking the seam. + +`maxResults` flows tool → seam → provider, and the bound is enforced on the way back: + +- `dsh-tool-web` owns the value and puts it on `WebSearchRequest.maxResults`. +- `ctx.web` passes the request through to the selected provider unchanged. +- A provider should apply `maxResults` at the request layer when its API supports it (Exa's `numResults`), as a cost/latency optimization. +- `ctx.web` enforces the bound on the result: if a provider returns more than `maxResults` sources — because its API has no result-count control (Perplexity) or ignored the hint — the seam truncates `sources[]` to `maxResults` and sets `WebSearchResult.truncated` to `true` before returning. This makes the bound a single cross-provider guarantee the model-facing layer can rely on, rather than something each provider must remember to honor. + +The seam request should not include provider-specific controls such as Perplexity model selection, search recency, domain filters, Exa `livecrawl`, Exa `type`, regional hints, generated-answer budgets, or search depth in the first version. Those fields should be added only when they have provider-neutral semantics that both the tool schema and selected providers can honor honestly. + +```ts +interface WebSearchRequest { + readonly query: string + /** Upper bound on returned sources; the seam truncates to it. Omitted = no bound. `dsh-tool-web` always sets it. */ + readonly maxResults?: number +} + +interface WebSearchResult { + readonly providerId: string + readonly query: string + readonly content?: string + readonly sources: readonly WebSearchSource[] + readonly truncated: boolean +} + +interface WebSearchSource { + readonly url: string + readonly title?: string + readonly snippet?: string + readonly publishedAt?: string +} +``` + +`content` is optional provider-generated answer text, search context, or summary. `sources[]` is the portable citation surface. A source always has a URL; title, snippet, and `publishedAt` are optional because not every provider returns them. `title` should not be required: Perplexity-style citations may provide only URLs, and forcing adapters to invent titles would make the seam lie. `dsh-tool-web` can render `title ?? hostname(url)` for display. `publishedAt` is an optional publication/crawl timestamp as an ISO-8601 string — Exa returns it as `publishedDate` on each result and Perplexity returns a `date` on search results, so it is real provider data, not derived; the seam carries it as a string and leaves date parsing to the consumer. + +Exa search should map each entry of the provider's flat `results[]` into a `WebSearchSource`: `url` ← `url`, `title` ← `title`, `snippet` ← the first `highlights[]` entry (an entry with no highlight has no portable snippet and is dropped), `publishedAt` ← `publishedDate`. Exa returns no provider-generated answer, so `content` is omitted. Perplexity search should map `choices[0].message.content` to `content` and prefer the structured top-level `search_results[]` for `sources[]` — `url` ← `url`, `title` ← `title`, `snippet` ← `snippet` (often empty), `publishedAt` ← `date` — falling back to the URL-only `citations[]` array only when `search_results` is absent (those sources carry just a `url`). If a provider returns fewer structured fields than the seam supports, the adapter omits those optional fields. + +Full page retrieval remains the job of `web_fetch(url)`. Search snippets are discovery context, not fetched page bodies. + +## Fetch request and result schema + +The first `web_fetch` implementation should be an anonymous public HTTP(S) fetch provider, likely `local-http`. It should fetch bytes from a concrete URL, apply the basic transport hygiene below (http/https-only, credential rejection, byte/time caps, cross-origin redirect blocking), decode textual content, and return only the minimal model-useful result: final URL, status code, body, and truncation. It should not carry browser cookies, editor credentials, git credentials, internal auth tokens, or implicit access to private services. (Full SSRF / private-network blocking is deferred — see [Deferred work](#deferred-work).) + +The first seam request should stay smaller than OpenCode's model-facing tool: + +- `url`: required HTTP(S) URL. +- `timeoutMs`: optional positive number capped by the provider. + +The seam request deliberately does not include `format`, `prompt`, or provider-specific extraction controls. `format` is a presentation decision over a fetched resource; `prompt` is a higher-level LLM summarization instruction; extraction APIs such as Firecrawl, Exa, Tavily, or Parallel may not expose a concrete HTTP response. If the product later needs provider-backed page extraction, add a separate `web_extract` capability or explicitly widen this RFC before implementation. Do not smuggle extract semantics into `web_fetch` by making every HTTP field optional. + +HTTP status is part of the fetched resource state, not automatically a tool failure. A successful network fetch of a `404` or `500` response should return `WebFetchResult` with the status code and a bounded decoded body when the content type is supported. `WebError` is for failures to safely retrieve or represent the resource: invalid or blocked URL, redirect policy violation, timeout, abort, response too large, unsupported content type, provider failure, or network failure. + +```ts +interface WebFetchRequest { + readonly url: string + readonly timeoutMs?: number +} + +interface WebFetchResult { + readonly providerId: string + readonly url: string + readonly statusCode: number + readonly body: WebFetchBody + readonly truncated: boolean +} + +type WebFetchBody = + | { readonly kind: 'html'; readonly content: string } + | { readonly kind: 'text'; readonly content: string } +``` + +`WebFetchResult.url` is the final URL after allowed redirects. The request URL is already present in `WebFetchRequest`, so the first version should not add separate `requestedUrl` and `finalUrl` fields. + +`WebFetchBody` is a CLOSED discriminated union owned by `dsh-web`, not a merge-extensible map. The merge-extensible pattern (`ContentBlockMap`) exists for variants that independent plugins introduce and the seam cannot foresee; body kinds are not that — `dsh-web` declares the kind, the fetch provider decodes it, and `dsh-tool-web` renders it, so a new kind is a coordinated change across three known packages, not a plugin extension. Keeping it closed buys compile-time exhaustiveness: consumers `switch` on `kind` ending in `default: assertNever(body, …)`, so adding a kind breaks compilation at every consumer that must render it (e.g. `tool-web`'s `html`→markdown vs `text` passthrough) until that arm is written. Each arm stays its own object literal even when the fields coincide today, leaving room for arm-specific fields (a future `pdf` body's `pageCount`, a `json` body's parsed value) without reshaping the type. Since the harness is unreleased, extending this closed union later is free (no migration, no compat shim). + +The provider owns safe resource retrieval: URL validation, HTTP transport, redirect policy, timeout, abort propagation, byte caps, charset decoding, content-type classification, and binary rejection. `dsh-tool-web` owns presentation: HTML-to-markdown, HTML-to-text, truncation formatting for the model, and future summaries. + +The fetch provider must define resource controls before the tool ships: + +- Accept only `http:` and `https:` URLs. +- Reject credentials in URLs. +- Enforce maximum URL length, response byte cap, decoded body character cap, timeout, and redirect hop cap. +- Propagate abort signals through network fetches and expensive decoding. +- Automatically follow only same-origin redirects. +- Fail cross-origin redirects with `WEB_REDIRECT_BLOCKED`, requiring a fresh tool call and therefore a fresh provider/permission decision. (Claude Code's WebFetch uses this same model — it does not auto-follow a cross-host redirect; it returns the redirect target to the model for a fresh call.) +- Use an explicit product user agent rather than silently impersonating a browser by default. + +SSRF / private-network protection (blocking private, loopback, link-local, multicast, and otherwise non-public destinations, with DNS-resolve-then-validate to defeat rebinding and per-hop re-validation on redirects) is **deferred** — see [Deferred work](#deferred-work). Until it lands, `web_fetch` is an SSRF primitive and must not be enabled in a deployment that can reach sensitive internal network targets. + +## Tool consumer behavior + +`dsh-tool-web` owns two `ToolDefinition`s: `web_search` and `web_fetch`. It owns model-facing JSON schemas, snake_case argument names, prompt sections, result rendering to `ContentBlock[]`, `presentCall`, and `presentResult`. + +`dsh-tool-web` must not enumerate providers or call provider `status()` directly. Its execution path is `ctx.web.search()` / `ctx.web.fetch()`, and any optional startup diagnostics should read only `ctx.web.searchStatus()` / `ctx.web.fetchStatus()`. That keeps provider selection in one layer; otherwise the tool package could decide one provider is usable while execution resolves a different state. + +Tool registration in the first version is a minimal stable sync: + +1. On plugin startup, read the product/app config that enables or disables web search and web fetch. +2. If web search is enabled, register `web_search` and keep that tool's disposer. +3. If web fetch is enabled, register `web_fetch` and keep that tool's disposer. +4. Do not dispose either tool merely because `ctx.web.searchStatus()` or `ctx.web.fetchStatus()` is unavailable. +5. Dispose registered tools when the `tool-web` fiber is disposed. + +Provider status changes affect execution results and diagnostics, not whether the model-facing schema exists. If a product wants no web tools at all, it disables `dsh-tool-web` or the individual web tool in config; if it wants web tools but the backend is misconfigured, the model sees a structured tool error at execution time. + +Prompt guidance should explain the semantic split: use `web_search` for discovery and current information, then use `web_fetch` when the model needs the content of a specific URL. The prompt and tool result should tell the model to cite relevant URLs with markdown links. + +The model-facing output should be text-first because current tool results are `ContentBlock[]`, but the seam outcome should stay structured so UI presentation and future adapters do not have to scrape rendered text. + +## Errors + +`dsh-web` should define `WebError extends HarnessError` with stable codes. Initial codes should include only states that callers may reasonably branch on: + +- `WEB_PROVIDER_UNAVAILABLE` +- `WEB_PROVIDER_CONFIGURED_MISSING` +- `WEB_PROVIDER_CONFIGURED_UNAVAILABLE` +- `WEB_PROVIDER_AMBIGUOUS` +- `WEB_DUPLICATE_PROVIDER` +- `WEB_INVALID_URL` +- `WEB_BLOCKED_URL` +- `WEB_REDIRECT_BLOCKED` +- `WEB_FETCH_TOO_LARGE` +- `WEB_FETCH_TIMEOUT` +- `WEB_ABORTED` +- `WEB_UNSUPPORTED_CONTENT_TYPE` +- `WEB_PROVIDER_ERROR` + +`WEB_DUPLICATE_PROVIDER` is thrown synchronously from `registerSearchProvider` / `registerFetchProvider` when an id is already registered for that capability kind (the analogue of `LlmService`'s `DUPLICATE_ADAPTER`); it is a registration-time programming error, not an execution outcome, but shares the `WebError` code space so callers see one taxonomy. `WEB_PROVIDER_ERROR` is the catch-all for a provider's own failure surfaced through the seam, including network/transport failure in `web-fetch-local` (DNS, connection refused, TLS); the first version does not split out a separate `WEB_NETWORK` code, but the provider should set a descriptive message so the model and logs can tell a network failure from a provider API failure. + +Tool execution should let these errors flow through `ToolRegistry.execute()`, which already converts `HarnessError` into an error tool result with structured metadata. The model gets a readable error message; hooks, tests, and UI code can route on the stable code. + +## Tests + +Tests should prove the seam contract without turning this RFC into an implementation checklist. + +`dsh-web` tests cover provider registration and disposal, duplicate provider ids, `web/providers-change` emission, rollback when a registration-time `web/providers-change` listener throws, `searchStatus()` and `fetchStatus()` for the selection table above, execution-time provider resolution, `maxResults` truncation of `sources[]` with `truncated` set when a provider over-returns, abort propagation through `WebExecContext.signal`, and structured `WebError` codes. + +Search provider tests cover request mapping, response parsing into `content` plus `sources[]`, missing credentials, provider errors, timeout/abort, truncation, and a self-skipping with-key smoke test for each real provider. Perplexity fixtures must include URL-only citations so the optional source fields stay honest. + +`dsh-web-fetch-local` tests cover real HTTP behavior using a local test server: valid text and HTML fetches, non-2xx HTTP responses returned as results, byte/decoded-body caps, timeout, abort, invalid URLs, credential-in-URL rejection, cross-origin redirect blocking, unsupported content types, and product user agent. (Private-destination/SSRF blocking tests come with that deferred work.) + +`dsh-tool-web` tests execute through the real tool registry. They verify schema registration follows product/app tool enablement rather than provider availability, unavailable or ambiguous providers produce structured execution errors, argument validation, formatting of successful search/fetch results, structured error propagation, and cleanup on disposal. + +Integration tests should load the real seam, provider, and tool packages together and execute through `ctx.tools.execute()` rather than calling providers directly. If wiring the tools into an ACP-facing example changes editor-visible transcripts, add or update the relevant snapshot scenario in the same change. + +At least one test must drive these packages through their REAL cordis Loader/export path, not a hand-built `ctx.plugin({...})` mount, so a broken export shape is caught (see [docs/postmortem/0001](../../../postmortem/0001-acp-default-export-drops-inject.md) and `packages/AGENTS.md` § plugin-export-shape). The two shapes need different guards: `dsh-web` and the provider packages are **services** (`export default` the class) and a stray extra export would surface as a missing service; `dsh-tool-web` is a **namespace plugin** (named `name`/`inject`/`apply`, NO default), and because it has `inject`, a stray `export default apply` makes `unwrapExports` drop the `inject` and the plugin throws `cannot get property … without inject` the moment it loads — so a Loader smoke that boots tool-web over `ctx.web` catches it. Prove the guard bites: add `export default apply` to `tool-web`, watch the smoke go red, revert. + +## Migration plan + +This is new capability work, so no compatibility migration is required while the harness is unreleased. + +Land the work in seam order: + +1. Add `packages/web/web` with `ctx.web`, provider registration, provider status, capability status, selection, request/result/error types, and contract tests. +2. Add `packages/web/web-search-exa` with parser/unit tests and a self-skipping real-provider smoke test. +3. Add `packages/web/web-search-perplexity` with parser/unit tests and a self-skipping real-provider smoke test. +4. Add `packages/web/web-fetch-local` with local HTTP behavior tests. +5. Add `packages/web/tool-web` with config-driven tool registration, prompt sections, model formatting, presentation, and tool-registry tests. +6. Wire product app/example configs only after package behavior is stable, because tool schemas and prompt sections affect agent behavior and snapshots. +7. Update `docs/architecture.md`, `packages/README.md`, package READMEs, generated Cordis catalogs if new events/services are added, and maintenance scripts. + +## Alternatives considered + +### Let each provider register its own model-facing tool + +This matches the most flexible provider-plugin systems: every provider can expose its full native schema. It is rejected for the harness because it gives provider packages ownership of model-facing names, descriptions, prompt guidance, and result formatting. Multiple search providers would produce duplicate tool names or provider-specific tool names, and the model would learn backend details instead of a stable product capability. + +### Put provider dispatch directly in `dsh-tool-web` + +This resembles OpenCode's local web search: one stable `websearch` tool dispatches to Exa or Parallel internally. It is acceptable for a small product path but wrong as a harness foundation. The tool package would own provider selection, credentials, request mapping, transport, response parsing, and presentation, making it hard to add Exa and Perplexity without baking their differences into the tool schema. + +### Split search and fetch into two seams (`dsh-search`, `dsh-fetch`) + +Tempting because the two halves share no request schema and no business logic, so each would map cleanly onto the bash/fs three-package template, and the `Search`/`Fetch` method-pair duplication on `WebService` would disappear. Rejected because the shared machinery — provider-id registry, registration-order-independent selection policy, abort propagation, the `WebError` taxonomy, and the product-facing "how this harness reaches the web" config surface — is real and would otherwise be duplicated across two near-identical seams. One `ctx.web` middle layer gives the product a single thing to inject and configure and gives provider selection one owner. The price is the parallel `searchX`/`fetchX` method pairs, which is accepted deliberately. + +### Choose the first registered provider + +Rejected. Registration order is not a product policy. It can change with config order, plugin loading, HMR, or refactors. Provider selection must be explicit, or automatic only when exactly one usable provider exists. + +### Treat Firecrawl/Exa/Tavily/Parallel extraction as fetch + +Rejected for the first version. Those providers often return extracted or summarized content rather than a concrete HTTP response. If the product needs extraction, design `web_extract` or deliberately widen the fetch seam later. + +### Mirror Claude Code's `url + prompt` WebFetch shape + +Rejected for the seam. `prompt` turns fetch into LLM summarization and couples public-web retrieval to a model provider. The harness seam should fetch and decode deterministically; `dsh-tool-web` can later offer summaries as a presentation mode without making `ctx.web` depend on `ctx.llm`. + +## Risks + +**The search schema may be too thin.** Exa and Perplexity both expose useful provider-specific controls. The first version should resist adding them until they can be defined provider-neutrally and enforced honestly by both tool registration and provider execution. + +**Perplexity citations may be sparse.** A citation may be only a URL. Making `title` and `snippet` optional keeps the seam truthful but means `tool-web` must render useful fallback labels. + +**Stable tool registration can defer misconfiguration to execution.** Keeping the tool visible is correct when the product enabled web access, but product apps that expect web search should surface `configured-missing`, `configured-unavailable`, and `ambiguous` loudly during startup diagnostics so users do not discover setup problems only after the model calls the tool. + +**Provider state can change after startup.** A tool can be visible in the request assembled at step start and lose its provider before execution. The execution path must resolve again and fail with a structured error. + +**Fetch is a network boundary, not just a read-only tool.** `web_fetch` can still reach sensitive network targets or exfiltrate data through URLs. The first version ships only the basic transport hygiene (http/https-only, credential rejection, byte/time caps, cross-origin redirect blocking); SSRF / private-network blocking is deferred (see [Deferred work](#deferred-work)), so until it lands `web_fetch` must not be enabled where it can reach internal targets. + +**Large web content can damage context quality.** Providers must enforce byte/character caps and report `truncated`; `tool-web` must format bounded model output with clear continuation or follow-up guidance. + +## Deferred work + +- SSRF / private-network protection for `web_fetch`: block private, loopback, link-local, multicast, and otherwise non-public destinations so `web_fetch` is not an SSRF primitive. Doing it correctly is more than a URL-string check — it needs DNS-resolve-then-connect-to-the-validated-IP (to defeat DNS rebinding / TOCTOU), per-hop re-validation across redirects, and IPv6 edge handling (private ranges, IPv4-mapped addresses). Neither reference implementation surveyed does IP-level blocking (OpenCode does a prefix check then fetches; Claude Code relies on a centralized hostname blocklist plus a "private URLs will fail" prompt), so there is no implementation to copy and this is the harness's only SSRF defense — it warrants its own focused design/spike. Until it lands, `web_fetch` must only be enabled in deployments that cannot reach sensitive internal targets. +- A `pdf` `WebFetchBody` kind: the `local-http` provider decodes text-extractable PDFs (best-effort, capped, `truncated`) into a `{ kind: 'pdf'; content; pageCount? }` arm, and `tool-web` renders it. This is fetch, not `web_extract` — PDF retrieval is a concrete HTTP 200 plus deterministic local decoding, not provider-side extraction of a non-HTTP resource. Adding it is a coordinated change across `dsh-web` (declare the arm), the provider (decode + narrow "binary rejection" to "reject binary except text-extractable PDF"; scanned/image PDFs needing OCR stay out of scope), and `tool-web` (render). The closed `WebFetchBody` union makes the consumer side fail to compile until the new arm is handled. +- Provider-backed extraction as a separate `web_extract` capability, rather than widening `web_fetch` silently. +- Permission policy integration once the deferred permission system lands. +- Provider-neutral search controls beyond `query` and `maxResults`, once Exa and Perplexity can both honor them honestly. + +## Open questions + +- Should product app packages treat `configured-missing`, `configured-unavailable`, and `ambiguous` as fatal startup errors when web is explicitly configured, or should `dsh-web` only report status and let apps decide? +- Where should permission policy for public web access live once the deferred permission system lands: a dedicated web permission plugin on `tools/execute`, provider config, or both? From d01f5f73b7866b457f00ffbe60b78af39273fc7a Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 25 Jun 2026 15:04:12 +0800 Subject: [PATCH 02/16] Add web capability seam: ctx.web, search/fetch providers, web tools Introduce web access as a first-class capability seam so the model-facing web tools stay stable while backends change. dsh-web owns ctx.web as a provider registry with registration-order-independent selection and the WebError taxonomy; dsh-web-search-exa, dsh-web-search-perplexity, and dsh-web-fetch-local register capabilities into it; dsh-tool-web is the sole owner of the model-facing web_search/web_fetch schemas, prompt sections, and HTML-to-markdown presentation. Search and fetch are deliberately one seam. Providers ship as namespace plugins that register into ctx.web (like an LlmAdapter into ctx.llm), not key-owning services, since multiple search providers cannot each own the key. Tool registration follows product enablement, not backend availability, so load order/credentials never enter the model contract; the seam resolves the provider at execution time and surfaces a structured WebError otherwise. Moves the RFC to implemented/ amended to match what shipped. Example/app configs are intentionally not wired yet (RFC migration step 6). --- .gitignore | 3 + docs/architecture.md | 7 + docs/rfc/README.md | 2 +- .../2026-06-24-web-capability-seam.md | 14 +- knip.json | 8 + packages/README.md | 11 + packages/web/README.md | 15 + packages/web/tool-web/README.md | 30 ++ packages/web/tool-web/package.json | 42 +++ packages/web/tool-web/src/fetch.ts | 87 ++++++ packages/web/tool-web/src/html.ts | 85 ++++++ packages/web/tool-web/src/index.ts | 59 ++++ packages/web/tool-web/src/search.ts | 105 +++++++ .../web/tool-web/tests/integration.spec.ts | 99 ++++++ packages/web/tool-web/tests/load-path.spec.ts | 49 +++ packages/web/tool-web/tests/tool-web.spec.ts | 281 ++++++++++++++++++ packages/web/tool-web/tsconfig.json | 17 ++ packages/web/tool-web/tsdown.config.ts | 18 ++ packages/web/web-fetch-local/README.md | 34 +++ packages/web/web-fetch-local/package.json | 33 ++ packages/web/web-fetch-local/src/index.ts | 77 +++++ packages/web/web-fetch-local/src/policy.ts | 59 ++++ packages/web/web-fetch-local/src/provider.ts | 233 +++++++++++++++ .../web-fetch-local/tests/fetch-local.spec.ts | 239 +++++++++++++++ packages/web/web-fetch-local/tsconfig.json | 24 ++ packages/web/web-search-exa/README.md | 23 ++ packages/web/web-search-exa/package.json | 33 ++ packages/web/web-search-exa/src/index.ts | 48 +++ packages/web/web-search-exa/src/provider.ts | 130 ++++++++ packages/web/web-search-exa/src/types.ts | 36 +++ packages/web/web-search-exa/tests/exa.e2e.ts | 19 ++ packages/web/web-search-exa/tests/exa.spec.ts | 193 ++++++++++++ packages/web/web-search-exa/tsconfig.json | 24 ++ packages/web/web-search-perplexity/README.md | 24 ++ .../web/web-search-perplexity/package.json | 33 ++ .../web/web-search-perplexity/src/index.ts | 52 ++++ .../web/web-search-perplexity/src/provider.ts | 138 +++++++++ .../web/web-search-perplexity/src/types.ts | 41 +++ .../tests/perplexity.e2e.ts | 23 ++ .../tests/perplexity.spec.ts | 192 ++++++++++++ .../web/web-search-perplexity/tsconfig.json | 24 ++ packages/web/web/README.md | 45 +++ packages/web/web/package.json | 33 ++ packages/web/web/src/index.ts | 270 +++++++++++++++++ packages/web/web/src/types.ts | 225 ++++++++++++++ packages/web/web/tests/web.spec.ts | 263 ++++++++++++++++ packages/web/web/tsconfig.json | 24 ++ pnpm-lock.yaml | 86 ++++++ tsconfig.base.json | 3 + tsconfig.build.json | 5 + 50 files changed, 3610 insertions(+), 8 deletions(-) rename docs/rfc/{proposed => implemented}/architecture/2026-06-24-web-capability-seam.md (96%) create mode 100644 packages/web/README.md create mode 100644 packages/web/tool-web/README.md create mode 100644 packages/web/tool-web/package.json create mode 100644 packages/web/tool-web/src/fetch.ts create mode 100644 packages/web/tool-web/src/html.ts create mode 100644 packages/web/tool-web/src/index.ts create mode 100644 packages/web/tool-web/src/search.ts create mode 100644 packages/web/tool-web/tests/integration.spec.ts create mode 100644 packages/web/tool-web/tests/load-path.spec.ts create mode 100644 packages/web/tool-web/tests/tool-web.spec.ts create mode 100644 packages/web/tool-web/tsconfig.json create mode 100644 packages/web/tool-web/tsdown.config.ts create mode 100644 packages/web/web-fetch-local/README.md create mode 100644 packages/web/web-fetch-local/package.json create mode 100644 packages/web/web-fetch-local/src/index.ts create mode 100644 packages/web/web-fetch-local/src/policy.ts create mode 100644 packages/web/web-fetch-local/src/provider.ts create mode 100644 packages/web/web-fetch-local/tests/fetch-local.spec.ts create mode 100644 packages/web/web-fetch-local/tsconfig.json create mode 100644 packages/web/web-search-exa/README.md create mode 100644 packages/web/web-search-exa/package.json create mode 100644 packages/web/web-search-exa/src/index.ts create mode 100644 packages/web/web-search-exa/src/provider.ts create mode 100644 packages/web/web-search-exa/src/types.ts create mode 100644 packages/web/web-search-exa/tests/exa.e2e.ts create mode 100644 packages/web/web-search-exa/tests/exa.spec.ts create mode 100644 packages/web/web-search-exa/tsconfig.json create mode 100644 packages/web/web-search-perplexity/README.md create mode 100644 packages/web/web-search-perplexity/package.json create mode 100644 packages/web/web-search-perplexity/src/index.ts create mode 100644 packages/web/web-search-perplexity/src/provider.ts create mode 100644 packages/web/web-search-perplexity/src/types.ts create mode 100644 packages/web/web-search-perplexity/tests/perplexity.e2e.ts create mode 100644 packages/web/web-search-perplexity/tests/perplexity.spec.ts create mode 100644 packages/web/web-search-perplexity/tsconfig.json create mode 100644 packages/web/web/README.md create mode 100644 packages/web/web/package.json create mode 100644 packages/web/web/src/index.ts create mode 100644 packages/web/web/src/types.ts create mode 100644 packages/web/web/tests/web.spec.ts create mode 100644 packages/web/web/tsconfig.json diff --git a/.gitignore b/.gitignore index b52f86cd61..2788817b23 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ examples/*/.sessions/ coverage/ .doc-typecheck-*/ .humanize/ +tmp/ +.claude/commands/ +.claude/settings.json .vscode/ .DS_Store .idea diff --git a/docs/architecture.md b/docs/architecture.md index c76d8f7ba4..dbaf76cd3f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -24,6 +24,9 @@ For a catalog of the **data structures** this architecture moves around — the │ @deepseek-ai/dsh-agent-loop (the ONE concrete plugin) │ │ @deepseek-ai/dsh-bash-local (bash impl) │ │ @deepseek-ai/dsh-tool-bash (bash tool schemas) │ +│ @deepseek-ai/dsh-web-search-exa (web search impl) │ +│ @deepseek-ai/dsh-web-fetch-local (web fetch impl) │ +│ @deepseek-ai/dsh-tool-web (web tool schemas) │ │ @deepseek-ai/dsh-session-persistence-jsonl (persistence impl)│ ├─────────────────────────────────────────────────────────────┤ │ @deepseek-ai/dsh-agent (vocabulary + registry) │ @@ -33,6 +36,7 @@ For a catalog of the **data structures** this architecture moves around — the │ @deepseek-ai/dsh-session-persistence (persistence seam) │ │ @deepseek-ai/dsh-llm (abstract model service) │ │ @deepseek-ai/dsh-bash (abstract bash executor) │ +│ @deepseek-ai/dsh-web (abstract web access) │ ├─────────────────────────────────────────────────────────────┤ │ vendor/: cordis, loader, include, group, timer, hmr, │ │ logger-console, cosmokit, schemastery │ @@ -54,6 +58,7 @@ Dependency rule: **extension** plugins depend on interface packages, never on `d | `ctx.agentLoop` | `AgentLoop` | dsh-agent-loop | creates `ReactLoopAgent`s and drives their loops | | `ctx.bash` | `BashExecutor` (abstract) | dsh-bash | bash execution seam: foreground runs + background tasks | | `ctx.compact` | `CompactService` (abstract) | dsh-compact | compaction seam: decide when history is too large, summarize an older range into a single surface node | +| `ctx.web` | `WebService` | dsh-web | web access seam: search/fetch provider registries, registration-order-independent selection, the `WebError` taxonomy | All registrations (`registerAdapter`, `section`, `tools`, `register`, …) go through `ctx.effect()` and return disposers, so plugin hot-reload (vendored HMR) and fiber disposal clean up automatically. @@ -69,6 +74,8 @@ Swappable capabilities are split into **three packages** so each part evolves in The LLM seam has the same topology folded differently: `dsh-llm` carries the interface (`LlmAdapter`) AND the consumer surface (`ctx.llm.stream()`), with adapters as implementation packages — there the consumer is the loop itself, not a swappable schema surface. Use the full three-package split when the consumer is independently replaceable; keep interface + consumer together when they are one concern. Don't split preemptively: a capability with one conceivable implementation and one consumer stays one package until proven otherwise. +The web capability uses the same three-package split but folds two capabilities onto one seam: `dsh-web` owns the abstract `ctx.web` service, which is a provider REGISTRY (`registerSearchProvider`/`registerFetchProvider`, registration-order-independent selection, the `WebError` taxonomy) rather than a single backend. Providers register capabilities, not tools — `dsh-web-search-exa`, `dsh-web-search-perplexity`, and `dsh-web-fetch-local` each register into `ctx.web` the way an `LlmAdapter` registers into `ctx.llm`, so they are namespace plugins (`inject: ['web']`), not key-owning services. `dsh-tool-web` is the single consumer that owns the model-facing `web_search`/`web_fetch` schemas, prompt sections, and presentation; it reads only the aggregated `ctx.web.searchStatus()`/`fetchStatus()` and executes through `ctx.web.search()`/`fetch()`, so provider selection has one owner. Search and fetch are deliberately one seam (one thing to inject and configure, one selection policy, one abort/error vocabulary) despite sharing no request schema — see the [web capability seam RFC](rfc/implemented/architecture/2026-06-24-web-capability-seam.md). + > **"Capability" — two unrelated meanings.** (1) The *seam pattern* above ("one plugin provides a capability, another needs it") is realized by plain Cordis **services + `inject`**: a provider registers a service (`ctx.bash`, declared in `interface Context`); a consumer declares `inject: ['bash']` and its fiber stays pending until the service exists, tearing down via HMR if it later vanishes. No extra library is needed. (2) `@cordisjs/plugin-capability` is a different axis entirely — a **permission/capability-security** service (named permissions with inheritance/dependency, tested against a session via `ctx.capability.test`). It is a candidate for the deferred permissions/sandbox work (the `tools/execute` veto seam), NOT a mechanism for swapping implementations. ## The vocabulary (dsh-llm) diff --git a/docs/rfc/README.md b/docs/rfc/README.md index 73a2675caa..ab3e97e4bc 100644 --- a/docs/rfc/README.md +++ b/docs/rfc/README.md @@ -59,7 +59,6 @@ Do NOT write one for a mechanical or local choice (a variable name, a one-file r |---|---| | [Runtime schemas for the event vocabulary (Zod vs the merge-extensible-map pattern)](proposed/architecture/2026-06-16-typed-event-schemas.md) | 2026-06-16 | | [Extract a generic long-running tool runtime](proposed/architecture/2026-06-20-generic-long-running-tool-runtime.md) | 2026-06-20 | -| [Web capability seam - provider registry and model-facing web tools](proposed/architecture/2026-06-24-web-capability-seam.md) | 2026-06-24 | ### Process @@ -120,6 +119,7 @@ Do NOT write one for a mechanical or local choice (a variable name, a one-file r | [Reorganize packages into a modular hierarchy](implemented/architecture/2026-06-20-package-hierarchy.md) | 2026-06-20 | | [Branded IDs everywhere they belong](implemented/architecture/2026-06-20-branded-ids.md) | 2026-06-20 | | [Extract example apps into packages](implemented/architecture/2026-06-20-extract-example-app-packages.md) | 2026-06-20 | +| [Web capability seam — provider registry and model-facing web tools](implemented/architecture/2026-06-24-web-capability-seam.md) | 2026-06-24 | ### Process diff --git a/docs/rfc/proposed/architecture/2026-06-24-web-capability-seam.md b/docs/rfc/implemented/architecture/2026-06-24-web-capability-seam.md similarity index 96% rename from docs/rfc/proposed/architecture/2026-06-24-web-capability-seam.md rename to docs/rfc/implemented/architecture/2026-06-24-web-capability-seam.md index e97e8380da..3b9fb166d0 100644 --- a/docs/rfc/proposed/architecture/2026-06-24-web-capability-seam.md +++ b/docs/rfc/implemented/architecture/2026-06-24-web-capability-seam.md @@ -1,6 +1,6 @@ # RFC: Web capability seam - stable tools over multiple providers -Status: proposed +Status: implemented ## Problem @@ -64,7 +64,7 @@ flowchart LR `@deepseek-ai/dsh-web` depends only on Cordis and low-level harness support. It declares `ctx.web`, provider interfaces, request/result types, status types, and error codes. It does not import tool, agent, session, LLM, or provider packages. -Provider packages depend on `@deepseek-ai/dsh-web` and Cordis. They own credentials, endpoint config, provider-specific request mapping, provider-specific response parsing, and provider-specific error translation into `WebError`. They issue network requests with the platform-native `fetch` (Node 24), mirroring `@deepseek-ai/dsh-llm-deepseek`'s adapter, NOT a cordis HTTP-client service (`ctx.http`/`@cordisjs/plugin-http`) — even where a Perplexity provider's request is shaped like an OpenAI-compatible chat completion, that wire shape is a provider-private detail and does not make the provider depend on `ctx.llm`. +Provider packages depend on `@deepseek-ai/dsh-web` and Cordis. They own credentials, endpoint config, provider-specific request mapping, provider-specific response parsing, and provider-specific error translation into `WebError`. They issue network requests with the platform-native `fetch` (Node 24), mirroring `@deepseek-ai/dsh-llm-deepseek`'s adapter, NOT a cordis HTTP-client service (`ctx.http`/`@cordisjs/plugin-http`) — even where a Perplexity provider's request is shaped like an OpenAI-compatible chat completion, that wire shape is a provider-private detail and does not make the provider depend on `ctx.llm`. A provider does NOT own the `ctx.web` key (two search providers cannot both own it): like `dsh-llm-deepseek`, each provider package is a function/namespace plugin (`inject: ['web']`) whose `apply` constructs the backend and calls `ctx.web.registerSearchProvider` / `registerFetchProvider`. `@deepseek-ai/dsh-web` is the `export default` service that owns the key. `@deepseek-ai/dsh-tool-web` depends on `@deepseek-ai/dsh-web`, `@deepseek-ai/dsh-tools`, `@deepseek-ai/dsh-system-prompt`, and Cordis. It never imports concrete provider packages. @@ -267,11 +267,11 @@ SSRF / private-network protection (blocking private, loopback, link-local, multi Tool registration in the first version is a minimal stable sync: -1. On plugin startup, read the product/app config that enables or disables web search and web fetch. -2. If web search is enabled, register `web_search` and keep that tool's disposer. -3. If web fetch is enabled, register `web_fetch` and keep that tool's disposer. +1. On plugin startup, read the `dsh-tool-web` `Config` (`search?: boolean`, `fetch?: boolean`, both default `true`) that enables or disables each web tool. +2. If web search is enabled, register `web_search` (its disposer is fiber-scoped via the effect-based registry). +3. If web fetch is enabled, register `web_fetch` (likewise fiber-scoped). 4. Do not dispose either tool merely because `ctx.web.searchStatus()` or `ctx.web.fetchStatus()` is unavailable. -5. Dispose registered tools when the `tool-web` fiber is disposed. +5. Disposing the `tool-web` fiber tears down its registrations automatically. Provider status changes affect execution results and diagnostics, not whether the model-facing schema exists. If a product wants no web tools at all, it disables `dsh-tool-web` or the individual web tool in config; if it wants web tools but the backend is misconfigured, the model sees a structured tool error at execution time. @@ -315,7 +315,7 @@ Search provider tests cover request mapping, response parsing into `content` plu Integration tests should load the real seam, provider, and tool packages together and execute through `ctx.tools.execute()` rather than calling providers directly. If wiring the tools into an ACP-facing example changes editor-visible transcripts, add or update the relevant snapshot scenario in the same change. -At least one test must drive these packages through their REAL cordis Loader/export path, not a hand-built `ctx.plugin({...})` mount, so a broken export shape is caught (see [docs/postmortem/0001](../../../postmortem/0001-acp-default-export-drops-inject.md) and `packages/AGENTS.md` § plugin-export-shape). The two shapes need different guards: `dsh-web` and the provider packages are **services** (`export default` the class) and a stray extra export would surface as a missing service; `dsh-tool-web` is a **namespace plugin** (named `name`/`inject`/`apply`, NO default), and because it has `inject`, a stray `export default apply` makes `unwrapExports` drop the `inject` and the plugin throws `cannot get property … without inject` the moment it loads — so a Loader smoke that boots tool-web over `ctx.web` catches it. Prove the guard bites: add `export default apply` to `tool-web`, watch the smoke go red, revert. +At least one test must drive these packages through their REAL cordis Loader/export path, not a hand-built `ctx.plugin({...})` mount, so a broken export shape is caught (see [docs/postmortem/0001](../../../postmortem/0001-acp-default-export-drops-inject.md) and `packages/AGENTS.md` § plugin-export-shape). The two shapes need different guards: `dsh-web` is a **service** (`export default` the class) and a stray extra export would surface as a missing service; the provider packages and `dsh-tool-web` are **namespace plugins** (named `name`/`inject`/`apply`, NO default), and because each has `inject`, a stray `export default apply` makes `unwrapExports` drop the `inject` and the plugin throws `cannot get property … without inject` the moment it loads — so a Loader smoke that boots tool-web over `ctx.web` catches it (and each provider's registration test mounts it the real way and asserts no default export). Prove the guard bites: add `export default apply` to `tool-web`, watch the smoke go red, revert. ## Migration plan diff --git a/knip.json b/knip.json index 67d99a861d..3f0a56097c 100644 --- a/knip.json +++ b/knip.json @@ -29,6 +29,14 @@ "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], "project": ["src/**/*.ts", "tests/**/*.ts"] }, + "packages/web/web-search-exa": { + "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], + "project": ["src/**/*.ts", "tests/**/*.ts"] + }, + "packages/web/web-search-perplexity": { + "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], + "project": ["src/**/*.ts", "tests/**/*.ts"] + }, "packages/ui/acp-agent": { "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], "project": ["src/**/*.ts", "tests/**/*.ts"] diff --git a/packages/README.md b/packages/README.md index 11cace9017..ab81df0a66 100644 --- a/packages/README.md +++ b/packages/README.md @@ -13,6 +13,7 @@ Packages are grouped by modular role at `packages///`. The group dir | [`bash/`](bash/README.md) | Bash capability family: the executor seam, a local impl, and the model-facing tool | Product — stable surface | | [`compact/`](compact/README.md) | Compaction capability family: the abstract seam (backend + tool deferred) | Product — stable surface | | [`subagent/`](subagent/README.md) | Subagent capability family: the provider-registry seam and the model-facing delegation tool | Product — stable surface | +| [`web/`](web/README.md) | Web capability family: the abstract seam, search/fetch provider impls, and the model-facing web tools | Product — stable surface | | [`session-persistence/`](session-persistence/README.md) | Persistence capability family: the seam + JSONL/SQLite backends | Product — stable surface | | [`ui/`](ui/README.md) | Editor/client integration surfaces (the ACP bridge) | Product — stable surface | | [`support/`](support/README.md) | Dev/test/example infrastructure (invariants, stdio UI, replay adapter) | Support — lower compatibility expectations | @@ -33,6 +34,11 @@ dsh-compact ← dsh-session, dsh-llm (abstract compaction s dsh-tools ← dsh-llm, dsh-system-prompt, dsh-agent dsh-bash-local ← dsh-bash (BashExecutor impl) dsh-tool-bash ← dsh-bash, dsh-tools (bash tool schemas) +dsh-web ← dsh-llm (abstract web seam; search/fetch registries, WebError) +dsh-web-search-exa ← dsh-web (Exa WebSearchProvider) +dsh-web-search-perplexity ← dsh-web (Perplexity WebSearchProvider) +dsh-web-fetch-local ← dsh-web (anonymous public HTTP(S) WebFetchProvider) +dsh-tool-web ← dsh-web, dsh-tools, dsh-system-prompt (web tool schemas) dsh-llm-deepseek ← dsh-llm (DeepSeek adapter) dsh-llm-pi-ai ← dsh-llm (pi-ai-backed adapter) dsh-agent-loop ← dsh-llm, dsh-session, dsh-system-prompt, dsh-tools, dsh-agent @@ -68,6 +74,11 @@ The rule: **extension** plugins depend on interfaces, never on the concrete loop | `bash-local/` | `bash` | Local-subprocess `BashExecutor` implementation | (registers `ctx.bash`) | | `tool-bash/` | `bash` | Model-facing `bash`/`bash_output`/`bash_kill` tool schemas | (registers on `ctx.tools`) | | `compact/` | `compact` | Abstract compaction seam + `compact/*` events + `CompactionResult` | `ctx.compact` | +| `web/` | `web` | Abstract web seam (search/fetch provider registries + selection + vocabulary + `WebError`) | `ctx.web` | +| `web-search-exa/` | `web` | Exa-backed `WebSearchProvider` | (registers on `ctx.web`) | +| `web-search-perplexity/` | `web` | Perplexity-backed `WebSearchProvider` | (registers on `ctx.web`) | +| `web-fetch-local/` | `web` | Anonymous public HTTP(S) `WebFetchProvider` | (registers on `ctx.web`) | +| `tool-web/` | `web` | Model-facing `web_search`/`web_fetch` tool schemas | (registers on `ctx.tools`) | | `llm-deepseek/` | `llm` | DeepSeek API adapter (hand-rolled fetch/SSE) | (registers on `ctx.llm`) | | `llm-pi-ai/` | `llm` | DeepSeek adapter via `@earendil-works/pi-ai` (design twin) | (registers on `ctx.llm`) | | `session-persistence/` | `session-persistence` | Persistence seam + write coordinator | `ctx.sessionPersistence` | diff --git a/packages/web/README.md b/packages/web/README.md new file mode 100644 index 0000000000..0742d9c2cc --- /dev/null +++ b/packages/web/README.md @@ -0,0 +1,15 @@ +# web/ - web capability family + +The web access capability seam: an abstract web interface, search/fetch provider implementations, and the model-facing web tools. All **product** packages. + +| Package | Role | ctx key | +|---|---|---| +| `web/` | Abstract web seam (search/fetch provider registries + selection + vocabulary + `WebError`) | `ctx.web` | +| `web-search-exa/` | Exa-backed `WebSearchProvider` | (registers on `ctx.web`) | +| `web-search-perplexity/` | Perplexity-backed `WebSearchProvider` | (registers on `ctx.web`) | +| `web-fetch-local/` | Anonymous public HTTP(S) `WebFetchProvider` | (registers on `ctx.web`) | +| `tool-web/` | Model-facing `web_search`/`web_fetch` tool schemas | (registers on `ctx.tools`) | + +The interface lives at `web/web/`. Unlike bash/fs, the seam spans **two capabilities** (search and fetch) with potentially multiple providers each: `ctx.web` is one web-access middle layer with one provider-selection policy, one abort/error vocabulary, and one product-facing "how this harness reaches the web" config surface. Providers register **capabilities**, not tools; `tool-web` is the only owner of model-facing names, schemas, prompt guidance, and presentation. A search provider swap does not change how the model asks for a query, and a fetch implementation swap does not change how the model asks for a URL. + +See the [web capability seam RFC](../../docs/rfc/implemented/architecture/2026-06-24-web-capability-seam.md) for the design rationale, including why search and fetch are deliberately one seam and why `web_fetch`'s SSRF protection is deferred. diff --git a/packages/web/tool-web/README.md b/packages/web/tool-web/README.md new file mode 100644 index 0000000000..762bfe0189 --- /dev/null +++ b/packages/web/tool-web/README.md @@ -0,0 +1,30 @@ +# @deepseek-ai/dsh-tool-web + +The model-facing web tool suite — `web_search` and `web_fetch` — over the [web capability seam](../web/README.md) (`ctx.web`). It owns model-facing concerns only: tool names, JSON schemas, snake_case argument names, prompt sections, the result-count bound, result formatting, HTML→markdown presentation, and `presentCall`. All web access goes through `ctx.web`; this package never imports a concrete provider. + +Each tool is also a subpath plugin (`@deepseek-ai/dsh-tool-web/search`, `/fetch`) for focused deployments. + +## Tools + +| Tool | Args | Behavior | +|---|---|---| +| `web_search` | `query` (string) | Discovery. Returns an optional answer plus source URLs. `max_results` is **not** model-facing — the tool sets the bound (`WEB_SEARCH_MAX_RESULTS = 8`) and passes it to the seam. | +| `web_fetch` | `url` (string), `timeout_ms` (number, optional) | Retrieves a specific URL. HTML bodies are rendered to markdown-ish text; text bodies pass through. A non-2xx status is reported, not an error. | + +## Config + +| Key | Default | Meaning | +|---|---|---| +| `search` | `true` | Register `web_search`. | +| `fetch` | `true` | Register `web_fetch`. | + +```yaml +- id: tool-web + name: '@deepseek-ai/dsh-tool-web' +``` + +## Stable registration + +Tool registration follows product **enablement**, not backend availability. A tool stays visible even when its selected provider is missing, misconfigured, ambiguous, or temporarily unavailable; the seam resolves the provider at execution time and execution fails with a structured `WebError` (e.g. `WEB_PROVIDER_UNAVAILABLE`, `WEB_PROVIDER_AMBIGUOUS`), which `ToolRegistry.execute()` turns into an error tool result the model can read and hooks/UI can route on. This keeps the model schema stable without making plugin load order, credential state, or HMR timing part of the model-facing contract. To remove a web tool entirely, disable it here in config. + +The tool reads only the aggregated `ctx.web.searchStatus()` / `fetchStatus()` for diagnostics — never each provider's `status()` directly — so provider selection has one owner. diff --git a/packages/web/tool-web/package.json b/packages/web/tool-web/package.json new file mode 100644 index 0000000000..8d46faa157 --- /dev/null +++ b/packages/web/tool-web/package.json @@ -0,0 +1,42 @@ +{ + "name": "@deepseek-ai/dsh-tool-web", + "description": "Model-facing web tools (web_search, web_fetch) over the DeepSeek Harness web capability seam (ctx.web)", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "exports": { + ".": { "types": "./lib/index.d.ts", "default": "./lib/index.js" }, + "./search": { "types": "./lib/search.d.ts", "default": "./lib/search.js" }, + "./fetch": { "types": "./lib/fetch.d.ts", "default": "./lib/fetch.js" }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "@deepseek-ai/dsh-llm": "^0.0.1", + "@deepseek-ai/dsh-system-prompt": "^0.0.1", + "@deepseek-ai/dsh-tools": "^0.0.1", + "@deepseek-ai/dsh-web": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "dependencies": { + "schemastery": "^3.18.0" + }, + "devDependencies": { + "@deepseek-ai/dsh-agent": "workspace:^", + "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-session": "workspace:^", + "@deepseek-ai/dsh-system-prompt": "workspace:^", + "@deepseek-ai/dsh-tools": "workspace:^", + "@deepseek-ai/dsh-web": "workspace:^", + "@deepseek-ai/dsh-web-fetch-local": "workspace:^", + "@deepseek-ai/dsh-web-search-exa": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/web/tool-web/src/fetch.ts b/packages/web/tool-web/src/fetch.ts new file mode 100644 index 0000000000..a48ad41414 --- /dev/null +++ b/packages/web/tool-web/src/fetch.ts @@ -0,0 +1,87 @@ +/** + * The model-facing `web_fetch` tool: retrieve the content of a specific URL. + * Execution goes through `ctx.web` — this module owns the model-facing schema, + * argument validation, and PRESENTATION (HTML→markdown, truncation formatting), + * while the fetch provider owns safe retrieval (transport, redirects, caps). + * + * @module @deepseek-ai/dsh-tool-web/fetch + */ + +import type { Context } from 'cordis' +import { defineTool } from '@deepseek-ai/dsh-tools' +import type { ToolCallPresentation } from '@deepseek-ai/dsh-tools' +import type { ContentBlock } from '@deepseek-ai/dsh-llm' +import type { WebFetchBody, WebFetchResult } from '@deepseek-ai/dsh-web' +import { assertNever } from '@deepseek-ai/dsh-llm' +import type {} from '@deepseek-ai/dsh-system-prompt' +import { htmlToMarkdown } from './html.ts' + +/** Validate value constraints the schema DSL can't express. */ +export function parseFetchArgs(args: { url: string; timeout_ms?: number }): { url: string; timeoutMs?: number } { + if (args.url.trim().length === 0) throw new Error('url must be a non-empty string') + if (args.timeout_ms !== undefined && (!Number.isFinite(args.timeout_ms) || args.timeout_ms <= 0)) { + throw new Error('timeout_ms must be a positive number') + } + return { url: args.url, ...args.timeout_ms !== undefined ? { timeoutMs: args.timeout_ms } : {} } +} + +/** Render a fetched body to model-facing markdown text. */ +export function renderBody(body: WebFetchBody): string { + switch (body.kind) { + case 'html': + return htmlToMarkdown(body.content) + case 'text': + return body.content + /* v8 ignore next 2 -- WebFetchBody is a closed union; this arm is unreachable and only makes adding a kind a compile error. */ + default: + return assertNever(body, 'unhandled web fetch body kind') + } +} + +/** Format a fetch result as one model-facing text block. */ +export function formatFetchOutput(result: WebFetchResult): string { + const header = `Fetched ${result.url} (HTTP ${result.statusCode})` + const footer = result.truncated ? '\n\n(Content truncated. Fetch a more specific URL or section for the full text.)' : '' + return `${header}\n\n${renderBody(result.body)}${footer}` +} + +/** Pending-call presentation: a fetch card titled by the URL. */ +export function presentFetchCall(args: { url: string; timeout_ms?: number }): ToolCallPresentation { + return { title: args.url, kind: 'fetch', rawInput: args.url } +} + +/** Register the `web_fetch` tool and its system-prompt guidance. */ +export function apply(ctx: Context): void { + ctx.systemPrompt.section({ + name: 'tool:web_fetch', + order: 111, + text: 'Use the web_fetch tool to retrieve the content of a specific HTTP(S) URL (for example a result from web_search). It returns the page content decoded to text. Cite the URL as a markdown link when you use its content.', + }) + + ctx.tools.register(defineTool({ + name: 'web_fetch', + description: 'Fetch the content of a specific HTTP(S) URL and return it decoded to text.', + parameters: { + url: { type: 'string', required: true, description: 'The HTTP(S) URL to fetch.' }, + timeout_ms: { type: 'number', description: 'Optional fetch timeout in milliseconds (capped by the provider).' }, + }, + async execute(args, exec): Promise { + const input = parseFetchArgs(args) + const result = await ctx.web.fetch( + { url: input.url, ...input.timeoutMs !== undefined ? { timeoutMs: input.timeoutMs } : {} }, + exec.signal ? { signal: exec.signal } : undefined, + ) + return [{ type: 'text', text: formatFetchOutput(result) }] + }, + presentCall: presentFetchCall, + })) +} + +/** Cordis plugin name used by loader diagnostics. */ +export const name = 'web-fetch' + +/** Services required by the `web_fetch` tool plugin. */ +export const inject = ['tools', 'web', 'systemPrompt'] + +/** Named helper for direct registration in the root plugin and tests. */ +export const applyWebFetchTool = apply diff --git a/packages/web/tool-web/src/html.ts b/packages/web/tool-web/src/html.ts new file mode 100644 index 0000000000..622be86fd5 --- /dev/null +++ b/packages/web/tool-web/src/html.ts @@ -0,0 +1,85 @@ +/** + * Minimal, dependency-free HTML→markdown-ish text conversion for `web_fetch` + * presentation. This is intentionally NOT a full HTML parser: it strips + * script/style/noscript, drops tags, decodes the common named/numeric entities, + * and collapses whitespace into a readable plain-text approximation with a few + * markdown affordances (headings, list bullets, links). A heavier converter can + * replace this without touching the seam or the tool schema. + * + * @module @deepseek-ai/dsh-tool-web/html + */ + +/** Decode the handful of HTML entities common in textual content. */ +function decodeEntities(text: string): string { + return text + .replace(/&(#[xX][0-9a-fA-F]+|#[0-9]+|[a-zA-Z]+);/g, (match, entity: string) => { + if (entity.startsWith('#x') || entity.startsWith('#X')) { + const code = Number.parseInt(entity.slice(2), 16) + return safeFromCodePoint(code, match) + } + if (entity.startsWith('#')) { + const code = Number.parseInt(entity.slice(1), 10) + return safeFromCodePoint(code, match) + } + return NAMED_ENTITIES[entity] ?? match + }) +} + +const NAMED_ENTITIES: Record = { + amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', + copy: '©', reg: '®', trade: '™', hellip: '…', mdash: '—', ndash: '–', +} + +function safeFromCodePoint(code: number, fallback: string): string { + try { + return String.fromCodePoint(code) + } catch { + // An out-of-range code point (RangeError) is the only failure here; keep the + // original entity text rather than throwing out of pure presentation. + return fallback + } +} + +/** + * Convert an HTML document to a readable markdown-ish text approximation. + * Best-effort and lossy by design — fidelity is the job of a future heavier + * converter, not this fallback. + */ +export function htmlToMarkdown(html: string): string { + let text = html + // Drop non-content elements entirely (including their contents). + .replace(/]*>[\s\S]*?<\/script>/gi, '') + .replace(/]*>[\s\S]*?<\/style>/gi, '') + .replace(/]*>[\s\S]*?<\/noscript>/gi, '') + .replace(//g, '') + + // Convert links to markdown before stripping tags. + text = text.replace(/]*\bhref\s*=\s*["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, (_match, href: string, label: string) => { + const cleanLabel = label.replace(/<[^>]+>/g, '').trim() + return cleanLabel.length > 0 ? `[${cleanLabel}](${href})` : href + }) + + // Headings → markdown hashes. + text = text.replace(/]*>([\s\S]*?)<\/h\1>/gi, (_match, level: string, body: string) => { + const hashes = '#'.repeat(Number(level)) + return `\n\n${hashes} ${body.replace(/<[^>]+>/g, '').trim()}\n\n` + }) + + // List items → bullets. + text = text.replace(/]*>([\s\S]*?)<\/li>/gi, (_match, body: string) => `\n- ${body.replace(/<[^>]+>/g, '').trim()}`) + + // Block-level breaks become paragraph breaks. + text = text + .replace(/<\/(p|div|section|article|header|footer|tr|table|ul|ol|blockquote)>/gi, '\n\n') + .replace(//gi, '\n') + + // Drop all remaining tags, decode entities, collapse whitespace. + text = text.replace(/<[^>]+>/g, '') + text = decodeEntities(text) + text = text + .replace(/[ \t\f\v]+/g, ' ') + .replace(/ *\n */g, '\n') + .replace(/\n{3,}/g, '\n\n') + .trim() + return text +} diff --git a/packages/web/tool-web/src/index.ts b/packages/web/tool-web/src/index.ts new file mode 100644 index 0000000000..031d7fe4cb --- /dev/null +++ b/packages/web/tool-web/src/index.ts @@ -0,0 +1,59 @@ +/** + * The model-facing web tool suite (`web_search`, `web_fetch`) over the `ctx.web` + * seam. This root plugin registers the tools the product has ENABLED, composing + * the per-tool registration helpers; each tool is also exposed as a subpath + * plugin (`@deepseek-ai/dsh-tool-web/search`, `/fetch`) for focused deployments. + * + * The package owns model-facing concerns only — tool names, JSON schemas, + * argument validation, prompt sections, result-cap constants, result formatting, + * HTML→markdown presentation. All web access goes through `ctx.web`; this + * package never imports a concrete provider package. + * + * Tool registration follows product/app ENABLEMENT, not backend availability: a + * tool stays visible even when its selected provider is missing/misconfigured, + * and execution fails with a structured `WebError` (resolved by the seam at call + * time). That keeps the model schema stable without making plugin load order, + * credential state, or HMR timing part of the model-facing contract. + * + * @module @deepseek-ai/dsh-tool-web + */ + +import type { Context } from 'cordis' +import z from 'schemastery' +import type {} from '@deepseek-ai/dsh-web' +import { applyWebSearchTool } from './search.ts' +import { applyWebFetchTool } from './fetch.ts' + +export { WEB_SEARCH_MAX_RESULTS, applyWebSearchTool, formatSearchOutput, parseSearchArgs, presentSearchCall } from './search.ts' +export { applyWebFetchTool, formatFetchOutput, parseFetchArgs, presentFetchCall, renderBody } from './fetch.ts' +export { htmlToMarkdown } from './html.ts' + +/** Cordis plugin name used by loader diagnostics. */ +export const name = 'tool-web' + +/** Services required by the web tool suite. */ +export const inject = ['tools', 'web', 'systemPrompt'] + +export interface Config { + /** Register `web_search`. Defaults to true. */ + search?: boolean + /** Register `web_fetch`. Defaults to true. */ + fetch?: boolean +} + +export const Config: z = z.object({ + search: z.boolean().default(true), + fetch: z.boolean().default(true), +}) + +/** + * Register the enabled web tools. `search`/`fetch` default to true; a product + * that wants only one disables the other in config. The tools' disposers are + * fiber-scoped (the effect-based registries clean up on dispose), so no manual + * teardown is needed. + */ +export function apply(ctx: Context, config: Config): void { + if (config.search !== false) applyWebSearchTool(ctx) + if (config.fetch !== false) applyWebFetchTool(ctx) +} + diff --git a/packages/web/tool-web/src/search.ts b/packages/web/tool-web/src/search.ts new file mode 100644 index 0000000000..6ed9991903 --- /dev/null +++ b/packages/web/tool-web/src/search.ts @@ -0,0 +1,105 @@ +/** + * The model-facing `web_search` tool: discover current information on the web. + * Execution goes through `ctx.web` — this module owns only the model-facing + * schema, argument validation, the result-count bound, and result formatting, + * never provider selection or network access. + * + * @module @deepseek-ai/dsh-tool-web/search + */ + +import type { Context } from 'cordis' +import { defineTool } from '@deepseek-ai/dsh-tools' +import type { ToolCallPresentation } from '@deepseek-ai/dsh-tools' +import type { ContentBlock } from '@deepseek-ai/dsh-llm' +import type { WebSearchResult } from '@deepseek-ai/dsh-web' +import type {} from '@deepseek-ai/dsh-system-prompt' + +/** + * Default upper bound on returned sources. Owned by the consumer (not the + * provider or model), mirroring `dsh-tool-fs`'s `READ_LIMIT`/`GREP_LIMIT`. The + * model just asks a question; the product controls how much context returns. + * The default `8` aligns with OpenCode's Exa default. + */ +export const WEB_SEARCH_MAX_RESULTS = 8 + +/** Validate value constraints the schema DSL can't express. */ +export function parseSearchArgs(args: { query: string }): { query: string } { + if (args.query.trim().length === 0) throw new Error('query must be a non-empty string') + return { query: args.query } +} + +/** Display label for a source: its title, else its hostname. */ +function sourceLabel(url: string, title: string | undefined): string { + if (title !== undefined && title.length > 0) return title + try { + return new URL(url).hostname + } catch { + // A provider should return a valid URL, but never let a malformed one throw + // out of pure formatting — fall back to the raw string. + return url + } +} + +/** Format a search result as one model-facing text block. */ +export function formatSearchOutput(result: WebSearchResult): string { + const parts: string[] = [] + if (result.content !== undefined && result.content.length > 0) parts.push(result.content) + + if (result.sources.length > 0) { + const lines = result.sources.map((source) => { + const label = sourceLabel(source.url, source.title) + const meta: string[] = [] + if (source.snippet !== undefined && source.snippet.length > 0) meta.push(source.snippet) + if (source.publishedAt !== undefined && source.publishedAt.length > 0) meta.push(`(${source.publishedAt})`) + const suffix = meta.length > 0 ? ` — ${meta.join(' ')}` : '' + return `- [${label}](${source.url})${suffix}` + }) + parts.push(`Sources:\n${lines.join('\n')}`) + } else if (result.content === undefined || result.content.length === 0) { + parts.push('No results found.') + } + + if (result.truncated) parts.push(`(Showing the first ${result.sources.length} sources. Refine the query for more.)`) + parts.push('Cite the relevant URLs above as markdown links in your answer.') + return parts.join('\n\n') +} + +/** Pending-call presentation: a search card titled by the query. */ +export function presentSearchCall(args: { query: string }): ToolCallPresentation { + return { title: args.query, kind: 'search', rawInput: args.query } +} + +/** Register the `web_search` tool and its system-prompt guidance. */ +export function apply(ctx: Context): void { + ctx.systemPrompt.section({ + name: 'tool:web_search', + order: 110, + text: 'Use the web_search tool to discover current information on the web. It returns an optional answer plus a list of source URLs. Follow up with web_fetch when you need the full content of a specific result, and cite the relevant URLs as markdown links.', + }) + + ctx.tools.register(defineTool({ + name: 'web_search', + description: 'Search the web for current information. Returns an optional summary answer and a list of source URLs.', + parameters: { + query: { type: 'string', required: true, description: 'The search query.' }, + }, + async execute(args, exec): Promise { + const input = parseSearchArgs(args) + const result = await ctx.web.search( + { query: input.query, maxResults: WEB_SEARCH_MAX_RESULTS }, + exec.signal ? { signal: exec.signal } : undefined, + ) + return [{ type: 'text', text: formatSearchOutput(result) }] + }, + presentCall: presentSearchCall, + })) +} + +/** Cordis plugin name used by loader diagnostics. */ +export const name = 'web-search' + +/** Services required by the `web_search` tool plugin. */ +export const inject = ['tools', 'web', 'systemPrompt'] + +/** Named helper for direct registration in the root plugin and tests. */ +export const applyWebSearchTool = apply diff --git a/packages/web/tool-web/tests/integration.spec.ts b/packages/web/tool-web/tests/integration.spec.ts new file mode 100644 index 0000000000..18604190c6 --- /dev/null +++ b/packages/web/tool-web/tests/integration.spec.ts @@ -0,0 +1,99 @@ +/** + * Integration: the real fetch backend (`dsh-web-fetch-local`) + a real search + * provider (`dsh-web-search-exa`) + the real seam (`dsh-web`) + the model tool + * (`dsh-tool-web`), exercised through `ctx.tools.execute()` — nothing bypasses + * the tool registry. Fetch hits a real loopback HTTP server (verifying the + * WORLD); search runs the real Exa provider over a stubbed global `fetch` (the + * network is the one boundary we mock). + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' +import { AddressInfo } from 'node:net' +import { Context } from 'cordis' +import { CallId } from '@deepseek-ai/dsh-llm' +import SystemPrompt from '@deepseek-ai/dsh-system-prompt' +import ToolRegistry from '@deepseek-ai/dsh-tools' +import WebService from '@deepseek-ai/dsh-web' +import * as WebFetchLocal from '@deepseek-ai/dsh-web-fetch-local' +import * as WebSearchExa from '@deepseek-ai/dsh-web-search-exa' +import * as ToolWeb from '@deepseek-ai/dsh-tool-web' + +type Handler = (req: IncomingMessage, res: ServerResponse) => void + +let server: Server +let base: string +let handler: Handler +let ctx: Context +let fiber: Awaited> + +beforeEach(async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/html' }); res.end('

Hello

World

') } + server = createServer((req, res) => { handler(req, res) }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + base = `http://127.0.0.1:${(server.address() as AddressInfo).port}` + + ctx = new Context() + await ctx.plugin(SystemPrompt) + await ctx.plugin(ToolRegistry) + await ctx.plugin(WebService, { searchProvider: WebSearchExa.EXA_PROVIDER_ID, fetchProvider: WebFetchLocal.LOCAL_FETCH_PROVIDER_ID }) + await ctx.plugin(WebFetchLocal, {}) + await ctx.plugin(WebSearchExa, { apiKey: 'exa-key', baseURL: 'https://api.exa.test' }) + fiber = await ctx.plugin(ToolWeb) +}) + +afterEach(async () => { + await fiber.dispose() + vi.unstubAllGlobals() + await new Promise(resolve => server.close(() => { resolve() })) +}) + +let counter = 0 +type ToolResult = { isError: boolean; content: { type: string; text?: string }[]; error?: { code: string } } +function call(name: string, args: unknown): Promise { + return ctx.tools.execute({ callId: CallId(`call-${++counter}`), name, arguments: args }) +} + +describe('web_fetch integration over the real backend', () => { + it('fetches an html page and renders it to markdown', async () => { + const out = await call('web_fetch', { url: base }) + expect(out.isError).toBe(false) + const text = out.content.map(b => b.text).join('') + expect(text).toContain(`Fetched ${base}`) + expect(text).toContain('# Hello') + expect(text).toContain('World') + }) + + it('reports a 404 as a result, not an error', async () => { + handler = (_req, res) => { res.writeHead(404, { 'content-type': 'text/plain' }); res.end('missing') } + const out = await call('web_fetch', { url: base }) + expect(out.isError).toBe(false) + expect(out.content.map(b => b.text).join('')).toContain('HTTP 404') + }) + + it('surfaces WEB_INVALID_URL as a structured tool error', async () => { + const out = await call('web_fetch', { url: 'ftp://example.com' }) + expect(out.isError).toBe(true) + expect(out.error?.code).toBe('WEB_INVALID_URL') + }) + + it('surfaces a blocked cross-origin redirect as WEB_REDIRECT_BLOCKED', async () => { + handler = (_req, res) => { res.writeHead(302, { location: 'https://example.com/' }); res.end() } + const out = await call('web_fetch', { url: base }) + expect(out.isError).toBe(true) + expect(out.error?.code).toBe('WEB_REDIRECT_BLOCKED') + }) +}) + +describe('web_search integration over the real Exa provider', () => { + it('runs web_search end-to-end and formats the provider result', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response( + JSON.stringify({ results: [{ url: 'https://result.test', title: 'Result', highlights: ['a highlight'] }] }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ))) + const out = await call('web_search', { query: 'deepseek' }) + expect(out.isError).toBe(false) + expect(out.content.map(b => b.text).join('')).toContain('[Result](https://result.test)') + }) +}) + diff --git a/packages/web/tool-web/tests/load-path.spec.ts b/packages/web/tool-web/tests/load-path.spec.ts new file mode 100644 index 0000000000..5c47f3ce59 --- /dev/null +++ b/packages/web/tool-web/tests/load-path.spec.ts @@ -0,0 +1,49 @@ +/** + * Real-load-path guard for @deepseek-ai/dsh-tool-web. `tool-web` is a NAMESPACE + * plugin with `inject` — so a stray `export default apply` would make the cordis + * Loader's `unwrapExports` (`exports.default ?? exports`) collapse the module to + * the bare `apply` function, DROPPING `inject`. The plugin would then read + * `ctx.web` without having injected it and throw `cannot get property … without + * inject` the moment it loads (postmortem 0001). + * + * A hand-built `ctx.plugin({ apply, inject })` mount CANNOT catch that — it + * bypasses `unwrapExports`. So this test unwraps the module through the REAL + * `Loader.prototype.unwrapExports` and mounts the result over `ctx.web`, + * exercising the exact path the Loader uses. Prove the guard bites: add + * `export default apply` to `src/index.ts`, watch this go red, revert. + */ + +import { describe, expect, it } from 'vitest' +import { Context } from 'cordis' +import Loader from '@cordisjs/plugin-loader' +import SystemPrompt from '@deepseek-ai/dsh-system-prompt' +import ToolRegistry from '@deepseek-ai/dsh-tools' +import WebService from '@deepseek-ai/dsh-web' +import * as toolWeb from '@deepseek-ai/dsh-tool-web' + +describe('dsh-tool-web real-load-path guard', () => { + it('has no default export and keeps name/inject/Config through unwrapExports', () => { + expect('default' in toolWeb).toBe(false) + + const loader = Object.create(Loader.prototype) as Loader + const unwrapped = loader.unwrapExports(toolWeb) as Record + expect(unwrapped).toBe(toolWeb) + expect(unwrapped.name).toBe('tool-web') + expect(unwrapped.inject).toEqual(['tools', 'web', 'systemPrompt']) + expect(typeof unwrapped.apply).toBe('function') + }) + + it('boots over ctx.web through the unwrapped module without an inject error', async () => { + const ctx = new Context() + await ctx.plugin(SystemPrompt) + await ctx.plugin(ToolRegistry) + await ctx.plugin(WebService, {}) + + const loader = Object.create(Loader.prototype) as Loader + const unwrapped = loader.unwrapExports(toolWeb) as Parameters[0] + // A collapsed export shape (dropped inject) would throw "without inject" here. + const fiber = await ctx.plugin(unwrapped) + expect(ctx.tools.schemas().map(s => s.name)).toEqual(expect.arrayContaining(['web_search', 'web_fetch'])) + await fiber.dispose() + }) +}) diff --git a/packages/web/tool-web/tests/tool-web.spec.ts b/packages/web/tool-web/tests/tool-web.spec.ts new file mode 100644 index 0000000000..2422c32ce3 --- /dev/null +++ b/packages/web/tool-web/tests/tool-web.spec.ts @@ -0,0 +1,281 @@ +import { describe, expect, it } from 'vitest' +import { Context } from 'cordis' +import { CallId } from '@deepseek-ai/dsh-llm' +import SystemPrompt from '@deepseek-ai/dsh-system-prompt' +import ToolRegistry from '@deepseek-ai/dsh-tools' +import WebService from '@deepseek-ai/dsh-web' +import type { WebSearchProvider, WebSearchResult, WebProviderStatus } from '@deepseek-ai/dsh-web' +import * as ToolWeb from '@deepseek-ai/dsh-tool-web' +import { + formatSearchOutput, + formatFetchOutput, + parseSearchArgs, + parseFetchArgs, + presentSearchCall, + presentFetchCall, + renderBody, + htmlToMarkdown, +} from '@deepseek-ai/dsh-tool-web' + +const available: WebProviderStatus = { available: true } + +function searchProvider(result: WebSearchResult, status: WebProviderStatus = available): WebSearchProvider { + return { id: 'stub-search', status: () => status, search: () => Promise.resolve(result) } +} + +/** Mount the real registry, seam, and tool-web; return an executor helper. */ +async function mountTools(opts: { + config?: ToolWeb.Config + webConfig?: ConstructorParameters[1] + search?: WebSearchProvider + fetchProvider?: import('@deepseek-ai/dsh-web').WebFetchProvider +} = {}): Promise<{ ctx: Context; fiber: Awaited>; call: (name: string, args: unknown) => Promise<{ isError: boolean; content: { type: string; text?: string }[]; error?: { code: string } }> }> { + const ctx = new Context() + await ctx.plugin(SystemPrompt) + await ctx.plugin(ToolRegistry) + await ctx.plugin(WebService, opts.webConfig ?? {}) + if (opts.search) ctx.web.registerSearchProvider(opts.search) + if (opts.fetchProvider) ctx.web.registerFetchProvider(opts.fetchProvider) + const fiber = await ctx.plugin(ToolWeb, opts.config ?? {}) + let counter = 0 + const call = (name: string, args: unknown) => ctx.tools.execute({ callId: CallId(`call-${++counter}`), name, arguments: args }) as never + return { ctx, fiber, call } +} + +describe('search formatting', () => { + it('renders content, sources with titles/hostnames, snippets, and a citation reminder', () => { + const out = formatSearchOutput({ + providerId: 'p', query: 'q', content: 'an answer', truncated: false, + sources: [ + { url: 'https://a.test/x', title: 'A', snippet: 'about a', publishedAt: '2026-01-01' }, + { url: 'https://b.test/y' }, + ], + }) + expect(out).toContain('an answer') + expect(out).toContain('[A](https://a.test/x) — about a (2026-01-01)') + expect(out).toContain('[b.test](https://b.test/y)') + expect(out).toContain('Cite the relevant URLs') + }) + + it('reports no results when there is neither content nor sources', () => { + expect(formatSearchOutput({ providerId: 'p', query: 'q', sources: [], truncated: false })) + .toContain('No results found.') + }) + + it('renders content alone when there are no sources', () => { + const out = formatSearchOutput({ providerId: 'p', query: 'q', content: 'just an answer', sources: [], truncated: false }) + expect(out).toContain('just an answer') + expect(out).not.toContain('No results found.') + expect(out).not.toContain('Sources:') + }) + + it('notes truncation', () => { + const out = formatSearchOutput({ providerId: 'p', query: 'q', sources: [{ url: 'https://a.test' }], truncated: true }) + expect(out).toContain('Showing the first 1 sources') + }) + + it('validates the query', () => { + expect(() => parseSearchArgs({ query: ' ' })).toThrow('non-empty') + expect(parseSearchArgs({ query: 'hi' })).toEqual({ query: 'hi' }) + }) + + it('presents a search call as a search-kind card titled by the query', () => { + expect(presentSearchCall({ query: 'find me' })).toEqual({ title: 'find me', kind: 'search', rawInput: 'find me' }) + }) +}) + +describe('fetch formatting', () => { + it('renders an html body to markdown text with a status header', () => { + const out = formatFetchOutput({ + providerId: 'p', url: 'https://a.test', statusCode: 200, truncated: false, + body: { kind: 'html', content: '

Title

Body text

' }, + }) + expect(out).toContain('Fetched https://a.test (HTTP 200)') + expect(out).toContain('# Title') + expect(out).toContain('Body text') + }) + + it('passes a text body through and notes truncation', () => { + const out = formatFetchOutput({ + providerId: 'p', url: 'https://a.test', statusCode: 200, truncated: true, + body: { kind: 'text', content: 'plain' }, + }) + expect(out).toContain('plain') + expect(out).toContain('Content truncated') + }) + + it('renderBody dispatches on kind', () => { + expect(renderBody({ kind: 'text', content: 'x' })).toBe('x') + expect(renderBody({ kind: 'html', content: '

y

' })).toBe('y') + }) + + it('validates url and timeout', () => { + expect(() => parseFetchArgs({ url: ' ' })).toThrow('non-empty') + expect(() => parseFetchArgs({ url: 'https://a.test', timeout_ms: -1 })).toThrow('positive') + expect(parseFetchArgs({ url: 'https://a.test', timeout_ms: 5 })).toEqual({ url: 'https://a.test', timeoutMs: 5 }) + }) + + it('presents a fetch call as a fetch-kind card titled by the url', () => { + expect(presentFetchCall({ url: 'https://a.test' })).toEqual({ title: 'https://a.test', kind: 'fetch', rawInput: 'https://a.test' }) + }) +}) + +describe('htmlToMarkdown', () => { + it('drops scripts/styles, keeps text, decodes entities, converts links', () => { + const md = htmlToMarkdown('

Tom & Jerry

link') + expect(md).not.toContain('bad()') + expect(md).not.toContain('.x{}') + expect(md).toContain('Tom & Jerry') + expect(md).toContain('[link](https://a.test)') + }) + + it('decodes numeric entities and collapses whitespace', () => { + expect(htmlToMarkdown('

a'b

')).toBe("a'b") + expect(htmlToMarkdown('
x
\n\n\n
y
')).toBe('x\n\ny') + }) + + it('decodes hex entities and named entities, and leaves unknown/out-of-range ones intact', () => { + expect(htmlToMarkdown('

AB

')).toBe('AB') + expect(htmlToMarkdown('

© —

')).toBe('© —') + expect(htmlToMarkdown('

¬areal;

')).toBe('¬areal;') + // An out-of-range code point keeps the original entity text (fromCodePoint fallback). + expect(htmlToMarkdown('

�

')).toBe('�') + expect(htmlToMarkdown('

�

')).toBe('�') + }) + + it('renders a link with an empty label as its bare href', () => { + expect(htmlToMarkdown('')).toBe('https://a.test') + }) + + it('converts headings and list items to markdown', () => { + expect(htmlToMarkdown('

Heading

after

')).toContain('## Heading') + const list = htmlToMarkdown('
  • one
  • two
') + expect(list).toContain('- one') + expect(list).toContain('- two') + }) + + it('falls back to the raw URL as a source label when the URL is unparseable', () => { + const out = formatSearchOutput({ providerId: 'p', query: 'q', truncated: false, sources: [{ url: 'not a url' }] }) + expect(out).toContain('[not a url](not a url)') + }) +}) + +describe('tool-web registration', () => { + it('registers both tools by default', async () => { + const { fiber, ctx } = await mountTools() + const names = ctx.tools.schemas().map(s => s.name) + expect(names).toContain('web_search') + expect(names).toContain('web_fetch') + await fiber.dispose() + expect(ctx.tools.schemas().map(s => s.name)).not.toContain('web_search') + }) + + it('registers only enabled tools', async () => { + const { fiber, ctx } = await mountTools({ config: { search: true, fetch: false } }) + const names = ctx.tools.schemas().map(s => s.name) + expect(names).toContain('web_search') + expect(names).not.toContain('web_fetch') + await fiber.dispose() + }) + + it('registers only web_fetch when search is disabled', async () => { + const { fiber, ctx } = await mountTools({ config: { search: false, fetch: true } }) + const names = ctx.tools.schemas().map(s => s.name) + expect(names).not.toContain('web_search') + expect(names).toContain('web_fetch') + await fiber.dispose() + }) + + it('registers web_search even when no provider is available (schema follows enablement, not availability)', async () => { + const { fiber, ctx } = await mountTools() + expect(ctx.tools.schemas().map(s => s.name)).toContain('web_search') + expect(ctx.web.searchStatus()).toEqual({ available: false, reason: 'none' }) + await fiber.dispose() + }) + + it('contributes prompt sections for the enabled tools', async () => { + const { fiber, ctx } = await mountTools() + const prompt = await ctx.systemPrompt.assemble() + const text = prompt.sections.map(s => (typeof s.text === 'function' ? s.text() : s.text)).join('\n') + expect(text).toContain('web_search') + expect(text).toContain('web_fetch') + await fiber.dispose() + }) +}) + +describe('tool-web execution through the real registry', () => { + it('executes web_search and formats the result', async () => { + const result: WebSearchResult = { + providerId: 'stub-search', query: 'q', content: 'answer', truncated: false, + sources: [{ url: 'https://a.test', title: 'A', snippet: 'snip' }], + } + const { fiber, call } = await mountTools({ webConfig: { searchProvider: 'stub-search' }, search: searchProvider(result) }) + const out = await call('web_search', { query: 'q' }) + expect(out.isError).toBe(false) + expect(out.content.map(b => b.text).join('')).toContain('[A](https://a.test)') + await fiber.dispose() + }) + + it('surfaces a structured WebError when no provider is available', async () => { + const { fiber, call } = await mountTools() + const out = await call('web_search', { query: 'q' }) + expect(out.isError).toBe(true) + expect(out.error?.code).toBe('WEB_PROVIDER_UNAVAILABLE') + await fiber.dispose() + }) + + it('surfaces WEB_PROVIDER_AMBIGUOUS for multiple unconfigured providers', async () => { + const { ctx, fiber, call } = await mountTools({ search: searchProvider({ providerId: 'stub-search', query: 'q', sources: [], truncated: false }) }) + ctx.web.registerSearchProvider({ id: 'other', status: () => available, search: () => Promise.resolve({ providerId: 'other', query: 'q', sources: [], truncated: false }) }) + const out = await call('web_search', { query: 'q' }) + expect(out.isError).toBe(true) + expect(out.error?.code).toBe('WEB_PROVIDER_AMBIGUOUS') + await fiber.dispose() + }) + + it('rejects invalid arguments with a structured INVALID_ARGS error', async () => { + const { fiber, call } = await mountTools({ webConfig: { searchProvider: 'stub-search' }, search: searchProvider({ providerId: 'stub-search', query: 'q', sources: [], truncated: false }) }) + const out = await call('web_search', { query: 123 }) + expect(out.isError).toBe(true) + expect(out.error?.code).toBe('INVALID_ARGS') + await fiber.dispose() + }) + + it('has no default export (namespace plugin export shape)', () => { + expect('default' in ToolWeb).toBe(false) + }) + + it('executes web_fetch, forwarding timeout_ms and the abort signal to the seam', async () => { + const seen: { request?: { url: string; timeoutMs?: number }; signal?: AbortSignal | undefined } = {} + const fetchProvider = { + id: 'stub-fetch', + status: () => available, + fetch: (request: { url: string; timeoutMs?: number }, exec?: { signal?: AbortSignal }) => { + seen.request = request + seen.signal = exec?.signal + return Promise.resolve({ providerId: 'stub-fetch', url: request.url, statusCode: 200, body: { kind: 'text' as const, content: 'ok' }, truncated: false }) + }, + } + const { ctx, fiber } = await mountTools({ webConfig: { fetchProvider: 'stub-fetch' }, fetchProvider }) + const controller = new AbortController() + const out = await ctx.tools.execute({ callId: CallId('fetch-1'), name: 'web_fetch', arguments: { url: 'https://a.test', timeout_ms: 1234 }, signal: controller.signal }) + expect(out.isError).toBe(false) + expect(seen.request).toEqual({ url: 'https://a.test', timeoutMs: 1234 }) + expect(seen.signal).toBe(controller.signal) + await fiber.dispose() + }) + + it('executes web_search, forwarding the abort signal to the seam', async () => { + const seen: { signal?: AbortSignal | undefined } = {} + const provider: WebSearchProvider = { + id: 'stub-search', + status: () => available, + search: (_request, exec) => { seen.signal = exec?.signal; return Promise.resolve({ providerId: 'stub-search', query: 'q', sources: [], truncated: false }) }, + } + const { ctx, fiber } = await mountTools({ webConfig: { searchProvider: 'stub-search' }, search: provider }) + const controller = new AbortController() + await ctx.tools.execute({ callId: CallId('search-1'), name: 'web_search', arguments: { query: 'q' }, signal: controller.signal }) + expect(seen.signal).toBe(controller.signal) + await fiber.dispose() + }) +}) diff --git a/packages/web/tool-web/tsconfig.json b/packages/web/tool-web/tsconfig.json new file mode 100644 index 0000000000..b4121a6c14 --- /dev/null +++ b/packages/web/tool-web/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib" + }, + "include": ["src"], + "references": [ + { "path": "../../../vendor/cosmokit" }, + { "path": "../../../vendor/cordis" }, + { "path": "../../../vendor/schemastery" }, + { "path": "../../llm/llm" }, + { "path": "../../core/tools" }, + { "path": "../../core/system-prompt" }, + { "path": "../web" } + ] +} diff --git a/packages/web/tool-web/tsdown.config.ts b/packages/web/tool-web/tsdown.config.ts new file mode 100644 index 0000000000..c4849939db --- /dev/null +++ b/packages/web/tool-web/tsdown.config.ts @@ -0,0 +1,18 @@ +import { defineConfig } from 'tsdown' + +/** + * tool-web exposes one package root plus one entry per tool plugin, so each tool + * can be loaded or replaced independently as a subpath plugin + * (`@deepseek-ai/dsh-tool-web/search`, `/fetch`). The root tsdown config only + * auto-discovers `src/index.ts`, so the subpath entries are declared here. + */ +export default defineConfig({ + entry: ['src/index.ts', 'src/search.ts', 'src/fetch.ts'], + outDir: 'lib', + format: ['esm'], + platform: 'node', + target: 'es2024', + fixedExtension: false, + dts: false, + clean: false, +}) diff --git a/packages/web/web-fetch-local/README.md b/packages/web/web-fetch-local/README.md new file mode 100644 index 0000000000..fd9150e46f --- /dev/null +++ b/packages/web/web-fetch-local/README.md @@ -0,0 +1,34 @@ +# @deepseek-ai/dsh-web-fetch-local + +An anonymous public HTTP(S) `WebFetchProvider` for the harness [web capability seam](../web/README.md) (`ctx.web`). It retrieves a concrete URL and returns a status code plus bounded decoded content. + +This is an **implementation** package: it registers a provider into `ctx.web`, it does not own the key and it does not register a model-facing tool. It is a function/namespace plugin (`inject: ['web']`). + +## Responsibility split + +The provider owns **safe resource retrieval**: URL validation, HTTP transport, redirect policy, timeout, abort propagation, byte caps, charset decoding, content-type classification, and binary rejection. `@deepseek-ai/dsh-tool-web` owns **presentation** (HTML→markdown, truncation formatting). A non-2xx HTTP response is a *result* (status code + decoded body), not an error; `WebError` is reserved for failures to safely retrieve or represent the resource. + +## Transport hygiene + +- Accepts only `http:` and `https:` URLs; rejects credentials in URLs (`WEB_BLOCKED_URL`) and over-long/malformed URLs (`WEB_INVALID_URL`). +- Enforces a max URL length, response byte cap (`WEB_FETCH_TOO_LARGE`), decoded body character cap, timeout (`WEB_FETCH_TIMEOUT`), and redirect hop cap. +- Propagates the caller's abort signal (`WEB_ABORTED`) into the network request and the streaming read. +- Follows only **same-origin** redirects; a cross-origin redirect fails with `WEB_REDIRECT_BLOCKED`, requiring a fresh tool call (the model of Claude Code's WebFetch). +- Sends an explicit product `User-Agent`, never a browser disguise. +- Rejects unsupported (e.g. binary) content types with `WEB_UNSUPPORTED_CONTENT_TYPE`. + +## Config + +| Key | Default | Meaning | +|---|---|---| +| `maxUrlLength` | `2048` | Maximum accepted request URL length. | +| `maxResponseBytes` | `5_000_000` | Maximum response body size in bytes. | +| `maxBodyChars` | `100_000` | Maximum decoded body length in characters. | +| `timeoutMs` | `30_000` | Default fetch timeout. | +| `maxTimeoutMs` | `120_000` | Upper bound for a per-request timeout override. | +| `maxRedirects` | `5` | Maximum same-origin redirect hops. | +| `userAgent` | `deepseek-harness/…` | `User-Agent` header. | + +## Security note + +SSRF / private-network protection (blocking private, loopback, link-local, multicast, and otherwise non-public destinations, with DNS-resolve-then-validate and per-hop re-validation) is **deferred** — see the [web capability seam RFC](../../../docs/rfc/implemented/architecture/2026-06-24-web-capability-seam.md). Until it lands, this provider is an SSRF primitive and **must not be enabled** in a deployment that can reach sensitive internal network targets. diff --git a/packages/web/web-fetch-local/package.json b/packages/web/web-fetch-local/package.json new file mode 100644 index 0000000000..7249697ec3 --- /dev/null +++ b/packages/web/web-fetch-local/package.json @@ -0,0 +1,33 @@ +{ + "name": "@deepseek-ai/dsh-web-fetch-local", + "description": "Anonymous public HTTP(S) fetch provider for the DeepSeek Harness web capability seam (ctx.web)", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "exports": { + ".": { + "types": "./lib/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "@deepseek-ai/dsh-web": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "dependencies": { + "schemastery": "^3.18.0" + }, + "devDependencies": { + "@deepseek-ai/dsh-web": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/web/web-fetch-local/src/index.ts b/packages/web/web-fetch-local/src/index.ts new file mode 100644 index 0000000000..1d57410d68 --- /dev/null +++ b/packages/web/web-fetch-local/src/index.ts @@ -0,0 +1,77 @@ +/** + * `@deepseek-ai/dsh-web-fetch-local`: registers an anonymous public HTTP(S) + * `WebFetchProvider` with `ctx.web`. A function/namespace plugin (NOT a + * default-export service): it registers INTO the seam's fetch registry, like the + * search providers register into the search registry. + * + * @module @deepseek-ai/dsh-web-fetch-local + */ + +import type { Context } from 'cordis' +import z from 'schemastery' +import type {} from '@deepseek-ai/dsh-web' +import { LocalFetchProvider } from './provider.ts' +import type { LocalFetchLimits } from './provider.ts' + +export { + LOCAL_FETCH_PROVIDER_ID, + LocalFetchProvider, +} from './provider.ts' +export type { LocalFetchLimits } from './provider.ts' +export { classifyContentType, isSameOrigin, validateFetchUrl } from './policy.ts' +export type { FetchableKind } from './policy.ts' + +/** Default `User-Agent`: an explicit product agent, never a browser disguise. */ +export const DEFAULT_USER_AGENT = 'deepseek-harness/0.0.1 (+https://github.com/deepseek-ai)' + +/** Cordis plugin name used by loader diagnostics. */ +export const name = 'web-fetch-local' + +/** The web seam this provider registers into. */ +export const inject = ['web'] + +export interface Config { + /** Maximum accepted request URL length. */ + maxUrlLength?: number + /** Maximum response body size in bytes. */ + maxResponseBytes?: number + /** Maximum decoded body length in characters. */ + maxBodyChars?: number + /** Default fetch timeout in milliseconds. */ + timeoutMs?: number + /** Upper bound for a per-request timeout override. */ + maxTimeoutMs?: number + /** Maximum number of same-origin redirect hops to follow. */ + maxRedirects?: number + /** `User-Agent` header sent on every request. */ + userAgent?: string +} + +export const Config: z = z.object({ + maxUrlLength: z.number().default(2048), + maxResponseBytes: z.number().default(5_000_000), + maxBodyChars: z.number().default(100_000), + timeoutMs: z.number().default(30_000), + maxTimeoutMs: z.number().default(120_000), + maxRedirects: z.number().default(5), + userAgent: z.string().default(DEFAULT_USER_AGENT), +}) + +/** The shape after schemastery applies its defaults to every field. */ +type ResolvedConfig = Required + +/** Register the local HTTP(S) fetch provider with `ctx.web`. */ +export function apply(ctx: Context, config: Config): void { + // schemastery (Config) has already filled every defaulted field. + const resolved = config as ResolvedConfig + const limits: LocalFetchLimits = { + maxUrlLength: resolved.maxUrlLength, + maxResponseBytes: resolved.maxResponseBytes, + maxBodyChars: resolved.maxBodyChars, + timeoutMs: resolved.timeoutMs, + maxTimeoutMs: resolved.maxTimeoutMs, + maxRedirects: resolved.maxRedirects, + userAgent: resolved.userAgent, + } + ctx.web.registerFetchProvider(new LocalFetchProvider(limits)) +} diff --git a/packages/web/web-fetch-local/src/policy.ts b/packages/web/web-fetch-local/src/policy.ts new file mode 100644 index 0000000000..8261c5c1ed --- /dev/null +++ b/packages/web/web-fetch-local/src/policy.ts @@ -0,0 +1,59 @@ +/** + * URL validation and content-type classification for the local HTTP(S) fetch + * provider — the pure, network-free half. The provider's `fetch()` composes + * these with transport (redirect following, byte caps, decoding). + * + * @module @deepseek-ai/dsh-web-fetch-local/policy + */ + +import { WebError } from '@deepseek-ai/dsh-web' + +/** The body kinds this provider decodes. */ +export type FetchableKind = 'html' | 'text' + +/** + * Validate a request URL against the basic transport hygiene the provider + * enforces before any network access: http(s) only, no embedded credentials, + * bounded length. Returns the parsed `URL`. Throws {@link WebError} otherwise. + * (SSRF / private-network blocking is deferred — see the package RFC.) + */ +export function validateFetchUrl(input: string, maxUrlLength: number): URL { + if (input.length > maxUrlLength) { + throw new WebError(`URL exceeds the maximum length of ${maxUrlLength}`, 'WEB_INVALID_URL') + } + let url: URL + try { + url = new URL(input) + } catch (error: unknown) { + throw new WebError(`invalid URL: ${input}`, 'WEB_INVALID_URL', { cause: error }) + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new WebError(`unsupported URL scheme "${url.protocol}" (only http and https are allowed)`, 'WEB_INVALID_URL') + } + if (url.username.length > 0 || url.password.length > 0) { + throw new WebError('credentials in URLs are not allowed', 'WEB_BLOCKED_URL') + } + return url +} + +/** + * Two URLs are same-origin when scheme, hostname, and port match. A redirect + * that crosses origins is refused so each new origin requires a fresh tool call + * (and thus a fresh provider/permission decision). + */ +export function isSameOrigin(a: URL, b: URL): boolean { + return a.protocol === b.protocol && a.hostname === b.hostname && a.port === b.port +} + +/** + * Classify a response `Content-Type` into a decodable body kind, or `undefined` + * for an unsupported (e.g. binary) type. `text/html` and `application/xhtml+xml` + * are `html`; other `text/*` plus a few structured text types are `text`. + */ +export function classifyContentType(contentType: string | null): FetchableKind | undefined { + const mime = (contentType ?? '').replace(/;.*$/s, '').trim().toLowerCase() + if (mime === 'text/html' || mime === 'application/xhtml+xml') return 'html' + if (mime.startsWith('text/')) return 'text' + if (mime === 'application/json' || mime === 'application/xml' || mime.endsWith('+json') || mime.endsWith('+xml')) return 'text' + return undefined +} diff --git a/packages/web/web-fetch-local/src/provider.ts b/packages/web/web-fetch-local/src/provider.ts new file mode 100644 index 0000000000..0622030af0 --- /dev/null +++ b/packages/web/web-fetch-local/src/provider.ts @@ -0,0 +1,233 @@ +/** + * `LocalFetchProvider`: a `WebFetchProvider` that retrieves a concrete public + * HTTP(S) URL with the platform-native `fetch` (Node 24) and returns a status + * code plus bounded decoded content. It owns SAFE RESOURCE RETRIEVAL — URL + * validation, redirect policy, timeout, abort, byte caps, charset decoding, + * content-type classification, binary rejection — but NOT presentation + * (HTML→markdown lives in `@deepseek-ai/dsh-tool-web`). + * + * Redirects are followed manually (`redirect: 'manual'`) so the provider can + * enforce a same-origin-only policy: a cross-origin redirect is refused with + * `WEB_REDIRECT_BLOCKED`, requiring a fresh tool call (Claude Code's WebFetch + * uses the same model). It does NOT carry browser cookies, editor/git + * credentials, or implicit access to private services. + * + * SSRF / private-network protection is DEFERRED (see the package RFC); until it + * lands this provider is an SSRF primitive and must not be enabled where it can + * reach sensitive internal targets. + * + * @module @deepseek-ai/dsh-web-fetch-local/provider + */ + +import { WebError } from '@deepseek-ai/dsh-web' +import type { WebFetchBody, WebFetchProvider, WebFetchRequest, WebFetchResult, WebProviderStatus } from '@deepseek-ai/dsh-web' +import { classifyContentType, isSameOrigin, validateFetchUrl } from './policy.ts' + +/** Resolved provider limits (the plugin's schemastery Config supplies defaults). */ +export interface LocalFetchLimits { + /** Maximum accepted request URL length. */ + maxUrlLength: number + /** Maximum response body size in bytes (read is aborted past this). */ + maxResponseBytes: number + /** Maximum decoded body length in characters (truncated past this). */ + maxBodyChars: number + /** Default fetch timeout in milliseconds. */ + timeoutMs: number + /** Upper bound for a per-request timeout override. */ + maxTimeoutMs: number + /** Maximum number of (same-origin) redirect hops to follow. */ + maxRedirects: number + /** `User-Agent` header sent on every request. */ + userAgent: string +} + +/** Stable id this provider registers under. */ +export const LOCAL_FETCH_PROVIDER_ID = 'local-http' + +/** The anonymous public HTTP(S) fetch provider. */ +export class LocalFetchProvider implements WebFetchProvider { + readonly id = LOCAL_FETCH_PROVIDER_ID + + constructor(private readonly limits: LocalFetchLimits) {} + + /** No credentials to check — an anonymous public fetcher is always usable. */ + status(): WebProviderStatus { + return { available: true } + } + + async fetch(request: WebFetchRequest, exec?: { readonly signal?: AbortSignal }): Promise { + const timeoutMs = request.timeoutMs !== undefined + ? Math.min(request.timeoutMs, this.limits.maxTimeoutMs) + : this.limits.timeoutMs + + // One controller drives both the caller's abort and our own timeout, so the + // network request and the streaming read both stop on either. + const controller = new AbortController() + const onAbort = (): void => { controller.abort() } + if (exec?.signal !== undefined) { + if (exec.signal.aborted) throw new WebError('web fetch aborted', 'WEB_ABORTED') + exec.signal.addEventListener('abort', onAbort, { once: true }) + } + const timer = setTimeout(() => { controller.abort(new WebError('web fetch timed out', 'WEB_FETCH_TIMEOUT')) }, timeoutMs) + + try { + return await this.followAndRead(request.url, controller, timeoutMs) + } finally { + clearTimeout(timer) + if (exec?.signal !== undefined) exec.signal.removeEventListener('abort', onAbort) + } + } + + /** Follow same-origin redirects up to the hop cap, then read the final response. */ + private async followAndRead(initialUrl: string, controller: AbortController, timeoutMs: number): Promise { + let currentUrl = validateFetchUrl(initialUrl, this.limits.maxUrlLength) + + for (let hop = 0; hop <= this.limits.maxRedirects; hop++) { + const response = await this.requestOnce(currentUrl, controller, timeoutMs) + + if (isRedirectStatus(response.status)) { + const location = response.headers.get('location') + if (location === null) { + // A redirect status with no Location is not a usable resource. + throw new WebError(`redirect response (HTTP ${response.status}) without a Location header`, 'WEB_PROVIDER_ERROR') + } + const target = resolveRedirect(location, currentUrl) + if (!isSameOrigin(target, currentUrl)) { + throw new WebError( + `cross-origin redirect to ${target.origin} is not followed automatically; retry against that URL directly`, + 'WEB_REDIRECT_BLOCKED', + ) + } + await response.body?.cancel() + currentUrl = target + continue + } + + return await this.readBody(response, currentUrl) + } + + throw new WebError(`exceeded the maximum of ${this.limits.maxRedirects} redirects`, 'WEB_REDIRECT_BLOCKED') + } + + private async requestOnce(url: URL, controller: AbortController, _timeoutMs: number): Promise { + try { + return await fetch(url, { + method: 'GET', + redirect: 'manual', + headers: { 'user-agent': this.limits.userAgent, 'accept': 'text/html,application/xhtml+xml,text/*;q=0.9,application/json;q=0.8' }, + signal: controller.signal, + }) + } catch (error: unknown) { + throw translateAbortOrNetwork(error) + } + } + + /** Read, byte-cap, classify, and decode the final response body. */ + private async readBody(response: Response, finalUrl: URL): Promise { + const kind = classifyContentType(response.headers.get('content-type')) + if (kind === undefined) { + await response.body?.cancel() + throw new WebError(`unsupported content type "${response.headers.get('content-type') ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE') + } + + const { bytes, truncatedByBytes } = await this.readCapped(response) + const decoded = new TextDecoder('utf-8').decode(bytes) + const truncatedByChars = decoded.length > this.limits.maxBodyChars + const content = truncatedByChars ? decoded.slice(0, this.limits.maxBodyChars) : decoded + const body: WebFetchBody = kind === 'html' ? { kind: 'html', content } : { kind: 'text', content } + + return { + providerId: this.id, + url: finalUrl.toString(), + statusCode: response.status, + body, + truncated: truncatedByBytes || truncatedByChars, + } + } + + /** + * Read the response stream up to `maxResponseBytes`. A `Content-Length` over + * the cap rejects immediately with `WEB_FETCH_TOO_LARGE`; a stream that grows + * past the cap is cut short (`truncatedByBytes`) rather than rejected, so a + * server that under-reports still yields a bounded usable body. + */ + private async readCapped(response: Response): Promise<{ bytes: Uint8Array; truncatedByBytes: boolean }> { + const declared = response.headers.get('content-length') + if (declared !== null) { + const length = Number(declared) + if (Number.isFinite(length) && length > this.limits.maxResponseBytes) { + await response.body?.cancel() + throw new WebError(`response exceeds the maximum of ${this.limits.maxResponseBytes} bytes`, 'WEB_FETCH_TOO_LARGE') + } + } + + /* v8 ignore next -- a 2xx Response from fetch always exposes a body stream; the null guard is defensive. */ + if (response.body === null) return { bytes: new Uint8Array(0), truncatedByBytes: false } + + const chunks: Uint8Array[] = [] + let total = 0 + let truncatedByBytes = false + const reader = response.body.getReader() + try { + for (;;) { + const { done, value } = await reader.read() + if (done) break + const remaining = this.limits.maxResponseBytes - total + if (value.byteLength >= remaining) { + chunks.push(value.subarray(0, remaining)) + total += remaining + truncatedByBytes = true + break + } + chunks.push(value) + total += value.byteLength + } + } catch (error: unknown) { + /* v8 ignore next -- mid-stream read fault needs a network drop after headers; translate path covered by request-phase tests. */ + throw translateAbortOrNetwork(error) + } finally { + /* v8 ignore next 4 -- cancel() after a completed/broken read settles without rejecting; unobserved best-effort cleanup. */ + await reader.cancel().catch(() => { + // Cancel after a successful read (or after we broke past the cap) is + // best-effort cleanup; the bytes we need are already collected. + }) + } + + const bytes = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + bytes.set(chunk, offset) + offset += chunk.byteLength + } + return { bytes, truncatedByBytes } + } +} + +/** HTTP redirect status codes that carry a `Location`. */ +function isRedirectStatus(status: number): boolean { + return status === 301 || status === 302 || status === 303 || status === 307 || status === 308 +} + +/** Resolve a (possibly relative) `Location` against the current URL. */ +function resolveRedirect(location: string, base: URL): URL { + try { + return new URL(location, base) + } catch (error: unknown) { + /* v8 ignore next 2 -- URL resolution against a valid absolute base effectively never throws; defensive guard. */ + throw new WebError(`invalid redirect Location "${location}"`, 'WEB_PROVIDER_ERROR', { cause: error }) + } +} + +/** + * Translate a thrown fetch/stream error into a `WebError`. Our own + * `WEB_FETCH_TIMEOUT` (passed to `controller.abort(reason)`) and any other + * already-typed `WebError` pass through; an `AbortError` becomes `WEB_ABORTED`; + * anything else is a transport/network failure (`WEB_PROVIDER_ERROR`). + */ +function translateAbortOrNetwork(error: unknown): WebError { + if (error instanceof WebError) return error + if (error instanceof DOMException && error.name === 'AbortError') { + return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error }) + } + return new WebError(`web fetch failed: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) +} diff --git a/packages/web/web-fetch-local/tests/fetch-local.spec.ts b/packages/web/web-fetch-local/tests/fetch-local.spec.ts new file mode 100644 index 0000000000..3ca48150e1 --- /dev/null +++ b/packages/web/web-fetch-local/tests/fetch-local.spec.ts @@ -0,0 +1,239 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' +import { AddressInfo } from 'node:net' +import { Context } from 'cordis' +import WebService from '@deepseek-ai/dsh-web' +import { LocalFetchProvider, LOCAL_FETCH_PROVIDER_ID, classifyContentType, isSameOrigin, validateFetchUrl } from '@deepseek-ai/dsh-web-fetch-local' +import type { LocalFetchLimits } from '@deepseek-ai/dsh-web-fetch-local' +import * as fetchPlugin from '@deepseek-ai/dsh-web-fetch-local' + +const limits: LocalFetchLimits = { + maxUrlLength: 2048, + maxResponseBytes: 5_000_000, + maxBodyChars: 100_000, + timeoutMs: 5_000, + maxTimeoutMs: 10_000, + maxRedirects: 5, + userAgent: 'test-agent/1.0', +} + +type Handler = (req: IncomingMessage, res: ServerResponse) => void + +let server: Server +let base: string +let handler: Handler + +beforeEach(async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('default') } + server = createServer((req, res) => { handler(req, res) }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + base = `http://127.0.0.1:${port}` +}) + +afterEach(async () => { + await new Promise(resolve => server.close(() => { resolve() })) +}) + +function provider(overrides: Partial = {}): LocalFetchProvider { + return new LocalFetchProvider({ ...limits, ...overrides }) +} + +describe('policy helpers', () => { + it('validates scheme, credentials, and length', () => { + expect(validateFetchUrl('https://example.com/x', 2048).hostname).toBe('example.com') + expect(() => validateFetchUrl('ftp://example.com', 2048)).toThrow(expect.objectContaining({ code: 'WEB_INVALID_URL' })) + expect(() => validateFetchUrl('not a url', 2048)).toThrow(expect.objectContaining({ code: 'WEB_INVALID_URL' })) + expect(() => validateFetchUrl('https://user:pass@example.com', 2048)).toThrow(expect.objectContaining({ code: 'WEB_BLOCKED_URL' })) + expect(() => validateFetchUrl(`https://example.com/${'a'.repeat(3000)}`, 2048)).toThrow(expect.objectContaining({ code: 'WEB_INVALID_URL' })) + }) + + it('classifies content types', () => { + expect(classifyContentType('text/html; charset=utf-8')).toBe('html') + expect(classifyContentType('application/xhtml+xml')).toBe('html') + expect(classifyContentType('text/plain')).toBe('text') + expect(classifyContentType('application/json')).toBe('text') + expect(classifyContentType('image/png')).toBeUndefined() + expect(classifyContentType(null)).toBeUndefined() + }) + + it('compares origins', () => { + expect(isSameOrigin(new URL('https://a.com/x'), new URL('https://a.com/y'))).toBe(true) + expect(isSameOrigin(new URL('https://a.com'), new URL('https://b.com'))).toBe(false) + expect(isSameOrigin(new URL('http://a.com'), new URL('https://a.com'))).toBe(false) + }) +}) + +describe('LocalFetchProvider success', () => { + it('fetches a text body', async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('hello world') } + const result = await provider().fetch({ url: base }) + expect(result.providerId).toBe(LOCAL_FETCH_PROVIDER_ID) + expect(result.statusCode).toBe(200) + expect(result.body).toEqual({ kind: 'text', content: 'hello world' }) + expect(result.truncated).toBe(false) + }) + + it('fetches an html body and classifies it as html', async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/html' }); res.end('

hi

') } + const result = await provider().fetch({ url: base }) + expect(result.body).toEqual({ kind: 'html', content: '

hi

' }) + }) + + it('sends the configured user agent', async () => { + let seen: string | undefined + handler = (req, res) => { seen = req.headers['user-agent']; res.writeHead(200, { 'content-type': 'text/plain' }); res.end('ok') } + await provider().fetch({ url: base }) + expect(seen).toBe('test-agent/1.0') + }) + + it('returns a non-2xx response as a result, not an error', async () => { + handler = (_req, res) => { res.writeHead(404, { 'content-type': 'text/plain' }); res.end('nope') } + const result = await provider().fetch({ url: base }) + expect(result.statusCode).toBe(404) + expect(result.body).toEqual({ kind: 'text', content: 'nope' }) + }) +}) + +describe('LocalFetchProvider caps', () => { + it('rejects an over-cap Content-Length with WEB_FETCH_TOO_LARGE', async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain', 'content-length': '999999' }); res.end('x'.repeat(999999)) } + await expect(provider({ maxResponseBytes: 10 }).fetch({ url: base })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_FETCH_TOO_LARGE' })) + }) + + it('truncates a stream that grows past the byte cap', async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('abcdefghij') } + const result = await provider({ maxResponseBytes: 4 }).fetch({ url: base }) + expect(result.body.content).toBe('abcd') + expect(result.truncated).toBe(true) + }) + + it('truncates a decoded body past the character cap', async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('abcdefghij') } + const result = await provider({ maxBodyChars: 3 }).fetch({ url: base }) + expect(result.body.content).toBe('abc') + expect(result.truncated).toBe(true) + }) + + it('rejects an unsupported content type', async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'image/png' }); res.end('binary') } + await expect(provider().fetch({ url: base })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + }) + + it('rejects a response with no content type at all', async () => { + handler = (_req, res) => { res.writeHead(200); res.end('no type') } + await expect(provider().fetch({ url: base })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + }) + + it('accepts a declared content-length within the cap', async () => { + handler = (_req, res) => { const body = 'sized'; res.writeHead(200, { 'content-type': 'text/plain', 'content-length': String(body.length) }); res.end(body) } + const result = await provider().fetch({ url: base }) + expect(result.body.content).toBe('sized') + }) +}) + +describe('LocalFetchProvider redirects', () => { + it('follows a same-origin redirect and reports the final URL', async () => { + handler = (req, res) => { + if (req.url === '/start') { res.writeHead(302, { location: '/end' }); res.end() } + else { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('arrived') } + } + const result = await provider().fetch({ url: `${base}/start` }) + expect(result.body.content).toBe('arrived') + expect(result.url).toBe(`${base}/end`) + }) + + it('blocks a cross-origin redirect with WEB_REDIRECT_BLOCKED', async () => { + handler = (_req, res) => { res.writeHead(302, { location: 'https://example.com/' }); res.end() } + await expect(provider().fetch({ url: base })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_REDIRECT_BLOCKED' })) + }) + + it('rejects exceeding the redirect hop cap', async () => { + handler = (req, res) => { + const n = Number(new URL(req.url ?? '/', base).searchParams.get('n') ?? '0') + res.writeHead(302, { location: `/?n=${n + 1}` }) + res.end() + } + await expect(provider({ maxRedirects: 2 }).fetch({ url: `${base}/?n=0` })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_REDIRECT_BLOCKED' })) + }) + + it('treats a redirect without a Location header as a provider error', async () => { + handler = (_req, res) => { res.writeHead(302); res.end() } + await expect(provider().fetch({ url: base })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) + + it('follows a relative same-origin redirect', async () => { + handler = (req, res) => { + if (req.url === '/a') { res.writeHead(301, { location: 'b' }); res.end() } + else { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('landed') } + } + const result = await provider().fetch({ url: `${base}/a` }) + expect(result.body.content).toBe('landed') + }) +}) + +describe('LocalFetchProvider invalid URLs and abort', () => { + it('rejects a non-http scheme before any network access', async () => { + await expect(provider().fetch({ url: 'ftp://example.com' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_INVALID_URL' })) + }) + + it('rejects credentials in the URL', async () => { + await expect(provider().fetch({ url: 'http://user:pass@127.0.0.1/' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_BLOCKED_URL' })) + }) + + it('honors a pre-aborted signal', async () => { + const controller = new AbortController() + controller.abort() + await expect(provider().fetch({ url: base }, { signal: controller.signal })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_ABORTED' })) + }) + + it('aborts an in-flight fetch via the signal', async () => { + handler = (_req, _res) => { /* never responds */ } + const controller = new AbortController() + const promise = provider().fetch({ url: base }, { signal: controller.signal }) + controller.abort() + await expect(promise).rejects.toThrow(expect.objectContaining({ code: 'WEB_ABORTED' })) + }) + + it('times out a slow response with WEB_FETCH_TIMEOUT', async () => { + handler = (_req, _res) => { /* never responds */ } + await expect(provider({ timeoutMs: 50 }).fetch({ url: base })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_FETCH_TIMEOUT' })) + }) + + it('maps a connection failure to WEB_PROVIDER_ERROR', async () => { + // Port 1 on loopback is not listening: a real connection failure (not abort). + await expect(provider().fetch({ url: 'http://127.0.0.1:1/' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) + + it('caps the per-request timeout at maxTimeoutMs', async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('ok') } + const result = await provider({ maxTimeoutMs: 10_000 }).fetch({ url: base, timeoutMs: 999_999 }) + expect(result.statusCode).toBe(200) + }) +}) + +describe('web-fetch-local plugin registration', () => { + it('registers the provider into ctx.web (HMR-safe)', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { fetchProvider: LOCAL_FETCH_PROVIDER_ID }) + const fiber = await ctx.plugin(fetchPlugin, {}) + expect(ctx.web.fetchStatus()).toEqual({ available: true, providerId: LOCAL_FETCH_PROVIDER_ID }) + await fiber.dispose() + expect(ctx.web.fetchStatus()).toEqual({ available: false, reason: 'configured-missing' }) + }) + + it('has no default export (namespace plugin export shape)', () => { + expect('default' in fetchPlugin).toBe(false) + }) +}) diff --git a/packages/web/web-fetch-local/tsconfig.json b/packages/web/web-fetch-local/tsconfig.json new file mode 100644 index 0000000000..cb9eb44552 --- /dev/null +++ b/packages/web/web-fetch-local/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../../vendor/cosmokit" + }, + { + "path": "../../../vendor/cordis" + }, + { + "path": "../../../vendor/schemastery" + }, + { + "path": "../web" + } + ] +} diff --git a/packages/web/web-search-exa/README.md b/packages/web/web-search-exa/README.md new file mode 100644 index 0000000000..7f39356e81 --- /dev/null +++ b/packages/web/web-search-exa/README.md @@ -0,0 +1,23 @@ +# @deepseek-ai/dsh-web-search-exa + +An [Exa](https://exa.ai)-backed `WebSearchProvider` for the harness [web capability seam](../web/README.md) (`ctx.web`). It calls Exa's `POST /search` endpoint with highlight contents and maps the flat `results[]` into the seam's normalized `WebSearchResult`. + +This is an **implementation** package: it registers a provider into `ctx.web`, it does not own the `ctx.web` key and it does not register a model-facing tool (that is `@deepseek-ai/dsh-tool-web`). Like `@deepseek-ai/dsh-llm-deepseek`, it is a function/namespace plugin (`inject: ['web']`) that registers its backend, not a default-export service. + +## Config + +| Key | Default | Meaning | +|---|---|---| +| `apiKey` | `$EXA_API_KEY` | Exa API key. Empty/absent → provider `status()` reports `missing-credential` (the seam reports `configured-unavailable`/`none`). | +| `baseURL` | `https://api.exa.ai` | Endpoint base; `/search` is appended. | + +```yaml +- id: web-search-exa + name: '@deepseek-ai/dsh-web-search-exa' + config: + apiKey: !!js process.env.EXA_API_KEY +``` + +## Mapping + +Exa returns a flat `results[]` and no generated answer, so `content` is omitted. Each result maps to a `WebSearchSource`: `url` ← `url`, `title` ← `title`, `snippet` ← the first non-empty `highlights[]` entry (a result with no highlight has no portable snippet and is dropped), `publishedAt` ← `publishedDate`. The provider passes `maxResults` through as Exa's `numResults` for a cost/latency optimization; the final bound is enforced by the seam. Provider failures (HTTP errors, network failure, unparseable bodies) surface as `WebError` `WEB_PROVIDER_ERROR`; an aborted request surfaces as `WEB_ABORTED`. diff --git a/packages/web/web-search-exa/package.json b/packages/web/web-search-exa/package.json new file mode 100644 index 0000000000..fe79af8ba8 --- /dev/null +++ b/packages/web/web-search-exa/package.json @@ -0,0 +1,33 @@ +{ + "name": "@deepseek-ai/dsh-web-search-exa", + "description": "Exa-backed search provider for the DeepSeek Harness web capability seam (ctx.web)", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "exports": { + ".": { + "types": "./lib/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "@deepseek-ai/dsh-web": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "dependencies": { + "schemastery": "^3.18.0" + }, + "devDependencies": { + "@deepseek-ai/dsh-web": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/web/web-search-exa/src/index.ts b/packages/web/web-search-exa/src/index.ts new file mode 100644 index 0000000000..f266474708 --- /dev/null +++ b/packages/web/web-search-exa/src/index.ts @@ -0,0 +1,48 @@ +/** + * `@deepseek-ai/dsh-web-search-exa`: registers an Exa-backed `WebSearchProvider` + * with `ctx.web`. A function/namespace plugin (NOT a default-export service): + * a search provider does not own the `ctx.web` key — it registers INTO the + * seam's provider registry, exactly as `@deepseek-ai/dsh-llm-deepseek` + * registers an adapter into `ctx.llm`. The key is owned by `@deepseek-ai/dsh-web`. + * + * @module @deepseek-ai/dsh-web-search-exa + */ + +import type { Context } from 'cordis' +import z from 'schemastery' +import type {} from '@deepseek-ai/dsh-web' +import { ExaSearchProvider, EXA_DEFAULT_BASE_URL } from './provider.ts' + +export { + EXA_DEFAULT_BASE_URL, + EXA_PROVIDER_ID, + ExaSearchProvider, + mapExaResponse, + mapExaResult, +} from './provider.ts' +export type { ExaSearchProviderOptions } from './provider.ts' + +/** Cordis plugin name used by loader diagnostics. */ +export const name = 'web-search-exa' + +/** The web seam this provider registers into. */ +export const inject = ['web'] + +export interface Config { + /** Exa API key. Falls back to `$EXA_API_KEY`. Empty → provider unavailable. */ + apiKey?: string + /** Endpoint base; `/search` is appended. Defaults to the public API. */ + baseURL?: string +} + +export const Config: z = z.object({ + apiKey: z.string(), + baseURL: z.string(), +}) + +/** Register the Exa search provider with `ctx.web`. */ +export function apply(ctx: Context, config: Config): void { + const apiKey = config.apiKey ?? process.env.EXA_API_KEY ?? '' + const baseURL = config.baseURL ?? EXA_DEFAULT_BASE_URL + ctx.web.registerSearchProvider(new ExaSearchProvider({ apiKey, baseURL })) +} diff --git a/packages/web/web-search-exa/src/provider.ts b/packages/web/web-search-exa/src/provider.ts new file mode 100644 index 0000000000..d7c464059a --- /dev/null +++ b/packages/web/web-search-exa/src/provider.ts @@ -0,0 +1,130 @@ +/** + * `ExaSearchProvider`: a `WebSearchProvider` backed by the Exa search API + * (`POST /search` with highlight contents). Maps Exa's flat `results[]` into the + * seam's normalized `WebSearchResult`. Exa returns no provider-generated answer, + * so `content` is omitted; each result maps to a `WebSearchSource` with `url`, + * `title`, the first highlight as `snippet`, and `publishedDate` as + * `publishedAt`. + * + * Network requests use platform-native `fetch` (Node 24), mirroring + * `@deepseek-ai/dsh-llm-deepseek`'s adapter — not a cordis HTTP-client service. + * + * @module @deepseek-ai/dsh-web-search-exa/provider + */ + +import { WebError } from '@deepseek-ai/dsh-web' +import type { + WebProviderStatus, + WebSearchProvider, + WebSearchRequest, + WebSearchResult, + WebSearchSource, +} from '@deepseek-ai/dsh-web' +import type { ExaError, ExaResult, ExaSearchResponse } from './types.ts' + +/** Stable id this provider registers under. */ +export const EXA_PROVIDER_ID = 'exa' + +/** Default Exa search endpoint; `/search` is the operation. */ +export const EXA_DEFAULT_BASE_URL = 'https://api.exa.ai' + +/** Attribution header sent on every request. Bump with the package version. */ +const USER_AGENT = 'deepseek-harness/0.0.1' + +export interface ExaSearchProviderOptions { + /** Exa API key. Empty/absent → `status()` reports `missing-credential`. */ + apiKey: string + /** Endpoint base; `/search` is appended. */ + baseURL: string +} + +/** + * Map one Exa result to a normalized source, or `undefined` when it carries no + * portable snippet (an entry with no highlight is dropped — the seam has no + * other field to derive a snippet from, and inventing one would lie). + */ +export function mapExaResult(result: ExaResult): WebSearchSource | undefined { + const snippet = result.highlights?.find(highlight => highlight.trim().length > 0) + if (snippet === undefined) return undefined + return { + url: result.url, + ...result.title != null && result.title.length > 0 ? { title: result.title } : {}, + snippet, + ...result.publishedDate != null && result.publishedDate.length > 0 ? { publishedAt: result.publishedDate } : {}, + } +} + +/** Map an Exa response envelope to a normalized search result. */ +export function mapExaResponse(query: string, response: ExaSearchResponse): WebSearchResult { + const sources = (response.results ?? []) + .map(mapExaResult) + .filter((source): source is WebSearchSource => source !== undefined) + // Exa returns no generated answer, so `content` is omitted. The seam owns the + // final `maxResults` truncation, so this provider reports `truncated: false`. + return { providerId: EXA_PROVIDER_ID, query, sources, truncated: false } +} + +/** The Exa-backed search provider. */ +export class ExaSearchProvider implements WebSearchProvider { + readonly id = EXA_PROVIDER_ID + + constructor(private readonly options: ExaSearchProviderOptions) {} + + status(): WebProviderStatus { + if (this.options.apiKey.length === 0) return { available: false, reason: 'missing-credential' } + return { available: true } + } + + async search(request: WebSearchRequest, exec?: { readonly signal?: AbortSignal }): Promise { + let response: Response + try { + response = await fetch(`${this.options.baseURL}/search`, { + method: 'POST', + headers: { + 'authorization': `Bearer ${this.options.apiKey}`, + 'content-type': 'application/json', + 'accept': 'application/json', + 'user-agent': USER_AGENT, + }, + body: JSON.stringify({ + query: request.query, + contents: { highlights: true }, + ...request.maxResults !== undefined ? { numResults: request.maxResults } : {}, + }), + ...exec?.signal ? { signal: exec.signal } : {}, + }) + } catch (error: unknown) { + if (isAbortError(error)) throw new WebError('Exa search aborted', 'WEB_ABORTED', { cause: error }) + throw new WebError(`Exa search request failed: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) + } + + if (!response.ok) { + const status = response.status + let message = `Exa API error (HTTP ${status})` + try { + const parsed = await response.json() as ExaError + const detail = parsed.error ?? parsed.message + if (detail !== undefined && detail.length > 0) message = detail + } catch { + // The HTTP status is already captured in `message` above; a malformed or + // non-JSON error body (normal for gateway 5xx/429s) can only cost a + // richer provider message, never the real error. `response.json()` is + // the sole statement and nothing else of consequence reaches here. + } + throw new WebError(message, 'WEB_PROVIDER_ERROR') + } + + let payload: ExaSearchResponse + try { + payload = await response.json() as ExaSearchResponse + } catch (error: unknown) { + throw new WebError(`Exa returned an unparseable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) + } + return mapExaResponse(request.query, payload) + } +} + +/** True for a fetch/`AbortSignal` abort, surfaced as `WEB_ABORTED`. */ +function isAbortError(error: unknown): boolean { + return error instanceof DOMException && error.name === 'AbortError' +} diff --git a/packages/web/web-search-exa/src/types.ts b/packages/web/web-search-exa/src/types.ts new file mode 100644 index 0000000000..a0bda5f768 --- /dev/null +++ b/packages/web/web-search-exa/src/types.ts @@ -0,0 +1,36 @@ +/** + * Wire types for the Exa search API (`POST https://api.exa.ai/search`). Types + * only — no runtime code. Exa returns a flat `results[]`; each entry carries a + * URL, optional title, optional `publishedDate`, and (when highlights are + * requested) a `highlights[]` array of salient sentences. + * + * @module @deepseek-ai/dsh-web-search-exa/types + */ + +/** Request body sent to Exa's search endpoint. */ +export interface ExaSearchRequest { + query: string + /** Exa's result-count control; the seam still enforces the bound on return. */ + numResults?: number + /** Ask Exa to return highlight sentences per result. */ + contents: { highlights: true } +} + +/** One entry of Exa's flat `results[]`. */ +export interface ExaResult { + url: string + title?: string | null + publishedDate?: string | null + highlights?: string[] +} + +/** Exa's search response envelope. */ +export interface ExaSearchResponse { + results?: ExaResult[] +} + +/** Exa's error response envelope (best-effort; fields vary by failure). */ +export interface ExaError { + error?: string + message?: string +} diff --git a/packages/web/web-search-exa/tests/exa.e2e.ts b/packages/web/web-search-exa/tests/exa.e2e.ts new file mode 100644 index 0000000000..78f11940e5 --- /dev/null +++ b/packages/web/web-search-exa/tests/exa.e2e.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest' +import { ExaSearchProvider, EXA_DEFAULT_BASE_URL } from '@deepseek-ai/dsh-web-search-exa' + +/** + * Real-API smoke for the Exa search provider. Self-skips without `$EXA_API_KEY` + * (CI has no secrets), per the with-key e2e policy in AGENTS.md § Secrets. + */ +const apiKey = process.env.EXA_API_KEY +const maybe = apiKey !== undefined && apiKey.length > 0 ? describe : describe.skip + +maybe('ExaSearchProvider real API', () => { + it('returns sources for a live query', async () => { + const provider = new ExaSearchProvider({ apiKey: apiKey!, baseURL: process.env.EXA_BASE_URL ?? EXA_DEFAULT_BASE_URL }) + const result = await provider.search({ query: 'DeepSeek coding agent', maxResults: 5 }) + expect(result.providerId).toBe('exa') + expect(result.sources.length).toBeGreaterThan(0) + for (const source of result.sources) expect(source.url).toMatch(/^https?:\/\//) + }, 30_000) +}) diff --git a/packages/web/web-search-exa/tests/exa.spec.ts b/packages/web/web-search-exa/tests/exa.spec.ts new file mode 100644 index 0000000000..3fdd878180 --- /dev/null +++ b/packages/web/web-search-exa/tests/exa.spec.ts @@ -0,0 +1,193 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Context } from 'cordis' +import WebService from '@deepseek-ai/dsh-web' +import { ExaSearchProvider, mapExaResponse, mapExaResult, EXA_PROVIDER_ID } from '@deepseek-ai/dsh-web-search-exa' +import * as exaPlugin from '@deepseek-ai/dsh-web-search-exa' + +const options = { apiKey: 'exa-key', baseURL: 'https://api.exa.test' } + +function jsonResponse(body: unknown, init: ResponseInit = {}): Response { + return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' }, ...init }) +} + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('Exa result mapping', () => { + it('maps a full result entry', () => { + expect(mapExaResult({ + url: 'https://a.test', + title: 'A', + publishedDate: '2026-01-01', + highlights: ['salient sentence', 'second'], + })).toEqual({ url: 'https://a.test', title: 'A', snippet: 'salient sentence', publishedAt: '2026-01-01' }) + }) + + it('drops a result with no usable highlight', () => { + expect(mapExaResult({ url: 'https://a.test', highlights: [] })).toBeUndefined() + expect(mapExaResult({ url: 'https://a.test' })).toBeUndefined() + expect(mapExaResult({ url: 'https://a.test', highlights: [' '] })).toBeUndefined() + }) + + it('omits null/empty optional fields rather than emitting them', () => { + expect(mapExaResult({ url: 'https://a.test', title: null, publishedDate: null, highlights: ['hi'] })) + .toEqual({ url: 'https://a.test', snippet: 'hi' }) + expect(mapExaResult({ url: 'https://a.test', title: '', publishedDate: '', highlights: ['hi'] })) + .toEqual({ url: 'https://a.test', snippet: 'hi' }) + }) + + it('maps a response to a result with no content and filtered sources', () => { + const result = mapExaResponse('q', { + results: [ + { url: 'https://a.test', highlights: ['one'] }, + { url: 'https://b.test' }, + { url: 'https://c.test', title: 'C', highlights: ['three'] }, + ], + }) + expect(result).toEqual({ + providerId: EXA_PROVIDER_ID, + query: 'q', + sources: [ + { url: 'https://a.test', snippet: 'one' }, + { url: 'https://c.test', title: 'C', snippet: 'three' }, + ], + truncated: false, + }) + expect(result.content).toBeUndefined() + }) + + it('tolerates a missing results array', () => { + expect(mapExaResponse('q', {}).sources).toEqual([]) + }) +}) + +describe('ExaSearchProvider status', () => { + it('is unavailable without a key', () => { + expect(new ExaSearchProvider({ apiKey: '', baseURL: options.baseURL }).status()) + .toEqual({ available: false, reason: 'missing-credential' }) + }) + + it('is available with a key', () => { + expect(new ExaSearchProvider(options).status()).toEqual({ available: true }) + }) +}) + +describe('ExaSearchProvider request mapping', () => { + it('sends query, highlights, numResults and bearer auth', async () => { + const fetchMock = vi.fn(async () => jsonResponse({ results: [{ url: 'https://a.test', highlights: ['hi'] }] })) + vi.stubGlobal('fetch', fetchMock) + + const provider = new ExaSearchProvider(options) + await provider.search({ query: 'hello', maxResults: 5 }) + + expect(fetchMock).toHaveBeenCalledOnce() + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe('https://api.exa.test/search') + expect((init.headers as Record)['authorization']).toBe('Bearer exa-key') + expect(JSON.parse(init.body as string)).toEqual({ query: 'hello', contents: { highlights: true }, numResults: 5 }) + }) + + it('omits numResults when maxResults is absent', async () => { + const fetchMock = vi.fn(async () => jsonResponse({ results: [] })) + vi.stubGlobal('fetch', fetchMock) + await new ExaSearchProvider(options).search({ query: 'q' }) + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(JSON.parse(init.body as string)).not.toHaveProperty('numResults') + }) + + it('forwards the abort signal', async () => { + const fetchMock = vi.fn(async () => jsonResponse({ results: [] })) + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + await new ExaSearchProvider(options).search({ query: 'q' }, { signal: controller.signal }) + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(init.signal).toBe(controller.signal) + }) +}) + +describe('ExaSearchProvider error handling', () => { + it('maps an HTTP error to WEB_PROVIDER_ERROR with the provider message', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({ error: 'bad key' }, { status: 401 }))) + await expect(new ExaSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR', message: 'bad key' })) + }) + + it('keeps a status-line message when the error body is not JSON', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('gateway down', { status: 502 }))) + await expect(new ExaSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR', message: 'Exa API error (HTTP 502)' })) + }) + + it('keeps the status-line message when the JSON error body carries no detail', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({}, { status: 500 }))) + await expect(new ExaSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ message: 'Exa API error (HTTP 500)' })) + }) + + it('maps a network failure to WEB_PROVIDER_ERROR', async () => { + vi.stubGlobal('fetch', vi.fn(() => Promise.reject(new TypeError('connection refused')))) + await expect(new ExaSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) + + it('maps an abort to WEB_ABORTED', async () => { + vi.stubGlobal('fetch', vi.fn(() => Promise.reject(new DOMException('aborted', 'AbortError')))) + await expect(new ExaSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_ABORTED' })) + }) + + it('maps an unparseable success body to WEB_PROVIDER_ERROR', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('not json', { status: 200 }))) + await expect(new ExaSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) +}) + +describe('web-search-exa plugin registration', () => { + it('registers the provider into ctx.web (HMR-safe)', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: EXA_PROVIDER_ID }) + const fiber = await ctx.plugin(exaPlugin, { apiKey: 'exa-key' }) + expect(ctx.web.searchStatus()).toEqual({ available: true, providerId: EXA_PROVIDER_ID }) + await fiber.dispose() + expect(ctx.web.searchStatus()).toEqual({ available: false, reason: 'configured-missing' }) + }) + + it('has no default export (namespace plugin export shape)', () => { + expect('default' in exaPlugin).toBe(false) + }) + + it('falls back to $EXA_API_KEY and the default base URL when config omits them', async () => { + const prev = process.env.EXA_API_KEY + process.env.EXA_API_KEY = 'env-key' + try { + const fetchMock = vi.fn(async () => jsonResponse({ results: [] })) + vi.stubGlobal('fetch', fetchMock) + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: EXA_PROVIDER_ID }) + const fiber = await ctx.plugin(exaPlugin, {}) + expect(ctx.web.searchStatus()).toEqual({ available: true, providerId: EXA_PROVIDER_ID }) + await ctx.web.search({ query: 'q' }) + const [url] = fetchMock.mock.calls[0] as unknown as [string] + expect(url).toBe('https://api.exa.ai/search') + await fiber.dispose() + } finally { + if (prev === undefined) delete process.env.EXA_API_KEY + else process.env.EXA_API_KEY = prev + } + }) + + it('is unavailable when neither config nor env supplies a key', async () => { + const prev = process.env.EXA_API_KEY + delete process.env.EXA_API_KEY + try { + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: EXA_PROVIDER_ID }) + await ctx.plugin(exaPlugin, {}) + expect(ctx.web.searchStatus()).toEqual({ available: false, reason: 'configured-unavailable' }) + } finally { + if (prev !== undefined) process.env.EXA_API_KEY = prev + } + }) +}) diff --git a/packages/web/web-search-exa/tsconfig.json b/packages/web/web-search-exa/tsconfig.json new file mode 100644 index 0000000000..cb9eb44552 --- /dev/null +++ b/packages/web/web-search-exa/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../../vendor/cosmokit" + }, + { + "path": "../../../vendor/cordis" + }, + { + "path": "../../../vendor/schemastery" + }, + { + "path": "../web" + } + ] +} diff --git a/packages/web/web-search-perplexity/README.md b/packages/web/web-search-perplexity/README.md new file mode 100644 index 0000000000..3850e5e9c1 --- /dev/null +++ b/packages/web/web-search-perplexity/README.md @@ -0,0 +1,24 @@ +# @deepseek-ai/dsh-web-search-perplexity + +A [Perplexity](https://perplexity.ai)-backed `WebSearchProvider` for the harness [web capability seam](../web/README.md) (`ctx.web`). It calls Perplexity's OpenAI-compatible `POST /chat/completions` endpoint and maps the generated answer plus citations into the seam's normalized `WebSearchResult`. + +This is an **implementation** package: it registers a provider into `ctx.web`, it does not own the key and it does not register a model-facing tool. Like `@deepseek-ai/dsh-llm-deepseek`, it is a function/namespace plugin (`inject: ['web']`). The OpenAI-compatible wire shape is a provider-private detail — it does **not** make this provider depend on `ctx.llm`. + +## Config + +| Key | Default | Meaning | +|---|---|---| +| `apiKey` | `$PERPLEXITY_API_KEY` | Perplexity API key. Empty/absent → provider `status()` reports `missing-credential`. | +| `baseURL` | `https://api.perplexity.ai` | Endpoint base; `/chat/completions` is appended. | +| `model` | `sonar` | Search model name. | + +```yaml +- id: web-search-perplexity + name: '@deepseek-ai/dsh-web-search-perplexity' + config: + apiKey: !!js process.env.PERPLEXITY_API_KEY +``` + +## Mapping + +`content` ← `choices[0].message.content` (the generated answer). `sources[]` prefers the structured `search_results[]` (`url`, `title`, `snippet`, `publishedAt` ← `date`), falling back to the URL-only `citations[]` array only when `search_results` is absent — those sources carry just a `url`, which is why `title`/`snippet`/`publishedAt` are optional on the seam. Provider failures surface as `WebError` `WEB_PROVIDER_ERROR`; an aborted request surfaces as `WEB_ABORTED`. Perplexity has no result-count control, so `maxResults` is enforced by the seam (truncating `sources[]` and setting `truncated`). diff --git a/packages/web/web-search-perplexity/package.json b/packages/web/web-search-perplexity/package.json new file mode 100644 index 0000000000..1d6229eae4 --- /dev/null +++ b/packages/web/web-search-perplexity/package.json @@ -0,0 +1,33 @@ +{ + "name": "@deepseek-ai/dsh-web-search-perplexity", + "description": "Perplexity-backed search provider for the DeepSeek Harness web capability seam (ctx.web)", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "exports": { + ".": { + "types": "./lib/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "@deepseek-ai/dsh-web": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "dependencies": { + "schemastery": "^3.18.0" + }, + "devDependencies": { + "@deepseek-ai/dsh-web": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/web/web-search-perplexity/src/index.ts b/packages/web/web-search-perplexity/src/index.ts new file mode 100644 index 0000000000..0fd46ffb71 --- /dev/null +++ b/packages/web/web-search-perplexity/src/index.ts @@ -0,0 +1,52 @@ +/** + * `@deepseek-ai/dsh-web-search-perplexity`: registers a Perplexity-backed + * `WebSearchProvider` with `ctx.web`. A function/namespace plugin (NOT a + * default-export service): it registers INTO the seam's provider registry, like + * `@deepseek-ai/dsh-llm-deepseek` registers an adapter into `ctx.llm`. + * + * @module @deepseek-ai/dsh-web-search-perplexity + */ + +import type { Context } from 'cordis' +import z from 'schemastery' +import type {} from '@deepseek-ai/dsh-web' +import { PerplexitySearchProvider, PERPLEXITY_DEFAULT_BASE_URL, PERPLEXITY_DEFAULT_MODEL } from './provider.ts' + +export { + PERPLEXITY_DEFAULT_BASE_URL, + PERPLEXITY_DEFAULT_MODEL, + PERPLEXITY_PROVIDER_ID, + PerplexitySearchProvider, + mapPerplexityResponse, + mapPerplexityResult, +} from './provider.ts' +export type { PerplexitySearchProviderOptions } from './provider.ts' + +/** Cordis plugin name used by loader diagnostics. */ +export const name = 'web-search-perplexity' + +/** The web seam this provider registers into. */ +export const inject = ['web'] + +export interface Config { + /** Perplexity API key. Falls back to `$PERPLEXITY_API_KEY`. Empty → unavailable. */ + apiKey?: string + /** Endpoint base; `/chat/completions` is appended. Defaults to the public API. */ + baseURL?: string + /** Search model name. Defaults to `sonar`. */ + model?: string +} + +export const Config: z = z.object({ + apiKey: z.string(), + baseURL: z.string(), + model: z.string(), +}) + +/** Register the Perplexity search provider with `ctx.web`. */ +export function apply(ctx: Context, config: Config): void { + const apiKey = config.apiKey ?? process.env.PERPLEXITY_API_KEY ?? '' + const baseURL = config.baseURL ?? PERPLEXITY_DEFAULT_BASE_URL + const model = config.model ?? PERPLEXITY_DEFAULT_MODEL + ctx.web.registerSearchProvider(new PerplexitySearchProvider({ apiKey, baseURL, model })) +} diff --git a/packages/web/web-search-perplexity/src/provider.ts b/packages/web/web-search-perplexity/src/provider.ts new file mode 100644 index 0000000000..2b596414dd --- /dev/null +++ b/packages/web/web-search-perplexity/src/provider.ts @@ -0,0 +1,138 @@ +/** + * `PerplexitySearchProvider`: a `WebSearchProvider` backed by the Perplexity + * search API (an OpenAI-compatible `POST /chat/completions`). Maps the generated + * answer (`choices[0].message.content`) into `content`, and prefers the + * structured `search_results[]` for `sources[]`, falling back to the URL-only + * `citations[]` when `search_results` is absent. + * + * Network requests use platform-native `fetch` (Node 24), mirroring + * `@deepseek-ai/dsh-llm-deepseek`'s adapter. The OpenAI-compatible request shape + * is a provider-private detail and does NOT make this provider depend on + * `ctx.llm`. + * + * @module @deepseek-ai/dsh-web-search-perplexity/provider + */ + +import { WebError } from '@deepseek-ai/dsh-web' +import type { + WebProviderStatus, + WebSearchProvider, + WebSearchRequest, + WebSearchResult, + WebSearchSource, +} from '@deepseek-ai/dsh-web' +import type { PerplexityError, PerplexityResponse, PerplexitySearchResult } from './types.ts' + +/** Stable id this provider registers under. */ +export const PERPLEXITY_PROVIDER_ID = 'perplexity' + +/** Default Perplexity endpoint; `/chat/completions` is the operation. */ +export const PERPLEXITY_DEFAULT_BASE_URL = 'https://api.perplexity.ai' + +/** Default search model. */ +export const PERPLEXITY_DEFAULT_MODEL = 'sonar' + +/** Attribution header sent on every request. Bump with the package version. */ +const USER_AGENT = 'deepseek-harness/0.0.1' + +export interface PerplexitySearchProviderOptions { + /** Perplexity API key. Empty/absent → `status()` reports `missing-credential`. */ + apiKey: string + /** Endpoint base; `/chat/completions` is appended. */ + baseURL: string + /** Search model name. */ + model: string +} + +/** Map one structured Perplexity search result to a normalized source. */ +export function mapPerplexityResult(result: PerplexitySearchResult): WebSearchSource { + return { + url: result.url, + ...result.title != null && result.title.length > 0 ? { title: result.title } : {}, + ...result.snippet != null && result.snippet.length > 0 ? { snippet: result.snippet } : {}, + ...result.date != null && result.date.length > 0 ? { publishedAt: result.date } : {}, + } +} + +/** + * Map a Perplexity response envelope to a normalized search result. Prefers + * structured `search_results[]`; falls back to URL-only `citations[]` (those + * sources carry just a `url`) only when `search_results` is absent. + */ +export function mapPerplexityResponse(query: string, response: PerplexityResponse): WebSearchResult { + const content = response.choices?.[0]?.message?.content + const sources: WebSearchSource[] = response.search_results !== undefined + ? response.search_results.map(mapPerplexityResult) + : (response.citations ?? []).map(url => ({ url })) + return { + providerId: PERPLEXITY_PROVIDER_ID, + query, + ...content != null && content.length > 0 ? { content } : {}, + sources, + truncated: false, + } +} + +/** The Perplexity-backed search provider. */ +export class PerplexitySearchProvider implements WebSearchProvider { + readonly id = PERPLEXITY_PROVIDER_ID + + constructor(private readonly options: PerplexitySearchProviderOptions) {} + + status(): WebProviderStatus { + if (this.options.apiKey.length === 0) return { available: false, reason: 'missing-credential' } + return { available: true } + } + + async search(request: WebSearchRequest, exec?: { readonly signal?: AbortSignal }): Promise { + let response: Response + try { + response = await fetch(`${this.options.baseURL}/chat/completions`, { + method: 'POST', + headers: { + 'authorization': `Bearer ${this.options.apiKey}`, + 'content-type': 'application/json', + 'accept': 'application/json', + 'user-agent': USER_AGENT, + }, + body: JSON.stringify({ + model: this.options.model, + messages: [{ role: 'user', content: request.query }], + }), + ...exec?.signal ? { signal: exec.signal } : {}, + }) + } catch (error: unknown) { + if (isAbortError(error)) throw new WebError('Perplexity search aborted', 'WEB_ABORTED', { cause: error }) + throw new WebError(`Perplexity search request failed: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) + } + + if (!response.ok) { + const status = response.status + let message = `Perplexity API error (HTTP ${status})` + try { + const parsed = await response.json() as PerplexityError + const detail = typeof parsed.error === 'string' ? parsed.error : parsed.error?.message ?? parsed.message + if (detail !== undefined && detail.length > 0) message = detail + } catch { + // The HTTP status is already captured in `message` above; a malformed or + // non-JSON error body (normal for gateway 5xx/429s) can only cost a + // richer provider message, never the real error. `response.json()` is + // the sole statement and nothing else of consequence reaches here. + } + throw new WebError(message, 'WEB_PROVIDER_ERROR') + } + + let payload: PerplexityResponse + try { + payload = await response.json() as PerplexityResponse + } catch (error: unknown) { + throw new WebError(`Perplexity returned an unparseable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) + } + return mapPerplexityResponse(request.query, payload) + } +} + +/** True for a fetch/`AbortSignal` abort, surfaced as `WEB_ABORTED`. */ +function isAbortError(error: unknown): boolean { + return error instanceof DOMException && error.name === 'AbortError' +} diff --git a/packages/web/web-search-perplexity/src/types.ts b/packages/web/web-search-perplexity/src/types.ts new file mode 100644 index 0000000000..7b1f2e32b0 --- /dev/null +++ b/packages/web/web-search-perplexity/src/types.ts @@ -0,0 +1,41 @@ +/** + * Wire types for the Perplexity search API + * (`POST https://api.perplexity.ai/chat/completions`, an OpenAI-compatible chat + * shape). Types only — no runtime code. Perplexity returns a generated answer in + * `choices[0].message.content` plus citation surfaces: a structured + * `search_results[]` (preferred) and a URL-only `citations[]` fallback. + * + * The OpenAI-compatible wire shape is a provider-private detail; it does not make + * this provider depend on `ctx.llm`. + * + * @module @deepseek-ai/dsh-web-search-perplexity/types + */ + +/** Request body sent to Perplexity's chat-completions endpoint. */ +export interface PerplexityRequest { + model: string + messages: { role: 'user'; content: string }[] +} + +/** One structured search result (the preferred citation surface). */ +export interface PerplexitySearchResult { + url: string + title?: string | null + snippet?: string | null + date?: string | null +} + +/** Perplexity's response envelope. */ +export interface PerplexityResponse { + choices?: { message?: { content?: string | null } }[] + /** Structured citation surface (preferred). */ + search_results?: PerplexitySearchResult[] + /** URL-only citation fallback. */ + citations?: string[] +} + +/** Perplexity's error response envelope (best-effort; fields vary). */ +export interface PerplexityError { + error?: { message?: string } | string + message?: string +} diff --git a/packages/web/web-search-perplexity/tests/perplexity.e2e.ts b/packages/web/web-search-perplexity/tests/perplexity.e2e.ts new file mode 100644 index 0000000000..a546acab70 --- /dev/null +++ b/packages/web/web-search-perplexity/tests/perplexity.e2e.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest' +import { PerplexitySearchProvider, PERPLEXITY_DEFAULT_BASE_URL, PERPLEXITY_DEFAULT_MODEL } from '@deepseek-ai/dsh-web-search-perplexity' + +/** + * Real-API smoke for the Perplexity search provider. Self-skips without + * `$PERPLEXITY_API_KEY`, per the with-key e2e policy in AGENTS.md § Secrets. + */ +const apiKey = process.env.PERPLEXITY_API_KEY +const maybe = apiKey !== undefined && apiKey.length > 0 ? describe : describe.skip + +maybe('PerplexitySearchProvider real API', () => { + it('returns a generated answer and sources for a live query', async () => { + const provider = new PerplexitySearchProvider({ + apiKey: apiKey!, + baseURL: process.env.PERPLEXITY_BASE_URL ?? PERPLEXITY_DEFAULT_BASE_URL, + model: process.env.PERPLEXITY_MODEL ?? PERPLEXITY_DEFAULT_MODEL, + }) + const result = await provider.search({ query: 'What is the DeepSeek coding agent?', maxResults: 5 }) + expect(result.providerId).toBe('perplexity') + expect(result.content ?? '').not.toBe('') + for (const source of result.sources) expect(source.url).toMatch(/^https?:\/\//) + }, 30_000) +}) diff --git a/packages/web/web-search-perplexity/tests/perplexity.spec.ts b/packages/web/web-search-perplexity/tests/perplexity.spec.ts new file mode 100644 index 0000000000..16557af888 --- /dev/null +++ b/packages/web/web-search-perplexity/tests/perplexity.spec.ts @@ -0,0 +1,192 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Context } from 'cordis' +import WebService from '@deepseek-ai/dsh-web' +import { + PerplexitySearchProvider, + mapPerplexityResponse, + PERPLEXITY_PROVIDER_ID, +} from '@deepseek-ai/dsh-web-search-perplexity' +import * as perplexityPlugin from '@deepseek-ai/dsh-web-search-perplexity' + +const options = { apiKey: 'pplx-key', baseURL: 'https://api.perplexity.test', model: 'sonar' } + +function jsonResponse(body: unknown, init: ResponseInit = {}): Response { + return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' }, ...init }) +} + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('Perplexity response mapping', () => { + it('maps the answer and prefers structured search_results', () => { + const result = mapPerplexityResponse('q', { + choices: [{ message: { content: 'the answer' } }], + search_results: [ + { url: 'https://a.test', title: 'A', snippet: 'snip', date: '2026-02-02' }, + { url: 'https://b.test' }, + ], + citations: ['https://ignored.test'], + }) + expect(result).toEqual({ + providerId: PERPLEXITY_PROVIDER_ID, + query: 'q', + content: 'the answer', + sources: [ + { url: 'https://a.test', title: 'A', snippet: 'snip', publishedAt: '2026-02-02' }, + { url: 'https://b.test' }, + ], + truncated: false, + }) + }) + + it('falls back to URL-only citations when search_results is absent', () => { + const result = mapPerplexityResponse('q', { + choices: [{ message: { content: 'answer' } }], + citations: ['https://a.test', 'https://b.test'], + }) + expect(result.sources).toEqual([{ url: 'https://a.test' }, { url: 'https://b.test' }]) + }) + + it('omits content when the answer is empty or missing', () => { + expect(mapPerplexityResponse('q', { citations: [] }).content).toBeUndefined() + expect(mapPerplexityResponse('q', { choices: [{ message: { content: '' } }] }).content).toBeUndefined() + expect(mapPerplexityResponse('q', { choices: [{ message: { content: null } }] }).content).toBeUndefined() + }) + + it('omits null/empty optional source fields', () => { + const result = mapPerplexityResponse('q', { + search_results: [{ url: 'https://a.test', title: null, snippet: '', date: null }], + }) + expect(result.sources).toEqual([{ url: 'https://a.test' }]) + }) + + it('yields no sources when neither search_results nor citations are present', () => { + expect(mapPerplexityResponse('q', { choices: [{ message: { content: 'a' } }] }).sources).toEqual([]) + }) +}) + +describe('PerplexitySearchProvider status', () => { + it('is unavailable without a key', () => { + expect(new PerplexitySearchProvider({ ...options, apiKey: '' }).status()) + .toEqual({ available: false, reason: 'missing-credential' }) + }) + + it('is available with a key', () => { + expect(new PerplexitySearchProvider(options).status()).toEqual({ available: true }) + }) +}) + +describe('PerplexitySearchProvider request mapping', () => { + it('sends a chat-completions request with the query as a user message', async () => { + const fetchMock = vi.fn(async () => jsonResponse({ choices: [{ message: { content: 'a' } }], citations: [] })) + vi.stubGlobal('fetch', fetchMock) + await new PerplexitySearchProvider(options).search({ query: 'hello' }) + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe('https://api.perplexity.test/chat/completions') + expect((init.headers as Record)['authorization']).toBe('Bearer pplx-key') + expect(JSON.parse(init.body as string)).toEqual({ model: 'sonar', messages: [{ role: 'user', content: 'hello' }] }) + }) + + it('forwards the abort signal', async () => { + const fetchMock = vi.fn(async () => jsonResponse({ citations: [] })) + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + await new PerplexitySearchProvider(options).search({ query: 'q' }, { signal: controller.signal }) + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(init.signal).toBe(controller.signal) + }) +}) + +describe('PerplexitySearchProvider error handling', () => { + it('maps an HTTP error to WEB_PROVIDER_ERROR with the provider message', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({ error: { message: 'rate limited' } }, { status: 429 }))) + await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR', message: 'rate limited' })) + }) + + it('handles a string-form error body', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({ error: 'bad request' }, { status: 400 }))) + await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ message: 'bad request' })) + }) + + it('keeps a status-line message when the error body is not JSON', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('upstream error', { status: 503 }))) + await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ message: 'Perplexity API error (HTTP 503)' })) + }) + + it('keeps the status-line message when the JSON error body carries no detail', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({}, { status: 500 }))) + await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ message: 'Perplexity API error (HTTP 500)' })) + }) + + it('maps an abort to WEB_ABORTED', async () => { + vi.stubGlobal('fetch', vi.fn(() => Promise.reject(new DOMException('aborted', 'AbortError')))) + await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_ABORTED' })) + }) + + it('maps an unparseable success body to WEB_PROVIDER_ERROR', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('not json', { status: 200 }))) + await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) + + it('maps a network failure to WEB_PROVIDER_ERROR', async () => { + vi.stubGlobal('fetch', vi.fn(() => Promise.reject(new TypeError('connection refused')))) + await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) +}) + +describe('web-search-perplexity plugin registration', () => { + it('registers the provider into ctx.web (HMR-safe)', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: PERPLEXITY_PROVIDER_ID }) + const fiber = await ctx.plugin(perplexityPlugin, { apiKey: 'pplx-key' }) + expect(ctx.web.searchStatus()).toEqual({ available: true, providerId: PERPLEXITY_PROVIDER_ID }) + await fiber.dispose() + expect(ctx.web.searchStatus()).toEqual({ available: false, reason: 'configured-missing' }) + }) + + it('has no default export (namespace plugin export shape)', () => { + expect('default' in perplexityPlugin).toBe(false) + }) + + it('falls back to env key and defaults for base URL and model when config omits them', async () => { + const prev = process.env.PERPLEXITY_API_KEY + process.env.PERPLEXITY_API_KEY = 'env-key' + try { + const fetchMock = vi.fn(async () => jsonResponse({ choices: [{ message: { content: 'a' } }], citations: [] })) + vi.stubGlobal('fetch', fetchMock) + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: PERPLEXITY_PROVIDER_ID }) + const fiber = await ctx.plugin(perplexityPlugin, {}) + expect(ctx.web.searchStatus()).toEqual({ available: true, providerId: PERPLEXITY_PROVIDER_ID }) + await ctx.web.search({ query: 'q' }) + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe('https://api.perplexity.ai/chat/completions') + expect(JSON.parse(init.body as string)).toMatchObject({ model: 'sonar' }) + await fiber.dispose() + } finally { + if (prev === undefined) delete process.env.PERPLEXITY_API_KEY + else process.env.PERPLEXITY_API_KEY = prev + } + }) + + it('is unavailable when neither config nor env supplies a key', async () => { + const prev = process.env.PERPLEXITY_API_KEY + delete process.env.PERPLEXITY_API_KEY + try { + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: PERPLEXITY_PROVIDER_ID }) + await ctx.plugin(perplexityPlugin, {}) + expect(ctx.web.searchStatus()).toEqual({ available: false, reason: 'configured-unavailable' }) + } finally { + if (prev !== undefined) process.env.PERPLEXITY_API_KEY = prev + } + }) +}) diff --git a/packages/web/web-search-perplexity/tsconfig.json b/packages/web/web-search-perplexity/tsconfig.json new file mode 100644 index 0000000000..cb9eb44552 --- /dev/null +++ b/packages/web/web-search-perplexity/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../../vendor/cosmokit" + }, + { + "path": "../../../vendor/cordis" + }, + { + "path": "../../../vendor/schemastery" + }, + { + "path": "../web" + } + ] +} diff --git a/packages/web/web/README.md b/packages/web/web/README.md new file mode 100644 index 0000000000..9b9e2ca333 --- /dev/null +++ b/packages/web/web/README.md @@ -0,0 +1,45 @@ +# @deepseek-ai/dsh-web + +The **web access seam**: an abstract `WebService` (`ctx.web`) defining WHAT web access the harness has — search the web, fetch a URL — over multiple providers, without binding the model contract to one vendor's API shape. + +This package is the interface third of the web capability. Unlike bash/fs it spans two capabilities (search and fetch) on one seam, with potentially multiple providers each: + +| Package | Role | +|---|---| +| `@deepseek-ai/dsh-web` (this) | the interface: the service, provider registries, selection policy, request/result vocabulary, the `WebError` taxonomy | +| `@deepseek-ai/dsh-web-search-exa` | a search implementation: Exa | +| `@deepseek-ai/dsh-web-search-perplexity` | a search implementation: Perplexity | +| `@deepseek-ai/dsh-web-fetch-local` | a fetch implementation: anonymous public HTTP(S) | +| `@deepseek-ai/dsh-tool-web` | the model-facing `web_search` / `web_fetch` tool schemas over `ctx.web` | + +Search and fetch share no request schema and no business logic, but they are deliberately one seam: `ctx.web` is a single web-access middle layer with one provider-selection policy owner, one abort/error vocabulary, and one product-facing "how this harness reaches the web" config surface. The cost is the parallel `Search`/`Fetch` method pairs; that parallelism is intentional, not a missed extraction. + +## Service API (`ctx.web`) + +| Member | Semantics | +|---|---| +| `registerSearchProvider(provider)` / `registerFetchProvider(provider)` | Register a backend. Throws `WebError` `WEB_DUPLICATE_PROVIDER` on a duplicate id within that capability kind. Returns a disposer; emits `web/providers-change` on register and on dispose. Disposed with the calling fiber. | +| `searchStatus()` / `fetchStatus()` | Derived (never stored) `WebCapabilityStatus`: whether the capability has a selected usable provider, or the broad category it fails in. Diagnostics + execution-resolution input. | +| `search(request, exec?)` | Resolve the search provider and run one search. Enforces `request.maxResults` on the result (truncates `sources[]`, sets `truncated`). Throws `WebError` when the capability cannot run. | +| `fetch(request, exec?)` | Resolve the fetch provider and retrieve one URL. A non-2xx response is a result, not a throw. Throws `WebError` for failures to safely retrieve or represent the resource. | + +Providers register **capabilities**, not tools. `dsh-tool-web` is the only owner of model-facing names, descriptions, prompt guidance, JSON schemas, and presentation. + +## Selection + +Selection never depends on registration, config, or HMR order. A capability has an explicit provider id (config `searchProvider`/`fetchProvider`, or env `$DSH_WEB_SEARCH_PROVIDER`/`$DSH_WEB_FETCH_PROVIDER` feeding the same fields), or auto-selects when exactly one usable provider is registered: + +| Situation | `WebCapabilityStatus` | Execution | +|---|---|---| +| configured id registered and `status().available` | `available` for it | runs | +| configured id not registered | `configured-missing` | `WEB_PROVIDER_CONFIGURED_MISSING` | +| configured id registered but unavailable | `configured-unavailable` | `WEB_PROVIDER_CONFIGURED_UNAVAILABLE` | +| no id, exactly one registered usable provider | `available` for it | runs | +| no id, no usable provider | `none` | `WEB_PROVIDER_UNAVAILABLE` | +| no id, multiple usable providers | `ambiguous` | `WEB_PROVIDER_AMBIGUOUS` | + +`WebCapabilityStatus` carries only `available` + a `reason` discriminant (plus the winning `providerId` on the available branch). The branchable per-reason detail lives in the thrown `WebError`, which is the surface callers route on — so the same fact never gets two homes that can disagree. A provider's own `status()` is a cheap local check (credential presence, parseable config) and **must not make network calls**; `dsh-tool-web` reads only the aggregated `searchStatus()`/`fetchStatus()`, never each provider's `status()` directly. + +## Vocabulary + +`WebSearchRequest` (`query`, `maxResults?`) → `WebSearchResult` (`providerId`, `query`, `content?`, `sources[]`, `truncated`); each `WebSearchSource` has a required `url` and optional `title`/`snippet`/`publishedAt` (Perplexity citations may be URL-only). `WebFetchRequest` (`url`, `timeoutMs?`) → `WebFetchResult` (`providerId`, final `url`, `statusCode`, `body`, `truncated`); `WebFetchBody` is a CLOSED discriminated union (`html` | `text`) owned here — consumers `switch` to exhaustiveness so a new kind breaks their compilation until handled. See `src/types.ts` for the full contracts and the `WebError` code taxonomy. diff --git a/packages/web/web/package.json b/packages/web/web/package.json new file mode 100644 index 0000000000..40ac10b715 --- /dev/null +++ b/packages/web/web/package.json @@ -0,0 +1,33 @@ +{ + "name": "@deepseek-ai/dsh-web", + "description": "Abstract web access capability seam (ctx.web) for the DeepSeek Harness — search/fetch provider registry, registration-order-independent selection, request/result vocabulary, and the WebError taxonomy", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "exports": { + ".": { + "types": "./lib/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "@deepseek-ai/dsh-llm": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "dependencies": { + "schemastery": "^3.18.0" + }, + "devDependencies": { + "@deepseek-ai/dsh-llm": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/web/web/src/index.ts b/packages/web/web/src/index.ts new file mode 100644 index 0000000000..172c1a0ecb --- /dev/null +++ b/packages/web/web/src/index.ts @@ -0,0 +1,270 @@ +/** + * The web access seam (`ctx.web`): a provider registry plus a provider-selecting + * execution surface for two capabilities — search and fetch. Provider packages + * register concrete backends with `registerSearchProvider` / + * `registerFetchProvider`; the model-facing consumer + * (`@deepseek-ai/dsh-tool-web`) reads capability status and executes through + * `search()` / `fetch()`. + * + * The registry half stays close to `LlmService`: a `Map` per + * capability kind, register methods that return disposers, duplicate ids that + * throw, and execution-time resolution that throws when the selected provider is + * absent or unusable. On top of that sits one small selection-status layer so + * diagnostics and execution can explain why a capability can or cannot run, + * independent of registration order. + * + * @module @deepseek-ai/dsh-web + */ + +import { Context, Service } from 'cordis' +import z from 'schemastery' +import type { + WebCapabilityStatus, + WebExecContext, + WebFetchProvider, + WebFetchRequest, + WebFetchResult, + WebProviderStatus, + WebSearchProvider, + WebSearchRequest, + WebSearchResult, +} from './types.ts' +import { WebError } from './types.ts' + +export { + WebError, +} from './types.ts' +export type { + WebCapabilityStatus, + WebErrorCode, + WebExecContext, + WebFetchBody, + WebFetchProvider, + WebFetchRequest, + WebFetchResult, + WebProviderStatus, + WebSearchProvider, + WebSearchRequest, + WebSearchResult, + WebSearchSource, +} from './types.ts' + +declare module 'cordis' { + interface Context { + web: WebService + } + + interface Events { + /** + * Fired after the provider registry changes — a search or fetch provider was + * registered or disposed. Carries no payload and no capability graph: it + * means only "the provider registry changed; observers may recompute status + * from `ctx.web`". `searchStatus()` / `fetchStatus()` stay derived, not + * stored. + * @mode emit + */ + 'web/providers-change'(this: WebService): void + } +} + +/** Selection inputs shared by the status query and execution resolution. */ +interface Selection

{ + /** The configured provider id for this capability, if any. */ + readonly configuredId?: string + /** Providers registered for this capability kind. */ + readonly providers: ReadonlyMap +} + +/** + * Config for the web seam. `searchProvider` / `fetchProvider` pin which provider + * wins for each capability; both are optional (a single registered usable + * provider auto-selects). Operational overrides such as environment variables + * must feed these same fields rather than introduce a hidden priority chain. + */ +export interface WebServiceConfig { + /** Explicit search provider id. Omitted = auto-select when exactly one usable. */ + readonly searchProvider?: string + /** Explicit fetch provider id. Omitted = auto-select when exactly one usable. */ + readonly fetchProvider?: string +} + +/** + * The web access service. Registered as `ctx.web` (one instance per context). + * + * Selection semantics (identical for status and execution, never order- + * dependent): + * - A configured id that is registered and `status().available` → that provider. + * - A configured id not registered → `configured-missing` / + * `WEB_PROVIDER_CONFIGURED_MISSING`. + * - A configured id registered but unavailable → `configured-unavailable` / + * `WEB_PROVIDER_CONFIGURED_UNAVAILABLE`. + * - No id configured, exactly one registered usable provider → that provider. + * - No id configured, multiple usable providers → `ambiguous` / + * `WEB_PROVIDER_AMBIGUOUS`. + * - No id configured, no usable provider → `none` / `WEB_PROVIDER_UNAVAILABLE`. + */ +export class WebService extends Service { + /** + * Provider selection config. Operational env overrides feed the SAME fields: + * `$DSH_WEB_SEARCH_PROVIDER` / `$DSH_WEB_FETCH_PROVIDER` are equivalent to + * `searchProvider` / `fetchProvider` and are NOT a hidden priority chain. + */ + static Config: z = z.object({ + searchProvider: z.string(), + fetchProvider: z.string(), + }) + + private searchProviders = new Map() + private fetchProviders = new Map() + private readonly searchProviderId: string | undefined + private readonly fetchProviderId: string | undefined + + constructor(ctx: Context, config: WebServiceConfig = {}) { + super(ctx, 'web') + this.searchProviderId = config.searchProvider ?? process.env.DSH_WEB_SEARCH_PROVIDER + this.fetchProviderId = config.fetchProvider ?? process.env.DSH_WEB_FETCH_PROVIDER + } + + /** + * Register a search provider. Throws {@link WebError} `WEB_DUPLICATE_PROVIDER` + * if its id is already registered for search. Returns a disposer; emits + * `web/providers-change` after a successful register and again on dispose. + * Disposed with the calling fiber. + */ + registerSearchProvider(provider: WebSearchProvider): () => void { + return this.registerProvider(this.searchProviders, provider) + } + + /** + * Register a fetch provider. Throws {@link WebError} `WEB_DUPLICATE_PROVIDER` + * if its id is already registered for fetch. Returns a disposer; emits + * `web/providers-change` after a successful register and again on dispose. + * Disposed with the calling fiber. + */ + registerFetchProvider(provider: WebFetchProvider): () => void { + return this.registerProvider(this.fetchProviders, provider) + } + + private registerProvider

(store: Map, provider: P): () => void { + if (store.has(provider.id)) { + throw new WebError(`a web provider with id "${provider.id}" is already registered`, 'WEB_DUPLICATE_PROVIDER') + } + const dispose = this.ctx.effect(function* (this: WebService) { + store.set(provider.id, provider) + // Yield the rollback BEFORE emitting `web/providers-change`: the generator + // effect collects each yielded disposer before the next step runs, so a + // throwing change listener removes the just-added provider instead of + // leaking it into the registry. + yield () => { + store.delete(provider.id) + this.ctx.emit('web/providers-change') + } + this.ctx.emit('web/providers-change') + }.bind(this), 'web.registerProvider()') + // ctx.effect's disposer returns Promise; our disposer API is + // synchronous fire-and-forget — discard the (always-resolved) promise. + return () => void dispose() + } + + /** Search-capability selection status, derived live (never stored). */ + searchStatus(): WebCapabilityStatus { + return resolveStatus({ + providers: this.searchProviders, + ...this.searchProviderId !== undefined ? { configuredId: this.searchProviderId } : {}, + }) + } + + /** Fetch-capability selection status, derived live (never stored). */ + fetchStatus(): WebCapabilityStatus { + return resolveStatus({ + providers: this.fetchProviders, + ...this.fetchProviderId !== undefined ? { configuredId: this.fetchProviderId } : {}, + }) + } + + /** + * Run one search through the selected provider. Resolves the provider at call + * time with the selection rules above; throws {@link WebError} when the + * capability cannot run. The seam enforces `request.maxResults` on the result: + * if the provider over-returns, `sources[]` is truncated and `truncated` set. + */ + async search(request: WebSearchRequest, exec?: WebExecContext): Promise { + const provider = resolveProvider({ + providers: this.searchProviders, + ...this.searchProviderId !== undefined ? { configuredId: this.searchProviderId } : {}, + }) + const result = await provider.search(request, exec) + return capSources(result, request.maxResults) + } + + /** + * Retrieve one URL through the selected provider. Resolves the provider at + * call time with the selection rules above; throws {@link WebError} when the + * capability cannot run. A non-2xx response is a result, not a throw. + */ + async fetch(request: WebFetchRequest, exec?: WebExecContext): Promise { + const provider = resolveProvider({ + providers: this.fetchProviders, + ...this.fetchProviderId !== undefined ? { configuredId: this.fetchProviderId } : {}, + }) + return provider.fetch(request, exec) + } +} + +interface ResolvableProvider { + readonly id: string + status(): WebProviderStatus +} + +/** Compute the capability status from configured id + registered providers. */ +function resolveStatus

(selection: Selection

): WebCapabilityStatus { + const { configuredId, providers } = selection + if (configuredId !== undefined) { + const provider = providers.get(configuredId) + if (!provider) return { available: false, reason: 'configured-missing' } + if (!provider.status().available) return { available: false, reason: 'configured-unavailable' } + return { available: true, providerId: configuredId } + } + const usable = [...providers.values()].filter(provider => provider.status().available) + const [single] = usable + if (single === undefined) return { available: false, reason: 'none' } + if (usable.length > 1) return { available: false, reason: 'ambiguous' } + return { available: true, providerId: single.id } +} + +/** + * Resolve the selected provider or throw the matching {@link WebError}. Shares + * the selection rules with {@link resolveStatus} so status and execution can + * never disagree. + */ +function resolveProvider

(selection: Selection

): P { + const { configuredId, providers } = selection + if (configuredId !== undefined) { + const provider = providers.get(configuredId) + if (!provider) { + throw new WebError(`configured web provider "${configuredId}" is not registered`, 'WEB_PROVIDER_CONFIGURED_MISSING') + } + if (!provider.status().available) { + throw new WebError(`configured web provider "${configuredId}" is registered but unavailable`, 'WEB_PROVIDER_CONFIGURED_UNAVAILABLE') + } + return provider + } + const usable = [...providers.values()].filter(provider => provider.status().available) + const [single] = usable + if (single === undefined) { + throw new WebError('no usable web provider is registered', 'WEB_PROVIDER_UNAVAILABLE') + } + if (usable.length > 1) { + const ids = usable.map(provider => provider.id).join(', ') + throw new WebError(`multiple usable web providers are registered (${ids}); configure one explicitly`, 'WEB_PROVIDER_AMBIGUOUS') + } + return single +} + +/** Enforce `maxResults` on a search result: truncate `sources[]` and flag it. */ +function capSources(result: WebSearchResult, maxResults: number | undefined): WebSearchResult { + if (maxResults === undefined || result.sources.length <= maxResults) return result + return { ...result, sources: result.sources.slice(0, maxResults), truncated: true } +} + +export default WebService diff --git a/packages/web/web/src/types.ts b/packages/web/web/src/types.ts new file mode 100644 index 0000000000..ec97101ae2 --- /dev/null +++ b/packages/web/web/src/types.ts @@ -0,0 +1,225 @@ +/** + * Vocabulary for the web capability seam (`ctx.web`): the search/fetch + * request/result shapes providers produce and consumers format, the provider + * and capability status discriminants selection reports, the execution-control + * context, and the typed error taxonomy. + * + * These types are shared by every provider backend + * (`@deepseek-ai/dsh-web-search-exa`, `@deepseek-ai/dsh-web-search-perplexity`, + * `@deepseek-ai/dsh-web-fetch-local`, and future backends) and by the + * model-facing consumer (`@deepseek-ai/dsh-tool-web`). Search and fetch share no + * request schema and no business logic, but they are deliberately one seam: + * `ctx.web` is a single web-access middle layer with one provider-selection + * policy, one abort/error vocabulary, and one product-facing configuration + * point. The cost is the parallel `Search`/`Fetch` shapes below; that + * parallelism is intentional. + * + * @module @deepseek-ai/dsh-web/types + */ + +import { HarnessError } from '@deepseek-ai/dsh-llm' + +/** + * Execution control threaded from the tool layer through the seam into a + * provider's network requests, stream readers, and expensive decoding. It is + * NOT business input: the first version carries only `signal` so `tool-web` can + * propagate turn cancellation, tool timeout, and agent disposal. It deliberately + * does NOT carry `ToolExecution`, which would make `dsh-web` depend on + * `dsh-tools`. + */ +export interface WebExecContext { + /** Abort signal a provider must honor for its network/decoding work. */ + readonly signal?: AbortSignal +} + +/** + * What one search-capable backend can return. The model-facing argument is just + * a query; `maxResults` is a `dsh-tool-web`-layer bound passed through unchanged + * and enforced on the way back by the seam (see {@link WebSearchResult}). + */ +export interface WebSearchRequest { + readonly query: string + /** + * Upper bound on returned sources; the seam truncates to it. Omitted = no + * bound. `dsh-tool-web` always sets it. A provider whose API supports a + * result-count control (Exa's `numResults`) should apply it at the request + * layer as a cost/latency optimization; the seam enforces the bound + * regardless. + */ + readonly maxResults?: number +} + +/** + * Normalized search outcome. `content` is optional provider-generated answer + * text or summary (Exa returns none; Perplexity returns a generated answer). + * `sources[]` is the portable citation surface. `truncated` is set by the seam + * when it cut `sources[]` down to `maxResults`. + */ +export interface WebSearchResult { + /** Id of the provider that produced this result. */ + readonly providerId: string + /** Echo of the query the provider answered. */ + readonly query: string + /** Optional provider-generated answer text, search context, or summary. */ + readonly content?: string + /** Citeable sources, already truncated to the request's `maxResults`. */ + readonly sources: readonly WebSearchSource[] + /** True when the seam dropped sources to honor `maxResults`. */ + readonly truncated: boolean +} + +/** + * One citeable source. A source always has a URL; `title`, `snippet`, and + * `publishedAt` are optional because not every provider returns them — forcing + * adapters to invent them would make the seam lie (Perplexity citations may be + * URL-only). `dsh-tool-web` renders `title ?? hostname(url)` for display. + */ +export interface WebSearchSource { + readonly url: string + readonly title?: string + readonly snippet?: string + /** Publication/crawl timestamp as a provider-supplied ISO-8601 string. */ + readonly publishedAt?: string +} + +/** + * What one fetch-capable backend is asked to retrieve. `timeoutMs` is an + * optional positive hint the provider caps. The request deliberately omits + * `format`, `prompt`, and extraction controls — those are presentation or + * higher-level LLM concerns, not safe-retrieval inputs. + */ +export interface WebFetchRequest { + readonly url: string + readonly timeoutMs?: number +} + +/** + * Normalized fetch outcome. A successful network fetch of a non-2xx response is + * a result, not an error: the status code is part of the fetched resource + * state. {@link WebError} is reserved for failures to safely retrieve or + * represent the resource. + */ +export interface WebFetchResult { + /** Id of the provider that produced this result. */ + readonly providerId: string + /** The final URL after allowed redirects (the request URL is in the request). */ + readonly url: string + /** HTTP status code of the fetched response. */ + readonly statusCode: number + /** Decoded body, classified by content kind. */ + readonly body: WebFetchBody + /** True when the provider capped the decoded body. */ + readonly truncated: boolean +} + +/** + * The decoded body of a fetched resource. A CLOSED discriminated union owned by + * `dsh-web`: the provider decodes the kind and `dsh-tool-web` renders it, so a + * new kind is a coordinated change across known packages, not a plugin + * extension. Consumers `switch` on `kind` ending in `default: assertNever(...)` + * so adding a kind breaks compilation at every consumer until handled. Each arm + * stays its own object literal even where fields coincide today, leaving room + * for arm-specific fields later (a `pdf` body's `pageCount`). + */ +export type WebFetchBody = + | { readonly kind: 'html'; readonly content: string } + | { readonly kind: 'text'; readonly content: string } + +/** + * Whether one concrete provider implementation is usable, by cheap local checks + * only (credential presence, parseable endpoint config). A provider `status()` + * must NOT make network calls. It is an input to selection, not a health system. + */ +export type WebProviderStatus = + | { readonly available: true } + | { readonly available: false; readonly reason: 'missing-credential' | 'misconfigured' } + +/** + * Whether a capability (search or fetch) has a selected usable provider, or the + * broad category in which selection fails. Intentionally small: it carries the + * winning `providerId` on the available branch (so diagnostics can report which + * provider won) but NOT the per-reason payload (the missing id, the ambiguous + * candidate set). That branchable detail lives in the {@link WebError} thrown at + * execution time — the surface callers route on — so the same fact does not get + * two homes that can disagree. + */ +export type WebCapabilityStatus = + | { readonly available: true; readonly providerId: string } + | { readonly available: false; readonly reason: 'none' | 'configured-missing' | 'configured-unavailable' | 'ambiguous' } + +/** + * A search-capable backend. Registered with `ctx.web.registerSearchProvider`. + * `id` is a stable string, unique within the search capability kind. + */ +export interface WebSearchProvider { + readonly id: string + /** Cheap local usability check; must not make network calls. */ + status(): WebProviderStatus + /** Run one search; honor `exec.signal` for cancellation. */ + search(request: WebSearchRequest, exec?: WebExecContext): Promise +} + +/** + * A fetch-capable backend. Registered with `ctx.web.registerFetchProvider`. + * `id` is a stable string, unique within the fetch capability kind. + */ +export interface WebFetchProvider { + readonly id: string + /** Cheap local usability check; must not make network calls. */ + status(): WebProviderStatus + /** Retrieve one URL; honor `exec.signal` for cancellation. */ + fetch(request: WebFetchRequest, exec?: WebExecContext): Promise +} + +/** + * Stable codes for {@link WebError}. Callers (hooks, tests, UI) route on these. + * + * - `WEB_PROVIDER_UNAVAILABLE`: no provider configured and none usable. + * - `WEB_PROVIDER_CONFIGURED_MISSING`: a configured id is not registered. + * - `WEB_PROVIDER_CONFIGURED_UNAVAILABLE`: a configured id is registered but its + * `status()` reports unavailable. + * - `WEB_PROVIDER_AMBIGUOUS`: no id configured and multiple usable providers + * exist (selection refuses to pick by registration order). + * - `WEB_DUPLICATE_PROVIDER`: a registration-time programming error — an id is + * already registered for that capability kind. + * - `WEB_INVALID_URL`: the fetch URL is malformed or not http(s). + * - `WEB_BLOCKED_URL`: the fetch URL is rejected by policy (credentials in URL). + * - `WEB_REDIRECT_BLOCKED`: a cross-origin redirect was refused. + * - `WEB_FETCH_TOO_LARGE`: the response exceeded the byte/character cap. + * - `WEB_FETCH_TIMEOUT`: the fetch exceeded its timeout. + * - `WEB_ABORTED`: the operation was aborted via `WebExecContext.signal`. + * - `WEB_UNSUPPORTED_CONTENT_TYPE`: the response content type cannot be decoded. + * - `WEB_PROVIDER_ERROR`: catch-all for a provider's own failure surfaced through + * the seam, including network/transport failure (DNS, connection refused, TLS). + */ +export type WebErrorCode = + | 'WEB_PROVIDER_UNAVAILABLE' + | 'WEB_PROVIDER_CONFIGURED_MISSING' + | 'WEB_PROVIDER_CONFIGURED_UNAVAILABLE' + | 'WEB_PROVIDER_AMBIGUOUS' + | 'WEB_DUPLICATE_PROVIDER' + | 'WEB_INVALID_URL' + | 'WEB_BLOCKED_URL' + | 'WEB_REDIRECT_BLOCKED' + | 'WEB_FETCH_TOO_LARGE' + | 'WEB_FETCH_TIMEOUT' + | 'WEB_ABORTED' + | 'WEB_UNSUPPORTED_CONTENT_TYPE' + | 'WEB_PROVIDER_ERROR' + +/** + * Typed web error. Extends {@link HarnessError} so it carries a stable + * {@link WebErrorCode} and chains `cause`. `dsh-web` owns this vocabulary so + * providers, the seam, and the tool layer raise the same codes instead of each + * inventing message strings. `ToolRegistry.execute()` converts a thrown + * `WebError` into an error tool result whose structured metadata exposes the + * code. + */ +export class WebError extends HarnessError { + override readonly code: WebErrorCode + + constructor(message: string, code: WebErrorCode, options?: ErrorOptions) { + super(message, code, options) + this.code = code + } +} diff --git a/packages/web/web/tests/web.spec.ts b/packages/web/web/tests/web.spec.ts new file mode 100644 index 0000000000..e97630ebab --- /dev/null +++ b/packages/web/web/tests/web.spec.ts @@ -0,0 +1,263 @@ +import { describe, expect, it, vi } from 'vitest' +import { Context } from 'cordis' +import WebService, { + WebError, + type WebFetchProvider, + type WebFetchResult, + type WebProviderStatus, + type WebSearchProvider, + type WebSearchRequest, + type WebSearchResult, +} from '@deepseek-ai/dsh-web' + +/** A scripted search provider for contract tests. */ +function makeSearchProvider( + id: string, + status: WebProviderStatus, + search: (request: WebSearchRequest) => Promise, +): WebSearchProvider { + return { id, status: () => status, search: request => search(request) } +} + +function makeFetchProvider(id: string, status: WebProviderStatus, result: WebFetchResult): WebFetchProvider { + return { id, status: () => status, fetch: () => Promise.resolve(result) } +} + +const available: WebProviderStatus = { available: true } +const unavailable: WebProviderStatus = { available: false, reason: 'missing-credential' } + +function searchResult(providerId: string, overrides: Partial = {}): WebSearchResult { + return { providerId, query: 'q', sources: [], truncated: false, ...overrides } +} + +function fetchResult(providerId: string): WebFetchResult { + return { providerId, url: 'https://example.com', statusCode: 200, body: { kind: 'text', content: 'hi' }, truncated: false } +} + +/** Mount a WebService on a fresh root context with the given config. */ +async function mountWeb(config: ConstructorParameters[1] = {}): Promise<{ ctx: Context; web: WebService }> { + const ctx = new Context() + await ctx.plugin(WebService, config) + return { ctx, web: ctx.web } +} + +describe('WebService registration', () => { + it('registers and disposes a search provider, emitting providers-change each way', async () => { + const { ctx, web } = await mountWeb() + const changed = vi.fn() + ctx.on('web/providers-change', changed) + + const dispose = web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa')))) + expect(changed).toHaveBeenCalledTimes(1) + expect(web.searchStatus()).toEqual({ available: true, providerId: 'exa' }) + + dispose() + expect(changed).toHaveBeenCalledTimes(2) + expect(web.searchStatus()).toEqual({ available: false, reason: 'none' }) + }) + + it('throws WEB_DUPLICATE_PROVIDER on a duplicate search id', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa')))) + expect(() => web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa'))))) + .toThrow(expect.objectContaining({ code: 'WEB_DUPLICATE_PROVIDER' })) + }) + + it('keeps search and fetch id namespaces independent', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('shared', available, () => Promise.resolve(searchResult('shared')))) + expect(() => web.registerFetchProvider(makeFetchProvider('shared', available, fetchResult('shared')))).not.toThrow() + }) + + it('rolls back a registration when a providers-change listener throws', async () => { + const { ctx, web } = await mountWeb() + ctx.on('web/providers-change', () => { throw new Error('listener boom') }) + expect(() => web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa'))))) + .toThrow('listener boom') + // The throwing listener must not leave the provider in the registry. + expect(web.searchStatus()).toEqual({ available: false, reason: 'none' }) + }) + + it('disposes provider registrations when the contributing fiber is disposed (HMR safety)', async () => { + const { ctx, web } = await mountWeb() + const fiber = await ctx.plugin(Object.assign((inner: Context) => { + inner.web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa')))) + }, { inject: ['web'] })) + expect(web.searchStatus()).toEqual({ available: true, providerId: 'exa' }) + await fiber.dispose() + expect(web.searchStatus()).toEqual({ available: false, reason: 'none' }) + }) +}) + +describe('WebService selection status', () => { + it('reports none when nothing is registered', async () => { + const { web } = await mountWeb() + expect(web.searchStatus()).toEqual({ available: false, reason: 'none' }) + expect(web.fetchStatus()).toEqual({ available: false, reason: 'none' }) + }) + + it('auto-selects the single usable provider when no id is configured', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa')))) + expect(web.searchStatus()).toEqual({ available: true, providerId: 'exa' }) + }) + + it('reports ambiguous when multiple usable providers exist and none is configured', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa')))) + web.registerSearchProvider(makeSearchProvider('perplexity', available, () => Promise.resolve(searchResult('perplexity')))) + expect(web.searchStatus()).toEqual({ available: false, reason: 'ambiguous' }) + }) + + it('ignores unusable providers when auto-selecting', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa')))) + web.registerSearchProvider(makeSearchProvider('perplexity', unavailable, () => Promise.resolve(searchResult('perplexity')))) + expect(web.searchStatus()).toEqual({ available: true, providerId: 'exa' }) + }) + + it('reports none when providers exist but none are usable', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('exa', unavailable, () => Promise.resolve(searchResult('exa')))) + expect(web.searchStatus()).toEqual({ available: false, reason: 'none' }) + }) + + it('honors a configured id over a different registered provider', async () => { + const { web } = await mountWeb({ searchProvider: 'perplexity' }) + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa')))) + web.registerSearchProvider(makeSearchProvider('perplexity', available, () => Promise.resolve(searchResult('perplexity')))) + expect(web.searchStatus()).toEqual({ available: true, providerId: 'perplexity' }) + }) + + it('reports configured-missing when the configured id is not registered', async () => { + const { web } = await mountWeb({ searchProvider: 'perplexity' }) + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa')))) + expect(web.searchStatus()).toEqual({ available: false, reason: 'configured-missing' }) + }) + + it('reports configured-unavailable when the configured id is registered but unusable', async () => { + const { web } = await mountWeb({ searchProvider: 'exa' }) + web.registerSearchProvider(makeSearchProvider('exa', unavailable, () => Promise.resolve(searchResult('exa')))) + expect(web.searchStatus()).toEqual({ available: false, reason: 'configured-unavailable' }) + }) + + it('does not let registration order change auto-selection', async () => { + const a = await mountWeb() + a.web.registerSearchProvider(makeSearchProvider('exa', unavailable, () => Promise.resolve(searchResult('exa')))) + a.web.registerSearchProvider(makeSearchProvider('perplexity', available, () => Promise.resolve(searchResult('perplexity')))) + expect(a.web.searchStatus()).toEqual({ available: true, providerId: 'perplexity' }) + + const b = await mountWeb() + b.web.registerSearchProvider(makeSearchProvider('perplexity', available, () => Promise.resolve(searchResult('perplexity')))) + b.web.registerSearchProvider(makeSearchProvider('exa', unavailable, () => Promise.resolve(searchResult('exa')))) + expect(b.web.searchStatus()).toEqual({ available: true, providerId: 'perplexity' }) + }) +}) + +describe('WebService execution resolution', () => { + it('throws WEB_PROVIDER_UNAVAILABLE when nothing is registered', async () => { + const { web } = await mountWeb() + await expect(web.search({ query: 'q' })).rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_UNAVAILABLE' })) + }) + + it('throws WEB_PROVIDER_CONFIGURED_MISSING for an unregistered configured id', async () => { + const { web } = await mountWeb({ searchProvider: 'perplexity' }) + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa')))) + await expect(web.search({ query: 'q' })).rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_CONFIGURED_MISSING' })) + }) + + it('throws WEB_PROVIDER_CONFIGURED_UNAVAILABLE for an unusable configured id', async () => { + const { web } = await mountWeb({ searchProvider: 'exa' }) + web.registerSearchProvider(makeSearchProvider('exa', unavailable, () => Promise.resolve(searchResult('exa')))) + await expect(web.search({ query: 'q' })).rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_CONFIGURED_UNAVAILABLE' })) + }) + + it('throws WEB_PROVIDER_AMBIGUOUS rather than picking by order', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa')))) + web.registerSearchProvider(makeSearchProvider('perplexity', available, () => Promise.resolve(searchResult('perplexity')))) + await expect(web.search({ query: 'q' })).rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_AMBIGUOUS' })) + }) + + it('runs the selected provider and returns its result', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve( + searchResult('exa', { content: 'answer', sources: [{ url: 'https://a' }] }), + ))) + const result = await web.search({ query: 'q' }) + expect(result.providerId).toBe('exa') + expect(result.content).toBe('answer') + expect(result.sources).toEqual([{ url: 'https://a' }]) + }) + + it('propagates the abort signal to the provider', async () => { + const { web } = await mountWeb() + const seen: (AbortSignal | undefined)[] = [] + web.registerSearchProvider({ + id: 'exa', + status: () => available, + search: (_request, exec) => { seen.push(exec?.signal); return Promise.resolve(searchResult('exa')) }, + }) + const controller = new AbortController() + await web.search({ query: 'q' }, { signal: controller.signal }) + expect(seen[0]).toBe(controller.signal) + }) +}) + +describe('WebService maxResults enforcement', () => { + it('truncates sources and sets truncated when a provider over-returns', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa', { + sources: [{ url: 'https://1' }, { url: 'https://2' }, { url: 'https://3' }], + })))) + const result = await web.search({ query: 'q', maxResults: 2 }) + expect(result.sources).toHaveLength(2) + expect(result.truncated).toBe(true) + }) + + it('leaves truncated false when within the bound', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa', { + sources: [{ url: 'https://1' }], + })))) + const result = await web.search({ query: 'q', maxResults: 8 }) + expect(result.sources).toHaveLength(1) + expect(result.truncated).toBe(false) + }) + + it('does not bound when maxResults is omitted', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa', { + sources: [{ url: 'https://1' }, { url: 'https://2' }], + })))) + const result = await web.search({ query: 'q' }) + expect(result.sources).toHaveLength(2) + expect(result.truncated).toBe(false) + }) +}) + +describe('WebService fetch capability', () => { + it('resolves and runs the fetch provider independently of search', async () => { + const { web } = await mountWeb() + web.registerFetchProvider(makeFetchProvider('local-http', available, fetchResult('local-http'))) + const result = await web.fetch({ url: 'https://example.com' }) + expect(result.providerId).toBe('local-http') + expect(result.statusCode).toBe(200) + }) + + it('throws WEB_PROVIDER_UNAVAILABLE for fetch when no fetch provider is registered', async () => { + const { web } = await mountWeb() + web.registerSearchProvider(makeSearchProvider('exa', available, () => Promise.resolve(searchResult('exa')))) + await expect(web.fetch({ url: 'https://example.com' })).rejects.toThrow( + expect.objectContaining({ code: 'WEB_PROVIDER_UNAVAILABLE' }), + ) + }) +}) + +describe('WebError', () => { + it('is a HarnessError carrying its code', () => { + const error = new WebError('boom', 'WEB_INVALID_URL') + expect(error.code).toBe('WEB_INVALID_URL') + expect(error.name).toBe('WebError') + }) +}) diff --git a/packages/web/web/tsconfig.json b/packages/web/web/tsconfig.json new file mode 100644 index 0000000000..b187cddf35 --- /dev/null +++ b/packages/web/web/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../../vendor/cosmokit" + }, + { + "path": "../../../vendor/cordis" + }, + { + "path": "../../../vendor/schemastery" + }, + { + "path": "../../llm/llm" + } + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2afc331514..a393b6c4b9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -703,6 +703,92 @@ importers: specifier: ^4.0.0-rc.6 version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/web/tool-web: + dependencies: + schemastery: + specifier: ^3.18.0 + version: 3.18.0 + devDependencies: + '@deepseek-ai/dsh-agent': + specifier: workspace:^ + version: link:../../core/agent + '@deepseek-ai/dsh-llm': + specifier: workspace:^ + version: link:../../llm/llm + '@deepseek-ai/dsh-session': + specifier: workspace:^ + version: link:../../core/session + '@deepseek-ai/dsh-system-prompt': + specifier: workspace:^ + version: link:../../core/system-prompt + '@deepseek-ai/dsh-tools': + specifier: workspace:^ + version: link:../../core/tools + '@deepseek-ai/dsh-web': + specifier: workspace:^ + version: link:../web + '@deepseek-ai/dsh-web-fetch-local': + specifier: workspace:^ + version: link:../web-fetch-local + '@deepseek-ai/dsh-web-search-exa': + specifier: workspace:^ + version: link:../web-search-exa + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + + packages/web/web: + dependencies: + schemastery: + specifier: ^3.18.0 + version: 3.18.0 + devDependencies: + '@deepseek-ai/dsh-llm': + specifier: workspace:^ + version: link:../../llm/llm + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + + packages/web/web-fetch-local: + dependencies: + schemastery: + specifier: ^3.18.0 + version: 3.18.0 + devDependencies: + '@deepseek-ai/dsh-web': + specifier: workspace:^ + version: link:../web + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + + packages/web/web-search-exa: + dependencies: + schemastery: + specifier: ^3.18.0 + version: 3.18.0 + devDependencies: + '@deepseek-ai/dsh-web': + specifier: workspace:^ + version: link:../web + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + + packages/web/web-search-perplexity: + dependencies: + schemastery: + specifier: ^3.18.0 + version: 3.18.0 + devDependencies: + '@deepseek-ai/dsh-web': + specifier: workspace:^ + version: link:../web + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + vendor/cordis: dependencies: '@cordisjs/plugin-include': diff --git a/tsconfig.base.json b/tsconfig.base.json index 7f46a9105a..bc4f13bdd5 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -34,6 +34,8 @@ "@cordisjs/plugin-timer": ["./vendor/timer/src"], "@cordisjs/plugin-hmr": ["./vendor/hmr/src"], "@cordisjs/plugin-logger-console": ["./vendor/logger-console/src"], + "@deepseek-ai/dsh-tool-web/search": ["./packages/web/tool-web/src/search.ts"], + "@deepseek-ai/dsh-tool-web/fetch": ["./packages/web/tool-web/src/fetch.ts"], // One wildcard maps every @deepseek-ai/dsh- to its source. Package // dir names are unique across groups, so first-on-disk-wins resolution is // unambiguous; adding a package under an existing group needs no edit @@ -45,6 +47,7 @@ "./packages/bash/*/src", "./packages/compact/*/src", "./packages/subagent/*/src", + "./packages/web/*/src", "./packages/session-persistence/*/src", "./packages/ui/*/src", "./packages/util/*/src", diff --git a/tsconfig.build.json b/tsconfig.build.json index 9d76a33385..fb7a058ea8 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -27,6 +27,11 @@ { "path": "./packages/llm/llm-pi-ai" }, { "path": "./packages/bash/bash-local" }, { "path": "./packages/bash/tool-bash" }, + { "path": "./packages/web/web" }, + { "path": "./packages/web/web-search-exa" }, + { "path": "./packages/web/web-search-perplexity" }, + { "path": "./packages/web/web-fetch-local" }, + { "path": "./packages/web/tool-web" }, { "path": "./packages/support/invariants" }, { "path": "./packages/ui/acp" }, { "path": "./packages/ui/acp-agent" }, From 567519184ba42ffd2204583327e08091ae3b5120 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 25 Jun 2026 15:28:15 +0800 Subject: [PATCH 03/16] fix: address codex review round 1 - Re-validate redirect targets through validateFetchUrl before following, so a same-origin Location carrying credentials (or a non-http(s)/over-long URL) cannot bypass the transport hygiene a direct request enforces. - Treat only DROPPED bytes as truncation: a body exactly at maxResponseBytes is no longer falsely flagged truncated (which emitted a spurious footer). - Honor the declared response charset: parse the Content-Type charset and decode with it (rejecting unsupported labels as WEB_UNSUPPORTED_CONTENT_TYPE) instead of always assuming UTF-8 and returning replacement characters. - Catalog the web seam vocabulary in docs/core-data-structures/web.md with type-equiv blocks + manifest entries, per the core-data-structures rule. --- docs/core-data-structures/core.md | 1 + docs/core-data-structures/web.md | 119 ++++++++++++++++++ packages/web/web-fetch-local/src/index.ts | 2 +- packages/web/web-fetch-local/src/policy.ts | 26 ++++ packages/web/web-fetch-local/src/provider.ts | 27 ++-- .../web-fetch-local/tests/fetch-local.spec.ts | 42 ++++++- scripts/type-equiv.manifest.json | 12 +- 7 files changed, 218 insertions(+), 11 deletions(-) create mode 100644 docs/core-data-structures/web.md diff --git a/docs/core-data-structures/core.md b/docs/core-data-structures/core.md index 6d20900c93..09058a596a 100644 --- a/docs/core-data-structures/core.md +++ b/docs/core-data-structures/core.md @@ -22,6 +22,7 @@ Everything else is documented on a **sub-page**, not here. The rule that draws t | [bash.md](bash.md) | the bash executor seam: `BashExecRequest`/`Spec`, `BashRunResult`, background `BashTask`s | | [compaction.md](compaction.md) | the compaction seam: the `compact/*` session events, `CompactionResult`, the `CompactService` interface | | [subagent.md](subagent.md) | the subagent seam: the named-provider registry, `SubagentStartRequest`/`Result`/`Run`, the start-time-vs-runtime capability split | +| [web.md](web.md) | the web access seam: `WebSearchRequest`/`Result`, `WebFetchRequest`/`Result`, `WebFetchBody`, provider/capability status, `WebErrorCode` | > Type definitions on this page are pasted **verbatim** from source and drift-checked by `pnpm run verify-type-equiv` (see [development.md](../development.md#documenting-types-verbatim-ts-type-equiv)). Inline JSDoc is omitted for readability; follow the source link for the full contracts. diff --git a/docs/core-data-structures/web.md b/docs/core-data-structures/web.md new file mode 100644 index 0000000000..f0ade276f7 --- /dev/null +++ b/docs/core-data-structures/web.md @@ -0,0 +1,119 @@ +# Web Access + +The web access seam — a [capability seam](../rfc/implemented/architecture/2026-06-24-web-capability-seam.md) that spans **two capabilities** (search and fetch) on one `ctx.web` service, split across packages: interface ([dsh-web](../../packages/web/web), `ctx.web` + the provider registries), implementations ([dsh-web-search-exa](../../packages/web/web-search-exa), [dsh-web-search-perplexity](../../packages/web/web-search-perplexity), [dsh-web-fetch-local](../../packages/web/web-fetch-local)), and consumer ([dsh-tool-web](../../packages/web/tool-web), the `web_search`/`web_fetch` tool schemas). Web is **one optional capability**, not part of the agent-loop spine — so its vocabulary lives here, not in [core.md](core.md). A search-provider swap does not change how the model asks for a query, and a fetch-implementation swap does not change how the model asks for a URL. + +Source: [`packages/web/web/src/types.ts`](../../packages/web/web/src/types.ts) + +## Why one seam for two capabilities + +Search and fetch share no request schema and no business logic, but they are deliberately one `ctx.web` middle layer: one provider-selection policy owner, one abort/error vocabulary, one product-facing "how this harness reaches the web" config surface. The cost is the parallel `searchX`/`fetchX` method pairs on the service; that parallelism is intentional, not a missed extraction. Providers register **capabilities** (a `WebSearchProvider` or `WebFetchProvider`), not tools; the model-facing names, schemas, prompt guidance, and presentation all live in the single `dsh-tool-web` consumer. + +## Search request and result + +The model-facing tool argument is just a `query`; `maxResults` is a consumer-owned bound (`dsh-tool-web`'s `WEB_SEARCH_MAX_RESULTS`, default `8`) passed through the seam and enforced on the way back — if a provider over-returns, the seam truncates `sources[]` and sets `truncated`. + +```ts type-equiv +interface WebSearchRequest { + readonly query: string + /** + * Upper bound on returned sources; the seam truncates to it. Omitted = no + * bound. `dsh-tool-web` always sets it. + */ + readonly maxResults?: number +} +``` + +```ts type-equiv +interface WebSearchResult { + readonly providerId: string + readonly query: string + readonly content?: string + readonly sources: readonly WebSearchSource[] + readonly truncated: boolean +} +``` + +`content` is optional provider-generated answer text (Exa returns none; Perplexity returns a generated answer). `sources[]` is the portable citation surface. A source always has a `url`; `title`/`snippet`/`publishedAt` are optional because not every provider returns them — Perplexity citations may be URL-only, and forcing adapters to invent the rest would make the seam lie. `dsh-tool-web` renders `title ?? hostname(url)`. + +```ts type-equiv +interface WebSearchSource { + readonly url: string + readonly title?: string + readonly snippet?: string + readonly publishedAt?: string +} +``` + +## Fetch request and result + +```ts type-equiv +interface WebFetchRequest { + readonly url: string + readonly timeoutMs?: number +} +``` + +HTTP status is part of the fetched resource state, not automatically a failure: a successful network fetch of a `404`/`500` returns a `WebFetchResult` with the status code and a bounded decoded body. `url` is the final URL after allowed redirects. `WebError` is reserved for failures to safely retrieve or represent the resource. + +```ts type-equiv +interface WebFetchResult { + readonly providerId: string + readonly url: string + readonly statusCode: number + readonly body: WebFetchBody + readonly truncated: boolean +} +``` + +`WebFetchBody` is a **closed** discriminated union owned by `dsh-web` (not a merge-extensible map): the provider decodes the kind and `dsh-tool-web` renders it, so a new kind is a coordinated change across known packages, not a plugin extension. Consumers `switch` on `kind` ending in `default: assertNever(...)`, so adding a kind breaks compilation at every consumer until handled. Each arm stays its own object literal even where fields coincide today, leaving room for arm-specific fields later (a future `pdf` body's `pageCount`). + +```ts type-equiv +type WebFetchBody = + | { readonly kind: 'html'; readonly content: string } + | { readonly kind: 'text'; readonly content: string } +``` + +## Provider and capability status + +A provider's `status()` is a cheap LOCAL check (credential presence, parseable config) and **must not make network calls**. It is an input to selection, not a health system. + +```ts type-equiv +type WebProviderStatus = + | { readonly available: true } + | { readonly available: false; readonly reason: 'missing-credential' | 'misconfigured' } +``` + +The service aggregates provider status into a `WebCapabilityStatus`: whether the capability has a selected usable provider, or the broad category in which selection fails. It carries the winning `providerId` on the available branch but NOT the per-reason payload (the missing id, the ambiguous set) — that branchable detail lives in the thrown `WebError`, the surface callers route on, so the same fact never gets two homes that can disagree. + +```ts type-equiv +type WebCapabilityStatus = + | { readonly available: true; readonly providerId: string } + | { readonly available: false; readonly reason: 'none' | 'configured-missing' | 'configured-unavailable' | 'ambiguous' } +``` + +Selection never depends on registration, config, or HMR order: a capability has an explicit provider id (config `searchProvider`/`fetchProvider`, or the matching env var feeding the same field), or auto-selects when exactly one usable provider is registered; multiple usable providers with no configured id is `ambiguous`, not first-wins. + +## Errors + +`WebError extends HarnessError` ([core.md](core.md) error taxonomy) with a stable `WebErrorCode`. `WEB_DUPLICATE_PROVIDER` is a registration-time programming error (the analogue of `LlmService`'s `DUPLICATE_ADAPTER`); the `WEB_PROVIDER_*` selection codes and the fetch transport codes are execution outcomes. `WEB_PROVIDER_ERROR` is the catch-all for a provider's own failure surfaced through the seam, including network/transport failure (DNS, connection refused, TLS). + +```ts type-equiv +type WebErrorCode = + | 'WEB_PROVIDER_UNAVAILABLE' + | 'WEB_PROVIDER_CONFIGURED_MISSING' + | 'WEB_PROVIDER_CONFIGURED_UNAVAILABLE' + | 'WEB_PROVIDER_AMBIGUOUS' + | 'WEB_DUPLICATE_PROVIDER' + | 'WEB_INVALID_URL' + | 'WEB_BLOCKED_URL' + | 'WEB_REDIRECT_BLOCKED' + | 'WEB_FETCH_TOO_LARGE' + | 'WEB_FETCH_TIMEOUT' + | 'WEB_ABORTED' + | 'WEB_UNSUPPORTED_CONTENT_TYPE' + | 'WEB_PROVIDER_ERROR' +``` + +## The service + +`WebService` (`ctx.web`, defined in [`packages/web/web/src/index.ts`](../../packages/web/web/src/index.ts)) is a provider registry plus a provider-selecting execution surface, close to `LlmService`'s shape: `registerSearchProvider`/`registerFetchProvider` (duplicate ids throw `WEB_DUPLICATE_PROVIDER`, return disposers, emit `web/providers-change`), `searchStatus`/`fetchStatus` (derived, never stored), and `search`/`fetch` (resolve the provider at call time, throw a structured `WebError` when the capability cannot run). Providers issue requests with the platform-native `fetch` (Node 24), mirroring `dsh-llm-deepseek`; the `dsh-web-fetch-local` provider owns safe retrieval (http/https-only, credential rejection, byte/char/timeout/redirect caps, same-origin-only redirects with per-hop re-validation, charset decoding) while `dsh-tool-web` owns presentation (HTML→markdown). SSRF / private-network blocking is deferred (see the RFC) — until it lands, `web_fetch` must not be enabled where it can reach sensitive internal targets. diff --git a/packages/web/web-fetch-local/src/index.ts b/packages/web/web-fetch-local/src/index.ts index 1d57410d68..eb3f8e4143 100644 --- a/packages/web/web-fetch-local/src/index.ts +++ b/packages/web/web-fetch-local/src/index.ts @@ -18,7 +18,7 @@ export { LocalFetchProvider, } from './provider.ts' export type { LocalFetchLimits } from './provider.ts' -export { classifyContentType, isSameOrigin, validateFetchUrl } from './policy.ts' +export { classifyContentType, decoderForCharset, isSameOrigin, parseCharset, validateFetchUrl } from './policy.ts' export type { FetchableKind } from './policy.ts' /** Default `User-Agent`: an explicit product agent, never a browser disguise. */ diff --git a/packages/web/web-fetch-local/src/policy.ts b/packages/web/web-fetch-local/src/policy.ts index 8261c5c1ed..7a76bd1af1 100644 --- a/packages/web/web-fetch-local/src/policy.ts +++ b/packages/web/web-fetch-local/src/policy.ts @@ -57,3 +57,29 @@ export function classifyContentType(contentType: string | null): FetchableKind | if (mime === 'application/json' || mime === 'application/xml' || mime.endsWith('+json') || mime.endsWith('+xml')) return 'text' return undefined } + +/** + * Extract the `charset` parameter from a response `Content-Type`, lower-cased, + * or `undefined` when absent. The provider feeds this label to `TextDecoder` + * so a non-UTF-8 response is decoded with its declared encoding rather than + * silently mangled into replacement characters. + */ +export function parseCharset(contentType: string | null): string | undefined { + const match = /;\s*charset\s*=\s*"?([^";]+)"?/i.exec(contentType ?? '') + return match?.[1]?.trim().toLowerCase() +} + +/** + * Build a `TextDecoder` for the declared charset, falling back to UTF-8 when + * none is declared. Throws {@link WebError} `WEB_UNSUPPORTED_CONTENT_TYPE` when + * the label is present but not a charset `TextDecoder` recognizes — better to + * fail loudly than return mojibake. + */ +export function decoderForCharset(charset: string | undefined): TextDecoder { + if (charset === undefined) return new TextDecoder('utf-8') + try { + return new TextDecoder(charset) + } catch (error: unknown) { + throw new WebError(`unsupported charset "${charset}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE', { cause: error }) + } +} diff --git a/packages/web/web-fetch-local/src/provider.ts b/packages/web/web-fetch-local/src/provider.ts index 0622030af0..c8b99d08b0 100644 --- a/packages/web/web-fetch-local/src/provider.ts +++ b/packages/web/web-fetch-local/src/provider.ts @@ -21,7 +21,7 @@ import { WebError } from '@deepseek-ai/dsh-web' import type { WebFetchBody, WebFetchProvider, WebFetchRequest, WebFetchResult, WebProviderStatus } from '@deepseek-ai/dsh-web' -import { classifyContentType, isSameOrigin, validateFetchUrl } from './policy.ts' +import { classifyContentType, decoderForCharset, isSameOrigin, parseCharset, validateFetchUrl } from './policy.ts' /** Resolved provider limits (the plugin's schemastery Config supplies defaults). */ export interface LocalFetchLimits { @@ -92,14 +92,18 @@ export class LocalFetchProvider implements WebFetchProvider { throw new WebError(`redirect response (HTTP ${response.status}) without a Location header`, 'WEB_PROVIDER_ERROR') } const target = resolveRedirect(location, currentUrl) - if (!isSameOrigin(target, currentUrl)) { + // Re-validate the target against the same transport hygiene a direct + // request gets: a redirect must not be a back door to a credentialed, + // non-http(s), or over-long URL that validateFetchUrl would reject. + const validatedTarget = validateFetchUrl(target.toString(), this.limits.maxUrlLength) + if (!isSameOrigin(validatedTarget, currentUrl)) { throw new WebError( - `cross-origin redirect to ${target.origin} is not followed automatically; retry against that URL directly`, + `cross-origin redirect to ${validatedTarget.origin} is not followed automatically; retry against that URL directly`, 'WEB_REDIRECT_BLOCKED', ) } await response.body?.cancel() - currentUrl = target + currentUrl = validatedTarget continue } @@ -124,14 +128,18 @@ export class LocalFetchProvider implements WebFetchProvider { /** Read, byte-cap, classify, and decode the final response body. */ private async readBody(response: Response, finalUrl: URL): Promise { - const kind = classifyContentType(response.headers.get('content-type')) + const contentType = response.headers.get('content-type') + const kind = classifyContentType(contentType) if (kind === undefined) { await response.body?.cancel() - throw new WebError(`unsupported content type "${response.headers.get('content-type') ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE') + throw new WebError(`unsupported content type "${contentType ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE') } + // Resolve the decoder BEFORE reading the body so an unsupported charset + // fails without consuming the stream. + const decoder = decoderForCharset(parseCharset(contentType)) const { bytes, truncatedByBytes } = await this.readCapped(response) - const decoded = new TextDecoder('utf-8').decode(bytes) + const decoded = decoder.decode(bytes) const truncatedByChars = decoded.length > this.limits.maxBodyChars const content = truncatedByChars ? decoded.slice(0, this.limits.maxBodyChars) : decoded const body: WebFetchBody = kind === 'html' ? { kind: 'html', content } : { kind: 'text', content } @@ -173,7 +181,10 @@ export class LocalFetchProvider implements WebFetchProvider { const { done, value } = await reader.read() if (done) break const remaining = this.limits.maxResponseBytes - total - if (value.byteLength >= remaining) { + // Only DROPPED bytes count as truncation: a chunk that exactly fills the + // remaining capacity keeps all its bytes and we read on to observe EOF, + // so an exactly-at-cap body is not falsely flagged truncated. + if (value.byteLength > remaining) { chunks.push(value.subarray(0, remaining)) total += remaining truncatedByBytes = true diff --git a/packages/web/web-fetch-local/tests/fetch-local.spec.ts b/packages/web/web-fetch-local/tests/fetch-local.spec.ts index 3ca48150e1..54e8de20c0 100644 --- a/packages/web/web-fetch-local/tests/fetch-local.spec.ts +++ b/packages/web/web-fetch-local/tests/fetch-local.spec.ts @@ -3,7 +3,7 @@ import { createServer, type IncomingMessage, type Server, type ServerResponse } import { AddressInfo } from 'node:net' import { Context } from 'cordis' import WebService from '@deepseek-ai/dsh-web' -import { LocalFetchProvider, LOCAL_FETCH_PROVIDER_ID, classifyContentType, isSameOrigin, validateFetchUrl } from '@deepseek-ai/dsh-web-fetch-local' +import { LocalFetchProvider, LOCAL_FETCH_PROVIDER_ID, classifyContentType, decoderForCharset, isSameOrigin, parseCharset, validateFetchUrl } from '@deepseek-ai/dsh-web-fetch-local' import type { LocalFetchLimits } from '@deepseek-ai/dsh-web-fetch-local' import * as fetchPlugin from '@deepseek-ai/dsh-web-fetch-local' @@ -62,6 +62,19 @@ describe('policy helpers', () => { expect(isSameOrigin(new URL('https://a.com'), new URL('https://b.com'))).toBe(false) expect(isSameOrigin(new URL('http://a.com'), new URL('https://a.com'))).toBe(false) }) + + it('parses the charset parameter', () => { + expect(parseCharset('text/html; charset=UTF-8')).toBe('utf-8') + expect(parseCharset('text/plain; charset="iso-8859-1"')).toBe('iso-8859-1') + expect(parseCharset('text/plain')).toBeUndefined() + expect(parseCharset(null)).toBeUndefined() + }) + + it('builds a decoder for a charset and defaults to UTF-8', () => { + expect(decoderForCharset(undefined).encoding).toBe('utf-8') + expect(decoderForCharset('iso-8859-1').encoding).toBe('windows-1252') + expect(() => decoderForCharset('not-a-charset')).toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + }) }) describe('LocalFetchProvider success', () => { @@ -109,6 +122,13 @@ describe('LocalFetchProvider caps', () => { expect(result.truncated).toBe(true) }) + it('does not flag a body that exactly fills the byte cap as truncated', async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('abcd') } + const result = await provider({ maxResponseBytes: 4 }).fetch({ url: base }) + expect(result.body.content).toBe('abcd') + expect(result.truncated).toBe(false) + }) + it('truncates a decoded body past the character cap', async () => { handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('abcdefghij') } const result = await provider({ maxBodyChars: 3 }).fetch({ url: base }) @@ -133,6 +153,19 @@ describe('LocalFetchProvider caps', () => { const result = await provider().fetch({ url: base }) expect(result.body.content).toBe('sized') }) + + it('decodes a non-UTF-8 declared charset', async () => { + // 0xE9 is "é" in ISO-8859-1; decoded as UTF-8 it would be a replacement char. + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain; charset=iso-8859-1' }); res.end(Buffer.from([0x63, 0x61, 0x66, 0xE9])) } + const result = await provider().fetch({ url: base }) + expect(result.body.content).toBe('café') + }) + + it('rejects an unsupported declared charset', async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain; charset=not-a-charset' }); res.end('x') } + await expect(provider().fetch({ url: base })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + }) }) describe('LocalFetchProvider redirects', () => { @@ -152,6 +185,13 @@ describe('LocalFetchProvider redirects', () => { .rejects.toThrow(expect.objectContaining({ code: 'WEB_REDIRECT_BLOCKED' })) }) + it('re-validates a redirect target, rejecting same-origin credentials in the Location', async () => { + const { port } = server.address() as AddressInfo + handler = (_req, res) => { res.writeHead(302, { location: `http://user:pass@127.0.0.1:${port}/` }); res.end() } + await expect(provider().fetch({ url: base })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_BLOCKED_URL' })) + }) + it('rejects exceeding the redirect hop cap', async () => { handler = (req, res) => { const n = Number(new URL(req.url ?? '/', base).searchParams.get('n') ?? '0') diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index 4008a7cc20..f250c5b383 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -48,6 +48,16 @@ { "doc": "docs/core-data-structures/subagent.md", "symbol": "SubagentResult", "source": "packages/subagent/subagent/src/types.ts" }, { "doc": "docs/core-data-structures/subagent.md", "symbol": "SubagentStopReasonMap", "source": "packages/subagent/subagent/src/types.ts" }, { "doc": "docs/core-data-structures/subagent.md", "symbol": "SubagentRun", "source": "packages/subagent/subagent/src/types.ts" }, - { "doc": "docs/core-data-structures/subagent.md", "symbol": "SubagentProvider", "source": "packages/subagent/subagent/src/types.ts" } + { "doc": "docs/core-data-structures/subagent.md", "symbol": "SubagentProvider", "source": "packages/subagent/subagent/src/types.ts" }, + + { "doc": "docs/core-data-structures/web.md", "symbol": "WebSearchRequest", "source": "packages/web/web/src/types.ts" }, + { "doc": "docs/core-data-structures/web.md", "symbol": "WebSearchResult", "source": "packages/web/web/src/types.ts" }, + { "doc": "docs/core-data-structures/web.md", "symbol": "WebSearchSource", "source": "packages/web/web/src/types.ts" }, + { "doc": "docs/core-data-structures/web.md", "symbol": "WebFetchRequest", "source": "packages/web/web/src/types.ts" }, + { "doc": "docs/core-data-structures/web.md", "symbol": "WebFetchResult", "source": "packages/web/web/src/types.ts" }, + { "doc": "docs/core-data-structures/web.md", "symbol": "WebFetchBody", "source": "packages/web/web/src/types.ts" }, + { "doc": "docs/core-data-structures/web.md", "symbol": "WebProviderStatus", "source": "packages/web/web/src/types.ts" }, + { "doc": "docs/core-data-structures/web.md", "symbol": "WebCapabilityStatus", "source": "packages/web/web/src/types.ts" }, + { "doc": "docs/core-data-structures/web.md", "symbol": "WebErrorCode", "source": "packages/web/web/src/types.ts" } ] } From 0930e483ecf98ef363fddb51e47bf0475d34b629 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 25 Jun 2026 16:03:55 +0800 Subject: [PATCH 04/16] fix: address codex review round 2 - Preserve abort errors while parsing search responses: when the caller's AbortSignal fires after headers but during response.json() (both the success and HTTP-error body parses), surface WEB_ABORTED instead of wrapping it as WEB_PROVIDER_ERROR, so agent cancel/dispose is not misreported as a provider failure. Applied to both the Exa and Perplexity providers. - Report a malformed baseURL as misconfigured in status() (URL.canParse), so selection diagnostics and execution agree (configured-unavailable up front rather than a late WEB_PROVIDER_ERROR). WebProviderStatus already had the reason. --- packages/web/web-search-exa/README.md | 2 +- packages/web/web-search-exa/src/provider.ts | 20 ++++++++++++++----- packages/web/web-search-exa/tests/exa.spec.ts | 19 ++++++++++++++++++ packages/web/web-search-perplexity/README.md | 2 +- .../web/web-search-perplexity/src/provider.ts | 15 +++++++++----- .../tests/perplexity.spec.ts | 19 ++++++++++++++++++ 6 files changed, 65 insertions(+), 12 deletions(-) diff --git a/packages/web/web-search-exa/README.md b/packages/web/web-search-exa/README.md index 7f39356e81..61da605df7 100644 --- a/packages/web/web-search-exa/README.md +++ b/packages/web/web-search-exa/README.md @@ -9,7 +9,7 @@ This is an **implementation** package: it registers a provider into `ctx.web`, i | Key | Default | Meaning | |---|---|---| | `apiKey` | `$EXA_API_KEY` | Exa API key. Empty/absent → provider `status()` reports `missing-credential` (the seam reports `configured-unavailable`/`none`). | -| `baseURL` | `https://api.exa.ai` | Endpoint base; `/search` is appended. | +| `baseURL` | `https://api.exa.ai` | Endpoint base; `/search` is appended. An unparseable value makes `status()` report `misconfigured`. | ```yaml - id: web-search-exa diff --git a/packages/web/web-search-exa/src/provider.ts b/packages/web/web-search-exa/src/provider.ts index d7c464059a..70e14cbf25 100644 --- a/packages/web/web-search-exa/src/provider.ts +++ b/packages/web/web-search-exa/src/provider.ts @@ -72,6 +72,7 @@ export class ExaSearchProvider implements WebSearchProvider { status(): WebProviderStatus { if (this.options.apiKey.length === 0) return { available: false, reason: 'missing-credential' } + if (!isValidBaseUrl(this.options.baseURL)) return { available: false, reason: 'misconfigured' } return { available: true } } @@ -105,11 +106,14 @@ export class ExaSearchProvider implements WebSearchProvider { const parsed = await response.json() as ExaError const detail = parsed.error ?? parsed.message if (detail !== undefined && detail.length > 0) message = detail - } catch { - // The HTTP status is already captured in `message` above; a malformed or - // non-JSON error body (normal for gateway 5xx/429s) can only cost a - // richer provider message, never the real error. `response.json()` is - // the sole statement and nothing else of consequence reaches here. + } catch (error: unknown) { + // An abort fired mid-body must surface as WEB_ABORTED, not be swallowed + // into a generic HTTP-error message — cancellation is not a provider + // error (the seam's cancellation contract). + if (isAbortError(error)) throw new WebError('Exa search aborted', 'WEB_ABORTED', { cause: error }) + // Otherwise: the HTTP status is already captured in `message` above; a + // malformed/non-JSON error body (normal for gateway 5xx/429s) can only + // cost a richer provider message, never the real error. } throw new WebError(message, 'WEB_PROVIDER_ERROR') } @@ -118,12 +122,18 @@ export class ExaSearchProvider implements WebSearchProvider { try { payload = await response.json() as ExaSearchResponse } catch (error: unknown) { + if (isAbortError(error)) throw new WebError('Exa search aborted', 'WEB_ABORTED', { cause: error }) throw new WebError(`Exa returned an unparseable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) } return mapExaResponse(request.query, payload) } } +/** True when `baseURL` parses as an absolute URL (a cheap local config check). */ +function isValidBaseUrl(baseURL: string): boolean { + return URL.canParse(baseURL) +} + /** True for a fetch/`AbortSignal` abort, surfaced as `WEB_ABORTED`. */ function isAbortError(error: unknown): boolean { return error instanceof DOMException && error.name === 'AbortError' diff --git a/packages/web/web-search-exa/tests/exa.spec.ts b/packages/web/web-search-exa/tests/exa.spec.ts index 3fdd878180..403198401e 100644 --- a/packages/web/web-search-exa/tests/exa.spec.ts +++ b/packages/web/web-search-exa/tests/exa.spec.ts @@ -71,6 +71,11 @@ describe('ExaSearchProvider status', () => { it('is available with a key', () => { expect(new ExaSearchProvider(options).status()).toEqual({ available: true }) }) + + it('is misconfigured when the base URL is unparseable', () => { + expect(new ExaSearchProvider({ apiKey: 'exa-key', baseURL: 'not a url' }).status()) + .toEqual({ available: false, reason: 'misconfigured' }) + }) }) describe('ExaSearchProvider request mapping', () => { @@ -142,6 +147,20 @@ describe('ExaSearchProvider error handling', () => { await expect(new ExaSearchProvider(options).search({ query: 'q' })) .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) }) + + it('surfaces an abort during success-body parse as WEB_ABORTED, not provider error', async () => { + const body = { json: () => Promise.reject(new DOMException('aborted', 'AbortError')), ok: true, status: 200 } + vi.stubGlobal('fetch', vi.fn(async () => body as unknown as Response)) + await expect(new ExaSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_ABORTED' })) + }) + + it('surfaces an abort during error-body parse as WEB_ABORTED', async () => { + const body = { json: () => Promise.reject(new DOMException('aborted', 'AbortError')), ok: false, status: 500 } + vi.stubGlobal('fetch', vi.fn(async () => body as unknown as Response)) + await expect(new ExaSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_ABORTED' })) + }) }) describe('web-search-exa plugin registration', () => { diff --git a/packages/web/web-search-perplexity/README.md b/packages/web/web-search-perplexity/README.md index 3850e5e9c1..e7093a1133 100644 --- a/packages/web/web-search-perplexity/README.md +++ b/packages/web/web-search-perplexity/README.md @@ -9,7 +9,7 @@ This is an **implementation** package: it registers a provider into `ctx.web`, i | Key | Default | Meaning | |---|---|---| | `apiKey` | `$PERPLEXITY_API_KEY` | Perplexity API key. Empty/absent → provider `status()` reports `missing-credential`. | -| `baseURL` | `https://api.perplexity.ai` | Endpoint base; `/chat/completions` is appended. | +| `baseURL` | `https://api.perplexity.ai` | Endpoint base; `/chat/completions` is appended. An unparseable value makes `status()` report `misconfigured`. | | `model` | `sonar` | Search model name. | ```yaml diff --git a/packages/web/web-search-perplexity/src/provider.ts b/packages/web/web-search-perplexity/src/provider.ts index 2b596414dd..69b4f794dd 100644 --- a/packages/web/web-search-perplexity/src/provider.ts +++ b/packages/web/web-search-perplexity/src/provider.ts @@ -81,6 +81,7 @@ export class PerplexitySearchProvider implements WebSearchProvider { status(): WebProviderStatus { if (this.options.apiKey.length === 0) return { available: false, reason: 'missing-credential' } + if (!URL.canParse(this.options.baseURL)) return { available: false, reason: 'misconfigured' } return { available: true } } @@ -113,11 +114,14 @@ export class PerplexitySearchProvider implements WebSearchProvider { const parsed = await response.json() as PerplexityError const detail = typeof parsed.error === 'string' ? parsed.error : parsed.error?.message ?? parsed.message if (detail !== undefined && detail.length > 0) message = detail - } catch { - // The HTTP status is already captured in `message` above; a malformed or - // non-JSON error body (normal for gateway 5xx/429s) can only cost a - // richer provider message, never the real error. `response.json()` is - // the sole statement and nothing else of consequence reaches here. + } catch (error: unknown) { + // An abort fired mid-body must surface as WEB_ABORTED, not be swallowed + // into a generic HTTP-error message — cancellation is not a provider + // error (the seam's cancellation contract). + if (isAbortError(error)) throw new WebError('Perplexity search aborted', 'WEB_ABORTED', { cause: error }) + // Otherwise: the HTTP status is already captured in `message` above; a + // malformed/non-JSON error body (normal for gateway 5xx/429s) can only + // cost a richer provider message, never the real error. } throw new WebError(message, 'WEB_PROVIDER_ERROR') } @@ -126,6 +130,7 @@ export class PerplexitySearchProvider implements WebSearchProvider { try { payload = await response.json() as PerplexityResponse } catch (error: unknown) { + if (isAbortError(error)) throw new WebError('Perplexity search aborted', 'WEB_ABORTED', { cause: error }) throw new WebError(`Perplexity returned an unparseable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) } return mapPerplexityResponse(request.query, payload) diff --git a/packages/web/web-search-perplexity/tests/perplexity.spec.ts b/packages/web/web-search-perplexity/tests/perplexity.spec.ts index 16557af888..c1f76a63fb 100644 --- a/packages/web/web-search-perplexity/tests/perplexity.spec.ts +++ b/packages/web/web-search-perplexity/tests/perplexity.spec.ts @@ -75,6 +75,11 @@ describe('PerplexitySearchProvider status', () => { it('is available with a key', () => { expect(new PerplexitySearchProvider(options).status()).toEqual({ available: true }) }) + + it('is misconfigured when the base URL is unparseable', () => { + expect(new PerplexitySearchProvider({ ...options, baseURL: 'not a url' }).status()) + .toEqual({ available: false, reason: 'misconfigured' }) + }) }) describe('PerplexitySearchProvider request mapping', () => { @@ -135,6 +140,20 @@ describe('PerplexitySearchProvider error handling', () => { .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) }) + it('surfaces an abort during success-body parse as WEB_ABORTED, not provider error', async () => { + const body = { json: () => Promise.reject(new DOMException('aborted', 'AbortError')), ok: true, status: 200 } + vi.stubGlobal('fetch', vi.fn(async () => body as unknown as Response)) + await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_ABORTED' })) + }) + + it('surfaces an abort during error-body parse as WEB_ABORTED', async () => { + const body = { json: () => Promise.reject(new DOMException('aborted', 'AbortError')), ok: false, status: 500 } + vi.stubGlobal('fetch', vi.fn(async () => body as unknown as Response)) + await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_ABORTED' })) + }) + it('maps a network failure to WEB_PROVIDER_ERROR', async () => { vi.stubGlobal('fetch', vi.fn(() => Promise.reject(new TypeError('connection refused')))) await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) From a1624530ee776c47413047527990e79a3bbb51bb Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 25 Jun 2026 16:32:39 +0800 Subject: [PATCH 05/16] fix: address codex review round 3 Resource-lifecycle and error-classification fixes in the local fetch provider: - Classify a timeout that fires DURING the body read as WEB_FETCH_TIMEOUT, not WEB_ABORTED: thread the controller signal into the body-read translate path and recover the timeout WebError from signal.reason, honoring the public WEB_FETCH_TIMEOUT contract for a stalled response body. - Cancel the response body before every blocked-redirect throw path (cross-origin, invalid target, missing Location), so a rejected redirect with a large or streaming body does not leak the socket after the tool returns WEB_REDIRECT_BLOCKED. - Cancel the body when charset validation fails, matching the unsupported-content-type and over-size paths (the round-1 charset check threw before readCapped owned the stream). --- packages/web/web-fetch-local/src/provider.ts | 73 +++++++++++++------ .../web-fetch-local/tests/fetch-local.spec.ts | 57 ++++++++++++++- 2 files changed, 108 insertions(+), 22 deletions(-) diff --git a/packages/web/web-fetch-local/src/provider.ts b/packages/web/web-fetch-local/src/provider.ts index c8b99d08b0..13ae7af708 100644 --- a/packages/web/web-fetch-local/src/provider.ts +++ b/packages/web/web-fetch-local/src/provider.ts @@ -71,7 +71,7 @@ export class LocalFetchProvider implements WebFetchProvider { const timer = setTimeout(() => { controller.abort(new WebError('web fetch timed out', 'WEB_FETCH_TIMEOUT')) }, timeoutMs) try { - return await this.followAndRead(request.url, controller, timeoutMs) + return await this.followAndRead(request.url, controller) } finally { clearTimeout(timer) if (exec?.signal !== undefined) exec.signal.removeEventListener('abort', onAbort) @@ -79,41 +79,50 @@ export class LocalFetchProvider implements WebFetchProvider { } /** Follow same-origin redirects up to the hop cap, then read the final response. */ - private async followAndRead(initialUrl: string, controller: AbortController, timeoutMs: number): Promise { + private async followAndRead(initialUrl: string, controller: AbortController): Promise { let currentUrl = validateFetchUrl(initialUrl, this.limits.maxUrlLength) for (let hop = 0; hop <= this.limits.maxRedirects; hop++) { - const response = await this.requestOnce(currentUrl, controller, timeoutMs) + const response = await this.requestOnce(currentUrl, controller) if (isRedirectStatus(response.status)) { const location = response.headers.get('location') if (location === null) { - // A redirect status with no Location is not a usable resource. + // A redirect status with no Location is not a usable resource. Cancel + // the (possibly streaming) body before throwing so no socket leaks. + await response.body?.cancel() throw new WebError(`redirect response (HTTP ${response.status}) without a Location header`, 'WEB_PROVIDER_ERROR') } const target = resolveRedirect(location, currentUrl) // Re-validate the target against the same transport hygiene a direct // request gets: a redirect must not be a back door to a credentialed, - // non-http(s), or over-long URL that validateFetchUrl would reject. - const validatedTarget = validateFetchUrl(target.toString(), this.limits.maxUrlLength) - if (!isSameOrigin(validatedTarget, currentUrl)) { - throw new WebError( - `cross-origin redirect to ${validatedTarget.origin} is not followed automatically; retry against that URL directly`, - 'WEB_REDIRECT_BLOCKED', - ) + // non-http(s), or over-long URL that validateFetchUrl would reject. A + // rejection here must still cancel the body first (see below). + let validatedTarget: URL + try { + validatedTarget = validateFetchUrl(target.toString(), this.limits.maxUrlLength) + if (!isSameOrigin(validatedTarget, currentUrl)) { + throw new WebError( + `cross-origin redirect to ${validatedTarget.origin} is not followed automatically; retry against that URL directly`, + 'WEB_REDIRECT_BLOCKED', + ) + } + } catch (error: unknown) { + await response.body?.cancel() + throw error } await response.body?.cancel() currentUrl = validatedTarget continue } - return await this.readBody(response, currentUrl) + return await this.readBody(response, currentUrl, controller.signal) } throw new WebError(`exceeded the maximum of ${this.limits.maxRedirects} redirects`, 'WEB_REDIRECT_BLOCKED') } - private async requestOnce(url: URL, controller: AbortController, _timeoutMs: number): Promise { + private async requestOnce(url: URL, controller: AbortController): Promise { try { return await fetch(url, { method: 'GET', @@ -122,12 +131,12 @@ export class LocalFetchProvider implements WebFetchProvider { signal: controller.signal, }) } catch (error: unknown) { - throw translateAbortOrNetwork(error) + throw translateAbortOrNetwork(error, controller.signal) } } /** Read, byte-cap, classify, and decode the final response body. */ - private async readBody(response: Response, finalUrl: URL): Promise { + private async readBody(response: Response, finalUrl: URL, signal: AbortSignal): Promise { const contentType = response.headers.get('content-type') const kind = classifyContentType(contentType) if (kind === undefined) { @@ -136,9 +145,16 @@ export class LocalFetchProvider implements WebFetchProvider { } // Resolve the decoder BEFORE reading the body so an unsupported charset - // fails without consuming the stream. - const decoder = decoderForCharset(parseCharset(contentType)) - const { bytes, truncatedByBytes } = await this.readCapped(response) + // fails without consuming the stream — but cancel the body on that failure + // so the socket does not leak (matching the unsupported-content-type path). + let decoder: TextDecoder + try { + decoder = decoderForCharset(parseCharset(contentType)) + } catch (error: unknown) { + await response.body?.cancel() + throw error + } + const { bytes, truncatedByBytes } = await this.readCapped(response, signal) const decoded = decoder.decode(bytes) const truncatedByChars = decoded.length > this.limits.maxBodyChars const content = truncatedByChars ? decoded.slice(0, this.limits.maxBodyChars) : decoded @@ -159,7 +175,7 @@ export class LocalFetchProvider implements WebFetchProvider { * past the cap is cut short (`truncatedByBytes`) rather than rejected, so a * server that under-reports still yields a bounded usable body. */ - private async readCapped(response: Response): Promise<{ bytes: Uint8Array; truncatedByBytes: boolean }> { + private async readCapped(response: Response, signal: AbortSignal): Promise<{ bytes: Uint8Array; truncatedByBytes: boolean }> { const declared = response.headers.get('content-length') if (declared !== null) { const length = Number(declared) @@ -195,7 +211,7 @@ export class LocalFetchProvider implements WebFetchProvider { } } catch (error: unknown) { /* v8 ignore next -- mid-stream read fault needs a network drop after headers; translate path covered by request-phase tests. */ - throw translateAbortOrNetwork(error) + throw translateAbortOrNetwork(error, signal) } finally { /* v8 ignore next 4 -- cancel() after a completed/broken read settles without rejecting; unobserved best-effort cleanup. */ await reader.cancel().catch(() => { @@ -235,9 +251,24 @@ function resolveRedirect(location: string, base: URL): URL { * already-typed `WebError` pass through; an `AbortError` becomes `WEB_ABORTED`; * anything else is a transport/network failure (`WEB_PROVIDER_ERROR`). */ -function translateAbortOrNetwork(error: unknown): WebError { +/** + * Translate a thrown fetch/stream error into a `WebError`. Our own + * `WEB_FETCH_TIMEOUT` (passed to `controller.abort(reason)`) and any other + * already-typed `WebError` pass through; an `AbortError` becomes `WEB_ABORTED`, + * UNLESS the abort was our timeout — the body-read reader surfaces a generic + * `AbortError` rather than the abort reason, so we recover the timeout's + * `WebError` from `signal.reason`; anything else is a transport/network failure + * (`WEB_PROVIDER_ERROR`). + */ +function translateAbortOrNetwork(error: unknown, signal?: AbortSignal): WebError { if (error instanceof WebError) return error if (error instanceof DOMException && error.name === 'AbortError') { + // A timeout abort carries its WebError as the signal reason; honor the + // WEB_FETCH_TIMEOUT contract instead of reporting a generic cancellation. + // (Node rejects WITH the reason — the WebError branch above — so this only + // fires on a runtime that surfaces a bare AbortError while reason is set.) + /* v8 ignore next */ + if (signal?.reason instanceof WebError) return signal.reason return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error }) } return new WebError(`web fetch failed: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) diff --git a/packages/web/web-fetch-local/tests/fetch-local.spec.ts b/packages/web/web-fetch-local/tests/fetch-local.spec.ts index 54e8de20c0..75a7cb1580 100644 --- a/packages/web/web-fetch-local/tests/fetch-local.spec.ts +++ b/packages/web/web-fetch-local/tests/fetch-local.spec.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' import { AddressInfo } from 'node:net' import { Context } from 'cordis' @@ -32,6 +32,7 @@ beforeEach(async () => { }) afterEach(async () => { + vi.unstubAllGlobals() await new Promise(resolve => server.close(() => { resolve() })) }) @@ -250,6 +251,20 @@ describe('LocalFetchProvider invalid URLs and abort', () => { .rejects.toThrow(expect.objectContaining({ code: 'WEB_FETCH_TIMEOUT' })) }) + it('classifies a timeout DURING the body read as WEB_FETCH_TIMEOUT, not WEB_ABORTED', async () => { + // Promise body that resolves headers (so fetch() returns) but a content-length + // that outlasts the bytes sent, so readCapped()'s reader awaits more and the + // timeout fires mid-read — the reader then surfaces a generic AbortError that + // must still be recovered as the timeout reason via signal.reason. + handler = (_req, res) => { + res.writeHead(200, { 'content-type': 'text/plain', 'content-length': '100' }) + res.write('partial') + // never send the remaining bytes nor end the response + } + await expect(provider({ timeoutMs: 80 }).fetch({ url: base })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_FETCH_TIMEOUT' })) + }) + it('maps a connection failure to WEB_PROVIDER_ERROR', async () => { // Port 1 on loopback is not listening: a real connection failure (not abort). await expect(provider().fetch({ url: 'http://127.0.0.1:1/' })) @@ -263,6 +278,46 @@ describe('LocalFetchProvider invalid URLs and abort', () => { }) }) +describe('LocalFetchProvider body cancellation on error paths', () => { + /** A fake Response whose body.cancel is observable. */ + type FakeInit = { status: number; headers: Record; location?: string } + function fakeResponse(init: FakeInit): { response: Response; cancelled: () => boolean } { + let cancelled = false + const headers = new Headers(init.headers) + if (init.location !== undefined) headers.set('location', init.location) + const response = { + status: init.status, + headers, + body: { cancel: () => { cancelled = true; return Promise.resolve() } }, + } as unknown as Response + return { response, cancelled: () => cancelled } + } + + it('cancels the body when a cross-origin redirect is blocked', async () => { + const { response, cancelled } = fakeResponse({ status: 302, headers: {}, location: 'https://elsewhere.test/' }) + vi.stubGlobal('fetch', vi.fn(async () => response)) + await expect(provider().fetch({ url: 'http://127.0.0.1:9/' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_REDIRECT_BLOCKED' })) + expect(cancelled()).toBe(true) + }) + + it('cancels the body when an unsupported charset is rejected', async () => { + const { response, cancelled } = fakeResponse({ status: 200, headers: { 'content-type': 'text/plain; charset=not-a-charset' } }) + vi.stubGlobal('fetch', vi.fn(async () => response)) + await expect(provider().fetch({ url: 'http://127.0.0.1:9/' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_UNSUPPORTED_CONTENT_TYPE' })) + expect(cancelled()).toBe(true) + }) + + it('cancels the body when a redirect has no Location header', async () => { + const { response, cancelled } = fakeResponse({ status: 302, headers: {} }) + vi.stubGlobal('fetch', vi.fn(async () => response)) + await expect(provider().fetch({ url: 'http://127.0.0.1:9/' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + expect(cancelled()).toBe(true) + }) +}) + describe('web-fetch-local plugin registration', () => { it('registers the provider into ctx.web (HMR-safe)', async () => { const ctx = new Context() From 1cd3a454da07ec028b0e98015868b88bec3d46d2 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 25 Jun 2026 18:00:07 +0800 Subject: [PATCH 06/16] fix: remove stale duplicate JSDoc on translateAbortOrNetwork The function carried two consecutive JSDoc blocks; the first was an outdated short version missing the timeout-recovery contract. Keep only the accurate detailed block. --- packages/web/web-fetch-local/src/provider.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/web/web-fetch-local/src/provider.ts b/packages/web/web-fetch-local/src/provider.ts index 13ae7af708..061eaf5e0c 100644 --- a/packages/web/web-fetch-local/src/provider.ts +++ b/packages/web/web-fetch-local/src/provider.ts @@ -245,12 +245,6 @@ function resolveRedirect(location: string, base: URL): URL { } } -/** - * Translate a thrown fetch/stream error into a `WebError`. Our own - * `WEB_FETCH_TIMEOUT` (passed to `controller.abort(reason)`) and any other - * already-typed `WebError` pass through; an `AbortError` becomes `WEB_ABORTED`; - * anything else is a transport/network failure (`WEB_PROVIDER_ERROR`). - */ /** * Translate a thrown fetch/stream error into a `WebError`. Our own * `WEB_FETCH_TIMEOUT` (passed to `controller.abort(reason)`) and any other From caef2529052d72b672b66665e0a864c185fbd4fc Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Fri, 26 Jun 2026 19:18:46 +0800 Subject: [PATCH 07/16] chore: adapt web seam to master's tsconfig + regenerate catalogs Register the five web packages in the root tsconfig.json project graph (master's typecheck moved to `tsc -b tsconfig.json` and dropped the separate tsconfig.typecheck.json), and regenerate the module graph and cordis catalog so they reflect the web packages on master. --- docs/cordis-catalog/events-and-services.md | 40 ++++++++++++++++++++-- docs/module-graph.md | 13 +++++++ tsconfig.json | 5 +++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/docs/cordis-catalog/events-and-services.md b/docs/cordis-catalog/events-and-services.md index 84ee2a893f..8134bd6546 100644 --- a/docs/cordis-catalog/events-and-services.md +++ b/docs/cordis-catalog/events-and-services.md @@ -11,7 +11,7 @@ The **harness tier** below (the `@deepseek-ai/dsh-*` packages) is the vocabulary ## Events -Dispatch modes: **emit** (fire-and-forget), **waterfall** (each listener gets `next()` and may transform or veto — see [waterfall semantics](../architecture.md#cordis-waterfall-semantics-important)), **parallel** (awaited fan-out, no veto). The harness declares 24 events across 6 scopes. +Dispatch modes: **emit** (fire-and-forget), **waterfall** (each listener gets `next()` and may transform or veto — see [waterfall semantics](../architecture.md#cordis-waterfall-semantics-important)), **parallel** (awaited fan-out, no veto). The harness declares 25 events across 7 scopes. ### `agent/*` @@ -299,9 +299,21 @@ Types: [ToolExecution](../core-data-structures/tools.md) · [ToolExecutionResult Source: [`packages/core/tools/src/index.ts:43`](../../packages/core/tools/src/index.ts) +### `web/*` + +#### `web/providers-change` — emit + +Fired after the provider registry changes — a search or fetch provider was registered or disposed. Carries no payload and no capability graph: it means only "the provider registry changed; observers may recompute status from `ctx.web`". `searchStatus()` / `fetchStatus()` stay derived, not stored. + +```ts cordis-catalog +'web/providers-change'(this: WebService): void +``` + +Source: [`packages/web/web/src/index.ts:66`](../../packages/web/web/src/index.ts) + ## Services -The 10 `ctx.` services the harness provides. An abstract seam (e.g. `ctx.bash`) is implemented by a separate package; the interface is what consumers code against. +The 11 `ctx.` services the harness provides. An abstract seam (e.g. `ctx.bash`) is implemented by a separate package; the interface is what consumers code against. ### `ctx.agentLoop` — `AgentLoop` @@ -472,6 +484,30 @@ Types: [ToolDefinition](../core-data-structures/tools.md) · [ToolExecution](../ Source: [`packages/core/tools/src/index.ts:277`](../../packages/core/tools/src/index.ts) +### `ctx.web` — `WebService` + +The web access service. Registered as `ctx.web` (one instance per context). + +Selection semantics (identical for status and execution, never order- dependent): + +- A configured id that is registered and `status().available` → that provider. +- A configured id not registered → `configured-missing` / `WEB_PROVIDER_CONFIGURED_MISSING`. +- A configured id registered but unavailable → `configured-unavailable` / `WEB_PROVIDER_CONFIGURED_UNAVAILABLE`. +- No id configured, exactly one registered usable provider → that provider. +- No id configured, multiple usable providers → `ambiguous` / `WEB_PROVIDER_AMBIGUOUS`. +- No id configured, no usable provider → `none` / `WEB_PROVIDER_UNAVAILABLE`. + +```ts cordis-catalog +registerSearchProvider(provider: WebSearchProvider): () => void +registerFetchProvider(provider: WebFetchProvider): () => void +searchStatus(): WebCapabilityStatus +fetchStatus(): WebCapabilityStatus +async search(request: WebSearchRequest, exec?: WebExecContext): Promise +async fetch(request: WebFetchRequest, exec?: WebExecContext): Promise +``` + +Source: [`packages/web/web/src/index.ts:106`](../../packages/web/web/src/index.ts) + ## Inherited tier (cordis core + loader/hmr/timer) The framework surface every plugin inherits, beyond the harness vocabulary above. This is pinned vendor source ([vendoring policy](../../vendor/README.md)); it is summarized here so the catalog is a complete picture of what `ctx` and the event bus offer, without elevating framework internals to the harness tier's prominence. diff --git a/docs/module-graph.md b/docs/module-graph.md index 346ef157fe..a17680cb6d 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -15,6 +15,7 @@ graph TD session --> brand session --> llm system-prompt --> llm + web --> llm agent --> brand agent --> llm agent --> session @@ -23,6 +24,9 @@ graph TD llm-replay --> llm llm-replay --> session session-persistence --> session + web-fetch-local --> web + web-search-exa --> web + web-search-perplexity --> web invariants --> agent invariants --> llm invariants --> session @@ -54,6 +58,10 @@ graph TD tool-bash --> bash tool-bash --> llm tool-bash --> tools + tool-web --> llm + tool-web --> system-prompt + tool-web --> tools + tool-web --> web agent-core --> agent agent-core --> agent-loop agent-core --> invariants @@ -102,10 +110,14 @@ graph TD | `llm-pi-ai` | `llm` | | `session` | `brand`, `llm` | | `system-prompt` | `llm` | +| `web` | `llm` | | `agent` | `brand`, `llm`, `session` | | `compact` | `llm`, `session` | | `llm-replay` | `llm`, `session` | | `session-persistence` | `session` | +| `web-fetch-local` | `web` | +| `web-search-exa` | `web` | +| `web-search-perplexity` | `web` | | `invariants` | `agent`, `llm`, `session` | | `session-persistence-jsonl` | `session`, `session-persistence` | | `session-persistence-sqlite` | `session`, `session-persistence` | @@ -115,6 +127,7 @@ graph TD | `agent-loop` | `agent`, `llm`, `session`, `session-persistence`, `system-prompt`, `tools` | | `subagent` | `agent`, `llm`, `tools` | | `tool-bash` | `agent`, `bash`, `llm`, `tools` | +| `tool-web` | `llm`, `system-prompt`, `tools`, `web` | | `agent-core` | `agent`, `agent-loop`, `invariants`, `llm`, `session`, `system-prompt`, `tool-bash`, `tools` | | `subagent-acp` | `agent`, `llm`, `subagent` | | `subagent-inprocess` | `agent`, `llm`, `session`, `subagent` | diff --git a/tsconfig.json b/tsconfig.json index 81f52357d5..120ad5a4fb 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -38,6 +38,11 @@ { "path": "./packages/bash/bash-local" }, { "path": "./packages/bash/tool-bash" }, { "path": "./packages/compact/compact" }, + { "path": "./packages/web/web" }, + { "path": "./packages/web/web-search-exa" }, + { "path": "./packages/web/web-search-perplexity" }, + { "path": "./packages/web/web-fetch-local" }, + { "path": "./packages/web/tool-web" }, { "path": "./packages/support/invariants" }, { "path": "./packages/ui/acp" }, { "path": "./packages/ui/acp-agent" }, From f843ea7701f8361e255cd38b0ce06eaa61ecc657 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Fri, 26 Jun 2026 19:30:08 +0800 Subject: [PATCH 08/16] fix: align web packages with master's two-stage build layout The web packages were authored against the old single-stage layout where tsc emitted directly to lib/. Master compiles declarations to lib/types/ via tsc -b, then bundles JS into lib/ via tsdown. Point every web package's tsc outDir at lib/types, update package.json types/exports/files to the lib/types declaration + lib/ bundle shape (matching dsh-bash/dsh-tool-bash), and bundle tool-web's subpath entries from lib/types/*.js rather than src. --- packages/web/tool-web/package.json | 23 +++++++++++++++---- packages/web/tool-web/tsconfig.json | 2 +- packages/web/tool-web/tsdown.config.ts | 7 +++--- packages/web/web-fetch-local/package.json | 8 ++++--- packages/web/web-fetch-local/tsconfig.json | 2 +- packages/web/web-search-exa/package.json | 8 ++++--- packages/web/web-search-exa/tsconfig.json | 2 +- .../web/web-search-perplexity/package.json | 8 ++++--- .../web/web-search-perplexity/tsconfig.json | 2 +- packages/web/web/package.json | 8 ++++--- packages/web/web/tsconfig.json | 2 +- 11 files changed, 47 insertions(+), 25 deletions(-) diff --git a/packages/web/tool-web/package.json b/packages/web/tool-web/package.json index 8d46faa157..c31c685e45 100644 --- a/packages/web/tool-web/package.json +++ b/packages/web/tool-web/package.json @@ -5,16 +5,29 @@ "private": true, "type": "module", "main": "lib/index.js", - "types": "lib/index.d.ts", + "types": "lib/types/index.d.ts", "exports": { - ".": { "types": "./lib/index.d.ts", "default": "./lib/index.js" }, - "./search": { "types": "./lib/search.d.ts", "default": "./lib/search.js" }, - "./fetch": { "types": "./lib/fetch.d.ts", "default": "./lib/fetch.js" }, + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./search": { + "types": "./lib/types/search.d.ts", + "default": "./lib/search.js" + }, + "./fetch": { + "types": "./lib/types/fetch.d.ts", + "default": "./lib/fetch.js" + }, "./src/*": "./src/*", "./package.json": "./package.json" }, "files": [ - "lib", + "lib/index.js", + "lib/search.js", + "lib/fetch.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", "src" ], "license": "BSD-3-Clause", diff --git a/packages/web/tool-web/tsconfig.json b/packages/web/tool-web/tsconfig.json index b4121a6c14..463a18dee9 100644 --- a/packages/web/tool-web/tsconfig.json +++ b/packages/web/tool-web/tsconfig.json @@ -2,7 +2,7 @@ "extends": "../../../tsconfig.base.json", "compilerOptions": { "rootDir": "src", - "outDir": "lib" + "outDir": "lib/types" }, "include": ["src"], "references": [ diff --git a/packages/web/tool-web/tsdown.config.ts b/packages/web/tool-web/tsdown.config.ts index c4849939db..0f75095d18 100644 --- a/packages/web/tool-web/tsdown.config.ts +++ b/packages/web/tool-web/tsdown.config.ts @@ -3,11 +3,12 @@ import { defineConfig } from 'tsdown' /** * tool-web exposes one package root plus one entry per tool plugin, so each tool * can be loaded or replaced independently as a subpath plugin - * (`@deepseek-ai/dsh-tool-web/search`, `/fetch`). The root tsdown config only - * auto-discovers `src/index.ts`, so the subpath entries are declared here. + * (`@deepseek-ai/dsh-tool-web/search`, `/fetch`). The root tsdown builds only + * `lib/types/index.js`, so this override adds the subpath entries. Declarations + * come from `tsc -b` (dts: false), matching every package. */ export default defineConfig({ - entry: ['src/index.ts', 'src/search.ts', 'src/fetch.ts'], + entry: ['lib/types/index.js', 'lib/types/search.js', 'lib/types/fetch.js'], outDir: 'lib', format: ['esm'], platform: 'node', diff --git a/packages/web/web-fetch-local/package.json b/packages/web/web-fetch-local/package.json index 7249697ec3..8d9a599a52 100644 --- a/packages/web/web-fetch-local/package.json +++ b/packages/web/web-fetch-local/package.json @@ -5,17 +5,19 @@ "private": true, "type": "module", "main": "lib/index.js", - "types": "lib/index.d.ts", + "types": "lib/types/index.d.ts", "exports": { ".": { - "types": "./lib/index.d.ts", + "types": "./lib/types/index.d.ts", "default": "./lib/index.js" }, "./src/*": "./src/*", "./package.json": "./package.json" }, "files": [ - "lib", + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", "src" ], "license": "BSD-3-Clause", diff --git a/packages/web/web-fetch-local/tsconfig.json b/packages/web/web-fetch-local/tsconfig.json index cb9eb44552..aa7c949fec 100644 --- a/packages/web/web-fetch-local/tsconfig.json +++ b/packages/web/web-fetch-local/tsconfig.json @@ -2,7 +2,7 @@ "extends": "../../../tsconfig.base.json", "compilerOptions": { "rootDir": "src", - "outDir": "lib" + "outDir": "lib/types" }, "include": [ "src" diff --git a/packages/web/web-search-exa/package.json b/packages/web/web-search-exa/package.json index fe79af8ba8..a111daa287 100644 --- a/packages/web/web-search-exa/package.json +++ b/packages/web/web-search-exa/package.json @@ -5,17 +5,19 @@ "private": true, "type": "module", "main": "lib/index.js", - "types": "lib/index.d.ts", + "types": "lib/types/index.d.ts", "exports": { ".": { - "types": "./lib/index.d.ts", + "types": "./lib/types/index.d.ts", "default": "./lib/index.js" }, "./src/*": "./src/*", "./package.json": "./package.json" }, "files": [ - "lib", + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", "src" ], "license": "BSD-3-Clause", diff --git a/packages/web/web-search-exa/tsconfig.json b/packages/web/web-search-exa/tsconfig.json index cb9eb44552..aa7c949fec 100644 --- a/packages/web/web-search-exa/tsconfig.json +++ b/packages/web/web-search-exa/tsconfig.json @@ -2,7 +2,7 @@ "extends": "../../../tsconfig.base.json", "compilerOptions": { "rootDir": "src", - "outDir": "lib" + "outDir": "lib/types" }, "include": [ "src" diff --git a/packages/web/web-search-perplexity/package.json b/packages/web/web-search-perplexity/package.json index 1d6229eae4..fde44ddd16 100644 --- a/packages/web/web-search-perplexity/package.json +++ b/packages/web/web-search-perplexity/package.json @@ -5,17 +5,19 @@ "private": true, "type": "module", "main": "lib/index.js", - "types": "lib/index.d.ts", + "types": "lib/types/index.d.ts", "exports": { ".": { - "types": "./lib/index.d.ts", + "types": "./lib/types/index.d.ts", "default": "./lib/index.js" }, "./src/*": "./src/*", "./package.json": "./package.json" }, "files": [ - "lib", + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", "src" ], "license": "BSD-3-Clause", diff --git a/packages/web/web-search-perplexity/tsconfig.json b/packages/web/web-search-perplexity/tsconfig.json index cb9eb44552..aa7c949fec 100644 --- a/packages/web/web-search-perplexity/tsconfig.json +++ b/packages/web/web-search-perplexity/tsconfig.json @@ -2,7 +2,7 @@ "extends": "../../../tsconfig.base.json", "compilerOptions": { "rootDir": "src", - "outDir": "lib" + "outDir": "lib/types" }, "include": [ "src" diff --git a/packages/web/web/package.json b/packages/web/web/package.json index 40ac10b715..8c68c58203 100644 --- a/packages/web/web/package.json +++ b/packages/web/web/package.json @@ -5,17 +5,19 @@ "private": true, "type": "module", "main": "lib/index.js", - "types": "lib/index.d.ts", + "types": "lib/types/index.d.ts", "exports": { ".": { - "types": "./lib/index.d.ts", + "types": "./lib/types/index.d.ts", "default": "./lib/index.js" }, "./src/*": "./src/*", "./package.json": "./package.json" }, "files": [ - "lib", + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", "src" ], "license": "BSD-3-Clause", diff --git a/packages/web/web/tsconfig.json b/packages/web/web/tsconfig.json index b187cddf35..e9de391ba1 100644 --- a/packages/web/web/tsconfig.json +++ b/packages/web/web/tsconfig.json @@ -2,7 +2,7 @@ "extends": "../../../tsconfig.base.json", "compilerOptions": { "rootDir": "src", - "outDir": "lib" + "outDir": "lib/types" }, "include": [ "src" From 70a8b57738d3c3e573b083b07549719a657a8b52 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Sun, 28 Jun 2026 17:02:06 +0800 Subject: [PATCH 09/16] fix: drop tool-web subpath exports, align with tool-bash single-entry shape The web tool package exposed ./search and ./fetch as standalone subpath plugins, but nothing consumed them, the RFC never called for them, and the sibling dsh-tool-bash (also a multi-tool consumer) ships a single entry and selects tools via config. The extra entries also tripped the workspace constraints gate, whose expected `files` list covers single-entry and bin packages but not a non-bin multi-entry one. Collapse to a single `.` entry: drop the ./search|./fetch exports and their lib/*.js from package.json files, delete the per-package tsdown override (the root config's lib/types/index.js entry now suffices), and remove the plugin-shaped name/inject exports from search.ts/fetch.ts (renaming each apply to its applyWeb{Search,Fetch}Tool helper, still composed by the root plugin and re-exported from the index). Selective enablement stays via the existing { search?, fetch? } config. Docs updated to match. --- packages/web/tool-web/README.md | 2 +- packages/web/tool-web/package.json | 10 ---------- packages/web/tool-web/src/fetch.ts | 13 +------------ packages/web/tool-web/src/index.ts | 3 +-- packages/web/tool-web/src/search.ts | 13 +------------ packages/web/tool-web/tsdown.config.ts | 19 ------------------- 6 files changed, 4 insertions(+), 56 deletions(-) delete mode 100644 packages/web/tool-web/tsdown.config.ts diff --git a/packages/web/tool-web/README.md b/packages/web/tool-web/README.md index 762bfe0189..f57f38d0d5 100644 --- a/packages/web/tool-web/README.md +++ b/packages/web/tool-web/README.md @@ -2,7 +2,7 @@ The model-facing web tool suite — `web_search` and `web_fetch` — over the [web capability seam](../web/README.md) (`ctx.web`). It owns model-facing concerns only: tool names, JSON schemas, snake_case argument names, prompt sections, the result-count bound, result formatting, HTML→markdown presentation, and `presentCall`. All web access goes through `ctx.web`; this package never imports a concrete provider. -Each tool is also a subpath plugin (`@deepseek-ai/dsh-tool-web/search`, `/fetch`) for focused deployments. +Each tool is registered independently; a product that wants only one disables the other via config (`{ search: false }` / `{ fetch: false }`). ## Tools diff --git a/packages/web/tool-web/package.json b/packages/web/tool-web/package.json index c31c685e45..8c22afa9a8 100644 --- a/packages/web/tool-web/package.json +++ b/packages/web/tool-web/package.json @@ -11,21 +11,11 @@ "types": "./lib/types/index.d.ts", "default": "./lib/index.js" }, - "./search": { - "types": "./lib/types/search.d.ts", - "default": "./lib/search.js" - }, - "./fetch": { - "types": "./lib/types/fetch.d.ts", - "default": "./lib/fetch.js" - }, "./src/*": "./src/*", "./package.json": "./package.json" }, "files": [ "lib/index.js", - "lib/search.js", - "lib/fetch.js", "lib/types/**/*.d.ts", "lib/types/**/*.d.ts.map", "src" diff --git a/packages/web/tool-web/src/fetch.ts b/packages/web/tool-web/src/fetch.ts index a48ad41414..85977fbea4 100644 --- a/packages/web/tool-web/src/fetch.ts +++ b/packages/web/tool-web/src/fetch.ts @@ -3,8 +3,6 @@ * Execution goes through `ctx.web` — this module owns the model-facing schema, * argument validation, and PRESENTATION (HTML→markdown, truncation formatting), * while the fetch provider owns safe retrieval (transport, redirects, caps). - * - * @module @deepseek-ai/dsh-tool-web/fetch */ import type { Context } from 'cordis' @@ -51,7 +49,7 @@ export function presentFetchCall(args: { url: string; timeout_ms?: number }): To } /** Register the `web_fetch` tool and its system-prompt guidance. */ -export function apply(ctx: Context): void { +export function applyWebFetchTool(ctx: Context): void { ctx.systemPrompt.section({ name: 'tool:web_fetch', order: 111, @@ -76,12 +74,3 @@ export function apply(ctx: Context): void { presentCall: presentFetchCall, })) } - -/** Cordis plugin name used by loader diagnostics. */ -export const name = 'web-fetch' - -/** Services required by the `web_fetch` tool plugin. */ -export const inject = ['tools', 'web', 'systemPrompt'] - -/** Named helper for direct registration in the root plugin and tests. */ -export const applyWebFetchTool = apply diff --git a/packages/web/tool-web/src/index.ts b/packages/web/tool-web/src/index.ts index 031d7fe4cb..072fc6018e 100644 --- a/packages/web/tool-web/src/index.ts +++ b/packages/web/tool-web/src/index.ts @@ -1,8 +1,7 @@ /** * The model-facing web tool suite (`web_search`, `web_fetch`) over the `ctx.web` * seam. This root plugin registers the tools the product has ENABLED, composing - * the per-tool registration helpers; each tool is also exposed as a subpath - * plugin (`@deepseek-ai/dsh-tool-web/search`, `/fetch`) for focused deployments. + * the per-tool registration helpers (`applyWebSearchTool`, `applyWebFetchTool`). * * The package owns model-facing concerns only — tool names, JSON schemas, * argument validation, prompt sections, result-cap constants, result formatting, diff --git a/packages/web/tool-web/src/search.ts b/packages/web/tool-web/src/search.ts index 6ed9991903..2aa93ef10e 100644 --- a/packages/web/tool-web/src/search.ts +++ b/packages/web/tool-web/src/search.ts @@ -3,8 +3,6 @@ * Execution goes through `ctx.web` — this module owns only the model-facing * schema, argument validation, the result-count bound, and result formatting, * never provider selection or network access. - * - * @module @deepseek-ai/dsh-tool-web/search */ import type { Context } from 'cordis' @@ -70,7 +68,7 @@ export function presentSearchCall(args: { query: string }): ToolCallPresentation } /** Register the `web_search` tool and its system-prompt guidance. */ -export function apply(ctx: Context): void { +export function applyWebSearchTool(ctx: Context): void { ctx.systemPrompt.section({ name: 'tool:web_search', order: 110, @@ -94,12 +92,3 @@ export function apply(ctx: Context): void { presentCall: presentSearchCall, })) } - -/** Cordis plugin name used by loader diagnostics. */ -export const name = 'web-search' - -/** Services required by the `web_search` tool plugin. */ -export const inject = ['tools', 'web', 'systemPrompt'] - -/** Named helper for direct registration in the root plugin and tests. */ -export const applyWebSearchTool = apply diff --git a/packages/web/tool-web/tsdown.config.ts b/packages/web/tool-web/tsdown.config.ts deleted file mode 100644 index 0f75095d18..0000000000 --- a/packages/web/tool-web/tsdown.config.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { defineConfig } from 'tsdown' - -/** - * tool-web exposes one package root plus one entry per tool plugin, so each tool - * can be loaded or replaced independently as a subpath plugin - * (`@deepseek-ai/dsh-tool-web/search`, `/fetch`). The root tsdown builds only - * `lib/types/index.js`, so this override adds the subpath entries. Declarations - * come from `tsc -b` (dts: false), matching every package. - */ -export default defineConfig({ - entry: ['lib/types/index.js', 'lib/types/search.js', 'lib/types/fetch.js'], - outDir: 'lib', - format: ['esm'], - platform: 'node', - target: 'es2024', - fixedExtension: false, - dts: false, - clean: false, -}) From b92a3c531a19710052a2a145e560a49e107b0d0e Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Mon, 29 Jun 2026 15:21:12 +0800 Subject: [PATCH 10/16] feat(web): add DeepSeek-backed web search provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add @deepseek-ai/dsh-web-search-deepseek: a WebSearchProvider that calls DeepSeek's Anthropic-compatible Messages API with the native web_search_20250305 server tool and parses the structured web_search_tool_result blocks into the ctx.web seam's WebSearchResult. - Namespace plugin (inject: ['web']), no default export — registers into ctx.web like dsh-llm-deepseek registers into ctx.llm. - Strict mode: a response with no web_search_tool_result block throws WEB_PROVIDER_ERROR rather than scraping URLs from model prose. - Reuses $DEEPSEEK_API_KEY; baseURL defaults to the Anthropic-compatible base (api.deepseek.com/anthropic/v1) and does NOT reuse $DEEPSEEK_BASE_URL, which belongs to the chat-completions LLM adapter. - snippet joined from text-block citations; sources deduped by url. - Two-stage build layout (outDir lib/types) matching the other web packages; registered in tsconfig.json, tsconfig.build.json, knip.json, and docs/module-graph.md. --- docs/module-graph.md | 2 + knip.json | 4 + packages/web/web-search-deepseek/README.md | 36 ++ packages/web/web-search-deepseek/package.json | 35 ++ packages/web/web-search-deepseek/src/index.ts | 81 +++++ .../web/web-search-deepseek/src/provider.ts | 217 ++++++++++++ packages/web/web-search-deepseek/src/types.ts | 58 ++++ .../web-search-deepseek/tests/deepseek.e2e.ts | 36 ++ .../tests/deepseek.spec.ts | 326 ++++++++++++++++++ .../web/web-search-deepseek/tsconfig.json | 24 ++ pnpm-lock.yaml | 13 + tsconfig.build.json | 1 + tsconfig.json | 1 + 13 files changed, 834 insertions(+) create mode 100644 packages/web/web-search-deepseek/README.md create mode 100644 packages/web/web-search-deepseek/package.json create mode 100644 packages/web/web-search-deepseek/src/index.ts create mode 100644 packages/web/web-search-deepseek/src/provider.ts create mode 100644 packages/web/web-search-deepseek/src/types.ts create mode 100644 packages/web/web-search-deepseek/tests/deepseek.e2e.ts create mode 100644 packages/web/web-search-deepseek/tests/deepseek.spec.ts create mode 100644 packages/web/web-search-deepseek/tsconfig.json diff --git a/docs/module-graph.md b/docs/module-graph.md index a17680cb6d..cddfabee14 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -25,6 +25,7 @@ graph TD llm-replay --> session session-persistence --> session web-fetch-local --> web + web-search-deepseek --> web web-search-exa --> web web-search-perplexity --> web invariants --> agent @@ -116,6 +117,7 @@ graph TD | `llm-replay` | `llm`, `session` | | `session-persistence` | `session` | | `web-fetch-local` | `web` | +| `web-search-deepseek` | `web` | | `web-search-exa` | `web` | | `web-search-perplexity` | `web` | | `invariants` | `agent`, `llm`, `session` | diff --git a/knip.json b/knip.json index 3f0a56097c..f0b8e44705 100644 --- a/knip.json +++ b/knip.json @@ -37,6 +37,10 @@ "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], "project": ["src/**/*.ts", "tests/**/*.ts"] }, + "packages/web/web-search-deepseek": { + "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], + "project": ["src/**/*.ts", "tests/**/*.ts"] + }, "packages/ui/acp-agent": { "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], "project": ["src/**/*.ts", "tests/**/*.ts"] diff --git a/packages/web/web-search-deepseek/README.md b/packages/web/web-search-deepseek/README.md new file mode 100644 index 0000000000..5b56601bbb --- /dev/null +++ b/packages/web/web-search-deepseek/README.md @@ -0,0 +1,36 @@ +# @deepseek-ai/dsh-web-search-deepseek + +A [DeepSeek](https://deepseek.com)-backed `WebSearchProvider` for the harness [web capability seam](../web/README.md) (`ctx.web`). It calls DeepSeek's **Anthropic-compatible Messages API** (`POST {baseURL}/messages`) with the native `web_search_20250305` server tool enabled, and maps the structured `web_search_tool_result` blocks DeepSeek returns into the seam's normalized `WebSearchResult`. + +This is an **implementation** package: it registers a provider into `ctx.web`, it does not own the key and it does not register a model-facing tool. Like `@deepseek-ai/dsh-llm-deepseek`, it is a function/namespace plugin (`inject: ['web']`). The Anthropic wire shape is a provider-private detail — it does **not** make this provider depend on `ctx.llm`. + +## How it differs from a dedicated search endpoint + +Exa and Perplexity expose dedicated search endpoints; DeepSeek does not. Instead this provider issues a **full Messages model call** carrying the `web_search` server tool, so one search costs a complete model turn in latency and tokens — heavier than a pure retrieval endpoint. DeepSeek runs the search server-side and returns **structured** `web_search_tool_result` blocks; the provider parses those blocks and **never scrapes URLs out of model prose**. + +**Strict mode**: if the response carries no `web_search_tool_result` block (native search did not trigger), the provider throws `WebError` `WEB_PROVIDER_ERROR` rather than degrading to prose-scraping — honest and debuggable. + +It reuses `$DEEPSEEK_API_KEY` (no new secret) but **not** `$DEEPSEEK_BASE_URL`: the search endpoint is the Anthropic-compatible base (`https://api.deepseek.com/anthropic/v1`), distinct from the chat-completions base (`https://api.deepseek.com`) the LLM adapter uses. + +## Config + +| Key | Default | Meaning | +|---|---|---| +| `apiKey` | `$DEEPSEEK_API_KEY` | DeepSeek API key. Empty/absent → provider `status()` reports `missing-credential`. Sent as both `x-api-key` and `Authorization: Bearer` (official vs Anthropic-compatible proxy). | +| `baseURL` | `https://api.deepseek.com/anthropic/v1` | Anthropic-compatible endpoint base; `/messages` is appended. Use a separate env var such as `$DEEPSEEK_SEARCH_BASE_URL` when overriding it; do not reuse `$DEEPSEEK_BASE_URL`, which belongs to the chat-completions LLM adapter. An unparseable value makes `status()` report `misconfigured`. | +| `model` | `deepseek-v4-flash` | Anthropic-format model name. | +| `apiVersion` | `2023-06-01` | `anthropic-version` header value. | +| `maxTokens` | `4096` | Upper bound on generated tokens for the Messages request. | +| `maxUses` | `5` | Maximum `web_search` server-tool uses per request. | + +```yaml +- id: web-search-deepseek + name: '@deepseek-ai/dsh-web-search-deepseek' + config: + apiKey: !!js process.env.DEEPSEEK_API_KEY + baseURL: !!js process.env.DEEPSEEK_SEARCH_BASE_URL +``` + +## Mapping + +DeepSeek returns no provider-generated answer surface this provider trusts as `content`, so `content` is omitted. `sources[]` is built from the `web_search_result` items inside `web_search_tool_result` blocks: `url` ← `url`, `title` ← `title`, `publishedAt` ← `page_age`. The per-source `snippet` lives separately in a `text` block's `citations[]` (a `cited_text` keyed by `url`), so the provider joins the two — a result with no citation excerpt simply has no `snippet`. Results are deduped by `url` (a `maxUses > 1` request can surface the same URL across searches). DeepSeek's `web_search` has no result-count knob (only `maxUses`), so `maxResults` is enforced by the seam (truncating `sources[]` and setting `truncated`). Provider failures surface as `WebError` `WEB_PROVIDER_ERROR`; an aborted request surfaces as `WEB_ABORTED`. diff --git a/packages/web/web-search-deepseek/package.json b/packages/web/web-search-deepseek/package.json new file mode 100644 index 0000000000..617e9f2768 --- /dev/null +++ b/packages/web/web-search-deepseek/package.json @@ -0,0 +1,35 @@ +{ + "name": "@deepseek-ai/dsh-web-search-deepseek", + "description": "DeepSeek-backed search provider (native web_search via the Anthropic-compatible API) for the DeepSeek Harness web capability seam (ctx.web)", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "@deepseek-ai/dsh-web": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "dependencies": { + "schemastery": "^3.18.0" + }, + "devDependencies": { + "@deepseek-ai/dsh-web": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/web/web-search-deepseek/src/index.ts b/packages/web/web-search-deepseek/src/index.ts new file mode 100644 index 0000000000..9a04955dd9 --- /dev/null +++ b/packages/web/web-search-deepseek/src/index.ts @@ -0,0 +1,81 @@ +/** + * `@deepseek-ai/dsh-web-search-deepseek`: registers a DeepSeek-backed + * `WebSearchProvider` with `ctx.web`. A function/namespace plugin (NOT a + * default-export service): it registers INTO the seam's provider registry, like + * `@deepseek-ai/dsh-llm-deepseek` registers an adapter into `ctx.llm`. + * + * The provider talks to DeepSeek's Anthropic-compatible Messages API with the + * native `web_search_20250305` server tool. It reuses `$DEEPSEEK_API_KEY` (no + * new secret) but NOT `$DEEPSEEK_BASE_URL` — the search endpoint is the + * Anthropic-compatible base, distinct from the chat-completions base the LLM + * adapter uses. + * + * @module @deepseek-ai/dsh-web-search-deepseek + */ + +import type { Context } from 'cordis' +import z from 'schemastery' +import type {} from '@deepseek-ai/dsh-web' +import { + DeepSeekSearchProvider, + DEEPSEEK_DEFAULT_API_VERSION, + DEEPSEEK_DEFAULT_BASE_URL, + DEEPSEEK_DEFAULT_MAX_TOKENS, + DEEPSEEK_DEFAULT_MAX_USES, + DEEPSEEK_DEFAULT_MODEL, +} from './provider.ts' + +export { + DeepSeekSearchProvider, + DEEPSEEK_DEFAULT_API_VERSION, + DEEPSEEK_DEFAULT_BASE_URL, + DEEPSEEK_DEFAULT_MAX_TOKENS, + DEEPSEEK_DEFAULT_MAX_USES, + DEEPSEEK_DEFAULT_MODEL, + DEEPSEEK_PROVIDER_ID, + citationSnippets, + mapAnthropicResponse, +} from './provider.ts' +export type { DeepSeekSearchProviderOptions } from './provider.ts' + +/** Cordis plugin name used by loader diagnostics. */ +export const name = 'web-search-deepseek' + +/** The web seam this provider registers into. */ +export const inject = ['web'] + +export interface Config { + /** DeepSeek API key. Falls back to `$DEEPSEEK_API_KEY`. Empty → unavailable. */ + apiKey?: string + /** Anthropic-compatible endpoint base; `/messages` is appended. */ + baseURL?: string + /** Anthropic-format model name. Defaults to `deepseek-v4-flash`. */ + model?: string + /** `anthropic-version` header value. Defaults to `2023-06-01`. */ + apiVersion?: string + /** Upper bound on generated tokens for the Messages request. Defaults to 4096. */ + maxTokens?: number + /** Maximum `web_search` server-tool uses per request. Defaults to 5. */ + maxUses?: number +} + +export const Config: z = z.object({ + apiKey: z.string(), + baseURL: z.string(), + model: z.string(), + apiVersion: z.string(), + maxTokens: z.natural(), + maxUses: z.natural(), +}) + +/** Register the DeepSeek search provider with `ctx.web`. */ +export function apply(ctx: Context, config: Config): void { + ctx.web.registerSearchProvider(new DeepSeekSearchProvider({ + apiKey: config.apiKey ?? process.env.DEEPSEEK_API_KEY ?? '', + baseURL: config.baseURL ?? DEEPSEEK_DEFAULT_BASE_URL, + model: config.model ?? DEEPSEEK_DEFAULT_MODEL, + apiVersion: config.apiVersion ?? DEEPSEEK_DEFAULT_API_VERSION, + maxTokens: config.maxTokens ?? DEEPSEEK_DEFAULT_MAX_TOKENS, + maxUses: config.maxUses ?? DEEPSEEK_DEFAULT_MAX_USES, + })) +} diff --git a/packages/web/web-search-deepseek/src/provider.ts b/packages/web/web-search-deepseek/src/provider.ts new file mode 100644 index 0000000000..5d02ad01ab --- /dev/null +++ b/packages/web/web-search-deepseek/src/provider.ts @@ -0,0 +1,217 @@ +/** + * `DeepSeekSearchProvider`: a `WebSearchProvider` backed by DeepSeek's + * Anthropic-compatible Messages API with the native `web_search_20250305` server + * tool enabled. + * + * Unlike a dedicated search endpoint (Exa's `POST /search`, Perplexity's + * `/chat/completions`), this issues a FULL Messages model call carrying a server + * tool, so a search costs a complete model turn in latency and tokens. In return + * DeepSeek runs the search server-side and returns STRUCTURED + * `web_search_tool_result` blocks — this provider parses those blocks and never + * scrapes URLs out of model prose. Strict mode: if the response carries no + * `web_search_tool_result` block (native search did not trigger), it throws + * `WEB_PROVIDER_ERROR` rather than degrading to prose-scraping. + * + * Network requests use platform-native `fetch` (Node 24), mirroring + * `@deepseek-ai/dsh-llm-deepseek`'s adapter — not a cordis HTTP-client service. + * The Anthropic wire shape is a provider-private detail and does NOT make this + * provider depend on `ctx.llm`. + * + * @module @deepseek-ai/dsh-web-search-deepseek/provider + */ + +import { WebError } from '@deepseek-ai/dsh-web' +import type { + WebProviderStatus, + WebSearchProvider, + WebSearchRequest, + WebSearchResult, + WebSearchSource, +} from '@deepseek-ai/dsh-web' +import type { + AnthropicError, + AnthropicResponse, + ContentBlock, + TextBlock, + WebSearchToolResultBlock, +} from './types.ts' + +/** Stable id this provider registers under. */ +export const DEEPSEEK_PROVIDER_ID = 'deepseek' + +/** + * Default endpoint: DeepSeek's Anthropic-compatible surface, `/v1` included + * (`/messages` is appended). This is NOT the chat-completions base + * (`https://api.deepseek.com`) `@deepseek-ai/dsh-llm-deepseek` uses, so this + * provider does NOT reuse `$DEEPSEEK_BASE_URL` — only the API key is shared. + */ +export const DEEPSEEK_DEFAULT_BASE_URL = 'https://api.deepseek.com/anthropic/v1' + +/** Default Anthropic-format model name (aligned with the repo's DeepSeek model vocabulary). */ +export const DEEPSEEK_DEFAULT_MODEL = 'deepseek-v4-flash' + +/** Default `anthropic-version` header value. */ +export const DEEPSEEK_DEFAULT_API_VERSION = '2023-06-01' + +/** Default upper bound on generated tokens for the Messages request. */ +export const DEEPSEEK_DEFAULT_MAX_TOKENS = 4096 + +/** Default maximum `web_search` server-tool uses per request. */ +export const DEEPSEEK_DEFAULT_MAX_USES = 5 + +/** Attribution header sent on every request. Bump with the package version. */ +const USER_AGENT = 'deepseek-harness/0.0.1' + +export interface DeepSeekSearchProviderOptions { + /** DeepSeek API key. Empty/absent → `status()` reports `missing-credential`. */ + apiKey: string + /** Endpoint base; `/messages` is appended. */ + baseURL: string + /** Anthropic-format model name. */ + model: string + /** `anthropic-version` header value. */ + apiVersion: string + /** Upper bound on generated tokens for the Messages request. */ + maxTokens: number + /** Maximum `web_search` server-tool uses per request. */ + maxUses: number +} + +/** + * Build a `url → cited_text` map from every `text` block's `citations[]`. This + * is the snippet surface: Anthropic `web_search_result` items carry + * `url`/`title`/`page_age` but typically NO inline snippet — the excerpt lives + * in a separate `text` block's citation, keyed by `url` (first occurrence wins). + */ +export function citationSnippets(blocks: readonly ContentBlock[]): Map { + const map = new Map() + for (const block of blocks) { + if (block.type !== 'text') continue + for (const cite of (block as TextBlock).citations ?? []) { + if (cite.url != null && cite.url.length > 0 && cite.cited_text != null && cite.cited_text.length > 0 && !map.has(cite.url)) { + map.set(cite.url, cite.cited_text) + } + } + } + return map +} + +/** + * Map a DeepSeek Anthropic Messages response to a normalized search result. + * Walks `web_search_tool_result` blocks for citeable `web_search_result` items, + * joins each to its citation excerpt as `snippet`, and dedupes by `url` (a + * `max_uses > 1` request can surface the same URL across searches). The seam + * owns the final `maxResults` truncation, so `truncated` is always `false` here. + * + * Throws `WEB_PROVIDER_ERROR` (strict mode) when no `web_search_tool_result` + * block is present — native search did not trigger, and prose-scraping is not a + * fallback. + */ +export function mapAnthropicResponse(query: string, response: AnthropicResponse): WebSearchResult { + const blocks = response.content ?? [] + const resultBlocks = blocks.filter( + (block): block is WebSearchToolResultBlock => block.type === 'web_search_tool_result', + ) + if (resultBlocks.length === 0) { + throw new WebError( + 'DeepSeek returned no web_search_tool_result blocks; the request may not have triggered native web search', + 'WEB_PROVIDER_ERROR', + ) + } + + const snippets = citationSnippets(blocks) + const seen = new Set() + const sources: WebSearchSource[] = [] + for (const block of resultBlocks) { + for (const item of block.content ?? []) { + if (item.type !== 'web_search_result' || item.url.length === 0 || seen.has(item.url)) continue + seen.add(item.url) + const snippet = snippets.get(item.url) + sources.push({ + url: item.url, + ...item.title != null && item.title.length > 0 ? { title: item.title } : {}, + ...snippet != null && snippet.length > 0 ? { snippet } : {}, + ...item.page_age != null && item.page_age.length > 0 ? { publishedAt: item.page_age } : {}, + }) + } + } + return { providerId: DEEPSEEK_PROVIDER_ID, query, sources, truncated: false } +} + +/** The DeepSeek-backed search provider. */ +export class DeepSeekSearchProvider implements WebSearchProvider { + readonly id = DEEPSEEK_PROVIDER_ID + + constructor(private readonly options: DeepSeekSearchProviderOptions) {} + + status(): WebProviderStatus { + if (this.options.apiKey.length === 0) return { available: false, reason: 'missing-credential' } + if (!URL.canParse(this.options.baseURL)) return { available: false, reason: 'misconfigured' } + return { available: true } + } + + async search(request: WebSearchRequest, exec?: { readonly signal?: AbortSignal }): Promise { + let response: Response + try { + response = await fetch(`${this.options.baseURL}/messages`, { + method: 'POST', + headers: { + // Official DeepSeek expects `x-api-key`; an Anthropic-compatible proxy + // may expect `Authorization: Bearer` — send both so either resolves. + 'x-api-key': this.options.apiKey, + 'authorization': `Bearer ${this.options.apiKey}`, + 'anthropic-version': this.options.apiVersion, + 'content-type': 'application/json', + 'accept': 'application/json', + 'user-agent': USER_AGENT, + }, + body: JSON.stringify({ + model: this.options.model, + max_tokens: this.options.maxTokens, + messages: [{ + role: 'user', + content: [{ type: 'text', text: `Perform a web search for the query: ${request.query}` }], + }], + tools: [{ type: 'web_search_20250305', name: 'web_search', max_uses: this.options.maxUses }], + }), + ...exec?.signal ? { signal: exec.signal } : {}, + }) + } catch (error: unknown) { + if (isAbortError(error)) throw new WebError('DeepSeek search aborted', 'WEB_ABORTED', { cause: error }) + throw new WebError(`DeepSeek search request failed: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) + } + + if (!response.ok) { + const status = response.status + let message = `DeepSeek API error (HTTP ${status})` + try { + const parsed = await response.json() as AnthropicError + const detail = typeof parsed.error === 'string' ? parsed.error : parsed.error?.message ?? parsed.message + if (detail !== undefined && detail.length > 0) message = detail + } catch (error: unknown) { + // An abort fired mid-body must surface as WEB_ABORTED, not be swallowed + // into a generic HTTP-error message — cancellation is not a provider + // error (the seam's cancellation contract). + if (isAbortError(error)) throw new WebError('DeepSeek search aborted', 'WEB_ABORTED', { cause: error }) + // Otherwise: the HTTP status is already captured in `message` above; a + // malformed/non-JSON error body (normal for gateway 5xx/429s) can only + // cost a richer provider message, never the real error. + } + throw new WebError(message, 'WEB_PROVIDER_ERROR') + } + + let payload: AnthropicResponse + try { + payload = await response.json() as AnthropicResponse + } catch (error: unknown) { + if (isAbortError(error)) throw new WebError('DeepSeek search aborted', 'WEB_ABORTED', { cause: error }) + throw new WebError(`DeepSeek returned an unparseable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) + } + return mapAnthropicResponse(request.query, payload) + } +} + +/** True for a fetch/`AbortSignal` abort, surfaced as `WEB_ABORTED`. */ +function isAbortError(error: unknown): boolean { + return error instanceof DOMException && error.name === 'AbortError' +} diff --git a/packages/web/web-search-deepseek/src/types.ts b/packages/web/web-search-deepseek/src/types.ts new file mode 100644 index 0000000000..bd88ed9663 --- /dev/null +++ b/packages/web/web-search-deepseek/src/types.ts @@ -0,0 +1,58 @@ +/** + * Wire types for DeepSeek's Anthropic-compatible Messages API + * (`POST {baseURL}/messages`) with the native `web_search_20250305` server tool + * enabled. Types only — no runtime code. + * + * DeepSeek returns structured content blocks: `web_search_tool_result` blocks + * carry the citeable `web_search_result` items (`url`/`title`/`page_age`), while + * the snippet/excerpt for a URL lives separately in a `text` block's + * `citations[]` (a `cited_text` keyed by `url`). The provider joins the two. + * + * The Anthropic wire shape is a provider-private detail; it does not make this + * provider depend on `ctx.llm`. + * + * @module @deepseek-ai/dsh-web-search-deepseek/types + */ + +/** A `web_search_result` item inside a `web_search_tool_result` block. */ +export interface WebSearchResultItem { + type: string + url: string + title?: string | null + /** Provider-supplied page age/recency string (mapped to `publishedAt`). */ + page_age?: string | null +} + +/** A `web_search_tool_result` content block: the citeable result surface. */ +export interface WebSearchToolResultBlock { + type: 'web_search_tool_result' + content?: WebSearchResultItem[] +} + +/** One citation location inside a `text` block (the snippet surface). */ +export interface CitationLocation { + type?: string + url?: string | null + cited_text?: string | null +} + +/** A `text` content block: the model's prose plus per-URL citations. */ +export interface TextBlock { + type: 'text' + text?: string | null + citations?: CitationLocation[] +} + +/** Any content block; only `web_search_tool_result` and `text` are consumed. */ +export type ContentBlock = WebSearchToolResultBlock | TextBlock | { type: string } + +/** DeepSeek's Anthropic Messages response envelope. */ +export interface AnthropicResponse { + content?: ContentBlock[] +} + +/** DeepSeek's error response envelope (best-effort; fields vary). */ +export interface AnthropicError { + error?: { message?: string } | string + message?: string +} diff --git a/packages/web/web-search-deepseek/tests/deepseek.e2e.ts b/packages/web/web-search-deepseek/tests/deepseek.e2e.ts new file mode 100644 index 0000000000..d06e384b31 --- /dev/null +++ b/packages/web/web-search-deepseek/tests/deepseek.e2e.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest' +import { + DeepSeekSearchProvider, + DEEPSEEK_DEFAULT_API_VERSION, + DEEPSEEK_DEFAULT_BASE_URL, + DEEPSEEK_DEFAULT_MAX_TOKENS, + DEEPSEEK_DEFAULT_MAX_USES, + DEEPSEEK_DEFAULT_MODEL, +} from '@deepseek-ai/dsh-web-search-deepseek' + +/** + * Real-API smoke for the DeepSeek search provider. Self-skips without + * `$DEEPSEEK_API_KEY`, per the with-key e2e policy in AGENTS.md § Secrets. This + * is the only test that proves DeepSeek's Anthropic-compatible endpoint actually + * triggers native `web_search` and returns the structured result blocks the + * provider parses — a mock cannot confirm the wire shape is real. + */ +const apiKey = process.env.DEEPSEEK_API_KEY +const maybe = apiKey !== undefined && apiKey.length > 0 ? describe : describe.skip + +maybe('DeepSeekSearchProvider real API', () => { + it('returns citeable sources for a live query via native web_search', async () => { + const provider = new DeepSeekSearchProvider({ + apiKey: apiKey!, + baseURL: process.env.DEEPSEEK_SEARCH_BASE_URL ?? DEEPSEEK_DEFAULT_BASE_URL, + model: process.env.DEEPSEEK_SEARCH_MODEL ?? DEEPSEEK_DEFAULT_MODEL, + apiVersion: DEEPSEEK_DEFAULT_API_VERSION, + maxTokens: DEEPSEEK_DEFAULT_MAX_TOKENS, + maxUses: DEEPSEEK_DEFAULT_MAX_USES, + }) + const result = await provider.search({ query: 'What is the DeepSeek coding agent?', maxResults: 5 }) + expect(result.providerId).toBe('deepseek') + expect(result.sources.length).toBeGreaterThan(0) + for (const source of result.sources) expect(source.url).toMatch(/^https?:\/\//) + }, 60_000) +}) diff --git a/packages/web/web-search-deepseek/tests/deepseek.spec.ts b/packages/web/web-search-deepseek/tests/deepseek.spec.ts new file mode 100644 index 0000000000..ff6b37cfa2 --- /dev/null +++ b/packages/web/web-search-deepseek/tests/deepseek.spec.ts @@ -0,0 +1,326 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Context } from 'cordis' +import Loader from '@cordisjs/plugin-loader' +import WebService from '@deepseek-ai/dsh-web' +import { + DeepSeekSearchProvider, + citationSnippets, + mapAnthropicResponse, + DEEPSEEK_PROVIDER_ID, +} from '@deepseek-ai/dsh-web-search-deepseek' +import * as deepseekPlugin from '@deepseek-ai/dsh-web-search-deepseek' +import type { AnthropicResponse } from '@deepseek-ai/dsh-web-search-deepseek/src/types.ts' + +const options = { + apiKey: 'ds-key', + baseURL: 'https://api.deepseek.test/anthropic/v1', + model: 'deepseek-chat', + apiVersion: '2023-06-01', + maxTokens: 4096, + maxUses: 5, +} + +function jsonResponse(body: unknown, init: ResponseInit = {}): Response { + return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' }, ...init }) +} + +/** A response with one result block plus a text block carrying the snippet. */ +function searchResponse(): AnthropicResponse { + return { + content: [ + { type: 'text', text: 'Here is what I found.', citations: [{ type: 'web_search_result_location', url: 'https://a.test', cited_text: 'excerpt for A' }] }, + { + type: 'web_search_tool_result', + content: [ + { type: 'web_search_result', url: 'https://a.test', title: 'A', page_age: '2026-02-02' }, + { type: 'web_search_result', url: 'https://b.test', title: 'B' }, + ], + }, + ], + } +} + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('citationSnippets', () => { + it('maps url → cited_text from text blocks, first occurrence wins', () => { + const map = citationSnippets([ + { type: 'text', citations: [{ url: 'https://a.test', cited_text: 'first' }, { url: 'https://a.test', cited_text: 'second' }] }, + { type: 'text', citations: [{ url: 'https://b.test', cited_text: 'b text' }] }, + ]) + expect(map.get('https://a.test')).toBe('first') + expect(map.get('https://b.test')).toBe('b text') + }) + + it('ignores citations missing url or cited_text', () => { + const map = citationSnippets([ + { type: 'text', citations: [{ url: 'https://a.test' }, { cited_text: 'orphan' }, { url: '', cited_text: 'empty url' }] }, + ]) + expect(map.size).toBe(0) + }) +}) + +describe('mapAnthropicResponse', () => { + it('joins result items to citation snippets and maps page_age to publishedAt', () => { + const result = mapAnthropicResponse('q', searchResponse()) + expect(result).toEqual({ + providerId: DEEPSEEK_PROVIDER_ID, + query: 'q', + sources: [ + { url: 'https://a.test', title: 'A', snippet: 'excerpt for A', publishedAt: '2026-02-02' }, + { url: 'https://b.test', title: 'B' }, + ], + truncated: false, + }) + }) + + it('dedupes repeated urls across result blocks (first wins)', () => { + const result = mapAnthropicResponse('q', { + content: [ + { type: 'web_search_tool_result', content: [{ type: 'web_search_result', url: 'https://a.test', title: 'first' }] }, + { type: 'web_search_tool_result', content: [{ type: 'web_search_result', url: 'https://a.test', title: 'second' }] }, + ], + }) + expect(result.sources).toEqual([{ url: 'https://a.test', title: 'first' }]) + }) + + it('skips non-result items and items with an empty url', () => { + const result = mapAnthropicResponse('q', { + content: [{ + type: 'web_search_tool_result', + content: [ + { type: 'web_search_result_error', url: 'https://err.test' }, + { type: 'web_search_result', url: '' }, + { type: 'web_search_result', url: 'https://ok.test' }, + ], + }], + }) + expect(result.sources).toEqual([{ url: 'https://ok.test' }]) + }) + + it('omits optional fields when absent or empty', () => { + const result = mapAnthropicResponse('q', { + content: [{ type: 'web_search_tool_result', content: [{ type: 'web_search_result', url: 'https://a.test', title: '', page_age: '' }] }], + }) + expect(result.sources).toEqual([{ url: 'https://a.test' }]) + }) + + it('tolerates a text block with no citations', () => { + const result = mapAnthropicResponse('q', { + content: [ + { type: 'text', text: 'no citations here' }, + { type: 'web_search_tool_result', content: [{ type: 'web_search_result', url: 'https://a.test', title: 'A' }] }, + ], + }) + expect(result.sources).toEqual([{ url: 'https://a.test', title: 'A' }]) + }) + + it('tolerates a result block with no content array', () => { + const result = mapAnthropicResponse('q', { + content: [ + { type: 'web_search_tool_result' }, + { type: 'web_search_tool_result', content: [{ type: 'web_search_result', url: 'https://a.test' }] }, + ], + }) + expect(result.sources).toEqual([{ url: 'https://a.test' }]) + }) + + it('throws WEB_PROVIDER_ERROR (strict mode) when no result block is present', () => { + expect(() => mapAnthropicResponse('q', { content: [{ type: 'text', text: 'just prose, no search' }] })) + .toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) + + it('throws WEB_PROVIDER_ERROR when content is absent entirely', () => { + expect(() => mapAnthropicResponse('q', {})) + .toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) +}) + +describe('DeepSeekSearchProvider status', () => { + it('is unavailable without a key', () => { + expect(new DeepSeekSearchProvider({ ...options, apiKey: '' }).status()) + .toEqual({ available: false, reason: 'missing-credential' }) + }) + + it('is available with a key', () => { + expect(new DeepSeekSearchProvider(options).status()).toEqual({ available: true }) + }) + + it('is misconfigured when the base URL is unparseable', () => { + expect(new DeepSeekSearchProvider({ ...options, baseURL: 'not a url' }).status()) + .toEqual({ available: false, reason: 'misconfigured' }) + }) +}) + +describe('DeepSeekSearchProvider request mapping', () => { + it('posts an Anthropic Messages request enabling the web_search server tool', async () => { + const fetchMock = vi.fn(async () => jsonResponse(searchResponse())) + vi.stubGlobal('fetch', fetchMock) + await new DeepSeekSearchProvider(options).search({ query: 'hello' }) + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe('https://api.deepseek.test/anthropic/v1/messages') + const headers = init.headers as Record + expect(headers['x-api-key']).toBe('ds-key') + expect(headers['authorization']).toBe('Bearer ds-key') + expect(headers['anthropic-version']).toBe('2023-06-01') + expect(JSON.parse(init.body as string)).toEqual({ + model: 'deepseek-chat', + max_tokens: 4096, + messages: [{ role: 'user', content: [{ type: 'text', text: 'Perform a web search for the query: hello' }] }], + tools: [{ type: 'web_search_20250305', name: 'web_search', max_uses: 5 }], + }) + }) + + it('forwards the abort signal', async () => { + const fetchMock = vi.fn(async () => jsonResponse(searchResponse())) + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + await new DeepSeekSearchProvider(options).search({ query: 'q' }, { signal: controller.signal }) + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(init.signal).toBe(controller.signal) + }) +}) + +describe('DeepSeekSearchProvider error handling', () => { + it('maps an HTTP error to WEB_PROVIDER_ERROR with the provider message', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({ error: { message: 'rate limited' } }, { status: 429 }))) + await expect(new DeepSeekSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR', message: 'rate limited' })) + }) + + it('handles a string-form error body', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({ error: 'bad request' }, { status: 400 }))) + await expect(new DeepSeekSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ message: 'bad request' })) + }) + + it('keeps a status-line message when the error body is not JSON', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('upstream error', { status: 503 }))) + await expect(new DeepSeekSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ message: 'DeepSeek API error (HTTP 503)' })) + }) + + it('keeps the status-line message when the JSON error body carries no detail', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({}, { status: 500 }))) + await expect(new DeepSeekSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ message: 'DeepSeek API error (HTTP 500)' })) + }) + + it('maps an abort to WEB_ABORTED', async () => { + vi.stubGlobal('fetch', vi.fn(() => Promise.reject(new DOMException('aborted', 'AbortError')))) + await expect(new DeepSeekSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_ABORTED' })) + }) + + it('maps an unparseable success body to WEB_PROVIDER_ERROR', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('not json', { status: 200 }))) + await expect(new DeepSeekSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) + + it('surfaces an abort during success-body parse as WEB_ABORTED', async () => { + const body = { json: () => Promise.reject(new DOMException('aborted', 'AbortError')), ok: true, status: 200 } + vi.stubGlobal('fetch', vi.fn(async () => body as unknown as Response)) + await expect(new DeepSeekSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_ABORTED' })) + }) + + it('surfaces an abort during error-body parse as WEB_ABORTED', async () => { + const body = { json: () => Promise.reject(new DOMException('aborted', 'AbortError')), ok: false, status: 500 } + vi.stubGlobal('fetch', vi.fn(async () => body as unknown as Response)) + await expect(new DeepSeekSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_ABORTED' })) + }) + + it('maps a network failure to WEB_PROVIDER_ERROR', async () => { + vi.stubGlobal('fetch', vi.fn(() => Promise.reject(new TypeError('connection refused')))) + await expect(new DeepSeekSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) + + it('strict mode flows through search(): a prose-only response throws WEB_PROVIDER_ERROR', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({ content: [{ type: 'text', text: 'no search happened' }] }))) + await expect(new DeepSeekSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) +}) + +describe('web-search-deepseek plugin registration', () => { + it('registers the provider into ctx.web (HMR-safe)', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: DEEPSEEK_PROVIDER_ID }) + const fiber = await ctx.plugin(deepseekPlugin, { apiKey: 'ds-key' }) + expect(ctx.web.searchStatus()).toEqual({ available: true, providerId: DEEPSEEK_PROVIDER_ID }) + await fiber.dispose() + expect(ctx.web.searchStatus()).toEqual({ available: false, reason: 'configured-missing' }) + }) + + it('has no default export (namespace plugin export shape)', () => { + expect('default' in deepseekPlugin).toBe(false) + }) + + it('survives the real Loader unwrapExports path keeping name/inject/Config', () => { + // A stray `export default apply` would make the cordis Loader's + // unwrapExports (`exports.default ?? exports`) collapse the module to the + // bare `apply` function, DROPPING `inject: ['web']` — the plugin would then + // read ctx.web without injecting it and throw "cannot get property … without + // inject" the moment it loads. A hand-built ctx.plugin(namespace) mount + // bypasses unwrapExports and cannot catch that, so drive the real path. + // Prove it bites: add `export default apply` to src/index.ts, watch this go + // red, revert. + const loader = Object.create(Loader.prototype) as Loader + const unwrapped = loader.unwrapExports(deepseekPlugin) as Record + expect(unwrapped).toBe(deepseekPlugin) + expect(unwrapped.name).toBe('web-search-deepseek') + expect(unwrapped.inject).toEqual(['web']) + expect(typeof unwrapped.apply).toBe('function') + }) + + it('boots over ctx.web through the unwrapped module without an inject error', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: DEEPSEEK_PROVIDER_ID }) + const loader = Object.create(Loader.prototype) as Loader + const unwrapped = loader.unwrapExports(deepseekPlugin) as Parameters[0] + // A collapsed export shape (dropped inject) would throw "without inject" here. + const fiber = await ctx.plugin(unwrapped, { apiKey: 'ds-key' }) + expect(ctx.web.searchStatus()).toEqual({ available: true, providerId: DEEPSEEK_PROVIDER_ID }) + await fiber.dispose() + }) + + it('falls back to the env key and defaults when config omits them', async () => { + const prev = process.env.DEEPSEEK_API_KEY + process.env.DEEPSEEK_API_KEY = 'env-key' + try { + const fetchMock = vi.fn(async () => jsonResponse(searchResponse())) + vi.stubGlobal('fetch', fetchMock) + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: DEEPSEEK_PROVIDER_ID }) + const fiber = await ctx.plugin(deepseekPlugin, {}) + expect(ctx.web.searchStatus()).toEqual({ available: true, providerId: DEEPSEEK_PROVIDER_ID }) + await ctx.web.search({ query: 'q' }) + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe('https://api.deepseek.com/anthropic/v1/messages') + expect((init.headers as Record)['x-api-key']).toBe('env-key') + expect(JSON.parse(init.body as string)).toMatchObject({ model: 'deepseek-v4-flash' }) + await fiber.dispose() + } finally { + if (prev === undefined) delete process.env.DEEPSEEK_API_KEY + else process.env.DEEPSEEK_API_KEY = prev + } + }) + + it('is unavailable when neither config nor env supplies a key', async () => { + const prev = process.env.DEEPSEEK_API_KEY + delete process.env.DEEPSEEK_API_KEY + try { + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: DEEPSEEK_PROVIDER_ID }) + await ctx.plugin(deepseekPlugin, {}) + expect(ctx.web.searchStatus()).toEqual({ available: false, reason: 'configured-unavailable' }) + } finally { + if (prev !== undefined) process.env.DEEPSEEK_API_KEY = prev + } + }) +}) diff --git a/packages/web/web-search-deepseek/tsconfig.json b/packages/web/web-search-deepseek/tsconfig.json new file mode 100644 index 0000000000..aa7c949fec --- /dev/null +++ b/packages/web/web-search-deepseek/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../../vendor/cosmokit" + }, + { + "path": "../../../vendor/cordis" + }, + { + "path": "../../../vendor/schemastery" + }, + { + "path": "../web" + } + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a393b6c4b9..a4e0811228 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -763,6 +763,19 @@ importers: specifier: ^4.0.0-rc.6 version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/web/web-search-deepseek: + dependencies: + schemastery: + specifier: ^3.18.0 + version: 3.18.0 + devDependencies: + '@deepseek-ai/dsh-web': + specifier: workspace:^ + version: link:../web + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/web/web-search-exa: dependencies: schemastery: diff --git a/tsconfig.build.json b/tsconfig.build.json index fb7a058ea8..840b9aee5f 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -30,6 +30,7 @@ { "path": "./packages/web/web" }, { "path": "./packages/web/web-search-exa" }, { "path": "./packages/web/web-search-perplexity" }, + { "path": "./packages/web/web-search-deepseek" }, { "path": "./packages/web/web-fetch-local" }, { "path": "./packages/web/tool-web" }, { "path": "./packages/support/invariants" }, diff --git a/tsconfig.json b/tsconfig.json index 120ad5a4fb..ba0af75248 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -41,6 +41,7 @@ { "path": "./packages/web/web" }, { "path": "./packages/web/web-search-exa" }, { "path": "./packages/web/web-search-perplexity" }, + { "path": "./packages/web/web-search-deepseek" }, { "path": "./packages/web/web-fetch-local" }, { "path": "./packages/web/tool-web" }, { "path": "./packages/support/invariants" }, From 8395722db555c2e44d3477d46df551b621d76b33 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Mon, 29 Jun 2026 17:23:11 +0800 Subject: [PATCH 11/16] fix: address codex review findings on web seam - search providers (exa/perplexity/deepseek): map the parsed response INSIDE the parse try, so a well-formed body of the wrong shape surfaces as WEB_PROVIDER_ERROR instead of escaping as a raw TypeError; a WebError the mapper throws on purpose is re-thrown untouched - web-fetch-local: validate numeric limits at plugin construction (positive finite caps; non-negative integer maxRedirects) rather than constructing a provider with nonsensical values - web-fetch-local: enforce the redirect budget BEFORE resolving each hop, so maxRedirects:N follows exactly N redirects and an over-limit hop reports "exceeded the maximum" rather than misdiagnosing a cross-origin block - drop the stale dsh-tool-web/search and /fetch path aliases (the package no longer declares those subpath exports) - strip trailing EOF blank lines flagged by git diff --check Each fix carries a regression test. --- packages/web/tool-web/src/index.ts | 1 - .../web/tool-web/tests/integration.spec.ts | 1 - packages/web/web-fetch-local/README.md | 4 +- packages/web/web-fetch-local/src/index.ts | 20 +++++ packages/web/web-fetch-local/src/provider.ts | 15 +++- .../web-fetch-local/tests/fetch-local.spec.ts | 90 +++++++++++++++++++ .../web/web-search-deepseek/src/provider.ts | 8 +- .../tests/deepseek.spec.ts | 6 ++ packages/web/web-search-exa/README.md | 2 +- packages/web/web-search-exa/src/provider.ts | 8 +- packages/web/web-search-exa/tests/exa.spec.ts | 7 ++ .../web/web-search-perplexity/src/provider.ts | 8 +- .../tests/perplexity.spec.ts | 6 ++ tsconfig.base.json | 2 - 14 files changed, 157 insertions(+), 21 deletions(-) diff --git a/packages/web/tool-web/src/index.ts b/packages/web/tool-web/src/index.ts index 072fc6018e..df8029466a 100644 --- a/packages/web/tool-web/src/index.ts +++ b/packages/web/tool-web/src/index.ts @@ -55,4 +55,3 @@ export function apply(ctx: Context, config: Config): void { if (config.search !== false) applyWebSearchTool(ctx) if (config.fetch !== false) applyWebFetchTool(ctx) } - diff --git a/packages/web/tool-web/tests/integration.spec.ts b/packages/web/tool-web/tests/integration.spec.ts index 18604190c6..50ae6c5624 100644 --- a/packages/web/tool-web/tests/integration.spec.ts +++ b/packages/web/tool-web/tests/integration.spec.ts @@ -96,4 +96,3 @@ describe('web_search integration over the real Exa provider', () => { expect(out.content.map(b => b.text).join('')).toContain('[Result](https://result.test)') }) }) - diff --git a/packages/web/web-fetch-local/README.md b/packages/web/web-fetch-local/README.md index fd9150e46f..58db557581 100644 --- a/packages/web/web-fetch-local/README.md +++ b/packages/web/web-fetch-local/README.md @@ -26,9 +26,11 @@ The provider owns **safe resource retrieval**: URL validation, HTTP transport, r | `maxBodyChars` | `100_000` | Maximum decoded body length in characters. | | `timeoutMs` | `30_000` | Default fetch timeout. | | `maxTimeoutMs` | `120_000` | Upper bound for a per-request timeout override. | -| `maxRedirects` | `5` | Maximum same-origin redirect hops. | +| `maxRedirects` | `5` | Maximum same-origin redirect hops (`0` follows none). | | `userAgent` | `deepseek-harness/…` | `User-Agent` header. | +The numeric limits are validated at plugin construction: every cap except `maxRedirects` must be a positive finite number, and `maxRedirects` must be a non-negative integer. An invalid value throws rather than silently constructing a provider with nonsensical limits. + ## Security note SSRF / private-network protection (blocking private, loopback, link-local, multicast, and otherwise non-public destinations, with DNS-resolve-then-validate and per-hop re-validation) is **deferred** — see the [web capability seam RFC](../../../docs/rfc/implemented/architecture/2026-06-24-web-capability-seam.md). Until it lands, this provider is an SSRF primitive and **must not be enabled** in a deployment that can reach sensitive internal network targets. diff --git a/packages/web/web-fetch-local/src/index.ts b/packages/web/web-fetch-local/src/index.ts index eb3f8e4143..7eb614f39a 100644 --- a/packages/web/web-fetch-local/src/index.ts +++ b/packages/web/web-fetch-local/src/index.ts @@ -60,10 +60,30 @@ export const Config: z = z.object({ /** The shape after schemastery applies its defaults to every field. */ type ResolvedConfig = Required +/** A resource limit (byte/char/length/timeout cap) must be a positive finite number. */ +function assertPositiveFinite(name: string, value: number): void { + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`web-fetch-local: ${name} must be a positive finite number`) + } +} + +/** The redirect hop cap must be a non-negative integer (0 follows no redirects). */ +function assertNonNegativeInteger(name: string, value: number): void { + if (!Number.isInteger(value) || value < 0) { + throw new Error(`web-fetch-local: ${name} must be a non-negative integer`) + } +} + /** Register the local HTTP(S) fetch provider with `ctx.web`. */ export function apply(ctx: Context, config: Config): void { // schemastery (Config) has already filled every defaulted field. const resolved = config as ResolvedConfig + assertPositiveFinite('maxUrlLength', resolved.maxUrlLength) + assertPositiveFinite('maxResponseBytes', resolved.maxResponseBytes) + assertPositiveFinite('maxBodyChars', resolved.maxBodyChars) + assertPositiveFinite('timeoutMs', resolved.timeoutMs) + assertPositiveFinite('maxTimeoutMs', resolved.maxTimeoutMs) + assertNonNegativeInteger('maxRedirects', resolved.maxRedirects) const limits: LocalFetchLimits = { maxUrlLength: resolved.maxUrlLength, maxResponseBytes: resolved.maxResponseBytes, diff --git a/packages/web/web-fetch-local/src/provider.ts b/packages/web/web-fetch-local/src/provider.ts index 061eaf5e0c..29b183b710 100644 --- a/packages/web/web-fetch-local/src/provider.ts +++ b/packages/web/web-fetch-local/src/provider.ts @@ -81,11 +81,21 @@ export class LocalFetchProvider implements WebFetchProvider { /** Follow same-origin redirects up to the hop cap, then read the final response. */ private async followAndRead(initialUrl: string, controller: AbortController): Promise { let currentUrl = validateFetchUrl(initialUrl, this.limits.maxUrlLength) + let redirectsFollowed = 0 - for (let hop = 0; hop <= this.limits.maxRedirects; hop++) { + for (;;) { const response = await this.requestOnce(currentUrl, controller) if (isRedirectStatus(response.status)) { + // The redirect budget is enforced BEFORE this hop's target is resolved + // or origin-checked, so `maxRedirects: N` follows at most N redirects + // exactly: the (N+1)th redirect is refused as "exceeded" regardless of + // where it points (a same-origin/cross-origin distinction on a hop we + // are not allowed to follow would be the wrong diagnosis). + if (redirectsFollowed >= this.limits.maxRedirects) { + await response.body?.cancel() + throw new WebError(`exceeded the maximum of ${this.limits.maxRedirects} redirects`, 'WEB_REDIRECT_BLOCKED') + } const location = response.headers.get('location') if (location === null) { // A redirect status with no Location is not a usable resource. Cancel @@ -113,13 +123,12 @@ export class LocalFetchProvider implements WebFetchProvider { } await response.body?.cancel() currentUrl = validatedTarget + redirectsFollowed++ continue } return await this.readBody(response, currentUrl, controller.signal) } - - throw new WebError(`exceeded the maximum of ${this.limits.maxRedirects} redirects`, 'WEB_REDIRECT_BLOCKED') } private async requestOnce(url: URL, controller: AbortController): Promise { diff --git a/packages/web/web-fetch-local/tests/fetch-local.spec.ts b/packages/web/web-fetch-local/tests/fetch-local.spec.ts index 75a7cb1580..27ed991c08 100644 --- a/packages/web/web-fetch-local/tests/fetch-local.spec.ts +++ b/packages/web/web-fetch-local/tests/fetch-local.spec.ts @@ -203,6 +203,60 @@ describe('LocalFetchProvider redirects', () => { .rejects.toThrow(expect.objectContaining({ code: 'WEB_REDIRECT_BLOCKED' })) }) + it('follows exactly maxRedirects hops: a chain landing on the Nth redirect succeeds', async () => { + // maxRedirects: 2 → /?n=0 → /?n=1 → /?n=2(200). Exactly 2 redirects + 1 + // final = 3 requests; the cap is inclusive of the landing request. + let requests = 0 + handler = (req, res) => { + requests++ + const n = Number(new URL(req.url ?? '/', base).searchParams.get('n') ?? '0') + if (n >= 2) { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('landed') } + else { res.writeHead(302, { location: `/?n=${n + 1}` }); res.end() } + } + const result = await provider({ maxRedirects: 2 }).fetch({ url: `${base}/?n=0` }) + expect(result.body.content).toBe('landed') + expect(requests).toBe(3) + }) + + it('makes exactly maxRedirects+1 requests before blocking an over-long chain', async () => { + // maxRedirects: 2 on an infinite chain: requests at n=0,1,2 (the 3rd is the + // over-limit redirect, refused before its Location is followed) = 3 total. + let requests = 0 + handler = (req, res) => { + requests++ + const n = Number(new URL(req.url ?? '/', base).searchParams.get('n') ?? '0') + res.writeHead(302, { location: `/?n=${n + 1}` }) + res.end() + } + await expect(provider({ maxRedirects: 2 }).fetch({ url: `${base}/?n=0` })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_REDIRECT_BLOCKED', message: 'exceeded the maximum of 2 redirects' })) + expect(requests).toBe(3) + }) + + it('reports an over-limit redirect as "exceeded", not cross-origin, even when the over-limit hop points cross-origin', async () => { + // The redirect budget is checked BEFORE the over-limit hop's target is + // origin-validated, so the diagnosis is "exceeded", not "cross-origin". + handler = (req, res) => { + const n = Number(new URL(req.url ?? '/', base).searchParams.get('n') ?? '0') + const location = n === 0 ? '/?n=1' : 'https://example.com/' + res.writeHead(302, { location }) + res.end() + } + await expect(provider({ maxRedirects: 1 }).fetch({ url: `${base}/?n=0` })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_REDIRECT_BLOCKED', message: 'exceeded the maximum of 1 redirects' })) + }) + + it('maxRedirects: 0 follows no redirect but still fetches a direct 200', async () => { + handler = (req, res) => { + if (req.url === '/r') { res.writeHead(302, { location: '/done' }); res.end() } + else { res.writeHead(200, { 'content-type': 'text/plain' }); res.end('direct') } + } + await expect(provider({ maxRedirects: 0 }).fetch({ url: `${base}/r` })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_REDIRECT_BLOCKED' })) + const direct = await provider({ maxRedirects: 0 }).fetch({ url: `${base}/done` }) + expect(direct.body.content).toBe('direct') + }) + it('treats a redirect without a Location header as a provider error', async () => { handler = (_req, res) => { res.writeHead(302); res.end() } await expect(provider().fetch({ url: base })) @@ -331,4 +385,40 @@ describe('web-fetch-local plugin registration', () => { it('has no default export (namespace plugin export shape)', () => { expect('default' in fetchPlugin).toBe(false) }) + + it('rejects a non-positive resource limit at construction', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { fetchProvider: LOCAL_FETCH_PROVIDER_ID }) + await expect(ctx.plugin(fetchPlugin, { maxResponseBytes: -1 })) + .rejects.toThrow(/maxResponseBytes must be a positive finite number/) + }) + + it('rejects a zero timeout at construction', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { fetchProvider: LOCAL_FETCH_PROVIDER_ID }) + await expect(ctx.plugin(fetchPlugin, { timeoutMs: 0 })) + .rejects.toThrow(/timeoutMs must be a positive finite number/) + }) + + it('rejects a fractional redirect cap at construction', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { fetchProvider: LOCAL_FETCH_PROVIDER_ID }) + await expect(ctx.plugin(fetchPlugin, { maxRedirects: 1.5 })) + .rejects.toThrow(/maxRedirects must be a non-negative integer/) + }) + + it('rejects a negative redirect cap at construction', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { fetchProvider: LOCAL_FETCH_PROVIDER_ID }) + await expect(ctx.plugin(fetchPlugin, { maxRedirects: -1 })) + .rejects.toThrow(/maxRedirects must be a non-negative integer/) + }) + + it('accepts maxRedirects: 0 (follow no redirects) as valid config', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { fetchProvider: LOCAL_FETCH_PROVIDER_ID }) + const fiber = await ctx.plugin(fetchPlugin, { maxRedirects: 0 }) + expect(ctx.web.fetchStatus()).toEqual({ available: true, providerId: LOCAL_FETCH_PROVIDER_ID }) + await fiber.dispose() + }) }) diff --git a/packages/web/web-search-deepseek/src/provider.ts b/packages/web/web-search-deepseek/src/provider.ts index 5d02ad01ab..b637d52d11 100644 --- a/packages/web/web-search-deepseek/src/provider.ts +++ b/packages/web/web-search-deepseek/src/provider.ts @@ -200,14 +200,14 @@ export class DeepSeekSearchProvider implements WebSearchProvider { throw new WebError(message, 'WEB_PROVIDER_ERROR') } - let payload: AnthropicResponse try { - payload = await response.json() as AnthropicResponse + const payload = await response.json() as AnthropicResponse + return mapAnthropicResponse(request.query, payload) } catch (error: unknown) { if (isAbortError(error)) throw new WebError('DeepSeek search aborted', 'WEB_ABORTED', { cause: error }) - throw new WebError(`DeepSeek returned an unparseable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) + if (error instanceof WebError) throw error + throw new WebError(`DeepSeek returned an unprocessable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) } - return mapAnthropicResponse(request.query, payload) } } diff --git a/packages/web/web-search-deepseek/tests/deepseek.spec.ts b/packages/web/web-search-deepseek/tests/deepseek.spec.ts index ff6b37cfa2..496b7f6a3c 100644 --- a/packages/web/web-search-deepseek/tests/deepseek.spec.ts +++ b/packages/web/web-search-deepseek/tests/deepseek.spec.ts @@ -220,6 +220,12 @@ describe('DeepSeekSearchProvider error handling', () => { .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) }) + it('maps a well-formed body of the wrong shape to WEB_PROVIDER_ERROR, not a raw TypeError', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({ content: {} }, { status: 200 }))) + await expect(new DeepSeekSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) + it('surfaces an abort during success-body parse as WEB_ABORTED', async () => { const body = { json: () => Promise.reject(new DOMException('aborted', 'AbortError')), ok: true, status: 200 } vi.stubGlobal('fetch', vi.fn(async () => body as unknown as Response)) diff --git a/packages/web/web-search-exa/README.md b/packages/web/web-search-exa/README.md index 61da605df7..6485d64c60 100644 --- a/packages/web/web-search-exa/README.md +++ b/packages/web/web-search-exa/README.md @@ -20,4 +20,4 @@ This is an **implementation** package: it registers a provider into `ctx.web`, i ## Mapping -Exa returns a flat `results[]` and no generated answer, so `content` is omitted. Each result maps to a `WebSearchSource`: `url` ← `url`, `title` ← `title`, `snippet` ← the first non-empty `highlights[]` entry (a result with no highlight has no portable snippet and is dropped), `publishedAt` ← `publishedDate`. The provider passes `maxResults` through as Exa's `numResults` for a cost/latency optimization; the final bound is enforced by the seam. Provider failures (HTTP errors, network failure, unparseable bodies) surface as `WebError` `WEB_PROVIDER_ERROR`; an aborted request surfaces as `WEB_ABORTED`. +Exa returns a flat `results[]` and no generated answer, so `content` is omitted. Each result maps to a `WebSearchSource`: `url` ← `url`, `title` ← `title`, `snippet` ← the first non-empty `highlights[]` entry (a result with no highlight has no portable snippet and is dropped), `publishedAt` ← `publishedDate`. The provider passes `maxResults` through as Exa's `numResults` for a cost/latency optimization; the final bound is enforced by the seam. Provider failures (HTTP errors, network failure, unparseable or wrong-shape bodies) surface as `WebError` `WEB_PROVIDER_ERROR`; an aborted request surfaces as `WEB_ABORTED`. diff --git a/packages/web/web-search-exa/src/provider.ts b/packages/web/web-search-exa/src/provider.ts index 70e14cbf25..3774bd5d58 100644 --- a/packages/web/web-search-exa/src/provider.ts +++ b/packages/web/web-search-exa/src/provider.ts @@ -118,14 +118,14 @@ export class ExaSearchProvider implements WebSearchProvider { throw new WebError(message, 'WEB_PROVIDER_ERROR') } - let payload: ExaSearchResponse try { - payload = await response.json() as ExaSearchResponse + const payload = await response.json() as ExaSearchResponse + return mapExaResponse(request.query, payload) } catch (error: unknown) { if (isAbortError(error)) throw new WebError('Exa search aborted', 'WEB_ABORTED', { cause: error }) - throw new WebError(`Exa returned an unparseable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) + if (error instanceof WebError) throw error + throw new WebError(`Exa returned an unprocessable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) } - return mapExaResponse(request.query, payload) } } diff --git a/packages/web/web-search-exa/tests/exa.spec.ts b/packages/web/web-search-exa/tests/exa.spec.ts index 403198401e..436e542d7c 100644 --- a/packages/web/web-search-exa/tests/exa.spec.ts +++ b/packages/web/web-search-exa/tests/exa.spec.ts @@ -60,6 +60,7 @@ describe('Exa result mapping', () => { it('tolerates a missing results array', () => { expect(mapExaResponse('q', {}).sources).toEqual([]) }) + }) describe('ExaSearchProvider status', () => { @@ -148,6 +149,12 @@ describe('ExaSearchProvider error handling', () => { .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) }) + it('maps a well-formed body of the wrong shape to WEB_PROVIDER_ERROR, not a raw TypeError', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({ results: {} }, { status: 200 }))) + await expect(new ExaSearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) + it('surfaces an abort during success-body parse as WEB_ABORTED, not provider error', async () => { const body = { json: () => Promise.reject(new DOMException('aborted', 'AbortError')), ok: true, status: 200 } vi.stubGlobal('fetch', vi.fn(async () => body as unknown as Response)) diff --git a/packages/web/web-search-perplexity/src/provider.ts b/packages/web/web-search-perplexity/src/provider.ts index 69b4f794dd..809086026a 100644 --- a/packages/web/web-search-perplexity/src/provider.ts +++ b/packages/web/web-search-perplexity/src/provider.ts @@ -126,14 +126,14 @@ export class PerplexitySearchProvider implements WebSearchProvider { throw new WebError(message, 'WEB_PROVIDER_ERROR') } - let payload: PerplexityResponse try { - payload = await response.json() as PerplexityResponse + const payload = await response.json() as PerplexityResponse + return mapPerplexityResponse(request.query, payload) } catch (error: unknown) { if (isAbortError(error)) throw new WebError('Perplexity search aborted', 'WEB_ABORTED', { cause: error }) - throw new WebError(`Perplexity returned an unparseable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) + if (error instanceof WebError) throw error + throw new WebError(`Perplexity returned an unprocessable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) } - return mapPerplexityResponse(request.query, payload) } } diff --git a/packages/web/web-search-perplexity/tests/perplexity.spec.ts b/packages/web/web-search-perplexity/tests/perplexity.spec.ts index c1f76a63fb..d84c34a328 100644 --- a/packages/web/web-search-perplexity/tests/perplexity.spec.ts +++ b/packages/web/web-search-perplexity/tests/perplexity.spec.ts @@ -116,6 +116,12 @@ describe('PerplexitySearchProvider error handling', () => { .rejects.toThrow(expect.objectContaining({ message: 'bad request' })) }) + it('maps a well-formed body of the wrong shape to WEB_PROVIDER_ERROR, not a raw TypeError', async () => { + vi.stubGlobal('fetch', vi.fn(async () => jsonResponse({ search_results: null }, { status: 200 }))) + await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) + .rejects.toThrow(expect.objectContaining({ code: 'WEB_PROVIDER_ERROR' })) + }) + it('keeps a status-line message when the error body is not JSON', async () => { vi.stubGlobal('fetch', vi.fn(async () => new Response('upstream error', { status: 503 }))) await expect(new PerplexitySearchProvider(options).search({ query: 'q' })) diff --git a/tsconfig.base.json b/tsconfig.base.json index bc4f13bdd5..f0fdcfe197 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -34,8 +34,6 @@ "@cordisjs/plugin-timer": ["./vendor/timer/src"], "@cordisjs/plugin-hmr": ["./vendor/hmr/src"], "@cordisjs/plugin-logger-console": ["./vendor/logger-console/src"], - "@deepseek-ai/dsh-tool-web/search": ["./packages/web/tool-web/src/search.ts"], - "@deepseek-ai/dsh-tool-web/fetch": ["./packages/web/tool-web/src/fetch.ts"], // One wildcard maps every @deepseek-ai/dsh- to its source. Package // dir names are unique across groups, so first-on-disk-wins resolution is // unambiguous; adding a package under an existing group needs no edit From bb8f7799cef81909a1b595ae79714cef0e41c6a1 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Mon, 29 Jun 2026 17:39:52 +0800 Subject: [PATCH 12/16] fix: drop unreachable WebError rethrow in exa/perplexity search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `if (error instanceof WebError) throw error` guard is dead code in the exa and perplexity providers: their mappers (mapExaResponse / mapPerplexityResponse) never throw a WebError — a wrong-shape body throws a TypeError, which the catch correctly translates to WEB_PROVIDER_ERROR. The guard was added for symmetry with the deepseek provider, whose mapper DOES throw a WebError in strict mode (no web_search_tool_result block), so it keeps the rethrow. The unreachable lines tripped the per-file 100% coverage gate. --- packages/web/web-search-exa/src/provider.ts | 1 - packages/web/web-search-perplexity/src/provider.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/web/web-search-exa/src/provider.ts b/packages/web/web-search-exa/src/provider.ts index 3774bd5d58..cfb41cf77f 100644 --- a/packages/web/web-search-exa/src/provider.ts +++ b/packages/web/web-search-exa/src/provider.ts @@ -123,7 +123,6 @@ export class ExaSearchProvider implements WebSearchProvider { return mapExaResponse(request.query, payload) } catch (error: unknown) { if (isAbortError(error)) throw new WebError('Exa search aborted', 'WEB_ABORTED', { cause: error }) - if (error instanceof WebError) throw error throw new WebError(`Exa returned an unprocessable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) } } diff --git a/packages/web/web-search-perplexity/src/provider.ts b/packages/web/web-search-perplexity/src/provider.ts index 809086026a..5b5feb897b 100644 --- a/packages/web/web-search-perplexity/src/provider.ts +++ b/packages/web/web-search-perplexity/src/provider.ts @@ -131,7 +131,6 @@ export class PerplexitySearchProvider implements WebSearchProvider { return mapPerplexityResponse(request.query, payload) } catch (error: unknown) { if (isAbortError(error)) throw new WebError('Perplexity search aborted', 'WEB_ABORTED', { cause: error }) - if (error instanceof WebError) throw error throw new WebError(`Perplexity returned an unprocessable response body: ${String(error)}`, 'WEB_PROVIDER_ERROR', { cause: error }) } } From cf71c0b215f93903d6798cf0fd56559ee7e480cd Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:08:53 +0800 Subject: [PATCH 13/16] fix: address web seam review findings --- docs/architecture.md | 4 ++- docs/core-data-structures/web.md | 4 +-- .../2026-06-24-web-capability-seam.md | 17 +++++++---- packages/README.md | 2 ++ packages/web/README.md | 1 + packages/web/web-search-deepseek/README.md | 4 +-- packages/web/web-search-deepseek/src/index.ts | 10 ++++--- .../web/web-search-deepseek/src/provider.ts | 6 ++++ .../tests/deepseek.spec.ts | 30 +++++++++++++++++++ 9 files changed, 64 insertions(+), 14 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index d50dedf5b4..f57d453067 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -25,6 +25,8 @@ For a catalog of the **data structures** this architecture moves around — the │ @deepseek-ai/dsh-bash-local (bash impl) │ │ @deepseek-ai/dsh-tool-bash (bash tool schemas) │ │ @deepseek-ai/dsh-web-search-exa (web search impl) │ +│ @deepseek-ai/dsh-web-search-perplexity (web search impl) │ +│ @deepseek-ai/dsh-web-search-deepseek (web search impl) │ │ @deepseek-ai/dsh-web-fetch-local (web fetch impl) │ │ @deepseek-ai/dsh-tool-web (web tool schemas) │ │ @deepseek-ai/dsh-subagent-* (subagent providers) │ @@ -78,7 +80,7 @@ Swappable capabilities are split into **three packages** so each part evolves in The LLM seam has the same topology folded differently: `dsh-llm` carries the interface (`LlmAdapter`) AND the consumer surface (`ctx.llm.stream()`), with adapters as implementation packages — there the consumer is the loop itself, not a swappable schema surface. Use the full three-package split when the consumer is independently replaceable; keep interface + consumer together when they are one concern. Don't split preemptively: a capability with one conceivable implementation and one consumer stays one package until proven otherwise. -The web capability uses the same three-package split but folds two capabilities onto one seam: `dsh-web` owns the abstract `ctx.web` service, which is a provider REGISTRY (`registerSearchProvider`/`registerFetchProvider`, registration-order-independent selection, the `WebError` taxonomy) rather than a single backend. Providers register capabilities, not tools — `dsh-web-search-exa`, `dsh-web-search-perplexity`, and `dsh-web-fetch-local` each register into `ctx.web` the way an `LlmAdapter` registers into `ctx.llm`, so they are namespace plugins (`inject: ['web']`), not key-owning services. `dsh-tool-web` is the single consumer that owns the model-facing `web_search`/`web_fetch` schemas, prompt sections, and presentation; it reads only the aggregated `ctx.web.searchStatus()`/`fetchStatus()` and executes through `ctx.web.search()`/`fetch()`, so provider selection has one owner. Search and fetch are deliberately one seam (one thing to inject and configure, one selection policy, one abort/error vocabulary) despite sharing no request schema — see the [web capability seam RFC](rfc/implemented/architecture/2026-06-24-web-capability-seam.md). +The web capability uses the same three-package split but folds two capabilities onto one seam: `dsh-web` owns the abstract `ctx.web` service, which is a provider REGISTRY (`registerSearchProvider`/`registerFetchProvider`, registration-order-independent selection, the `WebError` taxonomy) rather than a single backend. Providers register capabilities, not tools — `dsh-web-search-exa`, `dsh-web-search-perplexity`, `dsh-web-search-deepseek`, and `dsh-web-fetch-local` each register into `ctx.web` the way an `LlmAdapter` registers into `ctx.llm`, so they are namespace plugins (`inject: ['web']`), not key-owning services. `dsh-tool-web` is the single consumer that owns the model-facing `web_search`/`web_fetch` schemas, prompt sections, and presentation; it reads only the aggregated `ctx.web.searchStatus()`/`fetchStatus()` and executes through `ctx.web.search()`/`fetch()`, so provider selection has one owner. Search and fetch are deliberately one seam (one thing to inject and configure, one selection policy, one abort/error vocabulary) despite sharing no request schema — see the [web capability seam RFC](rfc/implemented/architecture/2026-06-24-web-capability-seam.md). > **"Capability" — two unrelated meanings.** (1) The *seam pattern* above ("one plugin provides a capability, another needs it") is realized by plain Cordis **services + `inject`**: a provider registers a service (`ctx.bash`, declared in `interface Context`); a consumer declares `inject: ['bash']` and its fiber stays pending until the service exists, tearing down via HMR if it later vanishes. No extra library is needed. (2) `@cordisjs/plugin-capability` is a different axis entirely — a **permission/capability-security** service (named permissions with inheritance/dependency, tested against a session via `ctx.capability.test`). It is a candidate for the deferred permissions/sandbox work (the `tools/execute` veto seam), NOT a mechanism for swapping implementations. diff --git a/docs/core-data-structures/web.md b/docs/core-data-structures/web.md index f0ade276f7..43ed4e7aeb 100644 --- a/docs/core-data-structures/web.md +++ b/docs/core-data-structures/web.md @@ -1,6 +1,6 @@ # Web Access -The web access seam — a [capability seam](../rfc/implemented/architecture/2026-06-24-web-capability-seam.md) that spans **two capabilities** (search and fetch) on one `ctx.web` service, split across packages: interface ([dsh-web](../../packages/web/web), `ctx.web` + the provider registries), implementations ([dsh-web-search-exa](../../packages/web/web-search-exa), [dsh-web-search-perplexity](../../packages/web/web-search-perplexity), [dsh-web-fetch-local](../../packages/web/web-fetch-local)), and consumer ([dsh-tool-web](../../packages/web/tool-web), the `web_search`/`web_fetch` tool schemas). Web is **one optional capability**, not part of the agent-loop spine — so its vocabulary lives here, not in [core.md](core.md). A search-provider swap does not change how the model asks for a query, and a fetch-implementation swap does not change how the model asks for a URL. +The web access seam — a [capability seam](../rfc/implemented/architecture/2026-06-24-web-capability-seam.md) that spans **two capabilities** (search and fetch) on one `ctx.web` service, split across packages: interface ([dsh-web](../../packages/web/web), `ctx.web` + the provider registries), implementations ([dsh-web-search-exa](../../packages/web/web-search-exa), [dsh-web-search-perplexity](../../packages/web/web-search-perplexity), [dsh-web-search-deepseek](../../packages/web/web-search-deepseek), [dsh-web-fetch-local](../../packages/web/web-fetch-local)), and consumer ([dsh-tool-web](../../packages/web/tool-web), the `web_search`/`web_fetch` tool schemas). Web is **one optional capability**, not part of the agent-loop spine — so its vocabulary lives here, not in [core.md](core.md). A search-provider swap does not change how the model asks for a query, and a fetch-implementation swap does not change how the model asks for a URL. Source: [`packages/web/web/src/types.ts`](../../packages/web/web/src/types.ts) @@ -33,7 +33,7 @@ interface WebSearchResult { } ``` -`content` is optional provider-generated answer text (Exa returns none; Perplexity returns a generated answer). `sources[]` is the portable citation surface. A source always has a `url`; `title`/`snippet`/`publishedAt` are optional because not every provider returns them — Perplexity citations may be URL-only, and forcing adapters to invent the rest would make the seam lie. `dsh-tool-web` renders `title ?? hostname(url)`. +`content` is optional provider-generated answer text (Exa and DeepSeek return none; Perplexity returns a generated answer). `sources[]` is the portable citation surface. A source always has a `url`; `title`/`snippet`/`publishedAt` are optional because not every provider returns them — Perplexity citations may be URL-only, and forcing adapters to invent the rest would make the seam lie. `dsh-tool-web` renders `title ?? hostname(url)`. ```ts type-equiv interface WebSearchSource { diff --git a/docs/rfc/implemented/architecture/2026-06-24-web-capability-seam.md b/docs/rfc/implemented/architecture/2026-06-24-web-capability-seam.md index 3b9fb166d0..55ada9d576 100644 --- a/docs/rfc/implemented/architecture/2026-06-24-web-capability-seam.md +++ b/docs/rfc/implemented/architecture/2026-06-24-web-capability-seam.md @@ -17,7 +17,7 @@ There is also a provider-selection question. Existing `tool-bash` and `tool-fs` Introduce web access as a first-class capability seam following [the capability-seam RFC](../../implemented/architecture/2026-06-13-capability-seams.md): 1. `@deepseek-ai/dsh-web` (`packages/web/web`) owns `ctx.web`, provider registration, provider selection, shared request/result vocabulary, and web-specific errors. -2. Provider packages implement concrete backends and register capabilities with `ctx.web`, for example `@deepseek-ai/dsh-web-search-exa`, `@deepseek-ai/dsh-web-search-perplexity`, and `@deepseek-ai/dsh-web-fetch-local`. +2. Provider packages implement concrete backends and register capabilities with `ctx.web`, for example `@deepseek-ai/dsh-web-search-exa`, `@deepseek-ai/dsh-web-search-perplexity`, `@deepseek-ai/dsh-web-search-deepseek`, and `@deepseek-ai/dsh-web-fetch-local`. 3. `@deepseek-ai/dsh-tool-web` (`packages/web/tool-web`) owns the model-facing `web_search` and `web_fetch` tool schemas, prompt sections, argument validation, result formatting, and tool-owned presentation over `ctx.web`. Providers do not register tools. Providers register capabilities. `dsh-tool-web` is the only owner of model-facing names, descriptions, prompt guidance, JSON schemas, and presentation. @@ -46,6 +46,8 @@ The dependency direction mirrors bash and filesystem: consumer interface implementation <--depends on-- @deepseek-ai/dsh-web-search-perplexity implementation + <--depends on-- @deepseek-ai/dsh-web-search-deepseek + implementation <--depends on-- @deepseek-ai/dsh-web-fetch-local implementation ``` @@ -56,6 +58,7 @@ At runtime, provider packages register capabilities with `ctx.web`; `tool-web` r flowchart LR exa["@deepseek-ai/dsh-web-search-exa"] -->|registerSearchProvider| web["@deepseek-ai/dsh-web / ctx.web"] perplexity["@deepseek-ai/dsh-web-search-perplexity"] -->|registerSearchProvider| web + deepseek["@deepseek-ai/dsh-web-search-deepseek"] -->|registerSearchProvider| web fetchLocal["@deepseek-ai/dsh-web-fetch-local"] -->|registerFetchProvider| web toolWeb["@deepseek-ai/dsh-tool-web"] -->|searchStatus/fetchStatus| web toolWeb -->|ctx.tools.register| webSearch["tool: web_search"] @@ -152,6 +155,9 @@ The "single provider auto-selects" rule is for tests, demos, and simple deployme - id: web-search-perplexity name: '@deepseek-ai/dsh-web-search-perplexity' +- id: web-search-deepseek + name: '@deepseek-ai/dsh-web-search-deepseek' + - id: web-fetch-local name: '@deepseek-ai/dsh-web-fetch-local' @@ -326,10 +332,11 @@ Land the work in seam order: 1. Add `packages/web/web` with `ctx.web`, provider registration, provider status, capability status, selection, request/result/error types, and contract tests. 2. Add `packages/web/web-search-exa` with parser/unit tests and a self-skipping real-provider smoke test. 3. Add `packages/web/web-search-perplexity` with parser/unit tests and a self-skipping real-provider smoke test. -4. Add `packages/web/web-fetch-local` with local HTTP behavior tests. -5. Add `packages/web/tool-web` with config-driven tool registration, prompt sections, model formatting, presentation, and tool-registry tests. -6. Wire product app/example configs only after package behavior is stable, because tool schemas and prompt sections affect agent behavior and snapshots. -7. Update `docs/architecture.md`, `packages/README.md`, package READMEs, generated Cordis catalogs if new events/services are added, and maintenance scripts. +4. Add `packages/web/web-search-deepseek` with parser/unit tests and a self-skipping real-provider smoke test. +5. Add `packages/web/web-fetch-local` with local HTTP behavior tests. +6. Add `packages/web/tool-web` with config-driven tool registration, prompt sections, model formatting, presentation, and tool-registry tests. +7. Wire product app/example configs only after package behavior is stable, because tool schemas and prompt sections affect agent behavior and snapshots. +8. Update `docs/architecture.md`, `packages/README.md`, package READMEs, generated Cordis catalogs if new events/services are added, and maintenance scripts. ## Alternatives considered diff --git a/packages/README.md b/packages/README.md index f19cb4cf6d..5461f7b1a1 100644 --- a/packages/README.md +++ b/packages/README.md @@ -38,6 +38,7 @@ dsh-tool-bash ← dsh-bash, dsh-tools (bash tool schemas) dsh-web ← dsh-llm (abstract web seam; search/fetch registries, WebError) dsh-web-search-exa ← dsh-web (Exa WebSearchProvider) dsh-web-search-perplexity ← dsh-web (Perplexity WebSearchProvider) +dsh-web-search-deepseek ← dsh-web (DeepSeek native-web-search WebSearchProvider) dsh-web-fetch-local ← dsh-web (anonymous public HTTP(S) WebFetchProvider) dsh-tool-web ← dsh-web, dsh-tools, dsh-system-prompt (web tool schemas) dsh-llm-deepseek ← dsh-llm (DeepSeek adapter) @@ -80,6 +81,7 @@ The rule: **extension** plugins depend on interfaces, never on the concrete loop | `web/` | `web` | Abstract web seam (search/fetch provider registries + selection + vocabulary + `WebError`) | `ctx.web` | | `web-search-exa/` | `web` | Exa-backed `WebSearchProvider` | (registers on `ctx.web`) | | `web-search-perplexity/` | `web` | Perplexity-backed `WebSearchProvider` | (registers on `ctx.web`) | +| `web-search-deepseek/` | `web` | DeepSeek-backed `WebSearchProvider` using native `web_search` through the Anthropic-compatible API | (registers on `ctx.web`) | | `web-fetch-local/` | `web` | Anonymous public HTTP(S) `WebFetchProvider` | (registers on `ctx.web`) | | `tool-web/` | `web` | Model-facing `web_search`/`web_fetch` tool schemas | (registers on `ctx.tools`) | | `llm-deepseek/` | `llm` | DeepSeek API adapter (hand-rolled fetch/SSE) | (registers on `ctx.llm`) | diff --git a/packages/web/README.md b/packages/web/README.md index 0742d9c2cc..c2d34e615f 100644 --- a/packages/web/README.md +++ b/packages/web/README.md @@ -7,6 +7,7 @@ The web access capability seam: an abstract web interface, search/fetch provider | `web/` | Abstract web seam (search/fetch provider registries + selection + vocabulary + `WebError`) | `ctx.web` | | `web-search-exa/` | Exa-backed `WebSearchProvider` | (registers on `ctx.web`) | | `web-search-perplexity/` | Perplexity-backed `WebSearchProvider` | (registers on `ctx.web`) | +| `web-search-deepseek/` | DeepSeek-backed `WebSearchProvider` using native `web_search` through the Anthropic-compatible API | (registers on `ctx.web`) | | `web-fetch-local/` | Anonymous public HTTP(S) `WebFetchProvider` | (registers on `ctx.web`) | | `tool-web/` | Model-facing `web_search`/`web_fetch` tool schemas | (registers on `ctx.tools`) | diff --git a/packages/web/web-search-deepseek/README.md b/packages/web/web-search-deepseek/README.md index 5b56601bbb..41b000d26a 100644 --- a/packages/web/web-search-deepseek/README.md +++ b/packages/web/web-search-deepseek/README.md @@ -20,8 +20,8 @@ It reuses `$DEEPSEEK_API_KEY` (no new secret) but **not** `$DEEPSEEK_BASE_URL`: | `baseURL` | `https://api.deepseek.com/anthropic/v1` | Anthropic-compatible endpoint base; `/messages` is appended. Use a separate env var such as `$DEEPSEEK_SEARCH_BASE_URL` when overriding it; do not reuse `$DEEPSEEK_BASE_URL`, which belongs to the chat-completions LLM adapter. An unparseable value makes `status()` report `misconfigured`. | | `model` | `deepseek-v4-flash` | Anthropic-format model name. | | `apiVersion` | `2023-06-01` | `anthropic-version` header value. | -| `maxTokens` | `4096` | Upper bound on generated tokens for the Messages request. | -| `maxUses` | `5` | Maximum `web_search` server-tool uses per request. | +| `maxTokens` | `4096` | Positive-integer upper bound on generated tokens for the Messages request. | +| `maxUses` | `5` | Positive-integer maximum `web_search` server-tool uses per request. | ```yaml - id: web-search-deepseek diff --git a/packages/web/web-search-deepseek/src/index.ts b/packages/web/web-search-deepseek/src/index.ts index 9a04955dd9..c993fa8808 100644 --- a/packages/web/web-search-deepseek/src/index.ts +++ b/packages/web/web-search-deepseek/src/index.ts @@ -64,18 +64,20 @@ export const Config: z = z.object({ baseURL: z.string(), model: z.string(), apiVersion: z.string(), - maxTokens: z.natural(), - maxUses: z.natural(), + maxTokens: z.number().step(1).min(1), + maxUses: z.number().step(1).min(1), }) /** Register the DeepSeek search provider with `ctx.web`. */ export function apply(ctx: Context, config: Config): void { + const maxTokens = config.maxTokens ?? DEEPSEEK_DEFAULT_MAX_TOKENS + const maxUses = config.maxUses ?? DEEPSEEK_DEFAULT_MAX_USES ctx.web.registerSearchProvider(new DeepSeekSearchProvider({ apiKey: config.apiKey ?? process.env.DEEPSEEK_API_KEY ?? '', baseURL: config.baseURL ?? DEEPSEEK_DEFAULT_BASE_URL, model: config.model ?? DEEPSEEK_DEFAULT_MODEL, apiVersion: config.apiVersion ?? DEEPSEEK_DEFAULT_API_VERSION, - maxTokens: config.maxTokens ?? DEEPSEEK_DEFAULT_MAX_TOKENS, - maxUses: config.maxUses ?? DEEPSEEK_DEFAULT_MAX_USES, + maxTokens, + maxUses, })) } diff --git a/packages/web/web-search-deepseek/src/provider.ts b/packages/web/web-search-deepseek/src/provider.ts index b637d52d11..40566b4f75 100644 --- a/packages/web/web-search-deepseek/src/provider.ts +++ b/packages/web/web-search-deepseek/src/provider.ts @@ -147,6 +147,7 @@ export class DeepSeekSearchProvider implements WebSearchProvider { status(): WebProviderStatus { if (this.options.apiKey.length === 0) return { available: false, reason: 'missing-credential' } if (!URL.canParse(this.options.baseURL)) return { available: false, reason: 'misconfigured' } + if (!isPositiveInteger(this.options.maxTokens) || !isPositiveInteger(this.options.maxUses)) return { available: false, reason: 'misconfigured' } return { available: true } } @@ -215,3 +216,8 @@ export class DeepSeekSearchProvider implements WebSearchProvider { function isAbortError(error: unknown): boolean { return error instanceof DOMException && error.name === 'AbortError' } + +/** True for DeepSeek request limits that can be sent to the Messages API. */ +function isPositiveInteger(value: number): boolean { + return Number.isInteger(value) && value > 0 +} diff --git a/packages/web/web-search-deepseek/tests/deepseek.spec.ts b/packages/web/web-search-deepseek/tests/deepseek.spec.ts index 496b7f6a3c..ef688b7ad2 100644 --- a/packages/web/web-search-deepseek/tests/deepseek.spec.ts +++ b/packages/web/web-search-deepseek/tests/deepseek.spec.ts @@ -152,6 +152,15 @@ describe('DeepSeekSearchProvider status', () => { expect(new DeepSeekSearchProvider({ ...options, baseURL: 'not a url' }).status()) .toEqual({ available: false, reason: 'misconfigured' }) }) + + it('is misconfigured when request limits are not positive integers', () => { + expect(new DeepSeekSearchProvider({ ...options, maxTokens: 0 }).status()) + .toEqual({ available: false, reason: 'misconfigured' }) + expect(new DeepSeekSearchProvider({ ...options, maxUses: 0 }).status()) + .toEqual({ available: false, reason: 'misconfigured' }) + expect(new DeepSeekSearchProvider({ ...options, maxUses: 1.5 }).status()) + .toEqual({ available: false, reason: 'misconfigured' }) + }) }) describe('DeepSeekSearchProvider request mapping', () => { @@ -263,6 +272,27 @@ describe('web-search-deepseek plugin registration', () => { expect(ctx.web.searchStatus()).toEqual({ available: false, reason: 'configured-missing' }) }) + it('rejects maxTokens: 0 at plugin construction', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: DEEPSEEK_PROVIDER_ID }) + await expect(ctx.plugin(deepseekPlugin, { apiKey: 'ds-key', maxTokens: 0 })) + .rejects.toThrow(/maxTokens expected number >= 1/) + }) + + it('rejects maxUses: 0 at plugin construction', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: DEEPSEEK_PROVIDER_ID }) + await expect(ctx.plugin(deepseekPlugin, { apiKey: 'ds-key', maxUses: 0 })) + .rejects.toThrow(/maxUses expected number >= 1/) + }) + + it('rejects a fractional maxUses at plugin construction', async () => { + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: DEEPSEEK_PROVIDER_ID }) + await expect(ctx.plugin(deepseekPlugin, { apiKey: 'ds-key', maxUses: 1.5 })) + .rejects.toThrow(/maxUses expected number multiple of 1/) + }) + it('has no default export (namespace plugin export shape)', () => { expect('default' in deepseekPlugin).toBe(false) }) From 744130725110d4efe535094652ac2c2b87bdc088 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Fri, 3 Jul 2026 15:52:05 +0800 Subject: [PATCH 14/16] fix(web): open WebError.code to string, aligning with other seams The closed WebErrorCode union leaked fetch-transport details (redirect, too-large, content-type) into the seam's shared vocabulary and made web the only seam with a closed error-code union. Drop it and let WebError carry an open code: string like LlmError/SubagentError; document the codes grouped by owner (seam-neutral vs dsh-web-fetch-local transport). Addresses tianyicui's leaky-abstraction review comment on WebErrorCode. --- docs/cordis-catalog/events-and-services.md | 4 +- docs/core-data-structures/core.md | 2 +- docs/core-data-structures/web.md | 19 +------- packages/web/web/src/index.ts | 1 - packages/web/web/src/types.ts | 55 ++++++++-------------- scripts/type-equiv.manifest.json | 3 +- 6 files changed, 25 insertions(+), 59 deletions(-) diff --git a/docs/cordis-catalog/events-and-services.md b/docs/cordis-catalog/events-and-services.md index c3843f944d..1bb243afc8 100644 --- a/docs/cordis-catalog/events-and-services.md +++ b/docs/cordis-catalog/events-and-services.md @@ -309,7 +309,7 @@ Fired after the provider registry changes — a search or fetch provider was reg 'web/providers-change'(this: WebService): void ``` -Source: [`packages/web/web/src/index.ts:66`](../../packages/web/web/src/index.ts) +Source: [`packages/web/web/src/index.ts:65`](../../packages/web/web/src/index.ts) ## Services @@ -506,7 +506,7 @@ async search(request: WebSearchRequest, exec?: WebExecContext): Promise ``` -Source: [`packages/web/web/src/index.ts:106`](../../packages/web/web/src/index.ts) +Source: [`packages/web/web/src/index.ts:105`](../../packages/web/web/src/index.ts) ## Inherited tier (cordis core + loader/hmr/timer) diff --git a/docs/core-data-structures/core.md b/docs/core-data-structures/core.md index fdfbcfbbf4..b6ce3d060f 100644 --- a/docs/core-data-structures/core.md +++ b/docs/core-data-structures/core.md @@ -22,7 +22,7 @@ Everything else is documented on a **sub-page**, not here. The rule that draws t | [bash.md](bash.md) | the bash executor seam: `BashExecRequest`/`Spec`, `BashRunResult`, background `BashTask`s | | [compaction.md](compaction.md) | the compaction seam: the `compact/*` session events, `CompactionResult`, the `CompactService` interface | | [subagent.md](subagent.md) | the subagent seam: the named-provider registry, `SubagentStartRequest`/`Result`/`Run`, the start-time-vs-runtime capability split | -| [web.md](web.md) | the web access seam: `WebSearchRequest`/`Result`, `WebFetchRequest`/`Result`, `WebFetchBody`, provider/capability status, `WebErrorCode` | +| [web.md](web.md) | the web access seam: `WebSearchRequest`/`Result`, `WebFetchRequest`/`Result`, `WebFetchBody`, provider/capability status, `WebError` | > Type definitions on this page are pasted **verbatim** from source and drift-checked by `pnpm run verify-type-equiv` (see [development.md](../development.md#documenting-types-verbatim-ts-type-equiv)). Inline JSDoc is omitted for readability; follow the source link for the full contracts. diff --git a/docs/core-data-structures/web.md b/docs/core-data-structures/web.md index 43ed4e7aeb..1adde3bd75 100644 --- a/docs/core-data-structures/web.md +++ b/docs/core-data-structures/web.md @@ -95,24 +95,7 @@ Selection never depends on registration, config, or HMR order: a capability has ## Errors -`WebError extends HarnessError` ([core.md](core.md) error taxonomy) with a stable `WebErrorCode`. `WEB_DUPLICATE_PROVIDER` is a registration-time programming error (the analogue of `LlmService`'s `DUPLICATE_ADAPTER`); the `WEB_PROVIDER_*` selection codes and the fetch transport codes are execution outcomes. `WEB_PROVIDER_ERROR` is the catch-all for a provider's own failure surfaced through the seam, including network/transport failure (DNS, connection refused, TLS). - -```ts type-equiv -type WebErrorCode = - | 'WEB_PROVIDER_UNAVAILABLE' - | 'WEB_PROVIDER_CONFIGURED_MISSING' - | 'WEB_PROVIDER_CONFIGURED_UNAVAILABLE' - | 'WEB_PROVIDER_AMBIGUOUS' - | 'WEB_DUPLICATE_PROVIDER' - | 'WEB_INVALID_URL' - | 'WEB_BLOCKED_URL' - | 'WEB_REDIRECT_BLOCKED' - | 'WEB_FETCH_TOO_LARGE' - | 'WEB_FETCH_TIMEOUT' - | 'WEB_ABORTED' - | 'WEB_UNSUPPORTED_CONTENT_TYPE' - | 'WEB_PROVIDER_ERROR' -``` +`WebError extends HarnessError` ([core.md](core.md) error taxonomy) with a `code: string` (open, like every other seam's error — `LlmError`, `SubagentError`), not a closed union: a provider may raise its own codes without editing `dsh-web`, and consumers must tolerate an unknown code. The codes split by owner. Seam-neutral codes are raised by `WebService` selection and the shared contract: `WEB_PROVIDER_UNAVAILABLE`, `WEB_PROVIDER_CONFIGURED_MISSING`, `WEB_PROVIDER_CONFIGURED_UNAVAILABLE`, `WEB_PROVIDER_AMBIGUOUS`, `WEB_DUPLICATE_PROVIDER` (a registration-time programming error, the analogue of `LlmService`'s `DUPLICATE_ADAPTER`), `WEB_ABORTED`, and `WEB_PROVIDER_ERROR` (the catch-all for a provider's own failure surfaced through the seam, including network/transport failure — DNS, connection refused, TLS). Fetch-transport codes are owned by the `dsh-web-fetch-local` implementation and a different fetch backend need not raise them: `WEB_INVALID_URL`, `WEB_BLOCKED_URL`, `WEB_REDIRECT_BLOCKED`, `WEB_FETCH_TOO_LARGE`, `WEB_FETCH_TIMEOUT`, `WEB_UNSUPPORTED_CONTENT_TYPE`. ## The service diff --git a/packages/web/web/src/index.ts b/packages/web/web/src/index.ts index 172c1a0ecb..50150f6961 100644 --- a/packages/web/web/src/index.ts +++ b/packages/web/web/src/index.ts @@ -36,7 +36,6 @@ export { } from './types.ts' export type { WebCapabilityStatus, - WebErrorCode, WebExecContext, WebFetchBody, WebFetchProvider, diff --git a/packages/web/web/src/types.ts b/packages/web/web/src/types.ts index ec97101ae2..6f85787d1b 100644 --- a/packages/web/web/src/types.ts +++ b/packages/web/web/src/types.ts @@ -172,8 +172,19 @@ export interface WebFetchProvider { } /** - * Stable codes for {@link WebError}. Callers (hooks, tests, UI) route on these. + * Typed web error. Extends {@link HarnessError} so it carries a stable, + * machine-routable `code` (a `string`, like every other seam's error) and + * chains `cause`. `ToolRegistry.execute()` converts a thrown `WebError` into an + * error tool result whose structured metadata exposes the code, so callers + * (hooks, tests, UI) route on it. * + * The `code` is an open `string`, NOT a closed union: a provider may raise its + * own codes without editing this package, and a consumer must tolerate an + * unknown code (a future provider will introduce ones this file never named). + * The codes split by who owns them — seam-neutral codes any provider may see, + * versus codes specific to a single implementation: + * + * Seam-neutral (raised by `WebService` selection and the shared contract): * - `WEB_PROVIDER_UNAVAILABLE`: no provider configured and none usable. * - `WEB_PROVIDER_CONFIGURED_MISSING`: a configured id is not registered. * - `WEB_PROVIDER_CONFIGURED_UNAVAILABLE`: a configured id is registered but its @@ -182,44 +193,18 @@ export interface WebFetchProvider { * exist (selection refuses to pick by registration order). * - `WEB_DUPLICATE_PROVIDER`: a registration-time programming error — an id is * already registered for that capability kind. + * - `WEB_ABORTED`: the operation was aborted via `WebExecContext.signal`. + * - `WEB_PROVIDER_ERROR`: catch-all for a provider's own failure surfaced + * through the seam, including network/transport failure (DNS, connection + * refused, TLS). + * + * Fetch-transport codes (owned by the `dsh-web-fetch-local` implementation; a + * different fetch backend need not raise these and may raise its own): * - `WEB_INVALID_URL`: the fetch URL is malformed or not http(s). * - `WEB_BLOCKED_URL`: the fetch URL is rejected by policy (credentials in URL). * - `WEB_REDIRECT_BLOCKED`: a cross-origin redirect was refused. * - `WEB_FETCH_TOO_LARGE`: the response exceeded the byte/character cap. * - `WEB_FETCH_TIMEOUT`: the fetch exceeded its timeout. - * - `WEB_ABORTED`: the operation was aborted via `WebExecContext.signal`. * - `WEB_UNSUPPORTED_CONTENT_TYPE`: the response content type cannot be decoded. - * - `WEB_PROVIDER_ERROR`: catch-all for a provider's own failure surfaced through - * the seam, including network/transport failure (DNS, connection refused, TLS). */ -export type WebErrorCode = - | 'WEB_PROVIDER_UNAVAILABLE' - | 'WEB_PROVIDER_CONFIGURED_MISSING' - | 'WEB_PROVIDER_CONFIGURED_UNAVAILABLE' - | 'WEB_PROVIDER_AMBIGUOUS' - | 'WEB_DUPLICATE_PROVIDER' - | 'WEB_INVALID_URL' - | 'WEB_BLOCKED_URL' - | 'WEB_REDIRECT_BLOCKED' - | 'WEB_FETCH_TOO_LARGE' - | 'WEB_FETCH_TIMEOUT' - | 'WEB_ABORTED' - | 'WEB_UNSUPPORTED_CONTENT_TYPE' - | 'WEB_PROVIDER_ERROR' - -/** - * Typed web error. Extends {@link HarnessError} so it carries a stable - * {@link WebErrorCode} and chains `cause`. `dsh-web` owns this vocabulary so - * providers, the seam, and the tool layer raise the same codes instead of each - * inventing message strings. `ToolRegistry.execute()` converts a thrown - * `WebError` into an error tool result whose structured metadata exposes the - * code. - */ -export class WebError extends HarnessError { - override readonly code: WebErrorCode - - constructor(message: string, code: WebErrorCode, options?: ErrorOptions) { - super(message, code, options) - this.code = code - } -} +export class WebError extends HarnessError {} diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index 2a499580a7..ed8d342e28 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -58,7 +58,6 @@ { "doc": "docs/core-data-structures/web.md", "symbol": "WebFetchResult", "source": "packages/web/web/src/types.ts" }, { "doc": "docs/core-data-structures/web.md", "symbol": "WebFetchBody", "source": "packages/web/web/src/types.ts" }, { "doc": "docs/core-data-structures/web.md", "symbol": "WebProviderStatus", "source": "packages/web/web/src/types.ts" }, - { "doc": "docs/core-data-structures/web.md", "symbol": "WebCapabilityStatus", "source": "packages/web/web/src/types.ts" }, - { "doc": "docs/core-data-structures/web.md", "symbol": "WebErrorCode", "source": "packages/web/web/src/types.ts" } + { "doc": "docs/core-data-structures/web.md", "symbol": "WebCapabilityStatus", "source": "packages/web/web/src/types.ts" } ] } From 580496b72aa385ac15eed89db3e52f726a69fe94 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Fri, 3 Jul 2026 16:21:12 +0800 Subject: [PATCH 15/16] feat(web): expose exa/perplexity search tuning as config The Exa and Perplexity providers hard-coded request parameters that deployments should control while defaults are still unsettled. Exa gains searchType, numResults, and highlightsPerResult; Perplexity gains maxTokens (it previously sent none) and an optional searchRecency. Each follows the deepseek provider's shape: a defaulted Config field, a DEFAULT_* constant, and a positive-integer status() check for numeric limits. The call-level maxResults still flows through WebSearchRequest and wins over the configured default, keeping the seam layering intact. Addresses tianyicui's "make everything configurable" review comment. --- packages/web/web-search-exa/README.md | 5 +- packages/web/web-search-exa/src/index.ts | 28 +++++++-- packages/web/web-search-exa/src/provider.ts | 26 +++++++- packages/web/web-search-exa/src/types.ts | 4 +- packages/web/web-search-exa/tests/exa.e2e.ts | 9 ++- packages/web/web-search-exa/tests/exa.spec.ts | 59 ++++++++++++++++--- packages/web/web-search-perplexity/README.md | 2 + .../web/web-search-perplexity/src/index.ts | 22 +++++-- .../web/web-search-perplexity/src/provider.ts | 18 ++++++ .../tests/perplexity.e2e.ts | 3 +- .../tests/perplexity.spec.ts | 23 +++++++- 11 files changed, 172 insertions(+), 27 deletions(-) diff --git a/packages/web/web-search-exa/README.md b/packages/web/web-search-exa/README.md index 6485d64c60..0bc58d6559 100644 --- a/packages/web/web-search-exa/README.md +++ b/packages/web/web-search-exa/README.md @@ -10,6 +10,9 @@ This is an **implementation** package: it registers a provider into `ctx.web`, i |---|---|---| | `apiKey` | `$EXA_API_KEY` | Exa API key. Empty/absent → provider `status()` reports `missing-credential` (the seam reports `configured-unavailable`/`none`). | | `baseURL` | `https://api.exa.ai` | Endpoint base; `/search` is appended. An unparseable value makes `status()` report `misconfigured`. | +| `searchType` | `auto` | Retrieval mode sent as Exa's `type`: `auto` (Exa decides), `keyword`, or `neural`. | +| `numResults` | (unset) | Default result count when a request carries no `maxResults`. Unset sends no default. Must be a positive integer. | +| `highlightsPerResult` | `1` | Highlight sentences requested per result (Exa's `highlightsPerUrl`). Must be a positive integer. | ```yaml - id: web-search-exa @@ -20,4 +23,4 @@ This is an **implementation** package: it registers a provider into `ctx.web`, i ## Mapping -Exa returns a flat `results[]` and no generated answer, so `content` is omitted. Each result maps to a `WebSearchSource`: `url` ← `url`, `title` ← `title`, `snippet` ← the first non-empty `highlights[]` entry (a result with no highlight has no portable snippet and is dropped), `publishedAt` ← `publishedDate`. The provider passes `maxResults` through as Exa's `numResults` for a cost/latency optimization; the final bound is enforced by the seam. Provider failures (HTTP errors, network failure, unparseable or wrong-shape bodies) surface as `WebError` `WEB_PROVIDER_ERROR`; an aborted request surfaces as `WEB_ABORTED`. +Exa returns a flat `results[]` and no generated answer, so `content` is omitted. Each result maps to a `WebSearchSource`: `url` ← `url`, `title` ← `title`, `snippet` ← the first non-empty `highlights[]` entry (a result with no highlight has no portable snippet and is dropped), `publishedAt` ← `publishedDate`. A request's `maxResults` wins over the configured `numResults` default and is sent as Exa's `numResults` for a cost/latency optimization; the final bound is enforced by the seam. Provider failures (HTTP errors, network failure, unparseable or wrong-shape bodies) surface as `WebError` `WEB_PROVIDER_ERROR`; an aborted request surfaces as `WEB_ABORTED`. diff --git a/packages/web/web-search-exa/src/index.ts b/packages/web/web-search-exa/src/index.ts index f266474708..39a20b16a4 100644 --- a/packages/web/web-search-exa/src/index.ts +++ b/packages/web/web-search-exa/src/index.ts @@ -11,10 +11,17 @@ import type { Context } from 'cordis' import z from 'schemastery' import type {} from '@deepseek-ai/dsh-web' -import { ExaSearchProvider, EXA_DEFAULT_BASE_URL } from './provider.ts' +import { + ExaSearchProvider, + EXA_DEFAULT_BASE_URL, + EXA_DEFAULT_HIGHLIGHTS_PER_RESULT, + EXA_DEFAULT_SEARCH_TYPE, +} from './provider.ts' export { EXA_DEFAULT_BASE_URL, + EXA_DEFAULT_HIGHLIGHTS_PER_RESULT, + EXA_DEFAULT_SEARCH_TYPE, EXA_PROVIDER_ID, ExaSearchProvider, mapExaResponse, @@ -33,16 +40,29 @@ export interface Config { apiKey?: string /** Endpoint base; `/search` is appended. Defaults to the public API. */ baseURL?: string + /** Retrieval mode sent as Exa's `type`. Defaults to `auto`. */ + searchType?: 'auto' | 'keyword' | 'neural' + /** Default result count when a request carries no `maxResults`. Omitted = none. */ + numResults?: number + /** Highlight sentences requested per result. Defaults to 1. */ + highlightsPerResult?: number } export const Config: z = z.object({ apiKey: z.string(), baseURL: z.string(), + searchType: z.union(['auto', 'keyword', 'neural'] as const), + numResults: z.number().step(1).min(1), + highlightsPerResult: z.number().step(1).min(1), }) /** Register the Exa search provider with `ctx.web`. */ export function apply(ctx: Context, config: Config): void { - const apiKey = config.apiKey ?? process.env.EXA_API_KEY ?? '' - const baseURL = config.baseURL ?? EXA_DEFAULT_BASE_URL - ctx.web.registerSearchProvider(new ExaSearchProvider({ apiKey, baseURL })) + ctx.web.registerSearchProvider(new ExaSearchProvider({ + apiKey: config.apiKey ?? process.env.EXA_API_KEY ?? '', + baseURL: config.baseURL ?? EXA_DEFAULT_BASE_URL, + searchType: config.searchType ?? EXA_DEFAULT_SEARCH_TYPE, + highlightsPerResult: config.highlightsPerResult ?? EXA_DEFAULT_HIGHLIGHTS_PER_RESULT, + ...config.numResults !== undefined ? { numResults: config.numResults } : {}, + })) } diff --git a/packages/web/web-search-exa/src/provider.ts b/packages/web/web-search-exa/src/provider.ts index cfb41cf77f..f187f90344 100644 --- a/packages/web/web-search-exa/src/provider.ts +++ b/packages/web/web-search-exa/src/provider.ts @@ -28,6 +28,12 @@ export const EXA_PROVIDER_ID = 'exa' /** Default Exa search endpoint; `/search` is the operation. */ export const EXA_DEFAULT_BASE_URL = 'https://api.exa.ai' +/** Default retrieval mode: let Exa pick between keyword and neural search. */ +export const EXA_DEFAULT_SEARCH_TYPE = 'auto' + +/** Default number of highlight sentences requested per result. */ +export const EXA_DEFAULT_HIGHLIGHTS_PER_RESULT = 1 + /** Attribution header sent on every request. Bump with the package version. */ const USER_AGENT = 'deepseek-harness/0.0.1' @@ -36,6 +42,12 @@ export interface ExaSearchProviderOptions { apiKey: string /** Endpoint base; `/search` is appended. */ baseURL: string + /** Retrieval mode sent as Exa's `type`. */ + searchType: 'auto' | 'keyword' | 'neural' + /** Default result count when a request carries no `maxResults`. */ + numResults?: number + /** Highlight sentences requested per result (Exa's `highlightsPerUrl`). */ + highlightsPerResult: number } /** @@ -73,10 +85,14 @@ export class ExaSearchProvider implements WebSearchProvider { status(): WebProviderStatus { if (this.options.apiKey.length === 0) return { available: false, reason: 'missing-credential' } if (!isValidBaseUrl(this.options.baseURL)) return { available: false, reason: 'misconfigured' } + if (!isPositiveInteger(this.options.highlightsPerResult)) return { available: false, reason: 'misconfigured' } + if (this.options.numResults !== undefined && !isPositiveInteger(this.options.numResults)) return { available: false, reason: 'misconfigured' } return { available: true } } async search(request: WebSearchRequest, exec?: { readonly signal?: AbortSignal }): Promise { + // A per-request bound wins over the configured default; either may be absent. + const numResults = request.maxResults ?? this.options.numResults let response: Response try { response = await fetch(`${this.options.baseURL}/search`, { @@ -89,8 +105,9 @@ export class ExaSearchProvider implements WebSearchProvider { }, body: JSON.stringify({ query: request.query, - contents: { highlights: true }, - ...request.maxResults !== undefined ? { numResults: request.maxResults } : {}, + type: this.options.searchType, + contents: { highlights: { highlightsPerUrl: this.options.highlightsPerResult } }, + ...numResults !== undefined ? { numResults } : {}, }), ...exec?.signal ? { signal: exec.signal } : {}, }) @@ -133,6 +150,11 @@ function isValidBaseUrl(baseURL: string): boolean { return URL.canParse(baseURL) } +/** True for a request limit that can be sent to Exa (a positive whole number). */ +function isPositiveInteger(value: number): boolean { + return Number.isInteger(value) && value > 0 +} + /** True for a fetch/`AbortSignal` abort, surfaced as `WEB_ABORTED`. */ function isAbortError(error: unknown): boolean { return error instanceof DOMException && error.name === 'AbortError' diff --git a/packages/web/web-search-exa/src/types.ts b/packages/web/web-search-exa/src/types.ts index a0bda5f768..fae42d07cd 100644 --- a/packages/web/web-search-exa/src/types.ts +++ b/packages/web/web-search-exa/src/types.ts @@ -10,10 +10,12 @@ /** Request body sent to Exa's search endpoint. */ export interface ExaSearchRequest { query: string + /** Retrieval mode: keyword, neural (embeddings), or auto (Exa decides). */ + type: 'auto' | 'keyword' | 'neural' /** Exa's result-count control; the seam still enforces the bound on return. */ numResults?: number /** Ask Exa to return highlight sentences per result. */ - contents: { highlights: true } + contents: { highlights: { highlightsPerUrl: number } } } /** One entry of Exa's flat `results[]`. */ diff --git a/packages/web/web-search-exa/tests/exa.e2e.ts b/packages/web/web-search-exa/tests/exa.e2e.ts index 78f11940e5..32da0a485c 100644 --- a/packages/web/web-search-exa/tests/exa.e2e.ts +++ b/packages/web/web-search-exa/tests/exa.e2e.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { ExaSearchProvider, EXA_DEFAULT_BASE_URL } from '@deepseek-ai/dsh-web-search-exa' +import { ExaSearchProvider, EXA_DEFAULT_BASE_URL, EXA_DEFAULT_HIGHLIGHTS_PER_RESULT, EXA_DEFAULT_SEARCH_TYPE } from '@deepseek-ai/dsh-web-search-exa' /** * Real-API smoke for the Exa search provider. Self-skips without `$EXA_API_KEY` @@ -10,7 +10,12 @@ const maybe = apiKey !== undefined && apiKey.length > 0 ? describe : describe.sk maybe('ExaSearchProvider real API', () => { it('returns sources for a live query', async () => { - const provider = new ExaSearchProvider({ apiKey: apiKey!, baseURL: process.env.EXA_BASE_URL ?? EXA_DEFAULT_BASE_URL }) + const provider = new ExaSearchProvider({ + apiKey: apiKey!, + baseURL: process.env.EXA_BASE_URL ?? EXA_DEFAULT_BASE_URL, + searchType: EXA_DEFAULT_SEARCH_TYPE, + highlightsPerResult: EXA_DEFAULT_HIGHLIGHTS_PER_RESULT, + }) const result = await provider.search({ query: 'DeepSeek coding agent', maxResults: 5 }) expect(result.providerId).toBe('exa') expect(result.sources.length).toBeGreaterThan(0) diff --git a/packages/web/web-search-exa/tests/exa.spec.ts b/packages/web/web-search-exa/tests/exa.spec.ts index 436e542d7c..dcb6fbea6d 100644 --- a/packages/web/web-search-exa/tests/exa.spec.ts +++ b/packages/web/web-search-exa/tests/exa.spec.ts @@ -4,7 +4,7 @@ import WebService from '@deepseek-ai/dsh-web' import { ExaSearchProvider, mapExaResponse, mapExaResult, EXA_PROVIDER_ID } from '@deepseek-ai/dsh-web-search-exa' import * as exaPlugin from '@deepseek-ai/dsh-web-search-exa' -const options = { apiKey: 'exa-key', baseURL: 'https://api.exa.test' } +const options = { apiKey: 'exa-key', baseURL: 'https://api.exa.test', searchType: 'auto' as const, highlightsPerResult: 1 } function jsonResponse(body: unknown, init: ResponseInit = {}): Response { return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' }, ...init }) @@ -65,7 +65,7 @@ describe('Exa result mapping', () => { describe('ExaSearchProvider status', () => { it('is unavailable without a key', () => { - expect(new ExaSearchProvider({ apiKey: '', baseURL: options.baseURL }).status()) + expect(new ExaSearchProvider({ ...options, apiKey: '' }).status()) .toEqual({ available: false, reason: 'missing-credential' }) }) @@ -74,27 +74,60 @@ describe('ExaSearchProvider status', () => { }) it('is misconfigured when the base URL is unparseable', () => { - expect(new ExaSearchProvider({ apiKey: 'exa-key', baseURL: 'not a url' }).status()) + expect(new ExaSearchProvider({ ...options, baseURL: 'not a url' }).status()) + .toEqual({ available: false, reason: 'misconfigured' }) + }) + + it('is misconfigured when highlightsPerResult is not a positive integer', () => { + expect(new ExaSearchProvider({ ...options, highlightsPerResult: 0 }).status()) + .toEqual({ available: false, reason: 'misconfigured' }) + expect(new ExaSearchProvider({ ...options, highlightsPerResult: 1.5 }).status()) + .toEqual({ available: false, reason: 'misconfigured' }) + }) + + it('is misconfigured when numResults is set but not a positive integer', () => { + expect(new ExaSearchProvider({ ...options, numResults: -1 }).status()) .toEqual({ available: false, reason: 'misconfigured' }) }) }) describe('ExaSearchProvider request mapping', () => { - it('sends query, highlights, numResults and bearer auth', async () => { + it('sends query, type, highlights, numResults and bearer auth', async () => { const fetchMock = vi.fn(async () => jsonResponse({ results: [{ url: 'https://a.test', highlights: ['hi'] }] })) vi.stubGlobal('fetch', fetchMock) - const provider = new ExaSearchProvider(options) + const provider = new ExaSearchProvider({ ...options, searchType: 'neural', highlightsPerResult: 3 }) await provider.search({ query: 'hello', maxResults: 5 }) expect(fetchMock).toHaveBeenCalledOnce() const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] expect(url).toBe('https://api.exa.test/search') expect((init.headers as Record)['authorization']).toBe('Bearer exa-key') - expect(JSON.parse(init.body as string)).toEqual({ query: 'hello', contents: { highlights: true }, numResults: 5 }) + expect(JSON.parse(init.body as string)).toEqual({ + query: 'hello', + type: 'neural', + contents: { highlights: { highlightsPerUrl: 3 } }, + numResults: 5, + }) }) - it('omits numResults when maxResults is absent', async () => { + it('falls back to the configured numResults when a request omits maxResults', async () => { + const fetchMock = vi.fn(async () => jsonResponse({ results: [] })) + vi.stubGlobal('fetch', fetchMock) + await new ExaSearchProvider({ ...options, numResults: 7 }).search({ query: 'q' }) + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(JSON.parse(init.body as string)).toMatchObject({ numResults: 7 }) + }) + + it('lets a request maxResults win over the configured numResults', async () => { + const fetchMock = vi.fn(async () => jsonResponse({ results: [] })) + vi.stubGlobal('fetch', fetchMock) + await new ExaSearchProvider({ ...options, numResults: 7 }).search({ query: 'q', maxResults: 2 }) + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(JSON.parse(init.body as string)).toMatchObject({ numResults: 2 }) + }) + + it('omits numResults when neither maxResults nor a configured default is set', async () => { const fetchMock = vi.fn(async () => jsonResponse({ results: [] })) vi.stubGlobal('fetch', fetchMock) await new ExaSearchProvider(options).search({ query: 'q' }) @@ -184,6 +217,18 @@ describe('web-search-exa plugin registration', () => { expect('default' in exaPlugin).toBe(false) }) + it('threads searchType and highlightsPerResult config into the request', async () => { + const fetchMock = vi.fn(async () => jsonResponse({ results: [] })) + vi.stubGlobal('fetch', fetchMock) + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: EXA_PROVIDER_ID }) + const fiber = await ctx.plugin(exaPlugin, { apiKey: 'exa-key', searchType: 'keyword', highlightsPerResult: 2 }) + await ctx.web.search({ query: 'q' }) + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(JSON.parse(init.body as string)).toMatchObject({ type: 'keyword', contents: { highlights: { highlightsPerUrl: 2 } } }) + await fiber.dispose() + }) + it('falls back to $EXA_API_KEY and the default base URL when config omits them', async () => { const prev = process.env.EXA_API_KEY process.env.EXA_API_KEY = 'env-key' diff --git a/packages/web/web-search-perplexity/README.md b/packages/web/web-search-perplexity/README.md index e7093a1133..f944413c96 100644 --- a/packages/web/web-search-perplexity/README.md +++ b/packages/web/web-search-perplexity/README.md @@ -11,6 +11,8 @@ This is an **implementation** package: it registers a provider into `ctx.web`, i | `apiKey` | `$PERPLEXITY_API_KEY` | Perplexity API key. Empty/absent → provider `status()` reports `missing-credential`. | | `baseURL` | `https://api.perplexity.ai` | Endpoint base; `/chat/completions` is appended. An unparseable value makes `status()` report `misconfigured`. | | `model` | `sonar` | Search model name. | +| `maxTokens` | `1024` | Upper bound on generated answer tokens (`max_tokens`). Must be a positive integer. | +| `searchRecency` | (unset) | Recency window sent as `search_recency_filter`: `day`, `week`, `month`, or `year`. Unset sends no filter. | ```yaml - id: web-search-perplexity diff --git a/packages/web/web-search-perplexity/src/index.ts b/packages/web/web-search-perplexity/src/index.ts index 0fd46ffb71..3d375eaabb 100644 --- a/packages/web/web-search-perplexity/src/index.ts +++ b/packages/web/web-search-perplexity/src/index.ts @@ -10,17 +10,18 @@ import type { Context } from 'cordis' import z from 'schemastery' import type {} from '@deepseek-ai/dsh-web' -import { PerplexitySearchProvider, PERPLEXITY_DEFAULT_BASE_URL, PERPLEXITY_DEFAULT_MODEL } from './provider.ts' +import { PerplexitySearchProvider, PERPLEXITY_DEFAULT_BASE_URL, PERPLEXITY_DEFAULT_MAX_TOKENS, PERPLEXITY_DEFAULT_MODEL } from './provider.ts' export { PERPLEXITY_DEFAULT_BASE_URL, + PERPLEXITY_DEFAULT_MAX_TOKENS, PERPLEXITY_DEFAULT_MODEL, PERPLEXITY_PROVIDER_ID, PerplexitySearchProvider, mapPerplexityResponse, mapPerplexityResult, } from './provider.ts' -export type { PerplexitySearchProviderOptions } from './provider.ts' +export type { PerplexityRecency, PerplexitySearchProviderOptions } from './provider.ts' /** Cordis plugin name used by loader diagnostics. */ export const name = 'web-search-perplexity' @@ -35,18 +36,27 @@ export interface Config { baseURL?: string /** Search model name. Defaults to `sonar`. */ model?: string + /** Upper bound on generated answer tokens. Defaults to 1024. */ + maxTokens?: number + /** Recency window sent as `search_recency_filter`. Omitted = no filter. */ + searchRecency?: 'day' | 'week' | 'month' | 'year' } export const Config: z = z.object({ apiKey: z.string(), baseURL: z.string(), model: z.string(), + maxTokens: z.number().step(1).min(1), + searchRecency: z.union(['day', 'week', 'month', 'year'] as const), }) /** Register the Perplexity search provider with `ctx.web`. */ export function apply(ctx: Context, config: Config): void { - const apiKey = config.apiKey ?? process.env.PERPLEXITY_API_KEY ?? '' - const baseURL = config.baseURL ?? PERPLEXITY_DEFAULT_BASE_URL - const model = config.model ?? PERPLEXITY_DEFAULT_MODEL - ctx.web.registerSearchProvider(new PerplexitySearchProvider({ apiKey, baseURL, model })) + ctx.web.registerSearchProvider(new PerplexitySearchProvider({ + apiKey: config.apiKey ?? process.env.PERPLEXITY_API_KEY ?? '', + baseURL: config.baseURL ?? PERPLEXITY_DEFAULT_BASE_URL, + model: config.model ?? PERPLEXITY_DEFAULT_MODEL, + maxTokens: config.maxTokens ?? PERPLEXITY_DEFAULT_MAX_TOKENS, + ...config.searchRecency !== undefined ? { searchRecency: config.searchRecency } : {}, + })) } diff --git a/packages/web/web-search-perplexity/src/provider.ts b/packages/web/web-search-perplexity/src/provider.ts index 5b5feb897b..ed72ea82c3 100644 --- a/packages/web/web-search-perplexity/src/provider.ts +++ b/packages/web/web-search-perplexity/src/provider.ts @@ -32,6 +32,12 @@ export const PERPLEXITY_DEFAULT_BASE_URL = 'https://api.perplexity.ai' /** Default search model. */ export const PERPLEXITY_DEFAULT_MODEL = 'sonar' +/** Default upper bound on generated answer tokens. */ +export const PERPLEXITY_DEFAULT_MAX_TOKENS = 1024 + +/** Recency filter values Perplexity accepts for `search_recency_filter`. */ +export type PerplexityRecency = 'day' | 'week' | 'month' | 'year' + /** Attribution header sent on every request. Bump with the package version. */ const USER_AGENT = 'deepseek-harness/0.0.1' @@ -42,6 +48,10 @@ export interface PerplexitySearchProviderOptions { baseURL: string /** Search model name. */ model: string + /** Upper bound on generated answer tokens (`max_tokens`). */ + maxTokens: number + /** Optional recency window sent as `search_recency_filter`; omitted = no filter. */ + searchRecency?: PerplexityRecency } /** Map one structured Perplexity search result to a normalized source. */ @@ -82,6 +92,7 @@ export class PerplexitySearchProvider implements WebSearchProvider { status(): WebProviderStatus { if (this.options.apiKey.length === 0) return { available: false, reason: 'missing-credential' } if (!URL.canParse(this.options.baseURL)) return { available: false, reason: 'misconfigured' } + if (!isPositiveInteger(this.options.maxTokens)) return { available: false, reason: 'misconfigured' } return { available: true } } @@ -98,7 +109,9 @@ export class PerplexitySearchProvider implements WebSearchProvider { }, body: JSON.stringify({ model: this.options.model, + max_tokens: this.options.maxTokens, messages: [{ role: 'user', content: request.query }], + ...this.options.searchRecency !== undefined ? { search_recency_filter: this.options.searchRecency } : {}, }), ...exec?.signal ? { signal: exec.signal } : {}, }) @@ -140,3 +153,8 @@ export class PerplexitySearchProvider implements WebSearchProvider { function isAbortError(error: unknown): boolean { return error instanceof DOMException && error.name === 'AbortError' } + +/** True for a request limit that can be sent to Perplexity (a positive whole number). */ +function isPositiveInteger(value: number): boolean { + return Number.isInteger(value) && value > 0 +} diff --git a/packages/web/web-search-perplexity/tests/perplexity.e2e.ts b/packages/web/web-search-perplexity/tests/perplexity.e2e.ts index a546acab70..9414d46937 100644 --- a/packages/web/web-search-perplexity/tests/perplexity.e2e.ts +++ b/packages/web/web-search-perplexity/tests/perplexity.e2e.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { PerplexitySearchProvider, PERPLEXITY_DEFAULT_BASE_URL, PERPLEXITY_DEFAULT_MODEL } from '@deepseek-ai/dsh-web-search-perplexity' +import { PerplexitySearchProvider, PERPLEXITY_DEFAULT_BASE_URL, PERPLEXITY_DEFAULT_MAX_TOKENS, PERPLEXITY_DEFAULT_MODEL } from '@deepseek-ai/dsh-web-search-perplexity' /** * Real-API smoke for the Perplexity search provider. Self-skips without @@ -14,6 +14,7 @@ maybe('PerplexitySearchProvider real API', () => { apiKey: apiKey!, baseURL: process.env.PERPLEXITY_BASE_URL ?? PERPLEXITY_DEFAULT_BASE_URL, model: process.env.PERPLEXITY_MODEL ?? PERPLEXITY_DEFAULT_MODEL, + maxTokens: PERPLEXITY_DEFAULT_MAX_TOKENS, }) const result = await provider.search({ query: 'What is the DeepSeek coding agent?', maxResults: 5 }) expect(result.providerId).toBe('perplexity') diff --git a/packages/web/web-search-perplexity/tests/perplexity.spec.ts b/packages/web/web-search-perplexity/tests/perplexity.spec.ts index d84c34a328..6d55f384dd 100644 --- a/packages/web/web-search-perplexity/tests/perplexity.spec.ts +++ b/packages/web/web-search-perplexity/tests/perplexity.spec.ts @@ -8,7 +8,7 @@ import { } from '@deepseek-ai/dsh-web-search-perplexity' import * as perplexityPlugin from '@deepseek-ai/dsh-web-search-perplexity' -const options = { apiKey: 'pplx-key', baseURL: 'https://api.perplexity.test', model: 'sonar' } +const options = { apiKey: 'pplx-key', baseURL: 'https://api.perplexity.test', model: 'sonar', maxTokens: 1024 } function jsonResponse(body: unknown, init: ResponseInit = {}): Response { return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' }, ...init }) @@ -80,17 +80,34 @@ describe('PerplexitySearchProvider status', () => { expect(new PerplexitySearchProvider({ ...options, baseURL: 'not a url' }).status()) .toEqual({ available: false, reason: 'misconfigured' }) }) + + it('is misconfigured when maxTokens is not a positive integer', () => { + expect(new PerplexitySearchProvider({ ...options, maxTokens: 0 }).status()) + .toEqual({ available: false, reason: 'misconfigured' }) + expect(new PerplexitySearchProvider({ ...options, maxTokens: 1.5 }).status()) + .toEqual({ available: false, reason: 'misconfigured' }) + }) }) describe('PerplexitySearchProvider request mapping', () => { - it('sends a chat-completions request with the query as a user message', async () => { + it('sends a chat-completions request with the query, model and max_tokens', async () => { const fetchMock = vi.fn(async () => jsonResponse({ choices: [{ message: { content: 'a' } }], citations: [] })) vi.stubGlobal('fetch', fetchMock) await new PerplexitySearchProvider(options).search({ query: 'hello' }) const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] expect(url).toBe('https://api.perplexity.test/chat/completions') expect((init.headers as Record)['authorization']).toBe('Bearer pplx-key') - expect(JSON.parse(init.body as string)).toEqual({ model: 'sonar', messages: [{ role: 'user', content: 'hello' }] }) + expect(JSON.parse(init.body as string)).toEqual({ model: 'sonar', max_tokens: 1024, messages: [{ role: 'user', content: 'hello' }] }) + }) + + it('sends search_recency_filter when configured, and omits it otherwise', async () => { + const fetchMock = vi.fn(async () => jsonResponse({ choices: [{ message: { content: 'a' } }], citations: [] })) + vi.stubGlobal('fetch', fetchMock) + await new PerplexitySearchProvider({ ...options, searchRecency: 'week' }).search({ query: 'q' }) + expect(JSON.parse((fetchMock.mock.calls[0] as unknown as [string, RequestInit])[1].body as string)).toMatchObject({ search_recency_filter: 'week' }) + + await new PerplexitySearchProvider(options).search({ query: 'q' }) + expect(JSON.parse((fetchMock.mock.calls[1] as unknown as [string, RequestInit])[1].body as string)).not.toHaveProperty('search_recency_filter') }) it('forwards the abort signal', async () => { From 0a595aea78742283d886accd6c803815bccbab48 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Fri, 3 Jul 2026 17:03:54 +0800 Subject: [PATCH 16/16] test(web): cover the config-present branch of exa/perplexity apply The numResults (exa) and searchRecency (perplexity) conditional spreads in apply() were only exercised on their absent side, leaving the 100% per-file branch gate red. Add plugin-registration tests that pass those config fields and assert they reach the request body. --- packages/web/web-search-exa/tests/exa.spec.ts | 6 +++--- .../web-search-perplexity/tests/perplexity.spec.ts | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/web/web-search-exa/tests/exa.spec.ts b/packages/web/web-search-exa/tests/exa.spec.ts index dcb6fbea6d..9cf31332f5 100644 --- a/packages/web/web-search-exa/tests/exa.spec.ts +++ b/packages/web/web-search-exa/tests/exa.spec.ts @@ -217,15 +217,15 @@ describe('web-search-exa plugin registration', () => { expect('default' in exaPlugin).toBe(false) }) - it('threads searchType and highlightsPerResult config into the request', async () => { + it('threads searchType, highlightsPerResult and numResults config into the request', async () => { const fetchMock = vi.fn(async () => jsonResponse({ results: [] })) vi.stubGlobal('fetch', fetchMock) const ctx = new Context() await ctx.plugin(WebService, { searchProvider: EXA_PROVIDER_ID }) - const fiber = await ctx.plugin(exaPlugin, { apiKey: 'exa-key', searchType: 'keyword', highlightsPerResult: 2 }) + const fiber = await ctx.plugin(exaPlugin, { apiKey: 'exa-key', searchType: 'keyword', highlightsPerResult: 2, numResults: 9 }) await ctx.web.search({ query: 'q' }) const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] - expect(JSON.parse(init.body as string)).toMatchObject({ type: 'keyword', contents: { highlights: { highlightsPerUrl: 2 } } }) + expect(JSON.parse(init.body as string)).toMatchObject({ type: 'keyword', contents: { highlights: { highlightsPerUrl: 2 } }, numResults: 9 }) await fiber.dispose() }) diff --git a/packages/web/web-search-perplexity/tests/perplexity.spec.ts b/packages/web/web-search-perplexity/tests/perplexity.spec.ts index 6d55f384dd..70a9a4c98b 100644 --- a/packages/web/web-search-perplexity/tests/perplexity.spec.ts +++ b/packages/web/web-search-perplexity/tests/perplexity.spec.ts @@ -198,6 +198,18 @@ describe('web-search-perplexity plugin registration', () => { expect('default' in perplexityPlugin).toBe(false) }) + it('threads maxTokens and searchRecency config into the request', async () => { + const fetchMock = vi.fn(async () => jsonResponse({ choices: [{ message: { content: 'a' } }], citations: [] })) + vi.stubGlobal('fetch', fetchMock) + const ctx = new Context() + await ctx.plugin(WebService, { searchProvider: PERPLEXITY_PROVIDER_ID }) + const fiber = await ctx.plugin(perplexityPlugin, { apiKey: 'pplx-key', maxTokens: 256, searchRecency: 'month' }) + await ctx.web.search({ query: 'q' }) + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(JSON.parse(init.body as string)).toMatchObject({ max_tokens: 256, search_recency_filter: 'month' }) + await fiber.dispose() + }) + it('falls back to env key and defaults for base URL and model when config omits them', async () => { const prev = process.env.PERPLEXITY_API_KEY process.env.PERPLEXITY_API_KEY = 'env-key'