From 45a5175e441ad073d82a8a0db688aa3572b90057 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 00:22:13 +0800 Subject: [PATCH] feat(tool-web): replace the regex HTML-to-markdown converter with turndown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the turndown Agent Note from the NIH dependency audit (full variant, not the minimal entities-only fallback): dsh-tool-web's fetch rendering now converts HTML through turndown + @joplin/turndown-plugin-gfm (atx headings, fenced code, dash bullets, GFM tables/strikethrough) over the real domino DOM, with script/style/noscript removed wholesale. The hand-rolled ~86-line regex converter html.ts and its entity tables are deleted; renderBody wraps the conversion in try/catch falling back to the raw HTML body, because turndown's recursive DOM walk overflows with a RangeError on pathological nesting (measured: 4k levels on the main thread, 8k in a worker) where the regex version could never throw. Closure weight, measured: tool-web IS in the single-exe runtime closure, and the exe asset globs would pack ~7.9 MB of the three new packages — but ~6 MB of that is domino's test corpus, with runtime lib/ at ~550 KB against a ~174 MB artifact (<0.5% either way), so the swap wins. Per testing policy the previously-missing keyless web_fetch snapshot ships in the same change: the acp-agent `web-fetch` scenario boots a new web.cordis.yml overlay (web seam + real dsh-web-fetch-local provider + tool-web fetch-only + a loopback HTTP fixture server on a fixed port serving deterministic HTML with entities, a GFM table, and nesting), so recording and keyless replay both drive the real HTTP fetch and real conversion end to end; the scenario pins the new `web` header class. The Agent Note moves proposed -> implemented and is rewritten per the lifecycle contract (Decision/Consequences/Testing, closure verdict and alternatives recorded); tool-web and acp-agent READMEs updated in both languages and pairs re-recorded. --- ...ndown-for-tool-web-html-markdown.i18n.yaml | 4 +- ...-26-turndown-for-tool-web-html-markdown.md | 37 ++ ...-turndown-for-tool-web-html-markdown.zh.md | 37 ++ ...-26-turndown-for-tool-web-html-markdown.md | 32 -- ...-turndown-for-tool-web-html-markdown.zh.md | 32 -- docs/config-catalog.md | 2 +- examples/acp-agent/README.i18n.yaml | 4 +- examples/acp-agent/README.md | 2 +- examples/acp-agent/README.zh.md | 2 +- examples/acp-agent/tests/acp.snapshot.ts | 7 + .../tests/snapshots/web-fetch/input.json | 7 + .../tests/snapshots/web-fetch/session.jsonl | 127 +++++ .../snapshots/web-fetch/stdout.expected.jsonl | 4 + .../web-fetch/system-prompt.expected.md | 27 + .../web-fetch/tool-schemas.expected.json | 489 ++++++++++++++++++ .../acp-agent/web-fetch-fixture-server.mjs | 52 ++ examples/acp-agent/web.cordis.snapshot.yml | 31 ++ examples/acp-agent/web.cordis.yml | 21 + examples/package.json | 1 + packages/web/tool-web/README.i18n.yaml | 4 +- packages/web/tool-web/README.md | 4 +- packages/web/tool-web/README.zh.md | 4 +- packages/web/tool-web/package.json | 5 +- packages/web/tool-web/src/fetch.ts | 34 +- packages/web/tool-web/src/html.ts | 86 --- packages/web/tool-web/src/index.ts | 1 - .../web/tool-web/src/turndown-plugin-gfm.d.ts | 12 + packages/web/tool-web/tests/tool-web.spec.ts | 69 +-- pnpm-lock.yaml | 35 ++ 29 files changed, 962 insertions(+), 210 deletions(-) rename .agents/notes/{proposed => implemented}/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml (60%) create mode 100644 .agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md create mode 100644 .agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md delete mode 100644 .agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md delete mode 100644 .agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md create mode 100644 examples/acp-agent/tests/snapshots/web-fetch/input.json create mode 100644 examples/acp-agent/tests/snapshots/web-fetch/session.jsonl create mode 100644 examples/acp-agent/tests/snapshots/web-fetch/stdout.expected.jsonl create mode 100644 examples/acp-agent/tests/snapshots/web-fetch/system-prompt.expected.md create mode 100644 examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json create mode 100644 examples/acp-agent/web-fetch-fixture-server.mjs create mode 100644 examples/acp-agent/web.cordis.snapshot.yml create mode 100644 examples/acp-agent/web.cordis.yml delete mode 100644 packages/web/tool-web/src/html.ts create mode 100644 packages/web/tool-web/src/turndown-plugin-gfm.d.ts diff --git a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml similarity index 60% rename from .agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml rename to .agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml index ced514a423..60a5d9aca7 100644 --- a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-26-turndown-for-tool-web-html-markdown.md: 7f25e51bf6e6fc9313a880abee737bca80a472af -2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 3a59b08e13fd392e4f34ac543f32f5b4648f3c1c +2026-07-26-turndown-for-tool-web-html-markdown.md: c72decc336055f3b78dafdf98f2be3771b833cdb +2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 30667b62538ec50608cae461b5cdf651b48e2731 diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md new file mode 100644 index 0000000000..c72decc336 --- /dev/null +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md @@ -0,0 +1,37 @@ +# Agent Note: Replace tool-web's regex HTML-to-markdown converter with turndown + +Status: implemented + +English | [中文](2026-07-26-turndown-for-tool-web-html-markdown.zh.md) + +## Problem + +`dsh-tool-web`'s `src/html.ts` (~86 lines, ~40 lines of dedicated tests; deleted by this change) converted fetched HTML to markdown with regexes: strip script/style/noscript/comments, convert ``/``/`
  • `, decode numeric entities plus a 12-entry named-entity table, collapse whitespace. The module's own JSDoc said "A richer converter can replace it without changing the seam or tool schema", and the README's Known Limitations documented it as "a minimal regex converter, not an HTML parser — tables, images, and nested formatting are lost." The [web capability seam note](../architecture/2026-06-24-web-capability-seam.md) assigns HTML→markdown to this package as presentation, so the swap point was exactly here. The converter's output is model-visible on every fetched HTML page; no keyless snapshot exercised `web_fetch`, so no expected outputs pinned it. + +## Decision + +`packages/web/tool-web/src/fetch.ts` owns a module-level [`turndown`](https://github.com/mixmark-io/turndown) instance (`headingStyle: 'atx'`, `codeBlockStyle: 'fenced'`, `bulletListMarker: '-'` — fixed model-facing presentation, not deployment tunables) with `@joplin/turndown-plugin-gfm`'s composite `gfm` plugin for tables/strikethrough and `remove(['script', 'style', 'noscript'])` replacing the old wholesale drops. `renderBody`'s `html` arm calls it in a try/catch falling back to the raw HTML body: the regex version could never throw, while turndown/domino's recursive DOM walk overflows with a `RangeError` at a few thousand nesting levels (measured: 4k throws on the main thread, 8k in a worker thread), and a degraded page beats an error for a body the provider already decoded. `html.ts` and its conversion tests are deleted; the fallback and the status-header/truncation-footer formatting are tested in `tests/tool-web.spec.ts`, and the README's Known Limitations trades the regex-converter caveat for the pathological-nesting fallback. The gfm plugin ships no types; `src/turndown-plugin-gfm.d.ts` declares the one imported export over `@types/turndown` (a devDependency). + +The dependency-weight question the proposal flagged resolves in favor of the swap: `@deepseek-ai/dsh-tool-web` is in the single-file-executable closure ([single-exe note](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)), and the exe's asset globs would pack ~7.9 MB of the three packages as published — but ~6 MB of that is `@mixmark-io/domino`'s test corpus (`test/**`), with runtime `lib/` at ~550 KB against a ~174 MB artifact, under 0.5% either way. + +## Snapshot coverage + +The previously-missing keyless `web_fetch` snapshot ships with the change as the acp-agent scenario `web-fetch`: `examples/acp-agent/web.cordis.yml` composes the web seam, the real `dsh-web-fetch-local` provider, `tool-web` with `search: false`, and `web-fetch-fixture-server.mjs` — a loopback HTTP fixture on a fixed port (the fetched URL is part of the recorded transcript) serving deterministic HTML with named entities, a GFM table, and nested formatting. Recording and keyless replay both drive the real HTTP fetch and conversion; the pinned tool result is the turndown output, and the scenario pins the `web` header class (the `web_fetch` schema and guidance). + +## Alternatives considered + +- **`@mozilla/readability` + a DOM.** Solves a different problem (content extraction, not conversion) and drags a heavier DOM dependency; the seam only asks for markdown rendering of whatever the fetch returned. +- **Keep the regex converter.** It was an explicit v1 placeholder per its own JSDoc; keeping it meant model-visible quality (tables, images, nested formatting) stayed lost for the cost of maintaining bespoke entity tables. +- **The minimal `entities`-only variant.** The proposal's fallback position: replace only the entity-decoding third of `html.ts` with the zero-dependency `entities` package, deleting less but avoiding the dependency-weight question. Not taken because the closure math above made the weight immaterial while the full swap deletes the whole hand-rolled converter and its documented quality gaps. +- **`turndown-plugin-gfm` (the original) instead of `@joplin/turndown-plugin-gfm`.** The original is unmaintained (last publish 2018); the Joplin fork is current against turndown 7 and actively released. + +## Consequences + +- **Bought**: full-fidelity model-visible markdown — tables, images, strikethrough, nested emphasis, fenced code blocks, and the complete named-entity set — plus the deletion of the bespoke converter and its entity tables, with the README's regex-converter caveat narrowed to one degenerate case. +- **Paid**: two runtime dependencies (`turndown` → `@mixmark-io/domino`) enter tool-web and therefore the exe closure (~550 KB of runtime code as measured above), and a new failure mode — pathological nesting — is handled by falling back to raw HTML rather than converting. +- Model-visible output changed on every fetched HTML page; nothing pinned the old output, and the new snapshot pins the new one. + +## Testing + +- `packages/web/tool-web/tests/tool-web.spec.ts` covers the turndown conversion surface (entities, links, tables, nesting, script/style/noscript removal) through `renderBody`, and the raw-HTML fallback with a measured reliably-overflowing 20k-level nesting input; per-file coverage on the package src is 100%. +- The `web-fetch` acp-agent snapshot pins the assembled behavior keylessly end to end (real Loader composition, real HTTP fetch, real conversion). diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md new file mode 100644 index 0000000000..30667b6253 --- /dev/null +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md @@ -0,0 +1,37 @@ +# Agent Note: 用 turndown 替换 tool-web 的正则 HTML 转 markdown 转换器 + +Status: implemented + +[English](2026-07-26-turndown-for-tool-web-html-markdown.md) | 中文 + +## 问题 + +`dsh-tool-web` 的 `src/html.ts`（约 86 行，另有约 40 行专属测试；已由本变更删除）曾用正则表达式把抓取到的 HTML 转成 markdown：剥离 script、style、noscript 标签与注释，转换 ``/``/`
  • `，解码数字实体外加一张 12 项的命名实体表，并折叠空白。该模块自身的 JSDoc 写明「A richer converter can replace it without changing the seam or tool schema」，README 的 Known Limitations 章节也把它记载为「a minimal regex converter, not an HTML parser — tables, images, and nested formatting are lost」。[web 能力 seam 决策记录](../architecture/2026-06-24-web-capability-seam.md)把 HTML 转 markdown 作为呈现职责划归本包（package），因此替换点恰好就在这里。每个抓取到的 HTML 页面上，该转换器的输出都对模型可见；此前没有任何无密钥快照执行到 `web_fetch`，因此没有预期输出固定它的行为。 + +## 决策 + +`packages/web/tool-web/src/fetch.ts` 持有一个模块级 [`turndown`](https://github.com/mixmark-io/turndown) 实例（`headingStyle: 'atx'`、`codeBlockStyle: 'fenced'`、`bulletListMarker: '-'`——固定的面向模型呈现方式，不是部署可调项），配合 `@joplin/turndown-plugin-gfm` 的组合 `gfm` 插件提供表格／删除线支持，并用 `remove(['script', 'style', 'noscript'])` 替代旧实现的整体剥离。`renderBody` 的 `html` 分支把调用包在 try/catch 中，失败时回退为原始 HTML 主体：正则版本从不可能抛异常，而 turndown/domino 的递归 DOM 遍历在数千层嵌套（实测：主线程 4k 层抛出，worker 线程 8k 层抛出）会以 `RangeError` 栈溢出，对提供方已经解码的主体来说，降级页面好过报错。`html.ts` 及其转换测试已删除；回退路径与状态头、截断页脚的格式化在 `tests/tool-web.spec.ts` 中有测试覆盖，README 的 Known Limitations 用病态嵌套回退条目替换了正则转换器的警示说明。gfm 插件不带类型声明；`src/turndown-plugin-gfm.d.ts` 基于 `@types/turndown`（devDependency）声明了唯一被导入的导出。 + +提案标记的依赖体积问题的裁决结果支持替换：`@deepseek-ai/dsh-tool-web` 在单文件可执行文件闭包内（[single-exe 决策记录](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)），可执行文件的资产 glob 会把这三个包按发布原样打入约 7.9 MB——但其中约 6 MB 是 `@mixmark-io/domino` 的测试语料（`test/**`），运行时 `lib/` 仅约 550 KB，相对约 174 MB 的产物，两种口径都不到 0.5%。 + +## 快照覆盖 + +此前缺失的无密钥 `web_fetch` 快照随本变更以 acp-agent 场景 `web-fetch` 落地：`examples/acp-agent/web.cordis.yml` 组合了 web seam、真实的 `dsh-web-fetch-local` 提供方、`search: false` 的 `tool-web`，以及 `web-fetch-fixture-server.mjs`——一个固定端口（抓取的 URL 是录制 transcript（文本记录）的一部分）上的回环 HTTP fixture，提供包含命名实体、GFM 表格与嵌套格式的确定性 HTML。录制与无密钥回放都驱动真实的 HTTP 抓取与转换；固定住的工具结果就是 turndown 的输出，该场景同时固定 `web` header 类（`web_fetch` 的 schema 与指引）。 + +## 曾考虑的替代方案 + +- **`@mozilla/readability` 加一个 DOM。** 它解决的是另一个问题（内容提取，而非格式转换），还会拖入更重的 DOM 依赖；这个 seam 只要求把抓取返回的内容渲染成 markdown。 +- **保留正则转换器。** 按其自身 JSDoc 的说法，它本来就是明确的 v1 占位实现；保留它意味着模型可见的质量（表格、图片、嵌套格式）继续缺失，代价还是维护一套自制实体表。 +- **仅引入 `entities` 的最小变体。** 提案中的退守方案：只用零依赖的 `entities` 包替换 `html.ts` 中的实体解码部分，删得更少但完全避开依赖体积问题。未采纳：上述闭包测算表明体积无关紧要，而完整替换能删掉整个手写转换器及其记录在案的质量缺口。 +- **用原版 `turndown-plugin-gfm` 而非 `@joplin/turndown-plugin-gfm`。** 原版已无人维护（最后发布于 2018 年）；Joplin 分叉与 turndown 7 保持同步并持续发布。 + +## 后果 + +- **收益**：模型可见的完整保真 markdown——表格、图片、删除线、嵌套强调、围栏代码块以及完整的命名实体集——并删除了自制转换器及其实体表，README 中的正则转换器警示收窄为一个退化用例。 +- **代价**：两个运行时依赖（`turndown` → `@mixmark-io/domino`）进入 tool-web 进而进入可执行文件闭包（如上实测约 550 KB 运行时代码），并新增一种失败模式——病态嵌套改为回退原始 HTML 而非转换。 +- 每个抓取到的 HTML 页面上模型可见的输出都已变化；旧输出本无任何固定，新快照固定了新输出。 + +## 测试 + +- `packages/web/tool-web/tests/tool-web.spec.ts` 通过 `renderBody` 覆盖 turndown 转换面（实体、链接、表格、嵌套、script/style/noscript 移除），并用实测可稳定溢出的 2 万层嵌套输入覆盖原始 HTML 回退；该包 src 的逐文件覆盖率为 100%。 +- acp-agent 的 `web-fetch` 快照无密钥地端到端固定组装后的行为（真实 Loader 组合、真实 HTTP 抓取、真实转换）。 diff --git a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md b/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md deleted file mode 100644 index 7f25e51bf6..0000000000 --- a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md +++ /dev/null @@ -1,32 +0,0 @@ -# Agent Note: Replace tool-web's regex HTML-to-markdown converter with turndown - -Status: proposed - -English | [中文](2026-07-26-turndown-for-tool-web-html-markdown.zh.md) - -## Problem - -`packages/web/tool-web/src/html.ts` (~86 lines, ~40 lines of dedicated tests) converts fetched HTML to markdown with regexes: strip script/style/noscript/comments, convert ``/``/`
  • `, decode numeric entities plus a 12-entry named-entity table, collapse whitespace. The module's own JSDoc says "A richer converter can replace it without changing the seam or tool schema", and the README's Known Limitations documents it as "a minimal regex converter, not an HTML parser — tables, images, and nested formatting are lost." The [web capability seam note](../../implemented/architecture/2026-06-24-web-capability-seam.md) assigns HTML→markdown to this package as presentation, so the swap point is exactly here. The converter's output is model-visible on every fetched HTML page; no keyless snapshot currently exercises `web_fetch`, so no expected outputs pin it. - -## Proposal - -Replace `htmlToMarkdown` with `turndown` (`new TurndownService().turndown(html)`), optionally with `turndown-plugin-gfm` for tables. The consumer switch in `fetch.ts` and the status-header/truncation-footer formatting stay. Wrap the call in try/catch falling back to the raw text path: the regex version could never throw; turndown on pathological HTML could. Delete `html.ts` and its conversion tests; keep tests for the fallback and the surrounding formatting. Update the README's Known Limitations to drop the regex-converter caveat. - -If the "deliberately minimal fallback" stance is preferred instead, a minimal variant still deletes the worst part: replace the entity-decoding third of the file (~30 lines: `decodeEntities`, `NAMED_ENTITIES`, `safeFromCodePoint`) with the zero-dependency `entities` package (already in the lockfile transitively), erasing the documented "about a dozen entities" limitation at near-zero risk. - -## Alternatives considered - -- **`@mozilla/readability` + a DOM.** Solves a different problem (content extraction, not conversion) and drags a heavier DOM dependency; the seam only asks for markdown rendering of whatever the fetch returned. -- **Keep the regex converter.** It was an explicit v1 placeholder per its own JSDoc; keeping it means model-visible quality (tables, images, nested formatting) stays lost for the cost of maintaining bespoke entity tables. -- **The minimal `entities`-only variant.** Kept in the proposal as the fallback position; it deletes less but avoids the dependency-weight question entirely. - -## Acceptance criteria - -- `web_fetch` renders tables/nested formatting via turndown (or, minimal variant: decodes all named entities), with the README limitation updated. -- Unit tests cover the fallback path; `pnpm run test` passes for the package. -- A keyless snapshot exercising `web_fetch` markdown rendering is added per testing policy (the missing snapshot coverage is part of the change, and it pins the new output). - -## Risks - -- Model-visible output changes on every fetched HTML page — transcript drift is acceptable pre-release, and nothing currently pins the old output. -- Dependency weight: turndown's one dependency (`@mixmark-io/domino`) is a ~200 KB DOM that would enter the single-file-executable closure if tool-web ships in it ([single-exe note](../../implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)); the minimal `entities` variant avoids this if closure size is the deciding factor. diff --git a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md b/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md deleted file mode 100644 index 3a59b08e13..0000000000 --- a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md +++ /dev/null @@ -1,32 +0,0 @@ -# Agent Note: 用 turndown 替换 tool-web 的正则 HTML 转 markdown 转换器 - -Status: proposed - -[English](2026-07-26-turndown-for-tool-web-html-markdown.md) | 中文 - -## 问题 - -`packages/web/tool-web/src/html.ts`（约 86 行，另有约 40 行专属测试）用正则表达式把抓取到的 HTML 转成 markdown：剥离 script、style、noscript 标签与注释，转换 ``/``/`
  • `，解码数字实体外加一张 12 项的命名实体表，并折叠空白。该模块自身的 JSDoc 写明「A richer converter can replace it without changing the seam or tool schema」，README 的 Known Limitations 章节也把它记载为「a minimal regex converter, not an HTML parser — tables, images, and nested formatting are lost」。[web 能力 seam 决策记录](../../implemented/architecture/2026-06-24-web-capability-seam.md)把 HTML 转 markdown 作为呈现职责划归本包（package），因此替换点恰好就在这里。每个抓取到的 HTML 页面上，该转换器的输出都对模型可见；当前没有任何无密钥快照执行到 `web_fetch`，因此没有预期输出固定它的行为。 - -## 提案 - -用 `turndown` 替换 `htmlToMarkdown`（`new TurndownService().turndown(html)`），可选择配合 `turndown-plugin-gfm` 支持表格。`fetch.ts` 中的消费方分支与状态头、截断页脚的格式化保持不变。把调用包在 try/catch 中，失败时回退到原始文本路径：正则版本从不可能抛异常，而 turndown 处理病态 HTML 时可能抛出。删除 `html.ts` 及其转换测试；保留回退路径与外围格式化的测试。更新 README 的 Known Limitations 章节，移除正则转换器的警示说明。 - -如果更倾向于「刻意保持最小回退实现」的立场，最小变体仍能删掉最糟的部分：用零依赖的 `entities` 包（已通过传递依赖存在于 lockfile 中）替换文件中占三分之一的实体解码部分（约 30 行：`decodeEntities`、`NAMED_ENTITIES`、`safeFromCodePoint`），以近乎为零的风险抹掉文档记载的「about a dozen entities」限制。 - -## 曾考虑的替代方案 - -- **`@mozilla/readability` 加一个 DOM。** 它解决的是另一个问题（内容提取，而非格式转换），还会拖入更重的 DOM 依赖；这个 seam 只要求把抓取返回的内容渲染成 markdown。 -- **保留正则转换器。** 按其自身 JSDoc 的说法，它本来就是明确的 v1 占位实现；保留它意味着模型可见的质量（表格、图片、嵌套格式）继续缺失，代价还是维护一套自制实体表。 -- **仅引入 `entities` 的最小变体。** 已作为退守方案保留在提案中；它删得更少，但完全避开了依赖体积问题。 - -## 验收标准 - -- `web_fetch` 经由 turndown 渲染表格与嵌套格式（或在最小变体下：解码全部命名实体），README 中的限制说明同步更新。 -- 单元测试覆盖回退路径；该包的 `pnpm run test` 通过。 -- 按测试政策补充一个执行 `web_fetch` markdown 渲染的无密钥快照（缺失的快照覆盖是本变更的一部分，它同时固定新输出）。 - -## 风险 - -- 模型可见的输出在每个抓取到的 HTML 页面上都会变化：预发布阶段的 transcript（文本记录）漂移可以接受，且当前没有任何东西固定旧输出。 -- 依赖体积：turndown 的唯一依赖（`@mixmark-io/domino`）是一个约 200 KB 的 DOM 实现，若 tool-web 进入单文件可执行文件，它会一并进入闭包（[single-exe 决策记录](../../implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)）；若闭包体积是决定因素，最小的 `entities` 变体可以避开这一点。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 9880761894..30499c7df6 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1672,7 +1672,7 @@ export interface Config { } ``` -Source: [`packages/web/tool-web/src/index.ts:29`](../packages/web/tool-web/src/index.ts) +Source: [`packages/web/tool-web/src/index.ts:28`](../packages/web/tool-web/src/index.ts) ## `@deepseek-ai/dsh-tool-workflow` diff --git a/examples/acp-agent/README.i18n.yaml b/examples/acp-agent/README.i18n.yaml index 22842fcd04..391967d91c 100644 --- a/examples/acp-agent/README.i18n.yaml +++ b/examples/acp-agent/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -README.md: 4b3d86b00613cc7c37a8898ef3b39d40a167e66b -README.zh.md: 5bcd85f2b4ae34a11b980bf196d3401f764004d8 +README.md: 0d63ec1f2d9165b9faf0817bd94fbe15b97fa961 +README.zh.md: 0c5f8866ea640843513fd9a4c15a17ed4db59d3b diff --git a/examples/acp-agent/README.md b/examples/acp-agent/README.md index 4b3d86b006..0d63ec1f2d 100644 --- a/examples/acp-agent/README.md +++ b/examples/acp-agent/README.md @@ -9,7 +9,7 @@ pnpm run demo:acp # needs DEEPSEEK_API_KEY (repo-root .env or env) pnpm run demo:code-mode acp # same protocol with the Code Mode tool transport ``` -The leaf loads the ACP app, DeepSeek adapter, sandboxed bash and filesystem stacks, one-shot approval policy, compaction, subagents, workflows, hooks, a derived session-query index, and repeat guard. The app creates one fresh agent per `session/new`, persists sessions to JSONL, and keeps stdout protocol-pure. [`session-query.cordis.yml`](session-query.cordis.yml) explicitly opts into the workspace-authorized query tools and generic timeout/spill policies for their dedicated snapshot; [`fs.cordis.yml`](fs.cordis.yml) adds spill storage for filesystem scenarios, while [`code-mode.cordis.yml`](code-mode.cordis.yml) adds `run_code` and its generated TypeScript SDK. +The leaf loads the ACP app, DeepSeek adapter, sandboxed bash and filesystem stacks, one-shot approval policy, compaction, subagents, workflows, hooks, a derived session-query index, and repeat guard. The app creates one fresh agent per `session/new`, persists sessions to JSONL, and keeps stdout protocol-pure. [`session-query.cordis.yml`](session-query.cordis.yml) explicitly opts into the workspace-authorized query tools and generic timeout/spill policies for their dedicated snapshot; [`fs.cordis.yml`](fs.cordis.yml) adds spill storage for filesystem scenarios, [`code-mode.cordis.yml`](code-mode.cordis.yml) adds `run_code` and its generated TypeScript SDK, and [`web.cordis.yml`](web.cordis.yml) adds the web seam, the local fetch provider, `web_fetch`, and a loopback HTML fixture server for the web-fetch snapshot. ## Protocol channel diff --git a/examples/acp-agent/README.zh.md b/examples/acp-agent/README.zh.md index 5bcd85f2b4..0c5f8866ea 100644 --- a/examples/acp-agent/README.zh.md +++ b/examples/acp-agent/README.zh.md @@ -9,7 +9,7 @@ pnpm run demo:acp # needs DEEPSEEK_API_KEY (repo-root .env or env) pnpm run demo:code-mode acp # same protocol with the Code Mode tool transport ``` -该叶节点加载 ACP 应用、DeepSeek 适配器、受沙箱限制的 bash 与文件系统栈、一次性批准策略、压缩（compaction）、subagent、工作流、钩子、派生会话查询索引和重复守卫。应用为每次 `session/new` 创建一个新 agent，将会话持久化到 JSONL，并保持 stdout 只含协议内容。[`session-query.cordis.yml`](session-query.cordis.yml) 为其专用快照显式选用 workspace 授权的查询工具和通用超时／溢出策略；[`fs.cordis.yml`](fs.cordis.yml) 为文件系统场景添加溢出存储，[`code-mode.cordis.yml`](code-mode.cordis.yml) 则添加 `run_code` 及其生成的 TypeScript SDK。 +该叶节点加载 ACP 应用、DeepSeek 适配器、受沙箱限制的 bash 与文件系统栈、一次性批准策略、压缩（compaction）、subagent、工作流、钩子、派生会话查询索引和重复守卫。应用为每次 `session/new` 创建一个新 agent，将会话持久化到 JSONL，并保持 stdout 只含协议内容。[`session-query.cordis.yml`](session-query.cordis.yml) 为其专用快照显式选用 workspace 授权的查询工具和通用超时／溢出策略；[`fs.cordis.yml`](fs.cordis.yml) 为文件系统场景添加溢出存储，[`code-mode.cordis.yml`](code-mode.cordis.yml) 添加 `run_code` 及其生成的 TypeScript SDK，[`web.cordis.yml`](web.cordis.yml) 则为 web-fetch 快照添加 web seam、本地抓取提供方、`web_fetch` 与一个回环 HTML fixture 服务器。 ## 协议通道 diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 8e74711a57..ed4a7d6a58 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -41,6 +41,7 @@ const PACKED_CHUNKS_CONFIG = fileURLToPath(new URL('../packed-chunks.cordis.yml' const SESSION_SANDBOX_ROOT_CONFIG = fileURLToPath(new URL('../session-sandbox-root.cordis.yml', import.meta.url)) const RETRY_CONFIG = fileURLToPath(new URL('../retry.cordis.yml', import.meta.url)) const LSP_CONFIG = fileURLToPath(new URL('./lsp.cordis.yml', import.meta.url)) +const WEB_CONFIG = fileURLToPath(new URL('../web.cordis.yml', import.meta.url)) const SNAPSHOTS_DIR = join(dirname(fileURLToPath(import.meta.url)), 'snapshots') const PACKED_CHUNKS_SOURCE = 'hook-cc-pretool-deny' @@ -110,6 +111,12 @@ const SCENARIOS: Scenario[] = [ { name: 'todo-write', hasModelTurn: true, recorded: true }, { name: 'skill-load', hasModelTurn: true, recorded: false, pinsHeader: true, headerClass: 'skill' }, { name: 'lsp-definition', hasModelTurn: true, recorded: false, pinsHeader: true, headerClass: 'lsp', configPath: LSP_CONFIG }, + // web_fetch markdown rendering end to end: the overlay's loopback fixture + // server supplies deterministic HTML (entities, a GFM table, nesting), the + // REAL local fetch provider retrieves it, and the tool result pins the + // turndown conversion. The fetched URL (fixed port) is part of the recorded + // transcript; replay re-executes the real fetch against the same fixture. + { name: 'web-fetch', hasModelTurn: true, recorded: true, pinsHeader: true, headerClass: 'web', configPath: WEB_CONFIG }, { name: 'workspace-edit', hasModelTurn: true, diff --git a/examples/acp-agent/tests/snapshots/web-fetch/input.json b/examples/acp-agent/tests/snapshots/web-fetch/input.json new file mode 100644 index 0000000000..dc1993235d --- /dev/null +++ b/examples/acp-agent/tests/snapshots/web-fetch/input.json @@ -0,0 +1,7 @@ +{ + "steps": [ + { "op": "initialize" }, + { "op": "newSession" }, + { "op": "prompt", "text": "Use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly DONE. Do not describe the content." } + ] +} diff --git a/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl b/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl new file mode 100644 index 0000000000..c6c34bc8e3 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl @@ -0,0 +1,127 @@ +{"type":"session","version":0,"id":"c12fa9af-1042-4a92-9ba4-4a968ff23495","createdAt":1785078727712,"cwd":"/tmp/acp-snap-cwd-hqkZWE","delegationDepth":0} +{"type":"turn/start","seq":0,"time":1785078727718,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}} +{"type":"user/message","seq":1,"time":1785078727719,"data":{"content":[{"type":"text","text":"Use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly DONE. Do not describe the content."}],"source":{"kind":"user"}},"surfaceOp":"append"} +{"type":"session/title","seq":2,"time":1785078727721,"data":{"title":"Use the web_fetch tool exactly","messageSeqs":[1],"source":{"kind":"fallback"}}} +{"type":"step/start","seq":3,"time":1785078727730,"data":{"turn":1,"step":1}} +{"type":"request/header","seq":4,"time":1785078727731,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-pro"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} +{"type":"assistant/chunk","seq":5,"time":1785078728804,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} +{"type":"assistant/chunk","seq":6,"time":1785078728805,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}} +{"type":"assistant/chunk","seq":7,"time":1785078728943,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}} +{"type":"assistant/chunk","seq":8,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" wants"}}} +{"type":"assistant/chunk","seq":9,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} +{"type":"assistant/chunk","seq":10,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} +{"type":"assistant/chunk","seq":11,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" use"}}} +{"type":"assistant/chunk","seq":12,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}} +{"type":"assistant/chunk","seq":13,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" web"}}} +{"type":"assistant/chunk","seq":14,"time":1785078729038,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"_f"}}} +{"type":"assistant/chunk","seq":15,"time":1785078729038,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"etch"}}} +{"type":"assistant/chunk","seq":16,"time":1785078729039,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" tool"}}} +{"type":"assistant/chunk","seq":17,"time":1785078729039,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} +{"type":"assistant/chunk","seq":18,"time":1785078729085,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" once"}}} +{"type":"assistant/chunk","seq":19,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} +{"type":"assistant/chunk","seq":20,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" fetch"}}} +{"type":"assistant/chunk","seq":21,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" http"}}} +{"type":"assistant/chunk","seq":22,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"://"}}} +{"type":"assistant/chunk","seq":23,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"127"}}} +{"type":"assistant/chunk","seq":24,"time":1785078729132,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"assistant/chunk","seq":25,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"0"}}} +{"type":"assistant/chunk","seq":26,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"assistant/chunk","seq":27,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"0"}}} +{"type":"assistant/chunk","seq":28,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"assistant/chunk","seq":29,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"1"}}} +{"type":"assistant/chunk","seq":30,"time":1785078729182,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":":"}}} +{"type":"assistant/chunk","seq":31,"time":1785078729182,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"431"}}} +{"type":"assistant/chunk","seq":32,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"17"}}} +{"type":"assistant/chunk","seq":33,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"/m"}}} +{"type":"assistant/chunk","seq":34,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"enu"}}} +{"type":"assistant/chunk","seq":35,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":".html"}}} +{"type":"assistant/chunk","seq":36,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":","}}} +{"type":"assistant/chunk","seq":37,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" then"}}} +{"type":"assistant/chunk","seq":38,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} +{"type":"assistant/chunk","seq":39,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} +{"type":"assistant/chunk","seq":40,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} +{"type":"assistant/chunk","seq":41,"time":1785078729231,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} +{"type":"assistant/chunk","seq":42,"time":1785078729276,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} +{"type":"assistant/chunk","seq":43,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} +{"type":"assistant/chunk","seq":44,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} +{"type":"assistant/chunk","seq":45,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" Let"}}} +{"type":"assistant/chunk","seq":46,"time":1785078729322,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} +{"type":"assistant/chunk","seq":47,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" do"}}} +{"type":"assistant/chunk","seq":48,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" that"}}} +{"type":"assistant/chunk","seq":49,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"assistant/chunk","seq":50,"time":1785078729463,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"tool-call"}}} +{"type":"assistant/chunk","seq":51,"time":1785078729464,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":""}}} +{"type":"assistant/chunk","seq":52,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"{"}}} +{"type":"assistant/chunk","seq":53,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} +{"type":"assistant/chunk","seq":54,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"url"}}} +{"type":"assistant/chunk","seq":55,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} +{"type":"assistant/chunk","seq":56,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":": "}}} +{"type":"assistant/chunk","seq":57,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} +{"type":"assistant/chunk","seq":58,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"http"}}} +{"type":"assistant/chunk","seq":59,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"://"}}} +{"type":"assistant/chunk","seq":60,"time":1785078729558,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"127"}}} +{"type":"assistant/chunk","seq":61,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} +{"type":"assistant/chunk","seq":62,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"0"}}} +{"type":"assistant/chunk","seq":63,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} +{"type":"assistant/chunk","seq":64,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"0"}}} +{"type":"assistant/chunk","seq":65,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} +{"type":"assistant/chunk","seq":66,"time":1785078729605,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"1"}}} +{"type":"assistant/chunk","seq":67,"time":1785078729651,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":":"}}} +{"type":"assistant/chunk","seq":68,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"431"}}} +{"type":"assistant/chunk","seq":69,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"17"}}} +{"type":"assistant/chunk","seq":70,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"/m"}}} +{"type":"assistant/chunk","seq":71,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"enu"}}} +{"type":"assistant/chunk","seq":72,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":".html"}}} +{"type":"assistant/chunk","seq":73,"time":1785078729697,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} +{"type":"assistant/chunk","seq":74,"time":1785078729698,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"}"}}} +{"type":"assistant/chunk","seq":75,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly \"DONE\". Let me do that."}}}} +{"type":"assistant/chunk","seq":76,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"tool-call","id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","arguments":"{\"url\": \"http://127.0.0.1:43117/menu.html\"}"}}}} +{"type":"assistant/chunk","seq":77,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":5405,"outputTokens":103,"cacheReadTokens":0,"reasoningTokens":44}}}} +{"type":"assistant/chunk","seq":78,"time":1785078729804,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} +{"type":"assistant/message","seq":79,"time":1785078729807,"data":{"turn":1,"step":1,"content":[{"type":"reasoning","text":"The user wants me to use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly \"DONE\". Let me do that."},{"type":"tool-call","id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","arguments":"{\"url\": \"http://127.0.0.1:43117/menu.html\"}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-pro"},"usage":{"inputTokens":5405,"outputTokens":103,"cacheReadTokens":0,"reasoningTokens":44}},"sourceEventSeqs":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78],"surfaceOp":"append"} +{"type":"tool/call","seq":80,"time":1785078729809,"data":{"turn":1,"step":1,"callId":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","arguments":"{\"url\": \"http://127.0.0.1:43117/menu.html\"}"}} +{"type":"tool/result","seq":81,"time":1785078729843,"data":{"turn":1,"step":1,"callId":"call_00_sxjOyfDYN07koiE7jiIa5326","content":[{"type":"text","text":"Fetched http://127.0.0.1:43117/menu.html (HTTP 200)\n\nMenu\n\n# Café menu\n\nPrices include **service & _tax_** — updated daily.\n\n- Espresso\n- Flat white\n\n| Drink | Price |\n| --- | --- |\n| Espresso | €2 |\n| Flat white | €3 |\n\nSee [today’s specials](https://fixture.invalid/specials)."}],"isError":false},"sourceEventSeqs":[80],"surfaceOp":"append"} +{"type":"step/end","seq":82,"time":1785078729847,"data":{"turn":1,"step":1}} +{"type":"step/start","seq":83,"time":1785078729848,"data":{"turn":1,"step":2}} +{"type":"assistant/chunk","seq":84,"time":1785078730611,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} +{"type":"assistant/chunk","seq":85,"time":1785078730612,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}} +{"type":"assistant/chunk","seq":86,"time":1785078730770,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}} +{"type":"assistant/chunk","seq":87,"time":1785078730824,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" asked"}}} +{"type":"assistant/chunk","seq":88,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} +{"type":"assistant/chunk","seq":89,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} +{"type":"assistant/chunk","seq":90,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" fetch"}}} +{"type":"assistant/chunk","seq":91,"time":1785078730861,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}} +{"type":"assistant/chunk","seq":92,"time":1785078730862,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" URL"}}} +{"type":"assistant/chunk","seq":93,"time":1785078730909,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":","}}} +{"type":"assistant/chunk","seq":94,"time":1785078730956,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" then"}}} +{"type":"assistant/chunk","seq":95,"time":1785078731002,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} +{"type":"assistant/chunk","seq":96,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} +{"type":"assistant/chunk","seq":97,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} +{"type":"assistant/chunk","seq":98,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} +{"type":"assistant/chunk","seq":99,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} +{"type":"assistant/chunk","seq":100,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} +{"type":"assistant/chunk","seq":101,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} +{"type":"assistant/chunk","seq":102,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" I"}}} +{"type":"assistant/chunk","seq":103,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"'ve"}}} +{"type":"assistant/chunk","seq":104,"time":1785078731051,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" fetched"}}} +{"type":"assistant/chunk","seq":105,"time":1785078731097,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" it"}}} +{"type":"assistant/chunk","seq":106,"time":1785078731140,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"assistant/chunk","seq":107,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" Now"}}} +{"type":"assistant/chunk","seq":108,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" I"}}} +{"type":"assistant/chunk","seq":109,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" just"}}} +{"type":"assistant/chunk","seq":110,"time":1785078731189,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} +{"type":"assistant/chunk","seq":111,"time":1785078731189,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} +{"type":"assistant/chunk","seq":112,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} +{"type":"assistant/chunk","seq":113,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} +{"type":"assistant/chunk","seq":114,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} +{"type":"assistant/chunk","seq":115,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} +{"type":"assistant/chunk","seq":116,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":1,"blockType":"text"}}} +{"type":"assistant/chunk","seq":117,"time":1785078731236,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":1,"text":"D"}}} +{"type":"assistant/chunk","seq":118,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":1,"text":"ONE"}}} +{"type":"assistant/chunk","seq":119,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user asked me to fetch the URL, then reply with exactly \"DONE\". I've fetched it. Now I just reply with \"DONE\"."}}}} +{"type":"assistant/chunk","seq":120,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"DONE"}}}} +{"type":"assistant/chunk","seq":121,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":239,"outputTokens":34,"cacheReadTokens":5376,"reasoningTokens":31}}}} +{"type":"assistant/chunk","seq":122,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} +{"type":"assistant/message","seq":123,"time":1785078731283,"data":{"turn":1,"step":2,"content":[{"type":"reasoning","text":"The user asked me to fetch the URL, then reply with exactly \"DONE\". I've fetched it. Now I just reply with \"DONE\"."},{"type":"text","text":"DONE"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-pro"},"usage":{"inputTokens":239,"outputTokens":34,"cacheReadTokens":5376,"reasoningTokens":31}},"sourceEventSeqs":[84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122],"surfaceOp":"append"} +{"type":"step/end","seq":124,"time":1785078731286,"data":{"turn":1,"step":2}} +{"type":"turn/end","seq":125,"time":1785078731286,"data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/examples/acp-agent/tests/snapshots/web-fetch/stdout.expected.jsonl b/examples/acp-agent/tests/snapshots/web-fetch/stdout.expected.jsonl new file mode 100644 index 0000000000..82ae8907ca --- /dev/null +++ b/examples/acp-agent/tests/snapshots/web-fetch/stdout.expected.jsonl @@ -0,0 +1,4 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"DONE"}}}} +{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/examples/acp-agent/tests/snapshots/web-fetch/system-prompt.expected.md b/examples/acp-agent/tests/snapshots/web-fetch/system-prompt.expected.md new file mode 100644 index 0000000000..45705db0a5 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/web-fetch/system-prompt.expected.md @@ -0,0 +1,27 @@ +You are an AI agent powered by the DeepSeek Harness SDK. + +You are a coding assistant powered by the deepseek-v4-pro model. Your working directory is {{cwd}}. Your bash tool runs under a file sandbox — a `[sandbox: file access denied …]` result is policy, not a command bug. + +Verify your work by running the code or tests. Keep answers brief and factual. + + +Use the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files. + +Use the write tool to create files or completely replace file contents. Existing files are overwritten, so read an existing file first (the default fs-policy requires it) and prefer edit for targeted changes. + +Use the edit tool for targeted changes to existing UTF-8 text files. It replaces literal old_string with new_string; by default old_string must appear exactly once. If old_string appears multiple times, provide a more specific old_string or set replace_all to true. Read the file first (the default fs-policy requires it), unless you just created or edited it in this session. + +Check the [exit code: N] marker on every bash result; investigate failures before moving on. + +Track every background task id you start. You are notified in-session when a task finishes — do not busy-poll or sleep on one; keep working on independent steps and do not duplicate a running task's work. Before giving a final answer, collect every still-relevant task with task_output (set wait: true only when you are genuinely blocked on it), and task_kill tasks that stopped mattering. + +Use the web_fetch tool to retrieve the content of a specific HTTP(S) URL (for example a result from web_search). It returns the page content decoded to text. Cite the URL as a markdown link when you use its content. + +Use goal tools for one long-running completion objective in the current session. create_goal may infer goal intent from a direct human request in any language; do not create a goal for routine single-turn work. Call get_goal before update_goal and copy its exact goal_id and revision. After session resume or fork, an active goal is disarmed: when a human asks to continue or resume in any wording or language, use update_goal action resume to rearm it. Mark complete only when the objective is actually achieved. Mark blocked only after the same blocking condition persists for at least 3 consecutive goal rounds, and report that concrete condition in blocked_reason; difficulty, uncertainty, or useful remaining work is not blocked. + +Approval prompts are disabled in this session: actions that require approval are rejected automatically — do not request sandbox escalation (do not set `sandbox_permissions`). + + +Use the workflow tool ONLY when the user explicitly asks for a workflow or for large multi-agent orchestration: you write a JavaScript script (the tool description documents the exact format) that fans work out across many subagents with phases and structured results. For one or two delegations, prefer plain subagent calls. + +Use the ralph tool ONLY when the direct human explicitly asks for a Ralph loop or fresh-agent iterative execution. Each Ralph round starts a fresh child with no conversation seed and uses the shared workspace as durable memory. Completion and blockers are worker reports, not independent evaluation. Use same-session goal tools for ordinary long-running objectives, and plain subagents or workflows for bounded delegation and fan-out. diff --git a/examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json b/examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json new file mode 100644 index 0000000000..1ee86b38ba --- /dev/null +++ b/examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json @@ -0,0 +1,489 @@ +{ + "initial": [ + { + "name": "bash", + "description": "Execute a bash command (`bash -c`) and return its stdout/stderr. Each call runs in a fresh shell: no state (cwd, variables, functions) persists between calls — pass `workdir` instead of using `cd`. Non-zero exits are reported as `[exit code: N]`. Current harness environment facts are exposed through managed `$DSH_*` variables; inspect them when needed. Commands may run under a file sandbox; a blocked file operation is reported as `[sandbox: file access denied under mode]` — a policy denial, not a bug in the command; do not retry another way. Long output is truncated to its tail; the full output is saved to a file whose path is reported when available. Set `run_in_background: true` for long-running commands: the call returns a task id immediately; read its output with `task_output` and stop it with `task_kill`. Attempting a command the sandbox may deny is safe and expected: run it and read the marker rather than assuming the denial. When a command is denied and a wider mode would let it succeed, escalate immediately in the same turn — the one sanctioned exception to a denial: retry the exact same command once with `sandbox_permissions` (the narrowest wider mode that suffices) plus a one-sentence `justification`. Do not detour through chat to ask permission first — the approval prompt raised by that retry is how the user consents. If the session states approval prompts are disabled, there is no exception: a denial is final — do not set `sandbox_permissions`. Never escalate speculatively: ground the request in a real denial — normally the one this command just hit; escalating up front is fine only when this session already denied the same access. A rejected escalation is final for that command — stop and explain, never work around it — but it does not forbid attempting or escalating other commands later.", + "parameters": { + "type": "object", + "properties": { + "command": { + "type": "string", + "description": "The bash command to execute." + }, + "description": { + "type": "string", + "description": "Clear, concise description of what this command does in active voice, 5-10 words (shown in the UI). Examples: \"ls\" → \"List files in current directory\"; \"git status\" → \"Show working tree status\"; \"npm install\" → \"Install package dependencies\"." + }, + "timeoutMs": { + "type": "number", + "description": "Timeout in milliseconds. The executor applies its configured default and cap, and kills the command on expiry." + }, + "workdir": { + "type": "string", + "description": "Working directory for this command. Defaults to the session workspace; a relative path is resolved against it." + }, + "run_in_background": { + "type": "boolean", + "description": "Run in the background and return a task id immediately (collect with task_output, stop with task_kill). No timeout applies." + }, + "sandbox_permissions": { + "type": "string", + "description": "The wider sandbox mode this command needs. Only valid as a one-shot retry of a command the sandbox just denied; requires justification and user approval.", + "enum": [ + "workspace-write", + "danger-full-access" + ] + }, + "justification": { + "type": "string", + "description": "Required with sandbox_permissions: one sentence for the user explaining why this exact command needs the wider access." + } + }, + "required": [ + "command", + "description" + ] + } + }, + { + "name": "create_goal", + "description": "Create one persisted same-session completion goal when the current direct human request is a long-running objective that should continue across autonomous goal rounds. You may infer that intent without requiring the user to say \"create a goal\". Do not use this for trivial single-turn work. Execution rejects non-human and subagent authority.", + "parameters": { + "type": "object", + "properties": { + "objective": { + "type": "string", + "description": "The concrete completion objective inferred from the direct human request." + }, + "max_goal_rounds": { + "type": "number", + "description": "Optional positive safe-integer limit on automatic continuation rounds." + } + }, + "required": [ + "objective" + ] + } + }, + { + "name": "edit", + "description": "Edit an existing UTF-8 text file by replacing literal text.", + "parameters": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Path to edit, resolved by the filesystem backend." + }, + "old_string": { + "type": "string", + "description": "Literal text to replace. Must match exactly." + }, + "new_string": { + "type": "string", + "description": "Literal replacement text. Use an empty string to delete the match." + }, + "replace_all": { + "type": "boolean", + "description": "Replace all matches. Defaults to false; when false, old_string must appear exactly once." + }, + "sandbox_permissions": { + "type": "string", + "description": "The wider sandbox mode this file operation needs. Only valid as a one-shot retry of an operation the sandbox just denied; requires justification and user approval.", + "enum": [ + "workspace-write", + "danger-full-access" + ] + }, + "justification": { + "type": "string", + "description": "Required with sandbox_permissions: one sentence for the user explaining why this exact file operation needs the wider access." + } + }, + "required": [ + "file_path", + "old_string", + "new_string" + ] + } + }, + { + "name": "get_goal", + "description": "Read the current same-session goal, including its exact id/revision, objective, phase, completed continuation rounds, round limit, blocker reason when present, and whether another continuation is armed. Call this before updating a goal.", + "parameters": { + "type": "object", + "properties": {} + } + }, + { + "name": "ralph", + "description": "Run a foreground fresh-agent Ralph loop toward one immutable objective. Use only when the direct human explicitly asks for Ralph or fresh-agent iteration. Each round opens a new child with no parent conversation or prior child session; the shared workspace is long-term memory, and only a bounded structured report crosses rounds. The call returns when a worker reports completion or a concrete blocker, or at the round limit. Ordinary long-running same-session work belongs to goal tools.", + "parameters": { + "type": "object", + "properties": { + "objective": { + "type": "string", + "description": "The immutable completion objective for every fresh Ralph round." + }, + "maxRounds": { + "type": "number", + "description": "Optional positive safe-integer round cap, bounded by the deployment ceiling." + } + }, + "required": [ + "objective" + ] + } + }, + { + "name": "read", + "description": "Read a UTF-8 text file and return line-numbered content.", + "parameters": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Path to read, resolved by the filesystem backend." + }, + "offset": { + "type": "number", + "description": "1-based first line to return. Defaults to 1." + }, + "limit": { + "type": "number", + "description": "Maximum number of lines to return. Defaults to 2000." + } + }, + "required": [ + "file_path" + ] + } + }, + { + "name": "skill", + "description": "Load the full instructions for an available skill. Call this with the exact skill name from the session skill catalog before acting on a task that names or clearly matches that skill.", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The exact skill name from the available skills list." + } + }, + "required": [ + "name" + ] + } + }, + { + "name": "subagent", + "description": "Delegate a self-contained task to a subagent (a separate agent that works in its own context) and return its final result. Use this to offload focused, independent work — research, a scoped implementation, an analysis — so it does not consume this conversation's context. The subagent runs to completion and you receive only its final answer, not its intermediate steps. Give it a complete, standalone prompt: it does not see this conversation. Set `run_in_background: true` to return a task id; collect with `task_output` and stop with `task_kill`.", + "parameters": { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "A short (3-5 word) description of the delegated task, for display." + }, + "prompt": { + "type": "string", + "description": "The complete, self-contained task for the subagent. It does not share this conversation's context, so include everything it needs." + }, + "run_in_background": { + "type": "boolean", + "description": "Run as a background task and return its id; collect with task_output or stop with task_kill." + } + }, + "required": [ + "description", + "prompt" + ] + } + }, + { + "name": "subagent_fork", + "description": "Delegate a task to a subagent that inherits this conversation: a child agent seeded with all completed turns so far (it does not see the current in-flight turn), returning only its final result. Use this when the subtask builds on this conversation's context — a follow-up analysis, a review, a continuation — without consuming this conversation's context for the work itself. You receive only its final answer, not its intermediate steps. Set `run_in_background: true` to return a task id; collect with `task_output` and stop with `task_kill`.", + "parameters": { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "A short (3-5 word) description of the delegated task, for display." + }, + "prompt": { + "type": "string", + "description": "The task for the subagent. It already sees this conversation's completed turns, so build on them freely and state only what is new." + }, + "run_in_background": { + "type": "boolean", + "description": "Run as a background task and return its id; collect with task_output or stop with task_kill." + } + }, + "required": [ + "description", + "prompt" + ] + } + }, + { + "name": "task_kill", + "description": "Request cancellation of a running background task by task id. Returns immediately; the task settles as killed once its work actually stops.", + "parameters": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "description": "Task id returned by the tool that started the background work." + }, + "reason": { + "type": "string", + "description": "Optional short reason, recorded in the log and forwarded to the task." + } + }, + "required": [ + "task_id" + ] + } + }, + { + "name": "task_list", + "description": "List your background tasks (running and finished) with their ids, kinds, and statuses.", + "parameters": { + "type": "object", + "properties": {} + } + }, + { + "name": "task_output", + "description": "Read a background task. Stream tasks return only output since the previous read; final-output tasks return their result after settlement. Every response ends with `[status: ...]`. Reads are non-blocking unless `wait: true`, which waits up to the configured cap.", + "parameters": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "description": "Task id returned by the tool that started the background work." + }, + "wait": { + "type": "boolean", + "description": "Block until the task reaches a terminal status or the timeout expires. A timed-out wait returns [status: running] and leaves the task alive." + }, + "timeout_ms": { + "type": "number", + "description": "Max wait in milliseconds (only meaningful with wait: true). Defaults to the configured wait timeout; capped by the configured maximum." + } + }, + "required": [ + "task_id" + ] + } + }, + { + "name": "todo_write", + "description": "Record and update a structured task list for the current work. Send the ENTIRE list every call — it REPLACES the previous list (there are no partial updates, no per-item edits). Use it to plan multi-step work and show progress: add one todo per concrete step before you start. Keep AT MOST ONE todo `in_progress` at a time; while work remains, exactly one active task should be `in_progress`. Mark a todo `completed` the moment it is done (do not batch completions), and allow no `in_progress` item only once all work is complete. Skip the list for trivial single-step tasks. Statuses: `pending` (not started), `in_progress` (being worked on now), `completed` (finished).", + "parameters": { + "type": "object", + "properties": { + "todos": { + "type": "array", + "description": "The COMPLETE task list, replacing any previous list.", + "items": { + "type": "object", + "additionalProperties": true, + "properties": { + "content": { + "type": "string", + "description": "What the task is — a short imperative line." + }, + "status": { + "type": "string", + "description": "pending (not started) | in_progress (now) | completed (done).", + "enum": [ + "pending", + "in_progress", + "completed" + ] + } + }, + "required": [ + "content", + "status" + ] + } + } + }, + "required": [ + "todos" + ] + } + }, + { + "name": "update_goal", + "description": "Update the exact current goal revision. edit, pause, and resume require a direct top-level human request. During an automatic continuation of the current goal, complete and blocked are also allowed. blocked is rejected before the configured minimum round count; the model remains responsible for judging that the same condition persisted across those rounds and must explain it in blocked_reason.", + "parameters": { + "type": "object", + "properties": { + "goal_id": { + "type": "string", + "description": "Exact id returned by get_goal." + }, + "revision": { + "type": "number", + "description": "Exact positive revision returned by get_goal." + }, + "action": { + "type": "string", + "description": "edit | pause | resume | complete | blocked", + "enum": [ + "edit", + "pause", + "resume", + "complete", + "blocked" + ] + }, + "objective": { + "type": "string", + "description": "Replacement objective; valid only with action edit." + }, + "max_goal_rounds": { + "type": "number", + "description": "Replacement cap; valid only with action edit." + }, + "blocked_reason": { + "type": "string", + "description": "Concrete blocking condition; required only with action blocked." + } + }, + "required": [ + "goal_id", + "revision", + "action" + ] + } + }, + { + "name": "web_fetch", + "description": "Fetch the content of a specific HTTP(S) URL and return it decoded to text.", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The HTTP(S) URL to fetch." + } + }, + "required": [ + "url" + ] + } + }, + { + "name": "workflow", + "description": "Run a JavaScript workflow script that orchestrates subagents at scale. Use this for work that fans out across many independent pieces — an audit over many files, a migration, multi-angle research, adversarial verification of findings — where you write the orchestration as a script instead of delegating turn by turn.\n\nThe workflow's identity rides the `meta` parameter as JSON: required `name` (short kebab-case) and `description` strings, optional `whenToUse` string and `phases` array (`{title, detail?, provider?, model?}`). The `script` parameter is the plain JavaScript body ONLY (NOT TypeScript, and NO `export const meta` statement — meta is a parameter, not code), running with top-level await; end with `return ` — the value must be JSON-serializable and is this tool's result.\n\nScript-body hooks:\n- `agent(prompt, opts?): Promise` — run one subagent to completion. Without `opts.schema` it resolves to the child's final text; with `opts.schema` (an object-rooted JSON Schema using ONLY type/properties/required/additionalProperties/items/enum/const/oneOf — no pattern/format/numeric bounds) it resolves to the validated object. Resolves `null` when the child fails (filter with `.filter(Boolean)`). Other opts: `label` (display), `phase` (progress group), and independent `provider`/`model` LLM target overrides (either may be provided alone). Anything else (`effort`/`isolation`/`agentType`) is rejected loudly.\n- `pipeline(items, ...stages): Promise` — run each item through the stages independently with NO barrier between stages (prefer this for multi-stage work). Each stage receives `(prev, item, index)`. An ordinary stage throw drops that ITEM to `null` and skips its remaining stages.\n- `parallel(thunks): Promise` — run zero-argument functions concurrently and await ALL of them (a barrier; use only when a stage genuinely needs every prior result together). A throwing thunk resolves to `null`.\n- `phase(title)` — start a progress phase; `log(message)` — narrate progress; `args` — the tool call's `args` input, verbatim.\n\nMisused hooks (bad arguments, unknown options, unsupported schemas, tripped caps) throw errors that ALWAYS kill the script — they never dissolve into a per-item `null`.\n\nConstraints: concurrency and total-agent caps apply; no filesystem, network, timers, or Node.js APIs are provided — the agents do the work, the script only coordinates them. The run executes in the foreground: this call returns when the whole script finishes.", + "parameters": { + "type": "object", + "properties": { + "script": { + "type": "string", + "description": "The plain-JS workflow script body (top-level await allowed; NO `export const meta` statement; end with `return `)." + }, + "meta": { + "type": "object", + "description": "The workflow identity block (plain JSON — never code).", + "additionalProperties": true, + "properties": { + "name": { + "type": "string", + "description": "Short kebab-case workflow name." + }, + "description": { + "type": "string", + "description": "One-line description of what the workflow does." + }, + "whenToUse": { + "type": "string", + "description": "Optional guidance on when this workflow applies." + }, + "phases": { + "type": "array", + "description": "Optional phase declarations matched by phase() calls.", + "items": { + "type": "object", + "additionalProperties": true, + "properties": { + "title": { + "type": "string", + "description": "The phase title phase() calls match by exact string." + }, + "detail": { + "type": "string", + "description": "Optional one-line description of the phase." + }, + "provider": { + "type": "string", + "description": "Optional provider override this phase is expected to use." + }, + "model": { + "type": "string", + "description": "Optional model override this phase is expected to use." + } + }, + "required": [ + "title" + ] + } + } + }, + "required": [ + "name", + "description" + ] + }, + "args": { + "type": "object", + "description": "Optional JSON input exposed to the script as the `args` global (wrap a bare list as a field, e.g. {\"files\": [...]}).", + "additionalProperties": true + } + }, + "required": [ + "script", + "meta" + ] + } + }, + { + "name": "write", + "description": "Create or fully replace a UTF-8 text file.", + "parameters": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Path to write, resolved by the filesystem backend." + }, + "content": { + "type": "string", + "description": "Full UTF-8 text content to write." + }, + "sandbox_permissions": { + "type": "string", + "description": "The wider sandbox mode this file operation needs. Only valid as a one-shot retry of an operation the sandbox just denied; requires justification and user approval.", + "enum": [ + "workspace-write", + "danger-full-access" + ] + }, + "justification": { + "type": "string", + "description": "Required with sandbox_permissions: one sentence for the user explaining why this exact file operation needs the wider access." + } + }, + "required": [ + "file_path", + "content" + ] + } + } + ], + "changes": [] +} diff --git a/examples/acp-agent/web-fetch-fixture-server.mjs b/examples/acp-agent/web-fetch-fixture-server.mjs new file mode 100644 index 0000000000..34a45fdd15 --- /dev/null +++ b/examples/acp-agent/web-fetch-fixture-server.mjs @@ -0,0 +1,52 @@ +/** + * Deterministic loopback HTTP fixture for the web-fetch snapshot scenario: a + * small HTML page (headings, named entities, a GFM table, nested formatting) + * on a fixed port, so recording and keyless replay drive the REAL + * `dsh-web-fetch-local` transport and `dsh-tool-web` markdown rendering + * without external network. The port is fixed because the fetched URL is part + * of the recorded model transcript. + */ +import { createServer } from 'node:http' + +/** Fixed loopback port the scenario prompt points `web_fetch` at. */ +const PORT = 43117 + +const PAGE = ` +Menu + +

    Café menu

    +

    Prices include service & tax — updated daily.

    +
    • Espresso
    • Flat white
    +
    DrinkPrice
    Espresso€2
    Flat white€3
    +

    See today’s specials.

    + +` + +/** Cordis plugin name. */ +export const name = 'web-fetch-fixture-server' + +/** + * Start the fixture server on 127.0.0.1 and register its shutdown. + * @param ctx - Cordis context; the effect disposes the server with the fiber. + */ +export async function apply(ctx) { + const server = createServer((req, res) => { + if (req.url === '/menu.html') { + res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }) + res.end(PAGE) + return + } + res.writeHead(404, { 'content-type': 'text/plain; charset=utf-8' }) + res.end('not found') + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(PORT, '127.0.0.1', () => resolve(undefined)) + }) + // The fixture must never hold the process open past protocol shutdown. + server.unref() + ctx.effect(() => () => { + server.close() + server.closeAllConnections() + }, 'web-fetch-fixture-server') +} diff --git a/examples/acp-agent/web.cordis.snapshot.yml b/examples/acp-agent/web.cordis.snapshot.yml new file mode 100644 index 0000000000..015e67e221 --- /dev/null +++ b/examples/acp-agent/web.cordis.snapshot.yml @@ -0,0 +1,31 @@ +# Keyless replay counterpart to web.cordis.yml: the web stack and loopback +# fixture server stay real (the tool call re-executes the actual HTTP fetch and +# markdown rendering); only the model adapter is replaced by replay. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - id: llm-deepseek + name: '@deepseek-ai/dsh-llm-deepseek' + disabled: true + - insert: + - id: web + name: '@deepseek-ai/dsh-web' + - id: web-fetch-local + name: '@deepseek-ai/dsh-web-fetch-local' + - id: web-fetch-fixture + name: './web-fetch-fixture-server.mjs' + - id: tool-web + name: '@deepseek-ai/dsh-tool-web' + config: + search: false + - id: llm-replay + name: '@deepseek-ai/dsh-llm-replay' + config: + providers: + - id: deepseek + name: DeepSeek + models: + - id: deepseek-v4-flash + - id: deepseek-v4-pro diff --git a/examples/acp-agent/web.cordis.yml b/examples/acp-agent/web.cordis.yml new file mode 100644 index 0000000000..1ed0b3efba --- /dev/null +++ b/examples/acp-agent/web.cordis.yml @@ -0,0 +1,21 @@ +# Web-fetch composition for the web-fetch snapshot scenario: the web seam, the +# real local HTTP fetch provider, the model-facing web tools (fetch only, so +# the pinned header carries exactly the surface under test), and the loopback +# fixture server the scenario prompt fetches — deterministic content, no +# external network, in recording and replay alike. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - insert: + - id: web + name: '@deepseek-ai/dsh-web' + - id: web-fetch-local + name: '@deepseek-ai/dsh-web-fetch-local' + - id: web-fetch-fixture + name: './web-fetch-fixture-server.mjs' + - id: tool-web + name: '@deepseek-ai/dsh-tool-web' + config: + search: false diff --git a/examples/package.json b/examples/package.json index 395c135a2d..3d0cc13718 100644 --- a/examples/package.json +++ b/examples/package.json @@ -63,6 +63,7 @@ "@deepseek-ai/dsh-tool-session-query": "workspace:*", "@deepseek-ai/dsh-tool-subagent": "workspace:*", "@deepseek-ai/dsh-tool-todo": "workspace:*", + "@deepseek-ai/dsh-tool-web": "workspace:*", "@deepseek-ai/dsh-tool-workflow": "workspace:*", "@deepseek-ai/dsh-tools": "workspace:*", "@deepseek-ai/dsh-user-approval": "workspace:*", diff --git a/packages/web/tool-web/README.i18n.yaml b/packages/web/tool-web/README.i18n.yaml index eb3fa4731d..1e746ed566 100644 --- a/packages/web/tool-web/README.i18n.yaml +++ b/packages/web/tool-web/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -README.md: 5e567115c386d14b7e412ed2502e7290826a5e5e -README.zh.md: b17fe4107908381806d4029481bbf03696c4f313 +README.md: 5fe48ced81a2cd02197cf8cc10a7d6567b17ffca +README.zh.md: 34ad08e290166ee6db2cd7b836746541d18aad52 diff --git a/packages/web/tool-web/README.md b/packages/web/tool-web/README.md index 5e567115c3..5fe48ced81 100644 --- a/packages/web/tool-web/README.md +++ b/packages/web/tool-web/README.md @@ -11,7 +11,7 @@ Each tool is registered independently; a product that wants only one disables th | Tool | Args | Behavior | |---|---|---| | `web_search` | `query` (string) | Discovery. Returns an optional answer plus source URLs. `max_results` is **not** model-facing — the tool sets the bound (the `searchMaxResults` config, default 8) and passes it to the seam. | -| `web_fetch` | `url` (string) | Retrieves a specific URL. HTML bodies are rendered to markdown-ish text; text bodies pass through. A non-2xx status is reported, not an error. The tool-call timeout is deployment policy (`dsh-timeout-policy`), not a model argument. | +| `web_fetch` | `url` (string) | Retrieves a specific URL. HTML bodies are rendered to markdown (turndown with GFM tables/strikethrough); text bodies pass through. A non-2xx status is reported, not an error. The tool-call timeout is deployment policy (`dsh-timeout-policy`), not a model argument. | Both tools opt into concurrent scheduling because provider reads return content without mutating parent-agent state. @@ -126,6 +126,6 @@ Append-only; newly visible content follows the reusable request prefix and does ## Known Limitations and Deferred Work -- **`htmlToMarkdown` is a minimal regex converter, not an HTML parser** — it strips script/style/noscript, keeps headings/bullets/links, and decodes about a dozen named entities; tables, images, and nested formatting are lost. +- **HTML→markdown conversion falls back to raw HTML on pathological input** — [turndown](https://github.com/mixmark-io/turndown) (with GFM tables/strikethrough) converts fetched HTML through a real DOM, but its recursive walk overflows on absurdly deep nesting (thousands of levels); such a body passes through unconverted rather than erroring ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)). - **The model-facing surface is minimal by design, with promotions deferred** — `max_results` stays a config bound (not a model argument), and `web_fetch` takes only `url` (no `format`/`prompt`/LLM-summarization mode); both are named later steps in [the seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md). - **No web-specific permission policy** — both tools execute without requesting `ctx.approval`; a deployment that needs confirmation must add a `tools/pre-execute` policy, and the package does not define persistent URL/domain grants. diff --git a/packages/web/tool-web/README.zh.md b/packages/web/tool-web/README.zh.md index b17fe41079..34ad08e290 100644 --- a/packages/web/tool-web/README.zh.md +++ b/packages/web/tool-web/README.zh.md @@ -11,7 +11,7 @@ | 工具 | 参数 | 行为 | |---|---|---| | `web_search` | `query`（string） | 发现。返回可选答案与源 URL。`max_results` **不** 面向模型：工具设置上限（`searchMaxResults` 配置，默认 8）并传给 seam。 | -| `web_fetch` | `url`（string） | 获取特定 URL。HTML 主体渲染为近似 markdown 的文本；文本主体原样通过。非 2xx 状态会报告，而非报错。工具调用超时是部署策略（`dsh-timeout-policy`），不是模型参数。 | +| `web_fetch` | `url`（string） | 获取特定 URL。HTML 主体渲染为 markdown（turndown，带 GFM 表格／删除线）；文本主体原样通过。非 2xx 状态会报告，而非报错。工具调用超时是部署策略（`dsh-timeout-policy`），不是模型参数。 | 两个工具都选择并发调度，因为提供方读取会返回内容，不会修改父 agent 状态。 @@ -126,6 +126,6 @@ Use the web_fetch tool to retrieve the content of a specific HTTP(S) URL (for ex ## 已知限制与暂缓事项 -- **`htmlToMarkdown` 是最小正则转换器，不是 HTML parser**：它会移除 script/style/noscript，保留标题／项目符号／链接，并解码约十余个命名 entity；表格、图片与嵌套格式会丢失。 +- **HTML→markdown 转换在病态输入上回退为原始 HTML**：[turndown](https://github.com/mixmark-io/turndown)（带 GFM 表格／删除线）通过真实 DOM 转换抓取到的 HTML，但其递归遍历在极深嵌套（数千层）上会栈溢出；此类主体不经转换原样通过，而非报错（[决策记录](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)）。 - **面向模型的表层有意保持最小，提升项暂缓**：`max_results` 保持为配置上限（不是模型参数），`web_fetch` 只接受 `url`（没有 `format`／`prompt`／LLM 摘要模式）；两项都列为 [seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md) 中的后续步骤。 - **没有 web 专用权限策略**：两个工具都不会请求 `ctx.approval` 就直接执行；需要确认的部署必须添加 `tools/pre-execute` 策略，该包不定义持久 URL／domain 授权。 diff --git a/packages/web/tool-web/package.json b/packages/web/tool-web/package.json index ec1d33f4d4..9e1a54b6cd 100644 --- a/packages/web/tool-web/package.json +++ b/packages/web/tool-web/package.json @@ -35,10 +35,13 @@ "cordis": "^4.0.0-rc.7" }, "dependencies": { - "schemastery": "^3.18.0" + "@joplin/turndown-plugin-gfm": "^1.0.67", + "schemastery": "^3.18.0", + "turndown": "^7.2.4" }, "devDependencies": { "@deepseek-ai/dsh-agent": "workspace:^", + "@types/turndown": "^5.0.6", "@deepseek-ai/dsh-invariants": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", "@deepseek-ai/dsh-session": "workspace:^", diff --git a/packages/web/tool-web/src/fetch.ts b/packages/web/tool-web/src/fetch.ts index cc2ae52970..60c0f33507 100644 --- a/packages/web/tool-web/src/fetch.ts +++ b/packages/web/tool-web/src/fetch.ts @@ -6,12 +6,29 @@ */ import type { Context } from 'cordis' +import TurndownService from 'turndown' +import { gfm } from '@joplin/turndown-plugin-gfm' import { defineTool } from '@deepseek-ai/dsh-tools' import type { GenericCallView } from '@deepseek-ai/dsh-tools' import type { WebFetchBody, WebFetchResult } from '@deepseek-ai/dsh-web' import { assertNever } from '@deepseek-ai/dsh-llm' import type {} from '@deepseek-ai/dsh-system-prompt' -import { htmlToMarkdown } from './html.ts' + +/** + * The shared HTML→markdown converter: turndown over its bundled domino DOM, + * with GitHub-flavored tables/strikethrough (`@joplin/turndown-plugin-gfm`). + * The style options are fixed model-facing presentation (matching the repo's + * markdown conventions), not deployment tunables. `remove` drops non-content + * elements wholesale — turndown's default keeps their text. The instance is + * stateless across `turndown()` calls and safe to share. + */ +const turndown = new TurndownService({ + headingStyle: 'atx', + codeBlockStyle: 'fenced', + bulletListMarker: '-', +}) +turndown.use(gfm) +turndown.remove(['script', 'style', 'noscript']) /** * Validate value constraints the schema DSL can't express: a non-blank `url`. @@ -30,14 +47,23 @@ export function parseFetchArgs(args: { url: string }): { url: string } { /** * Render a fetched body to model-facing markdown text. * - * @param body - the decoded body; `html` is converted via - * {@link htmlToMarkdown}, `text` passes through verbatim. + * @param body - the decoded body; `html` is converted via turndown, `text` + * passes through verbatim. When turndown throws (deeply pathological HTML + * overflows its recursive DOM walk), the raw HTML passes through instead — + * a degraded page beats an error for a body the provider already decoded. * @returns the text for the tool's output block. */ export function renderBody(body: WebFetchBody): string { switch (body.kind) { case 'html': - return htmlToMarkdown(body.content) + try { + return turndown.turndown(body.content) + } catch { + // turndown's DOM walk recurses per element; pathological nesting (a + // few thousand levels) throws RangeError. Provider errors stay + // structured WebErrors upstream; conversion failure downgrades to raw HTML. + return body.content + } case 'text': return body.content /* v8 ignore next 2 -- WebFetchBody is a closed union; this arm is unreachable and only makes adding a kind a compile error. */ diff --git a/packages/web/tool-web/src/html.ts b/packages/web/tool-web/src/html.ts deleted file mode 100644 index 1d6ffdb9a3..0000000000 --- a/packages/web/tool-web/src/html.ts +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Minimal dependency-free HTML-to-readable-text conversion for `web_fetch`, not a full parser. It - * removes non-content elements and tags, decodes common entities, collapses whitespace, and keeps - * basic headings, lists, and links. A richer converter can replace it without changing the seam or - * tool schema. - * @module @deepseek-ai/dsh-tool-web/html - */ - -/** Decode the handful of HTML entities common in textual content. */ -function decodeEntities(text: string): string { - return text - .replace(/&(#[xX][0-9a-fA-F]+|#[0-9]+|[a-zA-Z]+);/g, (match, entity: string) => { - if (entity.startsWith('#x') || entity.startsWith('#X')) { - const code = Number.parseInt(entity.slice(2), 16) - return safeFromCodePoint(code, match) - } - if (entity.startsWith('#')) { - const code = Number.parseInt(entity.slice(1), 10) - return safeFromCodePoint(code, match) - } - return NAMED_ENTITIES[entity] ?? match - }) -} - -const NAMED_ENTITIES: Record = { - amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', - copy: '©', reg: '®', trade: '™', hellip: '…', mdash: '—', ndash: '–', -} - -function safeFromCodePoint(code: number, fallback: string): string { - try { - return String.fromCodePoint(code) - } catch { - // An out-of-range code point (RangeError) is the only failure here; keep the - // original entity text rather than throwing out of pure presentation. - return fallback - } -} - -/** - * Convert an HTML document to a readable markdown-ish text approximation. - * Best-effort and lossy by design — fidelity is the job of a future heavier - * converter, not this fallback. - * - * @param html - the raw HTML source. - * @returns plain text with markdown headings, list bullets, and links; - * whitespace collapsed to at most one blank line and trimmed. - */ -export function htmlToMarkdown(html: string): string { - let text = html - // Drop non-content elements entirely (including their contents). - .replace(/]*>[\s\S]*?<\/script>/gi, '') - .replace(/]*>[\s\S]*?<\/style>/gi, '') - .replace(/]*>[\s\S]*?<\/noscript>/gi, '') - .replace(//g, '') - - // Convert links to markdown before stripping tags. - text = text.replace(/]*\bhref\s*=\s*["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, (_match, href: string, label: string) => { - const cleanLabel = label.replace(/<[^>]+>/g, '').trim() - return cleanLabel.length > 0 ? `[${cleanLabel}](${href})` : href - }) - - // Headings → markdown hashes. - text = text.replace(/]*>([\s\S]*?)<\/h\1>/gi, (_match, level: string, body: string) => { - const hashes = '#'.repeat(Number(level)) - return `\n\n${hashes} ${body.replace(/<[^>]+>/g, '').trim()}\n\n` - }) - - // List items → bullets. - text = text.replace(/]*>([\s\S]*?)<\/li>/gi, (_match, body: string) => `\n- ${body.replace(/<[^>]+>/g, '').trim()}`) - - // Block-level breaks become paragraph breaks. - text = text - .replace(/<\/(p|div|section|article|header|footer|tr|table|ul|ol|blockquote)>/gi, '\n\n') - .replace(//gi, '\n') - - // Drop all remaining tags, decode entities, collapse whitespace. - text = text.replace(/<[^>]+>/g, '') - text = decodeEntities(text) - text = text - .replace(/[ \t\f\v]+/g, ' ') - .replace(/ *\n */g, '\n') - .replace(/\n{3,}/g, '\n\n') - .trim() - return text -} diff --git a/packages/web/tool-web/src/index.ts b/packages/web/tool-web/src/index.ts index 7096371ed1..e7ac4b2453 100644 --- a/packages/web/tool-web/src/index.ts +++ b/packages/web/tool-web/src/index.ts @@ -14,7 +14,6 @@ import { applyWebFetchTool } from './fetch.ts' export { WEB_SEARCH_MAX_RESULTS, applyWebSearchTool, formatSearchOutput, parseSearchArgs, presentSearchCall } from './search.ts' export { applyWebFetchTool, formatFetchOutput, parseFetchArgs, presentFetchCall, renderBody } from './fetch.ts' -export { htmlToMarkdown } from './html.ts' /** Cordis plugin name used by loader diagnostics. */ export const name = 'tool-web' diff --git a/packages/web/tool-web/src/turndown-plugin-gfm.d.ts b/packages/web/tool-web/src/turndown-plugin-gfm.d.ts new file mode 100644 index 0000000000..66c9d929e4 --- /dev/null +++ b/packages/web/tool-web/src/turndown-plugin-gfm.d.ts @@ -0,0 +1,12 @@ +/** + * Ambient module declaration for `@joplin/turndown-plugin-gfm`, which ships no + * types and has no DefinitelyTyped package. Only the composite `gfm` plugin is + * declared; the package's individual plugins (`tables`, `strikethrough`, …) + * stay undeclared until something imports them. + */ +declare module '@joplin/turndown-plugin-gfm' { + import type TurndownService from 'turndown' + + /** The composite GitHub-flavored-markdown plugin (tables, strikethrough, task lists, highlighted code blocks). */ + export const gfm: TurndownService.Plugin +} diff --git a/packages/web/tool-web/tests/tool-web.spec.ts b/packages/web/tool-web/tests/tool-web.spec.ts index 093184c4a7..f9ffb1b5c5 100644 --- a/packages/web/tool-web/tests/tool-web.spec.ts +++ b/packages/web/tool-web/tests/tool-web.spec.ts @@ -14,7 +14,6 @@ import { presentSearchCall, presentFetchCall, renderBody, - htmlToMarkdown, WEB_SEARCH_MAX_RESULTS, } from '@deepseek-ai/dsh-tool-web' @@ -82,6 +81,11 @@ describe('search formatting', () => { expect(parseSearchArgs({ query: 'hi' })).toEqual({ query: 'hi' }) }) + it('falls back to the raw URL as a source label when the URL is unparseable', () => { + const out = formatSearchOutput({ truncated: false, sources: [{ url: 'not a url' }] }) + expect(out).toContain('[not a url](not a url)') + }) + it('presents a search call as a search-kind card titled by the query', () => { expect(presentSearchCall({ query: 'find me' })).toEqual({ card: 'generic', title: 'find me', kind: 'search', rawInput: 'find me' }) }) @@ -112,6 +116,29 @@ describe('fetch formatting', () => { expect(renderBody({ kind: 'html', content: '

    y

    ' })).toBe('y') }) + it('converts html via turndown: entities, links, tables, nesting; drops script/style/noscript', () => { + expect(renderBody({ + kind: 'html', + content: '

    Tom & Jerry © Résumé

    link', + })).toBe('Tom & Jerry © Résumé\n\n[link](https://a.test)') + expect(renderBody({ kind: 'html', content: '

    Heading

    • one
    • two
    ' })) + .toBe('## Heading\n\n- one\n- two') + expect(renderBody({ kind: 'html', content: '
    AB
    12
    ' })) + .toBe('| A | B |\n| --- | --- |\n| 1 | 2 |') + expect(renderBody({ kind: 'html', content: '

    bold italic

    quoted

    ' })) + .toBe('**bold _italic_**\n\n> quoted') + }) + + it('falls back to the raw html body when turndown throws on pathological nesting', { timeout: 60_000 }, () => { + // Nesting past V8's default stack overflows turndown/domino's recursive + // walk with a RangeError (measured: 4k levels throw on the main thread, + // 8k in a worker); 20k adds margin over either stack size. The raw body + // must pass through instead of throwing. + const depth = 20_000 + const pathological = '
    '.repeat(depth) + 'x' + '
    '.repeat(depth) + expect(renderBody({ kind: 'html', content: pathological })).toBe(pathological) + }) + it('validates url (non-empty), no timeout parameter', () => { expect(() => parseFetchArgs({ url: ' ' })).toThrow('non-empty') expect(parseFetchArgs({ url: 'https://a.test' })).toEqual({ url: 'https://a.test' }) @@ -122,46 +149,6 @@ describe('fetch formatting', () => { }) }) -describe('htmlToMarkdown', () => { - it('drops scripts/styles, keeps text, decodes entities, converts links', () => { - const md = htmlToMarkdown('

    Tom & Jerry

    link') - expect(md).not.toContain('bad()') - expect(md).not.toContain('.x{}') - expect(md).toContain('Tom & Jerry') - expect(md).toContain('[link](https://a.test)') - }) - - it('decodes numeric entities and collapses whitespace', () => { - expect(htmlToMarkdown('

    a'b

    ')).toBe("a'b") - expect(htmlToMarkdown('
    x
    \n\n\n
    y
    ')).toBe('x\n\ny') - }) - - it('decodes hex entities and named entities, and leaves unknown/out-of-range ones intact', () => { - expect(htmlToMarkdown('

    AB

    ')).toBe('AB') - expect(htmlToMarkdown('

    © —

    ')).toBe('© —') - expect(htmlToMarkdown('

    ¬areal;

    ')).toBe('¬areal;') - // An out-of-range code point keeps the original entity text (fromCodePoint fallback). - expect(htmlToMarkdown('

    �

    ')).toBe('�') - expect(htmlToMarkdown('

    �

    ')).toBe('�') - }) - - it('renders a link with an empty label as its bare href', () => { - expect(htmlToMarkdown('')).toBe('https://a.test') - }) - - it('converts headings and list items to markdown', () => { - expect(htmlToMarkdown('

    Heading

    after

    ')).toContain('## Heading') - const list = htmlToMarkdown('
    • one
    • two
    ') - expect(list).toContain('- one') - expect(list).toContain('- two') - }) - - it('falls back to the raw URL as a source label when the URL is unparseable', () => { - const out = formatSearchOutput({ truncated: false, sources: [{ url: 'not a url' }] }) - expect(out).toContain('[not a url](not a url)') - }) -}) - describe('tool-web registration', () => { it('registers both tools by default', async () => { const { fiber, ctx } = await mountTools() diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1946a841bb..430cf6ea0a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -523,6 +523,9 @@ importers: '@deepseek-ai/dsh-tool-todo': specifier: workspace:* version: link:../packages/todo/tool-todo + '@deepseek-ai/dsh-tool-web': + specifier: workspace:* + version: link:../packages/web/tool-web '@deepseek-ai/dsh-tool-workflow': specifier: workspace:* version: link:../packages/workflow/tool-workflow @@ -4259,9 +4262,15 @@ importers: packages/web/tool-web: dependencies: + '@joplin/turndown-plugin-gfm': + specifier: ^1.0.67 + version: 1.0.67 schemastery: specifier: ^3.18.0 version: 3.18.0 + turndown: + specifier: ^7.2.4 + version: 7.2.4 devDependencies: '@deepseek-ai/dsh-agent': specifier: workspace:^ @@ -4299,6 +4308,9 @@ importers: '@deepseek-ai/dsh-web-search-exa': specifier: workspace:^ version: link:../web-search-exa + '@types/turndown': + specifier: ^5.0.6 + version: 5.0.6 cordis: specifier: ^4.0.0-rc.7 version: 4.0.0-rc.7(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.5) @@ -5971,6 +5983,9 @@ packages: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@joplin/turndown-plugin-gfm@1.0.67': + resolution: {integrity: sha512-FZfW5EZfidhzd1IaY1uxHnIZPTVOxAdleMZ4/1U6Nt5b7+Qj5JThDnaIomuJtetnUBzuRNbe9FWMuqD4B3dlWA==} + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -6076,6 +6091,9 @@ packages: '@opentelemetry/api': optional: true + '@mixmark-io/domino@2.2.0': + resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==} + '@modelcontextprotocol/sdk@1.29.0': resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==} engines: {node: '>=18'} @@ -7005,6 +7023,9 @@ packages: '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} + '@types/turndown@5.0.6': + resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==} + '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -9463,6 +9484,10 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + turndown@7.2.4: + resolution: {integrity: sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==} + engines: {node: '>=18', npm: '>=9'} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -10860,6 +10885,8 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@joplin/turndown-plugin-gfm@1.0.67': {} + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -10951,6 +10978,8 @@ snapshots: - bufferutil - utf-8-validate + '@mixmark-io/domino@2.2.0': {} + '@modelcontextprotocol/sdk@1.29.0(zod@4.4.3)': dependencies: '@hono/node-server': 1.19.14(hono@4.12.29) @@ -11705,6 +11734,8 @@ snapshots: '@types/trusted-types@2.0.7': optional: true + '@types/turndown@5.0.6': {} + '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -14645,6 +14676,10 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + turndown@7.2.4: + dependencies: + '@mixmark-io/domino': 2.2.0 + type-check@0.4.0: dependencies: prelude-ls: 1.2.1