From b91b1fdefe2baa6f76530f7b90f34307efc53e1f Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sat, 8 Aug 2026 17:46:56 +0800 Subject: [PATCH] fix(repository-plugin): make GitHub source preparation self-contained --- ...-static-repository-plugin-format.i18n.yaml | 4 +- ...6-07-30-static-repository-plugin-format.md | 4 +- ...7-30-static-repository-plugin-format.zh.md | 4 +- ...it-repository-plugin-preparation.i18n.yaml | 6 ++ ...owned-git-repository-plugin-preparation.md | 42 +++++++++ ...ed-git-repository-plugin-preparation.zh.md | 42 +++++++++ .github/workflows/ci.yml | 2 + .../github-repository-plugin.built.e2e.ts | 89 +++++++++++++++++++ .../app-boot/tests/repository-cache.spec.ts | 46 ++++++---- .../repository-plugin/README.i18n.yaml | 4 +- .../repository-plugin/README.md | 9 +- .../repository-plugin/README.zh.md | 9 +- .../repository-plugin/src/format.ts | 7 +- .../repository-plugin/src/index.ts | 33 ++++--- .../repository-plugin/src/source.ts | 79 +++++++++++++++- .../tests/repository-plugin.spec.ts | 64 ++++++++++++- scripts/run-gates.spec.ts | 22 ++++- scripts/run-gates.ts | 15 ++++ vendor/README.md | 2 +- vendor/loader/src/repository.ts | 35 ++++++-- 20 files changed, 454 insertions(+), 64 deletions(-) create mode 100644 .agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.i18n.yaml create mode 100644 .agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.md create mode 100644 .agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.zh.md create mode 100644 apps/cli/tests/github-repository-plugin.built.e2e.ts diff --git a/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.i18n.yaml b/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.i18n.yaml index 6319b99fb2..40a0461378 100644 --- a/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.md -2026-07-30-static-repository-plugin-format.md: c9d755b925a6ea05eed71e75803397d2672df9f4 -2026-07-30-static-repository-plugin-format.zh.md: 361de64d2e98b9fb4ac42963e4ae48e77fbc7016 +2026-07-30-static-repository-plugin-format.md: 4823495dabee2101713ac72c8d0cee5bcc6b38d1 +2026-07-30-static-repository-plugin-format.zh.md: e5817ca982a71a609c86c5835b803b12f1317e1a diff --git a/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.md b/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.md index c9d755b925..4823495dab 100644 --- a/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.md +++ b/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.md @@ -14,7 +14,7 @@ The [package-manager-native repository cache](2026-07-30-package-manager-native- `@deepseek-ai/dsh-repository-plugin` owns a restricted `.dsh-plugin` package format with two contribution kinds only: skill roots and one common `.mcp.json`. Its package metadata uses `package.json#dsh.skills` for relative skill-root paths and `package.json#dsh.mcpServers` for the relative MCP document path. At least one is required. Each path may leave `.dsh-plugin` to reuse repository content but must remain beneath the directory containing that `.dsh-plugin`; a nested selectable Plugin therefore owns the adjacent subtree above its package without gaining access to unrelated host paths. -The `.dsh-plugin` package declares `dsh-plugin-prepare` as its ordinary package-manager `prepare` script. The helper validates metadata and source types, strictly parses `.mcp.json`, copies static assets into `dsh-plugin-assets`, and writes `dsh-plugin.mjs`. The `.mjs` extension avoids imposing `type: module` on repository-authored package metadata. The generated module is a fixed import-free template containing only a normalized manifest, an `inject` list derived from it (`loader`, plus `skills` and/or `tools` per the declared capabilities, so the wrapper fiber gates on the services its children need), and delegation to the `dsh-repository-plugin` Loader builtin. Preparation never discovers, transpiles, bundles, or preserves a custom repository entry point. +The `.dsh-plugin` package declares exact `scripts.prepack: "dsh-plugin-prepare"` metadata without depending on a DSH npm package. During Git installation, the standalone runtime temporarily supplies that command from its own build on the isolated lifecycle `PATH`; `prepack` runs after dependency installation and before pnpm packs a selected subdirectory, including a Plugin nested inside another package-manager workspace. The helper validates metadata and source types, strictly parses `.mcp.json`, copies static assets into `dsh-plugin-assets`, and writes `dsh-plugin.mjs`; the source loader revalidates the installed package's exact lifecycle metadata before importing that wrapper. The `.mjs` extension avoids imposing `type: module` on repository-authored package metadata. The generated module is a fixed import-free template containing only a normalized manifest, an `inject` list derived from it (`loader`, plus `skills` and/or `tools` per the declared capabilities, so the wrapper fiber gates on the services its children need), and delegation to the `dsh-repository-plugin` Loader builtin. Preparation never discovers, transpiles, bundles, or preserves a custom repository entry point. The host-owned command rationale is in the [Git source preparation repair](../bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.md). Loading the DSH package registers that builtin as an effect. A generated wrapper mounts the builtin as its child with `import.meta.url`, so all contributions belong to the wrapper fiber and disappear on Loader removal or rollback. The builtin revalidates the prepared manifest and path containment before reading assets. It composes the existing implementations rather than registering skills or MCP tools itself. @@ -46,4 +46,4 @@ Unknown MCP fields reject. This intentionally excludes OAuth, `auth` objects, `C ## Testing -Focused tests prepare skills and MCP metadata, prove the emitted wrapper contains no imports, reject Work IQ-style OAuth fields, map Expo-style HTTP and DataJunction-style stdio plus environment values, and exercise missing variables. A real Loader test mounts a generated wrapper through the registered builtin, reads its skill through `ctx.skills`, removes the Loader entry, and observes provider cleanup. The keyless headless example loads a checked-in prepared wrapper through its real `cordis.yml` and snapshots the repository skill's logged model catalog row. +Focused tests prepare skills and MCP metadata, prove the emitted wrapper contains no imports, reject Work IQ-style OAuth fields, map Expo-style HTTP and DataJunction-style stdio plus environment values, and exercise missing variables. A real Loader test mounts a generated wrapper through the registered builtin, reads its skill through `ctx.skills`, removes the Loader entry, and observes provider cleanup. The CI built-entry acceptance invokes `dsh run` with a GitHub source pinned to the pull request head, lets bundled pnpm fetch and prepare a private dependency-free fixture, then observes the copied skill in the real model request and the prepared wrapper in the immutable cache. diff --git a/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.zh.md b/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.zh.md index 361de64d2e..e5817ca982 100644 --- a/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.zh.md +++ b/.agents/notes/implemented/architecture/2026-07-30-static-repository-plugin-format.zh.md @@ -14,7 +14,7 @@ `@deepseek-ai/dsh-repository-plugin` 负责一个受限的 `.dsh-plugin` package 格式，且只允许两类贡献：skill 根和一个通用 `.mcp.json`。Package metadata 使用 `package.json#dsh.skills` 声明相对 skill 根路径，使用 `package.json#dsh.mcpServers` 声明相对 MCP 文档路径；两者至少需要一个。路径可以离开 `.dsh-plugin` 以复用仓库内容，但必须留在包含该 `.dsh-plugin` 的目录之下；因此，一个嵌套且可选择的 Plugin 可以拥有其 package 上方相邻的子树，却不能访问无关宿主路径。 -`.dsh-plugin` package 把 `dsh-plugin-prepare` 声明为普通 package-manager `prepare` 脚本。Helper 会校验 metadata 与源码类型，严格解析 `.mcp.json`，把静态资源复制到 `dsh-plugin-assets`，并写入 `dsh-plugin.mjs`。`.mjs` 扩展名避免强迫仓库作者在 package metadata 中设置 `type: module`。生成模块来自固定、无 import 的模板，只包含规范化 manifest、由 manifest 派生的 `inject` 列表（`loader`，加上按声明能力加入的 `skills`／`tools`，使包装 fiber 在其子插件所需服务上门控），以及对 `dsh-repository-plugin` Loader builtin 的委托。准备阶段永远不会发现、转译、打包或保留自定义仓库入口。 +`.dsh-plugin` 包声明精确的 `scripts.prepack: "dsh-plugin-prepare"` 元数据，且不依赖 DSH NPM 包。在 Git 安装期间，独立运行时会从自身构建产物中临时提供该命令，并将其放入隔离的生命周期 `PATH`；`prepack` 会在依赖安装后、pnpm 打包选定子目录前运行，即使插件嵌套在另一个包管理器工作区内也不例外。该辅助程序会校验元数据与源码类型，严格解析 `.mcp.json`，把静态资源复制到 `dsh-plugin-assets`，并写入 `dsh-plugin.mjs`；源码 loader 会在导入该包装层前重新校验已安装包的精确生命周期元数据。`.mjs` 扩展名避免强迫仓库作者在包元数据中设置 `type: module`。生成模块来自固定、无 import 的模板，只包含规范化 manifest、由 manifest 派生的 `inject` 列表（`loader`，加上按声明能力加入的 `skills`／`tools`，使包装 fiber 在其子插件所需服务上门控），以及对 `dsh-repository-plugin` Loader builtin 的委托。准备阶段永远不会发现、转译、打包或保留自定义仓库入口。宿主自有命令的设计依据见[Git 源准备修复](../bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.md)。 加载 DSH package 会以 effect 方式注册该 builtin。生成的包装模块使用 `import.meta.url` 把 builtin 挂载为自己的子级，因此所有贡献都归属于包装 fiber，并在 Loader 移除或回滚时消失。Builtin 会在读取资源前重新校验已准备 manifest 与路径包含关系。它只组合现有实现，而不自行注册 skills 或 MCP 工具。 @@ -46,4 +46,4 @@ ## 测试 -聚焦测试会准备 skills 与 MCP metadata，证明生成包装模块不含 import，拒绝 Work IQ 风格的 OAuth 字段，映射 Expo 风格 HTTP 与 DataJunction 风格 stdio 及环境变量，并覆盖缺失变量。真实 Loader 测试通过已注册 builtin 挂载生成包装模块，经 `ctx.skills` 读取其 skill，移除 Loader 条目并观察提供方清理。Keyless headless 示例通过真实 `cordis.yml` 加载一份签入的已准备包装模块，并快照 repository skill 写入日志的模型目录行。 +聚焦测试会准备 skills 与 MCP metadata，证明生成包装模块不含 import，拒绝 Work IQ 风格的 OAuth 字段，映射 Expo 风格 HTTP 与 DataJunction 风格 stdio 及环境变量，并覆盖缺失变量。真实 Loader 测试通过已注册 builtin 挂载生成包装模块，经 `ctx.skills` 读取其 skill，移除 Loader 条目并观察提供方清理。CI 的构建入口验收会用锁定到 PR（Pull Request）head 的 GitHub 源调用 `dsh run`，让随附 pnpm 获取并准备一个私有且不含依赖的 fixture（测试前置数据），然后在真实模型请求中观察已复制的 skill，并在不可变缓存中观察已准备的包装模块。 diff --git a/.agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.i18n.yaml new file mode 100644 index 0000000000..55d9045506 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.md +2026-08-08-host-owned-git-repository-plugin-preparation.md: 45e84f9c8a89bb9d1eb7e4521634d16789dea2f0 +2026-08-08-host-owned-git-repository-plugin-preparation.zh.md: ba88d5bb351427118d55c357209b496b39c2eb98 diff --git a/.agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.md b/.agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.md new file mode 100644 index 0000000000..45e84f9c8a --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.md @@ -0,0 +1,42 @@ +# Agent Note: Host-owned preparation makes GitHub repository Plugins installable + +Status: implemented + +English | [中文](2026-08-08-host-owned-git-repository-plugin-preparation.zh.md) + +## Problem + +The repository Plugin authoring contract depended on `scripts.prepare: "dsh-plugin-prepare"` and told source repositories to add `@deepseek-ai/dsh-repository-plugin` as a development dependency. That package is private and not published to npm, so an otherwise valid external GitHub repository could not obtain the helper in a clean install. + +The lifecycle choice also failed for a selectable `.dsh-plugin` inside a pnpm workspace. pnpm prepares a Git-hosted package by running the repository's preferred package manager before packing the selected subdirectory. A nested `pnpm install` joins the containing workspace and need not execute the unlisted `.dsh-plugin` package's `prepare` script. The install could therefore succeed and publish a cache generation containing only the source package metadata; real DSH startup failed later because `dsh-plugin.mjs` did not exist. + +The checked-in headless fixture did not catch either defect because it mounted an already prepared wrapper. It proved runtime composition, not GitHub acquisition or package preparation. + +## Decision + +The fixed authoring format now requires exact `scripts.prepack: "dsh-plugin-prepare"` metadata and no DSH dependency. pnpm's Git-hosted package preparation invokes `prepack` explicitly after its dependency-install step and before packlist selects the `.dsh-plugin` subtree, so the helper can still copy sibling repository assets such as `../skills` into the package. + +`@deepseek-ai/dsh-repository-plugin` materializes short-lived POSIX and Windows command wrappers that invoke its own built `dsh-plugin-prepare` entry. `RepositoryCache` accepts caller-owned executable directories, resolves them absolutely, and prepends them to the credential-scrubbed lifecycle `PATH` passed to bundled pnpm. The command directory exists only for the installation transaction and is removed on success or failure. The repository remains trusted package-manager input: DSH supplies one command, but other lifecycle scripts and dependencies still execute under the existing trust contract. + +The Node 24 consumer lane passes an exact source derived from the pull request head repository and SHA. Its built-entry acceptance launches the real `apps/cli/lib/bin.js run` command with a one-run patch selecting a `private: true`, dependency-free GitHub fixture. It requires the run to reach the mock LLM, finds the repository skill description in the actual model request, and verifies the generated wrapper and copied skill under the immutable DSH cache. The test fails if CI omits the exact source instead of silently skipping. + +## Alternatives considered + +**Publish the prepare helper to npm.** Rejected because the source package would acquire a release/version dependency solely to call code already owned by the running DSH installation, and the existing helper is intentionally private. + +**Keep `prepare` and only inject the command.** Rejected because command availability does not make a nested package's `prepare` lifecycle run when the Git repository's package manager treats it as part of another workspace. + +**Prepare after RepositoryCache installs the selected package.** Rejected because pnpm's packed subdirectory no longer contains sibling source assets referenced by paths such as `../skills`; preparation must happen before packlist. + +**Clone GitHub repositories in DSH and bypass pnpm's Git fetcher.** Rejected because it would duplicate ref resolution, subdirectory selection, dependency installation, packlist behavior, and cache integrity already owned by the pinned package manager. + +## Consequences + +- A repository author can commit the fixed `.dsh-plugin/package.json` and source assets to GitHub without publishing either the Plugin or its preparation helper to npm. +- `prepack`, not `prepare`, is part of the pre-release authoring format. Invalid lifecycle metadata fails during source preparation or installed-package validation instead of producing an ambiguous partial format. +- Exact source strings still identify immutable cache generations; a changed ref or source configuration selects another generation. +- This repair does not expand the contribution surface: prepared repository Plugins still contribute only declared skills and common MCP definitions, while arbitrary package lifecycle code remains trusted installation code rather than a model-facing Cordis Plugin API. + +## Testing + +`packages/ui/app-boot/tests/repository-cache.spec.ts` runs a local Git subpath through bundled pnpm with an injected command directory and proves that visible environment survives while credential-shaped variables are scrubbed. `packages/cordis/repository-plugin/tests/repository-plugin.spec.ts` pins the exact `prepack` metadata and temporary command cleanup. `apps/cli/tests/github-repository-plugin.built.e2e.ts` is the product acceptance: fresh DSH home, exact live GitHub source, actual built `dsh run`, real headless composition, mock LLM request observation, and prepared cache inspection. diff --git a/.agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.zh.md b/.agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.zh.md new file mode 100644 index 0000000000..ba88d5bb35 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.zh.md @@ -0,0 +1,42 @@ +# Agent Note: 宿主自有的准备机制使 GitHub repository 插件可安装 + +状态：已实现 + +[English](2026-08-08-host-owned-git-repository-plugin-preparation.md) | 中文 + +## 问题 + +repository 插件的创作契约依赖 `scripts.prepare: "dsh-plugin-prepare"`，并要求源码仓库将 `@deepseek-ai/dsh-repository-plugin` 添加为开发依赖。该包是私有包，且未发布到 NPM，因此即使外部 GitHub 仓库符合其他要求，也无法在全新安装中取得该辅助程序。 + +这种生命周期选择也无法支持 pnpm 工作区内可选的 `.dsh-plugin`。pnpm 会先运行 Git 托管仓库首选的包管理器，再打包选定的子目录，从而准备 Git 托管包。嵌套执行的 `pnpm install` 会加入外层工作区，而不一定执行未列入其中的 `.dsh-plugin` 包的 `prepare` 脚本。因此，安装可能成功并发布一个仅包含源包元数据的缓存 generation；随后真实 DSH 启动因 `dsh-plugin.mjs` 不存在而失败。 + +签入仓库的 headless fixture（测试前置数据）没有捕获任一缺陷，因为它挂载的是已准备好的包装层。它证明的是运行时组合，而不是 GitHub 获取或包准备。 + +## 决策 + +修复后的创作格式要求元数据中精确包含 `scripts.prepack: "dsh-plugin-prepare"`，且不包含 DSH 依赖。pnpm 针对 Git 托管包的准备流程会在依赖安装步骤之后、打包清单选择 `.dsh-plugin` 子树之前显式调用 `prepack`，因此辅助程序仍可将 `../skills` 等同仓库的相邻资源复制进包内。 + +`@deepseek-ai/dsh-repository-plugin` 会生成临时的 POSIX 和 Windows 命令包装脚本，用于调用其自有的已构建 `dsh-plugin-prepare` 入口。`RepositoryCache` 接受由调用方持有的可执行文件目录，将它们解析为绝对路径，再前置到传给随附 pnpm、已清除凭据的包生命周期 `PATH`。该命令目录仅存在于安装事务期间，无论成功还是失败都会被移除。仓库仍是受信任的包管理器输入：DSH 仅提供这一条命令，其他生命周期脚本和依赖仍会按既有信任契约执行。 + +Node 24 消费方 CI 任务会传入从 PR（Pull Request）head 仓库和 SHA 派生的精确源。其构建入口验收会启动真实的 `apps/cli/lib/bin.js run` 命令，并通过一个仅作用于当次运行的 patch 选择 `private: true`、不含依赖的 GitHub fixture。验收要求该次运行到达 mock LLM（大语言模型），在实际模型请求中找到 repository skill 描述，并验证不可变 DSH 缓存中的生成包装层和已复制 skill。如果 CI 遗漏精确源，测试会失败，而不是静默跳过。 + +## 考虑过的替代方案 + +**把准备辅助程序发布到 NPM。** 拒绝，因为源包会仅为了调用当前运行的 DSH 安装本就拥有的代码，而增加一个需要发布和管理版本的依赖；现有辅助程序又有意保持私有。 + +**保留 `prepare`，只注入命令。** 拒绝，因为当 Git 仓库的包管理器把嵌套包当作另一工作区的一部分时，即使命令可用，也不会使嵌套包的 `prepare` 生命周期得以运行。 + +**在 RepositoryCache 安装选定包后再准备。** 拒绝，因为 pnpm 打包后的子目录不再包含 `../skills` 等路径所引用的同仓库相邻资源；准备必须在生成打包清单前完成。 + +**在 DSH 中克隆 GitHub 仓库，并绕过 pnpm 的 Git 获取器。** 拒绝，因为这会重复实现已由锁定版本的包管理器负责的 ref 解析、子目录选择、依赖安装、打包清单行为和缓存完整性。 + +## 后果 + +- 仓库作者可以把修复后的 `.dsh-plugin/package.json` 和源资源提交到 GitHub，而无需把插件或其准备辅助程序发布到 NPM。 +- 预发布创作格式使用 `prepack` 而不是 `prepare`。无效的生命周期元数据会在源码准备或已安装包校验阶段导致失败，而不会留下状态不明的半成品格式。 +- 精确源字符串仍标识不可变缓存 generation；改变 ref 或源配置会选择另一个 generation。 +- 本次修复不扩大贡献范围：已准备的 repository 插件仍只贡献已声明的 skills 和通用 MCP 定义，而任意包生命周期代码仍是受信任的安装代码，不是面向模型的 Cordis 插件 API。 + +## 测试 + +`packages/ui/app-boot/tests/repository-cache.spec.ts` 会用注入的命令目录通过随附 pnpm 运行本地 Git 子路径，并证明可见环境变量得以保留，而名称符合凭据模式的变量会被清除。`packages/cordis/repository-plugin/tests/repository-plugin.spec.ts` 锁定精确的 `prepack` 元数据和临时命令清理行为。`apps/cli/tests/github-repository-plugin.built.e2e.ts` 是产品验收测试：全新的 DSH 主目录、精确的真实 GitHub 源、实际构建产物的 `dsh run`、真实 headless 组合、mock LLM 请求观测，以及对已准备缓存的检查。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46ab8e892f..5324df431e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,6 +175,8 @@ jobs: DSH_NODE_COMPAT_SKIP_TYPECHECK: '1' DSH_OXLINT_THREADS: '8' DSH_PUBLINT_CONCURRENCY: '8' + DSH_GITHUB_REPOSITORY_PLUGIN_SOURCE: >- + github:${{ github.event.pull_request.head.repo.full_name }}#${{ github.event.pull_request.head.sha }}&path:/apps/cli/tests/fixtures/github-repository-plugin/.dsh-plugin # Failover halves snapshot concurrency for the shared 64-core VM. DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ vars.DSH_CI_FAILOVER == 'selfhosted' && github.event.pull_request.user.login != 'dependabot[bot]' && '12' || '32' }} steps: diff --git a/apps/cli/tests/github-repository-plugin.built.e2e.ts b/apps/cli/tests/github-repository-plugin.built.e2e.ts new file mode 100644 index 0000000000..7262ddeb58 --- /dev/null +++ b/apps/cli/tests/github-repository-plugin.built.e2e.ts @@ -0,0 +1,89 @@ +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { startMockLlmServer } from '@deepseek-ai/dsh-llm-mock-server' +import { execa } from 'execa' +import { describe, expect, it } from 'vitest' + +const repoRoot = fileURLToPath(new URL('../../../', import.meta.url)) +const dshBin = join(repoRoot, 'apps/cli/lib/bin.js') +const source = process.env.DSH_GITHUB_REPOSITORY_PLUGIN_SOURCE +const required = process.env.DSH_REQUIRE_GITHUB_REPOSITORY_PLUGIN_E2E === '1' +const enabled = required || source !== undefined + +describe.skipIf(!enabled)('dsh run GitHub repository Plugin installation', () => { + it('installs a private exact GitHub source and exposes its skill to the model', async () => { + expect(existsSync(dshBin), 'the repository Plugin acceptance must run the built dsh entry').toBe(true) + expect(source, 'DSH_GITHUB_REPOSITORY_PLUGIN_SOURCE is required by this CI lane').toMatch( + /^github:[^/\s#&]+\/[^/\s#&]+#[0-9a-f]{40}&path:\/.*\/\.dsh-plugin$/u, + ) + + const apiKey = 'github-repository-plugin-e2e-key' + const server = await startMockLlmServer({ + sequence: ['success'], + apiKey, + successText: 'private GitHub repository Plugin reached dsh run', + }) + const home = mkdtempSync(join(tmpdir(), 'dsh-github-repository-plugin-')) + const patch = join(home, 'github-repository-plugin.cordis.patch.yml') + writeFileSync(patch, [ + '- id: repository-plugins', + ' config:', + ' repositories:', + ` - ${JSON.stringify(source)}`, + '', + ].join('\n')) + + try { + const result = await execa(process.execPath, [ + dshBin, + 'run', + '--patch', + patch, + 'prove the private GitHub repository Plugin is active', + ], { + cwd: repoRoot, + input: '', + timeout: 120_000, + killSignal: 'SIGKILL', + reject: false, + env: { + ...process.env, + DSH_HOME: home, + DSH_TELEMETRY_DISABLED: '1', + DEEPSEEK_API_KEY: apiKey, + DEEPSEEK_BASE_URL: server.baseURL, + }, + }) + if (result.timedOut) { + throw new Error(`dsh GitHub repository Plugin run did not exit within 120s. stdout:\n${result.stdout}\nstderr:\n${result.stderr}`) + } + expect(result.exitCode, `${result.stderr}\nstdout:\n${result.stdout}`).toBe(0) + expect(result.stdout).toBe('private GitHub repository Plugin reached dsh run') + expect(server.requests.length).toBeGreaterThan(0) + expect(JSON.stringify(server.requests.map(request => request.body))).toContain( + 'Proves that dsh installed a private repository Plugin from an exact GitHub source.', + ) + + const cacheRoot = join(home, 'cache', 'repository-plugins') + const generations = readdirSync(cacheRoot, { withFileTypes: true }).filter(entry => entry.isDirectory()) + expect(generations).toHaveLength(1) + const installed = join(cacheRoot, generations[0]!.name, 'node_modules', 'repository') + const manifest = JSON.parse(readFileSync(join(installed, 'package.json'), 'utf8')) as Record + expect(manifest).toMatchObject({ + name: 'dsh-github-repository-plugin-e2e-fixture', + private: true, + scripts: { prepack: 'dsh-plugin-prepare' }, + }) + expect(manifest).not.toHaveProperty('dependencies') + expect(manifest).not.toHaveProperty('devDependencies') + expect(readFileSync(join(installed, 'dsh-plugin-assets/skills/0/github-source-proof/SKILL.md'), 'utf8')) + .toContain('This skill exists only in the GitHub repository source fixture.') + expect(readFileSync(join(installed, 'dsh-plugin.mjs'), 'utf8')).toContain('dsh-repository-plugin') + } finally { + await server.close() + rmSync(home, { recursive: true, force: true }) + } + }, 130_000) +}) diff --git a/packages/boot/app-boot/tests/repository-cache.spec.ts b/packages/boot/app-boot/tests/repository-cache.spec.ts index b7d80470b8..186cbe1b43 100644 --- a/packages/boot/app-boot/tests/repository-cache.spec.ts +++ b/packages/boot/app-boot/tests/repository-cache.spec.ts @@ -36,14 +36,14 @@ describe('RepositoryCache', () => { calls.push(directory) await fakePackage(directory) } - const cache = new RepositoryCache(root, install) + const cache = new RepositoryCache(root, { install }) const specifier = 'github:owner/repository#0123456789abcdef' const [first, concurrent] = await Promise.all([cache.resolve(specifier), cache.resolve(specifier)]) expect(concurrent).toBe(first) expect(calls).toHaveLength(1) - const reopened = new RepositoryCache(root, async () => { throw new Error('cache miss') }) + const reopened = new RepositoryCache(root, { install: async () => { throw new Error('cache miss') } }) expect(await reopened.resolve(specifier)).toBe(first) expect(JSON.parse(await readFile(join(first, '..', '..', 'package.json'), 'utf8'))).toMatchObject({ packageManager: `pnpm@${BUNDLED_PNPM_VERSION}`, @@ -68,8 +68,8 @@ describe('RepositoryCache', () => { const specifier = 'github:owner/repository#race' const [first, second] = await Promise.all([ - new RepositoryCache(root, install).resolve(specifier), - new RepositoryCache(root, install).resolve(specifier), + new RepositoryCache(root, { install }).resolve(specifier), + new RepositoryCache(root, { install }).resolve(specifier), ]) expect(second).toBe(first) @@ -80,11 +80,11 @@ describe('RepositoryCache', () => { it('removes a failed staging tree and permits an exact retry', async () => { const root = await temporaryRoot('repository-retry') let attempts = 0 - const cache = new RepositoryCache(root, async (directory) => { + const cache = new RepositoryCache(root, { install: async (directory) => { attempts += 1 if (attempts === 1) throw new Error('install failed') await fakePackage(directory) - }) + } }) await expect(cache.resolve('github:owner/repository#ref')).rejects.toThrow('failed to prepare repository') expect(await readdir(root)).toEqual([]) @@ -94,7 +94,7 @@ describe('RepositoryCache', () => { it('rejects empty or padded specifiers before touching the cache', async () => { const root = await temporaryRoot('repository-input') - const cache = new RepositoryCache(root, fakePackage) + const cache = new RepositoryCache(root, { install: fakePackage }) expect(() => cache.resolve('')).toThrow('non-empty unpadded string') expect(() => cache.resolve(' github:owner/repository#ref')).toThrow('non-empty unpadded string') await expect(readdir(root)).resolves.toEqual([]) @@ -107,7 +107,7 @@ describe('RepositoryCache', () => { const entry = join(root, key) await mkdir(join(entry, 'node_modules', 'repository'), { recursive: true }) await writeFile(join(entry, '.repository-cache.json'), '{}\n') - const cache = new RepositoryCache(root, async () => { throw new Error('must not reinstall') }) + const cache = new RepositoryCache(root, { install: async () => { throw new Error('must not reinstall') } }) await expect(cache.resolve(specifier)).rejects.toThrow('repository cache marker is invalid') }) @@ -115,6 +115,22 @@ describe('RepositoryCache', () => { it('selects and prepares a root .dsh-plugin Git subpath through the bundled pnpm', { timeout: 60_000 }, async () => { const root = await temporaryRoot('repository-pnpm') const repository = join(root, 'source') + const executableDirectory = join(root, 'bin') + await mkdir(executableDirectory) + await writeFile(join(executableDirectory, 'dsh-plugin-prepare'), [ + '#!/usr/bin/env node', + "const { cpSync, mkdirSync, writeFileSync } = require('node:fs')", + "mkdirSync('dsh-plugin-assets/skills', { recursive: true })", + "cpSync('../skills', 'dsh-plugin-assets/skills/0', { recursive: true })", + "writeFileSync('dsh-plugin.mjs', 'export function apply() {}\\n')", + "writeFileSync('prepared.txt', `${process.env.REPOSITORY_TEST_VISIBLE ?? 'absent'}|${process.env.REPOSITORY_TEST_TOKEN ?? 'absent'}\\n`)", + '', + ].join('\n'), { mode: 0o700 }) + await writeFile(join(executableDirectory, 'dsh-plugin-prepare.cmd'), [ + '@echo off', + 'node "%~dp0\\dsh-plugin-prepare" %*', + '', + ].join('\r\n')) await mkdir(join(repository, '.dsh-plugin'), { recursive: true }) await mkdir(join(repository, 'skills', 'fixture'), { recursive: true }) await writeFile(join(repository, 'package.json'), `${JSON.stringify({ @@ -125,17 +141,9 @@ describe('RepositoryCache', () => { await writeFile(join(repository, '.dsh-plugin', 'package.json'), `${JSON.stringify({ name: 'repository-plugin-fixture', version: '1.0.0', - scripts: { prepare: 'node prepare.mjs' }, + scripts: { prepack: 'dsh-plugin-prepare' }, dsh: { skills: ['../skills'] }, })}\n`) - await writeFile(join(repository, '.dsh-plugin', 'prepare.mjs'), [ - "import { cp, mkdir, writeFile } from 'node:fs/promises'", - "await mkdir('dsh-plugin-assets/skills', { recursive: true })", - "await cp('../skills', 'dsh-plugin-assets/skills/0', { recursive: true })", - "await writeFile('dsh-plugin.mjs', 'export function apply() {}\\n')", - "await writeFile('prepared.txt', `${process.env.REPOSITORY_TEST_VISIBLE ?? 'absent'}|${process.env.REPOSITORY_TEST_TOKEN ?? 'absent'}\\n`)", - '', - ].join('\n')) await execFileAsync('git', ['init', '--quiet'], { cwd: repository }) await execFileAsync('git', ['add', '.'], { cwd: repository }) await execFileAsync('git', [ @@ -148,7 +156,9 @@ describe('RepositoryCache', () => { vi.stubEnv('REPOSITORY_TEST_VISIBLE', 'visible') vi.stubEnv('REPOSITORY_TEST_TOKEN', 'hidden') - const installed = await new RepositoryCache(join(root, 'cache')).resolve(specifier) + const installed = await new RepositoryCache(join(root, 'cache'), { + executableDirectories: [executableDirectory], + }).resolve(specifier) await expect(readFile(join(installed, 'prepared.txt'), 'utf8')).resolves.toBe('visible|absent\n') await expect(readFile(join(installed, 'dsh-plugin.mjs'), 'utf8')).resolves.toContain('export function apply') await expect(readFile(join(installed, 'dsh-plugin-assets/skills/0/fixture/SKILL.md'), 'utf8')) diff --git a/packages/self-modification/repository-plugin/README.i18n.yaml b/packages/self-modification/repository-plugin/README.i18n.yaml index e0ce60a641..c75c82dc84 100644 --- a/packages/self-modification/repository-plugin/README.i18n.yaml +++ b/packages/self-modification/repository-plugin/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/self-modification/repository-plugin/README.md -README.md: 33cd763d7dbe21b72f9e604b7b2e313081cf656f -README.zh.md: 903dfbe601cc76acb0c1e87453dc03ef0321409b +README.md: e0b45dc5fd40b5d1005598ae100d23ca7d0b6ec9 +README.zh.md: 30f200d2b7188d88e4a5e4e566cf3394f799933a diff --git a/packages/self-modification/repository-plugin/README.md b/packages/self-modification/repository-plugin/README.md index 33cd763d7d..e0b45dc5fd 100644 --- a/packages/self-modification/repository-plugin/README.md +++ b/packages/self-modification/repository-plugin/README.md @@ -14,10 +14,7 @@ Place an ordinary package in the repository's `.dsh-plugin` directory: "version": "0.0.0", "private": true, "scripts": { - "prepare": "dsh-plugin-prepare" - }, - "devDependencies": { - "@deepseek-ai/dsh-repository-plugin": "^0.0.1" + "prepack": "dsh-plugin-prepare" }, "dsh": { "skills": ["../skills"], @@ -26,7 +23,7 @@ Place an ordinary package in the repository's `.dsh-plugin` directory: } ``` -`dsh.skills` is an optional array of local skill roots. `dsh.mcpServers` is an optional path to one `.mcp.json`; at least one field is required. Paths are relative to `.dsh-plugin`, must stay under its parent source directory, and may therefore refer to existing repository assets such as `../skills`. A repository containing several Plugins gives each one its own `.dsh-plugin` package under a different selectable subdirectory. +`scripts.prepack` must be exactly `dsh-plugin-prepare`. DSH supplies that command from its own installed runtime while preparing Git source, so the repository package needs no DSH or npm dependency. `dsh.skills` is an optional array of local skill roots. `dsh.mcpServers` is an optional path to one `.mcp.json`; at least one field is required. Paths are relative to `.dsh-plugin`, must stay under its parent source directory, and may therefore refer to existing repository assets such as `../skills`. A repository containing several Plugins gives each one its own `.dsh-plugin` package under a different selectable subdirectory. ## Standalone app configuration @@ -47,7 +44,7 @@ Long-lived surfaces watch both `cordis.patch.yml` layers through Cordis HMR. A v ## Preparation -`dsh-plugin-prepare` validates `package.json#dsh`, verifies skill-root types, parses the MCP file, copies assets under `dsh-plugin-assets`, and writes `dsh-plugin.mjs`. The wrapper contains only the normalized static manifest and fixed code that looks up the `dsh-repository-plugin` Loader builtin. It neither discovers nor compiles repository JavaScript, and the runtime never imports another repository entry point. +During exact Git installation, DSH places a temporary host-owned `dsh-plugin-prepare` command on the isolated package lifecycle `PATH`; the command is not fetched from npm. The required `prepack` lifecycle runs after the Git package's dependency installation and before its selected subdirectory is packed, including when `.dsh-plugin` sits inside another package-manager workspace. The command validates `package.json#dsh`, verifies skill-root types, parses the MCP file, copies assets under `dsh-plugin-assets`, and writes `dsh-plugin.mjs`. Before importing that wrapper, DSH revalidates that the installed package retained the exact `prepack` declaration. The wrapper contains only the normalized static manifest and fixed code that looks up the `dsh-repository-plugin` Loader builtin. It neither discovers nor compiles repository JavaScript, and the runtime never imports another repository entry point. Failure to run or complete preparation fails installation before a cache generation is published. Rationale: [host-owned Git source preparation Agent Note](../../../.agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.md). The containing package manager still runs the configured repository package's lifecycle scripts. This restriction defines the supported DSH contribution surface; it is not a security boundary for a repository that the user chose to install as executable package-manager source. diff --git a/packages/self-modification/repository-plugin/README.zh.md b/packages/self-modification/repository-plugin/README.zh.md index 903dfbe601..30f200d2b7 100644 --- a/packages/self-modification/repository-plugin/README.zh.md +++ b/packages/self-modification/repository-plugin/README.zh.md @@ -14,10 +14,7 @@ "version": "0.0.0", "private": true, "scripts": { - "prepare": "dsh-plugin-prepare" - }, - "devDependencies": { - "@deepseek-ai/dsh-repository-plugin": "^0.0.1" + "prepack": "dsh-plugin-prepare" }, "dsh": { "skills": ["../skills"], @@ -26,7 +23,7 @@ } ``` -`dsh.skills` 是可选的本地 skill 根数组。`dsh.mcpServers` 是指向一个 `.mcp.json` 的可选路径；两者至少声明一个。路径相对于 `.dsh-plugin`，必须留在其父级源码目录下，因此可以引用 `../skills` 等仓库现有资源。一个仓库可以在不同的可选择子目录下放置多个各自独立的 `.dsh-plugin` 包。 +`scripts.prepack` 必须精确设为 `dsh-plugin-prepare`。DSH 会在准备 Git 源时由已安装的运行时提供该命令，因此仓库包无需添加 DSH 或 NPM 依赖。`dsh.skills` 是可选的本地 skill 根数组。`dsh.mcpServers` 是指向一个 `.mcp.json` 的可选路径；两者至少声明一个。路径相对于 `.dsh-plugin`，必须留在其父级源码目录下，因此可以引用 `../skills` 等仓库现有资源。一个仓库可以在不同的可选择子目录下放置多个各自独立的 `.dsh-plugin` 包。 ## 独立应用配置 @@ -47,7 +44,7 @@ ## 准备阶段 -`dsh-plugin-prepare` 校验 `package.json#dsh`、确认 skill 根类型、解析 MCP 文件、把资源复制到 `dsh-plugin-assets`，并写入 `dsh-plugin.mjs`。包装模块只包含规范化后的静态 manifest（元数据清单），以及查找 `dsh-repository-plugin` Loader builtin 的固定代码；它不会发现或编译仓库 JavaScript，运行时也不会导入仓库的其他入口。 +安装精确指定的 Git 源时，DSH 会把一个临时的宿主自有 `dsh-plugin-prepare` 命令放入隔离的包生命周期 `PATH`；该命令不从 NPM 获取。必需的 `prepack` 生命周期在 Git 包完成依赖安装后、选定子目录打包前运行，即使 `.dsh-plugin` 位于另一个包管理器工作区内也不例外。该命令校验 `package.json#dsh`、确认 skill 根类型、解析 MCP 文件、把资源复制到 `dsh-plugin-assets`，并写入 `dsh-plugin.mjs`。导入该包装模块前，DSH 会重新校验已安装包是否仍保留精确的 `prepack` 声明。包装模块只包含规范化后的静态 manifest（元数据清单），以及查找 `dsh-repository-plugin` Loader builtin 的固定代码；它不会发现或编译仓库 JavaScript，运行时也不会导入仓库的其他入口。准备阶段未运行或未完成时，安装会在发布缓存 generation 前失败。设计依据见[宿主自有 Git 源准备 Agent Note](../../../.agents/notes/implemented/bug-fix/2026-08-08-host-owned-git-repository-plugin-preparation.md)。 外层包管理器仍会运行已配置仓库包的生命周期脚本。这里的限制只定义 DSH 所支持的贡献表面；对于用户选择以可执行包管理器源安装的仓库，它并不是安全边界。 diff --git a/packages/self-modification/repository-plugin/src/format.ts b/packages/self-modification/repository-plugin/src/format.ts index 9d66321087..654e37c201 100644 --- a/packages/self-modification/repository-plugin/src/format.ts +++ b/packages/self-modification/repository-plugin/src/format.ts @@ -14,6 +14,8 @@ export const PREPARED_ENTRY_FILENAME = 'dsh-plugin.mjs' export const PREPARED_ASSET_DIRECTORY = 'dsh-plugin-assets' /** Loader builtin used by every generated import-free wrapper. */ export const REPOSITORY_PLUGIN_BUILTIN = 'dsh-repository-plugin' +/** Exact host-owned command required by the repository package `prepack` lifecycle. */ +export const REPOSITORY_PLUGIN_PREPARE_COMMAND = 'dsh-plugin-prepare' const sourceMetadataSchema = z.object({ skills: z.array(z.string().min(1)).default([]), @@ -23,6 +25,9 @@ const sourceMetadataSchema = z.object({ }) const sourcePackageSchema = z.looseObject({ name: z.string().min(1), + scripts: z.looseObject({ + prepack: z.literal(REPOSITORY_PLUGIN_PREPARE_COMMAND), + }), dsh: sourceMetadataSchema, }) const preparedManifestSchema = z.object({ @@ -148,7 +153,7 @@ export async function prepareDshPlugin(directory: string = process.cwd()): Promi throw new Error(`failed to read DSH plugin package metadata in ${pluginDirectory}`, { cause }) } const parsed = sourcePackageSchema.safeParse(packageValue) - if (!parsed.success) throw formatZodError('invalid package.json#dsh', parsed.error) + if (!parsed.success) throw formatZodError('invalid DSH plugin package.json', parsed.error) const sourceRoot = await realpath(dirname(pluginDirectory)) const skillSources: string[] = [] diff --git a/packages/self-modification/repository-plugin/src/index.ts b/packages/self-modification/repository-plugin/src/index.ts index 73eb2dc473..d3d287974f 100644 --- a/packages/self-modification/repository-plugin/src/index.ts +++ b/packages/self-modification/repository-plugin/src/index.ts @@ -20,6 +20,7 @@ import { } from './format.ts' import { parseMcpDocument, resolveMcpServers } from './mcp.ts' import { + createRepositoryPrepareCommand, loadPreparedRepository, resolveRepositoryCacheDirectory, resolveRepositorySpecifier, @@ -29,6 +30,7 @@ export { PREPARED_ASSET_DIRECTORY, PREPARED_ENTRY_FILENAME, REPOSITORY_PLUGIN_BUILTIN, + REPOSITORY_PLUGIN_PREPARE_COMMAND, prepareDshPlugin, type PreparedPluginManifest, } from './format.ts' @@ -129,17 +131,24 @@ export async function apply(ctx: Context, config: Config = {}): Promise { if (new Set(repositories).size !== repositories.length) { throw new Error('repository sources must resolve to unique exact specifiers') } - const cache = new RepositoryCache(resolveRepositoryCacheDirectory(config.cacheDir)) - await ctx.effect(async function* () { - ctx.loader.builtins[REPOSITORY_PLUGIN_BUILTIN] = preparedRuntime - yield () => { - if (ctx.loader.builtins[REPOSITORY_PLUGIN_BUILTIN] === preparedRuntime) { - Reflect.deleteProperty(ctx.loader.builtins, REPOSITORY_PLUGIN_BUILTIN) + const prepareCommand = repositories.length === 0 ? undefined : await createRepositoryPrepareCommand() + try { + const cache = new RepositoryCache(resolveRepositoryCacheDirectory(config.cacheDir), { + executableDirectories: prepareCommand === undefined ? [] : [prepareCommand.directory], + }) + await ctx.effect(async function* () { + ctx.loader.builtins[REPOSITORY_PLUGIN_BUILTIN] = preparedRuntime + yield () => { + if (ctx.loader.builtins[REPOSITORY_PLUGIN_BUILTIN] === preparedRuntime) { + Reflect.deleteProperty(ctx.loader.builtins, REPOSITORY_PLUGIN_BUILTIN) + } } - } - for (const repository of repositories) { - const plugin = await loadPreparedRepository(ctx, cache, repository) - yield plugin.dispose - } - }, 'repository-plugin runtime and sources') + for (const repository of repositories) { + const plugin = await loadPreparedRepository(ctx, cache, repository) + yield plugin.dispose + } + }, 'repository-plugin runtime and sources') + } finally { + await prepareCommand?.dispose() + } } diff --git a/packages/self-modification/repository-plugin/src/source.ts b/packages/self-modification/repository-plugin/src/source.ts index 0befb82e42..e8a9e9b9cc 100644 --- a/packages/self-modification/repository-plugin/src/source.ts +++ b/packages/self-modification/repository-plugin/src/source.ts @@ -3,12 +3,18 @@ * @module */ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' -import { pathToFileURL } from 'node:url' +import { fileURLToPath, pathToFileURL } from 'node:url' import type { Context, Fiber, FiberState, Plugin } from 'cordis' import type { RepositoryCache } from '@cordisjs/plugin-loader/repository' import { resolveDshHome } from '@deepseek-ai/dsh-paths' -import { PREPARED_ENTRY_FILENAME } from './format.ts' +import { z } from 'zod' +import { + PREPARED_ENTRY_FILENAME, + REPOSITORY_PLUGIN_PREPARE_COMMAND, +} from './format.ts' // Value mirror: Cordis's const enum has no runtime object to import. Keep // aligned with `packages/self-modification/tool-cordis/src/fiber-state.ts`. @@ -17,11 +23,66 @@ const FIBER_ACTIVE = 2 as FiberState.ACTIVE /** Directory under the Harness home containing immutable repository generations. */ export const DEFAULT_REPOSITORY_CACHE_DIRECTORY = 'repository-plugins' +/** Temporary host command supplied to repository package lifecycle scripts. */ +export interface RepositoryPrepareCommand { + /** Absolute directory to prepend to the isolated install's executable search path. */ + directory: string + /** Remove the temporary command directory. */ + dispose(): Promise +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'` +} + +function batchQuote(value: string): string { + return `"${value.replaceAll('%', '%%')}"` +} + +/** + * Materialize the DSH-owned prepare executable used only while pnpm packs Git source. + * @returns a command directory and its idempotent cleanup operation. + */ +export async function createRepositoryPrepareCommand(): Promise { + const directory = await mkdtemp(join(tmpdir(), 'dsh-repository-plugin-bin-')) + const target = fileURLToPath(new URL('../lib/bin.js', import.meta.url)) + try { + await Promise.all([ + writeFile(join(directory, REPOSITORY_PLUGIN_PREPARE_COMMAND), [ + '#!/bin/sh', + `exec ${shellQuote(process.execPath)} ${shellQuote(target)} "$@"`, + '', + ].join('\n'), { mode: 0o700 }), + writeFile(join(directory, `${REPOSITORY_PLUGIN_PREPARE_COMMAND}.cmd`), [ + '@echo off', + `${batchQuote(process.execPath)} ${batchQuote(target)} %*`, + '', + ].join('\r\n'), { mode: 0o700 }), + ]) + } catch (cause) { + /* v8 ignore next -- requires a host filesystem failure after mkdtemp; cleanup semantics are the contract under test. */ + await rm(directory, { recursive: true, force: true }) + /* v8 ignore next -- preserves that unstageable host failure after best-effort cleanup. */ + throw cause + } + return { + directory, + async dispose() { + await rm(directory, { recursive: true, force: true }) + }, + } +} + // The ref segment excludes `#` so `github:o/r#a#b` fails here — at the config // parser, with the syntax the error message promises — instead of inside the // cache's pnpm install ('misconfiguration fails loud at the earliest // resolvable point'). const GITHUB_SOURCE_PATTERN = /^github:([^/\s#&]+)\/([^/\s#&]+)#([^\s#&]+)(?:&path:(\/[^\s&]+))?$/ +const installedPackageSchema = z.looseObject({ + scripts: z.looseObject({ + prepack: z.literal(REPOSITORY_PLUGIN_PREPARE_COMMAND), + }), +}) function validPluginPath(path: string): boolean { const segments = path.split('/').slice(1) @@ -57,6 +118,19 @@ export function resolveRepositoryCacheDirectory(configured: string | undefined): return resolve(configured ?? join(resolveDshHome(), 'cache', DEFAULT_REPOSITORY_CACHE_DIRECTORY)) } +async function assertInstalledPackageMetadata(directory: string): Promise { + let value: unknown + try { + value = JSON.parse(await readFile(join(directory, 'package.json'), 'utf8')) as unknown + } catch (cause) { + throw new Error(`failed to read installed DSH plugin package metadata in ${directory}`, { cause }) + } + const result = installedPackageSchema.safeParse(value) + if (!result.success) { + throw new Error(`installed DSH plugin package must declare scripts.prepack as ${JSON.stringify(REPOSITORY_PLUGIN_PREPARE_COMMAND)}:\n${z.prettifyError(result.error)}`) + } +} + /** * Load one exact repository generation's generated wrapper as a child Cordis fiber. * @param ctx - repository runtime context that owns the child. @@ -73,6 +147,7 @@ export async function loadPreparedRepository( const directory = await cache.resolve(specifier) const filename = join(directory, PREPARED_ENTRY_FILENAME) try { + await assertInstalledPackageMetadata(directory) const plugin = await import(/* @vite-ignore */pathToFileURL(filename).href) as Plugin const fiber = ctx.plugin(plugin) await fiber diff --git a/packages/self-modification/repository-plugin/tests/repository-plugin.spec.ts b/packages/self-modification/repository-plugin/tests/repository-plugin.spec.ts index 39840fd1f3..d026a4e14c 100644 --- a/packages/self-modification/repository-plugin/tests/repository-plugin.spec.ts +++ b/packages/self-modification/repository-plugin/tests/repository-plugin.spec.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join, relative, resolve } from 'node:path' import { pathToFileURL } from 'node:url' @@ -14,6 +14,7 @@ import * as RepositoryPlugin from '@deepseek-ai/dsh-repository-plugin' import * as RepositoryPluginInvariant from '@deepseek-ai/dsh-repository-plugin/invariant' import { parsePreparedPluginConfig } from '../src/format.ts' import { + createRepositoryPrepareCommand, loadPreparedRepository, resolveRepositoryCacheDirectory, resolveRepositorySpecifier, @@ -30,7 +31,12 @@ async function temporaryDirectory(name: string): Promise { async function writePlugin(root: string, name: string, dsh: Record): Promise { const directory = join(root, '.dsh-plugin') await mkdir(directory, { recursive: true }) - await writeFile(join(directory, 'package.json'), `${JSON.stringify({ name, version: '0.0.0', dsh }, undefined, 2)}\n`) + await writeFile(join(directory, 'package.json'), `${JSON.stringify({ + name, + version: '0.0.0', + scripts: { prepack: RepositoryPlugin.REPOSITORY_PLUGIN_PREPARE_COMMAND }, + dsh, + }, undefined, 2)}\n`) return directory } @@ -102,6 +108,16 @@ describe('dsh-plugin-prepare', () => { await writeFile(join(malformed, 'package.json'), '{') await expect(RepositoryPlugin.prepareDshPlugin(malformed)).rejects.toThrow('failed to read DSH plugin package metadata') + const lifecycleRoot = await temporaryDirectory('wrong-lifecycle') + const lifecycle = join(lifecycleRoot, '.dsh-plugin') + await mkdir(lifecycle) + await writeFile(join(lifecycle, 'package.json'), JSON.stringify({ + name: 'wrong-lifecycle', + scripts: { prepare: 'dsh-plugin-prepare' }, + dsh: { skills: ['../skills'] }, + })) + await expect(RepositoryPlugin.prepareDshPlugin(lifecycle)).rejects.toThrow('prepack') + const emptyRoot = await temporaryDirectory('empty-metadata') const empty = await writePlugin(emptyRoot, 'empty', {}) await expect(RepositoryPlugin.prepareDshPlugin(empty)).rejects.toThrow('declare at least one skill root or mcpServers file') @@ -272,6 +288,17 @@ describe('prepared repository plugin Loader composition', () => { }) describe('configured GitHub repository sources', () => { + it('creates host-owned prepare commands and removes them idempotently', async () => { + const command = await createRepositoryPrepareCommand() + expect(await readFile(join(command.directory, RepositoryPlugin.REPOSITORY_PLUGIN_PREPARE_COMMAND), 'utf8')) + .toContain(process.execPath) + expect(await readFile(join(command.directory, `${RepositoryPlugin.REPOSITORY_PLUGIN_PREPARE_COMMAND}.cmd`), 'utf8')) + .toContain(process.execPath) + await command.dispose() + await command.dispose() + await expect(stat(command.directory)).rejects.toMatchObject({ code: 'ENOENT' }) + }) + it('defaults an omitted source list and rejects unknown configuration fields', () => { expect(RepositoryPlugin.Config.parse(undefined)).toEqual({ repositories: [] }) expect(RepositoryPlugin.Config.safeParse({ repositories: [], unexpected: true }).success).toBe(false) @@ -439,12 +466,43 @@ describe('configured GitHub repository sources', () => { it('labels a missing prepared wrapper with its exact source and path', async () => { const root = await temporaryDirectory('missing-wrapper') + const directory = await writePlugin(root, 'missing-wrapper', { skills: ['../skills'] }) const ctx = new Context() const specifier = 'github:owner/repository#missing&path:/.dsh-plugin' - await expect(loadPreparedRepository(ctx, { resolve: async () => root }, specifier)) + await expect(loadPreparedRepository(ctx, { resolve: async () => directory }, specifier)) .rejects.toThrow(`failed to load prepared repository Plugin ${JSON.stringify(specifier)}`) await ctx.fiber.dispose() }) + + it('rejects installed source with the obsolete prepare lifecycle', async () => { + const root = await temporaryDirectory('installed-lifecycle') + await writeFile(join(root, 'package.json'), JSON.stringify({ + name: 'installed-lifecycle', + scripts: { prepare: 'dsh-plugin-prepare' }, + })) + const ctx = new Context() + await expect(loadPreparedRepository(ctx, { resolve: async () => root }, 'github:owner/repository#old&path:/.dsh-plugin')) + .rejects.toMatchObject({ + cause: expect.objectContaining({ + message: expect.stringContaining('must declare scripts.prepack') as string, + }) as Error, + }) + await ctx.fiber.dispose() + }) + + it('labels missing installed package metadata with its source', async () => { + const root = await temporaryDirectory('missing-installed-metadata') + const ctx = new Context() + const specifier = 'github:owner/repository#damaged&path:/.dsh-plugin' + await expect(loadPreparedRepository(ctx, { resolve: async () => root }, specifier)) + .rejects.toMatchObject({ + message: expect.stringContaining(JSON.stringify(specifier)) as string, + cause: expect.objectContaining({ + message: expect.stringContaining('failed to read installed DSH plugin package metadata') as string, + }) as Error, + }) + await ctx.fiber.dispose() + }) }) describe('repository plugin invariant companion', () => { diff --git a/scripts/run-gates.spec.ts b/scripts/run-gates.spec.ts index c7b8a7d2c9..5c4ba9899a 100644 --- a/scripts/run-gates.spec.ts +++ b/scripts/run-gates.spec.ts @@ -218,7 +218,7 @@ describe('Node 24 lane ownership', () => { const subject = withPnpmEntrypoint(() => gatesForMode('ci-consumers')) expect(defaultConcurrency('ci-consumers', subject.length, 4)).toEqual({ - workers: 10, + workers: 11, source: 'ci-consumers gate count', }) expect(subject.map(item => item.id)).toEqual([ @@ -232,11 +232,19 @@ describe('Node 24 lane ownership', () => { 'doc-typecheck', 'node-next-types', 'built-bin-smoke', + 'github-repository-plugin-e2e', ]) expect(subject.find(item => item.id === 'publint')?.needs).toEqual(['build']) expect(subject.find(item => item.id === 'built-package-invariants')?.needs).toEqual(['publint']) expect(subject.find(item => item.id === 'lint-and-duplication')?.needs).toEqual(['built-package-invariants']) - for (const id of ['snapshot', 'web-snapshot', 'doc-typecheck', 'node-next-types', 'built-bin-smoke']) { + for (const id of [ + 'snapshot', + 'web-snapshot', + 'doc-typecheck', + 'node-next-types', + 'built-bin-smoke', + 'github-repository-plugin-e2e', + ]) { expect(subject.find(item => item.id === id)?.needs).toEqual(['built-package-invariants']) } expect(subject.find(item => item.id === 'snapshot')?.env).toEqual({ DSH_EXAMPLE_MODE: 'lib' }) @@ -249,6 +257,16 @@ describe('Node 24 lane ownership', () => { 'packages/subagent/subagent-claude-code/tests/loader-composition.e2e.ts', ]), ) + const githubRepositoryPlugin = subject.find(item => item.id === 'github-repository-plugin-e2e') + expect(githubRepositoryPlugin).toMatchObject({ + label: 'GitHub repository Plugin dsh run', + env: { + DSH_REQUIRE_GITHUB_REPOSITORY_PLUGIN_E2E: '1', + }, + }) + expect(githubRepositoryPlugin?.args).toEqual( + expect.arrayContaining(['apps/cli/tests/github-repository-plugin.built.e2e.ts']), + ) expect(subject.find(item => item.id === 'web-snapshot')).toMatchObject({ displayCommand: 'DSH_SNAPSHOT=replay pnpm run test:web:built', env: { DSH_SNAPSHOT: 'replay' }, diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index d99889d28b..3bd0e11987 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -406,6 +406,7 @@ function ciConsumerGates(): Gate[] { needs: validatedBuild, }), builtBinSmokeGate(validatedBuild), + githubRepositoryPluginE2eGate(validatedBuild), ] } @@ -636,6 +637,20 @@ function builtBinSmokeGate(needs: string[] = ['build']): Gate { }) } +function githubRepositoryPluginE2eGate(needs: string[]): Gate { + return pnpmExec('github-repository-plugin-e2e', [ + 'vitest', + 'run', + '--config', + 'vitest.e2e.config.ts', + 'apps/cli/tests/github-repository-plugin.built.e2e.ts', + ], { + label: 'GitHub repository Plugin dsh run', + needs, + env: { DSH_REQUIRE_GITHUB_REPOSITORY_PLUGIN_E2E: '1' }, + }) +} + /** * Reject a gate list whose graph cannot be executed unambiguously. * @param gates - complete aggregate to validate. diff --git a/vendor/README.md b/vendor/README.md index 02e33ea723..3cc2a313d2 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -39,7 +39,7 @@ Keep this log exhaustive — every divergence from upstream must be listed. 7. **`cordis/src/*.ts` JSDoc enrichment**: added `@param`/`@returns` tags and contract documentation (disposal semantics, waterfall veto, bail conditions, error cases) across the public plugin-author surface — `Context` (class, statics, and the `Context` interface properties incl. `root`), `EventsService`, `Fiber`, `RegistryService`, `ReflectService`, `Service`, `LoggerService` and their `declare module './context.ts'` overloads. Comment-only; no code changes. Motivation: the website API-reference generator renders these docs and hard-errors on undocumented members. Retire this entry when the enrichment is upstreamed to the fork. 8. **Transactional Loader/Include config reconciliation**: Loader imports a changed entry name before disposal, awaits lifecycle settlement, and restores the previous plugin or config when candidate application fails. Loader settlement rechecks service-gated fibers after current tasks drain, rejects failures, and leaves fibers with absent dependencies pending. Group updates start candidates concurrently, await every outcome, undo changes and additions on failure, await removal, preserve programmatic option identity, and persist direct or tree-level mutations only after success. Include reads and validates detached candidate content, applies patches to a clone, reconciles the tree, and only then commits its cached content/data; direct refresh failures propagate for the caller to contain. A non-array parse is invalid, patches re-apply on every file or Include-config update, an omitted patch list clears the overlay, and initial content falls back to `initial` only on `ENOENT`. Covered by `packages/boot/app-boot/tests/config-reload.spec.ts` and `packages/host/webserver/tests/webserver.spec.ts`. 9. **`hmr/src/index.ts` exact config watching**: `registerConfig()` watches one absolute config path outside module roots, including a path under missing parents, serializes and coalesces refreshes, and returns an async disposer that closes the watcher and drains active work. Refresh failures are normalized to `Error`, logged, and broadcast through the parallel `hmr/config-update-failed` event; observer failures are contained. Config-file changes discovered by the ordinary HMR watcher use the same serialized path. Covered by `packages/boot/app-boot/tests/hmr-config.spec.ts`. -10. **`loader/src/repository.ts`, `loader/tsdown.config.ts`, and the `@cordisjs/plugin-loader/repository` export**: the Node-only `RepositoryCache` installs one exact dependency specifier through the bundled `pnpm@11.7.0`, single-flights callers, and atomically publishes only a prepared package plus marker under the specifier hash. The subpath stays out of the browser-reachable Loader entry. Identical specifiers permanently reuse that entry; callers change the ref/specifier for another generation. The isolated workspace permits dependency build scripts because a configured repository is executable code, while the child drops ambient credential-shaped variables. Covered by `packages/boot/app-boot/tests/repository-cache.spec.ts`, including a keyless local-Git prepare run through the bundled pnpm. +10. **`loader/src/repository.ts`, `loader/tsdown.config.ts`, and the `@cordisjs/plugin-loader/repository` export**: the Node-only `RepositoryCache` installs one exact dependency specifier through the bundled `pnpm@11.7.0`, single-flights callers, and atomically publishes only a prepared package plus marker under the specifier hash. The subpath stays out of the browser-reachable Loader entry. Identical specifiers permanently reuse that entry; callers change the ref/specifier for another generation. Callers may prepend host-owned executable directories to the isolated package lifecycle `PATH`; all paths are resolved before the child starts. The isolated workspace permits dependency build scripts because a configured repository is executable code, while the child drops ambient credential-shaped variables. Covered by `packages/boot/app-boot/tests/repository-cache.spec.ts`, including a keyless local-Git `prepack` run through the bundled pnpm and an injected command directory. 11. **Vendored Node-compatible TypeScript**: marked erased imports explicitly across `cordis`, `loader`, `include`, `hmr`, and `schemastery` so Node's native TypeScript transform does not request types as runtime exports. Schemastery's source uses an ESM default export and its package declares `type: module`; its built ESM/CJS entries retain explicit `.mjs`/`.cjs` extensions. 12. **`include/src/index.ts` patch-semantics export**: extracted the private `applyPatches` body into the exported pure function `applyEntryPatches(data, patches, warn)` (the method delegates to it) and exported the `!!js` YAML dialect as `entryListSchema`, so `dsh --dump-config` composes and prints exactly what the include would mount without booting a tree. Behavior-preserving for mounting; the extraction exists because config tooling must never reimplement (and drift from) the patch algorithm. `applyEntryPatches` also indexes each `insert`ed entry as it is added, so a later patch in the same list can configure or disable a row an earlier patch inserted; upstream built the id index once before the patch loop, leaving inserted rows silently unpatchable. That matters because `dsh` composes an empty profile root with each bundle's patch layer, the profile's and the home-level `cordis.patch.yml`, and any `--patch` overlays as sibling patch lists at one include level — patches never cross an include boundary, so surface-only rows would otherwise be unreachable from user config. Covered by `packages/boot/app-boot/tests/config-reload.spec.ts`. 13. **`include/src/index.ts` serialized child-tree mutation and `hmr/src/index.ts` main-watcher initial-scan suppression**: every Include child-tree mutation (initial apply, refresh, `internal/update` patch re-application) runs through one per-Include queue, because the group's transactional `update` is not reentrant — two concurrent applies interleave create and rollback on the same entries and strand the Include fiber without ever settling. The HMR main watcher passes `ignoreInitial: true`: the initial scan re-announced files boot had just consumed, and its `add` for a config file refreshed an Include mid-initial-apply; once serialized, a failing initial apply's rollback disposed HMR, whose teardown drain waited on the queued refresh sitting behind that same apply — a deadlock that exited 13 with no diagnostic. `registerConfig()` keeps its own `ignoreInitial: false` watcher because a user patch layer present at registration must apply once. Covered by the patch-overlay boot-failure built-bin case in `apps/cli/tests/built-bin.e2e.ts`. diff --git a/vendor/loader/src/repository.ts b/vendor/loader/src/repository.ts index 94a0c5cf16..c0cd1589aa 100644 --- a/vendor/loader/src/repository.ts +++ b/vendor/loader/src/repository.ts @@ -8,7 +8,7 @@ import { spawn } from 'node:child_process' import { createHash } from 'node:crypto' import { mkdir, mkdtemp, readFile, rename, rm, stat, writeFile } from 'node:fs/promises' import { createRequire } from 'node:module' -import { dirname, join, resolve } from 'node:path' +import { delimiter, dirname, join, resolve } from 'node:path' /** Exact pnpm release shipped with the Loader for repository installation. */ export const BUNDLED_PNPM_VERSION = '11.7.0' @@ -21,6 +21,14 @@ const SENSITIVE_ENV_PATTERN = /KEY|PASSWORD|SECRET|TOKEN/i /** Injectable isolated-install boundary used by {@link RepositoryCache}. */ export type RepositoryInstall = (directory: string) => Promise +/** Installation controls for {@link RepositoryCache}. */ +export interface RepositoryCacheOptions { + /** Override the isolated package installation boundary. */ + install?: RepositoryInstall + /** Command directories resolved absolutely and prepended to package lifecycle `PATH`. */ + executableDirectories?: readonly string[] +} + interface CacheMarker { specifier: string } @@ -29,12 +37,26 @@ function scrubEnvironment(environment: NodeJS.ProcessEnv = process.env): NodeJS. return Object.fromEntries(Object.entries(environment).filter(([name]) => !SENSITIVE_ENV_PATTERN.test(name))) } +function installEnvironment(executableDirectories: readonly string[]): NodeJS.ProcessEnv { + const scrubbed = scrubEnvironment() + if (executableDirectories.length === 0) return scrubbed + const path = Object.entries(scrubbed).find(([name]) => name.toUpperCase() === 'PATH')?.[1] + const withoutPath = Object.fromEntries(Object.entries(scrubbed).filter(([name]) => name.toUpperCase() !== 'PATH')) + return { + ...withoutPath, + PATH: [...executableDirectories, ...(path === undefined ? [] : [path])].join(delimiter), + } +} + function appendOutput(current: string, chunk: Uint8Array): string { const combined = current + Buffer.from(chunk).toString('utf8') return combined.length <= MAX_ERROR_OUTPUT ? combined : combined.slice(-MAX_ERROR_OUTPUT) } -async function installWithBundledPnpm(directory: string): Promise { +async function installWithBundledPnpm( + directory: string, + executableDirectories: readonly string[], +): Promise { const require = createRequire(import.meta.url) const pnpmManifest = require.resolve('pnpm') const pnpmBin = join(dirname(pnpmManifest), 'bin', 'pnpm.mjs') @@ -47,7 +69,7 @@ async function installWithBundledPnpm(directory: string): Promise { '--reporter=append-only', ], { cwd: directory, - env: scrubEnvironment(), + env: installEnvironment(executableDirectories), shell: false, stdio: ['ignore', 'pipe', 'pipe'], }) @@ -122,13 +144,16 @@ export class RepositoryCache { readonly directory: string private readonly tasks = new Map>() + private readonly install: RepositoryInstall /** * @param directory - caller-owned persistent cache root. - * @param install - isolated package installation boundary; defaults to the bundled pnpm. + * @param options - isolated installer override and lifecycle command directories. */ - constructor(directory: string, private readonly install: RepositoryInstall = installWithBundledPnpm) { + constructor(directory: string, options: RepositoryCacheOptions = {}) { this.directory = resolve(directory) + const executableDirectories = (options.executableDirectories ?? []).map(entry => resolve(entry)) + this.install = options.install ?? (staging => installWithBundledPnpm(staging, executableDirectories)) } /**