From 8a9d882a11930a1348f039a5e8f783eecb6616fa Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:29:18 +0800 Subject: [PATCH] cleanup(build): minimize native payload handling --- python/README.i18n.yaml | 4 +- python/README.md | 2 +- python/README.zh.md | 2 +- python/sdk-runtime/hatch_build.py | 3 +- .../src/deepseek_harness_runtime/__init__.py | 4 +- python/sdk/tests/test_release_version.py | 54 +++++-------------- python/sdk/tests/test_runtime_resolution.py | 25 ++++----- scripts/build-exe-for-python-sdk.ts | 47 +++++----------- scripts/build-python-release.py | 25 +++------ 9 files changed, 47 insertions(+), 119 deletions(-) diff --git a/python/README.i18n.yaml b/python/README.i18n.yaml index f0d6c67967..59f3e836cf 100644 --- a/python/README.i18n.yaml +++ b/python/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write python/README.md -README.md: aee682e25fc33287c49131d0f5b92b136ed16bae -README.zh.md: 4404114fcdab78468991769a4657370f85997a88 +README.md: dfd9d909122f9245a19fe91d8a394156795b13ae +README.zh.md: 151b2cdab4f28384a20d16c86d398f669fbdc818 diff --git a/python/README.md b/python/README.md index aee682e25f..dfd9d90912 100644 --- a/python/README.md +++ b/python/README.md @@ -22,7 +22,7 @@ pnpm exec tsx scripts/build-exe-for-python-sdk.ts --skip-build # lib/ artifac pnpm exec tsx scripts/build-exe-for-python-sdk.ts --targets=node24-linux-x64,node24-linux-arm64,node24-macos-arm64 ``` -Products land in `dist-exe/` and are synced into this package as `sdk-runtime/src/deepseek_harness_runtime/runtime/dsh-jsonrpc-agent-pkg--` plus the matching `-spawn-helper` required by `node-pty` (platform: `linux`/`macos`; arch: `x64`/`arm64`) — after a local build the SDK finds the runtime with no further setup. The `build-exe-for-python-sdk` CI workflow (manual dispatch, or the `build-exe` PR label) exercises the same products. A full three-target run retains four release wheels; a subset dispatch retains the SDK wheel and selected platform wheels. Which plugins the exe bundles and how the carriers are organized: [sdk-runtime README](sdk-runtime/README.md); the build also refreshes the dev-only node carrier (see "against the Node source" below). +Products land in `dist-exe/` and are synced into this package as `sdk-runtime/src/deepseek_harness_runtime/runtime/dsh-jsonrpc-agent-pkg--` (platform: `linux`/`macos`; arch: `x64`/`arm64`); macOS builds also sync the matching `-spawn-helper` required by `node-pty`. After a local build the SDK finds the runtime with no further setup. The `build-exe-for-python-sdk` CI workflow (manual dispatch, or the `build-exe` PR label) exercises the same products. A full three-target run retains four release wheels; a subset dispatch retains the SDK wheel and selected platform wheels. Which plugins the exe bundles and how the carriers are organized: [sdk-runtime README](sdk-runtime/README.md); the build also refreshes the dev-only node carrier (see "against the Node source" below). ## Validating the SDK against the executable diff --git a/python/README.zh.md b/python/README.zh.md index 4404114fcd..151b2cdab4 100644 --- a/python/README.zh.md +++ b/python/README.zh.md @@ -22,7 +22,7 @@ pnpm exec tsx scripts/build-exe-for-python-sdk.ts --skip-build # lib/ artifac pnpm exec tsx scripts/build-exe-for-python-sdk.ts --targets=node24-linux-x64,node24-linux-arm64,node24-macos-arm64 ``` -产物落入 `dist-exe/`，并同步进本包的 `sdk-runtime/src/deepseek_harness_runtime/runtime/dsh-jsonrpc-agent-pkg--` 及 `node-pty` 所需的同名 `-spawn-helper` 伴随文件（platform：`linux`/`macos`；arch：`x64`/`arm64`），本地构建完成后 SDK 不需要额外设置就能找到运行时。`build-exe-for-python-sdk` CI 工作流（手动触发，或给 PR 打 `build-exe` 标签）会测试同样的产物。完整构建三个目标时保留 4 个发布用 wheel 包；手动选择部分目标时保留 SDK wheel 与所选平台的 wheel。exe 内置哪些插件、载体如何组织，见 [sdk-runtime README](sdk-runtime/README.md)；构建还会顺带刷新仅供开发使用的 `node` 载体（见下文「对着 Node 源码运行」）。 +产物落入 `dist-exe/`，并同步进本包的 `sdk-runtime/src/deepseek_harness_runtime/runtime/dsh-jsonrpc-agent-pkg--`（platform：`linux`/`macos`；arch：`x64`/`arm64`）；macOS 构建还会同步 `node-pty` 所需的同名 `-spawn-helper` 伴随文件。本地构建完成后 SDK 不需要额外设置就能找到运行时。`build-exe-for-python-sdk` CI 工作流（手动触发，或给 PR 打 `build-exe` 标签）会测试同样的产物。完整构建三个目标时保留 4 个发布用 wheel 包；手动选择部分目标时保留 SDK wheel 与所选平台的 wheel。exe 内置哪些插件、载体如何组织，见 [sdk-runtime README](sdk-runtime/README.md)；构建还会顺带刷新仅供开发使用的 `node` 载体（见下文「对着 Node 源码运行」）。 ## 用可执行文件验证 SDK diff --git a/python/sdk-runtime/hatch_build.py b/python/sdk-runtime/hatch_build.py index 19ec962257..b0f9a79550 100644 --- a/python/sdk-runtime/hatch_build.py +++ b/python/sdk-runtime/hatch_build.py @@ -13,7 +13,6 @@ _PLATFORMS = { "linux-arm64": ("manylinux_2_28_aarch64", "dsh-jsonrpc-agent-pkg-linux-arm64"), "macos-arm64": ("macosx_11_0_arm64", "dsh-jsonrpc-agent-pkg-macos-arm64"), } -_SPAWN_HELPER_SUFFIX = "-spawn-helper" def _host_platform_tag() -> str: @@ -50,7 +49,7 @@ class RuntimeBuildHook(BuildHookInterface): runtime_files = sorted(runtime_dir.glob("dsh-jsonrpc-agent-pkg-*") if runtime_dir.is_dir() else []) expected_files = [expected_executable] if "-macos-" in expected_executable: - expected_files.append(f"{expected_executable}{_SPAWN_HELPER_SUFFIX}") + expected_files.append(f"{expected_executable}-spawn-helper") found_files = [path.name for path in runtime_files] if found_files != expected_files: raise RuntimeError( diff --git a/python/sdk-runtime/src/deepseek_harness_runtime/__init__.py b/python/sdk-runtime/src/deepseek_harness_runtime/__init__.py index d6f8c497b6..a3aa53ae80 100644 --- a/python/sdk-runtime/src/deepseek_harness_runtime/__init__.py +++ b/python/sdk-runtime/src/deepseek_harness_runtime/__init__.py @@ -28,7 +28,6 @@ import sys from pathlib import Path PACKAGE_METADATA_FILENAME = "deepseek-harness-runtime.json" -SPAWN_HELPER_SUFFIX = "-spawn-helper" RUNTIME_MODE_ENV_VAR = "DSH_RUNTIME_MODE" @@ -85,7 +84,7 @@ def bundled_runtime_path() -> Path: + _EXE_ACQUISITION_HINT ) if tag.startswith("macos-"): - helper = Path(f"{path}{SPAWN_HELPER_SUFFIX}") + helper = Path(f"{path}-spawn-helper") if not helper.is_file(): raise FileNotFoundError( f"deepseek-harness-runtime-bin is missing the node-pty spawn helper at {helper}. " @@ -153,7 +152,6 @@ def _node_launch_args() -> tuple[str, str]: __all__ = [ "PACKAGE_METADATA_FILENAME", "RUNTIME_MODE_ENV_VAR", - "SPAWN_HELPER_SUFFIX", "bundled_default_config_path", "bundled_package_dir", "bundled_runtime_path", diff --git a/python/sdk/tests/test_release_version.py b/python/sdk/tests/test_release_version.py index 68a9aac993..7e5f660070 100644 --- a/python/sdk/tests/test_release_version.py +++ b/python/sdk/tests/test_release_version.py @@ -4,7 +4,6 @@ from __future__ import annotations import json import runpy -import stat from pathlib import Path from types import SimpleNamespace @@ -40,51 +39,22 @@ def test_repository_version_rejects_non_stable_versions(tmp_path: Path) -> None: build_python_release.repository_version(tmp_path) -def test_stage_runtime_copies_executable_and_spawn_helper(tmp_path: Path) -> None: - executable = tmp_path / "dsh-jsonrpc-agent-pkg-macos-arm64" - executable.write_bytes(b"runtime") - executable.chmod(0o755) - spawn_helper = Path(f"{executable}-spawn-helper") - spawn_helper.write_bytes(b"helper") - spawn_helper.chmod(0o751) - destination = tmp_path / "staging" - - build_python_release.stage_runtime( - destination, - "1.2.3", - executable, - executable.name, - ) - - runtime_dir = destination / "src" / "deepseek_harness_runtime" / "runtime" - assert (runtime_dir / executable.name).read_bytes() == b"runtime" - copied_helper = runtime_dir / spawn_helper.name - assert copied_helper.read_bytes() == b"helper" - assert copied_helper.stat().st_mode & stat.S_IXUSR - - -def test_stage_runtime_rejects_missing_spawn_helper(tmp_path: Path) -> None: - executable = tmp_path / "dsh-jsonrpc-agent-pkg-macos-arm64" - executable.write_bytes(b"runtime") - executable.chmod(0o755) - - with pytest.raises(FileNotFoundError, match="spawn-helper"): - build_python_release.stage_runtime( - tmp_path / "staging", - "1.2.3", - executable, - executable.name, - ) - - -def test_stage_runtime_copies_linux_executable_without_spawn_helper(tmp_path: Path) -> None: - executable = tmp_path / "dsh-jsonrpc-agent-pkg-linux-x64" +@pytest.mark.parametrize(("target", "with_helper"), [("linux-x64", False), ("macos-arm64", True)]) +def test_stage_runtime_copies_platform_payload( + tmp_path: Path, target: str, with_helper: bool +) -> None: + executable = tmp_path / f"dsh-jsonrpc-agent-pkg-{target}" executable.write_bytes(b"runtime") executable.chmod(0o755) + expected = {executable.name: b"runtime"} + if with_helper: + spawn_helper = Path(f"{executable}-spawn-helper") + spawn_helper.write_bytes(b"helper") + spawn_helper.chmod(0o755) + expected[spawn_helper.name] = b"helper" destination = tmp_path / "staging" build_python_release.stage_runtime(destination, "1.2.3", executable, executable.name) runtime_dir = destination / "src" / "deepseek_harness_runtime" / "runtime" - runtime_files = [path.name for path in runtime_dir.glob("dsh-jsonrpc-agent-pkg-*")] - assert runtime_files == [executable.name] + assert {path.name: path.read_bytes() for path in runtime_dir.glob("dsh-jsonrpc-agent-pkg-*")} == expected diff --git a/python/sdk/tests/test_runtime_resolution.py b/python/sdk/tests/test_runtime_resolution.py index e0411bb1fd..778203f4d1 100644 --- a/python/sdk/tests/test_runtime_resolution.py +++ b/python/sdk/tests/test_runtime_resolution.py @@ -44,25 +44,18 @@ def test_explicit_mode_wins_over_env_mode(monkeypatch: pytest.MonkeyPatch) -> No assert args[0].endswith(("-x64", "-arm64")) -@pytest.mark.parametrize( - ("platform_tag", "requires_helper"), - [("linux-x64", False), ("macos-arm64", True)], -) def test_runtime_requires_spawn_helper_only_on_macos( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - platform_tag: str, - requires_helper: bool, + tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: runtime_dir = tmp_path / "runtime" runtime_dir.mkdir() - executable = runtime_dir / f"dsh-jsonrpc-agent-pkg-{platform_tag}" - executable.touch() + linux = runtime_dir / "dsh-jsonrpc-agent-pkg-linux-x64" + linux.touch() + (runtime_dir / "dsh-jsonrpc-agent-pkg-macos-arm64").touch() monkeypatch.setattr(runtime, "bundled_package_dir", lambda: tmp_path) - monkeypatch.setattr(runtime, "_current_platform_tag", lambda: platform_tag) - if requires_helper: - with pytest.raises(FileNotFoundError, match="node-pty spawn helper"): - runtime.bundled_runtime_path() - else: - assert runtime.bundled_runtime_path() == executable + monkeypatch.setattr(runtime, "_current_platform_tag", lambda: "macos-arm64") + with pytest.raises(FileNotFoundError, match="node-pty spawn helper"): + runtime.bundled_runtime_path() + monkeypatch.setattr(runtime, "_current_platform_tag", lambda: "linux-x64") + assert runtime.bundled_runtime_path() == linux diff --git a/scripts/build-exe-for-python-sdk.ts b/scripts/build-exe-for-python-sdk.ts index bbfa2402d0..d2cdbeec4d 100644 --- a/scripts/build-exe-for-python-sdk.ts +++ b/scripts/build-exe-for-python-sdk.ts @@ -7,7 +7,7 @@ */ import { spawn } from 'node:child_process' -import { existsSync, mkdirSync, statSync } from 'node:fs' +import { existsSync, statSync } from 'node:fs' import { chmod, copyFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises' import { basename, dirname, join, resolve, sep } from 'node:path' import { parseArgs } from 'node:util' @@ -19,7 +19,6 @@ const DEPLOY_ROOT_PACKAGE = 'dsh-jsonrpc-agent-pkg' /** The app entry inside the deployed closure. */ const ENTRY_BIN = 'node_modules/@deepseek-ai/dsh-jsonrpc-demo/lib/bin.js' const OUTPUT_BASENAME = 'dsh-jsonrpc-agent-pkg' -const SPAWN_HELPER_SUFFIX = '-spawn-helper' /** Default Node major; SEA mode requires at least Node 22. */ const DEFAULT_NODE_RANGE = 'node24' /** Pinned for reproducible builds. */ @@ -295,7 +294,7 @@ class SingleExeBuild { async pack(target: Target): Promise { const product = join(this.outDir, `${OUTPUT_BASENAME}-${target.platform}-${target.arch}`) await this.prepareNativePty(target) - if (!this.cli.dryRun) mkdirSync(this.outDir, { recursive: true }) + if (!this.cli.dryRun) await mkdir(this.outDir, { recursive: true }) await this.run(`pkg ${target.spec}`, pnpmBin(), [ 'dlx', PKG_SPEC, @@ -310,13 +309,13 @@ class SingleExeBuild { throw new Error(`build-exe-for-python-sdk: product ${product} is missing after the pkg run; inspect ${this.outDir}.`) } if (target.platform !== 'macos') return [product] - const spawnHelper = `${product}${SPAWN_HELPER_SUFFIX}` + const spawnHelper = `${product}-spawn-helper` + const source = join(this.staging, 'node_modules', 'node-pty', 'prebuilds', `darwin-${target.arch}`, 'spawn-helper') if (this.cli.dryRun) { - console.log(`build-exe-for-python-sdk: [dry-run] copy target node-pty spawn-helper to ${spawnHelper}`) + console.log(`build-exe-for-python-sdk: [dry-run] cp ${source} ${spawnHelper}`) } else { - const source = this.resolveSpawnHelper(target) await copyFile(source, spawnHelper) - await chmod(spawnHelper, statSync(source).mode & 0o777) + await chmod(spawnHelper, 0o755) } return [product, spawnHelper] } @@ -327,47 +326,27 @@ class SingleExeBuild { * @param target - the pkg target whose native addon is being staged. */ private async prepareNativePty(target: Target): Promise { - const stagedRoot = join(this.staging, 'node_modules', 'node-pty') - const stagedBuild = join(stagedRoot, 'build') + const stagedBuild = join(this.staging, 'node_modules', 'node-pty', 'build') if (this.cli.dryRun) console.log(`build-exe-for-python-sdk: [dry-run] rm -rf ${stagedBuild}`) else await rm(stagedBuild, { recursive: true, force: true }) - + if (target.platform !== 'linux') return const source = join(root, 'packages', 'pty', 'pty-local', 'node_modules', 'node-pty', 'build', 'Release', 'pty.node') const destination = join(stagedBuild, 'Release', 'pty.node') if (this.cli.dryRun) { - if (target.platform === 'linux') console.log(`build-exe-for-python-sdk: [dry-run] cp ${source} ${destination}`) + console.log(`build-exe-for-python-sdk: [dry-run] cp ${source} ${destination}`) return } - if (target.platform === 'macos') return - const host = Target.host() - if (target.platform !== host.platform || target.arch !== host.arch || !existsSync(source)) { + if (target.platform !== host.platform || target.arch !== host.arch) { throw new Error( - `build-exe-for-python-sdk: node-pty native addon for ${target.platform}-${target.arch} is missing; ` - + `checked ${source}. Build the Linux runtime on its target architecture.`, + 'build-exe-for-python-sdk: build the Linux runtime on its target architecture; ' + + `target ${target.platform}-${target.arch} does not match host ${host.platform}-${host.arch}.`, ) } await mkdir(dirname(destination), { recursive: true }) await copyFile(source, destination) } - /** - * Resolve the node-pty helper that matches a pkg target. - * @param target - the pkg target whose helper must be shipped. - * @returns a physical executable outside pkg's virtual snapshot. - */ - private resolveSpawnHelper(target: Target): string { - const helper = join(this.staging, 'node_modules', 'node-pty', 'prebuilds', `darwin-${target.arch}`, 'spawn-helper') - if (!existsSync(helper)) { - throw new Error( - `build-exe-for-python-sdk: node-pty spawn-helper for ${target.platform}-${target.arch} is missing; ` - + `checked ${helper}. Build each runtime on its target platform and architecture.`, - ) - } - if (statSync(helper).mode & 0o111) return helper - throw new Error(`build-exe-for-python-sdk: node-pty spawn-helper is not executable: ${helper}`) - } - /** * Print each product path and, outside dry-run mode, its size. * @param products - the product paths returned by {@link pack}. @@ -397,7 +376,7 @@ class SingleExeBuild { } return } - mkdirSync(destDir, { recursive: true }) + await mkdir(destDir, { recursive: true }) for (const path of products) { const destination = join(destDir, basename(path)) await copyFile(path, destination) diff --git a/scripts/build-python-release.py b/scripts/build-python-release.py index 4fe7d62980..ec049cdd4f 100644 --- a/scripts/build-python-release.py +++ b/scripts/build-python-release.py @@ -22,7 +22,10 @@ PLATFORMS = { "linux-arm64": ("manylinux_2_28_aarch64", "dsh-jsonrpc-agent-pkg-linux-arm64"), "macos-arm64": ("macosx_11_0_arm64", "dsh-jsonrpc-agent-pkg-macos-arm64"), } -SPAWN_HELPER_SUFFIX = "-spawn-helper" + + +def runtime_suffixes(executable_name: str) -> tuple[str, ...]: + return ("", "-spawn-helper") if "-macos-" in executable_name else ("",) def main() -> None: @@ -133,24 +136,12 @@ def stage_sdk(destination: Path, version: str) -> None: def stage_runtime(destination: Path, version: str, executable: Path, executable_name: str) -> None: - payload = [(executable, executable_name)] - if "-macos-" in executable_name: - payload.append( - (Path(f"{executable}{SPAWN_HELPER_SUFFIX}"), f"{executable_name}{SPAWN_HELPER_SUFFIX}") - ) - for source, _ in payload: - if not source.is_file(): - raise FileNotFoundError(f"runtime file does not exist: {source}") - if source.stat().st_mode & stat.S_IXUSR == 0: - raise PermissionError(f"runtime file is not executable: {source}") copy_package(ROOT / "python" / "sdk-runtime", destination) rewrite_version(destination / "pyproject.toml", version) runtime_dir = destination / "src" / "deepseek_harness_runtime" / "runtime" runtime_dir.mkdir(parents=True, exist_ok=True) - for source, name in payload: - target = runtime_dir / name - shutil.copyfile(source, target) - target.chmod(source.stat().st_mode & 0o777) + for suffix in runtime_suffixes(executable_name): + shutil.copy2(Path(f"{executable}{suffix}"), runtime_dir / f"{executable_name}{suffix}") def verify_wheel( @@ -174,9 +165,7 @@ def verify_wheel( ] if package == "runtime": assert platform is not None - expected_files = [platform[1]] - if "-macos-" in platform[1]: - expected_files.append(f"{platform[1]}{SPAWN_HELPER_SUFFIX}") + expected_files = [f"{platform[1]}{suffix}" for suffix in runtime_suffixes(platform[1])] found_files = sorted(Path(name).name for name in runtime_files) if found_files != expected_files: raise RuntimeError(f"{wheel} runtime payload must be {expected_files}, found {found_files}")