diff --git a/python/sdk-runtime/hatch_build.py b/python/sdk-runtime/hatch_build.py index cf56dd4138..19ec962257 100644 --- a/python/sdk-runtime/hatch_build.py +++ b/python/sdk-runtime/hatch_build.py @@ -39,31 +39,24 @@ class RuntimeBuildHook(BuildHookInterface): ) platform_tag = os.environ.get("DSH_RUNTIME_PLATFORM_TAG") or _host_platform_tag() - matches = [(key, value) for key, value in _PLATFORMS.items() if value[0] == platform_tag] + matches = [value for value in _PLATFORMS.values() if value[0] == platform_tag] if len(matches) != 1: supported = ", ".join(value[0] for value in _PLATFORMS.values()) raise RuntimeError( f"unsupported DSH_RUNTIME_PLATFORM_TAG {platform_tag!r}; expected one of {supported}" ) - expected_target, (_, expected_executable) = matches[0] + expected_executable = matches[0][1] runtime_dir = Path(self.root) / "src" / "deepseek_harness_runtime" / "runtime" runtime_files = sorted(runtime_dir.glob("dsh-jsonrpc-agent-pkg-*") if runtime_dir.is_dir() else []) - executables = [path for path in runtime_files if not path.name.endswith(_SPAWN_HELPER_SUFFIX)] - helpers = [path for path in runtime_files if path.name.endswith(_SPAWN_HELPER_SUFFIX)] - if [path.name for path in executables] != [expected_executable]: - found = ", ".join(path.name for path in executables) or "none" + expected_files = [expected_executable] + if "-macos-" in expected_executable: + expected_files.append(f"{expected_executable}{_SPAWN_HELPER_SUFFIX}") + found_files = [path.name for path in runtime_files] + if found_files != expected_files: raise RuntimeError( - f"runtime wheel {platform_tag} must contain only {expected_executable}; found {found}" + f"runtime wheel {platform_tag} payload must be {expected_files}; found {found_files}" ) - expected_helper = f"{expected_executable}{_SPAWN_HELPER_SUFFIX}" - expected_helpers = [expected_helper] if expected_target.startswith("macos-") else [] - if [path.name for path in helpers] != expected_helpers: - expected = ", ".join(expected_helpers) or "none" - found = ", ".join(path.name for path in helpers) or "none" - raise RuntimeError( - f"runtime wheel {platform_tag} helper payload mismatch: expected {expected}; found {found}" - ) - for executable in [executables[0], *helpers]: + for executable in runtime_files: if executable.stat().st_mode & stat.S_IXUSR == 0: raise RuntimeError(f"runtime executable is not executable: {executable}") build_data["pure_python"] = False diff --git a/python/sdk/tests/test_release_version.py b/python/sdk/tests/test_release_version.py index b8fa5484c2..68a9aac993 100644 --- a/python/sdk/tests/test_release_version.py +++ b/python/sdk/tests/test_release_version.py @@ -68,7 +68,7 @@ def test_stage_runtime_rejects_missing_spawn_helper(tmp_path: Path) -> None: executable.write_bytes(b"runtime") executable.chmod(0o755) - with pytest.raises(FileNotFoundError, match="spawn helper"): + with pytest.raises(FileNotFoundError, match="spawn-helper"): build_python_release.stage_runtime( tmp_path / "staging", "1.2.3", @@ -77,32 +77,8 @@ def test_stage_runtime_rejects_missing_spawn_helper(tmp_path: Path) -> None: ) -def test_stage_runtime_rejects_unsupported_executable_name(tmp_path: Path) -> None: - executable = tmp_path / "custom-runtime" - executable.write_bytes(b"runtime") - executable.chmod(0o755) - - with pytest.raises( - ValueError, - match=( - "unsupported runtime executable 'custom-runtime'; expected one of: " - "dsh-jsonrpc-agent-pkg-linux-arm64, dsh-jsonrpc-agent-pkg-linux-x64, " - "dsh-jsonrpc-agent-pkg-macos-arm64" - ), - ): - build_python_release.stage_runtime( - tmp_path / "staging", - "1.2.3", - executable, - executable.name, - ) - - -@pytest.mark.parametrize("target", ["linux-x64", "linux-arm64"]) -def test_stage_runtime_copies_linux_executable_without_spawn_helper( - tmp_path: Path, target: str -) -> None: - executable = tmp_path / f"dsh-jsonrpc-agent-pkg-{target}" +def test_stage_runtime_copies_linux_executable_without_spawn_helper(tmp_path: Path) -> None: + executable = tmp_path / "dsh-jsonrpc-agent-pkg-linux-x64" executable.write_bytes(b"runtime") executable.chmod(0o755) destination = tmp_path / "staging" diff --git a/scripts/build-exe-for-python-sdk.ts b/scripts/build-exe-for-python-sdk.ts index e9ae59edbc..bbfa2402d0 100644 --- a/scripts/build-exe-for-python-sdk.ts +++ b/scripts/build-exe-for-python-sdk.ts @@ -53,15 +53,6 @@ const ARCHES = ['x64', 'arm64'] as const type Platform = (typeof PLATFORMS)[number] type Arch = (typeof ARCHES)[number] -interface RuntimeProduct { - executable: string - spawnHelper?: string -} - -function runtimeProductFiles(product: RuntimeProduct): string[] { - return [product.executable, ...(product.spawnHelper === undefined ? [] : [product.spawnHelper])] -} - function isPlatform(value: string): value is Platform { return (PLATFORMS as readonly string[]).includes(value) } @@ -299,9 +290,9 @@ class SingleExeBuild { /** * Package one target; SEA mode accepts one target per invocation. * @param target - the pkg target triple to build. - * @returns the canonical product path `/dsh-jsonrpc-agent-pkg--`. + * @returns the executable path and, on macOS, its helper path. */ - async pack(target: Target): Promise { + async pack(target: Target): Promise { const product = join(this.outDir, `${OUTPUT_BASENAME}-${target.platform}-${target.arch}`) await this.prepareNativePty(target) if (!this.cli.dryRun) mkdirSync(this.outDir, { recursive: true }) @@ -318,7 +309,7 @@ class SingleExeBuild { if (!this.cli.dryRun && !existsSync(product)) { throw new Error(`build-exe-for-python-sdk: product ${product} is missing after the pkg run; inspect ${this.outDir}.`) } - if (target.platform !== 'macos') return { executable: product } + if (target.platform !== 'macos') return [product] const spawnHelper = `${product}${SPAWN_HELPER_SUFFIX}` if (this.cli.dryRun) { console.log(`build-exe-for-python-sdk: [dry-run] copy target node-pty spawn-helper to ${spawnHelper}`) @@ -327,7 +318,7 @@ class SingleExeBuild { await copyFile(source, spawnHelper) await chmod(spawnHelper, statSync(source).mode & 0o777) } - return { executable: product, spawnHelper } + return [product, spawnHelper] } /** @@ -341,21 +332,19 @@ class SingleExeBuild { if (this.cli.dryRun) console.log(`build-exe-for-python-sdk: [dry-run] rm -rf ${stagedBuild}`) else await rm(stagedBuild, { recursive: true, force: true }) - const nativePlatform = target.platform === 'macos' ? 'darwin' : 'linux' - const prebuilt = join(stagedRoot, 'prebuilds', `${nativePlatform}-${target.arch}`, 'pty.node') const source = join(root, 'packages', 'pty', 'pty-local', 'node_modules', 'node-pty', 'build', 'Release', 'pty.node') const destination = join(stagedBuild, 'Release', 'pty.node') if (this.cli.dryRun) { if (target.platform === 'linux') console.log(`build-exe-for-python-sdk: [dry-run] cp ${source} ${destination}`) return } - if (existsSync(prebuilt)) return + if (target.platform === 'macos') return const host = Target.host() if (target.platform !== host.platform || target.arch !== host.arch || !existsSync(source)) { throw new Error( `build-exe-for-python-sdk: node-pty native addon for ${target.platform}-${target.arch} is missing; ` - + `checked ${prebuilt}, ${source}. Build the Linux runtime on its target architecture.`, + + `checked ${source}. Build the Linux runtime on its target architecture.`, ) } await mkdir(dirname(destination), { recursive: true }) @@ -368,19 +357,11 @@ class SingleExeBuild { * @returns a physical executable outside pkg's virtual snapshot. */ private resolveSpawnHelper(target: Target): string { - const nodePtyRoot = join(this.staging, 'node_modules', 'node-pty') - const candidates = [ - join(nodePtyRoot, 'prebuilds', `darwin-${target.arch}`, 'spawn-helper'), - ] - const host = Target.host() - if (target.platform === host.platform && target.arch === host.arch) { - candidates.push(join(root, 'packages', 'pty', 'pty-local', 'node_modules', 'node-pty', 'build', 'Release', 'spawn-helper')) - } - const helper = candidates.find(candidate => existsSync(candidate)) - if (helper === undefined) { + const helper = join(this.staging, 'node_modules', 'node-pty', 'prebuilds', `darwin-${target.arch}`, 'spawn-helper') + if (!existsSync(helper)) { throw new Error( `build-exe-for-python-sdk: node-pty spawn-helper for ${target.platform}-${target.arch} is missing; ` - + `checked ${candidates.join(', ')}. Build each runtime on its target platform and architecture.`, + + `checked ${helper}. Build each runtime on its target platform and architecture.`, ) } if (statSync(helper).mode & 0o111) return helper @@ -391,43 +372,37 @@ class SingleExeBuild { * Print each product path and, outside dry-run mode, its size. * @param products - the product paths returned by {@link pack}. */ - printProducts(products: RuntimeProduct[]): void { + printProducts(products: string[]): void { console.log(this.cli.dryRun ? 'build-exe-for-python-sdk: [dry-run] would produce:' : 'build-exe-for-python-sdk: products:') - for (const product of products) { + for (const path of products) { if (this.cli.dryRun) { - for (const path of runtimeProductFiles(product)) console.log(` ${path}`) + console.log(` ${path}`) continue } - for (const path of runtimeProductFiles(product)) { - const megabytes = statSync(path).size / (1024 * 1024) - console.log(` ${path} (${megabytes.toFixed(1)} MB)`) - } + const megabytes = statSync(path).size / (1024 * 1024) + console.log(` ${path} (${megabytes.toFixed(1)} MB)`) } } /** - * Copy each executable into the Python runtime package. The deployed node + * Copy each product into the Python runtime package. The deployed node * carrier is already in place, and `dist-exe/` retains upload copies. * @param products - the product paths returned by {@link pack}. */ - async syncToPythonRuntime(products: RuntimeProduct[]): Promise { + async syncToPythonRuntime(products: string[]): Promise { const destDir = resolve(root, PYTHON_RUNTIME_DIR) if (this.cli.dryRun) { - for (const product of products) { - for (const path of runtimeProductFiles(product)) { - console.log(`build-exe-for-python-sdk: [dry-run] cp ${path} ${join(destDir, basename(path))}`) - } + for (const path of products) { + console.log(`build-exe-for-python-sdk: [dry-run] cp ${path} ${join(destDir, basename(path))}`) } return } mkdirSync(destDir, { recursive: true }) - for (const product of products) { - for (const path of runtimeProductFiles(product)) { - const destination = join(destDir, basename(path)) - await copyFile(path, destination) - await chmod(destination, statSync(path).mode & 0o777) - console.log(`build-exe-for-python-sdk: synced ${destination}`) - } + for (const path of products) { + const destination = join(destDir, basename(path)) + await copyFile(path, destination) + await chmod(destination, statSync(path).mode & 0o777) + console.log(`build-exe-for-python-sdk: synced ${destination}`) } } @@ -476,8 +451,8 @@ async function main(): Promise { await pipeline.build() await pipeline.deployStaging() await pipeline.injectPkgConfig() - const products: RuntimeProduct[] = [] - for (const target of cli.targets) products.push(await pipeline.pack(target)) + const products: string[] = [] + for (const target of cli.targets) products.push(...await pipeline.pack(target)) pipeline.printProducts(products) await pipeline.syncToPythonRuntime(products) } diff --git a/scripts/build-python-release.py b/scripts/build-python-release.py index be5125cbcd..4fe7d62980 100644 --- a/scripts/build-python-release.py +++ b/scripts/build-python-release.py @@ -23,17 +23,6 @@ PLATFORMS = { "macos-arm64": ("macosx_11_0_arm64", "dsh-jsonrpc-agent-pkg-macos-arm64"), } SPAWN_HELPER_SUFFIX = "-spawn-helper" -EXECUTABLE_TARGETS = {value[1]: key for key, value in PLATFORMS.items()} - - -def executable_target(executable_name: str) -> str: - try: - return EXECUTABLE_TARGETS[executable_name] - except KeyError as error: - supported = ", ".join(sorted(EXECUTABLE_TARGETS)) - raise ValueError( - f"unsupported runtime executable {executable_name!r}; expected one of: {supported}" - ) from error def main() -> None: @@ -144,28 +133,24 @@ def stage_sdk(destination: Path, version: str) -> None: def stage_runtime(destination: Path, version: str, executable: Path, executable_name: str) -> None: - if not executable.is_file(): - raise FileNotFoundError(f"runtime executable does not exist: {executable}") - if executable.stat().st_mode & stat.S_IXUSR == 0: - raise PermissionError(f"runtime executable is not executable: {executable}") - expected_target = executable_target(executable_name) - spawn_helper = Path(f"{executable}{SPAWN_HELPER_SUFFIX}") - if expected_target.startswith("macos-"): - if not spawn_helper.is_file(): - raise FileNotFoundError(f"runtime spawn helper does not exist: {spawn_helper}") - if spawn_helper.stat().st_mode & stat.S_IXUSR == 0: - raise PermissionError(f"runtime spawn helper is not executable: {spawn_helper}") + payload = [(executable, executable_name)] + if "-macos-" in executable_name: + payload.append( + (Path(f"{executable}{SPAWN_HELPER_SUFFIX}"), f"{executable_name}{SPAWN_HELPER_SUFFIX}") + ) + for source, _ in payload: + if not source.is_file(): + raise FileNotFoundError(f"runtime file does not exist: {source}") + if source.stat().st_mode & stat.S_IXUSR == 0: + raise PermissionError(f"runtime file is not executable: {source}") copy_package(ROOT / "python" / "sdk-runtime", destination) rewrite_version(destination / "pyproject.toml", version) runtime_dir = destination / "src" / "deepseek_harness_runtime" / "runtime" runtime_dir.mkdir(parents=True, exist_ok=True) - destination_executable = runtime_dir / executable_name - shutil.copyfile(executable, destination_executable) - destination_executable.chmod(executable.stat().st_mode & 0o777) - if expected_target.startswith("macos-"): - destination_helper = runtime_dir / f"{executable_name}{SPAWN_HELPER_SUFFIX}" - shutil.copyfile(spawn_helper, destination_helper) - destination_helper.chmod(spawn_helper.stat().st_mode & 0o777) + for source, name in payload: + target = runtime_dir / name + shutil.copyfile(source, target) + target.chmod(source.stat().st_mode & 0o777) def verify_wheel( @@ -187,26 +172,18 @@ def verify_wheel( runtime_files = [ name for name in archive.namelist() if "/runtime/dsh-jsonrpc-agent-pkg-" in name ] - helpers = [name for name in runtime_files if name.endswith(SPAWN_HELPER_SUFFIX)] - executables = [name for name in runtime_files if not name.endswith(SPAWN_HELPER_SUFFIX)] if package == "runtime": assert platform is not None - if len(executables) != 1 or not executables[0].endswith(f"/runtime/{platform[1]}"): - raise RuntimeError(f"{wheel} must contain exactly {platform[1]}, found {executables}") - expected_target = executable_target(platform[1]) - expected_helper = f"{platform[1]}{SPAWN_HELPER_SUFFIX}" - expected_helpers = [expected_helper] if expected_target.startswith("macos-") else [] - found_helpers = [Path(helper).name for helper in helpers] - if found_helpers != expected_helpers: - expected = ", ".join(expected_helpers) or "none" - found = ", ".join(found_helpers) or "none" - raise RuntimeError( - f"{wheel} runtime helper payload mismatch: expected {expected}; found {found}" - ) - for executable in [executables[0], *helpers]: - mode = archive.getinfo(executable).external_attr >> 16 + expected_files = [platform[1]] + if "-macos-" in platform[1]: + expected_files.append(f"{platform[1]}{SPAWN_HELPER_SUFFIX}") + found_files = sorted(Path(name).name for name in runtime_files) + if found_files != expected_files: + raise RuntimeError(f"{wheel} runtime payload must be {expected_files}, found {found_files}") + for runtime_file in runtime_files: + mode = archive.getinfo(runtime_file).external_attr >> 16 if mode & stat.S_IXUSR == 0: - raise RuntimeError(f"{wheel} runtime executable lost its executable bit: {executable}") + raise RuntimeError(f"{wheel} runtime executable lost its executable bit: {runtime_file}") elif runtime_files: raise RuntimeError(f"SDK wheel unexpectedly contains runtime executables: {runtime_files}") if package == "sdk":