diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index affdd3c900..6c7a7489f3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -79,8 +79,14 @@ jobs: - name: Pack release tarballs run: pnpm run release:pack --family dsh --out dist/npm + # The harness packages declare the vendored framework as a peer, and this + # job has no credentials for the private registry, so the verification + # installs that family's pack output too. Only dist/npm is published. + - name: Pack the vendored framework for verification + run: pnpm run release:pack --family vendor --out dist/npm-vendor + - name: Verify packed install - run: pnpm run release:verify-packed-install --family dsh --from dist/npm + run: pnpm run release:verify-packed-install --family dsh --from dist/npm --from dist/npm-vendor - uses: actions/upload-artifact@v4 with: diff --git a/scripts/release/bump.ts b/scripts/release/bump.ts index 83e1b3a2da..9755fd00c4 100644 --- a/scripts/release/bump.ts +++ b/scripts/release/bump.ts @@ -13,11 +13,11 @@ * the tag after the commit merges. CI never writes to the repository. */ -import { spawnSync } from 'node:child_process' import { readFileSync, writeFileSync } from 'node:fs' import { join, matchesGlob } from 'node:path' import { parseArgs } from 'node:util' import { releaseFamily, type ReleaseFamily, type ReleaseMember } from './families.ts' +import { capture } from './process.ts' /** Files npm publishes whether or not `files` lists them. */ const ALWAYS_PUBLISHED = ['package.json', 'README*', 'LICENSE*', 'LICENCE*'] as const @@ -25,21 +25,6 @@ const ALWAYS_PUBLISHED = ['package.json', 'README*', 'LICENSE*', 'LICENCE*'] as /** Release types the dsh family accepts besides an explicit version. */ const RELEASE_TYPES = ['major', 'minor', 'patch'] as const -/** - * Run a command and fail the process on a non-zero exit. - * @param command - executable name. - * @param args - command arguments. - * @returns The captured stdout, trimmed. - */ -function run(command: string, args: readonly string[]): string { - const result = spawnSync(command, [...args], { encoding: 'utf8' }) - if (result.error !== undefined) throw result.error - if (result.status !== 0) { - throw new Error(`${command} ${args.join(' ')} exited with ${String(result.status)}:\n${result.stdout}\n${result.stderr}`) - } - return result.stdout.trim() -} - /** * Split a version into its release numbers, discarding any prerelease segment. * @param version - the current version. @@ -106,7 +91,7 @@ function reachesPayload(member: ReleaseMember, path: string): boolean { */ function lastPublishedTag(family: ReleaseFamily, member: ReleaseMember): string | undefined { const prefix = family.tagFor(member).replace(/-v[^-]*$/, '-v') - const tags = run('git', ['tag', '--list', `${prefix}*`, '--sort=-v:refname']).split('\n').filter(line => line !== '') + const tags = capture('git', ['tag', '--list', `${prefix}*`, '--sort=-v:refname']).split('\n').filter(line => line !== '') return tags[0] } @@ -119,7 +104,7 @@ function lastPublishedTag(family: ReleaseFamily, member: ReleaseMember): string function changedSincePublication(family: ReleaseFamily, member: ReleaseMember): boolean { const tag = lastPublishedTag(family, member) if (tag === undefined) return true - const changed = run('git', ['diff', '--name-only', `${tag}..HEAD`, '--', member.directory]) + const changed = capture('git', ['diff', '--name-only', `${tag}..HEAD`, '--', member.directory]) .split('\n').filter(line => line !== '') return changed.some(path => reachesPayload(member, path)) } @@ -176,7 +161,7 @@ function main(): void { const dryRun = values['dry-run'] if (!dryRun) { for (const { member, version } of planned) writeVersion(root, member, version) - run('pnpm', ['install', '--lockfile-only']) + capture('pnpm', ['install', '--lockfile-only']) } const summary = sharedVersion @@ -188,8 +173,8 @@ function main(): void { console.log('release bump: dry run, nothing written') return } - run('git', ['add', 'pnpm-lock.yaml', ...planned.map(entry => join(entry.member.directory, 'package.json'))]) - run('git', ['commit', '-m', `release(${family.id}): ${summary}`]) + capture('git', ['add', 'pnpm-lock.yaml', ...planned.map(entry => join(entry.member.directory, 'package.json'))]) + capture('git', ['commit', '-m', `release(${family.id}): ${summary}`]) // The dsh family tags once for its shared version; vendor tags each package. const tags = [...new Set(planned.map(entry => family.tagFor({ ...entry.member, version: entry.version })))] console.log('release bump: committed. After this merges to master, tag it:') diff --git a/scripts/release/pack.ts b/scripts/release/pack.ts index 1b128817b9..5c50ebca4b 100644 --- a/scripts/release/pack.ts +++ b/scripts/release/pack.ts @@ -7,41 +7,16 @@ * ([rationale](../../.agents/notes/proposed/process/2026-08-10-npm-release-sequences.md)). */ -import { spawnSync } from 'node:child_process' import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' import { join, resolve } from 'node:path' import { parseArgs } from 'node:util' import { releaseFamily, tarballName, type ReleaseFamily, type ReleaseMember } from './families.ts' +import { run } from './process.ts' +import { PUBLISH_ORDER_FILE, tarballFiles } from './tarball.ts' /** Where pack output lands when `--out` is omitted. */ const DEFAULT_OUTPUT = 'dist/npm' -/** Name of the file the publish step reads to learn the upload order. */ -export const PUBLISH_ORDER_FILE = 'publish-order.txt' - -/** - * Run a command, inheriting stdio, and fail the process on a non-zero exit. - * @param command - executable name. - * @param args - command arguments. - */ -function run(command: string, args: readonly string[]): void { - const result = spawnSync(command, [...args], { stdio: 'inherit' }) - if (result.error !== undefined) throw result.error - if (result.status !== 0) throw new Error(`${command} ${args.join(' ')} exited with ${String(result.status)}`) -} - -/** - * List a tarball's members. - * @param tarball - absolute tarball path. - * @returns Every path inside the archive. - */ -function tarballFiles(tarball: string): string[] { - const result = spawnSync('tar', ['-tzf', tarball], { encoding: 'utf8' }) - if (result.error !== undefined) throw result.error - if (result.status !== 0) throw new Error(`tar -tzf ${tarball} exited with ${String(result.status)}:\n${result.stderr}`) - return result.stdout.split('\n').filter(line => line !== '') -} - /** * Pack one member and check what its tarball carries. * @param family - the release family being packed. diff --git a/scripts/release/process.ts b/scripts/release/process.ts new file mode 100644 index 0000000000..3e8c6943bd --- /dev/null +++ b/scripts/release/process.ts @@ -0,0 +1,65 @@ +/** + * Process helpers shared by the release scripts: the release steps drive `git`, + * `pnpm`, `npm`, and `tar`, and each needs one of three failure behaviours. + */ + +import { spawnSync } from 'node:child_process' + +/** Where and with what environment a release step runs a command. */ +export interface RunOptions { + /** Working directory; defaults to the current one. */ + readonly cwd?: string + /** Child environment; defaults to this process's. */ + readonly env?: NodeJS.ProcessEnv +} + +/** What a command produced, for a caller that decides what a failure means. */ +export interface CommandResult { + /** Exit status, or null when a signal ended the process. */ + readonly status: number | null + /** Captured standard output. */ + readonly stdout: string + /** Captured standard error. */ + readonly stderr: string +} + +/** + * Run a command and capture its output without judging the exit status. + * @param command - executable name. + * @param args - command arguments. + * @param options - working directory and environment. + * @returns The exit status and captured streams. + */ +export function attempt(command: string, args: readonly string[], options: RunOptions = {}): CommandResult { + const result = spawnSync(command, [...args], { cwd: options.cwd, env: options.env, encoding: 'utf8' }) + if (result.error !== undefined) throw result.error + return { status: result.status, stdout: result.stdout, stderr: result.stderr } +} + +/** + * Run a command, capture its standard output, and fail on a non-zero exit. + * @param command - executable name. + * @param args - command arguments. + * @param options - working directory and environment. + * @returns The trimmed standard output. + */ +export function capture(command: string, args: readonly string[], options: RunOptions = {}): string { + const result = attempt(command, args, options) + if (result.status !== 0) { + throw new Error(`${command} ${args.join(' ')} exited with ${String(result.status)}:\n${result.stdout}\n${result.stderr}`) + } + return result.stdout.trim() +} + +/** + * Run a command with inherited streams, so its progress reaches the log, and + * fail on a non-zero exit. + * @param command - executable name. + * @param args - command arguments. + * @param options - working directory and environment. + */ +export function run(command: string, args: readonly string[], options: RunOptions = {}): void { + const result = spawnSync(command, [...args], { cwd: options.cwd, env: options.env, stdio: 'inherit' }) + if (result.error !== undefined) throw result.error + if (result.status !== 0) throw new Error(`${command} ${args.join(' ')} exited with ${String(result.status)}`) +} diff --git a/scripts/release/publish.ts b/scripts/release/publish.ts index 68dae1701a..bd0b2d8552 100644 --- a/scripts/release/publish.ts +++ b/scripts/release/publish.ts @@ -12,13 +12,13 @@ * the same artifact safe. */ -import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' import { readFileSync } from 'node:fs' import { join, resolve } from 'node:path' import { parseArgs } from 'node:util' import { releaseFamily } from './families.ts' -import { PUBLISH_ORDER_FILE } from './pack.ts' +import { attempt, run } from './process.ts' +import { packedIdentity, readPublishOrder } from './tarball.ts' /** npm access level for every package this repository publishes. */ const ACCESS = 'restricted' @@ -28,22 +28,6 @@ type RegistryState = | { readonly kind: 'absent' } | { readonly kind: 'present'; readonly integrity: string } -/** - * Read a packed tarball's own manifest. - * @param tarball - absolute tarball path. - * @returns The packed `package.json` name and version. - */ -function packedIdentity(tarball: string): { name: string; version: string } { - const result = spawnSync('tar', ['-xOzf', tarball, 'package/package.json'], { encoding: 'utf8' }) - if (result.error !== undefined) throw result.error - if (result.status !== 0) throw new Error(`cannot read ${tarball}:\n${result.stderr}`) - const manifest: unknown = JSON.parse(result.stdout) - if (manifest === null || typeof manifest !== 'object') throw new Error(`${tarball} has no manifest`) - const { name, version } = manifest as Record - if (typeof name !== 'string' || typeof version !== 'string') throw new Error(`${tarball} manifest lacks name/version`) - return { name, version } -} - /** * The subresource integrity string npm records for a tarball. * @param tarball - absolute tarball path. @@ -60,8 +44,7 @@ function integrityOf(tarball: string): string { * @returns The registry state for that version. */ function registryState(name: string, version: string): RegistryState { - const result = spawnSync('npm', ['view', `${name}@${version}`, 'dist.integrity', '--json'], { encoding: 'utf8' }) - if (result.error !== undefined) throw result.error + const result = attempt('npm', ['view', `${name}@${version}`, 'dist.integrity', '--json']) if (result.status !== 0) { const output = `${result.stdout}${result.stderr}` if (output.includes('E404') || output.includes('404 Not Found')) return { kind: 'absent' } @@ -74,19 +57,6 @@ function registryState(name: string, version: string): RegistryState { return { kind: 'present', integrity: parsed } } -/** - * Publish one tarball. - * @param tarball - absolute tarball path. - * @param version - the version being published; a prerelease never takes `latest`. - */ -function publish(tarball: string, version: string): void { - const args = ['publish', tarball, '--access', ACCESS] - if (version.includes('-')) args.push('--tag', 'next') - const result = spawnSync('npm', args, { stdio: 'inherit' }) - if (result.error !== undefined) throw result.error - if (result.status !== 0) throw new Error(`npm publish ${tarball} exited with ${String(result.status)}`) -} - /** Publish the family named by `--family` from the directory named by `--from`. */ function main(): void { const { values } = parseArgs({ @@ -99,11 +69,10 @@ function main(): void { const family = releaseFamily(values.family) const directory = resolve(process.cwd(), values.from) - const order = readFileSync(join(directory, PUBLISH_ORDER_FILE), 'utf8').split('\n').filter(line => line !== '') let published = 0 let skipped = 0 - for (const filename of order) { + for (const filename of readPublishOrder(directory)) { const tarball = join(directory, filename) const { name, version } = packedIdentity(tarball) const state = registryState(name, version) @@ -120,7 +89,9 @@ function main(): void { skipped += 1 continue } - publish(tarball, version) + // A prerelease version never takes the latest dist-tag. + const tagArgs = version.includes('-') ? ['--tag', 'next'] : [] + run('npm', ['publish', tarball, '--access', ACCESS, ...tagArgs]) published += 1 } diff --git a/scripts/release/tarball.ts b/scripts/release/tarball.ts new file mode 100644 index 0000000000..568c24e877 --- /dev/null +++ b/scripts/release/tarball.ts @@ -0,0 +1,53 @@ +/** + * Reading packed npm tarballs and the order file that accompanies them. + * + * The release steps after pack treat a directory of tarballs as the unit of + * work, so they read what a tarball declares rather than what the checkout + * currently says. + */ + +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { capture } from './process.ts' + +/** Name of the file recording the order in which a packed family uploads. */ +export const PUBLISH_ORDER_FILE = 'publish-order.txt' + +/** What a packed tarball calls itself. */ +export interface PackedIdentity { + /** Package name from the packed manifest. */ + readonly name: string + /** Package version from the packed manifest. */ + readonly version: string +} + +/** + * List a tarball's members. + * @param tarball - absolute tarball path. + * @returns Every path inside the archive. + */ +export function tarballFiles(tarball: string): string[] { + return capture('tar', ['-tzf', tarball]).split('\n').filter(line => line !== '') +} + +/** + * Read a packed tarball's own manifest. + * @param tarball - absolute tarball path. + * @returns The name and version the tarball declares. + */ +export function packedIdentity(tarball: string): PackedIdentity { + const manifest: unknown = JSON.parse(capture('tar', ['-xOzf', tarball, 'package/package.json'])) + if (manifest === null || typeof manifest !== 'object') throw new Error(`${tarball} has no manifest`) + const { name, version } = manifest as Record + if (typeof name !== 'string' || typeof version !== 'string') throw new Error(`${tarball} manifest lacks name/version`) + return { name, version } +} + +/** + * Read a packed directory's upload order. + * @param directory - absolute path of a pack output directory. + * @returns Tarball filenames in upload order. + */ +export function readPublishOrder(directory: string): string[] { + return readFileSync(join(directory, PUBLISH_ORDER_FILE), 'utf8').split('\n').filter(line => line !== '') +} diff --git a/scripts/release/verify-packed-install.ts b/scripts/release/verify-packed-install.ts index d50e876f06..7b970212ab 100644 --- a/scripts/release/verify-packed-install.ts +++ b/scripts/release/verify-packed-install.ts @@ -1,23 +1,28 @@ /** - * Install a packed release family into a throwaway consumer outside the - * repository and drive its installed executable with plain Node. + * Install packed tarballs into a throwaway consumer outside the repository and + * drive the installed executable with plain Node. * - * Everything the packed tarballs need comes from the tarballs themselves: the - * consumer declares every member as a `file:` dependency, so the only registry - * traffic is for external dependencies. What this proves is that `files` - * selected a complete payload and that the published dependency ranges resolve - * — a workspace link or a stale `lib/` in the checkout cannot stand in for a - * missing file here + * Every tarball the installed tree needs comes from `--from`, so the only + * registry traffic is for external dependencies. That matters beyond hermetic + * verification: the harness packages declare the vendored framework as a peer, + * and those packages live in another release sequence that this credential-free + * job cannot fetch from a private registry — so a dsh verification passes the + * vendored family's pack output too, while publishing only its own * ([rationale](../../.agents/notes/proposed/process/2026-08-10-npm-release-sequences.md)). + * + * What this proves is that `files` selected a complete payload and that the + * published dependency ranges resolve. A workspace link or a stale `lib/` in the + * checkout cannot stand in for a missing file here. */ -import { spawnSync } from 'node:child_process' import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' import { pathToFileURL } from 'node:url' import { parseArgs } from 'node:util' -import { releaseFamily, tarballName, type ReleaseMember } from './families.ts' +import { releaseFamily } from './families.ts' +import { capture } from './process.ts' +import { packedIdentity, readPublishOrder } from './tarball.ts' /** * Environment for the installed artifact: no host Node hooks, no host DeepSeek @@ -38,63 +43,61 @@ function consumerEnvironment(consumerRoot: string): NodeJS.ProcessEnv { } /** - * Run a command in the consumer and fail the process on a non-zero exit. - * @param command - executable name. - * @param args - command arguments. - * @param cwd - working directory. - * @param env - child environment. - * @returns The captured stdout, trimmed. + * Every packed tarball in the given directories, as `file:` dependency entries. + * @param directories - absolute pack output directories. + * @returns Package name to tarball file URL, and the version each carries. */ -function run(command: string, args: readonly string[], cwd: string, env: NodeJS.ProcessEnv): string { - const result = spawnSync(command, [...args], { cwd, env, encoding: 'utf8' }) - if (result.error !== undefined) throw result.error - if (result.status !== 0) { - throw new Error(`${command} ${args.join(' ')} exited with ${String(result.status)}:\n${result.stdout}\n${result.stderr}`) +function packedDependencies(directories: readonly string[]): Map { + const dependencies = new Map() + for (const directory of directories) { + for (const filename of readPublishOrder(directory)) { + const tarball = join(directory, filename) + const { name, version } = packedIdentity(tarball) + dependencies.set(name, { url: pathToFileURL(tarball).href, version }) + } } - return result.stdout.trim() + return dependencies } -/** Install the family named by `--family` from `--from` and drive its entry. */ +/** Install every tarball under `--from` and drive the `--family` entry. */ function main(): void { const { values } = parseArgs({ - options: { family: { type: 'string' }, from: { type: 'string' } }, + options: { family: { type: 'string' }, from: { type: 'string', multiple: true } }, allowPositionals: false, }) - if (values.family === undefined || values.from === undefined) { - throw new Error('usage: verify-packed-install.ts --family --from ') + if (values.family === undefined || values.from === undefined || values.from.length === 0) { + throw new Error('usage: verify-packed-install.ts --family --from [--from ...]') } const family = releaseFamily(values.family) const entry = family.installedEntry - const root = process.cwd() - const packed = resolve(root, values.from) - const members: ReleaseMember[] = family.members(root) - if (entry === undefined) { console.log(`release verify-packed-install: family ${family.id} publishes no executable, nothing to drive`) return } + const root = process.cwd() + const packed = packedDependencies(values.from.map(directory => resolve(root, directory))) + const expected = packed.get(entry.packageName) + if (expected === undefined) throw new Error(`${entry.packageName} is not among the packed tarballs`) + const consumerRoot = mkdtempSync(join(tmpdir(), `dsh-packed-${family.id}-`)) try { - const dependencies = Object.fromEntries(members.map(member => - [member.name, pathToFileURL(join(packed, tarballName(member))).href])) writeFileSync(join(consumerRoot, 'package.json'), `${JSON.stringify({ name: `dsh-packed-install-${family.id}`, version: '0.0.0', private: true, - dependencies, + dependencies: Object.fromEntries([...packed].map(([name, entryPacked]) => [name, entryPacked.url])), }, null, 2)}\n`) const environment = consumerEnvironment(consumerRoot) - console.log(`release verify-packed-install: installing ${String(members.length)} tarball(s) into ${consumerRoot}`) - run('npm', ['install', '--no-audit', '--no-fund', '--package-lock=false'], consumerRoot, environment) + console.log(`release verify-packed-install: installing ${String(packed.size)} tarball(s) into ${consumerRoot}`) + capture('npm', ['install', '--no-audit', '--no-fund', '--package-lock=false'], { cwd: consumerRoot, env: environment }) const bin = join(consumerRoot, 'node_modules', ...entry.packageName.split('/'), entry.binPath) - const version = run(process.execPath, [bin, '--version'], consumerRoot, environment) - const expected = members.find(member => member.name === entry.packageName)?.version - if (version !== expected) { - throw new Error(`installed ${entry.packageName} --version reported ${JSON.stringify(version)}, expected ${String(expected)}`) + const version = capture(process.execPath, [bin, '--version'], { cwd: consumerRoot, env: environment }) + if (version !== expected.version) { + throw new Error(`installed ${entry.packageName} --version reported ${JSON.stringify(version)}, expected ${expected.version}`) } console.log(`release verify-packed-install: installed ${entry.packageName} reports ${version}`) } finally {