From 72c83c771a52186a4971f09b5863078d7e318ae4 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:51:50 +0800 Subject: [PATCH 1/2] chore: parallelize pre-push gates --- docs/development.i18n.yaml | 4 +- docs/development.md | 2 +- docs/development.zh.md | 2 +- lefthook.yml | 58 ++++++++++++++++++-- scripts/publint-all.ts | 106 +++++++++++++++++++++++++++++++++---- 5 files changed, 153 insertions(+), 19 deletions(-) diff --git a/docs/development.i18n.yaml b/docs/development.i18n.yaml index cef11b8352..9de851345c 100644 --- a/docs/development.i18n.yaml +++ b/docs/development.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -development.md: f032764fff29baaca007211db8b69d9a5129078f -development.zh.md: 3a650d03ce7cafd0e34290ae918e5a303c2ad8a9 +development.md: 2c39e0c531638eb0fa3561ec32e6d5c00d87a8be +development.zh.md: 6c9c2a5480db3b71a2d9e25e89c6afd34d24fdb0 diff --git a/docs/development.md b/docs/development.md index f032764fff..2c39e0c531 100644 --- a/docs/development.md +++ b/docs/development.md @@ -59,7 +59,7 @@ DEEPSEEK_BASE_URL=https://... # optional lefthook is configured in `lefthook.yml` as an early local checkpoint before review: - `pre-commit` runs staged-file ESLint fixes, `pnpm run typecheck`, and the vendor manifest guard. -- `pre-push` runs `pnpm run test`, `pnpm run test:snapshot`, `pnpm run hygiene`, `pnpm run doc-sync`, and `pnpm run verify-module-graph`. +- `pre-push` runs unit tests, snapshot tests, module-graph freshness, and the member gates of `pnpm run hygiene` and `pnpm run doc-sync` as parallel lefthook jobs. The vendor manifest guard checks that changes under `vendor/*/src` are staged with the matching `vendor/README.md` manifest update. See `vendor/README.md` before editing vendored code. diff --git a/docs/development.zh.md b/docs/development.zh.md index 3a650d03ce..6c9c2a5480 100644 --- a/docs/development.zh.md +++ b/docs/development.zh.md @@ -59,7 +59,7 @@ DEEPSEEK_BASE_URL=https://... # optional lefthook 在 `lefthook.yml` 中配置，作为评审前的本地早期检查点： - `pre-commit` 运行对暂存文件的 ESLint 修复、`pnpm run typecheck` 和 vendor manifest 守卫。 -- `pre-push` 运行 `pnpm run test`、`pnpm run test:snapshot`、`pnpm run hygiene`、`pnpm run doc-sync` 和 `pnpm run verify-module-graph`。 +- `pre-push` 将单元测试、快照测试、module graph 新鲜度，以及 `pnpm run hygiene` 和 `pnpm run doc-sync` 的成员门禁拆成并行 lefthook job 运行。 vendor manifest 守卫检查 `vendor/*/src` 下的改动是否连同对应的 `vendor/README.md` manifest 更新一起暂存。编辑 vendor 代码前先看 `vendor/README.md`。 diff --git a/lefthook.yml b/lefthook.yml index 2a255424fa..3cf1f5a034 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -28,11 +28,61 @@ pre-push: - name: snapshot run: pnpm run test:snapshot - - name: hygiene - run: pnpm run hygiene + # Flatten `pnpm run hygiene` and `pnpm run doc-sync` so lefthook can run + # their independent member gates concurrently before a push. + - name: knip + run: pnpm run knip - - name: doc-sync - run: pnpm run doc-sync + - name: publint + run: pnpm run publint + + - name: constraints + run: pnpm run constraints + + - name: node-next types + run: pnpm run verify-node-next-types + + - name: doc typecheck + run: pnpm run doc-typecheck + + - name: cordis catalog + run: pnpm run verify-cordis-catalog + + - name: tool catalog + run: pnpm run verify-tool-catalog + + - name: persistence catalog + run: pnpm run verify-persistence-catalog + + - name: doc graphs + run: pnpm run verify-doc-graphs + + - name: markdown wrap + run: pnpm run verify-md-wrap + + - name: markdown links + run: pnpm run verify-md-links + + - name: doc refs + run: pnpm run verify-doc-refs + + - name: package paths + run: pnpm run verify-package-paths + + - name: mermaid + run: pnpm run verify-mermaid + + - name: rfc classification + run: pnpm run verify-rfc-classification + + - name: type equivalence + run: pnpm run verify-type-equiv + + - name: translation pairing + run: pnpm run verify-translation-pairing + + - name: doc budgets + run: pnpm run verify-doc-budgets - name: module-graph freshness run: pnpm run verify-module-graph diff --git a/scripts/publint-all.ts b/scripts/publint-all.ts index 0e06372c3f..126943c8dd 100644 --- a/scripts/publint-all.ts +++ b/scripts/publint-all.ts @@ -1,6 +1,11 @@ -import { execFileSync } from 'node:child_process' +import { execFile } from 'node:child_process' import { existsSync, readdirSync } from 'node:fs' +import { availableParallelism } from 'node:os' import { resolve } from 'node:path' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) +const CONCURRENCY_ENV = 'DSH_PUBLINT_CONCURRENCY' // publint every harness package. Packages live at packages// // (the group dirs — core/llm/bash/… — are pure containers); vendor/ is private @@ -9,15 +14,94 @@ import { resolve } from 'node:path' const root = resolve(import.meta.dirname, '..') const packagesRoot = resolve(root, 'packages') -const packages = readdirSync(packagesRoot, { withFileTypes: true }) - .filter(group => group.isDirectory()) - .flatMap(group => - readdirSync(resolve(packagesRoot, group.name), { withFileTypes: true }) - .filter(pkg => pkg.isDirectory()) - .filter(pkg => existsSync(resolve(packagesRoot, group.name, pkg.name, 'package.json'))) - .map(pkg => `packages/${group.name}/${pkg.name}`), - ) +type PublintResult = + | { path: string; status: 'passed'; stdout: string; stderr: string } + | { path: string; status: 'failed'; stdout: string; stderr: string; message: string } -for (const path of packages) { - execFileSync('node_modules/.bin/publint', [path], { cwd: root, stdio: 'inherit' }) +function workspacePackages(): string[] { + return readdirSync(packagesRoot, { withFileTypes: true }) + .filter(group => group.isDirectory()) + .flatMap(group => + readdirSync(resolve(packagesRoot, group.name), { withFileTypes: true }) + .filter(pkg => pkg.isDirectory()) + .filter(pkg => existsSync(resolve(packagesRoot, group.name, pkg.name, 'package.json'))) + .map(pkg => `packages/${group.name}/${pkg.name}`), + ) } + +function publintConcurrency(total: number): number { + if (total === 0) return 0 + + const raw = process.env[CONCURRENCY_ENV] + if (raw !== undefined) { + const parsed = Number.parseInt(raw, 10) + if (!Number.isSafeInteger(parsed) || parsed < 1) { + throw new Error(`publint-all: ${CONCURRENCY_ENV} must be a positive integer, got ${JSON.stringify(raw)}.`) + } + return Math.min(total, parsed) + } + + return Math.min(total, availableParallelism()) +} + +function outputText(value: unknown): string { + if (typeof value === 'string') return value + if (Buffer.isBuffer(value)) return value.toString() + return '' +} + +async function runPublint(path: string): Promise { + try { + const { stdout, stderr } = await execFileAsync('node_modules/.bin/publint', [path], { + cwd: root, + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024, + }) + return { path, status: 'passed', stdout, stderr } + } catch (error: unknown) { + const failed = error as { stdout?: unknown; stderr?: unknown; message?: string } + return { + path, + status: 'failed', + stdout: outputText(failed.stdout), + stderr: outputText(failed.stderr), + message: failed.message ?? 'publint failed', + } + } +} + +async function runAll(paths: string[], concurrency: number): Promise { + let next = 0 + const results: Array = [] + await Promise.all(Array.from({ length: concurrency }, async () => { + for (;;) { + const index = next + next += 1 + const path = paths[index] + if (path === undefined) return + results[index] = await runPublint(path) + } + })) + + return paths.map((path, index) => { + const result = results[index] + if (result === undefined) throw new Error(`publint-all: missing result for ${path}.`) + return result + }) +} + +function printResult(result: PublintResult): void { + console.log(`Running publint for ${result.path}...`) + process.stdout.write(result.stdout) + process.stderr.write(result.stderr) + if (result.status === 'failed') console.error(result.message) +} + +const packages = workspacePackages() +const concurrency = publintConcurrency(packages.length) +console.log(`publint-all: linting ${packages.length} package(s) with ${concurrency} worker(s).`) + +const results = await runAll(packages, concurrency) +for (const result of results) printResult(result) + +if (results.some(result => result.status === 'failed')) process.exit(1) From c0af59f80455a246ac830afe74fab5ff3e64eef3 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:04:06 +0800 Subject: [PATCH 2/2] docs(rfc): record parallel pre-push gates --- docs/rfc/INDEX.md | 1 + .../2026-07-06-parallel-pre-push-gates.md | 32 +++++++++++++++++++ lefthook.yml | 3 ++ 3 files changed, 36 insertions(+) create mode 100644 docs/rfc/implemented/process/2026-07-06-parallel-pre-push-gates.md diff --git a/docs/rfc/INDEX.md b/docs/rfc/INDEX.md index 0c5a11a4f9..44e1ea0c11 100644 --- a/docs/rfc/INDEX.md +++ b/docs/rfc/INDEX.md @@ -141,6 +141,7 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand; | [Generate the RFC index tables](implemented/process/2026-07-04-generate-rfc-index-tables.md) | 2026-07-04 | | [Generated persistence log event catalog](implemented/process/2026-07-04-persistence-log-catalog.md) | 2026-07-04 | | [One gated in-file format for RFCs](implemented/process/2026-07-05-uniform-rfc-format.md) | 2026-07-05 | +| [Parallel pre-push gates](implemented/process/2026-07-06-parallel-pre-push-gates.md) | 2026-07-06 | ### Testing diff --git a/docs/rfc/implemented/process/2026-07-06-parallel-pre-push-gates.md b/docs/rfc/implemented/process/2026-07-06-parallel-pre-push-gates.md new file mode 100644 index 0000000000..f17cf7e558 --- /dev/null +++ b/docs/rfc/implemented/process/2026-07-06-parallel-pre-push-gates.md @@ -0,0 +1,32 @@ +# RFC: Parallel pre-push gates + +Status: implemented + +## Problem + +The pre-push hook is the last local checkpoint before a branch leaves the machine, so its wall clock directly shapes whether contributors keep it enabled and trust its signal. Lefthook already runs top-level jobs in parallel, but aggregate jobs such as `pnpm run hygiene` and `pnpm run doc-sync` hide long sequential chains inside one job. The hook can therefore be configured as parallel while still waiting on serial subcommands whose members are independent. + +`publint` has the same shape one level lower. Each package is linted independently against its own manifest and built output, but the runner loops through every package in order. On this repo that makes one package-publication gate consume time proportional to the number of packages even though the checks do not share mutable state. + +## Decision + +[lefthook.yml](../../../../lefthook.yml) expands the pre-push hook into leaf jobs for the unit suite, snapshot suite, `hygiene` members, `doc-sync` members, and module-graph freshness. The leaf list keeps the same gate vocabulary as the package scripts, including RFC classification and RFC format, but lets lefthook schedule the independent checks concurrently and report each failure by its own job name. + +[scripts/publint-all.ts](../../../../scripts/publint-all.ts) discovers the package list from `packages//` and runs `publint` with a worker pool sized from `availableParallelism()`. `DSH_PUBLINT_CONCURRENCY` can cap or raise the worker count for local machines and CI runners with different resource profiles. Results are buffered per package and printed in deterministic package order, so parallel execution does not scramble each package's log block. + +The aggregate package scripts remain the source of truth for CI and ad hoc local runs. The hook is a parallel execution plan over their member gates, not a replacement vocabulary. + +## Alternatives considered + +- **Keep aggregate `hygiene` and `doc-sync` jobs in the hook** - simpler config, but it leaves most of the pre-push wall clock inside serial command chains that lefthook cannot see or schedule. +- **Background subcommands inside shell scripts** - can parallelize work, but it loses lefthook's job names, per-job timing, and failure grouping, and makes signal handling harder to reason about. +- **Declare one publint lefthook job per package** - exposes maximum parallelism, but it turns the hook into a hand-maintained package inventory that drifts exactly when new packages are added. +- **Run publint with unbounded concurrency** - minimizes elapsed time on small machines only by gambling with process count, memory pressure, package tarball creation, and readable logs. + +## Consequences + +The hook's critical path becomes the slowest real gate instead of the sum of hidden gate chains. Lefthook reports per-job timing for every leaf gate, so a slow local checkpoint points at the gate that actually dominates the run. + +The hook file is longer and must track the member list of `hygiene` and `doc-sync`. That duplication is acceptable because the purpose of the hook is scheduling, while `package.json` remains the command vocabulary that CI and humans call directly. + +`publint-all.ts` becomes asynchronous code and buffers command output instead of inheriting stdio live. The payoff is package-level parallelism with stable output order and one environment variable for resource tuning. diff --git a/lefthook.yml b/lefthook.yml index 3cf1f5a034..63d775ec9a 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -75,6 +75,9 @@ pre-push: - name: rfc classification run: pnpm run verify-rfc-classification + - name: rfc format + run: pnpm run verify-rfc-format + - name: type equivalence run: pnpm run verify-type-equiv