From 6ea7d7af53138cb5b28a8bf8dff147382e3de4f8 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:10:27 +0800 Subject: [PATCH] ci: parallelize real-api e2e files --- .github/workflows/e2e.yml | 5 ++-- .../2026-07-06-parallel-github-ci-gates.md | 4 ++-- .../testing/2026-06-19-real-api-e2e-ci.md | 4 ++-- examples/coding-agent/tests/compaction.e2e.ts | 2 +- examples/coding-agent/tests/full-loop.e2e.ts | 9 ++++++- examples/coding-agent/tests/todo-write.e2e.ts | 9 ++++++- vitest.e2e.config.ts | 24 +++++++++++++++---- 7 files changed, 44 insertions(+), 13 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ace789915b..934cbb8bda 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -54,8 +54,8 @@ jobs: if: >- github.event_name != 'pull_request' || !(github.event.pull_request.head.repo.fork || github.event.pull_request.user.login == 'dependabot[bot]') - # Serial files (fileParallelism: false), 120s/test, retry 2. 45m bounds a - # wedged run while leaving headroom for retry storms against a slow API. + # Bounded file parallelism (DSH_E2E_MAX_WORKERS), 120s/test, retry 2. 45m + # still bounds retry storms against a slow API while the happy path fans out. timeout-minutes: 45 steps: - uses: actions/checkout@v6 @@ -108,4 +108,5 @@ jobs: env: DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }} DEEPSEEK_BASE_URL: https://api.deepseek.com + DSH_E2E_MAX_WORKERS: 4 run: pnpm run test:e2e diff --git a/docs/rfc/implemented/process/2026-07-06-parallel-github-ci-gates.md b/docs/rfc/implemented/process/2026-07-06-parallel-github-ci-gates.md index 3c3be0782f..5a2621c1f9 100644 --- a/docs/rfc/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/docs/rfc/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -14,14 +14,14 @@ The hard part is the artifact boundary. `publint`, `verify-node-next-types`, and Build output is produced once by a `build` job and uploaded as a short-retention artifact. Artifact consumers run behind that boundary: the `artifact-gate` matrix downloads the built package tree and runs `pnpm run hygiene` plus the built-bin smoke tests. Node-version compatibility stays explicit but narrower: the Node 26 lane runs typecheck and unit tests, while the full quality/documentation/artifact surface runs on the package engine floor. -Both CI workflows cache the pnpm store after enabling Corepack. The real-API e2e workflow keeps its serial test execution because the e2e config deliberately disables file parallelism for API-quota stability; its speedup is dependency-cache reuse, not concurrent model calls. +Both CI workflows cache the pnpm store after enabling Corepack. The real-API e2e workflow also uses the shared `vitest.e2e.config.ts` bounded file pool (`DSH_E2E_MAX_WORKERS=4` in CI), so its speedup comes from dependency-cache reuse plus lower-level test-file fan-out instead of a separate GitHub job split. ## Alternatives considered - **Keep the full serial chain in a Node matrix** - simplest to reason about, but it duplicates repo-wide gates that do not produce Node-version-specific signal and leaves every PR waiting for the sum of all gates. - **Run every gate independently with no build artifact handoff** - maximizes fan-out, but the publication and built-bin checks are defined over built `lib/` outputs and would either fail, skip, or rebuild the same tree in several jobs. - **Build inside every artifact-dependent job** - preserves correctness but shifts the bottleneck from the serial chain to repeated `tsc -b` and bundling work. -- **Parallelize real-API e2e files** - rejected because the e2e suite's Vitest config uses `fileParallelism: false` to stay within shared API-key quota and avoid rate-limit flakes. +- **Use unbounded real-API e2e parallelism** - rejected because the suite includes many live model/tool scenarios; the worker pool needs an explicit `DSH_E2E_MAX_WORKERS` cap so CI and local runs can fan out without hiding quota or resource problems behind flaky rate-limit failures. ## Consequences diff --git a/docs/rfc/implemented/testing/2026-06-19-real-api-e2e-ci.md b/docs/rfc/implemented/testing/2026-06-19-real-api-e2e-ci.md index d23483eb88..b3812d10f7 100644 --- a/docs/rfc/implemented/testing/2026-06-19-real-api-e2e-ci.md +++ b/docs/rfc/implemented/testing/2026-06-19-real-api-e2e-ci.md @@ -20,7 +20,7 @@ ci.yml's value is that it is keyless, forkable, and always-green: any contributo ### Cost is not the constraint; reliability is -The usual reason to ration real-API CI — token cost — does not apply here: we are DeepSeek and internal inference is effectively free. So the design optimizes for *coverage and signal*, not for minimizing calls. The suite runs in full (all six `*.e2e.ts` files), on multiple triggers, on every trusted PR. This is the CI embodiment of the [docs/testing.md](../../../testing.md) with-key policy. +The usual reason to ration real-API CI — token cost — does not apply here: we are DeepSeek and internal inference is effectively free. So the design optimizes for *coverage and signal*, not for minimizing calls. The suite runs in full (all matching `*.e2e.ts` files), on multiple triggers, on every trusted PR. This is the CI embodiment of the [docs/testing.md](../../../testing.md) with-key policy. ### Triggers: trusted events only @@ -54,7 +54,7 @@ The repo secret is named `DEEPSEEK_API_KEY_EXTERNAL`; it is mapped to the `DEEPS ### Scope, runtime shape -Run **only** `test:e2e`. The keyless gates (typecheck/lint/coverage/snapshot/build/hygiene) already run in ci.yml on every push and PR; repeating them here would duplicate signal and slow the real-API job. No build step — e2e tests run unbuilt via tsx + the tsconfig paths map. Single Node 24 (the `engines` floor): these tests exercise *API integration*, not node-version compat, which ci.yml's `[24, 26]` matrix already owns; a second Node version would double real-API calls for no added signal. `timeout-minutes: 45` bounds a wedged run given serial files (`fileParallelism: false`), 120s/test, and `retry: 2`. `cancel-in-progress` is enabled only for `pull_request` runs — a superseded PR run is on a stale commit and worth cancelling, whereas a push/schedule run is already producing the post-merge/nightly signal and is never cancelled. +Run **only** `test:e2e`. The keyless gates (typecheck/lint/coverage/snapshot/build/hygiene) already run in ci.yml on every push and PR; repeating them here would duplicate signal and slow the real-API job. No build step — e2e tests run unbuilt via tsx + the tsconfig paths map. Single Node 24 (the `engines` floor): these tests exercise *API integration*, not node-version compat, which ci.yml's `[24, 26]` matrix already owns; a second Node version would double real-API calls for no added signal. `vitest.e2e.config.ts` runs files through a bounded worker pool (`DSH_E2E_MAX_WORKERS`, default and CI value `4`) so CI and local with-key runs parallelize independent files while retaining a one-line serial escape hatch for quota investigations. `timeout-minutes: 45` bounds a wedged run given 120s/test and `retry: 2`. `cancel-in-progress` is enabled only for `pull_request` runs — a superseded PR run is on a stale commit and worth cancelling, whereas a push/schedule run is already producing the post-merge/nightly signal and is never cancelled. ## Security diff --git a/examples/coding-agent/tests/compaction.e2e.ts b/examples/coding-agent/tests/compaction.e2e.ts index cb7ce43811..f306e20e98 100644 --- a/examples/coding-agent/tests/compaction.e2e.ts +++ b/examples/coding-agent/tests/compaction.e2e.ts @@ -62,7 +62,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('compaction: a long session compa maxTokens: 2048, compactionRetries: 1, }, - persistenceRoot: './.sessions', + persistenceRoot: join(workdir, '.sessions'), }) const agent = ctx.agentLoop.create(AgentId('e2e-compaction'), { model: 'deepseek-v4-flash' }) diff --git a/examples/coding-agent/tests/full-loop.e2e.ts b/examples/coding-agent/tests/full-loop.e2e.ts index 095d2a42a1..8718139ced 100644 --- a/examples/coding-agent/tests/full-loop.e2e.ts +++ b/examples/coding-agent/tests/full-loop.e2e.ts @@ -1,3 +1,6 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import type { Context } from 'cordis' import { AgentId } from '@deepseek-ai/dsh-agent' @@ -9,6 +12,7 @@ import { codingHarness, finalText, SYSTEM_PROMPT, waitForIdle } from './harness. */ let ctx: Context | undefined +let workdir: string | undefined afterEach(async () => { // Always dispose the harness, even on failure/retry/timeout: agent-loop @@ -16,11 +20,14 @@ afterEach(async () => { // process the model left behind. await ctx?.fiber.dispose() ctx = undefined + if (workdir !== undefined) await rm(workdir, { recursive: true, force: true }) + workdir = undefined }) describe.skipIf(!process.env.DEEPSEEK_API_KEY)('full loop: real model + real bash tool', () => { it('runs a bash command on request and reports its output', async () => { - ctx = await codingHarness(process.cwd(), { persona: SYSTEM_PROMPT }) + workdir = await mkdtemp(join(tmpdir(), 'dsh-full-loop-e2e-')) + ctx = await codingHarness(workdir, { persona: SYSTEM_PROMPT }) const agent = ctx.agentLoop.create(AgentId('e2e-loop'), { model: 'deepseek-v4-flash' }) agent.send([{ type: 'text', text: 'Run `echo e2e-ok` with the bash tool and tell me its exact output.' }]) diff --git a/examples/coding-agent/tests/todo-write.e2e.ts b/examples/coding-agent/tests/todo-write.e2e.ts index b100091a0f..698fbd9e9e 100644 --- a/examples/coding-agent/tests/todo-write.e2e.ts +++ b/examples/coding-agent/tests/todo-write.e2e.ts @@ -1,3 +1,6 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import type { Context } from 'cordis' import { AgentId } from '@deepseek-ai/dsh-agent' @@ -10,15 +13,19 @@ import { codingHarness, TODO_SYSTEM_PROMPT, waitForIdle } from './harness.ts' */ let ctx: Context | undefined +let workdir: string | undefined afterEach(async () => { await ctx?.fiber.dispose() ctx = undefined + if (workdir !== undefined) await rm(workdir, { recursive: true, force: true }) + workdir = undefined }) describe.skipIf(!process.env.DEEPSEEK_API_KEY)('todo_write: real model records a plan', () => { it('appends a todo/write event with the model-produced task list', async () => { - ctx = await codingHarness(process.cwd(), { persona: TODO_SYSTEM_PROMPT }) + workdir = await mkdtemp(join(tmpdir(), 'dsh-todo-write-e2e-')) + ctx = await codingHarness(workdir, { persona: TODO_SYSTEM_PROMPT }) const agent = ctx.agentLoop.create(AgentId('e2e-todo'), { model: 'deepseek-v4-flash' }) agent.send([{ type: 'text', text: diff --git a/vitest.e2e.config.ts b/vitest.e2e.config.ts index 63ccbb8a27..7283a6925d 100644 --- a/vitest.e2e.config.ts +++ b/vitest.e2e.config.ts @@ -19,6 +19,21 @@ try { // No .env — fine, the environment may already carry the variables. } +const DEFAULT_E2E_MAX_WORKERS = 4 + +function positiveIntFromEnv(name: string, fallback: number): number { + const raw = process.env[name] + if (raw === undefined || raw === '') return fallback + + const value = Number(raw) + if (!Number.isInteger(value) || value < 1) { + throw new Error(`${name} must be a positive integer, got ${JSON.stringify(raw)}`) + } + return value +} + +const e2eMaxWorkers = positiveIntFromEnv('DSH_E2E_MAX_WORKERS', DEFAULT_E2E_MAX_WORKERS) + export default defineConfig({ // Same resolution note as vitest.config.ts: bare workspace names resolve // through the root tsconfig paths map; the native option cannot do this. @@ -31,9 +46,10 @@ export default defineConfig({ testTimeout: 120_000, hookTimeout: 30_000, retry: 2, - // Run e2e files one at a time: the shared internal API key has a small - // concurrency quota, and parallel files issue enough simultaneous requests - // to trip it (manifesting as flaky rate-limit errors). - fileParallelism: false, + // Run files in a bounded pool: enough lower-level parallelism to keep CI + // and local with-key runs moving, while leaving a resource knob for shared + // API quotas (`DSH_E2E_MAX_WORKERS=1` restores serial execution). + fileParallelism: e2eMaxWorkers > 1, + maxWorkers: e2eMaxWorkers, }, })