ci: parallelize real-api e2e files

2026-07-06 01:10:27 +08:00
parent 08e09217bf
commit 6ea7d7af53
7 changed files with 44 additions and 13 deletions
@@ -54,8 +54,8 @@ jobs:
    if: >-
      github.event_name != 'pull_request'
      || !(github.event.pull_request.head.repo.fork || github.event.pull_request.user.login == 'dependabot[bot]')
-    # Serial files (fileParallelism: false), 120s/test, retry 2. 45m bounds a
-    # wedged run while leaving headroom for retry storms against a slow API.
+    # Bounded file parallelism (DSH_E2E_MAX_WORKERS), 120s/test, retry 2. 45m
+    # still bounds retry storms against a slow API while the happy path fans out.
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@v6
@@ -108,4 +108,5 @@ jobs:
        env:
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY_EXTERNAL }}
          DEEPSEEK_BASE_URL: https://api.deepseek.com
+          DSH_E2E_MAX_WORKERS: 4
        run: pnpm run test:e2e
@@ -14,14 +14,14 @@ The hard part is the artifact boundary. `publint`, `verify-node-next-types`, and

 Build output is produced once by a `build` job and uploaded as a short-retention artifact. Artifact consumers run behind that boundary: the `artifact-gate` matrix downloads the built package tree and runs `pnpm run hygiene` plus the built-bin smoke tests. Node-version compatibility stays explicit but narrower: the Node 26 lane runs typecheck and unit tests, while the full quality/documentation/artifact surface runs on the package engine floor.

-Both CI workflows cache the pnpm store after enabling Corepack. The real-API e2e workflow keeps its serial test execution because the e2e config deliberately disables file parallelism for API-quota stability; its speedup is dependency-cache reuse, not concurrent model calls.
+Both CI workflows cache the pnpm store after enabling Corepack. The real-API e2e workflow also uses the shared `vitest.e2e.config.ts` bounded file pool (`DSH_E2E_MAX_WORKERS=4` in CI), so its speedup comes from dependency-cache reuse plus lower-level test-file fan-out instead of a separate GitHub job split.

 ## Alternatives considered

 - **Keep the full serial chain in a Node matrix** - simplest to reason about, but it duplicates repo-wide gates that do not produce Node-version-specific signal and leaves every PR waiting for the sum of all gates.
 - **Run every gate independently with no build artifact handoff** - maximizes fan-out, but the publication and built-bin checks are defined over built `lib/` outputs and would either fail, skip, or rebuild the same tree in several jobs.
 - **Build inside every artifact-dependent job** - preserves correctness but shifts the bottleneck from the serial chain to repeated `tsc -b` and bundling work.
- **Parallelize real-API e2e files** - rejected because the e2e suite's Vitest config uses `fileParallelism: false` to stay within shared API-key quota and avoid rate-limit flakes.
+- **Use unbounded real-API e2e parallelism** - rejected because the suite includes many live model/tool scenarios; the worker pool needs an explicit `DSH_E2E_MAX_WORKERS` cap so CI and local runs can fan out without hiding quota or resource problems behind flaky rate-limit failures.

 ## Consequences

@@ -20,7 +20,7 @@ ci.yml's value is that it is keyless, forkable, and always-green: any contributo

 ### Cost is not the constraint; reliability is

-The usual reason to ration real-API CI — token cost — does not apply here: we are DeepSeek and internal inference is effectively free. So the design optimizes for *coverage and signal*, not for minimizing calls. The suite runs in full (all six `*.e2e.ts` files), on multiple triggers, on every trusted PR. This is the CI embodiment of the [docs/testing.md](../../../testing.md) with-key policy.
+The usual reason to ration real-API CI — token cost — does not apply here: we are DeepSeek and internal inference is effectively free. So the design optimizes for *coverage and signal*, not for minimizing calls. The suite runs in full (all matching `*.e2e.ts` files), on multiple triggers, on every trusted PR. This is the CI embodiment of the [docs/testing.md](../../../testing.md) with-key policy.

 ### Triggers: trusted events only

@@ -54,7 +54,7 @@ The repo secret is named `DEEPSEEK_API_KEY_EXTERNAL`; it is mapped to the `DEEPS

 ### Scope, runtime shape

-Run **only** `test:e2e`. The keyless gates (typecheck/lint/coverage/snapshot/build/hygiene) already run in ci.yml on every push and PR; repeating them here would duplicate signal and slow the real-API job. No build step — e2e tests run unbuilt via tsx + the tsconfig paths map. Single Node 24 (the `engines` floor): these tests exercise *API integration*, not node-version compat, which ci.yml's `[24, 26]` matrix already owns; a second Node version would double real-API calls for no added signal. `timeout-minutes: 45` bounds a wedged run given serial files (`fileParallelism: false`), 120s/test, and `retry: 2`. `cancel-in-progress` is enabled only for `pull_request` runs — a superseded PR run is on a stale commit and worth cancelling, whereas a push/schedule run is already producing the post-merge/nightly signal and is never cancelled.
+Run **only** `test:e2e`. The keyless gates (typecheck/lint/coverage/snapshot/build/hygiene) already run in ci.yml on every push and PR; repeating them here would duplicate signal and slow the real-API job. No build step — e2e tests run unbuilt via tsx + the tsconfig paths map. Single Node 24 (the `engines` floor): these tests exercise *API integration*, not node-version compat, which ci.yml's `[24, 26]` matrix already owns; a second Node version would double real-API calls for no added signal. `vitest.e2e.config.ts` runs files through a bounded worker pool (`DSH_E2E_MAX_WORKERS`, default and CI value `4`) so CI and local with-key runs parallelize independent files while retaining a one-line serial escape hatch for quota investigations. `timeout-minutes: 45` bounds a wedged run given 120s/test and `retry: 2`. `cancel-in-progress` is enabled only for `pull_request` runs — a superseded PR run is on a stale commit and worth cancelling, whereas a push/schedule run is already producing the post-merge/nightly signal and is never cancelled.

 ## Security

@@ -62,7 +62,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('compaction: a long session compa
        maxTokens: 2048,
        compactionRetries: 1,
      },
-      persistenceRoot: './.sessions',
+      persistenceRoot: join(workdir, '.sessions'),
    })
    const agent = ctx.agentLoop.create(AgentId('e2e-compaction'), { model: 'deepseek-v4-flash' })

@@ -1,3 +1,6 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
 import { afterEach, describe, expect, it } from 'vitest'
 import type { Context } from 'cordis'
 import { AgentId } from '@deepseek-ai/dsh-agent'
@@ -9,6 +12,7 @@ import { codingHarness, finalText, SYSTEM_PROMPT, waitForIdle } from './harness.
 */

 let ctx: Context | undefined
+let workdir: string | undefined

 afterEach(async () => {
  // Always dispose the harness, even on failure/retry/timeout: agent-loop
@@ -16,11 +20,14 @@ afterEach(async () => {
  // process the model left behind.
  await ctx?.fiber.dispose()
  ctx = undefined
+  if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
+  workdir = undefined
 })

 describe.skipIf(!process.env.DEEPSEEK_API_KEY)('full loop: real model + real bash tool', () => {
  it('runs a bash command on request and reports its output', async () => {
-    ctx = await codingHarness(process.cwd(), { persona: SYSTEM_PROMPT })
+    workdir = await mkdtemp(join(tmpdir(), 'dsh-full-loop-e2e-'))
+    ctx = await codingHarness(workdir, { persona: SYSTEM_PROMPT })
    const agent = ctx.agentLoop.create(AgentId('e2e-loop'), { model: 'deepseek-v4-flash' })

    agent.send([{ type: 'text', text: 'Run `echo e2e-ok` with the bash tool and tell me its exact output.' }])
@@ -1,3 +1,6 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
 import { afterEach, describe, expect, it } from 'vitest'
 import type { Context } from 'cordis'
 import { AgentId } from '@deepseek-ai/dsh-agent'
@@ -10,15 +13,19 @@ import { codingHarness, TODO_SYSTEM_PROMPT, waitForIdle } from './harness.ts'
 */

 let ctx: Context | undefined
+let workdir: string | undefined

 afterEach(async () => {
  await ctx?.fiber.dispose()
  ctx = undefined
+  if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
+  workdir = undefined
 })

 describe.skipIf(!process.env.DEEPSEEK_API_KEY)('todo_write: real model records a plan', () => {
  it('appends a todo/write event with the model-produced task list', async () => {
-    ctx = await codingHarness(process.cwd(), { persona: TODO_SYSTEM_PROMPT })
+    workdir = await mkdtemp(join(tmpdir(), 'dsh-todo-write-e2e-'))
+    ctx = await codingHarness(workdir, { persona: TODO_SYSTEM_PROMPT })
    const agent = ctx.agentLoop.create(AgentId('e2e-todo'), { model: 'deepseek-v4-flash' })

    agent.send([{ type: 'text', text:
@@ -19,6 +19,21 @@ try {
  // No .env — fine, the environment may already carry the variables.
 }

+const DEFAULT_E2E_MAX_WORKERS = 4
+
+function positiveIntFromEnv(name: string, fallback: number): number {
+  const raw = process.env[name]
+  if (raw === undefined || raw === '') return fallback
+
+  const value = Number(raw)
+  if (!Number.isInteger(value) || value < 1) {
+    throw new Error(`${name} must be a positive integer, got ${JSON.stringify(raw)}`)
+  }
+  return value
+}
+
+const e2eMaxWorkers = positiveIntFromEnv('DSH_E2E_MAX_WORKERS', DEFAULT_E2E_MAX_WORKERS)
+
 export default defineConfig({
  // Same resolution note as vitest.config.ts: bare workspace names resolve
  // through the root tsconfig paths map; the native option cannot do this.
@@ -31,9 +46,10 @@ export default defineConfig({
    testTimeout: 120_000,
    hookTimeout: 30_000,
    retry: 2,
-    // Run e2e files one at a time: the shared internal API key has a small
-    // concurrency quota, and parallel files issue enough simultaneous requests
-    // to trip it (manifesting as flaky rate-limit errors).
-    fileParallelism: false,
+    // Run files in a bounded pool: enough lower-level parallelism to keep CI
+    // and local with-key runs moving, while leaving a resource knob for shared
+    // API quotas (`DSH_E2E_MAX_WORKERS=1` restores serial execution).
+    fileParallelism: e2eMaxWorkers > 1,
+    maxWorkers: e2eMaxWorkers,
  },
 })