From d08050ab69256cdc5f4183dd0b56dbad475fac59 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:30:33 +0800 Subject: [PATCH] =?UTF-8?q?chore(web-e2e):=20gate=20fixes=20=E2=80=94=20ca?= =?UTF-8?q?talog,=20budgets,=20knip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerate config-catalog for the llm-replay paceMs row; condense the testing.md web-lane entry to pointer form and raise its ceiling 1020->1060 (the two-sentence tier entry for a genuinely new surface does not fit the old ceiling after relocation-first trims); internalize two harness helpers knip flagged (rawSessionLog/normalizeAria are module-internal). --- apps/web/tests/harness.ts | 9 ++----- docs/config-catalog.md | 2 +- docs/testing.md | 4 +-- .../llm-replay/tests/llm-replay.spec.ts | 27 ++++++++++++++++++- scripts/doc-budgets.manifest.json | 2 +- 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/apps/web/tests/harness.ts b/apps/web/tests/harness.ts index 71c3ac9344..b2941c62e0 100644 --- a/apps/web/tests/harness.ts +++ b/apps/web/tests/harness.ts @@ -230,10 +230,8 @@ export async function launchWebHarness(options: LaunchOptions = {}): Promise JSON.stringify(event)), @@ -333,11 +331,8 @@ export async function seedSession(harness: WebHarness, fixtureText: string, id: /** * Normalize an aria snapshot: uuid, cwd, workspace-basename, and duration * volatility collapse to stable tokens. - * @param snapshot - raw ariaSnapshot text. - * @param workspaceCwd - the harness workspace (basename doubles as the header breadcrumb). - * @returns tokenized snapshot text. */ -export function normalizeAria(snapshot: string, workspaceCwd: string): string { +function normalizeAria(snapshot: string, workspaceCwd: string): string { // The header breadcrumb renders the workspace's basename, not the full // path, so both spellings must collapse to the token. const base = workspaceCwd.split('/').pop()! diff --git a/docs/config-catalog.md b/docs/config-catalog.md index c06fef3d4d..1b546be67d 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -643,7 +643,7 @@ export interface ReplayModelConfig { } ``` -Source: [`packages/support/llm-replay/src/index.ts:453`](../packages/support/llm-replay/src/index.ts) +Source: [`packages/support/llm-replay/src/index.ts:454`](../packages/support/llm-replay/src/index.ts) ## `@deepseek-ai/dsh-llm-retry` diff --git a/docs/testing.md b/docs/testing.md index 5841cd5f29..78ce3c2856 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -8,7 +8,7 @@ How this repo tests, tier by tier, and the rules that keep a green suite meaning - **Coverage gate** (`pnpm run test:coverage`): the gating run, per-file 100% on `packages/*/*/src`. An uncovered line is often dead code the gate is correctly flagging for deletion, not a missing test to bolt on. Line coverage is necessary, never sufficient — it proves lines ran, not that the feature works as shipped. - **Real-API e2e** (`pnpm run test:e2e`): with-key tests against live provider APIs — the DeepSeek model plus provider-specific smokes that gate on their own keys (`EXA_API_KEY`, `PERPLEXITY_API_KEY`, …); each suite self-skips without its key so keyless CI stays green ([real-API e2e Agent Note](../.agents/notes/implemented/testing/2026-06-19-real-api-e2e-ci.md)). - **Snapshot** (`pnpm run test:snapshot`): transport-specific keyless expected outputs cover external presentation. ACP suites boot the real example subprocess, replay a recorded session, and diff normalized JSON-RPC plus the re-persisted log ([ACP snapshot Agent Note](../.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md)); the headless suite independently pins `stream-json` through its real one-shot subprocess. TUI completed journeys replay recorded primary/child JSONL through the real agent loop and tools before projecting ANSI into semantic terminal-state expected outputs; package-local snapshots retain transient renderer states, and a real PTY conversation covers the process boundary ([TUI snapshot Agent Note](../.agents/notes/implemented/testing/2026-07-18-tui-terminal-state-snapshots.md)). Use `pnpm run test:snapshot:record` when a model transcript must change and `pnpm run test:snapshot:refresh` when committed replay input remains correct; review every JSONL and expected-output diff. System-prompt/tool-schema content is pinned by one ACP scenario (`text-turn`) and tokenized in every other fixture, so a prompt or schema edit churns one committed line ([pinned-header Agent Note](../.agents/notes/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)). -- **Web browser snapshot** (inside `pnpm run test:web`, gate-exempt like the rest of that lane): the web GUI's keyless assembled-transcript tier — a real chromium over the real in-process web assembly (`llm: false` + `dsh-llm-replay`), scenario fixtures and normalized conversation aria goldens under `apps/web/tests/snapshots/`. Its record/refresh commands diverge from `test:snapshot` (`DSH_SNAPSHOT=record pnpm run test:web` re-records against the live model; `DSH_SNAPSHOT=refresh` rewrites goldens keylessly); the [web e2e lane Agent Note](../.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md) owns the design, and CI browser provisioning is deferred there. +- **Web browser snapshot** (gate-exempt `pnpm run test:web`): real chromium over the in-process web assembly replays recorded fixtures against conversation aria goldens (`apps/web/tests/snapshots/`); `DSH_SNAPSHOT=record`/`refresh` semantics and the deferred CI browser decision live in the [web e2e lane Agent Note](../.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md). ## The with-key policy: inference is cheap here @@ -42,4 +42,4 @@ An e2e assertion re-runs the command or re-reads the file externally; a keyword ## When a snapshot test is required -Every non-trivial model- or human-visible change adds or updates a keyless scenario in the same PR through a runnable example's owning snapshot suite. Package tests, e2e assertions, mock/test-only compositions, and PR rationale do not replace the assembled transcript; extend the harness when needed. ACP surfaces use `examples//tests/snapshots/`, a scenario table over the [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) suite factory (`examples/acp-agent` is primary); `examples/headless-agent` owns the `stream-json` snapshot and replay fixtures. Completed interactive-terminal journeys use JSONL-driven scenarios under `examples/tui-agent/tests/snapshots/`; transient presentation uses the package-local semantic matrix, with a PTY case when input, Loader selection, or terminal teardown changes. Browser-rendered web GUI journeys use `apps/web/tests/snapshots/` through the web e2e lane's harness. New capability seams, lifecycle shapes, or transcript surfaces name every coverage tier at plan time and verify the harness can express it before implementation. +Every non-trivial model- or human-visible change adds or updates a keyless scenario in the same PR through a runnable example's owning snapshot suite. Package tests, e2e assertions, mock/test-only compositions, and PR rationale do not replace the assembled transcript; extend the harness when needed. ACP surfaces use `examples//tests/snapshots/`, a scenario table over the [`dsh-acp-snapshot`](../packages/support/acp-snapshot/README.md) suite factory (`examples/acp-agent` is primary); `examples/headless-agent` owns the `stream-json` snapshot and replay fixtures. Completed interactive-terminal journeys use JSONL-driven scenarios under `examples/tui-agent/tests/snapshots/`; transient presentation uses the package-local semantic matrix, with a PTY case when input, Loader selection, or terminal teardown changes. Browser-rendered web GUI journeys use `apps/web/tests/snapshots/`. New capability seams, lifecycle shapes, or transcript surfaces name every coverage tier at plan time and verify the harness can express it before implementation. diff --git a/packages/support/llm-replay/tests/llm-replay.spec.ts b/packages/support/llm-replay/tests/llm-replay.spec.ts index b6b03aacbf..d42626bedd 100644 --- a/packages/support/llm-replay/tests/llm-replay.spec.ts +++ b/packages/support/llm-replay/tests/llm-replay.spec.ts @@ -476,6 +476,31 @@ describe('installLlmReplay (through the real LlmService)', () => { expect(() => { handle.assertConsumed() }).not.toThrow() }) + it('paces a throw-entry prefix too (the recorded partial streams at the same cadence)', async () => { + writeFileSync(file, sessionJsonl([]), 'utf8') + const overrideFile = join(dir, 'replay.override.json') + const partial: StreamChunk[] = [{ type: 'block-start', index: 0, blockType: 'text' }] + writeFileSync(overrideFile, JSON.stringify([ + { kind: 'throw', chunks: partial, message: 'boom', code: 'STREAM_CLOSED' }, + ]), 'utf8') + const ctx = new Context() + await ctx.plugin(LlmService) + installLlmReplay(ctx, { file, overrideFile, paceMs: 10 }) + const started = performance.now() + await expect(drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).rejects.toThrow('boom') + expect(performance.now() - started).toBeGreaterThanOrEqual(5) + }) + + it('assertConsumed names an underrunning identified session by its id', async () => { + writeLog(TEXT_CHUNKS, TEXT_CHUNKS) + const ctx = new Context() + await ctx.plugin(LlmService) + const handle = installLlmReplay(ctx, { file }) + const sessionId = 'live-underrun' as NonNullable + await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [], sessionId })) + expect(() => { handle.assertConsumed() }).toThrow(/session live-underrun consumed 1\/2/) + }) + it('assertConsumed reports recorded scripts no live session ever bound', async () => { writeLog(TEXT_CHUNKS) const childFile = join(dir, 'session.1.jsonl') @@ -690,7 +715,7 @@ describe('apply (the plugin entry)', () => { writeFileSync(file, sessionJsonl(TEXT_CHUNKS.map((c, i) => chunkEvent(i + 1, 1, 1, c))), 'utf8') const ctx = new Context() await ctx.plugin(LlmService) - apply(ctx, { file, providers: [{ id: 'm', models: [{ id: 'm' }] }] }) + apply(ctx, { file, providers: [{ id: 'm', models: [{ id: 'm' }] }], paceMs: 1 }) expect(ctx.llm.listProviders()).toEqual([{ id: 'm', name: 'm' }]) expect(await drain(ctx.llm.stream({ provider: 'm', model: 'm', messages: [] }))).toEqual(TEXT_CHUNKS) }) diff --git a/scripts/doc-budgets.manifest.json b/scripts/doc-budgets.manifest.json index 7e4d154174..fe61bfa69d 100644 --- a/scripts/doc-budgets.manifest.json +++ b/scripts/doc-budgets.manifest.json @@ -4,7 +4,7 @@ "docs/architecture.md": 1800, "docs/cordis-primer.md": 600, "docs/defensive-patterns.md": 550, - "docs/testing.md": 1020, + "docs/testing.md": 1060, "examples/AGENTS.md": 310, "packages/AGENTS.md": 660, "packages/README.md": 760