// Web e2e scenarios: live-turn interactions — cancellation, error surfacing, // and transient-retry recovery, all through the real composition and wire. // The model seam is dsh-llm-replay with override sidecars: `hang` (+ a // readyFile marker) makes mid-stream cancel deterministic by construction, // `throw` entries express provider failures by stable code, and `{ patches }` // augmentation injects a transient throw before the recorded success so // llm-retry's recovery is proven end-to-end in the browser. Sidecar CONTENT // is authored here (single-sourced against the fixture via deriveReplayScript // — no committed copy of recorded chunks); the file is a per-run artifact in // the temp workspace. One recorded base fixture serves all three scenarios. import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { existsSync } from 'node:fs' import { tmpdir } from 'node:os' import { fileURLToPath } from 'node:url' import { join } from 'node:path' import type { Browser, Page } from 'playwright' import { chromium } from 'playwright' import { afterEach, describe, expect, it, onTestFailed } from 'vitest' import { deriveReplayScript, parseSessionLog } from '@deepseek-ai/dsh-llm-replay' import type { ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay' import type { SessionEvent } from '@deepseek-ai/dsh-session' import { assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts, launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold, } from './scaffold.ts' import { connectFreshWorkspace, saveFailureShot } from './support.ts' const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/live-interactions', import.meta.url)) const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl') // One golden per interactive end-state: what the user is left looking at // after cancel, after a non-retryable failure (pins the FIXME(web-error-surface) // gap as a reviewable artifact: NO error copy in the tree), and after retry // recovery — three genuinely different terminal surfaces of one fixture. const CANCEL_EXPECTED = join(SNAPSHOT_DIR, 'cancel.expected.md') const ERROR_EXPECTED = join(SNAPSHOT_DIR, 'error-auth.expected.md') const RETRY_EXPECTED = join(SNAPSHOT_DIR, 'retry.expected.md') const MODE = webSnapshotMode() // The recorded base: one text-only turn whose derived script the sidecars // patch. Kept deliberately tool-free so the derived script is exactly one // model call. const PROMPT = 'Reply with a one-sentence description of event sourcing, then stop.' /** turn/end reasons observed, in order. */ function turnEndReasons(events: SessionEvent[]): string[] { return events .filter(e => e.type === 'turn/end') .map(e => (e as SessionEvent & { data: { reason: { kind: string } } }).data.reason.kind) } describe('web e2e: live-turn interactions (cancel / error / retry)', () => { let scaffold: WebScaffold | undefined let browser: Browser | undefined let page: Page let tripwire: ReturnType let sessionEvents: SessionEvent[] let sidecarDir: string | undefined afterEach(async () => { // scaffold.close() failures MUST fail the scenario: assertConsumed() is // the fixture-drift tripwire and cleanup problems are real defects. Run // every teardown step regardless, then rethrow what failed. const failures: unknown[] = [] await browser?.close().catch((error: unknown) => failures.push(error)) browser = undefined const closing = scaffold scaffold = undefined await closing?.close().catch((error: unknown) => failures.push(error)) if (sidecarDir !== undefined) await rm(sidecarDir, { recursive: true, force: true }).catch((error: unknown) => failures.push(error)) sidecarDir = undefined if (failures.length === 1) throw failures[0] if (failures.length > 1) throw new AggregateError(failures, 'live-interactions teardown failed') }) /** Boot scaffold + page with an optional override doc materialized per run. */ async function launch(buildOverride?: (sidecarHome: string) => ReplayOverrideDoc): Promise { sessionEvents = [] let overridePath: string | undefined if (buildOverride !== undefined) { // The sidecar CONTENT is authored in this spec; the file is a per-run // artifact minted in a spec-owned temp dir. It must exist BEFORE the // scaffold boots — installLlmReplay resolves the script at install. sidecarDir = await mkdtemp(join(tmpdir(), 'dsh-web-e2e-sidecar-')) overridePath = join(sidecarDir, 'replay.override.json') await writeFile(overridePath, JSON.stringify(buildOverride(sidecarDir))) } scaffold = await launchWebScaffold({ replayFixture: FIXTURE, ...(overridePath === undefined ? {} : { replayOverride: overridePath }), }) scaffold.ctx.on('session/event', (_session, event: SessionEvent) => { sessionEvents.push(event) }) browser = await chromium.launch() page = await browser.newPage({ viewport: { width: 1680, height: 1000 } }) tripwire = watchConsole(page) await page.goto(scaffold.baseUrl, { waitUntil: 'load' }) await page.waitForSelector('[class*="frame"]', { timeout: 30_000 }) // Fresh world: connect a Workspace so the composer scenarios start live. await connectFreshWorkspace(page) } /** * Type the recorded prompt and send, with the settled barrier pre-armed. * Returned WRAPPED ({ settled }) — a bare returned promise would be * flattened by the caller's await, blocking on turn/end before the caller * can act mid-turn (the cancel scenario's whole point). */ async function sendPrompt(timeoutMs?: number): Promise<{ settled: ReturnType }> { const input = page.locator('textarea').first() await input.waitFor({ timeout: 10_000 }) const settled = scaffold!.whenTurnSettled(timeoutMs) await input.fill(PROMPT) await input.press('Enter') return { settled } } it.skipIf(MODE !== 'record')('records the base fixture live through the composer', async () => { await launch() onTestFailed(() => saveFailureShot(page, 'web-e2e-interactions-record')) const { settled } = await sendPrompt(180_000) const sessionId = await settled await recordFixture(scaffold!, sessionId, FIXTURE) }, 200_000) it.skipIf(MODE === 'record')('cancels a hung stream deterministically via the readyFile marker', async () => { expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT]) let marker = '' await launch((sidecarHome) => { marker = join(sidecarHome, '.hang-ready') return { patches: [{ at: 0, entry: { kind: 'hang', readyFile: marker } }] } }) onTestFailed(() => saveFailureShot(page, 'web-e2e-cancel')) const { settled } = await sendPrompt() // The marker IS the synchronization: the stream is provably parked in the // hang (prefix chunks delivered to the loop) before the stop click. await expect.poll(() => existsSync(marker), { timeout: 15_000 }).toBe(true) await page.getByRole('button', { name: 'Stop generating' }).click() await settled expect(turnEndReasons(sessionEvents).at(-1)).toBe('aborted') // Composer recovered; no streaming node lingers. The host settled first // (awaited above), but the abort frame reaches the browser over SSE — the // frozen-partial swap is eventually consistent, so poll rather than count. await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true) await expect.poll(() => page.locator('[data-streaming="true"]').count(), { timeout: 10_000 }).toBe(0) // Golden of the aborted end-state: the prompt bubble plus the frozen // partial ('partial' is the hang entry's replayed prefix) and no more. const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd) await compareOrRefreshGolden(CANCEL_EXPECTED, snapshot, MODE) expect(tripwire.pageErrors).toEqual([]) expect(tripwire.warnings).toEqual([]) }, 120_000) it.skipIf(MODE === 'record')('surfaces a non-retryable AUTH failure without retrying', async () => { await launch(() => ({ patches: [{ at: 0, entry: { kind: 'throw', chunks: [], message: 'invalid api key', code: 'AUTH' } }], })) onTestFailed(() => saveFailureShot(page, 'web-e2e-error-auth')) const { settled } = await sendPrompt() await settled expect(turnEndReasons(sessionEvents).at(-1)).toBe('error') // AUTH is outside llm-retry's retryable set: no retry record. expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBe(0) // Product gap found by this lane, pinned as-is: the client consumes no // agent/error frames and a pre-chunk failure freezes no partial, so THIS // failure renders no error copy anywhere — the user sees the send simply // stop. FIXME(web-error-surface): assert visible error text here once the // web UI grows an error rendering; until then the pinned contract is // "no crash, composer recovers, turn logged as error". await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true) expect(await page.locator('[data-streaming="true"]').count()).toBe(0) // Golden of the same gap: the prompt bubble alone, no error copy in the // tree — the diff that changes when web-error-surface lands. const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd) await compareOrRefreshGolden(ERROR_EXPECTED, snapshot, MODE) expect(tripwire.pageErrors).toEqual([]) expect(tripwire.warnings).toEqual([]) }, 120_000) it.skipIf(MODE === 'record')('recovers a transient SERVER failure through llm-retry and completes', async () => { const derived = deriveReplayScript(parseSessionLog(await readFile(FIXTURE, 'utf8'))) expect(derived).toHaveLength(1) await launch(() => ({ patches: [ { at: 0, entry: { kind: 'throw', chunks: [], message: 'upstream 503', code: 'SERVER' } }, // Append the fixture's own success as the retry attempt — single- // sourced from the recording, never copied into a committed sidecar. { at: 1, entry: derived[0]! }, ], })) onTestFailed(() => saveFailureShot(page, 'web-e2e-retry')) // llm-retry backs off ~500ms before the second attempt. const { settled } = await sendPrompt(60_000) await settled expect(turnEndReasons(sessionEvents).at(-1)).toBe('completed') // The durable retry record proves the second attempt (request/header logs // only on change, so attempt count is invisible there). expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBeGreaterThanOrEqual(1) await expect.poll(() => page.getByText('event sourcing', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThan(0) // Golden of the recovered end-state: indistinguishable from a clean // completion — retries are deliberately invisible in the transcript. const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd) await compareOrRefreshGolden(RETRY_EXPECTED, snapshot, MODE) expect(tripwire.pageErrors).toEqual([]) expect(tripwire.warnings).toEqual([]) }, 120_000) it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => { await assertFixtureInventory(SNAPSHOT_DIR, [ 'session.jsonl', 'cancel.expected.md', 'error-auth.expected.md', 'retry.expected.md', ]) }) })