Tianyi Cui
e2882f486b
Validate replay sidecars and cross-copy failure facts, make browser console tripwires and macOS temp paths deterministic, and wait for asynchronous TUI resume details. Keep the owning docs, translations, and generated catalog aligned.
208 lines
11 KiB
TypeScript
208 lines
11 KiB
TypeScript
// Web e2e scenarios: live-turn interactions — cancellation, error surfacing,
|
|
// and transient-retry recovery, all through the real composition and wire.
|
|
// The model seam is dsh-llm-replay with override sidecars: `hang` (+ a
|
|
// readyFile marker) makes mid-stream cancel deterministic by construction,
|
|
// `throw` entries express provider failures by stable code, and `{ patches }`
|
|
// augmentation injects a transient throw before the recorded success so
|
|
// llm-retry's recovery is proven end-to-end in the browser. Sidecar CONTENT
|
|
// is authored here (single-sourced against the fixture via deriveReplayScript
|
|
// — no committed copy of recorded chunks); the file is a per-run artifact in
|
|
// the temp workspace. One recorded base fixture serves all three scenarios.
|
|
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
|
import { existsSync } from 'node:fs'
|
|
import { tmpdir } from 'node:os'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { join } from 'node:path'
|
|
import type { Browser, Page } from 'playwright'
|
|
import { chromium } from 'playwright'
|
|
import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
|
|
import { deriveReplayScript, parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
|
|
import type { ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
|
|
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
|
import {
|
|
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
|
|
launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold,
|
|
} from './scaffold.ts'
|
|
import { saveFailureShot } from './support.ts'
|
|
|
|
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/live-interactions', import.meta.url))
|
|
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
|
// One golden per interactive end-state: what the user is left looking at
|
|
// after cancel, after a non-retryable failure (pins the FIXME(web-error-surface)
|
|
// gap as a reviewable artifact: NO error copy in the tree), and after retry
|
|
// recovery — three genuinely different terminal surfaces of one fixture.
|
|
const CANCEL_EXPECTED = join(SNAPSHOT_DIR, 'cancel.expected.md')
|
|
const ERROR_EXPECTED = join(SNAPSHOT_DIR, 'error-auth.expected.md')
|
|
const RETRY_EXPECTED = join(SNAPSHOT_DIR, 'retry.expected.md')
|
|
const MODE = webSnapshotMode()
|
|
|
|
// The recorded base: one text-only turn whose derived script the sidecars
|
|
// patch. Kept deliberately tool-free so the derived script is exactly one
|
|
// model call.
|
|
const PROMPT = 'Reply with a one-sentence description of event sourcing, then stop.'
|
|
|
|
/** turn/end reasons observed, in order. */
|
|
function turnEndReasons(events: SessionEvent[]): string[] {
|
|
return events
|
|
.filter(e => e.type === 'turn/end')
|
|
.map(e => (e as SessionEvent & { data: { reason: { kind: string } } }).data.reason.kind)
|
|
}
|
|
|
|
describe('web e2e: live-turn interactions (cancel / error / retry)', () => {
|
|
let scaffold: WebScaffold | undefined
|
|
let browser: Browser | undefined
|
|
let page: Page
|
|
let tripwire: ReturnType<typeof watchConsole>
|
|
let sessionEvents: SessionEvent[]
|
|
let sidecarDir: string | undefined
|
|
|
|
afterEach(async () => {
|
|
// scaffold.close() failures MUST fail the scenario: assertConsumed() is
|
|
// the fixture-drift tripwire and cleanup problems are real defects. Run
|
|
// every teardown step regardless, then rethrow what failed.
|
|
const failures: unknown[] = []
|
|
await browser?.close().catch((error: unknown) => failures.push(error))
|
|
browser = undefined
|
|
const closing = scaffold
|
|
scaffold = undefined
|
|
await closing?.close().catch((error: unknown) => failures.push(error))
|
|
if (sidecarDir !== undefined) await rm(sidecarDir, { recursive: true, force: true }).catch((error: unknown) => failures.push(error))
|
|
sidecarDir = undefined
|
|
if (failures.length === 1) throw failures[0]
|
|
if (failures.length > 1) throw new AggregateError(failures, 'live-interactions teardown failed')
|
|
})
|
|
|
|
/** Boot scaffold + page with an optional override doc materialized per run. */
|
|
async function launch(buildOverride?: (sidecarHome: string) => ReplayOverrideDoc): Promise<void> {
|
|
sessionEvents = []
|
|
let overridePath: string | undefined
|
|
if (buildOverride !== undefined) {
|
|
// The sidecar CONTENT is authored in this spec; the file is a per-run
|
|
// artifact minted in a spec-owned temp dir. It must exist BEFORE the
|
|
// scaffold boots — installLlmReplay resolves the script at install.
|
|
sidecarDir = await mkdtemp(join(tmpdir(), 'dsh-web-e2e-sidecar-'))
|
|
overridePath = join(sidecarDir, 'replay.override.json')
|
|
await writeFile(overridePath, JSON.stringify(buildOverride(sidecarDir)))
|
|
}
|
|
scaffold = await launchWebScaffold({
|
|
replayFixture: FIXTURE,
|
|
...(overridePath === undefined ? {} : { replayOverride: overridePath }),
|
|
})
|
|
scaffold.ctx.on('session/event', (_session, event: SessionEvent) => { sessionEvents.push(event) })
|
|
browser = await chromium.launch()
|
|
page = await browser.newPage({ viewport: { width: 1680, height: 1000 } })
|
|
tripwire = watchConsole(page)
|
|
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
|
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
}
|
|
|
|
/**
|
|
* Type the recorded prompt and send, with the settled barrier pre-armed.
|
|
* Returned WRAPPED ({ settled }) — a bare returned promise would be
|
|
* flattened by the caller's await, blocking on turn/end before the caller
|
|
* can act mid-turn (the cancel scenario's whole point).
|
|
*/
|
|
async function sendPrompt(timeoutMs?: number): Promise<{ settled: ReturnType<WebScaffold['whenTurnSettled']> }> {
|
|
const input = page.locator('textarea').first()
|
|
await input.waitFor({ timeout: 10_000 })
|
|
const settled = scaffold!.whenTurnSettled(timeoutMs)
|
|
await input.fill(PROMPT)
|
|
await input.press('Enter')
|
|
return { settled }
|
|
}
|
|
|
|
it.skipIf(MODE !== 'record')('records the base fixture live through the composer', async () => {
|
|
await launch()
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-interactions-record'))
|
|
const { settled } = await sendPrompt(180_000)
|
|
const sessionId = await settled
|
|
await recordFixture(scaffold!, sessionId, FIXTURE)
|
|
}, 200_000)
|
|
|
|
it.skipIf(MODE === 'record')('cancels a hung stream deterministically via the readyFile marker', async () => {
|
|
expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT])
|
|
let marker = ''
|
|
await launch((sidecarHome) => {
|
|
marker = join(sidecarHome, '.hang-ready')
|
|
return { patches: [{ at: 0, entry: { kind: 'hang', readyFile: marker } }] }
|
|
})
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-cancel'))
|
|
const { settled } = await sendPrompt()
|
|
// The marker IS the synchronization: the stream is provably parked in the
|
|
// hang (prefix chunks delivered to the loop) before the stop click.
|
|
await expect.poll(() => existsSync(marker), { timeout: 15_000 }).toBe(true)
|
|
await page.getByRole('button', { name: 'Stop generating' }).click()
|
|
await settled
|
|
expect(turnEndReasons(sessionEvents).at(-1)).toBe('aborted')
|
|
// Composer recovered; no streaming node lingers.
|
|
await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true)
|
|
expect(await page.locator('[data-streaming="true"]').count()).toBe(0)
|
|
// Golden of the aborted end-state: the prompt bubble plus the frozen
|
|
// partial ('partial' is the hang entry's replayed prefix) and no more.
|
|
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
|
|
await compareOrRefreshGolden(CANCEL_EXPECTED, snapshot, MODE)
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
expect(tripwire.warnings).toEqual([])
|
|
}, 120_000)
|
|
|
|
it.skipIf(MODE === 'record')('surfaces a non-retryable AUTH failure without retrying', async () => {
|
|
await launch(() => ({
|
|
patches: [{ at: 0, entry: { kind: 'throw', chunks: [], message: 'invalid api key', code: 'AUTH' } }],
|
|
}))
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-error-auth'))
|
|
const { settled } = await sendPrompt()
|
|
await settled
|
|
expect(turnEndReasons(sessionEvents).at(-1)).toBe('error')
|
|
// AUTH is outside llm-retry's retryable set: no retry record.
|
|
expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBe(0)
|
|
// Product gap found by this lane, pinned as-is: the client consumes no
|
|
// agent/error frames and a pre-chunk failure freezes no partial, so THIS
|
|
// failure renders no error copy anywhere — the user sees the send simply
|
|
// stop. FIXME(web-error-surface): assert visible error text here once the
|
|
// web UI grows an error rendering; until then the pinned contract is
|
|
// "no crash, composer recovers, turn logged as error".
|
|
await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true)
|
|
expect(await page.locator('[data-streaming="true"]').count()).toBe(0)
|
|
// Golden of the same gap: the prompt bubble alone, no error copy in the
|
|
// tree — the diff that changes when web-error-surface lands.
|
|
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
|
|
await compareOrRefreshGolden(ERROR_EXPECTED, snapshot, MODE)
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
expect(tripwire.warnings).toEqual([])
|
|
}, 120_000)
|
|
|
|
it.skipIf(MODE === 'record')('recovers a transient SERVER failure through llm-retry and completes', async () => {
|
|
const derived = deriveReplayScript(parseSessionLog(await readFile(FIXTURE, 'utf8')))
|
|
expect(derived).toHaveLength(1)
|
|
await launch(() => ({
|
|
patches: [
|
|
{ at: 0, entry: { kind: 'throw', chunks: [], message: 'upstream 503', code: 'SERVER' } },
|
|
// Append the fixture's own success as the retry attempt — single-
|
|
// sourced from the recording, never copied into a committed sidecar.
|
|
{ at: 1, entry: derived[0]! },
|
|
],
|
|
}))
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-retry'))
|
|
// llm-retry backs off ~500ms before the second attempt.
|
|
const { settled } = await sendPrompt(60_000)
|
|
await settled
|
|
expect(turnEndReasons(sessionEvents).at(-1)).toBe('completed')
|
|
// The durable retry record proves the second attempt (request/header logs
|
|
// only on change, so attempt count is invisible there).
|
|
expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBeGreaterThanOrEqual(1)
|
|
await expect.poll(() => page.getByText('event sourcing', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThan(0)
|
|
// Golden of the recovered end-state: indistinguishable from a clean
|
|
// completion — retries are deliberately invisible in the transcript.
|
|
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
|
|
await compareOrRefreshGolden(RETRY_EXPECTED, snapshot, MODE)
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
expect(tripwire.warnings).toEqual([])
|
|
}, 120_000)
|
|
|
|
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
|
|
await assertFixtureInventory(SNAPSHOT_DIR, [
|
|
'session.jsonl', 'cancel.expected.md', 'error-auth.expected.md', 'retry.expected.md',
|
|
])
|
|
})
|
|
})
|