imccyu
ecdaf0dc24
The startup-selection flow leaves a fresh world (no Workspace) in the locked view state, so every e2e scenario that types into the composer now connects one first via the shared connectFreshWorkspace helper (hero picker create-by-name dialog; the default 'workspace' name keeps the session-header cwd assertions intact). Golden refreshes carry the current composer chrome: the plan/model control seats are empty until their owning plugins register (the seats shipped without occupants on this branch), the sidebar shows the connected workspace group pre-send, and the bash details material renders Input/code/Output as separate nodes. The cancel scenario polls the frozen-partial swap instead of counting synchronously — the abort frame reaches the browser over SSE after the host settles.
212 lines
11 KiB
TypeScript
212 lines
11 KiB
TypeScript
// Web e2e scenarios: live-turn interactions — cancellation, error surfacing,
|
|
// and transient-retry recovery, all through the real composition and wire.
|
|
// The model seam is dsh-llm-replay with override sidecars: `hang` (+ a
|
|
// readyFile marker) makes mid-stream cancel deterministic by construction,
|
|
// `throw` entries express provider failures by stable code, and `{ patches }`
|
|
// augmentation injects a transient throw before the recorded success so
|
|
// llm-retry's recovery is proven end-to-end in the browser. Sidecar CONTENT
|
|
// is authored here (single-sourced against the fixture via deriveReplayScript
|
|
// — no committed copy of recorded chunks); the file is a per-run artifact in
|
|
// the temp workspace. One recorded base fixture serves all three scenarios.
|
|
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
|
import { existsSync } from 'node:fs'
|
|
import { tmpdir } from 'node:os'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { join } from 'node:path'
|
|
import type { Browser, Page } from 'playwright'
|
|
import { chromium } from 'playwright'
|
|
import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
|
|
import { deriveReplayScript, parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
|
|
import type { ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
|
|
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
|
import {
|
|
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
|
|
launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold,
|
|
} from './scaffold.ts'
|
|
import { connectFreshWorkspace, saveFailureShot } from './support.ts'
|
|
|
|
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/live-interactions', import.meta.url))
|
|
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
|
// One golden per interactive end-state: what the user is left looking at
|
|
// after cancel, after a non-retryable failure (pins the FIXME(web-error-surface)
|
|
// gap as a reviewable artifact: NO error copy in the tree), and after retry
|
|
// recovery — three genuinely different terminal surfaces of one fixture.
|
|
const CANCEL_EXPECTED = join(SNAPSHOT_DIR, 'cancel.expected.md')
|
|
const ERROR_EXPECTED = join(SNAPSHOT_DIR, 'error-auth.expected.md')
|
|
const RETRY_EXPECTED = join(SNAPSHOT_DIR, 'retry.expected.md')
|
|
const MODE = webSnapshotMode()
|
|
|
|
// The recorded base: one text-only turn whose derived script the sidecars
|
|
// patch. Kept deliberately tool-free so the derived script is exactly one
|
|
// model call.
|
|
const PROMPT = 'Reply with a one-sentence description of event sourcing, then stop.'
|
|
|
|
/** turn/end reasons observed, in order. */
|
|
function turnEndReasons(events: SessionEvent[]): string[] {
|
|
return events
|
|
.filter(e => e.type === 'turn/end')
|
|
.map(e => (e as SessionEvent & { data: { reason: { kind: string } } }).data.reason.kind)
|
|
}
|
|
|
|
describe('web e2e: live-turn interactions (cancel / error / retry)', () => {
|
|
let scaffold: WebScaffold | undefined
|
|
let browser: Browser | undefined
|
|
let page: Page
|
|
let tripwire: ReturnType<typeof watchConsole>
|
|
let sessionEvents: SessionEvent[]
|
|
let sidecarDir: string | undefined
|
|
|
|
afterEach(async () => {
|
|
// scaffold.close() failures MUST fail the scenario: assertConsumed() is
|
|
// the fixture-drift tripwire and cleanup problems are real defects. Run
|
|
// every teardown step regardless, then rethrow what failed.
|
|
const failures: unknown[] = []
|
|
await browser?.close().catch((error: unknown) => failures.push(error))
|
|
browser = undefined
|
|
const closing = scaffold
|
|
scaffold = undefined
|
|
await closing?.close().catch((error: unknown) => failures.push(error))
|
|
if (sidecarDir !== undefined) await rm(sidecarDir, { recursive: true, force: true }).catch((error: unknown) => failures.push(error))
|
|
sidecarDir = undefined
|
|
if (failures.length === 1) throw failures[0]
|
|
if (failures.length > 1) throw new AggregateError(failures, 'live-interactions teardown failed')
|
|
})
|
|
|
|
/** Boot scaffold + page with an optional override doc materialized per run. */
|
|
async function launch(buildOverride?: (sidecarHome: string) => ReplayOverrideDoc): Promise<void> {
|
|
sessionEvents = []
|
|
let overridePath: string | undefined
|
|
if (buildOverride !== undefined) {
|
|
// The sidecar CONTENT is authored in this spec; the file is a per-run
|
|
// artifact minted in a spec-owned temp dir. It must exist BEFORE the
|
|
// scaffold boots — installLlmReplay resolves the script at install.
|
|
sidecarDir = await mkdtemp(join(tmpdir(), 'dsh-web-e2e-sidecar-'))
|
|
overridePath = join(sidecarDir, 'replay.override.json')
|
|
await writeFile(overridePath, JSON.stringify(buildOverride(sidecarDir)))
|
|
}
|
|
scaffold = await launchWebScaffold({
|
|
replayFixture: FIXTURE,
|
|
...(overridePath === undefined ? {} : { replayOverride: overridePath }),
|
|
})
|
|
scaffold.ctx.on('session/event', (_session, event: SessionEvent) => { sessionEvents.push(event) })
|
|
browser = await chromium.launch()
|
|
page = await browser.newPage({ viewport: { width: 1680, height: 1000 } })
|
|
tripwire = watchConsole(page)
|
|
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
|
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
// Fresh world: connect a Workspace so the composer scenarios start live.
|
|
await connectFreshWorkspace(page)
|
|
}
|
|
|
|
/**
|
|
* Type the recorded prompt and send, with the settled barrier pre-armed.
|
|
* Returned WRAPPED ({ settled }) — a bare returned promise would be
|
|
* flattened by the caller's await, blocking on turn/end before the caller
|
|
* can act mid-turn (the cancel scenario's whole point).
|
|
*/
|
|
async function sendPrompt(timeoutMs?: number): Promise<{ settled: ReturnType<WebScaffold['whenTurnSettled']> }> {
|
|
const input = page.locator('textarea').first()
|
|
await input.waitFor({ timeout: 10_000 })
|
|
const settled = scaffold!.whenTurnSettled(timeoutMs)
|
|
await input.fill(PROMPT)
|
|
await input.press('Enter')
|
|
return { settled }
|
|
}
|
|
|
|
it.skipIf(MODE !== 'record')('records the base fixture live through the composer', async () => {
|
|
await launch()
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-interactions-record'))
|
|
const { settled } = await sendPrompt(180_000)
|
|
const sessionId = await settled
|
|
await recordFixture(scaffold!, sessionId, FIXTURE)
|
|
}, 200_000)
|
|
|
|
it.skipIf(MODE === 'record')('cancels a hung stream deterministically via the readyFile marker', async () => {
|
|
expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT])
|
|
let marker = ''
|
|
await launch((sidecarHome) => {
|
|
marker = join(sidecarHome, '.hang-ready')
|
|
return { patches: [{ at: 0, entry: { kind: 'hang', readyFile: marker } }] }
|
|
})
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-cancel'))
|
|
const { settled } = await sendPrompt()
|
|
// The marker IS the synchronization: the stream is provably parked in the
|
|
// hang (prefix chunks delivered to the loop) before the stop click.
|
|
await expect.poll(() => existsSync(marker), { timeout: 15_000 }).toBe(true)
|
|
await page.getByRole('button', { name: 'Stop generating' }).click()
|
|
await settled
|
|
expect(turnEndReasons(sessionEvents).at(-1)).toBe('aborted')
|
|
// Composer recovered; no streaming node lingers. The host settled first
|
|
// (awaited above), but the abort frame reaches the browser over SSE — the
|
|
// frozen-partial swap is eventually consistent, so poll rather than count.
|
|
await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true)
|
|
await expect.poll(() => page.locator('[data-streaming="true"]').count(), { timeout: 10_000 }).toBe(0)
|
|
// Golden of the aborted end-state: the prompt bubble plus the frozen
|
|
// partial ('partial' is the hang entry's replayed prefix) and no more.
|
|
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
|
|
await compareOrRefreshGolden(CANCEL_EXPECTED, snapshot, MODE)
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
expect(tripwire.warnings).toEqual([])
|
|
}, 120_000)
|
|
|
|
it.skipIf(MODE === 'record')('surfaces a non-retryable AUTH failure without retrying', async () => {
|
|
await launch(() => ({
|
|
patches: [{ at: 0, entry: { kind: 'throw', chunks: [], message: 'invalid api key', code: 'AUTH' } }],
|
|
}))
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-error-auth'))
|
|
const { settled } = await sendPrompt()
|
|
await settled
|
|
expect(turnEndReasons(sessionEvents).at(-1)).toBe('error')
|
|
// AUTH is outside llm-retry's retryable set: no retry record.
|
|
expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBe(0)
|
|
// Product gap found by this lane, pinned as-is: the client consumes no
|
|
// agent/error frames and a pre-chunk failure freezes no partial, so THIS
|
|
// failure renders no error copy anywhere — the user sees the send simply
|
|
// stop. FIXME(web-error-surface): assert visible error text here once the
|
|
// web UI grows an error rendering; until then the pinned contract is
|
|
// "no crash, composer recovers, turn logged as error".
|
|
await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true)
|
|
expect(await page.locator('[data-streaming="true"]').count()).toBe(0)
|
|
// Golden of the same gap: the prompt bubble alone, no error copy in the
|
|
// tree — the diff that changes when web-error-surface lands.
|
|
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
|
|
await compareOrRefreshGolden(ERROR_EXPECTED, snapshot, MODE)
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
expect(tripwire.warnings).toEqual([])
|
|
}, 120_000)
|
|
|
|
it.skipIf(MODE === 'record')('recovers a transient SERVER failure through llm-retry and completes', async () => {
|
|
const derived = deriveReplayScript(parseSessionLog(await readFile(FIXTURE, 'utf8')))
|
|
expect(derived).toHaveLength(1)
|
|
await launch(() => ({
|
|
patches: [
|
|
{ at: 0, entry: { kind: 'throw', chunks: [], message: 'upstream 503', code: 'SERVER' } },
|
|
// Append the fixture's own success as the retry attempt — single-
|
|
// sourced from the recording, never copied into a committed sidecar.
|
|
{ at: 1, entry: derived[0]! },
|
|
],
|
|
}))
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-retry'))
|
|
// llm-retry backs off ~500ms before the second attempt.
|
|
const { settled } = await sendPrompt(60_000)
|
|
await settled
|
|
expect(turnEndReasons(sessionEvents).at(-1)).toBe('completed')
|
|
// The durable retry record proves the second attempt (request/header logs
|
|
// only on change, so attempt count is invisible there).
|
|
expect(sessionEvents.filter(e => e.type === 'llm/retry').length).toBeGreaterThanOrEqual(1)
|
|
await expect.poll(() => page.getByText('event sourcing', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThan(0)
|
|
// Golden of the recovered end-state: indistinguishable from a clean
|
|
// completion — retries are deliberately invisible in the transcript.
|
|
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold!.workspaceCwd)
|
|
await compareOrRefreshGolden(RETRY_EXPECTED, snapshot, MODE)
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
expect(tripwire.warnings).toEqual([])
|
|
}, 120_000)
|
|
|
|
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
|
|
await assertFixtureInventory(SNAPSHOT_DIR, [
|
|
'session.jsonl', 'cancel.expected.md', 'error-auth.expected.md', 'retry.expected.md',
|
|
])
|
|
})
|
|
})
|