// Opt-in browser benchmark for high-cardinality workspace and history // rendering. It reports measurements without timing assertions because host // speed is not a correctness contract; structural assertions keep the number // of workspaces and history entries from silently shrinking. import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { performance } from 'node:perf_hooks' import type { Browser, CDPSession, Locator, Page } from 'playwright' import { chromium } from 'playwright' import { afterAll, beforeAll, describe, expect, it } from 'vitest' import type { StreamChunk } from '@deepseek-ai/dsh-llm' import { CallId, createAssistantMessage, createToolResultMessage, createUserMessage, } from '@deepseek-ai/dsh-llm' import type { ReplayEntry, ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay' import type { SessionEvent } from '@deepseek-ai/dsh-session' import { SESSION_FORMAT_VERSION, Session, SessionId, } from '@deepseek-ai/dsh-session' // Carries the session/title event declaration into the fixture builder. import type {} from '@deepseek-ai/dsh-session-title' import { launchWebScaffold, seedSession, watchConsole, webSnapshotMode, type WebScaffold, } from './scaffold.ts' import { connectFreshWorkspace, newEnglishPage } from './support.ts' const SIDEBAR_SESSION_COUNT = 1_000 const LONG_SESSION_ID = 'perf-long-history' const LONG_SESSION_TITLE = 'LONG_PERF_SENTINEL 500-turn session' const LONG_HISTORY_TURNS = 500 const TOOL_TURN_INTERVAL = 10 const TOOLS_PER_TOOL_TURN = 10 const EXPECTED_TOOL_CALLS = LONG_HISTORY_TURNS / TOOL_TURN_INTERVAL * TOOLS_PER_TOOL_TURN const EXPECTED_TRAJECTORY_ROWS = 2_100 const DEFAULT_HISTORY_TURNS = 24 const PERF_REPLAY_CONTEXT_WINDOW = 10_000_000 const STREAM_PACE_MS = 8 const STREAM_DELTA_COUNT = 120 const COMPARISON_TURNS = 8 const COMPARISON_DELTA_COUNT = 24 const COMPARISON_TOOL_INTERVAL = 3 const SOAK_TURNS = 100 const POST_SOAK_RENDER_TURN = SOAK_TURNS + 1 const SOAK_DELTA_COUNT = 8 const SOAK_TOOL_INTERVAL = 10 const SOAK_CHECKPOINT_INTERVAL = 10 const LONG_CONTINUATION_USER_PREFIX = 'LONG_CONTINUATION_USER' const LONG_CONTINUATION_FIRST_PREFIX = 'LONG_CONTINUATION_FIRST' const LONG_CONTINUATION_DONE_PREFIX = 'LONG_CONTINUATION_DONE' const SOAK_USER_PREFIX = 'SOAK_CONVERSATION_USER' const SOAK_FIRST_PREFIX = 'SOAK_CONVERSATION_FIRST' const SOAK_DONE_PREFIX = 'SOAK_CONVERSATION_DONE' const LIVE_PROMPT_MARKER = 'STREAM_PERF_USER_INPUT' const STREAM_FIRST_MARKER = 'STREAM_PERF_FIRST' const STREAM_DONE_MARKER = 'STREAM_PERF_DONE' const LIVE_PROMPT = [ LIVE_PROMPT_MARKER, 'Analyze the following mixed-language project context and return a concise diagnostic.', ...Array.from( { length: 48 }, (_, index) => `Context ${String(index + 1).padStart(2, '0')}: 用户正在检查长会话中的增量渲染性能。` + ` Preserve item ${String(index)} and compare ${'payload'.repeat(8)}.`, ), '```ts', ...Array.from( { length: 40 }, (_, index) => `const sample_${String(index)} = ${JSON.stringify(`value-${String(index)}-${'x'.repeat(32)}`)}`, ), '```', ].join('\n') const STREAM_DELTAS = Array.from({ length: STREAM_DELTA_COUNT }, (_, index) => { if (index === 0) return `${STREAM_FIRST_MARKER} ` if (index === STREAM_DELTA_COUNT - 1) return `${STREAM_DONE_MARKER}.` return `chunk-${String(index).padStart(3, '0')} ${'response'.repeat(3)} ` }) interface ChromiumMetrics { readonly [name: string]: number } interface Measurement { readonly wallMs: number readonly taskMs: number readonly scriptMs: number readonly layoutMs: number readonly recalcStyleMs: number readonly devtoolsMs: number readonly nodesDelta: number readonly listenersDelta: number readonly heapDeltaMb: number readonly totalNodes: number readonly heapMb: number } interface MutationProbeResult { readonly batches: number readonly records: number } interface UserRenderProbeResult { readonly trustedClick: boolean readonly sendToDomMs: number readonly sendToPaintMs: number readonly domToPaintMs: number readonly mutationBatches: number readonly mutationRecords: number } interface RetainedBrowserState { readonly domElements: number readonly nodes: number readonly listeners: number readonly heapMb: number } interface ContinuedTurnReport { readonly ordinal: number readonly resultingTurns: number readonly kind: 'text' | 'tool' readonly promptChars: number readonly composerFill: Measurement readonly stream: MutationProbeResult & Measurement & { readonly paceMs: number readonly deltaChunks: number readonly persistedChunks: number readonly toolCalls: number readonly toolResults: number readonly clickToUserEchoMs: number readonly clickToFirstChunkMs: number readonly firstChunkToSettledMs: number } } interface ConversationTurnSpec { readonly prompt: string readonly deltas: readonly string[] readonly userMarker: string readonly firstMarker: string readonly doneMarker: string readonly toolResultMarker?: string } interface RetainedCheckpoint { readonly turns: number readonly state: RetainedBrowserState } interface ConversationReport { readonly startingTurns: number readonly turnsAdded: number readonly toolTurns: number readonly retainedBefore: RetainedBrowserState readonly retainedAfter: RetainedBrowserState readonly retainedDelta: RetainedBrowserState readonly checkpoints: readonly RetainedCheckpoint[] readonly turns: readonly ContinuedTurnReport[] } interface PerformanceWorld { readonly scaffold: WebScaffold readonly page: Page readonly tripwire: ReturnType readonly sessionEvents: SessionEvent[] readonly setupMs: number readonly replayDir?: string } interface PerformanceWorldOptions { readonly browser: Browser readonly replay?: ReplayOverrideDoc readonly sidebarSessions?: number readonly seedLongHistory?: boolean } function text(value: string): { type: 'text'; text: string }[] { return [{ type: 'text', text: value }] } function appendTitle(session: Session, title: string, messageSeq: number): void { session.append('session/title', { title, messageSeqs: [messageSeq], source: { kind: 'fallback' }, }) } function appendRequestHeader(session: Session, turn: number, step: number): void { session.append('request/header', { header: { config: { provider: 'deepseek-official', model: 'deepseek-v4-flash' }, system: `Synthetic performance system prompt for turn ${String(turn)}, step ${String(step)}.`, }, reason: turn === 1 && step === 1 ? 'initial' : 'change', }) } function appendAssistant( session: Session, turn: number, step: number, body: string, ): void { session.append('assistant/message', { turn, step, message: createAssistantMessage({ content: text(body), source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' }, }), usage: { inputTokens: 4_000 + turn * 10, outputTokens: 200 + step * 20, cacheReadTokens: turn % 2 === 0 ? 2_000 : 0, }, }, { surfaceOp: 'append' }) } function appendToolStep( session: Session, turn: number, step: number, toolCount: number, ): void { const calls = Array.from({ length: toolCount }, (_, index) => { const callId = CallId(`perf-call-${String(turn)}-${String(index)}`) const args = JSON.stringify({ turn, index, payload: 'x'.repeat(120), }) return { callId, index, args } }) session.append('assistant/message', { turn, step, message: createAssistantMessage({ content: [ { type: 'reasoning', text: `Dispatching ${String(toolCount)} synthetic tools for turn ${String(turn)}.`, }, ...calls.map(({ callId, args }) => ({ type: 'tool-call' as const, id: callId, name: 'synthetic_tool', arguments: args, })), ], source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' }, }), usage: { inputTokens: 6_000 + turn * 10, outputTokens: 500, cacheReadTokens: 3_000, reasoningTokens: 50, }, }, { surfaceOp: 'append' }) const callEvents = calls.map(({ callId, args }) => session.append('tool/call', { turn, step, callId, name: 'synthetic_tool', arguments: args, })) for (const [index, call] of calls.entries()) { const source = callEvents[index] if (source === undefined) throw new Error(`missing synthetic tool call ${String(index)}`) session.append('tool/result', { turn, step, message: createToolResultMessage({ callId: call.callId, content: text( `synthetic result turn=${String(turn)} index=${String(call.index)} ${'r'.repeat(400)}`, ), isError: false, }), }, { surfaceOp: 'append', sourceEventSeqs: [source.seq] }) } } function fencedCode(turn: number): string { if (turn % 25 !== 0) return '' const lines = Array.from( { length: 80 }, (_, index) => `const value_${String(index)} = ${String(turn + index)}`, ) return `\n\n\`\`\`ts\n${lines.join('\n')}\n\`\`\`` } function fixtureLog(session: Session): string { const header = { type: 'session', version: SESSION_FORMAT_VERSION, id: '{{sessionId}}', createdAt: Date.now() - 60_000, cwd: '{{cwd}}', } return [ JSON.stringify(header), ...session.events.map(event => JSON.stringify(event)), '', ].join('\n') } function smallSidebarFixture(): string { const session = Session.create(SessionId('perf-small-template')) session.append('turn/start', { turn: 1, }) const user = session.append('user/message', createUserMessage({ content: text('Inspect this compact synthetic session.'), source: { kind: 'user' }, }), { surfaceOp: 'append' }) appendTitle(session, 'Synthetic sidebar session', user.seq) session.append('step/start', { turn: 1, step: 1 }) appendRequestHeader(session, 1, 1) appendToolStep(session, 1, 1, 2) session.append('step/end', { turn: 1, step: 1 }) session.append('step/start', { turn: 1, step: 2 }) appendRequestHeader(session, 1, 2) appendAssistant(session, 1, 2, 'Synthetic sidebar fixture complete.') session.append('step/end', { turn: 1, step: 2 }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) return fixtureLog(session) } function longHistoryFixture(): string { const session = Session.create(SessionId(LONG_SESSION_ID)) for (let turn = 1; turn <= LONG_HISTORY_TURNS; turn += 1) { session.append('turn/start', { turn, }) const user = session.append('user/message', createUserMessage({ content: text( `LONG_PERF_SENTINEL turn ${String(turn)}: analyze payload ${'u'.repeat(200)}`, ), source: { kind: 'user' }, }), { surfaceOp: 'append' }) if (turn === 1) appendTitle(session, LONG_SESSION_TITLE, user.seq) session.append('step/start', { turn, step: 1 }) appendRequestHeader(session, turn, 1) if (turn % TOOL_TURN_INTERVAL === 0) { appendToolStep(session, turn, 1, TOOLS_PER_TOOL_TURN) session.append('step/end', { turn, step: 1 }) session.append('step/start', { turn, step: 2 }) appendRequestHeader(session, turn, 2) appendAssistant( session, turn, 2, `All synthetic tools completed for turn ${String(turn)}. ${'z'.repeat(320)}${fencedCode(turn)}`, ) session.append('step/end', { turn, step: 2 }) } else { appendAssistant( session, turn, 1, `Synthetic assistant response for turn ${String(turn)}. ${'a'.repeat(320)}${fencedCode(turn)}`, ) session.append('step/end', { turn, step: 1 }) } session.append('turn/end', { turn, reason: { kind: 'completed' } }) } return fixtureLog(session) } function textStream(deltas: readonly string[], inputTokens: number): StreamChunk[] { const response = deltas.join('') return [ { type: 'block-start', index: 0, blockType: 'text' }, ...deltas.map(text => ({ type: 'text-delta' as const, index: 0, text, })), { type: 'block-end', index: 0, block: { type: 'text', text: response }, }, { type: 'usage', usage: { inputTokens, outputTokens: Math.ceil(response.length / 4), }, }, { type: 'finish', reason: { kind: 'stop' } }, ] } function comparisonPrompt(index: number): string { if (index === COMPARISON_TURNS) return LIVE_PROMPT return (`${LONG_CONTINUATION_USER_PREFIX}_${String(index)} ` + `继续分析这个长会话的第 ${String(index)} 个增量问题,并保留当前滚动和输入响应。 ` + 'context '.repeat(80)).trimEnd() } function comparisonDeltas(index: number): string[] { if (index === COMPARISON_TURNS) return STREAM_DELTAS return Array.from({ length: COMPARISON_DELTA_COUNT }, (_, chunkIndex) => { if (chunkIndex === 0) return `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)} ` if (chunkIndex === COMPARISON_DELTA_COUNT - 1) { return `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}.` } return `turn-${String(index)}-chunk-${String(chunkIndex).padStart(2, '0')} ${'response'.repeat(2)} ` }) } function comparisonTurn(index: number): ConversationTurnSpec { const toolResultMarker = index % COMPARISON_TOOL_INTERVAL === 0 ? `LONG_CONTINUATION_TOOL_RESULT_${String(index)}` : undefined return { prompt: comparisonPrompt(index), deltas: comparisonDeltas(index), userMarker: index === COMPARISON_TURNS ? LIVE_PROMPT_MARKER : `${LONG_CONTINUATION_USER_PREFIX}_${String(index)}`, firstMarker: index === COMPARISON_TURNS ? STREAM_FIRST_MARKER : `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)}`, doneMarker: index === COMPARISON_TURNS ? STREAM_DONE_MARKER : `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}`, ...toolResultMarker === undefined ? {} : { toolResultMarker }, } } function soakTurn(index: number): ConversationTurnSpec { const suffix = String(index).padStart(3, '0') const toolResultMarker = index % SOAK_TOOL_INTERVAL === 0 ? `SOAK_CONVERSATION_TOOL_RESULT_${suffix}` : undefined return { prompt: (`${SOAK_USER_PREFIX}_${suffix} ` + `持续对话第 ${String(index)} 轮,检查增量渲染与保留状态。 ` + 'context '.repeat(20)).trimEnd(), deltas: Array.from({ length: SOAK_DELTA_COUNT }, (_, chunkIndex) => { if (chunkIndex === 0) return `${SOAK_FIRST_PREFIX}_${suffix} ` if (chunkIndex === SOAK_DELTA_COUNT - 1) return `${SOAK_DONE_PREFIX}_${suffix}.` return `soak-${suffix}-${String(chunkIndex).padStart(2, '0')} response ` }), userMarker: `${SOAK_USER_PREFIX}_${suffix}`, firstMarker: `${SOAK_FIRST_PREFIX}_${suffix}`, doneMarker: `${SOAK_DONE_PREFIX}_${suffix}`, ...toolResultMarker === undefined ? {} : { toolResultMarker }, } } function toolStream(index: number, marker: string): StreamChunk[] { const callId = CallId(`performance-tool-${marker.toLowerCase()}-${String(index)}`) const args = JSON.stringify({ command: `printf '${marker}\\n'`, description: `Emit performance marker ${String(index)}`, }) return [ { type: 'block-start', index: 0, blockType: 'tool-call' }, { type: 'tool-call-delta', index: 0, id: callId, name: 'bash', argumentsDelta: args, }, { type: 'block-end', index: 0, block: { type: 'tool-call', id: callId, name: 'bash', arguments: args }, }, { type: 'usage', usage: { inputTokens: 256, outputTokens: 32 } }, { type: 'finish', reason: { kind: 'tool-calls' } }, ] } function performanceReplayOverride( turnCount: number, turnSpec: (index: number) => ConversationTurnSpec, ): ReplayOverrideDoc { const continuationEntries = Array.from( { length: turnCount }, (_, offset): ReplayEntry[] => { const index = offset + 1 const spec = turnSpec(index) const finalResponse: ReplayEntry = { kind: 'chunks', chunks: textStream(spec.deltas, Math.ceil(spec.prompt.length / 4)), } return spec.toolResultMarker === undefined ? [finalResponse] : [{ kind: 'chunks', chunks: toolStream(index, spec.toolResultMarker) }, finalResponse] }, ).flat() return continuationEntries } function rounded(value: number): number { return Math.round(value * 1_000) / 1_000 } async function chromiumMetrics(cdp: CDPSession): Promise { const payload = await cdp.send('Performance.getMetrics') return Object.fromEntries(payload.metrics.map(metric => [metric.name, metric.value])) } async function retainedBrowserState( cdp: CDPSession, page: Page, ): Promise { await cdp.send('HeapProfiler.collectGarbage') const metrics = await chromiumMetrics(cdp) return { domElements: await page.evaluate(() => document.querySelectorAll('*').length), nodes: requiredMetric(metrics, 'Nodes'), listeners: requiredMetric(metrics, 'JSEventListeners'), heapMb: rounded(requiredMetric(metrics, 'JSHeapUsedSize') / 1_048_576), } } function requiredMetric(metrics: ChromiumMetrics, name: string): number { const value = metrics[name] if (value === undefined) throw new Error(`Chromium performance metric ${name} is unavailable`) return value } function metricDelta( before: ChromiumMetrics, after: ChromiumMetrics, wallMs: number, ): Measurement { return { wallMs: rounded(wallMs), taskMs: rounded((requiredMetric(after, 'TaskDuration') - requiredMetric(before, 'TaskDuration')) * 1_000), scriptMs: rounded((requiredMetric(after, 'ScriptDuration') - requiredMetric(before, 'ScriptDuration')) * 1_000), layoutMs: rounded((requiredMetric(after, 'LayoutDuration') - requiredMetric(before, 'LayoutDuration')) * 1_000), recalcStyleMs: rounded( (requiredMetric(after, 'RecalcStyleDuration') - requiredMetric(before, 'RecalcStyleDuration')) * 1_000, ), devtoolsMs: rounded( (requiredMetric(after, 'DevToolsCommandDuration') - requiredMetric(before, 'DevToolsCommandDuration')) * 1_000, ), nodesDelta: requiredMetric(after, 'Nodes') - requiredMetric(before, 'Nodes'), listenersDelta: requiredMetric(after, 'JSEventListeners') - requiredMetric(before, 'JSEventListeners'), heapDeltaMb: rounded( (requiredMetric(after, 'JSHeapUsedSize') - requiredMetric(before, 'JSHeapUsedSize')) / 1_048_576, ), totalNodes: requiredMetric(after, 'Nodes'), heapMb: rounded(requiredMetric(after, 'JSHeapUsedSize') / 1_048_576), } } async function measure( cdp: CDPSession, action: () => Promise, ): Promise<{ measurement: Measurement; value: T }> { const before = await chromiumMetrics(cdp) const started = performance.now() const value = await action() const wallMs = performance.now() - started const after = await chromiumMetrics(cdp) return { measurement: metricDelta(before, after, wallMs), value } } async function startMutationProbe(page: Page): Promise { await page.evaluate(() => { const target = document.querySelector('[class*="centerCol"]') if (target === null) throw new Error('stream mutation probe target is unavailable') const probe = { batches: 0, records: 0, observer: undefined as MutationObserver | undefined, } const observer = new MutationObserver((records) => { probe.batches += 1 probe.records += records.length }) probe.observer = observer observer.observe(target, { attributes: true, attributeFilter: ['data-streaming'], characterData: true, childList: true, subtree: true, }) Reflect.set(globalThis, '__dshPerfMutationProbe', probe) }) } async function stopMutationProbe(page: Page): Promise { return page.evaluate(() => { const probe = Reflect.get(globalThis, '__dshPerfMutationProbe') as | { batches: number; records: number; observer: MutationObserver } | undefined if (probe === undefined) throw new Error('stream mutation probe was not started') probe.observer.disconnect() Reflect.deleteProperty(globalThis, '__dshPerfMutationProbe') return { batches: probe.batches, records: probe.records } }) } async function startUserRenderProbe( page: Page, marker: string, ): Promise { const send = page.getByRole('button', { name: 'Send message', exact: true }) await send.waitFor({ timeout: 15_000 }) await expect.poll(() => send.isEnabled(), { timeout: 15_000 }).toBe(true) await page.evaluate((expectedMarker) => { const target = document.querySelector('[data-conversation-scroll]') if (target === null) throw new Error('user render probe target is unavailable') const probe: { sendAt?: number domAt?: number paintAt?: number trustedClick?: boolean batches: number records: number observer?: MutationObserver pollForMarker?: () => void } = { batches: 0, records: 0 } const transcriptContainsMarker = (): boolean => { // The composer projects the draft into a backdrop and hidden sizing // mirror; only a matching text node outside that card proves delivery. const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT) for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) { if (!node.textContent?.includes(expectedMarker)) continue const parent = node.parentElement if (parent !== null && parent.closest('[data-composer-card]') === null) return true } return false } const markDomVisible = (): void => { if ( probe.sendAt === undefined || probe.domAt !== undefined || !transcriptContainsMarker() ) return probe.domAt = globalThis.performance.now() // The second callback runs only after a render opportunity for the DOM // insertion observed before the first callback. requestAnimationFrame(() => { requestAnimationFrame(() => { probe.paintAt = globalThis.performance.now() observer.disconnect() }) }) } const pollForMarker = (): void => { markDomVisible() if (probe.domAt === undefined) requestAnimationFrame(pollForMarker) } const observer = new MutationObserver((records) => { if (probe.sendAt === undefined) return probe.batches += 1 probe.records += records.length markDomVisible() }) probe.observer = observer probe.pollForMarker = pollForMarker observer.observe(target, { attributes: true, attributeFilter: ['data-streaming'], characterData: true, childList: true, subtree: true, }) Reflect.set(globalThis, '__dshPerfUserRenderProbe', probe) }, marker) await send.evaluate((button) => { button.addEventListener('click', (event) => { const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as | { sendAt?: number; trustedClick?: boolean; pollForMarker?: () => void } | undefined if (probe?.pollForMarker === undefined) throw new Error('user render probe was not started') probe.trustedClick = event.isTrusted probe.sendAt = globalThis.performance.now() requestAnimationFrame(probe.pollForMarker) }, { capture: true, once: true }) }) } async function triggerUserRenderProbe(page: Page): Promise { const send = page.getByRole('button', { name: 'Send message', exact: true }) await send.click() } async function stopUserRenderProbe( page: Page, marker: string, ): Promise { try { await page.waitForFunction(() => { const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as | { paintAt?: number } | undefined return probe?.paintAt !== undefined }, undefined, { timeout: 15_000 }) } catch (error) { const diagnostic = await page.evaluate((expectedMarker) => { const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as | { sendAt?: number domAt?: number paintAt?: number trustedClick?: boolean batches: number records: number } | undefined const target = document.querySelector('[data-conversation-scroll]') let markerOutsideComposer = false if (target !== null) { const walker = document.createTreeWalker(target, NodeFilter.SHOW_TEXT) for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) { if (!node.textContent?.includes(expectedMarker)) continue const parent = node.parentElement if (parent !== null && parent.closest('[data-composer-card]') === null) { markerOutsideComposer = true break } } } return { probe, markerOutsideComposer, markerInComposer: document.querySelector('[data-composer-card]') ?.textContent?.includes(expectedMarker) ?? false, } }, marker) throw new Error(`user render probe timed out: ${JSON.stringify(diagnostic)}`, { cause: error }) } return page.evaluate(() => { const probe = Reflect.get(globalThis, '__dshPerfUserRenderProbe') as | { sendAt?: number domAt?: number paintAt?: number trustedClick?: boolean batches: number records: number } | undefined Reflect.deleteProperty(globalThis, '__dshPerfUserRenderProbe') if ( probe?.trustedClick !== true || probe.sendAt === undefined || probe.domAt === undefined || probe.paintAt === undefined ) { throw new Error('user render probe did not observe a trusted click, DOM insertion, and paint') } return { trustedClick: probe.trustedClick, sendToDomMs: probe.domAt - probe.sendAt, sendToPaintMs: probe.paintAt - probe.sendAt, domToPaintMs: probe.paintAt - probe.domAt, mutationBatches: probe.batches, mutationRecords: probe.records, } }) } async function stableCount( locator: Locator, accepts: (count: number) => boolean, timeoutMs = 60_000, ): Promise { const deadline = performance.now() + timeoutMs let previous = -1 let stableReads = 0 while (performance.now() < deadline) { const count = await locator.count() stableReads = accepts(count) && count === previous ? stableReads + 1 : 0 if (stableReads >= 4) return count previous = count await new Promise(resolve => setTimeout(resolve, 50)) } throw new Error(`browser row count did not stabilize; last count ${String(previous)}`) } async function conversationTurns(page: Page): Promise { const stats = page.getByText(/\d+ turns · \d+ steps/, { exact: true }).last() await stats.waitFor({ timeout: 15_000 }) const value = await stats.textContent() const match = value?.match(/^(\d+) turns · \d+ steps$/) if (match?.[1] === undefined) throw new Error(`unexpected conversation stats ${JSON.stringify(value)}`) return Number(match[1]) } function retainedDelta( before: RetainedBrowserState, after: RetainedBrowserState, ): RetainedBrowserState { return { domElements: after.domElements - before.domElements, nodes: after.nodes - before.nodes, listeners: after.listeners - before.listeners, heapMb: rounded(after.heapMb - before.heapMb), } } async function launchPerformanceWorld( options: PerformanceWorldOptions, ): Promise { const setupStarted = performance.now() let replayDir: string | undefined let scaffold: WebScaffold | undefined let page: Page | undefined try { if (options.replay === undefined) { scaffold = await launchWebScaffold() } else { replayDir = await mkdtemp(join(tmpdir(), 'dsh-web-perf-replay-')) const replayOverride = join(replayDir, 'replay.override.json') await writeFile(replayOverride, JSON.stringify(options.replay)) scaffold = await launchWebScaffold({ // The override-only fixture supplies one positional script for this // world's single live session. replayFixture: join(replayDir, 'override-only.jsonl'), replayOverride, paceMs: STREAM_PACE_MS, replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW, }) } const sessionEvents: SessionEvent[] = [] scaffold.ctx.on('session/event', (_session, event: SessionEvent) => { sessionEvents.push(event) }) if ((options.sidebarSessions ?? 0) > 0) { const small = smallSidebarFixture() for (let index = 0; index < (options.sidebarSessions ?? 0); index += 1) { await seedSession(scaffold, small, `perf-sidebar-${String(index).padStart(4, '0')}`) } } if (options.seedLongHistory === true) { await seedSession(scaffold, longHistoryFixture(), LONG_SESSION_ID) } const setupMs = performance.now() - setupStarted page = await newEnglishPage(options.browser) return { scaffold, page, tripwire: watchConsole(page), sessionEvents, setupMs, ...replayDir === undefined ? {} : { replayDir }, } } catch (error) { const failures: unknown[] = [error] if (page !== undefined) { try { await page.close() } catch (cleanupError) { failures.push(cleanupError) } } if (scaffold !== undefined) { try { await scaffold.close() } catch (cleanupError) { failures.push(cleanupError) } } if (replayDir !== undefined) { try { await rm(replayDir, { recursive: true, force: true }) } catch (cleanupError) { failures.push(cleanupError) } } if (failures.length === 1) throw error throw new AggregateError(failures, 'web performance setup and cleanup failed') } } async function closePerformanceWorld(world: PerformanceWorld): Promise { const failures: unknown[] = [] await world.page.close().catch((error: unknown) => failures.push(error)) await world.scaffold.close().catch((error: unknown) => failures.push(error)) if (world.replayDir !== undefined) { await rm(world.replayDir, { recursive: true, force: true }) .catch((error: unknown) => failures.push(error)) } if (failures.length === 1) throw failures[0] if (failures.length > 1) throw new AggregateError(failures, 'web performance teardown failed') } async function openPerformancePage( world: PerformanceWorld, expectedSessions: number, ): Promise { await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' }) await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 }) const group = world.page.getByRole('treeitem').first() await expect.poll(() => group.textContent(), { timeout: 30_000 }) .toContain(`${String(expectedSessions)} ${expectedSessions === 1 ? 'session' : 'sessions'}`) return group } async function openLongHistory(page: Page): Promise { await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true }) .fill('LONG_PERF_SENTINEL') const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem') await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1) await results.first().click() await page.getByRole('tab', { name: 'Chat', exact: true }).waitFor({ timeout: 30_000 }) return conversationTurns(page) } async function continueConversation( world: PerformanceWorld, cdp: CDPSession, options: { readonly startingTurns: number readonly turnCount: number readonly turnSpec: (index: number) => ConversationTurnSpec readonly expectedSessionId?: SessionId readonly checkpointInterval?: number }, ): Promise { const composer = world.page.locator('textarea:enabled').last() await composer.waitFor({ timeout: 15_000 }) const retainedBefore = await retainedBrowserState(cdp, world.page) const checkpoints: RetainedCheckpoint[] = [{ turns: options.startingTurns, state: retainedBefore }] const turns: ContinuedTurnReport[] = [] let liveSessionId = options.expectedSessionId for (let index = 1; index <= options.turnCount; index += 1) { const spec = options.turnSpec(index) const composerFill = await measure(cdp, async () => { await composer.fill(spec.prompt) await expect.poll(() => composer.inputValue()).toBe(spec.prompt) return (await composer.inputValue()).length }) expect(composerFill.value).toBe(spec.prompt.length) const eventStart = world.sessionEvents.length await startMutationProbe(world.page) const streamBefore = await chromiumMetrics(cdp) const streamStarted = performance.now() const settled = world.scaffold.whenTurnSettled(60_000) await world.page.getByRole('button', { name: 'Send message', exact: true }).click() await world.page.getByText(spec.userMarker, { exact: false }).last().waitFor({ timeout: 15_000 }) const clickToUserEchoMs = performance.now() - streamStarted await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 }) const clickToFirstChunkMs = performance.now() - streamStarted const settledSessionId = await settled if (liveSessionId === undefined) { liveSessionId = settledSessionId } else { expect(settledSessionId).toBe(liveSessionId) } await expect.poll( () => world.page.locator('[data-streaming="true"]').count(), { timeout: 15_000 }, ).toBe(0) await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 }) const clickToSettledMs = performance.now() - streamStarted const streamAfter = await chromiumMetrics(cdp) const mutations = await stopMutationProbe(world.page) const turnEvents = world.sessionEvents.slice(eventStart) const chunks = turnEvents.filter(event => event.type === 'assistant/chunk') const toolCalls = turnEvents.filter(event => event.type === 'tool/call') const toolResults = turnEvents.filter(event => event.type === 'tool/result') const toolTurn = spec.toolResultMarker !== undefined expect(chunks).toHaveLength(spec.deltas.length + (toolTurn ? 9 : 4)) expect(toolCalls).toHaveLength(toolTurn ? 1 : 0) expect(toolResults).toHaveLength(toolTurn ? 1 : 0) if (spec.toolResultMarker !== undefined) { expect(toolCalls[0]?.data.name).toBe('bash') const toolResult = toolResults[0] if (toolResult?.type !== 'tool/result') { throw new Error(`continued turn ${String(index)} did not log its tool result`) } const resultBlock = toolResult.data.message.content.find(block => block.type === 'tool-result') expect(resultBlock?.isError).toBe(false) expect(resultBlock?.content .filter(block => block.type === 'text') .map(block => block.text) .join('')).toContain(spec.toolResultMarker) } const user = turnEvents.find( event => event.type === 'user/message' && event.data.source.kind === 'user', ) if (user?.type !== 'user/message') { throw new Error(`continued turn ${String(index)} did not log its user message`) } expect(user.data.content .filter(block => block.type === 'text') .map(block => block.text) .join('')).toBe(spec.prompt) const resultingTurns = await conversationTurns(world.page) expect(resultingTurns).toBe(options.startingTurns + index) turns.push({ ordinal: index, resultingTurns, kind: toolTurn ? 'tool' : 'text', promptChars: spec.prompt.length, composerFill: composerFill.measurement, stream: { paceMs: STREAM_PACE_MS, deltaChunks: spec.deltas.length, persistedChunks: chunks.length, toolCalls: toolCalls.length, toolResults: toolResults.length, clickToUserEchoMs: rounded(clickToUserEchoMs), clickToFirstChunkMs: rounded(clickToFirstChunkMs), firstChunkToSettledMs: rounded(clickToSettledMs - clickToFirstChunkMs), ...mutations, ...metricDelta(streamBefore, streamAfter, clickToSettledMs), }, }) if (options.checkpointInterval !== undefined && index % options.checkpointInterval === 0) { checkpoints.push({ turns: options.startingTurns + index, state: await retainedBrowserState(cdp, world.page), }) } } const lastCheckpoint = checkpoints.at(-1) const retainedAfter = lastCheckpoint?.turns === options.startingTurns + options.turnCount ? lastCheckpoint.state : await retainedBrowserState(cdp, world.page) if (lastCheckpoint?.turns !== options.startingTurns + options.turnCount) { checkpoints.push({ turns: options.startingTurns + options.turnCount, state: retainedAfter }) } return { startingTurns: options.startingTurns, turnsAdded: options.turnCount, toolTurns: turns.filter(turn => turn.kind === 'tool').length, retainedBefore, retainedAfter, retainedDelta: retainedDelta(retainedBefore, retainedAfter), checkpoints, turns, } } async function measurePostSoakUserRender( world: PerformanceWorld, cdp: CDPSession, ): Promise { const spec = soakTurn(POST_SOAK_RENDER_TURN) if (spec.toolResultMarker !== undefined) { throw new Error('post-soak render probe must remain a text-only turn') } const composer = world.page.locator('textarea:enabled').last() const composerFill = await measure(cdp, async () => { await composer.fill(spec.prompt) await expect.poll(() => composer.inputValue()).toBe(spec.prompt) return (await composer.inputValue()).length }) expect(composerFill.value).toBe(spec.prompt.length) const eventStart = world.sessionEvents.length await startUserRenderProbe(world.page, spec.userMarker) const browserBefore = await chromiumMetrics(cdp) const fullTurnStarted = performance.now() const settled = world.scaffold.whenTurnSettled(60_000) await triggerUserRenderProbe(world.page) const browserTiming = await stopUserRenderProbe(world.page, spec.userMarker) const browserAfter = await chromiumMetrics(cdp) const browserAfterPaintSample = metricDelta( browserBefore, browserAfter, performance.now() - fullTurnStarted, ) await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 }) const settledSessionId = await settled await expect.poll( () => world.page.locator('[data-streaming="true"]').count(), { timeout: 15_000 }, ).toBe(0) await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 }) const fullTurnMs = performance.now() - fullTurnStarted const turnEvents = world.sessionEvents.slice(eventStart) const chunks = turnEvents.filter(event => event.type === 'assistant/chunk') const user = turnEvents.find( event => event.type === 'user/message' && event.data.source.kind === 'user', ) if (user?.type !== 'user/message') throw new Error('post-soak render probe did not log its user message') expect(user.data.content .filter(block => block.type === 'text') .map(block => block.text) .join('')).toBe(spec.prompt) expect(chunks).toHaveLength(spec.deltas.length + 4) expect(turnEvents.filter(event => event.type === 'tool/call')).toHaveLength(0) expect(turnEvents.filter(event => event.type === 'tool/result')).toHaveLength(0) expect(world.scaffold.ctx.agents.get(settledSessionId)).toBeDefined() expect(await conversationTurns(world.page)).toBe(POST_SOAK_RENDER_TURN) return { ordinal: POST_SOAK_RENDER_TURN, resultingTurns: POST_SOAK_RENDER_TURN, promptChars: spec.prompt.length, composerFill: composerFill.measurement, trustedClick: browserTiming.trustedClick, sendToDomMs: rounded(browserTiming.sendToDomMs), sendToPaintMs: rounded(browserTiming.sendToPaintMs), domToPaintMs: rounded(browserTiming.domToPaintMs), mutationBatchesThroughPaint: browserTiming.mutationBatches, mutationRecordsThroughPaint: browserTiming.mutationRecords, browserAfterPaintSample, fullTurnMs: rounded(fullTurnMs), persistedChunks: chunks.length, } } function average(values: readonly number[]): number { return rounded(values.reduce((sum, value) => sum + value, 0) / values.length) } function p95(values: readonly number[]): number { const sorted = [...values].sort((left, right) => left - right) return sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] ?? 0 } function summarizeTurnWindows( turns: readonly ContinuedTurnReport[], windowSize: number, ): object[] { const windows: object[] = [] for (let start = 0; start < turns.length; start += windowSize) { const window = turns.slice(start, start + windowSize) const streamWall = window.map(turn => turn.stream.wallMs) const userEcho = window.map(turn => turn.stream.clickToUserEchoMs) const firstChunk = window.map(turn => turn.stream.clickToFirstChunkMs) windows.push({ turns: `${String(start + 1)}-${String(start + window.length)}`, toolTurns: window.filter(turn => turn.kind === 'tool').length, average: { composerFillWallMs: average(window.map(turn => turn.composerFill.wallMs)), streamWallMs: average(streamWall), clickToUserEchoMs: average(userEcho), clickToFirstChunkMs: average(firstChunk), taskMs: average(window.map(turn => turn.stream.taskMs)), scriptMs: average(window.map(turn => turn.stream.scriptMs)), recalcStyleMs: average(window.map(turn => turn.stream.recalcStyleMs)), mutationBatches: average(window.map(turn => turn.stream.batches)), }, p95: { streamWallMs: rounded(p95(streamWall)), clickToUserEchoMs: rounded(p95(userEcho)), clickToFirstChunkMs: rounded(p95(firstChunk)), }, }) } return windows } describe('manual web performance: complex workspace and history', () => { let browser: Browser beforeAll(async () => { if (webSnapshotMode() === 'record') { throw new Error('manual web performance runs only with deterministic replay') } browser = await chromium.launch() }) afterAll(async () => { await browser?.close() }) it('reports workspace, history, and trajectory cardinality costs', async () => { const world = await launchPerformanceWorld({ browser, sidebarSessions: SIDEBAR_SESSION_COUNT, seedLongHistory: true, }) try { const bootStarted = performance.now() const group = await openPerformancePage(world, SIDEBAR_SESSION_COUNT + 1) const bootReadyMs = performance.now() - bootStarted const page = world.page const cdp = await page.context().newCDPSession(page) await cdp.send('Performance.enable') const firstContentfulPaintMs = await page.evaluate( () => globalThis.performance.getEntriesByName('first-contentful-paint')[0]?.startTime, ) const sidebar = await measure(cdp, async () => { await group.click() return stableCount( page.getByRole('treeitem'), count => count === SIDEBAR_SESSION_COUNT + 2, ) }) expect(sidebar.value).toBe(SIDEBAR_SESSION_COUNT + 2) await group.click() await expect.poll(() => page.getByRole('treeitem').count()).toBe(1) const contentSearch = await measure(cdp, async () => { await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true }) .fill('LONG_PERF_SENTINEL') const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem') await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1) await results.first().waitFor({ timeout: 60_000 }) return results.first() }) const opened = await measure(cdp, async () => { await contentSearch.value.click() await page.getByRole('tab', { name: 'Trajectory', exact: true }).waitFor({ timeout: 30_000 }) return conversationTurns(page) }) expect(opened.value).toBe(DEFAULT_HISTORY_TURNS) const trajectoryRows = page.getByRole('row') const coldTrajectory = await measure(cdp, async () => { await page.getByRole('tab', { name: 'Trajectory', exact: true }).click() return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS) }) expect(coldTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS) const collapseTurns = await measure(cdp, async () => { await page.getByRole('button', { name: 'Collapse turns', exact: true }).click() return stableCount(trajectoryRows, count => count > 0 && count < EXPECTED_TRAJECTORY_ROWS) }) expect(collapseTurns.value).toBeLessThan(EXPECTED_TRAJECTORY_ROWS) const trajectorySearch = await measure(cdp, async () => { await page.getByRole('searchbox', { name: 'Search trajectory', exact: true }).fill('turn 499') return stableCount(trajectoryRows, count => count > 0 && count < 20) }) expect(trajectorySearch.value).toBeLessThan(20) await page.getByRole('tab', { name: 'Chat', exact: true }).click() const historyPages: { turns: number; measurement: Measurement }[] = [] let turns = await conversationTurns(page) while (turns < LONG_HISTORY_TURNS) { const previousTurns = turns const older = await measure(cdp, async () => { await page.getByRole('button', { name: 'Load earlier', exact: true }).click() await expect.poll(() => conversationTurns(page), { timeout: 30_000 }) .toBeGreaterThan(previousTurns) return conversationTurns(page) }) turns = older.value historyPages.push({ turns, measurement: older.measurement }) } const warmTrajectory = await measure(cdp, async () => { await page.getByRole('tab', { name: 'Trajectory', exact: true }).click() return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS) }) expect(warmTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS) const warmConversation = await measure(cdp, async () => { await page.getByRole('tab', { name: 'Chat', exact: true }).click() return conversationTurns(page) }) expect(warmConversation.value).toBe(LONG_HISTORY_TURNS) console.info(`WEB_PERF_RESULT ${JSON.stringify({ scenario: 'workspace-history-trajectory', fixture: { sidebarSessions: SIDEBAR_SESSION_COUNT, totalSessions: SIDEBAR_SESSION_COUNT + 1, longHistoryTurns: LONG_HISTORY_TURNS, toolCalls: EXPECTED_TOOL_CALLS, trajectoryRows: EXPECTED_TRAJECTORY_ROWS, }, setupMs: rounded(world.setupMs), boot: { readyMs: rounded(bootReadyMs), firstContentfulPaintMs: firstContentfulPaintMs === undefined ? null : rounded(firstContentfulPaintMs), }, sidebarExpand: sidebar.measurement, contentSearch: contentSearch.measurement, openLongHistory: { initialTurns: opened.value, ...opened.measurement }, coldTrajectory: { rows: coldTrajectory.value, ...coldTrajectory.measurement }, collapseTurns: { rows: collapseTurns.value, ...collapseTurns.measurement }, trajectorySearch: { rows: trajectorySearch.value, ...trajectorySearch.measurement }, historyPages, warmTrajectory: { rows: warmTrajectory.value, ...warmTrajectory.measurement }, warmConversation: { turns: warmConversation.value, ...warmConversation.measurement }, }, null, 2)}`) expect(world.tripwire.warnings).toEqual([]) expect(world.tripwire.pageErrors).toEqual([]) } finally { await closePerformanceWorld(world) } }) it('reports default 24-turn history plus eight continued turns', async () => { const world = await launchPerformanceWorld({ browser, replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn), seedLongHistory: true, }) try { await openPerformancePage(world, 1) const cdp = await world.page.context().newCDPSession(world.page) await cdp.send('Performance.enable') const opened = await measure(cdp, () => openLongHistory(world.page)) expect(opened.value).toBe(DEFAULT_HISTORY_TURNS) const conversation = await continueConversation(world, cdp, { startingTurns: DEFAULT_HISTORY_TURNS, turnCount: COMPARISON_TURNS, turnSpec: comparisonTurn, expectedSessionId: SessionId(LONG_SESSION_ID), }) expect(conversation.toolTurns).toBe(2) console.info(`WEB_PERF_RESULT ${JSON.stringify({ scenario: 'default-resume-24-plus-8', setupMs: rounded(world.setupMs), replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW, openLongHistory: { turns: opened.value, ...opened.measurement }, conversation, }, null, 2)}`) expect(world.tripwire.warnings).toEqual([]) expect(world.tripwire.pageErrors).toEqual([]) } finally { await closePerformanceWorld(world) } }) it('reports fully expanded 500-turn history plus eight continued turns', async () => { const world = await launchPerformanceWorld({ browser, replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn), seedLongHistory: true, }) try { await openPerformancePage(world, 1) const cdp = await world.page.context().newCDPSession(world.page) await cdp.send('Performance.enable') expect(await openLongHistory(world.page)).toBe(DEFAULT_HISTORY_TURNS) const historyPages: { turns: number; measurement: Measurement }[] = [] let turns = DEFAULT_HISTORY_TURNS while (turns < LONG_HISTORY_TURNS) { const previousTurns = turns const older = await measure(cdp, async () => { await world.page.getByRole('button', { name: 'Load earlier', exact: true }).click() await expect.poll(() => conversationTurns(world.page), { timeout: 30_000 }) .toBeGreaterThan(previousTurns) return conversationTurns(world.page) }) turns = older.value historyPages.push({ turns, measurement: older.measurement }) } expect(turns).toBe(LONG_HISTORY_TURNS) const conversation = await continueConversation(world, cdp, { startingTurns: LONG_HISTORY_TURNS, turnCount: COMPARISON_TURNS, turnSpec: comparisonTurn, expectedSessionId: SessionId(LONG_SESSION_ID), }) expect(conversation.toolTurns).toBe(2) console.info(`WEB_PERF_RESULT ${JSON.stringify({ scenario: 'expanded-history-500-plus-8', setupMs: rounded(world.setupMs), replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW, historyPages, conversation, }, null, 2)}`) expect(world.tripwire.warnings).toEqual([]) expect(world.tripwire.pageErrors).toEqual([]) } finally { await closePerformanceWorld(world) } }) it('reports one hundred generated turns and the next user-message paint', async () => { const world = await launchPerformanceWorld({ browser, replay: performanceReplayOverride(POST_SOAK_RENDER_TURN, soakTurn), }) let testFailure: unknown try { await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' }) await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 }) await connectFreshWorkspace(world.page, world.scaffold.workspaceCwd, 'continuous-conversation-perf') const cdp = await world.page.context().newCDPSession(world.page) await cdp.send('Performance.enable') const conversation = await continueConversation(world, cdp, { startingTurns: 0, turnCount: SOAK_TURNS, turnSpec: soakTurn, checkpointInterval: SOAK_CHECKPOINT_INTERVAL, }) expect(conversation.toolTurns).toBe(SOAK_TURNS / SOAK_TOOL_INTERVAL) const postSoakUserRender = await measurePostSoakUserRender(world, cdp) const { turns, ...retained } = conversation console.info(`WEB_PERF_RESULT ${JSON.stringify({ scenario: 'continuous-100-turn-soak', setupMs: rounded(world.setupMs), replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW, ...retained, windows: summarizeTurnWindows(turns, SOAK_CHECKPOINT_INTERVAL), postSoakUserRender, }, null, 2)}`) expect(world.tripwire.warnings).toEqual([]) expect(world.tripwire.pageErrors).toEqual([]) } catch (error) { testFailure = error throw error } finally { try { await closePerformanceWorld(world) } catch (cleanupError) { if (testFailure === undefined) throw cleanupError throw new AggregateError( [testFailure, cleanupError], 'continuous conversation performance test and teardown failed', ) } } }) })