test(web): make real title smoke deterministic

This commit is contained in:
Tianyi Cui
2026-07-23 20:41:10 +08:00
parent 6cb0372d86
commit 2a62da8ff0
+62 -9
View File
@@ -77,6 +77,55 @@ async function rpc<T>(baseUrl: string, method: string, payload: unknown): Promis
return body.result.value
}
interface HistoryPage {
events: { event: { type: string; data: unknown } }[]
hasMore: boolean
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null
}
function providerTitle(page: HistoryPage): string | undefined {
for (let index = page.events.length - 1; index >= 0; index--) {
const event = page.events[index]!.event
if (event.type !== 'session/title' || !isRecord(event.data)) continue
const source = event.data.source
if (typeof event.data.title === 'string' && isRecord(source) && source.kind === 'provider') {
return event.data.title
}
}
return undefined
}
function hasAssistantMarker(page: HistoryPage, marker: string): boolean {
return page.events.some(({ event }) => {
if (event.type !== 'assistant/message' || !isRecord(event.data) || !Array.isArray(event.data.content)) return false
return event.data.content.some(block =>
isRecord(block) && block.type === 'text' && typeof block.text === 'string' && block.text.includes(marker))
})
}
async function history(baseUrl: string, sessionId: string): Promise<HistoryPage> {
return rpc<HistoryPage>(baseUrl, 'session.history', { sessionId, maxMessages: 10 })
}
async function waitForProviderTitle(baseUrl: string, sessionId: string): Promise<string> {
let observed: string | undefined
await expect.poll(async () => {
observed = providerTitle(await history(baseUrl, sessionId))
return observed
}, { timeout: 90_000 }).toEqual(expect.any(String))
if (observed === undefined) throw new Error('provider-backed session title was not observed')
return observed
}
async function waitForAssistantMarker(baseUrl: string, sessionId: string, marker: string): Promise<void> {
await expect.poll(async () => hasAssistantMarker(await history(baseUrl, sessionId), marker), {
timeout: 120_000,
}).toBe(true)
}
/** W5 screenshot: evidence for the figma comparison, not a failure artifact. */
async function screen(page: Page, name: string): Promise<void> {
await page.screenshot({ path: join(REPO_ROOT, '.artifacts', `w5-${name}.png`) })
@@ -282,7 +331,6 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
await input.waitFor({ timeout: 10_000 })
await screen(page, '02-empty-state')
const prompt = `Please answer this request carefully: explain event sourcing in two sentences, ending with exactly ${ROUND_DONE_MARKER}.`
const fallbackTitle = 'Please answer this request carefully:'
await input.fill(prompt)
await input.press('Enter')
// startSession chain: session mounts, composer moves to the bottom.
@@ -296,14 +344,18 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
undefined,
{ timeout: 15_000 },
)
await expect.poll(async () => (await rpc<{ items: { sessionId: string }[] }>(baseUrl, 'session.list', {})).items.length, {
timeout: 15_000,
}).toBe(1)
const sessions = await rpc<{ items: { sessionId: string }[] }>(baseUrl, 'session.list', {})
const sessionId = sessions.items[0]?.sessionId
if (sessionId === undefined) throw new Error('created Web session was not listed')
const durableTitle = await waitForProviderTitle(baseUrl, sessionId)
await page.waitForFunction(
expected => document.title !== `${expected} — DeepSeek Harness`
&& document.title.endsWith(' — DeepSeek Harness'),
fallbackTitle,
{ timeout: 90_000 },
expected => document.title === `${expected} — DeepSeek Harness`,
durableTitle,
{ timeout: 15_000 },
)
const durableTitle = (await page.title()).replace(/ — DeepSeek Harness$/, '')
expect(durableTitle).not.toBe(fallbackTitle)
const sessionTree = page.getByRole('tree', { name: 'Sessions' })
const projectRow = sessionTree.getByRole('treeitem').first()
if (await projectRow.getAttribute('aria-expanded') === 'false') await projectRow.click()
@@ -311,7 +363,8 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
sessionTree.getByText(durableTitle, { exact: true }).waitFor({ timeout: 10_000 }),
page.getByRole('navigation').getByText(durableTitle, { exact: true }).waitFor({ timeout: 10_000 }),
])
await page.waitForFunction(marker => document.body.innerText.includes(marker), ROUND_DONE_MARKER, { timeout: 120_000 })
await waitForAssistantMarker(baseUrl, sessionId, ROUND_DONE_MARKER)
await page.locator('p').filter({ hasText: ROUND_DONE_MARKER }).waitFor({ timeout: 10_000 })
await screen(page, '04-round-complete')
}, 150_000)
@@ -386,7 +439,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || notReady.length > 0)('web smoke
onTestFailed(() => saveFailureShot(page, 'w5-reload'))
await page.reload({ waitUntil: 'load' })
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
await page.waitForFunction(marker => document.body.innerText.includes(marker), ROUND_DONE_MARKER, { timeout: 30_000 })
await page.locator('p').filter({ hasText: ROUND_DONE_MARKER }).waitFor({ timeout: 30_000 })
await screen(page, '12-reload-recovery')
})