imccyu
c94de60a0c
The token-meter projections now subtract compact/summary's shadowedTokenCount verbatim, so the seeded-history fixture's hand-picked 10k count drove the message figure negative and killed the projections baseline. Realize the seed first (realizeSeedFixture, idempotent, shared with seedSession) and derive the count from the shadowed rows through the composed meter, reproducing the previous per-node fold bit for bit — the recorded aria goldens pass unchanged.
423 lines
21 KiB
TypeScript
423 lines
21 KiB
TypeScript
// Web e2e scenario: seeded history. A recorded session seeded cold through
|
|
// the REAL persistence API renders purely from the log — the surface nothing
|
|
// else covers: sidebar cold listing, the implicit resume/attach inside the
|
|
// history RPC, history-page tool views, and the client's log-ordered transcript
|
|
// events — with ZERO model calls in replay (no replay fixture; a stray stream
|
|
// fails loud on the open llm seam). The cold session also carries the one
|
|
// keyless command-row surface: an Access-chip pick runs `/permission` on the
|
|
// host, so the settled row's copy has a golden here. The seed is a recorded
|
|
// fixture under the
|
|
// same record discipline as every other: DSH_SNAPSHOT=record drives the turn
|
|
// live through the composer (real read tool against seeded workspace files)
|
|
// and harvests seed.jsonl; replay/refresh seed it cold and only render.
|
|
import { readFile, writeFile, mkdir } from 'node:fs/promises'
|
|
import { fileURLToPath } from 'node:url'
|
|
import type { Browser, Page } from 'playwright'
|
|
import { chromium } from 'playwright'
|
|
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
|
|
import { createUserMessage } from '@deepseek-ai/dsh-llm'
|
|
import type { ContentBlock, Message } from '@deepseek-ai/dsh-llm'
|
|
import { deriveEventMessage, SessionId } from '@deepseek-ai/dsh-session'
|
|
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
|
import type { TokenMeterService } from '@deepseek-ai/dsh-token-meter'
|
|
import { join } from 'node:path'
|
|
import {
|
|
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
|
|
launchWebScaffold, realizeSeedFixture, recordFixture, seedSession, watchConsole, webSnapshotMode, type WebScaffold,
|
|
} from './scaffold.ts'
|
|
import { newEnglishPage, saveFailureShot } from './support.ts'
|
|
|
|
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/seeded-history', import.meta.url))
|
|
const SEED = fileURLToPath(new URL('./snapshots/seeded-history/seed.jsonl', import.meta.url))
|
|
const UI_EXPECTED = fileURLToPath(new URL('./snapshots/seeded-history/ui.expected.md', import.meta.url))
|
|
// The command-row golden: the same conversation after one /permission switch,
|
|
// which is the only surface that shows a settled command row's copy.
|
|
const COMMAND_ROW_EXPECTED = fileURLToPath(new URL('./snapshots/seeded-history/command-row.expected.md', import.meta.url))
|
|
const MODE = webSnapshotMode()
|
|
const SEED_ID = 'seeded-history-web-e2e'
|
|
|
|
const PROMPT = 'Use the read tool twice in one assistant message: read a.txt and b.txt. Then reply with the single word DONE and stop.'
|
|
|
|
/**
|
|
* Append a complete, valid compaction transaction over the recorded turn's own
|
|
* surface. The recording stays model-authentic and reusable; replay adds this
|
|
* deterministic condition before seeding it cold, so the scenario pins the bug
|
|
* this change fixes — a landed compaction must not erase history the reader
|
|
* already saw — through the real host and the real browser.
|
|
* @param raw - the seed fixture text, already realized (placeholder-free) so
|
|
* the shadow price below is computed from the exact strings the host folds.
|
|
* @param meter - the composed token meter; the appended `compact/summary`'s
|
|
* shadow price must be the exact heuristic price of the shadowed nodes, the
|
|
* way compact-basic derives it, because the token-meter projections subtract
|
|
* it verbatim.
|
|
* @returns the fixture with a compacted turn appended.
|
|
*/
|
|
function withCompaction(raw: string, meter: TokenMeterService): string {
|
|
const lines = raw.trimEnd().split('\n')
|
|
const events = lines.slice(1).map(line => JSON.parse(line) as {
|
|
type: string
|
|
seq: number
|
|
time: number
|
|
surfaceOp?: unknown
|
|
data?: { turn?: unknown; message?: unknown; content?: unknown; callId?: unknown; isError?: unknown }
|
|
})
|
|
const surfaceSeqs = events
|
|
.filter(event => event.surfaceOp === 'append'
|
|
&& (event.type === 'user/message'
|
|
|| event.type === 'assistant/message'
|
|
|| event.type === 'tool/result'))
|
|
.map(event => event.seq)
|
|
const first = surfaceSeqs[0]
|
|
const last = surfaceSeqs.at(-1)
|
|
const tail = events.at(-1)
|
|
if (first === undefined || last === undefined || tail === undefined) {
|
|
throw new Error('seeded-history compaction requires a non-empty closed surface')
|
|
}
|
|
// The transaction opens the turn after the recording's last closed one; read
|
|
// it from the fixture so a re-recording with a different turn count stays
|
|
// valid instead of appending a duplicate turn number.
|
|
const lastTurn = events.filter(event => event.type === 'turn/end').at(-1)?.data?.turn
|
|
if (typeof lastTurn !== 'number') {
|
|
throw new Error('seeded-history compaction requires a recording ending on a closed turn')
|
|
}
|
|
const turn = lastTurn + 1
|
|
let seq = tail.seq + 1
|
|
let time = tail.time + 1
|
|
/**
|
|
* Append one event at the next seq/time.
|
|
* @param event - the event body, without seq/time.
|
|
* @returns the seq it took, so provenance cites the push instead of arithmetic over the push order below.
|
|
*/
|
|
const at = (event: Record<string, unknown>): number => {
|
|
const taken = seq++
|
|
lines.push(JSON.stringify({ ...event, seq: taken, time: time++ }))
|
|
return taken
|
|
}
|
|
at({ type: 'turn/start', data: { turn } })
|
|
const startSeq = at({ type: 'compact/start', data: { turn } })
|
|
// Load-bearing exactness: the projections subtract this count verbatim, so
|
|
// it must equal what the host's fold prices for these nodes. The estimator
|
|
// prices message CONTENT only, so a minimal wrapper per storage shape is
|
|
// exact — pre-identity rows carry bare `content` (the persistence read path
|
|
// upgrades them), a current row carries the full `message` envelope.
|
|
const priceRow = (row: (typeof events)[number]): number => {
|
|
if (row.data?.message !== undefined) {
|
|
const message = deriveEventMessage(row as unknown as SessionEvent)
|
|
return message === null ? 0 : meter.estimateMessage(message)
|
|
}
|
|
const content = row.data?.content as ContentBlock[]
|
|
if (row.type === 'tool/result') {
|
|
return meter.estimateMessage({
|
|
content: [{ type: 'tool-result', toolCallId: row.data?.callId, content, isError: row.data?.isError === true }],
|
|
} as unknown as Message)
|
|
}
|
|
// An empty-content assistant message derives no transcript entry.
|
|
if (row.type === 'assistant/message' && content.length === 0) return 0
|
|
return meter.estimateMessage({ content } as unknown as Message)
|
|
}
|
|
const shadowedTokenCount = surfaceSeqs.reduce((total, surfaceSeq) => {
|
|
const event = events.find(candidate => candidate.seq === surfaceSeq)
|
|
if (event === undefined) throw new Error(`seeded-history compaction: shadowed seq ${surfaceSeq} is not in the seed`)
|
|
return total + priceRow(event)
|
|
}, 0)
|
|
const summarySeq = at({
|
|
type: 'compact/summary',
|
|
data: {
|
|
summary: [{
|
|
type: 'text',
|
|
text: '## Cold resume compact summary\n\n- The exact summary remains available.',
|
|
}],
|
|
shadowedRange: { start: first, end: last },
|
|
shadowedSeqs: surfaceSeqs,
|
|
shadowedTokenCount,
|
|
provider: 'snapshot',
|
|
model: 'snapshot-compactor',
|
|
},
|
|
})
|
|
at({
|
|
type: 'user/message',
|
|
data: {
|
|
content: [{
|
|
type: 'text',
|
|
text: '<context_checkpoint>Model-only compact checkpoint.</context_checkpoint>',
|
|
}],
|
|
source: { kind: 'plugin', plugin: 'compact' },
|
|
},
|
|
surfaceOp: { op: 'replace', start: first, end: last },
|
|
sourceEventSeqs: [startSeq, summarySeq, ...surfaceSeqs],
|
|
})
|
|
at({ type: 'compact/end', data: { turn } })
|
|
at({ type: 'turn/end', data: { turn, reason: { kind: 'completed' } } })
|
|
return `${lines.join('\n')}\n`
|
|
}
|
|
|
|
describe('web e2e: seeded history renders through cold resume', () => {
|
|
let scaffold: WebScaffold
|
|
let browser: Browser
|
|
let page: Page
|
|
let tripwire: ReturnType<typeof watchConsole>
|
|
|
|
beforeAll(async () => {
|
|
scaffold = await launchWebScaffold({})
|
|
// The workspace-aware flow runs sessions in <workspaceCwd>/workspace
|
|
// (the composer's default draft name); the read-tool targets must live in
|
|
// that session cwd. Pre-creating the directory is safe: create-by-name
|
|
// adopts an existing directory.
|
|
const sessionCwd = join(scaffold.workspaceCwd, 'workspace')
|
|
await mkdir(sessionCwd, { recursive: true })
|
|
await writeFile(join(sessionCwd, 'a.txt'), 'alpha\n')
|
|
await writeFile(join(sessionCwd, 'b.txt'), 'beta\n')
|
|
if (MODE !== 'record') {
|
|
const raw = await readFile(SEED, 'utf8')
|
|
expect(fixtureUserPrompts(raw), 'seed fixture must carry exactly the drive prompt').toEqual([PROMPT])
|
|
const meter = scaffold.ctx.get('tokenMeter')
|
|
if (meter === undefined) throw new Error('seeded-history requires the composed token meter')
|
|
const realized = realizeSeedFixture(scaffold, raw, SEED_ID)
|
|
await seedSession(scaffold, withCompaction(realized, meter), SEED_ID)
|
|
}
|
|
browser = await chromium.launch()
|
|
page = await newEnglishPage(browser)
|
|
tripwire = watchConsole(page)
|
|
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
|
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
}, 120_000)
|
|
|
|
afterAll(async () => {
|
|
await browser?.close()
|
|
await scaffold?.close()
|
|
})
|
|
|
|
it.skipIf(MODE !== 'record')('records the seed turn live through the composer', async () => {
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-seeded-record'))
|
|
const input = page.locator('textarea').first()
|
|
await input.waitFor({ timeout: 10_000 })
|
|
const settled = scaffold.whenTurnSettled()
|
|
await input.fill(PROMPT)
|
|
await input.press('Enter')
|
|
const sessionId = await settled
|
|
await recordFixture(scaffold, sessionId, SEED)
|
|
}, 200_000)
|
|
|
|
it.skipIf(MODE === 'record')('serves the projections baseline on the real composition tail page', async () => {
|
|
// Composition regression tripwire: the projection registry must be a row
|
|
// in the SHIPPED cordis.yml — with it absent every domain unit's optional
|
|
// injection stays silent and this block disappears (no titles/todos on
|
|
// the web), while fixture-level suites stay green. Assert through the
|
|
// real HTTP wire against the booted real host.
|
|
const response = await fetch(`${scaffold.baseUrl}/api/session.history`, {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: JSON.stringify({
|
|
type: 'client-request', rpcId: 'seeded-projections', method: 'session.history',
|
|
payload: { sessionId: SEED_ID },
|
|
}),
|
|
})
|
|
expect(response.ok).toBe(true)
|
|
const body = await response.json() as {
|
|
result: { ok: boolean; value?: { projections?: { asOfSeq: number; values: Record<string, unknown> } } }
|
|
}
|
|
expect(body.result.ok).toBe(true)
|
|
const projections = body.result.value?.projections
|
|
expect(projections).toBeDefined()
|
|
expect(projections?.asOfSeq).toBeGreaterThanOrEqual(0)
|
|
// The seed carries a session/title event: the title unit must serve it.
|
|
expect(typeof projections?.values.title).toBe('string')
|
|
// tool-todo is composed but the seed has no todo/write: whole-value null,
|
|
// key PRESENT (absence would mean the unit never registered).
|
|
expect(projections?.values).toHaveProperty('todos', null)
|
|
})
|
|
|
|
it.skipIf(MODE === 'record')('lists the seeded session cold and renders its history from the log', async () => {
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-seeded-history'))
|
|
// The sidebar tree collapses workspace groups by default: click the group
|
|
// row (treeitem 0) to expand, then the revealed session row.
|
|
const groupRow = page.locator('[role="treeitem"]').first()
|
|
await groupRow.waitFor({ timeout: 15_000 })
|
|
await groupRow.click()
|
|
const sessionRow = page.locator('[role="treeitem"]').nth(1)
|
|
await sessionRow.waitFor({ timeout: 10_000 })
|
|
await sessionRow.click()
|
|
// Settled barrier for history: the recorded final assistant text renders.
|
|
await expect.poll(() => page.getByText('DONE', { exact: true }).count(), { timeout: 15_000 }).toBe(1)
|
|
await expect.poll(() => page.getByText('Context compacted', { exact: true }).count(), { timeout: 10_000 }).toBe(1)
|
|
// Tool cards render from logged tool/call + tool/result alone (views are
|
|
// host-recomputed per page; the generic card is the documented default).
|
|
const toolRows = page.locator('[data-variant], [data-sample]')
|
|
await expect.poll(() => toolRows.count(), { timeout: 10_000 }).toBeGreaterThanOrEqual(2)
|
|
expect(await page.getByText('a.txt', { exact: false }).count()).toBeGreaterThan(0)
|
|
// The bug this fixes: the compaction shadowed the whole recorded surface on
|
|
// the model side, and the prompt and full tool output are still on screen.
|
|
expect(await page.getByText(PROMPT, { exact: true }).count()).toBe(1)
|
|
|
|
const agent = scaffold.ctx.agents.get(SessionId(SEED_ID))
|
|
if (agent === undefined) throw new Error('seeded session did not attach an agent')
|
|
agent.session.append('user/message', createUserMessage({
|
|
content: [{
|
|
type: 'text',
|
|
text: '<system-reminder>\n'
|
|
+ 'The following workspace instructions may be relevant to your work. '
|
|
+ 'Use them as guidance when applicable.\n\n'
|
|
+ Array.from({ length: 24 }, (_, index) => `Instruction ${index + 1}: preserve the logged context contract.`).join('\n')
|
|
+ '\n</system-reminder>',
|
|
}],
|
|
source: {
|
|
kind: 'workspace-instructions',
|
|
baseline: true,
|
|
changes: [{
|
|
action: 'set',
|
|
scope: '.\u0000AGENTS.md',
|
|
path: 'AGENTS.md',
|
|
digest: 'context-injection-browser-snapshot',
|
|
}],
|
|
},
|
|
}), { surfaceOp: 'append' })
|
|
await page.getByRole('button', { name: 'Context injection' }).waitFor({ timeout: 10_000 })
|
|
}, 60_000)
|
|
|
|
it.skipIf(MODE === 'record')('matches the historical conversation aria golden', async () => {
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-seeded-aria'))
|
|
// This scenario deliberately leaves the LLM seam open to prove zero
|
|
// model calls. History still restores the routed id, but without an
|
|
// advertised catalog row the selector prompts for a listed replacement.
|
|
await page.getByRole('button', { name: 'Select model', exact: true })
|
|
.waitFor({ timeout: 10_000 })
|
|
const snapshot = (await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd))
|
|
.split(SEED_ID).join('{{seededId}}')
|
|
await compareOrRefreshGolden(UI_EXPECTED, snapshot, MODE)
|
|
})
|
|
|
|
it.skipIf(MODE === 'record')('matches the Figma context disclosure geometry', async () => {
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-context-injection'))
|
|
const disclosure = page.getByRole('button', { name: 'Context injection' })
|
|
expect(await disclosure.getAttribute('aria-expanded')).toBe('false')
|
|
const collapsedIcon = disclosure.locator('svg').first()
|
|
const collapsedIconBox = await collapsedIcon.boundingBox()
|
|
expect(collapsedIconBox?.width).toBe(14)
|
|
expect(collapsedIconBox?.height).toBe(14)
|
|
|
|
await disclosure.click()
|
|
await expect.poll(() => disclosure.getAttribute('aria-expanded')).toBe('true')
|
|
const body = page.locator('[data-context-injection-body]')
|
|
await body.waitFor({ timeout: 5_000 })
|
|
const headerBox = await disclosure.boundingBox()
|
|
const bodyBox = await body.boundingBox()
|
|
if (headerBox === null || bodyBox === null) throw new Error('context disclosure geometry is not measurable')
|
|
expect(headerBox.height).toBe(24)
|
|
expect(bodyBox.x - headerBox.x).toBe(22)
|
|
expect(bodyBox.y - headerBox.y - headerBox.height).toBe(4)
|
|
expect(bodyBox.height).toBe(141)
|
|
|
|
const style = await body.evaluate((element) => {
|
|
const computed = getComputedStyle(element)
|
|
return {
|
|
backgroundColor: computed.backgroundColor,
|
|
borderRadius: computed.borderRadius,
|
|
color: computed.color,
|
|
fontSize: computed.fontSize,
|
|
lineHeight: computed.lineHeight,
|
|
padding: [
|
|
computed.paddingTop,
|
|
computed.paddingRight,
|
|
computed.paddingBottom,
|
|
computed.paddingLeft,
|
|
],
|
|
scrolls: element.scrollHeight > element.clientHeight,
|
|
}
|
|
})
|
|
expect(style).toEqual({
|
|
backgroundColor: 'rgb(249, 250, 251)',
|
|
borderRadius: '8px',
|
|
color: 'rgb(129, 133, 140)',
|
|
fontSize: '11px',
|
|
lineHeight: '16px',
|
|
padding: ['10px', '16px', '12px', '12px'],
|
|
scrolls: true,
|
|
})
|
|
|
|
await disclosure.click()
|
|
await expect.poll(() => disclosure.getAttribute('aria-expanded')).toBe('false')
|
|
})
|
|
|
|
it.skipIf(MODE === 'record')('file-path tool rows rebuilt from the cold log stay details-inert', async () => {
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-seeded-toolrow'))
|
|
// Interaction over cold-resumed history: read summaries are host-open
|
|
// file links (not expand-in-place / not details). Runs after the golden
|
|
// capture; still zero model calls.
|
|
const fileLink = page.locator('[data-variant="read"] button').first()
|
|
await fileLink.waitFor({ timeout: 10_000 })
|
|
const frame = page.locator('[style*="grid-template-columns"]').first()
|
|
expect(await frame.getAttribute('data-details-collapsed')).toBe('true')
|
|
await fileLink.click()
|
|
await expect.poll(() => frame.getAttribute('data-details-collapsed'), { timeout: 5_000 }).toBe('true')
|
|
// Path label survives from the recorded args (a.txt).
|
|
await expect.poll(() => page.getByText('a.txt', { exact: false }).count(), { timeout: 5_000 }).toBeGreaterThan(0)
|
|
})
|
|
|
|
it.skipIf(MODE === 'record')('expands the cold-resumed compact summary', async () => {
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-seeded-compaction'))
|
|
const marker = page.getByRole('button', { name: /Context compacted/ })
|
|
await marker.waitFor({ timeout: 10_000 })
|
|
expect(await marker.getAttribute('aria-expanded')).toBe('false')
|
|
await marker.click()
|
|
await expect.poll(() => marker.getAttribute('aria-expanded'), { timeout: 5_000 }).toBe('true')
|
|
await expect.poll(() => page.getByRole('heading', { name: 'Cold resume compact summary' }).count(), {
|
|
timeout: 5_000,
|
|
}).toBe(1)
|
|
expect(await page.getByText('The exact summary remains available.', { exact: false }).count()).toBeGreaterThan(0)
|
|
// Restore the shared page state for any later case.
|
|
await marker.click()
|
|
await expect.poll(() => marker.getAttribute('aria-expanded'), { timeout: 5_000 }).toBe('false')
|
|
})
|
|
|
|
it.skipIf(MODE === 'record')('an Access-chip switch lands one command row: bare name, non-repeating settlement text', async () => {
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-seeded-command-row'))
|
|
// The Access chip submits `/permission <preset>` — a host command with no
|
|
// model call, so the settled row renders keylessly over this cold history.
|
|
// The row copy is the assertion: `permission · preset read-only`,
|
|
// where neither half repeats the other (the dispatched `/` and its
|
|
// argument stay out of the title, and the settlement text never restates
|
|
// the command's own name).
|
|
await page.getByRole('button', { name: 'Access mode, current: Workspace Write' }).click()
|
|
await page.getByRole('menuitem', { name: 'Read Only' }).click()
|
|
await page.getByRole('button', { name: 'Access mode, current: Read Only' }).waitFor({ timeout: 10_000 })
|
|
// Scoped to the row itself, so unrelated page text that happens to read
|
|
// `permission` (a future resident slash menu) cannot satisfy or break it.
|
|
const row = page.locator('[data-variant="others"]').filter({ hasText: 'preset read-only' })
|
|
await expect.poll(() => row.count(), { timeout: 10_000 }).toBe(1)
|
|
expect(await row.getByText('permission', { exact: true }).count()).toBe(1)
|
|
expect(await row.getByText('/permission read-only', { exact: true }).count()).toBe(0)
|
|
const snapshot = (await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd))
|
|
.split(SEED_ID).join('{{seededId}}')
|
|
await compareOrRefreshGolden(COMMAND_ROW_EXPECTED, snapshot, MODE)
|
|
}, 60_000)
|
|
|
|
it.skipIf(MODE === 'record')('fits short logged context without a scrollport', async () => {
|
|
const agent = scaffold.ctx.agents.get(SessionId(SEED_ID))
|
|
if (agent === undefined) throw new Error('seeded session did not attach an agent')
|
|
agent.session.append('user/message', createUserMessage({
|
|
content: [{ type: 'text', text: 'Short injected context.' }],
|
|
source: { kind: 'plugin', plugin: 'fixture' },
|
|
}), { surfaceOp: 'append' })
|
|
|
|
const disclosures = page.getByRole('button', { name: 'Context injection' })
|
|
await expect.poll(() => disclosures.count(), { timeout: 10_000 }).toBe(2)
|
|
const disclosure = disclosures.nth(1)
|
|
await disclosure.click()
|
|
await expect.poll(() => disclosure.getAttribute('aria-expanded')).toBe('true')
|
|
|
|
const body = page.locator('[data-context-injection-body]')
|
|
const bodyBox = await body.boundingBox()
|
|
if (bodyBox === null) throw new Error('short context disclosure geometry is not measurable')
|
|
expect(bodyBox.height).toBeLessThan(141)
|
|
expect(await body.evaluate(element => element.scrollHeight > element.clientHeight)).toBe(false)
|
|
})
|
|
|
|
it.skipIf(MODE === 'record')('issued zero model calls and stayed clean', async () => {
|
|
// No replay fixture was installed and the llm seam is open — any stray
|
|
// stream would have failed the turn loudly. Cleanliness pins the wire.
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
expect(tripwire.warnings).toEqual([])
|
|
await assertFixtureInventory(SNAPSHOT_DIR, ['command-row.expected.md', 'seed.jsonl', 'ui.expected.md'])
|
|
})
|
|
})
|