test(web): price the seeded compaction's shadow count exactly

The token-meter projections now subtract compact/summary's
shadowedTokenCount verbatim, so the seeded-history fixture's hand-picked
10k count drove the message figure negative and killed the projections
baseline. Realize the seed first (realizeSeedFixture, idempotent, shared
with seedSession) and derive the count from the shadowed rows through
the composed meter, reproducing the previous per-node fold bit for bit —
the recorded aria goldens pass unchanged.
This commit is contained in:
imccyu
2026-08-06 03:20:48 +08:00
parent 553898e529
commit c94de60a0c
2 changed files with 60 additions and 10 deletions
+17 -3
View File
@@ -476,15 +476,29 @@ export function fixtureUserPrompts(fixtureText: string): string[] {
* @param id - the seeded session id (stable for deterministic goldens).
* @returns the seeded id.
*/
export async function seedSession(scaffold: WebScaffold, fixtureText: string, id: string): Promise<SessionId> {
/**
* Realize a recorded seed fixture against one scaffold: substitute the
* `{{sessionId}}`/`{{cwd}}` placeholders and rewrite the recorded cwd to the
* scaffold's workspace. Idempotent, so a caller may realize early (e.g. to
* price content exactly as the host will fold it) and still pass the result
* through {@link seedSession}.
* @param scaffold - the booted scaffold whose workspace the seed targets.
* @param fixtureText - the committed seed fixture text.
* @param id - the session id the seed is realized for.
* @returns the realized fixture text.
*/
export function realizeSeedFixture(scaffold: WebScaffold, fixtureText: string, id: string): string {
const realized = fixtureText
.split('{{sessionId}}').join(id)
.split('{{cwd}}').join(scaffold.workspaceCwd)
const fixtureCwd = (JSON.parse(realized.split('\n', 1)[0]!) as { cwd?: string }).cwd
const rewritten = fixtureCwd === undefined
return fixtureCwd === undefined
? realized
: realized.split(fixtureCwd).join(scaffold.workspaceCwd)
const events = parseSessionLog(rewritten)
}
export async function seedSession(scaffold: WebScaffold, fixtureText: string, id: string): Promise<SessionId> {
const events = parseSessionLog(realizeSeedFixture(scaffold, fixtureText, id))
if (events.length === 0) throw new Error('seed fixture has no events')
const last = events[events.length - 1]!
// An open final turn would be mutated by resume's crash repair on first
+43 -7
View File
@@ -16,11 +16,14 @@ import type { Browser, Page } from 'playwright'
import { chromium } from 'playwright'
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
import { createUserMessage } from '@deepseek-ai/dsh-llm'
import { SessionId } from '@deepseek-ai/dsh-session'
import type { ContentBlock, Message } from '@deepseek-ai/dsh-llm'
import { deriveEventMessage, SessionId } from '@deepseek-ai/dsh-session'
import type { SessionEvent } from '@deepseek-ai/dsh-session'
import type { TokenMeterService } from '@deepseek-ai/dsh-token-meter'
import { join } from 'node:path'
import {
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
launchWebScaffold, recordFixture, seedSession, watchConsole, webSnapshotMode, type WebScaffold,
launchWebScaffold, realizeSeedFixture, recordFixture, seedSession, watchConsole, webSnapshotMode, type WebScaffold,
} from './scaffold.ts'
import { newEnglishPage, saveFailureShot } from './support.ts'
@@ -41,17 +44,22 @@ const PROMPT = 'Use the read tool twice in one assistant message: read a.txt and
* deterministic condition before seeding it cold, so the scenario pins the bug
* this change fixes — a landed compaction must not erase history the reader
* already saw — through the real host and the real browser.
* @param raw - the committed seed fixture text.
* @param raw - the seed fixture text, already realized (placeholder-free) so
* the shadow price below is computed from the exact strings the host folds.
* @param meter - the composed token meter; the appended `compact/summary`'s
* shadow price must be the exact heuristic price of the shadowed nodes, the
* way compact-basic derives it, because the token-meter projections subtract
* it verbatim.
* @returns the fixture with a compacted turn appended.
*/
function withCompaction(raw: string): string {
function withCompaction(raw: string, meter: TokenMeterService): string {
const lines = raw.trimEnd().split('\n')
const events = lines.slice(1).map(line => JSON.parse(line) as {
type: string
seq: number
time: number
surfaceOp?: unknown
data?: { turn?: unknown }
data?: { turn?: unknown; message?: unknown; content?: unknown; callId?: unknown; isError?: unknown }
})
const surfaceSeqs = events
.filter(event => event.surfaceOp === 'append'
@@ -87,6 +95,31 @@ function withCompaction(raw: string): string {
}
at({ type: 'turn/start', data: { turn } })
const startSeq = at({ type: 'compact/start', data: { turn } })
// Load-bearing exactness: the projections subtract this count verbatim, so
// it must equal what the host's fold prices for these nodes. The estimator
// prices message CONTENT only, so a minimal wrapper per storage shape is
// exact — pre-identity rows carry bare `content` (the persistence read path
// upgrades them), a current row carries the full `message` envelope.
const priceRow = (row: (typeof events)[number]): number => {
if (row.data?.message !== undefined) {
const message = deriveEventMessage(row as unknown as SessionEvent)
return message === null ? 0 : meter.estimateMessage(message)
}
const content = row.data?.content as ContentBlock[]
if (row.type === 'tool/result') {
return meter.estimateMessage({
content: [{ type: 'tool-result', toolCallId: row.data?.callId, content, isError: row.data?.isError === true }],
} as unknown as Message)
}
// An empty-content assistant message derives no transcript entry.
if (row.type === 'assistant/message' && content.length === 0) return 0
return meter.estimateMessage({ content } as unknown as Message)
}
const shadowedTokenCount = surfaceSeqs.reduce((total, surfaceSeq) => {
const event = events.find(candidate => candidate.seq === surfaceSeq)
if (event === undefined) throw new Error(`seeded-history compaction: shadowed seq ${surfaceSeq} is not in the seed`)
return total + priceRow(event)
}, 0)
const summarySeq = at({
type: 'compact/summary',
data: {
@@ -96,7 +129,7 @@ function withCompaction(raw: string): string {
}],
shadowedRange: { start: first, end: last },
shadowedSeqs: surfaceSeqs,
shadowedTokenCount: 10_000,
shadowedTokenCount,
provider: 'snapshot',
model: 'snapshot-compactor',
},
@@ -137,7 +170,10 @@ describe('web e2e: seeded history renders through cold resume', () => {
if (MODE !== 'record') {
const raw = await readFile(SEED, 'utf8')
expect(fixtureUserPrompts(raw), 'seed fixture must carry exactly the drive prompt').toEqual([PROMPT])
await seedSession(scaffold, withCompaction(raw), SEED_ID)
const meter = scaffold.ctx.get('tokenMeter')
if (meter === undefined) throw new Error('seeded-history requires the composed token meter')
const realized = realizeSeedFixture(scaffold, raw, SEED_ID)
await seedSession(scaffold, withCompaction(realized, meter), SEED_ID)
}
browser = await chromium.launch()
page = await newEnglishPage(browser)