test(jsonl): cover reusable decoder branches

This commit is contained in:
imccyu
2026-08-06 10:40:31 +08:00
parent 456d4319bf
commit 03bb663966
5 changed files with 147 additions and 20 deletions
@@ -280,6 +280,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi
signal?.throwIfAborted()
const headerFrame = decodedFrames.next()
signal?.throwIfAborted()
/* v8 ignore next -- a non-empty structural frame list makes the decoder yield its first frame or throw. */
if (headerFrame.done) throw new Error('empty or header-less Zstandard session log')
assertZstdHeaderFrame(headerFrame.value)
const scanner = new SessionLogScanner(headerFrame.value)
@@ -41,6 +41,7 @@ function privateZstdStream(
const errorKey = Reflect.ownKeys(stream).find((key): key is symbol => (
typeof key === 'symbol' && key.description === 'kError'
))
/* v8 ignore next -- one test runtime exposes one Node-private shape; the Node 22/24/26 matrix checks compatibility. */
if (
typeof handle !== 'object' || handle === null
|| typeof (handle as { writeSync?: unknown }).writeSync !== 'function'
@@ -48,6 +49,7 @@ function privateZstdStream(
|| candidate._writeState.length < 2
|| typeof candidate._defaultFlushFlag !== 'number'
|| errorKey === undefined
|| candidate[errorKey] !== null
) return undefined
return { stream: stream as NodeZstdPrivateStream, errorKey }
}
@@ -81,10 +83,13 @@ export class NodePrivateZstdFrameDecoder implements ZstdFrameDecoder {
static create(): NodePrivateZstdFrameDecoder | undefined {
const stream = createZstdDecompress({ chunkSize: DECODE_CHUNK_SIZE })
const privateAccess = privateZstdStream(stream)
/* v8 ignore next -- reached only when a supported Node release changes its private stream shape. */
if (privateAccess !== undefined) {
return new NodePrivateZstdFrameDecoder(privateAccess.stream, privateAccess.errorKey)
}
/* v8 ignore next -- the active Node runtime passed the private-shape probe above. */
stream.close()
/* v8 ignore next -- the active Node runtime passed the private-shape probe above. */
return undefined
}
@@ -111,6 +116,7 @@ export class NodePrivateZstdFrameDecoder implements ZstdFrameDecoder {
/** Decode one frame; its returned scratch view remains valid until the next call. */
private decodeFrame(input: Buffer): Buffer {
const handle = this.stream._handle
/* v8 ignore next -- decode() rejects closed instances before entering this private frame operation. */
if (this.closed || handle === null) throw new Error('cannot decode with a closed Zstandard frame decoder')
let inputOffset = 0
@@ -125,11 +131,11 @@ export class NodePrivateZstdFrameDecoder implements ZstdFrameDecoder {
inputRemaining,
this.output,
0,
DECODE_CHUNK_SIZE,
this.output.length,
)
if (this.decoderError !== undefined) throw this.decoderError
const internalError = this.stream[this.errorKey]
if (internalError !== undefined && internalError !== null) {
if (internalError !== null) {
if (internalError instanceof Error) throw internalError
throw new Error('Zstandard decoder exposed a non-Error internal failure')
}
@@ -137,21 +143,23 @@ export class NodePrivateZstdFrameDecoder implements ZstdFrameDecoder {
const outputAfter = this.stream._writeState[0]
const inputAfter = this.stream._writeState[1]
const consumed = inputRemaining - inputAfter
const produced = DECODE_CHUNK_SIZE - outputAfter
const produced = this.output.length - outputAfter
if (produced > 0) {
outputBytes += produced
/* v8 ignore next -- Buffer cannot materialize a frame beyond its own process-wide maximum length. */
if (outputBytes > bufferConstants.MAX_LENGTH) {
throw new Error(`Zstandard frame output exceeds ${bufferConstants.MAX_LENGTH} bytes`)
}
}
if (outputAfter !== 0) {
/* v8 ignore next -- structurally scanned ranges contain exactly one complete frame and no trailing bytes. */
if (inputAfter !== 0) throw new Error('Zstandard frame decoder left trailing input')
const finalChunk = this.output.subarray(0, produced)
if (fullChunks.length === 0) return finalChunk
if (produced > 0) fullChunks.push(Buffer.from(finalChunk))
const [onlyChunk] = fullChunks
return fullChunks.length === 1 && onlyChunk !== undefined
const onlyChunk = fullChunks[0] as Buffer
return fullChunks.length === 1
? onlyChunk
: Buffer.concat(fullChunks, outputBytes)
}
@@ -6,7 +6,7 @@
*/
import {
constants, zstdCompress, zstdDecompress, zstdDecompressSync, type ZstdOptions,
constants, zstdCompress, zstdDecompress, type ZstdOptions,
} from 'node:zlib'
import { promisify } from 'node:util'
import { NodePrivateZstdFrameDecoder } from './zstd-private-decoder.ts'
@@ -121,17 +121,6 @@ export async function decompressZstdFrame(input: Buffer): Promise<Buffer> {
return zstdDecompressAsync(input)
}
/**
* Synchronously decompress one complete frame and validate its checksum.
* Complete-log readers time-slice repeated calls so the event loop regains
* control without paying one asynchronous native dispatch per frame.
* @param input - one structurally complete Zstandard frame.
* @returns the frame plaintext.
*/
export function decompressZstdFrameSync(input: Buffer): Buffer {
return zstdDecompressSync(input)
}
/** Common lifecycle for interchangeable synchronous multi-frame decoders. */
export interface ZstdFrameDecoder {
/**
@@ -694,6 +694,46 @@ describe('SessionPersistenceJsonl: write path (session/event → flush)', () =>
describe('SessionPersistenceJsonl: scanLog unit', () => {
it('requires exactly one newline-terminated header record', () => {
const header = JSON.stringify(toHeaderLine(meta('scanner-header')))
expect(() => new SessionLogScanner(Buffer.alloc(0))).toThrow(/header-less/)
expect(() => new SessionLogScanner(Buffer.from(header))).toThrow(/header-less/)
expect(() => new SessionLogScanner(Buffer.from(`${header}\n${header}\n`))).toThrow(/header-less/)
})
it('handles empty writes, boundary newlines, torn fragments, and scanner completion', () => {
const header = Buffer.from(`${JSON.stringify(toHeaderLine(meta('scanner-lifecycle')))}\n`)
const event = Buffer.from(JSON.stringify(oneTurnLog()[0]))
const scanner = new SessionLogScanner(header)
scanner.write(Buffer.alloc(0))
scanner.write(event)
scanner.write(Buffer.from('\nignored torn tail'))
const result = scanner.finish()
expect(result.events).toEqual([oneTurnLog()[0]])
expect(result.committedBytes).toBe(header.length + event.length + 1)
expect(() =>{ scanner.write(Buffer.from('\n')) }).toThrow(/finished/)
})
it('keeps scanning after a tolerable corrupt suffix until a committed turn end appears', () => {
const header = Buffer.from(`${JSON.stringify(toHeaderLine(meta('scanner-corrupt-suffix')))}\n`)
const scanner = new SessionLogScanner(header)
scanner.write(Buffer.from([
JSON.stringify(oneTurnLog()[0]),
'{not json',
JSON.stringify({ type: 'step/start', seq: 1, time: 2, data: { turn: 1, step: 1 } }),
'',
].join('\n')))
expect(scanner.finish().events).toEqual([oneTurnLog()[0]])
const committed = new SessionLogScanner(header)
expect(() =>{ committed.write(Buffer.from([
JSON.stringify({ type: 'turn/end', seq: 1, time: 2, data: { turn: 1, reason: { kind: 'completed' } } }),
'',
].join('\n'))) }).toThrow(/seq gap in committed region/)
})
it('incrementally scans records split across reusable decoder chunks', () => {
const header = Buffer.from(`${JSON.stringify(toHeaderLine(meta('incremental')))}\n`)
const body = Buffer.from(`${oneTurnLog().map(event => JSON.stringify(event)).join('\n').replace('"hi"', '"你好"')}\n`)
@@ -10,8 +10,8 @@ import type { SessionEvent } from '@deepseek-ai/dsh-session'
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl'
import { logPath, scanLog, sessionDir, toHeaderLine, type JsonlCompression } from '../src/format.ts'
import {
compressZstdFrame, createZstdFrameDecoder, decompressZstdFrame, decompressZstdFrameSync, decompressZstdPrefix,
scanZstdFrames,
compressZstdFrame, createZstdFrameDecoder, decompressZstdFrame, decompressZstdPrefix, scanZstdFrames,
type ZstdFrameDecoder,
} from '../src/zstd.ts'
import { NodePrivateZstdFrameDecoder } from '../src/zstd-private-decoder.ts'
import { PublicZstdFrameDecoder } from '../src/zstd-public-decoder.ts'
@@ -157,7 +157,6 @@ describe('Zstandard frame structure', () => {
expect(first[4]! & 0x04).toBe(0x04)
expect(second[4]! & 0x04).toBe(0x04)
expect((await decompressZstdFrame(first)).toString()).toBe('header\n')
expect(decompressZstdFrameSync(second).toString()).toBe('event\n')
const decoder = createZstdFrameDecoder()
try {
const plaintext = Array.from(decoder.decode(stream, scanZstdFrames(stream).frames), chunk => Buffer.from(chunk))
@@ -185,6 +184,96 @@ describe('Zstandard frame structure', () => {
}
})
it('falls back to the public decoder when the private Node contract is unavailable', () => {
vi.spyOn(NodePrivateZstdFrameDecoder, 'create').mockReturnValue(undefined)
const decoder = createZstdFrameDecoder()
expect(decoder).toBeInstanceOf(PublicZstdFrameDecoder)
decoder.close()
})
it('enforces decoder lifecycle and checksum errors through both implementations', async () => {
const frame = await compressZstdFrame('frame\n')
const range = [{ start: 0, end: frame.length }]
const corrupt = Buffer.from(frame)
corrupt[corrupt.length - 1] = corrupt[corrupt.length - 1]! ^ 0xFF
const factories: Array<() => ZstdFrameDecoder> = [
() => new PublicZstdFrameDecoder(),
() => NodePrivateZstdFrameDecoder.create()!,
]
for (const create of factories) {
const interrupted = create()
const iterator = interrupted.decode(frame, range)
expect(iterator.next().value?.toString()).toBe('frame\n')
iterator.return()
expect(() => Array.from(interrupted.decode(frame, range))).toThrow(/already started/)
interrupted.close()
const closed = create()
closed.close()
closed.close()
expect(() => Array.from(closed.decode(frame, range))).toThrow(/closed/)
const invalid = create()
expect(() => Array.from(invalid.decode(corrupt, range))).toThrow(/frame at byte 0 failed validation/)
}
})
it('assembles private-decoder output at and beyond its reusable chunk boundary', async () => {
for (const length of [8, 9]) {
const plaintext = Buffer.alloc(length, 0x61)
const frame = await compressZstdFrame(plaintext)
const decoder = NodePrivateZstdFrameDecoder.create()!
;(decoder as unknown as { output: Buffer }).output = Buffer.allocUnsafe(8)
const [decoded] = Array.from(
decoder.decode(frame, [{ start: 0, end: frame.length }]),
chunk => Buffer.from(chunk),
)
expect(decoded).toEqual(plaintext)
}
})
it('normalizes private decoder stream failures', async () => {
interface PrivateDecoderInternals {
stream: {
[key: symbol]: unknown
emit(event: string, error: Error): boolean
}
errorKey: symbol
}
const frame = await compressZstdFrame('frame\n')
const range = [{ start: 0, end: frame.length }]
const emitted = NodePrivateZstdFrameDecoder.create()!
const emittedInternals = emitted as unknown as PrivateDecoderInternals
const first = new Error('first emitted decoder failure')
emittedInternals.stream.emit('error', first)
emittedInternals.stream.emit('error', new Error('later emitted decoder failure'))
try {
Array.from(emitted.decode(frame, range))
throw new Error('expected emitted decoder failure')
} catch (error) {
expect((error as Error).cause).toBe(first)
}
for (const internalFailure of [new Error('internal decoder failure'), 'not an Error']) {
const decoder = NodePrivateZstdFrameDecoder.create()!
const internals = decoder as unknown as PrivateDecoderInternals
internals.stream[internals.errorKey] = internalFailure
try {
Array.from(decoder.decode(frame, range))
throw new Error('expected internal decoder failure')
} catch (error) {
const cause = (error as Error).cause
if (internalFailure instanceof Error) {
expect(cause).toBe(internalFailure)
} else {
expect(cause).toMatchObject({ message: 'Zstandard decoder exposed a non-Error internal failure' })
}
}
}
})
it('distinguishes incomplete frame regions from invalid complete structure', () => {
expect(scanZstdFrames(MAGIC.subarray(0, 2))).toEqual({ frames: [], tornStart: 0 })
expect(scanZstdFrames(MAGIC)).toEqual({ frames: [], tornStart: 0 })