From 2f6d3b8539c482bb0fc3ce845eaa569479788284 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sun, 14 Jun 2026 00:06:25 +0800 Subject: [PATCH] test: property-based tests for protocol-shaped code (RFC 001) Adds fast-check + one tests/properties.spec.ts per protocol-shaped package (llm/BlockAssembler, session, tools/schema DSL, agent-loop scheduling). The tools suite includes the RFC 001<->005 composition property (generated args satisfying a spec pass validateArgs), closing the validator/InferArgs drift risk from ADR 0011. Loop properties are deterministic (settle on agent/status, no sleeps). The BlockAssembler suite found a real bug on first run: a duplicate block-end at the same index overwrote an already-flushed block, so the streamed prefix disagreed with final blocks(). Fixed (first close wins, matching the existing straggler rule) + regression test. Graduates RFC 001 -> ADR 0013. --- docs/adr/0013-property-based-testing.md | 23 +++ docs/adr/README.md | 1 + docs/rfc/001-property-based-testing.md | 2 +- docs/rfc/README.md | 2 +- package.json | 1 + packages/agent-loop/tests/properties.spec.ts | 138 ++++++++++++++++++ packages/llm/src/assembler.ts | 5 + packages/llm/tests/assembler.spec.ts | 35 +++++ packages/llm/tests/properties.spec.ts | 125 ++++++++++++++++ packages/session/tests/properties.spec.ts | 110 ++++++++++++++ packages/tools/tests/properties.spec.ts | 144 +++++++++++++++++++ yarn.lock | 17 +++ 12 files changed, 601 insertions(+), 2 deletions(-) create mode 100644 docs/adr/0013-property-based-testing.md create mode 100644 packages/agent-loop/tests/properties.spec.ts create mode 100644 packages/llm/tests/properties.spec.ts create mode 100644 packages/session/tests/properties.spec.ts create mode 100644 packages/tools/tests/properties.spec.ts diff --git a/docs/adr/0013-property-based-testing.md b/docs/adr/0013-property-based-testing.md new file mode 100644 index 0000000000..3d9a7e924f --- /dev/null +++ b/docs/adr/0013-property-based-testing.md @@ -0,0 +1,23 @@ +# ADR 0013: Property-based testing for protocol-shaped code + +Status: accepted (2026-06-14) + +## Context + +Example-based tests pin the cases we thought of. The harness's core is protocol-shaped — chunk streams, event logs, schema conversion, inbox scheduling — where the input space is combinatorial and the interesting bugs live in interleavings nobody wrote an example for. The motivating evidence: a `streamBlocks` ordering bug once survived 100% line coverage of the happy paths. Per-file 100% coverage proves every line ran, not that every interleaving is correct. + +## Decision + +Adopt `fast-check` (a root devDependency) with one `tests/properties.spec.ts` per protocol-shaped package, generators tuned for *realistic-but-adversarial* inputs (not uniform noise) and `numRuns` kept so the suite stays well under ~10s locally. Failures print a reproducible seed. + +- **dsh-llm / BlockAssembler:** arbitrary chunk streams (valid + malformed: duplicate indices, stragglers, missing block-start). Invariants: `flushReady()+flushRemaining() ≡ blocks()` in order; the streamed prefix is always a prefix of final `blocks()`; partial count ≤ distinct indices; re-assembly idempotent. +- **dsh-session:** arbitrary event logs. Invariants: `deriveMessages` deterministic; replay-from-seed identical; seq strictly monotonic; non-message events never affect derived history; derived content is decoupled from the log. +- **dsh-tools:** arbitrary `SchemaSpec`. Invariants: JSON Schema `required` equals the `required:true` keys at every level; conversion total; **and the RFC 001↔005 composition** — generated args satisfying a spec pass `validateArgs`, and targeted corruptions (dropped required key, non-object top level) are rejected. This closes the validator/`InferArgs` drift risk from ADR 0011. +- **dsh-agent-loop:** arbitrary send schedules against a never-exhausting adapter, driven through the `agent/status` settle signal (no wall-clock sleeps). Invariants: no message lost; turn numbers strictly increase; status transitions stay on the legal machine. + +## Consequences + +- Generator quality is the value lever — the generators bias toward small index pools and short strings so collisions and interleavings are common. +- **It already paid off:** the BlockAssembler stream found a real bug — a duplicate `block-end` at the same index overwrote an already-flushed block, so the streamed prefix disagreed with final `blocks()`. Fixed (first close wins, matching the existing straggler rule) with a dedicated regression test. +- A property flake from a timeout is a finding, not something to retry away. The loop properties are deterministic by construction (settle on `agent/status`), so a hang is a real defect. +- Property tests supplement, not replace, the example tests that pin specific branches for the 100%-coverage gate. diff --git a/docs/adr/README.md b/docs/adr/README.md index ee3ebde6c6..1842301e9e 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -24,3 +24,4 @@ Do NOT write an ADR for: a mechanical or local choice (a variable name, a one-fi | [0010](0010-twin-llm-adapters.md) | Two LLM adapters as a design-verification twin | accepted | | [0011](0011-runtime-arg-validation.md) | Runtime arg validation at the model boundary | accepted | | [0012](0012-dev-invariants-over-deep-readonly.md) | Dev-mode invariants over compile-time deep-readonly | accepted | +| [0013](0013-property-based-testing.md) | Property-based testing for protocol-shaped code | accepted | diff --git a/docs/rfc/001-property-based-testing.md b/docs/rfc/001-property-based-testing.md index cb519129de..17795ebfd7 100644 --- a/docs/rfc/001-property-based-testing.md +++ b/docs/rfc/001-property-based-testing.md @@ -1,6 +1,6 @@ # RFC 001: Property-based testing for protocol-shaped code -Status: proposed +Status: implemented — see [ADR 0013](../adr/0013-property-based-testing.md). (It found a real BlockAssembler duplicate-`block-end` bug on first run.) ## Problem diff --git a/docs/rfc/README.md b/docs/rfc/README.md index 4760bb98f6..fbbe442d31 100644 --- a/docs/rfc/README.md +++ b/docs/rfc/README.md @@ -4,7 +4,7 @@ Proposals for substantial future work — reviewed before implementation, unlike | # | Title | Status | |---|---|---| -| [001](001-property-based-testing.md) | Property-based testing for protocol-shaped code | proposed | +| [001](001-property-based-testing.md) | Property-based testing for protocol-shaped code | implemented | | [002](002-mutation-testing.md) | Mutation testing as the coverage counterweight | proposed | | [003](003-deterministic-and-stress-testing.md) | Deterministic tests + replay invariant fixture + race stress | proposed | | [004](004-architectural-conformance.md) | Architectural rules: dependency-cruiser, adapter conformance kit | proposed | diff --git a/package.json b/package.json index f4c3279082..4f752ac364 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "@vitest/coverage-v8": "^4.1.8", "@yarnpkg/types": "^4.0.1", "eslint": "^10.4.1", + "fast-check": "^4.8.0", "knip": "^6.16.1", "lefthook": "^2.1.9", "publint": "^0.3.21", diff --git a/packages/agent-loop/tests/properties.spec.ts b/packages/agent-loop/tests/properties.spec.ts new file mode 100644 index 0000000000..8e6768d402 --- /dev/null +++ b/packages/agent-loop/tests/properties.spec.ts @@ -0,0 +1,138 @@ +/** + * Property-based tests for the agent loop's inbox/turn scheduling (RFC 001 → + * ADR 0013). Deterministic by construction: schedules are driven through the + * `agent/status` settle signal (no wall-clock sleeps), so a flake is a finding, + * not timing noise. + * + * Invariants: every sent message appears exactly once in the log (none lost); + * turn numbers strictly increase; status transitions follow the legal machine + * idle→running→idle (and →disposed at teardown). + */ + +import { describe, expect, it } from 'vitest' +import { Context } from 'cordis' +import LlmService from '@deepseek-ai/dsh-llm' +import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm' +import { LlmAdapter } from '@deepseek-ai/dsh-llm' +import SessionStore from '@deepseek-ai/dsh-session' +import SystemPrompt from '@deepseek-ai/dsh-system-prompt' +import ToolRegistry from '@deepseek-ai/dsh-tools' +import AgentRegistry from '@deepseek-ai/dsh-agent' +import AgentLoop, { type LoopAgent } from '@deepseek-ai/dsh-agent-loop' +import fc from 'fast-check' + +/** A never-exhausting adapter: every model call returns the same short reply. */ +class EchoAdapter extends LlmAdapter { + async * stream(options: GenerateOptions): AsyncIterable { + if (options.signal?.aborted) throw new Error('aborted') + const text = 'ok' + yield { type: 'block-start', index: 0, blockType: 'text' } + yield { type: 'text-delta', index: 0, text } + yield { type: 'block-end', index: 0, block: { type: 'text', text } } + yield { type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } } + yield { type: 'finish', reason: { kind: 'stop' } } + } +} + +async function harness() { + const ctx = new Context() + await ctx.plugin(LlmService) + await ctx.plugin(SessionStore) + await ctx.plugin(SystemPrompt) + await ctx.plugin(ToolRegistry) + await ctx.plugin(AgentRegistry) + await ctx.plugin(AgentLoop, { agents: [] }) + ctx.llm.registerAdapter(['mock'], new EchoAdapter()) + return ctx +} + +/** Resolve on the agent's next transition to idle (event-based, not polled). */ +function nextIdle(ctx: Context, agent: LoopAgent): Promise { + return new Promise((resolve) => { + const dispose = ctx.on('agent/status', (subject, status) => { + if (subject === agent && status === 'idle') { + dispose() + resolve() + } + }) + }) +} + +/** Record every status transition for the legal-machine assertion. */ +function recordStatus(ctx: Context, agent: LoopAgent): string[] { + const seen: string[] = [] + ctx.on('agent/status', (subject, status) => { + if (subject === agent) seen.push(status) + }) + return seen +} + +function userMessageTexts(agent: LoopAgent): string[] { + return agent.session.events + .filter(e => e.type === 'user/message') + .map(e => (e.data as { content: { type: string; text?: string }[] }).content.map(b => b.text ?? '').join('')) +} + +function turnNumbers(agent: LoopAgent): number[] { + return agent.session.events + .filter(e => e.type === 'turn/start') + .map(e => (e.data as { turn: number }).turn) +} + +/** Assert a status trace is a legal run: idle/running alternating, ending idle. */ +function assertLegalStatusTrace(trace: string[]): void { + for (let i = 1; i < trace.length; i++) { + expect(trace[i]).not.toBe(trace[i - 1]) // no repeats (setStatus dedups) + } + for (const s of trace) expect(['idle', 'running']).toContain(s) +} + +describe('agent loop scheduling properties', () => { + it('a synchronous burst loses no message and uses strictly increasing turns', async () => { + await fc.assert(fc.asyncProperty( + fc.array(fc.string({ minLength: 1 }), { minLength: 1, maxLength: 6 }), + async (texts) => { + const ctx = await harness() + try { + const agent = ctx.agentLoop.create('a', { model: 'mock' }) + const trace = recordStatus(ctx, agent) + const idle = nextIdle(ctx, agent) + // Send all in one synchronous tick: they queue before the loop wakes. + for (const text of texts) agent.send([{ type: 'text', text }]) + await idle + + // No message lost: every send appears as a user/message, in order. + expect(userMessageTexts(agent)).toEqual(texts) + // Turn numbers strictly increase. + const turns = turnNumbers(agent) + for (let i = 1; i < turns.length; i++) expect(turns[i]!).toBeGreaterThan(turns[i - 1]!) + assertLegalStatusTrace(trace) + } finally { + await ctx.fiber.dispose() + } + }, + ), { numRuns: 25 }) + }) + + it('sequential sends each get their own turn with increasing numbers', async () => { + await fc.assert(fc.asyncProperty( + fc.array(fc.string({ minLength: 1 }), { minLength: 1, maxLength: 5 }), + async (texts) => { + const ctx = await harness() + try { + const agent = ctx.agentLoop.create('a', { model: 'mock' }) + for (const text of texts) { + const idle = nextIdle(ctx, agent) + agent.send([{ type: 'text', text }]) + await idle + } + // Each send was drained at a separate turn start: N turns, 1..N. + expect(turnNumbers(agent)).toEqual(texts.map((_, i) => i + 1)) + expect(userMessageTexts(agent)).toEqual(texts) + } finally { + await ctx.fiber.dispose() + } + }, + ), { numRuns: 20 }) + }) +}) diff --git a/packages/llm/src/assembler.ts b/packages/llm/src/assembler.ts index e18071d7e9..a61d6cf044 100644 --- a/packages/llm/src/assembler.ts +++ b/packages/llm/src/assembler.ts @@ -72,6 +72,11 @@ export class BlockAssembler { } case 'block-end': { const partial = this.ensure(chunk.index, chunk.block.type) + // First close wins: a second block-end for an already-closed index is + // a straggler (same rule as post-close deltas). Ignoring it keeps the + // streamed prefix and the final blocks() in agreement — otherwise a + // re-close could rewrite a block already flushed downstream. + if (partial.block) return partial.block = chunk.block return chunk.block } diff --git a/packages/llm/tests/assembler.spec.ts b/packages/llm/tests/assembler.spec.ts index e02e547e99..24639a04a2 100644 --- a/packages/llm/tests/assembler.spec.ts +++ b/packages/llm/tests/assembler.spec.ts @@ -166,3 +166,38 @@ describe('assertNever', () => { .toThrow('unreachable variant in BlockAssembler.push') }) }) + +describe('BlockAssembler regressions (property-test findings)', () => { + it('first block-end wins: a duplicate block-end for a closed index is ignored', () => { + // Found by fast-check (RFC 001): two block-ends at the same index made the + // streamed prefix (first block) disagree with final blocks() (second + // block). The first close must win — same straggler rule as post-close + // deltas — so streaming and one-shot assembly stay identical. + const chunks: StreamChunk[] = [ + { type: 'block-end', index: 0, block: { type: 'reasoning', text: 'first' } }, + { type: 'block-end', index: 0, block: { type: 'text', text: 'second' } }, + ] + const streaming = new BlockAssembler() + const flushed = [] + for (const chunk of chunks) { + streaming.push(chunk) + flushed.push(...streaming.flushReady()) + } + flushed.push(...streaming.flushRemaining()) + + const oneShot = new BlockAssembler() + for (const chunk of chunks) oneShot.push(chunk) + + expect(flushed).toEqual([{ type: 'reasoning', text: 'first' }]) + expect(oneShot.blocks()).toEqual([{ type: 'reasoning', text: 'first' }]) + expect(flushed).toEqual(oneShot.blocks()) + }) + + it('push returns undefined for a duplicate block-end (it closed nothing)', () => { + const a = new BlockAssembler() + expect(a.push({ type: 'block-end', index: 0, block: { type: 'text', text: 'x' } })) + .toEqual({ type: 'text', text: 'x' }) + expect(a.push({ type: 'block-end', index: 0, block: { type: 'text', text: 'y' } })) + .toBeUndefined() + }) +}) diff --git a/packages/llm/tests/properties.spec.ts b/packages/llm/tests/properties.spec.ts new file mode 100644 index 0000000000..60c546aa39 --- /dev/null +++ b/packages/llm/tests/properties.spec.ts @@ -0,0 +1,125 @@ +/** + * Property-based tests for the BlockAssembler (RFC 001 → ADR 0013). + * + * The assembler is protocol-shaped: arbitrary interleavings of block-start, + * deltas, block-end, usage, and finish — valid and malformed (duplicate + * indices, stragglers after block-end, missing block-start, delta-only). The + * invariants below are the contract the agent loop and LlmService rely on. + */ + +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { BlockAssembler } from '@deepseek-ai/dsh-llm' +import type { ContentBlock, StreamChunk } from '@deepseek-ai/dsh-llm' +import { CallId } from '@deepseek-ai/dsh-llm' + +// A small pool of indices so collisions (duplicate-index bugs) are common. +const indexArb = fc.integer({ min: 0, max: 4 }) + +const blockEndArb = (index: number): fc.Arbitrary => fc.oneof( + fc.record({ text: fc.string() }).map((r): StreamChunk => ( + { type: 'block-end', index, block: { type: 'text', text: r.text } } + )), + fc.record({ text: fc.string() }).map((r): StreamChunk => ( + { type: 'block-end', index, block: { type: 'reasoning', text: r.text } } + )), + fc.record({ id: fc.string({ minLength: 1 }), name: fc.string(), args: fc.string() }).map((r): StreamChunk => ( + { type: 'block-end', index, block: { type: 'tool-call', id: CallId(r.id), name: r.name, arguments: r.args } } + )), +) + +/** One arbitrary chunk over the small index pool — valid and malformed mixes. */ +const chunkArb: fc.Arbitrary = indexArb.chain(index => fc.oneof( + fc.constant({ type: 'block-start', index, blockType: 'text' }), + fc.constant({ type: 'block-start', index, blockType: 'reasoning' }), + fc.constant({ type: 'block-start', index, blockType: 'tool-call' }), + fc.string().map((text): StreamChunk => ({ type: 'text-delta', index, text })), + fc.string().map((text): StreamChunk => ({ type: 'reasoning-delta', index, text })), + fc.record({ id: fc.string({ minLength: 1 }), argumentsDelta: fc.string() }) + .map((r): StreamChunk => ({ type: 'tool-call-delta', index, id: CallId(r.id), argumentsDelta: r.argumentsDelta })), + blockEndArb(index), + fc.constant({ type: 'usage', usage: { inputTokens: 1, outputTokens: 1 } }), +)) + +/** A stream is a list of chunks; we add the terminal `finish` ourselves. */ +const streamArb = fc.array(chunkArb, { maxLength: 30 }) + +/** Feed a fresh assembler, return it. */ +function feed(chunks: StreamChunk[]): BlockAssembler { + const a = new BlockAssembler() + for (const chunk of chunks) a.push(chunk) + return a +} + +describe('BlockAssembler properties', () => { + it('flushReady() ++ flushRemaining() === blocks(), in order', () => { + fc.assert(fc.property(streamArb, (chunks) => { + const streaming = new BlockAssembler() + const flushed: ContentBlock[] = [] + for (const chunk of chunks) { + streaming.push(chunk) + flushed.push(...streaming.flushReady()) + } + flushed.push(...streaming.flushRemaining()) + + const oneShot = feed(chunks).blocks() + expect(flushed).toEqual(oneShot) + })) + }) + + it('streamBlocks-style flush never yields a block before an earlier open one', () => { + // flushReady is strict-order: once it stops at an open index, no later + // index may be emitted until that one closes. We assert the flushed prefix + // is always a prefix of the final blocks() order. + fc.assert(fc.property(streamArb, (chunks) => { + const streaming = new BlockAssembler() + const flushed: ContentBlock[] = [] + for (const chunk of chunks) { + streaming.push(chunk) + flushed.push(...streaming.flushReady()) + } + const finalSoFar = streaming.blocks() + // Everything flushed mid-stream is a prefix of the full ordered blocks. + expect(finalSoFar.slice(0, flushed.length)).toEqual(flushed) + })) + }) + + it('partials map size never exceeds the number of distinct indices seen', () => { + fc.assert(fc.property(streamArb, (chunks) => { + const distinct = new Set() + for (const chunk of chunks) { + if ('index' in chunk) distinct.add(chunk.index) + } + const a = feed(chunks) + // blocks() length equals the number of distinct indices that became + // partials (block-bearing chunks). It can never exceed distinct indices. + expect(a.blocks().length).toBeLessThanOrEqual(distinct.size) + })) + }) + + it('re-assembly is idempotent: blocks() is stable across repeated calls', () => { + fc.assert(fc.property(streamArb, (chunks) => { + const a = feed(chunks) + expect(a.blocks()).toEqual(a.blocks()) + // And message().content mirrors blocks(). + expect(a.message().content).toEqual(a.blocks()) + })) + }) + + it('blocks() never throws and yields only valid content-block tags', () => { + fc.assert(fc.property(streamArb, (chunks) => { + const blocks = feed(chunks).blocks() + for (const block of blocks) { + expect(['text', 'reasoning', 'tool-call', 'tool-result', 'image']).toContain(block.type) + } + })) + }) + + it('result().finish defaults to stop when no finish chunk arrives', () => { + fc.assert(fc.property(streamArb, (chunks) => { + const a = feed(chunks) + const hasFinish = chunks.some(c => c.type === 'finish') + if (!hasFinish) expect(a.finish).toEqual({ kind: 'stop' }) + })) + }) +}) diff --git a/packages/session/tests/properties.spec.ts b/packages/session/tests/properties.spec.ts new file mode 100644 index 0000000000..2b31d6e83b --- /dev/null +++ b/packages/session/tests/properties.spec.ts @@ -0,0 +1,110 @@ +/** + * Property-based tests for the Session event log (RFC 001 → ADR 0013). + * + * Generates arbitrary event logs and asserts the derivation invariants the + * agent loop and replay depend on: deriveMessages is deterministic and + * replay-from-seed reproduces it; seq is strictly monotonic; non-message + * events never affect derived history. + */ + +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { CallId } from '@deepseek-ai/dsh-llm' +import { Session, SessionId } from '@deepseek-ai/dsh-session' +import type { SessionEventMap, SessionEventType } from '@deepseek-ai/dsh-session' + +type Appendable = { [T in SessionEventType]: { type: T; data: SessionEventMap[T] } }[SessionEventType] + +const textContentArb = fc.array( + fc.record({ type: fc.constant<'text'>('text'), text: fc.string() }), + { maxLength: 3 }, +) + +// A message-producing event (these DO affect derived history). +const messageEventArb: fc.Arbitrary = fc.oneof( + textContentArb.map((content): Appendable => ({ type: 'user/message', data: { content, source: { kind: 'user' } } })), + textContentArb.map((content): Appendable => ({ type: 'assistant/message', data: { turn: 1, step: 1, content } })), + fc.record({ id: fc.string({ minLength: 1 }), content: textContentArb, isError: fc.boolean() }) + .map((r): Appendable => ({ type: 'tool/result', data: { turn: 1, step: 1, callId: CallId(r.id), content: r.content, isError: r.isError } })), +) + +// A non-message event (trace/replay data — must NOT affect derived history). +const nonMessageEventArb: fc.Arbitrary = fc.oneof( + fc.constant({ type: 'turn/start', data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }), + fc.constant({ type: 'turn/end', data: { turn: 1, reason: { kind: 'completed' } } }), + fc.constant({ type: 'step/start', data: { turn: 1, step: 1 } }), + fc.constant({ type: 'step/end', data: { turn: 1, step: 1 } }), + fc.string().map((text): Appendable => ({ type: 'assistant/chunk', data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text } } })), + fc.constant({ type: 'usage', data: { turn: 1, step: 1, usage: { inputTokens: 1, outputTokens: 1 } } }), + fc.constant({ type: 'error', data: { turn: 1, step: 1, message: 'x' } }), +) + +const anyEventArb = fc.oneof(messageEventArb, nonMessageEventArb) +const logArb = fc.array(anyEventArb, { maxLength: 25 }) + +let counter = 0 +function build(events: Appendable[]): Session { + const session = new Session(SessionId(`prop-${counter++}`)) + for (const e of events) session.append(e.type, e.data) + return session +} + +describe('Session properties', () => { + it('deriveMessages is deterministic (same log → identical derivation)', () => { + fc.assert(fc.property(logArb, (events) => { + const a = build(events) + expect(a.deriveMessages()).toEqual(a.deriveMessages()) + })) + }) + + it('seq is strictly monotonic and zero-based contiguous', () => { + fc.assert(fc.property(logArb, (events) => { + const session = build(events) + session.events.forEach((event, i) => { expect(event.seq).toBe(i) }) + expect(session.seq).toBe(events.length) + })) + }) + + it('replay-from-seed reproduces the derivation identically', () => { + fc.assert(fc.property(logArb, (events) => { + const original = build(events) + const replayed = new Session(SessionId(`replay-${counter++}`), [...original.events]) + expect(replayed.deriveMessages()).toEqual(original.deriveMessages()) + expect(replayed.seq).toBe(original.seq) + })) + }) + + it('non-message events never affect derived history', () => { + fc.assert(fc.property( + fc.array(messageEventArb, { maxLength: 12 }), + fc.array(nonMessageEventArb, { maxLength: 12 }), + (messages, noise) => { + // The same message events, with and without interleaved noise, derive + // the same history (noise is inserted at arbitrary positions). + const clean = build(messages).deriveMessages() + const interleaved: Appendable[] = [] + const maxLen = Math.max(messages.length, noise.length) + for (let i = 0; i < maxLen; i++) { + if (i < noise.length) interleaved.push(noise[i]!) + if (i < messages.length) interleaved.push(messages[i]!) + } + const withNoise = build(interleaved).deriveMessages() + expect(withNoise).toEqual(clean) + }, + )) + }) + + it('every derived message has a known role and decoupled content', () => { + fc.assert(fc.property(logArb, (events) => { + const session = build(events) + const messages = session.deriveMessages() + const before = structuredClone(session.events) + for (const m of messages) { + expect(['user', 'assistant', 'system']).toContain(m.role) + // Mutating derived content must not touch the log (append-only). + m.content.push({ type: 'text', text: 'mutation' }) + } + expect(session.events).toEqual(before) + })) + }) +}) diff --git a/packages/tools/tests/properties.spec.ts b/packages/tools/tests/properties.spec.ts new file mode 100644 index 0000000000..a5d44baf2d --- /dev/null +++ b/packages/tools/tests/properties.spec.ts @@ -0,0 +1,144 @@ +/** + * Property-based tests for the tool-schema DSL (RFC 001 → ADR 0013), including + * the RFC 001 ↔ 005 composition: generated args that satisfy a SchemaSpec must + * pass validateArgs, and targeted corruptions must be rejected. This closes the + * validator/InferArgs drift risk noted in ADR 0011. + */ + +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { schemaSpecToJsonSchema, validateArgs } from '@deepseek-ai/dsh-tools' +import type { SchemaProp, SchemaSpec } from '@deepseek-ai/dsh-tools' + +// A leaf prop arbitrary (no nesting) with optional required/enum. +function leafPropArb(): fc.Arbitrary { + return fc.oneof( + fc.record({ required: fc.boolean() }).map(({ required }): SchemaProp => ({ type: 'string', ...required ? { required: true } : {} })), + fc.record({ required: fc.boolean() }).map(({ required }): SchemaProp => ({ type: 'number', ...required ? { required: true } : {} })), + fc.record({ required: fc.boolean() }).map(({ required }): SchemaProp => ({ type: 'boolean', ...required ? { required: true } : {} })), + fc.record({ values: fc.uniqueArray(fc.string({ minLength: 1 }), { minLength: 1, maxLength: 3 }), required: fc.boolean() }) + .map(({ values, required }): SchemaProp => ({ type: 'string', enum: values, ...required ? { required: true } : {} })), + ) +} + +/** A prop arbitrary up to `depth` levels of nesting (objects and arrays). */ +function propArb(depth: number): fc.Arbitrary { + if (depth <= 0) return leafPropArb() + return fc.oneof( + { weight: 3, arbitrary: leafPropArb() }, + { + weight: 1, + arbitrary: fc.record({ properties: specArb(depth - 1), required: fc.boolean() }) + .map(({ properties, required }): SchemaProp => ({ type: 'object', properties, ...required ? { required: true } : {} })), + }, + { + weight: 1, + arbitrary: fc.record({ items: propArb(depth - 1), required: fc.boolean() }) + .map(({ items, required }): SchemaProp => ({ type: 'array', items, ...required ? { required: true } : {} })), + }, + ) +} + +function specArb(depth: number): fc.Arbitrary { + return fc.dictionary(fc.string({ minLength: 1, maxLength: 6 }), propArb(depth), { maxKeys: 4 }) +} + +/** Generate a value that satisfies a prop (used to build valid args). */ +function valueForProp(prop: SchemaProp): fc.Arbitrary { + switch (prop.type) { + case 'string': return prop.enum ? fc.constantFrom(...prop.enum) : fc.string() + case 'number': return fc.double({ noNaN: true }) + case 'boolean': return fc.boolean() + case 'object': return prop.properties ? validArgsForSpec(prop.properties) : fc.constant({}) + case 'array': return prop.items ? fc.array(valueForProp(prop.items), { maxLength: 3 }) : fc.constant([]) + } +} + +/** Generate args satisfying every required key of a spec (optionals included randomly). */ +function validArgsForSpec(spec: SchemaSpec): fc.Arbitrary> { + const entries = Object.entries(spec) + return fc.tuple(...entries.map(([key, prop]) => + fc.tuple( + fc.constant(key), + // required keys are always present; optional keys are present ~half the time + prop.required === true + ? valueForProp(prop).map(v => ({ include: true, value: v })) + : fc.oneof( + valueForProp(prop).map(v => ({ include: true, value: v })), + fc.constant({ include: false, value: undefined }), + ), + ), + )).map((pairs) => { + const out: Record = {} + for (const [key, { include, value }] of pairs) if (include) out[key] = value + return out + }) +} + +/** Collect the `required: true` keys at the top level of a spec. */ +function requiredKeys(spec: SchemaSpec): string[] { + return Object.entries(spec).filter(([, p]) => p.required === true).map(([k]) => k) +} + +describe('schema DSL properties', () => { + it('JSON Schema `required` equals the required:true keys at every level', () => { + fc.assert(fc.property(specArb(2), (spec) => { + const checkLevel = (s: SchemaSpec, json: { required?: string[]; properties: Record }) => { + expect(new Set(json.required ?? [])).toEqual(new Set(requiredKeys(s))) + for (const [key, prop] of Object.entries(s)) { + const propJson = json.properties[key] as Record + if (prop.type === 'object' && prop.properties) { + checkLevel(prop.properties, propJson as { required?: string[]; properties: Record }) + } + } + } + checkLevel(spec, schemaSpecToJsonSchema(spec)) + })) + }) + + it('conversion is total (never throws) for any spec', () => { + fc.assert(fc.property(specArb(3), (spec) => { + expect(() => schemaSpecToJsonSchema(spec)).not.toThrow() + })) + }) + + it('validateArgs is total (never throws) for any spec and any input', () => { + fc.assert(fc.property(specArb(2), fc.anything(), (spec, args) => { + expect(() => validateArgs(spec, args)).not.toThrow() + })) + }) + + it('RFC 001↔005: args satisfying the spec pass validateArgs', () => { + fc.assert(fc.property( + specArb(2).chain(spec => fc.tuple(fc.constant(spec), validArgsForSpec(spec))), + ([spec, args]) => { + expect(validateArgs(spec, args)).toEqual([]) + }, + )) + }) + + it('RFC 001↔005: dropping a required key is always rejected', () => { + fc.assert(fc.property( + specArb(1) + .filter(spec => requiredKeys(spec).length > 0) + .chain(spec => fc.tuple(fc.constant(spec), validArgsForSpec(spec))), + ([spec, args]) => { + const required = requiredKeys(spec) + const victim = required[0]! + const broken = Object.fromEntries(Object.entries(args).filter(([k]) => k !== victim)) + const violations = validateArgs(spec, broken) + expect(violations.some(v => v.includes(`"${victim}"`))).toBe(true) + }, + )) + }) + + it('RFC 001↔005: a non-object top level is always rejected', () => { + fc.assert(fc.property( + specArb(1), + fc.oneof(fc.string(), fc.integer(), fc.boolean(), fc.constant(null), fc.array(fc.anything())), + (spec, notAnObject) => { + expect(validateArgs(spec, notAnObject).length).toBeGreaterThan(0) + }, + )) + }) +}) diff --git a/yarn.lock b/yarn.lock index a57f97dadc..ee8d65477f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -673,6 +673,7 @@ __metadata: "@vitest/coverage-v8": "npm:^4.1.8" "@yarnpkg/types": "npm:^4.0.1" eslint: "npm:^10.4.1" + fast-check: "npm:^4.8.0" knip: "npm:^6.16.1" lefthook: "npm:^2.1.9" publint: "npm:^0.3.21" @@ -2829,6 +2830,15 @@ __metadata: languageName: node linkType: hard +"fast-check@npm:^4.8.0": + version: 4.8.0 + resolution: "fast-check@npm:4.8.0" + dependencies: + pure-rand: "npm:^8.0.0" + checksum: 10c0/f72556a29db4ff386a8b6e50d420b06c7e5eaafff7db5560a99136c57d8d4777998155eb02d1bbeff396f575cc0b1442c8a1c4ddb798c4a919b542de1a1904ff + languageName: node + linkType: hard + "fast-deep-equal@npm:^3.1.1, fast-deep-equal@npm:^3.1.3": version: 3.1.3 resolution: "fast-deep-equal@npm:3.1.3" @@ -4048,6 +4058,13 @@ __metadata: languageName: node linkType: hard +"pure-rand@npm:^8.0.0": + version: 8.4.0 + resolution: "pure-rand@npm:8.4.0" + checksum: 10c0/6414bbc1c6f45fb774173431c7205e79783b77cfae0e2145e741b6999363554dbd2f4210d2a5bc08683e0b2f6823198c9308766b1d0911e1dccd7beb8842f860 + languageName: node + linkType: hard + "quansync@npm:^1.0.0": version: 1.0.0 resolution: "quansync@npm:1.0.0"