Add generated persistence log event catalog with freshness + completeness gates

docs/persistence-catalog/log-events.md enumerates every SessionEventMap member — the owning dsh-session vocabulary plus the dsh-compact and dsh-hook-protocol declaration merges — with payload, surface/log-only badge, JSDoc prose, and declaration site. scripts/gen-persistence-catalog.ts is a pure AST pass in the gen-cordis-catalog mold: verify-persistence-catalog (--check) joins doc-sync, so a stale committed catalog fails pre-push and CI. The walk enforces JSDoc completeness (every member needs description prose; @mode is rejected as a category error — log events do not dispatch on the cordis bus), derives the surface badge from the SurfaceEventType union with a stale-member cross-check, and hard-errors on duplicate declarations. Payloads render through the TypeScript printer so newline-separated multi-line type literals still emit valid one-line fragments. Documented the five previously JSDoc-less core events (turn/step boundaries, tool/call), removed the two stray @mode tags on the hook/* merges, and replaced the hand-restated event enumerations (session.md hook/* table, compact README table, hook-protocol README bullets, session README name-list — whose merge note had already drifted) with links to the catalog. RFC: docs/rfc/implemented/process/2026-07-04-persistence-log-catalog.md.
2026-07-04 22:58:28 +08:00
parent c2c13529cd
commit 232f314c3a
16 changed files with 883 additions and 40 deletions
@@ -0,0 +1,388 @@
+/**
+ * Generate (and verify) the persistence log event catalog in
+ * docs/persistence-catalog/log-events.md.
+ *
+ * The catalog is the ON-DISK-vocabulary reference: every event type that can
+ * appear in a session's durable event log — every member of the
+ * merge-extensible `SessionEventMap`, across the owning declaration in
+ * `@deepseek-ai/dsh-session` and every plugin declaration merge. It complements
+ * the cordis events/services catalog (the live bus wiring — a log event is NOT
+ * a cordis event; it reaches listeners via the single `session/event` emit) and
+ * the core-data-structures session page (the `SessionEvent` envelope and
+ * derivation semantics): this page is the RECORDS a persisted log can contain.
+ *
+ *   `tsx scripts/gen-persistence-catalog.ts`          → write the catalog
+ *   `tsx scripts/gen-persistence-catalog.ts --check`  → exit 1 if the committed
+ *                                                       file is stale (CI /
+ *                                                       pre-push gate)
+ *
+ * Like its AST sibling `gen-cordis-catalog.ts` (and unlike the boot-based
+ * `gen-tool-catalog.ts`), this is a pure source pass: every log event is a
+ * string-literal-named property with a static type annotation, so the AST is
+ * the whole truth and a brand-new event (core or merged) appears in the next
+ * regenerate — an un-regenerated file fails `--check`. The walk enforces JSDoc
+ * COMPLETENESS on the whole vocabulary: every member carries description prose
+ * (it becomes the catalog entry), and an `@mode` tag on a member is a hard
+ * error — dispatch modes belong to cordis bus events, and a log event has none
+ * (see docs/rfc/implemented/process/2026-07-04-persistence-log-catalog.md).
+ * Violations aggregate into ONE error listing every offender.
+ *
+ * The surface/log-only badge is parsed from the `SurfaceEventType` union in the
+ * owning package (never hand-listed here), and every union member must name a
+ * collected event — a stale union member is a hard error.
+ *
+ * Payload fences use the ` ```ts persistence-catalog ` info string:
+ * doc-typecheck recognizes it and skips compilation (a bare payload fragment is
+ * not standalone-compilable), excluded from the opt-out ratio.
+ */
+
+import { globSync, readFileSync, writeFileSync } from 'node:fs'
+import { resolve } from 'node:path'
+import ts from 'typescript'
+
+const root = resolve(import.meta.dirname, '..')
+const OUT = 'docs/persistence-catalog/log-events.md'
+
+/** The fenced-block info string for generated payload blocks (skipped by
+ * doc-typecheck, since a bare payload fragment is not standalone-compilable). */
+const FENCE = 'ts persistence-catalog'
+
+/** The package whose module id plugin merges augment (`declare module '…'`). */
+const SESSION_MODULE = '@deepseek-ai/dsh-session'
+
+/**
+ * Cross-link map: a type name that appears in a payload → the
+ * core-data-structures page that documents it (path relative to OUT's folder).
+ * Hand-curated and catalog-owned, same policy as the cordis catalog's map: each
+ * name resolves to exactly one PRIMARY page. A payload type with no
+ * core-data-structures home (e.g. `HookDialect`, documented in its package)
+ * simply gets no link.
+ */
+const LINK_MAP: Record<string, string> = {
+  CallId: 'core.md',
+  ContentBlock: 'core.md',
+  MessageSource: 'core.md',
+  StreamChunk: 'llm-streaming.md',
+  TokenUsage: 'llm-streaming.md',
+  TodoItem: 'session.md',
+  TurnTrigger: 'session.md',
+  TurnEndReason: 'session.md',
+}
+
+/** One log event, extracted from a `SessionEventMap` declaration. */
+export interface LogEventEntry {
+  /** Scoped name, e.g. `turn/start`. */
+  name: string
+  /** The scope prefix, e.g. `turn` (everything before the first `/`). */
+  scope: string
+  /** Payload type text (the member's type annotation, whitespace-collapsed). */
+  payload: string
+  /** Description prose (the member's JSDoc), one line per paragraph. */
+  doc: string
+  /** Source pointer `packages/…/file.ts:line` of the declaration. */
+  source: string
+}
+
+/** A {@link LogEventEntry} plus its surface-eligibility badge. */
+export interface AnnotatedLogEventEntry extends LogEventEntry {
+  /** Whether the type is a `SurfaceEventType` member (may carry `surfaceOp`). */
+  surface: boolean
+}
+
+/** Repo-relative source pointer `file:line` for a node's first character. */
+function pointer(rel: string, sf: ts.SourceFile, node: ts.Node): string {
+  const { line } = sf.getLineAndCharacterOfPosition(node.getStart(sf))
+  return `${rel}:${line + 1}`
+}
+
+const printer = ts.createPrinter({ removeComments: true })
+
+/**
+ * One-line payload text for a member's type annotation. Printed through the
+ * TypeScript printer (not sliced from source text): the printer emits `;`
+ * member separators regardless of how the source separated them, so a
+ * multi-line newline-separated type literal still collapses to a VALID
+ * single-line fragment. The trailing `;` the printer puts before every `}` is
+ * dropped to match the repo's inline-literal style.
+ */
+function payloadText(type: ts.TypeNode, sf: ts.SourceFile): string {
+  return printer.printNode(ts.EmitHint.Unspecified, type, sf)
+    .replace(/\s+/g, ' ')
+    .replace(/;\s*\}/g, ' }')
+    .trim()
+}
+
+/** The raw `/** … */` JSDoc block immediately preceding a node, or '' if none. */
+function rawJsDoc(text: string, node: ts.Node): string {
+  const ranges = ts.getLeadingCommentRanges(text, node.getFullStart()) ?? []
+  const jsdoc = ranges.filter(r => text.slice(r.pos, r.pos + 3) === '/**').at(-1)
+  return jsdoc ? text.slice(jsdoc.pos, jsdoc.end) : ''
+}
+
+/**
+ * Parse a raw JSDoc block into description prose, flagging whether any `@mode`
+ * tag is present (forbidden on log events). Output obeys the repo's markdown
+ * conventions so the generated file passes verify-md-wrap: each prose paragraph
+ * collapses to ONE physical line, and a `-` bullet list is preserved with each
+ * item on its own single line (continuation lines folded in). `{@link Foo}`
+ * unwraps to `Foo`. Description prose ends at the FIRST block tag (standard
+ * JSDoc semantics): tag lines and their continuation lines are never prose.
+ */
+function parseJsDoc(raw: string): { doc: string; hasMode: boolean } {
+  const inner = raw
+    .replace(/^\/\*\*/, '')
+    .replace(/\*\/$/, '')
+    .split('\n')
+    .map(l => l.replace(/^\s*\*?\s?/, '').replace(/\s+$/, ''))
+  let hasMode = false
+  let inTags = false
+  const blocks: string[] = []
+  let para: string[] = []
+  let list: string[] = []
+  let item: string[] = []
+  const join = (parts: string[]): string => parts.join(' ').replace(/\s+/g, ' ').trim()
+  const flushItem = (): void => {
+    if (item.length) list.push(join(item))
+    item = []
+  }
+  const flushList = (): void => {
+    flushItem()
+    if (list.length) blocks.push(list.join('\n')) // one block, items on own lines
+    list = []
+  }
+  const flushPara = (): void => {
+    flushList()
+    if (para.length) blocks.push(join(para))
+    para = []
+  }
+  for (const line of inner) {
+    if (/^@mode\b/.test(line)) { hasMode = true; flushPara(); inTags = true; continue }
+    if (line.startsWith('@')) { flushPara(); inTags = true; continue }
+    if (inTags) continue // block-tag territory: continuations are never prose
+    if (line.trim() === '') { flushPara(); continue }
+    if (/^-\s+/.test(line)) {
+      // A list item starts: a pending paragraph (e.g. an intro line directly
+      // above the list, no blank between) flushes FIRST so it renders above.
+      flushItem()
+      if (para.length) { blocks.push(join(para)); para = [] }
+      item.push(line)
+      continue
+    }
+    if (item.length) { item.push(line); continue } // continuation of current item
+    para.push(line)
+  }
+  flushPara()
+  const doc = blocks.join('\n\n').replace(/\{@link\s+([^}]+)\}/g, '$1').trim()
+  return { doc, hasMode }
+}
+
+/**
+ * Throw one aggregate error for every completeness violation a walk collected.
+ * Aggregation is deliberate: a remediation pass sees the whole list at once
+ * instead of replaying the gate once per offender.
+ */
+function reportViolations(violations: string[]): void {
+  if (violations.length === 0) return
+  throw new Error(
+    `gen-persistence-catalog: ${violations.length} JSDoc completeness violation(s):\n`
+    + violations.map(v => `  ${v}`).join('\n'),
+  )
+}
+
+/**
+ * Every `interface SessionEventMap` declaration in a source file: the owning
+ * top-level declaration (in `@deepseek-ai/dsh-session`) and any declaration
+ * merge inside a `declare module '@deepseek-ai/dsh-session'` block. Both forms
+ * declare members of the SAME merged interface, so both are catalogued
+ * uniformly; nothing else in the repo may name an interface `SessionEventMap`.
+ */
+function sessionEventMapDecls(sf: ts.SourceFile): ts.InterfaceDeclaration[] {
+  const decls: ts.InterfaceDeclaration[] = []
+  for (const stmt of sf.statements) {
+    if (ts.isInterfaceDeclaration(stmt) && stmt.name.text === 'SessionEventMap') decls.push(stmt)
+    if (ts.isModuleDeclaration(stmt) && ts.isStringLiteral(stmt.name) && stmt.name.text === SESSION_MODULE
+      && stmt.body && ts.isModuleBlock(stmt.body)) {
+      for (const inner of stmt.body.statements) {
+        if (ts.isInterfaceDeclaration(inner) && inner.name.text === 'SessionEventMap') decls.push(inner)
+      }
+    }
+  }
+  return decls
+}
+
+/**
+ * Walk every `SessionEventMap` declaration (the owning interface plus every
+ * plugin declaration merge) and extract its events, hard-erroring (aggregated)
+ * on any completeness violation: a member without description prose, an
+ * `@mode` tag (a category error — log events have no dispatch mode), a
+ * non-literal member name, or the same event declared twice.
+ * `scanRoot` defaults to the repo root; tests pass a fixture dir.
+ */
+export function collectLogEvents(scanRoot: string = root): LogEventEntry[] {
+  const entries: LogEventEntry[] = []
+  const violations: string[] = []
+  const seen = new Map<string, string>()
+  for (const rel of globSync('packages/*/*/src/**/*.ts', { cwd: scanRoot }).sort()) {
+    const abs = resolve(scanRoot, rel)
+    const text = readFileSync(abs, 'utf8')
+    if (!text.includes('SessionEventMap')) continue
+    const sf = ts.createSourceFile(abs, text, ts.ScriptTarget.Latest, true)
+    for (const decl of sessionEventMapDecls(sf)) {
+      for (const member of decl.members) {
+        if (!ts.isPropertySignature(member) || !member.type) continue
+        const src = pointer(rel, sf, member)
+        if (!ts.isStringLiteral(member.name)) {
+          violations.push(`log event at ${src} has a non-literal name; the catalog needs string-literal event names.`)
+          continue
+        }
+        const name = member.name.text
+        const where = `log event '${name}' (${src})`
+        const prior = seen.get(name)
+        if (prior) {
+          violations.push(`${where} is already declared at ${prior}; an event type has exactly one declaration.`)
+          continue
+        }
+        seen.set(name, src)
+        const payload = payloadText(member.type, sf)
+        const { doc, hasMode } = parseJsDoc(rawJsDoc(text, member))
+        if (hasMode) {
+          violations.push(`${where} carries an @mode tag, but a log event has no dispatch mode (it is not a cordis bus event — it rides the 'session/event' emit). Remove the tag.`)
+        }
+        if (!doc) {
+          violations.push(`${where} has no description prose. Say what the event records and what its payload means — the JSDoc becomes the catalog entry.`)
+        }
+        entries.push({ name, scope: name.split('/')[0] ?? name, payload, doc, source: src })
+      }
+    }
+  }
+  reportViolations(violations)
+  return entries
+}
+
+/**
+ * Parse the `SurfaceEventType` union — the surface-eligible subset of event
+ * types — from source. Hard-errors when the alias is missing, declared more
+ * than once, or contains a non-string-literal member: the badge derivation
+ * relies on the union being a closed set of literal event names.
+ * `scanRoot` defaults to the repo root; tests pass a fixture dir.
+ */
+export function collectSurfaceEventTypes(scanRoot: string = root): string[] {
+  const found: { names: string[]; source: string }[] = []
+  for (const rel of globSync('packages/*/*/src/**/*.ts', { cwd: scanRoot }).sort()) {
+    const abs = resolve(scanRoot, rel)
+    const text = readFileSync(abs, 'utf8')
+    if (!text.includes('SurfaceEventType')) continue
+    const sf = ts.createSourceFile(abs, text, ts.ScriptTarget.Latest, true)
+    for (const stmt of sf.statements) {
+      if (!ts.isTypeAliasDeclaration(stmt) || stmt.name.text !== 'SurfaceEventType') continue
+      const src = pointer(rel, sf, stmt)
+      const members = ts.isUnionTypeNode(stmt.type) ? [...stmt.type.types] : [stmt.type]
+      const names: string[] = []
+      for (const m of members) {
+        if (ts.isLiteralTypeNode(m) && ts.isStringLiteral(m.literal)) names.push(m.literal.text)
+        else throw new Error(`gen-persistence-catalog: SurfaceEventType (${src}) has a non-string-literal member; the badge derivation needs a closed literal union.`)
+      }
+      found.push({ names, source: src })
+    }
+  }
+  const only = found[0]
+  if (!only) throw new Error('gen-persistence-catalog: no SurfaceEventType union found under packages/*/*/src.')
+  if (found.length > 1) throw new Error(`gen-persistence-catalog: SurfaceEventType is declared more than once (${found.map(f => f.source).join(', ')}); the surface subset has exactly one owner.`)
+  return only.names
+}
+
+/**
+ * Attach the surface/log-only badge to each event. Hard-errors when a
+ * `SurfaceEventType` union member names no collected event — a stale union
+ * member would otherwise silently badge nothing.
+ */
+export function annotateSurface(events: LogEventEntry[], surfaceTypes: string[]): AnnotatedLogEventEntry[] {
+  const names = new Set(events.map(e => e.name))
+  const stale = surfaceTypes.filter(t => !names.has(t))
+  if (stale.length > 0) {
+    throw new Error(`gen-persistence-catalog: SurfaceEventType member(s) ${stale.map(t => `'${t}'`).join(', ')} name no declared log event (stale union member?).`)
+  }
+  const surface = new Set(surfaceTypes)
+  return events.map(e => ({ ...e, surface: surface.has(e.name) }))
+}
+
+/** Render the cross-link "Types:" line for a payload, or '' if none apply. */
+function typeLinks(payload: string): string {
+  const seen = new Set<string>()
+  for (const name of Object.keys(LINK_MAP)) {
+    if (new RegExp(`\\b${name}\\b`).test(payload)) seen.add(name)
+  }
+  if (seen.size === 0) return ''
+  const links = [...seen].sort().map(n => `[${n}](../core-data-structures/${LINK_MAP[n]})`)
+  return `Types: ${links.join(' · ')}`
+}
+
+/** Render one log event entry. */
+function renderEvent(e: AnnotatedLogEventEntry): string[] {
+  const out = [`#### \`${e.name}\` — ${e.surface ? 'surface' : 'log-only'}`, '']
+  if (e.doc) out.push(e.doc, '')
+  out.push('```' + FENCE, `'${e.name}': ${e.payload}`, '```', '')
+  const links = typeLinks(e.payload)
+  if (links) out.push(links, '')
+  out.push(`Source: [\`${e.source}\`](../../${e.source.split(':')[0]})`, '')
+  return out
+}
+
+/** Render the full catalog (pure, deterministic given the collected inputs). */
+export function render(events: AnnotatedLogEventEntry[]): string {
+  const lines: string[] = [
+    '<!-- Generated by scripts/gen-persistence-catalog.ts — do not edit by hand.',
+    '     Run `pnpm run gen-persistence-catalog` to regenerate. -->',
+    '',
+    '# Persistence Log Event Catalog',
+    '',
+    'Every event type that can appear in a session\'s durable event log: each member of the merge-extensible `SessionEventMap` — the owning vocabulary in `@deepseek-ai/dsh-session` plus every plugin declaration merge in this repo — with the payload it carries, its surface badge, and the declaration it comes from. It complements [session.md](../core-data-structures/session.md) (the `SessionEvent` envelope, surface list, and `deriveMessages()` projection), [persistence.md](../core-data-structures/persistence.md) (how the log is made durable), and the [cordis catalog](../cordis-catalog/events-and-services.md) (the live bus wiring — a log event is NOT a cordis event; it reaches listeners via the single `session/event` emit).',
+    '',
+    'This file is GENERATED from source (`scripts/gen-persistence-catalog.ts`) and verified fresh by `pnpm run verify-persistence-catalog` (part of `doc-sync`) — do not edit it by hand. Payload blocks use a `ts persistence-catalog` fence (skipped by doc-typecheck, since a bare payload fragment is not standalone-compilable). Type names in a payload link to the page that documents them. See [the persistence-log-catalog RFC](../rfc/implemented/process/2026-07-04-persistence-log-catalog.md).',
+    '',
+    'The on-disk envelope around every payload is `SessionEvent` — `type`, monotonic `seq`, epoch-ms `time`, the `data` documented here, plus `surfaceOp`/`sourceEventSeqs` on **surface** events only ([envelope](../core-data-structures/session.md#sessioneventt--one-log-entry)). **surface** marks a `SurfaceEventType` member: it produces an LLM message and declares how it joins the surface list. **log-only** marks everything else: durable, replayable record with no derived-history contribution. Every payload is JSON-serializable (enforced at `Session.append`), and the whole format is pinned at `SESSION_FORMAT_VERSION = 0` — pre-release, no compatibility implied ([the version stance](../core-data-structures/persistence.md)). Scope: the packages in this repo; a downstream plugin can merge further event types, which are outside this catalog by construction.',
+    '',
+    '## Events',
+    '',
+  ]
+  const scopes = [...new Set(events.map(e => e.scope))].sort()
+  for (const scope of scopes) {
+    lines.push(`### \`${scope}/*\``, '')
+    for (const e of events.filter(x => x.scope === scope).sort((a, b) => a.name.localeCompare(b.name))) {
+      lines.push(...renderEvent(e))
+    }
+  }
+  return lines.join('\n')
+}
+
+/** CLI entry: default writes the catalog, `--check` fails if the committed copy
+ * is stale. Guarded behind an entry-point check so importing this module for
+ * tests neither regenerates the committed file nor calls process.exit. */
+function main(): void {
+  const content = render(annotateSurface(collectLogEvents(), collectSurfaceEventTypes()))
+  if (process.argv.includes('--check')) {
+    let committed: string | null = null
+    try {
+      committed = readFileSync(resolve(root, OUT), 'utf8')
+    } catch {
+      // Only ENOENT (not yet generated) is expected; a present-but-unreadable
+      // file is not a state this repo produces. Either way the remedy is the
+      // same — regenerate — so treat a read failure as "stale".
+      committed = null
+    }
+    if (committed === content) {
+      console.log(`gen-persistence-catalog: ${OUT} is up to date.`)
+      process.exit(0)
+    }
+    console.error(`gen-persistence-catalog: ${OUT} is stale. Run \`pnpm run gen-persistence-catalog\` and commit ${OUT}.`)
+    process.exit(1)
+  }
+
+  writeFileSync(resolve(root, OUT), content)
+  console.log(`gen-persistence-catalog: wrote ${OUT}.`)
+}
+
+// Run only when invoked as a script, not when imported by a test.
+if (process.argv[1] && import.meta.filename === resolve(process.argv[1])) {
+  main()
+}