Merge remote-tracking branch 'origin/master' into feat/adr0016-type-build-check
This commit is contained in:
imccyu
365 files changed
+13601
-7241
No files matched your search
@@ -0,0 +1,244 @@
|
||||
/**
|
||||
* SQLite durable session-persistence backend (`@deepseek-ai/dsh-session-persistence-sqlite`).
|
||||
*
|
||||
* A SECOND {@link SessionPersistence} implementation, built to validate that the
|
||||
* abstract seam + the shared `runPersistenceContract` suite are genuinely
|
||||
* backend-agnostic: the same append-only / contiguous-seq / lazy-materialization
|
||||
* / interrupted-turn-close-on-load semantics the JSONL backend expresses over
|
||||
* file bytes, expressed here over `node:sqlite` rows. Each `SessionEvent` maps
|
||||
* 1:1 onto a row `(session_id, seq, type, time, data)`.
|
||||
*
|
||||
* Like the JSONL backend it supplies ONLY the storage primitives (the
|
||||
* {@link PersistenceBackend} hooks below — INSERT/DELETE/SELECT inside
|
||||
* transactions); all the write-path orchestration lives in the backend-agnostic
|
||||
* {@link PersistenceCoordinator} this class composes. The four public
|
||||
* {@link SessionPersistence} methods delegate to the coordinator.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-session-persistence-sqlite
|
||||
*/
|
||||
|
||||
import { Context } from 'cordis'
|
||||
import z from 'schemastery'
|
||||
import { DatabaseSync } from 'node:sqlite'
|
||||
import { mkdir } from 'node:fs/promises'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import {
|
||||
SessionPersistence, PersistenceCoordinator,
|
||||
type PersistenceBackend, type StoredPrefix,
|
||||
} from '@deepseek-ai/dsh-session-persistence'
|
||||
import type { Session, SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
|
||||
import {
|
||||
openDatabase, rowToMeta, scanRows, type EventRow, type SessionRow,
|
||||
} from './schema'
|
||||
|
||||
export { SCHEMA_VERSION } from './schema'
|
||||
|
||||
/** Plugin configuration. */
|
||||
export interface Config {
|
||||
/**
|
||||
* Filesystem path to the SQLite database file. The special value `:memory:`
|
||||
* opens an in-process database (tests); a file path is created (with parent
|
||||
* dirs) on construction.
|
||||
*/
|
||||
path: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The SQLite persistence backend. Load as a plugin; it registers as
|
||||
* `ctx.sessionPersistence` and (via the coordinator) installs the write-path
|
||||
* listeners. Its torn-tail marker is the seq to delete from.
|
||||
*/
|
||||
export class SessionPersistenceSqlite extends SessionPersistence implements PersistenceBackend<number> {
|
||||
static inject = ['sessions']
|
||||
|
||||
static Config: z<Config> = z.object({
|
||||
path: z.string().required(),
|
||||
})
|
||||
|
||||
/**
|
||||
* Backend label for the coordinator's dispose diagnostics. Intentionally
|
||||
* shadows cordis `Service.name` (set to `'sessionPersistence'` by the base);
|
||||
* see the JSONL backend for why this does not affect service resolution.
|
||||
*/
|
||||
override readonly name = 'session-persistence-sqlite'
|
||||
|
||||
private db!: DatabaseSync
|
||||
private ready: Promise<void>
|
||||
private coordinator: PersistenceCoordinator<number>
|
||||
|
||||
constructor(ctx: Context, public config: Config) {
|
||||
super(ctx)
|
||||
// Open the database asynchronously (the parent directory may need creating);
|
||||
// every hook awaits `ready` first. Opening synchronously would force a sync
|
||||
// mkdir and block plugin apply.
|
||||
this.ready = this.openDb(config.path)
|
||||
this.coordinator = new PersistenceCoordinator<number>(this.ctx, this)
|
||||
}
|
||||
|
||||
private async openDb(path: string): Promise<void> {
|
||||
if (path !== ':memory:') {
|
||||
const abs = resolve(path)
|
||||
await mkdir(dirname(abs), { recursive: true, mode: 0o700 })
|
||||
this.db = openDatabase(abs)
|
||||
} else {
|
||||
this.db = openDatabase(path)
|
||||
}
|
||||
}
|
||||
|
||||
// --- SessionPersistence service surface (delegated to the coordinator) ---
|
||||
|
||||
create(meta: SessionHeader): Promise<void> {
|
||||
return this.coordinator.create(meta)
|
||||
}
|
||||
|
||||
append(id: SessionId, events: readonly SessionEvent[]): Promise<void> {
|
||||
return this.coordinator.append(id, events)
|
||||
}
|
||||
|
||||
load(id: SessionId): Promise<{ meta: SessionHeader; events: SessionEvent[] }> {
|
||||
return this.coordinator.load(id)
|
||||
}
|
||||
|
||||
// `list` is BOTH the public service method and the PersistenceBackend hook —
|
||||
// one method (the SELECT below). The coordinator adds no orchestration for
|
||||
// listing, so routing it through the coordinator would just recurse. Defined
|
||||
// once, in the "PersistenceBackend hooks" section.
|
||||
|
||||
/**
|
||||
* The per-session init promises, exposed for white-box tests that await a
|
||||
* specific session's onCreated (there is no public API to await one init).
|
||||
*/
|
||||
get inits(): Map<Session, Promise<void>> {
|
||||
return this.coordinator.inits
|
||||
}
|
||||
|
||||
// --- PersistenceBackend hooks (the SQLite storage primitives) ---
|
||||
|
||||
/** Read a stored prefix by id (ids are globally unique — no scope to scan). */
|
||||
loadStored(id: SessionId): Promise<StoredPrefix<number> | undefined> {
|
||||
return this.readPrefix(id)
|
||||
}
|
||||
|
||||
/** Read a stored prefix; `cwd` is ignored (the id is globally unique in SQLite). */
|
||||
loadLive(id: SessionId, _cwd: string | undefined): Promise<StoredPrefix<number> | undefined> {
|
||||
return this.readPrefix(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a session's row + ordered events into a {@link StoredPrefix}. The
|
||||
* torn-tail marker is the seq from which a never-committed tail must be deleted
|
||||
* (`scanRows` already returns it as `number | undefined`).
|
||||
*/
|
||||
private async readPrefix(id: SessionId): Promise<StoredPrefix<number> | undefined> {
|
||||
await this.ready
|
||||
const row = this.rowFor(id)
|
||||
if (row === undefined) return undefined
|
||||
const meta = rowToMeta(row)
|
||||
const eventRows = this.db
|
||||
.prepare('SELECT seq, type, time, data FROM events WHERE session_id = ? ORDER BY seq')
|
||||
.all(id) as unknown as EventRow[]
|
||||
const { preserved, tornFrom } = scanRows(eventRows)
|
||||
return { meta, events: preserved, ...tornFrom !== undefined ? { tornMarker: tornFrom } : {} }
|
||||
}
|
||||
|
||||
/**
|
||||
* Durably append a batch in ONE transaction: materialize the sessions row (if
|
||||
* lazy) and INSERT every event, or roll back entirely. The transaction is the
|
||||
* atomicity + durability boundary, so a mid-batch failure (a UNIQUE violation
|
||||
* on a duplicated seq) leaves the stored log untouched.
|
||||
*/
|
||||
async appendBatch(meta: SessionHeader, events: readonly SessionEvent[], isMaterialized: boolean): Promise<void> {
|
||||
await this.ready
|
||||
const insertEvent = this.db.prepare(
|
||||
'INSERT INTO events (session_id, seq, type, time, data) VALUES (?, ?, ?, ?, ?)',
|
||||
)
|
||||
this.db.exec('BEGIN')
|
||||
try {
|
||||
if (!isMaterialized) this.writeRow(meta)
|
||||
for (const event of events) {
|
||||
insertEvent.run(meta.id, event.seq, event.type, event.time, JSON.stringify(event.data))
|
||||
}
|
||||
this.db.exec('COMMIT')
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a crash repair durable in ONE transaction: DELETE the torn tail (from
|
||||
* `tornMarker`) and INSERT the synthetic `closers`. After COMMIT the stored rows
|
||||
* == the balanced log.
|
||||
*/
|
||||
async commitRepair(meta: SessionHeader, tornMarker: number | undefined, closers: readonly SessionEvent[]): Promise<void> {
|
||||
await this.ready
|
||||
this.db.exec('BEGIN')
|
||||
try {
|
||||
if (tornMarker !== undefined) {
|
||||
this.db.prepare('DELETE FROM events WHERE session_id = ? AND seq >= ?').run(meta.id, tornMarker)
|
||||
}
|
||||
if (closers.length > 0) {
|
||||
const insertEvent = this.db.prepare('INSERT INTO events (session_id, seq, type, time, data) VALUES (?, ?, ?, ?, ?)')
|
||||
for (const event of closers) {
|
||||
insertEvent.run(meta.id, event.seq, event.type, event.time, JSON.stringify(event.data))
|
||||
}
|
||||
}
|
||||
this.db.exec('COMMIT')
|
||||
} catch (error) {
|
||||
// The DELETE+INSERT cannot collide (a row at a closer's seq is preserved or
|
||||
// deleted as torn first); this rolls back a DB-level failure (disk full,
|
||||
// etc.), unreachable in test.
|
||||
/* v8 ignore start */
|
||||
this.db.exec('ROLLBACK')
|
||||
throw error
|
||||
/* v8 ignore stop */
|
||||
}
|
||||
}
|
||||
|
||||
/** List all materialized sessions' metadata (every row is a materialized session). */
|
||||
async list(): Promise<SessionHeader[]> {
|
||||
await this.ready
|
||||
const rows = this.db
|
||||
.prepare('SELECT * FROM sessions')
|
||||
.all() as unknown as SessionRow[]
|
||||
return rows.map(rowToMeta)
|
||||
}
|
||||
|
||||
/** Close the database handle (awaited by the coordinator's dispose, post-drain). */
|
||||
async close(): Promise<void> {
|
||||
await this.ready
|
||||
this.db.close()
|
||||
}
|
||||
|
||||
// --- row helpers ---
|
||||
|
||||
/** Fetch a session's row, or undefined if absent. */
|
||||
private rowFor(id: SessionId): SessionRow | undefined {
|
||||
return this.db.prepare('SELECT * FROM sessions WHERE id = ?').get(id) as unknown as SessionRow | undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert-or-replace a session's metadata row. The only caller is the first
|
||||
* materializing `appendBatch`, so writing the row IS the materialization (its
|
||||
* existence is the signal `list` reads).
|
||||
*/
|
||||
private writeRow(meta: SessionHeader): void {
|
||||
this.db.prepare(`
|
||||
INSERT INTO sessions (id, version, created_at, cwd, parent_session)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
version = excluded.version,
|
||||
created_at = excluded.created_at,
|
||||
cwd = excluded.cwd,
|
||||
parent_session = excluded.parent_session
|
||||
`).run(
|
||||
meta.id,
|
||||
meta.version,
|
||||
meta.createdAt,
|
||||
meta.cwd ?? null,
|
||||
meta.parentSession ?? null,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export default SessionPersistenceSqlite
|
||||
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* Schema + load-time helpers for the SQLite session-persistence backend: the
|
||||
* DDL (a `sessions` metadata table and a 1:1 `events` row per `SessionEvent`),
|
||||
* the database open/configure step, and the last-`turn/end` cut that gives the
|
||||
* SQLite backend the SAME crash-tail-on-load semantics as the JSONL backend.
|
||||
*
|
||||
* @module dsh-session-persistence-sqlite/schema
|
||||
*/
|
||||
|
||||
import { DatabaseSync } from 'node:sqlite'
|
||||
import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session'
|
||||
|
||||
/**
|
||||
* The on-disk schema version. Bumped only on a breaking change to the table
|
||||
* layout; orthogonal to a session's own `version` (which versions the EVENT
|
||||
* vocabulary, stored per session in the `sessions` row).
|
||||
*/
|
||||
export const SCHEMA_VERSION = 2
|
||||
|
||||
/**
|
||||
* A row of the `sessions` table — the out-of-log metadata ({@link SessionHeader}).
|
||||
* The row's EXISTENCE is the materialization signal: it is written only by the
|
||||
* first `append` (lazy materialization), so a created-but-never-appended
|
||||
* session has no row and is absent from `list`, mirroring the JSONL
|
||||
* backend's "no file until first append".
|
||||
*/
|
||||
export interface SessionRow {
|
||||
id: string
|
||||
version: number
|
||||
created_at: number
|
||||
cwd: string | null
|
||||
parent_session: string | null
|
||||
}
|
||||
|
||||
/** An `events` table row: one `SessionEvent` mapped 1:1 (`data` is JSON text). */
|
||||
export interface EventRow {
|
||||
seq: number
|
||||
type: string
|
||||
time: number
|
||||
data: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the database at `path` and apply the schema + pragmas. `foreign_keys`
|
||||
* makes `ON DELETE CASCADE` drop a session's events with its row; `journal_mode
|
||||
* = WAL` matches the durability model the ADR records (the row shape maps 1:1
|
||||
* onto `SessionEvent`; opencode runs this exact shape on SQLite/WAL).
|
||||
*
|
||||
* The table-layout version is persisted in SQLite's `PRAGMA user_version` and
|
||||
* checked on open: a fresh database (user_version 0) is stamped with the
|
||||
* current {@link SCHEMA_VERSION}; an existing database whose version is NOT the
|
||||
* current one (written by a different, incompatible build — older or newer) is
|
||||
* REJECTED rather than opened against a layout this build does not understand.
|
||||
* There are no migrations: v1 had a different `sessions` layout and is not
|
||||
* upgraded in place.
|
||||
*/
|
||||
export function openDatabase(path: string): DatabaseSync {
|
||||
const db = new DatabaseSync(path)
|
||||
db.exec('PRAGMA foreign_keys = ON')
|
||||
db.exec('PRAGMA journal_mode = WAL')
|
||||
// `PRAGMA user_version` always returns exactly one row { user_version }.
|
||||
const { user_version: onDisk } = db.prepare('PRAGMA user_version').get() as { user_version: number }
|
||||
if (onDisk !== 0 && onDisk !== SCHEMA_VERSION) {
|
||||
db.close()
|
||||
throw new Error(`session database at "${path}" has schema version ${onDisk}, incompatible with this build (${SCHEMA_VERSION})`)
|
||||
}
|
||||
if (onDisk === 0) {
|
||||
// Fresh (or pre-versioning) database: stamp the current layout version.
|
||||
// PRAGMA does not accept bound parameters, so interpolate the integer
|
||||
// constant (SCHEMA_VERSION is a trusted in-code number, not user input).
|
||||
db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`)
|
||||
}
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
version INTEGER NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
cwd TEXT,
|
||||
parent_session TEXT
|
||||
) STRICT
|
||||
`)
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
|
||||
seq INTEGER NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
time INTEGER NOT NULL,
|
||||
data TEXT NOT NULL,
|
||||
PRIMARY KEY (session_id, seq)
|
||||
) STRICT
|
||||
`)
|
||||
return db
|
||||
}
|
||||
|
||||
/** Reconstruct the {@link SessionHeader} from a `sessions` row. */
|
||||
export function rowToMeta(row: SessionRow): SessionHeader {
|
||||
return {
|
||||
version: row.version,
|
||||
id: row.id as SessionId,
|
||||
createdAt: row.created_at,
|
||||
...row.cwd !== null ? { cwd: row.cwd } : {},
|
||||
...row.parent_session !== null ? { parentSession: row.parent_session as SessionId } : {},
|
||||
}
|
||||
}
|
||||
|
||||
/** Reconstruct a {@link SessionEvent} from an `events` row (parses `data`). */
|
||||
export function rowToEvent(row: EventRow): SessionEvent {
|
||||
return {
|
||||
type: row.type,
|
||||
seq: row.seq,
|
||||
time: row.time,
|
||||
data: JSON.parse(row.data) as SessionEvent['data'],
|
||||
} as SessionEvent
|
||||
}
|
||||
|
||||
/**
|
||||
* The preserved prefix of an ordered event-row list (mirrors the JSONL
|
||||
* backend's `scanLog`): the longest prefix of complete, seq-contiguous,
|
||||
* parseable rows, PLUS the seq from which a never-committed torn tail must be
|
||||
* deleted (or `undefined` if the whole list is intact).
|
||||
*
|
||||
* A crash can leave a durable log whose final turn never closed: real,
|
||||
* fully-written rows sit after the last `turn/end`. Those are PRESERVED — a
|
||||
* single turn can be huge in a long-horizon task, so truncating it would
|
||||
* destroy real work; the backend closes the orphaned open turn with a synthetic
|
||||
* `turn/end {kind:'interrupted'}` on load (the session-persistence RFC). The ONLY thing excluded is
|
||||
* a torn trailing fragment — a row whose `data` never parses, or a seq gap —
|
||||
* AFTER the last committed `turn/end`; that bounds the preserved region and its
|
||||
* seq is returned as `tornFrom` so `load` can physically delete it.
|
||||
*
|
||||
* The last `turn/end` is computed from the `type` COLUMN (never parsing tail
|
||||
* `data`), so a malformed `data` in an uncommitted tail row is discarded rather
|
||||
* than making the session unloadable. A parse error or seq gap AT OR BEFORE the
|
||||
* last committed `turn/end` is committed-data corruption and throws.
|
||||
*
|
||||
* This relies on the session-log invariant that every event lives inside a turn
|
||||
* (`Session.append` enforces it): only the final turn can be open, so the
|
||||
* preserved tail is at most one unclosed turn.
|
||||
*/
|
||||
export function scanRows(rows: readonly EventRow[]): { preserved: SessionEvent[]; tornFrom?: number } {
|
||||
// Pass 1: parse each row's data; a row whose data is not valid JSON is a hole.
|
||||
// (The seq/type COLUMNS are always present even when `data` is corrupt.)
|
||||
interface Parsed { ok: boolean; event?: SessionEvent }
|
||||
const parsed: Parsed[] = rows.map((row) => {
|
||||
try {
|
||||
return { ok: true, event: rowToEvent(row) }
|
||||
} catch {
|
||||
return { ok: false }
|
||||
}
|
||||
})
|
||||
|
||||
// The last index that is a valid `turn/end` — the last fully-committed
|
||||
// boundary (the loop flushes only at turn/end).
|
||||
let lastTurnEnd = -1
|
||||
for (let i = parsed.length - 1; i >= 0; i--) {
|
||||
if (parsed[i]?.ok && rows[i]?.type === 'turn/end') { lastTurnEnd = i; break }
|
||||
}
|
||||
|
||||
// Walk the longest PREFIX of complete, seq-contiguous, parseable rows
|
||||
// (row i has seq === i). This includes the fully-written rows of an
|
||||
// interrupted final turn AFTER the last turn/end — real work, never
|
||||
// truncated. The walk stops at the first hole:
|
||||
// - at or before the last committed turn/end → committed corruption (throw);
|
||||
// - after it (or no committed turn/end) → tolerated torn tail (stop).
|
||||
const preserved: SessionEvent[] = []
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const p = parsed[i]
|
||||
if (!p?.ok || p.event === undefined) {
|
||||
if (i <= lastTurnEnd) throw new Error(`corrupt session log: unparsable committed event at seq ${rows[i]?.seq}`)
|
||||
break // torn tail fragment after the last turn/end — stop, tolerate
|
||||
}
|
||||
if (p.event.seq !== i) {
|
||||
if (i <= lastTurnEnd) throw new Error(`corrupt session log: seq gap in committed region (expected ${i}, got ${p.event.seq})`)
|
||||
break // gap after the last turn/end — torn tail, stop
|
||||
}
|
||||
preserved.push(p.event)
|
||||
}
|
||||
|
||||
// Any rows past the preserved prefix are a never-committed torn tail; their
|
||||
// first seq is the deletion point for load's physical repair.
|
||||
return preserved.length < rows.length ? { preserved, tornFrom: preserved.length } : { preserved }
|
||||
}
|
||||
Reference in New Issue
Block a user