From e15a6168d2c8d69ae004feff822c89b2adc78c36 Mon Sep 17 00:00:00 2001 From: Huanqi Cao Date: Sun, 5 Jul 2026 04:31:22 +0800 Subject: [PATCH] Support durable JSONL persistence on Windows --- docs/rfc/INDEX.md | 1 + ...026-07-05-windows-jsonl-durable-publish.md | 33 ++++ .../session-persistence-jsonl/README.md | 4 +- .../session-persistence-jsonl/package.json | 1 + .../session-persistence-jsonl/src/index.ts | 128 +++++++++---- .../session-persistence-jsonl/src/win32.ts | 150 ++++++++++++++++ .../tests/jsonl.spec.ts | 39 ++++ .../tests/win32.spec.ts | 168 ++++++++++++++++++ pnpm-lock.yaml | 144 +++++++++++++++ pnpm-workspace.yaml | 2 + 10 files changed, 636 insertions(+), 34 deletions(-) create mode 100644 docs/rfc/implemented/architecture/2026-07-05-windows-jsonl-durable-publish.md create mode 100644 packages/session-persistence/session-persistence-jsonl/src/win32.ts create mode 100644 packages/session-persistence/session-persistence-jsonl/tests/win32.spec.ts diff --git a/docs/rfc/INDEX.md b/docs/rfc/INDEX.md index ab39d4f314..32f11b744a 100644 --- a/docs/rfc/INDEX.md +++ b/docs/rfc/INDEX.md @@ -150,6 +150,7 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand; | [Prompt variables and tool-guidance ownership](implemented/architecture/2026-07-05-prompt-variables-and-tool-guidance-ownership.md) | 2026-07-05 | | [Every LLM request is reconstructable from the session log](implemented/architecture/2026-07-05-reconstructable-requests.md) | 2026-07-05 | | [Subagent provider-lifecycle events — `subagent/provider-added` / `subagent/provider-removed`](implemented/architecture/2026-07-05-subagent-provider-lifecycle-events.md) | 2026-07-05 | +| [Windows-native durable JSONL publication](implemented/architecture/2026-07-05-windows-jsonl-durable-publish.md) | 2026-07-05 | | [A shared timeout/deadline primitive, with hard-kill left to each capability](implemented/architecture/2026-07-06-timeout-deadline-library.md) | 2026-07-06 | | [Tool result retention library](implemented/architecture/2026-07-06-tool-result-retention-library.md) | 2026-07-06 | | [Tool-call timeout policy as a plugin](implemented/architecture/2026-07-07-tool-call-timeout-policy.md) | 2026-07-07 | diff --git a/docs/rfc/implemented/architecture/2026-07-05-windows-jsonl-durable-publish.md b/docs/rfc/implemented/architecture/2026-07-05-windows-jsonl-durable-publish.md new file mode 100644 index 0000000000..909f75132f --- /dev/null +++ b/docs/rfc/implemented/architecture/2026-07-05-windows-jsonl-durable-publish.md @@ -0,0 +1,33 @@ +# RFC: Windows-native durable JSONL publication + +Status: implemented + +## Problem + +`dsh-session-persistence-jsonl` publishes a session log lazily on the first append. The POSIX protocol writes a temp file, fsyncs it, links it to the final name, fsyncs the parent directory, and then removes the temp link. The parent-directory fsync is part of the durability contract: a crash after the namespace change must not lose the committed final name while leaving callers believing the session log materialized. + +Windows has atomic namespace operations, but Node does not expose a POSIX-equivalent parent-directory fsync contract there. Treating Windows directory sync failures as success would silently weaken a durable backend. The Windows path therefore needs a different publication primitive rather than a conditional inside the POSIX `syncDir` helper. + +## Decision + +The JSONL backend forks inside `materialize()` before any namespace mutation. Shared code computes the session directory, final log path, and initial JSONL bytes; POSIX and Windows then run separate publication protocols. + +POSIX keeps the existing protocol: create the root and cwd bucket with parent directory fsyncs, write and fsync a temp file, publish with `link()` so an existing final log is never overwritten, fsync the bucket directory, then remove the redundant temp hard link. + +Windows creates missing directories through a durable staging publish: create a random sibling directory, then publish it to the final directory name with `MoveFileExW(..., MOVEFILE_WRITE_THROUGH)` without `MOVEFILE_REPLACE_EXISTING` or `MOVEFILE_COPY_ALLOWED`. File materialization writes and fsyncs the temp log, then publishes that temp file to the final path with the same write-through `MoveFileExW` call and no replacement. `koffi` is the minimal Win32 bridge for this API surface; its install script is allowed in `pnpm-workspace.yaml` because the package ships the native loader and prebuilt platform modules. + +## Alternatives considered + +**Ignore Windows directory-sync failures.** Rejected because it reports a first append as durable without forcing the published namespace entry to stable storage. + +**Use `CreateHardLinkW`.** Rejected because hard links are filesystem-dependent, do not publish directories, and expose no write-through option. + +**Use replacement or transactional APIs.** `ReplaceFileW` has replacement semantics that conflict with same-id collision rejection, and Transactional NTFS is not recommended for new application designs. + +## Consequences + +The backend keeps one external contract across platforms: first append either publishes a complete log at the final name or fails without overwriting an existing log. The platform split is an implementation detail; `SessionPersistence` APIs and on-disk JSONL format do not change. + +Windows tests exercise the real Win32 publish path on native Windows. Power-loss behavior remains an API-contract property rather than something unit tests can prove; the testable invariants are that directory fsync is not called on Windows materialization, final-path collisions fail, temp logs are fsync'd before publication, and the resulting log loads normally. + +Append and repair still use ordinary file-handle fsyncs on both platforms. A failed append closes its append-only handle, reopens the log read/write, truncates it to the pre-append size, and fsyncs the rollback because Windows rejects `ftruncate` on append-only handles. diff --git a/packages/session-persistence/session-persistence-jsonl/README.md b/packages/session-persistence/session-persistence-jsonl/README.md index 43f4443107..097ad04b67 100644 --- a/packages/session-persistence/session-persistence-jsonl/README.md +++ b/packages/session-persistence/session-persistence-jsonl/README.md @@ -23,7 +23,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence ## Durability and crash semantics -- **Lazy materialization.** `create(meta)` writes nothing; on the first `append`, the backend writes and `fsync`s a temporary file, publishes it without overwrite via a hard link, then `fsync`s the directory. A created-but-never-appended session leaves nothing on disk and is absent from `list`. +- **Lazy materialization.** `create(meta)` writes nothing; the `.jsonl` (header + first batch) is written atomically on the first `append`: POSIX uses temp-write + file `fsync` + `link` + parent-directory `fsync`; Windows uses temp-write + file `fsync` + `MoveFileExW(..., MOVEFILE_WRITE_THROUGH)` and creates missing directories through the same write-through publish pattern. A created-but-never-appended session leaves nothing on disk and is absent from `list`. - **Append-only.** Committed events (at or below a flushed `turn/end`) are never rewritten. Subsequent appends are line appends at EOF + `fsync`. - **Crash recovery — preserve valid tail work.** `load` keeps the contiguous valid prefix of an interrupted final turn. It truncates from the first unparsable or sequence-gapped uncommitted record, then appends the synthetic tool, step, and turn closers required by the shared [persistence contract](../../../docs/rfc/implemented/architecture/2026-06-14-session-persistence.md); the same defect at or before the last committed `turn/end` rejects. - **Contiguous-seq.** `append` rejects a batch whose first `seq` does not continue the stored log, and rejects non-JSON-serializable `event.data` naming the offending event type. @@ -45,4 +45,4 @@ The plugin buffers frozen session events and drains them on flush or disposal. A - **Only the current `SESSION_FORMAT_VERSION` (v0) loads** — the on-disk format is pre-release/unstable: a breaking format change is absorbed at v0 and non-current logs are rejected; there is no migration. - **Nothing deletes session files** — logs accumulate under `root` until removed externally (the seam has no deletion surface). - **Single-process assumption** — per-session serialization and the write cursor live in this process; two processes appending to the same `root` are not coordinated. -- **Initial materialization requires hard-link support** — first append uses `link()` so same-id races fail instead of overwriting a committed log; a filesystem that cannot create hard links cannot host this backend. +- **POSIX materialization requires hard-link support** — its first append uses `link()` so same-id races fail instead of overwriting a committed log; Windows uses write-through rename without replacement. diff --git a/packages/session-persistence/session-persistence-jsonl/package.json b/packages/session-persistence/session-persistence-jsonl/package.json index ddb9f2af4d..2644e14efc 100644 --- a/packages/session-persistence/session-persistence-jsonl/package.json +++ b/packages/session-persistence/session-persistence-jsonl/package.json @@ -27,6 +27,7 @@ "cordis": "^4.0.0-rc.7" }, "dependencies": { + "koffi": "^3.1.0", "schemastery": "^3.18.0" }, "devDependencies": { diff --git a/packages/session-persistence/session-persistence-jsonl/src/index.ts b/packages/session-persistence/session-persistence-jsonl/src/index.ts index 0c8a2ee3ef..ac1c07860e 100644 --- a/packages/session-persistence/session-persistence-jsonl/src/index.ts +++ b/packages/session-persistence/session-persistence-jsonl/src/index.ts @@ -19,6 +19,7 @@ import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-se import { encodeSegment, eventLine, logPath, parseHeaderMeta, scanLog, sessionDir, toHeaderLine, } from './format.ts' +import { ensureDurableDirectoryWin32, publishNewFileWin32 } from './win32.ts' /** Plugin config: where the JSONL backend keeps its session logs (`root` is required — no default). */ export interface Config { @@ -160,33 +161,35 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi // --- materialization / append / repair (file mechanics) --- - /** Atomically write the header line + first batch (temp-write, fsync, collision-safe hard-link publish). */ + /** Atomically write the header line + first batch (temp-write, fsync, publish). */ private async materialize(meta: SessionHeader, events: readonly SessionEvent[]): Promise { const dir = sessionDir(this.root, meta.cwd) - await mkdir(this.root, { recursive: true, mode: 0o700 }) - await this.syncDir(dirname(this.root)) - await mkdir(dir, { recursive: true, mode: 0o700 }) - await this.syncDir(this.root) const finalPath = logPath(this.root, meta.cwd, meta.id) - // Materialization is the first write; an existing log is an id collision. - /* v8 ignore next 3 -- createCore guards collisions before materialize; this is a TOCTOU backstop */ - if (await this.exists(finalPath)) { - throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`) + const content = this.initialLogContent(meta, events) + /* v8 ignore next -- native Windows coverage exercises this platform dispatch; Linux covers the POSIX peer */ + if (process.platform === 'win32') { + await this.materializeWin32(dir, finalPath, meta.id, content) + } else { + await this.materializePosix(dir, finalPath, meta.id, content) } + } + + private initialLogContent(meta: SessionHeader, events: readonly SessionEvent[]): string { const header = JSON.stringify(toHeaderLine(meta)) const body = events.map(eventLine).join('\n') - const content = header + '\n' + body + '\n' + return header + '\n' + body + '\n' + } - const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp` - const handle = await open(tmp, 'wx', 0o600) - try { - await handle.writeFile(content) - await handle.sync() - } finally { - await handle.close() - } - // Publish with link()+unlink(): unlike rename(), link fails if another - // process materialized the same id first. + private async materializePosix(dir: string, finalPath: string, id: SessionId, content: string): Promise { + await mkdir(this.root, { recursive: true, mode: 0o700 }) + await this.syncDirPosix(dirname(this.root)) + await mkdir(dir, { recursive: true, mode: 0o700 }) + await this.syncDirPosix(this.root) + await this.rejectExistingLog(finalPath, id) + const tmp = await this.writeSyncedTempFile(finalPath, content) + // Publish via link()+unlink(), NOT rename(): link fails with EEXIST if the + // final path already exists, so two processes materializing the same id + // concurrently cannot clobber each other. rename() would silently overwrite. let linked = false try { await link(tmp, finalPath) @@ -197,10 +200,13 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi /* v8 ignore next -- link failure is the TOCTOU/IO race guarded above; not reachable in test */ if (!linked) await rm(tmp, { force: true }) } - // The published link becomes crash-durable only after its directory fsync. - await this.syncDir(dir) - // Best-effort temp cleanup: the log is already published and durable, so a failure to - // remove the (now-redundant) temp hard link must not reject the append. + // link() succeeded — the log is published. fsync the directory so the new + // entry survives a power loss: the new link is not crash-durable until the + // parent directory's metadata is synced. + await this.syncDirPosix(dir) + // Best-effort temp cleanup: the log is already published and durable, so a + // failure to remove the (now-redundant) temp hard link must NOT reject the + // append. Swallow only the rm failure; nothing else of consequence runs here. try { await rm(tmp, { force: true }) } catch { @@ -208,8 +214,47 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi } } - /** fsync a directory so a just-created or published entry inside it is crash-durable. */ - private async syncDir(dir: string): Promise { + /* v8 ignore start -- native Windows coverage exercises this integration path */ + private async materializeWin32(dir: string, finalPath: string, id: SessionId, content: string): Promise { + await ensureDurableDirectoryWin32(this.root) + await ensureDurableDirectoryWin32(dir) + await this.rejectExistingLog(finalPath, id) + const tmp = await this.writeSyncedTempFile(finalPath, content) + try { + await publishNewFileWin32(tmp, finalPath) + } catch (error) { + await rm(tmp, { force: true }) + throw error + } + } + /* v8 ignore stop */ + + private async rejectExistingLog(finalPath: string, id: SessionId): Promise { + // Never publish over an existing committed log: materialize is the FIRST + // write of a session the backend believes is new. A file here means a + // different session shares this id on disk — reject loudly. (createCore + // already guards the create path, so this is unreachable-in-practice TOCTOU + // defense.) + /* v8 ignore next 3 -- createCore guards collisions before materialize; this is a TOCTOU backstop */ + if (await this.exists(finalPath)) { + throw new Error(`refusing to materialize "${id}": a log already exists on disk (load/resume it instead)`) + } + } + + private async writeSyncedTempFile(finalPath: string, content: string): Promise { + const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp` + const handle = await open(tmp, 'wx', 0o600) + try { + await handle.writeFile(content) + await handle.sync() + } finally { + await handle.close() + } + return tmp + } + + /** fsync a POSIX directory so a just-created/renamed entry is crash-durable. */ + private async syncDirPosix(dir: string): Promise { const handle = await open(dir, 'r') try { await handle.sync() @@ -226,17 +271,37 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi private async appendLines(meta: SessionHeader, events: readonly SessionEvent[]): Promise { const path = logPath(this.root, meta.cwd, meta.id) const handle = await open(path, 'a') + let closed = false + const closeAppendHandle = async (): Promise => { + if (closed) return + closed = true + await handle.close() + } + try { const { size: before } = await handle.stat() try { await handle.writeFile(events.map(eventLine).join('\n') + '\n') await handle.sync() } catch (error) { - // Roll back whatever bytes landed so a retry starts from a clean EOF. - await handle.truncate(before) - await handle.sync() + try { + await closeAppendHandle() + await this.rollbackAppend(path, before) + } catch (rollbackError) { + throw new AggregateError([error, rollbackError], `failed to roll back append to "${path}"`) + } throw error } + } finally { + await closeAppendHandle() + } + } + + private async rollbackAppend(path: string, size: number): Promise { + const handle = await open(path, 'r+') + try { + await handle.truncate(size) + await handle.sync() } finally { await handle.close() } @@ -322,9 +387,8 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi await handle.close() return true } catch (error) { - // Only ENOENT means absent. A permission/I/O error must surface, not be - // collapsed to `false` — otherwise load() reports "not found" and collision - // checks proceed under a false absence assumption. + // Only ENOENT means absent. A permission/I/O error must surface rather + // than letting load or collision checks proceed under false absence. if (isENOENT(error)) return false throw error } diff --git a/packages/session-persistence/session-persistence-jsonl/src/win32.ts b/packages/session-persistence/session-persistence-jsonl/src/win32.ts new file mode 100644 index 0000000000..143f230ea3 --- /dev/null +++ b/packages/session-persistence/session-persistence-jsonl/src/win32.ts @@ -0,0 +1,150 @@ +/** + * Windows durable namespace helpers for the JSONL backend. + * + * POSIX publishes a newly-created log by creating a directory entry and then + * fsyncing the parent directory. Windows does not expose that parent-directory + * fsync contract through Node, so the Windows path uses the native durable + * namespace primitive instead: create a staging object in the target directory + * and publish it with `MoveFileExW(..., MOVEFILE_WRITE_THROUGH)` without + * replacement or cross-volume copy fallback. + * + * @module dsh-session-persistence-jsonl/win32 + */ + +import { mkdtemp, rm, stat } from 'node:fs/promises' +import { basename, join, parse, resolve, toNamespacedPath } from 'node:path' + +type MoveFileExW = (existing: string, replacement: string, flags: number) => boolean +type GetLastError = () => number + +interface Win32Bindings { + moveFileExW: MoveFileExW + getLastError: GetLastError +} + +interface Win32ErrnoException extends NodeJS.ErrnoException { + win32Code: number + dest: string +} + +const MOVEFILE_WRITE_THROUGH = 0x00000008 +const ERROR_FILE_NOT_FOUND = 2 +const ERROR_PATH_NOT_FOUND = 3 +const ERROR_ACCESS_DENIED = 5 +const ERROR_NOT_SAME_DEVICE = 17 +const ERROR_FILE_EXISTS = 80 +const ERROR_INVALID_NAME = 123 +const ERROR_ALREADY_EXISTS = 183 + +let bindings: Win32Bindings | undefined + +/** Load the small Win32 surface lazily so non-Windows processes never load Koffi. */ +async function win32(): Promise { + if (bindings !== undefined) return bindings + const koffi = (await import('koffi')).default + const kernel32 = koffi.load('kernel32.dll') + bindings = { + moveFileExW: kernel32.func('__stdcall', 'MoveFileExW', 'bool', ['str16', 'str16', 'uint']) as MoveFileExW, + getLastError: kernel32.func('__stdcall', 'GetLastError', 'uint', []) as GetLastError, + } + return bindings +} + +function errnoCode(win32Code: number): string { + switch (win32Code) { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + return 'ENOENT' + case ERROR_ACCESS_DENIED: + return 'EACCES' + case ERROR_NOT_SAME_DEVICE: + return 'EXDEV' + case ERROR_FILE_EXISTS: + case ERROR_ALREADY_EXISTS: + return 'EEXIST' + case ERROR_INVALID_NAME: + return 'EINVAL' + default: + return 'EIO' + } +} + +function win32Error(syscall: string, win32Code: number, path: string, dest: string): Win32ErrnoException { + const code = errnoCode(win32Code) + const error = new Error(`${syscall} ${code} (Win32 ${win32Code}): ${path} -> ${dest}`) as Win32ErrnoException + error.code = code + error.errno = win32Code + error.syscall = syscall + error.path = path + error.dest = dest + error.win32Code = win32Code + return error +} + +function isENOENT(error: unknown): boolean { + return (error as NodeJS.ErrnoException | null)?.code === 'ENOENT' +} + +function isEEXIST(error: unknown): boolean { + return (error as NodeJS.ErrnoException | null)?.code === 'EEXIST' +} + +async function assertDirectory(path: string): Promise { + try { + const info = await stat(path) + if (info.isDirectory()) return true + const error = new Error(`path exists but is not a directory: ${path}`) as NodeJS.ErrnoException + error.code = 'ENOTDIR' + error.path = path + throw error + } catch (error) { + if (isENOENT(error)) return false + throw error + } +} + +/** + * Publish `existing` at `replacement` with Windows write-through rename + * semantics. The destination must not already exist; the move must stay within + * the volume (no copy fallback flag is set). + * @param existing - the synced staging path to move. + * @param replacement - the final path, which must not already exist. + */ +export async function publishNewFileWin32(existing: string, replacement: string): Promise { + const api = await win32() + const ok = api.moveFileExW(toNamespacedPath(existing), toNamespacedPath(replacement), MOVEFILE_WRITE_THROUGH) + if (!ok) throw win32Error('MoveFileExW', api.getLastError(), existing, replacement) +} + +/** + * Create `target` and its missing ancestors with durable Windows namespace + * publication. Each missing directory is first created as a random staging + * sibling, then moved to its final name with `MOVEFILE_WRITE_THROUGH`; races + * with another creator are accepted only after verifying the winner is a + * directory. + * @param target - the absolute directory path to create durably when absent. + */ +export async function ensureDurableDirectoryWin32(target: string): Promise { + const absolute = resolve(target) + const root = parse(absolute).root + await assertDirectory(root) + + const segments = absolute.slice(root.length).split(/[\\/]+/).filter(part => part.length > 0) + let current = root + for (const segment of segments) { + const next = join(current, segment) + if (!await assertDirectory(next)) await createLeafDirectoryWin32(current, next) + current = next + } +} + +async function createLeafDirectoryWin32(parent: string, target: string): Promise { + const staging = await mkdtemp(join(parent, `.dsh-mkdir-${basename(target)}-`)) + try { + await publishNewFileWin32(staging, target) + } catch (error) { + await rm(staging, { recursive: true, force: true }) + if (isEEXIST(error) && await assertDirectory(target)) return + throw error + } +} diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index 4b279e3c6e..e7a65d3c8f 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -336,6 +336,45 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => { expect(loaded.events.map(e => e.seq)).toEqual([0, 1, 2, 3, 4, 5, 6, 7]) }) + it('reports both the append failure and a failed rollback', async () => { + const m = meta('rollback-failure') + await ctx.sessionPersistence.create(m) + await ctx.sessionPersistence.append(m.id, oneTurnLog()) + + const path = logPath(root, undefined, m.id) + const handle = await (await import('node:fs/promises')).open(path, 'r') + const proto = Object.getPrototypeOf(handle) as { sync: () => Promise } + await handle.close() + const realSync = proto.sync + let failed = false + const syncSpy = vi.spyOn(proto, 'sync').mockImplementation(async function (this: unknown) { + if (!failed) { failed = true; throw new Error('simulated append fsync failure') } + return realSync.call(this) + }) + const backend = ctx.sessionPersistence as unknown as { + rollbackAppend: (path: string, size: number) => Promise + } + const realRollback = backend.rollbackAppend.bind(backend) + backend.rollbackAppend = () => Promise.reject(new Error('simulated rollback failure')) + + try { + await ctx.sessionPersistence.append(m.id, [ + { type: 'turn/start', seq: 6, time: 9, data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } } }, + ] as SessionEvent[]) + throw new Error('expected append to reject') + } catch (error) { + expect(error).toBeInstanceOf(AggregateError) + const aggregate = error as AggregateError + expect(aggregate.message).toContain(`failed to roll back append to "${path}"`) + expect(aggregate.errors).toHaveLength(2) + expect(aggregate.errors[0]).toMatchObject({ message: 'simulated append fsync failure' }) + expect(aggregate.errors[1]).toMatchObject({ message: 'simulated rollback failure' }) + } finally { + backend.rollbackAppend = realRollback + syncSpy.mockRestore() + } + }) + it('load returns a meta copy: mutating it does not corrupt backend pathing', async () => { const m = meta('meta-copy', '/proj') await ctx.sessionPersistence.create(m) diff --git a/packages/session-persistence/session-persistence-jsonl/tests/win32.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/win32.spec.ts new file mode 100644 index 0000000000..760eb3d455 --- /dev/null +++ b/packages/session-persistence/session-persistence-jsonl/tests/win32.spec.ts @@ -0,0 +1,168 @@ +/** + * Unit tests for the Windows durable namespace helper with a mocked kernel32 + * binding. The real JSONL suite exercises the helper on native Windows; these + * tests keep the Win32 error mapping and race handling covered on every host. + */ + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const MOVEFILE_WRITE_THROUGH = 0x00000008 +const ERROR_FILE_NOT_FOUND = 2 +const ERROR_PATH_NOT_FOUND = 3 +const ERROR_ACCESS_DENIED = 5 +const ERROR_NOT_SAME_DEVICE = 17 +const ERROR_FILE_EXISTS = 80 +const ERROR_INVALID_NAME = 123 +const ERROR_ALREADY_EXISTS = 183 + +type MoveFileExW = (existing: string, replacement: string, flags: number, setLastError: (code: number) => void) => boolean + +const roots: string[] = [] + +function stripNamespace(path: string): string { + if (path.startsWith('\\\\?\\UNC\\')) return `\\\\${path.slice('\\\\?\\UNC\\'.length)}` + if (path.startsWith('\\\\?\\')) return path.slice('\\\\?\\'.length) + return path +} + +async function tempRoot(): Promise { + const dir = await mkdtemp(join(tmpdir(), 'dsh-jsonl-win32-')) + roots.push(dir) + return dir +} + +async function importWithMove(moveFileExW: MoveFileExW): Promise { + vi.resetModules() + vi.doMock('koffi', () => { + let lastError = 0 + const setLastError = (code: number): void => { lastError = code } + const move: MoveFileExW = (existing, replacement, flags, setError) => { + const ok = moveFileExW(existing, replacement, flags, setError) + lastError = ok ? 0 : lastError + return ok + } + return { + default: { + load: () => ({ + func: (_convention: string, name: string) => { + if (name === 'MoveFileExW') return (existing: string, replacement: string, flags: number) => { + const ok = move(existing, replacement, flags, setLastError) + return ok + } + return () => lastError + }, + }), + }, + } + }) + return import('../src/win32.ts') +} + +async function importWithError(code: number): Promise { + vi.resetModules() + vi.doMock('koffi', () => ({ + default: { + load: () => ({ + func: (_convention: string, name: string) => { + if (name === 'MoveFileExW') return () => false + return () => code + }, + }), + }, + })) + return import('../src/win32.ts') +} + +async function importWithFilesystemMove(): Promise { + return importWithMove((existing, replacement, flags, setLastError) => { + expect(flags).toBe(MOVEFILE_WRITE_THROUGH) + const from = stripNamespace(existing) + const to = stripNamespace(replacement) + if (!existsSync(from)) { setLastError(ERROR_FILE_NOT_FOUND); return false } + if (existsSync(to)) { setLastError(ERROR_ALREADY_EXISTS); return false } + renameSync(from, to) + return true + }) +} + +afterEach(async () => { + vi.doUnmock('koffi') + vi.resetModules() + for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }) +}) + +describe('Windows durable namespace helpers', () => { + it('publishes a new file with write-through MoveFileExW semantics', async () => { + const { publishNewFileWin32 } = await importWithFilesystemMove() + const root = await tempRoot() + const tmp = join(root, 'log.tmp') + const final = join(root, 'log.jsonl') + await writeFile(tmp, 'content') + + await publishNewFileWin32(tmp, final) + expect(existsSync(tmp)).toBe(false) + expect(readFileSync(final, 'utf8')).toBe('content') + }) + + it('maps Win32 publish failures to Node-style errno codes', async () => { + const cases = [ + [ERROR_FILE_NOT_FOUND, 'ENOENT'], + [ERROR_PATH_NOT_FOUND, 'ENOENT'], + [ERROR_ACCESS_DENIED, 'EACCES'], + [ERROR_NOT_SAME_DEVICE, 'EXDEV'], + [ERROR_FILE_EXISTS, 'EEXIST'], + [ERROR_ALREADY_EXISTS, 'EEXIST'], + [ERROR_INVALID_NAME, 'EINVAL'], + [9999, 'EIO'], + ] as const + for (const [win32Code, code] of cases) { + const { publishNewFileWin32 } = await importWithError(win32Code) + await expect(publishNewFileWin32('from', 'to')).rejects.toMatchObject({ code, win32Code, path: 'from', dest: 'to' }) + } + }) + + it('creates missing directories through staging siblings and tolerates an already-created race', async () => { + const root = await tempRoot() + const raced = join(root, 'raced') + const { ensureDurableDirectoryWin32 } = await importWithMove((existing, replacement, flags, setLastError) => { + expect(flags).toBe(MOVEFILE_WRITE_THROUGH) + const from = stripNamespace(existing) + const to = stripNamespace(replacement) + if (to === raced) { + mkdirSync(to) + setLastError(ERROR_ALREADY_EXISTS) + return false + } + if (!existsSync(from)) { setLastError(ERROR_FILE_NOT_FOUND); return false } + if (existsSync(to)) { setLastError(ERROR_ALREADY_EXISTS); return false } + renameSync(from, to) + return true + }) + + await ensureDurableDirectoryWin32(join(root, 'a', 'b')) + expect(existsSync(join(root, 'a', 'b'))).toBe(true) + await ensureDurableDirectoryWin32(join(root, 'a', 'b')) + await ensureDurableDirectoryWin32(raced) + expect(existsSync(raced)).toBe(true) + }) + + it('surfaces directory publication failures other than an existing-target race', async () => { + const { ensureDurableDirectoryWin32 } = await importWithError(ERROR_ACCESS_DENIED) + const root = await tempRoot() + + await expect(ensureDurableDirectoryWin32(join(root, 'denied'))).rejects.toMatchObject({ code: 'EACCES' }) + }) + + it('rejects a non-directory component instead of treating it as missing', async () => { + const { ensureDurableDirectoryWin32 } = await importWithFilesystemMove() + const root = await tempRoot() + const blocked = join(root, 'blocked') + writeFileSync(blocked, 'x') + + await expect(ensureDurableDirectoryWin32(join(blocked, 'child'))).rejects.toMatchObject({ code: 'ENOTDIR' }) + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9810a9a61e..9e6532f466 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1259,6 +1259,9 @@ importers: packages/session-persistence/session-persistence-jsonl: dependencies: + koffi: + specifier: ^3.1.0 + version: 3.1.1 schemastery: specifier: ^3.18.0 version: 3.18.0 @@ -3453,6 +3456,81 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@koromix/koffi-darwin-arm64@3.1.1': + resolution: {integrity: sha512-+Dl0zQDh1Wb55AWOn9hp7K30qgkODvrvN+ZNkFOh81Q0oFX/rpJQtocgjAuYk2zFAcajSeVDumkcHMPwnKSXzA==} + cpu: [arm64] + os: [darwin] + + '@koromix/koffi-darwin-x64@3.1.1': + resolution: {integrity: sha512-cDFAKn1qdZBFLrp7dAc9QUDw3l4xAhTJbOdPWWb0LxssVicUdHcRCLZGrDsmPW2tpH6LGNNeLgqRpAoD2Mo8iA==} + cpu: [x64] + os: [darwin] + + '@koromix/koffi-freebsd-arm64@3.1.1': + resolution: {integrity: sha512-zaP7FJISI/scQW9Wa5QicY3a09WmtKBWSbmC+5nfCqPzwWe7Hx2so74Er7mPsDfCiMMR0Ya+evKbJQDkfyXicg==} + cpu: [arm64] + os: [freebsd] + + '@koromix/koffi-freebsd-ia32@3.1.1': + resolution: {integrity: sha512-7GejVb688TLM8rbjfc0oezJrATxZc0dn801xWEDJekN2DgmRXu7HquGqWQ6z3NeSq7ZxEggz4T3xtlbCysQapA==} + cpu: [ia32] + os: [freebsd] + + '@koromix/koffi-freebsd-x64@3.1.1': + resolution: {integrity: sha512-XLiCFP9OFCyOoGTjAimtDKLhzhfo34WcP1ShVWxRzNCWDGjfz8BYjwd69cp/cDSUXZbxamqs4+/6vmkePq9wxA==} + cpu: [x64] + os: [freebsd] + + '@koromix/koffi-linux-arm64@3.1.1': + resolution: {integrity: sha512-HA9xINK7G4dRAkpfnBWD9VfuyIBgW1SuK+KPHjksUwRMOnhgqP8J/JqgrAzdzcDiefGBkqEacIP776OUwz7knQ==} + cpu: [arm64] + os: [linux] + + '@koromix/koffi-linux-ia32@3.1.1': + resolution: {integrity: sha512-jG7IFytmP8K5Qtbx0ro0ZeuX3JjSsLxmYhq+nmXDdrtOAlxIsWGynuiDLS6Jk3vOchVii2m6Y2f/L3GLG2fG5A==} + cpu: [ia32] + os: [linux] + + '@koromix/koffi-linux-loong64@3.1.1': + resolution: {integrity: sha512-CIsT1cNnih8FuU52Me/IVlJBpH28SQfoDeYPctJswgJzaARktusF7m4MUbtR1PBDjuquCVM4/vFyNdOzfPonvA==} + cpu: [loong64] + os: [linux] + + '@koromix/koffi-linux-riscv64@3.1.1': + resolution: {integrity: sha512-9D6RmqeKsSvs3U6jILJU9PcAjMwKKyn7yLxNBb5k6z9PCoUoGJ3/BrhXAX0qjrLLwEiIpP/hS/40RuXvH8Lc3Q==} + cpu: [riscv64] + os: [linux] + + '@koromix/koffi-linux-x64@3.1.1': + resolution: {integrity: sha512-pyTcX5fePeYbt7TZAwRby69wdlRx3PT+g15ra5IYdat/Pgh3qAKEYeZ+uu7WpPGOy43p/oSRqqZoa2kORzozlA==} + cpu: [x64] + os: [linux] + + '@koromix/koffi-openbsd-ia32@3.1.1': + resolution: {integrity: sha512-iPnPzvG2HOfdzaiG1drdkt86sAqmTPDv9mAf+5gL7mRzkeeQC88EVGboRy7eXwdXn7R+v0ntA3iQxdHrBn6yXw==} + cpu: [ia32] + os: [openbsd] + + '@koromix/koffi-openbsd-x64@3.1.1': + resolution: {integrity: sha512-/Xqc3R0SVoMCYjMPZnJ9bULtRo364+dKmnQhfDrI83tSpxUHRw7HRNf12vBeL+hPgKxSBjtMpWfQ/ZIyVyLFag==} + cpu: [x64] + os: [openbsd] + + '@koromix/koffi-win32-arm64@3.1.1': + resolution: {integrity: sha512-JhqHauEwQvdcWUERxrV5HH/DT9W7hY1A1eU6/o8tB+yck+D3kt5elpRDBt9KjpW6h+vHPy3V0sjDvO0CXyabTA==} + cpu: [arm64] + os: [win32] + + '@koromix/koffi-win32-ia32@3.1.1': + resolution: {integrity: sha512-ZRuyYmlGS/rCc966qqs0qREXDW4FRdul7rDF1VgSWHbVmdc196PUgUT+blq/GjZgTwqzeEXtMRgM+cU8krHjvA==} + cpu: [ia32] + os: [win32] + + '@koromix/koffi-win32-x64@3.1.1': + resolution: {integrity: sha512-KqHPmvj6QILhNyI/To8QSihHsijeVGIYYPBOUnXEpcnH2LuLbargY4Hd6dDeTN3Z90uUUxN+1FWz1UnhVzFOiA==} + cpu: [x64] + os: [win32] + '@mermaid-js/parser@1.2.0': resolution: {integrity: sha512-oYPyv8A4As1yH5Bx+04iQEQxXuIQDe0GKCNSRgao6z8AM9jixXIfP0vsppRLvGf+nKIOb9/LdpWA4YuJiVvESA==} @@ -5662,6 +5740,9 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} hasBin: true + koffi@3.1.1: + resolution: {integrity: sha512-mRX6AMeeKCxSOeOopqAcLAl5jcNvge7NAG8l7rF/8gGJATI0tdHFYjteIdE0mGOtWdsrJOij+PjnP8Q9c1gwgA==} + layout-base@1.0.2: resolution: {integrity: sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==} @@ -7842,6 +7923,51 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@koromix/koffi-darwin-arm64@3.1.1': + optional: true + + '@koromix/koffi-darwin-x64@3.1.1': + optional: true + + '@koromix/koffi-freebsd-arm64@3.1.1': + optional: true + + '@koromix/koffi-freebsd-ia32@3.1.1': + optional: true + + '@koromix/koffi-freebsd-x64@3.1.1': + optional: true + + '@koromix/koffi-linux-arm64@3.1.1': + optional: true + + '@koromix/koffi-linux-ia32@3.1.1': + optional: true + + '@koromix/koffi-linux-loong64@3.1.1': + optional: true + + '@koromix/koffi-linux-riscv64@3.1.1': + optional: true + + '@koromix/koffi-linux-x64@3.1.1': + optional: true + + '@koromix/koffi-openbsd-ia32@3.1.1': + optional: true + + '@koromix/koffi-openbsd-x64@3.1.1': + optional: true + + '@koromix/koffi-win32-arm64@3.1.1': + optional: true + + '@koromix/koffi-win32-ia32@3.1.1': + optional: true + + '@koromix/koffi-win32-x64@3.1.1': + optional: true + '@mermaid-js/parser@1.2.0': dependencies: '@chevrotain/types': 11.1.2 @@ -10072,6 +10198,24 @@ snapshots: yaml: 2.9.0 zod: 4.4.3 + koffi@3.1.1: + optionalDependencies: + '@koromix/koffi-darwin-arm64': 3.1.1 + '@koromix/koffi-darwin-x64': 3.1.1 + '@koromix/koffi-freebsd-arm64': 3.1.1 + '@koromix/koffi-freebsd-ia32': 3.1.1 + '@koromix/koffi-freebsd-x64': 3.1.1 + '@koromix/koffi-linux-arm64': 3.1.1 + '@koromix/koffi-linux-ia32': 3.1.1 + '@koromix/koffi-linux-loong64': 3.1.1 + '@koromix/koffi-linux-riscv64': 3.1.1 + '@koromix/koffi-linux-x64': 3.1.1 + '@koromix/koffi-openbsd-ia32': 3.1.1 + '@koromix/koffi-openbsd-x64': 3.1.1 + '@koromix/koffi-win32-arm64': 3.1.1 + '@koromix/koffi-win32-ia32': 3.1.1 + '@koromix/koffi-win32-x64': 3.1.1 + layout-base@1.0.2: {} layout-base@2.0.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8f93814899..2de5578f5d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -31,6 +31,8 @@ allowBuilds: '@google/genai': false protobufjs: false node-addon-require-builtin: false + # JSONL durability calls MoveFileExW with write-through publication on Windows. + koffi: true # The Landlock launcher family is our own sibling-repo release, consumed # fresh (hours old at each coordinated bump) — the release-age quarantine