diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 3a802231d7..e8a06e469e 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -613,7 +613,7 @@ export interface Config { export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' ``` -Source: [`packages/session-query/session-query-sqlite/src/index.ts:58`](../packages/session-query/session-query-sqlite/src/index.ts) +Source: [`packages/session-query/session-query-sqlite/src/index.ts:67`](../packages/session-query/session-query-sqlite/src/index.ts) ## `@deepseek-ai/dsh-skill` diff --git a/docs/cordis-catalog/services.md b/docs/cordis-catalog/services.md index fef691aae1..e83b96bd11 100644 --- a/docs/cordis-catalog/services.md +++ b/docs/cordis-catalog/services.md @@ -195,11 +195,12 @@ abstract create(meta: SessionHeader): Promise abstract append(id: SessionId, events: readonly SessionEvent[]): Promise abstract load(id: SessionId): Promise<{ meta: SessionHeader; events: SessionEvent[] }> abstract list(): Promise +abstract listSnapshots(): Promise ``` Types: [SessionEvent](../core-data-structures/core.md) -Source: [`packages/session-persistence/session-persistence/src/index.ts:102`](../../packages/session-persistence/session-persistence/src/index.ts) +Source: [`packages/session-persistence/session-persistence/src/index.ts:112`](../../packages/session-persistence/session-persistence/src/index.ts) ## `ctx.sessionQuery` — `SessionQueryService` @@ -207,12 +208,13 @@ Live-preferred logical-corpus and exact-event read service. ```ts cordis-catalog listSessions(): Promise +async filterSessions(filters: readonly SessionResultFilter[]): Promise async listEvents(sessionId: SessionId): Promise async filterEvents( sessionId: SessionId, filters: readonly SessionEventResultFilter[], ): Promise async readEvent(request: SessionEventReadRequest): Promise ``` -Source: [`packages/session-query/session-query/src/index.ts:83`](../../packages/session-query/session-query/src/index.ts) +Source: [`packages/session-query/session-query/src/index.ts:96`](../../packages/session-query/session-query/src/index.ts) ## `ctx.sessions` — `SessionStore` @@ -244,7 +246,7 @@ abstract searchSessions( request: SessionSearchRequest, exec?: SessionSearchExec abstract searchEvents( request: SessionEventSearchRequest, exec?: SessionSearchExecContext, ): Promise> ``` -Source: [`packages/session-query/session-query/src/index.ts:54`](../../packages/session-query/session-query/src/index.ts) +Source: [`packages/session-query/session-query/src/index.ts:67`](../../packages/session-query/session-query/src/index.ts) ## `ctx.skills` — `SkillService` diff --git a/docs/core-data-structures/persistence.md b/docs/core-data-structures/persistence.md index 7bf102924b..9535686f86 100644 --- a/docs/core-data-structures/persistence.md +++ b/docs/core-data-structures/persistence.md @@ -2,7 +2,7 @@ The **durability seam** for the event log. [session.md](session.md) describes the in-memory `Session` — the append-only `SessionEvent` log that is the source of truth. This page describes how that log is made durable: the abstract `SessionPersistence` service, its backends, the flush checkpoint, crash recovery, and the metadata header that travels alongside the log. The event vocabulary the log carries is enumerated, member by member, in the generated [persistence log event catalog](../persistence-catalog.md). -The seam is a textbook [capability seam](../rfc/implemented/architecture/2026-06-13-capability-seams.md): one abstract service ([dsh-session-persistence](../../packages/session-persistence/session-persistence), `ctx.sessionPersistence`) defining create/append/load/list over the existing `SessionEvent` — **no parallel persisted type** — and two interchangeable backends that pass the same `runPersistenceContract` suite. See the [session-persistence RFC](../rfc/implemented/architecture/2026-06-14-session-persistence.md). +The seam is a textbook [capability seam](../rfc/implemented/architecture/2026-06-13-capability-seams.md): one abstract service ([dsh-session-persistence](../../packages/session-persistence/session-persistence), `ctx.sessionPersistence`) defining create/append/load plus lightweight list/snapshot observation over the existing `SessionEvent` — **no parallel persisted event type** — and two interchangeable backends that pass the same `runPersistenceContract` suite. See the [session-persistence RFC](../rfc/implemented/architecture/2026-06-14-session-persistence.md). ## The flush checkpoint @@ -78,9 +78,24 @@ interface CreateSessionOptions { Replay/fork is therefore `ctx.sessions.create(id, { seed: seedEvents })`; resuming a *persisted* session into a live agent is `ctx.agents.resume({ resumeSessionId })`. +## Lightweight source revisions + +Consumers of derived state compare a cheap opaque revision before loading a full event log. The persistence backend owns its representation and changes it transactionally with append or mutating load repair; callers compare it only for equality. + +```ts type-equiv +export type SessionPersistenceRevision = Branded<'SessionPersistenceRevision'> +``` + +```ts type-equiv +export interface SessionPersistenceSnapshot { + header: SessionHeader + revision: SessionPersistenceRevision +} +``` + ## The backends -Both implement the same abstract `SessionPersistence` (create/append/load/list over `SessionEvent`) and pass `runPersistenceContract`, proving the seam is genuinely backend-agnostic: +Both implement the same abstract `SessionPersistence` (create/append/load/list/listSnapshots over `SessionEvent`) and pass `runPersistenceContract`, proving the seam is genuinely backend-agnostic: - **[dsh-session-persistence-jsonl](../../packages/session-persistence/session-persistence-jsonl)** — an append-only JSONL log per session with crash-safe atomic writes, the interrupted-turn crash recovery above, and a read/replay path. - **[dsh-session-persistence-sqlite](../../packages/session-persistence/session-persistence-sqlite)** — `node:sqlite`, one row per `SessionEvent`. The row shape `(session_id, seq, type, time, data, source_event_seqs, surface_op)` maps 1:1 onto the event, including optional surface metadata, so there is no parallel persisted schema to keep in sync. diff --git a/docs/core-data-structures/session-query.md b/docs/core-data-structures/session-query.md index 27ef9959de..d1ed41b775 100644 --- a/docs/core-data-structures/session-query.md +++ b/docs/core-data-structures/session-query.md @@ -58,19 +58,23 @@ export interface SessionEventSearchDocument extends SessionEventRecord { } ``` -`ctx.sessionQuery.filterEvents(sessionId, filters)` returns these documents in ascending seq order. Messages, reasoning, tool calls/results, blocked prompts, todos, and failure/status detail contribute semantic text; structural events and stream chunks do not. +`ctx.sessionQuery.filterSessions(filters)` applies `SessionResultFilter` to the complete logical corpus; `ctx.sessionQuery.filterEvents(sessionId, filters)` returns matching documents in ascending seq order. Messages, reasoning, tool calls/results, blocked prompts, todos, and failure/status detail contribute semantic text; structural events and stream chunks do not. ## Full-text search pages The independent `ctx.sessionSearch` seam has two scopes. `searchSessions()` groups the corpus by strongest matching event; `searchEvents()` searches one session. Requests bind an opaque cursor to the normalized query, metadata filters, and limit. The event text scan is intentionally absent from provider metadata filters. +```ts type-equiv +export type SessionSearchCursor = Branded<'SessionSearchCursor'> +``` + ```ts type-equiv export interface SessionSearchRequest { query: string sessionFilters?: readonly SessionResultFilter[] eventFilters?: readonly SessionEventMetadataFilter[] limit?: number - cursor?: string + cursor?: SessionSearchCursor } ``` @@ -80,14 +84,14 @@ export interface SessionEventSearchRequest { query: string filters?: readonly SessionEventMetadataFilter[] limit?: number - cursor?: string + cursor?: SessionSearchCursor } ``` ```ts type-equiv export interface SessionSearchPage { items: readonly T[] - nextCursor?: string + nextCursor?: SessionSearchCursor } ``` diff --git a/docs/module-graph.md b/docs/module-graph.md index 6fcec53dd4..7ad4cd8c70 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -151,6 +151,7 @@ flowchart TD pkg_web_search_deepseek --> pkg_web pkg_web_search_exa --> pkg_web pkg_web_search_perplexity --> pkg_web + pkg_session_persistence --> pkg_brand pkg_session_persistence --> pkg_session pkg_llm_replay --> pkg_llm pkg_llm_replay --> pkg_session @@ -168,6 +169,7 @@ flowchart TD pkg_session_persistence_jsonl --> pkg_session_persistence pkg_session_persistence_sqlite --> pkg_session pkg_session_persistence_sqlite --> pkg_session_persistence + pkg_session_query --> pkg_brand pkg_session_query --> pkg_llm pkg_session_query --> pkg_session pkg_session_query --> pkg_session_persistence @@ -361,7 +363,7 @@ flowchart TD | [`web-search-deepseek`](../packages/web/web-search-deepseek) | `web` | [`web`](../packages/web/web) | | [`web-search-exa`](../packages/web/web-search-exa) | `web` | [`web`](../packages/web/web) | | [`web-search-perplexity`](../packages/web/web-search-perplexity) | `web` | [`web`](../packages/web/web) | -| [`session-persistence`](../packages/session-persistence/session-persistence) | `session-persistence` | [`session`](../packages/core/session) | +| [`session-persistence`](../packages/session-persistence/session-persistence) | `session-persistence` | [`brand`](../packages/util/brand), [`session`](../packages/core/session) | | [`llm-replay`](../packages/support/llm-replay) | `support` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | | [`sandbox-local`](../packages/sandbox/sandbox-local) | `sandbox` | [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox) | | [`bash-local`](../packages/bash/bash-local) | `bash` | [`bash`](../packages/bash/bash), [`timeout`](../packages/util/timeout) | @@ -369,7 +371,7 @@ flowchart TD | [`hook-protocol`](../packages/hooks/hook-protocol) | `hooks` | [`bash`](../packages/bash/bash), [`session`](../packages/core/session) | | [`session-persistence-jsonl`](../packages/session-persistence/session-persistence-jsonl) | `session-persistence` | [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | | [`session-persistence-sqlite`](../packages/session-persistence/session-persistence-sqlite) | `session-persistence` | [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | -| [`session-query`](../packages/session-query/session-query) | `session-query` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | +| [`session-query`](../packages/session-query/session-query) | `session-query` | [`brand`](../packages/util/brand), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | | [`invariants`](../packages/support/invariants) | `support` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`session`](../packages/core/session) | | [`user-approval`](../packages/ui/user-approval) | `ui` | [`agent`](../packages/core/agent), [`brand`](../packages/util/brand), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt) | | [`user-interaction`](../packages/ui/user-interaction) | `ui` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm) | diff --git a/docs/rfc/implemented/feature/2026-07-10-session-query-service.md b/docs/rfc/implemented/feature/2026-07-10-session-query-service.md index be496311cc..10baa3e2c3 100644 --- a/docs/rfc/implemented/feature/2026-07-10-session-query-service.md +++ b/docs/rfc/implemented/feature/2026-07-10-session-query-service.md @@ -10,7 +10,7 @@ Full-text search is related but materially larger. Putting provider coordination ## Decision -`@deepseek-ai/dsh-session-query` owns `ctx.sessionQuery`, a small trusted exact-read service over one logical corpus. It exposes `listSessions()`, `listEvents(sessionId)`, provider-independent `filterEvents(sessionId, filters)`, and bounded `readEvent(request)`. The package also declares the separate abstract `ctx.sessionSearch` contract and shared semantic extraction used by the [SQLite search decision](2026-07-10-sqlite-session-query-provider.md), but `ctx.sessionQuery` does not coordinate providers or synchronize a derived index. +`@deepseek-ai/dsh-session-query` owns `ctx.sessionQuery`, a small trusted exact-read service over one logical corpus. It exposes `listSessions()`, provider-independent `filterSessions(filters)`, `listEvents(sessionId)`, `filterEvents(sessionId, filters)`, and bounded `readEvent(request)`. The package also declares the separate abstract `ctx.sessionSearch` contract and shared semantic extraction used by the [SQLite search decision](2026-07-10-sqlite-session-query-provider.md), but `ctx.sessionQuery` does not coordinate providers or synchronize a derived index. The service observes the optional `ctx.sessionPersistence` binding dynamically but retains no persisted cache or invalidation listener. Each cross-corpus list asks the active backend for authoritative metadata, then overlays a fresh live-store list. Matching ids become one `SessionRecord`: the live header wins and `live`/`persisted` independently report source availability. Immutable header disagreement is `SESSION_QUERY_SOURCE_CONFLICT`. diff --git a/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md b/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md index d620ece108..fafa7f3569 100644 --- a/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md +++ b/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md @@ -10,19 +10,19 @@ Splitting those concerns across a provider coordinator and a database implementa ## Decision -`@deepseek-ai/dsh-session-query` declares an independent abstract `ctx.sessionSearch` service without changing the exact-read `ctx.sessionQuery` key. `searchSessions(request, exec?)` returns cursor-paginated `SessionSearchHit`s grouped by each session's strongest matching event; `searchEvents(request, exec?)` returns `SessionEventSearchHit`s within one logical session. Both requests require `query`, accept `limit` and an opaque `cursor`, and support an optional abort signal. Session search accepts `sessionFilters` plus event metadata filters; event search accepts event metadata filters. Results expose bounded plain-text snippets but no provider identifier or numeric relevance score. +`@deepseek-ai/dsh-session-query` declares an independent abstract `ctx.sessionSearch` service without changing the exact-read `ctx.sessionQuery` key. `searchSessions(request, exec?)` returns cursor-paginated `SessionSearchHit`s grouped by each session's strongest matching event; `searchEvents(request, exec?)` returns `SessionEventSearchHit`s within one logical session. Both requests require `query`, accept `limit` and an owned branded `SessionSearchCursor`, and support an optional abort signal. Session search accepts `sessionFilters` plus event metadata filters; event search accepts event metadata filters. Results expose bounded plain-text snippets but no provider identifier or numeric relevance score. `@deepseek-ai/dsh-session-query-sqlite` is the sole concrete owner of `ctx.sessionSearch`. It depends on live `ctx.sessions`, observes optional `ctx.sessionPersistence` dynamically, and owns a dedicated derived SQLite database. There is no search-provider registry, coordinator, persistence event, or agent-loop integration. -The interface package also owns shared first-party semantic extraction and provider-independent filtering. `SessionResultFilter` covers id, nullable cwd, created-at range, nullable parent, and availability. `SessionEventResultFilter` covers seq/time ranges, event type, surface, and literal semantic text. Arrays are ANDed and list values are ORed. The text clause escapes caller input into a Unicode case-insensitive regular expression whose whitespace runs match one or more whitespace characters; it is available through `ctx.sessionQuery.filterEvents()` and is not delegated to an FTS provider. +The interface package also owns shared first-party semantic extraction and provider-independent filtering. `SessionResultFilter` covers id, nullable cwd, created-at range, nullable parent, and availability; `ctx.sessionQuery.filterSessions()` applies it without an FTS provider. `SessionEventResultFilter` covers seq/time ranges, event type, surface, and literal semantic text. Arrays are ANDed and list values are ORed. The text clause escapes caller input into a Unicode case-insensitive regular expression whose whitespace runs match one or more whitespace characters; it is available through `ctx.sessionQuery.filterEvents()` and is not delegated to an FTS provider. ## Search semantics Each semantic event is one FTS document carrying session metadata, event metadata, surface classification, and extracted text. All `current`, `shadowed`, and `log-only` documents participate unless a surface filter narrows them. Metadata filters compile to parameterized SQL before ranking. Session results partition matching documents by session and retain the strongest one. -Ordering is deterministic: BM25 ascending, event time descending, session id ascending for the cross-session scope, and seq descending. Provider scores remain private. Snippets normalize whitespace and are bounded by Unicode code points. Opaque cursors bind to the service instance, scope, canonical normalized request, offset, and relevant generation. Any corpus change invalidates cross-session cursors; a within-session cursor changes only when its target source/generation changes, so unrelated sessions do not invalidate it. Reopening creates a new service instance and invalidates old cursors. +Ordering is deterministic and comparable across the persistent and TEMP FTS tables: actual FTS5 highlighted-match span count descending, indexed document code-point length ascending, event time descending, session id ascending for the cross-session scope, and seq descending. Snippets use those actual highlight positions, strip the reserved markers, normalize whitespace, and bound by Unicode code points. Opaque cursors bind to the service instance, scope, canonical normalized request, offset, and relevant generation. Any corpus change invalidates cross-session cursors; a within-session cursor changes only when its target source/generation changes, so unrelated sessions do not invalidate it. Reopening creates a new service instance and invalidates old cursors. -Queries are trimmed, whitespace-normalized, and quoted as one literal FTS5 phrase. Embedded quotes are doubled before binding, so MATCH operators such as `OR`, `NEAR`, quotes, parentheses, and `*` remain data rather than executable query syntax. Phrase matching follows tokenizer tokens rather than arbitrary substrings. +Queries are trimmed, whitespace-normalized, and quoted as one literal FTS5 phrase. Embedded quotes are doubled before binding, so MATCH operators such as `OR`, `NEAR`, quotes, parentheses, and `*` remain data rather than executable query syntax. NUL is rejected before SQLite execution. Reserved highlight noncharacters and NUL in documents are normalized before indexing, making inserted presentation markers collision-free. Phrase matching follows tokenizer tokens rather than arbitrary substrings. ## Tokenizer choice @@ -32,11 +32,11 @@ Both persistent and live FTS5 tables use `unicode61`. The implementation experim The shared extractor includes message text, reasoning, nested tool-call/result content, tool names and arguments, blocked-prompt reasons, todo status/content, and error or terminal status detail. Structural boundaries, stream chunks, request headers, successful completion markers, and unknown declaration-merged event/content variants produce no document. Surface classification reuses `foldSurface()` so search agrees with model-history derivation. -One serialized operation observes complete persisted and live sources, computes stable fingerprints, reconciles rows in one transaction, and executes the query. Unchanged persisted sessions retain their rows and generation. New, changed, and deleted persisted sessions update on the next search. A source or extraction failure cannot mark a row current, and a transaction failure rolls back so a later search retries. +One serialized operation reads the provider-neutral `SessionPersistence` snapshot listing, compares each opaque revision with the revision stored beside the indexed session, loads only new or changed logs, reconciles rows in one transaction, and executes the query. Observation repeats when listing changes during a load; this incorporates a mutating load repair's refreshed revision before commit. Repeated queries and unchanged reopen load no full persisted logs. New, changed, and deleted sessions update on the next stable search. A source or extraction failure cannot mark a row current, and a transaction failure rolls back so a later search retries. Persisted documents survive restarts. Live sessions use connection-local TEMP tables, shadow the persisted base for the same id, and reveal that base on detach. Closing the database drops live rows. Unmounting persistence hides durable rows without treating absence as authoritative deletion; remounting observes and reconciles the backend again. Conflicting immutable live and durable headers fail rather than combining sources. -The derived schema has its own application id and monotonic schema version. A recognized incompatible version resets only this derived database. A database with a foreign application id or unrecognized user tables is refused, which prevents an accidentally configured canonical session database from being reset. +The derived schema has its own application id and monotonic schema version. A recognized incompatible version resets only this derived database. A database with a foreign application id or unrecognized user tables is refused before journal-mode mutation, which prevents an accidentally configured canonical session database from being changed. One service in one process exclusively owns a derived-index path; cross-process writers are unsupported because generations and live TEMP shadow state are connection-owned. Cancellation rejects queued operations and caller waits around asynchronous source observation without committing an aborted observation. Node's synchronous `DatabaseSync` MATCH call cannot be interrupted once it is executing on the JavaScript thread, so the service checks the signal at serialized boundaries but does not promise mid-statement preemption. @@ -46,11 +46,11 @@ Cancellation rejects queued operations and caller waits around asynchronous sour - **Add a phase-one provider registry and coordinator** — rejected because one implementation provides no evidence for registration semantics and would split one reconciliation lifecycle across two owners. - **Persist live overrides immediately** — rejected because live events are not canonical until the existing checkpoint commits. - **Use the FTS5 trigram tokenizer** — rejected because it omits useful queries shorter than three characters and measured about 2.1× the index size of `unicode61`; literal substring filtering remains available through the scan path. -- **Return BM25 scores** — rejected because provider-specific numeric scales are unstable across corpus changes. +- **Use FTS5 BM25 independently in each table** — rejected because scores from differently populated persistent and TEMP corpora are not comparable; actual matched spans and document length have one shared scale. ## Consequences -Search has a small provider-neutral API while its only backend owns every derived-index state transition. The separate database adds configuration and a reconciliation read before queries, but index corruption, reset, and tokenizer changes cannot endanger canonical logs. Persistent fingerprints avoid rewriting unchanged sessions; TEMP live overlays preserve current-session truth without making uncheckpointed events durable. +Search has a small provider-neutral API while its only backend owns every derived-index state transition. The separate database adds configuration and a lightweight snapshot read before queries, but index corruption, reset, and tokenizer changes cannot endanger canonical logs. Durable revisions avoid full-log reads and rewrites for unchanged sessions; TEMP live overlays preserve current-session truth without making uncheckpointed events durable. The chosen tokenizer supports short tokens with a smaller index but does not promise substring recall. Literal phrases make query syntax safe and predictable at the cost of excluding boolean/full MATCH expressions. Cancellation is effective while queued or awaiting sources, but synchronous SQLite execution remains a non-preemptible section. diff --git a/packages/cordis/tool-cordis/src/api-catalog.ts b/packages/cordis/tool-cordis/src/api-catalog.ts index 2859353cfa..7322698cdd 100644 --- a/packages/cordis/tool-cordis/src/api-catalog.ts +++ b/packages/cordis/tool-cordis/src/api-catalog.ts @@ -149,6 +149,7 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ 'abstract append(id: SessionId, events: readonly SessionEvent[]): Promise', 'abstract load(id: SessionId): Promise<{ meta: SessionHeader; events: SessionEvent[] }>', 'abstract list(): Promise', + 'abstract listSnapshots(): Promise', ], }, { @@ -156,6 +157,7 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ summary: 'Live-preferred logical-corpus and exact-event read service.', methods: [ 'listSessions(): Promise', + 'async filterSessions(filters: readonly SessionResultFilter[]): Promise', 'async listEvents(sessionId: SessionId): Promise', 'async filterEvents( sessionId: SessionId, filters: readonly SessionEventResultFilter[], ): Promise', 'async readEvent(request: SessionEventReadRequest): Promise', @@ -834,7 +836,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'SessionEventSearchRequest', - declaration: 'export interface SessionEventSearchRequest {\n sessionId: SessionId;\n query: string;\n filters?: readonly SessionEventMetadataFilter[];\n limit?: number;\n cursor?: string;\n}', + declaration: 'export interface SessionEventSearchRequest {\n sessionId: SessionId;\n query: string;\n filters?: readonly SessionEventMetadataFilter[];\n limit?: number;\n cursor?: SessionSearchCursor;\n}', }, { name: 'SessionEventSurface', @@ -860,6 +862,14 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SessionId', declaration: 'export type SessionId = Branded<\'SessionId\'>;', }, + { + name: 'SessionPersistenceRevision', + declaration: 'export type SessionPersistenceRevision = Branded<\'SessionPersistenceRevision\'>;', + }, + { + name: 'SessionPersistenceSnapshot', + declaration: 'export interface SessionPersistenceSnapshot {\n header: SessionHeader;\n revision: SessionPersistenceRevision;\n}', + }, { name: 'SessionRecord', declaration: 'export interface SessionRecord {\n header: SessionHeader;\n live: boolean;\n persisted: boolean;\n}', @@ -872,6 +882,10 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SessionResultRange', declaration: 'export interface SessionResultRange {\n from?: number;\n to?: number;\n}', }, + { + name: 'SessionSearchCursor', + declaration: 'export type SessionSearchCursor = Branded<\'SessionSearchCursor\'>;', + }, { name: 'SessionSearchExecContext', declaration: 'export interface SessionSearchExecContext {\n signal?: AbortSignal;\n}', @@ -882,11 +896,11 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'SessionSearchPage', - declaration: 'export interface SessionSearchPage {\n items: readonly T[];\n nextCursor?: string;\n}', + declaration: 'export interface SessionSearchPage {\n items: readonly T[];\n nextCursor?: SessionSearchCursor;\n}', }, { name: 'SessionSearchRequest', - declaration: 'export interface SessionSearchRequest {\n query: string;\n sessionFilters?: readonly SessionResultFilter[];\n eventFilters?: readonly SessionEventMetadataFilter[];\n limit?: number;\n cursor?: string;\n}', + declaration: 'export interface SessionSearchRequest {\n query: string;\n sessionFilters?: readonly SessionResultFilter[];\n eventFilters?: readonly SessionEventMetadataFilter[];\n limit?: number;\n cursor?: SessionSearchCursor;\n}', }, { name: 'SkillCandidate', diff --git a/packages/session-persistence/session-persistence-jsonl/README.md b/packages/session-persistence/session-persistence-jsonl/README.md index 990ad2fcc4..0be2c785b3 100644 --- a/packages/session-persistence/session-persistence-jsonl/README.md +++ b/packages/session-persistence/session-persistence-jsonl/README.md @@ -25,6 +25,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence - **Append-only.** Committed events (at or below a flushed `turn/end`) are never rewritten. Subsequent appends are line appends at EOF + `fsync`. - **Crash recovery — close, don't truncate.** A crash can leave a log whose final turn never closed (real events after the last `turn/end`). `load` PRESERVES those events (a turn can be huge — they are real work) and closes the orphaned turn by durably appending synthetic boundary events: an error `tool/result` for every `tool-call` the crash left unanswered (the loop logs the assistant message before running the tools, so a mid-tool crash leaves dangling calls — and `deriveMessages()` would replay an assistant tool-call with no result, which providers reject), then a `step/end` if a step was open, then `turn/end {kind:'interrupted'}`, returning a balanced log. Only a never-fully-written **torn tail fragment** (a final line with no newline / unparseable) is `ftruncate`d away before the closers are written. See [session persistence](../../../docs/rfc/implemented/architecture/2026-06-14-session-persistence.md). - **Contiguous-seq.** `load` rejects a mid-log parse error or `seq` gap (unloadable); `append` rejects a batch whose first `seq` does not continue the stored log, and rejects non-JSON-serializable `event.data` naming the offending event type. +- **Lightweight revisions.** `listSnapshots()` reads each header and returns an opaque identity from the file device/inode, size, and nanosecond mtime/ctime. Under the append-only single-writer contract, unchanged files retain revisions while append, repair, or replacement changes them without parsing event lines. - **Format version.** Only the current `SESSION_FORMAT_VERSION` (v0) is supported; `load` rejects any other version. While the harness is unreleased the on-disk format is pre-release/unstable: a breaking format change is absorbed at v0 (no bump until the first tagged release) and non-current logs are rejected — there is no migration (no persisted user data to preserve). ## Write path diff --git a/packages/session-persistence/session-persistence-jsonl/src/index.ts b/packages/session-persistence/session-persistence-jsonl/src/index.ts index bd31e0ae47..0aea7927d1 100644 --- a/packages/session-persistence/session-persistence-jsonl/src/index.ts +++ b/packages/session-persistence/session-persistence-jsonl/src/index.ts @@ -11,7 +11,7 @@ * (the `session/event` → buffer → `session/flush` drain, per-session * serialization, write cursors, fork-seed persistence, HMR live-adoption, * crash-repair sequencing, dispose quiescence) lives in the backend-agnostic - * {@link PersistenceCoordinator} this class composes. The four public + * {@link PersistenceCoordinator} this class composes. The stateful public * {@link SessionPersistence} methods delegate to the coordinator. * * @module @deepseek-ai/dsh-session-persistence-jsonl @@ -19,12 +19,12 @@ import { Context } from 'cordis' import z from 'schemastery' -import { open, mkdir, readFile, readdir, link, rm, truncate } from 'node:fs/promises' +import { open, mkdir, readFile, readdir, link, rm, stat, truncate } from 'node:fs/promises' import { dirname, resolve } from 'node:path' import { randomBytes } from 'node:crypto' import { - SessionPersistence, PersistenceCoordinator, - type PersistenceBackend, type StoredPrefix, + SessionPersistence, SessionPersistenceRevision, PersistenceCoordinator, + type PersistenceBackend, type SessionPersistenceSnapshot, type StoredPrefix, } from '@deepseek-ai/dsh-session-persistence' import type { Session, SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session' import { @@ -179,20 +179,44 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi /** List all stored sessions' metadata (header line only — no full-log parse). */ async list(): Promise { - const metas: SessionHeader[] = [] + return (await this.listArtifacts()).map(artifact => artifact.header) + } + + /** List metadata plus a stat-derived identity for each append-only log. */ + async listSnapshots(): Promise { + const snapshots: SessionPersistenceSnapshot[] = [] + for (const artifact of await this.listArtifacts()) { + const identity = await stat(artifact.path, { bigint: true }) + snapshots.push({ + header: artifact.header, + revision: SessionPersistenceRevision([ + identity.dev, + identity.ino, + identity.size, + identity.mtimeNs, + identity.ctimeNs, + ].join(':')), + }) + } + return snapshots + } + + private async listArtifacts(): Promise> { + const artifacts: Array<{ header: SessionHeader; path: string }> = [] for (const dir of await this.listCwdDirs()) { for (const name of await this.listJsonl(dir)) { // Read ONLY the header line, not the whole log: a session picker must // scale with the number of sessions, not the total size of every // conversation (the log persists every assistant/chunk verbatim). - const first = await this.readFirstLine(`${dir}/${name}`) + const path = `${dir}/${name}` + const first = await this.readFirstLine(path) if (first === undefined) continue // empty/half-written file const meta = parseHeaderMeta(first) if (meta === undefined) continue // not a session header - metas.push(meta) + artifacts.push({ header: meta, path }) } } - return metas + return artifacts } // --- materialization / append / repair (file mechanics) --- diff --git a/packages/session-persistence/session-persistence-sqlite/README.md b/packages/session-persistence/session-persistence-sqlite/README.md index 082b60af88..6e6006dc6b 100644 --- a/packages/session-persistence/session-persistence-sqlite/README.md +++ b/packages/session-persistence/session-persistence-sqlite/README.md @@ -6,7 +6,7 @@ A SQLite durable session-persistence backend — a second `SessionPersistence` i ## Storage model -Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [session surface](../../../docs/rfc/implemented/architecture/2026-06-18-session-surface.md)). Out-of-log metadata (`SessionHeader`) lives in a `sessions` row. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`list` reports exactly the sessions that have a row), so no separate column is needed. +Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [session surface](../../../docs/rfc/implemented/architecture/2026-06-18-session-surface.md)). Out-of-log metadata (`SessionHeader`) and a monotonic per-log revision live in a `sessions` row. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`list` reports exactly the sessions that have a row). The repo's `engines.node` is `^22.19.0 || >=24.0.0` (Node 22.19+ or 24+), matching the LTS floor required by the installed Pi adapter dependency; `node:sqlite` itself ships without the `--experimental-sqlite` flag from Node 22.13 (LTS) and 23.4 / 24 (Current) on. The range deliberately excludes Node 23 because that line is non-LTS/EOL and still has flagged runtime features before 23.6. The database opens with `foreign_keys = ON` (so `ON DELETE CASCADE` drops a session's events with its row) and the configured `journal_mode` (default `wal`; pick a rollback-journal mode like `delete` on filesystems where WAL's shared-memory files do not work, e.g. network mounts). The table-layout version is stored in `PRAGMA user_version` and checked on open: a fresh database is stamped with the current `SCHEMA_VERSION`; a database written by any other, incompatible build (a non-current `user_version`, older or newer) is rejected rather than opened against an unknown layout — there is no migration (unreleased software). @@ -14,6 +14,7 @@ The repo's `engines.node` is `^22.19.0 || >=24.0.0` (Node 22.19+ or 24+), matchi - **Append = a transaction.** `append` runs `BEGIN`/`COMMIT` around the batch: it materializes the `sessions` row (if still lazy) and INSERTs every event, asserting the contiguous-seq contract first (the first event's `seq` must equal the stored next-seq). A mid-batch failure (a UNIQUE violation on a duplicated seq) rolls back entirely, so the stored log and the in-memory cursor stay consistent. (`load()` already balanced the stored log, so `append` never has to repair a crash tail.) - **Lazy materialization.** `create()` records intent in memory only — no row is written until the first `append`. A created-but-never-appended session has no `sessions` row, so it is absent from `list()` (which reports exactly the sessions that have a row). +- **Lightweight revisions.** `listSnapshots()` reads the monotonic revision stored beside each session header. Append and mutating load repair increment it in the same transaction as their event changes, so unchanged observations are stable and no full-log count or parse is required. - **Interrupted-turn close on load.** `load()` reads every stored event ordered by `seq` and finds the longest seq-contiguous, parseable prefix — INCLUDING the real events of an interrupted final turn after the last `turn/end` (the loop only flushes at `turn/end`, so a process killed mid-turn leaves real, fully-written rows past it). A single turn can be huge in a long-horizon task, so those events are **preserved, never truncated**: `load()` CLOSES the orphaned turn by durably appending the minimal synthetic boundary events (an error `tool/result` for every assistant tool call left unanswered, a `step/end` if a step was open, then a `turn/end` carrying `{ kind: 'interrupted' }`), inside one transaction that also DELETEs any never-fully-written torn tail row. `load()` is therefore mutating — after it the stored rows are balanced and the cursor is truthful, so the next `append` continues cleanly. The boundary (last `turn/end`, torn-tail detection) is computed from the `seq`/`type` columns so a malformed `data` in a torn tail row is never parsed (discarded, not unloadable). A parse error or `seq` gap inside the committed region (at or before the last real `turn/end`) makes the session unloadable. A session whose only turn never closed keeps its metadata row and stays present in `list()` — the same as the JSONL backend, whose file likewise survives a first append that never reached `turn/end`. ## Configuration (schemastery) diff --git a/packages/session-persistence/session-persistence-sqlite/src/index.ts b/packages/session-persistence/session-persistence-sqlite/src/index.ts index 30387b4837..3a1c7ffbcc 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/index.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/index.ts @@ -11,7 +11,7 @@ * Like the JSONL backend it supplies ONLY the storage primitives (the * {@link PersistenceBackend} hooks below — INSERT/DELETE/SELECT inside * transactions); all the write-path orchestration lives in the backend-agnostic - * {@link PersistenceCoordinator} this class composes. The four public + * {@link PersistenceCoordinator} this class composes. The stateful public * {@link SessionPersistence} methods delegate to the coordinator. * * @module @deepseek-ai/dsh-session-persistence-sqlite @@ -23,8 +23,8 @@ import { DatabaseSync } from 'node:sqlite' import { mkdir } from 'node:fs/promises' import { dirname, resolve } from 'node:path' import { - SessionPersistence, PersistenceCoordinator, - type PersistenceBackend, type StoredPrefix, + SessionPersistence, SessionPersistenceRevision, PersistenceCoordinator, + type PersistenceBackend, type SessionPersistenceSnapshot, type StoredPrefix, } from '@deepseek-ai/dsh-session-persistence' import type { Session, SessionEvent, SurfaceEventType, SessionId, SessionHeader } from '@deepseek-ai/dsh-session' import { @@ -181,6 +181,7 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers const [surfaceSeqs, surfaceOp] = surfaceBindings(event) insertEvent.run(meta.id, event.seq, event.type, event.time, JSON.stringify(event.data), surfaceSeqs, surfaceOp) } + this.db.prepare('UPDATE sessions SET revision = revision + 1 WHERE id = ?').run(meta.id) this.db.exec('COMMIT') } catch (error) { this.db.exec('ROLLBACK') @@ -209,6 +210,9 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers insertEvent.run(meta.id, event.seq, event.type, event.time, JSON.stringify(event.data), surfaceSeqs, surfaceOp) } } + if (tornMarker !== undefined || closers.length > 0) { + this.db.prepare('UPDATE sessions SET revision = revision + 1 WHERE id = ?').run(meta.id) + } this.db.exec('COMMIT') } catch (error) { // The DELETE+INSERT cannot collide (a row at a closer's seq is preserved or @@ -230,6 +234,16 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers return rows.map(rowToMeta) } + /** List metadata with an append-only event-count revision per session. */ + async listSnapshots(): Promise { + await this.ready + const rows = this.db.prepare('SELECT * FROM sessions').all() as unknown as SessionRow[] + return rows.map(row => ({ + header: rowToMeta(row), + revision: SessionPersistenceRevision(`revision:${row.revision}`), + })) + } + /** Close the database handle (awaited by the coordinator's dispose, post-drain). */ async close(): Promise { await this.ready @@ -250,8 +264,8 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers */ private writeRow(meta: SessionHeader): void { this.db.prepare(` - INSERT INTO sessions (id, version, created_at, cwd, parent_session, seed_length) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO sessions (id, version, created_at, cwd, parent_session, seed_length, revision) + VALUES (?, ?, ?, ?, ?, ?, 0) ON CONFLICT(id) DO UPDATE SET version = excluded.version, created_at = excluded.created_at, diff --git a/packages/session-persistence/session-persistence-sqlite/src/schema.ts b/packages/session-persistence/session-persistence-sqlite/src/schema.ts index 2dacbe04a0..2f238b0131 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/schema.ts @@ -15,7 +15,7 @@ import type { SessionEvent, SessionId, SessionHeader, SurfaceOp } from '@deepsee * layout; orthogonal to a session's own `version` (which versions the EVENT * vocabulary, stored per session in the `sessions` row). */ -export const SCHEMA_VERSION = 4 +export const SCHEMA_VERSION = 5 /** * A row of the `sessions` table — the out-of-log metadata ({@link SessionHeader}). @@ -31,6 +31,8 @@ export interface SessionRow { cwd: string | null parent_session: string | null seed_length: number | null + /** Monotonic log-change token incremented in each mutating transaction. */ + revision: number } /** An `events` table row: one `SessionEvent` mapped 1:1 (`data` is JSON text). */ @@ -67,15 +69,9 @@ export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' * current {@link SCHEMA_VERSION}; an existing database whose version is NOT the * current one (written by a different, incompatible build — older or newer) is * REJECTED rather than opened against a layout this build does not understand. - * There are no migrations: an earlier layout is not upgraded in place — it is - * rejected. v1 had a different `sessions` shape; v2 lacked all of - * `seed_length`/`source_event_seqs`/`surface_op`. v3 is SKIPPED: two unmerged - * branches each shipped a DISTINCT v3 (one adding only `seed_length`, the other - * adding only the surface columns), so an on-disk v3 is ambiguous — it could be - * either sibling layout, neither of which has all of this build's columns. v4 - * is the merged layout carrying every column; bumping past the collided v3 - * makes the version check reject both sibling v3 databases instead of opening - * one against columns it does not have. + * There are no migrations: an incompatible layout is rejected. The current + * sessions row carries every header field plus its monotonic snapshot revision; + * the events row carries the complete surface metadata. * @param path - the SQLite database file to open (created when absent). * @param journalMode - the journal pragma to apply — a closed in-code union, validated by the plugin Config. * @returns the open handle with pragmas applied and both tables ensured. @@ -105,7 +101,8 @@ export function openDatabase(path: string, journalMode: JournalMode): DatabaseSy created_at INTEGER NOT NULL, cwd TEXT, parent_session TEXT, - seed_length INTEGER + seed_length INTEGER, + revision INTEGER NOT NULL ) STRICT `) db.exec(` diff --git a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts index a6eda72685..98924ca691 100644 --- a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts @@ -258,11 +258,11 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { it('rejects a sibling v3 database (the merge-collided version) rather than opening it against missing columns', async () => { // Two unmerged branches each shipped a DISTINCT layout under user_version 3 // (one added only `seed_length`, the other only the surface columns). The - // merged build is v4; an on-disk v3 is ambiguous and is missing at least one + // the current build rejects every older layout; an on-disk v3 is ambiguous and is missing at least one // of this build's columns, so it MUST be rejected, not opened. Stamp a v3 // database and confirm the version check refuses it. const path = await freshDbPath() - openDatabase(path, 'wal').close() // creates + stamps user_version = SCHEMA_VERSION (4) + openDatabase(path, 'wal').close() // creates + stamps user_version = SCHEMA_VERSION const db = openDatabase(path, 'wal') db.exec('PRAGMA user_version = 3') db.close() @@ -338,7 +338,18 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { }) it('exposes the schema version constant', () => { - expect(SCHEMA_VERSION).toBe(4) + expect(SCHEMA_VERSION).toBe(5) + }) + + it('keeps the revision stable for an empty repair hook', async () => { + const b = await backend() + const m = meta('empty-repair') + await b.ctx.sessionPersistence.create(m) + await b.ctx.sessionPersistence.append(m.id, oneTurnLog()) + const before = await b.ctx.sessionPersistence.listSnapshots() + await (b.ctx.sessionPersistence as SessionPersistenceSqlite).commitRepair(m, undefined, []) + expect(await b.ctx.sessionPersistence.listSnapshots()).toEqual(before) + await b.dispose() }) }) diff --git a/packages/session-persistence/session-persistence/README.md b/packages/session-persistence/session-persistence/README.md index bf7da03757..32958bfaaa 100644 --- a/packages/session-persistence/session-persistence/README.md +++ b/packages/session-persistence/session-persistence/README.md @@ -12,6 +12,7 @@ The persisted unit IS the existing `SessionEvent` (event-sourced model — the l | `append(id, events): Promise` | Durably persist a batch (from the `session/flush` drain). Append-only; first event `seq` == stored next-seq after any repair; rejects non-JSON-serializable data naming the offending type. | | `load(id): Promise<{ meta; events }>` | Reload meta + log. Preserves an interrupted (unclosed) final turn and closes it with synthetic closers — an error `tool/result` per unanswered `tool-call`, then `step/end?`+`turn/end {interrupted}` (a turn can be huge — never truncated); only a torn tail fragment is dropped. Events contiguous (`events[i].seq === i`); rejects a committed-region gap/parse error or unknown `version`. | | `list(): Promise` | Lightweight listing from metadata, no full-log parse. A zero-event lazily-materialized session is absent from `list`. | +| `listSnapshots(): Promise` | Lightweight metadata plus an opaque branded per-log revision, without loading event logs. A revision stays equal while that log is unchanged and changes after append or mutating load repair. | ## Invariants every backend must honor @@ -24,7 +25,7 @@ The persisted unit IS the existing `SessionEvent` (event-sourced model — the l The two first-party backends were byte-identical (or same-algorithm) for ALL of their write-path orchestration — the in-memory bookkeeping (per-id state, write-behind buffers, per-id serialization chains, per-session init promises), the `session/event` → buffer → `session/flush` drain, lazy materialization, crash-tail repair on load, the four `session/created` adoption cases (new / HMR-adopt / collision / ownerless-claim), and dispose-time quiescence. Only the STORAGE primitives differed (write bytes vs. INSERT rows). -`PersistenceCoordinator` owns that orchestration once. A first-party backend composes one (`new PersistenceCoordinator(ctx, this)`), implements the small `PersistenceBackend` hook interface, and delegates its four public service methods to the coordinator. This keeps the duplicated, correctness-heavy orchestration in a single place (it used to receive the same fixes twice). +`PersistenceCoordinator` owns that orchestration once. A first-party backend composes one (`new PersistenceCoordinator(ctx, this)`), implements the small `PersistenceBackend` hook interface, and delegates the stateful write/read methods to the coordinator. Lightweight snapshot listing remains a backend storage primitive because its revision identity is backend-owned. The `PersistenceBackend` hooks (the only seam between the coordinator and storage): @@ -38,11 +39,11 @@ The `PersistenceBackend` hooks (the only seam between the coordinato | `list()` | List all stored metadata. | | `close?()` | Optional lifecycle teardown (e.g. close a db handle), awaited after the dispose drain. | -The `tornMarker` is fully OPAQUE: the coordinator only tests `!== undefined` and round-trips it to `commitRepair`, never inspecting its value (the JSONL backend uses the byte offset to truncate to, the SQLite backend the seq to delete from). The public `SessionPersistence` service shape is unchanged, so a third-party backend MAY still implement the abstract service directly without the coordinator. See [the write-coordinator RFC](../../../docs/rfc/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md). +The `tornMarker` is fully OPAQUE: the coordinator only tests `!== undefined` and round-trips it to `commitRepair`, never inspecting its value (the JSONL backend uses the byte offset to truncate to, the SQLite backend the seq to delete from). A third-party backend MAY implement the abstract service directly without the coordinator, but it must also provide trustworthy lightweight snapshot revisions. See [the write-coordinator RFC](../../../docs/rfc/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md). ## Testing backends -Import `runPersistenceContract` from `tests/contract.ts` (the public-API contract) and `runCoordinatorContract` from `tests/coordinator-contract.ts` (the shared write-path orchestration: adoption, HMR, collision, dispose-drain, crash-tail repair) and call each with a fixture for your backend. Every backend is held to the same append-only / contiguous-seq / lazy-materialization / serializability semantics AND the same orchestration, so a backend's own spec is left with only storage-mechanics tests (path sanitization, fsync rollback; schema version, transaction rollback) on top. +Import `runPersistenceContract` from `tests/contract.ts` (the public API, including stable/change-sensitive lightweight revisions) and `runCoordinatorContract` from `tests/coordinator-contract.ts` (the shared write-path orchestration: adoption, HMR, collision, dispose-drain, crash-tail repair) and call each with a fixture for your backend. Every backend is held to the same append-only / contiguous-seq / lazy-materialization / serializability semantics AND the same orchestration, so a backend's own spec is left with only storage-mechanics tests (path sanitization, fsync rollback; schema version, transaction rollback) on top. Three backends run these suites: an in-memory reference (in `tests/`), `dsh-session-persistence-jsonl` (append-only file log) and `dsh-session-persistence-sqlite` (`node:sqlite`, each `SessionEvent` one row `(session_id, seq, type, time, data, source_event_seqs, surface_op)`). All passing the same contract + coordinator suite is the proof that the seam is genuinely backend-agnostic — lazy materialization, crash-tail-on-load, and contiguous-seq hold identically over file bytes and over a transactional store. diff --git a/packages/session-persistence/session-persistence/package.json b/packages/session-persistence/session-persistence/package.json index ed6c80dfd9..8b7140e221 100644 --- a/packages/session-persistence/session-persistence/package.json +++ b/packages/session-persistence/session-persistence/package.json @@ -22,10 +22,12 @@ ], "license": "BSD-3-Clause", "peerDependencies": { + "@deepseek-ai/dsh-brand": "^0.0.1", "@deepseek-ai/dsh-session": "^0.0.1", "cordis": "^4.0.0-rc.6" }, "devDependencies": { + "@deepseek-ai/dsh-brand": "workspace:^", "@deepseek-ai/dsh-session": "workspace:^", "cordis": "^4.0.0-rc.6" } diff --git a/packages/session-persistence/session-persistence/src/coordinator.ts b/packages/session-persistence/session-persistence/src/coordinator.ts index b1fc118a21..7857b57a3c 100644 --- a/packages/session-persistence/session-persistence/src/coordinator.ts +++ b/packages/session-persistence/session-persistence/src/coordinator.ts @@ -14,7 +14,7 @@ * {@link PersistenceBackend} hook object. * * The abstract {@link SessionPersistence} service's public API is independent of - * this: a backend IS a `SessionPersistence` (its four public methods delegate to + * this: a backend IS a `SessionPersistence` (its write/read methods delegate to * a coordinator it composes), so a third-party backend MAY implement the service * directly without using the coordinator at all. * @@ -146,7 +146,7 @@ async function settledErrors(promises: Iterable>): Promise + + /** + * List materialized sessions with cheap per-log change tokens. + * + * Repeated observations of an unchanged log return the same revision. A + * successful mutating {@link load} repair changes the next listed revision. + * @returns one header and opaque revision per materialized session without loading full logs. + */ + abstract listSnapshots(): Promise } export default SessionPersistence diff --git a/packages/session-persistence/session-persistence/src/revision.ts b/packages/session-persistence/session-persistence/src/revision.ts new file mode 100644 index 0000000000..41378eb3e4 --- /dev/null +++ b/packages/session-persistence/session-persistence/src/revision.ts @@ -0,0 +1,15 @@ +/** Opaque revision identity for lightweight persistence observations. */ + +import type { Branded } from '@deepseek-ai/dsh-brand' + +/** Backend-owned token that changes whenever one persisted session log changes. */ +export type SessionPersistenceRevision = Branded<'SessionPersistenceRevision'> + +/** + * Brand a backend revision for the provider-neutral persistence contract. + * @param value - backend-owned opaque revision representation. + * @returns the same runtime string with persistence-revision identity. + */ +export function SessionPersistenceRevision(value: string): SessionPersistenceRevision { + return value as SessionPersistenceRevision +} diff --git a/packages/session-persistence/session-persistence/tests/contract.ts b/packages/session-persistence/session-persistence/tests/contract.ts index 789aa72c91..d1699c58a5 100644 --- a/packages/session-persistence/session-persistence/tests/contract.ts +++ b/packages/session-persistence/session-persistence/tests/contract.ts @@ -102,11 +102,16 @@ export function runPersistenceContract(name: string, make: () => Promise snapshot.header.id === m.id)?.revision // load PRESERVES the interrupted turn's events (a turn can be huge — they // must not be truncated) and closes the orphaned turn with synthetic // boundary events: step/end (the step was open) then turn/end {interrupted}. const loaded = await persistence.load(m.id) + const afterRepair = (await persistence.listSnapshots()) + .find(snapshot => snapshot.header.id === m.id)?.revision + expect(afterRepair).not.toBe(beforeRepair) expect(loaded.events.map(e => e.type)).toEqual([ 'turn/start', 'user/message', 'step/start', 'assistant/message', 'step/end', 'turn/end', // turn 1 'turn/start', 'step/start', 'step/end', 'turn/end', // turn 2: real events + synthetic closers @@ -173,18 +178,33 @@ export function runPersistenceContract(name: string, make: () => Promise m.id)).not.toContain(SessionId('empty')) + expect((await persistence.listSnapshots()).map(snapshot => snapshot.header.id)) + .not.toContain(SessionId('empty')) } finally { await dispose() } }) - it('list() includes a session once it has events', async () => { + it('lists stable lightweight revisions that change after an append', async () => { const { persistence, dispose } = await make() try { const m = meta('s2') await persistence.create(m) await persistence.append(m.id, oneTurnLog()) expect((await persistence.list()).map(x => x.id)).toContain(m.id) + const first = (await persistence.listSnapshots()).find(snapshot => snapshot.header.id === m.id) + const repeated = (await persistence.listSnapshots()).find(snapshot => snapshot.header.id === m.id) + expect(first).toBeDefined() + expect(repeated?.revision).toBe(first?.revision) + + await persistence.append(m.id, [{ + type: 'turn/start', + seq: 6, + time: 7, + data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } }, + }]) + const changed = (await persistence.listSnapshots()).find(snapshot => snapshot.header.id === m.id) + expect(changed?.revision).not.toBe(first?.revision) } finally { await dispose() } diff --git a/packages/session-persistence/session-persistence/tests/persistence.spec.ts b/packages/session-persistence/session-persistence/tests/persistence.spec.ts index 9c766d4b66..00e9863bb8 100644 --- a/packages/session-persistence/session-persistence/tests/persistence.spec.ts +++ b/packages/session-persistence/session-persistence/tests/persistence.spec.ts @@ -3,8 +3,8 @@ import { Context } from 'cordis' import SessionStore, { SessionId, isJsonValue } from '@deepseek-ai/dsh-session' import type { Session, SessionEvent, SessionHeader } from '@deepseek-ai/dsh-session' import { - SessionPersistence, PersistenceCoordinator, assertSerializable, seedCoversPrefix, - type PersistenceBackend, type StoredPrefix, + SessionPersistence, SessionPersistenceRevision, PersistenceCoordinator, assertSerializable, seedCoversPrefix, + type PersistenceBackend, type SessionPersistenceSnapshot, type StoredPrefix, } from '../src/index.ts' import { runPersistenceContract, meta, oneTurnLog } from './contract.ts' import { runCoordinatorContract, type CoordinatorFixture } from './coordinator-contract.ts' @@ -109,6 +109,14 @@ class MemoryPersistence extends SessionPersistence implements PersistenceBackend async list(): Promise { return [...this.store.values()].map(e => structuredClone(e.meta)) } + + + async listSnapshots(): Promise { + return [...this.store.values()].map(entry => ({ + header: structuredClone(entry.meta), + revision: SessionPersistenceRevision(`events:${entry.events.length}`), + })) + } } // Run the shared contract against the in-memory backend. diff --git a/packages/session-persistence/session-persistence/tsconfig.json b/packages/session-persistence/session-persistence/tsconfig.json index e817086a6a..84c6f5ccb0 100644 --- a/packages/session-persistence/session-persistence/tsconfig.json +++ b/packages/session-persistence/session-persistence/tsconfig.json @@ -14,6 +14,9 @@ { "path": "../../../vendor/cordis" }, + { + "path": "../../util/brand" + }, { "path": "../../core/session" } diff --git a/packages/session-query/session-query-sqlite/README.md b/packages/session-query/session-query-sqlite/README.md index 1f3344887f..001ea87d7d 100644 --- a/packages/session-query/session-query-sqlite/README.md +++ b/packages/session-query/session-query-sqlite/README.md @@ -1,22 +1,22 @@ # @deepseek-ai/dsh-session-query-sqlite -SQLite FTS5 implementation of `ctx.sessionSearch`. The service searches the live-preferred logical session corpus, groups cross-session results by their strongest event, and keeps provider-specific BM25 scores private. +SQLite FTS5 implementation of `ctx.sessionSearch`. The service searches the live-preferred logical session corpus and groups cross-session results by their strongest event. ## Search contract `searchSessions(request, exec?)` returns `SessionSearchHit` pages across the corpus; `searchEvents(request, exec?)` returns `SessionEventSearchHit` pages within one session. Queries are required, trimmed, whitespace-normalized literal phrases. FTS5 syntax such as quotes, `OR`, `NEAR`, and `*` is treated as data rather than executable MATCH syntax. Metadata filters are parameterized SQL predicates applied before ranking. -Ordering is deterministic: relevance first, then event time, session id where applicable, and seq. Cross-session results expose the selected event as `bestMatch`; both scopes return plain-text snippets bounded in Unicode code points. Cursors are opaque, bind to the normalized request and service instance, and fail when the relevant generation changes. A within-session cursor survives unrelated-session changes; a cross-session cursor does not. +Relevance is source-comparable across persistent and TEMP tables: actual FTS5 highlighted-match span count descending, then stored document code-point length ascending. Event time, session id where applicable, and seq break remaining ties. Cross-session results expose the selected event as `bestMatch`; both scopes derive whitespace-normalized plain text from FTS5 highlight positions and bound it in Unicode code points. Cursors are opaque branded values, bind to the normalized request and service instance, and fail when the relevant generation changes. A within-session cursor survives unrelated-session changes; a cross-session cursor does not. All three surfaces (`current`, `shadowed`, and `log-only`) are searchable by default. Pass a surface filter to narrow them. ## Source and index lifecycle -The service requires `ctx.sessions` and observes optional `ctx.sessionPersistence` dynamically. One serialized state machine observes complete sources, extracts shared semantic documents, reconciles changes transactionally, and runs the query. Stable fingerprints preserve unchanged persisted rows and generations; new, changed, and deleted durable sessions reconcile on the next search. Source or transaction failure commits nothing, and the next search retries. +The service requires `ctx.sessions` and observes optional `ctx.sessionPersistence` dynamically. One serialized state machine compares lightweight durable snapshot revisions, loads only new or changed logs, extracts shared semantic documents, reconciles changes transactionally, and runs the query. Repeated queries and unchanged reopen load no full durable logs; new, changed, deleted, or load-repaired sources reconcile on the next stable observation. Source or transaction failure commits nothing, and the next search retries. Persisted FTS rows live in a dedicated derived database. Connection-local TEMP tables hold live rows, which shadow the durable base for the same session and reveal it when the live owner disappears. Unmounting persistence hides durable rows without discarding the cache; remounting reconciles it. Closing or reopening the database drops every live overlay while retaining persisted rows. -The database is disposable but reset is guarded: a recognized incompatible search schema rebuilds in place, while an unrelated or canonical database is refused. Never point `path` at the session-persistence database. +The database is disposable but reset is guarded: a recognized incompatible search schema rebuilds in place, while an unrelated or canonical database is refused before mutating journal mode. Never point `path` at the session-persistence database. Exactly one service in one process owns a derived-index path; external writers or a second process are unsupported because generations and TEMP shadow state are connection-owned. ## Configuration @@ -30,6 +30,6 @@ The database is disposable but reset is guarded: a recognized incompatible searc ## Tokenizer and limits -The index uses FTS5 `unicode61`. In the implementation experiment it supported the two-character query `AI` and produced an index about 2.1× smaller than the trigram alternative. The trade-off is token/phrase recall rather than arbitrary substring recall: `AI` does not match the token `BRAID`. Use `ctx.sessionQuery.filterEvents()` with a `text` clause when a literal whitespace-flexible substring scan is required. +The index uses FTS5 `unicode61`. In the implementation experiment it supported the two-character query `AI` and produced an index about 2.1× smaller than the trigram alternative. The trade-off is token/phrase recall rather than arbitrary substring recall: `AI` does not match the token `BRAID`. Use `ctx.sessionQuery.filterEvents()` with a `text` clause when a literal whitespace-flexible substring scan is required. NUL is rejected in queries; reserved highlight markers and NUL in documents are normalized before indexing so presentation markers cannot collide with source text. Abort signals stop queued work and caller waits around asynchronous source observation. Node's synchronous `DatabaseSync` API cannot interrupt a MATCH statement already executing on the JavaScript thread; the signal is checked immediately before and after the serialized observation/reconciliation boundary. diff --git a/packages/session-query/session-query-sqlite/src/index.ts b/packages/session-query/session-query-sqlite/src/index.ts index a6cba8d866..dd1ddb532a 100644 --- a/packages/session-query/session-query-sqlite/src/index.ts +++ b/packages/session-query/session-query-sqlite/src/index.ts @@ -6,12 +6,17 @@ import { createHash, randomUUID } from 'node:crypto' import { DatabaseSync } from 'node:sqlite' -import { Context } from 'cordis' +import { Context, type Fiber } from 'cordis' import z from 'schemastery' import type { Session, SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session' import type SessionPersistence from '@deepseek-ai/dsh-session-persistence' +import type { + SessionPersistenceRevision, + SessionPersistenceSnapshot, +} from '@deepseek-ai/dsh-session-persistence' import { SessionQueryError, + SessionSearchCursor, SessionSearchService, assertSessionHeadersCompatible, buildSessionEventSearchDocuments, @@ -22,6 +27,7 @@ import type { SessionEventSearchRequest, SessionSearchExecContext, SessionSearchHit, + SessionSearchCursor as SessionSearchCursorValue, SessionSearchPage, SessionSearchRequest, } from '@deepseek-ai/dsh-session-query' @@ -32,6 +38,8 @@ import { import { type NormalizedEventRequest, type NormalizedSessionRequest, + FTS_HIGHLIGHT_END, + FTS_HIGHLIGHT_START, buildEventWhere, buildSessionWhere, makeSnippet, @@ -39,6 +47,7 @@ import { normalizeSessionRequest, quoteFtsData, requestFingerprint, + sanitizeFtsText, } from './query.ts' export { @@ -78,19 +87,30 @@ interface ResolvedConfig { interface ObservedSession { header: SessionHeader - events: SessionEvent[] documents: SessionEventSearchDocument[] fingerprint: string } +interface ObservedPersistedSession { + header: SessionHeader + revision: SessionPersistenceRevision + loaded?: ObservedSession +} + interface Observation { persistence: SessionPersistence | undefined persistenceRevision: number - persisted: Map + persisted: Map live: Map } -interface IndexedRow { +interface IndexedPersistedRow { + id: string + revision: string + generation: number +} + +interface IndexedLiveRow { id: string fingerprint: string generation: number @@ -109,8 +129,9 @@ interface SearchRow { type: string time: number surface: string - text: string - score: number + marked_text: string + match_count: number + document_length: number } interface CursorPayload { @@ -149,27 +170,32 @@ export class SessionSearchSqlite extends SessionSearchService { private _localGeneration = 0 private _tail: Promise = Promise.resolve() private _closed = false + private _closePromise: Promise | undefined + private readonly _optionalPersistenceFiber: Fiber constructor(ctx: Context, config: Config) { super(ctx) this.config = resolveConfig(config) this._ready = this._open() - ctx.effect(() => { - const fiber = ctx.inject(['sessionPersistence'], (childCtx: Context) => { - const service = childCtx.sessionPersistence - const binding = {} - this._persistenceBinding = binding - this._persistence = service + // Attach a rejection observer immediately; callers still receive the same + // rejection from `_ready`, including when no search is ever attempted. + void this._ready.catch(() => undefined) + this._optionalPersistenceFiber = ctx.inject(['sessionPersistence'], (childCtx: Context) => { + const service = childCtx.sessionPersistence + const binding = {} + this._persistenceBinding = binding + this._persistence = service + this._persistenceRevision += 1 + childCtx.effect(() => () => { + /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */ + if (this._persistenceBinding !== binding) return + this._persistenceBinding = undefined + this._persistence = undefined this._persistenceRevision += 1 - childCtx.effect(() => () => { - /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */ - if (this._persistenceBinding !== binding) return - this._persistenceBinding = undefined - this._persistence = undefined - this._persistenceRevision += 1 - }, 'sessionSearchSqlite.persistenceBinding') - }) - return () => void fiber.dispose() + }, 'sessionSearchSqlite.persistenceBinding') + }) + ctx.effect(() => { + return () => this._optionalPersistenceFiber.dispose() }, 'sessionSearchSqlite.optionalPersistence') ctx.effect(() => async () => this.close(), 'sessionSearchSqlite.close') } @@ -179,17 +205,18 @@ export class SessionSearchSqlite extends SessionSearchService { exec?: SessionSearchExecContext, ): Promise> { const normalized = normalizeSessionRequest(request, this.config) - return this._serialized(exec?.signal, async () => { - await this._ensureReady(exec?.signal) - await this._reconcile(exec?.signal) - assertNotAborted(exec?.signal) + const signal = exec?.signal + return this._serialized(signal, async () => { + await this._ensureReady(signal) + await this._reconcile(signal) + assertNotAborted(signal) const generation = String(this._globalGeneration) const fingerprint = requestFingerprint(normalized) const offset = normalized.cursor === undefined ? 0 : decodeCursor(normalized.cursor, this._instance, 'sessions', fingerprint, generation) const rows = this._querySessions(normalized, offset) - return page(rows, normalized.limit, row => this._sessionHit(row, normalized.query), cursorOffset => encodeCursor({ + return page(rows, normalized.limit, row => this._sessionHit(row), cursorOffset => encodeCursor({ version: 1, instance: this._instance, scope: 'sessions', @@ -205,17 +232,18 @@ export class SessionSearchSqlite extends SessionSearchService { exec?: SessionSearchExecContext, ): Promise> { const normalized = normalizeEventRequest(request, this.config) - return this._serialized(exec?.signal, async () => { - await this._ensureReady(exec?.signal) - await this._reconcile(exec?.signal) - assertNotAborted(exec?.signal) + const signal = exec?.signal + return this._serialized(signal, async () => { + await this._ensureReady(signal) + await this._reconcile(signal) + assertNotAborted(signal) const generation = this._targetGeneration(normalized.sessionId) const fingerprint = requestFingerprint(normalized) const offset = normalized.cursor === undefined ? 0 : decodeCursor(normalized.cursor, this._instance, 'events', fingerprint, generation) const rows = this._queryEvents(normalized, offset) - return page(rows, normalized.limit, row => this._eventHit(row, normalized.query), cursorOffset => encodeCursor({ + return page(rows, normalized.limit, row => this._eventHit(row), cursorOffset => encodeCursor({ version: 1, instance: this._instance, scope: 'events', @@ -227,8 +255,12 @@ export class SessionSearchSqlite extends SessionSearchService { } /** Close the database after every accepted operation reaches quiescence. */ - async close(): Promise { - if (this._closed) return + close(): Promise { + this._closePromise ??= this._close() + return this._closePromise + } + + private async _close(): Promise { this._closed = true await this._tail try { @@ -287,20 +319,20 @@ export class SessionSearchSqlite extends SessionSearchService { } private async _reconcile(signal: AbortSignal | undefined): Promise { - const observation = await this._observeStable(signal) - assertNotAborted(signal) const db = this._requireDb() const persistedRows = db.prepare( - 'SELECT id, fingerprint, generation FROM persisted_sessions', - ).all() as unknown as IndexedRow[] + 'SELECT id, revision, generation FROM persisted_sessions', + ).all() as unknown as IndexedPersistedRow[] const liveRows = db.prepare( 'SELECT id, fingerprint, generation FROM temp.live_sessions', - ).all() as unknown as IndexedRow[] + ).all() as unknown as IndexedLiveRow[] const persistedById = new Map(persistedRows.map(row => [row.id as SessionId, row])) const liveById = new Map(liveRows.map(row => [row.id as SessionId, row])) + const observation = await this._observeStable(persistedById, signal) + assertNotAborted(signal) const persistentChanges = observation.persistence === undefined ? [] - : [...observation.persisted.values()].filter(entry => persistedById.get(entry.header.id)?.fingerprint !== entry.fingerprint) + : [...observation.persisted.values()].filter(entry => entry.loaded !== undefined) const persistentDeletes = observation.persistence === undefined ? [] : persistedRows.filter(row => !observation.persisted.has(row.id as SessionId)) @@ -327,13 +359,17 @@ export class SessionSearchSqlite extends SessionSearchService { db.exec('BEGIN IMMEDIATE') began = true for (const row of persistentDeletes) this._deleteSession('persisted', row.id as SessionId) - for (const entry of persistentChanges) this._replaceSession('persisted', entry, nextMainGeneration) + for (const entry of persistentChanges) { + /* v8 ignore next -- observation loads every entry whose revision differs */ + if (entry.loaded === undefined) throw new Error(`missing loaded revision for session "${entry.header.id}"`) + this._replacePersistedSession(entry.loaded, entry.revision, nextMainGeneration) + } if (persistentChanges.length > 0 || persistentDeletes.length > 0) { db.prepare('UPDATE search_state SET global_generation = ? WHERE singleton = 1').run(nextMainGeneration) } for (const row of liveDeletes) this._deleteSession('live', row.id as SessionId) for (const { entry, generation } of liveReplacements) { - this._replaceSession('live', entry, generation) + this._replaceLiveSession(entry, generation) } db.exec('COMMIT') } catch (error: unknown) { @@ -360,21 +396,39 @@ export class SessionSearchSqlite extends SessionSearchService { this._lastPersistenceRevision = observation.persistenceRevision } - private async _observeStable(signal: AbortSignal | undefined): Promise { + private async _observeStable( + indexed: ReadonlyMap, + signal: AbortSignal | undefined, + ): Promise { for (;;) { assertNotAborted(signal) const persistence = this._persistence const persistenceRevision = this._persistenceRevision - const persisted = new Map() + let persisted = new Map() if (persistence !== undefined) { try { - const headers = await waitWithAbort(persistence.list(), signal) - for (const listed of headers) { - const loaded = await waitWithAbort(persistence.load(listed.id), signal) - assertSessionHeadersCompatible(listed, loaded.meta) - persisted.set(listed.id, observeSession(loaded.meta, loaded.events)) + const canReuseIndexed = this._lastPersistenceRevision === undefined + || this._lastPersistenceRevision === persistenceRevision + const before = await waitWithAbort(persistence.listSnapshots(), signal) + persisted = materializePersistenceSnapshots(before) + for (const entry of persisted.values()) { + if (canReuseIndexed && indexed.get(entry.header.id)?.revision === entry.revision) continue + const loaded = await waitWithAbort(persistence.load(entry.header.id), signal) + assertSessionHeadersCompatible(entry.header, loaded.meta) + entry.loaded = observeSession(loaded.meta, loaded.events) } + const after = materializePersistenceSnapshots( + await waitWithAbort(persistence.listSnapshots(), signal), + ) + if (!samePersistenceSnapshots(persisted, after)) continue + if (this._persistenceRevision !== persistenceRevision) continue } catch (error: unknown) { + if (isAbort(error) || signal?.aborted) { + throw new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED', { + cause: error, + }) + } + if (this._persistenceRevision !== persistenceRevision) continue if (error instanceof SessionQueryError) throw error throw new SessionQueryError( `session-search persistence observation failed: ${errorMessage(error)}`, @@ -414,13 +468,50 @@ export class SessionSearchSqlite extends SessionSearchService { } } - private _replaceSession(source: 'persisted' | 'live', entry: ObservedSession, generation: number): void { - this._deleteSession(source, entry.header.id) + private _replacePersistedSession( + entry: ObservedSession, + revision: SessionPersistenceRevision, + generation: number, + ): void { + this._deleteSession('persisted', entry.header.id) const db = this._requireDb() - const sessionTable = source === 'persisted' ? 'persisted_sessions' : 'temp.live_sessions' - const docsTable = source === 'persisted' ? 'persisted_docs' : 'temp.live_docs' db.prepare(` - INSERT INTO ${sessionTable} + INSERT INTO persisted_sessions + (id, version, created_at, cwd, parent_session, seed_length, revision, generation) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `).run( + entry.header.id, + entry.header.version, + entry.header.createdAt, + entry.header.cwd ?? null, + entry.header.parentSession ?? null, + entry.header.seedLength ?? null, + revision, + generation, + ) + const insert = db.prepare(` + INSERT INTO persisted_docs (text, session_id, seq, type, time, surface, codepoint_length) + VALUES (?, ?, ?, ?, ?, ?, ?) + `) + for (const document of entry.documents) { + const text = sanitizeFtsText(document.text) + insert.run( + text, + document.sessionId, + document.seq, + document.type, + document.time, + document.surface, + Array.from(text).length, + ) + } + } + + private _replaceLiveSession(entry: ObservedSession, generation: number): void { + this._deleteSession('live', entry.header.id) + const db = this._requireDb() + db.prepare(` + INSERT INTO temp.live_sessions (id, version, created_at, cwd, parent_session, seed_length, fingerprint, generation) VALUES (?, ?, ?, ?, ?, ?, ?, ?) `).run( @@ -434,11 +525,20 @@ export class SessionSearchSqlite extends SessionSearchService { generation, ) const insert = db.prepare(` - INSERT INTO ${docsTable} (text, session_id, seq, type, time, surface) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO temp.live_docs (text, session_id, seq, type, time, surface, codepoint_length) + VALUES (?, ?, ?, ?, ?, ?, ?) `) for (const document of entry.documents) { - insert.run(document.text, document.sessionId, document.seq, document.type, document.time, document.surface) + const text = sanitizeFtsText(document.text) + insert.run( + text, + document.sessionId, + document.seq, + document.type, + document.time, + document.surface, + Array.from(text).length, + ) } } @@ -455,19 +555,16 @@ export class SessionSearchSqlite extends SessionSearchService { ranked AS ( SELECT *, ROW_NUMBER() OVER ( PARTITION BY session_id - ORDER BY score ASC, time DESC, seq DESC + ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC ) AS event_rank FROM filtered ) SELECT * FROM ranked WHERE event_rank = 1 - ORDER BY score ASC, time DESC, session_id ASC, seq DESC + ORDER BY match_count DESC, document_length ASC, time DESC, session_id ASC, seq DESC LIMIT ? OFFSET ? `).all( - quoteFtsData(request.query), - this._persistence === undefined ? 0 : 1, - this._persistence === undefined ? 0 : 1, - quoteFtsData(request.query), + ...selectedDocumentsParams(request.query, this._persistence !== undefined), ...sessionWhere.params, ...eventWhere.params, request.limit + 1, @@ -483,13 +580,10 @@ export class SessionSearchSqlite extends SessionSearchService { ${selected.sql} SELECT * FROM matched WHERE ${where} - ORDER BY score ASC, time DESC, seq DESC + ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC LIMIT ? OFFSET ? `).all( - quoteFtsData(request.query), - this._persistence === undefined ? 0 : 1, - this._persistence === undefined ? 0 : 1, - quoteFtsData(request.query), + ...selectedDocumentsParams(request.query, this._persistence !== undefined), request.sessionId, ...eventWhere.params, request.limit + 1, @@ -515,23 +609,23 @@ export class SessionSearchSqlite extends SessionSearchService { ) } - private _sessionHit(row: SearchRow, query: string): SessionSearchHit { + private _sessionHit(row: SearchRow): SessionSearchHit { return { header: rowHeader(row), live: row.live === 1, persisted: row.persisted === 1, - bestMatch: this._eventHit(row, query), + bestMatch: this._eventHit(row), } } - private _eventHit(row: SearchRow, query: string): SessionEventSearchHit { + private _eventHit(row: SearchRow): SessionEventSearchHit { return { sessionId: row.session_id as SessionId, seq: row.seq, type: row.type as SessionEventSearchHit['type'], time: row.time, surface: row.surface as SessionEventSearchHit['surface'], - snippet: makeSnippet(row.text, query, this.config.snippetChars), + snippet: makeSnippet(row.marked_text, this.config.snippetChars), } } @@ -548,7 +642,7 @@ export class SessionSearchSqlite extends SessionSearchService { function selectedDocumentsSql(): { sql: string } { return { - sql: `WITH matched AS ( + sql: `WITH candidates AS ( SELECT pd.session_id AS session_id, ps.version AS version, @@ -562,8 +656,8 @@ function selectedDocumentsSql(): { sql: string } { pd.type AS type, CAST(pd.time AS INTEGER) AS time, pd.surface AS surface, - pd.text AS text, - bm25(persisted_docs) AS score + highlight(persisted_docs, 0, ?, ?) AS marked_text, + CAST(pd.codepoint_length AS INTEGER) AS document_length FROM persisted_docs AS pd JOIN persisted_sessions AS ps ON ps.id = pd.session_id WHERE persisted_docs MATCH ? @@ -585,15 +679,39 @@ function selectedDocumentsSql(): { sql: string } { ld.type AS type, CAST(ld.time AS INTEGER) AS time, ld.surface AS surface, - ld.text AS text, - bm25(live_docs) AS score + highlight(live_docs, 0, ?, ?) AS marked_text, + CAST(ld.codepoint_length AS INTEGER) AS document_length FROM temp.live_docs AS ld JOIN temp.live_sessions AS ls ON ls.id = ld.session_id WHERE live_docs MATCH ? + ), matched AS ( + SELECT *, + ( + length(CAST(marked_text AS BLOB)) + - length(CAST(replace(marked_text, ?, '') AS BLOB)) + ) / ? AS match_count + FROM candidates )`, } } +function selectedDocumentsParams(query: string, persistenceVisible: boolean): Array { + const expression = quoteFtsData(query) + const visible = persistenceVisible ? 1 : 0 + return [ + FTS_HIGHLIGHT_START, + FTS_HIGHLIGHT_END, + expression, + visible, + visible, + FTS_HIGHLIGHT_START, + FTS_HIGHLIGHT_END, + expression, + FTS_HIGHLIGHT_START, + Buffer.byteLength(FTS_HIGHLIGHT_START, 'utf8'), + ] +} + function observeLive(session: Session): ObservedSession { return observeSession( structuredClone(session.header), @@ -606,7 +724,6 @@ function observeSession(header: SessionHeader, events: readonly SessionEvent[]): const detachedEvents = events.map(event => structuredClone(event)) return { header: detachedHeader, - events: detachedEvents, documents: buildSessionEventSearchDocuments(detachedHeader.id, detachedEvents), fingerprint: createHash('sha256') .update(JSON.stringify({ header: detachedHeader, events: detachedEvents })) @@ -614,6 +731,49 @@ function observeSession(header: SessionHeader, events: readonly SessionEvent[]): } } +function materializePersistenceSnapshots( + snapshots: readonly SessionPersistenceSnapshot[], +): Map { + if (!isRuntimeArray(snapshots)) throw new Error('persistence snapshots must be an array') + const result = new Map() + for (const snapshot of snapshots) { + if (typeof snapshot.revision !== 'string') { + throw new Error('persistence snapshot revision must be a string') + } + const header = structuredClone(snapshot.header) + if (result.has(header.id)) { + throw new Error(`persistence listed duplicate session "${header.id}"`) + } + result.set(header.id, { header, revision: snapshot.revision }) + } + return result +} + +function samePersistenceSnapshots( + before: ReadonlyMap, + after: ReadonlyMap, +): boolean { + if (before.size !== after.size) return false + for (const [id, first] of before) { + const second = after.get(id) + if ( + second === undefined + || first.revision !== second.revision + || !sameHeader(first.header, second.header) + ) return false + } + return true +} + +function sameHeader(a: SessionHeader, b: SessionHeader): boolean { + return a.version === b.version + && a.id === b.id + && a.createdAt === b.createdAt + && a.cwd === b.cwd + && a.parentSession === b.parentSession + && a.seedLength === b.seedLength +} + function rowHeader(row: SearchRow): SessionHeader { return { version: row.version, @@ -629,7 +789,7 @@ function page( rows: readonly Row[], limit: number, convert: (row: Row) => Item, - nextCursor: (offset: number) => string, + nextCursor: (offset: number) => SessionSearchCursorValue, offset: number, ): SessionSearchPage { const hasMore = rows.length > limit @@ -639,12 +799,12 @@ function page( } } -function encodeCursor(payload: CursorPayload): string { - return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url') +function encodeCursor(payload: CursorPayload): SessionSearchCursorValue { + return SessionSearchCursor(Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url')) } function decodeCursor( - cursor: string, + cursor: SessionSearchCursorValue, instance: string, scope: CursorPayload['scope'], fingerprint: string, @@ -762,4 +922,8 @@ function errorMessage(error: unknown): string { return error instanceof Error ? error.message : 'unknown error' } +function isRuntimeArray(value: unknown): boolean { + return Array.isArray(value) +} + export default SessionSearchSqlite diff --git a/packages/session-query/session-query-sqlite/src/query.ts b/packages/session-query/session-query-sqlite/src/query.ts index fd2b5156e6..9654f6ae70 100644 --- a/packages/session-query/session-query-sqlite/src/query.ts +++ b/packages/session-query/session-query-sqlite/src/query.ts @@ -2,16 +2,24 @@ import { SessionQueryError, - filterSessionEventDocuments, - filterSessionResults, + materializeSessionEventResultFilters, + materializeSessionResultFilters, } from '@deepseek-ai/dsh-session-query' import type { + SessionAvailability, SessionEventMetadataFilter, + SessionEventResultFilter, SessionEventSearchRequest, SessionResultFilter, + SessionSearchCursor, SessionSearchRequest, } from '@deepseek-ai/dsh-session-query' +/** Collision-free marker inserted before an FTS5 match by `highlight()`. */ +export const FTS_HIGHLIGHT_START = '\uFDD0' +/** Collision-free marker inserted after an FTS5 match by `highlight()`. */ +export const FTS_HIGHLIGHT_END = '\uFDD1' + /** Limit defaults needed to normalize a search request. */ export interface QueryLimits { /** Page size used when the request omits one. */ @@ -26,7 +34,7 @@ export interface NormalizedSessionRequest { sessionFilters: readonly SessionResultFilter[] eventFilters: readonly SessionEventMetadataFilter[] limit: number - cursor?: string + cursor?: SessionSearchCursor } /** Normalized within-session request. */ @@ -35,7 +43,7 @@ export interface NormalizedEventRequest { query: string filters: readonly SessionEventMetadataFilter[] limit: number - cursor?: string + cursor?: SessionSearchCursor } /** Parameterized SQL predicate fragment. */ @@ -56,16 +64,15 @@ export function normalizeSessionRequest( request: SessionSearchRequest, limits: QueryLimits, ): NormalizedSessionRequest { - const sessionFilters = request.sessionFilters ?? [] - const eventFilters = request.eventFilters ?? [] - filterSessionResults([], sessionFilters) - filterSessionEventDocuments([], eventFilters) + const sessionFilters = materializeSessionResultFilters(request.sessionFilters ?? []) + const eventFilters = materializeMetadataFilters(request.eventFilters ?? []) + const cursor = materializeCursor(request.cursor) return { query: normalizeQuery(request.query), sessionFilters, eventFilters, limit: normalizeLimit(request.limit, limits), - ...request.cursor === undefined ? {} : { cursor: request.cursor }, + ...cursor === undefined ? {} : { cursor }, } } @@ -79,14 +86,17 @@ export function normalizeEventRequest( request: SessionEventSearchRequest, limits: QueryLimits, ): NormalizedEventRequest { - const filters = request.filters ?? [] - filterSessionEventDocuments([], filters) + if (typeof request.sessionId !== 'string') { + throw new SessionQueryError('session-search session id must be text', 'SESSION_QUERY_INVALID_FILTER') + } + const filters = materializeMetadataFilters(request.filters ?? []) + const cursor = materializeCursor(request.cursor) return { sessionId: request.sessionId, query: normalizeQuery(request.query), filters, limit: normalizeLimit(request.limit, limits), - ...request.cursor === undefined ? {} : { cursor: request.cursor }, + ...cursor === undefined ? {} : { cursor }, } } @@ -115,9 +125,23 @@ export function buildSessionWhere(filters: readonly SessionResultFilter[]): SqlW case 'availability': { const availability = [...new Set(filter.values)] if (availability.length === 0) clauses.push('0') - else if (availability.length === 1) clauses.push(`${availability[0]} = 1`) + else if (availability.length === 1) { + const value = availability[0] as SessionAvailability + switch (value) { + case 'live': + clauses.push('live = 1') + break + case 'persisted': + clauses.push('persisted = 1') + break + default: + unknownAvailability(value) + } + } break } + default: + unknownFilter(filter) } } return { sql: clauses.join(' AND '), params } @@ -145,6 +169,8 @@ export function buildEventWhere(filters: readonly SessionEventMetadataFilter[]): case 'surface': addList(clauses, params, 'surface', filter.values) break + default: + unknownFilter(filter) } } return { sql: clauses.join(' AND '), params } @@ -159,6 +185,18 @@ export function quoteFtsData(query: string): string { return `"${query.replaceAll('"', '""')}"` } +/** + * Remove reserved marker collisions before text enters FTS5 or MATCH. + * @param text - extracted document text or normalized caller query. + * @returns text with reserved noncharacters mapped to replacement characters. + */ +export function sanitizeFtsText(text: string): string { + return text + .replaceAll('\0', '\uFFFD') + .replaceAll(FTS_HIGHLIGHT_START, '\uFFFD') + .replaceAll(FTS_HIGHLIGHT_END, '\uFFFD') +} + /** * Build the stable normalized request identity stored in opaque cursors. * @param request - normalized request whose filter ordering is canonicalized. @@ -185,19 +223,16 @@ export function requestFingerprint(request: NormalizedSessionRequest | Normalize /** * Build a whitespace-normalized excerpt no longer than `maxChars`. - * @param text - complete extracted semantic document. - * @param query - normalized literal query used to position the excerpt. + * @param markedText - complete document with FTS5 `highlight()` markers. * @param maxChars - maximum result length in Unicode code points. * @returns bounded plain-text snippet. */ -export function makeSnippet(text: string, query: string, maxChars: number): string { - const clean = text.replace(/\s+/gu, ' ').trim() +export function makeSnippet(markedText: string, maxChars: number): string { + const { text: clean, matchStart } = normalizeMarkedText(markedText) const characters = Array.from(clean) if (characters.length <= maxChars) return clean if (maxChars === 1) return '…' - const foundUnits = clean.toLowerCase().indexOf(query.toLowerCase()) - const found = foundUnits < 0 ? -1 : Array.from(clean.slice(0, foundUnits)).length - let start = found < 0 ? 0 : Math.max(0, found - Math.floor(maxChars / 3)) + let start = Math.max(0, matchStart - Math.floor(maxChars / 3)) let prefix = start > 0 ? '…' : '' let suffix = '…' let contentLength = maxChars - prefix.length - suffix.length @@ -216,6 +251,28 @@ export function makeSnippet(text: string, query: string, maxChars: number): stri return `${prefix}${characters.slice(start, end).join('')}${suffix}` } +function normalizeMarkedText(markedText: string): { text: string; matchStart: number } { + const characters: string[] = [] + let matchStart: number | undefined + for (const character of markedText) { + if (character === FTS_HIGHLIGHT_START) { + matchStart ??= characters.length + continue + } + if (character === FTS_HIGHLIGHT_END) continue + if (/\s/u.test(character)) { + if (characters.length > 0 && characters.at(-1) !== ' ') characters.push(' ') + } else { + characters.push(character) + } + } + if (characters.at(-1) === ' ') characters.pop() + return { + text: characters.join(''), + matchStart: matchStart ?? 0, + } +} + function normalizeQuery(value: string): string { if (typeof value !== 'string') { throw new SessionQueryError('session-search query must be text', 'SESSION_QUERY_INVALID_QUERY') @@ -227,7 +284,44 @@ function normalizeQuery(value: string): string { 'SESSION_QUERY_INVALID_QUERY', ) } - return query + if (query.includes('\0')) { + throw new SessionQueryError( + 'session-search query must not contain NUL', + 'SESSION_QUERY_INVALID_QUERY', + ) + } + return sanitizeFtsText(query) +} + +function materializeCursor(cursor: SessionSearchCursor | undefined): SessionSearchCursor | undefined { + if (cursor === undefined) return undefined + if (typeof cursor !== 'string') { + throw new SessionQueryError('session-search cursor must be text', 'SESSION_QUERY_INVALID_CURSOR') + } + return cursor +} + +function materializeMetadataFilters( + filters: readonly SessionEventMetadataFilter[], +): SessionEventMetadataFilter[] { + const candidates: readonly SessionEventResultFilter[] = filters + for (const filter of candidates) { + switch (filter.kind) { + case 'seq': + case 'time': + case 'type': + case 'surface': + break + case 'text': + throw new SessionQueryError( + 'session-search metadata filters do not accept text clauses', + 'SESSION_QUERY_INVALID_FILTER', + ) + default: + unknownFilter(filter) + } + } + return materializeSessionEventResultFilters(filters) as SessionEventMetadataFilter[] } function normalizeLimit(value: number | undefined, limits: QueryLimits): number { @@ -310,3 +404,18 @@ function compareNullable(a: string | null, b: string | null): number { if (b === null) return 1 return a.localeCompare(b) } + +function unknownAvailability(value: never): never { + throw new SessionQueryError( + `session availability filter contains unknown value "${String(value)}"`, + 'SESSION_QUERY_INVALID_FILTER', + ) +} + +function unknownFilter(filter: never): never { + const kind = (filter as { kind?: unknown }).kind + throw new SessionQueryError( + `session filter contains unknown kind ${typeof kind === 'string' ? `"${kind}"` : '(missing)'}`, + 'SESSION_QUERY_INVALID_FILTER', + ) +} diff --git a/packages/session-query/session-query-sqlite/src/schema.ts b/packages/session-query/session-query-sqlite/src/schema.ts index 1c9bd98791..8e5c85f676 100644 --- a/packages/session-query/session-query-sqlite/src/schema.ts +++ b/packages/session-query/session-query-sqlite/src/schema.ts @@ -5,7 +5,7 @@ import { mkdir } from 'node:fs/promises' import { dirname, resolve } from 'node:path' /** Current derived-index schema version. Incompatible versions reset in place. */ -export const SESSION_QUERY_SQLITE_SCHEMA_VERSION = 1 +export const SESSION_QUERY_SQLITE_SCHEMA_VERSION = 2 /** SQLite application id protecting unrelated databases from derived resets. */ export const SESSION_QUERY_SQLITE_APPLICATION_ID = 0x44534851 @@ -24,8 +24,6 @@ export async function openSearchDatabase(path: string, journalMode: JournalMode) if (actual !== ':memory:') await mkdir(dirname(actual), { recursive: true, mode: 0o700 }) const db = new DatabaseSync(actual) try { - // journalMode is a validated closed union, not caller-controlled SQL. - db.exec(`PRAGMA journal_mode = ${journalMode.toUpperCase()}`) const { application_id: applicationId } = db.prepare('PRAGMA application_id').get() as { application_id: number } const { user_version: version } = db.prepare('PRAGMA user_version').get() as { user_version: number } const userTables = listUserTables(db) @@ -38,6 +36,9 @@ export async function openSearchDatabase(path: string, journalMode: JournalMode) if (applicationId === SESSION_QUERY_SQLITE_APPLICATION_ID && version !== SESSION_QUERY_SQLITE_SCHEMA_VERSION) { resetDerivedSchema(db) } + // Apply mutating pragmas only after refusing foreign or canonical files. + // journalMode is a validated closed union, not caller-controlled SQL. + db.exec(`PRAGMA journal_mode = ${journalMode.toUpperCase()}`) ensurePersistentSchema(db) ensureTemporarySchema(db) return db @@ -78,7 +79,7 @@ function ensurePersistentSchema(db: DatabaseSync): void { cwd TEXT, parent_session TEXT, seed_length INTEGER, - fingerprint TEXT NOT NULL, + revision TEXT NOT NULL, generation INTEGER NOT NULL ) STRICT `) @@ -90,6 +91,7 @@ function ensurePersistentSchema(db: DatabaseSync): void { type UNINDEXED, time UNINDEXED, surface UNINDEXED, + codepoint_length UNINDEXED, tokenize = 'unicode61' ) `) @@ -117,6 +119,7 @@ function ensureTemporarySchema(db: DatabaseSync): void { type UNINDEXED, time UNINDEXED, surface UNINDEXED, + codepoint_length UNINDEXED, tokenize = 'unicode61' ) `) diff --git a/packages/session-query/session-query-sqlite/tests/query.spec.ts b/packages/session-query/session-query-sqlite/tests/query.spec.ts index 0faced72e4..f9c0a5d19e 100644 --- a/packages/session-query/session-query-sqlite/tests/query.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/query.spec.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from 'vitest' import { SessionId } from '@deepseek-ai/dsh-session' -import type { SessionQueryErrorCode } from '@deepseek-ai/dsh-session-query' +import { SessionSearchCursor, type SessionQueryErrorCode } from '@deepseek-ai/dsh-session-query' import { buildEventWhere, buildSessionWhere, + FTS_HIGHLIGHT_END, + FTS_HIGHLIGHT_START, makeSnippet, normalizeEventRequest, normalizeSessionRequest, @@ -32,13 +34,13 @@ describe('SQLite search request normalization', () => { sessionFilters: [{ kind: 'availability', values: ['live'] }], eventFilters: [{ kind: 'surface', values: ['current'] }], limit: 3, - cursor: 'next', + cursor: SessionSearchCursor('next'), }, limits)).toEqual({ query: 'needle', sessionFilters: [{ kind: 'availability', values: ['live'] }], eventFilters: [{ kind: 'surface', values: ['current'] }], limit: 3, - cursor: 'next', + cursor: SessionSearchCursor('next'), }) expect(normalizeEventRequest({ sessionId: SessionId('s'), query: 'needle' }, limits)).toEqual({ sessionId: SessionId('s'), @@ -50,13 +52,13 @@ describe('SQLite search request normalization', () => { sessionId: SessionId('s'), query: 'needle', filters: [{ kind: 'seq', from: 1 }], - cursor: 'next', + cursor: SessionSearchCursor('next'), }, limits)).toEqual({ sessionId: SessionId('s'), query: 'needle', filters: [{ kind: 'seq', from: 1 }], limit: 2, - cursor: 'next', + cursor: SessionSearchCursor('next'), }) }) @@ -65,11 +67,39 @@ describe('SQLite search request normalization', () => { .toThrow(expectCode('SESSION_QUERY_INVALID_QUERY')) expect(() => normalizeSessionRequest({ query: ' \n ' }, limits)) .toThrow(expectCode('SESSION_QUERY_INVALID_QUERY')) + expect(() => normalizeSessionRequest({ query: 'bad\0query' }, limits)) + .toThrow(expectCode('SESSION_QUERY_INVALID_QUERY')) + expect(() => normalizeEventRequest({ sessionId: 1 as never, query: 'x' }, limits)) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => normalizeEventRequest({ + sessionId: SessionId('s'), + query: 'x', + cursor: 1 as never, + }, limits)).toThrow(expectCode('SESSION_QUERY_INVALID_CURSOR')) + expect(() => normalizeSessionRequest({ + query: 'x', + eventFilters: [{ kind: 'text', text: 'x' } as never], + }, limits)).toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => normalizeSessionRequest({ + query: 'x', + eventFilters: [{} as never], + }, limits)).toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) for (const limit of [1.5, 0, 4]) { expect(() => normalizeEventRequest({ sessionId: SessionId('s'), query: 'x', limit }, limits)) .toThrow(expectCode('SESSION_QUERY_INVALID_LIMIT')) } }) + + it('materializes owned filter values during normalization', () => { + const values = ['live'] as Array<'live' | 'persisted'> + const filter = { kind: 'availability' as const, values } + const request = { query: 'needle', sessionFilters: [filter] } + const normalized = normalizeSessionRequest(request, limits) + + values[0] = 'persisted' + request.sessionFilters.push({ kind: 'availability', values: ['persisted'] }) + expect(normalized.sessionFilters).toEqual([{ kind: 'availability', values: ['live'] }]) + }) }) describe('SQLite search predicate compilation', () => { @@ -120,6 +150,17 @@ describe('SQLite search predicate compilation', () => { { kind: 'surface', values: [] }, ])).toEqual({ sql: '0 AND 0', params: [] }) }) + + it('rejects runtime-unknown filter discriminants in both SQL builders', () => { + expect(() => buildSessionWhere([{ kind: 'future' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => buildEventWhere([{ kind: 'future' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => buildSessionWhere([{ kind: 'availability', values: ['future'] } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => buildSessionWhere([{} as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + }) }) describe('SQLite query identity and presentation', () => { @@ -169,11 +210,13 @@ describe('SQLite query identity and presentation', () => { }) it('normalizes, bounds, and positions snippets by Unicode code point', () => { - expect(makeSnippet(' short\ntext ', 'absent', 20)).toBe('short text') - expect(makeSnippet('abcdef', 'f', 1)).toBe('…') - expect(makeSnippet('abcdefghij', 'absent', 5)).toBe('abcd…') - expect(makeSnippet('abcdefghij', 'c', 5)).toBe('…bcd…') - expect(makeSnippet('abcdef', 'f', 2)).toBe('a…') - expect(makeSnippet('abcdef', 'f', 5)).toBe('…cdef') + expect(makeSnippet(' short\ntext ', 20)).toBe('short text') + expect(makeSnippet(`abcde${FTS_HIGHLIGHT_START}f${FTS_HIGHLIGHT_END}`, 1)).toBe('…') + expect(makeSnippet('abcdefghij', 5)).toBe('abcd…') + expect(makeSnippet(`ab${FTS_HIGHLIGHT_START}c${FTS_HIGHLIGHT_END}defghij`, 5)).toBe('…bcd…') + expect(makeSnippet(`abcde${FTS_HIGHLIGHT_START}f${FTS_HIGHLIGHT_END}`, 2)).toBe('a…') + expect(makeSnippet(`abcde${FTS_HIGHLIGHT_START}f${FTS_HIGHLIGHT_END}`, 5)).toBe('…cdef') + expect(makeSnippet(` x—${FTS_HIGHLIGHT_START}café${FTS_HIGHLIGHT_END}\n y `, 20)) + .toBe('x—café y') }) }) diff --git a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts index 77d1e0125d..f4904e17d8 100644 --- a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts @@ -1,18 +1,25 @@ -import { afterEach, describe, expect, it } from 'vitest' -import { Context } from 'cordis' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Context, type Fiber } from 'cordis' import { DatabaseSync } from 'node:sqlite' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import SessionStore, { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' import type { SessionEvent, SessionHeader, SessionId as SessionIdType } from '@deepseek-ai/dsh-session' -import SessionPersistence from '@deepseek-ai/dsh-session-persistence' +import SessionPersistence, { SessionPersistenceRevision } from '@deepseek-ai/dsh-session-persistence' +import type { SessionPersistenceSnapshot } from '@deepseek-ai/dsh-session-persistence' import SessionPersistenceSqlite from '@deepseek-ai/dsh-session-persistence-sqlite' import SessionSearchSqlite, { SESSION_QUERY_SQLITE_APPLICATION_ID, SESSION_QUERY_SQLITE_SCHEMA_VERSION, } from '@deepseek-ai/dsh-session-query-sqlite' -import type { SessionQueryErrorCode } from '@deepseek-ai/dsh-session-query' +import { + SessionQueryError, + SessionSearchCursor, + type SessionAvailability, + type SessionQueryErrorCode, + type SessionSearchRequest, +} from '@deepseek-ai/dsh-session-query' const temporaryDirectories: string[] = [] @@ -48,19 +55,36 @@ function expectCode(code: SessionQueryErrorCode): Error { class TestPersistence extends SessionPersistence { static entries = new Map() + static revisions = new Map() + static nextRevision = 0 + static loads = new Map() + static loadEffect: ((entry: { meta: SessionHeader; events: SessionEvent[] }) => void) | undefined static listGate: Promise | undefined static listStarted: (() => void) | undefined + static snapshotEffect: (() => void | Promise) | undefined + static snapshotOverride: (() => SessionPersistenceSnapshot[]) | undefined static failure: unknown static reset(entries: readonly { meta: SessionHeader; events: SessionEvent[] }[] = []): void { - this.entries = new Map(entries.map(entry => [entry.meta.id, structuredClone(entry)])) + this.entries = new Map() + this.revisions = new Map() + this.loads = new Map() + this.loadEffect = undefined + for (const entry of entries) this.set(entry) this.listGate = undefined this.listStarted = undefined + this.snapshotEffect = undefined + this.snapshotOverride = undefined this.failure = undefined } + static set(entry: { meta: SessionHeader; events: SessionEvent[] }): void { + this.entries.set(entry.meta.id, structuredClone(entry)) + this.revisions.set(entry.meta.id, ++this.nextRevision) + } + create(meta: SessionHeader): Promise { - TestPersistence.entries.set(meta.id, { meta: structuredClone(meta), events: [] }) + TestPersistence.set({ meta, events: [] }) return Promise.resolve() } @@ -68,13 +92,21 @@ class TestPersistence extends SessionPersistence { const entry = TestPersistence.entries.get(id) if (entry === undefined) return Promise.reject(new Error('missing test session')) entry.events.push(...structuredClone(events)) + TestPersistence.revisions.set(id, ++TestPersistence.nextRevision) return Promise.resolve() } async load(id: SessionIdType): Promise<{ meta: SessionHeader; events: SessionEvent[] }> { + TestPersistence.loads.set(id, (TestPersistence.loads.get(id) ?? 0) + 1) if (TestPersistence.failure !== undefined) throw TestPersistence.failure const entry = TestPersistence.entries.get(id) if (entry === undefined) throw new Error('missing test session') + if (TestPersistence.loadEffect !== undefined) { + const effect = TestPersistence.loadEffect + TestPersistence.loadEffect = undefined + effect(entry) + TestPersistence.revisions.set(id, ++TestPersistence.nextRevision) + } return structuredClone(entry) } @@ -84,6 +116,20 @@ class TestPersistence extends SessionPersistence { if (TestPersistence.failure !== undefined) throw TestPersistence.failure return [...TestPersistence.entries.values()].map(entry => structuredClone(entry.meta)) } + + + async listSnapshots(): Promise { + TestPersistence.listStarted?.() + await TestPersistence.listGate + if (TestPersistence.failure !== undefined) throw TestPersistence.failure + const snapshots = TestPersistence.snapshotOverride?.() + ?? [...TestPersistence.entries.values()].map(entry => ({ + header: structuredClone(entry.meta), + revision: SessionPersistenceRevision(`test:${TestPersistence.revisions.get(entry.meta.id)}`), + })) + await TestPersistence.snapshotEffect?.() + return snapshots + } } async function liveContext(config: ConstructorParameters[1] = { path: ':memory:' }): Promise { @@ -175,6 +221,44 @@ describe('SQLite session search', () => { await expect(ctx.sessionSearch.searchSessions({ query: '*' })).resolves.toEqual({ items: [] }) }) + it('ranks live and persisted matches on one source-comparable contract', async () => { + const persisted = header('z-persisted') + TestPersistence.reset([ + { meta: persisted, events: messageEvents('needle needle', 10) }, + ...Array.from({ length: 12 }, (_, index) => ({ + meta: header(`filler-${index}`), + events: messageEvents('needle', 10), + })), + ]) + const ctx = await liveContext() + const persistence = await ctx.plugin(TestPersistence) + ctx.sessions.create(SessionId('a-live'), { + seed: messageEvents('needle needle', 10), + meta: { createdAt: persisted.createdAt }, + }) + + const result = await ctx.sessionSearch.searchSessions({ + query: 'needle', + sessionFilters: [{ kind: 'id', values: [SessionId('a-live'), persisted.id] }], + }) + expect(result.items.map(item => item.header.id)).toEqual([SessionId('a-live'), persisted.id]) + await persistence.dispose() + }) + + it('positions snippets from FTS5 matches across diacritics and punctuation', async () => { + const ctx = await liveContext({ path: ':memory:', snippetChars: 14 }) + const session = ctx.sessions.create(SessionId('snippet'), { + seed: messageEvents('long long long—café,\nnext value', 10), + }) + + const page = await ctx.sessionSearch.searchEvents({ sessionId: session.id, query: 'CAFE' }) + expect(page.items).toHaveLength(1) + expect(page.items[0]!.snippet).toContain('café') + expect(page.items[0]!.snippet).toContain('—') + expect(page.items[0]!.snippet).not.toContain('\n') + expect(Array.from(page.items[0]!.snippet).length).toBeLessThanOrEqual(14) + }) + it('binds cursors to requests and only invalidates within-session pages for target changes', async () => { const ctx = await liveContext({ path: ':memory:', defaultLimit: 1, maxLimit: 5 }) const target = ctx.sessions.create(SessionId('target'), { @@ -193,7 +277,7 @@ describe('SQLite session search', () => { if (eventPage.nextCursor === undefined || sessionPage.nextCursor === undefined) throw new Error('expected cursors') const eventKeys = eventPage.items.map(item => `${item.sessionId}:${item.seq}`) - let eventCursor: string | undefined = eventPage.nextCursor + let eventCursor: ReturnType | undefined = eventPage.nextCursor while (eventCursor !== undefined) { const next = await ctx.sessionSearch.searchEvents({ sessionId: target.id, @@ -208,7 +292,7 @@ describe('SQLite session search', () => { expect(new Set(eventKeys).size).toBe(eventKeys.length) const sessionIds = sessionPage.items.map(item => item.header.id) - let sessionCursor: string | undefined = sessionPage.nextCursor + let sessionCursor: ReturnType | undefined = sessionPage.nextCursor while (sessionCursor !== undefined) { const next = await ctx.sessionSearch.searchSessions({ query: 'needle', limit: 1, cursor: sessionCursor }) sessionIds.push(...next.items.map(item => item.header.id)) @@ -251,6 +335,7 @@ describe('SQLite session search', () => { { sessionId: session.id, query: 'needle', limit: 4 }, { sessionId: session.id, query: 'needle', filters: [{ kind: 'seq', from: 2, to: 1 }] }, { sessionId: session.id, query: 'needle', filters: [{ kind: 'surface', values: ['future'] }] }, + { sessionId: session.id, query: 'bad\0query' }, ] as const) { await expect(ctx.sessionSearch.searchEvents(request as never)).rejects.toBeInstanceOf(Error) } @@ -258,7 +343,24 @@ describe('SQLite session search', () => { query: 'needle', sessionFilters: [{ kind: 'availability', values: ['remote' as never] }], })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) - await expect(ctx.sessionSearch.searchEvents({ sessionId: session.id, query: 'needle', cursor: 'not-json' })) + await expect(ctx.sessionSearch.searchSessions({ + query: 'needle', + sessionFilters: [{ kind: 'future' } as never], + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionSearch.searchSessions({ + query: 'needle', + eventFilters: [{ kind: 'future' } as never], + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: session.id, + query: 'needle', + filters: [{ kind: 'future' } as never], + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: session.id, + query: 'needle', + cursor: SessionSearchCursor('not-json'), + })) .rejects.toThrow(expectCode('SESSION_QUERY_INVALID_CURSOR')) await expect(ctx.sessionSearch.searchEvents({ sessionId: SessionId('absent'), query: 'needle' })) .rejects.toThrow(expectCode('SESSION_QUERY_SESSION_NOT_FOUND')) @@ -280,6 +382,37 @@ describe('SQLite session search', () => { }) describe('SQLite reconciliation and source lifecycle', () => { + it('owns queued request and filter values before waiting for the serializer', async () => { + const durable = header('owned') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + const persistence = await ctx.plugin(TestPersistence) + let release!: () => void + TestPersistence.listGate = new Promise((resolve) => { release = resolve }) + let markStarted!: () => void + const started = new Promise((resolve) => { markStarted = resolve }) + TestPersistence.listStarted = () => { + TestPersistence.listStarted = undefined + markStarted() + } + const blocking = ctx.sessionSearch.searchSessions({ query: 'needle' }) + await started + + const availability: SessionAvailability[] = ['persisted'] + const request: SessionSearchRequest = { + query: 'needle', + sessionFilters: [{ kind: 'availability', values: availability }], + } + const queued = ctx.sessionSearch.searchSessions(request) + request.query = 'absent' + availability[0] = 'live' + release() + + await expect(blocking).resolves.toMatchObject({ items: [{ header: durable }] }) + await expect(queued).resolves.toMatchObject({ items: [{ header: durable }] }) + await persistence.dispose() + }) + it('mounts persistence dynamically, shadows with TEMP live rows, reveals, and hides on unmount', async () => { const shared = header('shared', 10, { cwd: '/work' }) const durable = header('durable', 5) @@ -311,7 +444,7 @@ describe('SQLite reconciliation and source lifecycle', () => { .rejects.toThrow(expectCode('SESSION_QUERY_SESSION_NOT_FOUND')) }) - it('restarts observation when persistence unmounts during an asynchronous list', async () => { + it('discards a stale list rejection when persistence unmounts during observation', async () => { const durable = header('racing') TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) const ctx = await liveContext() @@ -328,10 +461,140 @@ describe('SQLite reconciliation and source lifecycle', () => { const search = ctx.sessionSearch.searchSessions({ query: 'needle' }) await started await persistenceFiber.dispose() + TestPersistence.failure = new Error('stale backend rejection') release() await expect(search).resolves.toEqual({ items: [] }) }) + it('retries against a replacement after the prior binding rejects', async () => { + const durable = header('replacement') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + const prior = await ctx.plugin(TestPersistence) + let rejectPrior!: (reason: unknown) => void + TestPersistence.listGate = new Promise((_resolve, reject) => { rejectPrior = reject }) + let markStarted!: () => void + const started = new Promise((resolve) => { markStarted = resolve }) + TestPersistence.listStarted = () => { + TestPersistence.listStarted = undefined + markStarted() + } + + const search = ctx.sessionSearch.searchSessions({ query: 'needle' }) + await started + await prior.dispose() + TestPersistence.listGate = undefined + const replacement = await ctx.plugin(TestPersistence) + rejectPrior(new Error('stale prior binding')) + await expect(search).resolves.toMatchObject({ items: [{ header: durable }] }) + await replacement.dispose() + }) + + it('reloads a replacement source even when its opaque revisions collide', async () => { + const durable = header('colliding-replacement') + TestPersistence.reset([{ meta: durable, events: messageEvents('old content') }]) + const revision = TestPersistence.revisions.get(durable.id)! + const ctx = await liveContext() + const prior = await ctx.plugin(TestPersistence) + await expect(ctx.sessionSearch.searchSessions({ query: 'old' })) + .resolves.toMatchObject({ items: [{ header: durable }] }) + await prior.dispose() + + TestPersistence.set({ meta: durable, events: messageEvents('new needle') }) + TestPersistence.revisions.set(durable.id, revision) + const replacement = await ctx.plugin(TestPersistence) + const internals = ctx.sessionSearch as unknown as { + _lastPersistenceRevision: number + _persistenceRevision: number + } + expect(internals._persistenceRevision).not.toBe(internals._lastPersistenceRevision) + const page = await ctx.sessionSearch.searchSessions({ query: 'new needle' }) + expect(TestPersistence.loads.get(durable.id)).toBe(2) + expect(page).toMatchObject({ items: [{ header: durable }] }) + await expect(ctx.sessionSearch.searchSessions({ query: 'old' })).resolves.toEqual({ items: [] }) + expect(TestPersistence.loads.get(durable.id)).toBe(2) + await replacement.dispose() + }) + + it('retries when a successful observation belongs to a source unmounted during listing', async () => { + const durable = header('successful-unmount') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + const persistence = await ctx.plugin(TestPersistence) + let lists = 0 + TestPersistence.snapshotEffect = async () => { + lists += 1 + if (lists === 2) await persistence.dispose() + } + + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })).resolves.toEqual({ items: [] }) + expect(lists).toBe(2) + }) + + it('retries when the snapshot population changes during observation', async () => { + const first = header('first') + const added = header('added-during-list') + TestPersistence.reset([{ meta: first, events: messageEvents('first needle') }]) + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + TestPersistence.snapshotEffect = () => { + TestPersistence.snapshotEffect = undefined + TestPersistence.set({ meta: added, events: messageEvents('added needle') }) + } + + const page = await ctx.sessionSearch.searchSessions({ query: 'needle' }) + expect(page.items.map(item => item.header.id).sort()).toEqual([added.id, first.id].sort()) + expect(TestPersistence.loads.get(first.id)).toBe(2) + expect(TestPersistence.loads.get(added.id)).toBe(1) + }) + + it('retries if the source revision changes while live sessions are observed', async () => { + const durable = header('live-boundary-retry') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + const internals = ctx.sessionSearch as unknown as { _persistenceRevision: number } + const originalList = ctx.sessions.list.bind(ctx.sessions) + let bumped = false + const list = vi.spyOn(ctx.sessions, 'list').mockImplementation(() => { + if (!bumped) { + bumped = true + internals._persistenceRevision += 1 + } + return originalList() + }) + + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .resolves.toMatchObject({ items: [{ header: durable }] }) + expect(TestPersistence.loads.get(durable.id)).toBe(2) + list.mockRestore() + }) + + it('rejects malformed snapshots and preserves typed persistence failures', async () => { + const durable = header('invalid-snapshot') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + + TestPersistence.snapshotOverride = () => 'not-an-array' as never + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_PERSISTENCE_FAILED')) + TestPersistence.snapshotOverride = () => [{ header: durable, revision: 1 as never }] + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_PERSISTENCE_FAILED')) + TestPersistence.snapshotOverride = () => [ + { header: durable, revision: SessionPersistenceRevision('duplicate:1') }, + { header: durable, revision: SessionPersistenceRevision('duplicate:2') }, + ] + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_PERSISTENCE_FAILED')) + + TestPersistence.snapshotOverride = undefined + const typed = new SessionQueryError('typed persistence failure', 'SESSION_QUERY_PERSISTENCE_FAILED') + TestPersistence.failure = typed + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })).rejects.toBe(typed) + }) + it('rejects immutable header conflicts between live and persisted sources', async () => { const shared = header('conflict', 10) TestPersistence.reset([{ meta: shared, events: messageEvents('persisted needle') }]) @@ -358,6 +621,9 @@ describe('SQLite reconciliation and source lifecycle', () => { const firstPersistence = await first.plugin(TestPersistence) const firstSearch = await first.plugin(SessionSearchSqlite, { path }) await first.sessionSearch.searchSessions({ query: 'needle' }) + expect(Object.fromEntries(TestPersistence.loads)).toEqual({ unchanged: 1, changed: 1, deleted: 1 }) + await first.sessionSearch.searchSessions({ query: 'needle' }) + expect(Object.fromEntries(TestPersistence.loads)).toEqual({ unchanged: 1, changed: 1, deleted: 1 }) await firstSearch.dispose() await firstPersistence.dispose() @@ -368,14 +634,20 @@ describe('SQLite reconciliation and source lifecycle', () => { const added = header('added') TestPersistence.entries.delete(deleted.id) - TestPersistence.entries.set(changed.id, { meta: changed, events: messageEvents('changed needle') }) - TestPersistence.entries.set(added.id, { meta: added, events: messageEvents('added needle') }) + TestPersistence.set({ meta: changed, events: messageEvents('changed needle') }) + TestPersistence.set({ meta: added, events: messageEvents('added needle') }) const second = new Context() await second.plugin(SessionStore) const secondPersistence = await second.plugin(TestPersistence) const secondSearch = await second.plugin(SessionSearchSqlite, { path }) const result = await second.sessionSearch.searchSessions({ query: 'needle' }) expect(result.items.map(item => item.header.id).sort()).toEqual([added.id, changed.id, unchanged.id].sort()) + expect(Object.fromEntries(TestPersistence.loads)).toEqual({ + unchanged: 1, + changed: 2, + deleted: 1, + added: 1, + }) await secondSearch.dispose() await secondPersistence.dispose() @@ -409,10 +681,27 @@ describe('SQLite reconciliation and source lifecycle', () => { await expect(second.sessionSearch.searchSessions({ query: 'live' })).resolves.toEqual({ items: [] }) await expect(second.sessionSearch.searchSessions({ query: 'persisted' })) .resolves.toMatchObject({ items: [{ header: shared, live: false, persisted: true }] }) + expect(TestPersistence.loads.get(shared.id)).toBe(1) await searchAgain.dispose() await persistenceAgain.dispose() }) + it('refreshes the stored revision after a mutating load repair', async () => { + const durable = header('repair') + TestPersistence.reset([{ meta: durable, events: messageEvents('before repair') }]) + TestPersistence.loadEffect = (entry) => { + entry.events = messageEvents('repaired needle') + } + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + + await expect(ctx.sessionSearch.searchSessions({ query: 'repaired' })) + .resolves.toMatchObject({ items: [{ header: durable }] }) + expect(TestPersistence.loads.get(durable.id)).toBe(2) + await ctx.sessionSearch.searchSessions({ query: 'repaired' }) + expect(TestPersistence.loads.get(durable.id)).toBe(2) + }) + it('recovers on the next search after source and SQLite transaction failures', async () => { TestPersistence.reset([{ meta: header('durable'), events: messageEvents('durable needle') }]) const ctx = await liveContext() @@ -462,15 +751,18 @@ describe('SQLite schema, cancellation, and real persistence integration', () => const foreignPath = await temporaryPath('foreign.db') const foreign = new DatabaseSync(foreignPath) + foreign.exec('PRAGMA journal_mode = WAL') foreign.exec('CREATE TABLE canonical(value TEXT)') foreign.exec("INSERT INTO canonical VALUES ('safe')") foreign.close() - const foreignCtx = await liveContext({ path: foreignPath }) + const foreignCtx = await liveContext({ path: foreignPath, journalMode: 'delete' }) await expect(foreignCtx.sessionSearch.searchSessions({ query: 'needle' })) .rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) const stillForeign = new DatabaseSync(foreignPath) expect(stillForeign.prepare('SELECT value FROM canonical').get()).toEqual({ value: 'safe' }) + expect(stillForeign.prepare('PRAGMA journal_mode').get()).toEqual({ journal_mode: 'wal' }) stillForeign.close() + await (foreignCtx.sessionSearch as SessionSearchSqlite).close() const otherAppPath = await temporaryPath('other-app.db') const otherApp = new DatabaseSync(otherAppPath) @@ -479,6 +771,25 @@ describe('SQLite schema, cancellation, and real persistence integration', () => const otherAppCtx = await liveContext({ path: otherAppPath }) await expect(otherAppCtx.sessionSearch.searchSessions({ query: 'needle' })) .rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) + await (otherAppCtx.sessionSearch as SessionSearchSqlite).close() + }) + + it('observes asynchronous open rejection even when no query is made', async () => { + const path = await temporaryPath('never-queried.db') + const foreign = new DatabaseSync(path) + foreign.exec('CREATE TABLE canonical(value TEXT)') + foreign.close() + const unhandled: unknown[] = [] + const onUnhandled = (reason: unknown) => { unhandled.push(reason) } + process.on('unhandledRejection', onUnhandled) + try { + const ctx = await liveContext({ path }) + await new Promise((resolve) => { setImmediate(resolve) }) + expect(unhandled).toEqual([]) + await (ctx.sessionSearch as SessionSearchSqlite).close() + } finally { + process.off('unhandledRejection', onUnhandled) + } }) it('cancels both queued and in-flight source waits without committing them', async () => { @@ -518,7 +829,7 @@ describe('SQLite schema, cancellation, and real persistence integration', () => releaseBlocking() await expect(blocking).resolves.toEqual({ items: [] }) - TestPersistence.entries.set(SessionId('uncommitted'), { + TestPersistence.set({ meta: header('uncommitted'), events: messageEvents('durable needle'), }) @@ -560,14 +871,38 @@ describe('SQLite schema, cancellation, and real persistence integration', () => await started const queued = search.searchSessions({ query: 'needle' }) const closing = search.close() + const repeatedClose = search.close() + expect(repeatedClose).toBe(closing) release() await expect(accepted).resolves.toEqual({ items: [] }) await expect(queued).rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) - await closing + await Promise.all([closing, repeatedClose]) await expect(search.searchSessions({ query: 'needle' })) .rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) - await search.close() + expect(search.close()).toBe(closing) + }) + + it('awaits optional-persistence child-fiber quiescence on disposal', async () => { + TestPersistence.reset() + const ctx = new Context() + await ctx.plugin(SessionStore) + const search = await ctx.plugin(SessionSearchSqlite, { path: ':memory:' }) + const persistence = await ctx.plugin(TestPersistence) + const optional = (ctx.sessionSearch as unknown as { + _optionalPersistenceFiber: Fiber + })._optionalPersistenceFiber + let release!: () => void + const cleanup = new Promise((resolve) => { release = resolve }) + optional.ctx.effect(() => () => cleanup) + + let settled = false + const disposing = search.dispose().then(() => { settled = true }) + await Promise.resolve() + expect(settled).toBe(false) + release() + await disposing + await persistence.dispose() }) it('combines the real SQLite persistence backend with the real search service keylessly', async () => { diff --git a/packages/session-query/session-query/README.md b/packages/session-query/session-query/README.md index 6f9e9bbd7a..917984f0cd 100644 --- a/packages/session-query/session-query/README.md +++ b/packages/session-query/session-query/README.md @@ -7,6 +7,7 @@ This is trusted context-wide infrastructure. It performs no caller authorization ## Reads - `listSessions()` reads current persistence metadata, merges live records with live precedence, and returns cloned records in deterministic newest-first order. +- `filterSessions(filters)` applies provider-independent session metadata and availability predicates to that same cloned logical corpus. - `listEvents(sessionId)` loads the live-preferred raw log and classifies each event as `current`, `shadowed`, or `log-only` with the shared `dsh-session` surface fold. - `filterEvents(sessionId, filters)` extracts first-party semantic documents and applies provider-independent metadata and literal-text predicates in ascending seq order. - `readEvent(request)` returns a cloned header, the full target event, and a bounded raw-seq window. `before` and `after` default to zero and may not exceed `readWindowMax`. @@ -21,7 +22,7 @@ The text clause is deliberately independent of FTS providers: caller text is esc ## Full-text seam -`SessionSearchService` owns the independent `ctx.sessionSearch` key. `searchSessions(request, exec?)` groups the logical corpus by strongest matching event; `searchEvents(request, exec?)` searches one logical session. Both return opaque cursor pages, accept optional cancellation, and expose snippets without provider-specific numeric scores. Search requests accept only metadata event filters, because literal-text filtering is the scan path described above. +`SessionSearchService` owns the independent `ctx.sessionSearch` key. `searchSessions(request, exec?)` groups the logical corpus by strongest matching event; `searchEvents(request, exec?)` searches one logical session. Both return pages whose continuation is an owned branded `SessionSearchCursor`, accept optional cancellation, and expose snippets without provider-specific numeric scores. Search requests accept only metadata event filters, because literal-text filtering is the scan path described above. The package has no provider coordinator or registration protocol. A concrete backend owns observation, reconciliation, ranking, cursor generations, and query execution as one lifecycle; the first implementation is [`@deepseek-ai/dsh-session-query-sqlite`](../session-query-sqlite/README.md). diff --git a/packages/session-query/session-query/package.json b/packages/session-query/session-query/package.json index 9f78d4f1db..449610a5a4 100644 --- a/packages/session-query/session-query/package.json +++ b/packages/session-query/session-query/package.json @@ -22,6 +22,7 @@ ], "license": "BSD-3-Clause", "peerDependencies": { + "@deepseek-ai/dsh-brand": "^0.0.1", "@deepseek-ai/dsh-llm": "^0.0.1", "@deepseek-ai/dsh-session": "^0.0.1", "@deepseek-ai/dsh-session-persistence": "^0.0.1", @@ -36,6 +37,7 @@ "schemastery": "^3.18.0" }, "devDependencies": { + "@deepseek-ai/dsh-brand": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", "@deepseek-ai/dsh-session": "workspace:^", "@deepseek-ai/dsh-session-persistence": "workspace:^", diff --git a/packages/session-query/session-query/src/corpus.ts b/packages/session-query/session-query/src/corpus.ts index c8ddca3caa..4a27c34e58 100644 --- a/packages/session-query/session-query/src/corpus.ts +++ b/packages/session-query/session-query/src/corpus.ts @@ -1,6 +1,6 @@ /** Live/persisted logical-corpus resolution for session-query. */ -import type { Context } from 'cordis' +import type { Context, Fiber } from 'cordis' import type { Session, SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session' import type SessionPersistence from '@deepseek-ai/dsh-session-persistence' import type { SessionRecord } from './types.ts' @@ -18,18 +18,19 @@ export interface LogicalSession { /** Resolves a live-preferred corpus against the persistence service mounted now. */ export class SessionCorpus { private _persistence: SessionPersistence | undefined + private readonly _optionalPersistenceFiber: Fiber constructor(private readonly _ctx: Context) { + this._optionalPersistenceFiber = _ctx.inject(['sessionPersistence'], (childCtx: Context) => { + const service = childCtx.sessionPersistence + this._persistence = service + childCtx.effect(() => () => { + /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */ + if (this._persistence === service) this._persistence = undefined + }, 'sessionQuery.persistenceBinding') + }) _ctx.effect(() => { - const fiber = _ctx.inject(['sessionPersistence'], (childCtx: Context) => { - const service = childCtx.sessionPersistence - this._persistence = service - childCtx.effect(() => () => { - /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */ - if (this._persistence === service) this._persistence = undefined - }, 'sessionQuery.persistenceBinding') - }) - return () => void fiber.dispose() + return () => this._optionalPersistenceFiber.dispose() }, 'sessionQuery.optionalPersistence') } diff --git a/packages/session-query/session-query/src/cursor.ts b/packages/session-query/session-query/src/cursor.ts new file mode 100644 index 0000000000..8ee2a6660d --- /dev/null +++ b/packages/session-query/session-query/src/cursor.ts @@ -0,0 +1,15 @@ +/** Opaque cursor identity for session-search pagination. */ + +import type { Branded } from '@deepseek-ai/dsh-brand' + +/** Provider-owned opaque continuation token returned by session search. */ +export type SessionSearchCursor = Branded<'SessionSearchCursor'> + +/** + * Brand an encoded provider cursor for the public search contract. + * @param value - opaque encoded cursor value. + * @returns the same runtime string with session-search cursor identity. + */ +export function SessionSearchCursor(value: string): SessionSearchCursor { + return value as SessionSearchCursor +} diff --git a/packages/session-query/session-query/src/filters.ts b/packages/session-query/session-query/src/filters.ts index c7a7b40dd4..91ae640615 100644 --- a/packages/session-query/session-query/src/filters.ts +++ b/packages/session-query/session-query/src/filters.ts @@ -1,6 +1,12 @@ /** Pure provider-independent predicates for logical sessions and event text. */ -import type { SessionRecord, SessionEventSearchDocument, SessionEventResultFilter, SessionResultFilter, SessionResultRange } from './types.ts' +import type { + SessionEventResultFilter, + SessionEventSearchDocument, + SessionRecord, + SessionResultFilter, + SessionResultRange, +} from './types.ts' import { SessionQueryError } from './config.ts' /** @@ -31,6 +37,66 @@ export function filterSessionEventDocuments predicates.every(predicate => predicate(document))) } +/** + * Copy and validate logical-session filters before an asynchronous boundary. + * @param filters - caller-owned clauses to materialize. + * @returns detached validated clauses. + */ +export function materializeSessionResultFilters( + filters: readonly SessionResultFilter[], +): SessionResultFilter[] { + assertArray(filters) + return filters.map((filter) => { + switch (filter.kind) { + case 'id': + return { kind: filter.kind, values: copyStrings(filter.kind, filter.values) } + case 'cwd': + return { kind: filter.kind, values: copyNullableStrings(filter.kind, filter.values) } + case 'created-at': + return copyRange(filter.kind, filter) + case 'parent': + return { kind: filter.kind, values: copyNullableStrings(filter.kind, filter.values) } + case 'availability': { + const values = copyStrings(filter.kind, filter.values) + assertAllowedValues(filter.kind, values, ['live', 'persisted']) + return { kind: filter.kind, values } + } + default: + return unknownFilter(filter) + } + }) +} + +/** + * Copy and validate event filters before an asynchronous boundary. + * @param filters - caller-owned clauses to materialize. + * @returns detached validated clauses. + */ +export function materializeSessionEventResultFilters( + filters: readonly SessionEventResultFilter[], +): SessionEventResultFilter[] { + assertArray(filters) + return filters.map((filter) => { + switch (filter.kind) { + case 'seq': + case 'time': + return copyRange(filter.kind, filter) + case 'type': + return { kind: filter.kind, values: copyStrings(filter.kind, filter.values) } + case 'surface': { + const values = copyStrings(filter.kind, filter.values) + assertAllowedValues(filter.kind, values, ['current', 'shadowed', 'log-only']) + return { kind: filter.kind, values } + } + case 'text': + if (typeof filter.text !== 'string') throw invalidFilter('text filter text must be a string') + return { kind: filter.kind, text: filter.text } + default: + return unknownFilter(filter) + } + }) +} + /** * Compile a literal case-insensitive, whitespace-flexible semantic-text match. * @param text - caller-provided literal text. @@ -66,6 +132,8 @@ function sessionPredicate(filter: SessionResultFilter): (record: SessionRecord) case 'availability': assertAllowedValues(filter.kind, filter.values, ['live', 'persisted']) return record => filter.values.some(value => value === 'live' ? record.live : record.persisted) + default: + return unknownFilter(filter) } } @@ -88,9 +156,47 @@ function eventPredicate(filter: SessionEventResultFilter): (document: SessionEve const pattern = compileSessionTextFilter(filter.text) return document => pattern.test(document.text) } + default: + return unknownFilter(filter) } } +function copyStrings(name: string, values: readonly T[]): T[] { + if (!isRuntimeArray(values) || values.some(value => typeof value !== 'string')) { + throw invalidFilter(`${name} filter values must be an array of strings`) + } + return [...values] +} + +function assertArray(value: unknown): void { + if (!Array.isArray(value)) throw invalidFilter('filters must be an array') +} + +function copyNullableStrings(name: string, values: readonly (T | null)[]): Array { + if (!isRuntimeArray(values) || values.some(value => value !== null && typeof value !== 'string')) { + throw invalidFilter(`${name} filter values must be an array of strings or null`) + } + return [...values] +} + +function copyRange( + kind: K, + range: SessionResultRange, +): { kind: K } & SessionResultRange { + const copy = { + kind, + ...range.from === undefined ? {} : { from: range.from }, + ...range.to === undefined ? {} : { to: range.to }, + } + validateRange(kind, copy) + return copy +} + +function unknownFilter(filter: never): never { + const kind = (filter as { kind?: unknown }).kind + throw invalidFilter(`unknown filter kind ${typeof kind === 'string' ? `"${kind}"` : '(missing)'}`) +} + function assertAllowedValues( name: string, values: readonly string[], @@ -125,8 +231,13 @@ function matchesRange(value: number, range: SessionResultRange): boolean { } function invalidRange(name: string, detail: string): SessionQueryError { - return new SessionQueryError( - `session ${name} filter ${detail}`, - 'SESSION_QUERY_INVALID_FILTER', - ) + return invalidFilter(`${name} filter ${detail}`) +} + +function invalidFilter(detail: string): SessionQueryError { + return new SessionQueryError(`session ${detail}`, 'SESSION_QUERY_INVALID_FILTER') +} + +function isRuntimeArray(value: unknown): boolean { + return Array.isArray(value) } diff --git a/packages/session-query/session-query/src/index.ts b/packages/session-query/session-query/src/index.ts index 243c86746b..2c2155eebc 100644 --- a/packages/session-query/session-query/src/index.ts +++ b/packages/session-query/session-query/src/index.ts @@ -16,6 +16,7 @@ import type { SessionEventSearchRequest, SessionEventWindow, SessionRecord, + SessionResultFilter, SessionSearchExecContext, SessionSearchHit, SessionSearchPage, @@ -28,14 +29,26 @@ import { } from './config.ts' import { SessionCorpus } from './corpus.ts' import { buildSessionEventRecords, buildSessionEventSearchDocuments } from './documents.ts' -import { filterSessionEventDocuments } from './filters.ts' +import { + filterSessionEventDocuments, + filterSessionResults, + materializeSessionEventResultFilters, + materializeSessionResultFilters, +} from './filters.ts' export type * from './types.ts' +export { SessionSearchCursor } from './cursor.ts' export type { Config, SessionQueryErrorCode } from './config.ts' export { SESSION_QUERY_READ_WINDOW_MAX, SessionQueryError } from './config.ts' export { extractSessionEventText } from './extraction.ts' export { buildSessionEventRecords, buildSessionEventSearchDocuments } from './documents.ts' -export { compileSessionTextFilter, filterSessionEventDocuments, filterSessionResults } from './filters.ts' +export { + compileSessionTextFilter, + filterSessionEventDocuments, + filterSessionResults, + materializeSessionEventResultFilters, + materializeSessionResultFilters, +} from './filters.ts' export { assertSessionHeadersCompatible } from './sources.ts' declare module 'cordis' { @@ -109,6 +122,16 @@ export class SessionQueryService extends Service { return this._corpus.listSessions() } + /** + * Filter the complete logical corpus with provider-independent predicates. + * @param filters - ANDed session metadata and availability clauses. + * @returns matching cloned records in deterministic newest-first order. + */ + async filterSessions(filters: readonly SessionResultFilter[]): Promise { + const ownedFilters = materializeSessionResultFilters(filters) + return this._filterSessions(ownedFilters) + } + /** * List lightweight raw-log event records for one logical session. * @param sessionId - live-preferred session id to read. @@ -128,6 +151,18 @@ export class SessionQueryService extends Service { async filterEvents( sessionId: SessionId, filters: readonly SessionEventResultFilter[], + ): Promise { + const ownedFilters = materializeSessionEventResultFilters(filters) + return this._filterEvents(sessionId, ownedFilters) + } + + private async _filterSessions(filters: readonly SessionResultFilter[]): Promise { + return filterSessionResults(await this._corpus.listSessions(), filters) + } + + private async _filterEvents( + sessionId: SessionId, + filters: readonly SessionEventResultFilter[], ): Promise { const loaded = await this._corpus.load(sessionId) const documents = buildSessionEventSearchDocuments(sessionId, loaded.events) @@ -142,16 +177,27 @@ export class SessionQueryService extends Service { async readEvent(request: SessionEventReadRequest): Promise { const before = this._readWindow('before', request.before) const after = this._readWindow('after', request.after) - const loaded = await this._corpus.load(request.sessionId) - const target = loaded.events[request.seq] - if (target === undefined || target.seq !== request.seq) { + const sessionId = request.sessionId + const seq = request.seq + return this._readEvent(sessionId, seq, before, after) + } + + private async _readEvent( + sessionId: SessionId, + seq: number, + before: number, + after: number, + ): Promise { + const loaded = await this._corpus.load(sessionId) + const target = loaded.events[seq] + if (target === undefined || target.seq !== seq) { throw new SessionQueryError( - `session "${request.sessionId}" has no event at seq ${request.seq}`, + `session "${sessionId}" has no event at seq ${seq}`, 'SESSION_QUERY_EVENT_NOT_FOUND', ) } - const startSeq = Math.max(0, request.seq - before) - const endSeq = Math.min(loaded.events.length - 1, request.seq + after) + const startSeq = Math.max(0, seq - before) + const endSeq = Math.min(loaded.events.length - 1, seq + after) return { session: loaded.header, target, diff --git a/packages/session-query/session-query/src/types.ts b/packages/session-query/session-query/src/types.ts index d0de0dd48a..a223084f56 100644 --- a/packages/session-query/session-query/src/types.ts +++ b/packages/session-query/session-query/src/types.ts @@ -5,6 +5,9 @@ */ import type { SessionEvent, SessionEventType, SessionHeader, SessionId } from '@deepseek-ai/dsh-session' +import type { SessionSearchCursor } from './cursor.ts' + +export type { SessionSearchCursor } from './cursor.ts' /** Whether an event is current model context, replaced context, or raw-log-only. */ export type SessionEventSurface = 'current' | 'shadowed' | 'log-only' @@ -106,7 +109,7 @@ export interface SessionSearchPage { /** Results for this page in contract-defined order. */ items: readonly T[] /** Opaque continuation cursor, absent on the final page. */ - nextCursor?: string + nextCursor?: SessionSearchCursor } /** Controls shared by cross-session and within-session search calls. */ @@ -126,7 +129,7 @@ export interface SessionSearchRequest { /** Maximum sessions in this page. */ limit?: number /** Opaque cursor returned for the identical normalized request. */ - cursor?: string + cursor?: SessionSearchCursor } /** Within-session full-text search request. */ @@ -140,7 +143,7 @@ export interface SessionEventSearchRequest { /** Maximum events in this page. */ limit?: number /** Opaque cursor returned for the identical normalized request. */ - cursor?: string + cursor?: SessionSearchCursor } /** One event full-text search hit with a bounded plain-text excerpt. */ diff --git a/packages/session-query/session-query/tests/search-helpers.spec.ts b/packages/session-query/session-query/tests/search-helpers.spec.ts index e9cf857608..8ee9cf1ccf 100644 --- a/packages/session-query/session-query/tests/search-helpers.spec.ts +++ b/packages/session-query/session-query/tests/search-helpers.spec.ts @@ -10,6 +10,8 @@ import SessionQueryService, { extractSessionEventText, filterSessionEventDocuments, filterSessionResults, + materializeSessionEventResultFilters, + materializeSessionResultFilters, SessionSearchService, type SessionEventSearchHit, type SessionEventSearchRequest, @@ -177,6 +179,31 @@ describe('session-query document and filter helpers', () => { expect(() => buildSessionEventRecords(id, malformed)).toThrow(expectCode('SESSION_QUERY_INVALID_SURFACE')) }) + it('owns filters and rejects malformed runtime filter shapes deterministically', () => { + expect(materializeSessionResultFilters([{ kind: 'created-at', to: 2 }])) + .toEqual([{ kind: 'created-at', to: 2 }]) + expect(() => materializeSessionResultFilters('not-an-array' as never)) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionResultFilters([{ kind: 'id', values: 'bad' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionResultFilters([{ kind: 'id', values: [1] } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionResultFilters([{ kind: 'cwd', values: 'bad' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionResultFilters([{ kind: 'parent', values: [1] } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionResultFilters([{} as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionEventResultFilters([{ kind: 'text', text: 1 } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionEventResultFilters([{ kind: 'future' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => filterSessionResults([], [{ kind: 'future' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => filterSessionEventDocuments([], [{ kind: 'future' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + }) + it('exposes the scan path on the concrete exact-read service', async () => { const ctx = new Context() await ctx.plugin(SessionStore) diff --git a/packages/session-query/session-query/tests/session-query.spec.ts b/packages/session-query/session-query/tests/session-query.spec.ts index 3b50feee45..e1432722f2 100644 --- a/packages/session-query/session-query/tests/session-query.spec.ts +++ b/packages/session-query/session-query/tests/session-query.spec.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from 'vitest' -import { Context } from 'cordis' +import { Context, type Fiber } from 'cordis' import SessionStore, { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' import type { SessionEvent, SessionHeader, SessionId as SessionIdType } from '@deepseek-ai/dsh-session' -import SessionPersistence from '@deepseek-ai/dsh-session-persistence' +import SessionPersistence, { SessionPersistenceRevision } from '@deepseek-ai/dsh-session-persistence' import SessionQueryService, { + type SessionEventSurface, type SessionQueryErrorCode, } from '@deepseek-ai/dsh-session-query' @@ -59,6 +60,14 @@ class TestPersistence extends SessionPersistence { TestPersistence.afterList?.() return Promise.resolve(headers) } + + + async listSnapshots() { + return [...TestPersistence.entries.values()].map(entry => ({ + header: structuredClone(entry.meta), + revision: SessionPersistenceRevision(`events:${entry.events.length}`), + })) + } } async function liveContext(config: ConstructorParameters[1] = {}): Promise { @@ -94,6 +103,38 @@ describe('session-query exact reads', () => { expect(older.header.createdAt).toBe(1) }) + it('filters sessions symmetrically and owns mutable filter values immediately', async () => { + const durable = header('durable-filter', 1) + TestPersistence.reset([{ meta: durable, events: eventLog('durable') }]) + const ctx = await liveContext() + const live = ctx.sessions.create(SessionId('live-filter'), { meta: { createdAt: 2 } }) + live.append( + 'user/message', + { content: [{ type: 'text', text: 'live' }], source: { kind: 'user' } }, + { surfaceOp: 'append' }, + ) + const persistence = await ctx.plugin(TestPersistence) + + const ids = [durable.id] + const filtered = ctx.sessionQuery.filterSessions([{ kind: 'id', values: ids }]) + ids[0] = live.id + await expect(filtered).resolves.toEqual([{ + header: durable, + live: false, + persisted: true, + }]) + + const surfaces: SessionEventSurface[] = ['current'] + const events = ctx.sessionQuery.filterEvents(live.id, [{ kind: 'surface', values: surfaces }]) + surfaces[0] = 'shadowed' + await expect(events).resolves.toMatchObject([{ sessionId: live.id, surface: 'current', text: 'live' }]) + await expect(ctx.sessionQuery.filterSessions([{ kind: 'future' } as never])) + .rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionQuery.filterEvents(live.id, [{ kind: 'future' } as never])) + .rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await persistence.dispose() + }) + it('classifies current, shadowed, and raw-log-only events through foldSurface', async () => { const ctx = await liveContext() const session = ctx.sessions.create(SessionId('surface')) @@ -267,4 +308,26 @@ describe('session-query exact reads', () => { await fiber.dispose() expect(ctx.sessionQuery).toBeUndefined() }) + + it('awaits optional-persistence child-fiber quiescence on disposal', async () => { + TestPersistence.reset() + const ctx = new Context() + await ctx.plugin(SessionStore) + const query = await ctx.plugin(SessionQueryService) + const persistence = await ctx.plugin(TestPersistence) + const optional = (ctx.sessionQuery as unknown as { + _corpus: { _optionalPersistenceFiber: Fiber } + })._corpus._optionalPersistenceFiber + let release!: () => void + const cleanup = new Promise((resolve) => { release = resolve }) + optional.ctx.effect(() => () => cleanup) + + let settled = false + const disposing = query.dispose().then(() => { settled = true }) + await Promise.resolve() + expect(settled).toBe(false) + release() + await disposing + await persistence.dispose() + }) }) diff --git a/packages/session-query/session-query/tsconfig.json b/packages/session-query/session-query/tsconfig.json index 7153dae8bb..1a254e5379 100644 --- a/packages/session-query/session-query/tsconfig.json +++ b/packages/session-query/session-query/tsconfig.json @@ -17,6 +17,9 @@ { "path": "../../../vendor/schemastery" }, + { + "path": "../../util/brand" + }, { "path": "../../llm/llm" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 997f00ff5c..902c362355 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -720,6 +720,9 @@ importers: packages/session-persistence/session-persistence: devDependencies: + '@deepseek-ai/dsh-brand': + specifier: workspace:^ + version: link:../../util/brand '@deepseek-ai/dsh-session': specifier: workspace:^ version: link:../../core/session @@ -765,6 +768,9 @@ importers: specifier: ^3.18.0 version: 3.18.0 devDependencies: + '@deepseek-ai/dsh-brand': + specifier: workspace:^ + version: link:../../util/brand '@deepseek-ai/dsh-llm': specifier: workspace:^ version: link:../../llm/llm diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index 0d0a0b871d..ba2d75a8ff 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -45,6 +45,8 @@ { "doc": "docs/core-data-structures/persistence.md", "symbol": "SessionHeader", "source": "packages/core/session/src/types.ts" }, { "doc": "docs/core-data-structures/persistence.md", "symbol": "CreateSessionOptions", "source": "packages/core/session/src/types.ts" }, + { "doc": "docs/core-data-structures/persistence.md", "symbol": "SessionPersistenceRevision", "source": "packages/session-persistence/session-persistence/src/revision.ts" }, + { "doc": "docs/core-data-structures/persistence.md", "symbol": "SessionPersistenceSnapshot", "source": "packages/session-persistence/session-persistence/src/index.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventSurface", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionRecord", "source": "packages/session-query/session-query/src/types.ts" }, @@ -52,6 +54,7 @@ { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionResultFilter", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventResultFilter", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventSearchDocument", "source": "packages/session-query/session-query/src/types.ts" }, + { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionSearchCursor", "source": "packages/session-query/session-query/src/cursor.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionSearchRequest", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventSearchRequest", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionSearchPage", "source": "packages/session-query/session-query/src/types.ts" },