From ecf90ff382344b706a123a5db417869a5084d9d6 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Wed, 15 Jul 2026 10:51:38 +0800 Subject: [PATCH 001/113] feat(session-query): add SQLite full-text search --- docs/architecture.md | 1 + docs/capability-seams.md | 13 +- docs/config-catalog.md | 25 + docs/cordis-catalog/services.md | 16 +- docs/core-data-structures/core.md | 2 +- docs/core-data-structures/session-query.md | 82 +- docs/module-graph.md | 5 + docs/rfc/INDEX.md | 2 +- .../2026-07-10-session-query-service.md | 12 +- ...026-07-10-sqlite-session-query-provider.md | 57 ++ ...026-07-10-sqlite-session-query-provider.md | 51 -- packages/README.md | 2 +- .../cordis/tool-cordis/src/api-catalog.ts | 57 ++ packages/session-query/README.md | 7 +- .../session-query-sqlite/README.md | 35 + .../session-query-sqlite/package.json | 46 ++ .../session-query-sqlite/src/index.ts | 765 ++++++++++++++++++ .../session-query-sqlite/src/query.ts | 312 +++++++ .../session-query-sqlite/src/schema.ts | 127 +++ .../tests/load-path.e2e.ts | 60 ++ .../session-query-sqlite/tests/query.spec.ts | 179 ++++ .../session-query-sqlite/tests/sqlite.spec.ts | 594 ++++++++++++++ .../session-query-sqlite/tsconfig.json | 30 + .../session-query/session-query/README.md | 19 +- .../session-query/session-query/src/config.ts | 11 +- .../session-query/session-query/src/corpus.ts | 21 +- .../session-query/src/documents.ts | 74 ++ .../session-query/src/extraction.ts | 93 +++ .../session-query/src/filters.ts | 132 +++ .../session-query/session-query/src/index.ts | 92 ++- .../session-query/src/sources.ts | 25 + .../session-query/session-query/src/types.ts | 96 +++ .../tests/search-helpers.spec.ts | 209 +++++ pnpm-lock.yaml | 25 + scripts/gen-doc-graphs.ts | 14 +- scripts/type-equiv.manifest.json | 8 + tsconfig.build.json | 1 + tsconfig.json | 1 + 38 files changed, 3181 insertions(+), 120 deletions(-) create mode 100644 docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md delete mode 100644 docs/rfc/proposed/feature/2026-07-10-sqlite-session-query-provider.md create mode 100644 packages/session-query/session-query-sqlite/README.md create mode 100644 packages/session-query/session-query-sqlite/package.json create mode 100644 packages/session-query/session-query-sqlite/src/index.ts create mode 100644 packages/session-query/session-query-sqlite/src/query.ts create mode 100644 packages/session-query/session-query-sqlite/src/schema.ts create mode 100644 packages/session-query/session-query-sqlite/tests/load-path.e2e.ts create mode 100644 packages/session-query/session-query-sqlite/tests/query.spec.ts create mode 100644 packages/session-query/session-query-sqlite/tests/sqlite.spec.ts create mode 100644 packages/session-query/session-query-sqlite/tsconfig.json create mode 100644 packages/session-query/session-query/src/documents.ts create mode 100644 packages/session-query/session-query/src/extraction.ts create mode 100644 packages/session-query/session-query/src/filters.ts create mode 100644 packages/session-query/session-query/src/sources.ts create mode 100644 packages/session-query/session-query/tests/search-helpers.spec.ts diff --git a/docs/architecture.md b/docs/architecture.md index 8dc7f6f4b1..e7b6f25ac3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -35,6 +35,7 @@ A harness is one [Cordis](cordis-primer.md) context. Packages contribute service | `ctx.workflows` | [`workflow/`](../packages/workflow/README.md) | script-driven multi-agent orchestration | | `ctx.sessionPersistence` | [`session-persistence/`](../packages/session-persistence/README.md) | durable storage for session logs | | `ctx.sessionQuery` | [`session-query/`](../packages/session-query/README.md) | live-preferred logical-corpus and exact-event reads | +| `ctx.sessionSearch` | [`session-query/`](../packages/session-query/README.md) | SQLite full-text search | ## Event diff --git a/docs/capability-seams.md b/docs/capability-seams.md index 13af1cde1c..189b910149 100644 --- a/docs/capability-seams.md +++ b/docs/capability-seams.md @@ -19,6 +19,7 @@ flowchart LR pkg_agent["agent"] pkg_session_persistence["session-persistence"] pkg_session_query["session-query"] + pkg_session_query_sqlite["session-query-sqlite"] pkg_subagent_inprocess["subagent-inprocess"] pkg_invariants["invariants"] svc_sessionPersistence["ctx.sessionPersistence
Durable session persistence seam"] @@ -26,6 +27,7 @@ flowchart LR pkg_session_persistence_sqlite["session-persistence-sqlite"] pkg_acp["acp"] svc_sessionQuery["ctx.sessionQuery
Exact session-history reads"] + svc_sessionSearch["ctx.sessionSearch
Full-text session search"] pkg_system_prompt["system-prompt"] svc_systemPrompt["ctx.systemPrompt
System prompt assembly registry"] pkg_tools["tools"] @@ -108,6 +110,8 @@ flowchart LR pkg_session_persistence_jsonl --> svc_sessionPersistence pkg_session_persistence_sqlite --> svc_sessionPersistence pkg_session_query --> svc_sessionQuery + pkg_session_query --> svc_sessionSearch + pkg_session_query_sqlite --> svc_sessionSearch pkg_skill --> svc_skills pkg_skill_local --> svc_skills pkg_stdio_agent --> svc_userInteraction @@ -146,11 +150,13 @@ flowchart LR svc_sessionPersistence --> pkg_acp svc_sessionPersistence --> pkg_agent_loop svc_sessionPersistence --> pkg_session_query + svc_sessionPersistence --> pkg_session_query_sqlite svc_sessions --> pkg_agent svc_sessions --> pkg_agent_loop svc_sessions --> pkg_invariants svc_sessions --> pkg_session_persistence svc_sessions --> pkg_session_query + svc_sessions --> pkg_session_query_sqlite svc_sessions --> pkg_subagent_inprocess svc_skills --> pkg_tool_skill svc_subagents --> pkg_tool_subagent @@ -179,9 +185,10 @@ flowchart LR | ctx key | Role | Owner | Implementations | Direct consumers | Companion plugins | Note | | --- | --- | --- | --- | --- | --- | --- | | `ctx.llm` | `seam` | [`llm`](../packages/llm/llm) | [`llm-deepseek`](../packages/llm/llm-deepseek), [`llm-pi-ai`](../packages/llm/llm-pi-ai), [`llm-replay`](../packages/support/llm-replay) | [`agent-loop`](../packages/core/agent-loop), [`compact-basic`](../packages/compact/compact-basic) | - | Adapters register provider implementations; the loop and compaction call the provider-neutral stream service. | -| `ctx.sessions` | `core` | [`session`](../packages/core/session) | - | [`agent-loop`](../packages/core/agent-loop), [`agent`](../packages/core/agent), [`session-persistence`](../packages/session-persistence/session-persistence), [`session-query`](../packages/session-query/session-query), [`subagent-inprocess`](../packages/subagent/subagent-inprocess), [`invariants`](../packages/support/invariants) | - | Owns append-only Session instances and emits the durable session event feed. | -| `ctx.sessionPersistence` | `seam` | [`session-persistence`](../packages/session-persistence/session-persistence) | [`session-persistence-jsonl`](../packages/session-persistence/session-persistence-jsonl), [`session-persistence-sqlite`](../packages/session-persistence/session-persistence-sqlite) | [`agent-loop`](../packages/core/agent-loop), [`acp`](../packages/ui/acp), [`session-query`](../packages/session-query/session-query) | - | Backends persist the same SessionEvent vocabulary; apps choose a backend at composition time. | -| `ctx.sessionQuery` | `seam` | [`session-query`](../packages/session-query/session-query) | - | - | - | Resolves live and optional persisted logs into one logical corpus for exact reads. | +| `ctx.sessions` | `core` | [`session`](../packages/core/session) | - | [`agent-loop`](../packages/core/agent-loop), [`agent`](../packages/core/agent), [`session-persistence`](../packages/session-persistence/session-persistence), [`session-query`](../packages/session-query/session-query), [`session-query-sqlite`](../packages/session-query/session-query-sqlite), [`subagent-inprocess`](../packages/subagent/subagent-inprocess), [`invariants`](../packages/support/invariants) | - | Owns append-only Session instances and emits the durable session event feed. | +| `ctx.sessionPersistence` | `seam` | [`session-persistence`](../packages/session-persistence/session-persistence) | [`session-persistence-jsonl`](../packages/session-persistence/session-persistence-jsonl), [`session-persistence-sqlite`](../packages/session-persistence/session-persistence-sqlite) | [`agent-loop`](../packages/core/agent-loop), [`acp`](../packages/ui/acp), [`session-query`](../packages/session-query/session-query), [`session-query-sqlite`](../packages/session-query/session-query-sqlite) | - | Backends persist the same SessionEvent vocabulary; apps choose a backend at composition time. | +| `ctx.sessionQuery` | `seam` | [`session-query`](../packages/session-query/session-query) | - | - | - | Resolves live and optional persisted logs into one logical corpus for exact reads and semantic scans. | +| `ctx.sessionSearch` | `seam` | [`session-query`](../packages/session-query/session-query) | [`session-query-sqlite`](../packages/session-query/session-query-sqlite) | - | - | The concrete backend owns source reconciliation, ranking, snippets, and cursor generations as one lifecycle. | | `ctx.systemPrompt` | `core` | [`system-prompt`](../packages/core/system-prompt) | - | [`agent-loop`](../packages/core/agent-loop), [`tools`](../packages/core/tools), [`tool-fs`](../packages/fs/tool-fs), [`tool-web`](../packages/web/tool-web) | - | Collects prompt sections and model-facing tool schemas for each step. | | `ctx.tools` | `core` | [`tools`](../packages/core/tools) | - | [`agent-loop`](../packages/core/agent-loop), [`tool-ask-user`](../packages/ui/tool-ask-user), [`tool-bash`](../packages/bash/tool-bash), [`tool-cordis`](../packages/cordis/tool-cordis), [`tool-fs`](../packages/fs/tool-fs), [`tool-skill`](../packages/skill/tool-skill), [`tool-subagent`](../packages/subagent/tool-subagent), [`tool-todo`](../packages/todo/tool-todo), [`tool-web`](../packages/web/tool-web), [`acp`](../packages/ui/acp) | - | Registers capabilities, owns Code Mode transport, and routes calls through pre-policy, monotonic guards, around dispatch, post-policy, and final-result observation. | | `ctx.userInteraction` | `seam` | [`user-interaction`](../packages/ui/user-interaction) | [`stdio-agent`](../packages/ui/stdio-agent), [`acp`](../packages/ui/acp) | [`tool-ask-user`](../packages/ui/tool-ask-user), [`stdio-agent`](../packages/ui/stdio-agent), [`acp`](../packages/ui/acp) | - | UI front doors provide the active human-answer provider; tool-ask-user pauses a tool call on the provider-neutral ask() promise. | diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 24aee5adc2..3a802231d7 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -590,6 +590,31 @@ export interface Config { Source: [`packages/session-query/session-query/src/config.ts:9`](../packages/session-query/session-query/src/config.ts) +## `@deepseek-ai/dsh-session-query-sqlite` + +Requires: `sessions` + +```ts config-catalog +/** SQLite session-search configuration. */ +export interface Config { + /** Dedicated derived-index path; `:memory:` is supported for tests. */ + path: string + /** SQLite journal mode. Defaults to `wal`. */ + journalMode?: JournalMode + /** Page size when a request omits `limit`. Defaults to 20. */ + defaultLimit?: number + /** Largest accepted page size. Defaults to 100. */ + maxLimit?: number + /** Maximum snippet length in Unicode code points. Defaults to 240. */ + snippetChars?: number +} + +/** Supported SQLite journal modes. */ +export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' +``` + +Source: [`packages/session-query/session-query-sqlite/src/index.ts:58`](../packages/session-query/session-query-sqlite/src/index.ts) + ## `@deepseek-ai/dsh-skill` ```ts config-catalog diff --git a/docs/cordis-catalog/services.md b/docs/cordis-catalog/services.md index 6f8508f5ee..fef691aae1 100644 --- a/docs/cordis-catalog/services.md +++ b/docs/cordis-catalog/services.md @@ -208,10 +208,11 @@ Live-preferred logical-corpus and exact-event read service. ```ts cordis-catalog listSessions(): Promise async listEvents(sessionId: SessionId): Promise +async filterEvents( sessionId: SessionId, filters: readonly SessionEventResultFilter[], ): Promise async readEvent(request: SessionEventReadRequest): Promise ``` -Source: [`packages/session-query/session-query/src/index.ts:35`](../../packages/session-query/session-query/src/index.ts) +Source: [`packages/session-query/session-query/src/index.ts:83`](../../packages/session-query/session-query/src/index.ts) ## `ctx.sessions` — `SessionStore` @@ -232,6 +233,19 @@ fork(source: SessionForkSource, boundary?: number, childSessionId?: SessionId): Source: [`packages/core/session/src/index.ts:590`](../../packages/core/session/src/index.ts) +## `ctx.sessionSearch` — `SessionSearchService` (abstract seam) + +Abstract full-text search service implemented by one concrete backend. + +The implementation owns source observation, reconciliation, cursor generations, ranking, and query execution as one lifecycle. + +```ts cordis-catalog +abstract searchSessions( request: SessionSearchRequest, exec?: SessionSearchExecContext, ): Promise> +abstract searchEvents( request: SessionEventSearchRequest, exec?: SessionSearchExecContext, ): Promise> +``` + +Source: [`packages/session-query/session-query/src/index.ts:54`](../../packages/session-query/session-query/src/index.ts) + ## `ctx.skills` — `SkillService` Registry of skill providers. It merges provider catalogs with stable first-wins duplicate handling, exposes sorted model-visible summaries, and loads full skill bodies on demand. diff --git a/docs/core-data-structures/core.md b/docs/core-data-structures/core.md index 5536526aec..e632b36f4d 100644 --- a/docs/core-data-structures/core.md +++ b/docs/core-data-structures/core.md @@ -19,7 +19,7 @@ Everything else is documented on a **sub-page**, not here. The rule that draws t | [scope.md](scope.md) | scoped registration identity, dispatch carriers, and the owned `Scope` context | | [session.md](session.md) | the full `SessionEventMap` variant catalog, `TurnTrigger`/`TurnEndReason`, `deriveMessages()`, the turn-enclosure invariant | | [persistence.md](persistence.md) | the durability seam: `SessionPersistence`, JSONL + SQLite backends, `session/flush`, crash recovery, `SessionHeader` | -| [session-query.md](session-query.md) | logical session/event records and bounded exact-event reads | +| [session-query.md](session-query.md) | logical records, semantic filters/documents, exact reads, and full-text result pages | | [system-prompt.md](system-prompt.md) | per-assembly context, tool-provider results, prompt sections, and cooperative assembly | | [tools.md](tools.md) | `ToolDefinition` full fields, the schema DSL, `ToolExecution`/`ToolResult`, tool-presentation UI types, and the guarded execution pipeline | | [user-interaction.md](user-interaction.md) | the UI-backed human question/answer seam: `AskUserQuestionRequest`, answer/options vocabulary, provider API, error taxonomy | diff --git a/docs/core-data-structures/session-query.md b/docs/core-data-structures/session-query.md index ded8ca3f7e..27ef9959de 100644 --- a/docs/core-data-structures/session-query.md +++ b/docs/core-data-structures/session-query.md @@ -1,6 +1,6 @@ # Session Query -Exact reads over the live-preferred logical session corpus. The [package contract](../../packages/session-query/session-query) owns source precedence, dynamic optional persistence, cloning, surface classification, bounded windows, and typed failures. Full-text search is a separate proposed SQLite phase. +Query vocabulary over the live-preferred logical session corpus. The [interface package](../../packages/session-query/session-query) owns exact reads, source precedence, semantic extraction and provider-independent filters, while the [SQLite package](../../packages/session-query/session-query-sqlite) owns the concrete full-text index lifecycle. Source: [`packages/session-query/session-query/src/types.ts`](../../packages/session-query/session-query/src/types.ts) @@ -30,6 +30,79 @@ export interface SessionEventRecord { } ``` +## Provider-independent filters and documents + +Session and event filter arrays are ANDed; values inside one list clause are ORed. Ranges are inclusive. The event `text` clause is a literal Unicode case-insensitive, whitespace-flexible regular-expression scan over extracted semantic text, independent of full-text providers. + +```ts type-equiv +export type SessionResultFilter = + | { kind: 'id'; values: readonly SessionId[] } + | { kind: 'cwd'; values: readonly (string | null)[] } + | ({ kind: 'created-at' } & SessionResultRange) + | { kind: 'parent'; values: readonly (SessionId | null)[] } + | { kind: 'availability'; values: readonly SessionAvailability[] } +``` + +```ts type-equiv +export type SessionEventResultFilter = + | ({ kind: 'seq' } & SessionResultRange) + | ({ kind: 'time' } & SessionResultRange) + | { kind: 'type'; values: readonly SessionEventType[] } + | { kind: 'surface'; values: readonly SessionEventSurface[] } + | { kind: 'text'; text: string } +``` + +```ts type-equiv +export interface SessionEventSearchDocument extends SessionEventRecord { + text: string +} +``` + +`ctx.sessionQuery.filterEvents(sessionId, filters)` returns these documents in ascending seq order. Messages, reasoning, tool calls/results, blocked prompts, todos, and failure/status detail contribute semantic text; structural events and stream chunks do not. + +## Full-text search pages + +The independent `ctx.sessionSearch` seam has two scopes. `searchSessions()` groups the corpus by strongest matching event; `searchEvents()` searches one session. Requests bind an opaque cursor to the normalized query, metadata filters, and limit. The event text scan is intentionally absent from provider metadata filters. + +```ts type-equiv +export interface SessionSearchRequest { + query: string + sessionFilters?: readonly SessionResultFilter[] + eventFilters?: readonly SessionEventMetadataFilter[] + limit?: number + cursor?: string +} +``` + +```ts type-equiv +export interface SessionEventSearchRequest { + sessionId: SessionId + query: string + filters?: readonly SessionEventMetadataFilter[] + limit?: number + cursor?: string +} +``` + +```ts type-equiv +export interface SessionSearchPage { + items: readonly T[] + nextCursor?: string +} +``` + +```ts type-equiv +export interface SessionEventSearchHit extends SessionEventRecord { + snippet: string +} +``` + +```ts type-equiv +export interface SessionSearchHit extends SessionRecord { + bestMatch: SessionEventSearchHit +} +``` + ## Bounded event reads The request addresses one raw seq and optional neighboring counts. The result carries a `SessionHeader` rather than availability flags so a known live target can remain independent of persistence health. @@ -59,11 +132,18 @@ The closed code union distinguishes request validation, missing targets, malform ```ts type-equiv export type SessionQueryErrorCode = + | 'SESSION_QUERY_ABORTED' | 'SESSION_QUERY_EVENT_NOT_FOUND' + | 'SESSION_QUERY_INDEX_FAILED' | 'SESSION_QUERY_INVALID_CONFIG' + | 'SESSION_QUERY_INVALID_CURSOR' + | 'SESSION_QUERY_INVALID_FILTER' + | 'SESSION_QUERY_INVALID_LIMIT' + | 'SESSION_QUERY_INVALID_QUERY' | 'SESSION_QUERY_INVALID_SURFACE' | 'SESSION_QUERY_INVALID_WINDOW' | 'SESSION_QUERY_PERSISTENCE_FAILED' | 'SESSION_QUERY_SESSION_NOT_FOUND' + | 'SESSION_QUERY_STALE_CURSOR' | 'SESSION_QUERY_SOURCE_CONFLICT' ``` diff --git a/docs/module-graph.md b/docs/module-graph.md index 8a1ad9239d..6fcec53dd4 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -84,6 +84,7 @@ flowchart TD end subgraph group_session_query["packages/session-query"] pkg_session_query["session-query"] + pkg_session_query_sqlite["session-query-sqlite"] end subgraph group_support["packages/support"] pkg_acp_snapshot["acp-snapshot"] @@ -195,6 +196,9 @@ flowchart TD pkg_bash_sandbox --> pkg_bash pkg_bash_sandbox --> pkg_bash_local pkg_bash_sandbox --> pkg_sandbox + pkg_session_query_sqlite --> pkg_session + pkg_session_query_sqlite --> pkg_session_persistence + pkg_session_query_sqlite --> pkg_session_query pkg_agent_loop --> pkg_agent pkg_agent_loop --> pkg_llm pkg_agent_loop --> pkg_scope @@ -372,6 +376,7 @@ flowchart TD | [`workflow`](../packages/workflow/workflow) | `workflow` | [`agent`](../packages/core/agent), [`brand`](../packages/util/brand), [`llm`](../packages/llm/llm) | | [`tools`](../packages/core/tools) | `core` | [`agent`](../packages/core/agent), [`code-runtime`](../packages/code-runtime/code-runtime), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt), [`user-approval`](../packages/ui/user-approval) | | [`bash-sandbox`](../packages/bash/bash-sandbox) | `bash` | [`bash`](../packages/bash/bash), [`bash-local`](../packages/bash/bash-local), [`sandbox`](../packages/sandbox/sandbox) | +| [`session-query-sqlite`](../packages/session-query/session-query-sqlite) | `session-query` | [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence), [`session-query`](../packages/session-query/session-query) | | [`agent-loop`](../packages/core/agent-loop) | `core` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | | [`tool-bash`](../packages/bash/tool-bash) | `bash` | [`agent`](../packages/core/agent), [`bash`](../packages/bash/bash), [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-approval`](../packages/ui/user-approval) | | [`tool-fs`](../packages/fs/tool-fs) | `fs` | [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | diff --git a/docs/rfc/INDEX.md b/docs/rfc/INDEX.md index 54e59f7aa1..a3eb41b7b0 100644 --- a/docs/rfc/INDEX.md +++ b/docs/rfc/INDEX.md @@ -11,7 +11,6 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand; | [Pre-tool input rewrite — a consistent design](proposed/feature/2026-06-30-pre-tool-input-rewrite.md) | 2026-06-30 | | [Claude Code and Codex subagent backends (out-of-process delegation to external coding agents)](proposed/feature/2026-07-07-claude-code-and-codex-subagent-backends.md) | 2026-07-07 | | [Interactive side sessions and merge-back](proposed/feature/2026-07-08-interactive-side-sessions.md) | 2026-07-08 | -| [SQLite FTS5 session search](proposed/feature/2026-07-10-sqlite-session-query-provider.md) | 2026-07-10 | | [Stream workflow progress through tool calls](proposed/feature/2026-07-13-stream-workflow-progress-through-tool-calls.md) | 2026-07-13 | ### Simplification @@ -77,6 +76,7 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand; | [Repeat-tool-call guard plugin](implemented/feature/2026-07-08-repeat-tool-guard.md) | 2026-07-08 | | [The self-referential cordis toolset](implemented/feature/2026-07-08-self-referential-cordis-toolset.md) | 2026-07-08 | | [Exact session query service](implemented/feature/2026-07-10-session-query-service.md) | 2026-07-10 | +| [SQLite FTS5 session search](implemented/feature/2026-07-10-sqlite-session-query-provider.md) | 2026-07-10 | | [Configure subagent persona, tool visibility, and depth](implemented/feature/2026-07-12-subagent-persona-tool-filter-and-depth.md) | 2026-07-12 | ### Simplification diff --git a/docs/rfc/implemented/feature/2026-07-10-session-query-service.md b/docs/rfc/implemented/feature/2026-07-10-session-query-service.md index 7e13256669..be496311cc 100644 --- a/docs/rfc/implemented/feature/2026-07-10-session-query-service.md +++ b/docs/rfc/implemented/feature/2026-07-10-session-query-service.md @@ -6,11 +6,11 @@ Status: implemented Session history exists in two places: current `SessionStore` objects and an optional persistence backend. Consumers that need exact inspection would otherwise duplicate live-versus-persisted precedence, persistence lifecycle handling, raw-event surface classification, and defensive cloning. Durable state can lag the live log between checkpoints, so persistence alone is not a truthful current source. -Full-text search is related but materially larger. Designing provider registration, extraction, synchronization, invalidation, ranking, and cursor contracts before a real backend exists creates two speculative state machines: one in the interface service and another in the eventual database package. +Full-text search is related but materially larger. Putting provider coordination, synchronization, invalidation, ranking, and cursor state into the exact-read service would create a second state machine beside the concrete database owner. ## Decision -`@deepseek-ai/dsh-session-query` owns `ctx.sessionQuery`, a small trusted exact-read service over one logical corpus. It exposes `listSessions()`, `listEvents(sessionId)`, and bounded `readEvent(request)`. It does not expose filters, lineage or provenance traversals, text extractors, search requests, provider registration, or derived-index synchronization. +`@deepseek-ai/dsh-session-query` owns `ctx.sessionQuery`, a small trusted exact-read service over one logical corpus. It exposes `listSessions()`, `listEvents(sessionId)`, provider-independent `filterEvents(sessionId, filters)`, and bounded `readEvent(request)`. The package also declares the separate abstract `ctx.sessionSearch` contract and shared semantic extraction used by the [SQLite search decision](2026-07-10-sqlite-session-query-provider.md), but `ctx.sessionQuery` does not coordinate providers or synchronize a derived index. The service observes the optional `ctx.sessionPersistence` binding dynamically but retains no persisted cache or invalidation listener. Each cross-corpus list asks the active backend for authoritative metadata, then overlays a fresh live-store list. Matching ids become one `SessionRecord`: the live header wins and `live`/`persisted` independently report source availability. Immutable header disagreement is `SESSION_QUERY_SOURCE_CONFLICT`. @@ -31,11 +31,11 @@ The service is context-wide trusted infrastructure, not an authorization layer. - **Put logical-corpus resolution directly in every consumer** — rejected because source precedence, conflicts, optional-service lifecycle, cloning, and surface classification are shared correctness rules. - **Query only persistence** — rejected because checkpoints can lag the current live log. - **Cache persisted metadata and listen for writes/removals** — rejected because exact reads can ask the authoritative sources directly, while cache invalidation adds lifecycle and concurrency state before scale requires it. -- **Define a provider-neutral search protocol now** — rejected because no provider consumes it. The first SQLite FTS package should own one reconciliation/transaction state machine; a smaller shared seam can be extracted later only when a second implementation proves the boundary. -- **Include lineage, provenance, and generic filters in phase one** — rejected because no current consumer requires them and canonical logs remain sufficient to add them with evidence later. +- **Put provider registration into the exact-read service** — rejected because the SQLite package owns one reconciliation/transaction lifecycle; a registry would split that state without a second provider to justify it. +- **Include lineage and provenance traversal** — rejected because canonical logs remain sufficient to add those higher-level views when a concrete consumer requires them. ## Consequences -Phase one has one source-resolution state variable: the currently mounted persistence service. There are no provider queues, fingerprints, extractor registries, observation generations, or derived index updates. Exact reads remain usable in live-only deployments and deterministic when persistence is present. +Exact reads have one source-resolution state variable: the currently mounted persistence service. There are no provider queues, fingerprints, extractor registries, observation generations, or derived index updates in `ctx.sessionQuery`. Exact reads and semantic scans remain usable in live-only deployments and deterministic when persistence is present. -Cross-corpus listing and persisted exact reads perform backend I/O on each call. That is deliberate: correctness comes from current authoritative state, and scale-oriented search belongs to the phase-two database. Full-text search is unavailable until that package defines and implements its complete contract. +Cross-corpus listing and persisted exact reads perform backend I/O on each call. That is deliberate: correctness comes from current authoritative state, while scale-oriented full-text search uses the separately owned SQLite derived index. diff --git a/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md b/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md new file mode 100644 index 0000000000..d620ece108 --- /dev/null +++ b/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md @@ -0,0 +1,57 @@ +# RFC: SQLite FTS5 session search + +Status: implemented + +## Problem + +The exact-read `ctx.sessionQuery` service deliberately has no derived index. Large persisted histories need full-text search without scanning every event on every query, while current live sessions need an overlay newer than the last durability checkpoint. Search also needs concrete ranking, snippets, filters, pagination, cancellation, and rebuild behavior. + +Splitting those concerns across a provider coordinator and a database implementation would create two coupled reconciliation state machines. The first implementation needs to own source observation, extraction, SQLite transactions, generations, and query execution as one lifecycle while still exposing a small provider-neutral call contract. + +## Decision + +`@deepseek-ai/dsh-session-query` declares an independent abstract `ctx.sessionSearch` service without changing the exact-read `ctx.sessionQuery` key. `searchSessions(request, exec?)` returns cursor-paginated `SessionSearchHit`s grouped by each session's strongest matching event; `searchEvents(request, exec?)` returns `SessionEventSearchHit`s within one logical session. Both requests require `query`, accept `limit` and an opaque `cursor`, and support an optional abort signal. Session search accepts `sessionFilters` plus event metadata filters; event search accepts event metadata filters. Results expose bounded plain-text snippets but no provider identifier or numeric relevance score. + +`@deepseek-ai/dsh-session-query-sqlite` is the sole concrete owner of `ctx.sessionSearch`. It depends on live `ctx.sessions`, observes optional `ctx.sessionPersistence` dynamically, and owns a dedicated derived SQLite database. There is no search-provider registry, coordinator, persistence event, or agent-loop integration. + +The interface package also owns shared first-party semantic extraction and provider-independent filtering. `SessionResultFilter` covers id, nullable cwd, created-at range, nullable parent, and availability. `SessionEventResultFilter` covers seq/time ranges, event type, surface, and literal semantic text. Arrays are ANDed and list values are ORed. The text clause escapes caller input into a Unicode case-insensitive regular expression whose whitespace runs match one or more whitespace characters; it is available through `ctx.sessionQuery.filterEvents()` and is not delegated to an FTS provider. + +## Search semantics + +Each semantic event is one FTS document carrying session metadata, event metadata, surface classification, and extracted text. All `current`, `shadowed`, and `log-only` documents participate unless a surface filter narrows them. Metadata filters compile to parameterized SQL before ranking. Session results partition matching documents by session and retain the strongest one. + +Ordering is deterministic: BM25 ascending, event time descending, session id ascending for the cross-session scope, and seq descending. Provider scores remain private. Snippets normalize whitespace and are bounded by Unicode code points. Opaque cursors bind to the service instance, scope, canonical normalized request, offset, and relevant generation. Any corpus change invalidates cross-session cursors; a within-session cursor changes only when its target source/generation changes, so unrelated sessions do not invalidate it. Reopening creates a new service instance and invalidates old cursors. + +Queries are trimmed, whitespace-normalized, and quoted as one literal FTS5 phrase. Embedded quotes are doubled before binding, so MATCH operators such as `OR`, `NEAR`, quotes, parentheses, and `*` remain data rather than executable query syntax. Phrase matching follows tokenizer tokens rather than arbitrary substrings. + +## Tokenizer choice + +Both persistent and live FTS5 tables use `unicode61`. The implementation experiment found that this tokenizer supports the two-character token `AI` and produces an index about 2.1× smaller than the trigram alternative. The accepted limitation is token/phrase recall: `AI` does not match the larger token `BRAID`, and arbitrary substring search uses the provider-independent text scan instead. + +## Extraction and reconciliation + +The shared extractor includes message text, reasoning, nested tool-call/result content, tool names and arguments, blocked-prompt reasons, todo status/content, and error or terminal status detail. Structural boundaries, stream chunks, request headers, successful completion markers, and unknown declaration-merged event/content variants produce no document. Surface classification reuses `foldSurface()` so search agrees with model-history derivation. + +One serialized operation observes complete persisted and live sources, computes stable fingerprints, reconciles rows in one transaction, and executes the query. Unchanged persisted sessions retain their rows and generation. New, changed, and deleted persisted sessions update on the next search. A source or extraction failure cannot mark a row current, and a transaction failure rolls back so a later search retries. + +Persisted documents survive restarts. Live sessions use connection-local TEMP tables, shadow the persisted base for the same id, and reveal that base on detach. Closing the database drops live rows. Unmounting persistence hides durable rows without treating absence as authoritative deletion; remounting observes and reconciles the backend again. Conflicting immutable live and durable headers fail rather than combining sources. + +The derived schema has its own application id and monotonic schema version. A recognized incompatible version resets only this derived database. A database with a foreign application id or unrecognized user tables is refused, which prevents an accidentally configured canonical session database from being reset. + +Cancellation rejects queued operations and caller waits around asynchronous source observation without committing an aborted observation. Node's synchronous `DatabaseSync` MATCH call cannot be interrupted once it is executing on the JavaScript thread, so the service checks the signal at serialized boundaries but does not promise mid-statement preemption. + +## Alternatives considered + +- **Add FTS tables to the canonical persistence database** — rejected because a rebuildable index must not share the authoritative log's schema, reset, or failure boundary. +- **Add a phase-one provider registry and coordinator** — rejected because one implementation provides no evidence for registration semantics and would split one reconciliation lifecycle across two owners. +- **Persist live overrides immediately** — rejected because live events are not canonical until the existing checkpoint commits. +- **Use the FTS5 trigram tokenizer** — rejected because it omits useful queries shorter than three characters and measured about 2.1× the index size of `unicode61`; literal substring filtering remains available through the scan path. +- **Return BM25 scores** — rejected because provider-specific numeric scales are unstable across corpus changes. + +## Consequences + +Search has a small provider-neutral API while its only backend owns every derived-index state transition. The separate database adds configuration and a reconciliation read before queries, but index corruption, reset, and tokenizer changes cannot endanger canonical logs. Persistent fingerprints avoid rewriting unchanged sessions; TEMP live overlays preserve current-session truth without making uncheckpointed events durable. + +The chosen tokenizer supports short tokens with a smaller index but does not promise substring recall. Literal phrases make query syntax safe and predictable at the cost of excluding boolean/full MATCH expressions. Cancellation is effective while queued or awaiting sources, but synchronous SQLite execution remains a non-preemptible section. + +Unit coverage pins extraction, filters, both search scopes, all default surfaces, metadata-before-ranking, snippets, literal escaping, deterministic ties, complete pagination, scoped cursor invalidation, dynamic persistence mount/unmount, restart reconciliation, live shadow/reveal/reopen, schema safety, rollback retry, and queued/in-flight source-wait cancellation. A keyless real-Loader-path test combines the package with the real SQLite persistence backend. diff --git a/docs/rfc/proposed/feature/2026-07-10-sqlite-session-query-provider.md b/docs/rfc/proposed/feature/2026-07-10-sqlite-session-query-provider.md deleted file mode 100644 index acfdf23bee..0000000000 --- a/docs/rfc/proposed/feature/2026-07-10-sqlite-session-query-provider.md +++ /dev/null @@ -1,51 +0,0 @@ -# RFC: SQLite FTS5 session search - -Status: proposed - -## Problem - -The exact-read `ctx.sessionQuery` service deliberately has no derived index. Large persisted histories need full-text search without scanning every event on every query, while current live sessions need an overlay newer than the last durability checkpoint. Search also needs concrete ranking, snippets, filters, pagination, cancellation, and rebuild behavior. - -Splitting those concerns across a speculative provider coordinator and a database implementation would create two coupled reconciliation state machines. The first real implementation should own the source observation, extraction, SQLite transaction, generation, and query as one lifecycle. - -## Proposal - -Add `@deepseek-ai/dsh-session-query-sqlite` beside the exact-read package. The package will expose a search service or extend the family with the smallest API required by its actual consumers; phase one does not pre-commit a provider-registration protocol. It will depend on `ctx.sessions` and optional `ctx.sessionPersistence`, own a separate derived SQLite database, and reuse the canonical `foldSurface()` classification. - -The implementation owns one serialized reconciliation/DB transaction state machine. A transaction observes authoritative persisted metadata and live snapshots, extracts semantic documents, updates derived tables, advances relevant cursor generations, and executes or enables the corresponding query. No second service maintains parallel fingerprints, dirty flags, live-id sets, or invalidation generations. - -Persisted documents survive restarts. Live overrides are connection-local and shadow the persisted rows for the same session, then disappear when the live owner or database closes. The derived database remains separate from canonical persistence so index reset, corruption, tokenizer changes, and schema churn cannot endanger durable conversation logs. - -## Search semantics to decide with implementation - -The implementation must define both cross-session and within-session scopes from executable use cases. Each searchable event is one document with session metadata, event metadata, surface classification, normalized semantic text, and a bounded plain-text snippet. Session results group by their strongest matching event; numeric backend scores remain private. - -Filters compile to parameterized SQL before ranking. Query syntax is treated as data. Ordering includes stable tie fields. Opaque cursors bind to normalized request shape and the smallest relevant generation; unrelated session changes should not invalidate a within-session cursor. Cancellation must stop caller waiting and interrupt SQLite work where the runtime permits. - -Tokenizer choice remains an implementation experiment. FTS5 trigram supports substring recall but rejects useful terms shorter than three characters and increases index size; the proposal must benchmark that tradeoff against the default Unicode tokenizer before making it contract. - -## Extraction and reconciliation - -The package starts with first-party semantic extraction for messages, reasoning, tool calls/results, blocked prompts, context, steering, todos, and error/status detail. Structural events and stream chunks contribute no document. Unknown declaration-merged event/content types remain non-searchable unless a real extension consumer demonstrates the need for a public extractor registry. - -Reconciliation may use stable fingerprints to avoid rewriting unchanged persisted sessions, but the database package owns their calculation and storage. It must never report a row current when source observation or extraction failed. Provider-schema mismatch may reset only the derived database; ordinary source changes use transactional upsert/delete. Mounted but unreadable persistence fails affected searches without affecting canonical writes or known live exact reads. - -## Alternatives considered - -- **Add FTS tables to the canonical persistence database** — rejected because a rebuildable index must not share the authoritative log's schema/reset/failure boundary. -- **Reintroduce phase-one provider coordination** — rejected because there is one planned implementation and no evidence for a stable multi-provider seam. -- **Persist live overrides immediately** — rejected because live events are not canonical until the existing checkpoint commits. -- **Return BM25 scores** — rejected because provider-specific numeric scales are unstable across corpus changes. - -## Acceptance criteria - -- Restart tests cover unchanged, new, changed, and deleted persisted sessions without rebuilding the whole index. -- Reopening preserves persisted rows and removes live rows; live rows shadow and then reveal their persisted base. -- Tests cover both search scopes, metadata filters, surface defaults, snippets, escaping, deterministic ties, pagination, scoped stale cursors, cancellation, dynamic persistence mount/unmount, and recovery after a failed transaction. -- A schema mismatch resets only the derived database. -- A keyless end-to-end test combines a real persistence backend with the real SQLite search package. -- The RFC is amended to the measured tokenizer and public API actually implemented before moving to `implemented/`. - -## Risks - -A single owner is simpler but initially less reusable than a provider-neutral seam. That is intentional: a second real backend can reveal what to extract. SQLite runtime differences can affect FTS ranking and snippets, so tests must pin only contract-controlled ordering and presentation. The separate database adds configuration and lifecycle work, but preserves the canonical store's safety boundary. diff --git a/packages/README.md b/packages/README.md index d3dd6818df..f494b12222 100644 --- a/packages/README.md +++ b/packages/README.md @@ -25,7 +25,7 @@ Packages are grouped by modular role at `packages///`. The group dir | [`cordis/`](cordis/README.md) | Self-referential runtime toolset: inspect the live runtime's plugins and services, mount/unmount model-written plugins ([design](../docs/rfc/implemented/feature/2026-07-08-self-referential-cordis-toolset.md)) | Product — stable surface | | [`hooks/`](hooks/README.md) | Hook bridges + the shared Claude Code / Codex wire-protocol library | Product — stable surface | | [`session-persistence/`](session-persistence/README.md) | Persistence capability family: the seam + JSONL/SQLite backends | Product — stable surface | -| [`session-query/`](session-query/README.md) | Session retrieval family: logical corpus, surface records, and bounded exact reads | Product — stable surface | +| [`session-query/`](session-query/README.md) | Session retrieval family: exact reads, semantic filtering, and SQLite full-text search | Product — stable surface | | [`ui/`](ui/README.md) | Editor/client integration surfaces: ACP bridge, JSON-RPC SDK server, app packages, user-approval and user-interaction seams, ask-user tool | Product — stable surface | | [`support/`](support/README.md) | Dev/test/example infrastructure (invariants, replay adapter, subagent mock) | Support — lower compatibility expectations | | [`util/`](util/README.md) | Low-level zero-dependency utilities shared across groups (the `Branded` primitive) | Support — small, stable, harness-dep-free | diff --git a/packages/cordis/tool-cordis/src/api-catalog.ts b/packages/cordis/tool-cordis/src/api-catalog.ts index 06bf895e96..2859353cfa 100644 --- a/packages/cordis/tool-cordis/src/api-catalog.ts +++ b/packages/cordis/tool-cordis/src/api-catalog.ts @@ -157,6 +157,7 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ methods: [ 'listSessions(): Promise', 'async listEvents(sessionId: SessionId): Promise', + 'async filterEvents( sessionId: SessionId, filters: readonly SessionEventResultFilter[], ): Promise', 'async readEvent(request: SessionEventReadRequest): Promise', ], }, @@ -174,6 +175,14 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ 'fork(source: SessionForkSource, boundary?: number, childSessionId?: SessionId): Session', ], }, + { + key: 'sessionSearch', + summary: 'Abstract full-text search service implemented by one concrete backend.', + methods: [ + 'abstract searchSessions( request: SessionSearchRequest, exec?: SessionSearchExecContext, ): Promise>', + 'abstract searchEvents( request: SessionEventSearchRequest, exec?: SessionSearchExecContext, ): Promise>', + ], + }, { key: 'skills', summary: 'Registry of skill providers.', @@ -787,6 +796,10 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SendOptions', declaration: 'export interface SendOptions {\n source?: MessageSource;\n}', }, + { + name: 'SessionAvailability', + declaration: 'export type SessionAvailability = \'live\' | \'persisted\';', + }, { name: 'SessionEvent', declaration: 'export type SessionEvent = {\n [K in SessionEventType]: {\n type: K;\n seq: number;\n time: number;\n data: SessionEventMap[K];\n } & (K extends SurfaceEventType ? {\n sourceEventSeqs?: number[];\n surfaceOp?: SurfaceOp;\n } : object);\n}[T];', @@ -795,6 +808,10 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SessionEventMap', declaration: 'export interface SessionEventMap {\n \'turn/start\': {\n turn: number;\n trigger: TurnTrigger;\n };\n \'turn/end\': {\n turn: number;\n reason: TurnEndReason;\n };\n \'step/start\': {\n turn: number;\n step: number;\n };\n \'step/end\': {\n turn: number;\n step: number;\n };\n \'user/message\': {\n content: ContentBlock[];\n source: MessageSource;\n };\n \'prompt/blocked\': {\n content: ContentBlock[];\n source: MessageSource;\n reason: string;\n };\n \'context/message\': {\n content: ContentBlock[];\n source: MessageSource;\n };\n \'assistant/chunk\': {\n turn: number;\n step: number;\n chunk: StreamChunk;\n };\n \'assistant/message\': {\n turn: number;\n step: number;\n content: ContentBlock[];\n usage?: TokenUsage;\n };\n \'tool/call\': {\n turn: number;\n step: number;\n callId: CallId;\n name: string;\n arguments: string;\n };\n \'tool/result\': {\n turn: number;\n step: number;\n callId: CallId;\n content: ContentBlock[];\n isError: boolean;\n error?: {\n name: string;\n code: string;\n };\n meta?: unknown;\n };\n \'steering/message\': {\n turn: number;\n content: ContentBlock[];\n source: MessageSource;\n };\n \'todo/write\': {\n todos: TodoItem[];\n };\n \'request/header\': {\n header: E /* …truncated — full shape in source */', }, + { + name: 'SessionEventMetadataFilter', + declaration: 'export type SessionEventMetadataFilter = Exclude;', + }, { name: 'SessionEventReadRequest', declaration: 'export interface SessionEventReadRequest {\n sessionId: SessionId;\n seq: number;\n before?: number;\n after?: number;\n}', @@ -803,6 +820,22 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SessionEventRecord', declaration: 'export interface SessionEventRecord {\n sessionId: SessionId;\n seq: number;\n type: SessionEventType;\n time: number;\n surface: SessionEventSurface;\n}', }, + { + name: 'SessionEventResultFilter', + declaration: 'export type SessionEventResultFilter = ({\n kind: \'seq\';\n} & SessionResultRange) | ({\n kind: \'time\';\n} & SessionResultRange) | {\n kind: \'type\';\n values: readonly SessionEventType[];\n} | {\n kind: \'surface\';\n values: readonly SessionEventSurface[];\n} | {\n kind: \'text\';\n text: string;\n};', + }, + { + name: 'SessionEventSearchDocument', + declaration: 'export interface SessionEventSearchDocument extends SessionEventRecord {\n text: string;\n}', + }, + { + name: 'SessionEventSearchHit', + declaration: 'export interface SessionEventSearchHit extends SessionEventRecord {\n snippet: string;\n}', + }, + { + name: 'SessionEventSearchRequest', + declaration: 'export interface SessionEventSearchRequest {\n sessionId: SessionId;\n query: string;\n filters?: readonly SessionEventMetadataFilter[];\n limit?: number;\n cursor?: string;\n}', + }, { name: 'SessionEventSurface', declaration: 'export type SessionEventSurface = \'current\' | \'shadowed\' | \'log-only\';', @@ -831,6 +864,30 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SessionRecord', declaration: 'export interface SessionRecord {\n header: SessionHeader;\n live: boolean;\n persisted: boolean;\n}', }, + { + name: 'SessionResultFilter', + declaration: 'export type SessionResultFilter = {\n kind: \'id\';\n values: readonly SessionId[];\n} | {\n kind: \'cwd\';\n values: readonly (string | null)[];\n} | ({\n kind: \'created-at\';\n} & SessionResultRange) | {\n kind: \'parent\';\n values: readonly (SessionId | null)[];\n} | {\n kind: \'availability\';\n values: readonly SessionAvailability[];\n};', + }, + { + name: 'SessionResultRange', + declaration: 'export interface SessionResultRange {\n from?: number;\n to?: number;\n}', + }, + { + name: 'SessionSearchExecContext', + declaration: 'export interface SessionSearchExecContext {\n signal?: AbortSignal;\n}', + }, + { + name: 'SessionSearchHit', + declaration: 'export interface SessionSearchHit extends SessionRecord {\n bestMatch: SessionEventSearchHit;\n}', + }, + { + name: 'SessionSearchPage', + declaration: 'export interface SessionSearchPage {\n items: readonly T[];\n nextCursor?: string;\n}', + }, + { + name: 'SessionSearchRequest', + declaration: 'export interface SessionSearchRequest {\n query: string;\n sessionFilters?: readonly SessionResultFilter[];\n eventFilters?: readonly SessionEventMetadataFilter[];\n limit?: number;\n cursor?: string;\n}', + }, { name: 'SkillCandidate', declaration: 'export interface SkillCandidate extends SkillSummary {\n readonly rank: number;\n readonly locator: unknown;\n readonly path?: string;\n readonly metadata?: Readonly>;\n}', diff --git a/packages/session-query/README.md b/packages/session-query/README.md index 8b0c06a30c..0622858294 100644 --- a/packages/session-query/README.md +++ b/packages/session-query/README.md @@ -1,9 +1,10 @@ # session-query/ — session retrieval capability family -Trusted exact reads over live and durable session logs. Phase one contains one interface package that owns `ctx.sessionQuery`, logical-corpus precedence, surface classification, and bounded event reads. +Trusted exact reads, provider-independent semantic filtering, and SQLite full-text search over live and durable session logs. | Package | Role | ctx key | |---|---|---| -| [`session-query/`](session-query/README.md) | Logical-corpus and exact-event read service | `ctx.sessionQuery` | +| [`session-query/`](session-query/README.md) | Logical-corpus reads, semantic extraction/filtering, and the abstract search seam | `ctx.sessionQuery`, `ctx.sessionSearch` | +| [`session-query-sqlite/`](session-query-sqlite/README.md) | SQLite FTS5 search with persistent bases and live overlays | `ctx.sessionSearch` | -The family is independent of compaction: it reads the canonical session log but does not participate in compaction policy or execution. Full-text search remains proposed as a phase-two SQLite package rather than a speculative provider seam in this interface package. +The family is independent of compaction: it reads the canonical session log but does not participate in compaction policy or execution. Search uses one abstract service and one concrete owner, not a provider registry or coordinator. diff --git a/packages/session-query/session-query-sqlite/README.md b/packages/session-query/session-query-sqlite/README.md new file mode 100644 index 0000000000..1f3344887f --- /dev/null +++ b/packages/session-query/session-query-sqlite/README.md @@ -0,0 +1,35 @@ +# @deepseek-ai/dsh-session-query-sqlite + +SQLite FTS5 implementation of `ctx.sessionSearch`. The service searches the live-preferred logical session corpus, groups cross-session results by their strongest event, and keeps provider-specific BM25 scores private. + +## Search contract + +`searchSessions(request, exec?)` returns `SessionSearchHit` pages across the corpus; `searchEvents(request, exec?)` returns `SessionEventSearchHit` pages within one session. Queries are required, trimmed, whitespace-normalized literal phrases. FTS5 syntax such as quotes, `OR`, `NEAR`, and `*` is treated as data rather than executable MATCH syntax. Metadata filters are parameterized SQL predicates applied before ranking. + +Ordering is deterministic: relevance first, then event time, session id where applicable, and seq. Cross-session results expose the selected event as `bestMatch`; both scopes return plain-text snippets bounded in Unicode code points. Cursors are opaque, bind to the normalized request and service instance, and fail when the relevant generation changes. A within-session cursor survives unrelated-session changes; a cross-session cursor does not. + +All three surfaces (`current`, `shadowed`, and `log-only`) are searchable by default. Pass a surface filter to narrow them. + +## Source and index lifecycle + +The service requires `ctx.sessions` and observes optional `ctx.sessionPersistence` dynamically. One serialized state machine observes complete sources, extracts shared semantic documents, reconciles changes transactionally, and runs the query. Stable fingerprints preserve unchanged persisted rows and generations; new, changed, and deleted durable sessions reconcile on the next search. Source or transaction failure commits nothing, and the next search retries. + +Persisted FTS rows live in a dedicated derived database. Connection-local TEMP tables hold live rows, which shadow the durable base for the same session and reveal it when the live owner disappears. Unmounting persistence hides durable rows without discarding the cache; remounting reconciles it. Closing or reopening the database drops every live overlay while retaining persisted rows. + +The database is disposable but reset is guarded: a recognized incompatible search schema rebuilds in place, while an unrelated or canonical database is refused. Never point `path` at the session-persistence database. + +## Configuration + +| Key | Default | Contract | +|---|---:|---| +| `path` | required | Dedicated derived-index SQLite path; `:memory:` is supported. | +| `journalMode` | `wal` | `wal`, `delete`, `truncate`, or `persist`. | +| `defaultLimit` | `20` | Page size when a request omits `limit`. | +| `maxLimit` | `100` | Largest accepted request page size. | +| `snippetChars` | `240` | Maximum snippet length in Unicode code points. | + +## Tokenizer and limits + +The index uses FTS5 `unicode61`. In the implementation experiment it supported the two-character query `AI` and produced an index about 2.1× smaller than the trigram alternative. The trade-off is token/phrase recall rather than arbitrary substring recall: `AI` does not match the token `BRAID`. Use `ctx.sessionQuery.filterEvents()` with a `text` clause when a literal whitespace-flexible substring scan is required. + +Abort signals stop queued work and caller waits around asynchronous source observation. Node's synchronous `DatabaseSync` API cannot interrupt a MATCH statement already executing on the JavaScript thread; the signal is checked immediately before and after the serialized observation/reconciliation boundary. diff --git a/packages/session-query/session-query-sqlite/package.json b/packages/session-query/session-query-sqlite/package.json new file mode 100644 index 0000000000..de5677fd68 --- /dev/null +++ b/packages/session-query/session-query-sqlite/package.json @@ -0,0 +1,46 @@ +{ + "name": "@deepseek-ai/dsh-session-query-sqlite", + "description": "SQLite FTS5 implementation of ctx.sessionSearch", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "@deepseek-ai/dsh-session": "^0.0.1", + "@deepseek-ai/dsh-session-persistence": "^0.0.1", + "@deepseek-ai/dsh-session-query": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "peerDependenciesMeta": { + "@deepseek-ai/dsh-session-persistence": { + "optional": true + } + }, + "dependencies": { + "schemastery": "^3.18.0" + }, + "devDependencies": { + "@cordisjs/plugin-loader": "workspace:^", + "@deepseek-ai/dsh-session": "workspace:^", + "@deepseek-ai/dsh-session-persistence": "workspace:^", + "@deepseek-ai/dsh-session-persistence-sqlite": "workspace:^", + "@deepseek-ai/dsh-session-query": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/session-query/session-query-sqlite/src/index.ts b/packages/session-query/session-query-sqlite/src/index.ts new file mode 100644 index 0000000000..a6cba8d866 --- /dev/null +++ b/packages/session-query/session-query-sqlite/src/index.ts @@ -0,0 +1,765 @@ +/** + * SQLite FTS5 search over the live-preferred logical session corpus. + * + * @module @deepseek-ai/dsh-session-query-sqlite + */ + +import { createHash, randomUUID } from 'node:crypto' +import { DatabaseSync } from 'node:sqlite' +import { Context } from 'cordis' +import z from 'schemastery' +import type { Session, SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session' +import type SessionPersistence from '@deepseek-ai/dsh-session-persistence' +import { + SessionQueryError, + SessionSearchService, + assertSessionHeadersCompatible, + buildSessionEventSearchDocuments, +} from '@deepseek-ai/dsh-session-query' +import type { + SessionEventSearchDocument, + SessionEventSearchHit, + SessionEventSearchRequest, + SessionSearchExecContext, + SessionSearchHit, + SessionSearchPage, + SessionSearchRequest, +} from '@deepseek-ai/dsh-session-query' +import { + type JournalMode, + openSearchDatabase, +} from './schema.ts' +import { + type NormalizedEventRequest, + type NormalizedSessionRequest, + buildEventWhere, + buildSessionWhere, + makeSnippet, + normalizeEventRequest, + normalizeSessionRequest, + quoteFtsData, + requestFingerprint, +} from './query.ts' + +export { + SESSION_QUERY_SQLITE_APPLICATION_ID, + SESSION_QUERY_SQLITE_SCHEMA_VERSION, + type JournalMode, +} from './schema.ts' + +/** Default result page size. */ +export const SESSION_QUERY_SQLITE_DEFAULT_LIMIT = 20 +/** Maximum accepted result page size. */ +export const SESSION_QUERY_SQLITE_MAX_LIMIT = 100 +/** Default maximum snippet length in Unicode code points. */ +export const SESSION_QUERY_SQLITE_SNIPPET_CHARS = 240 + +/** SQLite session-search configuration. */ +export interface Config { + /** Dedicated derived-index path; `:memory:` is supported for tests. */ + path: string + /** SQLite journal mode. Defaults to `wal`. */ + journalMode?: JournalMode + /** Page size when a request omits `limit`. Defaults to 20. */ + defaultLimit?: number + /** Largest accepted page size. Defaults to 100. */ + maxLimit?: number + /** Maximum snippet length in Unicode code points. Defaults to 240. */ + snippetChars?: number +} + +interface ResolvedConfig { + path: string + journalMode: JournalMode + defaultLimit: number + maxLimit: number + snippetChars: number +} + +interface ObservedSession { + header: SessionHeader + events: SessionEvent[] + documents: SessionEventSearchDocument[] + fingerprint: string +} + +interface Observation { + persistence: SessionPersistence | undefined + persistenceRevision: number + persisted: Map + live: Map +} + +interface IndexedRow { + id: string + fingerprint: string + generation: number +} + +interface SearchRow { + session_id: string + version: number + created_at: number + cwd: string | null + parent_session: string | null + seed_length: number | null + live: number + persisted: number + seq: number + type: string + time: number + surface: string + text: string + score: number +} + +interface CursorPayload { + version: 1 + instance: string + scope: 'sessions' | 'events' + fingerprint: string + generation: string + offset: number +} + +/** Concrete SQLite owner of `ctx.sessionSearch`. */ +export class SessionSearchSqlite extends SessionSearchService { + static inject = ['sessions'] + + static Config: z = z.object({ + path: z.string().required(), + journalMode: z.union(['wal', 'delete', 'truncate', 'persist'] as const).default('wal'), + defaultLimit: z.number().step(1).min(1).default(SESSION_QUERY_SQLITE_DEFAULT_LIMIT), + maxLimit: z.number().step(1).min(1).default(SESSION_QUERY_SQLITE_MAX_LIMIT), + snippetChars: z.number().step(1).min(1).default(SESSION_QUERY_SQLITE_SNIPPET_CHARS), + }) + + /** Validated and defaulted backend configuration. */ + readonly config: ResolvedConfig + + private readonly _instance = randomUUID() + private readonly _ready: Promise + private _db: DatabaseSync | undefined + private _persistence: SessionPersistence | undefined + private _persistenceBinding: object | undefined + private _persistenceRevision = 0 + private _lastPersistenceRevision: number | undefined + private _persistenceEpoch = 0 + private _globalGeneration = 0 + private _localGeneration = 0 + private _tail: Promise = Promise.resolve() + private _closed = false + + constructor(ctx: Context, config: Config) { + super(ctx) + this.config = resolveConfig(config) + this._ready = this._open() + ctx.effect(() => { + const fiber = ctx.inject(['sessionPersistence'], (childCtx: Context) => { + const service = childCtx.sessionPersistence + const binding = {} + this._persistenceBinding = binding + this._persistence = service + this._persistenceRevision += 1 + childCtx.effect(() => () => { + /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */ + if (this._persistenceBinding !== binding) return + this._persistenceBinding = undefined + this._persistence = undefined + this._persistenceRevision += 1 + }, 'sessionSearchSqlite.persistenceBinding') + }) + return () => void fiber.dispose() + }, 'sessionSearchSqlite.optionalPersistence') + ctx.effect(() => async () => this.close(), 'sessionSearchSqlite.close') + } + + override async searchSessions( + request: SessionSearchRequest, + exec?: SessionSearchExecContext, + ): Promise> { + const normalized = normalizeSessionRequest(request, this.config) + return this._serialized(exec?.signal, async () => { + await this._ensureReady(exec?.signal) + await this._reconcile(exec?.signal) + assertNotAborted(exec?.signal) + const generation = String(this._globalGeneration) + const fingerprint = requestFingerprint(normalized) + const offset = normalized.cursor === undefined + ? 0 + : decodeCursor(normalized.cursor, this._instance, 'sessions', fingerprint, generation) + const rows = this._querySessions(normalized, offset) + return page(rows, normalized.limit, row => this._sessionHit(row, normalized.query), cursorOffset => encodeCursor({ + version: 1, + instance: this._instance, + scope: 'sessions', + fingerprint, + generation, + offset: cursorOffset, + }), offset) + }) + } + + override async searchEvents( + request: SessionEventSearchRequest, + exec?: SessionSearchExecContext, + ): Promise> { + const normalized = normalizeEventRequest(request, this.config) + return this._serialized(exec?.signal, async () => { + await this._ensureReady(exec?.signal) + await this._reconcile(exec?.signal) + assertNotAborted(exec?.signal) + const generation = this._targetGeneration(normalized.sessionId) + const fingerprint = requestFingerprint(normalized) + const offset = normalized.cursor === undefined + ? 0 + : decodeCursor(normalized.cursor, this._instance, 'events', fingerprint, generation) + const rows = this._queryEvents(normalized, offset) + return page(rows, normalized.limit, row => this._eventHit(row, normalized.query), cursorOffset => encodeCursor({ + version: 1, + instance: this._instance, + scope: 'events', + fingerprint, + generation, + offset: cursorOffset, + }), offset) + }) + } + + /** Close the database after every accepted operation reaches quiescence. */ + async close(): Promise { + if (this._closed) return + this._closed = true + await this._tail + try { + await this._ready + } catch { + // Opening already closed a partially-created handle; disposal only waits. + } + this._db?.close() + this._db = undefined + } + + private async _open(): Promise { + this._db = await openSearchDatabase(this.config.path, this.config.journalMode) + const state = this._db.prepare( + 'SELECT global_generation FROM search_state WHERE singleton = 1', + ).get() as { global_generation: number } + this._globalGeneration = state.global_generation + this._localGeneration = state.global_generation + } + + private async _ensureReady(signal: AbortSignal | undefined): Promise { + try { + await waitWithAbort(this._ready, signal) + } catch (error: unknown) { + if (isAbort(error)) throw error + throw new SessionQueryError( + `session-search SQLite index failed to open: ${errorMessage(error)}`, + 'SESSION_QUERY_INDEX_FAILED', + { cause: error }, + ) + } + } + + private async _serialized(signal: AbortSignal | undefined, operation: () => Promise): Promise { + if (this._isClosed()) throw indexClosed() + let release!: () => void + const gate = new Promise((resolve) => { release = resolve }) + const prior = this._tail + this._tail = prior.then(() => gate) + try { + await waitWithAbort(prior, signal) + } catch (error: unknown) { + release() + throw error + } + if (this._isClosed()) { + release() + throw indexClosed() + } + try { + assertNotAborted(signal) + return await operation() + } finally { + release() + } + } + + private async _reconcile(signal: AbortSignal | undefined): Promise { + const observation = await this._observeStable(signal) + assertNotAborted(signal) + const db = this._requireDb() + const persistedRows = db.prepare( + 'SELECT id, fingerprint, generation FROM persisted_sessions', + ).all() as unknown as IndexedRow[] + const liveRows = db.prepare( + 'SELECT id, fingerprint, generation FROM temp.live_sessions', + ).all() as unknown as IndexedRow[] + const persistedById = new Map(persistedRows.map(row => [row.id as SessionId, row])) + const liveById = new Map(liveRows.map(row => [row.id as SessionId, row])) + const persistentChanges = observation.persistence === undefined + ? [] + : [...observation.persisted.values()].filter(entry => persistedById.get(entry.header.id)?.fingerprint !== entry.fingerprint) + const persistentDeletes = observation.persistence === undefined + ? [] + : persistedRows.filter(row => !observation.persisted.has(row.id as SessionId)) + const liveChanges = [...observation.live.values()].filter(entry => liveById.get(entry.header.id)?.fingerprint !== entry.fingerprint) + const liveDeletes = liveRows.filter(row => !observation.live.has(row.id as SessionId)) + const pointerChanged = this._lastPersistenceRevision !== undefined + && this._lastPersistenceRevision !== observation.persistenceRevision + const hasWrites = persistentChanges.length > 0 + || persistentDeletes.length > 0 + || liveChanges.length > 0 + || liveDeletes.length > 0 + + let nextMainGeneration = this._mainGeneration() + let nextLocalGeneration = this._localGeneration + if (persistentChanges.length > 0 || persistentDeletes.length > 0) nextMainGeneration += 1 + const liveReplacements = liveChanges.map((entry) => { + nextLocalGeneration = Math.max(nextLocalGeneration, nextMainGeneration) + 1 + return { entry, generation: nextLocalGeneration } + }) + + if (hasWrites) { + let began = false + try { + db.exec('BEGIN IMMEDIATE') + began = true + for (const row of persistentDeletes) this._deleteSession('persisted', row.id as SessionId) + for (const entry of persistentChanges) this._replaceSession('persisted', entry, nextMainGeneration) + if (persistentChanges.length > 0 || persistentDeletes.length > 0) { + db.prepare('UPDATE search_state SET global_generation = ? WHERE singleton = 1').run(nextMainGeneration) + } + for (const row of liveDeletes) this._deleteSession('live', row.id as SessionId) + for (const { entry, generation } of liveReplacements) { + this._replaceSession('live', entry, generation) + } + db.exec('COMMIT') + } catch (error: unknown) { + /* v8 ignore next -- a BEGIN failure has no transaction to roll back; the common wrapper still reports it. */ + if (began) { + /* v8 ignore next 5 -- ROLLBACK failure requires a SQLite double fault; the original failure remains actionable. */ + try { + db.exec('ROLLBACK') + } catch { + // The original SQLite failure remains the actionable cause. + } + } + throw new SessionQueryError( + `session-search reconciliation failed: ${errorMessage(error)}`, + 'SESSION_QUERY_INDEX_FAILED', + { cause: error }, + ) + } + } + + if (hasWrites || pointerChanged) this._globalGeneration += 1 + if (pointerChanged) this._persistenceEpoch += 1 + this._localGeneration = nextLocalGeneration + this._lastPersistenceRevision = observation.persistenceRevision + } + + private async _observeStable(signal: AbortSignal | undefined): Promise { + for (;;) { + assertNotAborted(signal) + const persistence = this._persistence + const persistenceRevision = this._persistenceRevision + const persisted = new Map() + if (persistence !== undefined) { + try { + const headers = await waitWithAbort(persistence.list(), signal) + for (const listed of headers) { + const loaded = await waitWithAbort(persistence.load(listed.id), signal) + assertSessionHeadersCompatible(listed, loaded.meta) + persisted.set(listed.id, observeSession(loaded.meta, loaded.events)) + } + } catch (error: unknown) { + if (error instanceof SessionQueryError) throw error + throw new SessionQueryError( + `session-search persistence observation failed: ${errorMessage(error)}`, + 'SESSION_QUERY_PERSISTENCE_FAILED', + { cause: error }, + ) + } + } + const live = new Map() + for (const session of this.ctx.sessions.list()) { + const observed = observeLive(session) + const durable = persisted.get(session.id) + if (durable !== undefined) assertSessionHeadersCompatible(observed.header, durable.header) + live.set(session.id, observed) + } + if (this._persistenceRevision === persistenceRevision) { + return { persistence, persistenceRevision, persisted, live } + } + } + } + + private _mainGeneration(): number { + const row = this._requireDb().prepare( + 'SELECT global_generation FROM search_state WHERE singleton = 1', + ).get() as { global_generation: number } + return row.global_generation + } + + private _deleteSession(source: 'persisted' | 'live', id: SessionId): void { + const db = this._requireDb() + if (source === 'persisted') { + db.prepare('DELETE FROM persisted_docs WHERE session_id = ?').run(id) + db.prepare('DELETE FROM persisted_sessions WHERE id = ?').run(id) + } else { + db.prepare('DELETE FROM temp.live_docs WHERE session_id = ?').run(id) + db.prepare('DELETE FROM temp.live_sessions WHERE id = ?').run(id) + } + } + + private _replaceSession(source: 'persisted' | 'live', entry: ObservedSession, generation: number): void { + this._deleteSession(source, entry.header.id) + const db = this._requireDb() + const sessionTable = source === 'persisted' ? 'persisted_sessions' : 'temp.live_sessions' + const docsTable = source === 'persisted' ? 'persisted_docs' : 'temp.live_docs' + db.prepare(` + INSERT INTO ${sessionTable} + (id, version, created_at, cwd, parent_session, seed_length, fingerprint, generation) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `).run( + entry.header.id, + entry.header.version, + entry.header.createdAt, + entry.header.cwd ?? null, + entry.header.parentSession ?? null, + entry.header.seedLength ?? null, + entry.fingerprint, + generation, + ) + const insert = db.prepare(` + INSERT INTO ${docsTable} (text, session_id, seq, type, time, surface) + VALUES (?, ?, ?, ?, ?, ?) + `) + for (const document of entry.documents) { + insert.run(document.text, document.sessionId, document.seq, document.type, document.time, document.surface) + } + } + + private _querySessions(request: NormalizedSessionRequest, offset: number): SearchRow[] { + const selected = selectedDocumentsSql() + const sessionWhere = buildSessionWhere(request.sessionFilters) + const eventWhere = buildEventWhere(request.eventFilters) + const where = [sessionWhere.sql, eventWhere.sql].filter(Boolean).join(' AND ') + return this._requireDb().prepare(` + ${selected.sql}, + filtered AS ( + SELECT * FROM matched ${where.length === 0 ? '' : `WHERE ${where}`} + ), + ranked AS ( + SELECT *, ROW_NUMBER() OVER ( + PARTITION BY session_id + ORDER BY score ASC, time DESC, seq DESC + ) AS event_rank + FROM filtered + ) + SELECT * FROM ranked + WHERE event_rank = 1 + ORDER BY score ASC, time DESC, session_id ASC, seq DESC + LIMIT ? OFFSET ? + `).all( + quoteFtsData(request.query), + this._persistence === undefined ? 0 : 1, + this._persistence === undefined ? 0 : 1, + quoteFtsData(request.query), + ...sessionWhere.params, + ...eventWhere.params, + request.limit + 1, + offset, + ) as unknown as SearchRow[] + } + + private _queryEvents(request: NormalizedEventRequest, offset: number): SearchRow[] { + const selected = selectedDocumentsSql() + const eventWhere = buildEventWhere(request.filters) + const where = ['session_id = ?', eventWhere.sql].filter(Boolean).join(' AND ') + return this._requireDb().prepare(` + ${selected.sql} + SELECT * FROM matched + WHERE ${where} + ORDER BY score ASC, time DESC, seq DESC + LIMIT ? OFFSET ? + `).all( + quoteFtsData(request.query), + this._persistence === undefined ? 0 : 1, + this._persistence === undefined ? 0 : 1, + quoteFtsData(request.query), + request.sessionId, + ...eventWhere.params, + request.limit + 1, + offset, + ) as unknown as SearchRow[] + } + + private _targetGeneration(sessionId: SessionId): string { + const db = this._requireDb() + const live = db.prepare( + 'SELECT generation FROM temp.live_sessions WHERE id = ?', + ).get(sessionId) as { generation: number } | undefined + if (live !== undefined) return `live:${live.generation}` + if (this._persistence !== undefined) { + const persisted = db.prepare( + 'SELECT generation FROM persisted_sessions WHERE id = ?', + ).get(sessionId) as { generation: number } | undefined + if (persisted !== undefined) return `persisted:${this._persistenceEpoch}:${persisted.generation}` + } + throw new SessionQueryError( + `session "${sessionId}" not found`, + 'SESSION_QUERY_SESSION_NOT_FOUND', + ) + } + + private _sessionHit(row: SearchRow, query: string): SessionSearchHit { + return { + header: rowHeader(row), + live: row.live === 1, + persisted: row.persisted === 1, + bestMatch: this._eventHit(row, query), + } + } + + private _eventHit(row: SearchRow, query: string): SessionEventSearchHit { + return { + sessionId: row.session_id as SessionId, + seq: row.seq, + type: row.type as SessionEventSearchHit['type'], + time: row.time, + surface: row.surface as SessionEventSearchHit['surface'], + snippet: makeSnippet(row.text, query, this.config.snippetChars), + } + } + + private _requireDb(): DatabaseSync { + /* v8 ignore next -- callers await `_ready`; this guards lifecycle misuse */ + if (this._db === undefined) throw indexClosed() + return this._db + } + + private _isClosed(): boolean { + return this._closed + } +} + +function selectedDocumentsSql(): { sql: string } { + return { + sql: `WITH matched AS ( + SELECT + pd.session_id AS session_id, + ps.version AS version, + ps.created_at AS created_at, + ps.cwd AS cwd, + ps.parent_session AS parent_session, + ps.seed_length AS seed_length, + 0 AS live, + 1 AS persisted, + CAST(pd.seq AS INTEGER) AS seq, + pd.type AS type, + CAST(pd.time AS INTEGER) AS time, + pd.surface AS surface, + pd.text AS text, + bm25(persisted_docs) AS score + FROM persisted_docs AS pd + JOIN persisted_sessions AS ps ON ps.id = pd.session_id + WHERE persisted_docs MATCH ? + AND ? = 1 + AND NOT EXISTS (SELECT 1 FROM temp.live_sessions AS ls WHERE ls.id = pd.session_id) + UNION ALL + SELECT + ld.session_id AS session_id, + ls.version AS version, + ls.created_at AS created_at, + ls.cwd AS cwd, + ls.parent_session AS parent_session, + ls.seed_length AS seed_length, + 1 AS live, + CASE WHEN ? = 1 AND EXISTS ( + SELECT 1 FROM persisted_sessions AS ps WHERE ps.id = ld.session_id + ) THEN 1 ELSE 0 END AS persisted, + CAST(ld.seq AS INTEGER) AS seq, + ld.type AS type, + CAST(ld.time AS INTEGER) AS time, + ld.surface AS surface, + ld.text AS text, + bm25(live_docs) AS score + FROM temp.live_docs AS ld + JOIN temp.live_sessions AS ls ON ls.id = ld.session_id + WHERE live_docs MATCH ? + )`, + } +} + +function observeLive(session: Session): ObservedSession { + return observeSession( + structuredClone(session.header), + session.events.map(event => structuredClone(event)), + ) +} + +function observeSession(header: SessionHeader, events: readonly SessionEvent[]): ObservedSession { + const detachedHeader = structuredClone(header) + const detachedEvents = events.map(event => structuredClone(event)) + return { + header: detachedHeader, + events: detachedEvents, + documents: buildSessionEventSearchDocuments(detachedHeader.id, detachedEvents), + fingerprint: createHash('sha256') + .update(JSON.stringify({ header: detachedHeader, events: detachedEvents })) + .digest('base64url'), + } +} + +function rowHeader(row: SearchRow): SessionHeader { + return { + version: row.version, + id: row.session_id as SessionId, + createdAt: row.created_at, + ...row.cwd === null ? {} : { cwd: row.cwd }, + ...row.parent_session === null ? {} : { parentSession: row.parent_session as SessionId }, + ...row.seed_length === null ? {} : { seedLength: row.seed_length }, + } +} + +function page( + rows: readonly Row[], + limit: number, + convert: (row: Row) => Item, + nextCursor: (offset: number) => string, + offset: number, +): SessionSearchPage { + const hasMore = rows.length > limit + return { + items: rows.slice(0, limit).map(convert), + ...hasMore ? { nextCursor: nextCursor(offset + limit) } : {}, + } +} + +function encodeCursor(payload: CursorPayload): string { + return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url') +} + +function decodeCursor( + cursor: string, + instance: string, + scope: CursorPayload['scope'], + fingerprint: string, + generation: string, +): number { + let decoded: Partial + try { + decoded = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')) as Partial + } catch (error: unknown) { + throw invalidCursor(error) + } + if ( + decoded.version !== 1 + || decoded.instance !== instance + || decoded.scope !== scope + || decoded.fingerprint !== fingerprint + || !Number.isInteger(decoded.offset) + || decoded.offset === undefined + || decoded.offset < 0 + ) { + throw invalidCursor(new Error('cursor does not belong to this normalized request')) + } + if (decoded.generation !== generation) { + throw new SessionQueryError( + 'session-search cursor is stale because its relevant corpus changed', + 'SESSION_QUERY_STALE_CURSOR', + ) + } + return decoded.offset +} + +function invalidCursor(cause: unknown): SessionQueryError { + return new SessionQueryError( + 'session-search cursor is invalid', + 'SESSION_QUERY_INVALID_CURSOR', + { cause }, + ) +} + +function resolveConfig(config: Config): ResolvedConfig { + const resolved: ResolvedConfig = { + path: config.path, + journalMode: config.journalMode ?? 'wal', + defaultLimit: config.defaultLimit ?? SESSION_QUERY_SQLITE_DEFAULT_LIMIT, + maxLimit: config.maxLimit ?? SESSION_QUERY_SQLITE_MAX_LIMIT, + snippetChars: config.snippetChars ?? SESSION_QUERY_SQLITE_SNIPPET_CHARS, + } + if (typeof resolved.path !== 'string' || resolved.path.trim().length === 0) { + throw invalidConfig('path must not be blank') + } + assertPositiveInteger('defaultLimit', resolved.defaultLimit) + assertPositiveInteger('maxLimit', resolved.maxLimit) + assertPositiveInteger('snippetChars', resolved.snippetChars) + if (resolved.defaultLimit > resolved.maxLimit) { + throw invalidConfig('defaultLimit must be less than or equal to maxLimit') + } + const journalModes: readonly string[] = ['wal', 'delete', 'truncate', 'persist'] + if (!journalModes.includes(resolved.journalMode)) throw invalidConfig('journalMode is not supported') + return resolved +} + +function assertPositiveInteger(name: string, value: number): void { + if (!Number.isInteger(value) || value < 1) throw invalidConfig(`${name} must be a positive integer`) +} + +function invalidConfig(detail: string): SessionQueryError { + return new SessionQueryError( + `session-search SQLite config: ${detail}`, + 'SESSION_QUERY_INVALID_CONFIG', + ) +} + +function indexClosed(): SessionQueryError { + return new SessionQueryError('session-search SQLite index is closed', 'SESSION_QUERY_INDEX_FAILED') +} + +function assertNotAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) { + throw new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED') + } +} + +function waitWithAbort(promise: Promise, signal: AbortSignal | undefined): Promise { + if (signal === undefined) return promise + if (signal.aborted) return Promise.reject(new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED')) + return new Promise((resolve, reject) => { + const onAbort = () => { + reject(new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED')) + } + signal.addEventListener('abort', onAbort, { once: true }) + promise.then( + (value) => { + signal.removeEventListener('abort', onAbort) + resolve(value) + }, + (error: unknown) => { + signal.removeEventListener('abort', onAbort) + reject(asError(error)) + }, + ) + }) +} + +function isAbort(error: unknown): boolean { + return error instanceof SessionQueryError && error.code === 'SESSION_QUERY_ABORTED' +} + +function asError(error: unknown): Error { + return error instanceof Error + ? error + : new Error('session-search dependency rejected with a non-Error value', { cause: error }) +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : 'unknown error' +} + +export default SessionSearchSqlite diff --git a/packages/session-query/session-query-sqlite/src/query.ts b/packages/session-query/session-query-sqlite/src/query.ts new file mode 100644 index 0000000000..fd2b5156e6 --- /dev/null +++ b/packages/session-query/session-query-sqlite/src/query.ts @@ -0,0 +1,312 @@ +/** Request normalization, parameterized predicates, and result presentation. */ + +import { + SessionQueryError, + filterSessionEventDocuments, + filterSessionResults, +} from '@deepseek-ai/dsh-session-query' +import type { + SessionEventMetadataFilter, + SessionEventSearchRequest, + SessionResultFilter, + SessionSearchRequest, +} from '@deepseek-ai/dsh-session-query' + +/** Limit defaults needed to normalize a search request. */ +export interface QueryLimits { + /** Page size used when the request omits one. */ + defaultLimit: number + /** Largest accepted page size. */ + maxLimit: number +} + +/** Normalized cross-session request. */ +export interface NormalizedSessionRequest { + query: string + sessionFilters: readonly SessionResultFilter[] + eventFilters: readonly SessionEventMetadataFilter[] + limit: number + cursor?: string +} + +/** Normalized within-session request. */ +export interface NormalizedEventRequest { + sessionId: SessionEventSearchRequest['sessionId'] + query: string + filters: readonly SessionEventMetadataFilter[] + limit: number + cursor?: string +} + +/** Parameterized SQL predicate fragment. */ +export interface SqlWhere { + /** SQL without the leading `WHERE`. */ + sql: string + /** Bindings in placeholder order. */ + params: Array +} + +/** + * Validate and canonicalize a cross-session request. + * @param request - caller-provided query, filters, limit, and cursor. + * @param limits - configured default and maximum page sizes. + * @returns normalized request with explicit arrays and limit. + */ +export function normalizeSessionRequest( + request: SessionSearchRequest, + limits: QueryLimits, +): NormalizedSessionRequest { + const sessionFilters = request.sessionFilters ?? [] + const eventFilters = request.eventFilters ?? [] + filterSessionResults([], sessionFilters) + filterSessionEventDocuments([], eventFilters) + return { + query: normalizeQuery(request.query), + sessionFilters, + eventFilters, + limit: normalizeLimit(request.limit, limits), + ...request.cursor === undefined ? {} : { cursor: request.cursor }, + } +} + +/** + * Validate and canonicalize a within-session request. + * @param request - caller-provided target, query, filters, limit, and cursor. + * @param limits - configured default and maximum page sizes. + * @returns normalized request with an explicit filter array and limit. + */ +export function normalizeEventRequest( + request: SessionEventSearchRequest, + limits: QueryLimits, +): NormalizedEventRequest { + const filters = request.filters ?? [] + filterSessionEventDocuments([], filters) + return { + sessionId: request.sessionId, + query: normalizeQuery(request.query), + filters, + limit: normalizeLimit(request.limit, limits), + ...request.cursor === undefined ? {} : { cursor: request.cursor }, + } +} + +/** + * Compile logical-session predicates against selected-document columns. + * @param filters - validated ANDed logical-session clauses. + * @returns parameterized SQL fragment and ordered bindings. + */ +export function buildSessionWhere(filters: readonly SessionResultFilter[]): SqlWhere { + const clauses: string[] = [] + const params: Array = [] + for (const filter of filters) { + switch (filter.kind) { + case 'id': + addList(clauses, params, 'session_id', filter.values) + break + case 'cwd': + addNullableList(clauses, params, 'cwd', filter.values) + break + case 'created-at': + addRange(clauses, params, 'created_at', filter) + break + case 'parent': + addNullableList(clauses, params, 'parent_session', filter.values) + break + case 'availability': { + const availability = [...new Set(filter.values)] + if (availability.length === 0) clauses.push('0') + else if (availability.length === 1) clauses.push(`${availability[0]} = 1`) + break + } + } + } + return { sql: clauses.join(' AND '), params } +} + +/** + * Compile event metadata predicates against selected-document columns. + * @param filters - validated ANDed event metadata clauses. + * @returns parameterized SQL fragment and ordered bindings. + */ +export function buildEventWhere(filters: readonly SessionEventMetadataFilter[]): SqlWhere { + const clauses: string[] = [] + const params: Array = [] + for (const filter of filters) { + switch (filter.kind) { + case 'seq': + addRange(clauses, params, 'seq', filter) + break + case 'time': + addRange(clauses, params, 'time', filter) + break + case 'type': + addList(clauses, params, 'type', filter.values) + break + case 'surface': + addList(clauses, params, 'surface', filter.values) + break + } + } + return { sql: clauses.join(' AND '), params } +} + +/** + * Quote caller text as one FTS5 phrase so query syntax remains inert data. + * @param query - normalized caller query. + * @returns FTS5 expression containing one escaped literal phrase. + */ +export function quoteFtsData(query: string): string { + return `"${query.replaceAll('"', '""')}"` +} + +/** + * Build the stable normalized request identity stored in opaque cursors. + * @param request - normalized request whose filter ordering is canonicalized. + * @returns deterministic JSON identity for cursor binding. + */ +export function requestFingerprint(request: NormalizedSessionRequest | NormalizedEventRequest): string { + if ('sessionId' in request) { + return JSON.stringify({ + scope: 'events', + sessionId: request.sessionId, + query: request.query, + filters: canonicalFilters(request.filters), + limit: request.limit, + }) + } + return JSON.stringify({ + scope: 'sessions', + query: request.query, + sessionFilters: canonicalFilters(request.sessionFilters), + eventFilters: canonicalFilters(request.eventFilters), + limit: request.limit, + }) +} + +/** + * Build a whitespace-normalized excerpt no longer than `maxChars`. + * @param text - complete extracted semantic document. + * @param query - normalized literal query used to position the excerpt. + * @param maxChars - maximum result length in Unicode code points. + * @returns bounded plain-text snippet. + */ +export function makeSnippet(text: string, query: string, maxChars: number): string { + const clean = text.replace(/\s+/gu, ' ').trim() + const characters = Array.from(clean) + if (characters.length <= maxChars) return clean + if (maxChars === 1) return '…' + const foundUnits = clean.toLowerCase().indexOf(query.toLowerCase()) + const found = foundUnits < 0 ? -1 : Array.from(clean.slice(0, foundUnits)).length + let start = found < 0 ? 0 : Math.max(0, found - Math.floor(maxChars / 3)) + let prefix = start > 0 ? '…' : '' + let suffix = '…' + let contentLength = maxChars - prefix.length - suffix.length + if (contentLength < 1) { + start = 0 + prefix = '' + contentLength = maxChars - 1 + } + let end = Math.min(characters.length, start + contentLength) + if (end === characters.length) { + suffix = '' + contentLength = maxChars - prefix.length + start = Math.max(0, end - contentLength) + } + end = Math.min(characters.length, start + contentLength) + return `${prefix}${characters.slice(start, end).join('')}${suffix}` +} + +function normalizeQuery(value: string): string { + if (typeof value !== 'string') { + throw new SessionQueryError('session-search query must be text', 'SESSION_QUERY_INVALID_QUERY') + } + const query = value.trim().replace(/\s+/gu, ' ') + if (query.length === 0) { + throw new SessionQueryError( + 'session-search query must contain non-whitespace text', + 'SESSION_QUERY_INVALID_QUERY', + ) + } + return query +} + +function normalizeLimit(value: number | undefined, limits: QueryLimits): number { + const limit = value ?? limits.defaultLimit + if (!Number.isInteger(limit) || limit < 1 || limit > limits.maxLimit) { + throw new SessionQueryError( + `session-search limit must be an integer between 1 and ${limits.maxLimit}`, + 'SESSION_QUERY_INVALID_LIMIT', + ) + } + return limit +} + +function addList( + clauses: string[], + params: Array, + column: string, + values: readonly (string | number)[], +): void { + if (values.length === 0) { + clauses.push('0') + return + } + clauses.push(`${column} IN (${values.map(() => '?').join(', ')})`) + params.push(...values) +} + +function addNullableList( + clauses: string[], + params: Array, + column: string, + values: readonly (string | null)[], +): void { + if (values.length === 0) { + clauses.push('0') + return + } + const concrete = values.filter((value): value is string => value !== null) + const parts: string[] = [] + if (concrete.length > 0) { + parts.push(`${column} IN (${concrete.map(() => '?').join(', ')})`) + params.push(...concrete) + } + if (values.includes(null)) parts.push(`${column} IS NULL`) + clauses.push(`(${parts.join(' OR ')})`) +} + +function addRange( + clauses: string[], + params: Array, + column: string, + range: { from?: number; to?: number }, +): void { + if (range.from !== undefined) { + clauses.push(`CAST(${column} AS INTEGER) >= ?`) + params.push(range.from) + } + if (range.to !== undefined) { + clauses.push(`CAST(${column} AS INTEGER) <= ?`) + params.push(range.to) + } +} + +function canonicalFilters(filters: readonly (SessionResultFilter | SessionEventMetadataFilter)[]): unknown[] { + return filters.map((filter) => { + if ('values' in filter) { + return { ...filter, values: [...filter.values].sort(compareNullable) } + } + return { + kind: filter.kind, + from: filter.from ?? null, + to: filter.to ?? null, + } + }).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))) +} + +function compareNullable(a: string | null, b: string | null): number { + if (a === b) return 0 + if (a === null) return -1 + if (b === null) return 1 + return a.localeCompare(b) +} diff --git a/packages/session-query/session-query-sqlite/src/schema.ts b/packages/session-query/session-query-sqlite/src/schema.ts new file mode 100644 index 0000000000..1c9bd98791 --- /dev/null +++ b/packages/session-query/session-query-sqlite/src/schema.ts @@ -0,0 +1,127 @@ +/** SQLite schema for the disposable session full-text read model. */ + +import { DatabaseSync } from 'node:sqlite' +import { mkdir } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' + +/** Current derived-index schema version. Incompatible versions reset in place. */ +export const SESSION_QUERY_SQLITE_SCHEMA_VERSION = 1 + +/** SQLite application id protecting unrelated databases from derived resets. */ +export const SESSION_QUERY_SQLITE_APPLICATION_ID = 0x44534851 + +/** Supported SQLite journal modes. */ +export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' + +/** + * Open, validate, and initialize persistent and connection-local schemas. + * @param path - dedicated derived-index path or `:memory:`. + * @param journalMode - validated SQLite journal mode. + * @returns initialized database handle owned by the search service. + */ +export async function openSearchDatabase(path: string, journalMode: JournalMode): Promise { + const actual = path === ':memory:' ? path : resolve(path) + if (actual !== ':memory:') await mkdir(dirname(actual), { recursive: true, mode: 0o700 }) + const db = new DatabaseSync(actual) + try { + // journalMode is a validated closed union, not caller-controlled SQL. + db.exec(`PRAGMA journal_mode = ${journalMode.toUpperCase()}`) + const { application_id: applicationId } = db.prepare('PRAGMA application_id').get() as { application_id: number } + const { user_version: version } = db.prepare('PRAGMA user_version').get() as { user_version: number } + const userTables = listUserTables(db) + if (applicationId !== 0 && applicationId !== SESSION_QUERY_SQLITE_APPLICATION_ID) { + throw new Error(`session-search database at "${actual}" belongs to another application`) + } + if (applicationId === 0 && userTables.length > 0) { + throw new Error(`session-search database at "${actual}" is not an empty or recognized derived index`) + } + if (applicationId === SESSION_QUERY_SQLITE_APPLICATION_ID && version !== SESSION_QUERY_SQLITE_SCHEMA_VERSION) { + resetDerivedSchema(db) + } + ensurePersistentSchema(db) + ensureTemporarySchema(db) + return db + } catch (error: unknown) { + db.close() + throw error + } +} + +function listUserTables(db: DatabaseSync): string[] { + const rows = db.prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + ).all() as Array<{ name: string }> + return rows.map(row => row.name) +} + +function resetDerivedSchema(db: DatabaseSync): void { + for (const name of listUserTables(db)) { + db.exec(`DROP TABLE IF EXISTS ${quoteIdentifier(name)}`) + } + db.exec('PRAGMA user_version = 0') +} + +function ensurePersistentSchema(db: DatabaseSync): void { + db.exec(`PRAGMA application_id = ${SESSION_QUERY_SQLITE_APPLICATION_ID}`) + db.exec(` + CREATE TABLE IF NOT EXISTS search_state ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + global_generation INTEGER NOT NULL + ) STRICT + `) + db.exec('INSERT OR IGNORE INTO search_state (singleton, global_generation) VALUES (1, 0)') + db.exec(` + CREATE TABLE IF NOT EXISTS persisted_sessions ( + id TEXT PRIMARY KEY, + version INTEGER NOT NULL, + created_at INTEGER NOT NULL, + cwd TEXT, + parent_session TEXT, + seed_length INTEGER, + fingerprint TEXT NOT NULL, + generation INTEGER NOT NULL + ) STRICT + `) + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS persisted_docs USING fts5( + text, + session_id UNINDEXED, + seq UNINDEXED, + type UNINDEXED, + time UNINDEXED, + surface UNINDEXED, + tokenize = 'unicode61' + ) + `) + db.exec(`PRAGMA user_version = ${SESSION_QUERY_SQLITE_SCHEMA_VERSION}`) +} + +function ensureTemporarySchema(db: DatabaseSync): void { + db.exec(` + CREATE TEMP TABLE IF NOT EXISTS live_sessions ( + id TEXT PRIMARY KEY, + version INTEGER NOT NULL, + created_at INTEGER NOT NULL, + cwd TEXT, + parent_session TEXT, + seed_length INTEGER, + fingerprint TEXT NOT NULL, + generation INTEGER NOT NULL + ) STRICT + `) + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS temp.live_docs USING fts5( + text, + session_id UNINDEXED, + seq UNINDEXED, + type UNINDEXED, + time UNINDEXED, + surface UNINDEXED, + tokenize = 'unicode61' + ) + `) +} + +function quoteIdentifier(value: string): string { + return `"${value.replaceAll('"', '""')}"` +} diff --git a/packages/session-query/session-query-sqlite/tests/load-path.e2e.ts b/packages/session-query/session-query-sqlite/tests/load-path.e2e.ts new file mode 100644 index 0000000000..c20a501964 --- /dev/null +++ b/packages/session-query/session-query-sqlite/tests/load-path.e2e.ts @@ -0,0 +1,60 @@ +/** + * Keyless real-Loader-path smoke for the SQLite session-search service. + * + * @module @deepseek-ai/dsh-session-query-sqlite/tests/load-path + */ + +import { afterEach, describe, expect, it } from 'vitest' +import { Context } from 'cordis' +import Loader from '@cordisjs/plugin-loader' +import { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' +import SessionStore from '@deepseek-ai/dsh-session' +import SessionPersistenceSqlite from '@deepseek-ai/dsh-session-persistence-sqlite' +import SessionSearchSqlite, * as searchModule from '@deepseek-ai/dsh-session-query-sqlite' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const temporaryDirectories: string[] = [] + +afterEach(async () => { + for (const directory of temporaryDirectories.splice(0)) { + await rm(directory, { recursive: true, force: true }) + } +}) + +async function temporaryPath(name: string): Promise { + const directory = await mkdtemp(join(tmpdir(), 'dsh-session-search-loader-')) + temporaryDirectories.push(directory) + return join(directory, name) +} + +describe('dsh-session-query-sqlite real Loader path', () => { + it('unwraps, mounts, and searches the real persistence backend', async () => { + const persistencePath = await temporaryPath('canonical.db') + const searchPath = await temporaryPath('derived.db') + const ctx = new Context() + await ctx.plugin(SessionStore) + const persistence = await ctx.plugin(SessionPersistenceSqlite, { path: persistencePath }) + + const loader = Object.create(Loader.prototype) as Loader + const unwrapped = loader.unwrapExports(searchModule) as Parameters[0] + expect(unwrapped).toBe(SessionSearchSqlite) + const search = await ctx.plugin(unwrapped, { path: searchPath }) + + const id = SessionId('loader-path') + await ctx.sessionPersistence.create({ version: SESSION_FORMAT_VERSION, id, createdAt: 10 }) + await ctx.sessionPersistence.append(id, [{ + type: 'user/message', + seq: 0, + time: 10, + data: { content: [{ type: 'text', text: 'real Loader needle' }], source: { kind: 'user' } }, + surfaceOp: 'append', + }]) + + await expect(ctx.sessionSearch.searchSessions({ query: 'Loader needle' })) + .resolves.toMatchObject({ items: [{ header: { id }, persisted: true, live: false }] }) + await search.dispose() + await persistence.dispose() + }) +}) diff --git a/packages/session-query/session-query-sqlite/tests/query.spec.ts b/packages/session-query/session-query-sqlite/tests/query.spec.ts new file mode 100644 index 0000000000..0faced72e4 --- /dev/null +++ b/packages/session-query/session-query-sqlite/tests/query.spec.ts @@ -0,0 +1,179 @@ +import { describe, expect, it } from 'vitest' +import { SessionId } from '@deepseek-ai/dsh-session' +import type { SessionQueryErrorCode } from '@deepseek-ai/dsh-session-query' +import { + buildEventWhere, + buildSessionWhere, + makeSnippet, + normalizeEventRequest, + normalizeSessionRequest, + quoteFtsData, + requestFingerprint, + type NormalizedEventRequest, + type NormalizedSessionRequest, +} from '../src/query.ts' + +const limits = { defaultLimit: 2, maxLimit: 3 } + +function expectCode(code: SessionQueryErrorCode): Error { + return expect.objectContaining({ code }) as Error +} + +describe('SQLite search request normalization', () => { + it('normalizes both scopes, defaults arrays and limits, and preserves cursors', () => { + expect(normalizeSessionRequest({ query: ' alpha\n beta ' }, limits)).toEqual({ + query: 'alpha beta', + sessionFilters: [], + eventFilters: [], + limit: 2, + }) + expect(normalizeSessionRequest({ + query: 'needle', + sessionFilters: [{ kind: 'availability', values: ['live'] }], + eventFilters: [{ kind: 'surface', values: ['current'] }], + limit: 3, + cursor: 'next', + }, limits)).toEqual({ + query: 'needle', + sessionFilters: [{ kind: 'availability', values: ['live'] }], + eventFilters: [{ kind: 'surface', values: ['current'] }], + limit: 3, + cursor: 'next', + }) + expect(normalizeEventRequest({ sessionId: SessionId('s'), query: 'needle' }, limits)).toEqual({ + sessionId: SessionId('s'), + query: 'needle', + filters: [], + limit: 2, + }) + expect(normalizeEventRequest({ + sessionId: SessionId('s'), + query: 'needle', + filters: [{ kind: 'seq', from: 1 }], + cursor: 'next', + }, limits)).toEqual({ + sessionId: SessionId('s'), + query: 'needle', + filters: [{ kind: 'seq', from: 1 }], + limit: 2, + cursor: 'next', + }) + }) + + it('rejects non-text, blank, non-integer, non-positive, and oversized requests', () => { + expect(() => normalizeSessionRequest({ query: 1 as never }, limits)) + .toThrow(expectCode('SESSION_QUERY_INVALID_QUERY')) + expect(() => normalizeSessionRequest({ query: ' \n ' }, limits)) + .toThrow(expectCode('SESSION_QUERY_INVALID_QUERY')) + for (const limit of [1.5, 0, 4]) { + expect(() => normalizeEventRequest({ sessionId: SessionId('s'), query: 'x', limit }, limits)) + .toThrow(expectCode('SESSION_QUERY_INVALID_LIMIT')) + } + }) +}) + +describe('SQLite search predicate compilation', () => { + it('compiles all logical-session clauses including empty and nullable values', () => { + expect(buildSessionWhere([])).toEqual({ sql: '', params: [] }) + expect(buildSessionWhere([{ kind: 'id', values: [] }])).toEqual({ sql: '0', params: [] }) + expect(buildSessionWhere([{ kind: 'id', values: [SessionId('a'), SessionId('b')] }])).toEqual({ + sql: 'session_id IN (?, ?)', + params: [SessionId('a'), SessionId('b')], + }) + expect(buildSessionWhere([{ kind: 'cwd', values: [] }])).toEqual({ sql: '0', params: [] }) + expect(buildSessionWhere([{ kind: 'cwd', values: [null] }])).toEqual({ + sql: '(cwd IS NULL)', + params: [], + }) + expect(buildSessionWhere([{ kind: 'cwd', values: ['/a'] }])).toEqual({ + sql: '(cwd IN (?))', + params: ['/a'], + }) + expect(buildSessionWhere([{ kind: 'parent', values: [SessionId('p'), null] }])).toEqual({ + sql: '(parent_session IN (?) OR parent_session IS NULL)', + params: [SessionId('p')], + }) + expect(buildSessionWhere([ + { kind: 'created-at', from: 1, to: 2 }, + { kind: 'availability', values: [] }, + { kind: 'availability', values: ['live', 'live'] }, + { kind: 'availability', values: ['live', 'persisted'] }, + ])).toEqual({ + sql: 'CAST(created_at AS INTEGER) >= ? AND CAST(created_at AS INTEGER) <= ? AND 0 AND live = 1', + params: [1, 2], + }) + expect(buildSessionWhere([{ kind: 'created-at' }])).toEqual({ sql: '', params: [] }) + }) + + it('compiles every event clause and empty lists', () => { + expect(buildEventWhere([ + { kind: 'seq', from: 1 }, + { kind: 'time', to: 9 }, + { kind: 'type', values: ['user/message'] }, + { kind: 'surface', values: ['current', 'log-only'] }, + ])).toEqual({ + sql: 'CAST(seq AS INTEGER) >= ? AND CAST(time AS INTEGER) <= ? AND type IN (?) AND surface IN (?, ?)', + params: [1, 9, 'user/message', 'current', 'log-only'], + }) + expect(buildEventWhere([ + { kind: 'type', values: [] }, + { kind: 'surface', values: [] }, + ])).toEqual({ sql: '0 AND 0', params: [] }) + }) +}) + +describe('SQLite query identity and presentation', () => { + it('quotes all caller MATCH syntax as data', () => { + expect(quoteFtsData('say "needle" OR *')).toBe('"say ""needle"" OR *"') + }) + + it('canonicalizes request and filter ordering in both scopes', () => { + const sessionA: NormalizedSessionRequest = { + query: 'needle', + limit: 2, + sessionFilters: [ + { kind: 'cwd', values: ['/b', '/a'] }, + { kind: 'parent', values: [null, SessionId('p')] }, + { kind: 'id', values: [SessionId('same'), SessionId('same')] }, + { kind: 'created-at', from: 1 }, + ], + eventFilters: [{ kind: 'time', to: 9 }], + } + const sessionB: NormalizedSessionRequest = { + query: 'needle', + limit: 2, + sessionFilters: [ + { kind: 'created-at', from: 1 }, + { kind: 'id', values: [SessionId('same'), SessionId('same')] }, + { kind: 'parent', values: [SessionId('p'), null] }, + { kind: 'cwd', values: ['/a', '/b'] }, + ], + eventFilters: [{ kind: 'time', to: 9 }], + } + expect(requestFingerprint(sessionA)).toBe(requestFingerprint(sessionB)) + + const eventA: NormalizedEventRequest = { + sessionId: SessionId('s'), + query: 'needle', + limit: 2, + filters: [{ kind: 'seq' }, { kind: 'surface', values: ['shadowed', 'current'] }], + } + const eventB: NormalizedEventRequest = { + sessionId: SessionId('s'), + query: 'needle', + limit: 2, + filters: [{ kind: 'surface', values: ['current', 'shadowed'] }, { kind: 'seq' }], + } + expect(requestFingerprint(eventA)).toBe(requestFingerprint(eventB)) + expect(requestFingerprint(eventA)).not.toBe(requestFingerprint({ ...eventB, sessionId: SessionId('other') })) + }) + + it('normalizes, bounds, and positions snippets by Unicode code point', () => { + expect(makeSnippet(' short\ntext ', 'absent', 20)).toBe('short text') + expect(makeSnippet('abcdef', 'f', 1)).toBe('…') + expect(makeSnippet('abcdefghij', 'absent', 5)).toBe('abcd…') + expect(makeSnippet('abcdefghij', 'c', 5)).toBe('…bcd…') + expect(makeSnippet('abcdef', 'f', 2)).toBe('a…') + expect(makeSnippet('abcdef', 'f', 5)).toBe('…cdef') + }) +}) diff --git a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts new file mode 100644 index 0000000000..77d1e0125d --- /dev/null +++ b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts @@ -0,0 +1,594 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { Context } from 'cordis' +import { DatabaseSync } from 'node:sqlite' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import SessionStore, { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' +import type { SessionEvent, SessionHeader, SessionId as SessionIdType } from '@deepseek-ai/dsh-session' +import SessionPersistence from '@deepseek-ai/dsh-session-persistence' +import SessionPersistenceSqlite from '@deepseek-ai/dsh-session-persistence-sqlite' +import SessionSearchSqlite, { + SESSION_QUERY_SQLITE_APPLICATION_ID, + SESSION_QUERY_SQLITE_SCHEMA_VERSION, +} from '@deepseek-ai/dsh-session-query-sqlite' +import type { SessionQueryErrorCode } from '@deepseek-ai/dsh-session-query' + +const temporaryDirectories: string[] = [] + +afterEach(async () => { + for (const directory of temporaryDirectories.splice(0)) { + await rm(directory, { recursive: true, force: true }) + } +}) + +async function temporaryPath(name = 'search.db'): Promise { + const directory = await mkdtemp(join(tmpdir(), 'dsh-session-search-')) + temporaryDirectories.push(directory) + return join(directory, name) +} + +function header(id: string, createdAt = 1, extra: Partial = {}): SessionHeader { + return { version: SESSION_FORMAT_VERSION, id: SessionId(id), createdAt, ...extra } +} + +function messageEvents(text: string, time = 1): SessionEvent[] { + return [{ + type: 'user/message', + seq: 0, + time, + data: { content: [{ type: 'text', text }], source: { kind: 'user' } }, + surfaceOp: 'append', + }] +} + +function expectCode(code: SessionQueryErrorCode): Error { + return expect.objectContaining({ code }) as Error +} + +class TestPersistence extends SessionPersistence { + static entries = new Map() + static listGate: Promise | undefined + static listStarted: (() => void) | undefined + static failure: unknown + + static reset(entries: readonly { meta: SessionHeader; events: SessionEvent[] }[] = []): void { + this.entries = new Map(entries.map(entry => [entry.meta.id, structuredClone(entry)])) + this.listGate = undefined + this.listStarted = undefined + this.failure = undefined + } + + create(meta: SessionHeader): Promise { + TestPersistence.entries.set(meta.id, { meta: structuredClone(meta), events: [] }) + return Promise.resolve() + } + + append(id: SessionIdType, events: readonly SessionEvent[]): Promise { + const entry = TestPersistence.entries.get(id) + if (entry === undefined) return Promise.reject(new Error('missing test session')) + entry.events.push(...structuredClone(events)) + return Promise.resolve() + } + + async load(id: SessionIdType): Promise<{ meta: SessionHeader; events: SessionEvent[] }> { + if (TestPersistence.failure !== undefined) throw TestPersistence.failure + const entry = TestPersistence.entries.get(id) + if (entry === undefined) throw new Error('missing test session') + return structuredClone(entry) + } + + async list(): Promise { + TestPersistence.listStarted?.() + await TestPersistence.listGate + if (TestPersistence.failure !== undefined) throw TestPersistence.failure + return [...TestPersistence.entries.values()].map(entry => structuredClone(entry.meta)) + } +} + +async function liveContext(config: ConstructorParameters[1] = { path: ':memory:' }): Promise { + const ctx = new Context() + await ctx.plugin(SessionStore) + await ctx.plugin(SessionSearchSqlite, config) + return ctx +} + +describe('SQLite session search', () => { + it('searches two-character Unicode61 tokens in live-only sessions', async () => { + const ctx = await liveContext({ path: ':memory:', snippetChars: 20 }) + const session = ctx.sessions.create(SessionId('live'), { meta: { cwd: '/work', createdAt: 10, seedLength: 1 } }) + session.append( + 'user/message', + { content: [{ type: 'text', text: 'An AI helper' }], source: { kind: 'user' } }, + { surfaceOp: 'append' }, + ) + + await expect(ctx.sessionSearch.searchEvents({ sessionId: session.id, query: 'AI' })) + .resolves.toMatchObject({ items: [{ sessionId: session.id, seq: 0, snippet: 'An AI helper' }] }) + await expect(ctx.sessionSearch.searchSessions({ query: 'AI' })) + .resolves.toMatchObject({ items: [{ header: { ...session.header, seedLength: 1 }, live: true, persisted: false }] }) + }) + + it('searches all surfaces by default and applies metadata before ranking', async () => { + const ctx = await liveContext({ path: ':memory:', defaultLimit: 10, maxLimit: 20 }) + const parent = SessionId('parent') + const events: SessionEvent[] = [ + { type: 'user/message', seq: 0, time: 10, data: { content: [{ type: 'text', text: 'needle original' }], source: { kind: 'user' } }, surfaceOp: 'append' }, + { type: 'assistant/chunk', seq: 1, time: 11, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'needle raw' } } }, + { type: 'user/message', seq: 2, time: 12, data: { content: [{ type: 'text', text: 'needle summary' }], source: { kind: 'plugin', plugin: 'test' } }, surfaceOp: { op: 'replace', start: 0, end: 0 } }, + { type: 'turn/end', seq: 3, time: 13, data: { turn: 1, reason: { kind: 'error', step: 1, message: 'needle failure' } } }, + ] + ctx.sessions.create(SessionId('a'), { seed: events, meta: { cwd: '/a', parentSession: parent, createdAt: 20 } }) + ctx.sessions.create(SessionId('b'), { seed: messageEvents('needle peer', 12), meta: { createdAt: 20 } }) + + const all = await ctx.sessionSearch.searchEvents({ sessionId: SessionId('a'), query: 'needle' }) + expect(new Set(all.items.map(item => item.surface))).toEqual(new Set(['current', 'shadowed', 'log-only'])) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: SessionId('a'), + query: 'needle', + filters: [ + { kind: 'seq', from: 2, to: 2 }, + { kind: 'time', from: 12, to: 12 }, + { kind: 'type', values: ['user/message'] }, + { kind: 'surface', values: ['current'] }, + ], + })).resolves.toMatchObject({ items: [{ seq: 2, surface: 'current' }] }) + + const grouped = await ctx.sessionSearch.searchSessions({ + query: 'needle', + sessionFilters: [ + { kind: 'id', values: [SessionId('a')] }, + { kind: 'cwd', values: ['/a'] }, + { kind: 'created-at', from: 20, to: 20 }, + { kind: 'parent', values: [parent] }, + { kind: 'availability', values: ['live'] }, + ], + eventFilters: [{ kind: 'surface', values: ['shadowed'] }], + }) + expect(grouped.items).toHaveLength(1) + expect(grouped.items[0]).toMatchObject({ + header: { id: SessionId('a'), cwd: '/a', parentSession: parent }, + live: true, + persisted: false, + bestMatch: { seq: 0, surface: 'shadowed' }, + }) + }) + + it('uses literal phrase tokens, stable ties, and bounded Unicode snippets', async () => { + const ctx = await liveContext({ path: ':memory:', defaultLimit: 10, maxLimit: 10, snippetChars: 5 }) + ctx.sessions.create(SessionId('a'), { seed: messageEvents('😀😀 alpha beta BRAID 😀😀', 10), meta: { createdAt: 1 } }) + ctx.sessions.create(SessionId('b'), { seed: messageEvents('alpha beta', 10), meta: { createdAt: 1 } }) + ctx.sessions.create(SessionId('c'), { seed: messageEvents('alpha middle beta', 10), meta: { createdAt: 1 } }) + ctx.sessions.create(SessionId('d'), { seed: messageEvents('alpha beta', 10), meta: { createdAt: 1 } }) + ctx.sessions.create(SessionId('operator'), { seed: messageEvents('needle OR absent', 10), meta: { createdAt: 1 } }) + ctx.sessions.create(SessionId('only'), { seed: messageEvents('needle only', 10), meta: { createdAt: 1 } }) + ctx.sessions.create(SessionId('quote'), { seed: messageEvents('say "needle" exactly', 10), meta: { createdAt: 1 } }) + + const phrase = await ctx.sessionSearch.searchSessions({ query: 'alpha beta' }) + expect(phrase.items.map(item => item.header.id)).toEqual([SessionId('b'), SessionId('d'), SessionId('a')]) + expect(phrase.items.every(item => Array.from(item.bestMatch.snippet).length <= 5)).toBe(true) + await expect(ctx.sessionSearch.searchSessions({ query: 'AI' })).resolves.toEqual({ items: [] }) + await expect(ctx.sessionSearch.searchSessions({ query: 'needle OR absent' })) + .resolves.toMatchObject({ items: [{ header: { id: SessionId('operator') } }] }) + await expect(ctx.sessionSearch.searchSessions({ query: 'say "needle"' })) + .resolves.toMatchObject({ items: [{ header: { id: SessionId('quote') } }] }) + await expect(ctx.sessionSearch.searchSessions({ query: '*' })).resolves.toEqual({ items: [] }) + }) + + it('binds cursors to requests and only invalidates within-session pages for target changes', async () => { + const ctx = await liveContext({ path: ':memory:', defaultLimit: 1, maxLimit: 5 }) + const target = ctx.sessions.create(SessionId('target'), { + seed: [ + ...messageEvents('needle one', 10), + { ...messageEvents('needle two', 11)[0]!, seq: 1 }, + { ...messageEvents('needle three', 12)[0]!, seq: 2 }, + ], + }) + ctx.sessions.create(SessionId('other'), { seed: messageEvents('needle other', 10) }) + + const eventPage = await ctx.sessionSearch.searchEvents({ sessionId: target.id, query: 'needle', limit: 1 }) + const sessionPage = await ctx.sessionSearch.searchSessions({ query: 'needle', limit: 1 }) + expect(eventPage.nextCursor).toEqual(expect.any(String)) + expect(sessionPage.nextCursor).toEqual(expect.any(String)) + if (eventPage.nextCursor === undefined || sessionPage.nextCursor === undefined) throw new Error('expected cursors') + + const eventKeys = eventPage.items.map(item => `${item.sessionId}:${item.seq}`) + let eventCursor: string | undefined = eventPage.nextCursor + while (eventCursor !== undefined) { + const next = await ctx.sessionSearch.searchEvents({ + sessionId: target.id, + query: 'needle', + limit: 1, + cursor: eventCursor, + }) + eventKeys.push(...next.items.map(item => `${item.sessionId}:${item.seq}`)) + eventCursor = next.nextCursor + } + expect(eventKeys).toHaveLength(3) + expect(new Set(eventKeys).size).toBe(eventKeys.length) + + const sessionIds = sessionPage.items.map(item => item.header.id) + let sessionCursor: string | undefined = sessionPage.nextCursor + while (sessionCursor !== undefined) { + const next = await ctx.sessionSearch.searchSessions({ query: 'needle', limit: 1, cursor: sessionCursor }) + sessionIds.push(...next.items.map(item => item.header.id)) + sessionCursor = next.nextCursor + } + expect(sessionIds).toHaveLength(2) + expect(new Set(sessionIds).size).toBe(sessionIds.length) + + ctx.sessions.create(SessionId('unrelated'), { seed: messageEvents('needle unrelated', 20) }) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: target.id, + query: 'needle', + limit: 1, + cursor: eventPage.nextCursor, + })).resolves.toMatchObject({ items: [{ sessionId: target.id }] }) + await expect(ctx.sessionSearch.searchSessions({ query: 'needle', limit: 1, cursor: sessionPage.nextCursor })) + .rejects.toThrow(expectCode('SESSION_QUERY_STALE_CURSOR')) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: target.id, + query: 'different', + limit: 1, + cursor: eventPage.nextCursor, + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_CURSOR')) + + target.append('user/message', { content: [{ type: 'text', text: 'needle four' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: target.id, + query: 'needle', + limit: 1, + cursor: eventPage.nextCursor, + })).rejects.toThrow(expectCode('SESSION_QUERY_STALE_CURSOR')) + }) + + it('rejects invalid requests, filters, cursors, and direct config', async () => { + const ctx = await liveContext({ path: ':memory:', defaultLimit: 2, maxLimit: 3 }) + const session = ctx.sessions.create(SessionId('valid'), { seed: messageEvents('needle') }) + for (const request of [ + { sessionId: session.id, query: '' }, + { sessionId: session.id, query: 'needle', limit: 0 }, + { sessionId: session.id, query: 'needle', limit: 4 }, + { sessionId: session.id, query: 'needle', filters: [{ kind: 'seq', from: 2, to: 1 }] }, + { sessionId: session.id, query: 'needle', filters: [{ kind: 'surface', values: ['future'] }] }, + ] as const) { + await expect(ctx.sessionSearch.searchEvents(request as never)).rejects.toBeInstanceOf(Error) + } + await expect(ctx.sessionSearch.searchSessions({ + query: 'needle', + sessionFilters: [{ kind: 'availability', values: ['remote' as never] }], + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionSearch.searchEvents({ sessionId: session.id, query: 'needle', cursor: 'not-json' })) + .rejects.toThrow(expectCode('SESSION_QUERY_INVALID_CURSOR')) + await expect(ctx.sessionSearch.searchEvents({ sessionId: SessionId('absent'), query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_SESSION_NOT_FOUND')) + + for (const config of [ + { path: '' }, + { path: ':memory:', defaultLimit: 0 }, + { path: ':memory:', maxLimit: 0 }, + { path: ':memory:', snippetChars: 0 }, + { path: ':memory:', defaultLimit: 3, maxLimit: 2 }, + { path: ':memory:', journalMode: 'memory' }, + ]) { + const direct = new Context() + await direct.plugin(SessionStore) + expect(() => new SessionSearchSqlite(direct, config as never)) + .toThrow(expectCode('SESSION_QUERY_INVALID_CONFIG')) + } + }) +}) + +describe('SQLite reconciliation and source lifecycle', () => { + it('mounts persistence dynamically, shadows with TEMP live rows, reveals, and hides on unmount', async () => { + const shared = header('shared', 10, { cwd: '/work' }) + const durable = header('durable', 5) + TestPersistence.reset([ + { meta: shared, events: messageEvents('persisted needle') }, + { meta: durable, events: messageEvents('durable needle') }, + ]) + const ctx = await liveContext() + await expect(ctx.sessionSearch.searchSessions({ query: 'durable' })).resolves.toEqual({ items: [] }) + const persistenceFiber = await ctx.plugin(TestPersistence) + + await expect(ctx.sessionSearch.searchSessions({ query: 'durable' })) + .resolves.toMatchObject({ items: [{ header: durable, live: false, persisted: true }] }) + const live = ctx.sessions.prepare(shared.id, { meta: { createdAt: 10, cwd: '/work' } }) + live.append('user/message', { content: [{ type: 'text', text: 'live needle' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + const detach = ctx.sessions.enter(live) + ctx.sessions.announce(live) + + await expect(ctx.sessionSearch.searchSessions({ query: 'persisted' })).resolves.toEqual({ items: [] }) + await expect(ctx.sessionSearch.searchSessions({ query: 'live' })) + .resolves.toMatchObject({ items: [{ header: shared, live: true, persisted: true }] }) + detach() + await expect(ctx.sessionSearch.searchSessions({ query: 'persisted' })) + .resolves.toMatchObject({ items: [{ header: shared, live: false, persisted: true }] }) + + await persistenceFiber.dispose() + await expect(ctx.sessionSearch.searchSessions({ query: 'durable' })).resolves.toEqual({ items: [] }) + await expect(ctx.sessionSearch.searchEvents({ sessionId: durable.id, query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_SESSION_NOT_FOUND')) + }) + + it('restarts observation when persistence unmounts during an asynchronous list', async () => { + const durable = header('racing') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + const persistenceFiber = await ctx.plugin(TestPersistence) + let release!: () => void + TestPersistence.listGate = new Promise((resolve) => { release = resolve }) + let markStarted!: () => void + const started = new Promise((resolve) => { markStarted = resolve }) + TestPersistence.listStarted = () => { + TestPersistence.listStarted = undefined + markStarted() + } + + const search = ctx.sessionSearch.searchSessions({ query: 'needle' }) + await started + await persistenceFiber.dispose() + release() + await expect(search).resolves.toEqual({ items: [] }) + }) + + it('rejects immutable header conflicts between live and persisted sources', async () => { + const shared = header('conflict', 10) + TestPersistence.reset([{ meta: shared, events: messageEvents('persisted needle') }]) + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + ctx.sessions.create(shared.id, { seed: messageEvents('live needle'), meta: { createdAt: 11 } }) + + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_SOURCE_CONFLICT')) + }) + + it('preserves unchanged persisted generations while reconciling new, changed, and deleted rows', async () => { + const path = await temporaryPath() + const unchanged = header('unchanged') + const changed = header('changed') + const deleted = header('deleted') + TestPersistence.reset([ + { meta: unchanged, events: messageEvents('unchanged needle') }, + { meta: changed, events: messageEvents('old needle') }, + { meta: deleted, events: messageEvents('deleted needle') }, + ]) + const first = new Context() + await first.plugin(SessionStore) + const firstPersistence = await first.plugin(TestPersistence) + const firstSearch = await first.plugin(SessionSearchSqlite, { path }) + await first.sessionSearch.searchSessions({ query: 'needle' }) + await firstSearch.dispose() + await firstPersistence.dispose() + + const beforeDb = new DatabaseSync(path) + const beforeRows = beforeDb.prepare('SELECT id, generation FROM persisted_sessions ORDER BY id').all() as Array<{ id: string; generation: number }> + beforeDb.close() + const before = new Map(beforeRows.map(row => [row.id, row.generation])) + + const added = header('added') + TestPersistence.entries.delete(deleted.id) + TestPersistence.entries.set(changed.id, { meta: changed, events: messageEvents('changed needle') }) + TestPersistence.entries.set(added.id, { meta: added, events: messageEvents('added needle') }) + const second = new Context() + await second.plugin(SessionStore) + const secondPersistence = await second.plugin(TestPersistence) + const secondSearch = await second.plugin(SessionSearchSqlite, { path }) + const result = await second.sessionSearch.searchSessions({ query: 'needle' }) + expect(result.items.map(item => item.header.id).sort()).toEqual([added.id, changed.id, unchanged.id].sort()) + await secondSearch.dispose() + await secondPersistence.dispose() + + const afterDb = new DatabaseSync(path) + const afterRows = afterDb.prepare('SELECT id, generation FROM persisted_sessions ORDER BY id').all() as Array<{ id: string; generation: number }> + afterDb.close() + const after = new Map(afterRows.map(row => [row.id, row.generation])) + expect(after.get(unchanged.id)).toBe(before.get(unchanged.id)) + expect(after.get(changed.id)).toBeGreaterThan(before.get(changed.id)!) + expect(after.has(deleted.id)).toBe(false) + expect(after.has(added.id)).toBe(true) + }) + + it('drops connection-local live overlays on reopen and retains persistent bases', async () => { + const path = await temporaryPath() + const shared = header('shared', 10) + TestPersistence.reset([{ meta: shared, events: messageEvents('persisted needle') }]) + const first = new Context() + await first.plugin(SessionStore) + const persistence = await first.plugin(TestPersistence) + const live = first.sessions.create(shared.id, { seed: messageEvents('live needle'), meta: { createdAt: 10 } }) + const search = await first.plugin(SessionSearchSqlite, { path }) + await expect(first.sessionSearch.searchEvents({ sessionId: live.id, query: 'live' })).resolves.toMatchObject({ items: [{}] }) + await search.dispose() + await persistence.dispose() + + const second = new Context() + await second.plugin(SessionStore) + const persistenceAgain = await second.plugin(TestPersistence) + const searchAgain = await second.plugin(SessionSearchSqlite, { path }) + await expect(second.sessionSearch.searchSessions({ query: 'live' })).resolves.toEqual({ items: [] }) + await expect(second.sessionSearch.searchSessions({ query: 'persisted' })) + .resolves.toMatchObject({ items: [{ header: shared, live: false, persisted: true }] }) + await searchAgain.dispose() + await persistenceAgain.dispose() + }) + + it('recovers on the next search after source and SQLite transaction failures', async () => { + TestPersistence.reset([{ meta: header('durable'), events: messageEvents('durable needle') }]) + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + TestPersistence.failure = 'offline' + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_PERSISTENCE_FAILED')) + const signal = new AbortController().signal + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' }, { signal })) + .rejects.toThrow(expectCode('SESSION_QUERY_PERSISTENCE_FAILED')) + TestPersistence.failure = new Error('still offline') + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' }, { signal })) + .rejects.toThrow(expectCode('SESSION_QUERY_PERSISTENCE_FAILED')) + TestPersistence.failure = undefined + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })).resolves.toMatchObject({ items: [{}] }) + + const live = ctx.sessions.create(SessionId('live'), { seed: messageEvents('base') }) + await ctx.sessionSearch.searchEvents({ sessionId: live.id, query: 'base' }) + const db = (ctx.sessionSearch as unknown as { _db: DatabaseSync })._db + db.exec('PRAGMA query_only = ON') + live.append('user/message', { content: [{ type: 'text', text: 'retry needle' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + await expect(ctx.sessionSearch.searchEvents({ sessionId: live.id, query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) + db.exec('PRAGMA query_only = OFF') + await expect(ctx.sessionSearch.searchEvents({ sessionId: live.id, query: 'needle' })) + .resolves.toMatchObject({ items: [{ seq: 1 }] }) + }) +}) + +describe('SQLite schema, cancellation, and real persistence integration', () => { + it('resets a recognized incompatible derived schema but refuses a foreign database', async () => { + const stalePath = await temporaryPath('stale.db') + const stale = new DatabaseSync(stalePath) + stale.exec(`PRAGMA application_id = ${SESSION_QUERY_SQLITE_APPLICATION_ID}`) + stale.exec('PRAGMA user_version = 999') + stale.exec('CREATE TABLE stale(value TEXT)') + stale.close() + const staleCtx = await liveContext({ path: stalePath }) + staleCtx.sessions.create(SessionId('live'), { seed: messageEvents('needle') }) + await staleCtx.sessionSearch.searchSessions({ query: 'needle' }) + await (staleCtx.sessionSearch as SessionSearchSqlite).close() + const rebuilt = new DatabaseSync(stalePath) + expect((rebuilt.prepare('PRAGMA user_version').get() as { user_version: number }).user_version) + .toBe(SESSION_QUERY_SQLITE_SCHEMA_VERSION) + expect(rebuilt.prepare("SELECT name FROM sqlite_master WHERE name = 'stale'").get()).toBeUndefined() + rebuilt.close() + + const foreignPath = await temporaryPath('foreign.db') + const foreign = new DatabaseSync(foreignPath) + foreign.exec('CREATE TABLE canonical(value TEXT)') + foreign.exec("INSERT INTO canonical VALUES ('safe')") + foreign.close() + const foreignCtx = await liveContext({ path: foreignPath }) + await expect(foreignCtx.sessionSearch.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) + const stillForeign = new DatabaseSync(foreignPath) + expect(stillForeign.prepare('SELECT value FROM canonical').get()).toEqual({ value: 'safe' }) + stillForeign.close() + + const otherAppPath = await temporaryPath('other-app.db') + const otherApp = new DatabaseSync(otherAppPath) + otherApp.exec('PRAGMA application_id = 123') + otherApp.close() + const otherAppCtx = await liveContext({ path: otherAppPath }) + await expect(otherAppCtx.sessionSearch.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) + }) + + it('cancels both queued and in-flight source waits without committing them', async () => { + TestPersistence.reset() + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + + const boundaryController = new AbortController() + const boundary = ctx.sessionSearch.searchSessions({ query: 'needle' }, { signal: boundaryController.signal }) + queueMicrotask(() => { boundaryController.abort() }) + await expect(boundary).rejects.toThrow(expectCode('SESSION_QUERY_ABORTED')) + + const readyController = new AbortController() + readyController.abort() + const internals = ctx.sessionSearch as unknown as { + _ensureReady(signal: AbortSignal): Promise + } + await expect(internals._ensureReady(readyController.signal)) + .rejects.toThrow(expectCode('SESSION_QUERY_ABORTED')) + + let releaseBlocking!: () => void + TestPersistence.listGate = new Promise((resolve) => { releaseBlocking = resolve }) + let markBlockingStarted!: () => void + const blockingStarted = new Promise((resolve) => { markBlockingStarted = resolve }) + TestPersistence.listStarted = () => { + TestPersistence.listStarted = undefined + markBlockingStarted() + } + const blocking = ctx.sessionSearch.searchSessions({ query: 'needle' }) + await blockingStarted + + const queuedController = new AbortController() + const queued = ctx.sessionSearch.searchSessions({ query: 'needle' }, { signal: queuedController.signal }) + queuedController.abort() + await expect(queued).rejects.toThrow(expectCode('SESSION_QUERY_ABORTED')) + + releaseBlocking() + await expect(blocking).resolves.toEqual({ items: [] }) + + TestPersistence.entries.set(SessionId('uncommitted'), { + meta: header('uncommitted'), + events: messageEvents('durable needle'), + }) + let releaseActive!: () => void + TestPersistence.listGate = new Promise((resolve) => { releaseActive = resolve }) + let markActiveStarted!: () => void + const activeStarted = new Promise((resolve) => { markActiveStarted = resolve }) + TestPersistence.listStarted = () => { + TestPersistence.listStarted = undefined + markActiveStarted() + } + const activeController = new AbortController() + const active = ctx.sessionSearch.searchSessions({ query: 'needle' }, { signal: activeController.signal }) + await activeStarted + activeController.abort() + await expect(active).rejects.toThrow(expectCode('SESSION_QUERY_ABORTED')) + releaseActive() + + const db = (ctx.sessionSearch as unknown as { _db: DatabaseSync })._db + expect(db.prepare('SELECT COUNT(*) AS count FROM persisted_sessions').get()).toEqual({ count: 0 }) + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .resolves.toMatchObject({ items: [{ header: { id: SessionId('uncommitted') } }] }) + }) + + it('rejects queued and future work when close waits for an accepted operation', async () => { + TestPersistence.reset() + let release!: () => void + TestPersistence.listGate = new Promise((resolve) => { release = resolve }) + let markStarted!: () => void + const started = new Promise((resolve) => { markStarted = resolve }) + TestPersistence.listStarted = () => { + TestPersistence.listStarted = undefined + markStarted() + } + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + const search = ctx.sessionSearch as SessionSearchSqlite + const accepted = search.searchSessions({ query: 'needle' }) + await started + const queued = search.searchSessions({ query: 'needle' }) + const closing = search.close() + release() + + await expect(accepted).resolves.toEqual({ items: [] }) + await expect(queued).rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) + await closing + await expect(search.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) + await search.close() + }) + + it('combines the real SQLite persistence backend with the real search service keylessly', async () => { + const persistencePath = await temporaryPath('canonical.db') + const searchPath = await temporaryPath('derived.db') + const ctx = new Context() + await ctx.plugin(SessionStore) + const persistence = await ctx.plugin(SessionPersistenceSqlite, { path: persistencePath }) + const search = await ctx.plugin(SessionSearchSqlite, { path: searchPath }) + const meta = header('real', 10, { cwd: '/work' }) + await ctx.sessionPersistence.create(meta) + await ctx.sessionPersistence.append(meta.id, messageEvents('real SQLite needle')) + + await expect(ctx.sessionSearch.searchSessions({ query: 'SQLite needle' })) + .resolves.toMatchObject({ items: [{ header: meta, persisted: true, live: false }] }) + await expect(ctx.sessionSearch.searchEvents({ sessionId: meta.id, query: 'SQLite needle' })) + .resolves.toMatchObject({ items: [{ sessionId: meta.id, seq: 0 }] }) + await expect(ctx.sessionSearch.searchEvents({ sessionId: SessionId('absent'), query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_SESSION_NOT_FOUND')) + await search.dispose() + await expect(ctx.sessionPersistence.load(meta.id)).resolves.toMatchObject({ meta, events: [{ seq: 0 }] }) + await persistence.dispose() + }) +}) diff --git a/packages/session-query/session-query-sqlite/tsconfig.json b/packages/session-query/session-query-sqlite/tsconfig.json new file mode 100644 index 0000000000..ea16cdbe96 --- /dev/null +++ b/packages/session-query/session-query-sqlite/tsconfig.json @@ -0,0 +1,30 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../../vendor/cosmokit" + }, + { + "path": "../../../vendor/cordis" + }, + { + "path": "../../../vendor/schemastery" + }, + { + "path": "../../core/session" + }, + { + "path": "../../session-persistence/session-persistence" + }, + { + "path": "../session-query" + } + ] +} diff --git a/packages/session-query/session-query/README.md b/packages/session-query/session-query/README.md index 55f9b32fcd..6f9e9bbd7a 100644 --- a/packages/session-query/session-query/README.md +++ b/packages/session-query/session-query/README.md @@ -1,6 +1,6 @@ # @deepseek-ai/dsh-session-query -Exact session-history retrieval through `ctx.sessionQuery`. The service presents live `ctx.sessions` and an optional, dynamically mounted `ctx.sessionPersistence` as one logical corpus. Matching ids produce one record: live events win, while `live` and `persisted` report both source availabilities. Conflicting immutable headers fail with `SESSION_QUERY_SOURCE_CONFLICT`. +Session-history query contracts and provider-independent helpers. The concrete `ctx.sessionQuery` service presents live `ctx.sessions` and an optional, dynamically mounted `ctx.sessionPersistence` as one logical corpus for exact reads and semantic scans. The abstract `ctx.sessionSearch` service defines full-text search without introducing a provider registry. This is trusted context-wide infrastructure. It performs no caller authorization; a future model tool or UI must constrain which sessions its caller may inspect. @@ -8,11 +8,24 @@ This is trusted context-wide infrastructure. It performs no caller authorization - `listSessions()` reads current persistence metadata, merges live records with live precedence, and returns cloned records in deterministic newest-first order. - `listEvents(sessionId)` loads the live-preferred raw log and classifies each event as `current`, `shadowed`, or `log-only` with the shared `dsh-session` surface fold. +- `filterEvents(sessionId, filters)` extracts first-party semantic documents and applies provider-independent metadata and literal-text predicates in ascending seq order. - `readEvent(request)` returns a cloned header, the full target event, and a bounded raw-seq window. `before` and `after` default to zero and may not exceed `readWindowMax`. Persistence is optional and may mount or unmount dynamically. A cross-corpus list fails with `SESSION_QUERY_PERSISTENCE_FAILED` while mounted persistence is unreadable. A read targeting a known live session does not consult persistence, so durable backend health cannot make current in-memory history unreadable. Persisted exact reads list before loading, and reject a metadata mismatch rather than combining inconsistent observations. -`SessionQueryError.code` is a closed union: `SESSION_QUERY_EVENT_NOT_FOUND`, `SESSION_QUERY_INVALID_CONFIG`, `SESSION_QUERY_INVALID_SURFACE`, `SESSION_QUERY_INVALID_WINDOW`, `SESSION_QUERY_PERSISTENCE_FAILED`, `SESSION_QUERY_SESSION_NOT_FOUND`, and `SESSION_QUERY_SOURCE_CONFLICT`. +## Filtering and extraction + +`SessionResultFilter` covers id, nullable cwd, created-at range, nullable parent, and source availability. `SessionEventResultFilter` covers seq/time ranges, event type, surface, and semantic text. Filter arrays are ANDed; values within one list clause are ORed. Empty list values match nothing, ranges are inclusive, and malformed ranges or closed-union values fail with `SESSION_QUERY_INVALID_FILTER`. + +The text clause is deliberately independent of FTS providers: caller text is escaped into a Unicode, case-insensitive regular expression, and each whitespace run matches one or more whitespace characters. It is a literal semantic-text scan, not a full-text query. `extractSessionEventText()` and `buildSessionEventSearchDocuments()` define the shared first-party document projection; structural boundaries, stream chunks, request headers, and unknown declaration-merged variants produce no document. + +## Full-text seam + +`SessionSearchService` owns the independent `ctx.sessionSearch` key. `searchSessions(request, exec?)` groups the logical corpus by strongest matching event; `searchEvents(request, exec?)` searches one logical session. Both return opaque cursor pages, accept optional cancellation, and expose snippets without provider-specific numeric scores. Search requests accept only metadata event filters, because literal-text filtering is the scan path described above. + +The package has no provider coordinator or registration protocol. A concrete backend owns observation, reconciliation, ranking, cursor generations, and query execution as one lifecycle; the first implementation is [`@deepseek-ai/dsh-session-query-sqlite`](../session-query-sqlite/README.md). + +`SessionQueryError.code` is a closed union covering request validation, missing targets, malformed surfaces, source conflicts, persistence/index failures, cancellation, and invalid or stale cursors; the exact literals are defined in [`src/config.ts`](src/config.ts). ## Configuration @@ -20,4 +33,4 @@ Persistence is optional and may mount or unmount dynamically. A cross-corpus lis |---|---:|---| | `readWindowMax` | `50` | Maximum `before` or `after` raw-event count. | -This phase deliberately has no filters, lineage/provenance traversal, extraction registry, search-provider protocol, index synchronization, or model-facing tool. Full-text search belongs beside its first real implementation; the proposed SQLite package and its single transaction/reconciliation owner are described in the [phase-two RFC](../../../docs/rfc/proposed/feature/2026-07-10-sqlite-session-query-provider.md). +The package deliberately has no lineage/provenance traversal, extractor registry, search-provider registry, index synchronization, caller authorization, or model-facing tool. The SQLite ownership and tokenizer decisions are recorded in the [implemented search RFC](../../../docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md). diff --git a/packages/session-query/session-query/src/config.ts b/packages/session-query/session-query/src/config.ts index 2736f68cbd..296d3830c6 100644 --- a/packages/session-query/session-query/src/config.ts +++ b/packages/session-query/session-query/src/config.ts @@ -1,4 +1,4 @@ -/** Public configuration and typed failures for session-query. */ +/** Public configuration and typed failures for session-query and search. */ import { HarnessError } from '@deepseek-ai/dsh-llm' @@ -11,14 +11,21 @@ export interface Config { readWindowMax?: number } -/** Stable machine-routable failure taxonomy for exact session reads. */ +/** Stable machine-routable failure taxonomy for session reads and search. */ export type SessionQueryErrorCode = + | 'SESSION_QUERY_ABORTED' | 'SESSION_QUERY_EVENT_NOT_FOUND' + | 'SESSION_QUERY_INDEX_FAILED' | 'SESSION_QUERY_INVALID_CONFIG' + | 'SESSION_QUERY_INVALID_CURSOR' + | 'SESSION_QUERY_INVALID_FILTER' + | 'SESSION_QUERY_INVALID_LIMIT' + | 'SESSION_QUERY_INVALID_QUERY' | 'SESSION_QUERY_INVALID_SURFACE' | 'SESSION_QUERY_INVALID_WINDOW' | 'SESSION_QUERY_PERSISTENCE_FAILED' | 'SESSION_QUERY_SESSION_NOT_FOUND' + | 'SESSION_QUERY_STALE_CURSOR' | 'SESSION_QUERY_SOURCE_CONFLICT' /** Typed session-query failure whose `code` is one closed taxonomy member. */ diff --git a/packages/session-query/session-query/src/corpus.ts b/packages/session-query/session-query/src/corpus.ts index ebc3d92577..c8ddca3caa 100644 --- a/packages/session-query/session-query/src/corpus.ts +++ b/packages/session-query/session-query/src/corpus.ts @@ -5,6 +5,7 @@ import type { Session, SessionEvent, SessionHeader, SessionId } from '@deepseek- import type SessionPersistence from '@deepseek-ai/dsh-session-persistence' import type { SessionRecord } from './types.ts' import { SessionQueryError } from './config.ts' +import { assertSessionHeadersCompatible } from './sources.ts' /** Detached source selected for one exact read. */ export interface LogicalSession { @@ -45,7 +46,7 @@ export class SessionCorpus { } for (const session of this._ctx.sessions.list()) { const durable = records.get(session.id) - if (durable !== undefined) assertCompatibleHeaders(session.header, durable.header) + if (durable !== undefined) assertSessionHeadersCompatible(session.header, durable.header) records.set(session.id, { header: structuredClone(session.header), live: true, @@ -80,7 +81,7 @@ export class SessionCorpus { { cause: error }, ) } - assertCompatibleHeaders(loaded.meta, listed) + assertSessionHeadersCompatible(loaded.meta, listed) return { header: structuredClone(loaded.meta), events: loaded.events.map(event => structuredClone(event)), @@ -107,22 +108,6 @@ function snapshotLive(session: Session): LogicalSession { } } -function assertCompatibleHeaders(a: SessionHeader, b: SessionHeader): void { - if ( - a.version !== b.version - || a.id !== b.id - || a.createdAt !== b.createdAt - || a.cwd !== b.cwd - || a.parentSession !== b.parentSession - || a.seedLength !== b.seedLength - ) { - throw new SessionQueryError( - `live and persisted headers conflict for session "${a.id}"`, - 'SESSION_QUERY_SOURCE_CONFLICT', - ) - } -} - function compareSessions(a: SessionRecord, b: SessionRecord): number { return b.header.createdAt - a.header.createdAt || a.header.id.localeCompare(b.header.id) } diff --git a/packages/session-query/session-query/src/documents.ts b/packages/session-query/session-query/src/documents.ts new file mode 100644 index 0000000000..f58029ae67 --- /dev/null +++ b/packages/session-query/session-query/src/documents.ts @@ -0,0 +1,74 @@ +/** Shared event metadata and semantic-document projection. */ + +import { foldSurface } from '@deepseek-ai/dsh-session' +import type { SessionEvent, SessionId } from '@deepseek-ai/dsh-session' +import type { SessionEventRecord, SessionEventSearchDocument, SessionEventSurface } from './types.ts' +import { SessionQueryError } from './config.ts' +import { extractSessionEventText } from './extraction.ts' + +/** + * Project a raw log into lightweight surface-aware event records. + * @param sessionId - session that owns the log. + * @param events - complete contiguous raw event log. + * @returns one record per event in ascending seq order. + */ +export function buildSessionEventRecords( + sessionId: SessionId, + events: readonly SessionEvent[], +): SessionEventRecord[] { + const surfaceBySeq = classifySurface(events) + return events.map(event => ({ + sessionId, + seq: event.seq, + type: event.type, + time: event.time, + surface: surfaceBySeq.get(event.seq) ?? 'log-only', + })) +} + +/** + * Build first-party semantic documents for one complete raw event log. + * @param sessionId - session that owns the log. + * @param events - complete contiguous raw event log. + * @returns searchable documents in ascending seq order; structural events are omitted. + */ +export function buildSessionEventSearchDocuments( + sessionId: SessionId, + events: readonly SessionEvent[], +): SessionEventSearchDocument[] { + const surfaceBySeq = classifySurface(events) + const documents: SessionEventSearchDocument[] = [] + for (const event of events) { + const text = extractSessionEventText(event) + if (text.length === 0) continue + documents.push({ + sessionId, + seq: event.seq, + type: event.type, + time: event.time, + surface: surfaceBySeq.get(event.seq) ?? 'log-only', + text, + }) + } + return documents +} + +function classifySurface(events: readonly SessionEvent[]): Map { + let folded: ReturnType + try { + folded = foldSurface(events) + } catch (error: unknown) { + throw new SessionQueryError( + /* v8 ignore next -- foldSurface throws Error instances */ + `invalid session surface: ${error instanceof Error ? error.message : 'unknown error'}`, + 'SESSION_QUERY_INVALID_SURFACE', + { cause: error }, + ) + } + const result = new Map() + for (const node of folded.nodes) result.set(node.seq, 'current') + for (const replacement of folded.replacements) { + for (const seq of replacement.shadowedSeqs) result.set(seq, 'shadowed') + } + return result +} diff --git a/packages/session-query/session-query/src/extraction.ts b/packages/session-query/session-query/src/extraction.ts new file mode 100644 index 0000000000..96a0af247b --- /dev/null +++ b/packages/session-query/session-query/src/extraction.ts @@ -0,0 +1,93 @@ +/** First-party semantic text extraction for session-query consumers. */ + +import type { SessionEvent } from '@deepseek-ai/dsh-session' + +/** + * Extract searchable semantic text from one first-party session event. + * + * Structural boundaries, raw stream chunks, request envelopes, and unknown + * declaration-merged events contribute no text. + * @param event - event to inspect. + * @returns newline-joined semantic text, or an empty string when non-searchable. + */ +export function extractSessionEventText(event: SessionEvent): string { + switch (event.type) { + case 'user/message': + case 'assistant/message': + case 'context/message': + case 'steering/message': + return contentText(event.data.content) + case 'prompt/blocked': + return joinText([contentText(event.data.content), event.data.reason]) + case 'tool/call': + return joinText([event.data.name, event.data.arguments]) + case 'tool/result': + return joinText([ + contentText(event.data.content), + event.data.error?.name ?? '', + event.data.error?.code ?? '', + ]) + case 'todo/write': + return joinText(event.data.todos.flatMap(todo => [todo.status, todo.content])) + case 'turn/end': + return turnEndText(event.data.reason) + case 'turn/start': + case 'step/start': + case 'step/end': + case 'assistant/chunk': + case 'request/header': + case 'request/header-delta': + return '' + // SessionEventMap is merge-extensible. Unknown events remain + // non-searchable until a concrete first-party consumer defines semantics. + default: + return '' + } +} + +function turnEndText(reason: SessionEvent<'turn/end'>['data']['reason']): string { + switch (reason.kind) { + case 'error': + return joinText(['error', reason.message, reason.code ?? '']) + case 'aborted': + return joinText(['aborted', reason.reason ?? '']) + case 'rejected': + return joinText(['rejected', reason.reason]) + case 'disposed': + case 'max-tokens': + case 'interrupted': + return reason.kind + case 'completed': + return '' + // TurnEndReasonMap is merge-extensible. Unknown outcomes stay out until + // their owner defines which detail is semantic rather than structural. + default: + return '' + } +} + +type SessionContentBlock = SessionEvent<'user/message'>['data']['content'][number] + +function contentText(content: readonly SessionContentBlock[]): string { + return joinText(content.flatMap(blockText)) +} + +function blockText(block: SessionContentBlock): string[] { + switch (block.type) { + case 'text': + case 'reasoning': + return [block.text] + case 'tool-call': + return [block.name, block.arguments] + case 'tool-result': + return block.content.flatMap(blockText) + // ContentBlockMap is merge-extensible. Unknown blocks do not become + // searchable merely because their payload happens to contain strings. + default: + return [] + } +} + +function joinText(parts: readonly string[]): string { + return parts.map(part => part.trim()).filter(Boolean).join('\n') +} diff --git a/packages/session-query/session-query/src/filters.ts b/packages/session-query/session-query/src/filters.ts new file mode 100644 index 0000000000..c7a7b40dd4 --- /dev/null +++ b/packages/session-query/session-query/src/filters.ts @@ -0,0 +1,132 @@ +/** Pure provider-independent predicates for logical sessions and event text. */ + +import type { SessionRecord, SessionEventSearchDocument, SessionEventResultFilter, SessionResultFilter, SessionResultRange } from './types.ts' +import { SessionQueryError } from './config.ts' + +/** + * Apply ANDed logical-session filters while preserving input order. + * @param records - detached logical-session records to inspect. + * @param filters - clauses whose list values are ORed within each clause. + * @returns records accepted by every clause. + */ +export function filterSessionResults( + records: readonly T[], + filters: readonly SessionResultFilter[] = [], +): T[] { + const predicates = filters.map(sessionPredicate) + return records.filter(record => predicates.every(predicate => predicate(record))) +} + +/** + * Apply ANDed event filters to extracted semantic documents. + * @param documents - semantic documents produced by {@link buildSessionEventSearchDocuments}. + * @param filters - metadata and literal-text predicates. + * @returns documents accepted by every clause, in input order. + */ +export function filterSessionEventDocuments( + documents: readonly T[], + filters: readonly SessionEventResultFilter[] = [], +): T[] { + const predicates = filters.map(eventPredicate) + return documents.filter(document => predicates.every(predicate => predicate(document))) +} + +/** + * Compile a literal case-insensitive, whitespace-flexible semantic-text match. + * @param text - caller-provided literal text. + * @returns Unicode-aware regular expression safe from regex injection. + */ +export function compileSessionTextFilter(text: string): RegExp { + const trimmed = text.trim() + if (trimmed.length === 0) { + throw new SessionQueryError( + 'session text filter must contain non-whitespace text', + 'SESSION_QUERY_INVALID_FILTER', + ) + } + const pattern = trimmed + .split(/\s+/u) + .map(part => part.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&')) + .join('\\s+') + return new RegExp(pattern, 'iu') +} + +function sessionPredicate(filter: SessionResultFilter): (record: SessionRecord) => boolean { + switch (filter.kind) { + case 'id': + return record => filter.values.includes(record.header.id) + case 'cwd': + return record => filter.values.includes(record.header.cwd ?? null) + case 'created-at': { + const range = validateRange(filter.kind, filter) + return record => matchesRange(record.header.createdAt, range) + } + case 'parent': + return record => filter.values.includes(record.header.parentSession ?? null) + case 'availability': + assertAllowedValues(filter.kind, filter.values, ['live', 'persisted']) + return record => filter.values.some(value => value === 'live' ? record.live : record.persisted) + } +} + +function eventPredicate(filter: SessionEventResultFilter): (document: SessionEventSearchDocument) => boolean { + switch (filter.kind) { + case 'seq': { + const range = validateRange(filter.kind, filter) + return document => matchesRange(document.seq, range) + } + case 'time': { + const range = validateRange(filter.kind, filter) + return document => matchesRange(document.time, range) + } + case 'type': + return document => filter.values.includes(document.type) + case 'surface': + assertAllowedValues(filter.kind, filter.values, ['current', 'shadowed', 'log-only']) + return document => filter.values.includes(document.surface) + case 'text': { + const pattern = compileSessionTextFilter(filter.text) + return document => pattern.test(document.text) + } + } +} + +function assertAllowedValues( + name: string, + values: readonly string[], + allowed: readonly string[], +): void { + for (const value of values) { + if (!allowed.includes(value)) { + throw new SessionQueryError( + `session ${name} filter contains unknown value "${value}"`, + 'SESSION_QUERY_INVALID_FILTER', + ) + } + } +} + +function validateRange(name: string, range: SessionResultRange): SessionResultRange { + if (range.from !== undefined && !Number.isFinite(range.from)) { + throw invalidRange(name, 'from must be finite') + } + if (range.to !== undefined && !Number.isFinite(range.to)) { + throw invalidRange(name, 'to must be finite') + } + if (range.from !== undefined && range.to !== undefined && range.from > range.to) { + throw invalidRange(name, 'from must be less than or equal to to') + } + return range +} + +function matchesRange(value: number, range: SessionResultRange): boolean { + return (range.from === undefined || value >= range.from) + && (range.to === undefined || value <= range.to) +} + +function invalidRange(name: string, detail: string): SessionQueryError { + return new SessionQueryError( + `session ${name} filter ${detail}`, + 'SESSION_QUERY_INVALID_FILTER', + ) +} diff --git a/packages/session-query/session-query/src/index.ts b/packages/session-query/session-query/src/index.ts index 828fe2ec88..243c86746b 100644 --- a/packages/session-query/session-query/src/index.ts +++ b/packages/session-query/session-query/src/index.ts @@ -6,13 +6,20 @@ import { Context, Service } from 'cordis' import z from 'schemastery' -import { foldSurface } from '@deepseek-ai/dsh-session' -import type { SessionEvent, SessionId } from '@deepseek-ai/dsh-session' +import type { SessionId } from '@deepseek-ai/dsh-session' import type { + SessionEventResultFilter, SessionEventReadRequest, SessionEventRecord, + SessionEventSearchHit, + SessionEventSearchDocument, + SessionEventSearchRequest, SessionEventWindow, SessionRecord, + SessionSearchExecContext, + SessionSearchHit, + SessionSearchPage, + SessionSearchRequest, } from './types.ts' import { SESSION_QUERY_READ_WINDOW_MAX, @@ -20,17 +27,58 @@ import { type Config, } from './config.ts' import { SessionCorpus } from './corpus.ts' +import { buildSessionEventRecords, buildSessionEventSearchDocuments } from './documents.ts' +import { filterSessionEventDocuments } from './filters.ts' export type * from './types.ts' export type { Config, SessionQueryErrorCode } from './config.ts' export { SESSION_QUERY_READ_WINDOW_MAX, SessionQueryError } from './config.ts' +export { extractSessionEventText } from './extraction.ts' +export { buildSessionEventRecords, buildSessionEventSearchDocuments } from './documents.ts' +export { compileSessionTextFilter, filterSessionEventDocuments, filterSessionResults } from './filters.ts' +export { assertSessionHeadersCompatible } from './sources.ts' declare module 'cordis' { interface Context { sessionQuery: SessionQueryService + sessionSearch: SessionSearchService } } +/** + * Abstract full-text search service implemented by one concrete backend. + * + * The implementation owns source observation, reconciliation, cursor + * generations, ranking, and query execution as one lifecycle. + */ +export abstract class SessionSearchService extends Service { + constructor(ctx: Context) { + super(ctx, 'sessionSearch') + } + + /** + * Search the live-preferred logical corpus and group by session. + * @param request - query text, metadata filters, page size, and cursor. + * @param exec - optional cancellation control. + * @returns session hits ranked by their strongest matching event. + */ + abstract searchSessions( + request: SessionSearchRequest, + exec?: SessionSearchExecContext, + ): Promise> + + /** + * Search events within one live-preferred logical session. + * @param request - target session, query text, filters, page size, and cursor. + * @param exec - optional cancellation control. + * @returns matching event hits in deterministic relevance order. + */ + abstract searchEvents( + request: SessionEventSearchRequest, + exec?: SessionSearchExecContext, + ): Promise> +} + /** Live-preferred logical-corpus and exact-event read service. */ export class SessionQueryService extends Service { static inject = ['sessions'] @@ -68,7 +116,22 @@ export class SessionQueryService extends Service { */ async listEvents(sessionId: SessionId): Promise { const loaded = await this._corpus.load(sessionId) - return eventRecords(sessionId, loaded.events) + return buildSessionEventRecords(sessionId, loaded.events) + } + + /** + * Scan first-party semantic event documents with provider-independent filters. + * @param sessionId - live-preferred session id to scan. + * @param filters - ANDed metadata and literal-text predicates. + * @returns matching semantic documents in ascending seq order. + */ + async filterEvents( + sessionId: SessionId, + filters: readonly SessionEventResultFilter[], + ): Promise { + const loaded = await this._corpus.load(sessionId) + const documents = buildSessionEventSearchDocuments(sessionId, loaded.events) + return filterSessionEventDocuments(documents, filters) } /** @@ -110,27 +173,4 @@ export class SessionQueryService extends Service { } } -function eventRecords(sessionId: SessionId, events: readonly SessionEvent[]): SessionEventRecord[] { - let folded: ReturnType - try { - folded = foldSurface(events) - } catch (error: unknown) { - throw new SessionQueryError( - /* v8 ignore next -- foldSurface throws Error instances */ - `invalid session surface: ${error instanceof Error ? error.message : 'unknown error'}`, - 'SESSION_QUERY_INVALID_SURFACE', - { cause: error }, - ) - } - const current = new Set(folded.nodes.map(node => node.seq)) - const shadowed = new Set(folded.replacements.flatMap(replacement => replacement.shadowedSeqs)) - return events.map(event => ({ - sessionId, - seq: event.seq, - type: event.type, - time: event.time, - surface: current.has(event.seq) ? 'current' : shadowed.has(event.seq) ? 'shadowed' : 'log-only', - })) -} - export default SessionQueryService diff --git a/packages/session-query/session-query/src/sources.ts b/packages/session-query/session-query/src/sources.ts new file mode 100644 index 0000000000..00b08eae4e --- /dev/null +++ b/packages/session-query/session-query/src/sources.ts @@ -0,0 +1,25 @@ +/** Shared immutable-header checks for logical session source observers. */ + +import type { SessionHeader } from '@deepseek-ai/dsh-session' +import { SessionQueryError } from './config.ts' + +/** + * Reject incompatible observations of one logical session source. + * @param a - first live, listed, or loaded header observation. + * @param b - second header observation expected to identify the same source. + */ +export function assertSessionHeadersCompatible(a: SessionHeader, b: SessionHeader): void { + if ( + a.version !== b.version + || a.id !== b.id + || a.createdAt !== b.createdAt + || a.cwd !== b.cwd + || a.parentSession !== b.parentSession + || a.seedLength !== b.seedLength + ) { + throw new SessionQueryError( + `session source headers conflict for session "${a.id}"`, + 'SESSION_QUERY_SOURCE_CONFLICT', + ) + } +} diff --git a/packages/session-query/session-query/src/types.ts b/packages/session-query/session-query/src/types.ts index 5c49695dda..d0de0dd48a 100644 --- a/packages/session-query/session-query/src/types.ts +++ b/packages/session-query/session-query/src/types.ts @@ -58,3 +58,99 @@ export interface SessionEventWindow { /** Last seq included in `events`. */ endSeq: number } + +/** Inclusive numeric interval used by time and sequence filters. */ +export interface SessionResultRange { + /** Inclusive lower bound. */ + from?: number + /** Inclusive upper bound. */ + to?: number +} + +/** Source availability predicates understood by logical-session filters. */ +export type SessionAvailability = 'live' | 'persisted' + +/** + * One logical-session predicate. A filter array is ANDed; `values` within a + * clause are ORed. + */ +export type SessionResultFilter = + | { kind: 'id'; values: readonly SessionId[] } + | { kind: 'cwd'; values: readonly (string | null)[] } + | ({ kind: 'created-at' } & SessionResultRange) + | { kind: 'parent'; values: readonly (SessionId | null)[] } + | { kind: 'availability'; values: readonly SessionAvailability[] } + +/** + * One event predicate. A filter array is ANDed; list-valued clauses are ORed. + * Text is a literal, case-insensitive, whitespace-flexible semantic-text scan. + */ +export type SessionEventResultFilter = + | ({ kind: 'seq' } & SessionResultRange) + | ({ kind: 'time' } & SessionResultRange) + | { kind: 'type'; values: readonly SessionEventType[] } + | { kind: 'surface'; values: readonly SessionEventSurface[] } + | { kind: 'text'; text: string } + +/** Event predicates a full-text provider can apply before relevance ranking. */ +export type SessionEventMetadataFilter = Exclude + +/** Searchable semantic document derived from one session event. */ +export interface SessionEventSearchDocument extends SessionEventRecord { + /** First-party semantic text used by scan filters and full-text indexes. */ + text: string +} + +/** One cursor-paginated result page. */ +export interface SessionSearchPage { + /** Results for this page in contract-defined order. */ + items: readonly T[] + /** Opaque continuation cursor, absent on the final page. */ + nextCursor?: string +} + +/** Controls shared by cross-session and within-session search calls. */ +export interface SessionSearchExecContext { + /** Abort caller waiting and interrupt provider work where supported. */ + signal?: AbortSignal +} + +/** Cross-session full-text search request. */ +export interface SessionSearchRequest { + /** Full-text query interpreted as data, never executable FTS syntax. */ + query: string + /** Logical-session predicates applied before event ranking. */ + sessionFilters?: readonly SessionResultFilter[] + /** Event predicates applied before event ranking. */ + eventFilters?: readonly SessionEventMetadataFilter[] + /** Maximum sessions in this page. */ + limit?: number + /** Opaque cursor returned for the identical normalized request. */ + cursor?: string +} + +/** Within-session full-text search request. */ +export interface SessionEventSearchRequest { + /** Session whose live-preferred logical log is searched. */ + sessionId: SessionId + /** Full-text query interpreted as data, never executable FTS syntax. */ + query: string + /** Event predicates applied before ranking. */ + filters?: readonly SessionEventMetadataFilter[] + /** Maximum events in this page. */ + limit?: number + /** Opaque cursor returned for the identical normalized request. */ + cursor?: string +} + +/** One event full-text search hit with a bounded plain-text excerpt. */ +export interface SessionEventSearchHit extends SessionEventRecord { + /** Plain text excerpt selected around the match. */ + snippet: string +} + +/** One grouped cross-session hit, ranked by its strongest matching event. */ +export interface SessionSearchHit extends SessionRecord { + /** Strongest matching event for this session. */ + bestMatch: SessionEventSearchHit +} diff --git a/packages/session-query/session-query/tests/search-helpers.spec.ts b/packages/session-query/session-query/tests/search-helpers.spec.ts new file mode 100644 index 0000000000..e9cf857608 --- /dev/null +++ b/packages/session-query/session-query/tests/search-helpers.spec.ts @@ -0,0 +1,209 @@ +import { describe, expect, it } from 'vitest' +import { Context } from 'cordis' +import { CallId } from '@deepseek-ai/dsh-llm' +import SessionStore, { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' +import type { SessionEvent, SessionHeader } from '@deepseek-ai/dsh-session' +import SessionQueryService, { + buildSessionEventRecords, + buildSessionEventSearchDocuments, + compileSessionTextFilter, + extractSessionEventText, + filterSessionEventDocuments, + filterSessionResults, + SessionSearchService, + type SessionEventSearchHit, + type SessionEventSearchRequest, + type SessionQueryErrorCode, + type SessionSearchExecContext, + type SessionSearchHit, + type SessionSearchPage, + type SessionSearchRequest, +} from '@deepseek-ai/dsh-session-query' + +const id = SessionId('session') + +function header(value: string, extra: Partial = {}): SessionHeader { + return { version: SESSION_FORMAT_VERSION, id: SessionId(value), createdAt: 10, ...extra } +} + +function expectCode(code: SessionQueryErrorCode): Error { + return expect.objectContaining({ code }) as Error +} + +describe('session-query semantic extraction', () => { + it('extracts first-party message, tool, todo, and failure detail', () => { + const callId = CallId('call') + const messageContent: SessionEvent<'user/message'>['data']['content'] = [ + { type: 'text', text: ' visible ' }, + { type: 'reasoning', text: 'thought' }, + { type: 'tool-call', id: callId, name: 'read', arguments: '{"path":"a"}' }, + { + type: 'tool-result', + toolCallId: callId, + content: [{ type: 'text', text: 'nested' }], + isError: false, + }, + { type: 'future-content', payload: 'hidden' } as never, + ] + const events: SessionEvent[] = [ + { type: 'user/message', seq: 0, time: 1, data: { content: messageContent, source: { kind: 'user' } }, surfaceOp: 'append' }, + { type: 'assistant/message', seq: 1, time: 2, data: { turn: 1, step: 1, content: messageContent }, surfaceOp: 'append' }, + { type: 'context/message', seq: 2, time: 3, data: { content: messageContent, source: { kind: 'plugin', plugin: 'test' } }, surfaceOp: 'append' }, + { type: 'steering/message', seq: 3, time: 4, data: { turn: 1, content: messageContent, source: { kind: 'user' } }, surfaceOp: 'append' }, + { type: 'prompt/blocked', seq: 4, time: 5, data: { content: [{ type: 'text', text: 'unsafe' }], source: { kind: 'user' }, reason: 'policy' } }, + { type: 'tool/call', seq: 5, time: 6, data: { turn: 1, step: 1, callId, name: 'bash', arguments: '{"cmd":"pwd"}' } }, + { type: 'tool/result', seq: 6, time: 7, data: { turn: 1, step: 1, callId, content: [{ type: 'text', text: 'failed' }], isError: true, error: { name: 'Oops', code: 'E_OOPS' } }, surfaceOp: 'append' }, + { type: 'tool/result', seq: 7, time: 8, data: { turn: 1, step: 1, callId, content: [], isError: false }, surfaceOp: 'append' }, + { type: 'todo/write', seq: 8, time: 9, data: { todos: [{ status: 'in_progress', content: 'ship search' }] } }, + ] + + for (const event of events.slice(0, 4)) { + expect(extractSessionEventText(event)).toBe('visible\nthought\nread\n{"path":"a"}\nnested') + } + expect(extractSessionEventText(events[4]!)).toBe('unsafe\npolicy') + expect(extractSessionEventText(events[5]!)).toBe('bash\n{"cmd":"pwd"}') + expect(extractSessionEventText(events[6]!)).toBe('failed\nOops\nE_OOPS') + expect(extractSessionEventText(events[7]!)).toBe('') + expect(extractSessionEventText(events[8]!)).toBe('in_progress\nship search') + }) + + it('extracts meaningful turn outcomes and skips structural or unknown events', () => { + const reasons: Array<[SessionEvent<'turn/end'>['data']['reason'], string]> = [ + [{ kind: 'error', step: 2, message: 'boom', code: 'E' }, 'error\nboom\nE'], + [{ kind: 'error', step: 2, message: 'boom' }, 'error\nboom'], + [{ kind: 'aborted', reason: 'cancelled' }, 'aborted\ncancelled'], + [{ kind: 'aborted' }, 'aborted'], + [{ kind: 'rejected', reason: 'denied' }, 'rejected\ndenied'], + [{ kind: 'disposed' }, 'disposed'], + [{ kind: 'max-tokens' }, 'max-tokens'], + [{ kind: 'interrupted' }, 'interrupted'], + [{ kind: 'completed' }, ''], + [{ kind: 'future-status' } as never, ''], + ] + for (const [reason, text] of reasons) { + expect(extractSessionEventText({ type: 'turn/end', seq: 0, time: 1, data: { turn: 1, reason } })).toBe(text) + } + const structural: SessionEvent[] = [ + { type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }, + { type: 'step/start', seq: 1, time: 1, data: { turn: 1, step: 1 } }, + { type: 'step/end', seq: 2, time: 1, data: { turn: 1, step: 1 } }, + { type: 'assistant/chunk', seq: 3, time: 1, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'raw' } } }, + { type: 'request/header', seq: 4, time: 1, data: { header: { config: { model: 'test' } }, reason: 'initial' } }, + { type: 'request/header-delta', seq: 5, time: 1, data: {} }, + { type: 'future/event', seq: 6, time: 1, data: { text: 'hidden' } } as never, + ] + expect(structural.map(extractSessionEventText)).toEqual(['', '', '', '', '', '', '']) + }) +}) + +describe('session-query document and filter helpers', () => { + const events: SessionEvent[] = [ + { type: 'user/message', seq: 0, time: 10, data: { content: [{ type: 'text', text: 'Hello\n(AI)+' }], source: { kind: 'user' } }, surfaceOp: 'append' }, + { type: 'assistant/chunk', seq: 1, time: 11, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'raw' } } }, + { type: 'assistant/message', seq: 2, time: 12, data: { turn: 1, step: 1, content: [{ type: 'text', text: 'replacement' }] }, surfaceOp: { op: 'replace', start: 0, end: 0 } }, + { type: 'turn/end', seq: 3, time: 13, data: { turn: 1, reason: { kind: 'interrupted' } } }, + ] + + it('classifies every event and omits non-semantic documents', () => { + expect(buildSessionEventRecords(id, events).map(record => record.surface)) + .toEqual(['shadowed', 'log-only', 'current', 'log-only']) + const documents = buildSessionEventSearchDocuments(id, events) + expect(documents.map(document => [document.seq, document.text, document.surface])).toEqual([ + [0, 'Hello\n(AI)+', 'shadowed'], + [2, 'replacement', 'current'], + [3, 'interrupted', 'log-only'], + ]) + }) + + it('applies every session clause with OR values and validates closed values', () => { + const parent = SessionId('parent') + const records = [ + { header: header('a', { cwd: '/a', parentSession: parent }), live: true, persisted: false, marker: 1 }, + { header: header('b', { createdAt: 20 }), live: false, persisted: true, marker: 2 }, + ] + expect(filterSessionResults(records, [ + { kind: 'id', values: [SessionId('a'), SessionId('x')] }, + { kind: 'cwd', values: ['/a', null] }, + { kind: 'created-at', from: 5, to: 15 }, + { kind: 'parent', values: [parent, null] }, + { kind: 'availability', values: ['live'] }, + ])).toEqual([records[0]]) + expect(filterSessionResults(records, [{ kind: 'cwd', values: [null] }])).toEqual([records[1]]) + expect(filterSessionResults(records, [{ kind: 'parent', values: [null] }])).toEqual([records[1]]) + expect(filterSessionResults(records, [{ kind: 'availability', values: ['persisted'] }])).toEqual([records[1]]) + expect(() => filterSessionResults(records, [{ kind: 'availability', values: ['remote' as never] }])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + }) + + it('applies event metadata and safe literal text clauses', () => { + const documents = buildSessionEventSearchDocuments(id, events).map((document, marker) => ({ ...document, marker })) + expect(filterSessionEventDocuments(documents, [ + { kind: 'seq', from: 0, to: 1 }, + { kind: 'time', from: 9, to: 11 }, + { kind: 'type', values: ['user/message', 'tool/result'] }, + { kind: 'surface', values: ['shadowed'] }, + { kind: 'text', text: 'hello (ai)+' }, + ])).toEqual([documents[0]]) + expect(compileSessionTextFilter('CAFÉ').test('café')).toBe(true) + expect(filterSessionEventDocuments(documents)).toEqual(documents) + expect(filterSessionEventDocuments(documents, [{ kind: 'surface', values: [] }])).toEqual([]) + expect(() => filterSessionEventDocuments(documents, [{ kind: 'surface', values: ['future' as never] }])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => compileSessionTextFilter(' \n ')).toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + }) + + it('rejects malformed range filters and malformed surfaces', () => { + const documents = buildSessionEventSearchDocuments(id, events) + for (const filter of [ + { kind: 'seq', from: Number.NaN }, + { kind: 'seq', to: Number.POSITIVE_INFINITY }, + { kind: 'time', from: 2, to: 1 }, + ] as const) { + expect(() => filterSessionEventDocuments(documents, [filter])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + } + expect(() => filterSessionResults([], [{ kind: 'created-at', from: Number.NaN }])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => filterSessionResults([{ header: header('x'), live: true, persisted: false }], [ + { kind: 'created-at', from: Number.NaN }, + ])).toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + const malformed: SessionEvent[] = [{ + type: 'assistant/message', + seq: 0, + time: 1, + data: { turn: 1, step: 1, content: [{ type: 'text', text: 'bad' }] }, + surfaceOp: { op: 'replace', start: 9, end: 9 }, + }] + expect(() => buildSessionEventRecords(id, malformed)).toThrow(expectCode('SESSION_QUERY_INVALID_SURFACE')) + }) + + it('exposes the scan path on the concrete exact-read service', async () => { + const ctx = new Context() + await ctx.plugin(SessionStore) + await ctx.plugin(SessionQueryService) + const session = ctx.sessions.create(id) + session.append('user/message', { content: [{ type: 'text', text: 'Alpha\n beta' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + session.append('user/message', { content: [{ type: 'text', text: 'other' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + await expect(ctx.sessionQuery.filterEvents(id, [{ kind: 'text', text: 'alpha beta' }])) + .resolves.toMatchObject([{ seq: 0, text: 'Alpha\n beta' }]) + }) +}) + +class TestSearchService extends SessionSearchService { + searchSessions(_request: SessionSearchRequest, _exec?: SessionSearchExecContext): Promise> { + return Promise.resolve({ items: [] }) + } + + searchEvents(_request: SessionEventSearchRequest, _exec?: SessionSearchExecContext): Promise> { + return Promise.resolve({ items: [] }) + } +} + +it('registers the abstract search seam under its independent ctx key', async () => { + const ctx = new Context() + const fiber = await ctx.plugin(TestSearchService) + await expect(ctx.sessionSearch.searchSessions({ query: 'AI' })).resolves.toEqual({ items: [] }) + await expect(ctx.sessionSearch.searchEvents({ sessionId: id, query: 'AI' })).resolves.toEqual({ items: [] }) + await fiber.dispose() + expect(ctx.sessionSearch).toBeUndefined() +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 456ce0896b..997f00ff5c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -778,6 +778,31 @@ importers: specifier: ^4.0.0-rc.6 version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/session-query/session-query-sqlite: + dependencies: + schemastery: + specifier: ^3.18.0 + version: 3.18.0 + devDependencies: + '@cordisjs/plugin-loader': + specifier: workspace:^ + version: link:../../../vendor/loader + '@deepseek-ai/dsh-session': + specifier: workspace:^ + version: link:../../core/session + '@deepseek-ai/dsh-session-persistence': + specifier: workspace:^ + version: link:../../session-persistence/session-persistence + '@deepseek-ai/dsh-session-persistence-sqlite': + specifier: workspace:^ + version: link:../../session-persistence/session-persistence-sqlite + '@deepseek-ai/dsh-session-query': + specifier: workspace:^ + version: link:../session-query + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@vendor+loader) + packages/skill/skill: dependencies: schemastery: diff --git a/scripts/gen-doc-graphs.ts b/scripts/gen-doc-graphs.ts index 05eaaed337..413917aa9a 100644 --- a/scripts/gen-doc-graphs.ts +++ b/scripts/gen-doc-graphs.ts @@ -93,7 +93,7 @@ const SERVICE_ROLES: ServiceRole[] = [ pkg: 'session', title: 'In-memory session store', mode: 'core', - consumers: ['agent-loop', 'agent', 'session-persistence', 'session-query', 'subagent-inprocess', 'invariants'], + consumers: ['agent-loop', 'agent', 'session-persistence', 'session-query', 'session-query-sqlite', 'subagent-inprocess', 'invariants'], note: 'Owns append-only Session instances and emits the durable session event feed.', }, { @@ -102,7 +102,7 @@ const SERVICE_ROLES: ServiceRole[] = [ title: 'Durable session persistence seam', mode: 'seam', implementations: ['session-persistence-jsonl', 'session-persistence-sqlite'], - consumers: ['agent-loop', 'acp', 'session-query'], + consumers: ['agent-loop', 'acp', 'session-query', 'session-query-sqlite'], note: 'Backends persist the same SessionEvent vocabulary; apps choose a backend at composition time.', }, { @@ -110,7 +110,15 @@ const SERVICE_ROLES: ServiceRole[] = [ pkg: 'session-query', title: 'Exact session-history reads', mode: 'seam', - note: 'Resolves live and optional persisted logs into one logical corpus for exact reads.', + note: 'Resolves live and optional persisted logs into one logical corpus for exact reads and semantic scans.', + }, + { + key: 'sessionSearch', + pkg: 'session-query', + title: 'Full-text session search', + mode: 'seam', + implementations: ['session-query-sqlite'], + note: 'The concrete backend owns source reconciliation, ranking, snippets, and cursor generations as one lifecycle.', }, { key: 'systemPrompt', diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index 7b83f1004e..0d0a0b871d 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -49,6 +49,14 @@ { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventSurface", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionRecord", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventRecord", "source": "packages/session-query/session-query/src/types.ts" }, + { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionResultFilter", "source": "packages/session-query/session-query/src/types.ts" }, + { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventResultFilter", "source": "packages/session-query/session-query/src/types.ts" }, + { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventSearchDocument", "source": "packages/session-query/session-query/src/types.ts" }, + { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionSearchRequest", "source": "packages/session-query/session-query/src/types.ts" }, + { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventSearchRequest", "source": "packages/session-query/session-query/src/types.ts" }, + { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionSearchPage", "source": "packages/session-query/session-query/src/types.ts" }, + { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventSearchHit", "source": "packages/session-query/session-query/src/types.ts" }, + { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionSearchHit", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionQueryErrorCode", "source": "packages/session-query/session-query/src/config.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventReadRequest", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventWindow", "source": "packages/session-query/session-query/src/types.ts" }, diff --git a/tsconfig.build.json b/tsconfig.build.json index 7797998a30..d2470e0ff5 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -19,6 +19,7 @@ { "path": "./packages/session-persistence/session-persistence-jsonl" }, { "path": "./packages/session-persistence/session-persistence-sqlite" }, { "path": "./packages/session-query/session-query" }, + { "path": "./packages/session-query/session-query-sqlite" }, { "path": "./packages/core/system-prompt" }, { "path": "./packages/core/agent" }, { "path": "./packages/ui/user-interaction" }, diff --git a/tsconfig.json b/tsconfig.json index 778a26ff8a..c62cc70518 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -30,6 +30,7 @@ { "path": "./packages/session-persistence/session-persistence-jsonl" }, { "path": "./packages/session-persistence/session-persistence-sqlite" }, { "path": "./packages/session-query/session-query" }, + { "path": "./packages/session-query/session-query-sqlite" }, { "path": "./packages/core/system-prompt" }, { "path": "./packages/core/agent" }, { "path": "./packages/ui/user-interaction" }, From f88ca85ffdd5c3b86ff859bda2c9f05967ade11e Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Wed, 15 Jul 2026 12:10:24 +0800 Subject: [PATCH 002/113] fix(session-query): harden SQLite search reconciliation --- docs/config-catalog.md | 2 +- docs/cordis-catalog/services.md | 8 +- docs/core-data-structures/persistence.md | 19 +- docs/core-data-structures/session-query.md | 12 +- docs/module-graph.md | 6 +- .../2026-07-10-session-query-service.md | 2 +- ...026-07-10-sqlite-session-query-provider.md | 16 +- .../cordis/tool-cordis/src/api-catalog.ts | 20 +- .../session-persistence-jsonl/README.md | 1 + .../session-persistence-jsonl/src/index.ts | 40 +- .../session-persistence-sqlite/README.md | 3 +- .../session-persistence-sqlite/src/index.ts | 24 +- .../session-persistence-sqlite/src/schema.ts | 19 +- .../tests/sqlite.spec.ts | 17 +- .../session-persistence/README.md | 7 +- .../session-persistence/package.json | 2 + .../session-persistence/src/coordinator.ts | 4 +- .../session-persistence/src/index.ts | 19 + .../session-persistence/src/revision.ts | 15 + .../session-persistence/tests/contract.ts | 22 +- .../tests/persistence.spec.ts | 12 +- .../session-persistence/tsconfig.json | 3 + .../session-query-sqlite/README.md | 10 +- .../session-query-sqlite/src/index.ts | 326 ++++++++++++---- .../session-query-sqlite/src/query.ts | 151 ++++++- .../session-query-sqlite/src/schema.ts | 11 +- .../session-query-sqlite/tests/query.spec.ts | 65 +++- .../session-query-sqlite/tests/sqlite.spec.ts | 367 +++++++++++++++++- .../session-query/session-query/README.md | 3 +- .../session-query/session-query/package.json | 2 + .../session-query/session-query/src/corpus.ts | 21 +- .../session-query/session-query/src/cursor.ts | 15 + .../session-query/src/filters.ts | 121 +++++- .../session-query/session-query/src/index.ts | 62 ++- .../session-query/session-query/src/types.ts | 9 +- .../tests/search-helpers.spec.ts | 27 ++ .../session-query/tests/session-query.spec.ts | 67 +++- .../session-query/session-query/tsconfig.json | 3 + pnpm-lock.yaml | 6 + scripts/type-equiv.manifest.json | 3 + 40 files changed, 1315 insertions(+), 227 deletions(-) create mode 100644 packages/session-persistence/session-persistence/src/revision.ts create mode 100644 packages/session-query/session-query/src/cursor.ts diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 3a802231d7..e8a06e469e 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -613,7 +613,7 @@ export interface Config { export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' ``` -Source: [`packages/session-query/session-query-sqlite/src/index.ts:58`](../packages/session-query/session-query-sqlite/src/index.ts) +Source: [`packages/session-query/session-query-sqlite/src/index.ts:67`](../packages/session-query/session-query-sqlite/src/index.ts) ## `@deepseek-ai/dsh-skill` diff --git a/docs/cordis-catalog/services.md b/docs/cordis-catalog/services.md index fef691aae1..e83b96bd11 100644 --- a/docs/cordis-catalog/services.md +++ b/docs/cordis-catalog/services.md @@ -195,11 +195,12 @@ abstract create(meta: SessionHeader): Promise abstract append(id: SessionId, events: readonly SessionEvent[]): Promise abstract load(id: SessionId): Promise<{ meta: SessionHeader; events: SessionEvent[] }> abstract list(): Promise +abstract listSnapshots(): Promise ``` Types: [SessionEvent](../core-data-structures/core.md) -Source: [`packages/session-persistence/session-persistence/src/index.ts:102`](../../packages/session-persistence/session-persistence/src/index.ts) +Source: [`packages/session-persistence/session-persistence/src/index.ts:112`](../../packages/session-persistence/session-persistence/src/index.ts) ## `ctx.sessionQuery` — `SessionQueryService` @@ -207,12 +208,13 @@ Live-preferred logical-corpus and exact-event read service. ```ts cordis-catalog listSessions(): Promise +async filterSessions(filters: readonly SessionResultFilter[]): Promise async listEvents(sessionId: SessionId): Promise async filterEvents( sessionId: SessionId, filters: readonly SessionEventResultFilter[], ): Promise async readEvent(request: SessionEventReadRequest): Promise ``` -Source: [`packages/session-query/session-query/src/index.ts:83`](../../packages/session-query/session-query/src/index.ts) +Source: [`packages/session-query/session-query/src/index.ts:96`](../../packages/session-query/session-query/src/index.ts) ## `ctx.sessions` — `SessionStore` @@ -244,7 +246,7 @@ abstract searchSessions( request: SessionSearchRequest, exec?: SessionSearchExec abstract searchEvents( request: SessionEventSearchRequest, exec?: SessionSearchExecContext, ): Promise> ``` -Source: [`packages/session-query/session-query/src/index.ts:54`](../../packages/session-query/session-query/src/index.ts) +Source: [`packages/session-query/session-query/src/index.ts:67`](../../packages/session-query/session-query/src/index.ts) ## `ctx.skills` — `SkillService` diff --git a/docs/core-data-structures/persistence.md b/docs/core-data-structures/persistence.md index 7bf102924b..9535686f86 100644 --- a/docs/core-data-structures/persistence.md +++ b/docs/core-data-structures/persistence.md @@ -2,7 +2,7 @@ The **durability seam** for the event log. [session.md](session.md) describes the in-memory `Session` — the append-only `SessionEvent` log that is the source of truth. This page describes how that log is made durable: the abstract `SessionPersistence` service, its backends, the flush checkpoint, crash recovery, and the metadata header that travels alongside the log. The event vocabulary the log carries is enumerated, member by member, in the generated [persistence log event catalog](../persistence-catalog.md). -The seam is a textbook [capability seam](../rfc/implemented/architecture/2026-06-13-capability-seams.md): one abstract service ([dsh-session-persistence](../../packages/session-persistence/session-persistence), `ctx.sessionPersistence`) defining create/append/load/list over the existing `SessionEvent` — **no parallel persisted type** — and two interchangeable backends that pass the same `runPersistenceContract` suite. See the [session-persistence RFC](../rfc/implemented/architecture/2026-06-14-session-persistence.md). +The seam is a textbook [capability seam](../rfc/implemented/architecture/2026-06-13-capability-seams.md): one abstract service ([dsh-session-persistence](../../packages/session-persistence/session-persistence), `ctx.sessionPersistence`) defining create/append/load plus lightweight list/snapshot observation over the existing `SessionEvent` — **no parallel persisted event type** — and two interchangeable backends that pass the same `runPersistenceContract` suite. See the [session-persistence RFC](../rfc/implemented/architecture/2026-06-14-session-persistence.md). ## The flush checkpoint @@ -78,9 +78,24 @@ interface CreateSessionOptions { Replay/fork is therefore `ctx.sessions.create(id, { seed: seedEvents })`; resuming a *persisted* session into a live agent is `ctx.agents.resume({ resumeSessionId })`. +## Lightweight source revisions + +Consumers of derived state compare a cheap opaque revision before loading a full event log. The persistence backend owns its representation and changes it transactionally with append or mutating load repair; callers compare it only for equality. + +```ts type-equiv +export type SessionPersistenceRevision = Branded<'SessionPersistenceRevision'> +``` + +```ts type-equiv +export interface SessionPersistenceSnapshot { + header: SessionHeader + revision: SessionPersistenceRevision +} +``` + ## The backends -Both implement the same abstract `SessionPersistence` (create/append/load/list over `SessionEvent`) and pass `runPersistenceContract`, proving the seam is genuinely backend-agnostic: +Both implement the same abstract `SessionPersistence` (create/append/load/list/listSnapshots over `SessionEvent`) and pass `runPersistenceContract`, proving the seam is genuinely backend-agnostic: - **[dsh-session-persistence-jsonl](../../packages/session-persistence/session-persistence-jsonl)** — an append-only JSONL log per session with crash-safe atomic writes, the interrupted-turn crash recovery above, and a read/replay path. - **[dsh-session-persistence-sqlite](../../packages/session-persistence/session-persistence-sqlite)** — `node:sqlite`, one row per `SessionEvent`. The row shape `(session_id, seq, type, time, data, source_event_seqs, surface_op)` maps 1:1 onto the event, including optional surface metadata, so there is no parallel persisted schema to keep in sync. diff --git a/docs/core-data-structures/session-query.md b/docs/core-data-structures/session-query.md index 27ef9959de..d1ed41b775 100644 --- a/docs/core-data-structures/session-query.md +++ b/docs/core-data-structures/session-query.md @@ -58,19 +58,23 @@ export interface SessionEventSearchDocument extends SessionEventRecord { } ``` -`ctx.sessionQuery.filterEvents(sessionId, filters)` returns these documents in ascending seq order. Messages, reasoning, tool calls/results, blocked prompts, todos, and failure/status detail contribute semantic text; structural events and stream chunks do not. +`ctx.sessionQuery.filterSessions(filters)` applies `SessionResultFilter` to the complete logical corpus; `ctx.sessionQuery.filterEvents(sessionId, filters)` returns matching documents in ascending seq order. Messages, reasoning, tool calls/results, blocked prompts, todos, and failure/status detail contribute semantic text; structural events and stream chunks do not. ## Full-text search pages The independent `ctx.sessionSearch` seam has two scopes. `searchSessions()` groups the corpus by strongest matching event; `searchEvents()` searches one session. Requests bind an opaque cursor to the normalized query, metadata filters, and limit. The event text scan is intentionally absent from provider metadata filters. +```ts type-equiv +export type SessionSearchCursor = Branded<'SessionSearchCursor'> +``` + ```ts type-equiv export interface SessionSearchRequest { query: string sessionFilters?: readonly SessionResultFilter[] eventFilters?: readonly SessionEventMetadataFilter[] limit?: number - cursor?: string + cursor?: SessionSearchCursor } ``` @@ -80,14 +84,14 @@ export interface SessionEventSearchRequest { query: string filters?: readonly SessionEventMetadataFilter[] limit?: number - cursor?: string + cursor?: SessionSearchCursor } ``` ```ts type-equiv export interface SessionSearchPage { items: readonly T[] - nextCursor?: string + nextCursor?: SessionSearchCursor } ``` diff --git a/docs/module-graph.md b/docs/module-graph.md index 6fcec53dd4..7ad4cd8c70 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -151,6 +151,7 @@ flowchart TD pkg_web_search_deepseek --> pkg_web pkg_web_search_exa --> pkg_web pkg_web_search_perplexity --> pkg_web + pkg_session_persistence --> pkg_brand pkg_session_persistence --> pkg_session pkg_llm_replay --> pkg_llm pkg_llm_replay --> pkg_session @@ -168,6 +169,7 @@ flowchart TD pkg_session_persistence_jsonl --> pkg_session_persistence pkg_session_persistence_sqlite --> pkg_session pkg_session_persistence_sqlite --> pkg_session_persistence + pkg_session_query --> pkg_brand pkg_session_query --> pkg_llm pkg_session_query --> pkg_session pkg_session_query --> pkg_session_persistence @@ -361,7 +363,7 @@ flowchart TD | [`web-search-deepseek`](../packages/web/web-search-deepseek) | `web` | [`web`](../packages/web/web) | | [`web-search-exa`](../packages/web/web-search-exa) | `web` | [`web`](../packages/web/web) | | [`web-search-perplexity`](../packages/web/web-search-perplexity) | `web` | [`web`](../packages/web/web) | -| [`session-persistence`](../packages/session-persistence/session-persistence) | `session-persistence` | [`session`](../packages/core/session) | +| [`session-persistence`](../packages/session-persistence/session-persistence) | `session-persistence` | [`brand`](../packages/util/brand), [`session`](../packages/core/session) | | [`llm-replay`](../packages/support/llm-replay) | `support` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | | [`sandbox-local`](../packages/sandbox/sandbox-local) | `sandbox` | [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox) | | [`bash-local`](../packages/bash/bash-local) | `bash` | [`bash`](../packages/bash/bash), [`timeout`](../packages/util/timeout) | @@ -369,7 +371,7 @@ flowchart TD | [`hook-protocol`](../packages/hooks/hook-protocol) | `hooks` | [`bash`](../packages/bash/bash), [`session`](../packages/core/session) | | [`session-persistence-jsonl`](../packages/session-persistence/session-persistence-jsonl) | `session-persistence` | [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | | [`session-persistence-sqlite`](../packages/session-persistence/session-persistence-sqlite) | `session-persistence` | [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | -| [`session-query`](../packages/session-query/session-query) | `session-query` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | +| [`session-query`](../packages/session-query/session-query) | `session-query` | [`brand`](../packages/util/brand), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | | [`invariants`](../packages/support/invariants) | `support` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`session`](../packages/core/session) | | [`user-approval`](../packages/ui/user-approval) | `ui` | [`agent`](../packages/core/agent), [`brand`](../packages/util/brand), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt) | | [`user-interaction`](../packages/ui/user-interaction) | `ui` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm) | diff --git a/docs/rfc/implemented/feature/2026-07-10-session-query-service.md b/docs/rfc/implemented/feature/2026-07-10-session-query-service.md index be496311cc..10baa3e2c3 100644 --- a/docs/rfc/implemented/feature/2026-07-10-session-query-service.md +++ b/docs/rfc/implemented/feature/2026-07-10-session-query-service.md @@ -10,7 +10,7 @@ Full-text search is related but materially larger. Putting provider coordination ## Decision -`@deepseek-ai/dsh-session-query` owns `ctx.sessionQuery`, a small trusted exact-read service over one logical corpus. It exposes `listSessions()`, `listEvents(sessionId)`, provider-independent `filterEvents(sessionId, filters)`, and bounded `readEvent(request)`. The package also declares the separate abstract `ctx.sessionSearch` contract and shared semantic extraction used by the [SQLite search decision](2026-07-10-sqlite-session-query-provider.md), but `ctx.sessionQuery` does not coordinate providers or synchronize a derived index. +`@deepseek-ai/dsh-session-query` owns `ctx.sessionQuery`, a small trusted exact-read service over one logical corpus. It exposes `listSessions()`, provider-independent `filterSessions(filters)`, `listEvents(sessionId)`, `filterEvents(sessionId, filters)`, and bounded `readEvent(request)`. The package also declares the separate abstract `ctx.sessionSearch` contract and shared semantic extraction used by the [SQLite search decision](2026-07-10-sqlite-session-query-provider.md), but `ctx.sessionQuery` does not coordinate providers or synchronize a derived index. The service observes the optional `ctx.sessionPersistence` binding dynamically but retains no persisted cache or invalidation listener. Each cross-corpus list asks the active backend for authoritative metadata, then overlays a fresh live-store list. Matching ids become one `SessionRecord`: the live header wins and `live`/`persisted` independently report source availability. Immutable header disagreement is `SESSION_QUERY_SOURCE_CONFLICT`. diff --git a/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md b/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md index d620ece108..fafa7f3569 100644 --- a/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md +++ b/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md @@ -10,19 +10,19 @@ Splitting those concerns across a provider coordinator and a database implementa ## Decision -`@deepseek-ai/dsh-session-query` declares an independent abstract `ctx.sessionSearch` service without changing the exact-read `ctx.sessionQuery` key. `searchSessions(request, exec?)` returns cursor-paginated `SessionSearchHit`s grouped by each session's strongest matching event; `searchEvents(request, exec?)` returns `SessionEventSearchHit`s within one logical session. Both requests require `query`, accept `limit` and an opaque `cursor`, and support an optional abort signal. Session search accepts `sessionFilters` plus event metadata filters; event search accepts event metadata filters. Results expose bounded plain-text snippets but no provider identifier or numeric relevance score. +`@deepseek-ai/dsh-session-query` declares an independent abstract `ctx.sessionSearch` service without changing the exact-read `ctx.sessionQuery` key. `searchSessions(request, exec?)` returns cursor-paginated `SessionSearchHit`s grouped by each session's strongest matching event; `searchEvents(request, exec?)` returns `SessionEventSearchHit`s within one logical session. Both requests require `query`, accept `limit` and an owned branded `SessionSearchCursor`, and support an optional abort signal. Session search accepts `sessionFilters` plus event metadata filters; event search accepts event metadata filters. Results expose bounded plain-text snippets but no provider identifier or numeric relevance score. `@deepseek-ai/dsh-session-query-sqlite` is the sole concrete owner of `ctx.sessionSearch`. It depends on live `ctx.sessions`, observes optional `ctx.sessionPersistence` dynamically, and owns a dedicated derived SQLite database. There is no search-provider registry, coordinator, persistence event, or agent-loop integration. -The interface package also owns shared first-party semantic extraction and provider-independent filtering. `SessionResultFilter` covers id, nullable cwd, created-at range, nullable parent, and availability. `SessionEventResultFilter` covers seq/time ranges, event type, surface, and literal semantic text. Arrays are ANDed and list values are ORed. The text clause escapes caller input into a Unicode case-insensitive regular expression whose whitespace runs match one or more whitespace characters; it is available through `ctx.sessionQuery.filterEvents()` and is not delegated to an FTS provider. +The interface package also owns shared first-party semantic extraction and provider-independent filtering. `SessionResultFilter` covers id, nullable cwd, created-at range, nullable parent, and availability; `ctx.sessionQuery.filterSessions()` applies it without an FTS provider. `SessionEventResultFilter` covers seq/time ranges, event type, surface, and literal semantic text. Arrays are ANDed and list values are ORed. The text clause escapes caller input into a Unicode case-insensitive regular expression whose whitespace runs match one or more whitespace characters; it is available through `ctx.sessionQuery.filterEvents()` and is not delegated to an FTS provider. ## Search semantics Each semantic event is one FTS document carrying session metadata, event metadata, surface classification, and extracted text. All `current`, `shadowed`, and `log-only` documents participate unless a surface filter narrows them. Metadata filters compile to parameterized SQL before ranking. Session results partition matching documents by session and retain the strongest one. -Ordering is deterministic: BM25 ascending, event time descending, session id ascending for the cross-session scope, and seq descending. Provider scores remain private. Snippets normalize whitespace and are bounded by Unicode code points. Opaque cursors bind to the service instance, scope, canonical normalized request, offset, and relevant generation. Any corpus change invalidates cross-session cursors; a within-session cursor changes only when its target source/generation changes, so unrelated sessions do not invalidate it. Reopening creates a new service instance and invalidates old cursors. +Ordering is deterministic and comparable across the persistent and TEMP FTS tables: actual FTS5 highlighted-match span count descending, indexed document code-point length ascending, event time descending, session id ascending for the cross-session scope, and seq descending. Snippets use those actual highlight positions, strip the reserved markers, normalize whitespace, and bound by Unicode code points. Opaque cursors bind to the service instance, scope, canonical normalized request, offset, and relevant generation. Any corpus change invalidates cross-session cursors; a within-session cursor changes only when its target source/generation changes, so unrelated sessions do not invalidate it. Reopening creates a new service instance and invalidates old cursors. -Queries are trimmed, whitespace-normalized, and quoted as one literal FTS5 phrase. Embedded quotes are doubled before binding, so MATCH operators such as `OR`, `NEAR`, quotes, parentheses, and `*` remain data rather than executable query syntax. Phrase matching follows tokenizer tokens rather than arbitrary substrings. +Queries are trimmed, whitespace-normalized, and quoted as one literal FTS5 phrase. Embedded quotes are doubled before binding, so MATCH operators such as `OR`, `NEAR`, quotes, parentheses, and `*` remain data rather than executable query syntax. NUL is rejected before SQLite execution. Reserved highlight noncharacters and NUL in documents are normalized before indexing, making inserted presentation markers collision-free. Phrase matching follows tokenizer tokens rather than arbitrary substrings. ## Tokenizer choice @@ -32,11 +32,11 @@ Both persistent and live FTS5 tables use `unicode61`. The implementation experim The shared extractor includes message text, reasoning, nested tool-call/result content, tool names and arguments, blocked-prompt reasons, todo status/content, and error or terminal status detail. Structural boundaries, stream chunks, request headers, successful completion markers, and unknown declaration-merged event/content variants produce no document. Surface classification reuses `foldSurface()` so search agrees with model-history derivation. -One serialized operation observes complete persisted and live sources, computes stable fingerprints, reconciles rows in one transaction, and executes the query. Unchanged persisted sessions retain their rows and generation. New, changed, and deleted persisted sessions update on the next search. A source or extraction failure cannot mark a row current, and a transaction failure rolls back so a later search retries. +One serialized operation reads the provider-neutral `SessionPersistence` snapshot listing, compares each opaque revision with the revision stored beside the indexed session, loads only new or changed logs, reconciles rows in one transaction, and executes the query. Observation repeats when listing changes during a load; this incorporates a mutating load repair's refreshed revision before commit. Repeated queries and unchanged reopen load no full persisted logs. New, changed, and deleted sessions update on the next stable search. A source or extraction failure cannot mark a row current, and a transaction failure rolls back so a later search retries. Persisted documents survive restarts. Live sessions use connection-local TEMP tables, shadow the persisted base for the same id, and reveal that base on detach. Closing the database drops live rows. Unmounting persistence hides durable rows without treating absence as authoritative deletion; remounting observes and reconciles the backend again. Conflicting immutable live and durable headers fail rather than combining sources. -The derived schema has its own application id and monotonic schema version. A recognized incompatible version resets only this derived database. A database with a foreign application id or unrecognized user tables is refused, which prevents an accidentally configured canonical session database from being reset. +The derived schema has its own application id and monotonic schema version. A recognized incompatible version resets only this derived database. A database with a foreign application id or unrecognized user tables is refused before journal-mode mutation, which prevents an accidentally configured canonical session database from being changed. One service in one process exclusively owns a derived-index path; cross-process writers are unsupported because generations and live TEMP shadow state are connection-owned. Cancellation rejects queued operations and caller waits around asynchronous source observation without committing an aborted observation. Node's synchronous `DatabaseSync` MATCH call cannot be interrupted once it is executing on the JavaScript thread, so the service checks the signal at serialized boundaries but does not promise mid-statement preemption. @@ -46,11 +46,11 @@ Cancellation rejects queued operations and caller waits around asynchronous sour - **Add a phase-one provider registry and coordinator** — rejected because one implementation provides no evidence for registration semantics and would split one reconciliation lifecycle across two owners. - **Persist live overrides immediately** — rejected because live events are not canonical until the existing checkpoint commits. - **Use the FTS5 trigram tokenizer** — rejected because it omits useful queries shorter than three characters and measured about 2.1× the index size of `unicode61`; literal substring filtering remains available through the scan path. -- **Return BM25 scores** — rejected because provider-specific numeric scales are unstable across corpus changes. +- **Use FTS5 BM25 independently in each table** — rejected because scores from differently populated persistent and TEMP corpora are not comparable; actual matched spans and document length have one shared scale. ## Consequences -Search has a small provider-neutral API while its only backend owns every derived-index state transition. The separate database adds configuration and a reconciliation read before queries, but index corruption, reset, and tokenizer changes cannot endanger canonical logs. Persistent fingerprints avoid rewriting unchanged sessions; TEMP live overlays preserve current-session truth without making uncheckpointed events durable. +Search has a small provider-neutral API while its only backend owns every derived-index state transition. The separate database adds configuration and a lightweight snapshot read before queries, but index corruption, reset, and tokenizer changes cannot endanger canonical logs. Durable revisions avoid full-log reads and rewrites for unchanged sessions; TEMP live overlays preserve current-session truth without making uncheckpointed events durable. The chosen tokenizer supports short tokens with a smaller index but does not promise substring recall. Literal phrases make query syntax safe and predictable at the cost of excluding boolean/full MATCH expressions. Cancellation is effective while queued or awaiting sources, but synchronous SQLite execution remains a non-preemptible section. diff --git a/packages/cordis/tool-cordis/src/api-catalog.ts b/packages/cordis/tool-cordis/src/api-catalog.ts index 2859353cfa..7322698cdd 100644 --- a/packages/cordis/tool-cordis/src/api-catalog.ts +++ b/packages/cordis/tool-cordis/src/api-catalog.ts @@ -149,6 +149,7 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ 'abstract append(id: SessionId, events: readonly SessionEvent[]): Promise', 'abstract load(id: SessionId): Promise<{ meta: SessionHeader; events: SessionEvent[] }>', 'abstract list(): Promise', + 'abstract listSnapshots(): Promise', ], }, { @@ -156,6 +157,7 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ summary: 'Live-preferred logical-corpus and exact-event read service.', methods: [ 'listSessions(): Promise', + 'async filterSessions(filters: readonly SessionResultFilter[]): Promise', 'async listEvents(sessionId: SessionId): Promise', 'async filterEvents( sessionId: SessionId, filters: readonly SessionEventResultFilter[], ): Promise', 'async readEvent(request: SessionEventReadRequest): Promise', @@ -834,7 +836,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'SessionEventSearchRequest', - declaration: 'export interface SessionEventSearchRequest {\n sessionId: SessionId;\n query: string;\n filters?: readonly SessionEventMetadataFilter[];\n limit?: number;\n cursor?: string;\n}', + declaration: 'export interface SessionEventSearchRequest {\n sessionId: SessionId;\n query: string;\n filters?: readonly SessionEventMetadataFilter[];\n limit?: number;\n cursor?: SessionSearchCursor;\n}', }, { name: 'SessionEventSurface', @@ -860,6 +862,14 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SessionId', declaration: 'export type SessionId = Branded<\'SessionId\'>;', }, + { + name: 'SessionPersistenceRevision', + declaration: 'export type SessionPersistenceRevision = Branded<\'SessionPersistenceRevision\'>;', + }, + { + name: 'SessionPersistenceSnapshot', + declaration: 'export interface SessionPersistenceSnapshot {\n header: SessionHeader;\n revision: SessionPersistenceRevision;\n}', + }, { name: 'SessionRecord', declaration: 'export interface SessionRecord {\n header: SessionHeader;\n live: boolean;\n persisted: boolean;\n}', @@ -872,6 +882,10 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SessionResultRange', declaration: 'export interface SessionResultRange {\n from?: number;\n to?: number;\n}', }, + { + name: 'SessionSearchCursor', + declaration: 'export type SessionSearchCursor = Branded<\'SessionSearchCursor\'>;', + }, { name: 'SessionSearchExecContext', declaration: 'export interface SessionSearchExecContext {\n signal?: AbortSignal;\n}', @@ -882,11 +896,11 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'SessionSearchPage', - declaration: 'export interface SessionSearchPage {\n items: readonly T[];\n nextCursor?: string;\n}', + declaration: 'export interface SessionSearchPage {\n items: readonly T[];\n nextCursor?: SessionSearchCursor;\n}', }, { name: 'SessionSearchRequest', - declaration: 'export interface SessionSearchRequest {\n query: string;\n sessionFilters?: readonly SessionResultFilter[];\n eventFilters?: readonly SessionEventMetadataFilter[];\n limit?: number;\n cursor?: string;\n}', + declaration: 'export interface SessionSearchRequest {\n query: string;\n sessionFilters?: readonly SessionResultFilter[];\n eventFilters?: readonly SessionEventMetadataFilter[];\n limit?: number;\n cursor?: SessionSearchCursor;\n}', }, { name: 'SkillCandidate', diff --git a/packages/session-persistence/session-persistence-jsonl/README.md b/packages/session-persistence/session-persistence-jsonl/README.md index 990ad2fcc4..0be2c785b3 100644 --- a/packages/session-persistence/session-persistence-jsonl/README.md +++ b/packages/session-persistence/session-persistence-jsonl/README.md @@ -25,6 +25,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence - **Append-only.** Committed events (at or below a flushed `turn/end`) are never rewritten. Subsequent appends are line appends at EOF + `fsync`. - **Crash recovery — close, don't truncate.** A crash can leave a log whose final turn never closed (real events after the last `turn/end`). `load` PRESERVES those events (a turn can be huge — they are real work) and closes the orphaned turn by durably appending synthetic boundary events: an error `tool/result` for every `tool-call` the crash left unanswered (the loop logs the assistant message before running the tools, so a mid-tool crash leaves dangling calls — and `deriveMessages()` would replay an assistant tool-call with no result, which providers reject), then a `step/end` if a step was open, then `turn/end {kind:'interrupted'}`, returning a balanced log. Only a never-fully-written **torn tail fragment** (a final line with no newline / unparseable) is `ftruncate`d away before the closers are written. See [session persistence](../../../docs/rfc/implemented/architecture/2026-06-14-session-persistence.md). - **Contiguous-seq.** `load` rejects a mid-log parse error or `seq` gap (unloadable); `append` rejects a batch whose first `seq` does not continue the stored log, and rejects non-JSON-serializable `event.data` naming the offending event type. +- **Lightweight revisions.** `listSnapshots()` reads each header and returns an opaque identity from the file device/inode, size, and nanosecond mtime/ctime. Under the append-only single-writer contract, unchanged files retain revisions while append, repair, or replacement changes them without parsing event lines. - **Format version.** Only the current `SESSION_FORMAT_VERSION` (v0) is supported; `load` rejects any other version. While the harness is unreleased the on-disk format is pre-release/unstable: a breaking format change is absorbed at v0 (no bump until the first tagged release) and non-current logs are rejected — there is no migration (no persisted user data to preserve). ## Write path diff --git a/packages/session-persistence/session-persistence-jsonl/src/index.ts b/packages/session-persistence/session-persistence-jsonl/src/index.ts index bd31e0ae47..0aea7927d1 100644 --- a/packages/session-persistence/session-persistence-jsonl/src/index.ts +++ b/packages/session-persistence/session-persistence-jsonl/src/index.ts @@ -11,7 +11,7 @@ * (the `session/event` → buffer → `session/flush` drain, per-session * serialization, write cursors, fork-seed persistence, HMR live-adoption, * crash-repair sequencing, dispose quiescence) lives in the backend-agnostic - * {@link PersistenceCoordinator} this class composes. The four public + * {@link PersistenceCoordinator} this class composes. The stateful public * {@link SessionPersistence} methods delegate to the coordinator. * * @module @deepseek-ai/dsh-session-persistence-jsonl @@ -19,12 +19,12 @@ import { Context } from 'cordis' import z from 'schemastery' -import { open, mkdir, readFile, readdir, link, rm, truncate } from 'node:fs/promises' +import { open, mkdir, readFile, readdir, link, rm, stat, truncate } from 'node:fs/promises' import { dirname, resolve } from 'node:path' import { randomBytes } from 'node:crypto' import { - SessionPersistence, PersistenceCoordinator, - type PersistenceBackend, type StoredPrefix, + SessionPersistence, SessionPersistenceRevision, PersistenceCoordinator, + type PersistenceBackend, type SessionPersistenceSnapshot, type StoredPrefix, } from '@deepseek-ai/dsh-session-persistence' import type { Session, SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session' import { @@ -179,20 +179,44 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi /** List all stored sessions' metadata (header line only — no full-log parse). */ async list(): Promise { - const metas: SessionHeader[] = [] + return (await this.listArtifacts()).map(artifact => artifact.header) + } + + /** List metadata plus a stat-derived identity for each append-only log. */ + async listSnapshots(): Promise { + const snapshots: SessionPersistenceSnapshot[] = [] + for (const artifact of await this.listArtifacts()) { + const identity = await stat(artifact.path, { bigint: true }) + snapshots.push({ + header: artifact.header, + revision: SessionPersistenceRevision([ + identity.dev, + identity.ino, + identity.size, + identity.mtimeNs, + identity.ctimeNs, + ].join(':')), + }) + } + return snapshots + } + + private async listArtifacts(): Promise> { + const artifacts: Array<{ header: SessionHeader; path: string }> = [] for (const dir of await this.listCwdDirs()) { for (const name of await this.listJsonl(dir)) { // Read ONLY the header line, not the whole log: a session picker must // scale with the number of sessions, not the total size of every // conversation (the log persists every assistant/chunk verbatim). - const first = await this.readFirstLine(`${dir}/${name}`) + const path = `${dir}/${name}` + const first = await this.readFirstLine(path) if (first === undefined) continue // empty/half-written file const meta = parseHeaderMeta(first) if (meta === undefined) continue // not a session header - metas.push(meta) + artifacts.push({ header: meta, path }) } } - return metas + return artifacts } // --- materialization / append / repair (file mechanics) --- diff --git a/packages/session-persistence/session-persistence-sqlite/README.md b/packages/session-persistence/session-persistence-sqlite/README.md index 082b60af88..6e6006dc6b 100644 --- a/packages/session-persistence/session-persistence-sqlite/README.md +++ b/packages/session-persistence/session-persistence-sqlite/README.md @@ -6,7 +6,7 @@ A SQLite durable session-persistence backend — a second `SessionPersistence` i ## Storage model -Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [session surface](../../../docs/rfc/implemented/architecture/2026-06-18-session-surface.md)). Out-of-log metadata (`SessionHeader`) lives in a `sessions` row. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`list` reports exactly the sessions that have a row), so no separate column is needed. +Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [session surface](../../../docs/rfc/implemented/architecture/2026-06-18-session-surface.md)). Out-of-log metadata (`SessionHeader`) and a monotonic per-log revision live in a `sessions` row. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`list` reports exactly the sessions that have a row). The repo's `engines.node` is `^22.19.0 || >=24.0.0` (Node 22.19+ or 24+), matching the LTS floor required by the installed Pi adapter dependency; `node:sqlite` itself ships without the `--experimental-sqlite` flag from Node 22.13 (LTS) and 23.4 / 24 (Current) on. The range deliberately excludes Node 23 because that line is non-LTS/EOL and still has flagged runtime features before 23.6. The database opens with `foreign_keys = ON` (so `ON DELETE CASCADE` drops a session's events with its row) and the configured `journal_mode` (default `wal`; pick a rollback-journal mode like `delete` on filesystems where WAL's shared-memory files do not work, e.g. network mounts). The table-layout version is stored in `PRAGMA user_version` and checked on open: a fresh database is stamped with the current `SCHEMA_VERSION`; a database written by any other, incompatible build (a non-current `user_version`, older or newer) is rejected rather than opened against an unknown layout — there is no migration (unreleased software). @@ -14,6 +14,7 @@ The repo's `engines.node` is `^22.19.0 || >=24.0.0` (Node 22.19+ or 24+), matchi - **Append = a transaction.** `append` runs `BEGIN`/`COMMIT` around the batch: it materializes the `sessions` row (if still lazy) and INSERTs every event, asserting the contiguous-seq contract first (the first event's `seq` must equal the stored next-seq). A mid-batch failure (a UNIQUE violation on a duplicated seq) rolls back entirely, so the stored log and the in-memory cursor stay consistent. (`load()` already balanced the stored log, so `append` never has to repair a crash tail.) - **Lazy materialization.** `create()` records intent in memory only — no row is written until the first `append`. A created-but-never-appended session has no `sessions` row, so it is absent from `list()` (which reports exactly the sessions that have a row). +- **Lightweight revisions.** `listSnapshots()` reads the monotonic revision stored beside each session header. Append and mutating load repair increment it in the same transaction as their event changes, so unchanged observations are stable and no full-log count or parse is required. - **Interrupted-turn close on load.** `load()` reads every stored event ordered by `seq` and finds the longest seq-contiguous, parseable prefix — INCLUDING the real events of an interrupted final turn after the last `turn/end` (the loop only flushes at `turn/end`, so a process killed mid-turn leaves real, fully-written rows past it). A single turn can be huge in a long-horizon task, so those events are **preserved, never truncated**: `load()` CLOSES the orphaned turn by durably appending the minimal synthetic boundary events (an error `tool/result` for every assistant tool call left unanswered, a `step/end` if a step was open, then a `turn/end` carrying `{ kind: 'interrupted' }`), inside one transaction that also DELETEs any never-fully-written torn tail row. `load()` is therefore mutating — after it the stored rows are balanced and the cursor is truthful, so the next `append` continues cleanly. The boundary (last `turn/end`, torn-tail detection) is computed from the `seq`/`type` columns so a malformed `data` in a torn tail row is never parsed (discarded, not unloadable). A parse error or `seq` gap inside the committed region (at or before the last real `turn/end`) makes the session unloadable. A session whose only turn never closed keeps its metadata row and stays present in `list()` — the same as the JSONL backend, whose file likewise survives a first append that never reached `turn/end`. ## Configuration (schemastery) diff --git a/packages/session-persistence/session-persistence-sqlite/src/index.ts b/packages/session-persistence/session-persistence-sqlite/src/index.ts index 30387b4837..3a1c7ffbcc 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/index.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/index.ts @@ -11,7 +11,7 @@ * Like the JSONL backend it supplies ONLY the storage primitives (the * {@link PersistenceBackend} hooks below — INSERT/DELETE/SELECT inside * transactions); all the write-path orchestration lives in the backend-agnostic - * {@link PersistenceCoordinator} this class composes. The four public + * {@link PersistenceCoordinator} this class composes. The stateful public * {@link SessionPersistence} methods delegate to the coordinator. * * @module @deepseek-ai/dsh-session-persistence-sqlite @@ -23,8 +23,8 @@ import { DatabaseSync } from 'node:sqlite' import { mkdir } from 'node:fs/promises' import { dirname, resolve } from 'node:path' import { - SessionPersistence, PersistenceCoordinator, - type PersistenceBackend, type StoredPrefix, + SessionPersistence, SessionPersistenceRevision, PersistenceCoordinator, + type PersistenceBackend, type SessionPersistenceSnapshot, type StoredPrefix, } from '@deepseek-ai/dsh-session-persistence' import type { Session, SessionEvent, SurfaceEventType, SessionId, SessionHeader } from '@deepseek-ai/dsh-session' import { @@ -181,6 +181,7 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers const [surfaceSeqs, surfaceOp] = surfaceBindings(event) insertEvent.run(meta.id, event.seq, event.type, event.time, JSON.stringify(event.data), surfaceSeqs, surfaceOp) } + this.db.prepare('UPDATE sessions SET revision = revision + 1 WHERE id = ?').run(meta.id) this.db.exec('COMMIT') } catch (error) { this.db.exec('ROLLBACK') @@ -209,6 +210,9 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers insertEvent.run(meta.id, event.seq, event.type, event.time, JSON.stringify(event.data), surfaceSeqs, surfaceOp) } } + if (tornMarker !== undefined || closers.length > 0) { + this.db.prepare('UPDATE sessions SET revision = revision + 1 WHERE id = ?').run(meta.id) + } this.db.exec('COMMIT') } catch (error) { // The DELETE+INSERT cannot collide (a row at a closer's seq is preserved or @@ -230,6 +234,16 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers return rows.map(rowToMeta) } + /** List metadata with an append-only event-count revision per session. */ + async listSnapshots(): Promise { + await this.ready + const rows = this.db.prepare('SELECT * FROM sessions').all() as unknown as SessionRow[] + return rows.map(row => ({ + header: rowToMeta(row), + revision: SessionPersistenceRevision(`revision:${row.revision}`), + })) + } + /** Close the database handle (awaited by the coordinator's dispose, post-drain). */ async close(): Promise { await this.ready @@ -250,8 +264,8 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers */ private writeRow(meta: SessionHeader): void { this.db.prepare(` - INSERT INTO sessions (id, version, created_at, cwd, parent_session, seed_length) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO sessions (id, version, created_at, cwd, parent_session, seed_length, revision) + VALUES (?, ?, ?, ?, ?, ?, 0) ON CONFLICT(id) DO UPDATE SET version = excluded.version, created_at = excluded.created_at, diff --git a/packages/session-persistence/session-persistence-sqlite/src/schema.ts b/packages/session-persistence/session-persistence-sqlite/src/schema.ts index 2dacbe04a0..2f238b0131 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/schema.ts @@ -15,7 +15,7 @@ import type { SessionEvent, SessionId, SessionHeader, SurfaceOp } from '@deepsee * layout; orthogonal to a session's own `version` (which versions the EVENT * vocabulary, stored per session in the `sessions` row). */ -export const SCHEMA_VERSION = 4 +export const SCHEMA_VERSION = 5 /** * A row of the `sessions` table — the out-of-log metadata ({@link SessionHeader}). @@ -31,6 +31,8 @@ export interface SessionRow { cwd: string | null parent_session: string | null seed_length: number | null + /** Monotonic log-change token incremented in each mutating transaction. */ + revision: number } /** An `events` table row: one `SessionEvent` mapped 1:1 (`data` is JSON text). */ @@ -67,15 +69,9 @@ export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' * current {@link SCHEMA_VERSION}; an existing database whose version is NOT the * current one (written by a different, incompatible build — older or newer) is * REJECTED rather than opened against a layout this build does not understand. - * There are no migrations: an earlier layout is not upgraded in place — it is - * rejected. v1 had a different `sessions` shape; v2 lacked all of - * `seed_length`/`source_event_seqs`/`surface_op`. v3 is SKIPPED: two unmerged - * branches each shipped a DISTINCT v3 (one adding only `seed_length`, the other - * adding only the surface columns), so an on-disk v3 is ambiguous — it could be - * either sibling layout, neither of which has all of this build's columns. v4 - * is the merged layout carrying every column; bumping past the collided v3 - * makes the version check reject both sibling v3 databases instead of opening - * one against columns it does not have. + * There are no migrations: an incompatible layout is rejected. The current + * sessions row carries every header field plus its monotonic snapshot revision; + * the events row carries the complete surface metadata. * @param path - the SQLite database file to open (created when absent). * @param journalMode - the journal pragma to apply — a closed in-code union, validated by the plugin Config. * @returns the open handle with pragmas applied and both tables ensured. @@ -105,7 +101,8 @@ export function openDatabase(path: string, journalMode: JournalMode): DatabaseSy created_at INTEGER NOT NULL, cwd TEXT, parent_session TEXT, - seed_length INTEGER + seed_length INTEGER, + revision INTEGER NOT NULL ) STRICT `) db.exec(` diff --git a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts index a6eda72685..98924ca691 100644 --- a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts @@ -258,11 +258,11 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { it('rejects a sibling v3 database (the merge-collided version) rather than opening it against missing columns', async () => { // Two unmerged branches each shipped a DISTINCT layout under user_version 3 // (one added only `seed_length`, the other only the surface columns). The - // merged build is v4; an on-disk v3 is ambiguous and is missing at least one + // the current build rejects every older layout; an on-disk v3 is ambiguous and is missing at least one // of this build's columns, so it MUST be rejected, not opened. Stamp a v3 // database and confirm the version check refuses it. const path = await freshDbPath() - openDatabase(path, 'wal').close() // creates + stamps user_version = SCHEMA_VERSION (4) + openDatabase(path, 'wal').close() // creates + stamps user_version = SCHEMA_VERSION const db = openDatabase(path, 'wal') db.exec('PRAGMA user_version = 3') db.close() @@ -338,7 +338,18 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { }) it('exposes the schema version constant', () => { - expect(SCHEMA_VERSION).toBe(4) + expect(SCHEMA_VERSION).toBe(5) + }) + + it('keeps the revision stable for an empty repair hook', async () => { + const b = await backend() + const m = meta('empty-repair') + await b.ctx.sessionPersistence.create(m) + await b.ctx.sessionPersistence.append(m.id, oneTurnLog()) + const before = await b.ctx.sessionPersistence.listSnapshots() + await (b.ctx.sessionPersistence as SessionPersistenceSqlite).commitRepair(m, undefined, []) + expect(await b.ctx.sessionPersistence.listSnapshots()).toEqual(before) + await b.dispose() }) }) diff --git a/packages/session-persistence/session-persistence/README.md b/packages/session-persistence/session-persistence/README.md index bf7da03757..32958bfaaa 100644 --- a/packages/session-persistence/session-persistence/README.md +++ b/packages/session-persistence/session-persistence/README.md @@ -12,6 +12,7 @@ The persisted unit IS the existing `SessionEvent` (event-sourced model — the l | `append(id, events): Promise` | Durably persist a batch (from the `session/flush` drain). Append-only; first event `seq` == stored next-seq after any repair; rejects non-JSON-serializable data naming the offending type. | | `load(id): Promise<{ meta; events }>` | Reload meta + log. Preserves an interrupted (unclosed) final turn and closes it with synthetic closers — an error `tool/result` per unanswered `tool-call`, then `step/end?`+`turn/end {interrupted}` (a turn can be huge — never truncated); only a torn tail fragment is dropped. Events contiguous (`events[i].seq === i`); rejects a committed-region gap/parse error or unknown `version`. | | `list(): Promise` | Lightweight listing from metadata, no full-log parse. A zero-event lazily-materialized session is absent from `list`. | +| `listSnapshots(): Promise` | Lightweight metadata plus an opaque branded per-log revision, without loading event logs. A revision stays equal while that log is unchanged and changes after append or mutating load repair. | ## Invariants every backend must honor @@ -24,7 +25,7 @@ The persisted unit IS the existing `SessionEvent` (event-sourced model — the l The two first-party backends were byte-identical (or same-algorithm) for ALL of their write-path orchestration — the in-memory bookkeeping (per-id state, write-behind buffers, per-id serialization chains, per-session init promises), the `session/event` → buffer → `session/flush` drain, lazy materialization, crash-tail repair on load, the four `session/created` adoption cases (new / HMR-adopt / collision / ownerless-claim), and dispose-time quiescence. Only the STORAGE primitives differed (write bytes vs. INSERT rows). -`PersistenceCoordinator` owns that orchestration once. A first-party backend composes one (`new PersistenceCoordinator(ctx, this)`), implements the small `PersistenceBackend` hook interface, and delegates its four public service methods to the coordinator. This keeps the duplicated, correctness-heavy orchestration in a single place (it used to receive the same fixes twice). +`PersistenceCoordinator` owns that orchestration once. A first-party backend composes one (`new PersistenceCoordinator(ctx, this)`), implements the small `PersistenceBackend` hook interface, and delegates the stateful write/read methods to the coordinator. Lightweight snapshot listing remains a backend storage primitive because its revision identity is backend-owned. The `PersistenceBackend` hooks (the only seam between the coordinator and storage): @@ -38,11 +39,11 @@ The `PersistenceBackend` hooks (the only seam between the coordinato | `list()` | List all stored metadata. | | `close?()` | Optional lifecycle teardown (e.g. close a db handle), awaited after the dispose drain. | -The `tornMarker` is fully OPAQUE: the coordinator only tests `!== undefined` and round-trips it to `commitRepair`, never inspecting its value (the JSONL backend uses the byte offset to truncate to, the SQLite backend the seq to delete from). The public `SessionPersistence` service shape is unchanged, so a third-party backend MAY still implement the abstract service directly without the coordinator. See [the write-coordinator RFC](../../../docs/rfc/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md). +The `tornMarker` is fully OPAQUE: the coordinator only tests `!== undefined` and round-trips it to `commitRepair`, never inspecting its value (the JSONL backend uses the byte offset to truncate to, the SQLite backend the seq to delete from). A third-party backend MAY implement the abstract service directly without the coordinator, but it must also provide trustworthy lightweight snapshot revisions. See [the write-coordinator RFC](../../../docs/rfc/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md). ## Testing backends -Import `runPersistenceContract` from `tests/contract.ts` (the public-API contract) and `runCoordinatorContract` from `tests/coordinator-contract.ts` (the shared write-path orchestration: adoption, HMR, collision, dispose-drain, crash-tail repair) and call each with a fixture for your backend. Every backend is held to the same append-only / contiguous-seq / lazy-materialization / serializability semantics AND the same orchestration, so a backend's own spec is left with only storage-mechanics tests (path sanitization, fsync rollback; schema version, transaction rollback) on top. +Import `runPersistenceContract` from `tests/contract.ts` (the public API, including stable/change-sensitive lightweight revisions) and `runCoordinatorContract` from `tests/coordinator-contract.ts` (the shared write-path orchestration: adoption, HMR, collision, dispose-drain, crash-tail repair) and call each with a fixture for your backend. Every backend is held to the same append-only / contiguous-seq / lazy-materialization / serializability semantics AND the same orchestration, so a backend's own spec is left with only storage-mechanics tests (path sanitization, fsync rollback; schema version, transaction rollback) on top. Three backends run these suites: an in-memory reference (in `tests/`), `dsh-session-persistence-jsonl` (append-only file log) and `dsh-session-persistence-sqlite` (`node:sqlite`, each `SessionEvent` one row `(session_id, seq, type, time, data, source_event_seqs, surface_op)`). All passing the same contract + coordinator suite is the proof that the seam is genuinely backend-agnostic — lazy materialization, crash-tail-on-load, and contiguous-seq hold identically over file bytes and over a transactional store. diff --git a/packages/session-persistence/session-persistence/package.json b/packages/session-persistence/session-persistence/package.json index ed6c80dfd9..8b7140e221 100644 --- a/packages/session-persistence/session-persistence/package.json +++ b/packages/session-persistence/session-persistence/package.json @@ -22,10 +22,12 @@ ], "license": "BSD-3-Clause", "peerDependencies": { + "@deepseek-ai/dsh-brand": "^0.0.1", "@deepseek-ai/dsh-session": "^0.0.1", "cordis": "^4.0.0-rc.6" }, "devDependencies": { + "@deepseek-ai/dsh-brand": "workspace:^", "@deepseek-ai/dsh-session": "workspace:^", "cordis": "^4.0.0-rc.6" } diff --git a/packages/session-persistence/session-persistence/src/coordinator.ts b/packages/session-persistence/session-persistence/src/coordinator.ts index b1fc118a21..7857b57a3c 100644 --- a/packages/session-persistence/session-persistence/src/coordinator.ts +++ b/packages/session-persistence/session-persistence/src/coordinator.ts @@ -14,7 +14,7 @@ * {@link PersistenceBackend} hook object. * * The abstract {@link SessionPersistence} service's public API is independent of - * this: a backend IS a `SessionPersistence` (its four public methods delegate to + * this: a backend IS a `SessionPersistence` (its write/read methods delegate to * a coordinator it composes), so a third-party backend MAY implement the service * directly without using the coordinator at all. * @@ -146,7 +146,7 @@ async function settledErrors(promises: Iterable>): Promise + + /** + * List materialized sessions with cheap per-log change tokens. + * + * Repeated observations of an unchanged log return the same revision. A + * successful mutating {@link load} repair changes the next listed revision. + * @returns one header and opaque revision per materialized session without loading full logs. + */ + abstract listSnapshots(): Promise } export default SessionPersistence diff --git a/packages/session-persistence/session-persistence/src/revision.ts b/packages/session-persistence/session-persistence/src/revision.ts new file mode 100644 index 0000000000..41378eb3e4 --- /dev/null +++ b/packages/session-persistence/session-persistence/src/revision.ts @@ -0,0 +1,15 @@ +/** Opaque revision identity for lightweight persistence observations. */ + +import type { Branded } from '@deepseek-ai/dsh-brand' + +/** Backend-owned token that changes whenever one persisted session log changes. */ +export type SessionPersistenceRevision = Branded<'SessionPersistenceRevision'> + +/** + * Brand a backend revision for the provider-neutral persistence contract. + * @param value - backend-owned opaque revision representation. + * @returns the same runtime string with persistence-revision identity. + */ +export function SessionPersistenceRevision(value: string): SessionPersistenceRevision { + return value as SessionPersistenceRevision +} diff --git a/packages/session-persistence/session-persistence/tests/contract.ts b/packages/session-persistence/session-persistence/tests/contract.ts index 789aa72c91..d1699c58a5 100644 --- a/packages/session-persistence/session-persistence/tests/contract.ts +++ b/packages/session-persistence/session-persistence/tests/contract.ts @@ -102,11 +102,16 @@ export function runPersistenceContract(name: string, make: () => Promise snapshot.header.id === m.id)?.revision // load PRESERVES the interrupted turn's events (a turn can be huge — they // must not be truncated) and closes the orphaned turn with synthetic // boundary events: step/end (the step was open) then turn/end {interrupted}. const loaded = await persistence.load(m.id) + const afterRepair = (await persistence.listSnapshots()) + .find(snapshot => snapshot.header.id === m.id)?.revision + expect(afterRepair).not.toBe(beforeRepair) expect(loaded.events.map(e => e.type)).toEqual([ 'turn/start', 'user/message', 'step/start', 'assistant/message', 'step/end', 'turn/end', // turn 1 'turn/start', 'step/start', 'step/end', 'turn/end', // turn 2: real events + synthetic closers @@ -173,18 +178,33 @@ export function runPersistenceContract(name: string, make: () => Promise m.id)).not.toContain(SessionId('empty')) + expect((await persistence.listSnapshots()).map(snapshot => snapshot.header.id)) + .not.toContain(SessionId('empty')) } finally { await dispose() } }) - it('list() includes a session once it has events', async () => { + it('lists stable lightweight revisions that change after an append', async () => { const { persistence, dispose } = await make() try { const m = meta('s2') await persistence.create(m) await persistence.append(m.id, oneTurnLog()) expect((await persistence.list()).map(x => x.id)).toContain(m.id) + const first = (await persistence.listSnapshots()).find(snapshot => snapshot.header.id === m.id) + const repeated = (await persistence.listSnapshots()).find(snapshot => snapshot.header.id === m.id) + expect(first).toBeDefined() + expect(repeated?.revision).toBe(first?.revision) + + await persistence.append(m.id, [{ + type: 'turn/start', + seq: 6, + time: 7, + data: { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } }, + }]) + const changed = (await persistence.listSnapshots()).find(snapshot => snapshot.header.id === m.id) + expect(changed?.revision).not.toBe(first?.revision) } finally { await dispose() } diff --git a/packages/session-persistence/session-persistence/tests/persistence.spec.ts b/packages/session-persistence/session-persistence/tests/persistence.spec.ts index 9c766d4b66..00e9863bb8 100644 --- a/packages/session-persistence/session-persistence/tests/persistence.spec.ts +++ b/packages/session-persistence/session-persistence/tests/persistence.spec.ts @@ -3,8 +3,8 @@ import { Context } from 'cordis' import SessionStore, { SessionId, isJsonValue } from '@deepseek-ai/dsh-session' import type { Session, SessionEvent, SessionHeader } from '@deepseek-ai/dsh-session' import { - SessionPersistence, PersistenceCoordinator, assertSerializable, seedCoversPrefix, - type PersistenceBackend, type StoredPrefix, + SessionPersistence, SessionPersistenceRevision, PersistenceCoordinator, assertSerializable, seedCoversPrefix, + type PersistenceBackend, type SessionPersistenceSnapshot, type StoredPrefix, } from '../src/index.ts' import { runPersistenceContract, meta, oneTurnLog } from './contract.ts' import { runCoordinatorContract, type CoordinatorFixture } from './coordinator-contract.ts' @@ -109,6 +109,14 @@ class MemoryPersistence extends SessionPersistence implements PersistenceBackend async list(): Promise { return [...this.store.values()].map(e => structuredClone(e.meta)) } + + + async listSnapshots(): Promise { + return [...this.store.values()].map(entry => ({ + header: structuredClone(entry.meta), + revision: SessionPersistenceRevision(`events:${entry.events.length}`), + })) + } } // Run the shared contract against the in-memory backend. diff --git a/packages/session-persistence/session-persistence/tsconfig.json b/packages/session-persistence/session-persistence/tsconfig.json index e817086a6a..84c6f5ccb0 100644 --- a/packages/session-persistence/session-persistence/tsconfig.json +++ b/packages/session-persistence/session-persistence/tsconfig.json @@ -14,6 +14,9 @@ { "path": "../../../vendor/cordis" }, + { + "path": "../../util/brand" + }, { "path": "../../core/session" } diff --git a/packages/session-query/session-query-sqlite/README.md b/packages/session-query/session-query-sqlite/README.md index 1f3344887f..001ea87d7d 100644 --- a/packages/session-query/session-query-sqlite/README.md +++ b/packages/session-query/session-query-sqlite/README.md @@ -1,22 +1,22 @@ # @deepseek-ai/dsh-session-query-sqlite -SQLite FTS5 implementation of `ctx.sessionSearch`. The service searches the live-preferred logical session corpus, groups cross-session results by their strongest event, and keeps provider-specific BM25 scores private. +SQLite FTS5 implementation of `ctx.sessionSearch`. The service searches the live-preferred logical session corpus and groups cross-session results by their strongest event. ## Search contract `searchSessions(request, exec?)` returns `SessionSearchHit` pages across the corpus; `searchEvents(request, exec?)` returns `SessionEventSearchHit` pages within one session. Queries are required, trimmed, whitespace-normalized literal phrases. FTS5 syntax such as quotes, `OR`, `NEAR`, and `*` is treated as data rather than executable MATCH syntax. Metadata filters are parameterized SQL predicates applied before ranking. -Ordering is deterministic: relevance first, then event time, session id where applicable, and seq. Cross-session results expose the selected event as `bestMatch`; both scopes return plain-text snippets bounded in Unicode code points. Cursors are opaque, bind to the normalized request and service instance, and fail when the relevant generation changes. A within-session cursor survives unrelated-session changes; a cross-session cursor does not. +Relevance is source-comparable across persistent and TEMP tables: actual FTS5 highlighted-match span count descending, then stored document code-point length ascending. Event time, session id where applicable, and seq break remaining ties. Cross-session results expose the selected event as `bestMatch`; both scopes derive whitespace-normalized plain text from FTS5 highlight positions and bound it in Unicode code points. Cursors are opaque branded values, bind to the normalized request and service instance, and fail when the relevant generation changes. A within-session cursor survives unrelated-session changes; a cross-session cursor does not. All three surfaces (`current`, `shadowed`, and `log-only`) are searchable by default. Pass a surface filter to narrow them. ## Source and index lifecycle -The service requires `ctx.sessions` and observes optional `ctx.sessionPersistence` dynamically. One serialized state machine observes complete sources, extracts shared semantic documents, reconciles changes transactionally, and runs the query. Stable fingerprints preserve unchanged persisted rows and generations; new, changed, and deleted durable sessions reconcile on the next search. Source or transaction failure commits nothing, and the next search retries. +The service requires `ctx.sessions` and observes optional `ctx.sessionPersistence` dynamically. One serialized state machine compares lightweight durable snapshot revisions, loads only new or changed logs, extracts shared semantic documents, reconciles changes transactionally, and runs the query. Repeated queries and unchanged reopen load no full durable logs; new, changed, deleted, or load-repaired sources reconcile on the next stable observation. Source or transaction failure commits nothing, and the next search retries. Persisted FTS rows live in a dedicated derived database. Connection-local TEMP tables hold live rows, which shadow the durable base for the same session and reveal it when the live owner disappears. Unmounting persistence hides durable rows without discarding the cache; remounting reconciles it. Closing or reopening the database drops every live overlay while retaining persisted rows. -The database is disposable but reset is guarded: a recognized incompatible search schema rebuilds in place, while an unrelated or canonical database is refused. Never point `path` at the session-persistence database. +The database is disposable but reset is guarded: a recognized incompatible search schema rebuilds in place, while an unrelated or canonical database is refused before mutating journal mode. Never point `path` at the session-persistence database. Exactly one service in one process owns a derived-index path; external writers or a second process are unsupported because generations and TEMP shadow state are connection-owned. ## Configuration @@ -30,6 +30,6 @@ The database is disposable but reset is guarded: a recognized incompatible searc ## Tokenizer and limits -The index uses FTS5 `unicode61`. In the implementation experiment it supported the two-character query `AI` and produced an index about 2.1× smaller than the trigram alternative. The trade-off is token/phrase recall rather than arbitrary substring recall: `AI` does not match the token `BRAID`. Use `ctx.sessionQuery.filterEvents()` with a `text` clause when a literal whitespace-flexible substring scan is required. +The index uses FTS5 `unicode61`. In the implementation experiment it supported the two-character query `AI` and produced an index about 2.1× smaller than the trigram alternative. The trade-off is token/phrase recall rather than arbitrary substring recall: `AI` does not match the token `BRAID`. Use `ctx.sessionQuery.filterEvents()` with a `text` clause when a literal whitespace-flexible substring scan is required. NUL is rejected in queries; reserved highlight markers and NUL in documents are normalized before indexing so presentation markers cannot collide with source text. Abort signals stop queued work and caller waits around asynchronous source observation. Node's synchronous `DatabaseSync` API cannot interrupt a MATCH statement already executing on the JavaScript thread; the signal is checked immediately before and after the serialized observation/reconciliation boundary. diff --git a/packages/session-query/session-query-sqlite/src/index.ts b/packages/session-query/session-query-sqlite/src/index.ts index a6cba8d866..dd1ddb532a 100644 --- a/packages/session-query/session-query-sqlite/src/index.ts +++ b/packages/session-query/session-query-sqlite/src/index.ts @@ -6,12 +6,17 @@ import { createHash, randomUUID } from 'node:crypto' import { DatabaseSync } from 'node:sqlite' -import { Context } from 'cordis' +import { Context, type Fiber } from 'cordis' import z from 'schemastery' import type { Session, SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session' import type SessionPersistence from '@deepseek-ai/dsh-session-persistence' +import type { + SessionPersistenceRevision, + SessionPersistenceSnapshot, +} from '@deepseek-ai/dsh-session-persistence' import { SessionQueryError, + SessionSearchCursor, SessionSearchService, assertSessionHeadersCompatible, buildSessionEventSearchDocuments, @@ -22,6 +27,7 @@ import type { SessionEventSearchRequest, SessionSearchExecContext, SessionSearchHit, + SessionSearchCursor as SessionSearchCursorValue, SessionSearchPage, SessionSearchRequest, } from '@deepseek-ai/dsh-session-query' @@ -32,6 +38,8 @@ import { import { type NormalizedEventRequest, type NormalizedSessionRequest, + FTS_HIGHLIGHT_END, + FTS_HIGHLIGHT_START, buildEventWhere, buildSessionWhere, makeSnippet, @@ -39,6 +47,7 @@ import { normalizeSessionRequest, quoteFtsData, requestFingerprint, + sanitizeFtsText, } from './query.ts' export { @@ -78,19 +87,30 @@ interface ResolvedConfig { interface ObservedSession { header: SessionHeader - events: SessionEvent[] documents: SessionEventSearchDocument[] fingerprint: string } +interface ObservedPersistedSession { + header: SessionHeader + revision: SessionPersistenceRevision + loaded?: ObservedSession +} + interface Observation { persistence: SessionPersistence | undefined persistenceRevision: number - persisted: Map + persisted: Map live: Map } -interface IndexedRow { +interface IndexedPersistedRow { + id: string + revision: string + generation: number +} + +interface IndexedLiveRow { id: string fingerprint: string generation: number @@ -109,8 +129,9 @@ interface SearchRow { type: string time: number surface: string - text: string - score: number + marked_text: string + match_count: number + document_length: number } interface CursorPayload { @@ -149,27 +170,32 @@ export class SessionSearchSqlite extends SessionSearchService { private _localGeneration = 0 private _tail: Promise = Promise.resolve() private _closed = false + private _closePromise: Promise | undefined + private readonly _optionalPersistenceFiber: Fiber constructor(ctx: Context, config: Config) { super(ctx) this.config = resolveConfig(config) this._ready = this._open() - ctx.effect(() => { - const fiber = ctx.inject(['sessionPersistence'], (childCtx: Context) => { - const service = childCtx.sessionPersistence - const binding = {} - this._persistenceBinding = binding - this._persistence = service + // Attach a rejection observer immediately; callers still receive the same + // rejection from `_ready`, including when no search is ever attempted. + void this._ready.catch(() => undefined) + this._optionalPersistenceFiber = ctx.inject(['sessionPersistence'], (childCtx: Context) => { + const service = childCtx.sessionPersistence + const binding = {} + this._persistenceBinding = binding + this._persistence = service + this._persistenceRevision += 1 + childCtx.effect(() => () => { + /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */ + if (this._persistenceBinding !== binding) return + this._persistenceBinding = undefined + this._persistence = undefined this._persistenceRevision += 1 - childCtx.effect(() => () => { - /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */ - if (this._persistenceBinding !== binding) return - this._persistenceBinding = undefined - this._persistence = undefined - this._persistenceRevision += 1 - }, 'sessionSearchSqlite.persistenceBinding') - }) - return () => void fiber.dispose() + }, 'sessionSearchSqlite.persistenceBinding') + }) + ctx.effect(() => { + return () => this._optionalPersistenceFiber.dispose() }, 'sessionSearchSqlite.optionalPersistence') ctx.effect(() => async () => this.close(), 'sessionSearchSqlite.close') } @@ -179,17 +205,18 @@ export class SessionSearchSqlite extends SessionSearchService { exec?: SessionSearchExecContext, ): Promise> { const normalized = normalizeSessionRequest(request, this.config) - return this._serialized(exec?.signal, async () => { - await this._ensureReady(exec?.signal) - await this._reconcile(exec?.signal) - assertNotAborted(exec?.signal) + const signal = exec?.signal + return this._serialized(signal, async () => { + await this._ensureReady(signal) + await this._reconcile(signal) + assertNotAborted(signal) const generation = String(this._globalGeneration) const fingerprint = requestFingerprint(normalized) const offset = normalized.cursor === undefined ? 0 : decodeCursor(normalized.cursor, this._instance, 'sessions', fingerprint, generation) const rows = this._querySessions(normalized, offset) - return page(rows, normalized.limit, row => this._sessionHit(row, normalized.query), cursorOffset => encodeCursor({ + return page(rows, normalized.limit, row => this._sessionHit(row), cursorOffset => encodeCursor({ version: 1, instance: this._instance, scope: 'sessions', @@ -205,17 +232,18 @@ export class SessionSearchSqlite extends SessionSearchService { exec?: SessionSearchExecContext, ): Promise> { const normalized = normalizeEventRequest(request, this.config) - return this._serialized(exec?.signal, async () => { - await this._ensureReady(exec?.signal) - await this._reconcile(exec?.signal) - assertNotAborted(exec?.signal) + const signal = exec?.signal + return this._serialized(signal, async () => { + await this._ensureReady(signal) + await this._reconcile(signal) + assertNotAborted(signal) const generation = this._targetGeneration(normalized.sessionId) const fingerprint = requestFingerprint(normalized) const offset = normalized.cursor === undefined ? 0 : decodeCursor(normalized.cursor, this._instance, 'events', fingerprint, generation) const rows = this._queryEvents(normalized, offset) - return page(rows, normalized.limit, row => this._eventHit(row, normalized.query), cursorOffset => encodeCursor({ + return page(rows, normalized.limit, row => this._eventHit(row), cursorOffset => encodeCursor({ version: 1, instance: this._instance, scope: 'events', @@ -227,8 +255,12 @@ export class SessionSearchSqlite extends SessionSearchService { } /** Close the database after every accepted operation reaches quiescence. */ - async close(): Promise { - if (this._closed) return + close(): Promise { + this._closePromise ??= this._close() + return this._closePromise + } + + private async _close(): Promise { this._closed = true await this._tail try { @@ -287,20 +319,20 @@ export class SessionSearchSqlite extends SessionSearchService { } private async _reconcile(signal: AbortSignal | undefined): Promise { - const observation = await this._observeStable(signal) - assertNotAborted(signal) const db = this._requireDb() const persistedRows = db.prepare( - 'SELECT id, fingerprint, generation FROM persisted_sessions', - ).all() as unknown as IndexedRow[] + 'SELECT id, revision, generation FROM persisted_sessions', + ).all() as unknown as IndexedPersistedRow[] const liveRows = db.prepare( 'SELECT id, fingerprint, generation FROM temp.live_sessions', - ).all() as unknown as IndexedRow[] + ).all() as unknown as IndexedLiveRow[] const persistedById = new Map(persistedRows.map(row => [row.id as SessionId, row])) const liveById = new Map(liveRows.map(row => [row.id as SessionId, row])) + const observation = await this._observeStable(persistedById, signal) + assertNotAborted(signal) const persistentChanges = observation.persistence === undefined ? [] - : [...observation.persisted.values()].filter(entry => persistedById.get(entry.header.id)?.fingerprint !== entry.fingerprint) + : [...observation.persisted.values()].filter(entry => entry.loaded !== undefined) const persistentDeletes = observation.persistence === undefined ? [] : persistedRows.filter(row => !observation.persisted.has(row.id as SessionId)) @@ -327,13 +359,17 @@ export class SessionSearchSqlite extends SessionSearchService { db.exec('BEGIN IMMEDIATE') began = true for (const row of persistentDeletes) this._deleteSession('persisted', row.id as SessionId) - for (const entry of persistentChanges) this._replaceSession('persisted', entry, nextMainGeneration) + for (const entry of persistentChanges) { + /* v8 ignore next -- observation loads every entry whose revision differs */ + if (entry.loaded === undefined) throw new Error(`missing loaded revision for session "${entry.header.id}"`) + this._replacePersistedSession(entry.loaded, entry.revision, nextMainGeneration) + } if (persistentChanges.length > 0 || persistentDeletes.length > 0) { db.prepare('UPDATE search_state SET global_generation = ? WHERE singleton = 1').run(nextMainGeneration) } for (const row of liveDeletes) this._deleteSession('live', row.id as SessionId) for (const { entry, generation } of liveReplacements) { - this._replaceSession('live', entry, generation) + this._replaceLiveSession(entry, generation) } db.exec('COMMIT') } catch (error: unknown) { @@ -360,21 +396,39 @@ export class SessionSearchSqlite extends SessionSearchService { this._lastPersistenceRevision = observation.persistenceRevision } - private async _observeStable(signal: AbortSignal | undefined): Promise { + private async _observeStable( + indexed: ReadonlyMap, + signal: AbortSignal | undefined, + ): Promise { for (;;) { assertNotAborted(signal) const persistence = this._persistence const persistenceRevision = this._persistenceRevision - const persisted = new Map() + let persisted = new Map() if (persistence !== undefined) { try { - const headers = await waitWithAbort(persistence.list(), signal) - for (const listed of headers) { - const loaded = await waitWithAbort(persistence.load(listed.id), signal) - assertSessionHeadersCompatible(listed, loaded.meta) - persisted.set(listed.id, observeSession(loaded.meta, loaded.events)) + const canReuseIndexed = this._lastPersistenceRevision === undefined + || this._lastPersistenceRevision === persistenceRevision + const before = await waitWithAbort(persistence.listSnapshots(), signal) + persisted = materializePersistenceSnapshots(before) + for (const entry of persisted.values()) { + if (canReuseIndexed && indexed.get(entry.header.id)?.revision === entry.revision) continue + const loaded = await waitWithAbort(persistence.load(entry.header.id), signal) + assertSessionHeadersCompatible(entry.header, loaded.meta) + entry.loaded = observeSession(loaded.meta, loaded.events) } + const after = materializePersistenceSnapshots( + await waitWithAbort(persistence.listSnapshots(), signal), + ) + if (!samePersistenceSnapshots(persisted, after)) continue + if (this._persistenceRevision !== persistenceRevision) continue } catch (error: unknown) { + if (isAbort(error) || signal?.aborted) { + throw new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED', { + cause: error, + }) + } + if (this._persistenceRevision !== persistenceRevision) continue if (error instanceof SessionQueryError) throw error throw new SessionQueryError( `session-search persistence observation failed: ${errorMessage(error)}`, @@ -414,13 +468,50 @@ export class SessionSearchSqlite extends SessionSearchService { } } - private _replaceSession(source: 'persisted' | 'live', entry: ObservedSession, generation: number): void { - this._deleteSession(source, entry.header.id) + private _replacePersistedSession( + entry: ObservedSession, + revision: SessionPersistenceRevision, + generation: number, + ): void { + this._deleteSession('persisted', entry.header.id) const db = this._requireDb() - const sessionTable = source === 'persisted' ? 'persisted_sessions' : 'temp.live_sessions' - const docsTable = source === 'persisted' ? 'persisted_docs' : 'temp.live_docs' db.prepare(` - INSERT INTO ${sessionTable} + INSERT INTO persisted_sessions + (id, version, created_at, cwd, parent_session, seed_length, revision, generation) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `).run( + entry.header.id, + entry.header.version, + entry.header.createdAt, + entry.header.cwd ?? null, + entry.header.parentSession ?? null, + entry.header.seedLength ?? null, + revision, + generation, + ) + const insert = db.prepare(` + INSERT INTO persisted_docs (text, session_id, seq, type, time, surface, codepoint_length) + VALUES (?, ?, ?, ?, ?, ?, ?) + `) + for (const document of entry.documents) { + const text = sanitizeFtsText(document.text) + insert.run( + text, + document.sessionId, + document.seq, + document.type, + document.time, + document.surface, + Array.from(text).length, + ) + } + } + + private _replaceLiveSession(entry: ObservedSession, generation: number): void { + this._deleteSession('live', entry.header.id) + const db = this._requireDb() + db.prepare(` + INSERT INTO temp.live_sessions (id, version, created_at, cwd, parent_session, seed_length, fingerprint, generation) VALUES (?, ?, ?, ?, ?, ?, ?, ?) `).run( @@ -434,11 +525,20 @@ export class SessionSearchSqlite extends SessionSearchService { generation, ) const insert = db.prepare(` - INSERT INTO ${docsTable} (text, session_id, seq, type, time, surface) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO temp.live_docs (text, session_id, seq, type, time, surface, codepoint_length) + VALUES (?, ?, ?, ?, ?, ?, ?) `) for (const document of entry.documents) { - insert.run(document.text, document.sessionId, document.seq, document.type, document.time, document.surface) + const text = sanitizeFtsText(document.text) + insert.run( + text, + document.sessionId, + document.seq, + document.type, + document.time, + document.surface, + Array.from(text).length, + ) } } @@ -455,19 +555,16 @@ export class SessionSearchSqlite extends SessionSearchService { ranked AS ( SELECT *, ROW_NUMBER() OVER ( PARTITION BY session_id - ORDER BY score ASC, time DESC, seq DESC + ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC ) AS event_rank FROM filtered ) SELECT * FROM ranked WHERE event_rank = 1 - ORDER BY score ASC, time DESC, session_id ASC, seq DESC + ORDER BY match_count DESC, document_length ASC, time DESC, session_id ASC, seq DESC LIMIT ? OFFSET ? `).all( - quoteFtsData(request.query), - this._persistence === undefined ? 0 : 1, - this._persistence === undefined ? 0 : 1, - quoteFtsData(request.query), + ...selectedDocumentsParams(request.query, this._persistence !== undefined), ...sessionWhere.params, ...eventWhere.params, request.limit + 1, @@ -483,13 +580,10 @@ export class SessionSearchSqlite extends SessionSearchService { ${selected.sql} SELECT * FROM matched WHERE ${where} - ORDER BY score ASC, time DESC, seq DESC + ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC LIMIT ? OFFSET ? `).all( - quoteFtsData(request.query), - this._persistence === undefined ? 0 : 1, - this._persistence === undefined ? 0 : 1, - quoteFtsData(request.query), + ...selectedDocumentsParams(request.query, this._persistence !== undefined), request.sessionId, ...eventWhere.params, request.limit + 1, @@ -515,23 +609,23 @@ export class SessionSearchSqlite extends SessionSearchService { ) } - private _sessionHit(row: SearchRow, query: string): SessionSearchHit { + private _sessionHit(row: SearchRow): SessionSearchHit { return { header: rowHeader(row), live: row.live === 1, persisted: row.persisted === 1, - bestMatch: this._eventHit(row, query), + bestMatch: this._eventHit(row), } } - private _eventHit(row: SearchRow, query: string): SessionEventSearchHit { + private _eventHit(row: SearchRow): SessionEventSearchHit { return { sessionId: row.session_id as SessionId, seq: row.seq, type: row.type as SessionEventSearchHit['type'], time: row.time, surface: row.surface as SessionEventSearchHit['surface'], - snippet: makeSnippet(row.text, query, this.config.snippetChars), + snippet: makeSnippet(row.marked_text, this.config.snippetChars), } } @@ -548,7 +642,7 @@ export class SessionSearchSqlite extends SessionSearchService { function selectedDocumentsSql(): { sql: string } { return { - sql: `WITH matched AS ( + sql: `WITH candidates AS ( SELECT pd.session_id AS session_id, ps.version AS version, @@ -562,8 +656,8 @@ function selectedDocumentsSql(): { sql: string } { pd.type AS type, CAST(pd.time AS INTEGER) AS time, pd.surface AS surface, - pd.text AS text, - bm25(persisted_docs) AS score + highlight(persisted_docs, 0, ?, ?) AS marked_text, + CAST(pd.codepoint_length AS INTEGER) AS document_length FROM persisted_docs AS pd JOIN persisted_sessions AS ps ON ps.id = pd.session_id WHERE persisted_docs MATCH ? @@ -585,15 +679,39 @@ function selectedDocumentsSql(): { sql: string } { ld.type AS type, CAST(ld.time AS INTEGER) AS time, ld.surface AS surface, - ld.text AS text, - bm25(live_docs) AS score + highlight(live_docs, 0, ?, ?) AS marked_text, + CAST(ld.codepoint_length AS INTEGER) AS document_length FROM temp.live_docs AS ld JOIN temp.live_sessions AS ls ON ls.id = ld.session_id WHERE live_docs MATCH ? + ), matched AS ( + SELECT *, + ( + length(CAST(marked_text AS BLOB)) + - length(CAST(replace(marked_text, ?, '') AS BLOB)) + ) / ? AS match_count + FROM candidates )`, } } +function selectedDocumentsParams(query: string, persistenceVisible: boolean): Array { + const expression = quoteFtsData(query) + const visible = persistenceVisible ? 1 : 0 + return [ + FTS_HIGHLIGHT_START, + FTS_HIGHLIGHT_END, + expression, + visible, + visible, + FTS_HIGHLIGHT_START, + FTS_HIGHLIGHT_END, + expression, + FTS_HIGHLIGHT_START, + Buffer.byteLength(FTS_HIGHLIGHT_START, 'utf8'), + ] +} + function observeLive(session: Session): ObservedSession { return observeSession( structuredClone(session.header), @@ -606,7 +724,6 @@ function observeSession(header: SessionHeader, events: readonly SessionEvent[]): const detachedEvents = events.map(event => structuredClone(event)) return { header: detachedHeader, - events: detachedEvents, documents: buildSessionEventSearchDocuments(detachedHeader.id, detachedEvents), fingerprint: createHash('sha256') .update(JSON.stringify({ header: detachedHeader, events: detachedEvents })) @@ -614,6 +731,49 @@ function observeSession(header: SessionHeader, events: readonly SessionEvent[]): } } +function materializePersistenceSnapshots( + snapshots: readonly SessionPersistenceSnapshot[], +): Map { + if (!isRuntimeArray(snapshots)) throw new Error('persistence snapshots must be an array') + const result = new Map() + for (const snapshot of snapshots) { + if (typeof snapshot.revision !== 'string') { + throw new Error('persistence snapshot revision must be a string') + } + const header = structuredClone(snapshot.header) + if (result.has(header.id)) { + throw new Error(`persistence listed duplicate session "${header.id}"`) + } + result.set(header.id, { header, revision: snapshot.revision }) + } + return result +} + +function samePersistenceSnapshots( + before: ReadonlyMap, + after: ReadonlyMap, +): boolean { + if (before.size !== after.size) return false + for (const [id, first] of before) { + const second = after.get(id) + if ( + second === undefined + || first.revision !== second.revision + || !sameHeader(first.header, second.header) + ) return false + } + return true +} + +function sameHeader(a: SessionHeader, b: SessionHeader): boolean { + return a.version === b.version + && a.id === b.id + && a.createdAt === b.createdAt + && a.cwd === b.cwd + && a.parentSession === b.parentSession + && a.seedLength === b.seedLength +} + function rowHeader(row: SearchRow): SessionHeader { return { version: row.version, @@ -629,7 +789,7 @@ function page( rows: readonly Row[], limit: number, convert: (row: Row) => Item, - nextCursor: (offset: number) => string, + nextCursor: (offset: number) => SessionSearchCursorValue, offset: number, ): SessionSearchPage { const hasMore = rows.length > limit @@ -639,12 +799,12 @@ function page( } } -function encodeCursor(payload: CursorPayload): string { - return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url') +function encodeCursor(payload: CursorPayload): SessionSearchCursorValue { + return SessionSearchCursor(Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url')) } function decodeCursor( - cursor: string, + cursor: SessionSearchCursorValue, instance: string, scope: CursorPayload['scope'], fingerprint: string, @@ -762,4 +922,8 @@ function errorMessage(error: unknown): string { return error instanceof Error ? error.message : 'unknown error' } +function isRuntimeArray(value: unknown): boolean { + return Array.isArray(value) +} + export default SessionSearchSqlite diff --git a/packages/session-query/session-query-sqlite/src/query.ts b/packages/session-query/session-query-sqlite/src/query.ts index fd2b5156e6..9654f6ae70 100644 --- a/packages/session-query/session-query-sqlite/src/query.ts +++ b/packages/session-query/session-query-sqlite/src/query.ts @@ -2,16 +2,24 @@ import { SessionQueryError, - filterSessionEventDocuments, - filterSessionResults, + materializeSessionEventResultFilters, + materializeSessionResultFilters, } from '@deepseek-ai/dsh-session-query' import type { + SessionAvailability, SessionEventMetadataFilter, + SessionEventResultFilter, SessionEventSearchRequest, SessionResultFilter, + SessionSearchCursor, SessionSearchRequest, } from '@deepseek-ai/dsh-session-query' +/** Collision-free marker inserted before an FTS5 match by `highlight()`. */ +export const FTS_HIGHLIGHT_START = '\uFDD0' +/** Collision-free marker inserted after an FTS5 match by `highlight()`. */ +export const FTS_HIGHLIGHT_END = '\uFDD1' + /** Limit defaults needed to normalize a search request. */ export interface QueryLimits { /** Page size used when the request omits one. */ @@ -26,7 +34,7 @@ export interface NormalizedSessionRequest { sessionFilters: readonly SessionResultFilter[] eventFilters: readonly SessionEventMetadataFilter[] limit: number - cursor?: string + cursor?: SessionSearchCursor } /** Normalized within-session request. */ @@ -35,7 +43,7 @@ export interface NormalizedEventRequest { query: string filters: readonly SessionEventMetadataFilter[] limit: number - cursor?: string + cursor?: SessionSearchCursor } /** Parameterized SQL predicate fragment. */ @@ -56,16 +64,15 @@ export function normalizeSessionRequest( request: SessionSearchRequest, limits: QueryLimits, ): NormalizedSessionRequest { - const sessionFilters = request.sessionFilters ?? [] - const eventFilters = request.eventFilters ?? [] - filterSessionResults([], sessionFilters) - filterSessionEventDocuments([], eventFilters) + const sessionFilters = materializeSessionResultFilters(request.sessionFilters ?? []) + const eventFilters = materializeMetadataFilters(request.eventFilters ?? []) + const cursor = materializeCursor(request.cursor) return { query: normalizeQuery(request.query), sessionFilters, eventFilters, limit: normalizeLimit(request.limit, limits), - ...request.cursor === undefined ? {} : { cursor: request.cursor }, + ...cursor === undefined ? {} : { cursor }, } } @@ -79,14 +86,17 @@ export function normalizeEventRequest( request: SessionEventSearchRequest, limits: QueryLimits, ): NormalizedEventRequest { - const filters = request.filters ?? [] - filterSessionEventDocuments([], filters) + if (typeof request.sessionId !== 'string') { + throw new SessionQueryError('session-search session id must be text', 'SESSION_QUERY_INVALID_FILTER') + } + const filters = materializeMetadataFilters(request.filters ?? []) + const cursor = materializeCursor(request.cursor) return { sessionId: request.sessionId, query: normalizeQuery(request.query), filters, limit: normalizeLimit(request.limit, limits), - ...request.cursor === undefined ? {} : { cursor: request.cursor }, + ...cursor === undefined ? {} : { cursor }, } } @@ -115,9 +125,23 @@ export function buildSessionWhere(filters: readonly SessionResultFilter[]): SqlW case 'availability': { const availability = [...new Set(filter.values)] if (availability.length === 0) clauses.push('0') - else if (availability.length === 1) clauses.push(`${availability[0]} = 1`) + else if (availability.length === 1) { + const value = availability[0] as SessionAvailability + switch (value) { + case 'live': + clauses.push('live = 1') + break + case 'persisted': + clauses.push('persisted = 1') + break + default: + unknownAvailability(value) + } + } break } + default: + unknownFilter(filter) } } return { sql: clauses.join(' AND '), params } @@ -145,6 +169,8 @@ export function buildEventWhere(filters: readonly SessionEventMetadataFilter[]): case 'surface': addList(clauses, params, 'surface', filter.values) break + default: + unknownFilter(filter) } } return { sql: clauses.join(' AND '), params } @@ -159,6 +185,18 @@ export function quoteFtsData(query: string): string { return `"${query.replaceAll('"', '""')}"` } +/** + * Remove reserved marker collisions before text enters FTS5 or MATCH. + * @param text - extracted document text or normalized caller query. + * @returns text with reserved noncharacters mapped to replacement characters. + */ +export function sanitizeFtsText(text: string): string { + return text + .replaceAll('\0', '\uFFFD') + .replaceAll(FTS_HIGHLIGHT_START, '\uFFFD') + .replaceAll(FTS_HIGHLIGHT_END, '\uFFFD') +} + /** * Build the stable normalized request identity stored in opaque cursors. * @param request - normalized request whose filter ordering is canonicalized. @@ -185,19 +223,16 @@ export function requestFingerprint(request: NormalizedSessionRequest | Normalize /** * Build a whitespace-normalized excerpt no longer than `maxChars`. - * @param text - complete extracted semantic document. - * @param query - normalized literal query used to position the excerpt. + * @param markedText - complete document with FTS5 `highlight()` markers. * @param maxChars - maximum result length in Unicode code points. * @returns bounded plain-text snippet. */ -export function makeSnippet(text: string, query: string, maxChars: number): string { - const clean = text.replace(/\s+/gu, ' ').trim() +export function makeSnippet(markedText: string, maxChars: number): string { + const { text: clean, matchStart } = normalizeMarkedText(markedText) const characters = Array.from(clean) if (characters.length <= maxChars) return clean if (maxChars === 1) return '…' - const foundUnits = clean.toLowerCase().indexOf(query.toLowerCase()) - const found = foundUnits < 0 ? -1 : Array.from(clean.slice(0, foundUnits)).length - let start = found < 0 ? 0 : Math.max(0, found - Math.floor(maxChars / 3)) + let start = Math.max(0, matchStart - Math.floor(maxChars / 3)) let prefix = start > 0 ? '…' : '' let suffix = '…' let contentLength = maxChars - prefix.length - suffix.length @@ -216,6 +251,28 @@ export function makeSnippet(text: string, query: string, maxChars: number): stri return `${prefix}${characters.slice(start, end).join('')}${suffix}` } +function normalizeMarkedText(markedText: string): { text: string; matchStart: number } { + const characters: string[] = [] + let matchStart: number | undefined + for (const character of markedText) { + if (character === FTS_HIGHLIGHT_START) { + matchStart ??= characters.length + continue + } + if (character === FTS_HIGHLIGHT_END) continue + if (/\s/u.test(character)) { + if (characters.length > 0 && characters.at(-1) !== ' ') characters.push(' ') + } else { + characters.push(character) + } + } + if (characters.at(-1) === ' ') characters.pop() + return { + text: characters.join(''), + matchStart: matchStart ?? 0, + } +} + function normalizeQuery(value: string): string { if (typeof value !== 'string') { throw new SessionQueryError('session-search query must be text', 'SESSION_QUERY_INVALID_QUERY') @@ -227,7 +284,44 @@ function normalizeQuery(value: string): string { 'SESSION_QUERY_INVALID_QUERY', ) } - return query + if (query.includes('\0')) { + throw new SessionQueryError( + 'session-search query must not contain NUL', + 'SESSION_QUERY_INVALID_QUERY', + ) + } + return sanitizeFtsText(query) +} + +function materializeCursor(cursor: SessionSearchCursor | undefined): SessionSearchCursor | undefined { + if (cursor === undefined) return undefined + if (typeof cursor !== 'string') { + throw new SessionQueryError('session-search cursor must be text', 'SESSION_QUERY_INVALID_CURSOR') + } + return cursor +} + +function materializeMetadataFilters( + filters: readonly SessionEventMetadataFilter[], +): SessionEventMetadataFilter[] { + const candidates: readonly SessionEventResultFilter[] = filters + for (const filter of candidates) { + switch (filter.kind) { + case 'seq': + case 'time': + case 'type': + case 'surface': + break + case 'text': + throw new SessionQueryError( + 'session-search metadata filters do not accept text clauses', + 'SESSION_QUERY_INVALID_FILTER', + ) + default: + unknownFilter(filter) + } + } + return materializeSessionEventResultFilters(filters) as SessionEventMetadataFilter[] } function normalizeLimit(value: number | undefined, limits: QueryLimits): number { @@ -310,3 +404,18 @@ function compareNullable(a: string | null, b: string | null): number { if (b === null) return 1 return a.localeCompare(b) } + +function unknownAvailability(value: never): never { + throw new SessionQueryError( + `session availability filter contains unknown value "${String(value)}"`, + 'SESSION_QUERY_INVALID_FILTER', + ) +} + +function unknownFilter(filter: never): never { + const kind = (filter as { kind?: unknown }).kind + throw new SessionQueryError( + `session filter contains unknown kind ${typeof kind === 'string' ? `"${kind}"` : '(missing)'}`, + 'SESSION_QUERY_INVALID_FILTER', + ) +} diff --git a/packages/session-query/session-query-sqlite/src/schema.ts b/packages/session-query/session-query-sqlite/src/schema.ts index 1c9bd98791..8e5c85f676 100644 --- a/packages/session-query/session-query-sqlite/src/schema.ts +++ b/packages/session-query/session-query-sqlite/src/schema.ts @@ -5,7 +5,7 @@ import { mkdir } from 'node:fs/promises' import { dirname, resolve } from 'node:path' /** Current derived-index schema version. Incompatible versions reset in place. */ -export const SESSION_QUERY_SQLITE_SCHEMA_VERSION = 1 +export const SESSION_QUERY_SQLITE_SCHEMA_VERSION = 2 /** SQLite application id protecting unrelated databases from derived resets. */ export const SESSION_QUERY_SQLITE_APPLICATION_ID = 0x44534851 @@ -24,8 +24,6 @@ export async function openSearchDatabase(path: string, journalMode: JournalMode) if (actual !== ':memory:') await mkdir(dirname(actual), { recursive: true, mode: 0o700 }) const db = new DatabaseSync(actual) try { - // journalMode is a validated closed union, not caller-controlled SQL. - db.exec(`PRAGMA journal_mode = ${journalMode.toUpperCase()}`) const { application_id: applicationId } = db.prepare('PRAGMA application_id').get() as { application_id: number } const { user_version: version } = db.prepare('PRAGMA user_version').get() as { user_version: number } const userTables = listUserTables(db) @@ -38,6 +36,9 @@ export async function openSearchDatabase(path: string, journalMode: JournalMode) if (applicationId === SESSION_QUERY_SQLITE_APPLICATION_ID && version !== SESSION_QUERY_SQLITE_SCHEMA_VERSION) { resetDerivedSchema(db) } + // Apply mutating pragmas only after refusing foreign or canonical files. + // journalMode is a validated closed union, not caller-controlled SQL. + db.exec(`PRAGMA journal_mode = ${journalMode.toUpperCase()}`) ensurePersistentSchema(db) ensureTemporarySchema(db) return db @@ -78,7 +79,7 @@ function ensurePersistentSchema(db: DatabaseSync): void { cwd TEXT, parent_session TEXT, seed_length INTEGER, - fingerprint TEXT NOT NULL, + revision TEXT NOT NULL, generation INTEGER NOT NULL ) STRICT `) @@ -90,6 +91,7 @@ function ensurePersistentSchema(db: DatabaseSync): void { type UNINDEXED, time UNINDEXED, surface UNINDEXED, + codepoint_length UNINDEXED, tokenize = 'unicode61' ) `) @@ -117,6 +119,7 @@ function ensureTemporarySchema(db: DatabaseSync): void { type UNINDEXED, time UNINDEXED, surface UNINDEXED, + codepoint_length UNINDEXED, tokenize = 'unicode61' ) `) diff --git a/packages/session-query/session-query-sqlite/tests/query.spec.ts b/packages/session-query/session-query-sqlite/tests/query.spec.ts index 0faced72e4..f9c0a5d19e 100644 --- a/packages/session-query/session-query-sqlite/tests/query.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/query.spec.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from 'vitest' import { SessionId } from '@deepseek-ai/dsh-session' -import type { SessionQueryErrorCode } from '@deepseek-ai/dsh-session-query' +import { SessionSearchCursor, type SessionQueryErrorCode } from '@deepseek-ai/dsh-session-query' import { buildEventWhere, buildSessionWhere, + FTS_HIGHLIGHT_END, + FTS_HIGHLIGHT_START, makeSnippet, normalizeEventRequest, normalizeSessionRequest, @@ -32,13 +34,13 @@ describe('SQLite search request normalization', () => { sessionFilters: [{ kind: 'availability', values: ['live'] }], eventFilters: [{ kind: 'surface', values: ['current'] }], limit: 3, - cursor: 'next', + cursor: SessionSearchCursor('next'), }, limits)).toEqual({ query: 'needle', sessionFilters: [{ kind: 'availability', values: ['live'] }], eventFilters: [{ kind: 'surface', values: ['current'] }], limit: 3, - cursor: 'next', + cursor: SessionSearchCursor('next'), }) expect(normalizeEventRequest({ sessionId: SessionId('s'), query: 'needle' }, limits)).toEqual({ sessionId: SessionId('s'), @@ -50,13 +52,13 @@ describe('SQLite search request normalization', () => { sessionId: SessionId('s'), query: 'needle', filters: [{ kind: 'seq', from: 1 }], - cursor: 'next', + cursor: SessionSearchCursor('next'), }, limits)).toEqual({ sessionId: SessionId('s'), query: 'needle', filters: [{ kind: 'seq', from: 1 }], limit: 2, - cursor: 'next', + cursor: SessionSearchCursor('next'), }) }) @@ -65,11 +67,39 @@ describe('SQLite search request normalization', () => { .toThrow(expectCode('SESSION_QUERY_INVALID_QUERY')) expect(() => normalizeSessionRequest({ query: ' \n ' }, limits)) .toThrow(expectCode('SESSION_QUERY_INVALID_QUERY')) + expect(() => normalizeSessionRequest({ query: 'bad\0query' }, limits)) + .toThrow(expectCode('SESSION_QUERY_INVALID_QUERY')) + expect(() => normalizeEventRequest({ sessionId: 1 as never, query: 'x' }, limits)) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => normalizeEventRequest({ + sessionId: SessionId('s'), + query: 'x', + cursor: 1 as never, + }, limits)).toThrow(expectCode('SESSION_QUERY_INVALID_CURSOR')) + expect(() => normalizeSessionRequest({ + query: 'x', + eventFilters: [{ kind: 'text', text: 'x' } as never], + }, limits)).toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => normalizeSessionRequest({ + query: 'x', + eventFilters: [{} as never], + }, limits)).toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) for (const limit of [1.5, 0, 4]) { expect(() => normalizeEventRequest({ sessionId: SessionId('s'), query: 'x', limit }, limits)) .toThrow(expectCode('SESSION_QUERY_INVALID_LIMIT')) } }) + + it('materializes owned filter values during normalization', () => { + const values = ['live'] as Array<'live' | 'persisted'> + const filter = { kind: 'availability' as const, values } + const request = { query: 'needle', sessionFilters: [filter] } + const normalized = normalizeSessionRequest(request, limits) + + values[0] = 'persisted' + request.sessionFilters.push({ kind: 'availability', values: ['persisted'] }) + expect(normalized.sessionFilters).toEqual([{ kind: 'availability', values: ['live'] }]) + }) }) describe('SQLite search predicate compilation', () => { @@ -120,6 +150,17 @@ describe('SQLite search predicate compilation', () => { { kind: 'surface', values: [] }, ])).toEqual({ sql: '0 AND 0', params: [] }) }) + + it('rejects runtime-unknown filter discriminants in both SQL builders', () => { + expect(() => buildSessionWhere([{ kind: 'future' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => buildEventWhere([{ kind: 'future' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => buildSessionWhere([{ kind: 'availability', values: ['future'] } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => buildSessionWhere([{} as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + }) }) describe('SQLite query identity and presentation', () => { @@ -169,11 +210,13 @@ describe('SQLite query identity and presentation', () => { }) it('normalizes, bounds, and positions snippets by Unicode code point', () => { - expect(makeSnippet(' short\ntext ', 'absent', 20)).toBe('short text') - expect(makeSnippet('abcdef', 'f', 1)).toBe('…') - expect(makeSnippet('abcdefghij', 'absent', 5)).toBe('abcd…') - expect(makeSnippet('abcdefghij', 'c', 5)).toBe('…bcd…') - expect(makeSnippet('abcdef', 'f', 2)).toBe('a…') - expect(makeSnippet('abcdef', 'f', 5)).toBe('…cdef') + expect(makeSnippet(' short\ntext ', 20)).toBe('short text') + expect(makeSnippet(`abcde${FTS_HIGHLIGHT_START}f${FTS_HIGHLIGHT_END}`, 1)).toBe('…') + expect(makeSnippet('abcdefghij', 5)).toBe('abcd…') + expect(makeSnippet(`ab${FTS_HIGHLIGHT_START}c${FTS_HIGHLIGHT_END}defghij`, 5)).toBe('…bcd…') + expect(makeSnippet(`abcde${FTS_HIGHLIGHT_START}f${FTS_HIGHLIGHT_END}`, 2)).toBe('a…') + expect(makeSnippet(`abcde${FTS_HIGHLIGHT_START}f${FTS_HIGHLIGHT_END}`, 5)).toBe('…cdef') + expect(makeSnippet(` x—${FTS_HIGHLIGHT_START}café${FTS_HIGHLIGHT_END}\n y `, 20)) + .toBe('x—café y') }) }) diff --git a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts index 77d1e0125d..f4904e17d8 100644 --- a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts @@ -1,18 +1,25 @@ -import { afterEach, describe, expect, it } from 'vitest' -import { Context } from 'cordis' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Context, type Fiber } from 'cordis' import { DatabaseSync } from 'node:sqlite' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import SessionStore, { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' import type { SessionEvent, SessionHeader, SessionId as SessionIdType } from '@deepseek-ai/dsh-session' -import SessionPersistence from '@deepseek-ai/dsh-session-persistence' +import SessionPersistence, { SessionPersistenceRevision } from '@deepseek-ai/dsh-session-persistence' +import type { SessionPersistenceSnapshot } from '@deepseek-ai/dsh-session-persistence' import SessionPersistenceSqlite from '@deepseek-ai/dsh-session-persistence-sqlite' import SessionSearchSqlite, { SESSION_QUERY_SQLITE_APPLICATION_ID, SESSION_QUERY_SQLITE_SCHEMA_VERSION, } from '@deepseek-ai/dsh-session-query-sqlite' -import type { SessionQueryErrorCode } from '@deepseek-ai/dsh-session-query' +import { + SessionQueryError, + SessionSearchCursor, + type SessionAvailability, + type SessionQueryErrorCode, + type SessionSearchRequest, +} from '@deepseek-ai/dsh-session-query' const temporaryDirectories: string[] = [] @@ -48,19 +55,36 @@ function expectCode(code: SessionQueryErrorCode): Error { class TestPersistence extends SessionPersistence { static entries = new Map() + static revisions = new Map() + static nextRevision = 0 + static loads = new Map() + static loadEffect: ((entry: { meta: SessionHeader; events: SessionEvent[] }) => void) | undefined static listGate: Promise | undefined static listStarted: (() => void) | undefined + static snapshotEffect: (() => void | Promise) | undefined + static snapshotOverride: (() => SessionPersistenceSnapshot[]) | undefined static failure: unknown static reset(entries: readonly { meta: SessionHeader; events: SessionEvent[] }[] = []): void { - this.entries = new Map(entries.map(entry => [entry.meta.id, structuredClone(entry)])) + this.entries = new Map() + this.revisions = new Map() + this.loads = new Map() + this.loadEffect = undefined + for (const entry of entries) this.set(entry) this.listGate = undefined this.listStarted = undefined + this.snapshotEffect = undefined + this.snapshotOverride = undefined this.failure = undefined } + static set(entry: { meta: SessionHeader; events: SessionEvent[] }): void { + this.entries.set(entry.meta.id, structuredClone(entry)) + this.revisions.set(entry.meta.id, ++this.nextRevision) + } + create(meta: SessionHeader): Promise { - TestPersistence.entries.set(meta.id, { meta: structuredClone(meta), events: [] }) + TestPersistence.set({ meta, events: [] }) return Promise.resolve() } @@ -68,13 +92,21 @@ class TestPersistence extends SessionPersistence { const entry = TestPersistence.entries.get(id) if (entry === undefined) return Promise.reject(new Error('missing test session')) entry.events.push(...structuredClone(events)) + TestPersistence.revisions.set(id, ++TestPersistence.nextRevision) return Promise.resolve() } async load(id: SessionIdType): Promise<{ meta: SessionHeader; events: SessionEvent[] }> { + TestPersistence.loads.set(id, (TestPersistence.loads.get(id) ?? 0) + 1) if (TestPersistence.failure !== undefined) throw TestPersistence.failure const entry = TestPersistence.entries.get(id) if (entry === undefined) throw new Error('missing test session') + if (TestPersistence.loadEffect !== undefined) { + const effect = TestPersistence.loadEffect + TestPersistence.loadEffect = undefined + effect(entry) + TestPersistence.revisions.set(id, ++TestPersistence.nextRevision) + } return structuredClone(entry) } @@ -84,6 +116,20 @@ class TestPersistence extends SessionPersistence { if (TestPersistence.failure !== undefined) throw TestPersistence.failure return [...TestPersistence.entries.values()].map(entry => structuredClone(entry.meta)) } + + + async listSnapshots(): Promise { + TestPersistence.listStarted?.() + await TestPersistence.listGate + if (TestPersistence.failure !== undefined) throw TestPersistence.failure + const snapshots = TestPersistence.snapshotOverride?.() + ?? [...TestPersistence.entries.values()].map(entry => ({ + header: structuredClone(entry.meta), + revision: SessionPersistenceRevision(`test:${TestPersistence.revisions.get(entry.meta.id)}`), + })) + await TestPersistence.snapshotEffect?.() + return snapshots + } } async function liveContext(config: ConstructorParameters[1] = { path: ':memory:' }): Promise { @@ -175,6 +221,44 @@ describe('SQLite session search', () => { await expect(ctx.sessionSearch.searchSessions({ query: '*' })).resolves.toEqual({ items: [] }) }) + it('ranks live and persisted matches on one source-comparable contract', async () => { + const persisted = header('z-persisted') + TestPersistence.reset([ + { meta: persisted, events: messageEvents('needle needle', 10) }, + ...Array.from({ length: 12 }, (_, index) => ({ + meta: header(`filler-${index}`), + events: messageEvents('needle', 10), + })), + ]) + const ctx = await liveContext() + const persistence = await ctx.plugin(TestPersistence) + ctx.sessions.create(SessionId('a-live'), { + seed: messageEvents('needle needle', 10), + meta: { createdAt: persisted.createdAt }, + }) + + const result = await ctx.sessionSearch.searchSessions({ + query: 'needle', + sessionFilters: [{ kind: 'id', values: [SessionId('a-live'), persisted.id] }], + }) + expect(result.items.map(item => item.header.id)).toEqual([SessionId('a-live'), persisted.id]) + await persistence.dispose() + }) + + it('positions snippets from FTS5 matches across diacritics and punctuation', async () => { + const ctx = await liveContext({ path: ':memory:', snippetChars: 14 }) + const session = ctx.sessions.create(SessionId('snippet'), { + seed: messageEvents('long long long—café,\nnext value', 10), + }) + + const page = await ctx.sessionSearch.searchEvents({ sessionId: session.id, query: 'CAFE' }) + expect(page.items).toHaveLength(1) + expect(page.items[0]!.snippet).toContain('café') + expect(page.items[0]!.snippet).toContain('—') + expect(page.items[0]!.snippet).not.toContain('\n') + expect(Array.from(page.items[0]!.snippet).length).toBeLessThanOrEqual(14) + }) + it('binds cursors to requests and only invalidates within-session pages for target changes', async () => { const ctx = await liveContext({ path: ':memory:', defaultLimit: 1, maxLimit: 5 }) const target = ctx.sessions.create(SessionId('target'), { @@ -193,7 +277,7 @@ describe('SQLite session search', () => { if (eventPage.nextCursor === undefined || sessionPage.nextCursor === undefined) throw new Error('expected cursors') const eventKeys = eventPage.items.map(item => `${item.sessionId}:${item.seq}`) - let eventCursor: string | undefined = eventPage.nextCursor + let eventCursor: ReturnType | undefined = eventPage.nextCursor while (eventCursor !== undefined) { const next = await ctx.sessionSearch.searchEvents({ sessionId: target.id, @@ -208,7 +292,7 @@ describe('SQLite session search', () => { expect(new Set(eventKeys).size).toBe(eventKeys.length) const sessionIds = sessionPage.items.map(item => item.header.id) - let sessionCursor: string | undefined = sessionPage.nextCursor + let sessionCursor: ReturnType | undefined = sessionPage.nextCursor while (sessionCursor !== undefined) { const next = await ctx.sessionSearch.searchSessions({ query: 'needle', limit: 1, cursor: sessionCursor }) sessionIds.push(...next.items.map(item => item.header.id)) @@ -251,6 +335,7 @@ describe('SQLite session search', () => { { sessionId: session.id, query: 'needle', limit: 4 }, { sessionId: session.id, query: 'needle', filters: [{ kind: 'seq', from: 2, to: 1 }] }, { sessionId: session.id, query: 'needle', filters: [{ kind: 'surface', values: ['future'] }] }, + { sessionId: session.id, query: 'bad\0query' }, ] as const) { await expect(ctx.sessionSearch.searchEvents(request as never)).rejects.toBeInstanceOf(Error) } @@ -258,7 +343,24 @@ describe('SQLite session search', () => { query: 'needle', sessionFilters: [{ kind: 'availability', values: ['remote' as never] }], })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) - await expect(ctx.sessionSearch.searchEvents({ sessionId: session.id, query: 'needle', cursor: 'not-json' })) + await expect(ctx.sessionSearch.searchSessions({ + query: 'needle', + sessionFilters: [{ kind: 'future' } as never], + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionSearch.searchSessions({ + query: 'needle', + eventFilters: [{ kind: 'future' } as never], + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: session.id, + query: 'needle', + filters: [{ kind: 'future' } as never], + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: session.id, + query: 'needle', + cursor: SessionSearchCursor('not-json'), + })) .rejects.toThrow(expectCode('SESSION_QUERY_INVALID_CURSOR')) await expect(ctx.sessionSearch.searchEvents({ sessionId: SessionId('absent'), query: 'needle' })) .rejects.toThrow(expectCode('SESSION_QUERY_SESSION_NOT_FOUND')) @@ -280,6 +382,37 @@ describe('SQLite session search', () => { }) describe('SQLite reconciliation and source lifecycle', () => { + it('owns queued request and filter values before waiting for the serializer', async () => { + const durable = header('owned') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + const persistence = await ctx.plugin(TestPersistence) + let release!: () => void + TestPersistence.listGate = new Promise((resolve) => { release = resolve }) + let markStarted!: () => void + const started = new Promise((resolve) => { markStarted = resolve }) + TestPersistence.listStarted = () => { + TestPersistence.listStarted = undefined + markStarted() + } + const blocking = ctx.sessionSearch.searchSessions({ query: 'needle' }) + await started + + const availability: SessionAvailability[] = ['persisted'] + const request: SessionSearchRequest = { + query: 'needle', + sessionFilters: [{ kind: 'availability', values: availability }], + } + const queued = ctx.sessionSearch.searchSessions(request) + request.query = 'absent' + availability[0] = 'live' + release() + + await expect(blocking).resolves.toMatchObject({ items: [{ header: durable }] }) + await expect(queued).resolves.toMatchObject({ items: [{ header: durable }] }) + await persistence.dispose() + }) + it('mounts persistence dynamically, shadows with TEMP live rows, reveals, and hides on unmount', async () => { const shared = header('shared', 10, { cwd: '/work' }) const durable = header('durable', 5) @@ -311,7 +444,7 @@ describe('SQLite reconciliation and source lifecycle', () => { .rejects.toThrow(expectCode('SESSION_QUERY_SESSION_NOT_FOUND')) }) - it('restarts observation when persistence unmounts during an asynchronous list', async () => { + it('discards a stale list rejection when persistence unmounts during observation', async () => { const durable = header('racing') TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) const ctx = await liveContext() @@ -328,10 +461,140 @@ describe('SQLite reconciliation and source lifecycle', () => { const search = ctx.sessionSearch.searchSessions({ query: 'needle' }) await started await persistenceFiber.dispose() + TestPersistence.failure = new Error('stale backend rejection') release() await expect(search).resolves.toEqual({ items: [] }) }) + it('retries against a replacement after the prior binding rejects', async () => { + const durable = header('replacement') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + const prior = await ctx.plugin(TestPersistence) + let rejectPrior!: (reason: unknown) => void + TestPersistence.listGate = new Promise((_resolve, reject) => { rejectPrior = reject }) + let markStarted!: () => void + const started = new Promise((resolve) => { markStarted = resolve }) + TestPersistence.listStarted = () => { + TestPersistence.listStarted = undefined + markStarted() + } + + const search = ctx.sessionSearch.searchSessions({ query: 'needle' }) + await started + await prior.dispose() + TestPersistence.listGate = undefined + const replacement = await ctx.plugin(TestPersistence) + rejectPrior(new Error('stale prior binding')) + await expect(search).resolves.toMatchObject({ items: [{ header: durable }] }) + await replacement.dispose() + }) + + it('reloads a replacement source even when its opaque revisions collide', async () => { + const durable = header('colliding-replacement') + TestPersistence.reset([{ meta: durable, events: messageEvents('old content') }]) + const revision = TestPersistence.revisions.get(durable.id)! + const ctx = await liveContext() + const prior = await ctx.plugin(TestPersistence) + await expect(ctx.sessionSearch.searchSessions({ query: 'old' })) + .resolves.toMatchObject({ items: [{ header: durable }] }) + await prior.dispose() + + TestPersistence.set({ meta: durable, events: messageEvents('new needle') }) + TestPersistence.revisions.set(durable.id, revision) + const replacement = await ctx.plugin(TestPersistence) + const internals = ctx.sessionSearch as unknown as { + _lastPersistenceRevision: number + _persistenceRevision: number + } + expect(internals._persistenceRevision).not.toBe(internals._lastPersistenceRevision) + const page = await ctx.sessionSearch.searchSessions({ query: 'new needle' }) + expect(TestPersistence.loads.get(durable.id)).toBe(2) + expect(page).toMatchObject({ items: [{ header: durable }] }) + await expect(ctx.sessionSearch.searchSessions({ query: 'old' })).resolves.toEqual({ items: [] }) + expect(TestPersistence.loads.get(durable.id)).toBe(2) + await replacement.dispose() + }) + + it('retries when a successful observation belongs to a source unmounted during listing', async () => { + const durable = header('successful-unmount') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + const persistence = await ctx.plugin(TestPersistence) + let lists = 0 + TestPersistence.snapshotEffect = async () => { + lists += 1 + if (lists === 2) await persistence.dispose() + } + + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })).resolves.toEqual({ items: [] }) + expect(lists).toBe(2) + }) + + it('retries when the snapshot population changes during observation', async () => { + const first = header('first') + const added = header('added-during-list') + TestPersistence.reset([{ meta: first, events: messageEvents('first needle') }]) + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + TestPersistence.snapshotEffect = () => { + TestPersistence.snapshotEffect = undefined + TestPersistence.set({ meta: added, events: messageEvents('added needle') }) + } + + const page = await ctx.sessionSearch.searchSessions({ query: 'needle' }) + expect(page.items.map(item => item.header.id).sort()).toEqual([added.id, first.id].sort()) + expect(TestPersistence.loads.get(first.id)).toBe(2) + expect(TestPersistence.loads.get(added.id)).toBe(1) + }) + + it('retries if the source revision changes while live sessions are observed', async () => { + const durable = header('live-boundary-retry') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + const internals = ctx.sessionSearch as unknown as { _persistenceRevision: number } + const originalList = ctx.sessions.list.bind(ctx.sessions) + let bumped = false + const list = vi.spyOn(ctx.sessions, 'list').mockImplementation(() => { + if (!bumped) { + bumped = true + internals._persistenceRevision += 1 + } + return originalList() + }) + + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .resolves.toMatchObject({ items: [{ header: durable }] }) + expect(TestPersistence.loads.get(durable.id)).toBe(2) + list.mockRestore() + }) + + it('rejects malformed snapshots and preserves typed persistence failures', async () => { + const durable = header('invalid-snapshot') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + + TestPersistence.snapshotOverride = () => 'not-an-array' as never + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_PERSISTENCE_FAILED')) + TestPersistence.snapshotOverride = () => [{ header: durable, revision: 1 as never }] + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_PERSISTENCE_FAILED')) + TestPersistence.snapshotOverride = () => [ + { header: durable, revision: SessionPersistenceRevision('duplicate:1') }, + { header: durable, revision: SessionPersistenceRevision('duplicate:2') }, + ] + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_PERSISTENCE_FAILED')) + + TestPersistence.snapshotOverride = undefined + const typed = new SessionQueryError('typed persistence failure', 'SESSION_QUERY_PERSISTENCE_FAILED') + TestPersistence.failure = typed + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })).rejects.toBe(typed) + }) + it('rejects immutable header conflicts between live and persisted sources', async () => { const shared = header('conflict', 10) TestPersistence.reset([{ meta: shared, events: messageEvents('persisted needle') }]) @@ -358,6 +621,9 @@ describe('SQLite reconciliation and source lifecycle', () => { const firstPersistence = await first.plugin(TestPersistence) const firstSearch = await first.plugin(SessionSearchSqlite, { path }) await first.sessionSearch.searchSessions({ query: 'needle' }) + expect(Object.fromEntries(TestPersistence.loads)).toEqual({ unchanged: 1, changed: 1, deleted: 1 }) + await first.sessionSearch.searchSessions({ query: 'needle' }) + expect(Object.fromEntries(TestPersistence.loads)).toEqual({ unchanged: 1, changed: 1, deleted: 1 }) await firstSearch.dispose() await firstPersistence.dispose() @@ -368,14 +634,20 @@ describe('SQLite reconciliation and source lifecycle', () => { const added = header('added') TestPersistence.entries.delete(deleted.id) - TestPersistence.entries.set(changed.id, { meta: changed, events: messageEvents('changed needle') }) - TestPersistence.entries.set(added.id, { meta: added, events: messageEvents('added needle') }) + TestPersistence.set({ meta: changed, events: messageEvents('changed needle') }) + TestPersistence.set({ meta: added, events: messageEvents('added needle') }) const second = new Context() await second.plugin(SessionStore) const secondPersistence = await second.plugin(TestPersistence) const secondSearch = await second.plugin(SessionSearchSqlite, { path }) const result = await second.sessionSearch.searchSessions({ query: 'needle' }) expect(result.items.map(item => item.header.id).sort()).toEqual([added.id, changed.id, unchanged.id].sort()) + expect(Object.fromEntries(TestPersistence.loads)).toEqual({ + unchanged: 1, + changed: 2, + deleted: 1, + added: 1, + }) await secondSearch.dispose() await secondPersistence.dispose() @@ -409,10 +681,27 @@ describe('SQLite reconciliation and source lifecycle', () => { await expect(second.sessionSearch.searchSessions({ query: 'live' })).resolves.toEqual({ items: [] }) await expect(second.sessionSearch.searchSessions({ query: 'persisted' })) .resolves.toMatchObject({ items: [{ header: shared, live: false, persisted: true }] }) + expect(TestPersistence.loads.get(shared.id)).toBe(1) await searchAgain.dispose() await persistenceAgain.dispose() }) + it('refreshes the stored revision after a mutating load repair', async () => { + const durable = header('repair') + TestPersistence.reset([{ meta: durable, events: messageEvents('before repair') }]) + TestPersistence.loadEffect = (entry) => { + entry.events = messageEvents('repaired needle') + } + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + + await expect(ctx.sessionSearch.searchSessions({ query: 'repaired' })) + .resolves.toMatchObject({ items: [{ header: durable }] }) + expect(TestPersistence.loads.get(durable.id)).toBe(2) + await ctx.sessionSearch.searchSessions({ query: 'repaired' }) + expect(TestPersistence.loads.get(durable.id)).toBe(2) + }) + it('recovers on the next search after source and SQLite transaction failures', async () => { TestPersistence.reset([{ meta: header('durable'), events: messageEvents('durable needle') }]) const ctx = await liveContext() @@ -462,15 +751,18 @@ describe('SQLite schema, cancellation, and real persistence integration', () => const foreignPath = await temporaryPath('foreign.db') const foreign = new DatabaseSync(foreignPath) + foreign.exec('PRAGMA journal_mode = WAL') foreign.exec('CREATE TABLE canonical(value TEXT)') foreign.exec("INSERT INTO canonical VALUES ('safe')") foreign.close() - const foreignCtx = await liveContext({ path: foreignPath }) + const foreignCtx = await liveContext({ path: foreignPath, journalMode: 'delete' }) await expect(foreignCtx.sessionSearch.searchSessions({ query: 'needle' })) .rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) const stillForeign = new DatabaseSync(foreignPath) expect(stillForeign.prepare('SELECT value FROM canonical').get()).toEqual({ value: 'safe' }) + expect(stillForeign.prepare('PRAGMA journal_mode').get()).toEqual({ journal_mode: 'wal' }) stillForeign.close() + await (foreignCtx.sessionSearch as SessionSearchSqlite).close() const otherAppPath = await temporaryPath('other-app.db') const otherApp = new DatabaseSync(otherAppPath) @@ -479,6 +771,25 @@ describe('SQLite schema, cancellation, and real persistence integration', () => const otherAppCtx = await liveContext({ path: otherAppPath }) await expect(otherAppCtx.sessionSearch.searchSessions({ query: 'needle' })) .rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) + await (otherAppCtx.sessionSearch as SessionSearchSqlite).close() + }) + + it('observes asynchronous open rejection even when no query is made', async () => { + const path = await temporaryPath('never-queried.db') + const foreign = new DatabaseSync(path) + foreign.exec('CREATE TABLE canonical(value TEXT)') + foreign.close() + const unhandled: unknown[] = [] + const onUnhandled = (reason: unknown) => { unhandled.push(reason) } + process.on('unhandledRejection', onUnhandled) + try { + const ctx = await liveContext({ path }) + await new Promise((resolve) => { setImmediate(resolve) }) + expect(unhandled).toEqual([]) + await (ctx.sessionSearch as SessionSearchSqlite).close() + } finally { + process.off('unhandledRejection', onUnhandled) + } }) it('cancels both queued and in-flight source waits without committing them', async () => { @@ -518,7 +829,7 @@ describe('SQLite schema, cancellation, and real persistence integration', () => releaseBlocking() await expect(blocking).resolves.toEqual({ items: [] }) - TestPersistence.entries.set(SessionId('uncommitted'), { + TestPersistence.set({ meta: header('uncommitted'), events: messageEvents('durable needle'), }) @@ -560,14 +871,38 @@ describe('SQLite schema, cancellation, and real persistence integration', () => await started const queued = search.searchSessions({ query: 'needle' }) const closing = search.close() + const repeatedClose = search.close() + expect(repeatedClose).toBe(closing) release() await expect(accepted).resolves.toEqual({ items: [] }) await expect(queued).rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) - await closing + await Promise.all([closing, repeatedClose]) await expect(search.searchSessions({ query: 'needle' })) .rejects.toThrow(expectCode('SESSION_QUERY_INDEX_FAILED')) - await search.close() + expect(search.close()).toBe(closing) + }) + + it('awaits optional-persistence child-fiber quiescence on disposal', async () => { + TestPersistence.reset() + const ctx = new Context() + await ctx.plugin(SessionStore) + const search = await ctx.plugin(SessionSearchSqlite, { path: ':memory:' }) + const persistence = await ctx.plugin(TestPersistence) + const optional = (ctx.sessionSearch as unknown as { + _optionalPersistenceFiber: Fiber + })._optionalPersistenceFiber + let release!: () => void + const cleanup = new Promise((resolve) => { release = resolve }) + optional.ctx.effect(() => () => cleanup) + + let settled = false + const disposing = search.dispose().then(() => { settled = true }) + await Promise.resolve() + expect(settled).toBe(false) + release() + await disposing + await persistence.dispose() }) it('combines the real SQLite persistence backend with the real search service keylessly', async () => { diff --git a/packages/session-query/session-query/README.md b/packages/session-query/session-query/README.md index 6f9e9bbd7a..917984f0cd 100644 --- a/packages/session-query/session-query/README.md +++ b/packages/session-query/session-query/README.md @@ -7,6 +7,7 @@ This is trusted context-wide infrastructure. It performs no caller authorization ## Reads - `listSessions()` reads current persistence metadata, merges live records with live precedence, and returns cloned records in deterministic newest-first order. +- `filterSessions(filters)` applies provider-independent session metadata and availability predicates to that same cloned logical corpus. - `listEvents(sessionId)` loads the live-preferred raw log and classifies each event as `current`, `shadowed`, or `log-only` with the shared `dsh-session` surface fold. - `filterEvents(sessionId, filters)` extracts first-party semantic documents and applies provider-independent metadata and literal-text predicates in ascending seq order. - `readEvent(request)` returns a cloned header, the full target event, and a bounded raw-seq window. `before` and `after` default to zero and may not exceed `readWindowMax`. @@ -21,7 +22,7 @@ The text clause is deliberately independent of FTS providers: caller text is esc ## Full-text seam -`SessionSearchService` owns the independent `ctx.sessionSearch` key. `searchSessions(request, exec?)` groups the logical corpus by strongest matching event; `searchEvents(request, exec?)` searches one logical session. Both return opaque cursor pages, accept optional cancellation, and expose snippets without provider-specific numeric scores. Search requests accept only metadata event filters, because literal-text filtering is the scan path described above. +`SessionSearchService` owns the independent `ctx.sessionSearch` key. `searchSessions(request, exec?)` groups the logical corpus by strongest matching event; `searchEvents(request, exec?)` searches one logical session. Both return pages whose continuation is an owned branded `SessionSearchCursor`, accept optional cancellation, and expose snippets without provider-specific numeric scores. Search requests accept only metadata event filters, because literal-text filtering is the scan path described above. The package has no provider coordinator or registration protocol. A concrete backend owns observation, reconciliation, ranking, cursor generations, and query execution as one lifecycle; the first implementation is [`@deepseek-ai/dsh-session-query-sqlite`](../session-query-sqlite/README.md). diff --git a/packages/session-query/session-query/package.json b/packages/session-query/session-query/package.json index 9f78d4f1db..449610a5a4 100644 --- a/packages/session-query/session-query/package.json +++ b/packages/session-query/session-query/package.json @@ -22,6 +22,7 @@ ], "license": "BSD-3-Clause", "peerDependencies": { + "@deepseek-ai/dsh-brand": "^0.0.1", "@deepseek-ai/dsh-llm": "^0.0.1", "@deepseek-ai/dsh-session": "^0.0.1", "@deepseek-ai/dsh-session-persistence": "^0.0.1", @@ -36,6 +37,7 @@ "schemastery": "^3.18.0" }, "devDependencies": { + "@deepseek-ai/dsh-brand": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", "@deepseek-ai/dsh-session": "workspace:^", "@deepseek-ai/dsh-session-persistence": "workspace:^", diff --git a/packages/session-query/session-query/src/corpus.ts b/packages/session-query/session-query/src/corpus.ts index c8ddca3caa..4a27c34e58 100644 --- a/packages/session-query/session-query/src/corpus.ts +++ b/packages/session-query/session-query/src/corpus.ts @@ -1,6 +1,6 @@ /** Live/persisted logical-corpus resolution for session-query. */ -import type { Context } from 'cordis' +import type { Context, Fiber } from 'cordis' import type { Session, SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session' import type SessionPersistence from '@deepseek-ai/dsh-session-persistence' import type { SessionRecord } from './types.ts' @@ -18,18 +18,19 @@ export interface LogicalSession { /** Resolves a live-preferred corpus against the persistence service mounted now. */ export class SessionCorpus { private _persistence: SessionPersistence | undefined + private readonly _optionalPersistenceFiber: Fiber constructor(private readonly _ctx: Context) { + this._optionalPersistenceFiber = _ctx.inject(['sessionPersistence'], (childCtx: Context) => { + const service = childCtx.sessionPersistence + this._persistence = service + childCtx.effect(() => () => { + /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */ + if (this._persistence === service) this._persistence = undefined + }, 'sessionQuery.persistenceBinding') + }) _ctx.effect(() => { - const fiber = _ctx.inject(['sessionPersistence'], (childCtx: Context) => { - const service = childCtx.sessionPersistence - this._persistence = service - childCtx.effect(() => () => { - /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */ - if (this._persistence === service) this._persistence = undefined - }, 'sessionQuery.persistenceBinding') - }) - return () => void fiber.dispose() + return () => this._optionalPersistenceFiber.dispose() }, 'sessionQuery.optionalPersistence') } diff --git a/packages/session-query/session-query/src/cursor.ts b/packages/session-query/session-query/src/cursor.ts new file mode 100644 index 0000000000..8ee2a6660d --- /dev/null +++ b/packages/session-query/session-query/src/cursor.ts @@ -0,0 +1,15 @@ +/** Opaque cursor identity for session-search pagination. */ + +import type { Branded } from '@deepseek-ai/dsh-brand' + +/** Provider-owned opaque continuation token returned by session search. */ +export type SessionSearchCursor = Branded<'SessionSearchCursor'> + +/** + * Brand an encoded provider cursor for the public search contract. + * @param value - opaque encoded cursor value. + * @returns the same runtime string with session-search cursor identity. + */ +export function SessionSearchCursor(value: string): SessionSearchCursor { + return value as SessionSearchCursor +} diff --git a/packages/session-query/session-query/src/filters.ts b/packages/session-query/session-query/src/filters.ts index c7a7b40dd4..91ae640615 100644 --- a/packages/session-query/session-query/src/filters.ts +++ b/packages/session-query/session-query/src/filters.ts @@ -1,6 +1,12 @@ /** Pure provider-independent predicates for logical sessions and event text. */ -import type { SessionRecord, SessionEventSearchDocument, SessionEventResultFilter, SessionResultFilter, SessionResultRange } from './types.ts' +import type { + SessionEventResultFilter, + SessionEventSearchDocument, + SessionRecord, + SessionResultFilter, + SessionResultRange, +} from './types.ts' import { SessionQueryError } from './config.ts' /** @@ -31,6 +37,66 @@ export function filterSessionEventDocuments predicates.every(predicate => predicate(document))) } +/** + * Copy and validate logical-session filters before an asynchronous boundary. + * @param filters - caller-owned clauses to materialize. + * @returns detached validated clauses. + */ +export function materializeSessionResultFilters( + filters: readonly SessionResultFilter[], +): SessionResultFilter[] { + assertArray(filters) + return filters.map((filter) => { + switch (filter.kind) { + case 'id': + return { kind: filter.kind, values: copyStrings(filter.kind, filter.values) } + case 'cwd': + return { kind: filter.kind, values: copyNullableStrings(filter.kind, filter.values) } + case 'created-at': + return copyRange(filter.kind, filter) + case 'parent': + return { kind: filter.kind, values: copyNullableStrings(filter.kind, filter.values) } + case 'availability': { + const values = copyStrings(filter.kind, filter.values) + assertAllowedValues(filter.kind, values, ['live', 'persisted']) + return { kind: filter.kind, values } + } + default: + return unknownFilter(filter) + } + }) +} + +/** + * Copy and validate event filters before an asynchronous boundary. + * @param filters - caller-owned clauses to materialize. + * @returns detached validated clauses. + */ +export function materializeSessionEventResultFilters( + filters: readonly SessionEventResultFilter[], +): SessionEventResultFilter[] { + assertArray(filters) + return filters.map((filter) => { + switch (filter.kind) { + case 'seq': + case 'time': + return copyRange(filter.kind, filter) + case 'type': + return { kind: filter.kind, values: copyStrings(filter.kind, filter.values) } + case 'surface': { + const values = copyStrings(filter.kind, filter.values) + assertAllowedValues(filter.kind, values, ['current', 'shadowed', 'log-only']) + return { kind: filter.kind, values } + } + case 'text': + if (typeof filter.text !== 'string') throw invalidFilter('text filter text must be a string') + return { kind: filter.kind, text: filter.text } + default: + return unknownFilter(filter) + } + }) +} + /** * Compile a literal case-insensitive, whitespace-flexible semantic-text match. * @param text - caller-provided literal text. @@ -66,6 +132,8 @@ function sessionPredicate(filter: SessionResultFilter): (record: SessionRecord) case 'availability': assertAllowedValues(filter.kind, filter.values, ['live', 'persisted']) return record => filter.values.some(value => value === 'live' ? record.live : record.persisted) + default: + return unknownFilter(filter) } } @@ -88,9 +156,47 @@ function eventPredicate(filter: SessionEventResultFilter): (document: SessionEve const pattern = compileSessionTextFilter(filter.text) return document => pattern.test(document.text) } + default: + return unknownFilter(filter) } } +function copyStrings(name: string, values: readonly T[]): T[] { + if (!isRuntimeArray(values) || values.some(value => typeof value !== 'string')) { + throw invalidFilter(`${name} filter values must be an array of strings`) + } + return [...values] +} + +function assertArray(value: unknown): void { + if (!Array.isArray(value)) throw invalidFilter('filters must be an array') +} + +function copyNullableStrings(name: string, values: readonly (T | null)[]): Array { + if (!isRuntimeArray(values) || values.some(value => value !== null && typeof value !== 'string')) { + throw invalidFilter(`${name} filter values must be an array of strings or null`) + } + return [...values] +} + +function copyRange( + kind: K, + range: SessionResultRange, +): { kind: K } & SessionResultRange { + const copy = { + kind, + ...range.from === undefined ? {} : { from: range.from }, + ...range.to === undefined ? {} : { to: range.to }, + } + validateRange(kind, copy) + return copy +} + +function unknownFilter(filter: never): never { + const kind = (filter as { kind?: unknown }).kind + throw invalidFilter(`unknown filter kind ${typeof kind === 'string' ? `"${kind}"` : '(missing)'}`) +} + function assertAllowedValues( name: string, values: readonly string[], @@ -125,8 +231,13 @@ function matchesRange(value: number, range: SessionResultRange): boolean { } function invalidRange(name: string, detail: string): SessionQueryError { - return new SessionQueryError( - `session ${name} filter ${detail}`, - 'SESSION_QUERY_INVALID_FILTER', - ) + return invalidFilter(`${name} filter ${detail}`) +} + +function invalidFilter(detail: string): SessionQueryError { + return new SessionQueryError(`session ${detail}`, 'SESSION_QUERY_INVALID_FILTER') +} + +function isRuntimeArray(value: unknown): boolean { + return Array.isArray(value) } diff --git a/packages/session-query/session-query/src/index.ts b/packages/session-query/session-query/src/index.ts index 243c86746b..2c2155eebc 100644 --- a/packages/session-query/session-query/src/index.ts +++ b/packages/session-query/session-query/src/index.ts @@ -16,6 +16,7 @@ import type { SessionEventSearchRequest, SessionEventWindow, SessionRecord, + SessionResultFilter, SessionSearchExecContext, SessionSearchHit, SessionSearchPage, @@ -28,14 +29,26 @@ import { } from './config.ts' import { SessionCorpus } from './corpus.ts' import { buildSessionEventRecords, buildSessionEventSearchDocuments } from './documents.ts' -import { filterSessionEventDocuments } from './filters.ts' +import { + filterSessionEventDocuments, + filterSessionResults, + materializeSessionEventResultFilters, + materializeSessionResultFilters, +} from './filters.ts' export type * from './types.ts' +export { SessionSearchCursor } from './cursor.ts' export type { Config, SessionQueryErrorCode } from './config.ts' export { SESSION_QUERY_READ_WINDOW_MAX, SessionQueryError } from './config.ts' export { extractSessionEventText } from './extraction.ts' export { buildSessionEventRecords, buildSessionEventSearchDocuments } from './documents.ts' -export { compileSessionTextFilter, filterSessionEventDocuments, filterSessionResults } from './filters.ts' +export { + compileSessionTextFilter, + filterSessionEventDocuments, + filterSessionResults, + materializeSessionEventResultFilters, + materializeSessionResultFilters, +} from './filters.ts' export { assertSessionHeadersCompatible } from './sources.ts' declare module 'cordis' { @@ -109,6 +122,16 @@ export class SessionQueryService extends Service { return this._corpus.listSessions() } + /** + * Filter the complete logical corpus with provider-independent predicates. + * @param filters - ANDed session metadata and availability clauses. + * @returns matching cloned records in deterministic newest-first order. + */ + async filterSessions(filters: readonly SessionResultFilter[]): Promise { + const ownedFilters = materializeSessionResultFilters(filters) + return this._filterSessions(ownedFilters) + } + /** * List lightweight raw-log event records for one logical session. * @param sessionId - live-preferred session id to read. @@ -128,6 +151,18 @@ export class SessionQueryService extends Service { async filterEvents( sessionId: SessionId, filters: readonly SessionEventResultFilter[], + ): Promise { + const ownedFilters = materializeSessionEventResultFilters(filters) + return this._filterEvents(sessionId, ownedFilters) + } + + private async _filterSessions(filters: readonly SessionResultFilter[]): Promise { + return filterSessionResults(await this._corpus.listSessions(), filters) + } + + private async _filterEvents( + sessionId: SessionId, + filters: readonly SessionEventResultFilter[], ): Promise { const loaded = await this._corpus.load(sessionId) const documents = buildSessionEventSearchDocuments(sessionId, loaded.events) @@ -142,16 +177,27 @@ export class SessionQueryService extends Service { async readEvent(request: SessionEventReadRequest): Promise { const before = this._readWindow('before', request.before) const after = this._readWindow('after', request.after) - const loaded = await this._corpus.load(request.sessionId) - const target = loaded.events[request.seq] - if (target === undefined || target.seq !== request.seq) { + const sessionId = request.sessionId + const seq = request.seq + return this._readEvent(sessionId, seq, before, after) + } + + private async _readEvent( + sessionId: SessionId, + seq: number, + before: number, + after: number, + ): Promise { + const loaded = await this._corpus.load(sessionId) + const target = loaded.events[seq] + if (target === undefined || target.seq !== seq) { throw new SessionQueryError( - `session "${request.sessionId}" has no event at seq ${request.seq}`, + `session "${sessionId}" has no event at seq ${seq}`, 'SESSION_QUERY_EVENT_NOT_FOUND', ) } - const startSeq = Math.max(0, request.seq - before) - const endSeq = Math.min(loaded.events.length - 1, request.seq + after) + const startSeq = Math.max(0, seq - before) + const endSeq = Math.min(loaded.events.length - 1, seq + after) return { session: loaded.header, target, diff --git a/packages/session-query/session-query/src/types.ts b/packages/session-query/session-query/src/types.ts index d0de0dd48a..a223084f56 100644 --- a/packages/session-query/session-query/src/types.ts +++ b/packages/session-query/session-query/src/types.ts @@ -5,6 +5,9 @@ */ import type { SessionEvent, SessionEventType, SessionHeader, SessionId } from '@deepseek-ai/dsh-session' +import type { SessionSearchCursor } from './cursor.ts' + +export type { SessionSearchCursor } from './cursor.ts' /** Whether an event is current model context, replaced context, or raw-log-only. */ export type SessionEventSurface = 'current' | 'shadowed' | 'log-only' @@ -106,7 +109,7 @@ export interface SessionSearchPage { /** Results for this page in contract-defined order. */ items: readonly T[] /** Opaque continuation cursor, absent on the final page. */ - nextCursor?: string + nextCursor?: SessionSearchCursor } /** Controls shared by cross-session and within-session search calls. */ @@ -126,7 +129,7 @@ export interface SessionSearchRequest { /** Maximum sessions in this page. */ limit?: number /** Opaque cursor returned for the identical normalized request. */ - cursor?: string + cursor?: SessionSearchCursor } /** Within-session full-text search request. */ @@ -140,7 +143,7 @@ export interface SessionEventSearchRequest { /** Maximum events in this page. */ limit?: number /** Opaque cursor returned for the identical normalized request. */ - cursor?: string + cursor?: SessionSearchCursor } /** One event full-text search hit with a bounded plain-text excerpt. */ diff --git a/packages/session-query/session-query/tests/search-helpers.spec.ts b/packages/session-query/session-query/tests/search-helpers.spec.ts index e9cf857608..8ee9cf1ccf 100644 --- a/packages/session-query/session-query/tests/search-helpers.spec.ts +++ b/packages/session-query/session-query/tests/search-helpers.spec.ts @@ -10,6 +10,8 @@ import SessionQueryService, { extractSessionEventText, filterSessionEventDocuments, filterSessionResults, + materializeSessionEventResultFilters, + materializeSessionResultFilters, SessionSearchService, type SessionEventSearchHit, type SessionEventSearchRequest, @@ -177,6 +179,31 @@ describe('session-query document and filter helpers', () => { expect(() => buildSessionEventRecords(id, malformed)).toThrow(expectCode('SESSION_QUERY_INVALID_SURFACE')) }) + it('owns filters and rejects malformed runtime filter shapes deterministically', () => { + expect(materializeSessionResultFilters([{ kind: 'created-at', to: 2 }])) + .toEqual([{ kind: 'created-at', to: 2 }]) + expect(() => materializeSessionResultFilters('not-an-array' as never)) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionResultFilters([{ kind: 'id', values: 'bad' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionResultFilters([{ kind: 'id', values: [1] } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionResultFilters([{ kind: 'cwd', values: 'bad' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionResultFilters([{ kind: 'parent', values: [1] } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionResultFilters([{} as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionEventResultFilters([{ kind: 'text', text: 1 } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => materializeSessionEventResultFilters([{ kind: 'future' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => filterSessionResults([], [{ kind: 'future' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + expect(() => filterSessionEventDocuments([], [{ kind: 'future' } as never])) + .toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + }) + it('exposes the scan path on the concrete exact-read service', async () => { const ctx = new Context() await ctx.plugin(SessionStore) diff --git a/packages/session-query/session-query/tests/session-query.spec.ts b/packages/session-query/session-query/tests/session-query.spec.ts index 3b50feee45..e1432722f2 100644 --- a/packages/session-query/session-query/tests/session-query.spec.ts +++ b/packages/session-query/session-query/tests/session-query.spec.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from 'vitest' -import { Context } from 'cordis' +import { Context, type Fiber } from 'cordis' import SessionStore, { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' import type { SessionEvent, SessionHeader, SessionId as SessionIdType } from '@deepseek-ai/dsh-session' -import SessionPersistence from '@deepseek-ai/dsh-session-persistence' +import SessionPersistence, { SessionPersistenceRevision } from '@deepseek-ai/dsh-session-persistence' import SessionQueryService, { + type SessionEventSurface, type SessionQueryErrorCode, } from '@deepseek-ai/dsh-session-query' @@ -59,6 +60,14 @@ class TestPersistence extends SessionPersistence { TestPersistence.afterList?.() return Promise.resolve(headers) } + + + async listSnapshots() { + return [...TestPersistence.entries.values()].map(entry => ({ + header: structuredClone(entry.meta), + revision: SessionPersistenceRevision(`events:${entry.events.length}`), + })) + } } async function liveContext(config: ConstructorParameters[1] = {}): Promise { @@ -94,6 +103,38 @@ describe('session-query exact reads', () => { expect(older.header.createdAt).toBe(1) }) + it('filters sessions symmetrically and owns mutable filter values immediately', async () => { + const durable = header('durable-filter', 1) + TestPersistence.reset([{ meta: durable, events: eventLog('durable') }]) + const ctx = await liveContext() + const live = ctx.sessions.create(SessionId('live-filter'), { meta: { createdAt: 2 } }) + live.append( + 'user/message', + { content: [{ type: 'text', text: 'live' }], source: { kind: 'user' } }, + { surfaceOp: 'append' }, + ) + const persistence = await ctx.plugin(TestPersistence) + + const ids = [durable.id] + const filtered = ctx.sessionQuery.filterSessions([{ kind: 'id', values: ids }]) + ids[0] = live.id + await expect(filtered).resolves.toEqual([{ + header: durable, + live: false, + persisted: true, + }]) + + const surfaces: SessionEventSurface[] = ['current'] + const events = ctx.sessionQuery.filterEvents(live.id, [{ kind: 'surface', values: surfaces }]) + surfaces[0] = 'shadowed' + await expect(events).resolves.toMatchObject([{ sessionId: live.id, surface: 'current', text: 'live' }]) + await expect(ctx.sessionQuery.filterSessions([{ kind: 'future' } as never])) + .rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionQuery.filterEvents(live.id, [{ kind: 'future' } as never])) + .rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await persistence.dispose() + }) + it('classifies current, shadowed, and raw-log-only events through foldSurface', async () => { const ctx = await liveContext() const session = ctx.sessions.create(SessionId('surface')) @@ -267,4 +308,26 @@ describe('session-query exact reads', () => { await fiber.dispose() expect(ctx.sessionQuery).toBeUndefined() }) + + it('awaits optional-persistence child-fiber quiescence on disposal', async () => { + TestPersistence.reset() + const ctx = new Context() + await ctx.plugin(SessionStore) + const query = await ctx.plugin(SessionQueryService) + const persistence = await ctx.plugin(TestPersistence) + const optional = (ctx.sessionQuery as unknown as { + _corpus: { _optionalPersistenceFiber: Fiber } + })._corpus._optionalPersistenceFiber + let release!: () => void + const cleanup = new Promise((resolve) => { release = resolve }) + optional.ctx.effect(() => () => cleanup) + + let settled = false + const disposing = query.dispose().then(() => { settled = true }) + await Promise.resolve() + expect(settled).toBe(false) + release() + await disposing + await persistence.dispose() + }) }) diff --git a/packages/session-query/session-query/tsconfig.json b/packages/session-query/session-query/tsconfig.json index 7153dae8bb..1a254e5379 100644 --- a/packages/session-query/session-query/tsconfig.json +++ b/packages/session-query/session-query/tsconfig.json @@ -17,6 +17,9 @@ { "path": "../../../vendor/schemastery" }, + { + "path": "../../util/brand" + }, { "path": "../../llm/llm" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 997f00ff5c..902c362355 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -720,6 +720,9 @@ importers: packages/session-persistence/session-persistence: devDependencies: + '@deepseek-ai/dsh-brand': + specifier: workspace:^ + version: link:../../util/brand '@deepseek-ai/dsh-session': specifier: workspace:^ version: link:../../core/session @@ -765,6 +768,9 @@ importers: specifier: ^3.18.0 version: 3.18.0 devDependencies: + '@deepseek-ai/dsh-brand': + specifier: workspace:^ + version: link:../../util/brand '@deepseek-ai/dsh-llm': specifier: workspace:^ version: link:../../llm/llm diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index 0d0a0b871d..ba2d75a8ff 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -45,6 +45,8 @@ { "doc": "docs/core-data-structures/persistence.md", "symbol": "SessionHeader", "source": "packages/core/session/src/types.ts" }, { "doc": "docs/core-data-structures/persistence.md", "symbol": "CreateSessionOptions", "source": "packages/core/session/src/types.ts" }, + { "doc": "docs/core-data-structures/persistence.md", "symbol": "SessionPersistenceRevision", "source": "packages/session-persistence/session-persistence/src/revision.ts" }, + { "doc": "docs/core-data-structures/persistence.md", "symbol": "SessionPersistenceSnapshot", "source": "packages/session-persistence/session-persistence/src/index.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventSurface", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionRecord", "source": "packages/session-query/session-query/src/types.ts" }, @@ -52,6 +54,7 @@ { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionResultFilter", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventResultFilter", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventSearchDocument", "source": "packages/session-query/session-query/src/types.ts" }, + { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionSearchCursor", "source": "packages/session-query/session-query/src/cursor.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionSearchRequest", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionEventSearchRequest", "source": "packages/session-query/session-query/src/types.ts" }, { "doc": "docs/core-data-structures/session-query.md", "symbol": "SessionSearchPage", "source": "packages/session-query/session-query/src/types.ts" }, From 35edf2a825b40e4bac80d3e38d7c5334dbe1dd85 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Wed, 15 Jul 2026 12:32:18 +0800 Subject: [PATCH 003/113] fix(session-query): qualify persistence revisions by store --- docs/config-catalog.md | 2 +- ...026-07-10-sqlite-session-query-provider.md | 2 +- .../session-persistence-jsonl/README.md | 2 +- .../tests/jsonl.spec.ts | 23 ++++++++ .../session-persistence-sqlite/README.md | 4 +- .../session-persistence-sqlite/src/index.ts | 35 ++++++++--- .../session-persistence-sqlite/src/schema.ts | 38 +++++++++--- .../tests/sqlite.spec.ts | 59 +++++++++++++++++-- .../session-persistence/README.md | 2 +- .../session-persistence/src/index.ts | 4 +- .../session-persistence/src/revision.ts | 5 +- .../session-query-sqlite/README.md | 2 +- .../session-query-sqlite/tests/sqlite.spec.ts | 45 ++++++++++++++ 13 files changed, 192 insertions(+), 31 deletions(-) diff --git a/docs/config-catalog.md b/docs/config-catalog.md index e8a06e469e..aa37295146 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -574,7 +574,7 @@ export interface Config { export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' ``` -Source: [`packages/session-persistence/session-persistence-sqlite/src/index.ts:50`](../packages/session-persistence/session-persistence-sqlite/src/index.ts) +Source: [`packages/session-persistence/session-persistence-sqlite/src/index.ts:51`](../packages/session-persistence/session-persistence-sqlite/src/index.ts) ## `@deepseek-ai/dsh-session-query` diff --git a/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md b/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md index fafa7f3569..b77279949b 100644 --- a/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md +++ b/docs/rfc/implemented/feature/2026-07-10-sqlite-session-query-provider.md @@ -32,7 +32,7 @@ Both persistent and live FTS5 tables use `unicode61`. The implementation experim The shared extractor includes message text, reasoning, nested tool-call/result content, tool names and arguments, blocked-prompt reasons, todo status/content, and error or terminal status detail. Structural boundaries, stream chunks, request headers, successful completion markers, and unknown declaration-merged event/content variants produce no document. Surface classification reuses `foldSurface()` so search agrees with model-history derivation. -One serialized operation reads the provider-neutral `SessionPersistence` snapshot listing, compares each opaque revision with the revision stored beside the indexed session, loads only new or changed logs, reconciles rows in one transaction, and executes the query. Observation repeats when listing changes during a load; this incorporates a mutating load repair's refreshed revision before commit. Repeated queries and unchanged reopen load no full persisted logs. New, changed, and deleted sessions update on the next stable search. A source or extraction failure cannot mark a row current, and a transaction failure rolls back so a later search retries. +One serialized operation reads the provider-neutral `SessionPersistence` snapshot listing, compares each source-qualified opaque revision with the revision stored beside the indexed session, loads only new or changed logs, reconciles rows in one transaction, and executes the query. A revision identifies its backing persistence store as well as the backend-local log revision, so reopening against the same store reuses indexed rows while switching to an independent store cannot collide on a session id and local counter. Observation repeats when listing changes during a load; this incorporates a mutating load repair's refreshed revision before commit. Repeated queries and unchanged reopen load no full persisted logs. New, changed, and deleted sessions update on the next stable search. A source or extraction failure cannot mark a row current, and a transaction failure rolls back so a later search retries. Persisted documents survive restarts. Live sessions use connection-local TEMP tables, shadow the persisted base for the same id, and reveal that base on detach. Closing the database drops live rows. Unmounting persistence hides durable rows without treating absence as authoritative deletion; remounting observes and reconciles the backend again. Conflicting immutable live and durable headers fail rather than combining sources. diff --git a/packages/session-persistence/session-persistence-jsonl/README.md b/packages/session-persistence/session-persistence-jsonl/README.md index 0be2c785b3..2043967cb8 100644 --- a/packages/session-persistence/session-persistence-jsonl/README.md +++ b/packages/session-persistence/session-persistence-jsonl/README.md @@ -25,7 +25,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence - **Append-only.** Committed events (at or below a flushed `turn/end`) are never rewritten. Subsequent appends are line appends at EOF + `fsync`. - **Crash recovery — close, don't truncate.** A crash can leave a log whose final turn never closed (real events after the last `turn/end`). `load` PRESERVES those events (a turn can be huge — they are real work) and closes the orphaned turn by durably appending synthetic boundary events: an error `tool/result` for every `tool-call` the crash left unanswered (the loop logs the assistant message before running the tools, so a mid-tool crash leaves dangling calls — and `deriveMessages()` would replay an assistant tool-call with no result, which providers reject), then a `step/end` if a step was open, then `turn/end {kind:'interrupted'}`, returning a balanced log. Only a never-fully-written **torn tail fragment** (a final line with no newline / unparseable) is `ftruncate`d away before the closers are written. See [session persistence](../../../docs/rfc/implemented/architecture/2026-06-14-session-persistence.md). - **Contiguous-seq.** `load` rejects a mid-log parse error or `seq` gap (unloadable); `append` rejects a batch whose first `seq` does not continue the stored log, and rejects non-JSON-serializable `event.data` naming the offending event type. -- **Lightweight revisions.** `listSnapshots()` reads each header and returns an opaque identity from the file device/inode, size, and nanosecond mtime/ctime. Under the append-only single-writer contract, unchanged files retain revisions while append, repair, or replacement changes them without parsing event lines. +- **Lightweight revisions.** `listSnapshots()` reads each header and returns an opaque identity from the file device/inode, size, and nanosecond mtime/ctime. Under the append-only single-writer contract, unchanged files retain revisions while append, repair, replacement, or switching to an independent root changes them without parsing event lines. - **Format version.** Only the current `SESSION_FORMAT_VERSION` (v0) is supported; `load` rejects any other version. While the harness is unreleased the on-disk format is pre-release/unstable: a breaking format change is absorbed at v0 (no bump until the first tagged release) and non-current logs are rejected — there is no migration (no persisted user data to preserve). ## Write path diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index 757ba03aac..6f5aa81ee0 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -147,6 +147,29 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => { expect(loaded.events).toEqual(log) // chunks preserved, contiguous seqs }) + it('source-qualifies revisions across roots while preserving same-log reopen identity', async () => { + const m = meta('revision-source') + await ctx.sessionPersistence.create(m) + await ctx.sessionPersistence.append(m.id, oneTurnLog()) + const revision = (await ctx.sessionPersistence.listSnapshots())[0]?.revision + + const reopenedCtx = new Context() + await reopenedCtx.plugin(SessionStore) + await reopenedCtx.plugin(SessionPersistenceJsonl, { root }) + expect((await reopenedCtx.sessionPersistence.listSnapshots())[0]?.revision).toBe(revision) + + const otherRoot = await freshRoot() + const otherCtx = new Context() + await otherCtx.plugin(SessionStore) + await otherCtx.plugin(SessionPersistenceJsonl, { root: otherRoot }) + await otherCtx.sessionPersistence.create(m) + await otherCtx.sessionPersistence.append(m.id, oneTurnLog()) + expect((await otherCtx.sessionPersistence.listSnapshots())[0]?.revision).not.toBe(revision) + + await reopenedCtx.fiber.dispose() + await otherCtx.fiber.dispose() + }) + it('persists a forked child seed through the existing session write path', async () => { const source = ctx.sessions.create(SessionId('persist-parent'), { meta: { cwd: '/workspace' } }) appendClosedTurn(source) diff --git a/packages/session-persistence/session-persistence-sqlite/README.md b/packages/session-persistence/session-persistence-sqlite/README.md index 6e6006dc6b..e45f1d2371 100644 --- a/packages/session-persistence/session-persistence-sqlite/README.md +++ b/packages/session-persistence/session-persistence-sqlite/README.md @@ -6,7 +6,7 @@ A SQLite durable session-persistence backend — a second `SessionPersistence` i ## Storage model -Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [session surface](../../../docs/rfc/implemented/architecture/2026-06-18-session-surface.md)). Out-of-log metadata (`SessionHeader`) and a monotonic per-log revision live in a `sessions` row. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`list` reports exactly the sessions that have a row). +Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [session surface](../../../docs/rfc/implemented/architecture/2026-06-18-session-surface.md)). Out-of-log metadata (`SessionHeader`) and a monotonic per-log revision live in a `sessions` row; a singleton state row carries the immutable store id. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`list` reports exactly the sessions that have a row). The repo's `engines.node` is `^22.19.0 || >=24.0.0` (Node 22.19+ or 24+), matching the LTS floor required by the installed Pi adapter dependency; `node:sqlite` itself ships without the `--experimental-sqlite` flag from Node 22.13 (LTS) and 23.4 / 24 (Current) on. The range deliberately excludes Node 23 because that line is non-LTS/EOL and still has flagged runtime features before 23.6. The database opens with `foreign_keys = ON` (so `ON DELETE CASCADE` drops a session's events with its row) and the configured `journal_mode` (default `wal`; pick a rollback-journal mode like `delete` on filesystems where WAL's shared-memory files do not work, e.g. network mounts). The table-layout version is stored in `PRAGMA user_version` and checked on open: a fresh database is stamped with the current `SCHEMA_VERSION`; a database written by any other, incompatible build (a non-current `user_version`, older or newer) is rejected rather than opened against an unknown layout — there is no migration (unreleased software). @@ -14,7 +14,7 @@ The repo's `engines.node` is `^22.19.0 || >=24.0.0` (Node 22.19+ or 24+), matchi - **Append = a transaction.** `append` runs `BEGIN`/`COMMIT` around the batch: it materializes the `sessions` row (if still lazy) and INSERTs every event, asserting the contiguous-seq contract first (the first event's `seq` must equal the stored next-seq). A mid-batch failure (a UNIQUE violation on a duplicated seq) rolls back entirely, so the stored log and the in-memory cursor stay consistent. (`load()` already balanced the stored log, so `append` never has to repair a crash tail.) - **Lazy materialization.** `create()` records intent in memory only — no row is written until the first `append`. A created-but-never-appended session has no `sessions` row, so it is absent from `list()` (which reports exactly the sessions that have a row). -- **Lightweight revisions.** `listSnapshots()` reads the monotonic revision stored beside each session header. Append and mutating load repair increment it in the same transaction as their event changes, so unchanged observations are stable and no full-log count or parse is required. +- **Lightweight revisions.** `listSnapshots()` combines the database's immutable random store id and physical file identity with the monotonic revision stored beside each session header; an in-memory database uses the store id alone. Append and mutating load repair increment the local counter in the same transaction as their event changes, so unchanged same-file observations are stable, independent stores and file replacements cannot collide on a local counter, and no full-log count or parse is required. - **Interrupted-turn close on load.** `load()` reads every stored event ordered by `seq` and finds the longest seq-contiguous, parseable prefix — INCLUDING the real events of an interrupted final turn after the last `turn/end` (the loop only flushes at `turn/end`, so a process killed mid-turn leaves real, fully-written rows past it). A single turn can be huge in a long-horizon task, so those events are **preserved, never truncated**: `load()` CLOSES the orphaned turn by durably appending the minimal synthetic boundary events (an error `tool/result` for every assistant tool call left unanswered, a `step/end` if a step was open, then a `turn/end` carrying `{ kind: 'interrupted' }`), inside one transaction that also DELETEs any never-fully-written torn tail row. `load()` is therefore mutating — after it the stored rows are balanced and the cursor is truthful, so the next `append` continues cleanly. The boundary (last `turn/end`, torn-tail detection) is computed from the `seq`/`type` columns so a malformed `data` in a torn tail row is never parsed (discarded, not unloadable). A parse error or `seq` gap inside the committed region (at or before the last real `turn/end`) makes the session unloadable. A session whose only turn never closed keeps its metadata row and stays present in `list()` — the same as the JSONL backend, whose file likewise survives a first append that never reached `turn/end`. ## Configuration (schemastery) diff --git a/packages/session-persistence/session-persistence-sqlite/src/index.ts b/packages/session-persistence/session-persistence-sqlite/src/index.ts index 3a1c7ffbcc..546f285824 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/index.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/index.ts @@ -19,6 +19,7 @@ import { Context } from 'cordis' import z from 'schemastery' +import { statSync } from 'node:fs' import { DatabaseSync } from 'node:sqlite' import { mkdir } from 'node:fs/promises' import { dirname, resolve } from 'node:path' @@ -84,6 +85,7 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers override readonly name = 'session-persistence-sqlite' private db!: DatabaseSync + private storeIdentity!: string private ready: Promise private coordinator: PersistenceCoordinator @@ -98,12 +100,29 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers } private async openDb(path: string, journalMode: JournalMode): Promise { - if (path !== ':memory:') { - const abs = resolve(path) - await mkdir(dirname(abs), { recursive: true, mode: 0o700 }) - this.db = openDatabase(abs, journalMode) - } else { - this.db = openDatabase(path, journalMode) + const actual = path === ':memory:' ? path : resolve(path) + if (actual !== ':memory:') await mkdir(dirname(actual), { recursive: true, mode: 0o700 }) + this.db = openDatabase(actual, journalMode) + try { + const row = this.db.prepare( + 'SELECT store_id FROM persistence_state WHERE singleton = 1', + ).get() as { store_id: string } | undefined + /* v8 ignore next -- openDatabase inserts the singleton before returning. */ + if (row === undefined) { + throw new Error(`session database at "${actual}" has no store identity`) + } + if (row.store_id.length === 0) { + throw new Error(`session database at "${actual}" has no valid store identity`) + } + if (actual !== ':memory:') { + const identity = statSync(actual, { bigint: true }) + this.storeIdentity = `file:${identity.dev}:${identity.ino}:${identity.birthtimeNs}:store:${row.store_id}` + } else { + this.storeIdentity = `memory:store:${row.store_id}` + } + } catch (error: unknown) { + this.db.close() + throw error } } @@ -234,13 +253,13 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers return rows.map(rowToMeta) } - /** List metadata with an append-only event-count revision per session. */ + /** List metadata with a source-qualified monotonic revision per session. */ async listSnapshots(): Promise { await this.ready const rows = this.db.prepare('SELECT * FROM sessions').all() as unknown as SessionRow[] return rows.map(row => ({ header: rowToMeta(row), - revision: SessionPersistenceRevision(`revision:${row.revision}`), + revision: SessionPersistenceRevision(`${this.storeIdentity}:revision:${row.revision}`), })) } diff --git a/packages/session-persistence/session-persistence-sqlite/src/schema.ts b/packages/session-persistence/session-persistence-sqlite/src/schema.ts index 2f238b0131..caf1766f22 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/schema.ts @@ -1,12 +1,14 @@ /** * Schema + load-time helpers for the SQLite session-persistence backend: the - * DDL (a `sessions` metadata table and a 1:1 `events` row per `SessionEvent`), - * the database open/configure step, and the last-`turn/end` cut that gives the - * SQLite backend the SAME crash-tail-on-load semantics as the JSONL backend. + * DDL (a store-identity row, `sessions` metadata, and a 1:1 `events` row per + * `SessionEvent`), the database open/configure step, and the last-`turn/end` + * cut that gives the SQLite backend the SAME crash-tail-on-load semantics as + * the JSONL backend. * * @module dsh-session-persistence-sqlite/schema */ +import { randomUUID } from 'node:crypto' import { DatabaseSync } from 'node:sqlite' import type { SessionEvent, SessionId, SessionHeader, SurfaceOp } from '@deepseek-ai/dsh-session' @@ -15,7 +17,7 @@ import type { SessionEvent, SessionId, SessionHeader, SurfaceOp } from '@deepsee * layout; orthogonal to a session's own `version` (which versions the EVENT * vocabulary, stored per session in the `sessions` row). */ -export const SCHEMA_VERSION = 5 +export const SCHEMA_VERSION = 6 /** * A row of the `sessions` table — the out-of-log metadata ({@link SessionHeader}). @@ -70,14 +72,25 @@ export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' * current one (written by a different, incompatible build — older or newer) is * REJECTED rather than opened against a layout this build does not understand. * There are no migrations: an incompatible layout is rejected. The current - * sessions row carries every header field plus its monotonic snapshot revision; - * the events row carries the complete surface metadata. + * persistence-state row carries an immutable random store id, the sessions row + * carries every header field plus its monotonic snapshot revision, and the + * events row carries the complete surface metadata. * @param path - the SQLite database file to open (created when absent). * @param journalMode - the journal pragma to apply — a closed in-code union, validated by the plugin Config. - * @returns the open handle with pragmas applied and both tables ensured. + * @returns the open handle with pragmas applied and all three tables ensured. */ export function openDatabase(path: string, journalMode: JournalMode): DatabaseSync { const db = new DatabaseSync(path) + try { + configureDatabase(db, path, journalMode) + return db + } catch (error: unknown) { + db.close() + throw error + } +} + +function configureDatabase(db: DatabaseSync, path: string, journalMode: JournalMode): void { db.exec('PRAGMA foreign_keys = ON') // journalMode is a closed in-code union (validated by the plugin Config), not // user-controlled SQL — safe to interpolate (PRAGMA takes no bound params). @@ -85,7 +98,6 @@ export function openDatabase(path: string, journalMode: JournalMode): DatabaseSy // `PRAGMA user_version` always returns exactly one row { user_version }. const { user_version: onDisk } = db.prepare('PRAGMA user_version').get() as { user_version: number } if (onDisk !== 0 && onDisk !== SCHEMA_VERSION) { - db.close() throw new Error(`session database at "${path}" has schema version ${onDisk}, incompatible with this build (${SCHEMA_VERSION})`) } if (onDisk === 0) { @@ -94,6 +106,15 @@ export function openDatabase(path: string, journalMode: JournalMode): DatabaseSy // constant (SCHEMA_VERSION is a trusted in-code number, not user input). db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`) } + db.exec(` + CREATE TABLE IF NOT EXISTS persistence_state ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + store_id TEXT NOT NULL + ) STRICT + `) + db.prepare( + 'INSERT OR IGNORE INTO persistence_state (singleton, store_id) VALUES (1, ?)', + ).run(randomUUID()) db.exec(` CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, @@ -117,7 +138,6 @@ export function openDatabase(path: string, journalMode: JournalMode): DatabaseSy PRIMARY KEY (session_id, seq) ) STRICT `) - return db } /** diff --git a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts index 98924ca691..5b85f65908 100644 --- a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it } from 'vitest' import { Context } from 'cordis' import { existsSync } from 'node:fs' -import { mkdtemp, rm } from 'node:fs/promises' +import { mkdtemp, rm, symlink } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' @@ -245,12 +245,12 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { dbNewer.close() expect(() => openDatabase(path, 'wal')).toThrow(/incompatible with this build/) - // A stale OLDER version (e.g. a pre-summary-drop v1 DB) is also rejected — - // we do not migrate (unreleased software, no backward-compat). + // The immediately preceding layout lacks the required store identity and is + // rejected rather than migrated (unreleased software, no backward-compat). const olderPath = await freshDbPath() openDatabase(olderPath, 'wal').close() const dbOlder = openDatabase(olderPath, 'wal') - dbOlder.exec('PRAGMA user_version = 1') + dbOlder.exec(`PRAGMA user_version = ${SCHEMA_VERSION - 1}`) dbOlder.close() expect(() => openDatabase(olderPath, 'wal')).toThrow(/incompatible with this build/) }) @@ -337,8 +337,46 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { await fiber2.dispose() }) + it('source-qualifies revisions across stores while preserving same-file reopen identity', async () => { + const pathA = await freshDbPath() + const pathB = await freshDbPath() + const m = meta('revision-source') + const a = await backend(pathA) + await a.ctx.sessionPersistence.create(m) + await a.ctx.sessionPersistence.append(m.id, oneTurnLog()) + const revisionA = (await a.ctx.sessionPersistence.listSnapshots())[0]?.revision + await a.dispose() + + const probeA = openDatabase(pathA, 'wal') + const storeIdA = (probeA.prepare( + 'SELECT store_id FROM persistence_state WHERE singleton = 1', + ).get() as { store_id: string }).store_id + probeA.close() + + const aliasA = `${pathA}.alias` + await symlink(pathA, aliasA) + const reopenedA = await backend(aliasA) + expect((await reopenedA.ctx.sessionPersistence.listSnapshots())[0]?.revision).toBe(revisionA) + await reopenedA.dispose() + + const b = await backend(pathB) + await b.ctx.sessionPersistence.create(m) + await b.ctx.sessionPersistence.append(m.id, oneTurnLog()) + const revisionB = (await b.ctx.sessionPersistence.listSnapshots())[0]?.revision + const probeB = openDatabase(pathB, 'wal') + const storeIdB = (probeB.prepare( + 'SELECT store_id FROM persistence_state WHERE singleton = 1', + ).get() as { store_id: string }).store_id + probeB.close() + expect(storeIdB).not.toBe(storeIdA) + expect(revisionB).not.toBe(revisionA) + expect(String(revisionA)).toMatch(/:revision:1$/) + expect(String(revisionB)).toMatch(/:revision:1$/) + await b.dispose() + }) + it('exposes the schema version constant', () => { - expect(SCHEMA_VERSION).toBe(5) + expect(SCHEMA_VERSION).toBe(6) }) it('keeps the revision stable for an empty repair hook', async () => { @@ -354,6 +392,17 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { }) describe('SessionPersistenceSqlite: edge cases', () => { + it('rejects and closes a current-schema database with an invalid store identity', async () => { + const path = await freshDbPath() + const db = openDatabase(path, 'wal') + db.exec("UPDATE persistence_state SET store_id = '' WHERE singleton = 1") + db.close() + + const b = await backend(path) + await expect(b.ctx.sessionPersistence.listSnapshots()).rejects.toThrow(/no valid store identity/) + await expect(b.dispose()).resolves.toBeUndefined() + }) + it('append rolls back and rethrows when an event INSERT fails inside the transaction', async () => { const path = await freshDbPath() const m = meta('rollback-insert') diff --git a/packages/session-persistence/session-persistence/README.md b/packages/session-persistence/session-persistence/README.md index 32958bfaaa..d16bfcda40 100644 --- a/packages/session-persistence/session-persistence/README.md +++ b/packages/session-persistence/session-persistence/README.md @@ -12,7 +12,7 @@ The persisted unit IS the existing `SessionEvent` (event-sourced model — the l | `append(id, events): Promise` | Durably persist a batch (from the `session/flush` drain). Append-only; first event `seq` == stored next-seq after any repair; rejects non-JSON-serializable data naming the offending type. | | `load(id): Promise<{ meta; events }>` | Reload meta + log. Preserves an interrupted (unclosed) final turn and closes it with synthetic closers — an error `tool/result` per unanswered `tool-call`, then `step/end?`+`turn/end {interrupted}` (a turn can be huge — never truncated); only a torn tail fragment is dropped. Events contiguous (`events[i].seq === i`); rejects a committed-region gap/parse error or unknown `version`. | | `list(): Promise` | Lightweight listing from metadata, no full-log parse. A zero-event lazily-materialized session is absent from `list`. | -| `listSnapshots(): Promise` | Lightweight metadata plus an opaque branded per-log revision, without loading event logs. A revision stays equal while that log is unchanged and changes after append or mutating load repair. | +| `listSnapshots(): Promise` | Lightweight metadata plus an opaque branded per-log revision, without loading event logs. A revision stays equal while that log and its backing store are unchanged, changes after append or mutating load repair, and cannot collide solely because two stores use the same local counter. | ## Invariants every backend must honor diff --git a/packages/session-persistence/session-persistence/src/index.ts b/packages/session-persistence/session-persistence/src/index.ts index 84ab0f321a..b647befeee 100644 --- a/packages/session-persistence/session-persistence/src/index.ts +++ b/packages/session-persistence/session-persistence/src/index.ts @@ -34,7 +34,7 @@ export { SessionPersistenceRevision } from './revision.ts' export interface SessionPersistenceSnapshot { /** Detached metadata for one materialized session. */ header: SessionHeader - /** Opaque token that changes whenever this stored log changes. */ + /** Opaque source-qualified token that changes whenever this stored log changes. */ revision: SessionPersistenceRevision } @@ -172,6 +172,8 @@ export abstract class SessionPersistence extends Service { * * Repeated observations of an unchanged log return the same revision. A * successful mutating {@link load} repair changes the next listed revision. + * Revisions also distinguish independently backed stores so backend-local + * counters cannot compare equal across different persistence sources. * @returns one header and opaque revision per materialized session without loading full logs. */ abstract listSnapshots(): Promise diff --git a/packages/session-persistence/session-persistence/src/revision.ts b/packages/session-persistence/session-persistence/src/revision.ts index 41378eb3e4..cb037ffafc 100644 --- a/packages/session-persistence/session-persistence/src/revision.ts +++ b/packages/session-persistence/session-persistence/src/revision.ts @@ -2,7 +2,10 @@ import type { Branded } from '@deepseek-ai/dsh-brand' -/** Backend-owned token that changes whenever one persisted session log changes. */ +/** + * Backend-owned token that identifies both one storage source and one revision + * of a persisted session log. + */ export type SessionPersistenceRevision = Branded<'SessionPersistenceRevision'> /** diff --git a/packages/session-query/session-query-sqlite/README.md b/packages/session-query/session-query-sqlite/README.md index 001ea87d7d..d2f337cafd 100644 --- a/packages/session-query/session-query-sqlite/README.md +++ b/packages/session-query/session-query-sqlite/README.md @@ -12,7 +12,7 @@ All three surfaces (`current`, `shadowed`, and `log-only`) are searchable by def ## Source and index lifecycle -The service requires `ctx.sessions` and observes optional `ctx.sessionPersistence` dynamically. One serialized state machine compares lightweight durable snapshot revisions, loads only new or changed logs, extracts shared semantic documents, reconciles changes transactionally, and runs the query. Repeated queries and unchanged reopen load no full durable logs; new, changed, deleted, or load-repaired sources reconcile on the next stable observation. Source or transaction failure commits nothing, and the next search retries. +The service requires `ctx.sessions` and observes optional `ctx.sessionPersistence` dynamically. One serialized state machine compares source-qualified lightweight durable snapshot revisions, loads only new or changed logs, extracts shared semantic documents, reconciles changes transactionally, and runs the query. Repeated queries and unchanged same-store reopen load no full durable logs; switching stores, or observing new, changed, deleted, or load-repaired sources, reconciles on the next stable observation. Source or transaction failure commits nothing, and the next search retries. Persisted FTS rows live in a dedicated derived database. Connection-local TEMP tables hold live rows, which shadow the durable base for the same session and reveal it when the live owner disappears. Unmounting persistence hides durable rows without discarding the cache; remounting reconciles it. Closing or reopening the database drops every live overlay while retaining persisted rows. diff --git a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts index f4904e17d8..9897331549 100644 --- a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts @@ -926,4 +926,49 @@ describe('SQLite schema, cancellation, and real persistence integration', () => await expect(ctx.sessionPersistence.load(meta.id)).resolves.toMatchObject({ meta, events: [{ seq: 0 }] }) await persistence.dispose() }) + + it('reconciles colliding local revisions when a derived index reopens against another SQLite store', async () => { + const persistencePathA = await temporaryPath('canonical-a.db') + const persistencePathB = await temporaryPath('canonical-b.db') + const searchPath = await temporaryPath('derived-collision.db') + const shared = header('same-id', 10) + + const first = new Context() + await first.plugin(SessionStore) + const persistenceA = await first.plugin(SessionPersistenceSqlite, { path: persistencePathA }) + await first.sessionPersistence.create(shared) + await first.sessionPersistence.append(shared.id, messageEvents('alpha source')) + const loadA = vi.spyOn(first.sessionPersistence, 'load') + const searchA = await first.plugin(SessionSearchSqlite, { path: searchPath }) + await expect(first.sessionSearch.searchSessions({ query: 'alpha' })) + .resolves.toMatchObject({ items: [{ header: shared }] }) + expect(loadA).toHaveBeenCalledTimes(1) + await searchA.dispose() + await persistenceA.dispose() + + const reopened = new Context() + await reopened.plugin(SessionStore) + const persistenceAAgain = await reopened.plugin(SessionPersistenceSqlite, { path: persistencePathA }) + const reopenedLoad = vi.spyOn(reopened.sessionPersistence, 'load') + const searchAAgain = await reopened.plugin(SessionSearchSqlite, { path: searchPath }) + await expect(reopened.sessionSearch.searchSessions({ query: 'alpha' })) + .resolves.toMatchObject({ items: [{ header: shared }] }) + expect(reopenedLoad).not.toHaveBeenCalled() + await searchAAgain.dispose() + await persistenceAAgain.dispose() + + const second = new Context() + await second.plugin(SessionStore) + const persistenceB = await second.plugin(SessionPersistenceSqlite, { path: persistencePathB }) + await second.sessionPersistence.create(shared) + await second.sessionPersistence.append(shared.id, messageEvents('bravo source')) + const loadB = vi.spyOn(second.sessionPersistence, 'load') + const searchB = await second.plugin(SessionSearchSqlite, { path: searchPath }) + await expect(second.sessionSearch.searchSessions({ query: 'bravo' })) + .resolves.toMatchObject({ items: [{ header: shared }] }) + await expect(second.sessionSearch.searchSessions({ query: 'alpha' })).resolves.toEqual({ items: [] }) + expect(loadB).toHaveBeenCalledTimes(1) + await searchB.dispose() + await persistenceB.dispose() + }) }) From 9eb49f61a9334429837325849d34979f95fdd368 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Wed, 15 Jul 2026 12:41:46 +0800 Subject: [PATCH 004/113] chore(knip): include session query loader e2e --- knip.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/knip.json b/knip.json index 825980f205..03fcd1b30d 100644 --- a/knip.json +++ b/knip.json @@ -45,6 +45,10 @@ "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], "project": ["src/**/*.ts", "tests/**/*.ts"] }, + "packages/session-query/session-query-sqlite": { + "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], + "project": ["src/**/*.ts", "tests/**/*.ts"] + }, "packages/code-runtime/code-runtime-worker": { "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], "project": ["src/**/*.ts", "tests/**/*.ts"] From f1426511be657e9ca45663f9724678ffe3a79ca4 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Wed, 15 Jul 2026 16:35:17 +0800 Subject: [PATCH 005/113] test(hooks): wait for SubagentStop marker output --- packages/hooks/hooks-claude/tests/coverage.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/hooks/hooks-claude/tests/coverage.spec.ts b/packages/hooks/hooks-claude/tests/coverage.spec.ts index f376708688..9632d075dd 100644 --- a/packages/hooks/hooks-claude/tests/coverage.spec.ts +++ b/packages/hooks/hooks-claude/tests/coverage.spec.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import { mkdtempSync, rmSync, writeFileSync, chmodSync, existsSync } from 'node:fs' +import { mkdtempSync, rmSync, writeFileSync, chmodSync, existsSync, readFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Context } from 'cordis' @@ -612,7 +612,6 @@ describe('hooks-claude coverage — hook runs in the session cwd, not the server await waitForIdle(ctx, handle.agent as ReactLoopAgent) expect(existsSync(marker)).toBe(true) // the marker landed in the SESSION dir - const { readFileSync } = await import('node:fs') const where = readFileSync(marker, 'utf8').trim() // `pwd` may resolve symlinks (/var → /private/var etc.), so compare basenames. expect(where.endsWith(sessionDir.split('/').pop()!)).toBe(true) @@ -643,9 +642,10 @@ describe('hooks-claude coverage — hook runs in the session cwd, not the server const childHandle = await ctx.agents.create({ agentId: AgentId('child-stop'), sessionId: SessionId('child-stop-session'), meta: { cwd: childDir }, agentOptions: { model: 'mock' } }) ctx.emit('subagent/end', { provider: 'inproc', id: childHandle.agent.id, stopReason: 'completed' }) - await waitFor(() => existsSync(marker)) + // Redirection creates the marker before `pwd` writes it, so wait for the + // trailing newline that marks the command's complete output. + await waitFor(() => existsSync(marker) && readFileSync(marker, 'utf8').endsWith('\n')) expect(existsSync(marker)).toBe(true) // the marker landed in the CHILD dir - const { readFileSync } = await import('node:fs') const where = readFileSync(marker, 'utf8').trim() // `pwd` may resolve symlinks (/var → /private/var etc.), so compare basenames. expect(where.endsWith(childDir.split('/').pop()!)).toBe(true) From 4505c6c55a515b5eed9a22a7ae112de5de413ba6 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Thu, 16 Jul 2026 13:55:23 +0800 Subject: [PATCH 006/113] refactor(session-query): simplify persistence binding (round 1) --- .../session-query-sqlite/src/index.ts | 58 +++++++++---------- .../session-query-sqlite/tests/sqlite.spec.ts | 29 +++++++--- 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/packages/session-query/session-query-sqlite/src/index.ts b/packages/session-query/session-query-sqlite/src/index.ts index dd1ddb532a..4a510e5324 100644 --- a/packages/session-query/session-query-sqlite/src/index.ts +++ b/packages/session-query/session-query-sqlite/src/index.ts @@ -97,9 +97,12 @@ interface ObservedPersistedSession { loaded?: ObservedSession } +interface PersistenceBinding { + readonly service?: SessionPersistence +} + interface Observation { - persistence: SessionPersistence | undefined - persistenceRevision: number + persistenceBinding: PersistenceBinding persisted: Map live: Map } @@ -161,10 +164,8 @@ export class SessionSearchSqlite extends SessionSearchService { private readonly _instance = randomUUID() private readonly _ready: Promise private _db: DatabaseSync | undefined - private _persistence: SessionPersistence | undefined - private _persistenceBinding: object | undefined - private _persistenceRevision = 0 - private _lastPersistenceRevision: number | undefined + private _persistenceBinding: PersistenceBinding = {} + private _lastPersistenceBinding: PersistenceBinding | undefined private _persistenceEpoch = 0 private _globalGeneration = 0 private _localGeneration = 0 @@ -182,16 +183,12 @@ export class SessionSearchSqlite extends SessionSearchService { void this._ready.catch(() => undefined) this._optionalPersistenceFiber = ctx.inject(['sessionPersistence'], (childCtx: Context) => { const service = childCtx.sessionPersistence - const binding = {} + const binding = { service } this._persistenceBinding = binding - this._persistence = service - this._persistenceRevision += 1 childCtx.effect(() => () => { /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */ if (this._persistenceBinding !== binding) return - this._persistenceBinding = undefined - this._persistence = undefined - this._persistenceRevision += 1 + this._persistenceBinding = {} }, 'sessionSearchSqlite.persistenceBinding') }) ctx.effect(() => { @@ -330,16 +327,16 @@ export class SessionSearchSqlite extends SessionSearchService { const liveById = new Map(liveRows.map(row => [row.id as SessionId, row])) const observation = await this._observeStable(persistedById, signal) assertNotAborted(signal) - const persistentChanges = observation.persistence === undefined + const persistentChanges = observation.persistenceBinding.service === undefined ? [] : [...observation.persisted.values()].filter(entry => entry.loaded !== undefined) - const persistentDeletes = observation.persistence === undefined + const persistentDeletes = observation.persistenceBinding.service === undefined ? [] : persistedRows.filter(row => !observation.persisted.has(row.id as SessionId)) const liveChanges = [...observation.live.values()].filter(entry => liveById.get(entry.header.id)?.fingerprint !== entry.fingerprint) const liveDeletes = liveRows.filter(row => !observation.live.has(row.id as SessionId)) - const pointerChanged = this._lastPersistenceRevision !== undefined - && this._lastPersistenceRevision !== observation.persistenceRevision + const pointerChanged = this._lastPersistenceBinding !== undefined + && this._lastPersistenceBinding !== observation.persistenceBinding const hasWrites = persistentChanges.length > 0 || persistentDeletes.length > 0 || liveChanges.length > 0 @@ -393,7 +390,7 @@ export class SessionSearchSqlite extends SessionSearchService { if (hasWrites || pointerChanged) this._globalGeneration += 1 if (pointerChanged) this._persistenceEpoch += 1 this._localGeneration = nextLocalGeneration - this._lastPersistenceRevision = observation.persistenceRevision + this._lastPersistenceBinding = observation.persistenceBinding } private async _observeStable( @@ -402,13 +399,13 @@ export class SessionSearchSqlite extends SessionSearchService { ): Promise { for (;;) { assertNotAborted(signal) - const persistence = this._persistence - const persistenceRevision = this._persistenceRevision + const persistenceBinding = this._persistenceBinding + const persistence = persistenceBinding.service let persisted = new Map() if (persistence !== undefined) { try { - const canReuseIndexed = this._lastPersistenceRevision === undefined - || this._lastPersistenceRevision === persistenceRevision + const canReuseIndexed = this._lastPersistenceBinding === undefined + || this._lastPersistenceBinding === persistenceBinding const before = await waitWithAbort(persistence.listSnapshots(), signal) persisted = materializePersistenceSnapshots(before) for (const entry of persisted.values()) { @@ -421,14 +418,14 @@ export class SessionSearchSqlite extends SessionSearchService { await waitWithAbort(persistence.listSnapshots(), signal), ) if (!samePersistenceSnapshots(persisted, after)) continue - if (this._persistenceRevision !== persistenceRevision) continue + if (this._persistenceBinding !== persistenceBinding) continue } catch (error: unknown) { if (isAbort(error) || signal?.aborted) { throw new SessionQueryError('session-search aborted', 'SESSION_QUERY_ABORTED', { cause: error, }) } - if (this._persistenceRevision !== persistenceRevision) continue + if (this._persistenceBinding !== persistenceBinding) continue if (error instanceof SessionQueryError) throw error throw new SessionQueryError( `session-search persistence observation failed: ${errorMessage(error)}`, @@ -444,8 +441,8 @@ export class SessionSearchSqlite extends SessionSearchService { if (durable !== undefined) assertSessionHeadersCompatible(observed.header, durable.header) live.set(session.id, observed) } - if (this._persistenceRevision === persistenceRevision) { - return { persistence, persistenceRevision, persisted, live } + if (this._persistenceBinding === persistenceBinding) { + return { persistenceBinding, persisted, live } } } } @@ -564,7 +561,7 @@ export class SessionSearchSqlite extends SessionSearchService { ORDER BY match_count DESC, document_length ASC, time DESC, session_id ASC, seq DESC LIMIT ? OFFSET ? `).all( - ...selectedDocumentsParams(request.query, this._persistence !== undefined), + ...selectedDocumentsParams(request.query, this._persistenceBinding.service !== undefined), ...sessionWhere.params, ...eventWhere.params, request.limit + 1, @@ -583,7 +580,7 @@ export class SessionSearchSqlite extends SessionSearchService { ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC LIMIT ? OFFSET ? `).all( - ...selectedDocumentsParams(request.query, this._persistence !== undefined), + ...selectedDocumentsParams(request.query, this._persistenceBinding.service !== undefined), request.sessionId, ...eventWhere.params, request.limit + 1, @@ -597,7 +594,7 @@ export class SessionSearchSqlite extends SessionSearchService { 'SELECT generation FROM temp.live_sessions WHERE id = ?', ).get(sessionId) as { generation: number } | undefined if (live !== undefined) return `live:${live.generation}` - if (this._persistence !== undefined) { + if (this._persistenceBinding.service !== undefined) { const persisted = db.prepare( 'SELECT generation FROM persisted_sessions WHERE id = ?', ).get(sessionId) as { generation: number } | undefined @@ -713,10 +710,7 @@ function selectedDocumentsParams(query: string, persistenceVisible: boolean): Ar } function observeLive(session: Session): ObservedSession { - return observeSession( - structuredClone(session.header), - session.events.map(event => structuredClone(event)), - ) + return observeSession(session.header, session.events) } function observeSession(header: SessionHeader, events: readonly SessionEvent[]): ObservedSession { diff --git a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts index 9897331549..3de325b35f 100644 --- a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts @@ -326,6 +326,24 @@ describe('SQLite session search', () => { })).rejects.toThrow(expectCode('SESSION_QUERY_STALE_CURSOR')) }) + it('invalidates session cursors after transient persistence topology changes', async () => { + TestPersistence.reset() + const ctx = await liveContext({ path: ':memory:', defaultLimit: 1, maxLimit: 5 }) + ctx.sessions.create(SessionId('first'), { seed: messageEvents('needle first') }) + ctx.sessions.create(SessionId('second'), { seed: messageEvents('needle second') }) + const page = await ctx.sessionSearch.searchSessions({ query: 'needle', limit: 1 }) + if (page.nextCursor === undefined) throw new Error('expected cursor') + + const persistence = await ctx.plugin(TestPersistence) + await persistence.dispose() + + await expect(ctx.sessionSearch.searchSessions({ + query: 'needle', + limit: 1, + cursor: page.nextCursor, + })).rejects.toThrow(expectCode('SESSION_QUERY_STALE_CURSOR')) + }) + it('rejects invalid requests, filters, cursors, and direct config', async () => { const ctx = await liveContext({ path: ':memory:', defaultLimit: 2, maxLimit: 3 }) const session = ctx.sessions.create(SessionId('valid'), { seed: messageEvents('needle') }) @@ -503,11 +521,6 @@ describe('SQLite reconciliation and source lifecycle', () => { TestPersistence.set({ meta: durable, events: messageEvents('new needle') }) TestPersistence.revisions.set(durable.id, revision) const replacement = await ctx.plugin(TestPersistence) - const internals = ctx.sessionSearch as unknown as { - _lastPersistenceRevision: number - _persistenceRevision: number - } - expect(internals._persistenceRevision).not.toBe(internals._lastPersistenceRevision) const page = await ctx.sessionSearch.searchSessions({ query: 'new needle' }) expect(TestPersistence.loads.get(durable.id)).toBe(2) expect(page).toMatchObject({ items: [{ header: durable }] }) @@ -553,13 +566,15 @@ describe('SQLite reconciliation and source lifecycle', () => { TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) const ctx = await liveContext() await ctx.plugin(TestPersistence) - const internals = ctx.sessionSearch as unknown as { _persistenceRevision: number } + const internals = ctx.sessionSearch as unknown as { + _persistenceBinding: { service?: SessionPersistence } + } const originalList = ctx.sessions.list.bind(ctx.sessions) let bumped = false const list = vi.spyOn(ctx.sessions, 'list').mockImplementation(() => { if (!bumped) { bumped = true - internals._persistenceRevision += 1 + internals._persistenceBinding = { ...internals._persistenceBinding } } return originalList() }) From de863aab9ab674a1528f575695fd62d1b151966a Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Thu, 16 Jul 2026 14:04:49 +0800 Subject: [PATCH 007/113] fix(session-query): release stale persistence binding (round 2) --- .../session-query-sqlite/src/index.ts | 19 ++++++++++--------- .../session-query-sqlite/tests/sqlite.spec.ts | 7 +++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/session-query/session-query-sqlite/src/index.ts b/packages/session-query/session-query-sqlite/src/index.ts index 4a510e5324..1590a1d61c 100644 --- a/packages/session-query/session-query-sqlite/src/index.ts +++ b/packages/session-query/session-query-sqlite/src/index.ts @@ -98,6 +98,7 @@ interface ObservedPersistedSession { } interface PersistenceBinding { + readonly identity: symbol readonly service?: SessionPersistence } @@ -164,8 +165,8 @@ export class SessionSearchSqlite extends SessionSearchService { private readonly _instance = randomUUID() private readonly _ready: Promise private _db: DatabaseSync | undefined - private _persistenceBinding: PersistenceBinding = {} - private _lastPersistenceBinding: PersistenceBinding | undefined + private _persistenceBinding: PersistenceBinding = { identity: Symbol() } + private _lastPersistenceIdentity: symbol | undefined private _persistenceEpoch = 0 private _globalGeneration = 0 private _localGeneration = 0 @@ -183,12 +184,12 @@ export class SessionSearchSqlite extends SessionSearchService { void this._ready.catch(() => undefined) this._optionalPersistenceFiber = ctx.inject(['sessionPersistence'], (childCtx: Context) => { const service = childCtx.sessionPersistence - const binding = { service } + const binding = { identity: Symbol(), service } this._persistenceBinding = binding childCtx.effect(() => () => { /* v8 ignore next -- a stale optional-service disposer cannot clear a replacement */ if (this._persistenceBinding !== binding) return - this._persistenceBinding = {} + this._persistenceBinding = { identity: Symbol() } }, 'sessionSearchSqlite.persistenceBinding') }) ctx.effect(() => { @@ -335,8 +336,8 @@ export class SessionSearchSqlite extends SessionSearchService { : persistedRows.filter(row => !observation.persisted.has(row.id as SessionId)) const liveChanges = [...observation.live.values()].filter(entry => liveById.get(entry.header.id)?.fingerprint !== entry.fingerprint) const liveDeletes = liveRows.filter(row => !observation.live.has(row.id as SessionId)) - const pointerChanged = this._lastPersistenceBinding !== undefined - && this._lastPersistenceBinding !== observation.persistenceBinding + const pointerChanged = this._lastPersistenceIdentity !== undefined + && this._lastPersistenceIdentity !== observation.persistenceBinding.identity const hasWrites = persistentChanges.length > 0 || persistentDeletes.length > 0 || liveChanges.length > 0 @@ -390,7 +391,7 @@ export class SessionSearchSqlite extends SessionSearchService { if (hasWrites || pointerChanged) this._globalGeneration += 1 if (pointerChanged) this._persistenceEpoch += 1 this._localGeneration = nextLocalGeneration - this._lastPersistenceBinding = observation.persistenceBinding + this._lastPersistenceIdentity = observation.persistenceBinding.identity } private async _observeStable( @@ -404,8 +405,8 @@ export class SessionSearchSqlite extends SessionSearchService { let persisted = new Map() if (persistence !== undefined) { try { - const canReuseIndexed = this._lastPersistenceBinding === undefined - || this._lastPersistenceBinding === persistenceBinding + const canReuseIndexed = this._lastPersistenceIdentity === undefined + || this._lastPersistenceIdentity === persistenceBinding.identity const before = await waitWithAbort(persistence.listSnapshots(), signal) persisted = materializePersistenceSnapshots(before) for (const entry of persisted.values()) { diff --git a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts index 3de325b35f..0a62fff1b3 100644 --- a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts @@ -567,14 +567,17 @@ describe('SQLite reconciliation and source lifecycle', () => { const ctx = await liveContext() await ctx.plugin(TestPersistence) const internals = ctx.sessionSearch as unknown as { - _persistenceBinding: { service?: SessionPersistence } + _persistenceBinding: { identity: symbol; service?: SessionPersistence } } const originalList = ctx.sessions.list.bind(ctx.sessions) let bumped = false const list = vi.spyOn(ctx.sessions, 'list').mockImplementation(() => { if (!bumped) { bumped = true - internals._persistenceBinding = { ...internals._persistenceBinding } + internals._persistenceBinding = { + ...internals._persistenceBinding, + identity: Symbol(), + } } return originalList() }) From 92fd92fa697639ece53fcbc8175daf8589b263e4 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Thu, 16 Jul 2026 14:12:50 +0800 Subject: [PATCH 008/113] test(session-query): name binding retry precisely (round 3) --- .../session-query/session-query-sqlite/tests/sqlite.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts index 0a62fff1b3..b4a69acbbd 100644 --- a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts @@ -561,7 +561,7 @@ describe('SQLite reconciliation and source lifecycle', () => { expect(TestPersistence.loads.get(added.id)).toBe(1) }) - it('retries if the source revision changes while live sessions are observed', async () => { + it('retries if the persistence binding changes while live sessions are observed', async () => { const durable = header('live-boundary-retry') TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) const ctx = await liveContext() From 60fcb494a763dec5ad8e431f922e016799cb529f Mon Sep 17 00:00:00 2001 From: ZiyaZhang <199893125+ZiyaZhang@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:19:25 -0700 Subject: [PATCH 009/113] docs(i18n): restore prompt-v4 as the pipeline baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按 jingtingxiang 拍板将 prompt 回到 v4 基线：模板正文恢复内嵌的 格式/语气/句式/词汇/标点全量约束与 11 组正误例（量词规则按术语表 现行裁定 package→包 写作「由三个包构成的 seam」），不再注入 translation-rules.md——该文件约束人和 agent，不进模板；占位符收敛 为 source_lang/target_lang/terminology 三个，切换行由模型按文档 自身拼写。渲染器、解析器、conformance 门禁与单测同步回 v4 契约： 三段裸 XML（translation/review/final 顺序唯一），容忍整体 ```xml 围栏回显；saxes 依赖随 CDATA 协议一并移除。 --- docs/i18n/translation-prompt.md | 168 +++++++++++++++++++-------- package.json | 1 - pnpm-lock.yaml | 22 ++-- scripts/translation-prompt.spec.ts | 87 +++++--------- scripts/translation-prompt.ts | 144 +++++++---------------- scripts/verify-translation-prompt.ts | 28 ++--- 6 files changed, 213 insertions(+), 237 deletions(-) diff --git a/docs/i18n/translation-prompt.md b/docs/i18n/translation-prompt.md index 8bc15d6e8e..d59170b00a 100644 --- a/docs/i18n/translation-prompt.md +++ b/docs/i18n/translation-prompt.md @@ -1,6 +1,6 @@ # Translation prompt (pipeline asset) -本文件是自动翻译流水线使用的 prompt 模板；从 `# Translation Prompt` 开始的正文会逐字进入模型请求，因此本文件不参与双语配对（见 [README.md](README.md) 排除清单）。渲染时会把 [translation-rules.md](translation-rules.md) 全文填入 `{{translation_rules}}`，把 [terminology.md](terminology.md) 整表填入 `{{terminology}}`，以免模板另存一份规则而日后失去同步。[style-samples.md](style-samples.md) 定义文体，模板中的 Examples 只用于说明典型问题；术语表、忠实性和结构规则优先于样例，样例只在这些硬性约束内决定文体。修改本文件会改变翻译行为，需正常经过 PR 评审。 +本文件是自动翻译流水线的 prompt 模板；从 `# Translation Prompt` 开始的正文会逐字进入模型请求，因此本文件不参与双语配对（见 [README.md](README.md) 排除清单）。模板正文与内嵌 few-shot 正误例由 jingtingxiang 基于对存量译文的质量评审撰写，是流水线行为的拍板基线。渲染时把 [terminology.md](terminology.md) 整表填入 `{{terminology}}`；除此之外不注入任何其他仓库文件（translation-rules.md 约束人和 agent 的翻译工作，不注入本模板）。[style-samples.md](style-samples.md) 定义文体，模板中的 Examples 只用于说明典型问题，两者冲突时以文体样例为准。修改本文件会改变翻译行为，需正常经过 PR 评审。 ## 占位符契约 @@ -10,18 +10,13 @@ |---|---|---| | `{{source_lang}}` | 源语言名（`English` / `Chinese`） | 由改动侧文件推断：`.zh.md` 被改则为 `Chinese` | | `{{target_lang}}` | 目标语言名（`Chinese` / `English`） | 与 `{{source_lang}}` 相对 | -| `{{translation_rules}}` | [translation-rules.md](translation-rules.md) 全文（Markdown 原文） | 渲染时读取仓库当前版本，不缓存 | | `{{terminology}}` | [terminology.md](terminology.md) 的完整表格（Markdown 原文） | 渲染时读取仓库当前版本，不缓存 | -| `{{source_filename}}` | 源文档的 basename（如 `foo.md` 或 `foo.zh.md`） | 由流水线从待译文件路径取得 | -| `{{source_filename_zh}}` | 中文侧 basename（如 `foo.zh.md`） | 英文源追加 `.zh`；中文源使用自身 basename | -例如，英译中时若源文件是 `foo.md`，`{{source_filename}}` 填 `foo.md`，`{{source_filename_zh}}` 填 `foo.zh.md`；中译英时若源文件是 `foo.zh.md`，两个占位符都填 `foo.zh.md`。 - -流水线只识别上表中的占位符，并且一次翻译整篇文档。它不支持 `{{to}}`、`{{title_prompt}}`、`{{summary_prompt}}`、`{{terms_prompt}}`、`{{imt_style_guide}}` 或 `%%` 分段协议。输出必须是一个以 `` 为根元素的 XML 文档；三个子元素中的 Markdown 内容都放在 CDATA 中。内容出现 `]]>` 时写成 `]]]]>`，XML 解析后仍会还原为原文。 +流水线只识别上表中的占位符，并且一次翻译整篇文档。它不支持 `{{to}}`、`{{title_prompt}}`、`{{summary_prompt}}`、`{{terms_prompt}}`、`{{imt_style_guide}}`、`{{translation_rules}}` 或 `%%` 分段协议；输出采用模板正文规定的三段 XML，流水线解析取 `` 段。 ## Few-shot 金标 -流水线使用**整篇文档**的中英对照作为 few-shot，不是模板内嵌的句子级正误例。以下 5 组配对文档均经过人工评审，并以仓库当前版本为准，随仓库一同更新： +流水线使用**整篇文档**的中英对照作为 few-shot，不是模板内嵌的句子级正误例。以下 5 组配对文档均经过人工评审，以仓库当前版本为准、随仓库更新： - `README.md` ↔ `README.zh.md` - `docs/development.md` ↔ `docs/development.zh.md` @@ -29,58 +24,131 @@ - `docs/i18n/translation-rules.md` ↔ `docs/i18n/translation-rules.zh.md` - `docs/rfc/implemented/process/2026-07-02-bilingual-docs-and-pairing-gate.md` ↔ 对应 `.zh.md` -注入时按当前翻译方向选择每组的源侧与目标侧：user 消息包含源文档全文，assistant 消息采用模板正文规定的 XML 协议；`translation` 与 `final` 都放入目标文档全文，`review` 填 `- [None] No corrections.`。CDATA 遵循上文的 `]]>` 拆分规则。上下文不足时，按上列顺序从后往前删减示例组数。这 5 组也是评审校准锚点；改动任何一组都会改变流水线行为。 +注入方式：在系统消息（本模板）之后、待译文档之前，每组作为一轮示例对话——user 消息为源文档全文，assistant 消息为定稿译文全文（裸文本，不带三段 XML 包装；只有真实请求要求三段输出）。上下文不足时按上列顺序从后往前删减组数。这 5 组也是评审校准锚点（见 [style-samples.md](style-samples.md)），改动任何一组即改变流水线行为。 ## 模板正文 ````text # Translation Prompt -You are a senior technical translator specializing in LLM and agent development documentation. Translate the complete source document from {{source_lang}} to {{target_lang}} as natural, professional technical prose. +You are a senior technical translator specializing in LLM and agent development documentation. Your task is to translate the given source document from {{source_lang}} to {{target_lang}}, producing natural, professional technical prose. -## Binding Translation Rules +## Quality Requirements -The canonical repository rules below are injected verbatim. Apply every direction-appropriate requirement. In those rules, the authored document is the source for this request and the generated document is its counterpart. +### Structure and Format Preservation +- Output a complete translated document that maintains exactly the same structure as the source: heading hierarchy, list shape, table columns, link targets, and code blocks. +- Fenced code blocks must be byte-identical to the source, including ALL comments inside them. Do NOT translate comments inside code blocks. This is a hard rule with no exceptions. +- Inline code spans (commands, flags, paths, API names, version numbers) must be kept verbatim. Never translate or reformat them. +- Every relative link must point to the same target as in the source. Link text is translated; link targets are not. +- Language switcher line: when translating into Chinese, write `[English](source-filename.md) | 中文`. When translating into English, write `English | [中文](source-filename.zh.md)`. Do NOT copy the switcher line from the source file unchanged — you must flip the link direction. +- After a closing bold marker `**`, always insert a space before the next character. -{{translation_rules}} +### Tone and Style +- The translation must read as if originally written in the target language by a native speaker. If an expression sounds like a word-for-word rendering from the source language, rephrase it. +- Write in a professional, formal tone appropriate for developer documentation. Never use colloquial or casual expressions. +- Use polite imperative forms where the text instructs the reader to do something. +- Keep the author's register: concise stays concise, detailed stays detailed. -## Request-Specific Structure +### Sentence Structure +- Break long sentences with commas or semicolons. Avoid run-on sentences. +- Prefer active voice. Convert passive constructions to active if it reads more naturally. +- Translate meaning, not words. Restructure sentences where the target language grammar requires it. +- Do not invent words or expressions that do not exist in natural technical writing of the target language. -- The source basename is `{{source_filename}}`. When translating into Chinese, write `[English]({{source_filename}}) | 中文` immediately after the H1. When translating into English, write `English | [中文]({{source_filename_zh}})` immediately after the H1. -- Emit the switcher for a new pair and flip an existing switcher; never copy it unchanged. +### Word Choice +- Prefer precise, formal vocabulary over casual or colloquial alternatives. +- When multiple synonyms exist, choose the one most commonly used in professional technical documentation of the target language. +- Avoid slang, internal jargon, or overly literal translations that would not be recognized by the general developer audience. +- Do not use the same word to translate two different source-language terms that carry distinct meanings. +- Avoid repeating the same verb in close proximity; vary word choice for readability. -## Binding Terminology +#### When translating into Chinese +- When a number modifies a noun, always include a Chinese classifier or measure word (量词). For example: "three-package seam" → "由三个包构成的 seam", not "三包 seam". -Apply the current table below exactly as required by the injected translation rules. +### Punctuation + +#### When translating into Chinese +- Use full-width Chinese punctuation in prose: `，。：；？！（）「」`. +- Strongly prefer replacing all em-dashes (——) with colons, periods, commas, or parentheses. Keep an em-dash only if no other punctuation works at all. +- Use enumeration commas (、) between parallel items, not regular commas. +- List item endings: use semicolons or no punctuation. Do not end list items with commas. +- Put one half-width space between Chinese text and Latin words/numbers. +- For RFC 2119 keywords (MUST, MUST NOT, SHOULD, MAY), render the corresponding Chinese term in italics: *必须*、*禁止*、*应当*、*可以*. + +#### When translating into English +(To be added.) + +## Terminology + +A terminology table is provided below. Follow it strictly: +- Render every listed term exactly as specified. +- First occurrence: write as shown in the "首次出现" column (with parenthetical gloss). Subsequent occurrences: write only the part before the parentheses. +- If a term has already been glossed as part of a compound term, do not gloss it again when it appears alone later. +- NEVER use translations listed in the "不要译作" column. +- For technical terms not in the table: keep them in the source language. Do not invent a translation. This rule applies to terminology only; for general prose, freely restructure and paraphrase for natural expression. {{terminology}} ## Output Format -Return exactly one well-formed XML document with this root and these three child elements. Do not wrap it in a Markdown code fence. Put all Markdown and review text inside CDATA. If any content contains the CDATA terminator, split it as `]]]]>` so XML parsing reconstructs the original `]]>` sequence. +Produce your output in three XML sections: ```xml - - - - - + +(Complete translation of the source document) + + + +(Self-review notes, one correction per line with category tag, e.g.) +- [Tone] "旁挂记录" → "伴随记录"（生造词） +- [Sentence] 第 3 段补充逗号断句 +- [Punctuation] 两处破折号替换为冒号 +- 无修正 + + + +(Final translation after corrections) + ``` ## Self-Review Instructions -After writing ``, re-read it in the target language without looking at the source. Then apply the injected translation rules as a clause-by-clause comparison against the source and record actual corrections in English inside ``. Apply every recorded correction in ``. If no correction is needed, write only `- [None] No corrections.` in `` and copy `` unchanged into ``. +After writing ``, re-read it in the target language only, without looking at the source. Check by category: + +**Structure** +- Is the heading hierarchy, list shape, and code block content identical to the source? +- Are ALL comments inside code blocks left untranslated (byte-identical to source)? +- Is the language switcher line correctly flipped (not copied from source)? +- Are link targets preserved and bold markers followed by a space? + +**Tone & Style** +- Does every sentence read as if originally written by a native speaker? +- Is there any colloquial, casual, or overly informal phrasing? + +**Sentence Structure** +- Are there run-on sentences that need breaking? +- Are there stiff passive constructions that should be converted to active voice? + +**Word Choice** +- Are there overly literal translations that sound unnatural? +- Is the same target-language word used to translate two distinct source concepts? +- Is any slang or internal jargon present? + +**Terminology** +- Are first-occurrence glosses correctly applied (not missing, not repeated)? +- Are any "不要译作" forbidden translations present? +- Are unlisted terms correctly kept in the source language? + +**Punctuation** (when target is Chinese) +- Are there em-dashes that should be replaced with colons, periods, or commas? +- Are list items ending with commas instead of semicolons? +- Are RFC 2119 keywords rendered in italics? + +Record corrections in `` with category tags. Then output the corrected version in ``. If no corrections are needed, write "无修正" in `` and copy the translation unchanged into ``. ## Examples -Follow the Good versions; these sentence-level examples illustrate error categories, not the assistant-message wire format. +Below are representative examples of common problems and their corrections. Follow the "Good" versions. ### Colloquial verb → Professional verb - Source: `The repo pins pnpm@11.7.0 in package.json` @@ -102,40 +170,40 @@ Follow the Good versions; these sentence-level examples illustrate error categor - Bad: `旁挂记录两侧 blob hash，使一致性可检查` - Good: `伴随记录保存两侧 blob hash，使一致性可检查` +### Em-dash → Colon/period +- Source: `FIXME — an issue that should block a new release. A release should not ship with an open FIXME unless reviewers explicitly agree the change can be merged anyway.` +- Bad: `FIXME——应当阻塞新版本发布的问题。除非评审者明确同意可以照常合入，发布不应带着未解决的 FIXME 出门。` +- Good: `FIXME：应当阻塞新版本发布的问题。除非评审者明确同意该更改可以合并，否则发布版本不应包含未解决的 FIXME。` + ### Overly literal → Meaningful rendering - Source: `awkward phrasing is easier to hear without the source anchoring you` - Bad: `没有源文锚着，别扭的表述更容易被听出来` - Good: `不对照原文时，更容易察觉别扭的表达` -### Terminology — keep the binding English form +### Terminology — do not translate what should be kept in English - Source: `typed service seams, and explicit extension points` - Bad: `类型化的服务 seam（扩展点）与显式扩展点` - Good: `类型化的服务 seam 与显式扩展点` -### Slang → Professional phrasing +### Slang/jargon → Professional phrasing - Source: `The committed agent workflow lives in .agents/skills/dsh-translate-docs` - Bad: `进仓的 agent 工作流见 .agents/skills/dsh-translate-docs` - Good: `仓库内置的 agent 工作流见 .agents/skills/dsh-translate-docs` -### Chinese → English — idiomatic subject and predicate -- Source: `门禁绿并不代表译文内容正确。` -- Bad: `The gate green does not represent that the translation content is correct.` -- Good: `A green gate does not mean the translation is correct.` +### "For humans" — translate the intent, not the word +- Source: `For humans, start with the development guide` +- Bad: `对于人工读者，请先从开发指南开始`（"人工读者"生硬） +- Good: `面向开发者：请先阅读开发指南`（"开发者"自然，且中文里冒号在此处更自然） -### Code block comments — never translate +### Code block comments — NEVER translate - Source code block contains: `# REPL agent demo (needs DEEPSEEK_API_KEY)` - Bad: `# REPL agent 演示（需要 DEEPSEEK_API_KEY）` -- Good: `# REPL agent demo (needs DEEPSEEK_API_KEY)` (byte-identical) +- Good: `# REPL agent demo (needs DEEPSEEK_API_KEY)` (keep exactly as-is, byte-for-byte) -### Language switcher — English to Chinese -- Source: `English | [中文](README.zh.md)` -- Bad: `English | [中文](README.zh.md)` -- Good: `[English](README.md) | 中文` - -### Language switcher — Chinese to English -- Source: `[English](README.md) | 中文` -- Bad: `[English](README.md) | 中文` -- Good: `English | [中文](README.zh.md)` +### Language switcher — flip direction +- Source file (English) has: `English | [中文](README.zh.md)` +- Bad (copying source unchanged): `English | [中文](README.zh.md)` +- Good (flipped for Chinese file): `[English](README.md) | 中文` --- diff --git a/package.json b/package.json index aaec1e62c4..51fd789a2b 100644 --- a/package.json +++ b/package.json @@ -100,7 +100,6 @@ "mermaid": "11.16.0", "micromark-extension-gfm": "^3.0.0", "publint": "^0.3.21", - "saxes": "^6.0.0", "tsdown": "^0.22.2", "tsx": "^4.22.4", "typescript": "^6.0.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b1a3a78030..0e71da2ea1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,9 +68,6 @@ importers: publint: specifier: ^0.3.21 version: 0.3.21 - saxes: - specifier: ^6.0.0 - version: 6.0.0 tsdown: specifier: ^0.22.2 version: 0.22.2(oxc-resolver@11.20.0)(publint@0.3.21)(tsx@4.22.4)(typescript@6.0.3) @@ -819,7 +816,7 @@ importers: dependencies: '@earendil-works/pi-ai': specifier: ^0.79.1 - version: 0.79.3(ws@8.21.0)(zod@4.4.3) + version: 0.79.3(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) schemastery: specifier: ^3.18.0 version: 3.18.0 @@ -3059,6 +3056,10 @@ packages: cpu: [x64] os: [win32] + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + '@protobufjs/aspromise@1.1.2': resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} @@ -6137,11 +6138,11 @@ snapshots: '@csstools/css-tokenizer@4.0.0': {} - '@earendil-works/pi-ai@0.79.3(ws@8.21.0)(zod@4.4.3)': + '@earendil-works/pi-ai@0.79.3(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) '@aws-sdk/client-bedrock-runtime': 3.1048.0 - '@google/genai': 1.52.0 + '@google/genai': 1.52.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3)) '@mistralai/mistralai': 2.2.1 '@smithy/node-http-handler': 4.7.3 http-proxy-agent: 7.0.2 @@ -6299,12 +6300,14 @@ snapshots: '@exodus/bytes@1.15.1': {} - '@google/genai@1.52.0': + '@google/genai@1.52.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))': dependencies: google-auth-library: 10.7.0 p-retry: 4.6.2 protobufjs: 7.6.4 ws: 8.21.0 + optionalDependencies: + '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) transitivePeerDependencies: - bufferutil - supports-color @@ -6563,6 +6566,9 @@ snapshots: '@oxc-resolver/binding-win32-x64-msvc@11.20.0': optional: true + '@pkgjs/parseargs@0.11.0': + optional: true + '@protobufjs/aspromise@1.1.2': {} '@protobufjs/base64@1.1.2': {} @@ -8034,6 +8040,8 @@ snapshots: jackspeak@3.4.3: dependencies: '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 jiti@2.7.0: {} diff --git a/scripts/translation-prompt.spec.ts b/scripts/translation-prompt.spec.ts index 93754a79a1..db3f8b31a0 100644 --- a/scripts/translation-prompt.spec.ts +++ b/scripts/translation-prompt.spec.ts @@ -1,5 +1,7 @@ -/** Regression tests for the executable translation prompt contract. */ +/** Unit tests for the prompt-v4 renderer and three-section response parser. */ +import { readFileSync } from 'node:fs' +import { join, resolve } from 'node:path' import { describe, expect, it } from 'vitest' import { parseTranslationResponse, @@ -7,70 +9,43 @@ import { renderTranslationResponse, } from './translation-prompt.ts' -const document = `# Wrapper - -## 模板正文 - -\`\`\`\`text -{{source_lang}} to {{target_lang}} -{{translation_rules}} -{{terminology}} -[English]({{source_filename}}) | [中文]({{source_filename_zh}}) -\`\`\`\` -` +const root = resolve(import.meta.dirname, '..') +const document = readFileSync(join(root, 'docs/i18n/translation-prompt.md'), 'utf8') +const terminology = '| English | 中文 |\n|---|---|\n| agent | agent |' describe('translation prompt rendering', () => { - it('renders every supported placeholder without recursively rewriting injected rules', () => { - const rendered = renderTranslationPrompt(document, { - sourceLanguage: 'English', - sourceFilename: 'guide.md', - translationRules: 'A literal {{source_lang}} in injected rules.', - terminology: '| English | 中文 |', - }) - expect(rendered).toContain('English to Chinese') - expect(rendered).toContain('A literal {{source_lang}} in injected rules.') - expect(rendered).toContain('[English](guide.md) | [中文](guide.zh.md)') + it('renders both directions with every placeholder resolved', () => { + const en = renderTranslationPrompt(document, { sourceLanguage: 'English', terminology }) + expect(en).toContain('from English to Chinese') + expect(en).toContain(terminology) + expect(en).not.toContain('{{') + const zh = renderTranslationPrompt(document, { sourceLanguage: 'Chinese', terminology }) + expect(zh).toContain('from Chinese to English') }) - it('rejects a filename whose suffix contradicts the source language', () => { - expect(() => renderTranslationPrompt(document, { - sourceLanguage: 'Chinese', - sourceFilename: 'guide.md', - translationRules: 'rules', - terminology: 'terms', - })).toThrow('does not match source language Chinese') - }) - - it('rejects malformed template placeholders before injecting rule contents', () => { - expect(() => renderTranslationPrompt(document.replace('{{source_lang}}', '{{source-lang}}'), { - sourceLanguage: 'English', - sourceFilename: 'guide.md', - translationRules: 'A literal {{source_lang}} in injected rules.', - terminology: '| English | 中文 |', - })).toThrow('template contains malformed placeholder syntax') + it('rejects a template with unknown or missing placeholders', () => { + const alien = document.replaceAll('{{terminology}}', '{{terms_prompt}}') + expect(() => renderTranslationPrompt(alien, { sourceLanguage: 'English', terminology })).toThrow(/unsupported placeholder/) + const missing = document.replaceAll('{{terminology}}', '') + expect(() => renderTranslationPrompt(missing, { sourceLanguage: 'English', terminology })).toThrow(/required placeholder/) }) }) -describe('translation response XML', () => { - it('round-trips Markdown and the CDATA terminator', () => { - const response = { - translation: '# Draft\n\nA ]]> marker.', - review: '- [Tone] Fixed.', - final: '# Final\n\nA ]]> marker.', - } +describe('translation response sections', () => { + it('round-trips Markdown bodies', () => { + const response = { translation: '# 标题\n\n正文 **加粗**。', review: '- [Tone] 修正一处。\n- 无修正', final: '# 标题\n\n定稿。' } expect(parseTranslationResponse(renderTranslationResponse(response))).toEqual(response) }) - it('rejects missing, reordered, nested, attributed, or non-CDATA children', () => { - expect(() => parseTranslationResponse('')).toThrow('translation, review, and final') - expect(() => parseTranslationResponse('')) - .toThrow('expected translation, got review') - expect(() => parseTranslationResponse(renderTranslationResponse({ translation: 'x', review: 'y', final: 'z' }) - .replace('', ''))) - .toThrow('nested element b is not allowed') - expect(() => parseTranslationResponse(renderTranslationResponse({ translation: 'x', review: 'y', final: 'z' }).replace('', ''))) - .toThrow('review must not have attributes') - expect(() => parseTranslationResponse(renderTranslationResponse({ translation: 'x', review: 'y', final: 'z' }).replace('', 'x'))) - .toThrow('all response field content must be inside CDATA') + it('tolerates a fenced xml wrapper around the whole response', () => { + const fenced = '```xml\n\nA\n\n\n\n- 无修正\n\n\n\nA\n\n```' + expect(parseTranslationResponse(fenced).final).toBe('A') + }) + + it('rejects missing, unterminated, or duplicated sections', () => { + expect(() => parseTranslationResponse('\nA\n')).toThrow(/missing /) + expect(() => parseTranslationResponse('\nA')).toThrow(/unterminated /) + const dup = '\nA\n\n\nR\n\n\nF\n\n\nG\n' + expect(() => parseTranslationResponse(dup)).toThrow(/duplicate /) }) }) diff --git a/scripts/translation-prompt.ts b/scripts/translation-prompt.ts index e30c962498..0556ff39fa 100644 --- a/scripts/translation-prompt.ts +++ b/scripts/translation-prompt.ts @@ -1,20 +1,16 @@ /** - * Executable renderer and strict response parser for the committed - * documentation-translation prompt contract. + * Executable renderer and response parser for the committed + * documentation-translation prompt contract (prompt-v4). + * + * The v4 contract: three placeholders (`source_lang`, `target_lang`, + * `terminology`), whole-document translation, and a three-section response + * (``, ``, `` in order, bare XML tags with raw + * Markdown bodies). The switcher filename is spelled out by the model from + * the document itself; the pipeline injects no other repository file. */ -import { basename } from 'node:path' -import { SaxesParser } from 'saxes' - /** Placeholder names supported by the committed translation prompt. */ -export const TRANSLATION_PROMPT_PLACEHOLDERS = [ - 'source_lang', - 'target_lang', - 'translation_rules', - 'terminology', - 'source_filename', - 'source_filename_zh', -] as const +export const TRANSLATION_PROMPT_PLACEHOLDERS = ['source_lang', 'target_lang', 'terminology'] as const type TranslationPromptPlaceholder = (typeof TRANSLATION_PROMPT_PLACEHOLDERS)[number] @@ -24,15 +20,11 @@ type TranslationLanguage = 'English' | 'Chinese' /** Inputs that vary for one rendered translation request. */ export interface TranslationPromptInput { sourceLanguage: TranslationLanguage - /** Source basename, including `.md` or `.zh.md`. */ - sourceFilename: string - /** Complete current `translation-rules.md` contents. */ - translationRules: string /** Complete current `terminology.md` contents. */ terminology: string } -/** Parsed contents of the three-element XML response. */ +/** Parsed contents of the three-section response. */ export interface TranslationResponse { translation: string review: string @@ -42,7 +34,7 @@ export interface TranslationResponse { const PLACEHOLDER = /{{([a-z_]+)}}/g const TEMPLATE_OPEN = '## 模板正文\n\n````text\n' const TEMPLATE_CLOSE = '\n````' -const RESPONSE_CHILDREN = ['translation', 'review', 'final'] as const +const RESPONSE_SECTIONS = ['translation', 'review', 'final'] as const /** Extract the machine-consumed text fence from `translation-prompt.md`. */ function extractTranslationPrompt(document: string): string { @@ -61,31 +53,15 @@ export function documentedTranslationPromptPlaceholders(document: string): strin return [...document.slice(0, preambleEnd).matchAll(/^\| `{{([a-z_]+)}}` \|/gm)].map(match => match[1] ?? '') } -/** Render one system prompt from the checked-in template and canonical rules. */ +/** Render one system prompt from the checked-in template. */ export function renderTranslationPrompt(document: string, input: TranslationPromptInput): string { - if (basename(input.sourceFilename) !== input.sourceFilename) { - throw new Error(`translation prompt: sourceFilename must be a basename; got ${JSON.stringify(input.sourceFilename)}`) - } - const sourceIsChinese = input.sourceFilename.endsWith('.zh.md') - if (input.sourceLanguage === 'Chinese' ? !sourceIsChinese : sourceIsChinese || !input.sourceFilename.endsWith('.md')) { - throw new Error(`translation prompt: ${input.sourceFilename} does not match source language ${input.sourceLanguage}`) - } - const targetLanguage: TranslationLanguage = input.sourceLanguage === 'English' ? 'Chinese' : 'English' - const sourceFilenameZh = sourceIsChinese ? input.sourceFilename : input.sourceFilename.replace(/\.md$/, '.zh.md') const values: Record = { source_lang: input.sourceLanguage, target_lang: targetLanguage, - translation_rules: input.translationRules, terminology: input.terminology, - source_filename: input.sourceFilename, - source_filename_zh: sourceFilenameZh, } const template = extractTranslationPrompt(document) - const placeholderFreeTemplate = template.replace(PLACEHOLDER, '') - if (placeholderFreeTemplate.includes('{{') || placeholderFreeTemplate.includes('}}')) { - throw new Error('translation prompt: template contains malformed placeholder syntax') - } const names = [...template.matchAll(PLACEHOLDER)].map(match => match[1] ?? '') const unknown = names.filter(name => !TRANSLATION_PROMPT_PLACEHOLDERS.includes(name as TranslationPromptPlaceholder)) if (unknown.length > 0) throw new Error(`translation prompt: unsupported placeholder(s): ${[...new Set(unknown)].join(', ')}`) @@ -95,77 +71,37 @@ export function renderTranslationPrompt(document: string, input: TranslationProm return template.replace(PLACEHOLDER, (_token, name: string) => values[name as TranslationPromptPlaceholder]) } -/** Escape one value so it remains byte-identical inside an XML CDATA field. */ -function escapeTranslationCdata(value: string): string { - return value.replaceAll(']]>', ']]]]>') -} - -/** Serialize a response using the exact XML wire contract in the prompt. */ +/** Serialize a response in the exact three-section shape the prompt requests. */ export function renderTranslationResponse(response: TranslationResponse): string { - return [ - '', - ``, - ``, - ``, - '', - ].join('\n') + return RESPONSE_SECTIONS.map(section => `<${section}>\n${response[section]}\n`).join('\n\n') } -/** Parse and validate the exact XML response shape emitted by the model. */ -export function parseTranslationResponse(xml: string): TranslationResponse { - const values: TranslationResponse = { translation: '', review: '', final: '' } - const stack: string[] = [] - const cdataFields = new Set() - let rootSeen = false - let childIndex = 0 - const fail = (message: string): never => { - throw new Error(`translation response: ${message}`) - } - const parser = new SaxesParser({ xmlns: false }) +/** + * Parse the three-section response. Sections must each appear exactly once + * and in order; bodies are raw Markdown taken verbatim between the tags. + * A fenced ```xml wrapper around the whole response is tolerated, matching + * the shape some models echo back from the prompt's own example. + */ +export function parseTranslationResponse(text: string): TranslationResponse { + let body = text.trim() + const fenced = /^```(?:xml)?\n([\s\S]*?)\n```$/.exec(body) + if (fenced?.[1] !== undefined) body = fenced[1].trim() - parser.on('opentag', (tag) => { - if (stack.length === 0) { - if (rootSeen) fail('contains more than one root element') - if (tag.name !== 'dsh-translation-response') fail(`expected dsh-translation-response root, got ${tag.name}`) - const attributes = Object.keys(tag.attributes) - if (attributes.length !== 1 || tag.attributes.version !== '1') fail('root must have only version="1"') - rootSeen = true - } else if (stack.length === 1) { - const expected = RESPONSE_CHILDREN[childIndex] - if (tag.name !== expected) fail(`expected ${expected ?? 'no more children'}, got ${tag.name}`) - if (Object.keys(tag.attributes).length !== 0) fail(`${tag.name} must not have attributes`) - childIndex++ - } else { - fail(`nested element ${tag.name} is not allowed`) - } - stack.push(tag.name) - }) - parser.on('text', (value) => { - if (stack.length <= 1 && value.trim() === '') return - fail('all response field content must be inside CDATA') - }) - parser.on('cdata', (value) => { - const field = stack.at(-1) - if (field === undefined || !RESPONSE_CHILDREN.includes(field as (typeof RESPONSE_CHILDREN)[number])) { - fail('CDATA is allowed only inside translation, review, or final') - } - const key = field as (typeof RESPONSE_CHILDREN)[number] - values[key] += value - cdataFields.add(key) - }) - parser.on('closetag', (tag) => { - const expected = stack.pop() - if (expected !== tag.name) fail(`closing ${tag.name} does not match ${expected ?? 'nothing'}`) - }) - parser.on('comment', () => fail('comments are not allowed')) - parser.on('doctype', () => fail('doctypes are not allowed')) - parser.on('processinginstruction', () => fail('processing instructions are not allowed')) - parser.on('error', error => fail(`invalid XML: ${error.message}`)) - parser.write(xml).close() - - if (childIndex !== RESPONSE_CHILDREN.length) fail('translation, review, and final must each appear exactly once and in order') - for (const field of RESPONSE_CHILDREN) { - if (!cdataFields.has(field)) fail(`${field} must contain a CDATA section`) + const values: Partial> = {} + let cursor = 0 + for (const section of RESPONSE_SECTIONS) { + const open = `<${section}>` + const close = `` + const start = body.indexOf(open, cursor) + if (start === -1) throw new Error(`translation response: missing <${section}> section`) + const end = body.indexOf(close, start + open.length) + if (end === -1) throw new Error(`translation response: unterminated <${section}> section`) + values[section] = body.slice(start + open.length, end).replace(/^\n/, '').replace(/\n$/, '') + cursor = end + close.length } - return values + for (const section of RESPONSE_SECTIONS) { + const again = body.indexOf(`<${section}>`, cursor) + if (again !== -1) throw new Error(`translation response: duplicate <${section}> section`) + } + return values as TranslationResponse } diff --git a/scripts/verify-translation-prompt.ts b/scripts/verify-translation-prompt.ts index 66d83e47ad..df72ac73ce 100644 --- a/scripts/verify-translation-prompt.ts +++ b/scripts/verify-translation-prompt.ts @@ -18,37 +18,27 @@ function read(path: string): string { try { const document = read('docs/i18n/translation-prompt.md') - const translationRules = read('docs/i18n/translation-rules.md') const terminology = read('docs/i18n/terminology.md') const documented = documentedTranslationPromptPlaceholders(document) if (documented.join('\n') !== TRANSLATION_PROMPT_PLACEHOLDERS.join('\n')) { throw new Error(`placeholder table must list exactly: ${TRANSLATION_PROMPT_PLACEHOLDERS.join(', ')}`) } - const englishSource = renderTranslationPrompt(document, { - sourceLanguage: 'English', - sourceFilename: 'example.md', - translationRules, - terminology, - }) - const chineseSource = renderTranslationPrompt(document, { - sourceLanguage: 'Chinese', - sourceFilename: 'example.zh.md', - translationRules, - terminology, - }) - if (!englishSource.includes('[English](example.md) | 中文')) throw new Error('English-source render does not carry the Chinese switcher instruction') - if (!chineseSource.includes('English | [中文](example.zh.md)')) throw new Error('Chinese-source render does not carry the English switcher instruction') + const englishSource = renderTranslationPrompt(document, { sourceLanguage: 'English', terminology }) + const chineseSource = renderTranslationPrompt(document, { sourceLanguage: 'Chinese', terminology }) + if (englishSource.includes('{{') || chineseSource.includes('{{')) throw new Error('rendered prompt contains an unresolved placeholder') + if (!englishSource.includes('from English to Chinese')) throw new Error('English-source render does not translate into Chinese') + if (!chineseSource.includes('from Chinese to English')) throw new Error('Chinese-source render does not translate into English') const example = /```xml\n([\s\S]*?)\n```/.exec(englishSource)?.[1] - if (example === undefined) throw new Error('rendered prompt has no XML response example') + if (example === undefined) throw new Error('rendered prompt has no three-section response example') parseTranslationResponse(example) - const roundTrip = { translation: 'first ]]> pass', review: '- [None] No corrections.', final: 'final ]]> text' } + const roundTrip = { translation: 'first pass\n\nwith **markdown**', review: '- 无修正', final: 'final text' } const parsed = parseTranslationResponse(renderTranslationResponse(roundTrip)) - if (JSON.stringify(parsed) !== JSON.stringify(roundTrip)) throw new Error('CDATA split rule does not round-trip response content') + if (JSON.stringify(parsed) !== JSON.stringify(roundTrip)) throw new Error('three-section response does not round-trip') - console.log('verify-translation-prompt: both directions render and the XML response contract parses.') + console.log('verify-translation-prompt: both directions render and the three-section response contract parses.') } catch (error) { const message = error instanceof Error ? error.message : String(error) console.error(`verify-translation-prompt: ${message}`) From 75e9958f11e941ef33753eba037915f6d92be6ec Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Fri, 17 Jul 2026 09:39:39 +0800 Subject: [PATCH 010/113] fix(session-query): close review edge cases (round 4) --- docs/config-catalog.md | 8 +-- .../session-persistence-jsonl/src/index.ts | 26 ++++---- .../tests/jsonl.spec.ts | 33 ++++++++++ .../session-persistence-sqlite/README.md | 4 +- .../session-persistence-sqlite/src/index.ts | 11 +++- .../session-persistence-sqlite/src/schema.ts | 5 +- .../tests/sqlite.spec.ts | 25 +++++++- .../session-query-sqlite/README.md | 4 +- .../session-query-sqlite/src/index.ts | 62 +++++++++++++------ .../session-query-sqlite/src/query.ts | 25 +++++--- .../session-query-sqlite/tests/query.spec.ts | 10 ++- .../session-query-sqlite/tests/sqlite.spec.ts | 51 +++++++++++++++ 12 files changed, 213 insertions(+), 51 deletions(-) diff --git a/docs/config-catalog.md b/docs/config-catalog.md index c4d5ecc523..6d141185ca 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -627,7 +627,7 @@ export interface Config { export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' ``` -Source: [`packages/session-persistence/session-persistence-sqlite/src/index.ts:39`](../packages/session-persistence/session-persistence-sqlite/src/index.ts) +Source: [`packages/session-persistence/session-persistence-sqlite/src/index.ts:40`](../packages/session-persistence/session-persistence-sqlite/src/index.ts) ## `@deepseek-ai/dsh-session-query` @@ -654,9 +654,9 @@ export interface Config { path: string /** SQLite journal mode. Defaults to `wal`. */ journalMode?: JournalMode - /** Page size when a request omits `limit`. Defaults to 20. */ + /** Page size when a request omits `limit`. At most `Number.MAX_SAFE_INTEGER - 1`; defaults to 20. */ defaultLimit?: number - /** Largest accepted page size. Defaults to 100. */ + /** Largest accepted page size. At most `Number.MAX_SAFE_INTEGER - 1`; defaults to 100. */ maxLimit?: number /** Maximum snippet length in Unicode code points. Defaults to 240. */ snippetChars?: number @@ -666,7 +666,7 @@ export interface Config { export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' ``` -Source: [`packages/session-query/session-query-sqlite/src/index.ts:67`](../packages/session-query/session-query-sqlite/src/index.ts) +Source: [`packages/session-query/session-query-sqlite/src/index.ts:72`](../packages/session-query/session-query-sqlite/src/index.ts) ## `@deepseek-ai/dsh-skill` diff --git a/packages/session-persistence/session-persistence-jsonl/src/index.ts b/packages/session-persistence/session-persistence-jsonl/src/index.ts index d45f31e3df..b47766e186 100644 --- a/packages/session-persistence/session-persistence-jsonl/src/index.ts +++ b/packages/session-persistence/session-persistence-jsonl/src/index.ts @@ -145,17 +145,21 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi async listSnapshots(): Promise { const snapshots: SessionPersistenceSnapshot[] = [] for (const artifact of await this.listArtifacts()) { - const identity = await stat(artifact.path, { bigint: true }) - snapshots.push({ - header: artifact.header, - revision: SessionPersistenceRevision([ - identity.dev, - identity.ino, - identity.size, - identity.mtimeNs, - identity.ctimeNs, - ].join(':')), - }) + try { + const identity = await stat(artifact.path, { bigint: true }) + snapshots.push({ + header: artifact.header, + revision: SessionPersistenceRevision([ + identity.dev, + identity.ino, + identity.size, + identity.mtimeNs, + identity.ctimeNs, + ].join(':')), + }) + } catch (error: unknown) { + if (!isENOENT(error)) throw error + } } return snapshots } diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index 0cb4ed3fea..3c826db2eb 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -179,6 +179,39 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => { await otherCtx.fiber.dispose() }) + it('omits a snapshot artifact removed after discovery', async () => { + const m = meta('vanishing-snapshot') + await ctx.sessionPersistence.create(m) + await ctx.sessionPersistence.append(m.id, oneTurnLog()) + const persistence = ctx.sessionPersistence as unknown as { + listArtifacts(): Promise> + } + const listArtifacts = persistence.listArtifacts.bind(persistence) + const discovery = vi.spyOn(persistence, 'listArtifacts').mockImplementation(async () => { + const artifacts = await listArtifacts() + await rm(artifacts[0]!.path) + return artifacts + }) + + await expect(ctx.sessionPersistence.listSnapshots()).resolves.toEqual([]) + discovery.mockRestore() + }) + + it('surfaces non-ENOENT snapshot stat failures after discovery', async () => { + const blocker = join(root, 'snapshot-not-a-directory') + await writeFile(blocker, 'x') + const persistence = ctx.sessionPersistence as unknown as { + listArtifacts(): Promise> + } + const discovery = vi.spyOn(persistence, 'listArtifacts').mockResolvedValue([{ + header: meta('snapshot-stat-failure'), + path: join(blocker, 'session.jsonl'), + }]) + + await expect(ctx.sessionPersistence.listSnapshots()).rejects.toThrow(/ENOTDIR/) + discovery.mockRestore() + }) + it('persists a forked child seed through the existing session write path', async () => { const source = ctx.sessions.create(SessionId('persist-parent'), { meta: { cwd: '/workspace' } }) appendClosedTurn(source) diff --git a/packages/session-persistence/session-persistence-sqlite/README.md b/packages/session-persistence/session-persistence-sqlite/README.md index a4a0e645c4..cae28d2bd4 100644 --- a/packages/session-persistence/session-persistence-sqlite/README.md +++ b/packages/session-persistence/session-persistence-sqlite/README.md @@ -6,7 +6,7 @@ A SQLite durable session-persistence backend — a second `SessionPersistence` i ## Storage model -Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [session surface](../../../docs/rfc/implemented/architecture/2026-06-18-session-surface.md)). Out-of-log metadata (`SessionHeader`) and a monotonic per-log revision live in a `sessions` row; a singleton state row carries the immutable store id. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`list` reports exactly the sessions that have a row). +Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [session surface](../../../docs/rfc/implemented/architecture/2026-06-18-session-surface.md)). Out-of-log metadata (`SessionHeader`), a per-materialization incarnation id, and a monotonic per-log revision live in a `sessions` row; a singleton state row carries the immutable store id. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`list` reports exactly the sessions that have a row). The repository's Node range supports unflagged `node:sqlite`. The database enables foreign keys and uses the configured journal mode (`wal` by default; use a rollback mode where WAL shared-memory files are unsuitable). `PRAGMA user_version` stores the table-layout version; databases with any other version are rejected because this unreleased format has no migrations. @@ -15,7 +15,7 @@ The repository's Node range supports unflagged `node:sqlite`. The database enabl - **Append = a transaction.** `append` runs `BEGIN`/`COMMIT` around the batch: it materializes the `sessions` row (if still lazy) and INSERTs every event, asserting the contiguous-seq contract first (the first event's `seq` must equal the stored next-seq). A mid-batch failure (a UNIQUE violation on a duplicated seq) rolls back entirely, so the stored log and the in-memory cursor stay consistent. (`load()` already balanced the stored log, so `append` never has to repair a crash tail.) - **Lazy materialization.** `create()` records intent in memory only — no row is written until the first `append`. A created-but-never-appended session has no `sessions` row, so it is absent from `list()` (which reports exactly the sessions that have a row). - **Interrupted-turn close on load.** `load()` implements the shared [crash-recovery contract](../../../docs/rfc/implemented/architecture/2026-06-14-session-persistence.md): preserve the valid interrupted turn, append its synthetic closing events in one transaction, and remove only a torn tail row. Committed parse errors or sequence gaps make the session unloadable. Because recovery mutates stored rows, the next append starts from a balanced log and accurate cursor. -- **Lightweight revisions.** `listSnapshots()` combines an immutable store identity, the database file identity, and a per-session counter incremented in each mutating transaction. This keeps unchanged observations stable without parsing event rows and prevents independent stores from sharing a revision accidentally. +- **Lightweight revisions.** `listSnapshots()` combines the immutable store and database-file identity, a per-materialization incarnation id, and a per-session counter incremented in each mutating transaction. This keeps unchanged observations stable without parsing event rows and distinguishes independent stores and recreated same-id logs. ## Configuration (schemastery) diff --git a/packages/session-persistence/session-persistence-sqlite/src/index.ts b/packages/session-persistence/session-persistence-sqlite/src/index.ts index 6759704d7f..2f3b941f8b 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/index.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/index.ts @@ -7,6 +7,7 @@ import { Context } from 'cordis' import z from 'schemastery' +import { randomUUID } from 'node:crypto' import { statSync } from 'node:fs' import { DatabaseSync } from 'node:sqlite' import { mkdir } from 'node:fs/promises' @@ -235,7 +236,9 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers const rows = this.db.prepare('SELECT * FROM sessions').all() as unknown as SessionRow[] return rows.map(row => ({ header: rowToMeta(row), - revision: SessionPersistenceRevision(`${this.storeIdentity}:revision:${row.revision}`), + revision: SessionPersistenceRevision( + `${this.storeIdentity}:incarnation:${row.incarnation}:revision:${row.revision}`, + ), })) } @@ -259,8 +262,9 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers */ private writeRow(meta: SessionHeader): void { this.db.prepare(` - INSERT INTO sessions (id, version, created_at, cwd, parent_session, seed_length, revision) - VALUES (?, ?, ?, ?, ?, ?, 0) + INSERT INTO sessions + (id, version, created_at, cwd, parent_session, seed_length, incarnation, revision) + VALUES (?, ?, ?, ?, ?, ?, ?, 0) ON CONFLICT(id) DO UPDATE SET version = excluded.version, created_at = excluded.created_at, @@ -274,6 +278,7 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers meta.cwd ?? null, meta.parentSession ?? null, meta.seedLength ?? null, + randomUUID(), ) } } diff --git a/packages/session-persistence/session-persistence-sqlite/src/schema.ts b/packages/session-persistence/session-persistence-sqlite/src/schema.ts index fe7db15d67..fc397ff742 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/schema.ts @@ -17,7 +17,7 @@ import type { SessionEvent, SessionId, SessionHeader, SurfaceOp } from '@deepsee * layout; orthogonal to a session's own `version` (which versions the EVENT * vocabulary, stored per session in the `sessions` row). */ -export const SCHEMA_VERSION = 6 +export const SCHEMA_VERSION = 7 /** * A row of the `sessions` table — the out-of-log metadata ({@link SessionHeader}). @@ -33,6 +33,8 @@ export interface SessionRow { cwd: string | null parent_session: string | null seed_length: number | null + /** Stable identity assigned when this log is materialized. */ + incarnation: string /** Monotonic log-change token incremented in each mutating transaction. */ revision: number } @@ -108,6 +110,7 @@ function configureDatabase(db: DatabaseSync, path: string, journalMode: JournalM cwd TEXT, parent_session TEXT, seed_length INTEGER, + incarnation TEXT NOT NULL, revision INTEGER NOT NULL ) STRICT `) diff --git a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts index 26b39e22d4..c1d4f8e74f 100644 --- a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts @@ -378,8 +378,31 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { await b.dispose() }) + it('changes revisions when a deleted session id is materialized again in the same database', async () => { + const path = await freshDbPath() + const m = meta('recreated-revision') + const first = await backend(path) + await first.ctx.sessionPersistence.create(m) + await first.ctx.sessionPersistence.append(m.id, oneTurnLog()) + const before = (await first.ctx.sessionPersistence.listSnapshots())[0]?.revision + await first.dispose() + + const cleanup = openDatabase(path, 'wal') + cleanup.prepare('DELETE FROM sessions WHERE id = ?').run(m.id) + cleanup.close() + + const second = await backend(path) + await second.ctx.sessionPersistence.create(m) + await second.ctx.sessionPersistence.append(m.id, oneTurnLog()) + const after = (await second.ctx.sessionPersistence.listSnapshots())[0]?.revision + expect(after).not.toBe(before) + expect(String(before)).toMatch(/:revision:1$/) + expect(String(after)).toMatch(/:revision:1$/) + await second.dispose() + }) + it('exposes the schema version constant', () => { - expect(SCHEMA_VERSION).toBe(6) + expect(SCHEMA_VERSION).toBe(7) }) it('keeps the revision stable for an empty repair hook', async () => { diff --git a/packages/session-query/session-query-sqlite/README.md b/packages/session-query/session-query-sqlite/README.md index 101f03eeca..dba2132f43 100644 --- a/packages/session-query/session-query-sqlite/README.md +++ b/packages/session-query/session-query-sqlite/README.md @@ -24,8 +24,8 @@ The database is disposable but reset is guarded: a recognized incompatible searc |---|---:|---| | `path` | required | Dedicated derived-index SQLite path; `:memory:` is supported. | | `journalMode` | `wal` | `wal`, `delete`, `truncate`, or `persist`. | -| `defaultLimit` | `20` | Page size when a request omits `limit`. | -| `maxLimit` | `100` | Largest accepted request page size. | +| `defaultLimit` | `20` | Page size when a request omits `limit`; at most `Number.MAX_SAFE_INTEGER - 1`. | +| `maxLimit` | `100` | Largest accepted request page size; at most `Number.MAX_SAFE_INTEGER - 1`. | | `snippetChars` | `240` | Maximum snippet length in Unicode code points. | ## Tokenizer and limits diff --git a/packages/session-query/session-query-sqlite/src/index.ts b/packages/session-query/session-query-sqlite/src/index.ts index 1590a1d61c..4265405098 100644 --- a/packages/session-query/session-query-sqlite/src/index.ts +++ b/packages/session-query/session-query-sqlite/src/index.ts @@ -48,6 +48,7 @@ import { quoteFtsData, requestFingerprint, sanitizeFtsText, + SQLITE_MAX_PAGE_LIMIT, } from './query.ts' export { @@ -63,15 +64,19 @@ export const SESSION_QUERY_SQLITE_MAX_LIMIT = 100 /** Default maximum snippet length in Unicode code points. */ export const SESSION_QUERY_SQLITE_SNIPPET_CHARS = 240 +// A serialized search tolerates one transient source change; repeated churn +// fails instead of monopolizing the operation queue. +const STABLE_OBSERVATION_ATTEMPTS = 2 + /** SQLite session-search configuration. */ export interface Config { /** Dedicated derived-index path; `:memory:` is supported for tests. */ path: string /** SQLite journal mode. Defaults to `wal`. */ journalMode?: JournalMode - /** Page size when a request omits `limit`. Defaults to 20. */ + /** Page size when a request omits `limit`. At most `Number.MAX_SAFE_INTEGER - 1`; defaults to 20. */ defaultLimit?: number - /** Largest accepted page size. Defaults to 100. */ + /** Largest accepted page size. At most `Number.MAX_SAFE_INTEGER - 1`; defaults to 100. */ maxLimit?: number /** Maximum snippet length in Unicode code points. Defaults to 240. */ snippetChars?: number @@ -154,8 +159,8 @@ export class SessionSearchSqlite extends SessionSearchService { static Config: z = z.object({ path: z.string().required(), journalMode: z.union(['wal', 'delete', 'truncate', 'persist'] as const).default('wal'), - defaultLimit: z.number().step(1).min(1).default(SESSION_QUERY_SQLITE_DEFAULT_LIMIT), - maxLimit: z.number().step(1).min(1).default(SESSION_QUERY_SQLITE_MAX_LIMIT), + defaultLimit: z.number().step(1).min(1).max(SQLITE_MAX_PAGE_LIMIT).default(SESSION_QUERY_SQLITE_DEFAULT_LIMIT), + maxLimit: z.number().step(1).min(1).max(SQLITE_MAX_PAGE_LIMIT).default(SESSION_QUERY_SQLITE_MAX_LIMIT), snippetChars: z.number().step(1).min(1).default(SESSION_QUERY_SQLITE_SNIPPET_CHARS), }) @@ -206,14 +211,14 @@ export class SessionSearchSqlite extends SessionSearchService { const signal = exec?.signal return this._serialized(signal, async () => { await this._ensureReady(signal) - await this._reconcile(signal) + const persistenceBinding = await this._reconcile(signal) assertNotAborted(signal) const generation = String(this._globalGeneration) const fingerprint = requestFingerprint(normalized) const offset = normalized.cursor === undefined ? 0 : decodeCursor(normalized.cursor, this._instance, 'sessions', fingerprint, generation) - const rows = this._querySessions(normalized, offset) + const rows = this._querySessions(normalized, offset, persistenceBinding) return page(rows, normalized.limit, row => this._sessionHit(row), cursorOffset => encodeCursor({ version: 1, instance: this._instance, @@ -233,14 +238,14 @@ export class SessionSearchSqlite extends SessionSearchService { const signal = exec?.signal return this._serialized(signal, async () => { await this._ensureReady(signal) - await this._reconcile(signal) + const persistenceBinding = await this._reconcile(signal) assertNotAborted(signal) - const generation = this._targetGeneration(normalized.sessionId) + const generation = this._targetGeneration(normalized.sessionId, persistenceBinding) const fingerprint = requestFingerprint(normalized) const offset = normalized.cursor === undefined ? 0 : decodeCursor(normalized.cursor, this._instance, 'events', fingerprint, generation) - const rows = this._queryEvents(normalized, offset) + const rows = this._queryEvents(normalized, offset, persistenceBinding) return page(rows, normalized.limit, row => this._eventHit(row), cursorOffset => encodeCursor({ version: 1, instance: this._instance, @@ -316,7 +321,7 @@ export class SessionSearchSqlite extends SessionSearchService { } } - private async _reconcile(signal: AbortSignal | undefined): Promise { + private async _reconcile(signal: AbortSignal | undefined): Promise { const db = this._requireDb() const persistedRows = db.prepare( 'SELECT id, revision, generation FROM persisted_sessions', @@ -392,13 +397,14 @@ export class SessionSearchSqlite extends SessionSearchService { if (pointerChanged) this._persistenceEpoch += 1 this._localGeneration = nextLocalGeneration this._lastPersistenceIdentity = observation.persistenceBinding.identity + return observation.persistenceBinding } private async _observeStable( indexed: ReadonlyMap, signal: AbortSignal | undefined, ): Promise { - for (;;) { + for (let attempt = 0; attempt < STABLE_OBSERVATION_ATTEMPTS; attempt += 1) { assertNotAborted(signal) const persistenceBinding = this._persistenceBinding const persistence = persistenceBinding.service @@ -446,6 +452,10 @@ export class SessionSearchSqlite extends SessionSearchService { return { persistenceBinding, persisted, live } } } + throw new SessionQueryError( + 'session-search persistence observation did not stabilize after one retry', + 'SESSION_QUERY_PERSISTENCE_FAILED', + ) } private _mainGeneration(): number { @@ -540,7 +550,11 @@ export class SessionSearchSqlite extends SessionSearchService { } } - private _querySessions(request: NormalizedSessionRequest, offset: number): SearchRow[] { + private _querySessions( + request: NormalizedSessionRequest, + offset: number, + persistenceBinding: PersistenceBinding, + ): SearchRow[] { const selected = selectedDocumentsSql() const sessionWhere = buildSessionWhere(request.sessionFilters) const eventWhere = buildEventWhere(request.eventFilters) @@ -562,7 +576,7 @@ export class SessionSearchSqlite extends SessionSearchService { ORDER BY match_count DESC, document_length ASC, time DESC, session_id ASC, seq DESC LIMIT ? OFFSET ? `).all( - ...selectedDocumentsParams(request.query, this._persistenceBinding.service !== undefined), + ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined), ...sessionWhere.params, ...eventWhere.params, request.limit + 1, @@ -570,7 +584,11 @@ export class SessionSearchSqlite extends SessionSearchService { ) as unknown as SearchRow[] } - private _queryEvents(request: NormalizedEventRequest, offset: number): SearchRow[] { + private _queryEvents( + request: NormalizedEventRequest, + offset: number, + persistenceBinding: PersistenceBinding, + ): SearchRow[] { const selected = selectedDocumentsSql() const eventWhere = buildEventWhere(request.filters) const where = ['session_id = ?', eventWhere.sql].filter(Boolean).join(' AND ') @@ -581,7 +599,7 @@ export class SessionSearchSqlite extends SessionSearchService { ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC LIMIT ? OFFSET ? `).all( - ...selectedDocumentsParams(request.query, this._persistenceBinding.service !== undefined), + ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined), request.sessionId, ...eventWhere.params, request.limit + 1, @@ -589,13 +607,13 @@ export class SessionSearchSqlite extends SessionSearchService { ) as unknown as SearchRow[] } - private _targetGeneration(sessionId: SessionId): string { + private _targetGeneration(sessionId: SessionId, persistenceBinding: PersistenceBinding): string { const db = this._requireDb() const live = db.prepare( 'SELECT generation FROM temp.live_sessions WHERE id = ?', ).get(sessionId) as { generation: number } | undefined if (live !== undefined) return `live:${live.generation}` - if (this._persistenceBinding.service !== undefined) { + if (persistenceBinding.service !== undefined) { const persisted = db.prepare( 'SELECT generation FROM persisted_sessions WHERE id = ?', ).get(sessionId) as { generation: number } | undefined @@ -850,8 +868,8 @@ function resolveConfig(config: Config): ResolvedConfig { if (typeof resolved.path !== 'string' || resolved.path.trim().length === 0) { throw invalidConfig('path must not be blank') } - assertPositiveInteger('defaultLimit', resolved.defaultLimit) - assertPositiveInteger('maxLimit', resolved.maxLimit) + assertPageLimit('defaultLimit', resolved.defaultLimit) + assertPageLimit('maxLimit', resolved.maxLimit) assertPositiveInteger('snippetChars', resolved.snippetChars) if (resolved.defaultLimit > resolved.maxLimit) { throw invalidConfig('defaultLimit must be less than or equal to maxLimit') @@ -865,6 +883,12 @@ function assertPositiveInteger(name: string, value: number): void { if (!Number.isInteger(value) || value < 1) throw invalidConfig(`${name} must be a positive integer`) } +function assertPageLimit(name: string, value: number): void { + if (!Number.isSafeInteger(value) || value < 1 || value > SQLITE_MAX_PAGE_LIMIT) { + throw invalidConfig(`${name} must be an integer between 1 and ${SQLITE_MAX_PAGE_LIMIT}`) + } +} + function invalidConfig(detail: string): SessionQueryError { return new SessionQueryError( `session-search SQLite config: ${detail}`, diff --git a/packages/session-query/session-query-sqlite/src/query.ts b/packages/session-query/session-query-sqlite/src/query.ts index 9654f6ae70..5eb3ed8380 100644 --- a/packages/session-query/session-query-sqlite/src/query.ts +++ b/packages/session-query/session-query-sqlite/src/query.ts @@ -20,6 +20,9 @@ export const FTS_HIGHLIGHT_START = '\uFDD0' /** Collision-free marker inserted after an FTS5 match by `highlight()`. */ export const FTS_HIGHLIGHT_END = '\uFDD1' +/** Largest page size whose internal lookahead remains an exact SQLite integer binding. */ +export const SQLITE_MAX_PAGE_LIMIT = Number.MAX_SAFE_INTEGER - 1 + /** Limit defaults needed to normalize a search request. */ export interface QueryLimits { /** Page size used when the request omits one. */ @@ -232,14 +235,17 @@ export function makeSnippet(markedText: string, maxChars: number): string { const characters = Array.from(clean) if (characters.length <= maxChars) return clean if (maxChars === 1) return '…' - let start = Math.max(0, matchStart - Math.floor(maxChars / 3)) - let prefix = start > 0 ? '…' : '' + const matchedIndex = Math.min(matchStart, characters.length - 1) + let start = Math.max(0, matchedIndex - Math.floor(maxChars / 3)) + const prefix = start > 0 ? '…' : '' let suffix = '…' let contentLength = maxChars - prefix.length - suffix.length if (contentLength < 1) { - start = 0 - prefix = '' - contentLength = maxChars - 1 + start = matchedIndex + suffix = '' + contentLength = maxChars - prefix.length - suffix.length + } else if (matchedIndex >= start + contentLength) { + start = matchedIndex - contentLength + 1 } let end = Math.min(characters.length, start + contentLength) if (end === characters.length) { @@ -326,9 +332,14 @@ function materializeMetadataFilters( function normalizeLimit(value: number | undefined, limits: QueryLimits): number { const limit = value ?? limits.defaultLimit - if (!Number.isInteger(limit) || limit < 1 || limit > limits.maxLimit) { + const maxLimit = Math.min(limits.maxLimit, SQLITE_MAX_PAGE_LIMIT) + if ( + !Number.isSafeInteger(limit) + || limit < 1 + || limit > maxLimit + ) { throw new SessionQueryError( - `session-search limit must be an integer between 1 and ${limits.maxLimit}`, + `session-search limit must be an integer between 1 and ${maxLimit}`, 'SESSION_QUERY_INVALID_LIMIT', ) } diff --git a/packages/session-query/session-query-sqlite/tests/query.spec.ts b/packages/session-query/session-query-sqlite/tests/query.spec.ts index f9c0a5d19e..14e84aae75 100644 --- a/packages/session-query/session-query-sqlite/tests/query.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/query.spec.ts @@ -11,6 +11,7 @@ import { normalizeSessionRequest, quoteFtsData, requestFingerprint, + SQLITE_MAX_PAGE_LIMIT, type NormalizedEventRequest, type NormalizedSessionRequest, } from '../src/query.ts' @@ -88,6 +89,12 @@ describe('SQLite search request normalization', () => { expect(() => normalizeEventRequest({ sessionId: SessionId('s'), query: 'x', limit }, limits)) .toThrow(expectCode('SESSION_QUERY_INVALID_LIMIT')) } + expect(() => normalizeEventRequest({ + sessionId: SessionId('s'), + query: 'x', + limit: SQLITE_MAX_PAGE_LIMIT + 1, + }, { defaultLimit: 1, maxLimit: SQLITE_MAX_PAGE_LIMIT + 1 })) + .toThrow(expectCode('SESSION_QUERY_INVALID_LIMIT')) }) it('materializes owned filter values during normalization', () => { @@ -214,7 +221,8 @@ describe('SQLite query identity and presentation', () => { expect(makeSnippet(`abcde${FTS_HIGHLIGHT_START}f${FTS_HIGHLIGHT_END}`, 1)).toBe('…') expect(makeSnippet('abcdefghij', 5)).toBe('abcd…') expect(makeSnippet(`ab${FTS_HIGHLIGHT_START}c${FTS_HIGHLIGHT_END}defghij`, 5)).toBe('…bcd…') - expect(makeSnippet(`abcde${FTS_HIGHLIGHT_START}f${FTS_HIGHLIGHT_END}`, 2)).toBe('a…') + expect(makeSnippet(`ab${FTS_HIGHLIGHT_START}c${FTS_HIGHLIGHT_END}defghij`, 3)).toBe('…c…') + expect(makeSnippet(`abcde${FTS_HIGHLIGHT_START}f${FTS_HIGHLIGHT_END}`, 2)).toBe('…f') expect(makeSnippet(`abcde${FTS_HIGHLIGHT_START}f${FTS_HIGHLIGHT_END}`, 5)).toBe('…cdef') expect(makeSnippet(` x—${FTS_HIGHLIGHT_START}café${FTS_HIGHLIGHT_END}\n y `, 20)) .toBe('x—café y') diff --git a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts index b4a69acbbd..0895b6f08e 100644 --- a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts @@ -387,6 +387,8 @@ describe('SQLite session search', () => { { path: '' }, { path: ':memory:', defaultLimit: 0 }, { path: ':memory:', maxLimit: 0 }, + { path: ':memory:', defaultLimit: 1e100 }, + { path: ':memory:', maxLimit: 1e100 }, { path: ':memory:', snippetChars: 0 }, { path: ':memory:', defaultLimit: 3, maxLimit: 2 }, { path: ':memory:', journalMode: 'memory' }, @@ -462,6 +464,39 @@ describe('SQLite reconciliation and source lifecycle', () => { .rejects.toThrow(expectCode('SESSION_QUERY_SESSION_NOT_FOUND')) }) + it('uses the reconciled persistence binding through the query boundary', async () => { + const durable = header('post-reconcile-unmount') + TestPersistence.reset([{ meta: durable, events: [ + ...messageEvents('durable needle', 1), + { ...messageEvents('durable needle again', 2)[0]!, seq: 1 }, + ] }]) + const ctx = await liveContext({ path: ':memory:', defaultLimit: 1, maxLimit: 2 }) + const persistence = await ctx.plugin(TestPersistence) + const internals = ctx.sessionSearch as unknown as { + _reconcile(signal: AbortSignal | undefined): Promise<{ + identity: symbol + service?: SessionPersistence + }> + } + const reconcile = internals._reconcile.bind(internals) + const boundary = vi.spyOn(internals, '_reconcile').mockImplementation(async (signal) => { + const binding = await reconcile(signal) + await persistence.dispose() + return binding + }) + + const page = await ctx.sessionSearch.searchEvents({ + sessionId: durable.id, + query: 'needle', + limit: 1, + }) + expect(page.items).toMatchObject([{ sessionId: durable.id }]) + expect(page.nextCursor).toEqual(expect.any(String)) + boundary.mockRestore() + await expect(ctx.sessionSearch.searchEvents({ sessionId: durable.id, query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_SESSION_NOT_FOUND')) + }) + it('discards a stale list rejection when persistence unmounts during observation', async () => { const durable = header('racing') TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) @@ -561,6 +596,22 @@ describe('SQLite reconciliation and source lifecycle', () => { expect(TestPersistence.loads.get(added.id)).toBe(1) }) + it('fails after one retry when persistence snapshots keep changing', async () => { + const durable = header('continuous-mutation') + TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) + const ctx = await liveContext() + await ctx.plugin(TestPersistence) + let lists = 0 + TestPersistence.snapshotEffect = () => { + lists += 1 + TestPersistence.set({ meta: durable, events: messageEvents(`durable needle ${lists}`) }) + } + + await expect(ctx.sessionSearch.searchSessions({ query: 'needle' })) + .rejects.toThrow(expectCode('SESSION_QUERY_PERSISTENCE_FAILED')) + expect(lists).toBe(4) + }) + it('retries if the persistence binding changes while live sessions are observed', async () => { const durable = header('live-boundary-retry') TestPersistence.reset([{ meta: durable, events: messageEvents('durable needle') }]) From 220076e5e2e74248f56bb8b4bf23de9b792f72bc Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Fri, 17 Jul 2026 10:15:48 +0800 Subject: [PATCH 011/113] fix(session-query): preserve typed query failures (round 5) --- .../session-query-sqlite/README.md | 2 +- .../session-query-sqlite/src/index.ts | 47 +++++++++++++------ .../session-query-sqlite/tests/sqlite.spec.ts | 46 ++++++++++++++++++ 3 files changed, 79 insertions(+), 16 deletions(-) diff --git a/packages/session-query/session-query-sqlite/README.md b/packages/session-query/session-query-sqlite/README.md index dba2132f43..1a5ee97e29 100644 --- a/packages/session-query/session-query-sqlite/README.md +++ b/packages/session-query/session-query-sqlite/README.md @@ -4,7 +4,7 @@ SQLite FTS5 implementation of `ctx.sessionSearch`. The service searches the live ## Search contract -`searchSessions(request, exec?)` returns `SessionSearchHit` pages across the corpus; `searchEvents(request, exec?)` returns `SessionEventSearchHit` pages within one session. Queries are required, trimmed, whitespace-normalized literal phrases. FTS5 syntax such as quotes, `OR`, `NEAR`, and `*` is treated as data rather than executable MATCH syntax. Metadata filters are parameterized SQL predicates applied before ranking. +`searchSessions(request, exec?)` returns `SessionSearchHit` pages across the corpus; `searchEvents(request, exec?)` returns `SessionEventSearchHit` pages within one session. Queries are required, trimmed, whitespace-normalized literal phrases. FTS5 syntax such as quotes, `OR`, `NEAR`, and `*` is treated as data rather than executable MATCH syntax. Metadata filters are parameterized SQL predicates applied before ranking. A request exceeding SQLite's portable limit of 32,766 total bindings, including fixed query and pagination values, fails with `SESSION_QUERY_INVALID_FILTER` before statement preparation. Relevance is source-comparable across persistent and TEMP tables: actual FTS5 highlighted-match span count descending, then stored document code-point length ascending. Event time, session id where applicable, and seq break remaining ties. Cross-session results expose the selected event as `bestMatch`; both scopes derive whitespace-normalized plain text from FTS5 highlight positions and bound it in Unicode code points. Cursors are opaque branded values, bind to the normalized request and service instance, and fail when the relevant generation changes. A within-session cursor survives unrelated-session changes; a cross-session cursor does not. diff --git a/packages/session-query/session-query-sqlite/src/index.ts b/packages/session-query/session-query-sqlite/src/index.ts index 4265405098..4428d7ca00 100644 --- a/packages/session-query/session-query-sqlite/src/index.ts +++ b/packages/session-query/session-query-sqlite/src/index.ts @@ -559,6 +559,14 @@ export class SessionSearchSqlite extends SessionSearchService { const sessionWhere = buildSessionWhere(request.sessionFilters) const eventWhere = buildEventWhere(request.eventFilters) const where = [sessionWhere.sql, eventWhere.sql].filter(Boolean).join(' AND ') + const bindings = [ + ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined), + ...sessionWhere.params, + ...eventWhere.params, + request.limit + 1, + offset, + ] + assertPortableBindingCount(bindings) return this._requireDb().prepare(` ${selected.sql}, filtered AS ( @@ -575,13 +583,7 @@ export class SessionSearchSqlite extends SessionSearchService { WHERE event_rank = 1 ORDER BY match_count DESC, document_length ASC, time DESC, session_id ASC, seq DESC LIMIT ? OFFSET ? - `).all( - ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined), - ...sessionWhere.params, - ...eventWhere.params, - request.limit + 1, - offset, - ) as unknown as SearchRow[] + `).all(...bindings) as unknown as SearchRow[] } private _queryEvents( @@ -592,19 +594,21 @@ export class SessionSearchSqlite extends SessionSearchService { const selected = selectedDocumentsSql() const eventWhere = buildEventWhere(request.filters) const where = ['session_id = ?', eventWhere.sql].filter(Boolean).join(' AND ') + const bindings = [ + ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined), + request.sessionId, + ...eventWhere.params, + request.limit + 1, + offset, + ] + assertPortableBindingCount(bindings) return this._requireDb().prepare(` ${selected.sql} SELECT * FROM matched WHERE ${where} ORDER BY match_count DESC, document_length ASC, time DESC, seq DESC LIMIT ? OFFSET ? - `).all( - ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined), - request.sessionId, - ...eventWhere.params, - request.limit + 1, - offset, - ) as unknown as SearchRow[] + `).all(...bindings) as unknown as SearchRow[] } private _targetGeneration(sessionId: SessionId, persistenceBinding: PersistenceBinding): string { @@ -728,6 +732,19 @@ function selectedDocumentsParams(query: string, persistenceVisible: boolean): Ar ] } +// SQLite builds may raise this ceiling; supported modern versions share 32,766 +// as the portable host-parameter limit. +const SQLITE_PORTABLE_VARIABLE_LIMIT = 32_766 + +function assertPortableBindingCount(bindings: readonly (string | number)[]): void { + if (bindings.length > SQLITE_PORTABLE_VARIABLE_LIMIT) { + throw new SessionQueryError( + `session-search request requires ${bindings.length} SQLite bindings; reduce filters to stay within the portable ${SQLITE_PORTABLE_VARIABLE_LIMIT}-variable limit`, + 'SESSION_QUERY_INVALID_FILTER', + ) + } +} + function observeLive(session: Session): ObservedSession { return observeSession(session.header, session.events) } @@ -834,7 +851,7 @@ function decodeCursor( || decoded.instance !== instance || decoded.scope !== scope || decoded.fingerprint !== fingerprint - || !Number.isInteger(decoded.offset) + || !Number.isSafeInteger(decoded.offset) || decoded.offset === undefined || decoded.offset < 0 ) { diff --git a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts index 0895b6f08e..4b875931c4 100644 --- a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts @@ -53,6 +53,16 @@ function expectCode(code: SessionQueryErrorCode): Error { return expect.objectContaining({ code }) as Error } +function replaceCursorOffset( + cursor: ReturnType, + offset: number, +): ReturnType { + const payload = JSON.parse( + Buffer.from(cursor, 'base64url').toString('utf8'), + ) as Record + return SessionSearchCursor(Buffer.from(JSON.stringify({ ...payload, offset }), 'utf8').toString('base64url')) +} + class TestPersistence extends SessionPersistence { static entries = new Map() static revisions = new Map() @@ -276,6 +286,14 @@ describe('SQLite session search', () => { expect(sessionPage.nextCursor).toEqual(expect.any(String)) if (eventPage.nextCursor === undefined || sessionPage.nextCursor === undefined) throw new Error('expected cursors') + const unsafeOffsetCursor = replaceCursorOffset(eventPage.nextCursor, 1e100) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: target.id, + query: 'needle', + limit: 1, + cursor: unsafeOffsetCursor, + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_CURSOR')) + const eventKeys = eventPage.items.map(item => `${item.sessionId}:${item.seq}`) let eventCursor: ReturnType | undefined = eventPage.nextCursor while (eventCursor !== undefined) { @@ -399,6 +417,34 @@ describe('SQLite session search', () => { .toThrow(expectCode('SESSION_QUERY_INVALID_CONFIG')) } }) + + it('rejects aggregate filter bindings above SQLite\'s portable variable limit', async () => { + const ctx = await liveContext() + const session = ctx.sessions.create(SessionId('binding-limit'), { seed: messageEvents('needle') }) + // Each clause is below the ceiling; combined with its sibling and fixed + // query bindings, the complete statement is not portable. + const halfPortableLimit = 16_383 + const ids = Array.from( + { length: halfPortableLimit }, + (_, index) => SessionId(`binding-${index}`), + ) + const types = Array.from({ length: halfPortableLimit }, () => 'user/message' as const) + const surfaces = Array.from({ length: halfPortableLimit }, () => 'current' as const) + + await expect(ctx.sessionSearch.searchSessions({ + query: 'needle', + sessionFilters: [{ kind: 'id', values: ids }], + eventFilters: [{ kind: 'type', values: types }], + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: session.id, + query: 'needle', + filters: [ + { kind: 'type', values: types }, + { kind: 'surface', values: surfaces }, + ], + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + }) }) describe('SQLite reconciliation and source lifecycle', () => { From f401528941e39c4b96957a9d910af5ced7ab08de Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Fri, 17 Jul 2026 10:29:49 +0800 Subject: [PATCH 012/113] fix(session-query): preflight SQLite bindings (round 6) --- .../session-query-sqlite/src/index.ts | 21 +++--------- .../session-query-sqlite/src/query.ts | 33 ++++++++++++++++--- .../session-query-sqlite/tests/sqlite.spec.ts | 13 ++++++++ 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/packages/session-query/session-query-sqlite/src/index.ts b/packages/session-query/session-query-sqlite/src/index.ts index 4428d7ca00..05b5edace4 100644 --- a/packages/session-query/session-query-sqlite/src/index.ts +++ b/packages/session-query/session-query-sqlite/src/index.ts @@ -40,6 +40,7 @@ import { type NormalizedSessionRequest, FTS_HIGHLIGHT_END, FTS_HIGHLIGHT_START, + assertPortableBindingCount, buildEventWhere, buildSessionWhere, makeSnippet, @@ -64,8 +65,7 @@ export const SESSION_QUERY_SQLITE_MAX_LIMIT = 100 /** Default maximum snippet length in Unicode code points. */ export const SESSION_QUERY_SQLITE_SNIPPET_CHARS = 240 -// A serialized search tolerates one transient source change; repeated churn -// fails instead of monopolizing the operation queue. +// One transient source change gets a retry; repeated churn fails rather than monopolizing the queue. const STABLE_OBSERVATION_ATTEMPTS = 2 /** SQLite session-search configuration. */ @@ -566,7 +566,7 @@ export class SessionSearchSqlite extends SessionSearchService { request.limit + 1, offset, ] - assertPortableBindingCount(bindings) + assertPortableBindingCount(bindings.length) return this._requireDb().prepare(` ${selected.sql}, filtered AS ( @@ -601,7 +601,7 @@ export class SessionSearchSqlite extends SessionSearchService { request.limit + 1, offset, ] - assertPortableBindingCount(bindings) + assertPortableBindingCount(bindings.length) return this._requireDb().prepare(` ${selected.sql} SELECT * FROM matched @@ -732,19 +732,6 @@ function selectedDocumentsParams(query: string, persistenceVisible: boolean): Ar ] } -// SQLite builds may raise this ceiling; supported modern versions share 32,766 -// as the portable host-parameter limit. -const SQLITE_PORTABLE_VARIABLE_LIMIT = 32_766 - -function assertPortableBindingCount(bindings: readonly (string | number)[]): void { - if (bindings.length > SQLITE_PORTABLE_VARIABLE_LIMIT) { - throw new SessionQueryError( - `session-search request requires ${bindings.length} SQLite bindings; reduce filters to stay within the portable ${SQLITE_PORTABLE_VARIABLE_LIMIT}-variable limit`, - 'SESSION_QUERY_INVALID_FILTER', - ) - } -} - function observeLive(session: Session): ObservedSession { return observeSession(session.header, session.events) } diff --git a/packages/session-query/session-query-sqlite/src/query.ts b/packages/session-query/session-query-sqlite/src/query.ts index 5eb3ed8380..409c20f028 100644 --- a/packages/session-query/session-query-sqlite/src/query.ts +++ b/packages/session-query/session-query-sqlite/src/query.ts @@ -23,6 +23,22 @@ export const FTS_HIGHLIGHT_END = '\uFDD1' /** Largest page size whose internal lookahead remains an exact SQLite integer binding. */ export const SQLITE_MAX_PAGE_LIMIT = Number.MAX_SAFE_INTEGER - 1 +/** Portable host-parameter ceiling shared by predicate and statement builders. */ +export const SQLITE_PORTABLE_VARIABLE_LIMIT = 32_766 + +/** + * Reject prospective SQLite binding growth beyond the portable ceiling. + * @param count - binding count at the current construction boundary. + */ +export function assertPortableBindingCount(count: number): void { + if (count > SQLITE_PORTABLE_VARIABLE_LIMIT) { + throw new SessionQueryError( + `session-search request exceeds SQLite's portable ${SQLITE_PORTABLE_VARIABLE_LIMIT}-variable limit; reduce filter values`, + 'SESSION_QUERY_INVALID_FILTER', + ) + } +} + /** Limit defaults needed to normalize a search request. */ export interface QueryLimits { /** Page size used when the request omits one. */ @@ -356,8 +372,7 @@ function addList( clauses.push('0') return } - clauses.push(`${column} IN (${values.map(() => '?').join(', ')})`) - params.push(...values) + clauses.push(`${column} IN (${appendListBindings(params, values)})`) } function addNullableList( @@ -373,8 +388,7 @@ function addNullableList( const concrete = values.filter((value): value is string => value !== null) const parts: string[] = [] if (concrete.length > 0) { - parts.push(`${column} IN (${concrete.map(() => '?').join(', ')})`) - params.push(...concrete) + parts.push(`${column} IN (${appendListBindings(params, concrete)})`) } if (values.includes(null)) parts.push(`${column} IS NULL`) clauses.push(`(${parts.join(' OR ')})`) @@ -387,15 +401,26 @@ function addRange( range: { from?: number; to?: number }, ): void { if (range.from !== undefined) { + assertPortableBindingCount(params.length + 1) clauses.push(`CAST(${column} AS INTEGER) >= ?`) params.push(range.from) } if (range.to !== undefined) { + assertPortableBindingCount(params.length + 1) clauses.push(`CAST(${column} AS INTEGER) <= ?`) params.push(range.to) } } +function appendListBindings( + params: Array, + values: readonly (string | number)[], +): string { + assertPortableBindingCount(params.length + values.length) + for (const value of values) params.push(value) + return values.map(() => '?').join(', ') +} + function canonicalFilters(filters: readonly (SessionResultFilter | SessionEventMetadataFilter)[]): unknown[] { return filters.map((filter) => { if ('values' in filter) { diff --git a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts index 4b875931c4..8db1df5666 100644 --- a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts @@ -445,6 +445,19 @@ describe('SQLite session search', () => { ], })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) }) + + it('rejects one 125,000-value filter list with a typed error', async () => { + const ctx = await liveContext() + const ids = Array.from( + { length: 125_000 }, + (_, index) => SessionId(`oversized-binding-${index}`), + ) + + await expect(ctx.sessionSearch.searchSessions({ + query: 'needle', + sessionFilters: [{ kind: 'id', values: ids }], + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + }) }) describe('SQLite reconciliation and source lifecycle', () => { From e8abfd6482b6d7050e161915de5cb98486e3e14a Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Fri, 17 Jul 2026 11:13:03 +0800 Subject: [PATCH 013/113] fix(session-query): guard FTS predicate planning (round 7) --- docs/config-catalog.md | 2 +- .../session-query-sqlite/README.md | 2 +- .../session-query-sqlite/src/index.ts | 3 + .../session-query-sqlite/src/query.ts | 24 +++++++- .../session-query-sqlite/tests/query.spec.ts | 42 ++++++++++++-- .../session-query-sqlite/tests/sqlite.spec.ts | 55 +++++++++++++++++++ 6 files changed, 119 insertions(+), 9 deletions(-) diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 6d141185ca..de1a316cc6 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -666,7 +666,7 @@ export interface Config { export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' ``` -Source: [`packages/session-query/session-query-sqlite/src/index.ts:72`](../packages/session-query/session-query-sqlite/src/index.ts) +Source: [`packages/session-query/session-query-sqlite/src/index.ts:73`](../packages/session-query/session-query-sqlite/src/index.ts) ## `@deepseek-ai/dsh-skill` diff --git a/packages/session-query/session-query-sqlite/README.md b/packages/session-query/session-query-sqlite/README.md index 1a5ee97e29..1beb479d5e 100644 --- a/packages/session-query/session-query-sqlite/README.md +++ b/packages/session-query/session-query-sqlite/README.md @@ -4,7 +4,7 @@ SQLite FTS5 implementation of `ctx.sessionSearch`. The service searches the live ## Search contract -`searchSessions(request, exec?)` returns `SessionSearchHit` pages across the corpus; `searchEvents(request, exec?)` returns `SessionEventSearchHit` pages within one session. Queries are required, trimmed, whitespace-normalized literal phrases. FTS5 syntax such as quotes, `OR`, `NEAR`, and `*` is treated as data rather than executable MATCH syntax. Metadata filters are parameterized SQL predicates applied before ranking. A request exceeding SQLite's portable limit of 32,766 total bindings, including fixed query and pagination values, fails with `SESSION_QUERY_INVALID_FILTER` before statement preparation. +`searchSessions(request, exec?)` returns `SessionSearchHit` pages across the corpus; `searchEvents(request, exec?)` returns `SessionEventSearchHit` pages within one session. Queries are required, trimmed, whitespace-normalized literal phrases. FTS5 syntax such as quotes, `OR`, `NEAR`, and `*` is treated as data rather than executable MATCH syntax. Metadata filters are parameterized SQL predicates applied before ranking. To keep SQLite FTS5 MATCH in a supported outer-predicate context, cross-session requests may compile at most 14 combined session and event filter predicates; within-session requests may compile at most 13 filter predicates because the fixed target-session predicate consumes one slot. Each range endpoint compiles as one predicate. A request exceeding either predicate budget or SQLite's portable limit of 32,766 total bindings, including fixed query and pagination values, fails with `SESSION_QUERY_INVALID_FILTER` before statement preparation. Relevance is source-comparable across persistent and TEMP tables: actual FTS5 highlighted-match span count descending, then stored document code-point length ascending. Event time, session id where applicable, and seq break remaining ties. Cross-session results expose the selected event as `bestMatch`; both scopes derive whitespace-normalized plain text from FTS5 highlight positions and bound it in Unicode code points. Cursors are opaque branded values, bind to the normalized request and service instance, and fail when the relevant generation changes. A within-session cursor survives unrelated-session changes; a cross-session cursor does not. diff --git a/packages/session-query/session-query-sqlite/src/index.ts b/packages/session-query/session-query-sqlite/src/index.ts index 05b5edace4..2bccb6f269 100644 --- a/packages/session-query/session-query-sqlite/src/index.ts +++ b/packages/session-query/session-query-sqlite/src/index.ts @@ -40,6 +40,7 @@ import { type NormalizedSessionRequest, FTS_HIGHLIGHT_END, FTS_HIGHLIGHT_START, + assertFts5OuterPredicateCount, assertPortableBindingCount, buildEventWhere, buildSessionWhere, @@ -558,6 +559,7 @@ export class SessionSearchSqlite extends SessionSearchService { const selected = selectedDocumentsSql() const sessionWhere = buildSessionWhere(request.sessionFilters) const eventWhere = buildEventWhere(request.eventFilters) + assertFts5OuterPredicateCount(sessionWhere.predicateCount + eventWhere.predicateCount) const where = [sessionWhere.sql, eventWhere.sql].filter(Boolean).join(' AND ') const bindings = [ ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined), @@ -593,6 +595,7 @@ export class SessionSearchSqlite extends SessionSearchService { ): SearchRow[] { const selected = selectedDocumentsSql() const eventWhere = buildEventWhere(request.filters) + assertFts5OuterPredicateCount(1 + eventWhere.predicateCount) const where = ['session_id = ?', eventWhere.sql].filter(Boolean).join(' AND ') const bindings = [ ...selectedDocumentsParams(request.query, persistenceBinding.service !== undefined), diff --git a/packages/session-query/session-query-sqlite/src/query.ts b/packages/session-query/session-query-sqlite/src/query.ts index 409c20f028..5a67653911 100644 --- a/packages/session-query/session-query-sqlite/src/query.ts +++ b/packages/session-query/session-query-sqlite/src/query.ts @@ -26,6 +26,9 @@ export const SQLITE_MAX_PAGE_LIMIT = Number.MAX_SAFE_INTEGER - 1 /** Portable host-parameter ceiling shared by predicate and statement builders. */ export const SQLITE_PORTABLE_VARIABLE_LIMIT = 32_766 +/** Supported outer-predicate budget that keeps SQLite FTS5 MATCH usable. */ +export const SQLITE_FTS5_OUTER_PREDICATE_LIMIT = 14 + /** * Reject prospective SQLite binding growth beyond the portable ceiling. * @param count - binding count at the current construction boundary. @@ -39,6 +42,19 @@ export function assertPortableBindingCount(count: number): void { } } +/** + * Reject compiled outer predicates beyond the supported FTS5 planner budget. + * @param count - predicate count including fixed statement predicates. + */ +export function assertFts5OuterPredicateCount(count: number): void { + if (count > SQLITE_FTS5_OUTER_PREDICATE_LIMIT) { + throw new SessionQueryError( + `session-search request exceeds the supported SQLite FTS5 outer-predicate budget of ${SQLITE_FTS5_OUTER_PREDICATE_LIMIT}; reduce filters`, + 'SESSION_QUERY_INVALID_FILTER', + ) + } +} + /** Limit defaults needed to normalize a search request. */ export interface QueryLimits { /** Page size used when the request omits one. */ @@ -71,6 +87,8 @@ export interface SqlWhere { sql: string /** Bindings in placeholder order. */ params: Array + /** Number of compiled predicates in `sql`. */ + predicateCount: number } /** @@ -163,7 +181,8 @@ export function buildSessionWhere(filters: readonly SessionResultFilter[]): SqlW unknownFilter(filter) } } - return { sql: clauses.join(' AND '), params } + assertFts5OuterPredicateCount(clauses.length) + return { sql: clauses.join(' AND '), params, predicateCount: clauses.length } } /** @@ -192,7 +211,8 @@ export function buildEventWhere(filters: readonly SessionEventMetadataFilter[]): unknownFilter(filter) } } - return { sql: clauses.join(' AND '), params } + assertFts5OuterPredicateCount(clauses.length) + return { sql: clauses.join(' AND '), params, predicateCount: clauses.length } } /** diff --git a/packages/session-query/session-query-sqlite/tests/query.spec.ts b/packages/session-query/session-query-sqlite/tests/query.spec.ts index 14e84aae75..b40489d429 100644 --- a/packages/session-query/session-query-sqlite/tests/query.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/query.spec.ts @@ -11,6 +11,7 @@ import { normalizeSessionRequest, quoteFtsData, requestFingerprint, + SQLITE_FTS5_OUTER_PREDICATE_LIMIT, SQLITE_MAX_PAGE_LIMIT, type NormalizedEventRequest, type NormalizedSessionRequest, @@ -111,24 +112,36 @@ describe('SQLite search request normalization', () => { describe('SQLite search predicate compilation', () => { it('compiles all logical-session clauses including empty and nullable values', () => { - expect(buildSessionWhere([])).toEqual({ sql: '', params: [] }) - expect(buildSessionWhere([{ kind: 'id', values: [] }])).toEqual({ sql: '0', params: [] }) + expect(buildSessionWhere([])).toEqual({ sql: '', params: [], predicateCount: 0 }) + expect(buildSessionWhere([{ kind: 'id', values: [] }])).toEqual({ + sql: '0', + params: [], + predicateCount: 1, + }) expect(buildSessionWhere([{ kind: 'id', values: [SessionId('a'), SessionId('b')] }])).toEqual({ sql: 'session_id IN (?, ?)', params: [SessionId('a'), SessionId('b')], + predicateCount: 1, + }) + expect(buildSessionWhere([{ kind: 'cwd', values: [] }])).toEqual({ + sql: '0', + params: [], + predicateCount: 1, }) - expect(buildSessionWhere([{ kind: 'cwd', values: [] }])).toEqual({ sql: '0', params: [] }) expect(buildSessionWhere([{ kind: 'cwd', values: [null] }])).toEqual({ sql: '(cwd IS NULL)', params: [], + predicateCount: 1, }) expect(buildSessionWhere([{ kind: 'cwd', values: ['/a'] }])).toEqual({ sql: '(cwd IN (?))', params: ['/a'], + predicateCount: 1, }) expect(buildSessionWhere([{ kind: 'parent', values: [SessionId('p'), null] }])).toEqual({ sql: '(parent_session IN (?) OR parent_session IS NULL)', params: [SessionId('p')], + predicateCount: 1, }) expect(buildSessionWhere([ { kind: 'created-at', from: 1, to: 2 }, @@ -138,8 +151,13 @@ describe('SQLite search predicate compilation', () => { ])).toEqual({ sql: 'CAST(created_at AS INTEGER) >= ? AND CAST(created_at AS INTEGER) <= ? AND 0 AND live = 1', params: [1, 2], + predicateCount: 4, + }) + expect(buildSessionWhere([{ kind: 'created-at' }])).toEqual({ + sql: '', + params: [], + predicateCount: 0, }) - expect(buildSessionWhere([{ kind: 'created-at' }])).toEqual({ sql: '', params: [] }) }) it('compiles every event clause and empty lists', () => { @@ -151,11 +169,25 @@ describe('SQLite search predicate compilation', () => { ])).toEqual({ sql: 'CAST(seq AS INTEGER) >= ? AND CAST(time AS INTEGER) <= ? AND type IN (?) AND surface IN (?, ?)', params: [1, 9, 'user/message', 'current', 'log-only'], + predicateCount: 4, }) expect(buildEventWhere([ { kind: 'type', values: [] }, { kind: 'surface', values: [] }, - ])).toEqual({ sql: '0 AND 0', params: [] }) + ])).toEqual({ sql: '0 AND 0', params: [], predicateCount: 2 }) + }) + + it('rejects predicate builders above the supported FTS5 outer budget', () => { + const filters = Array.from( + { length: SQLITE_FTS5_OUTER_PREDICATE_LIMIT }, + () => ({ kind: 'id' as const, values: [SessionId('safe')] }), + ) + + expect(buildSessionWhere(filters).predicateCount).toBe(SQLITE_FTS5_OUTER_PREDICATE_LIMIT) + expect(() => buildSessionWhere([ + ...filters, + { kind: 'id', values: [SessionId('over')] }, + ])).toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) }) it('rejects runtime-unknown filter discriminants in both SQL builders', () => { diff --git a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts index 8db1df5666..043bf3cf6b 100644 --- a/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts +++ b/packages/session-query/session-query-sqlite/tests/sqlite.spec.ts @@ -210,6 +210,61 @@ describe('SQLite session search', () => { }) }) + it('searches at the supported FTS5 outer-predicate boundary in both scopes', async () => { + const ctx = await liveContext() + const session = ctx.sessions.create(SessionId('predicate-boundary'), { + seed: messageEvents('needle'), + meta: { cwd: '/work' }, + }) + const sessionFilters = Array.from( + { length: 14 }, + () => ({ kind: 'cwd' as const, values: ['/work', null] }), + ) + const eventFilters = Array.from( + { length: 13 }, + () => ({ kind: 'type' as const, values: ['user/message' as const] }), + ) + + await expect(ctx.sessionSearch.searchSessions({ query: 'needle', sessionFilters })) + .resolves.toMatchObject({ items: [{ header: { id: session.id } }] }) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: session.id, + query: 'needle', + filters: eventFilters, + })).resolves.toMatchObject({ items: [{ sessionId: session.id, seq: 0 }] }) + }) + + it('rejects unsupported FTS5 outer-predicate counts with typed errors', async () => { + const ctx = await liveContext() + const session = ctx.sessions.create(SessionId('predicate-limit'), { seed: messageEvents('needle') }) + const sessionFilters = Array.from( + { length: 1_100 }, + () => ({ kind: 'id' as const, values: [session.id] }), + ) + const eventFilters = Array.from( + { length: 1_100 }, + () => ({ kind: 'type' as const, values: ['user/message' as const] }), + ) + + await expect(ctx.sessionSearch.searchSessions({ query: 'needle', sessionFilters })) + .rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: session.id, + query: 'needle', + filters: eventFilters, + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionSearch.searchSessions({ + query: 'needle', + sessionFilters: sessionFilters.slice(0, 7), + eventFilters: eventFilters.slice(0, 8), + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + await expect(ctx.sessionSearch.searchEvents({ + sessionId: session.id, + query: 'needle', + filters: eventFilters.slice(0, 14), + })).rejects.toThrow(expectCode('SESSION_QUERY_INVALID_FILTER')) + }) + it('uses literal phrase tokens, stable ties, and bounded Unicode snippets', async () => { const ctx = await liveContext({ path: ':memory:', defaultLimit: 10, maxLimit: 10, snippetChars: 5 }) ctx.sessions.create(SessionId('a'), { seed: messageEvents('😀😀 alpha beta BRAID 😀😀', 10), meta: { createdAt: 1 } }) From 73e3f658c609a5b43e2acf292315a0260d87b36a Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:40:10 +0800 Subject: [PATCH 014/113] fix(persistence): bind JSONL identity before mutation JSONL discovered a log by the requested session id but later routed repair and append from the parsed header. A log selected for session A could therefore declare session B and redirect mutation to B. Validate the requested id and exact header-derived cwd-bucket path before returning a stored prefix, reject duplicate ids across buckets, and repeat the id/cwd guards in the coordinator before repair or state publication. Collapse the redundant loadLive hook into loadStored while retaining the existing bucket layout and one-live-writer topology, avoiding flat-layout churn and a locator generic that SQLite and test backends do not need. --- ...18-shared-persistence-write-coordinator.md | 9 +-- ...026-07-20-jsonl-storage-identity.i18n.yaml | 6 ++ .../2026-07-20-jsonl-storage-identity.md | 29 +++++++ .../2026-07-20-jsonl-storage-identity.zh.md | 29 +++++++ .../session-persistence-jsonl/README.md | 7 +- .../session-persistence-jsonl/src/index.ts | 71 ++++++++++-------- .../tests/jsonl.spec.ts | 75 ++++++++++++++++--- .../session-persistence-sqlite/src/index.ts | 5 -- .../session-persistence/README.md | 5 +- .../session-persistence/src/coordinator.ts | 59 ++++++++------- .../tests/persistence.spec.ts | 48 +++++++++--- 11 files changed, 245 insertions(+), 98 deletions(-) create mode 100644 .agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.i18n.yaml create mode 100644 .agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.md create mode 100644 .agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.zh.md diff --git a/.agents/notes/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md b/.agents/notes/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md index 7c73cf24a4..8f26aea91b 100644 --- a/.agents/notes/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md +++ b/.agents/notes/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md @@ -16,11 +16,10 @@ The coordinator retires each live session from its `session/disposed` notificati ### The hook interface (`PersistenceBackend`) -Six methods (five required + an optional lifecycle hook) — the only seam between the coordinator and storage: +Five required members plus an optional lifecycle hook form the only boundary between the coordinator and storage: - `name` — backend label for the dispose-failure `AggregateError`. -- `loadStored(id)` — read a stored prefix by id, scanning ANY storage scope (every JSONL cwd bucket; SQLite's id is globally unique). Used by resume/load and, via `!== undefined`, the create-collision probe. -- `loadLive(id, cwd)` — read a stored prefix SCOPED to `cwd`. **Deliberately distinct from `loadStored`**: HMR live-adoption must only adopt a persisted log at the SAME cwd as the live session; a same-id log at a different cwd is a collision, not a resume. Collapsing the two reintroduces a cross-cwd adoption bug. SQLite ignores `cwd`. +- `loadStored(id)` — read one stored prefix by id across every storage scope (every JSONL cwd bucket; SQLite's id is globally unique). Resume/load, live adoption, and the create-collision probe share this lookup. The coordinator asserts the returned id and rejects a stored/live cwd mismatch before repair or state publication. - `appendBatch(meta, events, isMaterialized)` — durably append a contiguous batch, lazily materializing the session ATOMICALLY when not yet materialized (the materialize-write and the first event batch must commit together — a crash between them must not leave a materialized-but-empty session; this is why there is no separate `materialize` hook). - `commitRepair(meta, tornMarker, closers)` — make a crash repair durable: truncate the torn tail (iff `tornMarker !== undefined`) and append `closers`. **NOT required to be atomic** — JSONL legitimately truncates-then-appends in two fsync'd steps, SQLite does DELETE+INSERT in one transaction. Used by `load` (truncate + synthetic closers) and live-adoption (truncate only, `closers = []`). - `list()` — list all stored metadata. @@ -37,8 +36,8 @@ The shared `runPersistenceContract` (public-API contract) keeps running for ever ## Alternatives considered - **A base class the backends extend** — rejected for composition: a backend exposes only the hooks, cannot reach the coordinator's private orchestration state, and a third-party backend may still implement the abstract service directly without the coordinator at all. -- **A wider hook surface** — each candidate hook folded away: there is no separate `materialize` hook (the materialize-write must commit atomically with the first event batch inside `appendBatch`), no separate create-collision probe (it is `loadStored(id) !== undefined`), and no coordinator pass-through for `list()` (listing needs none of the orchestration). +- **A wider hook surface** — each candidate hook folds away: there is no scope-specific live lookup because `loadStored` plus the coordinator's cwd check preserves the collision boundary, no storage-locator generic because validated JSONL metadata reproduces its path while SQLite is already id-bound, no separate `materialize` hook because the first batch must commit atomically with materialization, no separate create-collision probe because it is `loadStored(id) !== undefined`, and no coordinator pass-through for `list()` because listing needs none of the orchestration. ## Consequences -The coordinator adds one indirection, an opaque torn marker, and detached session-retirement tasks, but centralizes correctness-heavy orchestration previously duplicated by every backend. Session disposal remains an observe-only event, so the session owner does not await persistence retirement; the coordinator contains failures, preserves uncommitted buffers, and makes backend teardown the quiescence boundary. Its hook surface stays narrow: collision checks reuse `loadStored`, materialization stays atomic inside `appendBatch`, and listing bypasses the coordinator. New backends implement storage primitives rather than copy the event-buffer-flush lifecycle. +The coordinator adds one indirection, an opaque torn marker, and detached session-retirement tasks, but centralizes correctness-heavy orchestration previously duplicated by every backend. Session disposal remains an observe-only event, so the session owner does not await persistence retirement; the coordinator contains failures, preserves uncommitted buffers, and makes backend teardown the quiescence boundary. Its hook surface stays narrow: identity, adoption, and collision checks reuse `loadStored`; materialization stays atomic inside `appendBatch`; and listing bypasses the coordinator. New backends implement storage primitives rather than copy the event-buffer-flush lifecycle. diff --git a/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.i18n.yaml new file mode 100644 index 0000000000..2feb8bdf82 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write +2026-07-20-jsonl-storage-identity.md: c22377834244e5749993952a5fe8018b89130c1c +2026-07-20-jsonl-storage-identity.zh.md: 8b9a291772ba7e079c06e0d9e3ab9f285ac1ad7e diff --git a/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.md b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.md new file mode 100644 index 0000000000..c223778342 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.md @@ -0,0 +1,29 @@ +# Agent Note: Bind JSONL session identity before mutation + +Status: implemented + +English | [中文](2026-07-20-jsonl-storage-identity.zh.md) + +## Problem + +JSONL lookup selects a physical log from the requested session id across cwd buckets, while the parsed `SessionHeader` supplies the metadata used by later repair and append operations. Without binding those two facts, a log selected for session A can declare session B's id or cwd and redirect a repair or later append to B's path. The bucket scan also needs a defined result when the same encoded id exists in more than one bucket. SQLite does not share this ambiguity because its primary-key query binds metadata and events to the requested id. + +## Decision + +`loadStored(id)` is the coordinator's single stored-prefix lookup. The JSONL backend scans every cwd bucket, requires at most one matching encoded filename, parses that file, then validates both `header.id === id` and `selectedPath === logPath(root, header.cwd, header.id)` before returning metadata. `list()` applies the same path validation and rejects duplicate ids across buckets. + +The coordinator independently asserts the returned id and compares the stored cwd with a live session's cwd before repair, state publication, or suffix persistence. It keeps a detached copy of validated metadata; JSONL append and repair derive their path from that copy. The `PersistenceBackend` interface therefore needs neither a scope-specific live lookup nor a storage-locator type. + +The backend supports one live writer per session; another backend instance or process must not mutate that session until the owner finishes disposal and all writes stop. + +## Alternatives considered + +**Flatten storage by session id.** A flat namespace makes duplicate publication collide on one path, but path validation and duplicate rejection close the identity defect without changing the project-grouped cwd layout or its consumers. + +**Carry an opaque storage locator through the coordinator.** A locator binds JSONL mutations directly to a selected path, but JSONL can reproduce that path from metadata it has already validated. Adding another generic and argument to SQLite, test backends, append, and repair makes every implementation carry a concept only the file backend needs. + +**Coordinate multiple live writers.** A dedicated coordination service, process-global registry, or cross-process lock would define a new deployment topology rather than repair identity validation. The supported topology has one live writer; no-overwrite hard-link publication still arbitrates an initial same-id creation race. + +## Consequences + +Mismatched, misplaced, and duplicate JSONL logs fail before repair or coordinator state mutation. The cwd-bucket format stays unchanged and needs no migration. Lookup remains proportional to the number of buckets, and one-live-writer ownership remains an explicit limitation. Coordinator and JSONL tests pin rejection before repair, unchanged bytes for both affected logs, path validation during listing, duplicate-id rejection, and cwd collision handling. diff --git a/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.zh.md b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.zh.md new file mode 100644 index 0000000000..8b9a291772 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.zh.md @@ -0,0 +1,29 @@ +# Agent Note: 在变更前绑定 JSONL 会话身份 + +Status: implemented + +[English](2026-07-20-jsonl-storage-identity.md) | 中文 + +## 问题 + +JSONL 查找会根据请求的会话 id 在各个 cwd 分桶目录中选出物理日志，而解析得到的 `SessionHeader` 会提供后续修复和追加操作使用的元数据。如果这两个事实没有绑定，为会话 A 选中的日志就能声明会话 B 的 id 或 cwd，并将修复或后续追加重定向到 B 的路径。当同一个编码后 id 出现在多个分桶目录中时，分桶扫描也必须给出确定的结果。SQLite 不存在这种歧义，因为主键查询会将元数据和事件绑定到请求的 id。 + +## 决策 + +`loadStored(id)` 是协调器唯一的已存前缀查找操作。JSONL 后端扫描所有 cwd 分桶目录，要求匹配编码文件名的日志至多有一个，解析该文件，然后在返回元数据前同时验证 `header.id === id` 和 `selectedPath === logPath(root, header.cwd, header.id)`。`list()` 执行相同的路径验证，并拒绝跨分桶目录重复的 id。 + +协调器会独立断言返回的 id，并在修复、发布状态或持久化后缀之前比较已存 cwd 和活动会话的 cwd。协调器保留一份已验证元数据的独立副本；JSONL 的追加和修复操作根据该副本派生路径。因此，`PersistenceBackend` 接口既不需要限定范围的活动会话查找，也不需要存储定位器类型。 + +后端对每个会话只支持一个活动写入方；在所有者完成资源释放且所有写入停止之前，另一个后端实例或进程不得变更该会话。 + +## 考虑过的替代方案 + +**按会话 id 扁平化存储。** 扁平命名空间会让重复发布在同一路径上冲突，但路径验证和重复项拒绝无需改变按项目分组的 cwd 布局及其消费方，也能消除身份缺陷。 + +**通过协调器传递不透明存储定位器。** 定位器可以将 JSONL 变更直接绑定到选定路径，但 JSONL 可以根据已经验证的元数据重新得到该路径。为 SQLite、测试后端、追加和修复操作增加一个泛型和参数，会让每个实现都承担只有文件后端需要的概念。 + +**协调多个活动写入方。** 专用协调服务、进程级全局注册表或跨进程锁会定义新的部署拓扑，而不是修复身份验证。受支持的拓扑只有一个活动写入方；禁止覆盖的硬链接发布仍会裁决初始的同 id 创建竞争。 + +## 后果 + +JSONL 日志的身份不匹配、位置错误和重复会在修复或协调器状态变更前失败。cwd 分桶格式保持不变，无需迁移。查找开销仍与分桶目录数量成正比，单一活动写入方的所有权仍是明确限制。协调器和 JSONL 测试固定了修复前拒绝、两个受影响日志的字节均保持不变、列出时的路径验证、重复 id 拒绝以及 cwd 冲突处理。 diff --git a/packages/session-persistence/session-persistence-jsonl/README.md b/packages/session-persistence/session-persistence-jsonl/README.md index cf733b85d4..a97fbfd206 100644 --- a/packages/session-persistence/session-persistence-jsonl/README.md +++ b/packages/session-persistence/session-persistence-jsonl/README.md @@ -11,7 +11,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence ``` - The first `.jsonl` line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession?, seedLength? }`; every subsequent line is one `SessionEvent` JSON, **verbatim including `assistant/chunk`** so `seq` stays contiguous (`events[i].seq === i`). -- Session ids are unvalidated branded strings, so they are percent-encoded to a single safe path segment before use (no traversal, no collision). +- Session ids are unvalidated branded strings, so they are injectively encoded as one safe path segment before use (no traversal, no collision). ## Config @@ -23,6 +23,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence ## Durability and crash semantics +- **Bound storage identity.** Lookup requires one matching encoded filename across the cwd buckets, then verifies that the header id equals the requested id and that the header's id/cwd derive the selected path. Listing applies the same path check and rejects duplicate ids. Identity failures occur before repair or append. - **Lazy materialization.** `create(meta)` writes nothing; on the first `append`, the backend writes and `fsync`s a temporary file, publishes it without overwrite via a hard link, then `fsync`s the directory when the host supports it. A created-but-never-appended session leaves nothing on disk and is absent from `list`. - **Append-only.** Committed events (at or below a flushed `turn/end`) are never rewritten. Subsequent appends are line appends at EOF + `fsync`. - **Crash recovery — preserve valid tail work.** `load` keeps the contiguous valid prefix of an interrupted final turn. It truncates from the first unparsable or sequence-gapped uncommitted record, then appends the synthetic tool, step, and turn closers required by the shared [persistence contract](../../../.agents/notes/implemented/architecture/2026-06-14-session-persistence.md); the same defect at or before the last committed `turn/end` rejects. @@ -30,7 +31,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence ## Write path -The plugin buffers frozen session events and drains them on flush or disposal. A per-session cursor prevents resumed sessions from re-appending stored events, and live sessions are seeded when the plugin loads. Operations for one session are serialized; disposal waits for initialization and the final drain so no write lands after teardown. +The plugin buffers frozen session events and drains them on flush or disposal. A per-session cursor prevents resumed sessions from re-appending stored events, and live sessions are seeded when the plugin loads. The owning backend instance serializes operations for one session; disposal waits for initialization and the final drain so no write lands after teardown. ## Model Experience @@ -52,6 +53,6 @@ JSONL storage does not mutate live request prefixes. A resumed loop can reuse pr - **Only the current `SESSION_FORMAT_VERSION` (v0) loads** — the on-disk format is pre-release/unstable: a breaking format change is absorbed at v0 and non-current logs are rejected; there is no migration. - **Nothing deletes session files** — logs accumulate under `root` until removed externally (the seam has no deletion surface). -- **Single-process assumption** — per-session serialization and the write cursor live in this process; two processes appending to the same `root` are not coordinated. +- **One live writer per session** — append and repair are coordinated only inside the owning backend instance. Another backend instance or process must not write the same session until that owner reaches quiescent disposal; initial same-id publication remains collision-safe through the no-overwrite hard link. - **Initial materialization requires hard-link support** — first append uses `link()` so same-id races fail instead of overwriting a committed log; a filesystem that cannot create hard links cannot host this backend. - **Windows cannot `fsync` directory handles through Node** — the backend tolerates only Windows `EPERM` from directory `fsync`; file-content `fsync` remains mandatory, but a crash can lose a newly published directory entry on a host without an equivalent directory-sync primitive. diff --git a/packages/session-persistence/session-persistence-jsonl/src/index.ts b/packages/session-persistence/session-persistence-jsonl/src/index.ts index 4e52cb0b9e..84db4ac541 100644 --- a/packages/session-persistence/session-persistence-jsonl/src/index.ts +++ b/packages/session-persistence/session-persistence-jsonl/src/index.ts @@ -9,7 +9,7 @@ import { Context } from 'cordis' import z from 'schemastery' import { open, mkdir, readFile, readdir, link, rm, truncate } from 'node:fs/promises' -import { dirname, resolve } from 'node:path' +import { dirname, join, resolve } from 'node:path' import { randomBytes } from 'node:crypto' import { SessionPersistence, PersistenceCoordinator, @@ -97,28 +97,19 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi /** Read a stored prefix by id across all cwd buckets when cwd is unknown. */ async loadStored(id: SessionId): Promise | undefined> { - const file = await this.findLog(id) - if (file === undefined) return undefined - return this.readPrefix(file.path) - } - - /** - * Read a stored prefix within one cwd for HMR adoption. `undefined` names the - * no-cwd bucket rather than an unknown cwd, so this never scans other buckets. - */ - async loadLive(id: SessionId, cwd: string | undefined): Promise | undefined> { - const path = logPath(this.root, cwd, id) - if (!await this.exists(path)) return undefined - return this.readPrefix(path) + const path = await this.findLog(id) + if (path === undefined) return undefined + return this.readPrefix(path, id) } /** * Read a stored prefix and convert torn-tail state to the byte offset the * coordinator can round-trip without knowing the file format. */ - private async readPrefix(path: string): Promise> { + private async readPrefix(path: string, expectedId: SessionId): Promise> { const buffer = await readFile(path) const { meta, events, committedBytes } = scanLog(buffer) + this.assertStoredIdentity(path, meta, expectedId) return { meta, events, @@ -145,16 +136,23 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi if (closers.length > 0) await this.appendLines(meta, closers) } - /** List all stored sessions' metadata (header line only — no full-log parse). */ + /** List valid unique stored sessions' metadata (header line only — no full-log parse). */ async list(): Promise { const metas: SessionHeader[] = [] + const ids = new Set() for (const dir of await this.listCwdDirs()) { for (const name of await this.listJsonl(dir)) { + const path = join(dir, name) // Read only headers so listing scales with session count, not log size. - const first = await this.readFirstLine(`${dir}/${name}`) + const first = await this.readFirstLine(path) if (first === undefined) continue // empty/half-written file const meta = parseHeaderMeta(first) if (meta === undefined) continue // not a session header + this.assertStoredIdentity(path, meta) + if (ids.has(meta.id)) { + throw new Error(`duplicate JSONL session id "${meta.id}" appears in multiple cwd buckets`) + } + ids.add(meta.id) metas.push(meta) } } @@ -292,28 +290,41 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi } } - /** - * Find a session by id across cwd buckets for resume. Cwd-scoped HMR adoption - * bypasses this scan so a no-cwd session cannot claim another bucket. - */ - private async findLog(id: SessionId): Promise<{ path: string; cwd: string | undefined } | undefined> { + /** Find the unique physical log for an id across every cwd bucket. */ + private async findLog(id: SessionId): Promise { const target = encodeSegment(id) + '.jsonl' + const matches: string[] = [] for (const dir of await this.listCwdDirs()) { - const path = `${dir}/${target}` - if (await this.exists(path)) { - // Recover the cwd from the header so the caller has the session's bucket. - const { meta } = scanLog(await readFile(path)) - return { path, cwd: meta.cwd } - } + const path = join(dir, target) + if (await this.exists(path)) matches.push(path) + } + if (matches.length > 1) { + throw new Error(`duplicate JSONL session id "${id}" appears in multiple cwd buckets`) + } + return matches[0] + } + + /** Reject metadata that does not identify the selected physical log. */ + private assertStoredIdentity(path: string, meta: SessionHeader, expectedId?: SessionId): void { + if (expectedId !== undefined && meta.id !== expectedId) { + throw new Error(`corrupt session log "${path}": requested id "${expectedId}" does not match header id "${meta.id}"`) + } + let expectedPath: string + try { + expectedPath = logPath(this.root, meta.cwd, meta.id) + } catch (error) { + throw new Error(`corrupt session log "${path}": header id cannot name a storage path`, { cause: error }) + } + if (path !== expectedPath) { + throw new Error(`corrupt session log "${path}": header id "${meta.id}" and cwd belong at "${expectedPath}"`) } - return undefined } /** The cwd-bucket directories under the root (absolute paths). */ private async listCwdDirs(): Promise { try { const entries = await readdir(this.root, { withFileTypes: true }) - return entries.filter(e => e.isDirectory()).map(e => `${this.root}/${e.name}`) + return entries.filter(e => e.isDirectory()).map(e => join(this.root, e.name)) } catch (error) { // Only an absent root means no sessions; rethrow every other I/O failure. if (isENOENT(error)) return [] diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index e7dc469132..ab62bce938 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -21,6 +21,15 @@ function mutableHeader(header: SessionHeader): MutableSessionHeader { return header } +/** Rewrite only a stored header while preserving every event byte below it. */ +async function rewriteHeader(path: string, update: (header: Record) => void): Promise { + const lines = (await readFile(path, 'utf8')).split('\n') + const header = JSON.parse(lines[0] as string) as Record + update(header) + lines[0] = JSON.stringify(header) + await writeFile(path, lines.join('\n')) +} + async function expectParallelFlushError(promise: Promise, message: RegExp): Promise { try { await promise @@ -393,6 +402,31 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => { expect(reloaded.events.map(e => e.seq)).toEqual([0, 1, 2, 3, 4, 5, 6, 7]) }) + it('rejects a mismatched header before repairing either session log', async () => { + const a = meta('identity-a', '/same') + const b = meta('identity-b', '/same') + await ctx.sessionPersistence.create(a) + await ctx.sessionPersistence.append(a.id, [{ + type: 'turn/start', + seq: 0, + time: 1, + data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }, + }]) + await ctx.sessionPersistence.create(b) + await ctx.sessionPersistence.append(b.id, oneTurnLog()) + + const aPath = logPath(root, a.cwd, a.id) + const bPath = logPath(root, b.cwd, b.id) + await rewriteHeader(aPath, (header) => { header.id = b.id }) + const beforeA = await readFile(aPath) + const beforeB = await readFile(bPath) + + await expect(ctx.sessionPersistence.load(a.id)) + .rejects.toThrow(/requested id "identity-a" does not match header id "identity-b"/) + expect(await readFile(aPath)).toEqual(beforeA) + expect(await readFile(bPath)).toEqual(beforeB) + }) + it('rejects a re-append of an already-stored seq', async () => { const m = meta('reappend') await ctx.sessionPersistence.create(m) @@ -599,6 +633,28 @@ describe('SessionPersistenceJsonl: edge cases', () => { expect(ids).toContain('big') }) + it('list rejects a header whose cwd does not identify its physical log', async () => { + const m = meta('misplaced', '/stored') + await ctx.sessionPersistence.create(m) + await ctx.sessionPersistence.append(m.id, oneTurnLog()) + await rewriteHeader(logPath(root, m.cwd, m.id), (header) => { header.cwd = '/elsewhere' }) + + await expect(ctx.sessionPersistence.list()).rejects.toThrow(/and cwd belong at/) + }) + + it('load and list reject one id materialized in multiple cwd buckets', async () => { + const id = SessionId('duplicate') + for (const cwd of ['/a', '/b']) { + const m = meta(id, cwd) + await mkdir(sessionDir(root, cwd), { recursive: true }) + const content = [JSON.stringify(toHeaderLine(m)), ...oneTurnLog().map(event => JSON.stringify(event))].join('\n') + '\n' + await writeFile(logPath(root, cwd, id), content) + } + + await expect(ctx.sessionPersistence.load(id)).rejects.toThrow(/appears in multiple cwd buckets/) + await expect(ctx.sessionPersistence.list()).rejects.toThrow(/appears in multiple cwd buckets/) + }) + it('a DIFFERENT live session object reusing a disposed id gets its own init (no stale cache)', async () => { // Session A materializes a log under id "reuse". const sessFiberA = await ctx.plugin(Object.assign((inner: Context) => { @@ -619,18 +675,16 @@ describe('SessionPersistenceJsonl: edge cases', () => { await expect(ctx.sessions.flush(b)).rejects.toThrow(/already bound to a different live session|already has a persisted log on disk/) }) - it('a NO-CWD live session does NOT cross-cwd-adopt a same-id log from a real cwd bucket (loadLive is scope-exact)', async () => { + it('a no-cwd live session cannot adopt a same-id log from another cwd', async () => { // Backend 1: materialize a log under id "x" in the cwd "/w" bucket, then // dispose the WHOLE backend (so backend 2 mounts with an EMPTY states map — - // the HMR/reload path where onCreated goes through loadLive, not a tracked - // collision). + // the HMR/reload path with no tracked collision state). await ctx.sessionPersistence.create(meta('x', '/w')) await ctx.sessionPersistence.append(SessionId('x'), oneTurnLog()) await ctx.fiber.dispose() - // Backend 2 creates a no-cwd session whose id exists only in `/w`. Exact `loadLive(id, - // undefined)` must not adopt across buckets; the any-cwd collision check then rejects instead - // of grafting no-cwd events onto a log with mismatched cwd. + // Backend 2 creates a no-cwd session whose id exists only in `/w`. The + // stored cwd check rejects instead of grafting no-cwd events onto that log. const ctx2 = new Context() await ctx2.plugin(SessionStore) await ctx2.plugin(SessionPersistenceJsonl, { root }) @@ -638,7 +692,7 @@ describe('SessionPersistenceJsonl: edge cases', () => { await ctx2.plugin(Object.assign((inner: Context) => { b = inner.sessions.create(SessionId('x')) // no cwd }, { inject: ['sessions'] })) - await expect(ctx2.sessions.flush(b)).rejects.toThrow(/already has a persisted log on disk/) + await expect(ctx2.sessions.flush(b)).rejects.toThrow(/different cwd|id collision/) // The "/w" log is untouched — no no-cwd events were grafted onto it, and no // `_no-cwd` log for "x" was created. @@ -706,9 +760,7 @@ describe('SessionPersistenceJsonl: edge cases', () => { await ctx2.fiber.dispose() }) - it('loadLive surfaces a non-ENOENT lookup error (ENOTDIR) instead of reporting absent', async () => { - // A non-ENOENT per-id open error must surface rather than become "not found" and permit false - // live adoption. Making the cwd bucket a regular file forces ENOTDIR for its child log path. + it('materialization surfaces a cwd-bucket storage fault', async () => { const cwd = '/x' const ctx2 = new Context() await ctx2.plugin(SessionStore) @@ -717,8 +769,9 @@ describe('SessionPersistenceJsonl: edge cases', () => { let s!: Session await ctx2.plugin(Object.assign((inner: Context) => { s = inner.sessions.create(SessionId('exists-fault'), { meta: { cwd } }) + appendClosedTurn(s) }, { inject: ['sessions'] })) - await expect(ctx2.sessions.flush(s)).rejects.toThrow(/ENOTDIR/) + await expect(ctx2.sessions.flush(s)).rejects.toThrow(/EEXIST|ENOTDIR/) await ctx2.fiber.dispose() }) diff --git a/packages/session-persistence/session-persistence-sqlite/src/index.ts b/packages/session-persistence/session-persistence-sqlite/src/index.ts index 4661b41309..c7d7770642 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/index.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/index.ts @@ -144,11 +144,6 @@ export class SessionPersistenceSqlite extends SessionPersistence implements Pers return this.readPrefix(id) } - /** Read a stored prefix; `cwd` is ignored (the id is globally unique in SQLite). */ - loadLive(id: SessionId, _cwd: string | undefined): Promise | undefined> { - return this.readPrefix(id) - } - /** * Read a session's row + ordered events into a {@link StoredPrefix}. The * torn-tail marker is the seq from which a never-committed tail must be deleted diff --git a/packages/session-persistence/session-persistence/README.md b/packages/session-persistence/session-persistence/README.md index ca04cd1ab8..7ed713a3d6 100644 --- a/packages/session-persistence/session-persistence/README.md +++ b/packages/session-persistence/session-persistence/README.md @@ -34,14 +34,13 @@ The `PersistenceBackend` hooks (the only seam between the coordinato | Hook | Role | |---|---| | `name` | Backend label for the dispose-failure `AggregateError`. | -| `loadStored(id)` | Read a stored prefix by id, scanning ANY storage scope. Used by resume/load and, via `!== undefined`, the create-collision probe. Returns an opaque `tornMarker` iff a torn tail must be truncated. | -| `loadLive(id, cwd)` | Read a stored prefix SCOPED to `cwd` (HMR live-adoption must only adopt a log at the SAME cwd; a same-id log elsewhere is a collision, not a resume). A globally-unique-id backend ignores `cwd`. | +| `loadStored(id)` | Read a stored prefix by id across every storage scope. Used by resume/load, live adoption, and the create-collision probe. Returned metadata identifies `id`; an opaque `tornMarker` is present iff a torn tail must be truncated. | | `appendBatch(meta, events, isMaterialized)` | Durably append a contiguous batch, lazily materializing ATOMICALLY when not yet materialized. | | `commitRepair(meta, tornMarker, closers)` | Make a crash repair durable: truncate the torn tail (iff `tornMarker !== undefined` — a marker may be falsy, e.g. seq/offset `0`) and append `closers`. NOT required to be atomic. Used by load (truncate + closers) and live-adoption (truncate only). | | `list()` | List all stored metadata. | | `close?()` | Optional lifecycle teardown (e.g. close a db handle), awaited after the dispose drain. | -The `tornMarker` is fully OPAQUE: the coordinator only tests `!== undefined` and round-trips it to `commitRepair`, never inspecting its value (the JSONL backend uses the byte offset to truncate to, the SQLite backend the seq to delete from). The public `SessionPersistence` service shape is unchanged, so a third-party backend MAY still implement the abstract service directly without the coordinator. See [the write-coordinator Agent Note](../../../.agents/notes/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md). +The coordinator asserts the stored id and compares stored/live cwd before repair or live adoption. The `tornMarker` is fully OPAQUE: the coordinator only tests `!== undefined` and round-trips it to `commitRepair`, never inspecting its value (the JSONL backend uses the byte offset to truncate to, the SQLite backend the seq to delete from). The public `SessionPersistence` service shape is unchanged, so a third-party backend MAY still implement the abstract service directly without the coordinator. See [the write-coordinator Agent Note](../../../.agents/notes/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md). ## Testing backends diff --git a/packages/session-persistence/session-persistence/src/coordinator.ts b/packages/session-persistence/session-persistence/src/coordinator.ts index beb5fca483..11e003147c 100644 --- a/packages/session-persistence/session-persistence/src/coordinator.ts +++ b/packages/session-persistence/session-persistence/src/coordinator.ts @@ -35,22 +35,14 @@ export interface PersistenceBackend { readonly name: string /** - * Read a stored prefix by id, scanning ANY storage scope (for JSONL: every - * cwd bucket). Returns `undefined` if no stored artifact exists. Used by - * resume/load, and — via `!== undefined` — by the create-collision probe. - * The returned `tornMarker` is present iff there is a torn tail to truncate. + * Read a stored prefix by id, scanning every backend storage scope. Returns + * `undefined` if no stored artifact exists. Returned metadata must identify + * `id` before repair or state publication. Used by resume/load, live adoption, + * and — via `!== undefined` — the create-collision probe. The returned + * `tornMarker` is present iff there is a torn tail to truncate. */ loadStored(id: SessionId): Promise | undefined> - /** - * Read a stored prefix SCOPED to `cwd`. Deliberately distinct from - * {@link loadStored}: HMR live-adoption must only adopt a persisted log at the - * SAME cwd as the live session (a same-id log at a different cwd is a - * collision, not a resume) — conflating the two reintroduces a cross-cwd - * adoption bug. For a globally-unique-id backend (SQLite) `cwd` is ignored. - */ - loadLive(id: SessionId, cwd: string | undefined): Promise | undefined> - /** * Durably append a CONTIGUOUS batch, lazily materializing the session first * when `!isMaterialized`. The materialize-write and the first event batch MUST @@ -259,6 +251,7 @@ export class PersistenceCoordinator { const stored = await this.backend.loadStored(id) if (stored === undefined) throw new Error(`session "${id}" not found`) const { meta, events, tornMarker } = stored + this.assertStoredId(id, meta) this.assertVersion(meta) assertSupportedEvents(events, id) @@ -317,6 +310,13 @@ export class PersistenceCoordinator { } } + /** Reject backend metadata that is not bound to the requested session id. */ + private assertStoredId(id: SessionId, meta: SessionHeader): void { + if (meta.id !== id) { + throw new Error(`stored session identity mismatch: requested "${id}", header contains "${meta.id}"`) + } + } + // --- write path (session/event → flush drain) --- private installWritePath(): void { @@ -439,6 +439,7 @@ export class PersistenceCoordinator { const stored = await this.backend.loadStored(id) /* v8 ignore next -- a cursor > 0 means the session was materialized, so it exists */ if (stored === undefined) return false + this.assertStoredId(id, stored.meta) return seedCoversPrefix(seed, stored.events.slice(0, cursor)) } @@ -448,9 +449,10 @@ export class PersistenceCoordinator { * Cases, by whether this backend tracks the id and whether an artifact exists: * 1. Already tracked → no-op (or claim ownerless state if the seed matches, * or reclaim a truly-abandoned id, else reject as a collision). - * 2. Not tracked, an artifact EXISTS at this cwd and is a seq-aligned PREFIX - * of the live events → ADOPT it (HMR/reload), persisting any live suffix. - * 3. Not tracked, an artifact EXISTS but is NOT a prefix → REJECT (collision). + * 2. Not tracked, an artifact EXISTS at the same cwd and is a seq-aligned + * PREFIX of the live events → ADOPT it, persisting any live suffix. + * 3. Not tracked, an artifact EXISTS at another cwd or is NOT a prefix → + * REJECT (collision). * 4. Not tracked and NO artifact → a genuinely new session: register meta * (lazy) and persist its seed once. */ @@ -464,14 +466,11 @@ export class PersistenceCoordinator { if (tracked.owner === undefined) { // Ownerless state from the public create()/load() API. The FIRST live // session claims it — but ONLY if BOTH the cwd scope and the seed match. - // The cwd guard mirrors case-2's cwd-scoped loadLive(): a same-id - // ownerless artifact at a DIFFERENT cwd is a collision, not a claim - // (claiming it would append the live cwd's events under the stored - // header's cwd, the exact cross-cwd corruption the loadLive scope - // prevents). The seed guard then ensures the live events reproduce the - // persisted prefix (else a fresh, unrelated session reusing the id would - // have its seq 0..cursor-1 events filtered as already-written and - // grafted on). + // A same-id ownerless artifact at a different cwd is a collision, not a + // claim: accepting it would append this live session's events through + // the stored header's cwd. The seed guard then ensures the live events + // reproduce the persisted prefix; otherwise a fresh session reusing the + // id could have its leading events filtered as already written. if (tracked.meta.cwd !== session.header.cwd) { throw new Error(`session "${id}" is already persisted at a different cwd (persisted: ${String(tracked.meta.cwd)}, live: ${String(session.header.cwd)}) (id collision)`) } @@ -495,11 +494,9 @@ export class PersistenceCoordinator { } } - // case 2/3: an artifact at THIS cwd is adopted as a live prefix (or rejected - // as a collision inside adoptLivePrefix). cwd-scoped (loadLive), never - // any-scope: a same-id artifact at a different cwd is a collision, not a - // resume. - const live = await this.backend.loadLive(id, session.header.cwd) + // case 2/3: resolve the id once across storage, then let adoption reject a + // cwd mismatch before repair or state publication. + const live = await this.backend.loadStored(id) if (live !== undefined) { // Do NOT route through loadCore(): that crash-repairs open turns as // interrupted, which is wrong for HMR while the live Session is still the @@ -528,6 +525,10 @@ export class PersistenceCoordinator { */ private async adoptLivePrefix(session: Session, seed: readonly SessionEvent[], stored: StoredPrefix): Promise { const { meta, events, tornMarker } = stored + this.assertStoredId(session.header.id, meta) + if (meta.cwd !== session.header.cwd) { + throw new Error(`session "${session.header.id}" is already persisted at a different cwd (persisted: ${String(meta.cwd)}, live: ${String(session.header.cwd)}) (id collision)`) + } this.assertVersion(meta) assertSupportedEvents(events, session.header.id) if (!seedCoversPrefix(seed, events)) { diff --git a/packages/session-persistence/session-persistence/tests/persistence.spec.ts b/packages/session-persistence/session-persistence/tests/persistence.spec.ts index e083ac3543..21d878a6a7 100644 --- a/packages/session-persistence/session-persistence/tests/persistence.spec.ts +++ b/packages/session-persistence/session-persistence/tests/persistence.spec.ts @@ -63,7 +63,7 @@ class MemoryPersistence extends SessionPersistence implements PersistenceBackend super(ctx) // Assign the store BEFORE constructing the coordinator: the coordinator's // constructor installs the write path and synchronously seeds existing live - // sessions (onCreated → loadLive → this.store), so store must exist first. + // sessions through loadStored(), so store must exist first. this.store = config?.store ?? new Map() this.coordinator = new PersistenceCoordinator(this.ctx, this) } @@ -88,18 +88,13 @@ class MemoryPersistence extends SessionPersistence implements PersistenceBackend // --- PersistenceBackend hooks (the Map storage primitives) --- - // A Map-backed store has no torn tails, so `tornMarker` is never set. Ids are - // globally unique, so loadStored and loadLive are identical (cwd is ignored). + // A Map-backed store has no torn tails, so `tornMarker` is never set. async loadStored(id: SessionId): Promise | undefined> { const entry = this.store.get(id) if (!entry) return undefined return { meta: structuredClone(entry.meta), events: structuredClone(entry.events) } } - loadLive(id: SessionId, _cwd: string | undefined): Promise | undefined> { - return this.loadStored(id) - } - async appendBatch(m: SessionHeader, events: readonly SessionEvent[], _isMaterialized: boolean): Promise { // Defense-in-depth: the coordinator already validates serializability, but a // durable store must reject non-JSON data at its own boundary too. @@ -137,6 +132,7 @@ class ControlledBackend implements PersistenceBackend { readonly lifecycle: string[] = [] appendAttempts = 0 loadAttempts = 0 + repairAttempts = 0 beforeAppend?: (attempt: number) => Promise beforeLoadStored?: (attempt: number) => Promise @@ -147,10 +143,6 @@ class ControlledBackend implements PersistenceBackend { return { meta: structuredClone(entry.meta), events: structuredClone(entry.events) } } - loadLive(id: SessionId, _cwd: string | undefined): Promise | undefined> { - return this.loadStored(id) - } - async appendBatch(m: SessionHeader, events: readonly SessionEvent[], _isMaterialized: boolean): Promise { const attempt = ++this.appendAttempts await this.beforeAppend?.(attempt) @@ -162,7 +154,9 @@ class ControlledBackend implements PersistenceBackend { } } - async commitRepair(_m: SessionHeader, _tornMarker: undefined, _closers: readonly SessionEvent[]): Promise {} + async commitRepair(_m: SessionHeader, _tornMarker: undefined, _closers: readonly SessionEvent[]): Promise { + this.repairAttempts += 1 + } async list(): Promise { return [...this.store.values()].map(entry => structuredClone(entry.meta)) @@ -194,6 +188,36 @@ runCoordinatorContract('memory', async (): Promise => { } }) +describe('PersistenceCoordinator stored identity', () => { + it('rejects a mismatched backend header before repair or state publication', async () => { + const ctx = new Context() + await ctx.plugin(SessionStore) + const backend = new ControlledBackend() + const requested = SessionId('requested') + backend.store.set(requested, { + meta: meta('different'), + events: [{ + type: 'turn/start', + seq: 0, + time: 1, + data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }, + }], + }) + let coordinator!: PersistenceCoordinator + const fiber = await ctx.plugin(Object.assign((inner: Context) => { + coordinator = new PersistenceCoordinator(inner, backend) + }, { inject: ['sessions'] })) + try { + await expect(coordinator.load(requested)).rejects.toThrow(/stored session identity mismatch/) + expect(backend.repairAttempts).toBe(0) + expect((coordinator as unknown as CoordinatorInternals).states.size).toBe(0) + } finally { + await fiber.dispose() + await ctx.fiber.dispose() + } + }) +}) + describe('PersistenceCoordinator retirement', () => { it('a retiring unmaterialized owner without buffered events releases its id', async () => { const ctx = new Context() From c9d3d5d557afb3f5376c50f13d44e66a451ca630 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:41:40 +0800 Subject: [PATCH 015/113] fix(jsonl): reject unusable roots at plugin load A configured root that already exists as a file or unreadable directory cannot host cwd buckets, but the backend previously mounted and deferred that deterministic configuration error until a later list or write. Probe the resolved root while the plugin loads, surface every error except ENOENT, and keep an absent root valid for lazy first materialization. Document the timing contract, regenerate the config catalog, and pin the non-directory case at the load boundary. --- .../2026-07-20-jsonl-storage-identity.i18n.yaml | 4 ++-- .../bug-fix/2026-07-20-jsonl-storage-identity.md | 4 ++-- .../2026-07-20-jsonl-storage-identity.zh.md | 4 ++-- docs/config-catalog.md | 6 ++++-- .../session-persistence-jsonl/README.md | 2 +- .../session-persistence-jsonl/src/index.ts | 16 +++++++++++++++- .../tests/jsonl.spec.ts | 7 ++----- 7 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.i18n.yaml index 2feb8bdf82..f907cf276b 100644 --- a/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.i18n.yaml +++ b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-20-jsonl-storage-identity.md: c22377834244e5749993952a5fe8018b89130c1c -2026-07-20-jsonl-storage-identity.zh.md: 8b9a291772ba7e079c06e0d9e3ab9f285ac1ad7e +2026-07-20-jsonl-storage-identity.md: 1ada16791f411a54fbcf9271c7d7963223bbe683 +2026-07-20-jsonl-storage-identity.zh.md: 8027c51dbf6c7d01463b7851d859a40890bf03e1 diff --git a/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.md b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.md index c223778342..1ada16791f 100644 --- a/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.md +++ b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.md @@ -14,7 +14,7 @@ JSONL lookup selects a physical log from the requested session id across cwd buc The coordinator independently asserts the returned id and compares the stored cwd with a live session's cwd before repair, state publication, or suffix persistence. It keeps a detached copy of validated metadata; JSONL append and repair derive their path from that copy. The `PersistenceBackend` interface therefore needs neither a scope-specific live lookup nor a storage-locator type. -The backend supports one live writer per session; another backend instance or process must not mutate that session until the owner finishes disposal and all writes stop. +An existing configured JSONL root must be a readable directory when the plugin loads. An absent root remains valid and is created on first materialization. The backend supports one live writer per session; another backend instance or process must not mutate that session until the owner finishes disposal and all writes stop. ## Alternatives considered @@ -26,4 +26,4 @@ The backend supports one live writer per session; another backend instance or pr ## Consequences -Mismatched, misplaced, and duplicate JSONL logs fail before repair or coordinator state mutation. The cwd-bucket format stays unchanged and needs no migration. Lookup remains proportional to the number of buckets, and one-live-writer ownership remains an explicit limitation. Coordinator and JSONL tests pin rejection before repair, unchanged bytes for both affected logs, path validation during listing, duplicate-id rejection, and cwd collision handling. +Mismatched, misplaced, and duplicate JSONL logs fail before repair or coordinator state mutation. The cwd-bucket format stays unchanged and needs no migration. Lookup remains proportional to the number of buckets, and one-live-writer ownership remains an explicit limitation. Coordinator and JSONL tests pin rejection before repair, unchanged bytes for both affected logs, path validation during listing, duplicate-id rejection, cwd collision handling, and load-time root validation. diff --git a/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.zh.md b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.zh.md index 8b9a291772..8027c51dbf 100644 --- a/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.zh.md +++ b/.agents/notes/implemented/bug-fix/2026-07-20-jsonl-storage-identity.zh.md @@ -14,7 +14,7 @@ JSONL 查找会根据请求的会话 id 在各个 cwd 分桶目录中选出物 协调器会独立断言返回的 id，并在修复、发布状态或持久化后缀之前比较已存 cwd 和活动会话的 cwd。协调器保留一份已验证元数据的独立副本；JSONL 的追加和修复操作根据该副本派生路径。因此，`PersistenceBackend` 接口既不需要限定范围的活动会话查找，也不需要存储定位器类型。 -后端对每个会话只支持一个活动写入方；在所有者完成资源释放且所有写入停止之前，另一个后端实例或进程不得变更该会话。 +如果配置的 JSONL 根目录已存在，插件加载时该路径必须是可读目录。根目录不存在仍然是有效配置，首次物化时会创建该目录。后端对每个会话只支持一个活动写入方；在所有者完成资源释放且所有写入停止之前，另一个后端实例或进程不得变更该会话。 ## 考虑过的替代方案 @@ -26,4 +26,4 @@ JSONL 查找会根据请求的会话 id 在各个 cwd 分桶目录中选出物 ## 后果 -JSONL 日志的身份不匹配、位置错误和重复会在修复或协调器状态变更前失败。cwd 分桶格式保持不变，无需迁移。查找开销仍与分桶目录数量成正比，单一活动写入方的所有权仍是明确限制。协调器和 JSONL 测试固定了修复前拒绝、两个受影响日志的字节均保持不变、列出时的路径验证、重复 id 拒绝以及 cwd 冲突处理。 +JSONL 日志的身份不匹配、位置错误和重复会在修复或协调器状态变更前失败。cwd 分桶格式保持不变，无需迁移。查找开销仍与分桶目录数量成正比，单一活动写入方的所有权仍是明确限制。协调器和 JSONL 测试固定了修复前拒绝、两个受影响日志的字节均保持不变、列出时的路径验证、重复 id 拒绝、cwd 冲突处理以及加载时的根目录验证。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index c2d31f93e3..c0ccd5f8ee 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -735,13 +735,15 @@ export interface Config { /** * Root directory for all session files. Required (no default): a default of * `process.cwd()` would scatter session files as the process's cwd changes - * (bash calls, subprocesses). Sessions group under per-cwd subdirectories. + * (bash calls, subprocesses). Sessions group under per-cwd subdirectories. An + * existing root must be a readable directory; an absent root is created on + * first materialization. */ root: string } ``` -Source: [`packages/session-persistence/session-persistence-jsonl/src/index.ts:24`](../packages/session-persistence/session-persistence-jsonl/src/index.ts) +Source: [`packages/session-persistence/session-persistence-jsonl/src/index.ts:25`](../packages/session-persistence/session-persistence-jsonl/src/index.ts) ## `@deepseek-ai/dsh-session-persistence-sqlite` diff --git a/packages/session-persistence/session-persistence-jsonl/README.md b/packages/session-persistence/session-persistence-jsonl/README.md index a97fbfd206..f2a6a0a9b7 100644 --- a/packages/session-persistence/session-persistence-jsonl/README.md +++ b/packages/session-persistence/session-persistence-jsonl/README.md @@ -17,7 +17,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence | Key | Type | Notes | |---|---|---| -| `root` | `string` (required) | Root directory for all session files. **No default** — a `process.cwd()` default would scatter files as the process's cwd changes (bash calls, subprocesses). | +| `root` | `string` (required) | Root directory for all session files. **No default** — a `process.cwd()` default would scatter files as the process's cwd changes (bash calls, subprocesses). An existing root must be a readable directory; an absent root is created on first materialization. | `locate(meta)` returns `{ kind: 'jsonl', path }` using the resolved absolute root and the same cwd-bucket/id encoding as materialization. It performs no filesystem I/O: the target can be returned before the file exists, and an existing file contains only the last flushed prefix. diff --git a/packages/session-persistence/session-persistence-jsonl/src/index.ts b/packages/session-persistence/session-persistence-jsonl/src/index.ts index 84db4ac541..a3fdb8b440 100644 --- a/packages/session-persistence/session-persistence-jsonl/src/index.ts +++ b/packages/session-persistence/session-persistence-jsonl/src/index.ts @@ -8,6 +8,7 @@ import { Context } from 'cordis' import z from 'schemastery' +import { readdirSync } from 'node:fs' import { open, mkdir, readFile, readdir, link, rm, truncate } from 'node:fs/promises' import { dirname, join, resolve } from 'node:path' import { randomBytes } from 'node:crypto' @@ -25,7 +26,9 @@ export interface Config { /** * Root directory for all session files. Required (no default): a default of * `process.cwd()` would scatter session files as the process's cwd changes - * (bash calls, subprocesses). Sessions group under per-cwd subdirectories. + * (bash calls, subprocesses). Sessions group under per-cwd subdirectories. An + * existing root must be a readable directory; an absent root is created on + * first materialization. */ root: string } @@ -64,6 +67,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi super(ctx) // Resolve once so later process.cwd() changes cannot split one backend across roots. this.root = resolve(config.root) + this.assertUsableRoot() this.coordinator = new PersistenceCoordinator(this.ctx, this) } @@ -304,6 +308,16 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi return matches[0] } + /** Require an existing configured root to be a readable directory. */ + private assertUsableRoot(): void { + try { + readdirSync(this.root) + } catch (error) { + if (isENOENT(error)) return + throw error + } + } + /** Reject metadata that does not identify the selected physical log. */ private assertStoredIdentity(path: string, meta: SessionHeader, expectedId?: SessionId): void { if (expectedId !== undefined && meta.id !== expectedId) { diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index ab62bce938..d68a48a72d 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -748,15 +748,12 @@ describe('SessionPersistenceJsonl: edge cases', () => { await ctx2.fiber.dispose() }) - it('list surfaces a non-ENOENT root error (ENOTDIR) instead of reporting no sessions', async () => { - // A durable backend must not collapse a storage fault to "no sessions". Making the root a - // regular file forces ENOTDIR from `readdir`, which must propagate. + it('plugin load rejects an existing root that is not a directory', async () => { const filePath = join(root, 'not-a-dir') await writeFile(filePath, 'x') const ctx2 = new Context() await ctx2.plugin(SessionStore) - await ctx2.plugin(SessionPersistenceJsonl, { root: filePath }) - await expect(ctx2.sessionPersistence.list()).rejects.toThrow(/ENOTDIR/) + await expect(ctx2.plugin(SessionPersistenceJsonl, { root: filePath })).rejects.toThrow(/ENOTDIR/) await ctx2.fiber.dispose() }) From f4c9e53a2a466abff7eeb463dfe643538d9c2d24 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:42:03 +0800 Subject: [PATCH 016/113] docs(persistence): state backend ownership precisely The shared coordinator serializes operations within one backend instance; it does not coordinate multiple instances writing the same on-disk session. Remove prose that implied unsupported shared-writer semantics. Also update the older seam-simplification note to describe the surviving loadStored existence probe instead of the removed loadLive hook, so implemented documentation matches the current contract. --- .../simplification/2026-06-20-prune-dead-seam-methods.md | 2 +- docs/core-data-structures/persistence.md | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.agents/notes/implemented/simplification/2026-06-20-prune-dead-seam-methods.md b/.agents/notes/implemented/simplification/2026-06-20-prune-dead-seam-methods.md index 782ffe891e..f74de250ef 100644 --- a/.agents/notes/implemented/simplification/2026-06-20-prune-dead-seam-methods.md +++ b/.agents/notes/implemented/simplification/2026-06-20-prune-dead-seam-methods.md @@ -12,7 +12,7 @@ A capability seam ([interface / implementation / consumer](../architecture/2026- The abstract service declared its operations beyond create/append: `load`, `list`, `has`, `delete`. Production consumers of `ctx.sessionPersistence` use only two: the agent-loop resume path calls `load()` ([packages/core/agent-loop/src/index.ts:176](../../../../packages/core/agent-loop/src/index.ts)), and the ACP bridge calls `list()` for `session/list` ([packages/ui/acp/src/index.ts:494](../../../../packages/ui/acp/src/index.ts)). Grepping every `sessionPersistence.*` / `persistence.*` use across `packages/*/src` and `examples/` finds no `has(` and no `delete(` on the service. The `.has(`/`.delete(` calls in `packages/ui/acp/src/index.ts` are on the in-memory `SessionStore` and a local `Set` of loading ids, not persistence. The only callers of `has`/`delete` were the contract suites and per-backend specs. -`has()` was not just unused — it was the most intricate branch in the shared coordinator: a tracked-vs-untracked dual-probe (`loadLive(id, cwd)` for a live-tracked session vs `loadStored(id)` for an untracked one) with a multi-line rationale. `delete()` dragged the `deleteStored` backend hook that every backend had to implement. This is the [drop-mutable-session-summary](2026-06-19-drop-mutable-session-summary.md) pattern: a contract test exercised both, but no shipping code asks "is this session persisted?" or removes one. +`has()` was not just unused: it added a tracked-vs-untracked coordinator probe and a contract branch even though `loadStored(id)` already owns durable existence checks. `delete()` dragged the `deleteStored` backend hook that every backend had to implement. This is the [drop-mutable-session-summary](2026-06-19-drop-mutable-session-summary.md) pattern: a contract test exercised both, but no shipping code asks "is this session persisted?" or removes one. ## Decision diff --git a/docs/core-data-structures/persistence.md b/docs/core-data-structures/persistence.md index a80b9bc896..c88df10294 100644 --- a/docs/core-data-structures/persistence.md +++ b/docs/core-data-structures/persistence.md @@ -97,5 +97,3 @@ Both implement the same abstract `SessionPersistence` (locate/create/append/load - **[dsh-session-persistence-jsonl](../../packages/session-persistence/session-persistence-jsonl)** — an append-only JSONL log per session with crash-safe atomic writes, the interrupted-turn crash recovery above, and a read/replay path. - **[dsh-session-persistence-sqlite](../../packages/session-persistence/session-persistence-sqlite)** — `node:sqlite`, one row per `SessionEvent`. The row shape `(session_id, seq, type, time, data, source_event_seqs, surface_op)` maps 1:1 onto the event, including optional surface metadata, so there is no parallel persisted schema to keep in sync. - -Multiple backends sharing one on-disk session coordinate writes through the [shared persistence write-coordinator](../../.agents/notes/implemented/architecture/2026-06-18-shared-persistence-write-coordinator.md). From 7a9177d624ac06830a1d242c4a969ed0a97acd64 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:46:51 +0800 Subject: [PATCH 017/113] test(jsonl): pin corruption and storage-fault rejection Identity validation must reject a header id that cannot derive a path, and only ENOENT may mean that storage is absent. Other root or per-path failures must remain visible instead of becoming an empty list or false miss. Exercise those branches with narrow storage-mechanics cases, preserving per-file 100% coverage without restoring the flat-layout or multi-writer tests removed from the replacement design. --- .../tests/jsonl.spec.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index d68a48a72d..8589fcd2f9 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -642,6 +642,16 @@ describe('SessionPersistenceJsonl: edge cases', () => { await expect(ctx.sessionPersistence.list()).rejects.toThrow(/and cwd belong at/) }) + it('list rejects a session header whose id cannot name a storage path', async () => { + const bucket = sessionDir(root, undefined) + await mkdir(bucket, { recursive: true }) + await writeFile(join(bucket, 'invalid-id.jsonl'), JSON.stringify({ + type: 'session', version: 0, id: '', createdAt: 1, + }) + '\n') + + await expect(ctx.sessionPersistence.list()).rejects.toThrow(/header id cannot name a storage path/) + }) + it('load and list reject one id materialized in multiple cwd buckets', async () => { const id = SessionId('duplicate') for (const cwd of ['/a', '/b']) { @@ -757,6 +767,21 @@ describe('SessionPersistenceJsonl: edge cases', () => { await ctx2.fiber.dispose() }) + it('list surfaces a root that becomes unusable after plugin load', async () => { + await rm(root, { recursive: true }) + await writeFile(root, 'not a directory') + + await expect(ctx.sessionPersistence.list()).rejects.toThrow(/ENOTDIR/) + }) + + it('per-id lookup surfaces non-ENOENT storage errors', async () => { + const blocker = join(root, 'not-a-directory') + await writeFile(blocker, 'x') + const backend = ctx.sessionPersistence as unknown as { exists(path: string): Promise } + + await expect(backend.exists(join(blocker, 'child.jsonl'))).rejects.toThrow(/ENOTDIR/) + }) + it('materialization surfaces a cwd-bucket storage fault', async () => { const cwd = '/x' const ctx2 = new Context() From 64d70670e8e5fe0d36151f3ad6cb4f7d20b96a7d Mon Sep 17 00:00:00 2001 From: ZiyaZhang <199893125+ZiyaZhang@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:46:46 -0700 Subject: [PATCH 018/113] docs(i18n): address ds-review-bot on the v4 restoration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 解析器：三段闭合标签改为行首锚定（正文提及 不再截 断）、重复段全文扫描并校验段序（补两条回归测试）。切换行：v4 模板 保持字面占位不变，资产文档写明全新配对由流水线在解析后按目标文件 名机械插入、配对门禁兜底。加粗后空格限定于字母/数字/汉字、标点前 一律不加；RFC 2119 关键词改为保留源侧强调标记（斜体归斜体、加粗 归加粗）。i18n README 双侧同步 v4 契约描述（不再承诺 CDATA 协议与 规则注入）并重录配对。 --- docs/i18n/README.i18n.yaml | 4 ++-- docs/i18n/README.md | 2 +- docs/i18n/README.zh.md | 2 +- docs/i18n/translation-prompt.md | 6 ++++-- scripts/translation-prompt.spec.ts | 14 ++++++++++++-- scripts/translation-prompt.ts | 24 ++++++++++++------------ 6 files changed, 32 insertions(+), 20 deletions(-) diff --git a/docs/i18n/README.i18n.yaml b/docs/i18n/README.i18n.yaml index ab1c9024ad..a1ff3a701e 100644 --- a/docs/i18n/README.i18n.yaml +++ b/docs/i18n/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -README.md: 17bb1eeb67b4f5119a698fca23f12490c9378a7f -README.zh.md: c957a82bf420a942e2249942a2d9afc54ad950cf +README.md: 3980aef52545aeb7c8ec44856cb95c7c0f7c7f22 +README.zh.md: 39e03cb6d33c008d2f310915db6b6bebd638cded diff --git a/docs/i18n/README.md b/docs/i18n/README.md index 17bb1eeb67..3980aef525 100644 --- a/docs/i18n/README.md +++ b/docs/i18n/README.md @@ -49,4 +49,4 @@ The gate's limit, stated plainly: **a green gate means the pair was confirmed co ## Division of labor -Counterparts here are produced by an agent running [dsh-translate-docs](../../.agents/skills/dsh-translate-docs/SKILL.md) and reviewed by a human — inference is cheap here, review attention is the scarce resource. The gate checks pair completeness, recorded hashes, switchers, and its documented structural signature. Review still owns translation quality, terminology, and structural requirements that the signature does not encode. The prompt contract is executable: [scripts/translation-prompt.ts](../../scripts/translation-prompt.ts) renders the canonical rules into either direction and strictly parses the three-field XML response, while `verify-translation-prompt` exercises both render directions, the checked-in example, and the CDATA split rule in `doc-sync`. +Counterparts here are produced by an agent running [dsh-translate-docs](../../.agents/skills/dsh-translate-docs/SKILL.md) and reviewed by a human — inference is cheap here, review attention is the scarce resource. The gate checks pair completeness, recorded hashes, switchers, and its documented structural signature. Review still owns translation quality, terminology, and structural requirements that the signature does not encode. The prompt contract is executable: [scripts/translation-prompt.ts](../../scripts/translation-prompt.ts) renders the committed template (terminology injected; the template carries its own calibrated rules) into either direction and parses the three-section response, while `verify-translation-prompt` exercises both render directions and the checked-in example in `doc-sync`. diff --git a/docs/i18n/README.zh.md b/docs/i18n/README.zh.md index c957a82bf4..39e03cb6d3 100644 --- a/docs/i18n/README.zh.md +++ b/docs/i18n/README.zh.md @@ -49,4 +49,4 @@ ## 分工 -对侧译文由运行 [dsh-translate-docs](../../.agents/skills/dsh-translate-docs/SKILL.md) 的 agent 生成，再由人评审：在这里推理（inference）很便宜，评审注意力才是稀缺资源。门禁负责检查配对是否完整、记录的 hash、语言切换行以及本文列出的结构签名；翻译质量、术语和签名未涵盖的结构要求仍由评审把关。prompt 契约也有可执行实现：[scripts/translation-prompt.ts](../../scripts/translation-prompt.ts) 会把权威规则渲染到英译中或中译英的 prompt 中，并严格解析包含三个字段的 XML 响应；`doc-sync` 中的 `verify-translation-prompt` 会检查两个渲染方向、仓库内示例与 CDATA 拆分规则。 +对侧译文由运行 [dsh-translate-docs](../../.agents/skills/dsh-translate-docs/SKILL.md) 的 agent 生成，再由人评审：在这里推理（inference）很便宜，评审注意力才是稀缺资源。门禁负责检查配对是否完整、记录的 hash、语言切换行以及本文列出的结构签名；翻译质量、术语和签名未涵盖的结构要求仍由评审把关。prompt 契约也有可执行实现：[scripts/translation-prompt.ts](../../scripts/translation-prompt.ts) 会把进仓模板（注入术语表；模板自带经人工校准的规则）渲染到英译中或中译英的 prompt 中，并解析三段式响应；`doc-sync` 中的 `verify-translation-prompt` 会检查两个渲染方向与仓库内示例。 diff --git a/docs/i18n/translation-prompt.md b/docs/i18n/translation-prompt.md index d59170b00a..91b52bbb51 100644 --- a/docs/i18n/translation-prompt.md +++ b/docs/i18n/translation-prompt.md @@ -14,6 +14,8 @@ 流水线只识别上表中的占位符，并且一次翻译整篇文档。它不支持 `{{to}}`、`{{title_prompt}}`、`{{summary_prompt}}`、`{{terms_prompt}}`、`{{imt_style_guide}}`、`{{translation_rules}}` 或 `%%` 分段协议；输出采用模板正文规定的三段 XML，流水线解析取 `` 段。 +语言切换行：已有配对的源文件自带切换行，模型按模板规则翻转即可。全新配对的源文件没有切换行，模型也无从得知文件名——此时由流水线在解析 `` 后按目标文件名插入或校正切换行（机械后处理，配对门禁兜底校验）。 + ## Few-shot 金标 流水线使用**整篇文档**的中英对照作为 few-shot，不是模板内嵌的句子级正误例。以下 5 组配对文档均经过人工评审，以仓库当前版本为准、随仓库更新： @@ -41,7 +43,7 @@ You are a senior technical translator specializing in LLM and agent development - Inline code spans (commands, flags, paths, API names, version numbers) must be kept verbatim. Never translate or reformat them. - Every relative link must point to the same target as in the source. Link text is translated; link targets are not. - Language switcher line: when translating into Chinese, write `[English](source-filename.md) | 中文`. When translating into English, write `English | [中文](source-filename.zh.md)`. Do NOT copy the switcher line from the source file unchanged — you must flip the link direction. -- After a closing bold marker `**`, always insert a space before the next character. +- After a closing bold marker `**`, insert a space before the next character when that character is a Latin letter, digit, or CJK ideograph. Never insert a space before any punctuation (full-width or half-width). ### Tone and Style - The translation must read as if originally written in the target language by a native speaker. If an expression sounds like a word-for-word rendering from the source language, rephrase it. @@ -73,7 +75,7 @@ You are a senior technical translator specializing in LLM and agent development - Use enumeration commas (、) between parallel items, not regular commas. - List item endings: use semicolons or no punctuation. Do not end list items with commas. - Put one half-width space between Chinese text and Latin words/numbers. -- For RFC 2119 keywords (MUST, MUST NOT, SHOULD, MAY), render the corresponding Chinese term in italics: *必须*、*禁止*、*应当*、*可以*. +- For RFC 2119 keywords (MUST, MUST NOT, SHOULD, MAY), translate to the corresponding Chinese term (必须、禁止、应当、可以) and keep the SOURCE emphasis marker: plain source stays plain in italics (*必须*), bold source stays bold (**必须**). #### When translating into English (To be added.) diff --git a/scripts/translation-prompt.spec.ts b/scripts/translation-prompt.spec.ts index db3f8b31a0..45ba97b8e7 100644 --- a/scripts/translation-prompt.spec.ts +++ b/scripts/translation-prompt.spec.ts @@ -42,9 +42,19 @@ describe('translation response sections', () => { expect(parseTranslationResponse(fenced).final).toBe('A') }) + it('keeps an inline close tag inside prose from terminating the section', () => { + const doc = { translation: 'the wire format uses as its close tag', review: '- 无修正', final: 'F' } + expect(parseTranslationResponse(renderTranslationResponse(doc))).toEqual(doc) + }) + + it('rejects a duplicate section appearing before final', () => { + const early = '\nA\n\n\nB\n\n\nR\n\n\nF\n' + expect(() => parseTranslationResponse(early)).toThrow(/duplicate /) + }) + it('rejects missing, unterminated, or duplicated sections', () => { - expect(() => parseTranslationResponse('\nA\n')).toThrow(/missing /) - expect(() => parseTranslationResponse('\nA')).toThrow(/unterminated /) + expect(() => parseTranslationResponse('\nA\n')).toThrow(/missing or unterminated /) + expect(() => parseTranslationResponse('\nA')).toThrow(/missing or unterminated /) const dup = '\nA\n\n\nR\n\n\nF\n\n\nG\n' expect(() => parseTranslationResponse(dup)).toThrow(/duplicate /) }) diff --git a/scripts/translation-prompt.ts b/scripts/translation-prompt.ts index 0556ff39fa..25f1d9ae09 100644 --- a/scripts/translation-prompt.ts +++ b/scripts/translation-prompt.ts @@ -81,6 +81,10 @@ export function renderTranslationResponse(response: TranslationResponse): string * and in order; bodies are raw Markdown taken verbatim between the tags. * A fenced ```xml wrapper around the whole response is tolerated, matching * the shape some models echo back from the prompt's own example. + * + * Section close tags are matched at line starts (the wire shape the prompt + * example establishes), so a tag mentioned inline in translated prose does + * not terminate its section early. */ export function parseTranslationResponse(text: string): TranslationResponse { let body = text.trim() @@ -88,20 +92,16 @@ export function parseTranslationResponse(text: string): TranslationResponse { if (fenced?.[1] !== undefined) body = fenced[1].trim() const values: Partial> = {} - let cursor = 0 for (const section of RESPONSE_SECTIONS) { - const open = `<${section}>` - const close = `` - const start = body.indexOf(open, cursor) - if (start === -1) throw new Error(`translation response: missing <${section}> section`) - const end = body.indexOf(close, start + open.length) - if (end === -1) throw new Error(`translation response: unterminated <${section}> section`) - values[section] = body.slice(start + open.length, end).replace(/^\n/, '').replace(/\n$/, '') - cursor = end + close.length + const pattern = new RegExp(`^<${section}>\\n?([\\s\\S]*?)\\n?^$`, 'gm') + const first = pattern.exec(body) + if (first?.[1] === undefined) throw new Error(`translation response: missing or unterminated <${section}> section`) + if (pattern.exec(body) !== null) throw new Error(`translation response: duplicate <${section}> section`) + values[section] = first[1] } - for (const section of RESPONSE_SECTIONS) { - const again = body.indexOf(`<${section}>`, cursor) - if (again !== -1) throw new Error(`translation response: duplicate <${section}> section`) + const order = RESPONSE_SECTIONS.map(section => body.search(new RegExp(`^<${section}>`, 'm'))) + if (!(order[0]! < order[1]! && order[1]! < order[2]!)) { + throw new Error('translation response: sections must appear in translation, review, final order') } return values as TranslationResponse } From 0517d7487538331b9468ab0fe1ff6e5f655f48d7 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:03:23 +0800 Subject: [PATCH 019/113] test(persistence): use stored-prefix lookup --- .../session-persistence-jsonl/tests/zstd.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/session-persistence/session-persistence-jsonl/tests/zstd.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/zstd.spec.ts index 830e17ffc7..43ef0b62c2 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/zstd.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/zstd.spec.ts @@ -460,7 +460,7 @@ describe('SessionPersistenceJsonl: encoding selection', () => { '', ].join('\n')) await expect(ctx.sessionPersistence.load(loadHeader.id)).rejects.toThrow(/uses \.jsonl/) - await expect((ctx.sessionPersistence as SessionPersistenceJsonl).loadLive(loadHeader.id, loadHeader.cwd)) + await expect((ctx.sessionPersistence as SessionPersistenceJsonl).loadStored(loadHeader.id)) .rejects.toThrow(/uses \.jsonl/) await expect(ctx.sessionPersistence.list()).rejects.toThrow(/uses \.jsonl/) }) From 642ef353ece7a0d028b3ec6f4003b2718944f3f5 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:40:23 +0800 Subject: [PATCH 020/113] docs(config): refresh catalog after merge --- docs/config-catalog.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 92e45d5b65..b3acd460a3 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -894,7 +894,7 @@ export interface Config { export type JsonlCompression = 'zstd' | 'none' ``` -Source: [`packages/session-persistence/session-persistence-jsonl/src/index.ts:37`](../packages/session-persistence/session-persistence-jsonl/src/index.ts) +Source: [`packages/session-persistence/session-persistence-jsonl/src/index.ts:38`](../packages/session-persistence/session-persistence-jsonl/src/index.ts) ## `@deepseek-ai/dsh-session-persistence-sqlite` @@ -1495,7 +1495,7 @@ export interface TuiConfig { } ``` -Source: [`packages/ui/tui/src/index.ts:128`](../packages/ui/tui/src/index.ts) +Source: [`packages/ui/tui/src/index.ts:129`](../packages/ui/tui/src/index.ts) ## `@deepseek-ai/dsh-tui-demo` From 57a47b1fb3812488c6cbec4c5a6242fc543baf1f Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 22:37:20 +0800 Subject: [PATCH 021/113] fix(pty): close review lifecycle gaps --- ...06-20-generic-long-running-tool-runtime.md | 10 +- ...26-07-16-persistent-pty-sessions.i18n.yaml | 4 +- .../2026-07-16-persistent-pty-sessions.md | 31 ++-- .../2026-07-16-persistent-pty-sessions.zh.md | 31 ++-- docs/config-catalog.md | 21 ++- docs/cordis-catalog/services.md | 11 +- docs/core-data-structures/tasks.md | 7 + docs/event-producer-consumer.md | 2 +- docs/module-graph.md | 10 +- examples/acp-agent/pty.cordis.snapshot.yml | 2 + .../tests/snapshots/pty-tools/session.jsonl | 2 +- .../snapshots/pty-tools/stdout.expected.jsonl | 2 +- .../headless-agent/pty.cordis.snapshot.yml | 2 + .../tests/snapshots/pty-tools/session.jsonl | 2 +- .../pty-tools/stream-json.expected.jsonl | 2 +- .../cordis/tool-cordis/src/api-catalog.ts | 8 +- packages/pty/pty-local/README.md | 6 +- packages/pty/pty-local/package.json | 2 + packages/pty/pty-local/src/index.ts | 20 ++- packages/pty/pty-local/src/sanitize.ts | 41 ++++- packages/pty/pty-local/src/session.ts | 163 ++++++++++++------ packages/pty/pty-local/tests/index.spec.ts | 101 ++++++++++- packages/pty/pty-local/tests/local.spec.ts | 33 ++++ packages/pty/pty-local/tests/sanitize.spec.ts | 16 +- packages/pty/pty-local/tests/session.spec.ts | 99 ++++++++++- packages/pty/pty-local/tsconfig.json | 6 + packages/pty/pty/README.md | 4 +- packages/pty/pty/src/index.ts | 36 +++- packages/pty/pty/tests/service.spec.ts | 40 ++++- packages/pty/tool-pty/README.md | 15 +- packages/pty/tool-pty/package.json | 5 + packages/pty/tool-pty/src/index.ts | 54 ++++-- packages/pty/tool-pty/src/render.ts | 92 ++++++++-- packages/pty/tool-pty/tests/render.spec.ts | 54 ++++-- packages/pty/tool-pty/tests/tools.spec.ts | 51 +++++- packages/pty/tool-pty/tsconfig.json | 3 + packages/tasks/tasks/README.md | 4 +- packages/tasks/tasks/src/index.ts | 7 + packages/tasks/tasks/src/types.ts | 7 + packages/tasks/tasks/tests/tasks.spec.ts | 23 ++- packages/tasks/tool-tasks/README.md | 4 +- packages/tasks/tool-tasks/package.json | 8 +- packages/tasks/tool-tasks/src/index.ts | 52 +++++- .../tasks/tool-tasks/tests/tool-tasks.spec.ts | 22 ++- packages/tasks/tool-tasks/tsconfig.json | 3 + pnpm-lock.yaml | 13 ++ python/sdk-runtime/package.json | 1 + 47 files changed, 940 insertions(+), 192 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-06-20-generic-long-running-tool-runtime.md b/.agents/notes/implemented/architecture/2026-06-20-generic-long-running-tool-runtime.md index 4db0d78910..11425939c3 100644 --- a/.agents/notes/implemented/architecture/2026-06-20-generic-long-running-tool-runtime.md +++ b/.agents/notes/implemented/architecture/2026-06-20-generic-long-running-tool-runtime.md @@ -21,7 +21,9 @@ Long-running tools are producers. `dsh-tool-bash` adapts a `BashProcess` into in ## Runtime contract -The literal types live in the [task data-structure catalog](../../../../docs/core-data-structures/tasks.md). A producer calls `ctx.tasks.start()` with a kind, label, optional owning `Agent`, and a `run()` function. The runtime completes all failable preflight work before calling `run()` and invokes it once. After `run()` returns hooks, registration commits without another failable step; a producer cannot start work that lacks a collectable task id. +The literal types live in the [task data-structure catalog](../../../../docs/core-data-structures/tasks.md). A producer calls `ctx.tasks.start()` with a kind, label, optional owning `Agent`, optional positive `outputLimitBytes`, and a `run()` function. The runtime completes all failable preflight work before calling `run()` and invokes it once. After `run()` returns hooks, registration commits without another failable step; a producer cannot start work that lacks a collectable task id. + +`outputLimitBytes` is producer-owned presentation policy, not a registry buffer. The registry validates and projects it unchanged into `TaskSnapshot`; generic control surfaces apply the cap to complete model-facing output after adding their own status or notice metadata. Omitting it preserves the existing surface behavior, so the runtime does not impose a hidden default on unrelated producer families. The producer hooks define three responsibilities: @@ -73,11 +75,11 @@ Stream reads share one task-scoped consuming cursor because the owning model is The system prompt tells the model to retain task ids, continue independent work instead of busy-polling or duplicating a running task, collect relevant tasks before its final answer, and kill work that no longer matters. Completion injects a logged `context/message` into the exact owner's session; it becomes durable context for the next request but does not wake an idle agent. -The runtime marks a terminal task `reported` when a read or wait delivers it, when a live waiter has claimed delivery at settlement, or when the model explicitly kills it. Reported tasks do not inject redundant completion notices. Listener failures are logged independently, do not stop later listeners, and are not awaited by waiters or teardown. +The runtime marks a terminal task `reported` when a read or wait delivers it, when a live waiter has claimed delivery at settlement, or when the model explicitly kills it. Reported tasks do not inject redundant completion notices. Listener failures are logged independently, do not stop later listeners, and are not awaited by waiters or teardown. When a snapshot carries `outputLimitBytes`, `dsh-tool-tasks` reserves space for status or notice suffixes, preserves UTF-8 boundaries, and reuses an existing producer truncation marker rather than duplicating it. ## Producer opt-in -Each producer owns whether its schema exposes `run_in_background` through defaulted config. `dsh-tool-bash` and each `dsh-tool-subagent` instance use `enableRunInBackground`, defaulting to true. A disabled instance omits the parameter and also rejects a forced background argument at execution because the generic argument validator permits undeclared keys. Schema omission advertises the capability; the execution check enforces it. +Each producer owns whether its schema exposes `run_in_background` through defaulted config. `dsh-tool-bash`, `dsh-tool-pty`, and each `dsh-tool-subagent` instance use `enableRunInBackground`, defaulting to true. A disabled instance omits the parameter and also rejects a forced background argument at execution because the generic argument validator permits undeclared keys. Schema omission advertises the capability; the execution check enforces it. `ctx.tasks` does not rewrite producer schemas. A bundle forwards configuration only for producers it owns. If a background call reaches `start()` without an attached surface, the runtime fence fails before execution. @@ -119,7 +121,7 @@ Authorization, not unguessability, is the access boundary, and ids do not derive ## Testing -Unit coverage pins preflight atomicity, per-kind ids, stream and final reads, wait timeout and abort races, cancellation, first-wins settlement, listener containment, notice suppression, owner isolation, stale owner instances, owner cleanup, service teardown, and the no-surface fence. Producer tests cover bash process mapping, subagent startup cancellation, terminal mapping, and disposal. Snapshot coverage pins the control-tool schemas and prompt guidance. +Unit coverage pins preflight atomicity, per-kind ids, output-limit validation and projection, complete UTF-8 result bounds, stream and final reads, wait timeout and abort races, cancellation, first-wins settlement, listener containment, notice suppression, owner isolation, stale owner instances, owner cleanup, service teardown, and the no-surface fence. Producer tests cover bash process mapping, subagent startup cancellation, terminal mapping, and disposal. Snapshot coverage pins the control-tool schemas and prompt guidance. ## Consequences diff --git a/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.i18n.yaml b/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.i18n.yaml index f58ed600f0..627e46ace5 100644 --- a/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-16-persistent-pty-sessions.md: 76354891f557974b953a1b0b805d94330c9f6e69 -2026-07-16-persistent-pty-sessions.zh.md: 86200d70c6eda815a440c44e8b55457b0424edb6 +2026-07-16-persistent-pty-sessions.md: 8d279fea2e606894e4e8856a706113c0ea173e98 +2026-07-16-persistent-pty-sessions.zh.md: 9e81cad7357bc37856dc74ed5654744d70981a06 diff --git a/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.md b/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.md index 76354891f5..8d279fea2e 100644 --- a/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.md +++ b/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.md @@ -34,14 +34,14 @@ Idle detection is backend behavior, not a second public seam. A remote or contai There are no plugin-load auto-start sessions. `terminal_open` creates a session only during an agent tool call, when ownership and the owning event-sourced session are known. A future declarative startup feature must compose through unpublished agent setup rather than create shared global terminals. -Agent-scope disposal closes registrations first, then awaits quiescent teardown of every owned PTY. Backend or tool-plugin reload does not orphan sessions: ownership lives in `PtyService` until the agent ends, following the same service-owned-record pattern as [`ctx.tasks`](../../../../packages/tasks/tasks/README.md). +Agent-scope disposal closes registrations first, then awaits quiescent teardown of every owned PTY. Backend or tool-plugin reload does not orphan sessions: ownership lives in `PtyService` until the agent ends, following the same service-owned-record pattern as [`ctx.tasks`](../../../../packages/tasks/tasks/README.md). The service reserves the session synchronously for one active send before returning its operation, including before a background task id becomes visible; a second send fails with `SEND_ACTIVE`, so output and cancellation cannot cross operation ownership. ### Security and process boundary A registered `shell` backend constrains how a terminal starts; it does not constrain commands typed after startup. `dsh-pty-local` therefore applies two protections before spawning: - It builds a scrubbed child environment using the same credential-shaped-name policy as `bash-local`, removing ambient `*KEY*`, `*SECRET*`, `*TOKEN*`, and harness-managed variables unless an explicit trusted mapping supplies them. -- It requires `ctx.sandbox` and the shared `ctx.sandboxPolicy`. At spawn, the backend resolves the owner's effective session mode over the deployment default and wraps the shell argv once; that mode and workspace root remain the process boundary for the PTY lifetime. `danger-full-access` is the existing explicit unconfined choice rather than a PTY-specific bypass. +- It requires `ctx.sandbox` and the shared `ctx.sandboxPolicy`. At spawn, the backend resolves the owner's effective session mode over the deployment default and wraps the shell argv once; that mode and workspace root remain the process boundary for the PTY lifetime. A write that would change the effective `sandbox/mode` is rejected before commit while the owner has any open PTY or unpublished spawn, with an instruction to wait for creation to settle and close those sessions first; same-effective-mode writes remain valid. The pending reservation spans backend setup through publication, so there is no race in which a wider terminal appears after a downgrade. `danger-full-access` is the existing explicit unconfined choice rather than a PTY-specific bypass. Sandboxing confines local process effects but does not make arbitrary shell input safe: network calls and other external side effects remain governed by deployment policy. Tool descriptions state that PTY sessions are less auditable than one-shot tools and should be used only when persistence or interactive stdin is necessary. @@ -58,19 +58,21 @@ The implementation uses only public `node-pty` capabilities: child PID, `data` a | `terminal_close` | Close one session and await process-tree quiescence | `{ killed }` | | `terminal_list` | List the caller's live sessions | owner-scoped session summaries | -`terminal_send({ sessionId, text, submit?, run_in_background? })` treats `text` as UTF-8 bytes and resolves `submit` to `true` in the tool implementation. When `submit` is true it writes the platform Enter sequence after the text; when false it writes only the text, allowing control characters and REPL fragments without hidden content heuristics. +The ACP render contract is exact and location-free. `terminal_send` uses terminal call/result cards only for foreground sends; its background form is generic `execute`. `terminal_open`, `terminal_read`, `terminal_signal`, `terminal_close`, and `terminal_list` use generic `execute`, `read`, `execute`, `delete`, and `read` cards respectively. No PTY tool emits `locations`. -Foreground sends return a bounded rendered delta and two independent facts: `waitReason` (`stdin_read | inferred_idle | timeout | session_exit`) and `sessionStatus` (`running` or `exited` with exit code or signal). `session_exit` refers to the PTY's top-level shell process, not an arbitrary foreground command whose status the shell consumes. A timeout never implies process exit. +`terminal_send({ sessionId, text, submit?, run_in_background? })` treats `text` as UTF-8 bytes and resolves `submit` to `true` in the tool implementation. When `submit` is true it writes the platform Enter sequence after the text; when false it writes only the text, allowing control characters and REPL fragments without hidden content heuristics. `enableRunInBackground` defaults to true; false removes `run_in_background` from the schema and rejects the same undeclared argument if a caller forces it through execution. -With `run_in_background: true`, `dsh-tool-pty` registers the in-flight send on `ctx.tasks` and returns immediately with `taskId`. `task_output(wait: true)` waits, reads incremental output, and records the final result; `task_kill` forwards cancellation as `SIGINT` and escalates only through the PTY backend's owned teardown path. If the task surface is absent, background mode fails before writing input. No PTY-specific `sleep` tool or general wake-up seam is added. +Foreground sends return a bounded rendered delta and two independent facts: `waitReason` (`stdin_read | inferred_idle | timeout | session_exit`) and `sessionStatus` (`running` or `exited` with exit code or signal). `session_exit` refers to the PTY's top-level shell process, not an arbitrary foreground command whose status the shell consumes. A timeout never implies process exit. `dsh-tool-pty.maxResultBytes` defaults to 262144 and caps the complete UTF-8 result after wait, session, pagination, truncation, and generic task-status wrappers; the renderer reserves suffix space and preserves code-point boundaries instead of treating the backend payload cap as the final model bound. -`terminal_read` pages backward from the newest retained line. The backend enforces both line and UTF-8 byte caps on retained scrollback and the complete returned value, so one oversized line cannot bypass the bound. `truncated` distinguishes retention loss from an ordinary viewport delta. +With `run_in_background: true`, `dsh-tool-pty` registers the in-flight send on `ctx.tasks` and returns immediately with `taskId`. The producer places `maxResultBytes` on the task snapshot so `task_output`, terminal kill status, and completion notices enforce the same complete-result cap after generic metadata. `task_output(wait: true)` waits, reads incremental output, and records the final result; `task_kill` resolves the current foreground PGID and delivers a real `SIGINT`, including when the application has disabled terminal `ISIG`, and escalates only through the PTY backend's owned teardown path. If the task surface is absent, background mode fails before writing input. No PTY-specific `sleep` tool or general wake-up seam is added. + +`terminal_read` pages backward from the newest retained line. The backend enforces both line and UTF-8 byte caps on retained scrollback and the returned page payload, so one oversized line cannot bypass the backend bound; the tool then caps the fully rendered page including pagination and truncation metadata. `truncated` distinguishes retention loss from an ordinary viewport delta. `terminal_signal` accepts the closed set `SIGINT | SIGTERM | SIGKILL | SIGTSTP | SIGHUP`. The backend resolves the terminal foreground process group at execution time. `SIGKILL` is rejected when that group is the top-level shell, directing the caller to `terminal_close`; a failed group lookup fails the operation instead of signaling a guessed PID. ### Local readiness detection -The local backend first recognizes a private OSC prompt marker emitted by its controlled bash startup, then runs three bounded fallback tiers. The marker is removed before output reaches the model and avoids a fixed silence delay for ordinary shell commands on both platforms. Unpublished startup does not accept zero-output silence as readiness; timeout rejects the spawn. All timings are validated config fields: `pollIntervalMs`, `exactProbeAfterMs`, `idleSilenceMs`, and `timeoutMs`. +The local backend first recognizes a private OSC prompt marker emitted by its controlled bash startup, then requires printable prompt text after that marker before declaring prompt readiness and runs three bounded fallback tiers. Carrying that state across data callbacks covers macOS delivery where the OSC marker and `PS1` arrive separately; the marker alone can no longer publish an empty MOTD. The marker is removed before output reaches the model and avoids a fixed silence delay for ordinary shell commands on both platforms. Unpublished startup does not accept zero-output silence as readiness; timeout rejects the spawn. All timings are validated config fields: `pollIntervalMs`, `exactProbeAfterMs`, `idleSilenceMs`, and `timeoutMs`. On Linux, the inspector reads the shell's terminal foreground PGID from `/proc//stat`, enumerates every process and thread in that process group, and probes their current syscalls. A positive Tier 1 result requires an observed stdin wait: direct `read(0)`, a permitted read of a `select`/`pselect6` or `poll`/`ppoll` argument containing fd 0, or an epoll interest list containing fd 0. Unreadable process memory and unrecognized syscalls are misses, never positive guesses. Architecture tables contain only syscall numbers defined by the corresponding Linux UAPI; unsupported architectures skip Tier 1. @@ -78,7 +80,7 @@ On macOS there is no exact syscall tier. Output silence returns `inferred_idle` Tier 2 returns `inferred_idle` after `idleSilenceMs` without output. A sleeping or network-blocked command can therefore look ready. Tier 3 returns `timeout` after `timeoutMs` so a foreground tool call cannot hold the agent indefinitely. The result preserves the distinction; callers may wait through `ctx.tasks`, signal the foreground group, or inspect from another session. -`node-pty` data notifications feed one streaming decoder and terminal parser. Parser carry state handles UTF-8 and terminal query sequences split across chunks. The implementation normalizes line-oriented output and detects alternate-screen entry, but it does not promise correct interaction with a full-screen application. +`node-pty` data notifications feed one terminal parser. Parser carry state handles control sequences and a trailing carriage return split across callbacks, so a divided CRLF produces one newline rather than a pagination-changing blank line. The implementation normalizes line-oriented output, but it does not promise correct interaction with a full-screen application. ### Model-visible output and durability @@ -88,9 +90,9 @@ Background sends use the existing task completion notice and `task_output` resul ### Process-tree teardown -The top-level `node-pty` child is the ownership anchor. On close, the backend stops callbacks, snapshots that PID and its transitive descendants by parent PID in children-first order, sends `SIGTERM`, closes the PTY, waits for quiescence, then sends `SIGKILL` to verified survivors after configurable `disposeGraceMs` and waits for them to leave the process table. Every captured PID includes process-start identity so reuse cannot redirect escalation. +The top-level `node-pty` child is the ownership anchor. On close, the backend stops callbacks, snapshots its transitive descendants by parent PID in children-first order, sends `SIGTERM`, waits, rescans for children forked during shutdown, sends `SIGKILL` to the remaining descendant tree, and verifies that every descendant left the process table while the shell is still alive to reap it. Only then does it stop the shell with its own TERM/grace/KILL sequence. Every captured PID includes process-start identity so reuse cannot redirect escalation. -Teardown reports root exit and survivor cleanup independently. It does not claim success merely because the shell exited; disposal resolves only after no captured tree member remains or returns a structured cleanup failure naming the survivors. Service disposal still clears its backend, reservation, and owner-detacher registries when a close fails. It never broadens ownership to every member of the root PID's POSIX session. +Teardown reports root exit and survivor cleanup independently. It does not claim success merely because the shell exited; disposal resolves only after no captured tree member remains or returns a cleanup failure naming the survivors. A failed close is not cached forever: the registry and local session retain their closing fence but allow a later close to retry after the external survivor condition changes. Service disposal still clears its backend, reservation, and owner-detacher registries when a close fails. It never broadens ownership to every member of the root PID's POSIX session. ### Composition and rollout @@ -115,9 +117,12 @@ plugins: timeoutMs: 30000 disposeGraceMs: 3000 '@deepseek-ai/dsh-tool-pty': + config: + enableRunInBackground: true + maxResultBytes: 262144 ``` -The package ships concise tool guidance explaining persistent state, owner isolation, uncertain idle results, cleanup, and the preference for existing one-shot tools when interaction is unnecessary. It does not add a global system-prompt recommendation or mount PTY in shipped defaults; dedicated ACP and headless snapshot overlays exercise the opt-in composition. +The package ships concise tool guidance explaining persistent state, owner isolation, uncertain idle results, cleanup, and the preference for existing one-shot tools when interaction is unnecessary. It does not mount PTY in the base shipped examples: PTY is opt-in through the dedicated composition, while ACP and headless snapshot overlays exercise it. Within an enabled `dsh-tool-pty` instance, the six tools and `run_in_background` are enabled by default; deployments may disable only the background argument with config. ### Deferred work @@ -147,9 +152,9 @@ The package ships concise tool guidance explaining persistent state, owner isola ## Verification -- Per-file coverage pins owner fencing, concurrent reservations, lifecycle cleanup, readiness tiers, sanitizer carry state, UTF-8 bounds, task integration, schemas, and render intents. +- Per-file coverage pins owner fencing, concurrent reservations, sandbox-mode change rejection, retriable lifecycle cleanup, readiness tiers, sanitizer carry state, complete UTF-8 bounds, task integration, schemas, and exact render intents. - Linux process fixtures cover non-leader and non-main-thread stdin waits, unreadable process state, supported syscall tables, unsupported architectures, and false-positive rejection; macOS inspector logic is injected into the same unit suite. -- Real `node-pty` tests exercise shell state, shared sandbox policy, environment scrubbing, signals, a TERM-ignoring descendant, and immediate post-disposal quiescence on supported hosts. +- Real `node-pty` tests exercise shell state, shared sandbox policy, environment scrubbing, raw-mode foreground `SIGINT`, a TERM-ignoring descendant, and immediate post-disposal quiescence on supported hosts. - A Loader-driven `cordis.yml` test mounts the real three-package composition, while ACP and headless snapshots pin the six schemas, bounded results, error rendering, and terminal/generic cards through opt-in overlays. - Package contracts, the architecture map, core data structures, generated catalogs, and the website API describe the same shipped surface. - The repository CI-equivalent sequence owns type, lint, coverage, snapshot, documentation, build, hygiene, demo, and built-entry verification. diff --git a/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.zh.md b/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.zh.md index 86200d70c6..9e81cad735 100644 --- a/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.zh.md +++ b/.agents/notes/implemented/feature/2026-07-16-persistent-pty-sessions.zh.md @@ -34,14 +34,14 @@ idle 检测属于后端行为，不是第二条公共 seam。远程或容器后 实现不提供插件加载期 auto-start 会话。`terminal_open` 只在 agent 工具调用期间创建会话，此时所有权和所属的事件溯源会话都已确定。未来的声明式启动功能必须通过尚未发布的 agent setup 组合，而不能创建全局共享终端。 -agent scope dispose 时先关闭注册，再等待全部所属 PTY 静默退出。后端或工具插件 reload 不会遗留会话：所有权持续存放在 `PtyService` 中，直到 agent 结束，与 [`ctx.tasks`](../../../../packages/tasks/tasks/README.md) 的服务持有记录模式一致。 +agent scope dispose 时先关闭注册，再等待全部所属 PTY 静默退出。后端或工具插件 reload 不会遗留会话：所有权持续存放在 `PtyService` 中，直到 agent 结束，与 [`ctx.tasks`](../../../../packages/tasks/tasks/README.md) 的服务持有记录模式一致。服务会先同步把会话预留给一次活跃发送，再返回该操作；后台发送同样会在 task id 对外可见前完成预留。第二次发送会以 `SEND_ACTIVE` 失败，因此输出与取消无法跨越操作所有权。 ### 安全与进程边界 注册的 `shell` 后端只约束终端如何启动，不约束启动后输入的命令。因此 `dsh-pty-local` 在 spawn 前应用两层保护： - 它使用与 `bash-local` 相同的凭证形态名称策略构建清洗后的子进程环境，移除环境中的 `*KEY*`、`*SECRET*`、`*TOKEN*` 和 harness 管理的变量，除非显式的可信映射提供这些值。 -- 它要求 `ctx.sandbox` 和共享的 `ctx.sandboxPolicy`。后端在 spawn 时，以部署默认值为底折叠 owner 的有效 session mode，并只包装一次 shell argv；该 mode 与 workspace root 在 PTY 的整个生命周期中充当进程边界。`danger-full-access` 是现有的显式无约束选择，不另设 PTY 私有 bypass。 +- 它要求 `ctx.sandbox` 和共享的 `ctx.sandboxPolicy`。后端在 spawn 时，以部署默认值为底折叠 owner 的有效 session mode，并只包装一次 shell argv；该 mode 与 workspace root 在 PTY 的整个生命周期中充当进程边界。只要 owner 有任何已打开的 PTY 或尚未发布的 spawn，任何会改变生效 `sandbox/mode` 的写入都会在提交前被拒绝，并提示先等待创建操作结算，再关闭这些会话；不会改变生效模式的写入仍然有效。这项进行中的预留从后端 setup 持续到发布完成，因此不存在降级后又出现权限更宽的终端这一竞态。`danger-full-access` 是现有的显式无约束选择，不另设 PTY 私有 bypass。 沙箱限制本地进程副作用，但不会让任意 shell 输入自动安全：网络调用和其他外部副作用仍由部署策略治理。工具描述会说明 PTY 会话比一次性工具更难审计，只应在确实需要持久状态或交互式 stdin 时使用。 @@ -58,19 +58,21 @@ agent scope dispose 时先关闭注册，再等待全部所属 PTY 静默退出 | `terminal_close` | 关闭一个会话并等待进程树静默退出 | `{ killed }` | | `terminal_list` | 列出调用方的活会话 | 按 owner 隔离的会话摘要 | -`terminal_send({ sessionId, text, submit?, run_in_background? })` 将 `text` 视为 UTF-8 字节，并由工具实现在解析阶段把 `submit` 默认成 `true`。`submit` 为 true 时先写入文本，再写入平台 Enter 序列；为 false 时只写文本，使控制字符和 REPL 片段无需隐藏的内容启发式即可发送。 +ACP 渲染契约精确且不携带位置信息。`terminal_send` 只为前台发送使用 terminal 调用卡片和结果卡片；后台形式使用通用 `execute` 卡片。`terminal_open`、`terminal_read`、`terminal_signal`、`terminal_close` 和 `terminal_list` 分别使用通用 `execute`、`read`、`execute`、`delete` 和 `read` 卡片。所有 PTY 工具都不发出 `locations`。 -前台发送返回有界的渲染增量和两个独立事实：`waitReason`（`stdin_read | inferred_idle | timeout | session_exit`）与 `sessionStatus`（`running`，或携带退出码或信号的 `exited`）。`session_exit` 指 PTY 顶层 shell 进程退出，不指由 shell 消费状态的任意前台命令。timeout 从不意味着进程已经退出。 +`terminal_send({ sessionId, text, submit?, run_in_background? })` 将 `text` 视为 UTF-8 字节，并由工具实现在解析阶段把 `submit` 默认成 `true`。`submit` 为 true 时先写入文本，再写入平台 Enter 序列；为 false 时只写文本，使控制字符和 REPL 片段无需隐藏的内容启发式即可发送。`enableRunInBackground` 默认为 true；设为 false 时，schema 中会移除 `run_in_background`，调用方即使强行把这个未声明参数传入执行流程，也会被拒绝。 -当 `run_in_background: true` 时，`dsh-tool-pty` 在 `ctx.tasks` 上注册进行中的发送，并立即返回 `taskId`。`task_output(wait: true)` 负责等待、读取增量输出并记录最终结果；`task_kill` 将取消转发为 `SIGINT`，只有 PTY 后端拥有的 teardown 路径可以升级信号。若 task 对外接口不存在，后台模式必须在写入输入前失败。设计不新增 PTY 专用的 `sleep` 工具或通用唤醒 seam。 +前台发送返回有界的渲染增量和两个独立事实：`waitReason`（`stdin_read | inferred_idle | timeout | session_exit`）与 `sessionStatus`（`running`，或携带退出码或信号的 `exited`）。`session_exit` 指 PTY 顶层 shell 进程退出，不指由 shell 消费状态的任意前台命令。timeout 从不意味着进程已经退出。`dsh-tool-pty.maxResultBytes` 默认为 262144；完整 UTF-8 结果在加入等待与会话状态、分页与截断元数据以及通用 task 状态包装后，仍受该值限制。渲染器会为后缀预留空间并保持代码点边界，而不会把后端载荷上限当作面向模型结果的最终上限。 -`terminal_read` 从最新保留行向后分页。后端同时对保留的 scrollback 和完整返回值执行行数与 UTF-8 字节上限，因此单个超长行无法绕过限制。`truncated` 用于区分保留数据丢失与普通 viewport 增量。 +当 `run_in_background: true` 时，`dsh-tool-pty` 在 `ctx.tasks` 上注册进行中的发送，并立即返回 `taskId`。生产方把 `maxResultBytes` 写入 task 快照，使 `task_output`、kill 返回的终态状态和完成通知在加上通用元数据后，仍对完整结果执行同一上限。`task_output(wait: true)` 负责等待、读取增量输出并记录最终结果；`task_kill` 会解析当前前台 PGID 并发送真正的 `SIGINT`，即使应用已禁用终端 `ISIG` 也同样如此，且后续升级仍只通过 PTY 后端拥有的 teardown 路径进行。若 task 对外接口不存在，后台模式必须在写入输入前失败。设计不新增 PTY 专用的 `sleep` 工具或通用唤醒 seam。 + +`terminal_read` 从最新保留行向后分页。后端同时对保留的 scrollback 和返回页载荷执行行数与 UTF-8 字节上限，因此单个超长行无法绕过后端上限；工具随后再限制包含分页与截断元数据的完整渲染页。`truncated` 用于区分保留数据丢失与普通 viewport 增量。 `terminal_signal` 接受闭合集 `SIGINT | SIGTERM | SIGKILL | SIGTSTP | SIGHUP`。后端在执行时解析终端前台进程组。当目标组是顶层 shell 时拒绝 `SIGKILL`，并指引调用方使用 `terminal_close`；进程组解析失败时操作直接失败，而不是向猜测的 PID 发送信号。 ### 本地就绪检测 -本地后端先识别受控 bash 启动时发出的私有 OSC prompt marker，再执行 3 个有界 fallback 层级。marker 在输出到达模型前被移除，使两个平台上的普通 shell 命令都无需固定等待静默阈值。尚未发布的 startup 不会把零输出静默视为就绪；timeout 会拒绝 spawn。所有时间参数都是经校验的配置字段：`pollIntervalMs`、`exactProbeAfterMs`、`idleSilenceMs` 和 `timeoutMs`。 +本地后端先识别受控 bash 启动时发出的私有 OSC prompt marker，并且只有在该 marker 后出现可打印的 prompt 文本时才据此声明 prompt 就绪；除此之外，它还运行 3 个有界 fallback 层级。在 data callback 之间保留这项状态，可以适配 macOS 分开交付 OSC marker 与 `PS1` 的情况；单独的 marker 不会发布空 MOTD。marker 在输出到达模型前被移除，使两个平台上的普通 shell 命令都无需固定等待静默阈值。尚未发布的 startup 不会把零输出静默视为就绪；timeout 会拒绝 spawn。所有时间参数都是经校验的配置字段：`pollIntervalMs`、`exactProbeAfterMs`、`idleSilenceMs` 和 `timeoutMs`。 在 Linux 上，检查器从 `/proc//stat` 读取 shell 的终端前台 PGID，枚举该进程组中的每个进程与线程，并检查它们当前的 syscall。Tier 1 只有观察到 stdin 等待才返回正结果：直接 `read(0)`、获准读取且含 fd 0 的 `select`/`pselect6` 或 `poll`/`ppoll` 参数，或者含 fd 0 的 epoll interest list。无法读取的进程内存和未识别的 syscall 都是 miss，绝不作为正向猜测。架构表只包含对应 Linux UAPI 定义的 syscall number；不支持的架构跳过 Tier 1。 @@ -78,7 +80,7 @@ macOS 没有精确 syscall 层。任何前台进程组输出静默都会返回 ` Tier 2 在持续 `idleSilenceMs` 没有输出后返回 `inferred_idle`，因此 sleep 或网络阻塞的命令可能看似 ready。Tier 3 在 `timeoutMs` 后返回 `timeout`，避免前台工具调用无限占住 agent。结果保留这些区别；调用方可以通过 `ctx.tasks` 等待、向前台组发信号，或从另一个会话排查。 -`node-pty` data 通知进入同一个流式 decoder 和终端 parser。parser 的 carry 状态处理跨 chunk 的 UTF-8 与终端查询序列。当前实现只规范化行式输出并检测 alternate-screen 进入，不承诺正确操作全屏应用。 +`node-pty` data 通知进入同一个终端 parser。parser 的 carry state 会处理跨 callback 的控制序列和位于 callback 末尾的回车；因此，即使 CRLF 被拆开，也只会生成一个换行，而不会产生改变分页的空行。实现会规范化行式输出，但不承诺正确操作全屏应用。 ### 模型可见输出与持久性 @@ -88,9 +90,9 @@ Tier 2 在持续 `idleSilenceMs` 没有输出后返回 `inferred_idle`，因此 ### 进程树 teardown -顶层 `node-pty` 子进程是所有权锚点。关闭时，后端先停止 callback，再按父 PID 以子进程优先顺序捕获该 PID 及其传递子进程、发送 `SIGTERM`、关闭 PTY 并等待静默，然后在可配置的 `disposeGraceMs` 后向已验证的存活者发送 `SIGKILL`，并等待它们离开进程表。每个捕获的 PID 都包含进程启动身份，避免 PID 复用把升级信号发给无关进程。 +顶层 `node-pty` 子进程是所有权锚点。关闭时，后端先停止 callback，再按父 PID 以子进程优先顺序捕获其传递子进程、发送 `SIGTERM` 并等待，然后重新扫描关停期间 fork 出的子进程，向剩余子孙进程树发送 `SIGKILL`，并在 shell 仍存活、可以回收这些进程时，验证每个子孙进程都已离开进程表。完成这些步骤后，后端才用 shell 自身的 TERM、宽限等待、KILL 序列停止 shell。每个捕获的 PID 都包含进程启动身份，避免 PID 复用把升级信号发给无关进程。 -teardown 独立报告根进程退出与存活进程清理。它不会只因 shell 退出就声称成功；dispose 只有在已捕获的进程树成员全部消失后才完成，否则返回结构化清理失败并列出存活者。即使某个 close 失败，服务 dispose 仍会清空其后端、预留与 owner detacher 注册表。所有权绝不会扩大到根 PID 所属 POSIX 会话的全部成员。 +teardown 独立报告根进程退出与存活进程清理。它不会只因 shell 退出就声称成功；dispose 只有在已捕获的进程树成员全部消失后才完成，否则返回清理失败并列出存活者。失败的 close 不会永久缓存：注册表与本地会话会保留关闭围栏，但在外部存活进程状态改变后允许后续 close 重试。即使某个 close 失败，服务 dispose 仍会清空其后端、预留与 owner detacher 注册表。所有权绝不会扩大到根 PID 所属 POSIX 会话的全部成员。 ### 组合与推行 @@ -115,9 +117,12 @@ plugins: timeoutMs: 30000 disposeGraceMs: 3000 '@deepseek-ai/dsh-tool-pty': + config: + enableRunInBackground: true + maxResultBytes: 262144 ``` -包提供简洁的工具指引，说明持久状态、owner 隔离、不确定的 idle 结果、清理，以及无需交互时优先使用现有一次性工具。它不增加全局 system prompt 推荐，也不在已发布的默认配置中挂载 PTY；专用 ACP 与 headless 快照 overlay 覆盖 opt-in 组合。 +包提供简洁的工具指引，说明持久状态、owner 隔离、不确定的 idle 结果、清理，以及无需交互时优先使用现有一次性工具。已发布的基础示例不挂载 PTY：PTY 仅通过专用组合 opt-in，ACP 与 headless 快照 overlay 覆盖该组合。`dsh-tool-pty` 实例一旦启用，6 个工具和 `run_in_background` 就会默认启用；部署可通过配置仅禁用后台参数。 ### 推迟的工作 @@ -147,9 +152,9 @@ plugins: ## 验证 -- 每文件覆盖率固定 owner 隔离、并发预留、生命周期清理、就绪层级、sanitizer carry state、UTF-8 上限、task 集成、schema 和 render intent。 +- 每文件覆盖率固定 owner 隔离、并发预留、沙箱模式变更拒绝、可重试的生命周期清理、就绪层级、sanitizer carry state、完整 UTF-8 结果上限、task 集成、schema 和精确 render intent。 - Linux 进程 fixture 覆盖非 leader 与非主线程的 stdin 等待、不可读进程状态、受支持的 syscall 表、不支持的架构和误报拒绝；同一单元测试套件通过注入覆盖 macOS 检查器逻辑。 -- 真实 `node-pty` 测试在受支持宿主上覆盖 shell 状态、共享沙箱策略、环境清洗、信号、忽略 `SIGTERM` 的子进程，以及 dispose 返回后立即静默。 +- 真实 `node-pty` 测试在受支持宿主上覆盖 shell 状态、共享沙箱策略、环境清洗、raw mode 下的前台 `SIGINT`、忽略 `SIGTERM` 的子进程，以及 dispose 返回后立即静默。 - Loader 驱动的 `cordis.yml` 测试挂载真实三包组合；ACP 与 headless 快照通过 opt-in overlay 固定 6 个 schema、有界结果、错误渲染和 terminal/generic card。 - 包契约、架构图、核心数据结构、生成目录和 website API 描述同一个已发布接口。 - 仓库 CI 等价序列负责类型、lint、覆盖率、快照、文档、构建、hygiene、demo 和 built-entry 验证。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 76afd7d68a..34d69c5227 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -802,7 +802,7 @@ Source: [`packages/plan/plan-mode/src/index.ts:57`](../packages/plan/plan-mode/s ## `@deepseek-ai/dsh-pty-local` -Requires: `pty` · `sandbox` · `sandboxPolicy` +Requires: `agents` · `pty` · `sandbox` · `sandboxPolicy` ```ts config-catalog /** Public plugin configuration. */ @@ -1339,6 +1339,22 @@ export interface Config { Source: [`packages/lsp/tool-lsp/src/index.ts:58`](../packages/lsp/tool-lsp/src/index.ts) +## `@deepseek-ai/dsh-tool-pty` + +Requires: `pty` · `tools` · `systemPrompt` + +```ts config-catalog +/** Model-facing terminal tool configuration. */ +export interface Config { + /** Expose `run_in_background` and accept background sends (default true). */ + enableRunInBackground?: boolean + /** Maximum UTF-8 bytes in one complete terminal or task-output result. */ + maxResultBytes?: number +} +``` + +Source: [`packages/pty/tool-pty/src/index.ts:33`](../packages/pty/tool-pty/src/index.ts) + ## `@deepseek-ai/dsh-tool-ralph` Requires: `tools` · `workflows` · `subagents` · `systemPrompt` @@ -1443,7 +1459,7 @@ export interface Config { } ``` -Source: [`packages/tasks/tool-tasks/src/index.ts:21`](../packages/tasks/tool-tasks/src/index.ts) +Source: [`packages/tasks/tool-tasks/src/index.ts:22`](../packages/tasks/tool-tasks/src/index.ts) ## `@deepseek-ai/dsh-tool-web` @@ -1840,7 +1856,6 @@ These load from a `cordis.yml` entry with no `config:` block; they declare no co - `@deepseek-ai/dsh-tasks` ([`packages/tasks/tasks/src/index.ts`](../packages/tasks/tasks/src/index.ts)) - `@deepseek-ai/dsh-timeout-policy` — requires `tools` ([`packages/timeout/timeout-policy/src/index.ts`](../packages/timeout/timeout-policy/src/index.ts)) - `@deepseek-ai/dsh-tool-ask-user` — requires `tools` · `userInteraction` ([`packages/ui/tool-ask-user/src/index.ts`](../packages/ui/tool-ask-user/src/index.ts)) -- `@deepseek-ai/dsh-tool-pty` — requires `pty` · `tools` · `systemPrompt` ([`packages/pty/tool-pty/src/index.ts`](../packages/pty/tool-pty/src/index.ts)) - `@deepseek-ai/dsh-tool-todo` — requires `tools` ([`packages/todo/tool-todo/src/index.ts`](../packages/todo/tool-todo/src/index.ts)) - `@deepseek-ai/dsh-user-interaction` ([`packages/ui/user-interaction/src/index.ts`](../packages/ui/user-interaction/src/index.ts)) diff --git a/docs/cordis-catalog/services.md b/docs/cordis-catalog/services.md index d7ca6cb602..15d32e9089 100644 --- a/docs/cordis-catalog/services.md +++ b/docs/cordis-catalog/services.md @@ -787,6 +787,13 @@ listBackends(): string[] */ async spawn(owner: Agent, request: PtySpawnRequest, signal?: AbortSignal): Promise +/** + * Test whether an exact owner has a published session or unpublished spawn. + * @param owner - exact live owner to inspect. + * @returns true across the entire spawn-to-close interval, with no publication gap. + */ +hasOwnerActivity(owner: Agent): boolean + /** * Start one exclusive interactive send. * @param owner - exact session owner. @@ -833,7 +840,7 @@ list(owner: Agent): PtySessionSnapshot[] Types: [Agent](../core-data-structures/core.md) · [PtyBackend](../core-data-structures/pty.md) · [PtyReadRequest](../core-data-structures/pty.md) · [PtyReadResult](../core-data-structures/pty.md) · [PtySendOperation](../core-data-structures/pty.md) · [PtySendRequest](../core-data-structures/pty.md) · [PtySessionId](../core-data-structures/pty.md) · [PtySessionSnapshot](../core-data-structures/pty.md) · [PtySignal](../core-data-structures/pty.md) · [PtySignalResult](../core-data-structures/pty.md) · [PtySpawnRequest](../core-data-structures/pty.md) · [PtySpawnResult](../core-data-structures/pty.md) -Source: [`packages/pty/pty/src/index.ts:95`](../../packages/pty/pty/src/index.ts) +Source: [`packages/pty/pty/src/index.ts:91`](../../packages/pty/pty/src/index.ts) ## `ctx.sandbox` — `SandboxProvider` (abstract seam) @@ -1404,7 +1411,7 @@ attachSurface(name: string): () => void Types: [Agent](../core-data-structures/core.md) · [TaskDoneListener](../core-data-structures/tasks.md) · [TaskId](../core-data-structures/tasks.md) · [TaskRead](../core-data-structures/tasks.md) · [TaskSnapshot](../core-data-structures/tasks.md) · [TaskStart](../core-data-structures/tasks.md) -Source: [`packages/tasks/tasks/src/index.ts:76`](../../packages/tasks/tasks/src/index.ts) +Source: [`packages/tasks/tasks/src/index.ts:77`](../../packages/tasks/tasks/src/index.ts) ## `ctx.tokenMeter` — `TokenMeterService` diff --git a/docs/core-data-structures/tasks.md b/docs/core-data-structures/tasks.md index 491f380166..2c7555b84d 100644 --- a/docs/core-data-structures/tasks.md +++ b/docs/core-data-structures/tasks.md @@ -34,6 +34,11 @@ interface TaskStart { kind: TaskKind /** One-line model-facing label (the command; the delegation description). */ label: string + /** + * Optional UTF-8 byte cap for each complete model-facing completion notice or + * output read, including control-surface status metadata. + */ + outputLimitBytes?: number /** * Owning live agent. Access is fenced by its session id, and agent disposal * cancels and awaits the task. The instance must be the one currently @@ -104,6 +109,8 @@ interface TaskSnapshot { kind: TaskKind /** The producer-supplied one-line label. */ label: string + /** Producer-owned cap for complete model-facing notices and output reads. */ + outputLimitBytes?: number /** * Owner session id used for authorization and correlation; absent for * unowned tasks. Completion listeners receive the exact {@link Agent} diff --git a/docs/event-producer-consumer.md b/docs/event-producer-consumer.md index d7f4123b91..100362892f 100644 --- a/docs/event-producer-consumer.md +++ b/docs/event-producer-consumer.md @@ -57,7 +57,7 @@ This matrix shows which packages dispatch each harness-owned event and which pac | Event string | Dispatchers | Listeners | | --- | --- | --- | -| `internal/dispatch` | - | [`compact`](../packages/compact/compact), [`fs`](../packages/fs/fs), [`goal`](../packages/goal/goal), [`goal-session`](../packages/goal/goal-session), [`hook-protocol`](../packages/hooks/hook-protocol), [`llm-retry`](../packages/llm/llm-retry), [`permission`](../packages/ui/permission), [`plan-mode`](../packages/plan/plan-mode), `runtime`, [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`scope`](../packages/core/scope), [`session`](../packages/core/session), [`subagent`](../packages/subagent/subagent), [`time-context`](../packages/context/time-context), [`tool-todo`](../packages/todo/tool-todo), [`tools`](../packages/core/tools), [`user-approval`](../packages/ui/user-approval), [`workflow`](../packages/workflow/workflow) | +| `internal/dispatch` | - | [`compact`](../packages/compact/compact), [`fs`](../packages/fs/fs), [`goal`](../packages/goal/goal), [`goal-session`](../packages/goal/goal-session), [`hook-protocol`](../packages/hooks/hook-protocol), [`llm-retry`](../packages/llm/llm-retry), [`permission`](../packages/ui/permission), [`plan-mode`](../packages/plan/plan-mode), [`pty-local`](../packages/pty/pty-local), `runtime`, [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`scope`](../packages/core/scope), [`session`](../packages/core/session), [`subagent`](../packages/subagent/subagent), [`time-context`](../packages/context/time-context), [`tool-todo`](../packages/todo/tool-todo), [`tools`](../packages/core/tools), [`user-approval`](../packages/ui/user-approval), [`workflow`](../packages/workflow/workflow) | | `internal/plugin` | - | `webserver` | | `internal/status` | - | [`agent`](../packages/core/agent) | | `slots/changed` | `runtime` (`emit`) | - | diff --git a/docs/module-graph.md b/docs/module-graph.md index ca4848477f..86b2ed9fbe 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -434,10 +434,12 @@ flowchart TD pkg_permission --> pkg_sandbox_policy pkg_permission --> pkg_session pkg_permission --> pkg_user_approval + pkg_pty_local --> pkg_agent pkg_pty_local --> pkg_invariants pkg_pty_local --> pkg_pty pkg_pty_local --> pkg_sandbox pkg_pty_local --> pkg_sandbox_policy + pkg_pty_local --> pkg_session pkg_agent_loop --> pkg_agent pkg_agent_loop --> pkg_invariants pkg_agent_loop --> pkg_llm @@ -563,11 +565,13 @@ flowchart TD pkg_tool_pty --> pkg_invariants pkg_tool_pty --> pkg_llm pkg_tool_pty --> pkg_pty + pkg_tool_pty --> pkg_retention pkg_tool_pty --> pkg_system_prompt pkg_tool_pty --> pkg_tasks pkg_tool_pty --> pkg_tools pkg_tool_tasks --> pkg_agent pkg_tool_tasks --> pkg_invariants + pkg_tool_tasks --> pkg_retention pkg_tool_tasks --> pkg_system_prompt pkg_tool_tasks --> pkg_tasks pkg_tool_tasks --> pkg_tools @@ -808,7 +812,7 @@ flowchart TD | [`session-title-all-messages-llm`](../packages/session-title/session-title-all-messages-llm) | `session-title` | [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session-title/session-title), [`session-title-llm`](../packages/session-title/session-title-llm) | | [`session-title-first-message-llm`](../packages/session-title/session-title-first-message-llm) | `session-title` | [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session-title/session-title), [`session-title-llm`](../packages/session-title/session-title-llm) | | [`permission`](../packages/ui/permission) | `ui` | [`bash`](../packages/bash/bash), [`invariants`](../packages/support/invariants), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session`](../packages/core/session), [`user-approval`](../packages/ui/user-approval) | -| [`pty-local`](../packages/pty/pty-local) | `pty` | [`invariants`](../packages/support/invariants), [`pty`](../packages/pty/pty), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy) | +| [`pty-local`](../packages/pty/pty-local) | `pty` | [`agent`](../packages/core/agent), [`invariants`](../packages/support/invariants), [`pty`](../packages/pty/pty), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session`](../packages/core/session) | | [`agent-loop`](../packages/core/agent-loop) | `core` | [`agent`](../packages/core/agent), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | | [`tool-goal`](../packages/goal/tool-goal) | `goal` | [`agent`](../packages/core/agent), [`goal`](../packages/goal/goal), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | | [`tool-bash`](../packages/bash/tool-bash) | `bash` | [`agent`](../packages/core/agent), [`bash`](../packages/bash/bash), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`paths`](../packages/util/paths), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session-persistence`](../packages/session-persistence/session-persistence), [`system-prompt`](../packages/core/system-prompt), [`tasks`](../packages/tasks/tasks), [`tools`](../packages/core/tools), [`user-approval`](../packages/ui/user-approval) | @@ -829,8 +833,8 @@ flowchart TD | [`repeat-tool-guard`](../packages/guard/repeat-tool-guard) | `guard` | [`agent`](../packages/core/agent), [`invariants`](../packages/support/invariants), [`tools`](../packages/core/tools) | | [`tool-lsp`](../packages/lsp/tool-lsp) | `lsp` | [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`lsp`](../packages/lsp/lsp), [`system-prompt`](../packages/core/system-prompt), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | | [`mcp-client`](../packages/mcp/mcp-client) | `mcp` | [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`tools`](../packages/core/tools) | -| [`tool-pty`](../packages/pty/tool-pty) | `pty` | [`agent`](../packages/core/agent), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`pty`](../packages/pty/pty), [`system-prompt`](../packages/core/system-prompt), [`tasks`](../packages/tasks/tasks), [`tools`](../packages/core/tools) | -| [`tool-tasks`](../packages/tasks/tool-tasks) | `tasks` | [`agent`](../packages/core/agent), [`invariants`](../packages/support/invariants), [`system-prompt`](../packages/core/system-prompt), [`tasks`](../packages/tasks/tasks), [`tools`](../packages/core/tools) | +| [`tool-pty`](../packages/pty/tool-pty) | `pty` | [`agent`](../packages/core/agent), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`pty`](../packages/pty/pty), [`retention`](../packages/util/retention), [`system-prompt`](../packages/core/system-prompt), [`tasks`](../packages/tasks/tasks), [`tools`](../packages/core/tools) | +| [`tool-tasks`](../packages/tasks/tool-tasks) | `tasks` | [`agent`](../packages/core/agent), [`invariants`](../packages/support/invariants), [`retention`](../packages/util/retention), [`system-prompt`](../packages/core/system-prompt), [`tasks`](../packages/tasks/tasks), [`tools`](../packages/core/tools) | | [`tool-workflow`](../packages/workflow/tool-workflow) | `workflow` | [`agent`](../packages/core/agent), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`workflow`](../packages/workflow/workflow) | | [`subagent-acp`](../packages/subagent/subagent-acp) | `subagent` | [`agent`](../packages/core/agent), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`subagent`](../packages/subagent/subagent), [`subagent-subprocess`](../packages/subagent/subagent-subprocess) | | [`subagent-inprocess`](../packages/subagent/subagent-inprocess) | `subagent` | [`agent`](../packages/core/agent), [`invariants`](../packages/support/invariants), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`subagent`](../packages/subagent/subagent), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | diff --git a/examples/acp-agent/pty.cordis.snapshot.yml b/examples/acp-agent/pty.cordis.snapshot.yml index 9ef3ff6418..07e4605375 100644 --- a/examples/acp-agent/pty.cordis.snapshot.yml +++ b/examples/acp-agent/pty.cordis.snapshot.yml @@ -14,6 +14,8 @@ name: './pty-snapshot-backend.mjs' - id: tool-pty name: '@deepseek-ai/dsh-tool-pty' + config: + maxResultBytes: 64 - id: llm-replay name: '@deepseek-ai/dsh-llm-replay' config: diff --git a/examples/acp-agent/tests/snapshots/pty-tools/session.jsonl b/examples/acp-agent/tests/snapshots/pty-tools/session.jsonl index f3157d811b..5694ae4343 100644 --- a/examples/acp-agent/tests/snapshots/pty-tools/session.jsonl +++ b/examples/acp-agent/tests/snapshots/pty-tools/session.jsonl @@ -21,7 +21,7 @@ {"type":"assistant/chunk","seq":19,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} {"type":"assistant/message","seq":20,"time":0,"data":{"turn":1,"step":2,"content":[{"type":"tool-call","id":"pty-send","name":"terminal_send","arguments":"{\"sessionId\":\"pty-1\",\"text\":\"printf 'PTY_OK\\\\n'\"}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-pro"},"usage":{"inputTokens":10,"outputTokens":5}},"sourceEventSeqs":[15,16,17,18,19],"surfaceOp":"append"} {"type":"tool/call","seq":21,"time":0,"data":{"turn":1,"step":2,"callId":"pty-send","name":"terminal_send","arguments":"{\"sessionId\":\"pty-1\",\"text\":\"printf 'PTY_OK\\\\n'\"}"}} -{"type":"tool/result","seq":22,"time":0,"data":{"turn":1,"step":2,"callId":"pty-send","content":[{"type":"text","text":"printf 'PTY_OK\\n'\nPTY_OK\ndsh> \n[wait: stdin_read]\n[session: running]"}],"isError":false,"meta":{"viewport":"printf 'PTY_OK\\n'\nPTY_OK\ndsh> ","waitReason":"stdin_read","sessionStatus":{"kind":"running"},"truncated":false}},"sourceEventSeqs":[21],"surfaceOp":"append"} +{"type":"tool/result","seq":22,"time":0,"data":{"turn":1,"step":2,"callId":"pty-send","content":[{"type":"text","text":"K\ndsh> \n[wait: stdin_read]\n[session: running]\n[output truncated]"}],"isError":false,"meta":{"viewport":"printf 'PTY_OK\\n'\nPTY_OK\ndsh> ","waitReason":"stdin_read","sessionStatus":{"kind":"running"},"truncated":false}},"sourceEventSeqs":[21],"surfaceOp":"append"} {"type":"step/end","seq":23,"time":0,"data":{"turn":1,"step":2}} {"type":"step/start","seq":24,"time":0,"data":{"turn":1,"step":3}} {"type":"assistant/chunk","seq":25,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}} diff --git a/examples/acp-agent/tests/snapshots/pty-tools/stdout.expected.jsonl b/examples/acp-agent/tests/snapshots/pty-tools/stdout.expected.jsonl index 94cb1f180e..6ecaac24c6 100644 --- a/examples/acp-agent/tests/snapshots/pty-tools/stdout.expected.jsonl +++ b/examples/acp-agent/tests/snapshots/pty-tools/stdout.expected.jsonl @@ -5,7 +5,7 @@ {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"pty-spawn","title":"Open terminal main","kind":"execute","status":"in_progress"}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"pty-spawn","status":"completed","content":[{"type":"content","content":{"type":"text","text":"started terminal session pty-1 (main) [type: shell]\ndsh> "}}]}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"pty-send","title":"printf 'PTY_OK\\n'","kind":"execute","status":"in_progress","rawInput":"printf 'PTY_OK\\n'","content":[{"type":"content","content":{"type":"text","text":"Terminal pty-1"}},{"type":"terminal","terminalId":"pty-send"}],"_meta":{"terminal_info":{"terminal_id":"pty-send","cwd":"{{cwd}}"}}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"pty-send","status":"completed","_meta":{"terminal_output":{"terminal_id":"pty-send","data":"printf 'PTY_OK\\n'\nPTY_OK\ndsh> \n[wait: stdin_read]\n[session: running]"}}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"pty-send","status":"completed","_meta":{"terminal_output":{"terminal_id":"pty-send","data":"K\ndsh> \n[wait: stdin_read]\n[session: running]\n[output truncated]"}}}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"pty-read","title":"Read terminal pty-1","kind":"read","status":"in_progress","rawInput":{"sessionId":"pty-1","offset":0,"count":20}}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"pty-read","status":"completed","content":[{"type":"content","content":{"type":"text","text":"dsh> printf 'PTY_OK\\n'\nPTY_OK\ndsh> \n[lines: 0-3 of 3]"}}]}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"pty-signal","title":"Signal terminal pty-missing","kind":"execute","status":"in_progress","rawInput":{"sessionId":"pty-missing","signal":"SIGINT"}}}} diff --git a/examples/headless-agent/pty.cordis.snapshot.yml b/examples/headless-agent/pty.cordis.snapshot.yml index f7fcea389a..0de292a8f9 100644 --- a/examples/headless-agent/pty.cordis.snapshot.yml +++ b/examples/headless-agent/pty.cordis.snapshot.yml @@ -14,5 +14,7 @@ name: '../acp-agent/pty-snapshot-backend.mjs' - id: tool-pty name: '@deepseek-ai/dsh-tool-pty' + config: + maxResultBytes: 64 - id: llm-replay name: '@deepseek-ai/dsh-llm-replay' diff --git a/examples/headless-agent/tests/snapshots/pty-tools/session.jsonl b/examples/headless-agent/tests/snapshots/pty-tools/session.jsonl index bad2f0353d..d91782e1e2 100644 --- a/examples/headless-agent/tests/snapshots/pty-tools/session.jsonl +++ b/examples/headless-agent/tests/snapshots/pty-tools/session.jsonl @@ -21,7 +21,7 @@ {"type":"assistant/chunk","seq":19,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} {"type":"assistant/message","seq":20,"time":0,"data":{"turn":1,"step":2,"content":[{"type":"tool-call","id":"pty-send","name":"terminal_send","arguments":"{\"sessionId\":\"pty-1\",\"text\":\"printf 'PTY_OK\\\\n'\"}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":10,"outputTokens":5}},"sourceEventSeqs":[15,16,17,18,19],"surfaceOp":"append"} {"type":"tool/call","seq":21,"time":0,"data":{"turn":1,"step":2,"callId":"pty-send","name":"terminal_send","arguments":"{\"sessionId\":\"pty-1\",\"text\":\"printf 'PTY_OK\\\\n'\"}"}} -{"type":"tool/result","seq":22,"time":0,"data":{"turn":1,"step":2,"callId":"pty-send","content":[{"type":"text","text":"printf 'PTY_OK\\n'\nPTY_OK\ndsh> \n[wait: stdin_read]\n[session: running]"}],"isError":false,"meta":{"viewport":"printf 'PTY_OK\\n'\nPTY_OK\ndsh> ","waitReason":"stdin_read","sessionStatus":{"kind":"running"},"truncated":false}},"sourceEventSeqs":[21],"surfaceOp":"append"} +{"type":"tool/result","seq":22,"time":0,"data":{"turn":1,"step":2,"callId":"pty-send","content":[{"type":"text","text":"K\ndsh> \n[wait: stdin_read]\n[session: running]\n[output truncated]"}],"isError":false,"meta":{"viewport":"printf 'PTY_OK\\n'\nPTY_OK\ndsh> ","waitReason":"stdin_read","sessionStatus":{"kind":"running"},"truncated":false}},"sourceEventSeqs":[21],"surfaceOp":"append"} {"type":"step/end","seq":23,"time":0,"data":{"turn":1,"step":2}} {"type":"step/start","seq":24,"time":0,"data":{"turn":1,"step":3}} {"type":"assistant/chunk","seq":25,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}} diff --git a/examples/headless-agent/tests/snapshots/pty-tools/stream-json.expected.jsonl b/examples/headless-agent/tests/snapshots/pty-tools/stream-json.expected.jsonl index b4db490cb2..ab37662dde 100644 --- a/examples/headless-agent/tests/snapshots/pty-tools/stream-json.expected.jsonl +++ b/examples/headless-agent/tests/snapshots/pty-tools/stream-json.expected.jsonl @@ -20,7 +20,7 @@ {"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":19,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}} {"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":20,"time":0,"data":{"turn":1,"step":2,"content":[{"type":"tool-call","id":"pty-send","name":"terminal_send","arguments":"{\"sessionId\":\"pty-1\",\"text\":\"printf 'PTY_OK\\\\n'\"}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":10,"outputTokens":5}},"sourceEventSeqs":[15,16,17,18,19],"surfaceOp":"append"}} {"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/call","seq":21,"time":0,"data":{"turn":1,"step":2,"callId":"pty-send","name":"terminal_send","arguments":"{\"sessionId\":\"pty-1\",\"text\":\"printf 'PTY_OK\\\\n'\"}"}}} -{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/result","seq":22,"time":0,"data":{"turn":1,"step":2,"callId":"pty-send","content":[{"type":"text","text":"printf 'PTY_OK\\n'\nPTY_OK\ndsh> \n[wait: stdin_read]\n[session: running]"}],"isError":false,"meta":{"viewport":"printf 'PTY_OK\\n'\nPTY_OK\ndsh> ","waitReason":"stdin_read","sessionStatus":{"kind":"running"},"truncated":false}},"sourceEventSeqs":[21],"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/result","seq":22,"time":0,"data":{"turn":1,"step":2,"callId":"pty-send","content":[{"type":"text","text":"K\ndsh> \n[wait: stdin_read]\n[session: running]\n[output truncated]"}],"isError":false,"meta":{"viewport":"printf 'PTY_OK\\n'\nPTY_OK\ndsh> ","waitReason":"stdin_read","sessionStatus":{"kind":"running"},"truncated":false}},"sourceEventSeqs":[21],"surfaceOp":"append"}} {"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":23,"time":0,"data":{"turn":1,"step":2}}} {"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":24,"time":0,"data":{"turn":1,"step":3}}} {"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":25,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}} diff --git a/packages/cordis/tool-cordis/src/api-catalog.ts b/packages/cordis/tool-cordis/src/api-catalog.ts index 7a8e8abdf3..a454623486 100644 --- a/packages/cordis/tool-cordis/src/api-catalog.ts +++ b/packages/cordis/tool-cordis/src/api-catalog.ts @@ -402,6 +402,10 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ signature: 'async spawn(owner: Agent, request: PtySpawnRequest, signal?: AbortSignal): Promise', jsDoc: '/**\n * Create and publish one owner-scoped session after backend setup succeeds.\n * @param owner - exact registered Agent that owns access and cleanup.\n * @param request - backend type plus optional owner-local name and cwd.\n * @param signal - cancellation of unpublished setup.\n * @returns published identity, metadata, status, and MOTD.\n */', }, + { + signature: 'hasOwnerActivity(owner: Agent): boolean', + jsDoc: '/**\n * Test whether an exact owner has a published session or unpublished spawn.\n * @param owner - exact live owner to inspect.\n * @returns true across the entire spawn-to-close interval, with no publication gap.\n */', + }, { signature: 'startSend(owner: Agent, id: PtySessionId, request: PtySendRequest): PtySendOperation', jsDoc: '/**\n * Start one exclusive interactive send.\n * @param owner - exact session owner.\n * @param id - target PTY identity.\n * @param request - explicit text, submit behavior, and cancellation.\n * @returns live operation handle for foreground await or task registration.\n */', @@ -1854,11 +1858,11 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'TaskSnapshot', - declaration: 'export interface TaskSnapshot {\n id: TaskId;\n kind: TaskKind;\n label: string;\n ownerSession?: SessionId;\n status: TaskStatus;\n detail?: string;\n startedAt: number;\n finishedAt?: number;\n reported: boolean;\n}', + declaration: 'export interface TaskSnapshot {\n id: TaskId;\n kind: TaskKind;\n label: string;\n outputLimitBytes?: number;\n ownerSession?: SessionId;\n status: TaskStatus;\n detail?: string;\n startedAt: number;\n finishedAt?: number;\n reported: boolean;\n}', }, { name: 'TaskStart', - declaration: 'export interface TaskStart {\n kind: TaskKind;\n label: string;\n owner?: Agent;\n run(): TaskHooks;\n}', + declaration: 'export interface TaskStart {\n kind: TaskKind;\n label: string;\n outputLimitBytes?: number;\n owner?: Agent;\n run(): TaskHooks;\n}', }, { name: 'TaskStatus', diff --git a/packages/pty/pty-local/README.md b/packages/pty/pty-local/README.md index 00cd1b8b00..32b2721074 100644 --- a/packages/pty/pty-local/README.md +++ b/packages/pty/pty-local/README.md @@ -4,9 +4,11 @@ Local `node-pty` backend for `ctx.pty`. It starts an interactive shell under the ## Plugin (`pty-local`) -The plugin injects `pty`, `sandbox`, and `sandboxPolicy`, then registers the configured backend type (`shell`). `danger-full-access` starts the shell directly; confined modes wrap the exact shell argv through `ctx.sandbox`. The current session-level sandbox override is resolved at spawn and remains fixed for the PTY lifetime. +The plugin injects `agents`, `pty`, `sandbox`, and `sandboxPolicy`, then registers the configured backend type (`shell`). `danger-full-access` starts the shell directly; confined modes wrap the exact shell argv through `ctx.sandbox`. The effective session mode is resolved at spawn. A change to a different effective mode is rejected before its `sandbox/mode` event commits while that owner has an open PTY or a spawn in progress; wait for creation to settle and close the sessions before changing modes, so a terminal opened with wider access cannot survive a downgrade. -Linux readiness combines a foreground-verified private bash prompt marker, foreground-process-group syscall inspection, silence fallback, and absolute timeout. macOS uses the verified prompt marker plus silence/timeout because it has no `/proc` syscall surface. Unrecognized or unreadable process state is never a positive exact-idle signal. During unpublished startup, a fallback requires observed output; zero-output silence cannot publish an empty session, and timeout rejects the spawn. Incomplete terminal-control sequences are bounded by `maxReadBytes` and discarded through their terminator after crossing that limit. +Linux readiness combines a foreground-verified private bash prompt marker, foreground-process-group syscall inspection, silence fallback, and absolute timeout. macOS uses the verified prompt marker plus silence/timeout because it has no `/proc` syscall surface. A marker is not ready until printable prompt text arrives, including when the OSC marker and `PS1` are split across data callbacks. Unrecognized or unreadable process state is never a positive exact-idle signal. During unpublished startup, a fallback requires observed output; zero-output silence cannot publish an empty session, and timeout rejects the spawn. Incomplete terminal-control sequences are bounded by `maxReadBytes` and discarded through their terminator after crossing that limit; a trailing carriage return is carried across callbacks so split CRLF becomes one newline. + +Send cancellation resolves the current foreground process group and delivers a real `SIGINT`; it never emulates interruption by writing `\x03`, so raw-mode programs remain cancellable. Close sends `SIGTERM` to descendants, waits, rescans and sends `SIGKILL` to the remaining tree, verifies that descendants left the process table while the shell can still reap them, and only then stops the shell. A survivor failure does not cache a permanently rejected close; a later close retries the teardown. ## Model Experience diff --git a/packages/pty/pty-local/package.json b/packages/pty/pty-local/package.json index 339c3916ad..fb26d845e5 100644 --- a/packages/pty/pty-local/package.json +++ b/packages/pty/pty-local/package.json @@ -30,10 +30,12 @@ }, "license": "BSD-3-Clause", "peerDependencies": { + "@deepseek-ai/dsh-agent": "^0.0.1", "@deepseek-ai/dsh-invariants": "^0.0.1", "@deepseek-ai/dsh-pty": "^0.0.1", "@deepseek-ai/dsh-sandbox": "^0.0.1", "@deepseek-ai/dsh-sandbox-policy": "^0.0.1", + "@deepseek-ai/dsh-session": "^0.0.1", "cordis": "^4.0.0-rc.7" }, "dependencies": { diff --git a/packages/pty/pty-local/src/index.ts b/packages/pty/pty-local/src/index.ts index 0b99d1b058..706a1bba7a 100644 --- a/packages/pty/pty-local/src/index.ts +++ b/packages/pty/pty-local/src/index.ts @@ -7,6 +7,7 @@ import { Context } from 'cordis' import * as nodePty from 'node-pty' import type { IPtyForkOptions } from 'node-pty' +import type { Session, SessionEvent } from '@deepseek-ai/dsh-session' import type { PtyBackend, PtyBackendSpawnSpec } from '@deepseek-ai/dsh-pty' import type { SandboxMode } from '@deepseek-ai/dsh-sandbox' import { effectiveSandboxMode } from '@deepseek-ai/dsh-sandbox-policy' @@ -20,8 +21,8 @@ export type { Config as PtyLocalConfig } from './config.ts' /** Cordis plugin name. */ export const name = 'pty-local' -/** Required services: registry plus the one shared confinement policy. */ -export const inject = ['pty', 'sandbox', 'sandboxPolicy'] +/** Required services: owner/PTY registries plus the one shared confinement policy. */ +export const inject = ['agents', 'pty', 'sandbox', 'sandboxPolicy'] const SENSITIVE_ENV_PATTERN = /KEY|SECRET|TOKEN/i @@ -73,7 +74,7 @@ export class LocalPtyBackend implements PtyBackend { } async spawn(spec: PtyBackendSpawnSpec): Promise { - if (spec.signal?.aborted === true) throw new Error('PTY spawn aborted') + spec.signal?.throwIfAborted() const argv = spawnArgv(this.ctx, this.config, spec) const file = argv[0] if (file === undefined) throw new Error('pty-local: sandbox returned empty argv') @@ -105,4 +106,17 @@ export function apply(ctx: Context, config: Config): void { validateConfig(config) const inspector = createProcessInspector() ctx.pty.registerBackend(new LocalPtyBackend(ctx, config, inspector)) + ctx.on('internal/dispatch', (_mode, eventName, args) => { + if (eventName !== 'session/event') return + const [session, event] = args as [Session, SessionEvent] + if (event.type !== 'sandbox/mode') return + const currentMode = effectiveSandboxMode(session.events) ?? ctx.sandboxPolicy.defaultMode + if (event.data.mode === currentMode) return + const owner = ctx.agents.get(session.id) + if (owner === undefined) return + if (!ctx.pty.hasOwnerActivity(owner)) return + throw new Error( + `cannot change sandbox mode from "${currentMode}" to "${event.data.mode}" while persistent terminal sessions are open or being created; wait for creation to settle and close them first`, + ) + }, { global: true }) } diff --git a/packages/pty/pty-local/src/sanitize.ts b/packages/pty/pty-local/src/sanitize.ts index 6e109f6115..cc22c29c01 100644 --- a/packages/pty/pty-local/src/sanitize.ts +++ b/packages/pty/pty-local/src/sanitize.ts @@ -9,6 +9,8 @@ export const PROMPT_MARKER_PREFIX = '133;D;' export interface SanitizedChunk { text: string prompt: boolean + /** Present when printable text followed the latest owned prompt marker. */ + promptText?: true } /** @@ -20,6 +22,8 @@ export class TerminalSanitizer { private pending = '' private discardMode: 'osc' | 'csi' | undefined private discardOscEscape = false + private trailingCarriageReturn = false + private awaitingPromptText = false constructor(private readonly maxPendingBytes: number) {} @@ -32,15 +36,24 @@ export class TerminalSanitizer { this.pending += this.discardPrefix(chunk) let text = '' let prompt = false + let promptText = false let index = 0 + const appendText = (value: string): boolean => { + text += value + if (this.awaitingPromptText && value.replace(/[\r\n\x07]/g, '').length > 0) { + this.awaitingPromptText = false + return true + } + return false + } while (index < this.pending.length) { const escape = this.pending.indexOf('\x1b', index) if (escape < 0) { - text += this.pending.slice(index) + promptText = appendText(this.pending.slice(index)) || promptText index = this.pending.length break } - text += this.pending.slice(index, escape) + promptText = appendText(this.pending.slice(index, escape)) || promptText if (escape + 1 >= this.pending.length) { index = escape break @@ -59,7 +72,11 @@ export class TerminalSanitizer { } const terminatorBytes = this.pending[end - 1] === '\x07' ? 1 : 2 const content = this.pending.slice(escape + 2, end - terminatorBytes) - if (content.startsWith(PROMPT_MARKER_PREFIX)) prompt = true + if (content.startsWith(PROMPT_MARKER_PREFIX)) { + prompt = true + promptText = false + this.awaitingPromptText = true + } index = end continue } @@ -82,7 +99,7 @@ export class TerminalSanitizer { } this.pending = this.pending.slice(index) this.enforcePendingBound() - return { text: normalizeTerminalText(text), prompt } + return { text: this.normalizeText(text), prompt, ...promptText ? { promptText: true } : {} } } /** @@ -94,7 +111,21 @@ export class TerminalSanitizer { this.pending = '' this.discardMode = undefined this.discardOscEscape = false - return normalizeTerminalText(text) + this.awaitingPromptText = false + const normalized = this.normalizeText(text) + if (!this.trailingCarriageReturn) return normalized + this.trailingCarriageReturn = false + return `${normalized}\n` + } + + private normalizeText(text: string): string { + let complete = this.trailingCarriageReturn ? `\r${text}` : text + this.trailingCarriageReturn = false + if (complete.endsWith('\r')) { + complete = complete.slice(0, -1) + this.trailingCarriageReturn = true + } + return normalizeTerminalText(complete) } private enforcePendingBound(): void { diff --git a/packages/pty/pty-local/src/session.ts b/packages/pty/pty-local/src/session.ts index 52863db674..a1e638e0d2 100644 --- a/packages/pty/pty-local/src/session.ts +++ b/packages/pty/pty-local/src/session.ts @@ -17,7 +17,7 @@ import type { PtyWaitReason, } from '@deepseek-ai/dsh-pty' import type { ResolvedConfig } from './config.ts' -import type { ProcessInspector } from './process-inspector.ts' +import type { ProcessIdentity, ProcessInspector } from './process-inspector.ts' import { TerminalSanitizer } from './sanitize.ts' function delay(ms: number): Promise { @@ -148,9 +148,11 @@ export class LocalPtySession implements PtyBackendSession { private activeTimer: NodeJS.Timeout | undefined private activeAbort: (() => void) | undefined private promptSeen = false + private promptTextSeen = false private shellPgid: number | undefined private initializing = false private lastOutputAt = Date.now() + private closing = false private closePromise: Promise | undefined constructor( @@ -190,21 +192,20 @@ export class LocalPtySession implements PtyBackendSession { } startSend(request: PtySendRequest): PtySendOperation { - if (this.closePromise !== undefined) throw new Error('PTY session is closing') + if (this.closing) throw new Error('PTY session is closing') if (this.statusValue.kind === 'exited') throw new Error('PTY session has exited') if (this.active !== undefined) throw new Error('PTY session already has an active send') if (request.signal?.aborted === true) throw new Error('PTY send aborted before write') - const operation = new LocalSendOperation(this.config.maxReadBytes, Date.now(), () => { - try { - this.terminal.write('\x03') - } catch (error: unknown) { - operation.fail(error) - } - }) + const operation = new LocalSendOperation( + this.config.maxReadBytes, + Date.now(), + () => { this.interrupt(operation) }, + ) this.active = operation this.lastOutputAt = Date.now() this.promptSeen = false + this.promptTextSeen = false if (request.signal !== undefined) { const onAbort = (): void => { operation.cancel() } @@ -267,8 +268,15 @@ export class LocalPtySession implements PtyBackendSession { } close(reason: string): Promise { - this.closePromise ??= this.closeOnce(reason) - return this.closePromise + this.closing = true + if (this.closePromise !== undefined) return this.closePromise + const closing = this.closeOnce(reason).catch((error: unknown) => { + this.closePromise = undefined + this.failActive(error) + throw error + }) + this.closePromise = closing + return closing } private onData(data: string): void { @@ -279,8 +287,11 @@ export class LocalPtySession implements PtyBackendSession { if (this.shellPgid === undefined) this.shellPgid = foregroundPgid if (foregroundPgid !== undefined && foregroundPgid === this.shellPgid) { this.promptSeen = true + this.promptTextSeen = sanitized.promptText === true this.lastOutputAt = Date.now() } + } else if (this.promptSeen && sanitized.promptText === true) { + this.promptTextSeen = true } } @@ -297,7 +308,7 @@ export class LocalPtySession implements PtyBackendSession { this.settleActive('session_exit') return } - if (this.promptSeen && Date.now() - this.lastOutputAt >= this.config.pollIntervalMs) { + if (this.promptSeen && this.promptTextSeen && Date.now() - this.lastOutputAt >= this.config.pollIntervalMs) { this.settleActive('stdin_read') return } @@ -337,58 +348,98 @@ export class LocalPtySession implements PtyBackendSession { this.active = undefined } + private failActive(error: unknown): void { + const operation = this.active + if (operation === undefined) return + this.clearActive() + operation.fail(error) + } + + private interrupt(operation: LocalSendOperation): void { + if (this.active !== operation) return + try { + const pgid = this.inspector.foregroundPgid(this.pid) + if (pgid === undefined) throw new Error(`cannot resolve foreground process group for PTY ${this.pid}`) + this.inspector.signalGroup(pgid, 'SIGINT') + } catch (error: unknown) { + this.failActive(error) + } + } + + private survivors(members: ProcessIdentity[]): ProcessIdentity[] { + return members.filter(member => this.inspector.isAlive(member)) + } + + private descendants(): ProcessIdentity[] { + return this.inspector.processTree(this.pid).filter(member => member.pid !== this.pid) + } + + private async waitForExit(members: ProcessIdentity[]): Promise { + const deadline = Date.now() + this.config.disposeGraceMs + let survivors = this.survivors(members) + while (survivors.length > 0 && Date.now() < deadline) { + await delay(Math.min(25, Math.max(1, deadline - Date.now()))) + survivors = this.survivors(members) + } + return survivors + } + + private signalMembers(members: ProcessIdentity[], signal: 'SIGTERM' | 'SIGKILL'): void { + for (const member of members) { + try { + this.inspector.signalProcess(member, signal) + } catch (_alreadyExitedDuringSignal) { + // Identity is rechecked by the inspector; a same-tick exit is success. + } + } + } + + private async stopDescendants(): Promise { + let members = this.descendants() + this.signalMembers(members, 'SIGTERM') + await this.waitForExit(members) + // A TERM-handling descendant may have forked while winding down. Rescan + // while the shell can still reap every member, then kill the fresh tree. + members = this.descendants() + this.signalMembers(members, 'SIGKILL') + await this.waitForExit(members) + return this.descendants().filter(member => this.inspector.isAlive(member)) + } + + private async stopShell(): Promise { + try { + this.terminal.kill('SIGTERM') + } catch (_topLevelAlreadyExitedDuringTerm) { + // The exit notification remains authoritative. + } + if (this.statusValue.kind === 'running') { + await Promise.race([this.exitPromise.promise, delay(this.config.disposeGraceMs)]) + } + if (this.statusValue.kind === 'running') { + try { + this.terminal.kill('SIGKILL') + } catch (_topLevelAlreadyExitedDuringKill) { + // The exit notification remains authoritative. + } + await Promise.race([this.exitPromise.promise, delay(this.config.disposeGraceMs)]) + } + if (this.statusValue.kind === 'running') { + throw new Error(`PTY cleanup failed; surviving pids: ${this.pid}`) + } + } + private async closeOnce(reason: string): Promise { this.dataDisposable.dispose() // Stop readiness polling but retain the active operation: teardown settles // it as session_exit below, so an in-flight send is never mis-settled as // stdin_read/inferred_idle/timeout during the grace period. this.stopPolling() - const members = this.inspector.processTree(this.pid) - for (const member of members) { - try { - this.inspector.signalProcess(member, 'SIGTERM') - } catch (_alreadyExitedDuringTerm) { - // Identity is rechecked by the inspector; a same-tick exit is success. - } - } - try { - this.terminal.kill('SIGTERM') - } catch (_topLevelAlreadyExited) { - // onExit or identity checks below remain authoritative. - } - - const deadline = Date.now() + this.config.disposeGraceMs - let survivors = members.filter(member => this.inspector.isAlive(member)) - while (survivors.length > 0 && Date.now() < deadline) { - await delay(Math.min(25, this.config.disposeGraceMs)) - survivors = members.filter(member => this.inspector.isAlive(member)) - } - for (const survivor of survivors) { - try { - this.inspector.signalProcess(survivor, 'SIGKILL') - } catch (_alreadyExitedDuringKill) { - // Final identity check below decides success. - } - } - try { - this.terminal.kill('SIGKILL') - } catch (_topLevelAlreadyKilled) { - // The root may already have delivered onExit. - } - - const killDeadline = Date.now() + this.config.disposeGraceMs - survivors = members.filter(member => this.inspector.isAlive(member)) - while (survivors.length > 0 && Date.now() < killDeadline) { - await delay(Math.min(25, this.config.disposeGraceMs)) - survivors = members.filter(member => this.inspector.isAlive(member)) - } - const exitWaitMs = Math.max(0, killDeadline - Date.now()) - await Promise.race([this.exitPromise.promise, delay(exitWaitMs)]) - survivors = members.filter(member => this.inspector.isAlive(member)) - this.settleActive('session_exit') - this.exitDisposable.dispose() + const survivors = await this.stopDescendants() if (survivors.length > 0) { throw new Error(`PTY cleanup failed (${reason}); surviving pids: ${survivors.map(member => member.pid).join(', ')}`) } + await this.stopShell() + this.settleActive('session_exit') + this.exitDisposable.dispose() } } diff --git a/packages/pty/pty-local/tests/index.spec.ts b/packages/pty/pty-local/tests/index.spec.ts index 1273824033..55cc134787 100644 --- a/packages/pty/pty-local/tests/index.spec.ts +++ b/packages/pty/pty-local/tests/index.spec.ts @@ -2,12 +2,14 @@ import { describe, expect, it, vi } from 'vitest' import type { IPty, IPtyForkOptions } from 'node-pty' import { Context } from 'cordis' import Loader from '@cordisjs/plugin-loader' -import { Session, SessionId } from '@deepseek-ai/dsh-session' +import SessionStore, { Session, SessionId } from '@deepseek-ai/dsh-session' +import AgentRegistry from '@deepseek-ai/dsh-agent' import type { Agent } from '@deepseek-ai/dsh-agent' import SandboxProvider from '@deepseek-ai/dsh-sandbox' import type { ConfinedArgv, SandboxPolicy } from '@deepseek-ai/dsh-sandbox' -import SandboxPolicyService from '@deepseek-ai/dsh-sandbox-policy' +import SandboxPolicyService, { setSandboxMode } from '@deepseek-ai/dsh-sandbox-policy' import PtyService, { PtySessionId } from '@deepseek-ai/dsh-pty' +import type { PtyBackendSession } from '@deepseek-ai/dsh-pty' import { LocalPtyBackend } from '@deepseek-ai/dsh-pty-local' import * as ptyLocal from '@deepseek-ai/dsh-pty-local' import type { ResolvedConfig } from '@deepseek-ai/dsh-pty-local/src/config.ts' @@ -69,8 +71,9 @@ describe('LocalPtyBackend startup rollback', () => { await ctx.plugin(SandboxPolicyService, { mode: 'read-only', workspaceRoot: '/tmp' }) const backend = new LocalPtyBackend(ctx, config(), inspector) const controller = new AbortController() - controller.abort() - await expect(backend.spawn(spec(agent(ctx), controller.signal))).rejects.toThrow('spawn aborted') + const abortReason = new Error('spawn aborted') + controller.abort(abortReason) + await expect(backend.spawn(spec(agent(ctx), controller.signal))).rejects.toBe(abortReason) await expect(backend.spawn(spec(agent(ctx)))).rejects.toThrow('empty argv') }) @@ -169,12 +172,13 @@ describe('pty-local plugin shape', () => { const loader = Object.create(Loader.prototype) as Loader const unwrapped = loader.unwrapExports(ptyLocal) as Record expect(unwrapped.name).toBe('pty-local') - expect(unwrapped.inject).toEqual(['pty', 'sandbox', 'sandboxPolicy']) + expect(unwrapped.inject).toEqual(['agents', 'pty', 'sandbox', 'sandboxPolicy']) expect(unwrapped.Config).toBeDefined() }) it('validates config and registers the configured backend', async () => { const ctx = new Context() + await ctx.plugin(AgentRegistry) await ctx.plugin(PtyService) await ctx.plugin(EmptySandbox) await ctx.plugin(SandboxPolicyService, { mode: 'danger-full-access', workspaceRoot: '/tmp' }) @@ -183,4 +187,91 @@ describe('pty-local plugin shape', () => { await fiber.dispose() expect(ctx.pty.listBackends()).toEqual([]) }) + + it('ignores unrelated session events and mode changes without a live owner', async () => { + const ctx = new Context() + await ctx.plugin(SessionStore) + await ctx.plugin(AgentRegistry) + await ctx.plugin(PtyService) + await ctx.plugin(EmptySandbox) + await ctx.plugin(SandboxPolicyService, { mode: 'danger-full-access', workspaceRoot: '/tmp' }) + await ctx.plugin(ptyLocal, config()) + + const session = ctx.sessions.create(SessionId('unowned-mode')) + expect(() => { + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + }).not.toThrow() + expect(() => { setSandboxMode(session, 'read-only') }).not.toThrow() + }) + + it('rejects an effective sandbox-mode change until the owner closes live terminals', async () => { + const ctx = new Context() + await ctx.plugin(SessionStore) + await ctx.plugin(AgentRegistry) + await ctx.plugin(PtyService) + await ctx.plugin(EmptySandbox) + await ctx.plugin(SandboxPolicyService, { mode: 'danger-full-access', workspaceRoot: '/tmp' }) + await ctx.plugin(ptyLocal, config()) + + const session = ctx.sessions.create(SessionId('mode-owner')) + const owner: Agent = { + id: session.id, options: {}, session, status: 'idle', ctx, + send() {}, steer() {}, inject() {}, cancel() {}, whenIdle: () => Promise.resolve(), + } + ctx.agents.register(owner) + const backendSession = { + motd: '', + startSend: () => { throw new Error('unused') }, + read: () => { throw new Error('unused') }, + signal: () => Promise.resolve({ delivered: true, targetPgid: 1 }), + status: () => ({ kind: 'running' as const }), + close: () => Promise.resolve(), + } satisfies PtyBackendSession + ctx.pty.registerBackend({ type: 'stub', spawn: () => Promise.resolve(backendSession) }) + const created = await ctx.pty.spawn(owner, { type: 'stub' }) + + expect(() => { setSandboxMode(session, 'danger-full-access') }).not.toThrow() + expect(() => { setSandboxMode(session, 'read-only') }).toThrow( + 'cannot change sandbox mode from "danger-full-access" to "read-only" while persistent terminal sessions are open or being created; wait for creation to settle and close them first', + ) + expect(session.events.filter(event => event.type === 'sandbox/mode')).toHaveLength(1) + + await ctx.pty.kill(owner, created.sessionId) + expect(() => { setSandboxMode(session, 'read-only') }).not.toThrow() + expect(session.events.filter(event => event.type === 'sandbox/mode')).toHaveLength(2) + }) + + it('also fences sandbox-mode changes across unpublished PTY creation', async () => { + const ctx = new Context() + await ctx.plugin(SessionStore) + await ctx.plugin(AgentRegistry) + await ctx.plugin(PtyService) + await ctx.plugin(EmptySandbox) + await ctx.plugin(SandboxPolicyService, { mode: 'danger-full-access', workspaceRoot: '/tmp' }) + await ctx.plugin(ptyLocal, config()) + + const session = ctx.sessions.create(SessionId('pending-mode-owner')) + const owner: Agent = { + id: session.id, options: {}, session, status: 'idle', ctx, + send() {}, steer() {}, inject() {}, cancel() {}, whenIdle: () => Promise.resolve(), + } + ctx.agents.register(owner) + const gate = Promise.withResolvers() + ctx.pty.registerBackend({ type: 'slow', spawn: () => gate.promise }) + const spawning = ctx.pty.spawn(owner, { type: 'slow' }) + + expect(ctx.pty.hasOwnerActivity(owner)).toBe(true) + expect(() => { setSandboxMode(session, 'read-only') }).toThrow('open or being created') + gate.resolve({ + motd: '', + startSend: () => { throw new Error('unused') }, + read: () => { throw new Error('unused') }, + signal: () => Promise.resolve({ delivered: true, targetPgid: 1 }), + status: () => ({ kind: 'running' as const }), + close: () => Promise.resolve(), + }) + const created = await spawning + await ctx.pty.kill(owner, created.sessionId) + expect(ctx.pty.hasOwnerActivity(owner)).toBe(false) + }) }) diff --git a/packages/pty/pty-local/tests/local.spec.ts b/packages/pty/pty-local/tests/local.spec.ts index 0ff5aebf35..182e2aae19 100644 --- a/packages/pty/pty-local/tests/local.spec.ts +++ b/packages/pty/pty-local/tests/local.spec.ts @@ -7,6 +7,7 @@ import { Session, SessionId } from '@deepseek-ai/dsh-session' import AgentRegistry from '@deepseek-ai/dsh-agent' import type { Agent } from '@deepseek-ai/dsh-agent' import PtyService from '@deepseek-ai/dsh-pty' +import type { PtySendOperation } from '@deepseek-ai/dsh-pty' import SandboxProvider from '@deepseek-ai/dsh-sandbox' import type { ConfinedArgv, SandboxPolicy } from '@deepseek-ai/dsh-sandbox' import SandboxPolicyService from '@deepseek-ai/dsh-sandbox-policy' @@ -62,6 +63,16 @@ async function harness(mode: 'danger-full-access' | 'workspace-write') { return { ctx, root, agent, fiber, sandbox: ctx.sandbox as PassthroughSandbox } } +async function waitForOutput(operation: PtySendOperation, expected: string): Promise { + const deadline = Date.now() + 2_000 + let output = '' + while (!output.includes(expected) && Date.now() < deadline) { + output += operation.readOutput().delta + if (!output.includes(expected)) await new Promise(resolve => setTimeout(resolve, 10)) + } + expect(output).toContain(expected) +} + describe('pty-local real shell', () => { it('persists cwd and environment across sends, scrubs secrets, and closes', async () => { const previous = process.env.DSH_TEST_SECRET @@ -119,4 +130,26 @@ describe('pty-local real shell', () => { await ctx.pty.kill(agent, created.sessionId) expect(() => process.kill(pid, 0)).toThrow() }, 10_000) + + it('cancels a raw-mode foreground process with a real SIGINT', async () => { + const { ctx, agent } = await harness('danger-full-access') + const created = await ctx.pty.spawn(agent, { type: 'shell' }) + const controller = new AbortController() + const foreground = ctx.pty.startSend(agent, created.sessionId, { + text: 'python3 -c \'import signal,sys,termios,time; signal.signal(signal.SIGINT, lambda *_: (print("SIGINT_SEEN", flush=True), sys.exit(0))); attrs=termios.tcgetattr(0); attrs[3] &= ~termios.ISIG; termios.tcsetattr(0, termios.TCSANOW, attrs); print("RAW_READY", flush=True); time.sleep(60)\'', + submit: true, + signal: controller.signal, + }) + await waitForOutput(foreground, 'RAW_READY') + controller.abort() + const result = await foreground.done + expect(result.waitReason).toBe('stdin_read') + const after = await ctx.pty.startSend(agent, created.sessionId, { + text: 'echo AFTER_SIGINT', + submit: true, + }).done + expect(after.viewport).toContain('AFTER_SIGINT') + expect(after.waitReason).toBe('stdin_read') + await ctx.pty.kill(agent, created.sessionId) + }, 10_000) }) diff --git a/packages/pty/pty-local/tests/sanitize.spec.ts b/packages/pty/pty-local/tests/sanitize.spec.ts index eee994e1e5..4da4ab0d73 100644 --- a/packages/pty/pty-local/tests/sanitize.spec.ts +++ b/packages/pty/pty-local/tests/sanitize.spec.ts @@ -7,7 +7,7 @@ describe('TerminalSanitizer', () => { expect(sanitizer.push('red\x1b[3')).toEqual({ text: 'red', prompt: false }) expect(sanitizer.push('1m text\x1b[0m\r\n')).toEqual({ text: ' text\n', prompt: false }) expect(sanitizer.push('\x1b]133;')).toEqual({ text: '', prompt: false }) - expect(sanitizer.push('D;0\x07dsh> ')).toEqual({ text: 'dsh> ', prompt: true }) + expect(sanitizer.push('D;0\x07dsh> ')).toEqual({ text: 'dsh> ', prompt: true, promptText: true }) }) it('drops unrelated OSC, short escapes, BEL, and incomplete trailing escape', () => { @@ -25,6 +25,20 @@ describe('TerminalSanitizer', () => { expect(normalizeTerminalText('a\r\nb\rc\x07')).toBe('a\nb\nc') }) + it('carries a trailing carriage return across data chunks and flushes standalone CR', () => { + const sanitizer = new TerminalSanitizer(64) + expect(sanitizer.push('a\r')).toEqual({ text: 'a', prompt: false }) + expect(sanitizer.push('\nb')).toEqual({ text: '\nb', prompt: false }) + expect(sanitizer.push('\r')).toEqual({ text: '', prompt: false }) + expect(sanitizer.flush()).toBe('\n') + }) + + it('reports printable prompt text that follows a marker in a later chunk', () => { + const sanitizer = new TerminalSanitizer(64) + expect(sanitizer.push('\x1b]133;D;0\x07')).toEqual({ text: '', prompt: true }) + expect(sanitizer.push('dsh> ')).toEqual({ text: 'dsh> ', prompt: false, promptText: true }) + }) + it('bounds and discards unterminated control sequences through their terminators', () => { const oscBel = new TerminalSanitizer(8) expect(oscBel.push(`\x1b]0;${'x'.repeat(16)}`)).toEqual({ text: '', prompt: false }) diff --git a/packages/pty/pty-local/tests/session.spec.ts b/packages/pty/pty-local/tests/session.spec.ts index 562aaf4eb3..42d634e368 100644 --- a/packages/pty/pty-local/tests/session.spec.ts +++ b/packages/pty/pty-local/tests/session.spec.ts @@ -15,6 +15,7 @@ class FakeTerminal { kills: string[] = [] throwWrite = false throwKill = false + autoExitOnKill = true private dataListeners = new Set<(data: string) => void>() private exitListeners = new Set<(event: { exitCode: number; signal?: number }) => void>() @@ -44,7 +45,7 @@ class FakeTerminal { kill(signal?: string): void { if (this.throwKill) throw new Error('kill failed') this.kills.push(signal ?? 'SIGHUP') - this.emitExit(0, signal === 'SIGKILL' ? 9 : 15) + if (this.autoExitOnKill) this.emitExit(0, signal === 'SIGKILL' ? 9 : 15) } resize() {} @@ -148,7 +149,7 @@ describe('LocalPtySession readiness and output', () => { expect(() => session.startSend({ text: '', submit: false })).toThrow('has exited') }) - it('cancels with Ctrl-C, observes AbortSignal, and contains write failures', async () => { + it('cancels with foreground-group SIGINT, observes AbortSignal, and contains write failures', async () => { vi.useFakeTimers() const terminal = new FakeTerminal() const inspector = new FakeInspector() @@ -159,7 +160,8 @@ describe('LocalPtySession readiness and output', () => { const operation = session.startSend({ text: 'sleep', submit: true, signal: controller.signal }) expect(() => session.startSend({ text: 'again', submit: true })).toThrow('active send') controller.abort() - expect(terminal.writes.at(-1)).toBe('\x03') + expect(inspector.groups).toContainEqual([456, 'SIGINT']) + expect(terminal.writes).not.toContain('\x03') terminal.emitData('\x1b]133;D;130\x07dsh> ') await vi.advanceTimersByTimeAsync(10) await operation.done @@ -196,11 +198,13 @@ describe('LocalPtySession readiness and output', () => { operationInternal.append('') const sessionInternal = session as unknown as { pollReadiness(operation: PtySendOperation): void + interrupt(operation: PtySendOperation): void statusValue: PtySessionStatus appendOutput(text: string): void } sessionInternal.appendOutput('') sessionInternal.pollReadiness({} as PtySendOperation) + sessionInternal.interrupt({} as PtySendOperation) sessionInternal.statusValue = { kind: 'exited', exitCode: 2, signal: null } sessionInternal.pollReadiness(operation) await operation.done @@ -212,12 +216,23 @@ describe('LocalPtySession readiness and output', () => { expect(unknown.status()).toEqual({ kind: 'exited', exitCode: 1, signal: null }) const cancelTerminal = new FakeTerminal() - const cancel = new LocalPtySession(cancelTerminal.asPty(), new FakeInspector(), config()) + const cancelInspector = new FakeInspector() + const cancel = new LocalPtySession(cancelTerminal.asPty(), cancelInspector, config()) await initialize(cancel, cancelTerminal) const cancellable = cancel.startSend({ text: '', submit: false }) - cancelTerminal.throwWrite = true + cancelInspector.throwGroup = true expect(cancellable.cancel()).toBe(true) - await expect(cancellable.done).rejects.toThrow('write failed') + await expect(cancellable.done).rejects.toThrow('group failed') + expect(cancellable.cancel()).toBe(false) + + const missingGroupTerminal = new FakeTerminal() + const missingGroupInspector = new FakeInspector() + const missingGroup = new LocalPtySession(missingGroupTerminal.asPty(), missingGroupInspector, config()) + await initialize(missingGroup, missingGroupTerminal) + missingGroupInspector.pgid = undefined + const unresolved = missingGroup.startSend({ text: '', submit: false }) + expect(unresolved.cancel()).toBe(true) + await expect(unresolved.done).rejects.toThrow('cannot resolve foreground process group') }) it('does not treat zero-output startup silence as readiness and fails on startup timeout', async () => { @@ -239,6 +254,23 @@ describe('LocalPtySession readiness and output', () => { await timedOut }) + it('waits for printable prompt text when the startup marker is split from PS1', async () => { + vi.useFakeTimers() + const terminal = new FakeTerminal() + const session = new LocalPtySession(terminal.asPty(), new FakeInspector(), config()) + let settled = false + const initializing = session.initialize().then(() => { settled = true }) + + terminal.emitData('\x1b]133;D;0\x07') + await vi.advanceTimersByTimeAsync(20) + expect(settled).toBe(false) + + terminal.emitData('dsh> ') + await vi.advanceTimersByTimeAsync(10) + await initializing + expect(session.motd).toBe('dsh> ') + }) + it('trusts prompt markers only while the startup shell owns the foreground group', async () => { vi.useFakeTimers() const terminal = new FakeTerminal() @@ -328,14 +360,15 @@ describe('LocalPtySession bounds, signals, and teardown', () => { // readiness poll would otherwise mis-settle this as stdin_read once close // begins, so teardown must stop polling before its grace period. terminal.emitData('\x1b]133;D;0\x07dsh> ') - terminal.throwKill = true + terminal.autoExitOnKill = false const closing = session.close('mid-send') - await vi.advanceTimersByTimeAsync(60) + await vi.advanceTimersByTimeAsync(20) + terminal.emitExit(0, 15) expect((await operation.done).waitReason).toBe('session_exit') await closing }) - it('waits for SIGKILL recipients to leave the process table after the shell exits', async () => { + it('keeps the shell alive until SIGKILL recipients leave the process table', async () => { vi.useFakeTimers() const terminal = new FakeTerminal() const inspector = new FakeInspector() @@ -348,11 +381,59 @@ describe('LocalPtySession bounds, signals, and teardown', () => { const closing = session.close('test').then(() => { settled = true }) await vi.advanceTimersByTimeAsync(20) expect(inspector.processes).toContainEqual([124, 'SIGKILL']) + expect(terminal.kills).toEqual([]) expect(settled).toBe(false) inspector.alive.delete(124) await vi.advanceTimersByTimeAsync(20) await closing + expect(terminal.kills).toEqual(['SIGTERM']) expect(settled).toBe(true) }) + + it('rescans for descendants forked during TERM before stopping the shell', async () => { + const terminal = new FakeTerminal() + const inspector = new FakeInspector() + let reads = 0 + inspector.processTree = () => { + reads += 1 + if (reads === 1) { + inspector.alive.add(124) + return [{ pid: 124, started: 'first' }] + } + if (reads === 2) { + inspector.alive.add(125) + return [{ pid: 125, started: 'late' }] + } + return [] + } + const session = new LocalPtySession(terminal.asPty(), inspector, config()) + + await session.close('test') + + expect(inspector.processes).toEqual([[124, 'SIGTERM'], [125, 'SIGKILL']]) + expect(terminal.kills).toEqual(['SIGTERM']) + }) + + it('allows teardown to retry after a descendant-survivor failure', async () => { + vi.useFakeTimers() + const terminal = new FakeTerminal() + const inspector = new FakeInspector() + inspector.members = [{ pid: 124, started: 'child' }] + inspector.alive.add(124) + inspector.removeOnSignal = false + const session = new LocalPtySession(terminal.asPty(), inspector, config({ disposeGraceMs: 10 })) + + const first = session.close('first') + const rejected = expect(first).rejects.toThrow('surviving pids: 124') + await vi.advanceTimersByTimeAsync(25) + await rejected + expect(terminal.kills).toEqual([]) + + inspector.alive.delete(124) + const second = session.close('retry') + expect(second).not.toBe(first) + await second + expect(terminal.kills).toEqual(['SIGTERM']) + }) }) diff --git a/packages/pty/pty-local/tsconfig.json b/packages/pty/pty-local/tsconfig.json index 06b5dcd4e7..45a03248db 100644 --- a/packages/pty/pty-local/tsconfig.json +++ b/packages/pty/pty-local/tsconfig.json @@ -17,6 +17,12 @@ { "path": "../../../vendor/schemastery" }, + { + "path": "../../core/agent" + }, + { + "path": "../../core/session" + }, { "path": "../pty" }, diff --git a/packages/pty/pty/README.md b/packages/pty/pty/README.md index 3620d8eaab..6916a4ef82 100644 --- a/packages/pty/pty/README.md +++ b/packages/pty/pty/README.md @@ -5,10 +5,12 @@ Owner-scoped persistent PTY seam. `PtyService` registers as `ctx.pty`, mints opa ## Contract - Backends register one stable `type` and return an unpublished `PtyBackendSession`; failed or cancelled setup must clean partial resources. +- Spawn cancellation preserves the caller's exact abort reason. Service disposal and owner loss remain distinct machine-routable failures after backend setup. +- `hasOwnerActivity(owner)` spans unpublished setup through final close, so lifecycle policy can fence the exact owner without a publication race. - A successful spawn publishes one `PtySessionId`. The optional `name` is owner-local display metadata, never authority. - One session accepts at most one live send operation. Reads and signals may observe it; another send fails until the operation settles. - `PtySendResult.waitReason` and `sessionStatus` are independent. `session_exit` describes the top-level PTY process, not an arbitrary foreground command. -- `kill()` and disposal resolve only after the backend's captured process tree is quiescent. A cleanup failure rejects instead of claiming success. +- `kill()` and disposal resolve only after the backend's captured process tree is quiescent. A cleanup failure rejects instead of claiming success and leaves the close retriable. The seam contains no `node-pty`, sandbox, tool-schema, prompt, task, or terminal-rendering policy. Implementations own terminal mechanics; consumers own model presentation and optional background-task registration. diff --git a/packages/pty/pty/src/index.ts b/packages/pty/pty/src/index.ts index 7bef5f709f..e809f50e3e 100644 --- a/packages/pty/pty/src/index.ts +++ b/packages/pty/pty/src/index.ts @@ -77,10 +77,6 @@ export function PtySessionId(value: string): PtySessionId { return value as PtySessionId } -function isAborted(signal: AbortSignal | undefined): boolean { - return signal?.aborted === true -} - interface SessionRecord { readonly id: PtySessionId readonly owner: Agent @@ -96,6 +92,7 @@ export class PtyService extends Service { private readonly backends = new Map() private readonly sessions = new Map() private readonly reservedNames = new Map>() + private readonly pendingSpawns = new Map() private readonly ownerCleanups = new Map Promise | void>() private readonly disposedOwners = new WeakSet() private nextId = 0 @@ -142,13 +139,13 @@ export class PtyService extends Service { */ async spawn(owner: Agent, request: PtySpawnRequest, signal?: AbortSignal): Promise { this.assertActive() + signal?.throwIfAborted() this.ensureOwnerCleanup(owner) const backend = this.backends.get(request.type) if (backend === undefined) throw new PtyError(`no PTY backend registered for "${request.type}"`, 'NO_BACKEND') if (request.name !== undefined && request.name.length === 0) throw new Error('PTY session name must be non-empty') - if (isAborted(signal)) throw new Error('PTY spawn aborted') - const releaseName = this.reserveName(owner, request.name) + const releaseSpawn = this.reserveSpawn(owner) const sessionId = PtySessionId(`pty-${++this.nextId}`) let session: PtyBackendSession | undefined try { @@ -160,7 +157,11 @@ export class PtyService extends Service { ...request.cwd !== undefined ? { cwd: request.cwd } : {}, ...signal !== undefined ? { signal } : {}, }) - if (this.disposing || isAborted(signal) || !this.isLiveOwner(owner)) { + signal?.throwIfAborted() + if (this.disposing) { + throw new PtyError('PTY service is disposing', 'SERVICE_DISPOSING') + } + if (!this.isLiveOwner(owner)) { throw new PtyError('PTY owner is no longer live', 'OWNER_NOT_LIVE') } const record: SessionRecord = { @@ -184,10 +185,21 @@ export class PtyService extends Service { } throw error } finally { + releaseSpawn() releaseName() } } + /** + * Test whether an exact owner has a published session or unpublished spawn. + * @param owner - exact live owner to inspect. + * @returns true across the entire spawn-to-close interval, with no publication gap. + */ + hasOwnerActivity(owner: Agent): boolean { + return (this.pendingSpawns.get(owner) ?? 0) > 0 + || [...this.sessions.values()].some(record => record.owner === owner) + } + /** * Start one exclusive interactive send. * @param owner - exact session owner. @@ -302,6 +314,15 @@ export class PtyService extends Service { } } + private reserveSpawn(owner: Agent): () => void { + this.pendingSpawns.set(owner, (this.pendingSpawns.get(owner) ?? 0) + 1) + return () => { + const remaining = (this.pendingSpawns.get(owner) ?? 1) - 1 + if (remaining === 0) this.pendingSpawns.delete(owner) + else this.pendingSpawns.set(owner, remaining) + } + } + private expectOwned(owner: Agent, id: PtySessionId): SessionRecord { const record = this.sessions.get(id) if (record === undefined) throw new PtyError(`unknown PTY session ${id}`, 'NO_SESSION') @@ -339,6 +360,7 @@ export class PtyService extends Service { } finally { this.backends.clear() this.reservedNames.clear() + this.pendingSpawns.clear() const cleanups = [...this.ownerCleanups.values()] this.ownerCleanups.clear() await Promise.all(cleanups.map(cleanup => Promise.resolve(cleanup()))) diff --git a/packages/pty/pty/tests/service.spec.ts b/packages/pty/pty/tests/service.spec.ts index 17b0302ea6..21587163f6 100644 --- a/packages/pty/pty/tests/service.spec.ts +++ b/packages/pty/pty/tests/service.spec.ts @@ -178,8 +178,9 @@ describe('PtyService ownership and lifecycle', () => { const created = await ctx.pty.spawn(owner, { type: 'stub', name: 'main' }) await expect(ctx.pty.spawn(owner, { type: 'stub', name: '' })).rejects.toThrow('must be non-empty') const aborted = new AbortController() - aborted.abort() - await expect(ctx.pty.spawn(owner, { type: 'stub' }, aborted.signal)).rejects.toThrow('spawn aborted') + const abortReason = new Error('spawn aborted') + aborted.abort(abortReason) + await expect(ctx.pty.spawn(owner, { type: 'stub' }, aborted.signal)).rejects.toBe(abortReason) await expect(ctx.pty.spawn(owner, { type: 'stub', name: 'main' })).rejects.toMatchObject({ code: 'DUPLICATE_NAME' }) const operation = ctx.pty.startSend(owner, created.sessionId, { text: 'echo hi', submit: true }) @@ -211,6 +212,41 @@ describe('PtyService ownership and lifecycle', () => { expect(session.closed).toEqual(['PTY spawn rolled back']) }) + it('preserves caller cancellation when a pending backend spawn completes', async () => { + const ctx = await harness() + const gate = Promise.withResolvers() + const session = new StubSession() + ctx.pty.registerBackend({ type: 'slow', spawn: () => gate.promise }) + const owner = stubAgent(ctx, 'owner') + ctx.agents.register(owner) + const controller = new AbortController() + const reason = new Error('cancelled by caller') + + const pending = ctx.pty.spawn(owner, { type: 'slow' }, controller.signal) + controller.abort(reason) + gate.resolve(session) + + await expect(pending).rejects.toBe(reason) + expect(session.closed).toEqual(['PTY spawn rolled back']) + expect(ctx.agents.get(owner.id)).toBe(owner) + }) + + it('rolls back an unpublished backend session when service disposal wins', async () => { + const ctx = await harness() + const gate = Promise.withResolvers() + const session = new StubSession() + ctx.pty.registerBackend({ type: 'slow', spawn: () => gate.promise }) + const owner = stubAgent(ctx, 'owner') + ctx.agents.register(owner) + + const pending = ctx.pty.spawn(owner, { type: 'slow' }) + await disposePtyService(ctx) + gate.resolve(session) + + await expect(pending).rejects.toMatchObject({ code: 'SERVICE_DISPOSING' }) + expect(session.closed).toEqual(['PTY spawn rolled back']) + }) + it('keeps independent reservations and handles provider failure before publication', async () => { const ctx = await harness() const firstGate = Promise.withResolvers() diff --git a/packages/pty/tool-pty/README.md b/packages/pty/tool-pty/README.md index 5a0edca33a..0857c5d3ee 100644 --- a/packages/pty/tool-pty/README.md +++ b/packages/pty/tool-pty/README.md @@ -2,7 +2,16 @@ Six model-facing tools over `ctx.pty`: `terminal_open`, `terminal_send`, `terminal_read`, `terminal_signal`, `terminal_close`, and `terminal_list`. Every operation requires the exact initiating `Agent`, so a model cannot address another agent's terminal even if it learns the id. -`terminal_send(run_in_background: true)` reuses `ctx.tasks`; task preflight occurs before any terminal write, completion is collected with `task_output`, and `task_kill` requests `Ctrl-C`. Foreground sends use terminal ACP cards; lifecycle, history, signal, and list calls use generic cards. +`terminal_send(run_in_background: true)` reuses `ctx.tasks`; task preflight and the PTY service's exclusive per-session send reservation occur before the task id is returned, completion is collected with `task_output`, and `task_kill` delivers `SIGINT` to the foreground process group. Foreground sends use terminal ACP call/result cards. Background sends use a generic execute card; open, read, signal, close, and list use generic `execute`, `read`, `execute`, `delete`, and `read` cards respectively. None declares source locations. + +## Config + +| key | default | meaning | +|---|---:|---| +| `enableRunInBackground` | `true` | expose and accept `run_in_background`; false omits the schema field and rejects a forced undeclared argument | +| `maxResultBytes` | `262144` | UTF-8 cap for each complete terminal result or PTY task output after wait, session, pagination, truncation, and task-status metadata | + +Both values are validated at load. When a result exceeds `maxResultBytes`, rendering reserves space for control metadata and a truncation marker when they fit; cuts preserve UTF-8 boundaries. ## Model Experience @@ -44,11 +53,11 @@ Prefix-stable while tool visibility and definitions are unchanged. #### What the model sees -Spawn returns the id and bounded MOTD. Send/read return bounded terminal text plus readiness/history markers. Background mode returns a generic task id. Results remain in session history until compaction; incremental task reads do not repeat consumed output. +Spawn returns the id and bounded MOTD. Send/read return bounded terminal text plus readiness/history markers. Background mode returns a generic task id. Every complete result is capped by `maxResultBytes`, including generic task status text. Results remain in session history until compaction; incremental task reads do not repeat consumed output. #### Token effect -Data-dependent and bounded by the backend; each returned result remains in history until compaction. +Data-dependent and bounded by `maxResultBytes`; each returned result remains in history until compaction. #### KV Cache effect diff --git a/packages/pty/tool-pty/package.json b/packages/pty/tool-pty/package.json index 6bc602699c..2d36fb5c9b 100644 --- a/packages/pty/tool-pty/package.json +++ b/packages/pty/tool-pty/package.json @@ -26,11 +26,15 @@ "src" ], "license": "BSD-3-Clause", + "dependencies": { + "schemastery": "^3.18.0" + }, "peerDependencies": { "@deepseek-ai/dsh-agent": "^0.0.1", "@deepseek-ai/dsh-invariants": "^0.0.1", "@deepseek-ai/dsh-llm": "^0.0.1", "@deepseek-ai/dsh-pty": "^0.0.1", + "@deepseek-ai/dsh-retention": "^0.0.1", "@deepseek-ai/dsh-system-prompt": "^0.0.1", "@deepseek-ai/dsh-tasks": "^0.0.1", "@deepseek-ai/dsh-tools": "^0.0.1", @@ -44,6 +48,7 @@ "@deepseek-ai/dsh-llm": "workspace:^", "@deepseek-ai/dsh-pty": "workspace:^", "@deepseek-ai/dsh-pty-local": "workspace:^", + "@deepseek-ai/dsh-retention": "workspace:^", "@deepseek-ai/dsh-sandbox": "workspace:^", "@deepseek-ai/dsh-sandbox-policy": "workspace:^", "@deepseek-ai/dsh-session": "workspace:^", diff --git a/packages/pty/tool-pty/src/index.ts b/packages/pty/tool-pty/src/index.ts index f145102dea..edb8102a79 100644 --- a/packages/pty/tool-pty/src/index.ts +++ b/packages/pty/tool-pty/src/index.ts @@ -5,6 +5,7 @@ */ import { Context } from 'cordis' +import z from 'schemastery' import type { Agent } from '@deepseek-ai/dsh-agent' import type { ContentBlock } from '@deepseek-ai/dsh-llm' import { PtySessionId } from '@deepseek-ai/dsh-pty' @@ -12,7 +13,7 @@ import type { PtySendResult, PtySessionId as PtySessionIdType, PtySignal } from import type {} from '@deepseek-ai/dsh-tasks' import { defineTool } from '@deepseek-ai/dsh-tools' import type { ToolExecutionResult, ToolResult } from '@deepseek-ai/dsh-tools' -import { renderList, renderRead, renderSend, renderSendRead, renderSpawn } from './render.ts' +import { boundTerminalText, renderList, renderRead, renderSend, renderSendRead, renderSpawn } from './render.ts' declare module '@deepseek-ai/dsh-tasks' { interface TaskKindMap { @@ -25,6 +26,23 @@ export const name = 'tool-pty' /** Required capability, registry, and prompt services. */ export const inject = ['pty', 'tools', 'systemPrompt'] +/** Default cap for one complete model-facing terminal result. */ +export const DEFAULT_MAX_RESULT_BYTES = 256 * 1024 + +/** Model-facing terminal tool configuration. */ +export interface Config { + /** Expose `run_in_background` and accept background sends (default true). */ + enableRunInBackground?: boolean + /** Maximum UTF-8 bytes in one complete terminal or task-output result. */ + maxResultBytes?: number +} + +/** Schemastery configuration for the terminal tool consumer. */ +export const Config: z = z.object({ + enableRunInBackground: z.boolean().default(true), + maxResultBytes: z.number().step(1).min(1).max(Number.MAX_SAFE_INTEGER).default(DEFAULT_MAX_RESULT_BYTES), +}) + interface SpawnArgs { type: string name?: string @@ -62,8 +80,8 @@ function sessionId(args: SessionArgs): PtySessionIdType { return PtySessionId(args.sessionId) } -function textResult(text: string): ContentBlock[] { - return [{ type: 'text', text }] +function textResult(text: string, maxBytes: number): ContentBlock[] { + return [{ type: 'text', text: boundTerminalText(text, maxBytes) }] } function rawResultText(result: ToolResult): string | undefined { @@ -79,7 +97,12 @@ function sendDetail(result: PtySendResult): string { } /** Register all terminal tools and the minimal usage guidance. */ -export function apply(ctx: Context): void { +export function apply(ctx: Context, config: Config = {}): void { + const enableRunInBackground = config.enableRunInBackground ?? true + const maxResultBytes = config.maxResultBytes ?? DEFAULT_MAX_RESULT_BYTES + if (!Number.isSafeInteger(maxResultBytes) || maxResultBytes <= 0) { + throw new Error('tool-pty: maxResultBytes must be a positive safe integer') + } ctx.systemPrompt.section({ name: 'tool:pty', order: 106, @@ -101,7 +124,7 @@ export function apply(ctx: Context): void { ...args.name !== undefined ? { name: args.name } : {}, ...args.cwd !== undefined ? { cwd: args.cwd } : {}, }, exec.signal) - return textResult(renderSpawn(result)) + return textResult(renderSpawn(result, maxResultBytes), maxResultBytes) }, presentCall: (args) => { const parsed = args @@ -111,18 +134,22 @@ export function apply(ctx: Context): void { ctx.tools.register(defineTool({ name: 'terminal_send', - description: 'Send text to a persistent terminal. By default Enter is submitted and the call waits for a prompt, stdin wait, output silence, timeout, or session exit. Background mode returns a task id for task_output/task_kill.', + description: 'Send text to a persistent terminal. By default Enter is submitted and the call waits for a prompt, stdin wait, output silence, timeout, or session exit.' + + (enableRunInBackground ? ' Background mode returns a task id for task_output/task_kill.' : ''), parameters: { sessionId: { type: 'string', required: true, description: 'Terminal session id returned by terminal_open or terminal_list.' }, text: { type: 'string', required: true, description: 'UTF-8 text to write to the terminal.' }, submit: { type: 'boolean', description: 'Submit Enter after text (default true). Set false for control characters or incomplete REPL input.' }, - run_in_background: { type: 'boolean', description: 'Return a task id immediately; collect with task_output or stop with task_kill.' }, + ...enableRunInBackground + ? { run_in_background: { type: 'boolean' as const, description: 'Return a task id immediately; collect with task_output or stop with task_kill.' } } + : {}, }, async execute(args: SendArgs, exec): Promise { const owner = requireAgent(exec.agent) const id = sessionId(args) const request = { text: args.text, submit: args.submit ?? true } if (args.run_in_background === true) { + if (!enableRunInBackground) throw new Error('background terminal sends are disabled by tool-pty configuration') const tasks = ctx.get('tasks') if (tasks === undefined) throw new Error('background terminal sends require @deepseek-ai/dsh-tasks and @deepseek-ai/dsh-tool-tasks') let cancelRequested = false @@ -130,6 +157,7 @@ export function apply(ctx: Context): void { kind: 'pty-send', label: `${id}: ${args.text || '(input)'}`, owner, + outputLimitBytes: maxResultBytes, run: () => { const operation = ctx.pty.startSend(owner, id, request) return { @@ -145,12 +173,12 @@ export function apply(ctx: Context): void { } }, }) - return { content: textResult(`started background task ${taskId}`), isError: false } + return { content: textResult(`started background task ${taskId}`, maxResultBytes), isError: false } } const operation = ctx.pty.startSend(owner, id, { ...request, signal: exec.signal }) const result = await operation.done if (exec.signal.aborted) throw new Error('terminal send aborted') - return { content: textResult(renderSend(result)), isError: false, meta: result } + return { content: textResult(renderSend(result, maxResultBytes), maxResultBytes), isError: false, meta: result } }, presentCall(args) { const parsed = args as Partial @@ -179,7 +207,7 @@ export function apply(ctx: Context): void { ...args.offset !== undefined ? { offset: args.offset } : {}, ...args.count !== undefined ? { count: args.count } : {}, }) - return Promise.resolve(textResult(renderRead(result))) + return Promise.resolve(textResult(renderRead(result, maxResultBytes), maxResultBytes)) }, presentCall: args => ({ card: 'generic', title: `Read terminal ${(args).sessionId}`, kind: 'read', rawInput: args }), })) @@ -193,7 +221,7 @@ export function apply(ctx: Context): void { }, async execute(args: SignalArgs, exec) { const result = await ctx.pty.signal(requireAgent(exec.agent), sessionId(args), args.signal) - return textResult(`delivered ${args.signal} to foreground process group ${result.targetPgid}`) + return textResult(`delivered ${args.signal} to foreground process group ${result.targetPgid}`, maxResultBytes) }, presentCall: args => ({ card: 'generic', title: `Signal terminal ${(args as SignalArgs).sessionId}`, kind: 'execute', rawInput: args }), })) @@ -207,7 +235,7 @@ export function apply(ctx: Context): void { async execute(args: SessionArgs, exec) { const id = sessionId(args) const closed = await ctx.pty.kill(requireAgent(exec.agent), id) - return textResult(closed ? `closed terminal session ${id}` : `terminal session ${id} was already closing`) + return textResult(closed ? `closed terminal session ${id}` : `terminal session ${id} was already closing`, maxResultBytes) }, presentCall: args => ({ card: 'generic', title: `Close terminal ${(args).sessionId}`, kind: 'delete' }), })) @@ -217,7 +245,7 @@ export function apply(ctx: Context): void { description: 'List persistent terminal sessions owned by the current agent.', parameters: {}, execute(_args: Record, exec) { - return Promise.resolve(textResult(renderList(ctx.pty.list(requireAgent(exec.agent))))) + return Promise.resolve(textResult(renderList(ctx.pty.list(requireAgent(exec.agent)), maxResultBytes), maxResultBytes)) }, presentCall: () => ({ card: 'generic', title: 'List terminal sessions', kind: 'read' }), })) diff --git a/packages/pty/tool-pty/src/render.ts b/packages/pty/tool-pty/src/render.ts index bed176e890..0930d205ad 100644 --- a/packages/pty/tool-pty/src/render.ts +++ b/packages/pty/tool-pty/src/render.ts @@ -1,57 +1,128 @@ /** Model and ACP rendering for persistent terminal tool results. */ +import { TextRetainer } from '@deepseek-ai/dsh-retention' import type { PtyReadResult, PtySendRead, PtySendResult, PtySessionSnapshot, PtySpawnResult } from '@deepseek-ai/dsh-pty' +const encoder = new TextEncoder() +const TRUNCATED = '\n[output truncated]' + +function byteLength(text: string): number { + return encoder.encode(text).byteLength +} + +function retain(text: string, maxBytes: number, kind: 'head' | 'tail'): string { + const retainer = new TextRetainer({ kind, maxBytes }) + retainer.push(text) + return retainer.finish().text +} + +function fitWithSuffix(content: string, suffix: string, maxBytes: number): string { + const fixedBytes = byteLength(suffix) + if (fixedBytes >= maxBytes) return retain(suffix, maxBytes, 'tail') + return `${retain(content, maxBytes - fixedBytes, 'tail')}${suffix}` +} + +function fitWithPrefix(prefix: string, content: string, maxBytes: number): string { + const fixed = `${prefix}${TRUNCATED}` + const fixedBytes = byteLength(fixed) + if (fixedBytes >= maxBytes) return retain(fixed, maxBytes, 'head') + return `${prefix}${retain(content, maxBytes - fixedBytes, 'tail')}${TRUNCATED}` +} + +function boundBodyWithSuffix( + content: string, + metadata: string, + upstreamTruncated: boolean, + maxBytes: number, +): string { + const suffix = `${metadata}${upstreamTruncated ? TRUNCATED : ''}` + const complete = `${content}${suffix}` + if (byteLength(complete) <= maxBytes) return complete + return fitWithSuffix(content, `${metadata}${TRUNCATED}`, maxBytes) +} + +/** + * Bound one complete terminal acknowledgement while preserving UTF-8 cuts. + * @param text - complete acknowledgement text. + * @param maxBytes - positive final result cap. + * @returns bounded text with a truncation marker when it fits. + */ +export function boundTerminalText(text: string, maxBytes: number): string { + if (byteLength(text) <= maxBytes) return text + const markerBytes = byteLength(TRUNCATED) + if (markerBytes >= maxBytes) return retain(TRUNCATED, maxBytes, 'tail') + return `${retain(text, maxBytes - markerBytes, 'head')}${TRUNCATED}` +} + /** * Render one created session and its bounded MOTD. * @param result - published spawn result. + * @param maxBytes - complete UTF-8 result cap. * @returns Model-facing session acknowledgement. */ -export function renderSpawn(result: PtySpawnResult): string { +export function renderSpawn(result: PtySpawnResult, maxBytes: number): string { const label = result.name === undefined ? result.sessionId : `${result.sessionId} (${result.name})` - return `started terminal session ${label} [type: ${result.type}]\n${result.motd || '(no startup output)'}` + const prefix = `started terminal session ${label} [type: ${result.type}]\n` + const motd = result.motd || '(no startup output)' + const complete = `${prefix}${motd}` + return byteLength(complete) <= maxBytes ? complete : fitWithPrefix(prefix, motd, maxBytes) } /** * Render one settled interactive send. * @param result - settled send outcome. + * @param maxBytes - complete UTF-8 result cap. * @returns Terminal output plus wait/session markers. */ -export function renderSend(result: PtySendResult): string { +export function renderSend(result: PtySendResult, maxBytes: number): string { const output = result.viewport || '(no new output)' const status = result.sessionStatus.kind === 'running' ? 'running' : `exited code=${result.sessionStatus.exitCode ?? 'null'} signal=${result.sessionStatus.signal ?? 'null'}` - return `${output}\n[wait: ${result.waitReason}]\n[session: ${status}]${result.truncated ? '\n[output truncated]' : ''}` + return boundBodyWithSuffix( + output, + `\n[wait: ${result.waitReason}]\n[session: ${status}]`, + result.truncated, + maxBytes, + ) } /** * Render one incremental background operation read. * @param read - consuming operation delta. - * @returns Delta plus truncation marker when needed. + * @returns Delta plus its upstream truncation marker. The generic task control + * applies the producer's complete-result cap after adding task status. */ export function renderSendRead(read: PtySendRead): string { - return `${read.delta}${read.truncated ? `${read.delta.endsWith('\n') || read.delta.length === 0 ? '' : '\n'}[output truncated]` : ''}` + const separator = read.delta.endsWith('\n') || read.delta.length === 0 ? '' : '\n' + return `${read.delta}${read.truncated ? `${separator}[output truncated]` : ''}` } /** * Render one bounded historical page. * @param result - retained scrollback page. + * @param maxBytes - complete UTF-8 result cap. * @returns Page text plus pagination and truncation markers. */ -export function renderRead(result: PtyReadResult): string { +export function renderRead(result: PtyReadResult, maxBytes: number): string { const output = result.text || '(no retained output)' - return `${output}\n[lines: ${result.lineBegin}-${result.lineEnd} of ${result.totalLines}]${result.truncated ? '\n[output truncated]' : ''}` + return boundBodyWithSuffix( + output, + `\n[lines: ${result.lineBegin}-${result.lineEnd} of ${result.totalLines}]`, + result.truncated, + maxBytes, + ) } /** * Render owner-visible live sessions. * @param sessions - fresh owner-scoped snapshots. + * @param maxBytes - complete UTF-8 result cap. * @returns One line per session or the empty marker. */ -export function renderList(sessions: PtySessionSnapshot[]): string { +export function renderList(sessions: PtySessionSnapshot[], maxBytes: number): string { if (sessions.length === 0) return '(no terminal sessions)' - return sessions.map((session) => { + const text = sessions.map((session) => { const name = session.name === undefined ? '' : ` (${session.name})` const pid = session.pid === undefined ? '' : ` pid=${session.pid}` const status = session.status.kind === 'running' @@ -59,4 +130,5 @@ export function renderList(sessions: PtySessionSnapshot[]): string { : `exited code=${session.status.exitCode ?? 'null'} signal=${session.status.signal ?? 'null'}` return `${session.sessionId}${name} [${session.type}] ${status}${pid}` }).join('\n') + return boundBodyWithSuffix(text, '', false, maxBytes) } diff --git a/packages/pty/tool-pty/tests/render.spec.ts b/packages/pty/tool-pty/tests/render.spec.ts index 33b288ab5f..b02ba3b8ef 100644 --- a/packages/pty/tool-pty/tests/render.spec.ts +++ b/packages/pty/tool-pty/tests/render.spec.ts @@ -1,23 +1,23 @@ import { describe, expect, it } from 'vitest' import { PtySessionId } from '@deepseek-ai/dsh-pty' -import { renderList, renderRead, renderSend, renderSendRead, renderSpawn } from '@deepseek-ai/dsh-tool-pty/src/render.ts' +import { boundTerminalText, renderList, renderRead, renderSend, renderSendRead, renderSpawn } from '@deepseek-ai/dsh-tool-pty/src/render.ts' describe('tool-pty rendering', () => { it('renders spawn with and without names or MOTD', () => { - expect(renderSpawn({ sessionId: PtySessionId('pty-1'), type: 'shell', status: { kind: 'running' }, motd: '' })) + expect(renderSpawn({ sessionId: PtySessionId('pty-1'), type: 'shell', status: { kind: 'running' }, motd: '' }, 1024)) .toBe('started terminal session pty-1 [type: shell]\n(no startup output)') - expect(renderSpawn({ sessionId: PtySessionId('pty-2'), name: 'main', type: 'shell', pid: 2, status: { kind: 'running' }, motd: 'ready' })) + expect(renderSpawn({ sessionId: PtySessionId('pty-2'), name: 'main', type: 'shell', pid: 2, status: { kind: 'running' }, motd: 'ready' }, 1024)) .toContain('pty-2 (main)') }) it('renders running, exited, empty, and truncated sends', () => { - expect(renderSend({ viewport: '', waitReason: 'timeout', sessionStatus: { kind: 'running' }, truncated: true })) + expect(renderSend({ viewport: '', waitReason: 'timeout', sessionStatus: { kind: 'running' }, truncated: true }, 1024)) .toBe('(no new output)\n[wait: timeout]\n[session: running]\n[output truncated]') - expect(renderSend({ viewport: 'bye', waitReason: 'session_exit', sessionStatus: { kind: 'exited', exitCode: null, signal: 'SIGTERM' }, truncated: false })) + expect(renderSend({ viewport: 'bye', waitReason: 'session_exit', sessionStatus: { kind: 'exited', exitCode: null, signal: 'SIGTERM' }, truncated: false }, 1024)) .toContain('exited code=null signal=SIGTERM') - expect(renderSend({ viewport: 'bye', waitReason: 'session_exit', sessionStatus: { kind: 'exited', exitCode: 2, signal: null }, truncated: false })) + expect(renderSend({ viewport: 'bye', waitReason: 'session_exit', sessionStatus: { kind: 'exited', exitCode: 2, signal: null }, truncated: false }, 1024)) .toContain('exited code=2 signal=null') - expect(renderSend({ viewport: 'bye', waitReason: 'session_exit', sessionStatus: { kind: 'exited', exitCode: null, signal: null }, truncated: false })) + expect(renderSend({ viewport: 'bye', waitReason: 'session_exit', sessionStatus: { kind: 'exited', exitCode: null, signal: null }, truncated: false }, 1024)) .toContain('exited code=null signal=null') expect(renderSendRead({ delta: '', truncated: true })).toBe('[output truncated]') expect(renderSendRead({ delta: 'x', truncated: true })).toBe('x\n[output truncated]') @@ -26,14 +26,48 @@ describe('tool-pty rendering', () => { }) it('renders history and every list status shape', () => { - expect(renderRead({ text: '', totalLines: 0, lineBegin: 0, lineEnd: 0, truncated: true })) + expect(renderRead({ text: '', totalLines: 0, lineBegin: 0, lineEnd: 0, truncated: true }, 1024)) .toBe('(no retained output)\n[lines: 0-0 of 0]\n[output truncated]') - expect(renderList([])).toBe('(no terminal sessions)') + expect(renderList([], 1024)).toBe('(no terminal sessions)') expect(renderList([ { sessionId: PtySessionId('pty-1'), type: 'shell', status: { kind: 'running' } }, { sessionId: PtySessionId('pty-2'), name: 'done', type: 'shell', pid: 9, status: { kind: 'exited', exitCode: 2, signal: null } }, { sessionId: PtySessionId('pty-3'), type: 'shell', status: { kind: 'exited', exitCode: null, signal: 'SIGTERM' } }, { sessionId: PtySessionId('pty-4'), type: 'shell', status: { kind: 'exited', exitCode: null, signal: null } }, - ])).toBe('pty-1 [shell] running\npty-2 (done) [shell] exited code=2 signal=null pid=9\npty-3 [shell] exited code=null signal=SIGTERM\npty-4 [shell] exited code=null signal=null') + ], 1024)).toBe('pty-1 [shell] running\npty-2 (done) [shell] exited code=2 signal=null pid=9\npty-3 [shell] exited code=null signal=SIGTERM\npty-4 [shell] exited code=null signal=null') + }) + + it('bounds complete UTF-8 results while retaining terminal metadata when it fits', () => { + const send = renderSend({ + viewport: `prefix-${'界'.repeat(40)}`, + waitReason: 'stdin_read', + sessionStatus: { kind: 'running' }, + truncated: false, + }, 64) + expect(Buffer.byteLength(send)).toBeLessThanOrEqual(64) + expect(send).toContain('[wait: stdin_read]') + expect(send).toContain('[output truncated]') + + const read = renderRead({ + text: 'x'.repeat(200), totalLines: 20, lineBegin: 0, lineEnd: 10, truncated: false, + }, 48) + expect(Buffer.byteLength(read)).toBeLessThanOrEqual(48) + expect(read).toContain('[lines: 0-10 of 20]') + + expect(Buffer.byteLength(renderSpawn({ + sessionId: PtySessionId('pty-1'), type: 'shell', status: { kind: 'running' }, motd: 'x'.repeat(200), + }, 32))).toBeLessThanOrEqual(32) + + const boundedSpawn = renderSpawn({ + sessionId: PtySessionId('pty-1'), type: 'shell', status: { kind: 'running' }, motd: 'x'.repeat(200), + }, 96) + expect(boundedSpawn).toContain('started terminal session pty-1') + expect(boundedSpawn).toContain('[output truncated]') + + expect(Buffer.byteLength(renderSend({ + viewport: 'x'.repeat(200), waitReason: 'stdin_read', sessionStatus: { kind: 'running' }, truncated: false, + }, 8))).toBeLessThanOrEqual(8) + expect(boundTerminalText('x'.repeat(200), 8)).toHaveLength(8) + expect(boundTerminalText('x'.repeat(200), 32).endsWith('[output truncated]')).toBe(true) }) }) diff --git a/packages/pty/tool-pty/tests/tools.spec.ts b/packages/pty/tool-pty/tests/tools.spec.ts index 5adcaea441..d6023a4d17 100644 --- a/packages/pty/tool-pty/tests/tools.spec.ts +++ b/packages/pty/tool-pty/tests/tools.spec.ts @@ -31,20 +31,23 @@ class StubSession implements PtyBackendSession { autoSettle = true rejectOperation = false closeGate: PromiseWithResolvers | undefined + viewport = 'command output' + delta = 'live output' + deltaTruncated = false startSend(_request: PtySendRequest): PtySendOperation { let settle!: () => void let reject!: (error: unknown) => void let cancelled = false const done = new Promise((resolve, rejectPromise) => { settle = resolve; reject = rejectPromise }).then(() => ({ - viewport: cancelled ? '^C' : 'command output', + viewport: cancelled ? '^C' : this.viewport, waitReason: 'stdin_read' as const, sessionStatus: this.statusValue, truncated: false, })) const operation: PtySendOperation = { done, - readOutput: () => ({ delta: 'live output', truncated: false }), + readOutput: () => ({ delta: this.delta, truncated: this.deltaTruncated }), cancel: () => { if (cancelled) return false cancelled = true @@ -87,7 +90,13 @@ function stubBackend() { return { backend, sessions } } -async function setup(tasks: boolean) { +async function setup(tasks: boolean, config: ToolPty.Config = {}) { + const base = await setupBase(tasks) + await base.ctx.plugin(ToolPty, config) + return base +} + +async function setupBase(tasks: boolean) { const ctx = new Context() await ctx.plugin(SystemPrompt) await ctx.plugin(ToolRegistry) @@ -99,7 +108,6 @@ async function setup(tasks: boolean) { await ctx.plugin(TaskService) await ctx.plugin(ToolTasks) } - await ctx.plugin(ToolPty) return { ctx, stub, agent: fakeAgent(ctx, tasks ? 'with-tasks' : 'foreground') } } @@ -172,6 +180,24 @@ describe('tool-pty foreground surface', () => { expect(ctx.tools.get('terminal_close')?.presentCall?.({ sessionId: 'pty-1' })).toMatchObject({ card: 'generic', title: 'Close terminal pty-1' }) expect(ctx.tools.get('terminal_list')?.presentCall?.({})).toMatchObject({ card: 'generic', title: 'List terminal sessions' }) }) + + it('configuration-gates background sends and validates the final result bound', async () => { + const disabled = await setup(true, { enableRunInBackground: false }) + const definition = disabled.ctx.tools.get('terminal_send') + expect(definition?.parameters).not.toHaveProperty('properties.run_in_background') + expect(definition?.description).not.toContain('Background mode') + await call(disabled.ctx, 'terminal_open', { type: 'stub' }, disabled.agent) + expect((await call(disabled.ctx, 'terminal_send', { + sessionId: 'pty-1', text: 'work', run_in_background: true, + }, disabled.agent)).isError).toBe(true) + + const defaults = await setupBase(false) + ToolPty.apply(defaults.ctx) + expect(defaults.ctx.tools.get('terminal_send')?.parameters).toHaveProperty('properties.run_in_background') + + const invalid = await setupBase(false) + expect(() => { ToolPty.apply(invalid.ctx, { maxResultBytes: 0 }) }).toThrow('maxResultBytes') + }) }) describe('tool-pty task integration', () => { @@ -184,6 +210,23 @@ describe('tool-pty task integration', () => { expect(text(output)).toContain('[status: completed, wait: stdin_read]') }) + it('bounds foreground and background results after terminal and task metadata', async () => { + const { ctx, agent, stub } = await setup(true, { maxResultBytes: 64 }) + await call(ctx, 'terminal_open', { type: 'stub' }, agent) + stub.sessions[0]!.viewport = '界'.repeat(100) + const foreground = await call(ctx, 'terminal_send', { sessionId: 'pty-1', text: 'foreground' }, agent) + expect(Buffer.byteLength(text(foreground))).toBeLessThanOrEqual(64) + + stub.sessions[0]!.delta = '界'.repeat(100) + stub.sessions[0]!.deltaTruncated = true + await call(ctx, 'terminal_send', { sessionId: 'pty-1', text: 'background', run_in_background: true }, agent) + const background = await call(ctx, 'task_output', { task_id: 'pty-send-1', wait: true }, agent) + expect(Buffer.byteLength(text(background))).toBeLessThanOrEqual(64) + expect(text(background)).toContain('[status: completed') + expect(text(background).match(/\[output truncated\]/g)).toHaveLength(1) + expect(text(background)).toContain('[output truncated]\n[status: completed') + }) + it('rejects pre-aborted background calls, maps task cancellation, and contains operation failure', async () => { const { ctx, agent, stub } = await setup(true) await call(ctx, 'terminal_open', { type: 'stub' }, agent) diff --git a/packages/pty/tool-pty/tsconfig.json b/packages/pty/tool-pty/tsconfig.json index 7ba9633c2c..9674519034 100644 --- a/packages/pty/tool-pty/tsconfig.json +++ b/packages/pty/tool-pty/tsconfig.json @@ -14,6 +14,9 @@ { "path": "../../../vendor/cordis" }, + { + "path": "../../util/retention" + }, { "path": "../pty" }, diff --git a/packages/tasks/tasks/README.md b/packages/tasks/tasks/README.md index 37d342e0fe..1d9ce2b249 100644 --- a/packages/tasks/tasks/README.md +++ b/packages/tasks/tasks/README.md @@ -4,7 +4,7 @@ The process-local background task registry (`ctx.tasks`). It gives long-running ## Service API -- `start(spec): TaskId` validates the control surface, spec, and exact live owner before calling the producer's `run()` once. A starter throw leaves nothing registered; successful return commits without another failable step. +- `start(spec): TaskId` validates the control surface, spec, exact live owner, and optional positive `outputLimitBytes` before calling the producer's `run()` once. A starter throw leaves nothing registered; successful return commits without another failable step. - `get(id, caller?)` and `list(caller?)` return non-consuming snapshots. Listing includes only caller-owned and unowned tasks. - `read(id, caller?)` consumes the single cursor for stream tasks and reads terminal output idempotently for final-output tasks. - `kill(id, caller?, reason?)` invokes producer cancellation before changing status. A cancellation throw leaves the task running; success changes it to `stopping` and marks terminal delivery reported. @@ -14,6 +14,8 @@ The process-local background task registry (`ctx.tasks`). It gives long-running Owned access compares the task's `SessionId` with the caller's. Ids such as `bash-1` are predictable, so this fence is the boundary. Unowned tasks are open to callers and last until service disposal. +`outputLimitBytes` is producer-owned model-presentation policy carried unchanged into snapshots. A control surface applies it after adding status or notice metadata; the registry does not rewrite producer output or invent a default for producers that omit it. + ## Lifecycle Tasks belong to their owner and backend, not the producer tool fiber, so producer and surface reloads do not stop them. The first task for an owner attaches one awaited effect to the exact `Agent` scope. Owner disposal cancels that object's tasks, awaits producer quiescence, and removes their snapshots; reused agent or session ids cannot redirect an old cleanup. diff --git a/packages/tasks/tasks/src/index.ts b/packages/tasks/tasks/src/index.ts index 457f5473a3..16f0807656 100644 --- a/packages/tasks/tasks/src/index.ts +++ b/packages/tasks/tasks/src/index.ts @@ -42,6 +42,7 @@ interface TrackedTask { id: TaskId kind: TaskKind label: string + outputLimitBytes: number | undefined /** Exact lifecycle owner; session-id authorization is derived from it. */ owner: Agent | undefined cancel: (reason?: string) => void @@ -104,6 +105,10 @@ export class TaskService extends Service { } if (spec.kind.length === 0) throw new Error('invalid task kind: expected a non-empty string') if (spec.label.length === 0) throw new Error('invalid task label: expected a non-empty string') + if (spec.outputLimitBytes !== undefined + && (!Number.isSafeInteger(spec.outputLimitBytes) || spec.outputLimitBytes <= 0)) { + throw new Error(`invalid outputLimitBytes: expected a positive safe integer, got ${JSON.stringify(spec.outputLimitBytes)}`) + } if (spec.owner !== undefined) this.ensureOwnerCleanup(spec.owner) const hooks = spec.run() @@ -117,6 +122,7 @@ export class TaskService extends Service { id, kind: spec.kind, label: spec.label, + outputLimitBytes: spec.outputLimitBytes, owner: spec.owner, cancel: hooks.cancel.bind(hooks), readOutput: hooks.readOutput?.bind(hooks), @@ -329,6 +335,7 @@ export class TaskService extends Service { id: task.id, kind: task.kind, label: task.label, + ...task.outputLimitBytes !== undefined ? { outputLimitBytes: task.outputLimitBytes } : {}, ...ownerSession !== undefined ? { ownerSession } : {}, status: task.status, ...task.detail !== undefined ? { detail: task.detail } : {}, diff --git a/packages/tasks/tasks/src/types.ts b/packages/tasks/tasks/src/types.ts index 96316260ec..d722f3fce6 100644 --- a/packages/tasks/tasks/src/types.ts +++ b/packages/tasks/tasks/src/types.ts @@ -61,6 +61,11 @@ export interface TaskStart { kind: TaskKind /** One-line model-facing label (the command; the delegation description). */ label: string + /** + * Optional UTF-8 byte cap for each complete model-facing completion notice or + * output read, including control-surface status metadata. + */ + outputLimitBytes?: number /** * Owning live agent. Access is fenced by its session id, and agent disposal * cancels and awaits the task. The instance must be the one currently @@ -109,6 +114,8 @@ export interface TaskSnapshot { kind: TaskKind /** The producer-supplied one-line label. */ label: string + /** Producer-owned cap for complete model-facing notices and output reads. */ + outputLimitBytes?: number /** * Owner session id used for authorization and correlation; absent for * unowned tasks. Completion listeners receive the exact {@link Agent} diff --git a/packages/tasks/tasks/tests/tasks.spec.ts b/packages/tasks/tasks/tests/tasks.spec.ts index 0d3eae8338..34506b3a47 100644 --- a/packages/tasks/tasks/tests/tasks.spec.ts +++ b/packages/tasks/tasks/tests/tasks.spec.ts @@ -44,13 +44,19 @@ function producer(overrides: Partial & TaskHooks> = {}) { let settle!: (outcome: TaskOutcome) => void let reject!: (error: unknown) => void const cancels: (string | undefined)[] = [] - const { kind = 'bash', label = 'sleep 60', owner, ...hookOverrides } = overrides + const { kind = 'bash', label = 'sleep 60', owner, outputLimitBytes, ...hookOverrides } = overrides const hooks: TaskHooks = { cancel(reason) { cancels.push(reason) }, done: new Promise((res, rej) => { settle = res; reject = rej }), ...hookOverrides, } - const spec: TaskStart = { kind, label, ...owner !== undefined ? { owner } : {}, run: () => hooks } + const spec: TaskStart = { + kind, + label, + ...owner !== undefined ? { owner } : {}, + ...outputLimitBytes !== undefined ? { outputLimitBytes } : {}, + run: () => hooks, + } return { spec, settle, reject, cancels } } @@ -85,10 +91,11 @@ describe('TaskService.start', () => { .toThrow('background tasks unavailable: no control surface is attached (load @deepseek-ai/dsh-tool-tasks)') }) - it('rejects an empty kind and an empty label', async () => { + it('rejects an empty kind, empty label, and invalid output limit', async () => { const ctx = await harness() expect(() => ctx.tasks.start(producer({ kind: '' as TaskKind }).spec)).toThrow('invalid task kind') expect(() => ctx.tasks.start(producer({ label: '' }).spec)).toThrow('invalid task label') + expect(() => ctx.tasks.start(producer({ outputLimitBytes: 0 }).spec)).toThrow('outputLimitBytes') }) it('issues kind-prefixed ids from per-kind counters', async () => { @@ -118,6 +125,16 @@ describe('TaskService reads and settlement', () => { expect(read.snapshot.finishedAt).toBeTypeOf('number') }) + it('projects a producer-owned model output limit into reads and snapshots', async () => { + const ctx = await harness() + const p = producer({ outputLimitBytes: 64, readOutput: () => 'delta' }) + const id = ctx.tasks.start(p.spec) + expect(ctx.tasks.read(id)).toMatchObject({ + text: 'delta', snapshot: { outputLimitBytes: 64 }, + }) + expect(ctx.tasks.get(id)).toMatchObject({ outputLimitBytes: 64 }) + }) + it('final-output kinds read empty while live, the outcome output idempotently once settled', async () => { const ctx = await harness() const p = producer({ kind: 'subagent', label: 'research task' }) diff --git a/packages/tasks/tool-tasks/README.md b/packages/tasks/tool-tasks/README.md index 7e1ab82971..f4a3475786 100644 --- a/packages/tasks/tool-tasks/README.md +++ b/packages/tasks/tool-tasks/README.md @@ -10,6 +10,8 @@ The model-facing control surface for `ctx.tasks`: three kind-independent tools, All three use generic ACP cards: `read` for output and list, `execute` for kill. +When a producer supplies `outputLimitBytes`, `task_output`, terminal `task_kill`, and completion notices cap the complete UTF-8 result after adding status or notice text. The output tail and control suffix are retained when they fit; an existing producer truncation marker is reused rather than duplicated. Producers that omit the field retain the existing unbounded control-surface behavior. + ## Completion notices An unreported completion injects `background task (: