feat(telemetry): add feedback-gated OTEL modes
This commit is contained in:
41 files changed
+635
-182
No files matched your search
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/telemetry/session-telemetry/README.md
|
||||
README.md: 272c9abe78849be3d2bba2c54cd7e25bcbe2d4c2
|
||||
README.zh.md: e6f077c1d12d00e746147908560d05381fde11c3
|
||||
README.md: d38433a728c699c7fb3cc0512bb6a2d977dd4cc6
|
||||
README.zh.md: 3a86b01321fc7dfd33d39530ee7fa38a6ee1f2dc
|
||||
@@ -2,23 +2,23 @@
|
||||
|
||||
English | [中文](README.zh.md)
|
||||
|
||||
The telemetry seam: the CAPTURE side of session-event reporting, behind a backend contract any reporting SDK satisfies with zero bending. The boundary axiom that shapes everything here: **this package's aspect ends at `emit()`** — batching, retry, queueing, and loss policy belong to the backend's SDK and are neither specified nor wrapped. Rationale and rejected alternatives: [the revival Agent Note](../../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md).
|
||||
The telemetry seam: the capture side of session-event reporting, behind a backend contract any reporting SDK satisfies with zero bending. Capture can hand each redacted record over immediately or hold a per-session prefix for an explicit release. The boundary axiom that shapes everything here: **this package's aspect ends at `emit()`** — batching, retry, queueing, and loss policy belong to the backend's SDK and are neither specified nor wrapped. Rationale and rejected alternatives: [the revival Agent Note](../../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md) and [feedback-gated delivery](../../../.agents/notes/implemented/feature/2026-08-05-feedback-gated-session-telemetry.md).
|
||||
|
||||
## The backend contract
|
||||
|
||||
`TelemetryBackend` is three members: `emit(record)` (MUST be a non-blocking enqueue — it runs synchronously on the `session/event` hot path), optional `flush()` (a turn-boundary hint, fire-and-forget; most backends leave it unimplemented and let their SDK's batching cadence govern export timing — an implementer owns the interaction between concurrent flushes and `shutdown()`'s drain), and `shutdown()` (the lifecycle forward: drain-and-quiesce, awaited at dispose). `Telemetry` is its service-registered form under the `telemetry` context key — one implementation per context, duplicate load throws. A backend composes `TelemetryCoordinator` in its constructor.
|
||||
`TelemetryBackend` is three members: `emit(record)` (MUST be a non-blocking enqueue — it runs synchronously on the `session/event` hot path, either at capture or held-prefix release), optional `flush()` (a turn-boundary hint, fire-and-forget; most backends leave it unimplemented and let their SDK's batching cadence govern export timing — an implementer owns the interaction between concurrent flushes and `shutdown()`'s drain), and `shutdown()` (the lifecycle forward: drain-and-quiesce, awaited at dispose). `Telemetry` is its service-registered form under the `telemetry` context key — one implementation per context, duplicate load throws. A backend composes `TelemetryCoordinator` with `immediate` delivery or `held` delivery and calls `release(session)` at its owning trigger.
|
||||
|
||||
## Capture points
|
||||
|
||||
The coordinator registers, all through the composing fiber's effects: `session/created` (adopt: record the header, read the log back through the projection from the construction boundary — constructor seeds from fork/resume never re-emit on the firehose and never re-export), `session/event` (project, deep-copy, redact, hand off; zero I/O), `session/flush` (forward the optional `flush()` hint and return void — the loop's awaited parallel must never wait on telemetry), `session/disposed` (emit the session's `shutdown` operational record at its own termination edge — where receivers key crash detection — then retire it, so a long-lived backend neither retains closed sessions nor re-marks them at unload), `agent/error` (the one live-bus relay; the session event vocabulary intentionally has no operational-error record), a dispose effect (mark each session still alive at teardown, then await the backend's `shutdown()`; failures warn instead of throwing), and an adoption sweep of `ctx.sessions.list()` (a hot reload does not replay `session/created`).
|
||||
The coordinator registers, all through the composing fiber's effects: `session/created` (adopt: record the header, read the log back through the projection from the construction boundary — constructor seeds from fork/resume never re-emit on the firehose and never re-export), `session/event` (project, deep-copy, redact, then hand off or hold; zero I/O), `session/flush` (forward the optional `flush()` hint and return void — the loop's awaited parallel must never wait on telemetry), `session/disposed` (capture the session's `shutdown` operational record at its termination edge, then retire it), `agent/error` (the one live-bus relay; the session event vocabulary intentionally has no operational-error record), a dispose effect (capture shutdown for each still-live session, then await the backend's `shutdown()`; failures warn instead of throwing), and an adoption sweep of `ctx.sessions.list()` (a hot reload does not replay `session/created`). Immediate delivery hands lifecycle records over; held delivery leaves any suffix after the last release local, including its later shutdown marker.
|
||||
|
||||
## The redact waterfall
|
||||
|
||||
Every record passes the `telemetry/record` waterfall between projection and `emit()` — the seam's scrubbing extension point. The seam ships NO rules of its own: the innermost `next()` passes the record through unchanged, so with no listener mounted records reach the backend exactly as captured, and exported data is precisely as clean as the rules a deployment mounts. Listeners stack by transforming `next()`'s return value; returning without `next()` replaces everything beneath, and a throwing listener withholds that one record fail-closed inside the coordinator's containment. Redaction applies to the exported copy only; the canonical session log is never rewritten.
|
||||
Every record passes the `telemetry/record` waterfall immediately after projection — the seam's scrubbing extension point. The seam ships NO rules of its own: the innermost `next()` passes the record through unchanged, so with no listener mounted records reach the backend exactly as captured, and exported data is precisely as clean as the rules a deployment mounts. Listeners stack by transforming `next()`'s return value; returning without `next()` replaces everything beneath, and a throwing listener withholds that one record fail-closed inside the coordinator's containment. Held delivery stores only the waterfall result, so later policy removal cannot expose the original capture. Redaction applies to the outbound copy only; the canonical session log is never rewritten.
|
||||
|
||||
## The handoff cursor
|
||||
|
||||
A module-scope `WeakMap<Session, seq>` marks the highest seq HANDED OFF (not delivered) per session, advanced at emit time. It survives reloads that do not re-evaluate this module — config re-applies and backend source reloads, which is where iteration happens; that asymmetry is why the cursor lives in the seam. On re-adoption the coordinator re-hands only events past the cursor (events at or below it still rebuild the chunk-projection state); a missing cursor safely degrades to a re-hand from the session's construction boundary (`Session.firstLiveSeq` — seq 0 for a session born in this process), absorbed by receiver-side dedupe on `(session.id, event.seq)`. Constructor seeds never re-export: a resumed session's history shipped from the previous process under the same id, and a fork's inherited prefix lives in the parent's stream (receivers stitch on `session.parent_id` + `session.seed_length`). The accepted cost, consistent with at-most-once delivery: a resume does not backfill records a previous process failed to deliver — a deployment with a backfill requirement needs the deferred outbox, not replay. This is a deliberate, narrow exception to the registrations-are-effects discipline: entries die with their sessions, the value is a monotonic watermark, and losing it is never an error.
|
||||
A module-scope `WeakMap<Session, seq>` marks the highest seq HANDED OFF (not delivered) per session. Immediate delivery advances it at capture; held delivery advances it only when `release(session)` hands that record to the backend. An unreleased prefix therefore survives a coordinator reload through deterministic re-adoption instead of disappearing with its in-memory copy. On re-adoption the coordinator re-hands only events past the cursor (events at or below it still rebuild the chunk-projection state); a missing cursor safely degrades to a re-hand from the session's construction boundary (`Session.firstLiveSeq` — seq 0 for a session born in this process), absorbed by receiver-side dedupe on `(session.id, event.seq)`. Constructor seeds never re-export: a resumed session's history shipped from the previous process under the same id, and a fork's inherited prefix lives in the parent's stream (receivers stitch on `session.parent_id` + `session.seed_length`). The accepted cost, consistent with at-most-once delivery: a resume does not backfill records a previous process failed to deliver — a deployment with a backfill requirement needs the deferred outbox, not replay. This is a deliberate, narrow exception to the registrations-are-effects discipline: entries die with their sessions, the value is a monotonic watermark, and losing it is never an error.
|
||||
|
||||
## The fixed chunk projection
|
||||
|
||||
@@ -40,3 +40,4 @@ None; this package neither assembles nor sends a provider request.
|
||||
|
||||
- **Best-effort delivery** — the cursor marks handed-off, not delivered; a session torn down inside a reload window cannot be re-adopted; whatever sits in a backend queue at crash time is lost. A durable outbox (spool, per-sink cursors, at-least-once) is deferred until a deployment states a crash-loss requirement — see [the revival Agent Note](../../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md).
|
||||
- **No built-in redaction rules** — with no `telemetry/record` listener mounted, records leave the process exactly as captured, including any credentials embedded in file contents or command output; a deployment exporting to a shared collector owns its rule set.
|
||||
- **Held prefixes duplicate memory** — held delivery retains one deep-copied, redacted record per projected event until release or session collection. It adds no durable outbox and intentionally trades memory for a simple no-upload-before-trigger boundary.
|
||||
@@ -2,23 +2,23 @@
|
||||
|
||||
[English](README.md) | 中文
|
||||
|
||||
遥测(telemetry)seam:会话事件上报的捕获侧,隔在一个后端契约之后,任何上报 SDK 都无需变形即可满足该契约。塑造本包(package)一切设计的边界公理:**本包的职责止于 `emit()`**。批处理、重试、排队与丢失策略都属于后端自身的 SDK,本包既不为其立规,也不做包装。设计依据与被否决的替代方案见[复活 Agent Note(agent 决策记录)](../../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md)。
|
||||
遥测(telemetry)seam:会话事件上报的捕获侧,隔在一个后端契约之后,任何上报 SDK 都无需变形即可满足该契约。捕获侧可立即交接每条已脱敏记录,也可按会话暂存一个前缀,等待显式释放。塑造本包(package)一切设计的边界公理:**本包的职责止于 `emit()`**。批处理、重试、排队与丢失策略都属于后端自身的 SDK,本包既不为其立规,也不做包装。设计依据与被否决的替代方案见[复活 Agent Note(agent 决策记录)](../../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md)与[反馈门控投递](../../../.agents/notes/implemented/feature/2026-08-05-feedback-gated-session-telemetry.md)。
|
||||
|
||||
## 后端契约
|
||||
|
||||
`TelemetryBackend` 只有三个成员:`emit(record)`(必须是非阻塞入队;它在 `session/event` 热路径上同步执行)、可选的 `flush()`(轮次边界提示,触发后不等待结果;多数后端不实现它,而由其 SDK 的批处理节奏决定导出时机;并发 flush 与 `shutdown()` 的排空之间的交互由实现方自行负责)、以及 `shutdown()`(生命周期转发点:排空并完全停稳,在 dispose(资源释放)时被等待)。`Telemetry` 是它注册在 `telemetry` 上下文键下的服务形态:每个上下文只允许一个实现,重复加载会抛出异常。后端在其构造函数中组合 `TelemetryCoordinator`。
|
||||
`TelemetryBackend` 只有三个成员:`emit(record)`(必须是非阻塞入队;它会在捕获或暂存前缀释放时,于 `session/event` 热路径上同步执行)、可选的 `flush()`(轮次边界提示,触发后不等待结果;多数后端不实现它,而由其 SDK 的批处理节奏决定导出时机;并发 flush 与 `shutdown()` 的排空之间的交互由实现方自行负责)、以及 `shutdown()`(生命周期转发点:排空并完全停稳,在 dispose(资源释放)时被等待)。`Telemetry` 是它注册在 `telemetry` 上下文键下的服务形态:每个上下文只允许一个实现,重复加载会抛出异常。后端以 `immediate` 或 `held` 投递模式组合 `TelemetryCoordinator`,并在自身所属的触发器中调用 `release(session)`。
|
||||
|
||||
## 捕获点
|
||||
|
||||
协调器的全部注册都经由组合方 fiber 的 effect 完成:`session/created`(收养:记录 header,并经投影从构造边界起回读日志;来自 fork 或恢复的构造函数种子绝不会在 firehose 上再次发出,也绝不会再次导出)、`session/event`(投影、深拷贝、脱敏、交接;零 I/O)、`session/flush`(转发可选的 `flush()` 提示并返回 void;循环所等待的并行任务绝不能等待遥测)、`session/disposed`(在会话自身的终止边缘发出该会话的 `shutdown` 运维记录,接收端正是在这个边缘锚定崩溃检测;随后将该会话退役,因此长生命周期的后端既不会保留已关闭的会话,也不会在卸载时再次标记它们)、`agent/error`(唯一的实时总线转发;会话事件词汇有意不包含运维错误记录)、一个 dispose effect(拆卸时先标记每个仍存活的会话,再等待后端的 `shutdown()`;失败只发出警告而不抛出),以及对 `ctx.sessions.list()` 的收养扫描(热重载不会重放 `session/created`)。
|
||||
协调器的全部注册都经由组合方 fiber 的 effect 完成:`session/created`(收养:记录 header,并经投影从构造边界起回读日志;来自 fork 或恢复的构造函数种子绝不会在 firehose 上再次发出,也绝不会再次导出)、`session/event`(投影、深拷贝、脱敏,再交接或暂存;零 I/O)、`session/flush`(转发可选的 `flush()` 提示并返回 void;循环所等待的并行任务绝不能等待遥测)、`session/disposed`(在会话自身的终止边缘捕获该会话的 `shutdown` 运维记录,然后将其退役)、`agent/error`(唯一的实时总线转发;会话事件词汇有意不包含运维错误记录)、一个 dispose effect(捕获每个仍存活会话的 shutdown,再等待后端的 `shutdown()`;失败只发出警告而不抛出),以及对 `ctx.sessions.list()` 的收养扫描(热重载不会重放 `session/created`)。即时投递会交接生命周期记录;暂存投递会将上次释放后的任何后缀留在本地,包括随后的 shutdown 标记。
|
||||
|
||||
## 脱敏 waterfall(瀑布式事件)
|
||||
|
||||
每条记录在投影与 `emit()` 之间都要经过 `telemetry/record` waterfall,这是该 seam 的脱敏扩展点。seam 自身不带任何规则:最内层的 `next()` 原样透传记录,因此未挂载监听器时,记录以捕获时的原样到达后端;导出数据能干净到什么程度,恰恰取决于部署方挂载了什么规则。监听器通过变换 `next()` 的返回值来堆叠;不调用 `next()` 就返回,即替换其下方的全部逻辑;抛出异常的监听器会在协调器的隔离范围内以 fail-closed 方式拦下这一条记录。脱敏只作用于导出副本;权威会话日志永不改写。
|
||||
每条记录在投影后立即经过 `telemetry/record` waterfall,这是该 seam 的脱敏扩展点。seam 自身不带任何规则:最内层的 `next()` 原样透传记录,因此未挂载监听器时,记录以捕获时的原样到达后端;导出数据能干净到什么程度,恰恰取决于部署方挂载了什么规则。监听器通过变换 `next()` 的返回值来堆叠;不调用 `next()` 就返回,即替换其下方的全部逻辑;抛出异常的监听器会在协调器的隔离范围内以 fail-closed 方式拦下这一条记录。暂存投递只保留 waterfall 的结果,因此后续移除策略也无法暴露捕获时的原始内容。脱敏只作用于外发副本;权威会话日志永不改写。
|
||||
|
||||
## handoff 游标
|
||||
|
||||
一个模块作用域的 `WeakMap<Session, seq>` 记录每个会话已交接(而非已投递)的最高 seq,在 emit 时推进。游标在不重新求值本模块的重载(配置重新应用、后端源码重载)中存活,而迭代恰恰发生在这类重载中;这种不对称正是游标放在 seam 一侧的原因。重新收养时,协调器只重新交接游标之后的事件(游标及其之前的事件仍用于重建分片投影状态);游标缺失时安全退化为从会话构造边界起的重新交接(`Session.firstLiveSeq`,对在本进程中诞生的会话即 seq 0),由接收端基于 `(session.id, event.seq)` 的去重吸收。构造函数种子绝不会再次导出:恢复会话的历史已由上一个进程以同一 id 发出,fork 继承的前缀则位于父会话的流中(接收端基于 `session.parent_id` + `session.seed_length` 拼接)。由此接受的代价与至多一次(at-most-once)投递一致:恢复不会回填上一个进程未能投递的记录;有回填要求的部署需要的是已推迟的 outbox,而不是回放。这是对「注册即 effect」纪律的一次有意且范围极窄的例外:条目随其会话消亡,值是单调水位线,丢失它绝不是错误。
|
||||
一个模块作用域的 `WeakMap<Session, seq>` 记录每个会话已交接(而非已投递)的最高 seq。即时投递在捕获时推进游标;暂存投递只有在 `release(session)` 将记录交给后端时才推进游标。因此,重建协调器后会通过确定性重新收养恢复未释放的前缀,而不会随其内存副本一同消失。重新收养时,协调器只重新交接游标之后的事件(游标及其之前的事件仍用于重建分片投影状态);游标缺失时安全退化为从会话构造边界起的重新交接(`Session.firstLiveSeq`,对在本进程中诞生的会话即 seq 0),由接收端基于 `(session.id, event.seq)` 的去重吸收。构造函数种子绝不会再次导出:恢复会话的历史已由上一个进程以同一 id 发出,fork 继承的前缀则位于父会话的流中(接收端基于 `session.parent_id` + `session.seed_length` 拼接)。由此接受的代价与至多一次(at-most-once)投递一致:恢复不会回填上一个进程未能投递的记录;有回填要求的部署需要的是已推迟的 outbox,而不是回放。这是对「注册即 effect」纪律的一次有意且范围极窄的例外:条目随其会话消亡,值是单调水位线,丢失它绝不是错误。
|
||||
|
||||
## 固定分片投影
|
||||
|
||||
@@ -40,3 +40,4 @@
|
||||
|
||||
- **尽力而为的投递**:游标标记的是已交接而非已投递;在重载窗口内被拆除的会话无法重新收养;崩溃时留在后端队列中的内容会丢失。持久化 outbox(spool、每 sink 游标、at-least-once)推迟到有部署方提出明确的崩溃丢失要求时再实现;见[复活 Agent Note](../../../.agents/notes/implemented/feature/2026-07-23-session-telemetry-otel-revival.md)。
|
||||
- **不内置脱敏规则**:未挂载 `telemetry/record` 监听器时,记录以捕获时的原样离开进程,包括文件内容或命令输出中内嵌的任何凭据;向共享 collector 导出的部署方自行负责其规则集。
|
||||
- **暂存前缀会重复占用内存**:暂存投递会为每个已投影事件保留一份深拷贝且已脱敏的记录,直到释放或回收会话。它不增加持久化 outbox,而是有意以内存换取简单的「触发前不上传」边界。
|
||||
@@ -3,10 +3,11 @@
|
||||
* firehose plus the one live-bus relay (`agent/error`), applies the fixed
|
||||
* chunk projection, builds logical records, runs each through the
|
||||
* `telemetry/record` waterfall (deployment-mounted redaction rules;
|
||||
* pass-through when none), and hands the result to the backend — synchronously, with every
|
||||
* handler self-contained so a failing backend can never starve other
|
||||
* subscribers (cordis `emit` is stop-on-throw) or touch the agent loop.
|
||||
* Composed by a backend in its constructor.
|
||||
* pass-through when none), then hands the result to the backend immediately
|
||||
* or holds it for explicit release. Every synchronous handler is
|
||||
* self-contained so a failing backend can never starve other subscribers
|
||||
* (cordis `emit` is stop-on-throw) or touch the agent loop. Composed by a
|
||||
* backend in its constructor.
|
||||
*
|
||||
* @module @deepseek-ai/dsh-session-telemetry/coordinator
|
||||
*/
|
||||
@@ -16,6 +17,16 @@ import type { Session, SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import type { Agent } from '@deepseek-ai/dsh-agent'
|
||||
import type { TelemetryBackend, TelemetryRecord, TelemetrySeverity } from './index.ts'
|
||||
|
||||
/** Whether capture hands records over immediately or holds them for an explicit release. */
|
||||
export type TelemetryDelivery = 'immediate' | 'held'
|
||||
|
||||
/** One redacted record waiting at the capture boundary. */
|
||||
interface PendingRecord {
|
||||
readonly record: TelemetryRecord
|
||||
/** Ledger cursor advanced only after the backend accepts this record. */
|
||||
readonly seq?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* The handoff cursor: per session, the highest `seq` handed to a backend.
|
||||
* Deliberately MODULE-scope ambient state — a narrow, documented exception
|
||||
@@ -35,14 +46,13 @@ const handoffCursor = new WeakMap<Session, number>()
|
||||
* Registers the persistence-coordinator listener set plus the `agent/error`
|
||||
* relay, all through `ctx.effect()`/`ctx.on()` on the composing fiber, and
|
||||
* sweeps already-live sessions (a hot reload does not replay
|
||||
* `session/created`). A `session/disposed` emits the session's `shutdown`
|
||||
* operational record — the marker rides the session's own termination edge,
|
||||
* where receivers key crash detection — and retires it from the adopted set,
|
||||
* so a long-lived backend neither retains closed sessions (and their frozen
|
||||
* event logs) nor re-marks them at unload. Disposal marks the sessions still
|
||||
* alive at teardown (their own edge would fire unobserved) and then awaits
|
||||
* the backend's `shutdown()`; a failure there warns instead of throwing —
|
||||
* best-effort reporting must not fail application teardown.
|
||||
* `session/created`). A `session/disposed` captures the session's `shutdown`
|
||||
* operational record at its own termination edge and retires it from the
|
||||
* adopted set. Immediate delivery hands that marker over; held delivery keeps
|
||||
* it local without another explicit release. Disposal captures the same
|
||||
* marker for sessions still alive, then awaits the backend's `shutdown()`; a
|
||||
* failure there warns instead of throwing — best-effort reporting must not
|
||||
* fail application teardown.
|
||||
*/
|
||||
export class TelemetryCoordinator {
|
||||
/**
|
||||
@@ -53,28 +63,30 @@ export class TelemetryCoordinator {
|
||||
private readonly adopted = new Set<Session>()
|
||||
/** Per session, the `turn:step` keys whose first chunk already shipped; rebuilt from the log on re-adoption. */
|
||||
private readonly chunkSeen = new WeakMap<Session, Set<string>>()
|
||||
/** Redacted records retained until {@link release}; weak keys do not extend session lifetime. */
|
||||
private readonly held = new WeakMap<Session, PendingRecord[]>()
|
||||
|
||||
/**
|
||||
* @param ctx - the composing backend's context; listeners bind to its fiber.
|
||||
* @param backend - the backend receiving records; owned elsewhere, never disposed here beyond `shutdown()` forwarding.
|
||||
* @param delivery - immediate handoff, or held delivery released explicitly per session.
|
||||
*/
|
||||
constructor(
|
||||
private readonly ctx: Context,
|
||||
private readonly backend: TelemetryBackend,
|
||||
private readonly delivery: TelemetryDelivery = 'immediate',
|
||||
) {
|
||||
ctx.on('session/created', (session) => {
|
||||
this.adopt(session)
|
||||
})
|
||||
// The session's own termination edge: emit the shutdown marker HERE —
|
||||
// receivers classify a session with activity and no marker as crashed,
|
||||
// so a normally closed session in a long-running host must get its
|
||||
// marker at disposal, not never. Then retire: the projection/cursor
|
||||
// WeakMaps die with the Session object; only the strong adopted set
|
||||
// needs the explicit release.
|
||||
// Capture the shutdown marker at the session's own termination edge.
|
||||
// Immediate delivery preserves crash classification; held delivery does
|
||||
// not let a later lifecycle edge extend a user-released prefix. Then
|
||||
// retire the only strong reference owned by this coordinator.
|
||||
ctx.on('session/disposed', (session) => {
|
||||
this.contain(() => {
|
||||
if (!this.adopted.delete(session)) return
|
||||
this.handOff(shutdownRecord(session))
|
||||
this.submit(session, { record: this.redact(shutdownRecord(session)) })
|
||||
})
|
||||
})
|
||||
ctx.on('session/event', (session, event) => {
|
||||
@@ -95,13 +107,12 @@ export class TelemetryCoordinator {
|
||||
})
|
||||
})
|
||||
ctx.effect(() => async () => {
|
||||
// Sessions still adopted here are alive through a whole-application
|
||||
// teardown (their own disposal edge will fire after telemetry is gone,
|
||||
// unobserved) — mark them now so the receiver sees a clean stop of
|
||||
// observation rather than a crash-shaped silence.
|
||||
// Sessions still adopted here are alive through whole-application
|
||||
// teardown, so capture the marker before the backend quiesces. Held
|
||||
// delivery intentionally leaves it local without another release.
|
||||
for (const session of this.adopted) {
|
||||
this.contain(() => {
|
||||
this.handOff(shutdownRecord(session))
|
||||
this.submit(session, { record: this.redact(shutdownRecord(session)) })
|
||||
})
|
||||
}
|
||||
try {
|
||||
@@ -115,6 +126,23 @@ export class TelemetryCoordinator {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hand the records currently held for one session to the backend in capture order.
|
||||
* Records captured after this call form a new held prefix. Backend failures remain
|
||||
* contained per record and do not starve later records in the same release.
|
||||
* @param session - session whose pending capture prefix may leave the process.
|
||||
*/
|
||||
release(session: Session): void {
|
||||
const pending = this.held.get(session)
|
||||
if (pending === undefined) return
|
||||
this.held.delete(session)
|
||||
for (const record of pending) {
|
||||
this.contain(() => {
|
||||
this.deliver(session, record)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adopt a session: replay its log THROUGH the projection from the handoff
|
||||
* cursor, then rely on the firehose for everything after. When no cursor
|
||||
@@ -153,7 +181,7 @@ export class TelemetryCoordinator {
|
||||
}
|
||||
}
|
||||
|
||||
/** Project one event and hand it to the backend, advancing the cursor on handoff. */
|
||||
/** Project and redact one event, then submit it under the delivery policy. */
|
||||
private capture(session: Session, event: SessionEvent): void {
|
||||
if (event.type === 'assistant/chunk') {
|
||||
const key = `${event.data.turn}:${event.data.step}`
|
||||
@@ -165,27 +193,47 @@ export class TelemetryCoordinator {
|
||||
if (seen.has(key)) return
|
||||
seen.add(key)
|
||||
}
|
||||
this.handOff({
|
||||
channel: 'ledger',
|
||||
time: event.time,
|
||||
severity: severityOf(event),
|
||||
attributes: identityOf(session, event),
|
||||
// The live event object is mutable and the backend serializes later;
|
||||
// append-time validation guarantees this clone cannot throw.
|
||||
body: structuredClone(event.data),
|
||||
this.submit(session, {
|
||||
record: this.redact({
|
||||
channel: 'ledger',
|
||||
time: event.time,
|
||||
severity: severityOf(event),
|
||||
attributes: identityOf(session, event),
|
||||
// The live event object is mutable and the backend serializes later;
|
||||
// append-time validation guarantees this clone cannot throw.
|
||||
body: structuredClone(event.data),
|
||||
}),
|
||||
seq: event.seq,
|
||||
})
|
||||
handoffCursor.set(session, event.seq)
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the `telemetry/record` waterfall over one record and hand the result
|
||||
* to the backend. The innermost `next` passes the record through unchanged
|
||||
* — the seam ships no rules; exported data is as clean as the listeners a
|
||||
* deployment mounts. Callers run inside {@link contain}, so a throwing
|
||||
* rule withholds the record instead of reaching the loop (fail-closed).
|
||||
* Run the `telemetry/record` waterfall at capture time. The innermost `next`
|
||||
* passes the record through unchanged — the seam ships no rules; exported
|
||||
* data is as clean as the listeners a deployment mounts. Callers run inside
|
||||
* {@link contain}, so a throwing rule withholds the record instead of
|
||||
* reaching the loop (fail-closed). Held delivery stores only this result, so
|
||||
* a later policy reload cannot expose the pre-redaction capture.
|
||||
*/
|
||||
private handOff(record: TelemetryRecord): void {
|
||||
this.backend.emit(this.ctx.waterfall('telemetry/record', record, () => record))
|
||||
private redact(record: TelemetryRecord): TelemetryRecord {
|
||||
return this.ctx.waterfall('telemetry/record', record, () => record)
|
||||
}
|
||||
|
||||
/** Hold one redacted record or deliver it immediately under the configured policy. */
|
||||
private submit(session: Session, pending: PendingRecord): void {
|
||||
if (this.delivery === 'held') {
|
||||
let records = this.held.get(session)
|
||||
if (records === undefined) this.held.set(session, records = [])
|
||||
records.push(pending)
|
||||
return
|
||||
}
|
||||
this.deliver(session, pending)
|
||||
}
|
||||
|
||||
/** Hand one redacted record to the backend, then advance its ledger cursor. */
|
||||
private deliver(session: Session, pending: PendingRecord): void {
|
||||
this.backend.emit(pending.record)
|
||||
if (pending.seq !== undefined) handoffCursor.set(session, pending.seq)
|
||||
}
|
||||
|
||||
/** Forward the turn-end boundary to the backend's optional flush hint. */
|
||||
@@ -196,19 +244,21 @@ export class TelemetryCoordinator {
|
||||
/** Relay one `agent/error` bus emission as an `agent-error` operational record. */
|
||||
private relayAgentError(agent: Agent, turn: number, step: number, error: unknown): void {
|
||||
const detail = errorDetail(error)
|
||||
this.handOff({
|
||||
channel: 'ops',
|
||||
time: Date.now(),
|
||||
severity: 'error',
|
||||
attributes: {
|
||||
'telemetry.op': 'agent-error',
|
||||
'session.id': String(agent.session.id),
|
||||
'agent.id': agent.id,
|
||||
'error.name': detail.name,
|
||||
turn,
|
||||
step,
|
||||
},
|
||||
body: detail,
|
||||
this.submit(agent.session, {
|
||||
record: this.redact({
|
||||
channel: 'ops',
|
||||
time: Date.now(),
|
||||
severity: 'error',
|
||||
attributes: {
|
||||
'telemetry.op': 'agent-error',
|
||||
'session.id': String(agent.session.id),
|
||||
'agent.id': agent.id,
|
||||
'error.name': detail.name,
|
||||
turn,
|
||||
step,
|
||||
},
|
||||
body: detail,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
*
|
||||
* The seam owns the CAPTURE side of session-event reporting — which records
|
||||
* exist (the chunk projection), what they carry (the logical record), when
|
||||
* they are handed over (adoption, the per-append firehose, lifecycle
|
||||
* forwarding), and the HMR handoff cursor. Everything downstream of
|
||||
* they are captured (adoption, the per-append firehose, lifecycle
|
||||
* forwarding), immediate versus explicitly released handoff, and the HMR
|
||||
* cursor. Everything downstream of
|
||||
* {@link Telemetry.emit} — batching, retry, queueing, loss policy — is the
|
||||
* reporting SDK's territory and is deliberately not modelled here. The
|
||||
* design and its trade-offs are pinned in
|
||||
@@ -94,9 +95,10 @@ export interface TelemetryBackend {
|
||||
/**
|
||||
* Hand one record to the backend's pipeline. MUST be a non-blocking
|
||||
* enqueue — the coordinator calls this synchronously from the
|
||||
* `session/event` hot path, so anything slower than a queue push would tax
|
||||
* the agent loop. Errors thrown here are contained by the coordinator and
|
||||
* logged; they never reach the loop.
|
||||
* `session/event` hot path, either at capture or while releasing a held
|
||||
* prefix, so anything slower than a queue push would tax the agent loop.
|
||||
* Errors thrown here are contained by the coordinator and logged; they
|
||||
* never reach the loop.
|
||||
* @param record - the logical record to report; owned by the backend after the call.
|
||||
*/
|
||||
emit(record: TelemetryRecord): void
|
||||
@@ -121,6 +123,9 @@ export interface TelemetryBackend {
|
||||
* coordinator emits its dispose-time `shutdown` markers immediately before
|
||||
* calling this). Awaited by the coordinator's dispose; a rejection is
|
||||
* logged as a warning and never fails application teardown.
|
||||
* The coordinator captures dispose-time shutdown markers immediately
|
||||
* before this call; immediate delivery enqueues them, while held delivery
|
||||
* leaves an unreleased suffix local.
|
||||
* @returns resolves when the backend's pipeline has quiesced.
|
||||
*/
|
||||
shutdown(): Promise<void>
|
||||
@@ -153,4 +158,4 @@ export abstract class Telemetry extends Service implements TelemetryBackend {
|
||||
abstract shutdown(): Promise<void>
|
||||
}
|
||||
|
||||
export { TelemetryCoordinator } from './coordinator.ts'
|
||||
export { TelemetryCoordinator, type TelemetryDelivery } from './coordinator.ts'
|
||||
@@ -10,7 +10,12 @@ import { describe, expect, it, vi } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import SessionStore, { SessionId, type Session, type SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import type { Agent } from '@deepseek-ai/dsh-agent'
|
||||
import { TelemetryCoordinator, type TelemetryBackend, type TelemetryRecord } from '../src/index.ts'
|
||||
import {
|
||||
TelemetryCoordinator,
|
||||
type TelemetryBackend,
|
||||
type TelemetryDelivery,
|
||||
type TelemetryRecord,
|
||||
} from '../src/index.ts'
|
||||
|
||||
declare module '@deepseek-ai/dsh-session' {
|
||||
interface SessionEventMap {
|
||||
@@ -54,15 +59,21 @@ class FakeBackend implements TelemetryBackend {
|
||||
}
|
||||
}
|
||||
|
||||
async function setup(backend: FakeBackend = new FakeBackend()) {
|
||||
async function setup(
|
||||
backend: FakeBackend = new FakeBackend(),
|
||||
delivery: TelemetryDelivery = 'immediate',
|
||||
) {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(SessionStore)
|
||||
let coordinator!: TelemetryCoordinator
|
||||
const fiber = await ctx.plugin({
|
||||
name: 'fake-telemetry',
|
||||
inject: ['sessions'],
|
||||
apply: (inner: Context) => void new TelemetryCoordinator(inner, backend),
|
||||
apply: (inner: Context) => {
|
||||
coordinator = new TelemetryCoordinator(inner, backend, delivery)
|
||||
},
|
||||
})
|
||||
return { ctx, backend, fiber }
|
||||
return { ctx, backend, coordinator, fiber }
|
||||
}
|
||||
|
||||
function liveSession(ctx: Context, id = `s-${Math.random().toString(36).slice(2)}`): Session {
|
||||
@@ -167,6 +178,80 @@ describe('TelemetryCoordinator capture', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('TelemetryCoordinator held delivery', () => {
|
||||
it('releases one pending prefix at a time without handing later records over early', async () => {
|
||||
const { ctx, backend, coordinator } = await setup(new FakeBackend(), 'held')
|
||||
const session = liveSession(ctx, 'held-prefix')
|
||||
appendTurn(session)
|
||||
expect(backend.records).toEqual([])
|
||||
|
||||
coordinator.release(session)
|
||||
expect(backend.ledger().map(record => record.attributes['event.type'])).toEqual([
|
||||
'turn/start',
|
||||
'user/message',
|
||||
])
|
||||
|
||||
session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
|
||||
expect(backend.ledger()).toHaveLength(2)
|
||||
coordinator.release(session)
|
||||
coordinator.release(session)
|
||||
expect(backend.ledger().map(record => record.attributes['event.type'])).toEqual([
|
||||
'turn/start',
|
||||
'user/message',
|
||||
'turn/end',
|
||||
])
|
||||
})
|
||||
|
||||
it('stores the capture-time redacted copy rather than re-running policy at release', async () => {
|
||||
const { ctx, backend, coordinator } = await setup(new FakeBackend(), 'held')
|
||||
const disposeRule = ctx.on('telemetry/record', (_record, next) => ({
|
||||
...next(),
|
||||
body: { scrubbed: true },
|
||||
}))
|
||||
const session = liveSession(ctx, 'held-redacted')
|
||||
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
|
||||
disposeRule()
|
||||
|
||||
coordinator.release(session)
|
||||
expect(backend.ledger()[0]!.body).toEqual({ scrubbed: true })
|
||||
})
|
||||
|
||||
it('contains each backend failure independently while releasing a batch', async () => {
|
||||
const backend = new FakeBackend()
|
||||
backend.rejectSeq = 1
|
||||
const { ctx, coordinator } = await setup(backend, 'held')
|
||||
const warn = vi.spyOn(ctx.logger, 'warn').mockImplementation(() => {})
|
||||
const session = liveSession(ctx, 'held-failure')
|
||||
appendTurn(session)
|
||||
session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
|
||||
|
||||
coordinator.release(session)
|
||||
expect(backend.ledger().map(record => record.attributes['event.seq'])).toEqual([0, 2])
|
||||
expect(warn).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rebuilds an unreleased prefix after coordinator reload', async () => {
|
||||
const first = new FakeBackend()
|
||||
const { ctx, fiber } = await setup(first, 'held')
|
||||
const session = liveSession(ctx, 'held-reload')
|
||||
session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
|
||||
await fiber.dispose()
|
||||
expect(first.records).toEqual([])
|
||||
|
||||
const second = new FakeBackend()
|
||||
let coordinator!: TelemetryCoordinator
|
||||
await ctx.plugin({
|
||||
name: 'fake-telemetry-after-held-reload',
|
||||
inject: ['sessions'],
|
||||
apply: (inner: Context) => {
|
||||
coordinator = new TelemetryCoordinator(inner, second, 'held')
|
||||
},
|
||||
})
|
||||
coordinator.release(session)
|
||||
expect(second.ledger().map(record => record.attributes['event.seq'])).toEqual([0])
|
||||
})
|
||||
})
|
||||
|
||||
describe('TelemetryCoordinator adoption', () => {
|
||||
it('exports an unpublished suffix without re-exporting constructor history', async () => {
|
||||
const backend = new FakeBackend()
|
||||
|
||||
Reference in New Issue
Block a user