import { createServer } from 'node:http' import type { IncomingMessage, Server, ServerResponse } from 'node:http' import { afterEach, describe, expect, it, vi } from 'vitest' import { Context } from 'cordis' import LlmService, { CONTEXT_WINDOW_EXCEEDED_CODE, LlmError, ReasoningEffortId, userAgent } from '@deepseek-ai/dsh-llm' import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai' import { PiAiAdapter } from '@deepseek-ai/dsh-llm-pi-ai' import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' import { getBuiltinModels } from '@earendil-works/pi-ai/providers/all' import { resolveProfiles } from '../src/config.ts' import { assemble } from './assemble.ts' interface MockServer { url: string paths: string[] requests: unknown[] headers: IncomingMessage['headers'][] readonly closedResponses: number responseClosed: Promise } const servers: Server[] = [] afterEach(async () => { vi.unstubAllEnvs() await Promise.all(servers.splice(0).map(server => new Promise(resolve => server.close(resolve)))) }) async function mockServer(script: { status?: number events?: string[] body?: string delayMs?: number headers?: Record }[]): Promise { const paths: string[] = [] const requests: unknown[] = [] const headers: IncomingMessage['headers'][] = [] let closedResponses = 0 const responseClosed = Promise.withResolvers() const server = createServer((request: IncomingMessage, response: ServerResponse) => { response.on('close', () => { closedResponses += 1 responseClosed.resolve(undefined) }) let body = '' request.on('data', (chunk: Buffer) => { body += chunk.toString('utf8') }) request.on('end', () => { paths.push(request.url ?? '') requests.push(body.length === 0 ? undefined : JSON.parse(body)) headers.push(request.headers) const behavior = script.shift() ?? { status: 500, body: 'script exhausted' } if (behavior.status !== undefined && behavior.status !== 200) { response.writeHead(behavior.status, { 'content-type': 'application/json', ...behavior.headers }) response.end(behavior.body ?? '{}') return } response.writeHead(200, { 'content-type': 'text/event-stream' }) let index = 0 const writeNext = (): void => { const event = behavior.events?.[index++] if (event === undefined) { response.end(); return } response.write(`data: ${event}\n\n`) if (behavior.delayMs === undefined) writeNext() else setTimeout(writeNext, behavior.delayMs) } writeNext() }) }) servers.push(server) await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) const address = server.address() if (address === null || typeof address === 'string') throw new Error('no port') return { url: `http://127.0.0.1:${address.port}`, paths, requests, headers, responseClosed: responseClosed.promise, get closedResponses() { return closedResponses }, } } const textEvents = [ '{"choices":[{"delta":{"role":"assistant","content":""},"index":0,"finish_reason":null}]}', '{"choices":[{"delta":{"content":"hello"},"index":0,"finish_reason":null}]}', '{"choices":[{"delta":{},"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}', '[DONE]', ] async function harness(baseURL: string, overrides: Record = {}): Promise { const ctx = new Context() await ctx.plugin(LlmService) await ctx.plugin(LlmPiAi, { providers: [{ provider: 'deepseek', apiKey: 'test-key', baseURL, ...overrides }], }) return ctx } describe('PiAiAdapter provider routing', () => { it('resolves a catalog model dynamically and uses a private endpoint', async () => { const server = await mockServer([{ events: textEvents }]) const ctx = await harness(server.url) const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }], }) expect(result.message.content).toEqual([{ type: 'text', text: 'hello' }]) expect(result.finish).toEqual({ kind: 'stop' }) expect(result.usage).toEqual({ inputTokens: 3, outputTokens: 1 }) expect(server.paths).toEqual(['/chat/completions']) }) it('merges profile headers with Harness attribution winning', async () => { const server = await mockServer([{ events: textEvents }]) const ctx = await harness(server.url, { headers: { 'x-company': 'private', 'User-Agent': 'wrong' }, }) await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] }) expect(server.headers[0]?.['x-company']).toBe('private') expect(server.headers[0]?.['user-agent']).toBe(userAgent()) }) it('forwards common stream options and profile reasoning', async () => { const server = await mockServer([{ events: textEvents }]) const ctx = await harness(server.url, { reasoning: 'max', cacheRetention: 'none', transport: 'sse', timeoutMs: 5000, websocketConnectTimeoutMs: 3000, streamIdleTimeoutMs: 10_000, thinkingBudgets: { high: 2048 }, }) await assemble(ctx, { model: 'deepseek-v4-flash', messages: [], temperature: 0.2, maxTokens: 77, sessionId: 'session-for-pi' as never, }) expect(server.requests[0]).toMatchObject({ model: 'deepseek-v4-flash', temperature: 0.2, max_completion_tokens: 77, thinking: { type: 'enabled' }, reasoning_effort: 'max', }) }) it('uses a dynamic request effort and rejects unsupported efforts before network I/O', async () => { const server = await mockServer([{ events: textEvents }, { events: textEvents }]) const ctx = await harness(server.url, { reasoning: 'max' }) await assemble(ctx, { model: 'deepseek-v4-flash', reasoningEffort: ReasoningEffortId('high'), messages: [], }) expect(server.requests[0]).toMatchObject({ reasoning_effort: 'high' }) await assemble(ctx, { model: 'deepseek-v4-flash', reasoningEffort: ReasoningEffortId('off'), messages: [], }) expect(server.requests[1]).toMatchObject({ thinking: { type: 'disabled' } }) expect(server.requests[1]).not.toHaveProperty('reasoning_effort') await expect(assemble(ctx, { model: 'deepseek-v4-flash', reasoningEffort: ReasoningEffortId('xhigh'), messages: [], })).rejects.toMatchObject({ code: 'UNSUPPORTED_REASONING_EFFORT' }) expect(server.requests).toHaveLength(2) }) it('preserves omitted profile options when constructing the adapter directly', async () => { const server = await mockServer([{ events: textEvents }]) const ctx = new Context() await ctx.plugin(LlmService) ctx.llm.registerAdapter(['deepseek'], new PiAiAdapter({ profiles: [{ provider: 'deepseek', apiKey: 'test-key', baseURL: server.url }], })) const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] }) expect(result.message.content).toEqual([{ type: 'text', text: 'hello' }]) }) it('rejects stop sequences rather than silently ignoring them', async () => { const server = await mockServer([]) const ctx = await harness(server.url) await expect(assemble(ctx, { model: 'deepseek-v4-flash', messages: [], stop: ['END'] })) .rejects.toMatchObject({ code: 'UNSUPPORTED_OPTION' }) expect(server.requests).toEqual([]) }) it('rejects unknown catalog models before network I/O', async () => { const server = await mockServer([]) const ctx = await harness(server.url) await expect(assemble(ctx, { model: 'not-in-the-catalog', messages: [] })) .rejects.toMatchObject({ code: 'UNKNOWN_MODEL' }) expect(server.requests).toEqual([]) }) it('uses the catalog API implementation, including OpenAI Responses', async () => { const server = await mockServer([{ status: 401, body: JSON.stringify({ error: { message: 'expected mock failure' } }) }]) const ctx = new Context() await ctx.plugin(LlmService) await ctx.plugin(LlmPiAi, { providers: [{ provider: 'openai', apiKey: 'test-key', baseURL: `${server.url}/v1` }], }) const result = await assemble(ctx, { provider: 'openai', model: 'gpt-4.1', messages: [] }) expect(result.finish.kind).toBe('error') expect(server.paths).toEqual(['/v1/responses']) }) it('forces one wire request for an SDK-retryable provider failure', async () => { const server = await mockServer([ { status: 429, headers: { 'retry-after-ms': '1' }, body: JSON.stringify({ error: { message: 'retryable provider failure' } }), }, { status: 500, body: JSON.stringify({ error: { message: 'hidden SDK retry' } }) }, { status: 500, body: JSON.stringify({ error: { message: 'second hidden SDK retry' } }) }, ]) const ctx = new Context() await ctx.plugin(LlmService) await ctx.plugin(LlmPiAi, { providers: [{ provider: 'openai', apiKey: 'test-key', baseURL: `${server.url}/v1` }], }) const result = await assemble(ctx, { provider: 'openai', model: 'gpt-4.1', messages: [] }) expect(result.finish).toMatchObject({ kind: 'error' }) expect(server.paths).toEqual(['/v1/responses']) }) it('uses OpenAI Responses against an Azure project v1 path with its API key header', async () => { const server = await mockServer([{ status: 401, body: JSON.stringify({ error: { message: 'expected mock failure' } }) }]) const ctx = new Context() await ctx.plugin(LlmService) await ctx.plugin(LlmPiAi, { providers: [{ provider: 'openai', apiKey: 'test-key', baseURL: `${server.url}/api/projects/openai/openai/v1`, headers: { 'api-key': 'test-key', Authorization: '' }, }], }) const result = await assemble(ctx, { provider: 'openai', model: 'gpt-5.5', messages: [] }) expect(result.finish.kind).toBe('error') expect(server.paths).toEqual(['/api/projects/openai/openai/v1/responses']) expect(server.headers[0]?.['api-key']).toBe('test-key') expect(server.headers[0]?.authorization).toBe('') }) it.each([ [401, 'AUTH'], [400, 'INVALID_REQUEST'], [429, 'RATE_LIMIT'], [500, 'SERVER'], ] as const)('maps HTTP %s failures to %s', async (status, code) => { const server = await mockServer([{ status, body: JSON.stringify({ error: { message: `provider ${status}` } }) }]) const ctx = await harness(server.url) const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] }) expect(result.finish).toMatchObject({ kind: 'error', failure: { code } }) expect(server.paths).toEqual(['/chat/completions']) }) it('uses the resolved catalog context window for usage-based overflow detection', async () => { const model = getBuiltinModels('deepseek').find(candidate => candidate.id === 'deepseek-v4-flash') if (model === undefined) throw new Error('deepseek-v4-flash missing from pi-ai test catalog') const events = [ '{"choices":[{"delta":{"role":"assistant","content":""},"index":0,"finish_reason":null}]}', JSON.stringify({ choices: [{ delta: {}, index: 0, finish_reason: 'stop' }], usage: { prompt_tokens: model.contextWindow + 1, completion_tokens: 0 }, }), '[DONE]', ] const server = await mockServer([{ events }]) const ctx = await harness(server.url) const result = await assemble(ctx, { model: model.id, messages: [] }) expect(result.finish).toEqual({ kind: 'error', failure: { message: `pi-ai detected context overflow for model "${model.id}"`, code: CONTEXT_WINDOW_EXCEEDED_CODE, }, }) }) it('stops the SDK request when the adapter idle watchdog expires', async () => { const server = await mockServer([{ events: textEvents, delayMs: 200 }]) const ctx = await harness(server.url, { streamIdleTimeoutMs: 20 }) await expect(assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })) .rejects.toMatchObject({ code: 'TIMEOUT' }) await Promise.race([ server.responseClosed, new Promise((_resolve, reject) => { setTimeout(() => { reject(new Error('SDK request did not close after idle timeout')) }, 100) }), ]) expect(server.paths).toEqual(['/chat/completions']) expect(server.closedResponses).toBe(1) }) }) describe('provider profile lifecycle', () => { it('keeps adapter helpers off the package root', () => { for (const helper of [ 'resolveProfiles', 'toPiContext', 'toPiReplayState', 'toPiAssistant', 'mapStopReason', 'mapUsage', 'toStreamChunks', ]) expect(LlmPiAi).not.toHaveProperty(helper) }) it('registers every profile atomically and unregisters on dispose', async () => { const ctx = new Context() await ctx.plugin(LlmService) const fiber = await ctx.plugin(LlmPiAi, { providers: [{ provider: 'openai' }, { provider: 'anthropic' }], }) expect(ctx.llm.listProviders()).toEqual([ { id: 'openai', name: 'openai' }, { id: 'anthropic', name: 'anthropic' }, ]) await fiber.dispose() expect(ctx.llm.listProviders()).toEqual([]) }) it('exposes the installed pi-ai model catalog through provider-neutral metadata', async () => { const ctx = new Context() await ctx.plugin(LlmService) await ctx.plugin(LlmPiAi, { providers: [{ provider: 'openai' }] }) const models = await ctx.llm.listModels('openai') expect(models.find(model => model.id === 'gpt-4.1')).toEqual({ provider: 'openai', id: 'gpt-4.1', name: 'GPT-4.1', }) expect(models.every(model => model.provider === 'openai')).toBe(true) const info = await ctx.llm.resolveModelInfo('openai', 'gpt-4.1') expect(typeof info.context?.contextWindow).toBe('number') }) it('exposes pi-ai model thinking levels verbatim without inventing a provider default', async () => { const ctx = new Context() await ctx.plugin(LlmService) await ctx.plugin(LlmPiAi, { providers: [{ provider: 'deepseek' }, { provider: 'openai' }], }) await expect(ctx.llm.resolveModelInfo('deepseek', 'deepseek-v4-flash')) .resolves.toMatchObject({ reasoning: { efforts: [ { id: ReasoningEffortId('off'), name: 'Off' }, { id: ReasoningEffortId('high'), name: 'High' }, { id: ReasoningEffortId('max'), name: 'Max' }, ], }, }) const extended = await ctx.llm.resolveModelInfo('openai', 'gpt-5.6-sol') expect(extended.reasoning?.efforts.map(effort => effort.id)).toEqual([ ReasoningEffortId('off'), ReasoningEffortId('minimal'), ReasoningEffortId('low'), ReasoningEffortId('medium'), ReasoningEffortId('high'), ReasoningEffortId('xhigh'), ReasoningEffortId('max'), ]) await expect(ctx.llm.resolveModelInfo('openai', 'gpt-4.1')) .resolves.toMatchObject({ reasoning: { efforts: [{ id: ReasoningEffortId('off'), name: 'Off' }], }, }) }) it('uses a supported profile reasoning value as the model default and rejects an unsupported one', async () => { const supported = new Context() await supported.plugin(LlmService) await supported.plugin(LlmPiAi, { providers: [{ provider: 'deepseek', reasoning: 'max' }], }) await expect(supported.llm.resolveModelInfo('deepseek', 'deepseek-v4-flash')) .resolves.toMatchObject({ reasoning: { defaultEffort: ReasoningEffortId('max') } }) const unsupported = new Context() await unsupported.plugin(LlmService) await unsupported.plugin(LlmPiAi, { providers: [{ provider: 'deepseek', reasoning: 'medium' }], }) await expect(unsupported.llm.resolveModelInfo('deepseek', 'deepseek-v4-flash')) .rejects.toMatchObject({ code: 'UNSUPPORTED_REASONING_EFFORT' }) const disabled = new Context() await disabled.plugin(LlmService) await disabled.plugin(LlmPiAi, { providers: [{ provider: 'deepseek', reasoning: 'off' }], }) await expect(disabled.llm.resolveModelInfo('deepseek', 'deepseek-v4-flash')) .resolves.toMatchObject({ reasoning: { defaultEffort: ReasoningEffortId('off') } }) }) it('accepts absent credentials for pi-ai ambient authentication', async () => { vi.stubEnv('DEEPSEEK_API_KEY', 'ambient-key') const server = await mockServer([{ events: textEvents }]) const ctx = await harness(server.url, { apiKey: undefined }) await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] }) expect(server.headers[0]?.authorization).toBe('Bearer ambient-key') }) it('validates empty, duplicate, unknown, and explicitly blank profiles', () => { expect(() => resolveProfiles([])).toThrow(/at least one/) expect(() => resolveProfiles([{ provider: '' }])).toThrow(/non-empty/) expect(() => resolveProfiles([{ provider: 'not-real' }])).toThrow(/unknown/) expect(() => resolveProfiles([{ provider: 'openai' }, { provider: 'openai' }])).toThrow(/duplicate/) expect(() => resolveProfiles([{ provider: 'openai', apiKey: '' }])).toThrow(/empty apiKey/) expect(() => resolveProfiles([{ provider: 'openai', apiKey: ' ' }])).toThrow(/empty apiKey/) expect(() => resolveProfiles([{ provider: 'openai', baseURL: '' }])).toThrow(/empty baseURL/) }) it.each(['maxRetries', 'maxRetryDelayMs'] as const)( 'rejects removed profile field %s instead of silently restoring hidden SDK retries', async (field) => { const legacy = { provider: 'openai', [field]: 2 } expect(() => resolveProfiles([legacy as never])).toThrow(/removed.*agent recovery/i) const ctx = new Context() await ctx.plugin(LlmService) await expect(ctx.plugin(LlmPiAi, { providers: [legacy as never] })) .rejects.toThrow(/removed.*agent recovery/i) }, ) it('rejects invalid stream tunables at plugin load', async () => { const invalid = [ { timeoutMs: -1 }, { websocketConnectTimeoutMs: -1 }, { streamIdleTimeoutMs: 0 }, { streamIdleTimeoutMs: Number.NaN }, { streamIdleTimeoutMs: MAX_TIMER_DELAY_MS + 1 }, ] for (const entry of invalid) { const ctx = new Context() await ctx.plugin(LlmService) await expect(ctx.plugin(LlmPiAi, { providers: [{ provider: 'openai', ...entry }] })) .rejects.toThrow() } }) it('constructs the adapter directly and rejects routes it does not own', async () => { const adapter = new PiAiAdapter({ profiles: [{ provider: 'openai' }] }) await expect(adapter.listModels('anthropic')).rejects.toMatchObject({ code: 'NO_ADAPTER' }) await expect(adapter.resolveModel('anthropic', 'claude-sonnet-4')) .rejects.toMatchObject({ code: 'NO_ADAPTER' }) await expect(adapter.resolveModel('openai', 'not-a-catalog-model')) .rejects.toMatchObject({ code: 'UNKNOWN_MODEL' }) await expect((async () => { for await (const _chunk of adapter.stream({ provider: 'anthropic', model: 'claude-sonnet-4', messages: [] })) { /* drain */ } })()).rejects.toMatchObject({ code: 'NO_ADAPTER' }) expect(new LlmError('x', 'X')).toBeInstanceOf(Error) }) it('validates direct-constructor profiles at the embedding boundary', () => { expect(() => new PiAiAdapter({ profiles: [{ provider: 'openai', streamIdleTimeoutMs: 0 }], })).toThrow(/streamIdleTimeoutMs.*positive finite/) expect(() => new PiAiAdapter({ profiles: [{ provider: 'openai', streamIdleTimeoutMs: MAX_TIMER_DELAY_MS + 1 }], })).toThrow(/streamIdleTimeoutMs.*no greater/) }) }) describe('abort wiring', () => { it('preserves an unknown pre-dispatch adapter Error exactly', async () => { const original = new Error('SDK context conversion exploded') const message = Object.defineProperty({}, 'role', { get() { throw original }, }) const adapter = new PiAiAdapter({ profiles: [{ provider: 'deepseek', apiKey: 'test-key' }] }) const drain = async (): Promise => { for await (const _chunk of adapter.stream({ provider: 'deepseek', model: 'deepseek-v4-flash', messages: [message as never], })) { /* drain */ } } await expect(drain()).rejects.toBe(original) }) it('lets a concurrent caller abort classify a pre-dispatch adapter failure', async () => { const controller = new AbortController() const original = new Error('conversion lost its caller') const message = Object.defineProperty({}, 'role', { get() { controller.abort('caller cancelled during conversion') throw original }, }) const adapter = new PiAiAdapter({ profiles: [{ provider: 'deepseek', apiKey: 'test-key' }] }) const drain = async (): Promise => { for await (const _chunk of adapter.stream({ provider: 'deepseek', model: 'deepseek-v4-flash', messages: [message as never], signal: controller.signal, })) { /* drain */ } } await expect(drain()).rejects.toMatchObject({ code: 'ABORTED', cause: original }) }) it('resolves catalog endpoints without an override before honoring pre-abort', async () => { const adapter = new PiAiAdapter({ profiles: [{ provider: 'deepseek', apiKey: 'test-key' }] }) const controller = new AbortController() controller.abort('already stopped') const chunks = [] for await (const chunk of adapter.stream({ provider: 'deepseek', model: 'deepseek-v4-flash', messages: [], signal: controller.signal, })) chunks.push(chunk) expect(chunks.at(-1)).toMatchObject({ type: 'finish', reason: { kind: 'aborted' } }) }) it('honors a pre-aborted caller signal', async () => { const server = await mockServer([{ events: textEvents, delayMs: 20 }]) const ctx = await harness(server.url) const controller = new AbortController() controller.abort('already stopped') const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [], signal: controller.signal }) expect(result.finish.kind).toBe('aborted') }) it('forwards an abort that arrives while provider streaming is active', async () => { const server = await mockServer([{ events: textEvents, delayMs: 30 }]) const ctx = await harness(server.url) const controller = new AbortController() const resultPromise = assemble(ctx, { model: 'deepseek-v4-flash', messages: [], signal: controller.signal, }) setTimeout(() => { controller.abort('stopped during stream') }, 10) const result = await resultPromise expect(result.finish.kind).toBe('aborted') }) it('aborts upstream when a consumer stops early', async () => { const server = await mockServer([{ events: textEvents, delayMs: 30 }]) const ctx = await harness(server.url) for await (const chunk of ctx.llm.stream({ provider: 'deepseek', model: 'deepseek-v4-flash', messages: [] })) { if (chunk.type === 'block-start') break } await new Promise(resolve => setTimeout(resolve, 20)) expect(server.requests).toHaveLength(1) }) })