477 lines
19 KiB
TypeScript
477 lines
19 KiB
TypeScript
import { createServer } from 'node:http'
|
|
import type { IncomingMessage, Server, ServerResponse } from 'node:http'
|
|
import { afterEach, describe, expect, it, vi } from 'vitest'
|
|
import { Context } from 'cordis'
|
|
import LlmService, { CONTEXT_WINDOW_EXCEEDED_CODE, LlmError, userAgent } from '@deepseek-ai/dsh-llm'
|
|
import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai'
|
|
import { PiAiAdapter } from '@deepseek-ai/dsh-llm-pi-ai'
|
|
import { getModels } from '@earendil-works/pi-ai'
|
|
import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout'
|
|
import { resolveProfiles } from '../src/config.ts'
|
|
import { assemble } from './assemble.ts'
|
|
|
|
interface MockServer {
|
|
url: string
|
|
paths: string[]
|
|
requests: unknown[]
|
|
headers: IncomingMessage['headers'][]
|
|
readonly closedResponses: number
|
|
responseClosed: Promise<void>
|
|
}
|
|
|
|
const servers: Server[] = []
|
|
|
|
afterEach(async () => {
|
|
vi.unstubAllEnvs()
|
|
await Promise.all(servers.splice(0).map(server => new Promise(resolve => server.close(resolve))))
|
|
})
|
|
|
|
async function mockServer(script: {
|
|
status?: number
|
|
events?: string[]
|
|
body?: string
|
|
delayMs?: number
|
|
headers?: Record<string, string>
|
|
}[]): Promise<MockServer> {
|
|
const paths: string[] = []
|
|
const requests: unknown[] = []
|
|
const headers: IncomingMessage['headers'][] = []
|
|
let closedResponses = 0
|
|
const responseClosed = Promise.withResolvers<undefined>()
|
|
const server = createServer((request: IncomingMessage, response: ServerResponse) => {
|
|
response.on('close', () => {
|
|
closedResponses += 1
|
|
responseClosed.resolve(undefined)
|
|
})
|
|
let body = ''
|
|
request.on('data', (chunk: Buffer) => { body += chunk.toString('utf8') })
|
|
request.on('end', () => {
|
|
paths.push(request.url ?? '')
|
|
requests.push(body.length === 0 ? undefined : JSON.parse(body))
|
|
headers.push(request.headers)
|
|
const behavior = script.shift() ?? { status: 500, body: 'script exhausted' }
|
|
if (behavior.status !== undefined && behavior.status !== 200) {
|
|
response.writeHead(behavior.status, { 'content-type': 'application/json', ...behavior.headers })
|
|
response.end(behavior.body ?? '{}')
|
|
return
|
|
}
|
|
response.writeHead(200, { 'content-type': 'text/event-stream' })
|
|
let index = 0
|
|
const writeNext = (): void => {
|
|
const event = behavior.events?.[index++]
|
|
if (event === undefined) { response.end(); return }
|
|
response.write(`data: ${event}\n\n`)
|
|
if (behavior.delayMs === undefined) writeNext()
|
|
else setTimeout(writeNext, behavior.delayMs)
|
|
}
|
|
writeNext()
|
|
})
|
|
})
|
|
servers.push(server)
|
|
await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))
|
|
const address = server.address()
|
|
if (address === null || typeof address === 'string') throw new Error('no port')
|
|
return {
|
|
url: `http://127.0.0.1:${address.port}`,
|
|
paths,
|
|
requests,
|
|
headers,
|
|
responseClosed: responseClosed.promise,
|
|
get closedResponses() { return closedResponses },
|
|
}
|
|
}
|
|
|
|
const textEvents = [
|
|
'{"choices":[{"delta":{"role":"assistant","content":""},"index":0,"finish_reason":null}]}',
|
|
'{"choices":[{"delta":{"content":"hello"},"index":0,"finish_reason":null}]}',
|
|
'{"choices":[{"delta":{},"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}',
|
|
'[DONE]',
|
|
]
|
|
|
|
async function harness(baseURL: string, overrides: Record<string, unknown> = {}): Promise<Context> {
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: [{ provider: 'deepseek', apiKey: 'test-key', baseURL, ...overrides }],
|
|
})
|
|
return ctx
|
|
}
|
|
|
|
describe('PiAiAdapter provider routing', () => {
|
|
it('resolves a catalog model dynamically and uses a private endpoint', async () => {
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = await harness(server.url)
|
|
const result = await assemble(ctx, {
|
|
model: 'deepseek-v4-flash',
|
|
messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }],
|
|
})
|
|
expect(result.message.content).toEqual([{ type: 'text', text: 'hello' }])
|
|
expect(result.finish).toEqual({ kind: 'stop' })
|
|
expect(result.usage).toEqual({ inputTokens: 3, outputTokens: 1 })
|
|
expect(server.paths).toEqual(['/chat/completions'])
|
|
})
|
|
|
|
it('merges profile headers with Harness attribution winning', async () => {
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = await harness(server.url, {
|
|
headers: { 'x-company': 'private', 'User-Agent': 'wrong' },
|
|
})
|
|
await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
expect(server.headers[0]?.['x-company']).toBe('private')
|
|
expect(server.headers[0]?.['user-agent']).toBe(userAgent())
|
|
})
|
|
|
|
it('forwards common stream options and profile reasoning', async () => {
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = await harness(server.url, {
|
|
reasoning: 'xhigh',
|
|
cacheRetention: 'none',
|
|
transport: 'sse',
|
|
timeoutMs: 5000,
|
|
websocketConnectTimeoutMs: 3000,
|
|
streamIdleTimeoutMs: 10_000,
|
|
thinkingBudgets: { high: 2048 },
|
|
})
|
|
await assemble(ctx, {
|
|
model: 'deepseek-v4-flash',
|
|
messages: [],
|
|
temperature: 0.2,
|
|
maxTokens: 77,
|
|
sessionId: 'session-for-pi' as never,
|
|
})
|
|
expect(server.requests[0]).toMatchObject({
|
|
model: 'deepseek-v4-flash',
|
|
temperature: 0.2,
|
|
max_completion_tokens: 77,
|
|
thinking: { type: 'enabled' },
|
|
reasoning_effort: 'max',
|
|
})
|
|
})
|
|
|
|
it('preserves omitted profile options when constructing the adapter directly', async () => {
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
ctx.llm.registerAdapter(['deepseek'], new PiAiAdapter({
|
|
profiles: [{ provider: 'deepseek', apiKey: 'test-key', baseURL: server.url }],
|
|
}))
|
|
|
|
const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
|
|
expect(result.message.content).toEqual([{ type: 'text', text: 'hello' }])
|
|
})
|
|
|
|
it('rejects stop sequences rather than silently ignoring them', async () => {
|
|
const server = await mockServer([])
|
|
const ctx = await harness(server.url)
|
|
await expect(assemble(ctx, { model: 'deepseek-v4-flash', messages: [], stop: ['END'] }))
|
|
.rejects.toMatchObject({ code: 'UNSUPPORTED_OPTION' })
|
|
expect(server.requests).toEqual([])
|
|
})
|
|
|
|
it('rejects unknown catalog models before network I/O', async () => {
|
|
const server = await mockServer([])
|
|
const ctx = await harness(server.url)
|
|
await expect(assemble(ctx, { model: 'not-in-the-catalog', messages: [] }))
|
|
.rejects.toMatchObject({ code: 'UNKNOWN_MODEL' })
|
|
expect(server.requests).toEqual([])
|
|
})
|
|
|
|
it('uses the catalog API implementation, including OpenAI Responses', async () => {
|
|
const server = await mockServer([{ status: 401, body: JSON.stringify({ error: { message: 'expected mock failure' } }) }])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: [{ provider: 'openai', apiKey: 'test-key', baseURL: `${server.url}/v1` }],
|
|
})
|
|
const result = await assemble(ctx, { provider: 'openai', model: 'gpt-4.1', messages: [] })
|
|
expect(result.finish.kind).toBe('error')
|
|
expect(server.paths).toEqual(['/v1/responses'])
|
|
})
|
|
|
|
it('forces one wire request for an SDK-retryable provider failure', async () => {
|
|
const server = await mockServer([
|
|
{
|
|
status: 429,
|
|
headers: { 'retry-after-ms': '1' },
|
|
body: JSON.stringify({ error: { message: 'retryable provider failure' } }),
|
|
},
|
|
{ status: 500, body: JSON.stringify({ error: { message: 'hidden SDK retry' } }) },
|
|
{ status: 500, body: JSON.stringify({ error: { message: 'second hidden SDK retry' } }) },
|
|
])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: [{ provider: 'openai', apiKey: 'test-key', baseURL: `${server.url}/v1` }],
|
|
})
|
|
|
|
const result = await assemble(ctx, { provider: 'openai', model: 'gpt-4.1', messages: [] })
|
|
|
|
expect(result.finish).toMatchObject({ kind: 'error' })
|
|
expect(server.paths).toEqual(['/v1/responses'])
|
|
})
|
|
|
|
it('uses OpenAI Responses against an Azure project v1 path with its API key header', async () => {
|
|
const server = await mockServer([{ status: 401, body: JSON.stringify({ error: { message: 'expected mock failure' } }) }])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: [{
|
|
provider: 'openai',
|
|
apiKey: 'test-key',
|
|
baseURL: `${server.url}/api/projects/openai/openai/v1`,
|
|
headers: { 'api-key': 'test-key', Authorization: '' },
|
|
}],
|
|
})
|
|
const result = await assemble(ctx, { provider: 'openai', model: 'gpt-5.5', messages: [] })
|
|
expect(result.finish.kind).toBe('error')
|
|
expect(server.paths).toEqual(['/api/projects/openai/openai/v1/responses'])
|
|
expect(server.headers[0]?.['api-key']).toBe('test-key')
|
|
expect(server.headers[0]?.authorization).toBe('')
|
|
})
|
|
|
|
it.each([
|
|
[401, 'AUTH'],
|
|
[400, 'INVALID_REQUEST'],
|
|
[429, 'RATE_LIMIT'],
|
|
[500, 'SERVER'],
|
|
] as const)('maps HTTP %s failures to %s', async (status, code) => {
|
|
const server = await mockServer([{ status, body: JSON.stringify({ error: { message: `provider ${status}` } }) }])
|
|
const ctx = await harness(server.url)
|
|
const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
expect(result.finish).toMatchObject({ kind: 'error', failure: { code } })
|
|
expect(server.paths).toEqual(['/chat/completions'])
|
|
})
|
|
|
|
it('uses the resolved catalog context window for usage-based overflow detection', async () => {
|
|
const model = getModels('deepseek').find(candidate => candidate.id === 'deepseek-v4-flash')
|
|
if (model === undefined) throw new Error('deepseek-v4-flash missing from pi-ai test catalog')
|
|
const events = [
|
|
'{"choices":[{"delta":{"role":"assistant","content":""},"index":0,"finish_reason":null}]}',
|
|
JSON.stringify({
|
|
choices: [{ delta: {}, index: 0, finish_reason: 'stop' }],
|
|
usage: { prompt_tokens: model.contextWindow + 1, completion_tokens: 0 },
|
|
}),
|
|
'[DONE]',
|
|
]
|
|
const server = await mockServer([{ events }])
|
|
const ctx = await harness(server.url)
|
|
|
|
const result = await assemble(ctx, { model: model.id, messages: [] })
|
|
|
|
expect(result.finish).toEqual({
|
|
kind: 'error',
|
|
failure: {
|
|
message: `pi-ai detected context overflow for model "${model.id}"`,
|
|
code: CONTEXT_WINDOW_EXCEEDED_CODE,
|
|
},
|
|
})
|
|
})
|
|
|
|
it('stops the SDK request when the adapter idle watchdog expires', async () => {
|
|
const server = await mockServer([{ events: textEvents, delayMs: 200 }])
|
|
const ctx = await harness(server.url, { streamIdleTimeoutMs: 20 })
|
|
|
|
await expect(assemble(ctx, { model: 'deepseek-v4-flash', messages: [] }))
|
|
.rejects.toMatchObject({ code: 'TIMEOUT' })
|
|
await Promise.race([
|
|
server.responseClosed,
|
|
new Promise<never>((_resolve, reject) => {
|
|
setTimeout(() => { reject(new Error('SDK request did not close after idle timeout')) }, 100)
|
|
}),
|
|
])
|
|
|
|
expect(server.paths).toEqual(['/chat/completions'])
|
|
expect(server.closedResponses).toBe(1)
|
|
})
|
|
})
|
|
|
|
describe('provider profile lifecycle', () => {
|
|
it('keeps adapter helpers off the package root', () => {
|
|
for (const helper of [
|
|
'resolveProfiles',
|
|
'toPiContext',
|
|
'toPiReplayState',
|
|
'toPiAssistant',
|
|
'mapStopReason',
|
|
'mapUsage',
|
|
'toStreamChunks',
|
|
]) expect(LlmPiAi).not.toHaveProperty(helper)
|
|
})
|
|
|
|
it('registers every profile atomically and unregisters on dispose', async () => {
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
const fiber = await ctx.plugin(LlmPiAi, {
|
|
providers: [{ provider: 'openai' }, { provider: 'anthropic' }],
|
|
})
|
|
expect(ctx.llm.listProviders()).toEqual([
|
|
{ id: 'openai', name: 'openai' },
|
|
{ id: 'anthropic', name: 'anthropic' },
|
|
])
|
|
await fiber.dispose()
|
|
expect(ctx.llm.listProviders()).toEqual([])
|
|
})
|
|
|
|
it('exposes the installed pi-ai model catalog through provider-neutral metadata', async () => {
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, { providers: [{ provider: 'openai' }] })
|
|
const models = await ctx.llm.listModels('openai')
|
|
expect(models.find(model => model.id === 'gpt-4.1')).toEqual({
|
|
provider: 'openai', id: 'gpt-4.1', name: 'GPT-4.1',
|
|
})
|
|
expect(models.every(model => model.provider === 'openai')).toBe(true)
|
|
})
|
|
|
|
it('accepts absent credentials for pi-ai ambient authentication', async () => {
|
|
vi.stubEnv('DEEPSEEK_API_KEY', 'ambient-key')
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = await harness(server.url, { apiKey: undefined })
|
|
await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
expect(server.headers[0]?.authorization).toBe('Bearer ambient-key')
|
|
})
|
|
|
|
it('validates empty, duplicate, unknown, and explicitly blank profiles', () => {
|
|
expect(() => resolveProfiles([])).toThrow(/at least one/)
|
|
expect(() => resolveProfiles([{ provider: '' }])).toThrow(/non-empty/)
|
|
expect(() => resolveProfiles([{ provider: 'not-real' }])).toThrow(/unknown/)
|
|
expect(() => resolveProfiles([{ provider: 'openai' }, { provider: 'openai' }])).toThrow(/duplicate/)
|
|
expect(() => resolveProfiles([{ provider: 'openai', apiKey: '' }])).toThrow(/empty apiKey/)
|
|
expect(() => resolveProfiles([{ provider: 'openai', apiKey: ' ' }])).toThrow(/empty apiKey/)
|
|
expect(() => resolveProfiles([{ provider: 'openai', baseURL: '' }])).toThrow(/empty baseURL/)
|
|
})
|
|
|
|
it.each(['maxRetries', 'maxRetryDelayMs'] as const)(
|
|
'rejects removed profile field %s instead of silently restoring hidden SDK retries',
|
|
async (field) => {
|
|
const legacy = { provider: 'openai', [field]: 2 }
|
|
expect(() => resolveProfiles([legacy as never])).toThrow(/removed.*agent recovery/i)
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await expect(ctx.plugin(LlmPiAi, { providers: [legacy as never] }))
|
|
.rejects.toThrow(/removed.*agent recovery/i)
|
|
},
|
|
)
|
|
|
|
it('rejects invalid stream tunables at plugin load', async () => {
|
|
const invalid = [
|
|
{ timeoutMs: -1 },
|
|
{ websocketConnectTimeoutMs: -1 },
|
|
{ streamIdleTimeoutMs: 0 },
|
|
{ streamIdleTimeoutMs: Number.NaN },
|
|
{ streamIdleTimeoutMs: MAX_TIMER_DELAY_MS + 1 },
|
|
]
|
|
for (const entry of invalid) {
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await expect(ctx.plugin(LlmPiAi, { providers: [{ provider: 'openai', ...entry }] }))
|
|
.rejects.toThrow()
|
|
}
|
|
})
|
|
|
|
it('constructs the adapter directly and rejects routes it does not own', async () => {
|
|
const adapter = new PiAiAdapter({ profiles: [{ provider: 'openai' }] })
|
|
await expect(adapter.listModels('anthropic')).rejects.toMatchObject({ code: 'NO_ADAPTER' })
|
|
await expect((async () => {
|
|
for await (const _chunk of adapter.stream({ provider: 'anthropic', model: 'claude-sonnet-4', messages: [] })) { /* drain */ }
|
|
})()).rejects.toMatchObject({ code: 'NO_ADAPTER' })
|
|
expect(new LlmError('x', 'X')).toBeInstanceOf(Error)
|
|
})
|
|
|
|
it('validates direct-constructor profiles at the embedding boundary', () => {
|
|
expect(() => new PiAiAdapter({
|
|
profiles: [{ provider: 'openai', streamIdleTimeoutMs: 0 }],
|
|
})).toThrow(/streamIdleTimeoutMs.*positive finite/)
|
|
expect(() => new PiAiAdapter({
|
|
profiles: [{ provider: 'openai', streamIdleTimeoutMs: MAX_TIMER_DELAY_MS + 1 }],
|
|
})).toThrow(/streamIdleTimeoutMs.*no greater/)
|
|
})
|
|
})
|
|
|
|
describe('abort wiring', () => {
|
|
it('preserves an unknown pre-dispatch adapter Error exactly', async () => {
|
|
const original = new Error('SDK context conversion exploded')
|
|
const message = Object.defineProperty({}, 'role', {
|
|
get() { throw original },
|
|
})
|
|
const adapter = new PiAiAdapter({ profiles: [{ provider: 'deepseek', apiKey: 'test-key' }] })
|
|
const drain = async (): Promise<void> => {
|
|
for await (const _chunk of adapter.stream({
|
|
provider: 'deepseek',
|
|
model: 'deepseek-v4-flash',
|
|
messages: [message as never],
|
|
})) { /* drain */ }
|
|
}
|
|
|
|
await expect(drain()).rejects.toBe(original)
|
|
})
|
|
|
|
it('lets a concurrent caller abort classify a pre-dispatch adapter failure', async () => {
|
|
const controller = new AbortController()
|
|
const original = new Error('conversion lost its caller')
|
|
const message = Object.defineProperty({}, 'role', {
|
|
get() {
|
|
controller.abort('caller cancelled during conversion')
|
|
throw original
|
|
},
|
|
})
|
|
const adapter = new PiAiAdapter({ profiles: [{ provider: 'deepseek', apiKey: 'test-key' }] })
|
|
const drain = async (): Promise<void> => {
|
|
for await (const _chunk of adapter.stream({
|
|
provider: 'deepseek',
|
|
model: 'deepseek-v4-flash',
|
|
messages: [message as never],
|
|
signal: controller.signal,
|
|
})) { /* drain */ }
|
|
}
|
|
|
|
await expect(drain()).rejects.toMatchObject({ code: 'ABORTED', cause: original })
|
|
})
|
|
|
|
it('resolves catalog endpoints without an override before honoring pre-abort', async () => {
|
|
const adapter = new PiAiAdapter({ profiles: [{ provider: 'deepseek', apiKey: 'test-key' }] })
|
|
const controller = new AbortController()
|
|
controller.abort('already stopped')
|
|
const chunks = []
|
|
for await (const chunk of adapter.stream({
|
|
provider: 'deepseek',
|
|
model: 'deepseek-v4-flash',
|
|
messages: [],
|
|
signal: controller.signal,
|
|
})) chunks.push(chunk)
|
|
expect(chunks.at(-1)).toMatchObject({ type: 'finish', reason: { kind: 'aborted' } })
|
|
})
|
|
|
|
it('honors a pre-aborted caller signal', async () => {
|
|
const server = await mockServer([{ events: textEvents, delayMs: 20 }])
|
|
const ctx = await harness(server.url)
|
|
const controller = new AbortController()
|
|
controller.abort('already stopped')
|
|
const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [], signal: controller.signal })
|
|
expect(result.finish.kind).toBe('aborted')
|
|
})
|
|
|
|
it('forwards an abort that arrives while provider streaming is active', async () => {
|
|
const server = await mockServer([{ events: textEvents, delayMs: 30 }])
|
|
const ctx = await harness(server.url)
|
|
const controller = new AbortController()
|
|
const resultPromise = assemble(ctx, {
|
|
model: 'deepseek-v4-flash', messages: [], signal: controller.signal,
|
|
})
|
|
setTimeout(() => { controller.abort('stopped during stream') }, 10)
|
|
const result = await resultPromise
|
|
expect(result.finish.kind).toBe('aborted')
|
|
})
|
|
|
|
it('aborts upstream when a consumer stops early', async () => {
|
|
const server = await mockServer([{ events: textEvents, delayMs: 30 }])
|
|
const ctx = await harness(server.url)
|
|
for await (const chunk of ctx.llm.stream({ provider: 'deepseek', model: 'deepseek-v4-flash', messages: [] })) {
|
|
if (chunk.type === 'block-start') break
|
|
}
|
|
await new Promise(resolve => setTimeout(resolve, 20))
|
|
expect(server.requests).toHaveLength(1)
|
|
})
|
|
})
|