791 lines
31 KiB
TypeScript
791 lines
31 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
|
import { Context } from 'cordis'
|
|
import LlmService, { createUserMessage, CONTEXT_WINDOW_EXCEEDED_CODE, LlmError, ReasoningEffortId, userAgent } from '@deepseek-ai/dsh-llm'
|
|
import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai'
|
|
import { PiAiAdapter } from '@deepseek-ai/dsh-llm-pi-ai'
|
|
import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout'
|
|
import { getBuiltinModels } from '@earendil-works/pi-ai/providers/all'
|
|
import { resolveProfiles } from '../src/config.ts'
|
|
import { assemble } from './assemble.ts'
|
|
import { closeMockServers, mockServer, textEvents } from './mock-server.ts'
|
|
|
|
afterEach(async () => {
|
|
vi.unstubAllEnvs()
|
|
await closeMockServers()
|
|
})
|
|
|
|
async function harness(baseURL: string, overrides: Record<string, unknown> = {}): Promise<Context> {
|
|
vi.stubEnv('PI_TEST_KEY', 'test-key')
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: { deepseek: { apiKeyEnv: 'PI_TEST_KEY', baseURL, ...overrides } },
|
|
})
|
|
return ctx
|
|
}
|
|
|
|
/** Direct adapter over the real profile resolver, with a fixed key per call. */
|
|
function adapterOf(
|
|
providers: Record<string, LlmPiAi.PiAiProviderProfile>,
|
|
apiKey: string | undefined = 'test-key',
|
|
): PiAiAdapter {
|
|
return new PiAiAdapter({
|
|
profiles: () => resolveProfiles(providers),
|
|
resolveApiKey: () => Promise.resolve(apiKey),
|
|
})
|
|
}
|
|
|
|
beforeEach(() => {
|
|
// Configuration carries only the reference; these mounts resolve it from
|
|
// the environment, which is the whole credential plane without a seam.
|
|
vi.stubEnv('PI_TEST_KEY', 'test-key')
|
|
})
|
|
|
|
describe('PiAiAdapter provider routing', () => {
|
|
it('resolves a catalog model dynamically and uses a private endpoint', async () => {
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = await harness(server.url)
|
|
const result = await assemble(ctx, {
|
|
model: 'deepseek-v4-flash',
|
|
messages: [createUserMessage({
|
|
content: [{ type: 'text', text: 'hi' }],
|
|
source: { kind: 'plugin', plugin: 'test' },
|
|
})],
|
|
})
|
|
expect(result.message.content).toEqual([{ type: 'text', text: 'hello' }])
|
|
expect(result.finish).toEqual({ kind: 'stop' })
|
|
expect(result.usage).toEqual({ inputTokens: 3, outputTokens: 1 })
|
|
expect(server.paths).toEqual(['/chat/completions'])
|
|
})
|
|
|
|
it('merges profile headers with Harness attribution winning', async () => {
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = await harness(server.url, {
|
|
headers: { 'x-company': 'private', 'User-Agent': 'wrong' },
|
|
})
|
|
await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
expect(server.headers[0]?.['x-company']).toBe('private')
|
|
expect(server.headers[0]?.['user-agent']).toBe(userAgent())
|
|
})
|
|
|
|
it('forwards common stream options and profile reasoning', async () => {
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = await harness(server.url, {
|
|
reasoning: 'max',
|
|
cacheRetention: 'none',
|
|
transport: 'sse',
|
|
timeoutMs: 5000,
|
|
websocketConnectTimeoutMs: 3000,
|
|
streamIdleTimeoutMs: 10_000,
|
|
thinkingBudgets: { high: 2048 },
|
|
})
|
|
await assemble(ctx, {
|
|
model: 'deepseek-v4-flash',
|
|
messages: [],
|
|
temperature: 0.2,
|
|
maxTokens: 77,
|
|
sessionId: 'session-for-pi' as never,
|
|
})
|
|
expect(server.requests[0]).toMatchObject({
|
|
model: 'deepseek-v4-flash',
|
|
temperature: 0.2,
|
|
max_completion_tokens: 77,
|
|
thinking: { type: 'enabled' },
|
|
reasoning_effort: 'max',
|
|
})
|
|
})
|
|
|
|
it('uses a dynamic request effort and reports unsupported efforts before network I/O', async () => {
|
|
const server = await mockServer([{ events: textEvents }, { events: textEvents }])
|
|
const ctx = await harness(server.url, { reasoning: 'max' })
|
|
|
|
await assemble(ctx, {
|
|
model: 'deepseek-v4-flash',
|
|
reasoningEffort: ReasoningEffortId('high'),
|
|
messages: [],
|
|
})
|
|
expect(server.requests[0]).toMatchObject({ reasoning_effort: 'high' })
|
|
|
|
await assemble(ctx, {
|
|
model: 'deepseek-v4-flash',
|
|
reasoningEffort: ReasoningEffortId('off'),
|
|
messages: [],
|
|
})
|
|
expect(server.requests[1]).toMatchObject({ thinking: { type: 'disabled' } })
|
|
expect(server.requests[1]).not.toHaveProperty('reasoning_effort')
|
|
|
|
const unsupported = await assemble(ctx, {
|
|
model: 'deepseek-v4-flash',
|
|
reasoningEffort: ReasoningEffortId('xhigh'),
|
|
messages: [],
|
|
})
|
|
expect(unsupported.finish).toMatchObject({
|
|
kind: 'error',
|
|
failure: { code: 'UNSUPPORTED_REASONING_EFFORT' },
|
|
})
|
|
expect(server.requests).toHaveLength(2)
|
|
})
|
|
|
|
it('preserves omitted profile options when constructing the adapter directly', async () => {
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
ctx.llm.registerAdapter(['deepseek'], adapterOf({
|
|
deepseek: { apiKeyEnv: 'PI_TEST_KEY', baseURL: server.url },
|
|
}))
|
|
|
|
const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
|
|
expect(result.message.content).toEqual([{ type: 'text', text: 'hello' }])
|
|
})
|
|
|
|
it('names a route by its displayName, and by its own key once the profiles drop it', () => {
|
|
const adapter = adapterOf({ 'acme-gateway': {
|
|
displayName: 'Acme Gateway',
|
|
api: 'openai-completions',
|
|
baseURL: 'https://acme.test/v1',
|
|
models: [{ id: 'acme-large' }],
|
|
} })
|
|
expect(adapter.providerInfo('acme-gateway')).toEqual({ id: 'acme-gateway', name: 'Acme Gateway' })
|
|
|
|
// The registry and the profiles can disagree for a moment: a refused
|
|
// registration swap leaves the previous routes serving while resolution
|
|
// has already moved on, so a selector may ask about a route the current
|
|
// profiles no longer describe. It gets the key rather than nothing.
|
|
expect(adapter.providerInfo('departed')).toEqual({ id: 'departed', name: 'departed' })
|
|
})
|
|
|
|
it('reports unsupported stop sequences rather than silently ignoring them', async () => {
|
|
const server = await mockServer([])
|
|
const ctx = await harness(server.url)
|
|
const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [], stop: ['END'] })
|
|
expect(result.finish).toMatchObject({ kind: 'error', failure: { code: 'UNSUPPORTED_OPTION' } })
|
|
expect(server.requests).toEqual([])
|
|
})
|
|
|
|
it('reports unknown catalog models before network I/O', async () => {
|
|
const server = await mockServer([])
|
|
const ctx = await harness(server.url)
|
|
const result = await assemble(ctx, { model: 'not-in-the-catalog', messages: [] })
|
|
expect(result.finish).toMatchObject({ kind: 'error', failure: { code: 'UNKNOWN_MODEL' } })
|
|
expect(server.requests).toEqual([])
|
|
})
|
|
|
|
it('uses the catalog API implementation, including OpenAI Responses', async () => {
|
|
const server = await mockServer([{ status: 401, body: JSON.stringify({ error: { message: 'expected mock failure' } }) }])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: { openai: { apiKeyEnv: 'PI_TEST_KEY', baseURL: `${server.url}/v1` } },
|
|
})
|
|
const result = await assemble(ctx, { provider: 'openai', model: 'gpt-4.1', messages: [] })
|
|
expect(result.finish.kind).toBe('error')
|
|
expect(server.paths).toEqual(['/v1/responses'])
|
|
})
|
|
|
|
it('forces one wire request for an SDK-retryable provider failure', async () => {
|
|
const server = await mockServer([
|
|
{
|
|
status: 429,
|
|
headers: { 'retry-after-ms': '1' },
|
|
body: JSON.stringify({ error: { message: 'retryable provider failure' } }),
|
|
},
|
|
{ status: 500, body: JSON.stringify({ error: { message: 'hidden SDK retry' } }) },
|
|
{ status: 500, body: JSON.stringify({ error: { message: 'second hidden SDK retry' } }) },
|
|
])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: { openai: { apiKeyEnv: 'PI_TEST_KEY', baseURL: `${server.url}/v1` } },
|
|
})
|
|
|
|
const result = await assemble(ctx, { provider: 'openai', model: 'gpt-4.1', messages: [] })
|
|
|
|
expect(result.finish).toMatchObject({ kind: 'error' })
|
|
expect(server.paths).toEqual(['/v1/responses'])
|
|
})
|
|
|
|
it('uses OpenAI Responses against an Azure project v1 path with its API key header', async () => {
|
|
const server = await mockServer([{ status: 401, body: JSON.stringify({ error: { message: 'expected mock failure' } }) }])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: {
|
|
openai: {
|
|
apiKeyEnv: 'PI_TEST_KEY',
|
|
baseURL: `${server.url}/api/projects/openai/openai/v1`,
|
|
headers: { 'api-key': 'test-key', Authorization: '' },
|
|
},
|
|
},
|
|
})
|
|
const result = await assemble(ctx, { provider: 'openai', model: 'gpt-5.5', messages: [] })
|
|
expect(result.finish.kind).toBe('error')
|
|
expect(server.paths).toEqual(['/api/projects/openai/openai/v1/responses'])
|
|
expect(server.headers[0]?.['api-key']).toBe('test-key')
|
|
expect(server.headers[0]?.authorization).toBe('')
|
|
})
|
|
|
|
it.each([
|
|
[401, 'AUTH'],
|
|
[400, 'INVALID_REQUEST'],
|
|
[429, 'RATE_LIMIT'],
|
|
[500, 'SERVER'],
|
|
] as const)('maps HTTP %s failures to %s', async (status, code) => {
|
|
const server = await mockServer([{ status, body: JSON.stringify({ error: { message: `provider ${status}` } }) }])
|
|
const ctx = await harness(server.url)
|
|
const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
expect(result.finish).toMatchObject({ kind: 'error', failure: { code } })
|
|
expect(server.paths).toEqual(['/chat/completions'])
|
|
})
|
|
|
|
it('uses the resolved catalog context window for usage-based overflow detection', async () => {
|
|
const model = getBuiltinModels('deepseek').find(candidate => candidate.id === 'deepseek-v4-flash')
|
|
if (model === undefined) throw new Error('deepseek-v4-flash missing from pi-ai test catalog')
|
|
const events = [
|
|
'{"choices":[{"delta":{"role":"assistant","content":""},"index":0,"finish_reason":null}]}',
|
|
JSON.stringify({
|
|
choices: [{ delta: {}, index: 0, finish_reason: 'stop' }],
|
|
usage: { prompt_tokens: model.contextWindow + 1, completion_tokens: 0 },
|
|
}),
|
|
'[DONE]',
|
|
]
|
|
const server = await mockServer([{ events }])
|
|
const ctx = await harness(server.url)
|
|
|
|
const result = await assemble(ctx, { model: model.id, messages: [] })
|
|
|
|
expect(result.finish).toEqual({
|
|
kind: 'error',
|
|
failure: {
|
|
message: `pi-ai detected context overflow for model "${model.id}"`,
|
|
code: CONTEXT_WINDOW_EXCEEDED_CODE,
|
|
},
|
|
})
|
|
})
|
|
|
|
it('stops the SDK request when the adapter idle watchdog expires', async () => {
|
|
const server = await mockServer([{ events: textEvents, delayMs: 200 }])
|
|
const ctx = await harness(server.url, { streamIdleTimeoutMs: 20 })
|
|
|
|
const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
expect(result.finish).toMatchObject({ kind: 'error', failure: { code: 'TIMEOUT' } })
|
|
await Promise.race([
|
|
server.responseClosed,
|
|
new Promise<never>((_resolve, reject) => {
|
|
setTimeout(() => { reject(new Error('SDK request did not close after idle timeout')) }, 1_000)
|
|
}),
|
|
])
|
|
|
|
expect(server.paths).toEqual(['/chat/completions'])
|
|
expect(server.closedResponses).toBe(1)
|
|
})
|
|
})
|
|
|
|
describe('provider profile lifecycle', () => {
|
|
it('keeps adapter helpers off the package root', () => {
|
|
for (const helper of [
|
|
'resolveProfiles',
|
|
'toPiContext',
|
|
'toPiReplayState',
|
|
'toPiAssistant',
|
|
'mapStopReason',
|
|
'mapUsage',
|
|
'toStreamChunks',
|
|
]) expect(LlmPiAi).not.toHaveProperty(helper)
|
|
})
|
|
|
|
it('registers every profile atomically and unregisters on dispose', async () => {
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
const fiber = await ctx.plugin(LlmPiAi, {
|
|
providers: {
|
|
openai: {
|
|
retryPolicy: {
|
|
mode: 'always',
|
|
backoff: { initialDelayMs: 25, maxDelayMs: 100, jitterRatio: 0.2 },
|
|
},
|
|
},
|
|
anthropic: {},
|
|
},
|
|
})
|
|
expect(ctx.llm.listProviders()).toEqual([
|
|
{ id: 'openai', name: 'openai' },
|
|
{ id: 'anthropic', name: 'anthropic' },
|
|
])
|
|
expect(ctx.llm.providerRetryPolicy('openai')).toEqual({
|
|
mode: 'always',
|
|
initialDelayMs: 25,
|
|
maxDelayMs: 100,
|
|
jitterRatio: 0.2,
|
|
})
|
|
expect(ctx.llm.providerRetryPolicy('anthropic')).toMatchObject({
|
|
mode: 'normal',
|
|
maxRetries: 2,
|
|
})
|
|
await fiber.dispose()
|
|
expect(ctx.llm.listProviders()).toEqual([])
|
|
})
|
|
|
|
it('exposes the installed pi-ai model catalog through provider-neutral metadata', async () => {
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, { providers: { openai: {} } })
|
|
const models = await ctx.llm.listModels('openai')
|
|
expect(models.find(model => model.id === 'gpt-4.1')).toEqual({
|
|
provider: 'openai', id: 'gpt-4.1', name: 'GPT-4.1',
|
|
})
|
|
expect(models.every(model => model.provider === 'openai')).toBe(true)
|
|
const info = await ctx.llm.resolveModelInfo('openai', 'gpt-4.1')
|
|
expect(typeof info.context?.contextWindow).toBe('number')
|
|
})
|
|
|
|
it('exposes pi-ai model thinking levels verbatim without inventing a provider default', async () => {
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: { deepseek: {}, openai: {} },
|
|
})
|
|
|
|
await expect(ctx.llm.resolveModelInfo('deepseek', 'deepseek-v4-flash'))
|
|
.resolves.toMatchObject({
|
|
reasoning: {
|
|
efforts: [
|
|
{ id: ReasoningEffortId('off'), name: 'Off' },
|
|
{ id: ReasoningEffortId('high'), name: 'High' },
|
|
{ id: ReasoningEffortId('max'), name: 'Max' },
|
|
],
|
|
},
|
|
})
|
|
const extended = await ctx.llm.resolveModelInfo('openai', 'gpt-5.6-sol')
|
|
expect(extended.reasoning?.efforts.map(effort => effort.id)).toEqual([
|
|
ReasoningEffortId('off'),
|
|
ReasoningEffortId('low'),
|
|
ReasoningEffortId('medium'),
|
|
ReasoningEffortId('high'),
|
|
ReasoningEffortId('xhigh'),
|
|
ReasoningEffortId('max'),
|
|
])
|
|
// A catalog model without reasoning is the same case as a hand-declared
|
|
// one: pi-ai reports the single level `off`, which translates to omitting
|
|
// the reasoning option — exactly what naming no effort already does. The
|
|
// capability is reported unavailable rather than offering that control.
|
|
expect((await ctx.llm.resolveModelInfo('openai', 'gpt-4.1')).reasoning).toBeUndefined()
|
|
})
|
|
|
|
it('uses a supported profile reasoning value as the model default and rejects an unsupported one', async () => {
|
|
const supported = new Context()
|
|
await supported.plugin(LlmService)
|
|
await supported.plugin(LlmPiAi, {
|
|
providers: { deepseek: { reasoning: 'max' } },
|
|
})
|
|
await expect(supported.llm.resolveModelInfo('deepseek', 'deepseek-v4-flash'))
|
|
.resolves.toMatchObject({ reasoning: { defaultEffort: ReasoningEffortId('max') } })
|
|
|
|
// A profile level this model cannot take DESCRIBES as no default rather
|
|
// than failing: resolveModelInfo builds the model catalog, and a catalog
|
|
// that throws takes its whole provider out of every picker — one mis-set
|
|
// field would hide every model on the route, including the ones that do
|
|
// support the level. The request path below is where it is refused.
|
|
const unsupported = new Context()
|
|
await unsupported.plugin(LlmService)
|
|
await unsupported.plugin(LlmPiAi, {
|
|
providers: { deepseek: { reasoning: 'medium' } },
|
|
})
|
|
const described = await unsupported.llm.resolveModelInfo('deepseek', 'deepseek-v4-flash')
|
|
expect(described.reasoning?.defaultEffort).toBeUndefined()
|
|
expect(described.reasoning?.efforts.length).toBeGreaterThan(0)
|
|
await expect(assemble(unsupported, {
|
|
provider: 'deepseek', model: 'deepseek-v4-flash', messages: [],
|
|
})).resolves.toMatchObject({
|
|
finish: { kind: 'error', failure: { code: 'UNSUPPORTED_REASONING_EFFORT' } },
|
|
})
|
|
|
|
const disabled = new Context()
|
|
await disabled.plugin(LlmService)
|
|
await disabled.plugin(LlmPiAi, {
|
|
providers: { deepseek: { reasoning: 'off' } },
|
|
})
|
|
await expect(disabled.llm.resolveModelInfo('deepseek', 'deepseek-v4-flash'))
|
|
.resolves.toMatchObject({ reasoning: { defaultEffort: ReasoningEffortId('off') } })
|
|
})
|
|
|
|
it('serves declared reasoning efforts to selectors and honours the profile default', async () => {
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: {
|
|
'acme-gateway': {
|
|
apiKeyEnv: 'PI_TEST_KEY',
|
|
api: 'openai-completions',
|
|
baseURL: 'https://acme.test/v1',
|
|
reasoning: 'high',
|
|
models: [{
|
|
id: 'acme-think',
|
|
contextWindow: 65_536,
|
|
maxTokens: 4096,
|
|
reasoningEfforts: { off: null, low: 'low', high: 'high' },
|
|
}],
|
|
},
|
|
},
|
|
})
|
|
|
|
// Declared levels reach the same seam catalog metadata does, so the
|
|
// effort picker works for a model pi-ai has never heard of.
|
|
await expect(ctx.llm.resolveModelInfo('acme-gateway', 'acme-think')).resolves.toMatchObject({
|
|
reasoning: {
|
|
efforts: [
|
|
{ id: ReasoningEffortId('off'), name: 'Off' },
|
|
{ id: ReasoningEffortId('low'), name: 'Low' },
|
|
{ id: ReasoningEffortId('high'), name: 'High' },
|
|
],
|
|
defaultEffort: ReasoningEffortId('high'),
|
|
},
|
|
})
|
|
})
|
|
|
|
it('sends the declared wire spelling and refuses undeclared levels before network I/O', async () => {
|
|
vi.stubEnv('PI_TEST_KEY', 'test-key')
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: {
|
|
'acme-gateway': {
|
|
apiKeyEnv: 'PI_TEST_KEY',
|
|
api: 'openai-completions',
|
|
baseURL: `${server.url}/v1`,
|
|
models: [{
|
|
id: 'acme-think',
|
|
contextWindow: 65_536,
|
|
maxTokens: 4096,
|
|
reasoningEfforts: { off: null, high: 'ultra' },
|
|
}],
|
|
},
|
|
},
|
|
})
|
|
|
|
await assemble(ctx, {
|
|
provider: 'acme-gateway',
|
|
model: 'acme-think',
|
|
reasoningEffort: ReasoningEffortId('high'),
|
|
messages: [],
|
|
})
|
|
// The declared value, not the canonical level name, goes on the wire.
|
|
expect(server.requests[0]).toMatchObject({ reasoning_effort: 'ultra' })
|
|
|
|
const undeclared = await assemble(ctx, {
|
|
provider: 'acme-gateway',
|
|
model: 'acme-think',
|
|
reasoningEffort: ReasoningEffortId('max'),
|
|
messages: [],
|
|
})
|
|
expect(undeclared.finish).toMatchObject({
|
|
kind: 'error',
|
|
failure: { code: 'UNSUPPORTED_REASONING_EFFORT' },
|
|
})
|
|
expect(server.requests).toHaveLength(1)
|
|
})
|
|
|
|
it('dispatches the compat-switched dialect on a declared route', async () => {
|
|
vi.stubEnv('PI_TEST_KEY', 'test-key')
|
|
const server = await mockServer([{ events: textEvents }, { events: textEvents }])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: {
|
|
'acme-gateway': {
|
|
apiKeyEnv: 'PI_TEST_KEY',
|
|
api: 'openai-completions',
|
|
baseURL: `${server.url}/v1`,
|
|
// Without the switch pi-ai guesses the dialect from the endpoint
|
|
// URL, and a private gateway's URL says nothing.
|
|
compat: { thinkingFormat: 'deepseek' },
|
|
models: [{
|
|
id: 'acme-think',
|
|
contextWindow: 65_536,
|
|
maxTokens: 4096,
|
|
reasoningEfforts: { off: null, high: 'high' },
|
|
}],
|
|
},
|
|
},
|
|
})
|
|
const prompt = (effort: string): Promise<unknown> => assemble(ctx, {
|
|
provider: 'acme-gateway',
|
|
model: 'acme-think',
|
|
reasoningEffort: ReasoningEffortId(effort),
|
|
messages: [],
|
|
})
|
|
|
|
await prompt('high')
|
|
expect(server.requests[0]).toMatchObject({ thinking: { type: 'enabled' }, reasoning_effort: 'high' })
|
|
|
|
await prompt('off')
|
|
expect(server.requests[1]).toMatchObject({ thinking: { type: 'disabled' } })
|
|
expect(server.requests[1]).not.toHaveProperty('reasoning_effort')
|
|
})
|
|
|
|
it('sends a declared off value as the effort parameter instead of omitting it', async () => {
|
|
vi.stubEnv('PI_TEST_KEY', 'test-key')
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: {
|
|
'acme-gateway': {
|
|
apiKeyEnv: 'PI_TEST_KEY',
|
|
api: 'openai-completions',
|
|
baseURL: `${server.url}/v1`,
|
|
models: [{
|
|
id: 'acme-think',
|
|
contextWindow: 65_536,
|
|
maxTokens: 4096,
|
|
reasoningEfforts: { off: 'none', high: 'high' },
|
|
}],
|
|
},
|
|
},
|
|
})
|
|
|
|
// The adapter strips a selected Off to "no reasoning option", and pi-ai's
|
|
// dispatch reads thinkingLevelMap.off exactly then — so the declared value
|
|
// still reaches the wire, which is the README's promise for `off: none`.
|
|
await assemble(ctx, {
|
|
provider: 'acme-gateway',
|
|
model: 'acme-think',
|
|
reasoningEffort: ReasoningEffortId('off'),
|
|
messages: [],
|
|
})
|
|
expect(server.requests[0]).toMatchObject({ reasoning_effort: 'none' })
|
|
})
|
|
|
|
it('holds back reasoning_effort when the endpoint cannot take it', async () => {
|
|
vi.stubEnv('PI_TEST_KEY', 'test-key')
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await ctx.plugin(LlmPiAi, {
|
|
providers: {
|
|
'acme-gateway': {
|
|
apiKeyEnv: 'PI_TEST_KEY',
|
|
api: 'openai-completions',
|
|
baseURL: `${server.url}/v1`,
|
|
compat: { supportsReasoningEffort: false },
|
|
models: [{
|
|
id: 'acme-think',
|
|
contextWindow: 65_536,
|
|
maxTokens: 4096,
|
|
reasoningEfforts: { off: null, high: 'high' },
|
|
}],
|
|
},
|
|
},
|
|
})
|
|
|
|
await assemble(ctx, {
|
|
provider: 'acme-gateway',
|
|
model: 'acme-think',
|
|
reasoningEffort: ReasoningEffortId('high'),
|
|
messages: [],
|
|
})
|
|
expect(server.requests[0]).not.toHaveProperty('reasoning_effort')
|
|
})
|
|
|
|
it('accepts absent credentials for pi-ai ambient authentication', async () => {
|
|
vi.stubEnv('DEEPSEEK_API_KEY', 'ambient-key')
|
|
const server = await mockServer([{ events: textEvents }])
|
|
// A profile that names no reference at all is the one case that defers to
|
|
// pi-ai's own provider-native discovery.
|
|
const ctx = await harness(server.url, { apiKeyEnv: undefined })
|
|
await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
expect(server.headers[0]?.authorization).toBe('Bearer ambient-key')
|
|
})
|
|
|
|
it('falls back to the ambient environment for apiKeyEnv without the credentials seam', async () => {
|
|
vi.stubEnv('PI_CUSTOM_REF_KEY', 'custom-ref-key')
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = await harness(server.url, { apiKey: undefined, apiKeyEnv: 'PI_CUSTOM_REF_KEY' })
|
|
await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
expect(server.headers[0]?.authorization).toBe('Bearer custom-ref-key')
|
|
})
|
|
|
|
it('fails a named-but-missing apiKeyEnv instead of using another ambient key', async () => {
|
|
// The exact confusion this guards: the named reference is empty while an
|
|
// unrelated provider key sits in the environment. Deferring to pi-ai's own
|
|
// discovery here would authenticate as another tenant.
|
|
vi.stubEnv('PI_CUSTOM_REF_KEY', '')
|
|
vi.stubEnv('DEEPSEEK_API_KEY', 'ambient-key')
|
|
const server = await mockServer([{ events: textEvents }])
|
|
const ctx = await harness(server.url, { apiKey: undefined, apiKeyEnv: 'PI_CUSTOM_REF_KEY' })
|
|
const first = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
expect(first.finish).toMatchObject({ kind: 'error', failure: { code: 'MISSING_CREDENTIAL' } })
|
|
const second = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] })
|
|
expect(second.finish.kind).toBe('error')
|
|
if (second.finish.kind !== 'error') throw new Error('expected an error finish')
|
|
expect(second.finish.failure.message).toMatch(/provider route "deepseek".*PI_CUSTOM_REF_KEY/s)
|
|
expect(server.requests).toHaveLength(0)
|
|
})
|
|
|
|
it('validates empty, underspecified, legacy-shaped, and explicitly blank profiles', () => {
|
|
// Empty and omitted dicts are the dormant zero-route posture, not errors.
|
|
expect(resolveProfiles({}).size).toBe(0)
|
|
expect(resolveProfiles(undefined).size).toBe(0)
|
|
expect(() => resolveProfiles({ '': {} })).toThrow(/non-empty/)
|
|
// A route the installed catalog does not ship is allowed, but it has no
|
|
// defaults to fall back on: it must describe its own models.
|
|
expect(() => resolveProfiles({ 'not-real': {} })).toThrow(/resolves no models/)
|
|
// The pre-release array shape and its per-profile provider field fail
|
|
// loud with migration directions instead of half-working.
|
|
expect(() => resolveProfiles([{ provider: 'openai' }] as never)).toThrow(/dict keyed by provider/)
|
|
expect(() => resolveProfiles({ openai: { provider: 'openai' } as never })).toThrow(/moved to the providers dict key/)
|
|
expect(() => resolveProfiles({ openai: { baseURL: '' } })).toThrow(/empty baseURL/)
|
|
expect(() => resolveProfiles({ openai: { apiKeyEnv: 'not-a-var!' } })).toThrow(/must match/)
|
|
})
|
|
|
|
it.each(['maxRetries', 'maxRetryDelayMs'] as const)(
|
|
'rejects removed profile field %s instead of silently restoring hidden SDK retries',
|
|
async (field) => {
|
|
const legacy = { [field]: 2 }
|
|
expect(() => resolveProfiles({ openai: legacy })).toThrow(/removed.*agent recovery/i)
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await expect(ctx.plugin(LlmPiAi, { providers: { openai: legacy } }))
|
|
.rejects.toThrow(/removed.*agent recovery/i)
|
|
},
|
|
)
|
|
|
|
it('rejects invalid stream tunables at plugin load', async () => {
|
|
const invalid = [
|
|
{ timeoutMs: -1 },
|
|
{ websocketConnectTimeoutMs: -1 },
|
|
{ streamIdleTimeoutMs: 0 },
|
|
{ streamIdleTimeoutMs: Number.NaN },
|
|
{ streamIdleTimeoutMs: MAX_TIMER_DELAY_MS + 1 },
|
|
]
|
|
for (const entry of invalid) {
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await expect(ctx.plugin(LlmPiAi, { providers: { openai: { ...entry } } }))
|
|
.rejects.toThrow()
|
|
}
|
|
})
|
|
|
|
it('rejects invalid nested retryPolicy at the provider-profile boundary', async () => {
|
|
expect(() => resolveProfiles({
|
|
openai: { retryPolicy: { mode: 'always', backoff: { jitterRatio: -1 } } },
|
|
})).toThrow(/retryPolicy\.backoff\.jitterRatio/)
|
|
|
|
const ctx = new Context()
|
|
await ctx.plugin(LlmService)
|
|
await expect(ctx.plugin(LlmPiAi, {
|
|
providers: { openai: { retryPolicy: { mode: 'normal', maxRetries: -1 } } },
|
|
})).rejects.toThrow(/retryPolicy/)
|
|
expect(ctx.llm.listProviders()).toEqual([])
|
|
})
|
|
|
|
it('constructs the adapter directly and rejects routes it does not own', async () => {
|
|
const adapter = adapterOf({ openai: {} })
|
|
await expect(adapter.listModels('anthropic')).rejects.toMatchObject({ code: 'NO_ADAPTER' })
|
|
await expect(adapter.resolveModel('anthropic', 'claude-sonnet-4'))
|
|
.rejects.toMatchObject({ code: 'NO_ADAPTER' })
|
|
await expect(adapter.resolveModel('openai', 'not-a-catalog-model'))
|
|
.rejects.toMatchObject({ code: 'UNKNOWN_MODEL' })
|
|
await expect((async () => {
|
|
for await (const _chunk of adapter.stream({ provider: 'anthropic', model: 'claude-sonnet-4', messages: [] })) { /* drain */ }
|
|
})()).rejects.toMatchObject({ code: 'NO_ADAPTER' })
|
|
expect(new LlmError('x', 'X')).toBeInstanceOf(Error)
|
|
})
|
|
|
|
it('validates profiles at the shared resolver boundary', () => {
|
|
expect(() => resolveProfiles({
|
|
openai: { streamIdleTimeoutMs: 0 },
|
|
})).toThrow(/streamIdleTimeoutMs.*positive finite/)
|
|
expect(() => resolveProfiles({
|
|
openai: { streamIdleTimeoutMs: MAX_TIMER_DELAY_MS + 1 },
|
|
})).toThrow(/streamIdleTimeoutMs.*no greater/)
|
|
})
|
|
})
|
|
|
|
describe('abort wiring', () => {
|
|
it('preserves an unknown pre-dispatch adapter Error exactly', async () => {
|
|
const original = new Error('SDK context conversion exploded')
|
|
const message = Object.defineProperty({}, 'role', {
|
|
get() { throw original },
|
|
})
|
|
const adapter = adapterOf({ deepseek: {} })
|
|
const drain = async (): Promise<void> => {
|
|
for await (const _chunk of adapter.stream({
|
|
provider: 'deepseek',
|
|
model: 'deepseek-v4-flash',
|
|
messages: [message as never],
|
|
})) { /* drain */ }
|
|
}
|
|
|
|
await expect(drain()).rejects.toBe(original)
|
|
})
|
|
|
|
it('lets a concurrent caller abort classify a pre-dispatch adapter failure', async () => {
|
|
const controller = new AbortController()
|
|
const original = new Error('conversion lost its caller')
|
|
const message = Object.defineProperty({}, 'role', {
|
|
get() {
|
|
controller.abort('caller cancelled during conversion')
|
|
throw original
|
|
},
|
|
})
|
|
const adapter = adapterOf({ deepseek: {} })
|
|
const drain = async (): Promise<void> => {
|
|
for await (const _chunk of adapter.stream({
|
|
provider: 'deepseek',
|
|
model: 'deepseek-v4-flash',
|
|
messages: [message as never],
|
|
signal: controller.signal,
|
|
})) { /* drain */ }
|
|
}
|
|
|
|
await expect(drain()).rejects.toMatchObject({ code: 'ABORTED', cause: original })
|
|
})
|
|
|
|
it('resolves catalog endpoints without an override before honoring pre-abort', async () => {
|
|
const adapter = adapterOf({ deepseek: {} })
|
|
const controller = new AbortController()
|
|
controller.abort('already stopped')
|
|
const chunks = []
|
|
for await (const chunk of adapter.stream({
|
|
provider: 'deepseek',
|
|
model: 'deepseek-v4-flash',
|
|
messages: [],
|
|
signal: controller.signal,
|
|
})) chunks.push(chunk)
|
|
expect(chunks.at(-1)).toMatchObject({ type: 'finish', reason: { kind: 'aborted' } })
|
|
})
|
|
|
|
it('honors a pre-aborted caller signal', async () => {
|
|
const server = await mockServer([{ events: textEvents, delayMs: 20 }])
|
|
const ctx = await harness(server.url)
|
|
const controller = new AbortController()
|
|
controller.abort('already stopped')
|
|
const result = await assemble(ctx, { model: 'deepseek-v4-flash', messages: [], signal: controller.signal })
|
|
expect(result.finish.kind).toBe('aborted')
|
|
})
|
|
|
|
it('forwards an abort that arrives while provider streaming is active', async () => {
|
|
const server = await mockServer([{ events: textEvents, delayMs: 30 }])
|
|
const ctx = await harness(server.url)
|
|
const controller = new AbortController()
|
|
const resultPromise = assemble(ctx, {
|
|
model: 'deepseek-v4-flash', messages: [], signal: controller.signal,
|
|
})
|
|
setTimeout(() => { controller.abort('stopped during stream') }, 10)
|
|
const result = await resultPromise
|
|
expect(result.finish.kind).toBe('aborted')
|
|
})
|
|
|
|
it('aborts upstream when a consumer stops early', async () => {
|
|
const server = await mockServer([{ events: textEvents, delayMs: 30 }])
|
|
const ctx = await harness(server.url)
|
|
for await (const chunk of ctx.llm.stream({ provider: 'deepseek', model: 'deepseek-v4-flash', messages: [] })) {
|
|
if (chunk.type === 'block-start') break
|
|
}
|
|
await new Promise(resolve => setTimeout(resolve, 20))
|
|
expect(server.requests).toHaveLength(1)
|
|
})
|
|
})
|