ci: add manual pi-ai provider e2e
This commit is contained in:
Yichen Jiang
@@ -0,0 +1,77 @@
|
||||
name: E2E (pi-ai OpenAI and Anthropic)
|
||||
|
||||
# This suite spends tokens against two external providers and is intentionally
|
||||
# opt-in. It has no push, pull_request, schedule, or workflow_call trigger.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
openai_model:
|
||||
description: OpenAI model from pi-ai's installed catalog
|
||||
required: true
|
||||
default: gpt-5.5
|
||||
type: string
|
||||
anthropic_model:
|
||||
description: Anthropic model from pi-ai's installed catalog
|
||||
required: true
|
||||
default: claude-opus-4-8
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
runs-on: ubuntu-latest
|
||||
name: OpenAI Responses + Anthropic Messages
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
- name: Enable corepack (pnpm)
|
||||
run: corepack enable
|
||||
|
||||
- name: Resolve pnpm store path
|
||||
id: pnpm-store
|
||||
run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ${{ steps.pnpm-store.outputs.path }}
|
||||
key: ${{ runner.os }}-node-24-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-node-24-pnpm-
|
||||
|
||||
- name: Install (immutable)
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
# The tests self-skip locally when a credential is absent. A manually
|
||||
# dispatched CI run must fail instead of reporting an all-skipped green.
|
||||
- name: Preflight (require provider API keys)
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_EXTERNAL }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY_EXTERNAL }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
missing=0
|
||||
for name in OPENAI_API_KEY ANTHROPIC_API_KEY; do
|
||||
if [ -z "${!name:-}" ]; then
|
||||
echo "::error::${name} is empty. Configure the corresponding *_EXTERNAL repository secret."
|
||||
missing=1
|
||||
fi
|
||||
done
|
||||
exit "$missing"
|
||||
|
||||
- name: E2E tests (real OpenAI and Anthropic APIs)
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_EXTERNAL }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY_EXTERNAL }}
|
||||
DSH_PI_AI_OPENAI_MODEL: ${{ inputs.openai_model }}
|
||||
DSH_PI_AI_ANTHROPIC_MODEL: ${{ inputs.anthropic_model }}
|
||||
DSH_E2E_MAX_WORKERS: 2
|
||||
run: >-
|
||||
pnpm exec vitest run --config vitest.e2e.config.ts
|
||||
packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts
|
||||
@@ -0,0 +1,146 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import LlmService, { CallId } from '@deepseek-ai/dsh-llm'
|
||||
import type { Message, ToolSchema } from '@deepseek-ai/dsh-llm'
|
||||
import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai'
|
||||
import type { PiAiReplayState } from '@deepseek-ai/dsh-llm-pi-ai'
|
||||
import { assemble, type AssembledResult } from './assemble.ts'
|
||||
|
||||
interface ProviderCase {
|
||||
provider: 'openai' | 'anthropic'
|
||||
api: 'openai-responses' | 'anthropic-messages'
|
||||
model: string
|
||||
apiKey?: string
|
||||
}
|
||||
|
||||
const providerCases: ProviderCase[] = [
|
||||
{
|
||||
provider: 'openai',
|
||||
api: 'openai-responses',
|
||||
model: process.env.DSH_PI_AI_OPENAI_MODEL ?? 'gpt-5.5',
|
||||
...process.env.OPENAI_API_KEY ? { apiKey: process.env.OPENAI_API_KEY } : {},
|
||||
},
|
||||
{
|
||||
provider: 'anthropic',
|
||||
api: 'anthropic-messages',
|
||||
model: process.env.DSH_PI_AI_ANTHROPIC_MODEL ?? 'claude-opus-4-8',
|
||||
...process.env.ANTHROPIC_API_KEY ? { apiKey: process.env.ANTHROPIC_API_KEY } : {},
|
||||
},
|
||||
]
|
||||
|
||||
const contexts: Context[] = []
|
||||
|
||||
async function harness(): Promise<Context> {
|
||||
const ctx = new Context()
|
||||
contexts.push(ctx)
|
||||
await ctx.plugin(LlmService)
|
||||
await ctx.plugin(LlmPiAi, {
|
||||
providers: providerCases.map(profile => ({
|
||||
provider: profile.provider,
|
||||
...profile.apiKey === undefined ? {} : { apiKey: profile.apiKey },
|
||||
})),
|
||||
})
|
||||
return ctx
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(contexts.splice(0).map(ctx => ctx.fiber.dispose()))
|
||||
})
|
||||
|
||||
function ask(text: string): Message[] {
|
||||
return [{ role: 'user', content: [{ type: 'text', text }] }]
|
||||
}
|
||||
|
||||
function textOf(result: AssembledResult): string {
|
||||
return result.message.content
|
||||
.filter(block => block.type === 'text')
|
||||
.map(block => block.text)
|
||||
.join('')
|
||||
}
|
||||
|
||||
function expectNativeReplay(result: AssembledResult, profile: ProviderCase): PiAiReplayState {
|
||||
const replayState = result.message.provenance?.replayState
|
||||
expect(replayState).toMatchObject({
|
||||
kind: 'pi-ai',
|
||||
version: 1,
|
||||
api: profile.api,
|
||||
provider: profile.provider,
|
||||
model: profile.model,
|
||||
})
|
||||
return replayState as PiAiReplayState
|
||||
}
|
||||
|
||||
const lookupTool: ToolSchema = {
|
||||
name: 'lookup_code',
|
||||
description: 'Look up the word represented by a short code.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: { code: { type: 'string', description: 'The code to look up.' } },
|
||||
required: ['code'],
|
||||
},
|
||||
}
|
||||
|
||||
for (const profile of providerCases) {
|
||||
describe.skipIf(profile.apiKey === undefined)(
|
||||
`llm-pi-ai ${profile.provider} e2e (${profile.api})`,
|
||||
() => {
|
||||
it('streams text with usage and native replay metadata', async () => {
|
||||
const ctx = await harness()
|
||||
const result = await assemble(ctx, {
|
||||
provider: profile.provider,
|
||||
model: profile.model,
|
||||
messages: ask('Reply with exactly the word: pong'),
|
||||
maxTokens: 64,
|
||||
})
|
||||
|
||||
expect(result.finish.kind).toBe('stop')
|
||||
expect(textOf(result).toLowerCase()).toContain('pong')
|
||||
expect(result.usage?.inputTokens).toBeGreaterThan(0)
|
||||
expect(result.usage?.outputTokens).toBeGreaterThan(0)
|
||||
expect(expectNativeReplay(result, profile).stopReason).toBe('stop')
|
||||
})
|
||||
|
||||
it('round-trips a tool call with provider-native replay metadata', async () => {
|
||||
const ctx = await harness()
|
||||
const prompt = ask('Use lookup_code with code "blue". Do not answer without calling the tool.')
|
||||
const first = await assemble(ctx, {
|
||||
provider: profile.provider,
|
||||
model: profile.model,
|
||||
messages: prompt,
|
||||
tools: [lookupTool],
|
||||
maxTokens: 256,
|
||||
})
|
||||
|
||||
expect(first.finish.kind).toBe('tool-calls')
|
||||
const call = first.message.content.find(block => block.type === 'tool-call')
|
||||
expect(call).toBeDefined()
|
||||
expect(call!.name).toBe('lookup_code')
|
||||
expect(JSON.parse(call!.arguments)).toMatchObject({ code: 'blue' })
|
||||
expect(expectNativeReplay(first, profile).stopReason).toBe('toolUse')
|
||||
|
||||
const second = await assemble(ctx, {
|
||||
provider: profile.provider,
|
||||
model: profile.model,
|
||||
messages: [
|
||||
...prompt,
|
||||
first.message,
|
||||
{
|
||||
role: 'user',
|
||||
content: [{
|
||||
type: 'tool-result',
|
||||
toolCallId: CallId(call!.id),
|
||||
content: [{ type: 'text', text: 'The code blue means ocean.' }],
|
||||
}],
|
||||
},
|
||||
],
|
||||
tools: [lookupTool],
|
||||
maxTokens: 256,
|
||||
})
|
||||
|
||||
expect(second.finish.kind).toBe('stop')
|
||||
expect(textOf(second).toLowerCase()).toContain('ocean')
|
||||
expect(expectNativeReplay(second, profile).stopReason).toBe('stop')
|
||||
})
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -2,8 +2,9 @@ import tsconfigPaths from 'vite-tsconfig-paths'
|
||||
import { defineConfig } from 'vitest/config'
|
||||
|
||||
// Real-API suite, separate because it spends tokens. Each test self-skips without
|
||||
// DEEPSEEK_API_KEY for keyless CI; the credentialed workflow preflights the secret. Values may come
|
||||
// from the environment or gitignored root `.env`, with optional DEEPSEEK_BASE_URL.
|
||||
// its provider credential for keyless CI; credentialed workflows preflight the
|
||||
// secrets they require. Values may come from the environment or gitignored root
|
||||
// `.env`, with provider-specific endpoint overrides where supported.
|
||||
try {
|
||||
// Node >= 21.7 native; throws when the file does not exist.
|
||||
process.loadEnvFile(new URL('.env', import.meta.url).pathname)
|
||||
|
||||
Reference in New Issue
Block a user