Merge pull request #367 from deepseek-harness/worktree/pi-ai-manual-e2e
ci: add manual pi-ai Azure OpenAI and Anthropic e2e workflow
This commit is contained in:
Tianyi Cui
GitHub
@@ -0,0 +1,78 @@
|
||||
name: E2E (pi-ai Azure OpenAI and Anthropic)
|
||||
|
||||
# This suite spends tokens against two external providers and is intentionally
|
||||
# opt-in. It has no push, pull_request, schedule, or workflow_call trigger.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
azure_openai_model:
|
||||
description: Azure OpenAI model from pi-ai's installed catalog
|
||||
required: true
|
||||
default: gpt-5.5
|
||||
type: string
|
||||
anthropic_model:
|
||||
description: Anthropic model from pi-ai's installed catalog
|
||||
required: true
|
||||
default: claude-opus-4-8
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
e2e:
|
||||
runs-on: ubuntu-latest
|
||||
name: Azure OpenAI Responses + Anthropic Messages
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
- name: Enable corepack (pnpm)
|
||||
run: corepack enable
|
||||
|
||||
- name: Resolve pnpm store path
|
||||
id: pnpm-store
|
||||
run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ${{ steps.pnpm-store.outputs.path }}
|
||||
key: ${{ runner.os }}-node-24-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-node-24-pnpm-
|
||||
|
||||
- name: Install (immutable)
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
# The tests self-skip locally when a credential is absent. A manually
|
||||
# dispatched CI run must fail instead of reporting an all-skipped green.
|
||||
- name: Preflight (require provider API keys)
|
||||
env:
|
||||
AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY_EXTERNAL }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY_EXTERNAL }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
missing=0
|
||||
for name in AZURE_OPENAI_API_KEY ANTHROPIC_API_KEY; do
|
||||
if [ -z "${!name:-}" ]; then
|
||||
echo "::error::${name} is empty. Configure the corresponding *_EXTERNAL repository secret."
|
||||
missing=1
|
||||
fi
|
||||
done
|
||||
exit "$missing"
|
||||
|
||||
- name: E2E tests (real Azure OpenAI and Anthropic APIs)
|
||||
env:
|
||||
AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY_EXTERNAL }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY_EXTERNAL }}
|
||||
DSH_PI_AI_OPENAI_MODEL: ${{ inputs.azure_openai_model }}
|
||||
DSH_PI_AI_OPENAI_BASE_URL: https://openai-routerhub-resource.services.ai.azure.com/api/projects/openai/openai/v1
|
||||
DSH_PI_AI_ANTHROPIC_MODEL: ${{ inputs.anthropic_model }}
|
||||
DSH_E2E_MAX_WORKERS: 2
|
||||
run: >-
|
||||
pnpm exec vitest run --config vitest.e2e.config.ts
|
||||
packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts
|
||||
@@ -168,6 +168,26 @@ describe('PiAiAdapter provider routing', () => {
|
||||
expect(server.paths).toEqual(['/v1/responses'])
|
||||
})
|
||||
|
||||
it('uses OpenAI Responses against an Azure project v1 path with its API key header', async () => {
|
||||
const server = await mockServer([{ status: 401, body: JSON.stringify({ error: { message: 'expected mock failure' } }) }])
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
await ctx.plugin(LlmPiAi, {
|
||||
providers: [{
|
||||
provider: 'openai',
|
||||
apiKey: 'test-key',
|
||||
baseURL: `${server.url}/api/projects/openai/openai/v1`,
|
||||
headers: { 'api-key': 'test-key', Authorization: '' },
|
||||
maxRetries: 0,
|
||||
}],
|
||||
})
|
||||
const result = await assemble(ctx, { provider: 'openai', model: 'gpt-5.5', messages: [] })
|
||||
expect(result.finish.kind).toBe('error')
|
||||
expect(server.paths).toEqual(['/api/projects/openai/openai/v1/responses'])
|
||||
expect(server.headers[0]?.['api-key']).toBe('test-key')
|
||||
expect(server.headers[0]?.authorization).toBe('')
|
||||
})
|
||||
|
||||
it.each([
|
||||
[401, 'AUTH'],
|
||||
[400, 'INVALID_REQUEST'],
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import LlmService, { CallId } from '@deepseek-ai/dsh-llm'
|
||||
import type { Message, ToolSchema } from '@deepseek-ai/dsh-llm'
|
||||
import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai'
|
||||
import type { PiAiReplayState } from '../src/replay.ts'
|
||||
import { assemble, type AssembledResult } from './assemble.ts'
|
||||
|
||||
interface ProviderCase {
|
||||
provider: 'openai' | 'anthropic'
|
||||
api: 'openai-responses' | 'anthropic-messages'
|
||||
model: string
|
||||
apiKey?: string
|
||||
baseURL?: string
|
||||
headers?: Record<string, string>
|
||||
}
|
||||
|
||||
const openAIBaseURL = process.env.DSH_PI_AI_OPENAI_BASE_URL
|
||||
const azureOpenAIKey = process.env.AZURE_OPENAI_API_KEY
|
||||
|
||||
const providerCases: ProviderCase[] = [
|
||||
{
|
||||
provider: 'openai',
|
||||
api: 'openai-responses',
|
||||
model: process.env.DSH_PI_AI_OPENAI_MODEL ?? 'gpt-5.5',
|
||||
...azureOpenAIKey
|
||||
? { apiKey: azureOpenAIKey, headers: { 'api-key': azureOpenAIKey, Authorization: '' } }
|
||||
: {},
|
||||
...openAIBaseURL ? { baseURL: openAIBaseURL } : {},
|
||||
},
|
||||
{
|
||||
provider: 'anthropic',
|
||||
api: 'anthropic-messages',
|
||||
model: process.env.DSH_PI_AI_ANTHROPIC_MODEL ?? 'claude-opus-4-8',
|
||||
...process.env.ANTHROPIC_API_KEY ? { apiKey: process.env.ANTHROPIC_API_KEY } : {},
|
||||
},
|
||||
]
|
||||
|
||||
const contexts: Context[] = []
|
||||
|
||||
async function harness(): Promise<Context> {
|
||||
const ctx = new Context()
|
||||
contexts.push(ctx)
|
||||
await ctx.plugin(LlmService)
|
||||
await ctx.plugin(LlmPiAi, {
|
||||
providers: providerCases.map(profile => ({
|
||||
provider: profile.provider,
|
||||
...profile.apiKey === undefined ? {} : { apiKey: profile.apiKey },
|
||||
...profile.baseURL === undefined ? {} : { baseURL: profile.baseURL },
|
||||
...profile.headers === undefined ? {} : { headers: profile.headers },
|
||||
})),
|
||||
})
|
||||
return ctx
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(contexts.splice(0).map(ctx => ctx.fiber.dispose()))
|
||||
})
|
||||
|
||||
function ask(text: string): Message[] {
|
||||
return [{ role: 'user', content: [{ type: 'text', text }] }]
|
||||
}
|
||||
|
||||
function textOf(result: AssembledResult): string {
|
||||
return result.message.content
|
||||
.filter(block => block.type === 'text')
|
||||
.map(block => block.text)
|
||||
.join('')
|
||||
}
|
||||
|
||||
function expectFinish(result: AssembledResult, expected: 'stop' | 'tool-calls'): void {
|
||||
if (result.finish.kind === 'error') {
|
||||
throw new Error(`provider request failed (${result.finish.code ?? 'unknown'}): ${result.finish.message}`)
|
||||
}
|
||||
expect(result.finish.kind).toBe(expected)
|
||||
}
|
||||
|
||||
function expectNativeReplay(result: AssembledResult, profile: ProviderCase): PiAiReplayState {
|
||||
const replayState = result.message.provenance?.replayState
|
||||
expect(replayState).toMatchObject({
|
||||
kind: 'pi-ai',
|
||||
version: 1,
|
||||
api: profile.api,
|
||||
provider: profile.provider,
|
||||
model: profile.model,
|
||||
})
|
||||
return replayState as PiAiReplayState
|
||||
}
|
||||
|
||||
const lookupTool: ToolSchema = {
|
||||
name: 'lookup_code',
|
||||
description: 'Look up the word represented by a short code.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: { code: { type: 'string', description: 'The code to look up.' } },
|
||||
required: ['code'],
|
||||
},
|
||||
}
|
||||
|
||||
for (const profile of providerCases) {
|
||||
describe.skipIf(profile.apiKey === undefined)(
|
||||
`llm-pi-ai ${profile.provider} e2e (${profile.api})`,
|
||||
() => {
|
||||
it('streams text with usage and native replay metadata', async () => {
|
||||
const ctx = await harness()
|
||||
const result = await assemble(ctx, {
|
||||
provider: profile.provider,
|
||||
model: profile.model,
|
||||
messages: ask('Reply with exactly the word: pong'),
|
||||
maxTokens: 1024,
|
||||
})
|
||||
|
||||
expectFinish(result, 'stop')
|
||||
expect(textOf(result).toLowerCase()).toContain('pong')
|
||||
expect(result.usage?.inputTokens).toBeGreaterThan(0)
|
||||
expect(result.usage?.outputTokens).toBeGreaterThan(0)
|
||||
expect(expectNativeReplay(result, profile).stopReason).toBe('stop')
|
||||
})
|
||||
|
||||
it('round-trips a tool call with provider-native replay metadata', async () => {
|
||||
const ctx = await harness()
|
||||
const prompt = ask('Use lookup_code with code "blue". Do not answer without calling the tool.')
|
||||
const first = await assemble(ctx, {
|
||||
provider: profile.provider,
|
||||
model: profile.model,
|
||||
messages: prompt,
|
||||
tools: [lookupTool],
|
||||
maxTokens: 2048,
|
||||
})
|
||||
|
||||
expectFinish(first, 'tool-calls')
|
||||
const call = first.message.content.find(block => block.type === 'tool-call')
|
||||
expect(call).toBeDefined()
|
||||
expect(call!.name).toBe('lookup_code')
|
||||
expect(JSON.parse(call!.arguments)).toMatchObject({ code: 'blue' })
|
||||
expect(expectNativeReplay(first, profile).stopReason).toBe('toolUse')
|
||||
|
||||
const second = await assemble(ctx, {
|
||||
provider: profile.provider,
|
||||
model: profile.model,
|
||||
messages: [
|
||||
...prompt,
|
||||
first.message,
|
||||
{
|
||||
role: 'user',
|
||||
content: [{
|
||||
type: 'tool-result',
|
||||
toolCallId: CallId(call!.id),
|
||||
content: [{ type: 'text', text: 'The code blue means ocean.' }],
|
||||
}],
|
||||
},
|
||||
],
|
||||
tools: [lookupTool],
|
||||
maxTokens: 2048,
|
||||
})
|
||||
|
||||
expectFinish(second, 'stop')
|
||||
expect(textOf(second).toLowerCase()).toContain('ocean')
|
||||
expect(expectNativeReplay(second, profile).stopReason).toBe('stop')
|
||||
})
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -2,8 +2,9 @@ import tsconfigPaths from 'vite-tsconfig-paths'
|
||||
import { defineConfig } from 'vitest/config'
|
||||
|
||||
// Real-API suite, separate because it spends tokens. Each test self-skips without
|
||||
// DEEPSEEK_API_KEY for keyless CI; the credentialed workflow preflights the secret. Values may come
|
||||
// from the environment or gitignored root `.env`, with optional DEEPSEEK_BASE_URL.
|
||||
// its provider credential for keyless CI; credentialed workflows preflight the
|
||||
// secrets they require. Values may come from the environment or gitignored root
|
||||
// `.env`, with provider-specific endpoint overrides where supported.
|
||||
try {
|
||||
// Node >= 21.7 native; throws when the file does not exist.
|
||||
process.loadEnvFile(new URL('.env', import.meta.url).pathname)
|
||||
|
||||
Reference in New Issue
Block a user