diff --git a/.github/workflows/pi-ai-provider-e2e.yml b/.github/workflows/pi-ai-provider-e2e.yml new file mode 100644 index 0000000000..7a273c52f3 --- /dev/null +++ b/.github/workflows/pi-ai-provider-e2e.yml @@ -0,0 +1,77 @@ +name: E2E (pi-ai OpenAI and Anthropic) + +# This suite spends tokens against two external providers and is intentionally +# opt-in. It has no push, pull_request, schedule, or workflow_call trigger. +on: + workflow_dispatch: + inputs: + openai_model: + description: OpenAI model from pi-ai's installed catalog + required: true + default: gpt-5.5 + type: string + anthropic_model: + description: Anthropic model from pi-ai's installed catalog + required: true + default: claude-opus-4-8 + type: string + +permissions: + contents: read + +jobs: + e2e: + runs-on: ubuntu-latest + name: OpenAI Responses + Anthropic Messages + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Enable corepack (pnpm) + run: corepack enable + + - name: Resolve pnpm store path + id: pnpm-store + run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-node-24-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-24-pnpm- + + - name: Install (immutable) + run: pnpm install --frozen-lockfile + + # The tests self-skip locally when a credential is absent. A manually + # dispatched CI run must fail instead of reporting an all-skipped green. + - name: Preflight (require provider API keys) + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_EXTERNAL }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY_EXTERNAL }} + run: | + set -euo pipefail + missing=0 + for name in OPENAI_API_KEY ANTHROPIC_API_KEY; do + if [ -z "${!name:-}" ]; then + echo "::error::${name} is empty. Configure the corresponding *_EXTERNAL repository secret." + missing=1 + fi + done + exit "$missing" + + - name: E2E tests (real OpenAI and Anthropic APIs) + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_EXTERNAL }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY_EXTERNAL }} + DSH_PI_AI_OPENAI_MODEL: ${{ inputs.openai_model }} + DSH_PI_AI_ANTHROPIC_MODEL: ${{ inputs.anthropic_model }} + DSH_E2E_MAX_WORKERS: 2 + run: >- + pnpm exec vitest run --config vitest.e2e.config.ts + packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts diff --git a/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts b/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts new file mode 100644 index 0000000000..7fffba4618 --- /dev/null +++ b/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts @@ -0,0 +1,146 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { Context } from 'cordis' +import LlmService, { CallId } from '@deepseek-ai/dsh-llm' +import type { Message, ToolSchema } from '@deepseek-ai/dsh-llm' +import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai' +import type { PiAiReplayState } from '@deepseek-ai/dsh-llm-pi-ai' +import { assemble, type AssembledResult } from './assemble.ts' + +interface ProviderCase { + provider: 'openai' | 'anthropic' + api: 'openai-responses' | 'anthropic-messages' + model: string + apiKey?: string +} + +const providerCases: ProviderCase[] = [ + { + provider: 'openai', + api: 'openai-responses', + model: process.env.DSH_PI_AI_OPENAI_MODEL ?? 'gpt-5.5', + ...process.env.OPENAI_API_KEY ? { apiKey: process.env.OPENAI_API_KEY } : {}, + }, + { + provider: 'anthropic', + api: 'anthropic-messages', + model: process.env.DSH_PI_AI_ANTHROPIC_MODEL ?? 'claude-opus-4-8', + ...process.env.ANTHROPIC_API_KEY ? { apiKey: process.env.ANTHROPIC_API_KEY } : {}, + }, +] + +const contexts: Context[] = [] + +async function harness(): Promise { + const ctx = new Context() + contexts.push(ctx) + await ctx.plugin(LlmService) + await ctx.plugin(LlmPiAi, { + providers: providerCases.map(profile => ({ + provider: profile.provider, + ...profile.apiKey === undefined ? {} : { apiKey: profile.apiKey }, + })), + }) + return ctx +} + +afterEach(async () => { + await Promise.all(contexts.splice(0).map(ctx => ctx.fiber.dispose())) +}) + +function ask(text: string): Message[] { + return [{ role: 'user', content: [{ type: 'text', text }] }] +} + +function textOf(result: AssembledResult): string { + return result.message.content + .filter(block => block.type === 'text') + .map(block => block.text) + .join('') +} + +function expectNativeReplay(result: AssembledResult, profile: ProviderCase): PiAiReplayState { + const replayState = result.message.provenance?.replayState + expect(replayState).toMatchObject({ + kind: 'pi-ai', + version: 1, + api: profile.api, + provider: profile.provider, + model: profile.model, + }) + return replayState as PiAiReplayState +} + +const lookupTool: ToolSchema = { + name: 'lookup_code', + description: 'Look up the word represented by a short code.', + parameters: { + type: 'object', + properties: { code: { type: 'string', description: 'The code to look up.' } }, + required: ['code'], + }, +} + +for (const profile of providerCases) { + describe.skipIf(profile.apiKey === undefined)( + `llm-pi-ai ${profile.provider} e2e (${profile.api})`, + () => { + it('streams text with usage and native replay metadata', async () => { + const ctx = await harness() + const result = await assemble(ctx, { + provider: profile.provider, + model: profile.model, + messages: ask('Reply with exactly the word: pong'), + maxTokens: 64, + }) + + expect(result.finish.kind).toBe('stop') + expect(textOf(result).toLowerCase()).toContain('pong') + expect(result.usage?.inputTokens).toBeGreaterThan(0) + expect(result.usage?.outputTokens).toBeGreaterThan(0) + expect(expectNativeReplay(result, profile).stopReason).toBe('stop') + }) + + it('round-trips a tool call with provider-native replay metadata', async () => { + const ctx = await harness() + const prompt = ask('Use lookup_code with code "blue". Do not answer without calling the tool.') + const first = await assemble(ctx, { + provider: profile.provider, + model: profile.model, + messages: prompt, + tools: [lookupTool], + maxTokens: 256, + }) + + expect(first.finish.kind).toBe('tool-calls') + const call = first.message.content.find(block => block.type === 'tool-call') + expect(call).toBeDefined() + expect(call!.name).toBe('lookup_code') + expect(JSON.parse(call!.arguments)).toMatchObject({ code: 'blue' }) + expect(expectNativeReplay(first, profile).stopReason).toBe('toolUse') + + const second = await assemble(ctx, { + provider: profile.provider, + model: profile.model, + messages: [ + ...prompt, + first.message, + { + role: 'user', + content: [{ + type: 'tool-result', + toolCallId: CallId(call!.id), + content: [{ type: 'text', text: 'The code blue means ocean.' }], + }], + }, + ], + tools: [lookupTool], + maxTokens: 256, + }) + + expect(second.finish.kind).toBe('stop') + expect(textOf(second).toLowerCase()).toContain('ocean') + expect(expectNativeReplay(second, profile).stopReason).toBe('stop') + }) + }, + ) +} diff --git a/vitest.e2e.config.ts b/vitest.e2e.config.ts index adc8b3f3b2..6a84b13daf 100644 --- a/vitest.e2e.config.ts +++ b/vitest.e2e.config.ts @@ -2,8 +2,9 @@ import tsconfigPaths from 'vite-tsconfig-paths' import { defineConfig } from 'vitest/config' // Real-API suite, separate because it spends tokens. Each test self-skips without -// DEEPSEEK_API_KEY for keyless CI; the credentialed workflow preflights the secret. Values may come -// from the environment or gitignored root `.env`, with optional DEEPSEEK_BASE_URL. +// its provider credential for keyless CI; credentialed workflows preflight the +// secrets they require. Values may come from the environment or gitignored root +// `.env`, with provider-specific endpoint overrides where supported. try { // Node >= 21.7 native; throws when the file does not exist. process.loadEnvFile(new URL('.env', import.meta.url).pathname)