diff --git a/examples/headless-agent/credentials.cordis.snapshot.yml b/examples/headless-agent/credentials.cordis.snapshot.yml new file mode 100644 index 0000000000..7e85b90df7 --- /dev/null +++ b/examples/headless-agent/credentials.cordis.snapshot.yml @@ -0,0 +1,27 @@ +# Keyless dynamic-configuration composition: the settings and credentials +# providers live under the run cwd, no API key exists anywhere, and the +# deepseek route still registers — so the prompt fails with the actionable +# MISSING_CREDENTIAL guidance this snapshot pins as first-run UX. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - id: llm-deepseek + name: '@deepseek-ai/dsh-llm-deepseek' + disabled: true + - insert: + - id: settings + name: '@deepseek-ai/dsh-settings-local' + config: + dshHome: ./.dsh + debounceMs: 10 + - id: credentials + name: '@deepseek-ai/dsh-credentials-local' + config: + dshHome: ./.dsh + # The endpoint is never dialed: credential resolution fails first. + - id: llm-deepseek-keyless + name: '@deepseek-ai/dsh-llm-deepseek' + config: + baseURL: 'http://127.0.0.1:9' diff --git a/examples/headless-agent/tests/headless.snapshot.ts b/examples/headless-agent/tests/headless.snapshot.ts index 2581d8a042..fba7bf7338 100644 --- a/examples/headless-agent/tests/headless.snapshot.ts +++ b/examples/headless-agent/tests/headless.snapshot.ts @@ -27,6 +27,8 @@ const goalScenarioDir = join(snapshotsDir, 'goal-tools') const goalConfigPath = fileURLToPath(new URL('../goal.cordis.snapshot.yml', import.meta.url)) const retryScenarioDir = join(snapshotsDir, 'provider-retry') const retryConfigPath = fileURLToPath(new URL('../retry.cordis.snapshot.yml', import.meta.url)) +const credentialsScenarioDir = join(snapshotsDir, 'missing-credential') +const credentialsConfigPath = fileURLToPath(new URL('../credentials.cordis.snapshot.yml', import.meta.url)) const ralphScenarioDir = join(snapshotsDir, 'ralph-loop') const ralphConfigPath = fileURLToPath(new URL('../ralph.cordis.snapshot.yml', import.meta.url)) const binScript = fileURLToPath(new URL('../../../packages/examples/cli-demo/src/bin.ts', import.meta.url)) @@ -168,6 +170,37 @@ describe('headless stream-json snapshots', () => { expect(normalized).toBe(await readFile(streamExpected, 'utf8')) }, LOADER_SMOKE_TEST_TIMEOUT_MS) + it('surfaces actionable missing-credential guidance through the one-shot app', async () => { + const streamExpected = join(credentialsScenarioDir, 'stream-json.expected.jsonl') + let runCwd = '' + const result = await runLoaderSmoke({ + label: 'missing-credential headless stream-json snapshot', + tempDirPrefix: 'headless-snapshot-missing-credential-', + binScript, + configPath: credentialsConfigPath, + binArgs: ['--config', credentialsConfigPath, '--output-format', 'stream-json', 'say pong'], + tsconfigPath, + env: { + // First-run posture: no key in the environment, none under ./.dsh. + DEEPSEEK_API_KEY: '', + DEEPSEEK_BASE_URL: '', + NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '), + }, + // The designed failure surface: the one-shot app reports the failed turn. + expectedExitCode: 1, + prepare: (cwd) => { runCwd = cwd }, + }) + + expect(result.stderr).toBe( + 'dsh-cli-demo: turn 1 failed at step 1: llm-deepseek: no API key for provider route "deepseek";' + + ' set the llm-deepseek "apiKey" setting, store DEEPSEEK_API_KEY with the credentials service,' + + ' or export DEEPSEEK_API_KEY\n', + ) + const normalized = normalizeHeadlessStream(result.stdout, runCwd) + if (refreshing) await writeFile(streamExpected, normalized) + expect(normalized).toBe(await readFile(streamExpected, 'utf8')) + }, LOADER_SMOKE_TEST_TIMEOUT_MS) + it('logs the model default and a dynamic next-step reasoning effort', async () => { const result = await runLoaderSmoke({ label: 'reasoning effort headless stream-json snapshot', diff --git a/examples/headless-agent/tests/snapshots/missing-credential/stream-json.expected.jsonl b/examples/headless-agent/tests/snapshots/missing-credential/stream-json.expected.jsonl new file mode 100644 index 0000000000..d7d72f6a86 --- /dev/null +++ b/examples/headless-agent/tests/snapshots/missing-credential/stream-json.expected.jsonl @@ -0,0 +1,8 @@ +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/start","seq":0,"time":0,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"user/message","seq":1,"time":0,"data":{"content":[{"type":"text","text":"say pong"}],"source":{"kind":"user"},"role":"user","id":"{{sessionId}}"},"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"session/title","seq":2,"time":0,"data":{"title":"say pong","messageSeqs":[1],"source":{"kind":"fallback"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":3,"time":0,"data":{"turn":1,"step":1}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"request/header","seq":4,"time":0,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash","reasoningEffort":"high"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":5,"time":0,"data":{"turn":1,"step":1}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/end","seq":6,"time":0,"data":{"turn":1,"reason":{"kind":"error","step":1,"failure":{"message":"llm-deepseek: no API key for provider route \"deepseek\"; set the llm-deepseek \"apiKey\" setting, store DEEPSEEK_API_KEY with the credentials service, or export DEEPSEEK_API_KEY","code":"MISSING_CREDENTIAL"}}}}} +{"type":"result","success":false,"sessionId":"{{sessionId}}","turn":1,"result":"","reason":{"kind":"error","step":1,"failure":{"message":"llm-deepseek: no API key for provider route \"deepseek\"; set the llm-deepseek \"apiKey\" setting, store DEEPSEEK_API_KEY with the credentials service, or export DEEPSEEK_API_KEY","code":"MISSING_CREDENTIAL"}}} diff --git a/examples/package.json b/examples/package.json index 51fc48b8fa..35d9a74e3e 100644 --- a/examples/package.json +++ b/examples/package.json @@ -3,7 +3,7 @@ "private": true, "version": "0.0.1", "type": "module", - "description": "Workspace umbrella for runnable demos and example-owned test compositions: declares their cordis.yml packages so plain Node resolves real exports→lib. Not a build target.", + "description": "Workspace umbrella for runnable demos and example-owned test compositions: declares their cordis.yml packages so plain Node resolves real exports\u2192lib. Not a build target.", "dependencies": { "@cordisjs/plugin-hmr": "workspace:*", "@cordisjs/plugin-include": "workspace:*", @@ -16,6 +16,7 @@ "@deepseek-ai/dsh-code-runtime-worker": "workspace:*", "@deepseek-ai/dsh-compact-basic": "workspace:*", "@deepseek-ai/dsh-compact-tool-result-prune": "workspace:*", + "@deepseek-ai/dsh-credentials-local": "workspace:*", "@deepseek-ai/dsh-fs-local": "workspace:*", "@deepseek-ai/dsh-fs-policy": "workspace:*", "@deepseek-ai/dsh-fs-sandbox": "workspace:^", @@ -32,7 +33,6 @@ "@deepseek-ai/dsh-lsp-local": "workspace:*", "@deepseek-ai/dsh-permission": "workspace:*", "@deepseek-ai/dsh-plan-mode": "workspace:*", - "@deepseek-ai/dsh-subprocess-local": "workspace:*", "@deepseek-ai/dsh-pty": "workspace:*", "@deepseek-ai/dsh-pty-local": "workspace:*", "@deepseek-ai/dsh-repeat-tool-guard": "workspace:*", @@ -44,13 +44,15 @@ "@deepseek-ai/dsh-session-query-sqlite": "workspace:*", "@deepseek-ai/dsh-session-telemetry-otel": "workspace:*", "@deepseek-ai/dsh-session-title-first-message-llm": "workspace:*", + "@deepseek-ai/dsh-settings-local": "workspace:*", "@deepseek-ai/dsh-spill-local": "workspace:*", "@deepseek-ai/dsh-spill-policy": "workspace:*", "@deepseek-ai/dsh-subagent": "workspace:*", "@deepseek-ai/dsh-subagent-acp": "workspace:*", - "@deepseek-ai/dsh-subagent-fork": "workspace:*", "@deepseek-ai/dsh-subagent-dsh-sdk": "workspace:*", + "@deepseek-ai/dsh-subagent-fork": "workspace:*", "@deepseek-ai/dsh-subagent-spawn": "workspace:*", + "@deepseek-ai/dsh-subprocess-local": "workspace:*", "@deepseek-ai/dsh-tasks-local": "workspace:*", "@deepseek-ai/dsh-time-context": "workspace:*", "@deepseek-ai/dsh-timeout-policy": "workspace:*", diff --git a/packages/llm/llm-deepseek/tests/adapter.e2e.ts b/packages/llm/llm-deepseek/tests/adapter.e2e.ts index f6d97031c4..e59af1185d 100644 --- a/packages/llm/llm-deepseek/tests/adapter.e2e.ts +++ b/packages/llm/llm-deepseek/tests/adapter.e2e.ts @@ -1,7 +1,11 @@ -import { afterEach, describe, expect, it } from 'vitest' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' import { Context } from 'cordis' import LlmService, { createUserMessage, CallId, ReasoningEffortId , createMessage } from '@deepseek-ai/dsh-llm' import type { Message, ToolSchema } from '@deepseek-ai/dsh-llm' +import { CredentialsLocal } from '@deepseek-ai/dsh-credentials-local' import * as LlmDeepSeek from '@deepseek-ai/dsh-llm-deepseek' import type { Config } from '@deepseek-ai/dsh-llm-deepseek' import { assemble, type AssembledResult } from './assemble.ts' @@ -53,6 +57,34 @@ const weatherTool: ToolSchema = { } describe.skipIf(!process.env.DEEPSEEK_API_KEY)('llm-deepseek e2e (real API)', () => { + it('serves a real request with the key held only by a credentials-local document', async () => { + const key = process.env.DEEPSEEK_API_KEY + if (key === undefined) throw new Error('e2e ran without DEEPSEEK_API_KEY') + const dir = await mkdtemp(join(tmpdir(), 'dsh-e2e-credentials-')) + try { + await writeFile(join(dir, '.env'), `DEEPSEEK_API_KEY=${key}\n`, { mode: 0o600 }) + // Scrub the ambient variable so only the credential seam can supply the + // key: this request proves the per-request resolution path end to end. + vi.stubEnv('DEEPSEEK_API_KEY', '') + const ctx = new Context() + contexts.push(ctx) + await ctx.plugin(LlmService) + await ctx.plugin(CredentialsLocal, { path: join(dir, '.env'), watch: false }) + await ctx.plugin(LlmDeepSeek, {}) + + const result = await assemble(ctx, { + model: FLASH, + messages: ask('Reply with exactly the word: pong'), + maxTokens: 50, + }) + expect(result.finish.kind).toBe('stop') + expect(textOf(result).toLowerCase()).toContain('pong') + } finally { + vi.unstubAllEnvs() + await rm(dir, { recursive: true, force: true }) + } + }) + it('flash dynamically switches from off to high', async () => { const ctx = await harness(FLASH, { reasoningEffort: 'off' }) const withoutThinking = await assemble(ctx,{ diff --git a/packages/llm/llm-deepseek/tests/loader-composition.spec.ts b/packages/llm/llm-deepseek/tests/loader-composition.spec.ts new file mode 100644 index 0000000000..9ca8c87367 --- /dev/null +++ b/packages/llm/llm-deepseek/tests/loader-composition.spec.ts @@ -0,0 +1,142 @@ +/** + * Real-composition guard for the dynamic-configuration chain: LlmService, + * settings-local, credentials-local, and llm-deepseek boot from a test-only + * cordis.yml through the actual Loader + Include path, external edits of + * settings.yaml and .env hot-publish through their providers, and the very + * next request carries the fresh base URL and credential. The same adapter + * composition without settings or credentials entries keeps entry-config + * behavior — the documented optional-inject fallback. + */ + +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Context } from 'cordis' +import Loader from '@cordisjs/plugin-loader' +import Include from '@cordisjs/plugin-include' +import LlmService from '@deepseek-ai/dsh-llm' +import { credentialRef } from '@deepseek-ai/dsh-credentials' +import CredentialsLocal from '@deepseek-ai/dsh-credentials-local' +import { settingsNamespace } from '@deepseek-ai/dsh-settings' +import SettingsLocal from '@deepseek-ai/dsh-settings-local' +import * as LlmDeepSeek from '@deepseek-ai/dsh-llm-deepseek' +import { assemble } from './assemble.ts' +import { closeMockServers, mockServer, textEvents } from './mock-server.ts' + +const NS = settingsNamespace('llm-deepseek') +const KEY_REF = credentialRef('DEEPSEEK_API_KEY') + +let root: string | undefined +let context: Context | undefined + +afterEach(async () => { + await context?.fiber.dispose() + context = undefined + if (root !== undefined) await rm(root, { recursive: true, force: true }) + root = undefined + await closeMockServers() + vi.unstubAllEnvs() +}) + +async function loadComposition( + options: { withDynamic: boolean; baseURL: string }, +): Promise<{ ctx: Context; settingsPath: string; envPath: string }> { + root = await mkdtemp(join(tmpdir(), 'dsh-llm-composition-')) + const settingsPath = join(root, 'settings.yaml') + const envPath = join(root, '.env') + if (options.withDynamic) { + await writeFile(settingsPath, '# personal settings\n') + await writeFile(envPath, 'DEEPSEEK_API_KEY=boot-key\n') + } + + const configPath = join(root, 'cordis.yml') + await writeFile(configPath, [ + '- id: llm', + " name: 'test-llm-service'", + ...options.withDynamic + ? [ + '- id: settings', + " name: '@deepseek-ai/dsh-settings-local'", + ' config:', + ` path: ${JSON.stringify(settingsPath)}`, + ' debounceMs: 10', + '- id: credentials', + " name: '@deepseek-ai/dsh-credentials-local'", + ' config:', + ` path: ${JSON.stringify(envPath)}`, + ' debounceMs: 10', + ] + : [], + '- id: llm-deepseek', + " name: '@deepseek-ai/dsh-llm-deepseek'", + ' config:', + ` baseURL: ${JSON.stringify(options.baseURL)}`, + ...options.withDynamic ? [] : [' apiKey: entry-key'], + '', + ].join('\n')) + + const ctx = new Context() + context = ctx + ctx.baseUrl = pathToFileURL(root).href + '/' + await ctx.plugin(Loader) + ctx.loader.builtins.include = Include + const modules = new Map([ + ['test-llm-service', LlmService], + ['@deepseek-ai/dsh-settings-local', SettingsLocal], + ['@deepseek-ai/dsh-credentials-local', CredentialsLocal], + ['@deepseek-ai/dsh-llm-deepseek', LlmDeepSeek], + ]) + ctx.loader.internal = { + version: 'v2', + async import(specifier: string) { + if (!modules.has(specifier)) throw new Error(`unexpected Loader import: ${specifier}`) + return modules.get(specifier) + }, + } as unknown as NonNullable + await ctx.loader.create({ + name: 'cordis:include', + config: { path: pathToFileURL(configPath).href }, + }) + await ctx.loader.await() + return { ctx, settingsPath, envPath } +} + +describe('llm-deepseek real dynamic composition', () => { + it('boots from cordis.yml and routes the next request after external settings and .env edits', async () => { + vi.stubEnv('DEEPSEEK_API_KEY', '') + const serverA = await mockServer([{ kind: 'sse', events: textEvents }]) + const serverB = await mockServer([{ kind: 'sse', events: textEvents }]) + const { ctx, settingsPath, envPath } = await loadComposition({ withDynamic: true, baseURL: serverA.url }) + + expect(ctx.get('settings')!.describe().map(entry => entry.ns)).toEqual([NS]) + await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] }) + expect(serverA.headers[0]?.authorization).toBe('Bearer boot-key') + + // External edits, exactly as a user or the web UI would leave them on disk. + await writeFile(settingsPath, `llm-deepseek:\n baseURL: ${serverB.url}\n`) + await vi.waitFor(() => { + expect((ctx.get('settings')!.get(NS) as { baseURL?: string }).baseURL).toBe(serverB.url) + }, { timeout: 5000 }) + await writeFile(envPath, 'DEEPSEEK_API_KEY=rotated-key\n') + await vi.waitFor(async () => { + expect(await ctx.get('credentials')!.resolve(KEY_REF)).toEqual({ value: 'rotated-key', source: 'file' }) + }, { timeout: 5000 }) + + await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] }) + expect(serverA.requests).toHaveLength(1) + expect(serverB.headers[0]?.authorization).toBe('Bearer rotated-key') + }) + + it('boots the same adapter without settings or credentials entries on entry config alone', async () => { + vi.stubEnv('DEEPSEEK_API_KEY', '') + const server = await mockServer([{ kind: 'sse', events: textEvents }]) + const { ctx } = await loadComposition({ withDynamic: false, baseURL: server.url }) + + expect(ctx.get('settings')).toBeUndefined() + expect(ctx.get('credentials')).toBeUndefined() + await assemble(ctx, { model: 'deepseek-v4-flash', messages: [] }) + expect(server.headers[0]?.authorization).toBe('Bearer entry-key') + }) +}) diff --git a/packages/support/loader-smoke/src/index.ts b/packages/support/loader-smoke/src/index.ts index e573684a4e..4ee5cfec7d 100644 --- a/packages/support/loader-smoke/src/index.ts +++ b/packages/support/loader-smoke/src/index.ts @@ -141,6 +141,13 @@ export interface LoaderSmokeOptions { readonly prepare?: (cwd: string) => Promise | void /** Optional world-state assertion run in the isolated cwd before cleanup. */ readonly inspect?: (cwd: string) => Promise | void + /** + * Exact process exit code this smoke expects; defaults to `0`. Scenarios + * pinning a designed failure surface (a one-shot turn ending in an error + * result) declare its nonzero exit here, and a run that exits any other + * way — including succeeding — still fails the smoke. + */ + readonly expectedExitCode?: number } /** Captured output from a Loader smoke that exited successfully. */ @@ -187,8 +194,9 @@ export async function runLoaderSmoke(options: LoaderSmokeOptions): Promise { libBinScript: fixture('fail'), configPath, tsconfigPath, - })).rejects.toThrow('failure fixture exited 7. stdout:\n\nstderr:\nfixture failed') + })).rejects.toThrow('failure fixture exited 7 (expected 0). stdout:\n\nstderr:\nfixture failed') + }) + + it('accepts a declared expected failure exit and rejects any other outcome', async () => { + // A scenario pinning a designed failure surface declares its exit code… + const declared = await runLoaderSmoke({ + label: 'declared failure fixture', + tempDirPrefix: 'loader-smoke-declared-fail-', + binScript: fixture('fail'), + libBinScript: fixture('fail'), + configPath, + tsconfigPath, + expectedExitCode: 7, + }) + expect(declared.stderr).toBe('fixture failed\n') + + // …and a run that succeeds instead still fails the smoke. + await expect(runLoaderSmoke({ + label: 'unexpectedly clean fixture', + tempDirPrefix: 'loader-smoke-clean-', + binScript: fixture('success'), + libBinScript: fixture('success'), + configPath, + tsconfigPath, + expectedExitCode: 7, + })).rejects.toThrow(/exited 0 \(expected 7\)/) }) it('kills a process at its deadline and reports captured output', async () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1f28171d9e..de87482bb1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -448,6 +448,9 @@ importers: '@deepseek-ai/dsh-compact-tool-result-prune': specifier: workspace:* version: link:../packages/compact/compact-tool-result-prune + '@deepseek-ai/dsh-credentials-local': + specifier: workspace:* + version: link:../packages/credentials/credentials-local '@deepseek-ai/dsh-fs-local': specifier: workspace:* version: link:../packages/fs/fs-local @@ -529,6 +532,9 @@ importers: '@deepseek-ai/dsh-session-title-first-message-llm': specifier: workspace:* version: link:../packages/session-title/session-title-first-message-llm + '@deepseek-ai/dsh-settings-local': + specifier: workspace:* + version: link:../packages/settings/settings-local '@deepseek-ai/dsh-spill-local': specifier: workspace:* version: link:../packages/spill/spill-local