import { spawnSync } from 'node:child_process' import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import type { Context } from 'cordis' import { AgentId } from '@deepseek-ai/dsh-agent' import { codingHarness, finalText, SYSTEM_PROMPT, waitForIdle } from './harness.ts' /** * The swebench-style smoke test: a real model fixes a real bug in a temp * directory using only the bash tool, and the fix is verified OUTSIDE the * agent by re-running the test script. Key-gated. */ const TEST_FILE = [ "const assert = require('node:assert');", "const { add } = require('./add.js');", 'assert.strictEqual(add(2, 3), 5);', 'assert.strictEqual(add(-1, 1), 0);', "console.log('PASS');", '', ].join('\n') const BUGGY_ADD = [ '// A tiny module with an obvious bug.', 'function add(a, b) {', ' return a - b;', '}', 'module.exports = { add };', '', ].join('\n') let workdir: string | undefined let ctx: Context | undefined afterEach(async () => { // Dispose the harness even on failure/retry: agent-loop teardown stops the // loop and LocalBashExecutor teardown kills anything the model left running. await ctx?.fiber.dispose() ctx = undefined if (workdir !== undefined) await rm(workdir, { recursive: true, force: true }) workdir = undefined }) describe.skipIf(!process.env.DEEPSEEK_API_KEY)('coding task: fix a failing test via bash', () => { it('repairs add.js so node add.test.js passes', async () => { workdir = await mkdtemp(join(tmpdir(), 'dsh-coding-task-')) await writeFile(join(workdir, 'add.js'), BUGGY_ADD) await writeFile(join(workdir, 'add.test.js'), TEST_FILE) // Confirm the fixture actually fails before the agent touches it. const before = spawnSync('node', ['add.test.js'], { cwd: workdir }) expect(before.status).not.toBe(0) ctx = await codingHarness(workdir) const agent = ctx.agentLoop.create(AgentId('e2e-task'), { model: 'deepseek-v4-flash', systemPrompt: SYSTEM_PROMPT, }) agent.send([{ type: 'text', text: 'In the current directory, `node add.test.js` fails because add.js has a bug. ' + 'Fix add.js so the test passes, run `node add.test.js` to verify, and report the result. ' + 'Do not modify add.test.js.', }]) await waitForIdle(ctx, agent) // The agent claims success… const summary = finalText([...agent.session.events]).toLowerCase() expect(summary.length).toBeGreaterThan(0) // …and the world agrees: the test passes when WE run it, and the test // file is byte-identical (an agent that neutered the test instead of // fixing the bug fails here, not just on a keyword probe). const untouchedTest = await readFile(join(workdir, 'add.test.js'), 'utf8') expect(untouchedTest).toBe(TEST_FILE) const after = spawnSync('node', ['add.test.js'], { cwd: workdir, encoding: 'utf8' }) expect(after.stdout).toContain('PASS') expect(after.status).toBe(0) const fixed = await readFile(join(workdir, 'add.js'), 'utf8') expect(fixed).not.toMatch(/a\s*-\s*b/) }, 180_000) })