kevin/deepseek-harness
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(python): package the minimal runtime closure

Browse Source
This commit is contained in:
Yichen Jiang
2026-08-10 20:55:02 +08:00
parent 62f4da95f5
commit 4481637684
20 changed files with 1008 additions and 579 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/smoke-python-runtime.py
+79 -100
View File
@@ -17,7 +17,7 @@ from pathlib import Path
from typing import TYPE_CHECKING, Callable
if TYPE_CHECKING:
from deepseek_harness import TurnResult
from deepseek_harness import RunResult
EXPECTED_TEXT = "runtime smoke ok"
@@ -25,10 +25,14 @@ CODE_PROMPT = "Use run_code to compute the packaged worker smoke value."
CODE_WORKER_TEXT = "code worker smoke ok"
WORKFLOW_PROMPT = "Use workflow to compute the packaged worker smoke value without agents."
WORKFLOW_WORKER_TEXT = "workflow worker smoke ok"
PERSISTENT_TOOLS_PROMPT = "Exercise the packaged persistent Bash and string-replacement editor."
PERSISTENT_TOOLS_TEXT = "persistent tools smoke ok"
PERSISTENT_EDITOR_PATH_PREFIX = "Editor path: "
PERSISTENT_BASH_COMMAND = (
MINIMAL_PROMPT = "Exercise the packaged minimal agent's persistent Bash and string-replacement editor."
MINIMAL_TEXT = "minimal agent smoke ok"
MINIMAL_EDITOR_PATH_PREFIX = "Editor path: "
MINIMAL_SYSTEM_PROMPT = "You are a helpful software engineer assistant."
MINIMAL_CORDIS = (
Path(__file__).resolve().parent.parent / "examples" / "jsonrpc-agent" / "minimal.cordis.yml"
)
MINIMAL_BASH_COMMAND = (
"counter=$(( ${counter:-0} + 1 )); export counter; "
"printf 'COUNT=%s CWD=%s\\n' \"$counter\" \"$PWD\"; "
"if [ \"$counter\" -eq 1 ]; then cd /tmp; fi"
@@ -103,51 +107,6 @@ CUSTOM_CORDIS = """\
- id: cordis-tool
name: '@deepseek-ai/dsh-tool-cordis'
"""
PERSISTENT_TOOLS_CORDIS = """\
- id: jsonrpc
name: '@deepseek-ai/dsh-jsonrpc'
- id: llm
name: '@deepseek-ai/dsh-llm-deepseek'
config:
apiKey: !!js process.env.DEEPSEEK_API_KEY
baseURL: !!js process.env.DEEPSEEK_BASE_URL
- id: sandbox
name: '@deepseek-ai/dsh-sandbox-local'
- id: sandbox-policy
name: '@deepseek-ai/dsh-sandbox-policy'
config:
mode: danger-full-access
workspaceRoot: !!js process.env.DSH_CWD
- id: pty
name: '@deepseek-ai/dsh-pty'
- id: pty-local
name: '@deepseek-ai/dsh-pty-local'
- id: fs
name: '@deepseek-ai/dsh-fs-local'
config:
cwd: !!js process.env.DSH_CWD
- id: agent-core
name: '@deepseek-ai/dsh-agent-spine-demo'
config:
includeHarnessIdentity: false
persona: 'You are a helpful software engineer assistant.'
workspaceContext: false
skills:
enabled: false
toolBash: false
toolTasks: false
- id: sessions
name: '@deepseek-ai/dsh-session-persistence-jsonl'
config:
root: !!js process.env.DSH_SESSION_ROOT
compression: 'none'
- id: persistent-bash
name: '@deepseek-ai/dsh-tool-bash-persistent'
- id: str-replace-editor
name: '@deepseek-ai/dsh-tool-str-replace-editor'
"""
class MockModelHandler(BaseHTTPRequestHandler):
"""Return deterministic text, worker, and orchestration completions."""
@@ -182,9 +141,9 @@ def completion_chunks(body: dict[str, object]) -> list[dict[str, object]]:
if latest.get("role") == "tool":
call_id, tool_name = latest_tool_call(messages)
tool_text = message_text(latest.get("content"))
persistent = persistent_tool_followup(body, call_id, tool_name, tool_text)
if persistent is not None:
return persistent
minimal = minimal_tool_followup(body, call_id, tool_name, tool_text)
if minimal is not None:
return minimal
advanced = advanced_tool_followup(body, call_id, tool_name, tool_text)
if advanced is not None:
return advanced
@@ -196,16 +155,35 @@ def completion_chunks(body: dict[str, object]) -> list[dict[str, object]]:
return text_chunks(WORKFLOW_WORKER_TEXT)
raise AssertionError(f"unexpected tool follow-up: {tool_name}")
prompt = message_text(latest.get("content"))
if prompt.startswith(f"{PERSISTENT_TOOLS_PROMPT}\n{PERSISTENT_EDITOR_PATH_PREFIX}"):
minimal_prompt = next(
(
message_text(message.get("content"))
for message in reversed(messages)
if isinstance(message, dict)
and message.get("role") == "user"
and message_text(message.get("content")).startswith(
f"{MINIMAL_PROMPT}\n{MINIMAL_EDITOR_PATH_PREFIX}"
)
),
None,
)
if minimal_prompt is not None:
names = advertised_tool_names(body)
if names != {"bash", "str_replace_editor"}:
raise AssertionError(f"persistent tools smoke advertised unexpected tools: {names}")
raise AssertionError(f"minimal agent smoke advertised unexpected tools: {names}")
system_prompts = [
message_text(message.get("content"))
for message in messages
if isinstance(message, dict) and message.get("role") == "system"
]
if system_prompts != [MINIMAL_SYSTEM_PROMPT]:
raise AssertionError(f"minimal agent smoke assembled unexpected system prompts: {system_prompts}")
return tool_call_chunks(
"persistent-bash-1",
"minimal-bash-1",
"bash",
{"command": PERSISTENT_BASH_COMMAND},
{"command": MINIMAL_BASH_COMMAND},
)
prompt = message_text(latest.get("content"))
if prompt == SNAPSHOT_DIRECT_CHILD_PROMPT:
return text_chunks("DIRECT_CHILD_OK")
if prompt == SNAPSHOT_WORKFLOW_CHILD_PROMPT:
@@ -240,24 +218,24 @@ def completion_chunks(body: dict[str, object]) -> list[dict[str, object]]:
return text_chunks(EXPECTED_TEXT)
def persistent_tool_followup(
def minimal_tool_followup(
body: dict[str, object],
call_id: str,
tool_name: str,
tool_text: str,
) -> list[dict[str, object]] | None:
"""Verify packaged PTY persistence, then invoke the packaged editor."""
if not call_id.startswith("persistent-"):
"""Verify the checked-in minimal composition's PTY and editor."""
if not call_id.startswith("minimal-"):
return None
if call_id == "persistent-bash-1" and tool_name == "bash":
if call_id == "minimal-bash-1" and tool_name == "bash":
if "COUNT=1" not in tool_text:
raise AssertionError(f"first persistent bash call lost its output: {tool_text}")
return tool_call_chunks(
"persistent-bash-2",
"minimal-bash-2",
"bash",
{"command": PERSISTENT_BASH_COMMAND},
{"command": MINIMAL_BASH_COMMAND},
)
if call_id == "persistent-bash-2" and tool_name == "bash":
if call_id == "minimal-bash-2" and tool_name == "bash":
if "COUNT=2 CWD=/tmp" not in tool_text:
raise AssertionError(f"persistent bash did not retain state: {tool_text}")
messages = body.get("messages")
@@ -265,18 +243,18 @@ def persistent_tool_followup(
raise AssertionError("persistent editor smoke request has no messages")
editor_path = next(
(
text.split(PERSISTENT_EDITOR_PATH_PREFIX, 1)[1].strip()
text.split(MINIMAL_EDITOR_PATH_PREFIX, 1)[1].strip()
for message in messages
if isinstance(message, dict) and message.get("role") == "user"
for text in [message_text(message.get("content"))]
if PERSISTENT_EDITOR_PATH_PREFIX in text
if MINIMAL_EDITOR_PATH_PREFIX in text
),
None,
)
if editor_path is None:
raise AssertionError("persistent editor smoke prompt has no editor path")
return tool_call_chunks(
"persistent-editor",
"minimal-editor",
"str_replace_editor",
{
"command": "create",
@@ -284,11 +262,11 @@ def persistent_tool_followup(
"file_text": "created by packaged editor\n",
},
)
if call_id == "persistent-editor" and tool_name == "str_replace_editor":
if call_id == "minimal-editor" and tool_name == "str_replace_editor":
if "New file created successfully" not in tool_text:
raise AssertionError(f"packaged editor did not create its file: {tool_text}")
return text_chunks(PERSISTENT_TOOLS_TEXT)
raise AssertionError(f"unexpected persistent-tools follow-up: {call_id} {tool_name}: {tool_text}")
return text_chunks(MINIMAL_TEXT)
raise AssertionError(f"unexpected minimal-agent follow-up: {call_id} {tool_name}: {tool_text}")
def advanced_tool_followup(
@@ -470,14 +448,14 @@ def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--scenario",
choices=("all", "sdk-default", "sdk-custom", "sdk-persistent", "sdk-snapshot", "direct"),
choices=("all", "sdk-default", "sdk-custom", "sdk-minimal", "sdk-snapshot", "direct"),
default="all",
)
parser.add_argument("--exe", type=Path)
parser.add_argument("--update-snapshots", action="store_true")
args = parser.parse_args()
if args.scenario in {"all", "sdk-custom", "sdk-persistent", "sdk-snapshot", "direct"} and args.exe is None:
parser.error("--exe is required for custom, persistent, snapshot, and direct scenarios")
if args.scenario in {"all", "sdk-custom", "sdk-minimal", "sdk-snapshot", "direct"} and args.exe is None:
parser.error("--exe is required for custom, minimal, snapshot, and direct scenarios")
if args.update_snapshots and args.scenario not in {"all", "sdk-snapshot"}:
parser.error("--update-snapshots requires --scenario sdk-snapshot or all")
if args.exe is not None and not args.exe.is_file():
@@ -489,9 +467,9 @@ def main() -> None:
if args.scenario in {"all", "sdk-custom"}:
assert args.exe is not None
smoke_sdk_custom(model.url, args.exe.resolve())
if args.scenario in {"all", "sdk-persistent"}:
if args.scenario in {"all", "sdk-minimal"}:
assert args.exe is not None
smoke_sdk_persistent_tools(model.url, args.exe.resolve())
smoke_sdk_minimal(model.url, args.exe.resolve())
if args.scenario in {"all", "sdk-snapshot"}:
assert args.exe is not None
smoke_sdk_snapshot(model.url, args.exe.resolve(), args.update_snapshots)
@@ -519,7 +497,6 @@ def smoke_sdk_default(base_url: str) -> None:
request_timeout_seconds=60,
) as harness:
result = harness.run("reply with the smoke text", session_id="default-smoke")
assert result.status == "ok", result
assert result.final_response == EXPECTED_TEXT, result.final_response
assert_zstd_session_log(sessions)
@@ -546,46 +523,40 @@ def smoke_sdk_custom(base_url: str, executable: Path) -> None:
text_result = harness.run("reply with the smoke text", session_id="custom-smoke")
code_result = harness.run(CODE_PROMPT, session_id="custom-smoke")
workflow_result = harness.run(WORKFLOW_PROMPT, session_id="custom-smoke")
assert text_result.status == "ok", text_result
assert text_result.final_response == EXPECTED_TEXT, text_result.final_response
assert code_result.status == "ok", code_result
assert code_result.final_response == CODE_WORKER_TEXT, code_result.final_response
assert workflow_result.status == "ok", workflow_result
assert workflow_result.final_response == WORKFLOW_WORKER_TEXT, workflow_result.final_response
assert_session_log(sessions, root, EXPECTED_TEXT, CODE_WORKER_TEXT, WORKFLOW_WORKER_TEXT)
def smoke_sdk_persistent_tools(base_url: str, executable: Path) -> None:
"""Exercise native PTY state and the editor through the packaged executable."""
def smoke_sdk_minimal(base_url: str, executable: Path) -> None:
"""Exercise the checked-in minimal composition through the packaged executable."""
from deepseek_harness import DeepSeekHarness
with tempfile.TemporaryDirectory(prefix="dsh-sdk-persistent-tools-") as temporary:
with tempfile.TemporaryDirectory(prefix="dsh-sdk-minimal-") as temporary:
root = Path(temporary).resolve()
editor_path = root / "created.txt"
prompt = f"{PERSISTENT_TOOLS_PROMPT}\n{PERSISTENT_EDITOR_PATH_PREFIX}{editor_path}"
prompt = f"{MINIMAL_PROMPT}\n{MINIMAL_EDITOR_PATH_PREFIX}{editor_path}"
sessions = root / "sessions"
cordis = root / "cordis.yml"
cordis.write_text(PERSISTENT_TOOLS_CORDIS)
with DeepSeekHarness(
provider="deepseek",
provider="deepseek-official",
model="smoke-model",
cwd=str(root),
session_root=str(sessions),
cordis=str(cordis),
cordis=str(MINIMAL_CORDIS),
runtime_bin=str(executable),
api_key="sk-keyless-smoke",
base_url=base_url,
request_timeout_seconds=60,
) as harness:
result = harness.run(prompt, session_id="persistent-tools-smoke")
result = harness.run(prompt, session_id="minimal-agent-smoke")
assert result.status == "ok", result
event_text = json.dumps(result.events)
if PERSISTENT_TOOLS_TEXT not in event_text:
raise AssertionError(f"packaged tools run emitted no final response: {result.events}")
if MINIMAL_TEXT not in event_text:
raise AssertionError(f"minimal agent run emitted no final response: {result.events}")
if editor_path.read_text() != "created by packaged editor\n":
raise AssertionError(f"packaged editor wrote unexpected content: {editor_path.read_text()!r}")
assert_session_log(sessions, root, PERSISTENT_TOOLS_TEXT, "COUNT=1", "COUNT=2 CWD=/tmp")
assert_session_log(sessions, root, MINIMAL_TEXT, "COUNT=1", "COUNT=2 CWD=/tmp")
def smoke_sdk_snapshot(base_url: str, executable: Path, update_snapshots: bool) -> None:
@@ -610,7 +581,6 @@ def smoke_sdk_snapshot(base_url: str, executable: Path, update_snapshots: bool)
) as harness:
result = harness.run(SNAPSHOT_PROMPT, session_id=SNAPSHOT_SESSION_ID)
assert result.status == "ok", result
assert result.final_response == SNAPSHOT_FINAL_TEXT, result.final_response
methods = [notification.method for notification in result.notifications]
if methods.count("subagent.started") != 2 or methods.count("subagent.finished") != 2:
@@ -657,8 +627,8 @@ def smoke_direct(base_url: str, executable: Path) -> None:
"params": {"sessionId": "direct-smoke", "contentBlocks": [{"type": "text", "text": "reply with the smoke text"}]},
})
messages = peer.read_until(lambda message: message.get("id") == "prompt")
if not any(message.get("method") == "session.finished" and message.get("params", {}).get("status") == "ok" for message in messages):
messages.extend(peer.read_until(lambda message: message.get("method") == "session.finished"))
if not any(is_idle_notification(message) for message in messages):
messages.extend(peer.read_until(is_idle_notification))
event_text = json.dumps(messages)
if EXPECTED_TEXT not in event_text:
raise AssertionError(f"direct runtime emitted no final response: {messages}")
@@ -669,6 +639,16 @@ def smoke_direct(base_url: str, executable: Path) -> None:
assert_session_log(sessions, root, EXPECTED_TEXT)
def is_idle_notification(message: dict[str, object]) -> bool:
"""Return whether a JSON-RPC notification marks a session idle."""
params = message.get("params")
return (
message.get("method") == "session.status"
and isinstance(params, dict)
and params.get("status") == "idle"
)
class RuntimePeer:
def __init__(self, argv: list[str], cwd: Path, environment: dict[str, str]) -> None:
self.process = subprocess.Popen(
@@ -776,7 +756,7 @@ def read_session_logs(sessions: Path) -> dict[str, list[dict[str, object]]]:
return logs
def snapshot_child_ids(result: "TurnResult") -> list[str]:
def snapshot_child_ids(result: "RunResult") -> list[str]:
"""Return the two child session ids in their SDK notification order."""
child_ids: list[str] = []
for notification in result.notifications:
@@ -794,7 +774,7 @@ def snapshot_child_ids(result: "TurnResult") -> list[str]:
def build_snapshot_files(
result: "TurnResult",
result: "RunResult",
logs: dict[str, list[dict[str, object]]],
child_ids: list[str],
cwd: Path,
@@ -809,7 +789,6 @@ def build_snapshot_files(
result_value = {
"session_id": result.session_id,
"status": result.status,
"final_response": result.final_response,
"events": result.events,
"notifications": [
@@ -834,7 +813,7 @@ def build_snapshot_files(
return files
def snapshot_agent_id(result: "TurnResult", child_id: str) -> str:
def snapshot_agent_id(result: "RunResult", child_id: str) -> str:
"""Find the successful subagent id paired with one child session."""
for notification in result.notifications:
if notification.method != "subagent.finished":