kevin/deepseek-harness
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(skill): snapshot invocation policy surfaces

Browse Source
This commit is contained in:
Tianyi Cui
2026-07-29 21:58:21 +08:00
parent 80ab0aa548
commit 7ff1246842
6 files changed
+146 -9
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/web/tests/slash-flow.snapshot.ts
+20 -3
View File
@@ -2,8 +2,9 @@
// Assembled keyless snapshot of the slash/input/session convergence under the
// agent-parity model: the New Session view state locks the composer until a
// Workspace is picked (connectWorkspace materializes the full Session+Agent),
// the '/' menu serves the session's wire command catalog (sessions are always
// agent-backed — no draft/materialized split), a leadingInput command claims,
// the '/' menu serves the session's filtered skill and wire command catalogs
// (sessions are always agent-backed — no draft/materialized split), a skill
// pick inserts its reference, a leadingInput command claims,
// submits over the wire, and notices its result, and the SAME composer
// textarea then carries the first plain send, whose ACCEPTANCE (not attempt)
// flips blank and surfaces the session in lists. This is the user-visible
@@ -114,7 +115,7 @@ async function typeComposer(composer: HTMLTextAreaElement, value: string): Promi
await waitFor(() => { expect(composer.value).toBe(value) })
}
it('locked view state, connectWorkspace unlock, /echo claim chain, and blank-on-acceptance ride one resident composer', async () => {
it('locked view state, skill discovery, /echo claim chain, and blank-on-acceptance ride one resident composer', async () => {
boot('?fixture=empty')
// View state: no session entity — the composer renders locked; only the
@@ -140,6 +141,18 @@ it('locked view state, connectWorkspace unlock, /echo claim chain, and blank-on-
)
expect(composer.disabled).toBe(false)
// The built skill plugin prewarms the session-addressed catalog; querying
// it exercises the assembled discovery path and picking inserts the literal
// reference into the resident composer.
await typeComposer(composer, '/fixture')
const skillMenu = await screen.findByRole('listbox', { name: 'Trigger suggestions' })
const skillOption = await within(skillMenu).findByRole('option', { name: /fixture-demo/ })
const skillMenuText = visibleText(skillMenu)
fireEvent.mouseDown(skillOption)
await waitFor(() => { expect(composer.value).toBe('/fixture-demo ') })
const pickedSkill = composer.value
await typeComposer(composer, '')
// '/' opens the menu with the session's wire command catalog (the session
// is agent-backed from birth — the catalog is the single-address list).
await typeComposer(composer, '/')
@@ -178,6 +191,8 @@ it('locked view state, connectWorkspace unlock, /echo claim chain, and blank-on-
menuHadEcho: menuText.includes('echo'),
menuHadCompact: menuText.includes('compact'),
composerSurvivedConversion: after === before,
skillMenuHadFixtureDemo: skillMenuText.includes('fixture-demo'),
skillPickInserted: pickedSkill,
sessionListed: visibleText(within(tree).getByText('1 session').closest('[role="treeitem"]')!),
}).toMatchInlineSnapshot(`
{
@@ -185,6 +200,8 @@ it('locked view state, connectWorkspace unlock, /echo claim chain, and blank-on-
"menuHadCompact": true,
"menuHadEcho": true,
"sessionListed": "nova1 session",
"skillMenuHadFixtureDemo": true,
"skillPickInserted": "/fixture-demo ",
}
`)
})
examples/tui-agent/tests/snapshots/skill-invocation-policy/session.jsonl
+1
View File
@@ -0,0 +1 @@
{"type":"session","version":0,"id":"31f63cc0-0198-4ab2-bfde-79a4eb4f1867","createdAt":1783352180000,"cwd":"{{cwd}}","delegationDepth":0}
examples/tui-agent/tests/snapshots/skill-invocation-policy/terminal.expected.txt
+55
View File
@@ -0,0 +1,55 @@
=== skill autocomplete ===
terminal 100x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "DSH TUI snapshot"
cursor hidden column=13 viewportRow=5 bufferRow=5
buffer
0| " DEEPSEEK HARNESS"
style 1-8 fg=bright-magenta bold
style 10-16 bold
1| " Recorded replay: skill-invocation-policy"
style 1-40 dim
2| " main-session"
style 1-12 dim
3| <blank>
4| "/workspace/project deepseek-v4-flash ↑0 ↓0 0% context"
style 0-51 fg=bright-magenta bold
style 54-70 dim
style 73-77 dim
style 80-89 dim
5| " dsh > /skill "
style 1-3 fg=bright-magenta bold
style 5-6 dim
style 13-13 inverse
6| " → skill:user-only-skill (project) — User-only assembled snapshot skill. "
style 7-78 fg=bright-magenta
7-35| <blank>
=== denied exact invocation ===
terminal 100x36 buffer=normal length=36 base=0 viewport=0
lifecycle started=1 stopped=0 progress=inactive
title "DSH TUI snapshot"
cursor hidden column=7 viewportRow=7 bufferRow=7
buffer
0| " DEEPSEEK HARNESS"
style 1-8 fg=bright-magenta bold
style 10-16 bold
1| " Recorded replay: skill-invocation-policy"
style 1-40 dim
2| " main-session"
style 1-12 dim
3| <blank>
4| "Skill \"model-only-skill\" is not available for user invocation. "
style 0-61 fg=yellow
5| <blank>
6| "/workspace/project deepseek-v4-flash ↑0 ↓0 0% context"
style 0-51 fg=bright-magenta bold
style 54-70 dim
style 73-77 dim
style 80-89 dim
7| " dsh > "
style 1-3 fg=bright-magenta bold
style 5-6 dim
style 7-7 inverse
8-35| <blank>
examples/tui-agent/tests/snapshots/skill-invocation-policy/workspace/.agents/skills/model-only-skill/SKILL.md
+8
View File
@@ -0,0 +1,8 @@
---
name: model-only-skill
description: Model-only assembled snapshot skill.
disable-model-invocation: false
user-invocable: false
---
MODEL-ONLY BODY MUST NOT LOAD
examples/tui-agent/tests/snapshots/skill-invocation-policy/workspace/.agents/skills/user-only-skill/SKILL.md
+8
View File
@@ -0,0 +1,8 @@
---
name: user-only-skill
description: User-only assembled snapshot skill.
disable-model-invocation: true
user-invocable: true
---
USER-ONLY BODY
examples/tui-agent/tests/tui.snapshot.ts
+54 -6
View File
@@ -65,6 +65,8 @@ interface Scenario {
* preview + locator while the program value stays whole.
*/
spillMaxInlineBytes?: number
/** Exercise user-facing skill discovery and exact policy denial without opening a model turn. */
skillInvocationPolicy?: boolean
}
const SCENARIOS: Scenario[] = [
@@ -98,6 +100,14 @@ const SCENARIOS: Scenario[] = [
recorded: true,
seedWorkspace: true,
},
{
name: 'skill-invocation-policy',
composition: 'native',
expectedTools: [],
recorded: false,
seedWorkspace: true,
skillInvocationPolicy: true,
},
{
name: 'code-mode',
composition: 'code',
@@ -269,9 +279,10 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
const dir = scenarioDir(scenario)
const fixtureFile = join(dir, 'session.jsonl')
const childFiles = childFixturePaths(scenario)
const fixture = await readFile(fixtureFile, 'utf8')
const prompts = userPrompts(fixture)
expect(prompts.length, `${scenario.name} must carry at least one recorded user prompt`).toBeGreaterThan(0)
const prompts = userPrompts(await readFile(fixtureFile, 'utf8'))
if (scenario.skillInvocationPolicy !== true) {
expect(prompts.length, `${scenario.name} must carry at least one recorded user prompt`).toBeGreaterThan(0)
}
const cwd = await mkdtemp(join(SNAPSHOT_TMP_ROOT, `dsh-tui-snapshot-${scenario.name}-`))
const displayCwd = `/tmp/${basename(cwd)}`
@@ -310,6 +321,40 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
})
await settleTerminal(terminal)
let skillPolicySnapshot: string | undefined
if (scenario.skillInvocationPolicy === true) {
terminal.send('/skill')
await settleTerminal(terminal)
const discovery = normalizeTerminalSnapshot(
await terminal.snapshot({ includeScrollback: true }),
cwd,
displayCwd,
)
expect(discovery).toContain('user-only-skill')
expect(discovery).not.toContain('model-only-skill')
terminal.send('\x03')
await settleTerminal(terminal)
terminal.send('/skill:model-only-skill')
terminal.send('\r')
await settleTerminal(terminal)
const denied = normalizeTerminalSnapshot(
await terminal.snapshot({ includeScrollback: true }),
cwd,
displayCwd,
)
expect(denied).toContain('model-only-skill')
expect(denied).toContain('not available for user invocation.')
expect(denied).not.toContain('MODEL-ONLY BODY MUST NOT LOAD')
skillPolicySnapshot = [
'=== skill autocomplete ===',
discovery,
'',
'=== denied exact invocation ===',
denied,
].join('\n')
}
let remainingPrompts = prompts
if (scenario.enterPlanMode === true) {
const firstPrompt = prompts[0]!
@@ -335,8 +380,10 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
const events: SessionEvent[] = [...agent.session.events]
const firstHeader = events.find(event => event.type === 'request/header')
expect(firstHeader?.type === 'request/header' && firstHeader.data.header.system)
.toContain(FILE_REFERENCE_PROMPT)
if (scenario.skillInvocationPolicy !== true) {
expect(firstHeader?.type === 'request/header' && firstHeader.data.header.system)
.toContain(FILE_REFERENCE_PROMPT)
}
expect(events.filter(event => event.type === 'tool/call').map(event => event.data.name)).toEqual(scenario.expectedTools)
for (const [type, count] of Object.entries(scenario.expectedEventCounts ?? {})) {
expect(events.filter(event => event.type === type), `${scenario.name} must emit ${type}`).toHaveLength(count)
@@ -392,11 +439,12 @@ async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
}
expect(terminal.themeViolations(), `${scenario.name} must remain theme-agnostic`).toEqual([])
const snapshot = normalizeTerminalSnapshot(
const terminalSnapshot = normalizeTerminalSnapshot(
await terminal.snapshot({ includeScrollback: true }),
cwd,
displayCwd,
)
const snapshot = skillPolicySnapshot ?? terminalSnapshot
await handle.dispose()
const children = disposedSessions
.filter(session => session !== agent.session)