test(web): cover user-only skill invocation end to end
The policy scenario now expects the user-only quadrant in the menu with its marker (riding the description — the hint field is claim-state ghost text, which the menu never renders), and a new skill-user-invoke scenario drives /name args through the composer against the real host: the claim lands skill.invoke, the transcript shows the dedicated card with the collapsed <skill_content> body, and a paced replay answers the injected turn deterministically.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
// Web e2e scenario: the real host filters skill.list to the model-and-user
|
||||
// intersection before the browser slash source renders candidates. A real
|
||||
// Web e2e scenario: the real host serves every user-invocable skill to the
|
||||
// browser slash source — user-only (disable-model-invocation) entries appear
|
||||
// with their marker while user-disabled quadrants stay hidden. A real
|
||||
// chromium connects a fresh workspace seeded with all four policy quadrants;
|
||||
// no model call is issued, so a stray stream fails loud on the open LLM seam.
|
||||
import { mkdir, writeFile } from 'node:fs/promises'
|
||||
@@ -92,7 +93,7 @@ describe('web e2e: skill invocation policy through the real host', () => {
|
||||
await scaffold?.close()
|
||||
})
|
||||
|
||||
it('renders only the model-and-user intersection in slash candidates', async () => {
|
||||
it('renders every user-invocable skill and marks the user-only entry', async () => {
|
||||
onTestFailed(() => saveFailureShot(page, 'web-e2e-skill-invocation-policy'))
|
||||
const input = page.locator('textarea').first()
|
||||
await input.fill('/policy')
|
||||
@@ -102,8 +103,10 @@ describe('web e2e: skill invocation policy through the real host', () => {
|
||||
{ timeout: 10_000 },
|
||||
).toBe(1)
|
||||
|
||||
// The user-only quadrant is invocable here — its only entry point — and
|
||||
// wears the user-only marker; both user-disabled quadrants stay hidden.
|
||||
expect(await menu.getByRole('option', { name: /policy-user-only user-only · / }).count()).toBe(1)
|
||||
expect(await menu.getByRole('option', { name: /policy-model-only/ }).count()).toBe(0)
|
||||
expect(await menu.getByRole('option', { name: /policy-user-only/ }).count()).toBe(0)
|
||||
expect(await menu.getByRole('option', { name: /policy-trusted-only/ }).count()).toBe(0)
|
||||
|
||||
const snapshot = await captureStableAria(page, '[role="listbox"]', scaffold.workspaceCwd)
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
// Web e2e scenario: a user invokes a disable-model-invocation skill through
|
||||
// the composer (issue #1470). The entered `/name args` line claims into
|
||||
// skill.invoke: the real host renders the skill body, injects it as a
|
||||
// user-role message carrying the skill-invocation source, and starts a turn
|
||||
// answered by the replay seam. The transcript shows the dedicated invocation
|
||||
// card (chip + args, body collapsed) and the model's reply.
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { join } from 'node:path'
|
||||
import type { Browser, Page } from 'playwright'
|
||||
import { chromium } from 'playwright'
|
||||
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
|
||||
import type { ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay'
|
||||
import {
|
||||
assertFixtureInventory,
|
||||
captureStableAria,
|
||||
compareOrRefreshGolden,
|
||||
launchWebScaffold,
|
||||
watchConsole,
|
||||
webSnapshotMode,
|
||||
type WebScaffold,
|
||||
} from './scaffold.ts'
|
||||
import { connectFreshWorkspace, newEnglishPage, saveFailureShot } from './support.ts'
|
||||
|
||||
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/skill-user-invoke', import.meta.url))
|
||||
const UI_EXPECTED = join(SNAPSHOT_DIR, 'ui.expected.md')
|
||||
const MODE = webSnapshotMode()
|
||||
|
||||
const SKILL_NAME = 'user-invoke-demo'
|
||||
const ARGS_TEXT = 'and confirm the fixture wiring'
|
||||
const REPLY = 'USER_INVOKE_REPLY acknowledged; following the injected skill.'
|
||||
|
||||
async function seedUserOnlySkill(workspaceCwd: string): Promise<void> {
|
||||
const directory = join(workspaceCwd, 'workspace', '.agents', 'skills', SKILL_NAME)
|
||||
await mkdir(directory, { recursive: true })
|
||||
await writeFile(join(directory, 'SKILL.md'), [
|
||||
'---',
|
||||
`name: ${SKILL_NAME}`,
|
||||
'description: Prove user-explicit invocation of a model-hidden skill',
|
||||
'disable-model-invocation: true',
|
||||
'---',
|
||||
'',
|
||||
'Reply with the fixture acknowledgement line.',
|
||||
'',
|
||||
].join('\n'))
|
||||
}
|
||||
|
||||
const REPLAY: ReplayOverrideDoc = [{
|
||||
kind: 'chunks',
|
||||
chunks: [
|
||||
{ type: 'block-start', index: 0, blockType: 'text' },
|
||||
{ type: 'text-delta', index: 0, text: REPLY },
|
||||
{ type: 'block-end', index: 0, block: { type: 'text', text: REPLY } },
|
||||
{ type: 'usage', usage: { inputTokens: 256, outputTokens: 16 } },
|
||||
{ type: 'finish', reason: { kind: 'stop' } },
|
||||
],
|
||||
}]
|
||||
|
||||
describe.skipIf(MODE === 'record')('web e2e: user-explicit skill invocation through the composer', () => {
|
||||
let scaffold: WebScaffold
|
||||
let browser: Browser
|
||||
let page: Page
|
||||
let replayDir: string
|
||||
let tripwire: ReturnType<typeof watchConsole>
|
||||
|
||||
beforeAll(async () => {
|
||||
replayDir = await mkdtemp(join(tmpdir(), 'dsh-skill-user-invoke-replay-'))
|
||||
const replayOverride = join(replayDir, 'replay.override.json')
|
||||
await writeFile(replayOverride, JSON.stringify(REPLAY))
|
||||
scaffold = await launchWebScaffold({
|
||||
replayFixture: join(replayDir, 'override-only.jsonl'),
|
||||
replayOverride,
|
||||
// Paced replay keeps the timing-derived chrome (TTFT / tok/s) present
|
||||
// deterministically; instant playback races it in and out of the golden.
|
||||
paceMs: 10,
|
||||
})
|
||||
await seedUserOnlySkill(scaffold.workspaceCwd)
|
||||
browser = await chromium.launch()
|
||||
page = await newEnglishPage(browser)
|
||||
tripwire = watchConsole(page)
|
||||
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
||||
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
||||
await connectFreshWorkspace(page, scaffold.workspaceCwd)
|
||||
}, 120_000)
|
||||
|
||||
afterAll(async () => {
|
||||
const failures: unknown[] = []
|
||||
await browser?.close().catch((error: unknown) => failures.push(error))
|
||||
await scaffold?.close().catch((error: unknown) => failures.push(error))
|
||||
if (replayDir !== undefined) {
|
||||
await rm(replayDir, { recursive: true, force: true })
|
||||
.catch((error: unknown) => failures.push(error))
|
||||
}
|
||||
if (failures.length === 1) throw failures[0]
|
||||
if (failures.length > 1) throw new AggregateError(failures, 'skill-user-invoke e2e cleanup failed')
|
||||
})
|
||||
|
||||
it('claims /name args into an injection card and a replayed answer', async () => {
|
||||
onTestFailed(() => saveFailureShot(page, 'web-e2e-skill-user-invoke'))
|
||||
const composer = page.locator('textarea:enabled').last()
|
||||
await composer.waitFor({ timeout: 15_000 })
|
||||
|
||||
// The menu lists the user-only skill (its only entry point) before enter.
|
||||
await composer.fill(`/${SKILL_NAME}`)
|
||||
const menu = page.getByRole('listbox', { name: 'Trigger suggestions' })
|
||||
await expect.poll(
|
||||
() => menu.getByRole('option', { name: new RegExp(SKILL_NAME) }).count(),
|
||||
{ timeout: 10_000 },
|
||||
).toBe(1)
|
||||
|
||||
await composer.fill(`/${SKILL_NAME} ${ARGS_TEXT}`)
|
||||
await composer.press('Enter')
|
||||
|
||||
// The injection card presents the gesture from source metadata: chip plus
|
||||
// args, with the rendered <skill_content> collapsed behind a disclosure.
|
||||
const card = page.locator('[data-skill-invocation]')
|
||||
await card.waitFor({ timeout: 15_000 })
|
||||
const chip = card.locator('[data-ref-chip="skill"]')
|
||||
expect(await chip.textContent()).toBe(`/${SKILL_NAME}`)
|
||||
expect(await card.textContent()).toContain(ARGS_TEXT)
|
||||
|
||||
const disclosure = card.locator('details')
|
||||
expect(await disclosure.getAttribute('open')).toBeNull()
|
||||
await card.locator('summary').click()
|
||||
const body = card.locator('pre')
|
||||
await body.waitFor()
|
||||
expect(await body.textContent()).toContain(`<skill_content name="${SKILL_NAME}">`)
|
||||
expect(await body.textContent()).toContain('Reply with the fixture acknowledgement line.')
|
||||
expect(await body.textContent()).toContain(ARGS_TEXT)
|
||||
await card.locator('summary').click()
|
||||
|
||||
// The injection started a turn; the replay seam answers it.
|
||||
await page.getByText('USER_INVOKE_REPLY', { exact: false }).first().waitFor({ timeout: 20_000 })
|
||||
|
||||
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
||||
await compareOrRefreshGolden(UI_EXPECTED, snapshot, MODE)
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
expect(tripwire.warnings).toEqual([])
|
||||
}, 60_000)
|
||||
|
||||
it('keeps its snapshot inventory closed', async () => {
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['ui.expected.md'])
|
||||
})
|
||||
})
|
||||
@@ -1,3 +1,4 @@
|
||||
- listbox "Trigger suggestions":
|
||||
- text: Skills
|
||||
- option "policy-shared Available to both model and user invocation" [selected]
|
||||
- option "policy-user-only user-only · Available only to user invocation"
|
||||
@@ -0,0 +1,31 @@
|
||||
- banner:
|
||||
- navigation "Session hierarchy":
|
||||
- button "workspace" [disabled]
|
||||
- tablist:
|
||||
- tab "Chat" [selected]
|
||||
- tab "Trajectory"
|
||||
- text: /user-invoke-demo and confirm the fixture wiring
|
||||
- group: View injected skill content
|
||||
- text: {{clock}}
|
||||
- button "Copy":
|
||||
- img
|
||||
- button "Context injection @deepseek-ai/dsh-system-prompt":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection @deepseek-ai/dsh-system-prompt
|
||||
- paragraph: USER_INVOKE_REPLY acknowledged; following the injected skill.
|
||||
- button "Copy":
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
- 'button "Access mode, current: Workspace Write"': Workspace Write
|
||||
- button "Select model, current DeepSeek-V4-Flash":
|
||||
- text: DeepSeek-V4-Flash
|
||||
- img
|
||||
- button "0% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 1 steps LLM {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 0% Input 256 tok · Output 16 tok
|
||||
@@ -157,8 +157,9 @@ export function apply(ctx: ClientContext): void {
|
||||
.filter(skill => skill.name.startsWith(query))
|
||||
.map(skill => ({
|
||||
name: skill.name,
|
||||
description: skill.description,
|
||||
...skill.modelInvocable ? {} : { hint: userOnlyHint() },
|
||||
// The user-only marker rides the description (the menu's only
|
||||
// secondary text); `hint` is the claim-state ghost text, not a badge.
|
||||
description: skill.modelInvocable ? skill.description : `${userOnlyHint()} · ${skill.description}`,
|
||||
}))
|
||||
},
|
||||
warm(session) {
|
||||
|
||||
@@ -388,7 +388,7 @@ describe('adjudication', () => {
|
||||
})
|
||||
|
||||
describe('user-only marking', () => {
|
||||
it('carries the user-only hint on candidates the model cannot invoke', async () => {
|
||||
it('prefixes the description of candidates the model cannot invoke', async () => {
|
||||
const rows: SkillRow[] = [
|
||||
{ name: 'shared-skill', description: 'both surfaces', modelInvocable: true },
|
||||
{ name: 'user-only-skill', description: 'user surface only', modelInvocable: false },
|
||||
@@ -397,7 +397,7 @@ describe('user-only marking', () => {
|
||||
const candidates = await source.candidates(proj('s1'), req(''))
|
||||
expect(candidates).toEqual([
|
||||
{ name: 'shared-skill', description: 'both surfaces' },
|
||||
{ name: 'user-only-skill', description: 'user surface only', hint: '仅用户' },
|
||||
{ name: 'user-only-skill', description: '仅用户 · user surface only' },
|
||||
])
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user