diff --git a/apps/web/tests/skill-invocation-policy.e2e.ts b/apps/web/tests/skill-invocation-policy.e2e.ts new file mode 100644 index 0000000000..925ff924ca --- /dev/null +++ b/apps/web/tests/skill-invocation-policy.e2e.ts @@ -0,0 +1,115 @@ +// Web e2e scenario: the real host filters skill.list to the model-and-user +// intersection before the browser slash source renders candidates. A real +// chromium connects a fresh workspace seeded with all four policy quadrants; +// no model call is issued, so a stray stream fails loud on the open LLM seam. +import { mkdir, writeFile } from 'node:fs/promises' +import { fileURLToPath } from 'node:url' +import { join } from 'node:path' +import type { Browser, Page } from 'playwright' +import { chromium } from 'playwright' +import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest' +import { + assertFixtureInventory, + captureStableAria, + compareOrRefreshGolden, + launchWebScaffold, + watchConsole, + webSnapshotMode, + type WebScaffold, +} from './scaffold.ts' +import { connectFreshWorkspace, saveFailureShot } from './support.ts' + +const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/skill-invocation-policy', import.meta.url)) +const MENU_EXPECTED = join(SNAPSHOT_DIR, 'menu.expected.md') +const MODE = webSnapshotMode() + +interface SeedSkill { + name: string + description: string + frontmatter: string +} + +const SKILLS: readonly SeedSkill[] = [ + { + name: 'policy-shared', + description: 'Available to both model and user invocation', + frontmatter: '', + }, + { + name: 'policy-model-only', + description: 'Available only to model invocation', + frontmatter: 'user-invocable: false\n', + }, + { + name: 'policy-user-only', + description: 'Available only to user invocation', + frontmatter: 'disable-model-invocation: true\n', + }, + { + name: 'policy-trusted-only', + description: 'Available only to trusted internal callers', + frontmatter: 'disable-model-invocation: true\nuser-invocable: false\n', + }, +] + +async function seedSkills(workspaceCwd: string): Promise { + for (const skill of SKILLS) { + const directory = join(workspaceCwd, 'workspace', '.agents', 'skills', skill.name) + await mkdir(directory, { recursive: true }) + const policyLines = skill.frontmatter === '' ? [] : skill.frontmatter.trimEnd().split('\n') + await writeFile(join(directory, 'SKILL.md'), [ + '---', + `name: ${skill.name}`, + `description: ${skill.description}`, + ...policyLines, + '---', + '', + `# ${skill.name}`, + '', + ].join('\n')) + } +} + +describe('web e2e: skill invocation policy through the real host', () => { + let scaffold: WebScaffold + let browser: Browser + let page: Page + let tripwire: ReturnType + + beforeAll(async () => { + scaffold = await launchWebScaffold({}) + await seedSkills(scaffold.workspaceCwd) + browser = await chromium.launch() + page = await browser.newPage({ viewport: { width: 1680, height: 1000 } }) + tripwire = watchConsole(page) + await page.goto(scaffold.baseUrl, { waitUntil: 'load' }) + await page.waitForSelector('[class*="frame"]', { timeout: 30_000 }) + await connectFreshWorkspace(page) + }, 120_000) + + afterAll(async () => { + await browser?.close() + await scaffold?.close() + }) + + it('renders only the model-and-user intersection in slash candidates', async () => { + onTestFailed(() => saveFailureShot(page, 'web-e2e-skill-invocation-policy')) + const input = page.locator('textarea').first() + await input.fill('/policy') + const menu = page.getByRole('listbox', { name: 'Trigger suggestions' }) + await expect.poll( + () => menu.getByRole('option', { name: /policy-shared/ }).count(), + { timeout: 10_000 }, + ).toBe(1) + + expect(await menu.getByRole('option', { name: /policy-model-only/ }).count()).toBe(0) + expect(await menu.getByRole('option', { name: /policy-user-only/ }).count()).toBe(0) + expect(await menu.getByRole('option', { name: /policy-trusted-only/ }).count()).toBe(0) + + const snapshot = await captureStableAria(page, '[role="listbox"]', scaffold.workspaceCwd) + await compareOrRefreshGolden(MENU_EXPECTED, snapshot, MODE) + expect(tripwire.pageErrors).toEqual([]) + expect(tripwire.warnings).toEqual([]) + await assertFixtureInventory(SNAPSHOT_DIR, ['menu.expected.md']) + }) +}) diff --git a/apps/web/tests/slash-flow.snapshot.ts b/apps/web/tests/slash-flow.snapshot.ts index 68c0cdbb36..b4a7624e73 100644 --- a/apps/web/tests/slash-flow.snapshot.ts +++ b/apps/web/tests/slash-flow.snapshot.ts @@ -2,7 +2,7 @@ // Assembled keyless snapshot of the slash/input/session convergence under the // agent-parity model: the New Session view state locks the composer until a // Workspace is picked (connectWorkspace materializes the full Session+Agent), -// the '/' menu serves the session's filtered skill and wire command catalogs +// the '/' menu renders the session's skill and wire command catalogs // (sessions are always agent-backed — no draft/materialized split), a skill // pick inserts its reference, a leadingInput command claims, // submits over the wire, and notices its result, and the SAME composer @@ -141,9 +141,10 @@ it('locked view state, skill discovery, /echo claim chain, and blank-on-acceptan ) expect(composer.disabled).toBe(false) - // The built skill plugin prewarms the session-addressed catalog; querying - // it exercises the assembled discovery path and picking inserts the literal - // reference into the resident composer. + // The built skill plugin prewarms the fixture's session-addressed catalog; + // this pins client rendering and picking, while the real-host browser lane + // owns policy filtering. Picking inserts the literal reference into the + // resident composer. await typeComposer(composer, '/fixture') const skillMenu = await screen.findByRole('listbox', { name: 'Trigger suggestions' }) const skillOption = await within(skillMenu).findByRole('option', { name: /fixture-demo/ }) diff --git a/apps/web/tests/snapshots/skill-invocation-policy/menu.expected.md b/apps/web/tests/snapshots/skill-invocation-policy/menu.expected.md new file mode 100644 index 0000000000..c25b68a30e --- /dev/null +++ b/apps/web/tests/snapshots/skill-invocation-policy/menu.expected.md @@ -0,0 +1,2 @@ +- listbox "Trigger suggestions": + - option "policy-shared Available to both model and user invocation" [selected] diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index 276251910c..6bc412bbe9 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -36,7 +36,8 @@ "tests/sidebar-scrollbar.e2e.ts", "tests/code-mode-round.e2e.ts", "tests/cordis-tool-round.e2e.ts", - "tests/message-actions.e2e.ts" + "tests/message-actions.e2e.ts", + "tests/skill-invocation-policy.e2e.ts" ], "references": [ { diff --git a/tsconfig.host.json b/tsconfig.host.json index 51f6ad0fa7..7188bf6dec 100644 --- a/tsconfig.host.json +++ b/tsconfig.host.json @@ -24,6 +24,7 @@ "apps/web/tests/code-mode-round.e2e.ts", "apps/web/tests/cordis-tool-round.e2e.ts", "apps/web/tests/message-actions.e2e.ts", + "apps/web/tests/skill-invocation-policy.e2e.ts", "apps/cli/tests/**/*.ts", "examples/*/src/**/*.ts", "examples/*/start.ts",