refactor(llm): drop the image content block until a path can honor it

ImageBlock had no production producer and every consumer dropped it:
the deepseek serializer skipped it, the pi-ai converter skipped it as
unrepresentable, the ACP bridge neither advertises image prompt
capability nor forwards image blocks, and compact-basic charged a flat
85-token estimate and rendered an [image] placeholder. A block
constructed today would silently vanish from the wire — the vocabulary
advertised a capability no path honors, the silent-data-loss shape the
defensive patterns warn against. The only constructors were tests
pinning the skip/estimate branches.

Remove ImageBlock and its ContentBlockMap entry (its cache?: CacheHint
field leaves with it; CacheHint itself and the other two cache? fields
are out of scope). compact-basic loses its explicit image estimate and
placeholder arms (the merge-extensible default arms absorb the case);
the deepseek serializer, pi-ai converter, and ACP codec already handled
image in their default arms, so only their image-naming comments
change. The codec's inbound rejection of ACP-protocol image prompt
content stays — that guards wire content a client can send regardless
of our vocabulary.

Tests that constructed harness image blocks to pin the removed branches
are dropped (the 85-token estimate pin) or retargeted onto plugin-added
block types / other non-text blocks, which the surviving default arms
own. Docs, the type-equiv pastes, and the content-block vocabulary
RFC's block list and multimodal-home consequence are updated in the
same change; the RFC moves to implemented/ and the index is
regenerated. A real multimodal feature reintroduces image via
declaration merging together with the adapter mapping, ACP
advertisement, and compaction pricing that honor it.
This commit is contained in:
Tianyi Cui
2026-07-04 17:21:13 +08:00
parent 226a8b5e4c
commit 2745879132
25 changed files with 97 additions and 107 deletions
@@ -811,11 +811,6 @@ describe('BasicCompactService token estimation (char/4 heuristic)', () => {
])).toBe(10)
})
it('estimates image blocks at fixed 85 tokens', () => {
const svc = new BasicCompactService(new Context(), cfg({ auto: false }))
expect(svc.estimateContentTokens([{ type: 'image', url: 'https://example.com/img.png' }])).toBe(85)
})
it('returns 0 for empty content blocks', () => {
const svc = new BasicCompactService(new Context(), cfg({ auto: false }))
expect(svc.estimateContentTokens([])).toBe(0)
@@ -1324,7 +1319,7 @@ describe('BasicCompactService edge cases', () => {
s.append('assistant/message', {
turn: 1, step: 1,
content: [
{ type: 'tool-result', toolCallId: CallId('n1'), content: [{ type: 'image', url: 'https://x/n.png' }] },
{ type: 'tool-result', toolCallId: CallId('n1'), content: [{ type: 'chart', data: 'x' } as unknown as ContentBlock] },
{ type: 'custom-widget', payload: 'x' } as unknown as ContentBlock,
{ type: 'tool-call', id: CallId('b1'), name: 'bash', arguments: '{}' },
],
@@ -1343,7 +1338,7 @@ describe('BasicCompactService edge cases', () => {
const nodes = s.surface.nodes
await compactRegion(svc, s, nodes[0]!.seq, nodes[nodes.length - 1]!.seq, 'm')
const { text } = svc.summarizeCalls[0]!
expect(text).toContain('[tool-result: [image]]') // nested tool-result with content
expect(text).toContain('[tool-result: [chart]]') // nested tool-result with content
expect(text).toContain('[custom-widget]') // unknown block placeholder
expect(text).toContain('Tool result (call b1): [tool-result]') // empty nested → bare placeholder
})
@@ -1510,25 +1505,28 @@ describe('BasicCompactService edge cases', () => {
it('renders non-text blocks as type-tagged placeholders across all message kinds', async () => {
const svc = createTestService()
const s = new Session(SessionId('placeholders'))
// A plugin-added block type (merge-extensible ContentBlockMap) — the
// placeholder path must cover every message kind, not just assistant.
const chart = (id: string): ContentBlock => ({ type: 'chart', data: id } as unknown as ContentBlock)
s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } })
s.append('step/start', { turn: 1, step: 1 })
// user/message with only an image block → '[image]' placeholder.
s.append('user/message', { content: [{ type: 'image', url: 'https://x/y.png' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
// assistant/message with an image block AND the tool-call its tool/result
// answers (so the surface is tool-pairing balanced) → '[image]' placeholder.
// user/message with only a plugin-added block → '[chart]' placeholder.
s.append('user/message', { content: [chart('y')], source: { kind: 'user' } }, { surfaceOp: 'append' })
// assistant/message with a plugin-added block AND the tool-call its
// tool/result answers (so the surface is tool-pairing balanced).
s.append('assistant/message', {
turn: 1, step: 1,
content: [
{ type: 'image', url: 'https://x/z.png' },
chart('z'),
{ type: 'tool-call', id: CallId('e1'), name: 'bash', arguments: '{}' },
],
}, { surfaceOp: 'append' })
// tool/result with an image block → '[image]' placeholder.
// tool/result with a plugin-added block → '[chart]' placeholder.
s.append('tool/call', { turn: 1, step: 1, callId: CallId('e1'), name: 'bash', arguments: '{}' })
s.append('tool/result', { turn: 1, step: 1, callId: CallId('e1'), content: [{ type: 'image', url: 'https://x/r.png' }], isError: false }, { surfaceOp: 'append' })
// context/message and steering/message with image content.
s.append('context/message', { content: [{ type: 'image', url: 'https://x/c.png' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
s.append('steering/message', { turn: 1, content: [{ type: 'image', url: 'https://x/s.png' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
s.append('tool/result', { turn: 1, step: 1, callId: CallId('e1'), content: [chart('r')], isError: false }, { surfaceOp: 'append' })
// context/message and steering/message with plugin-added content.
s.append('context/message', { content: [chart('c')], source: { kind: 'user' } }, { surfaceOp: 'append' })
s.append('steering/message', { turn: 1, content: [chart('s')], source: { kind: 'user' } }, { surfaceOp: 'append' })
s.append('step/end', { turn: 1, step: 1 })
s.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
s.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } })
@@ -1537,11 +1535,11 @@ describe('BasicCompactService edge cases', () => {
await compactRegion(svc, s, nodes[0]!.seq, nodes[nodes.length - 1]!.seq, 'm')
const { text } = svc.summarizeCalls[0]!
// Every non-text block surfaces as a placeholder rather than being dropped.
expect(text).toContain('User: [image]')
expect(text).toContain('Assistant: [image]')
expect(text).toContain('Tool result (call e1): [image]')
expect(text).toContain('[Context: [image]]')
expect(text).toContain('[Steering: [image]]')
expect(text).toContain('User: [chart]')
expect(text).toContain('Assistant: [chart]')
expect(text).toContain('Tool result (call e1): [chart]')
expect(text).toContain('[Context: [chart]]')
expect(text).toContain('[Steering: [chart]]')
})
})