Tianyi Cui
1fd9e2a9ab
Every session.jsonl fixture embedded the full composed system prompt and
complete tool-schema list in its request/header event (~8 KB on one line,
identical across the suite), so any prompt or tool-schema edit forced a
re-record or hand-edit of every fixture — see the dynamic-workflows PR for
the churn pattern this removes.
Now exactly one scenario (text-turn, flagged pinsHeader) commits and
compares that content verbatim; every other fixture stores and compares it
as {{system}}/{{tools}} tokens via the new pure scrubRequestHeaders
normalizer (applied to both compare sides and to record-mode writes, so a
re-record cannot reintroduce the content). request/header-delta payloads
are scrubbed the same way; config/reason stay verbatim — a model swap
SHOULD churn every fixture, a prompt edit should not. Replay is unaffected:
script derivation reads only assistant/chunk events.
Fixture meta-guards enforce the split: non-pinning fixtures must be fixed
points of the scrub, the pinning fixture must not be, and exactly one
scenario pins. Committed fixtures migrated through the same function.
Docs: pinned-header RFC (implemented/testing), base snapshot RFC + testing
policy + llm-replay module doc/README updated.
78 lines
12 KiB
JSON
78 lines
12 KiB
JSON
{"type":"session","version":0,"id":"e55a6b3d-3cad-479a-ace8-65c5bb14b1d0","createdAt":1783279410878,"cwd":"/tmp/acp-snap-cwd-yKv3Ie","parentSession":"9b5104dc-98eb-4055-a88e-e67c881ce44f","seedLength":33}
|
|
{"type":"turn/start","seq":0,"time":1783279408071,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}}
|
|
{"type":"user/message","seq":1,"time":1783279408071,"data":{"content":[{"type":"text","text":"Remember this fact for later: the project codeword is MARMALADE. Reply with the single word OK and stop. Do not use any tools."}],"source":{"kind":"user"}},"surfaceOp":"append"}
|
|
{"type":"step/start","seq":2,"time":1783279408072,"data":{"turn":1,"step":1}}
|
|
{"type":"request/header","seq":3,"time":1783279408073,"data":{"header":{"config":{"model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}
|
|
{"type":"assistant/chunk","seq":4,"time":1783279408758,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}}
|
|
{"type":"assistant/chunk","seq":5,"time":1783279408758,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}}
|
|
{"type":"assistant/chunk","seq":6,"time":1783279408906,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}}
|
|
{"type":"assistant/chunk","seq":7,"time":1783279408935,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" is"}}}
|
|
{"type":"assistant/chunk","seq":8,"time":1783279408935,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" asking"}}}
|
|
{"type":"assistant/chunk","seq":9,"time":1783279408935,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}}
|
|
{"type":"assistant/chunk","seq":10,"time":1783279408936,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}}
|
|
{"type":"assistant/chunk","seq":11,"time":1783279408936,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" remember"}}}
|
|
{"type":"assistant/chunk","seq":12,"time":1783279408936,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" a"}}}
|
|
{"type":"assistant/chunk","seq":13,"time":1783279408961,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" fact"}}}
|
|
{"type":"assistant/chunk","seq":14,"time":1783279408962,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" for"}}}
|
|
{"type":"assistant/chunk","seq":15,"time":1783279408962,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" later"}}}
|
|
{"type":"assistant/chunk","seq":16,"time":1783279408989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" and"}}}
|
|
{"type":"assistant/chunk","seq":17,"time":1783279408989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" then"}}}
|
|
{"type":"assistant/chunk","seq":18,"time":1783279409024,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}}
|
|
{"type":"assistant/chunk","seq":19,"time":1783279409024,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}}
|
|
{"type":"assistant/chunk","seq":20,"time":1783279409024,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" just"}}}
|
|
{"type":"assistant/chunk","seq":21,"time":1783279409047,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}}
|
|
{"type":"assistant/chunk","seq":22,"time":1783279409048,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"OK"}}}
|
|
{"type":"assistant/chunk","seq":23,"time":1783279409048,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}}
|
|
{"type":"assistant/chunk","seq":24,"time":1783279409073,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"text"}}}
|
|
{"type":"assistant/chunk","seq":25,"time":1783279409073,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"OK"}}}
|
|
{"type":"assistant/chunk","seq":26,"time":1783279409074,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user is asking me to remember a fact for later and then reply with just \"OK\"."}}}}
|
|
{"type":"assistant/chunk","seq":27,"time":1783279409074,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"OK"}}}}
|
|
{"type":"assistant/chunk","seq":28,"time":1783279409074,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":2113,"outputTokens":21,"cacheReadTokens":0,"reasoningTokens":19}}}}
|
|
{"type":"assistant/chunk","seq":29,"time":1783279409074,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}
|
|
{"type":"assistant/message","seq":30,"time":1783279409076,"data":{"turn":1,"step":1,"content":[{"type":"reasoning","text":"The user is asking me to remember a fact for later and then reply with just \"OK\"."},{"type":"text","text":"OK"}],"usage":{"inputTokens":2113,"outputTokens":21,"cacheReadTokens":0,"reasoningTokens":19}},"sourceEventSeqs":[4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29],"surfaceOp":"append"}
|
|
{"type":"step/end","seq":31,"time":1783279409077,"data":{"turn":1,"step":1}}
|
|
{"type":"turn/end","seq":32,"time":1783279409077,"data":{"turn":1,"reason":{"kind":"completed"}}}
|
|
{"type":"turn/start","seq":33,"time":1783279410879,"data":{"turn":2,"trigger":{"kind":"message","source":{"kind":"user"}}}}
|
|
{"type":"user/message","seq":34,"time":1783279410880,"data":{"content":[{"type":"text","text":"What is the project codeword mentioned earlier in this conversation? Reply with exactly that one word and nothing else."}],"source":{"kind":"user"}},"surfaceOp":"append"}
|
|
{"type":"step/start","seq":35,"time":1783279410880,"data":{"turn":2,"step":1}}
|
|
{"type":"request/header","seq":36,"time":1783279410880,"data":{"header":{"config":{"model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"resume"}}
|
|
{"type":"assistant/chunk","seq":37,"time":1783279411585,"data":{"turn":2,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}}
|
|
{"type":"assistant/chunk","seq":38,"time":1783279411586,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}}
|
|
{"type":"assistant/chunk","seq":39,"time":1783279411711,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}}
|
|
{"type":"assistant/chunk","seq":40,"time":1783279411739,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" is"}}}
|
|
{"type":"assistant/chunk","seq":41,"time":1783279411740,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" asking"}}}
|
|
{"type":"assistant/chunk","seq":42,"time":1783279411740,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" about"}}}
|
|
{"type":"assistant/chunk","seq":43,"time":1783279411807,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}}
|
|
{"type":"assistant/chunk","seq":44,"time":1783279411807,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" project"}}}
|
|
{"type":"assistant/chunk","seq":45,"time":1783279411807,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" cod"}}}
|
|
{"type":"assistant/chunk","seq":46,"time":1783279411807,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"ew"}}}
|
|
{"type":"assistant/chunk","seq":47,"time":1783279411807,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"ord"}}}
|
|
{"type":"assistant/chunk","seq":48,"time":1783279411808,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" mentioned"}}}
|
|
{"type":"assistant/chunk","seq":49,"time":1783279411808,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" earlier"}}}
|
|
{"type":"assistant/chunk","seq":50,"time":1783279411808,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" in"}}}
|
|
{"type":"assistant/chunk","seq":51,"time":1783279411808,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}}
|
|
{"type":"assistant/chunk","seq":52,"time":1783279411808,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" conversation"}}}
|
|
{"type":"assistant/chunk","seq":53,"time":1783279411808,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}}
|
|
{"type":"assistant/chunk","seq":54,"time":1783279411808,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" I"}}}
|
|
{"type":"assistant/chunk","seq":55,"time":1783279411822,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" recall"}}}
|
|
{"type":"assistant/chunk","seq":56,"time":1783279411850,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" it"}}}
|
|
{"type":"assistant/chunk","seq":57,"time":1783279411850,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" was"}}}
|
|
{"type":"assistant/chunk","seq":58,"time":1783279411850,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}}
|
|
{"type":"assistant/chunk","seq":59,"time":1783279411878,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"M"}}}
|
|
{"type":"assistant/chunk","seq":60,"time":1783279411878,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"ARM"}}}
|
|
{"type":"assistant/chunk","seq":61,"time":1783279411878,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"AL"}}}
|
|
{"type":"assistant/chunk","seq":62,"time":1783279411879,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"ADE"}}}
|
|
{"type":"assistant/chunk","seq":63,"time":1783279411879,"data":{"turn":2,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}}
|
|
{"type":"assistant/chunk","seq":64,"time":1783279411907,"data":{"turn":2,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"text"}}}
|
|
{"type":"assistant/chunk","seq":65,"time":1783279411907,"data":{"turn":2,"step":1,"chunk":{"type":"text-delta","index":1,"text":"M"}}}
|
|
{"type":"assistant/chunk","seq":66,"time":1783279411907,"data":{"turn":2,"step":1,"chunk":{"type":"text-delta","index":1,"text":"ARM"}}}
|
|
{"type":"assistant/chunk","seq":67,"time":1783279411907,"data":{"turn":2,"step":1,"chunk":{"type":"text-delta","index":1,"text":"AL"}}}
|
|
{"type":"assistant/chunk","seq":68,"time":1783279411907,"data":{"turn":2,"step":1,"chunk":{"type":"text-delta","index":1,"text":"ADE"}}}
|
|
{"type":"assistant/chunk","seq":69,"time":1783279411908,"data":{"turn":2,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user is asking about the project codeword mentioned earlier in the conversation. I recall it was \"MARMALADE\"."}}}}
|
|
{"type":"assistant/chunk","seq":70,"time":1783279411908,"data":{"turn":2,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"MARMALADE"}}}}
|
|
{"type":"assistant/chunk","seq":71,"time":1783279411908,"data":{"turn":2,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":93,"outputTokens":31,"cacheReadTokens":2048,"reasoningTokens":26}}}}
|
|
{"type":"assistant/chunk","seq":72,"time":1783279411908,"data":{"turn":2,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}
|
|
{"type":"assistant/message","seq":73,"time":1783279411908,"data":{"turn":2,"step":1,"content":[{"type":"reasoning","text":"The user is asking about the project codeword mentioned earlier in the conversation. I recall it was \"MARMALADE\"."},{"type":"text","text":"MARMALADE"}],"usage":{"inputTokens":93,"outputTokens":31,"cacheReadTokens":2048,"reasoningTokens":26}},"sourceEventSeqs":[37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72],"surfaceOp":"append"}
|
|
{"type":"step/end","seq":74,"time":1783279411908,"data":{"turn":2,"step":1}}
|
|
{"type":"turn/end","seq":75,"time":1783279411908,"data":{"turn":2,"reason":{"kind":"completed"}}}
|