evidence-hook: every claim with a receipt

A developer released evidence-hook, an open-source hook for Claude Code that blocks a session from stopping when the assistant's claims lack supporting tool-call evidence in the transcript. The hook tracks UserPromptSubmit and Stop events, reads Claude Code's JSONL transcripts, and returns a block decision with a reason when a claim has no receipt; the accompanying test suite runs offline against fixture transcripts in a temp directory.

| | /usr/bin/env node | | | // Tests for evidence-hook.mjs. Run: node evidence-hook.test.mjs | | | // Offline: fixture transcripts in the JSONL shape Claude Code writes, in a temp folder. | | | import fs from 'node:fs'; | | | import os from 'node:os'; | | | import path from 'node:path'; | | | import { spawnSync } from 'node:child process'; | | | import { fileURLToPath } from 'node:url'; | | | | | | const ROOT = fs.mkdtempSync path.join os.tmpdir , 'evidence-hook-test-' ; | | | Object.assign process.env, { TMPDIR: ROOT, TEMP: ROOT, TMP: ROOT } ; // the hook keeps its state under os.tmpdir | | | delete process.env.CLAUDE PROJECT DIR; // the run layer cases set it themselves | | | fs.writeFileSync path.join ROOT, 'gitconfig' , '' ; // git sees only the test repos' own config | | | Object.assign process.env, { GIT CONFIG GLOBAL: path.join ROOT, 'gitconfig' , GIT CONFIG NOSYSTEM: '1' } ; | | | const TDIR = path.join ROOT, 't' ; fs.mkdirSync TDIR ; // transcripts, apart from the files the cases read | | | const repo = ...kv = { const d = fs.mkdtempSync path.join ROOT, 'repo-' ; spawnSync 'git', 'init', '-q', d ; for const k, v of kv spawnSync 'git', 'config', k, v , { cwd: d } ; return d; }; | | | const DIR = path.dirname fileURLToPath import.meta.url ; | | | const HOOK = path.join DIR, 'evidence-hook.mjs' ; | | | const H = await import HOOK ; | | | | | | let passed = 0; let total = 0; | | | function check name, ok, detail = '' { total++; if ok passed++; console.log ${ok ? 'PASS' : 'FAIL'} ${name}${ok ? '' : ${detail} } ; } | | | | | | // Transcript entries, in the shape Claude Code writes them. | | | const P = Date.now ; | | | const at = k = new Date P + 1000 + k 1000 .toISOString ; | | | const old = k = new Date P - 3600e3 + k 1000 .toISOString ; | | | const use = id, name, input, t = { type: 'assistant', timestamp: t, isSidechain: false, message: { role: 'assistant', content: { type: 'tool use', id, name, input } } } ; | | | const res = id, text, t, extra = {} = { | | | const block = { type: 'tool result', tool use id: id, content: extra.array ? { type: 'text', text } : text }; | | | if extra.isError block.is error = true; | | | const e = { type: 'user', timestamp: t, isSidechain: false, cwd: extra.cwd \|\| '/r', message: { role: 'user', content: block } }; | | | if extra.toolUseResult e.toolUseResult = extra.toolUseResult; | | | return e; | | | }; | | | const human = text, t = { type: 'user', timestamp: t, isSidechain: false, message: { role: 'user', content: text }, origin: { kind: 'human' } } ; | | | const say = text, t = { type: 'assistant', timestamp: t, isSidechain: false, message: { role: 'assistant', content: { type: 'text', text } } } ; | | | | | | let n = 0; | | | // One case: a user prompt UserPromptSubmit , a transcript, then Stop. | | | function stop { entries = , reply, sid, noPrompt, promptAt, cwd = '/r', opts = {} } { | | | sid = sid \|\| case-${++n} ; | | | if noPrompt H.run { hook event name: 'UserPromptSubmit', session id: sid, prompt: 'go on' } ; | | | if promptAt fs.writeFileSync path.join ROOT, 'evidence-hook', ${sid}.json , JSON.stringify { promptAt, blocks: 0 } ; | | | const file = path.join TDIR, ${sid}.jsonl ; | | | fs.writeFileSync file, entries.map e = JSON.stringify e .join '\n' + '\n' ; | | | const out = H.run { hook event name: 'Stop', session id: sid, transcript path: file, cwd, last assistant message: reply }, { retryMs: 0, ...opts } ; | | | return { out, sid }; | | | } | | | const blocked = o = o.out.includes '"decision":"block"' ; | | | const reason = o = blocked o ? JSON.parse o.out .reason : '' ; | | | | | | const RD = 'line 7: settings change only on a branch'; | | | const readAt = k = use r${k} , 'Read', { file path: '/r/a.md' }, at k , res r${k} , RD, at k + 1 ; | | | | | | { | | | const a = stop { entries: readAt 1 , reply: Settings change on a branch.\nEVIDENCE read \| "${RD}" } ; | | | check 'valid read via Read passes', blocked a , a.out ; | | | const b = stop { entries: readAt 1 , reply: 'EVIDENCE read \| "this passage appears nowhere at all"' } ; | | | check 'passage in no tool result blocks', blocked b && reason b .includes 'found in no tool result of this turn' , b.out ; | | | const c = stop { entries: use 'o', 'Read', { file path: '/r/a.md' }, old 1 , res 'o', RD, old 2 , human 'next', old 3 , reply: EVIDENCE read \| "${RD}" } ; | | | check 'passage only in an earlier turn blocks', blocked c && reason c .includes 'found in no tool result of this turn' , c.out ; | | | const d = stop { entries: readAt 1 , reply: EVIDENCE rule \| "${RD}"\nEvidence read \| "${RD}"\nEVIDENCE read \| "too short" } ; | | | check 'malformed lines block unknown layer, wrong case, passage under 12 characters , with the form line', | | | blocked d && reason d .split '\n' .filter l = l.startsWith 'malformed:' .length === 3 && reason d .includes 'Write: EVIDENCE <layer ' , d.out ; | | | const e = stop { entries: readAt 1 , reply: - EVIDENCE read \| "${RD}"\n EVIDENCE read \| "${RD}" } ; | | | check 'markdown list and quote marks before a well-formed line are accepted', blocked e , e.out ; | | | const f = stop { entries: readAt 1 , reply: 'Evidence shows the bug is in X.\nEvidence is thin \| so I read it again.\nEVIDENCES differ here.' } ; | | | check 'prose led by Evidence is not an EVIDENCE line', f.out === '', f.out ; | | | const g = stop { entries: readAt 1 , reply: evidence: read \| "${RD}"\n Evidence: read \| "${RD}"\nEVIDENCE read "${RD}" } ; | | | check 'evidence, an optional colon, one word and a \| any case , or the word EVIDENCE and a space, is malformed', | | | blocked g && reason g .split '\n' .filter l = l.startsWith 'malformed:' .length === 3, g.out ; | | | } | | | { | | | const t = 'all suites green, 41 of 41 here'; | | | const a = stop { entries: use 'e', 'Bash', { command: echo "${t}" }, at 1 , res 'e', t, at 2 , reply: EVIDENCE read \| "${t}" } ; | | | check 'echo output is not read', blocked a && reason a .includes 'found only as layer none' , a.out ; | | | const g = 'abc1234 fix the build script'; | | | const b = stop { entries: use 'g', 'Bash', { command: 'git log --oneline -3' }, at 1 , res 'g', g, at 2 , { cwd: repo } , reply: EVIDENCE read \| "${g}" } ; | | | check 'allowlisted git log is read', blocked b , b.out ; | | | const bad = 'cat a; echo x', 'cat a \| head', 'cat $HOME', 'cat x ', 'cat a b', 'git log --format x', 'git log --pretty=x', 'find .', 'stat f', 'git -c a=b log', 'node t.mjs' ; | | | check 'chains, redirects, expansions, caller-text options and other programs are not read', bad.every c = H.bashRead c === null , bad.filter c = H.bashRead c == null .join ' \| ' ; | | | const body = 'model-written text, two lines long'; | | | const c = stop { entries: use 'w', 'Write', { file path: '/r/new.md', content: body }, at 1 , res 'w', 'File created successfully', at 2 , | | | use 'c', 'Bash', { command: 'cat new.md' }, at 3 , res 'c', body, at 4 , reply: EVIDENCE read \| "${body}" } ; | | | check 'a Bash read of a path written earlier this turn is not read', blocked c && reason c .includes 'found only as layer none' , c.out ; | | | const refused = 'cat fo .txt', 'cat a?.md', 'cat ab .md', 'cat a.md x', 'cat a\\ b.md', 'cat ~/a.md', 'ls ~', 'ls -D %s a', 'ls --time-style x a', | | | 'git log --date x', 'git log --format x', 'git log --pretty x', 'git log --output x', 'git -C /x log', 'git -c x log', 'git diff --ext-diff', | | | 'git show --textconv', 'git diff --no-index a b', 'grep --label x y a.md', 'tail -f a.md', 'head -n x a.md', 'git branch new' ; | | | check 'allowlist: globs, escapes, ~ at word start and options off the tables are refused', refused.every x = H.bashRead x === null , refused.filter x = H.bashRead x == null .join ' \| ' ; | | | const kept = 'git log --oneline HEAD~1', 'HEAD~1' , 'cat "a b.md"', 'a b.md' , 'grep -n -e x a.md', 'a.md' , 'grep -rn x', '.' , | | | 'head -n 5 a.md', 'a.md' , 'tail -n +3 a.md', 'a.md' , 'git show -s --stat HEAD', 'HEAD' , 'git branch -l x', 'x' , "grep 'a b' c.md", 'c.md' ; | | | check 'allowlist: table options pass and give the path words HEAD~1, quoted words kept ', kept.every x, w = JSON.stringify H.bashRead x === JSON.stringify w , | | | kept.filter x, w = JSON.stringify H.bashRead x == JSON.stringify w .map x = ${x} - ${JSON.stringify H.bashRead x } .join ' \| ' ; | | | const g2 = 'model-written text, read back by a glob'; | | | const d = stop { entries: use 'w2', 'Write', { file path: '/r/foo.txt', content: g2 }, at 1 , res 'w2', 'File created successfully', at 2 , | | | use 'c2', 'Bash', { command: 'cat fo .txt' }, at 3 , res 'c2', g2, at 4 , reply: EVIDENCE read \| "${g2}" } ; | | | check 'a file written this turn read back through a glob is not read', blocked d && reason d .includes 'found only as layer none' , d.out ; | | | } | | | { | | | const proj = fs.mkdtempSync path.join ROOT, 'proj-' ; | | | const cfg = dir, body, when = old 0 = { const f = path.join dir, '.claude', 'evidence-commands.json' ; fs.mkdirSync path.dirname f , { recursive: true } ; | | | fs.writeFileSync f, body ; fs.utimesSync f, new Date when , new Date when ; return dir; }; // an mtime from before the turn, unless given | | | cfg proj, JSON.stringify { commands: 'npm test' } ; | | | const out1 = 'Tests: 2 failed, 9 passed, 11 total'; const out2 = 'Tests: 11 passed, 11 total'; | | | const runAt = id, cmd, t, k, isError = use id, 'Bash', { command: cmd }, at k , res id, t, at k + 1 , { isError } ; | | | process.env.CLAUDE PROJECT DIR = proj; | | | const a = stop { entries: runAt 't1', 'npm test', out1, 1, true , reply: EVIDENCE run \| "${out1}" } ; | | | check 'run: a listed command error result is run, from CLAUDE PROJECT DIR', blocked a , a.out ; | | | const b = stop { entries: ...runAt 't1', 'npm test', out1, 1, true , ...runAt 't2', 'npm test', out2, 3 , reply: EVIDENCE run \| "${out1}"\nEVIDENCE run \| "${out2}" } ; | | | check 'run: only the newest run of a command is fresh', blocked b && reason b .split '\n' .filter l = l.includes 'older run' .length === 1 && reason b .includes out2 , b.out ; | | | const c = stop { entries: ...runAt 't3', 'npm test -- x', out2, 1 , ...runAt 't4', 'npm test', out1, 3 , reply: EVIDENCE run \| "${out2}"\nEVIDENCE read \| "${out1}" } ; | | | check 'run: a command that only starts with an entry is not run, and a run is not read', blocked c && reason c .includes 'found only as layer none' && reason c .includes 'found only as layer run' , c.out ; | | | delete process.env.CLAUDE PROJECT DIR; | | | const d = stop { cwd: proj, entries: runAt 't1', 'npm test', out1, 1 , reply: EVIDENCE run \| "${out1}" } ; | | | check 'run: without CLAUDE PROJECT DIR the file is read from the input cwd', blocked d , d.out ; | | | const missing = fs.mkdtempSync path.join ROOT, 'none-' ; const bad = cfg fs.mkdtempSync path.join ROOT, 'bad-' , '{"commands": "npm test"' ; | | | const badShape = cfg fs.mkdtempSync path.join ROOT, 'shape-' , JSON.stringify { commands: 'npm test', 3 } ; | | | const e = missing, bad, badShape .map cwd = stop { cwd, entries: runAt 't1', 'npm test', out1, 1 , reply: EVIDENCE run \| "${out1}" } ; | | | check 'run: a missing, unparsable or wrongly shaped file gives no run layer', e.every o = blocked o && reason o .includes 'found only as layer none' , e.map o = o.out .join ' \| ' ; | | | const f = at 0 , at 2 .map when = stop { cwd: cfg fs.mkdtempSync path.join ROOT, 'late-' , JSON.stringify { commands: 'npm test' } , when , promptAt: at 0 , | | | entries: runAt 't1', 'npm test', out1, 1 , reply: EVIDENCE run \| "${out1}" } ; | | | check 'A run list: a commands file with an mtime at or after the turn start gives no run layer', f.every o = blocked o && reason o .includes 'found only as layer none' , f.map o = o.out .join ' \| ' ; | | | } | | | { | | | const g = 'abc1234 fix the build script'; | | | const gitRead = cwd = use 'g', 'Bash', { command: 'git log --oneline -3' }, at 1 , res 'g', g, at 2 , { cwd } ; | | | const bad = 'diff.external', 'x' , 'diff.x.textconv', 'x' , 'format.pretty', 'x' , 'log.date', 'format:%s' , 'core.fsmonitor', './watch.sh' , 'filter.x.clean', 'git-lfs clean -- %f' ; | | | const a = bad.map kv = stop { entries: gitRead repo kv , reply: EVIDENCE read \| "${g}" } ; | | | check 'B git config: a git read where the cwd config sets an external diff, textconv, pretty format, format: date, fsmonitor or a non-lfs filter is not read', | | | a.every o = blocked o && reason o .includes 'found only as layer none' , a.map o, i = blocked o ? '' : bad i 0 .join ' ' ; | | | const stock = repo 'filter.lfs.clean', 'git-lfs clean -- %f' , 'filter.lfs.smudge', 'git-lfs smudge -- %f' , 'filter.lfs.process', 'git-lfs filter-process' , | | | 'log.date', 'iso' , 'core.fsmonitor', 'false' , 'log.showSignature', 'false' ; | | | const b = stop { entries: gitRead stock , reply: EVIDENCE read \| "${g}" } ; | | | check 'B git config: stock git-lfs lines and harmless values keep a git read read', blocked b , b.out ; | | | } | | | { | | | const dir = fs.mkdtempSync path.join ROOT, 'files-' ; const file = path.join dir, 'c.md' ; const body = 'line 3: the cache is cleared on start'; | | | fs.writeFileSync file, body ; const touch = t = fs.utimesSync file, new Date t , new Date t ; | | | const cases = = use 'c1', 'Read', { file path: file }, at 1 , res 'c1', body, at 2 , use 'c2', 'Bash', { command: 'cat c.md' }, at 1 , res 'c2', body, at 2 , { cwd: dir } | | | .map entries = stop { entries, reply: EVIDENCE read \| "${body}" } ; | | | touch old 0 ; const a = cases ; | | | check 'C mtime: a Read or Bash read of a file unchanged since its result is read', a.every o = blocked o , a.map o = o.out .join ' \| ' ; | | | touch at 3 ; const b = cases ; | | | check 'C mtime: a Read or Bash read of a file with an mtime after its result is stale', b.every o = blocked o && reason o .includes 'stale read' , b.map o = o.out .join ' \| ' ; | | | } | | | { | | | const said = 'I ran the suite: 41 of 41 green'; | | | const tr = sid = path.join TDIR, ${sid}.jsonl ; | | | const cases = 'tr-read', 'Read', { file path: tr 'tr-read' } , 'tr-sub', 'Read', { file path: path.join TDIR, 'tr-sub', 'subagents', 'x.jsonl' } , | | | 'tr-grep', 'Grep', { pattern: '41 of 41', path: TDIR } , 'tr-up', 'Grep', { pattern: '41 of 41', path: ROOT } , 'tr-bash', 'Bash', { command: grep -n green ${tr 'tr-bash' } } ; | | | const a = cases.map sid, name, input = stop { sid, entries: use 'x', name, input, at 1 , res 'x', said, at 2 , reply: EVIDENCE read \| "${said}" } ; | | | check 'D transcript: a Read, Grep or Bash read of the transcript, a file under its folder, or a folder above it is not read', | | | a.every o = blocked o && reason o .includes 'found only as layer none' , a.map o, i = blocked o ? '' : cases i 0 .join ' ' ; | | | } | | | { | | | const miss = 'cat: TOTAL 41 of 41 passed: No such file or directory'; const p = 'TOTAL 41 of 41 passed'; | | | const a = stop { entries: use 'x', 'Bash', { command: cat "${p}" }, at 1 , res 'x', miss, at 2 , { isError: true } , reply: EVIDENCE read \| "${p}" } ; | | | check 'E is error: an allowlisted Bash read whose result is an error is not read', blocked a && reason a .includes 'found only as layer none' , a.out ; | | | } | | | { | | | const page = 'issue 42 is open, labeled bug, 3 comments'; | | | const mcp = isError = use 'm1', 'mcp tracker get issue', { id: 42 }, at 1 , res 'm1', page, at 2 , { isError } ; | | | const a = stop { entries: mcp false , reply: EVIDENCE mcp \| "${page}" } ; | | | check 'mcp: an error-free mcp tool result is mcp', blocked a , a.out ; | | | const b = stop { entries: mcp true , reply: EVIDENCE mcp \| "${page}" } ; | | | check 'mcp: an mcp error result is tool, not mcp', blocked b && reason b .includes 'found only as layer tool' , b.out ; | | | const c = stop { entries: mcp false , reply: EVIDENCE read \| "${page}" } ; | | | check 'mcp: an mcp result cited as read blocks', blocked c && reason c .includes 'found only as layer mcp' , c.out ; | | | } | | | { // the gist's companion files, when they sit next to the tests | | | const snip = path.join DIR, 'settings-snippet.json' ; const readme = path.join DIR, 'README.md' ; | | | if fs.existsSync snip { | | | const s = JSON.parse fs.readFileSync snip, 'utf8' ; const h = s.hooks; const want = 'node "${CLAUDE PROJECT DIR}/.claude/hooks/evidence-hook.mjs"'; | | | const cmds = 'UserPromptSubmit', 'Stop' .flatMap k = h k \|\| .flatMap g = g.hooks.map x = x.command ; | | | check 'settings snippet runs the hook by ${CLAUDE PROJECT DIR} on both events', cmds.length === 2 && cmds.every x = x === want , JSON.stringify cmds ; | | | const deny = s.permissions && s.permissions.deny \|\| ; | | | check 'A settings snippet denies Claude edits to the run list', deny.includes 'Edit /.claude/evidence-commands.json ' , JSON.stringify deny ; | | | } | | | if fs.existsSync readme { | | | const md = fs.readFileSync readme, 'utf8' ; const part = a, b = md.slice md.indexOf a , b ? md.indexOf b : undefined ; | | | const rule = part ' The rule', ' What it checks' ; const checks = part ' What it checks', ' Known limits' ; const limits = part ' Known limits' ; | | | const ok = H.LAYERS.every l = rule.includes - ${l}: && checks.includes - ${l}: && md.includes 'evidence-commands.json' && md.includes 'relative to the project root' | | | && 'last assistant message', 'then read with Read', 'echoes its input', 'script or test file', 'parsed whole', 'fires on each user message' .every s = limits.includes s ; | | | check 'README lists all six layers in the rule and the checks, the commands file and the limits', ok ; | | | const need = 'Edit /.claude/evidence-commands.json ', 'all built-in tools that edit files', 'primary working directory', 'mtime', 'git config', 'transcript' ; | | | check 'F README explains the run list deny rule and the new checks', need.every x = md.includes x , need.filter x = md.includes x .join ' \| ' ; | | | } | | | } | | | { | | | const edit = k = use e${k} , 'Edit', { file path: '/r/a.md', old string: 'a', new string: 'b' }, at k , res e${k} , 'The file has been updated successfully.', at k + 1 ; | | | const a = stop { entries: ...readAt 1 , ...edit 3 , reply: EVIDENCE read \| "${RD}" } ; | | | check 'Read then Edit of the same path is stale', blocked a && reason a .includes 'stale read' , a.out ; | | | const b = stop { entries: ...edit 1 , ...readAt 3 , reply: EVIDENCE read \| "${RD}" } ; | | | check 'Edit then Read of the same path passes', blocked b , b.out ; | | | } | | | { | | | const rep = 'Builder reports: suite green, 12 of 12'; | | | const a = stop { entries: use 'ag', 'Agent', { prompt: 'x' }, at 1 , res 'ag', rep, at 2 , { array: true, toolUseResult: { status: 'completed' } } , reply: EVIDENCE report \| "${rep}" } ; | | | check 'Agent result is report', blocked a , a.out ; | | | const b = stop { entries: use 'ag', 'Agent', { prompt: 'x' }, at 1 , res 'ag', rep, at 2 , { array: true, toolUseResult: { status: 'completed' } } , reply: EVIDENCE read \| "${rep}" } ; | | | check 'Agent result cited as read blocks', blocked b && reason b .includes 'found only as layer report' , b.out ; | | | const tn = { type: 'user', timestamp: at 3 , isSidechain: false, message: { role: 'user', content: <task-notification <result ${rep}</result </task-notification }, origin: { kind: 'task-notification' } }; | | | const c = stop { entries: tn , reply: EVIDENCE report \| "${rep}" } ; | | | check 'task-notification result is report', blocked c , c.out ; | | | const err = 'File does not exist at that path now'; | | | const d = stop { entries: use 'm', 'Read', { file path: '/r/x' }, at 1 , res 'm', err, at 2 , { isError: true } , reply: EVIDENCE tool \| "${err}" } ; | | | check 'error result of a non-Bash tool is tool', blocked d , d.out ; | | | const e = stop { entries: use 'm', 'Bash', { command: 'ls x' }, at 1 , res 'm', err, at 2 , { isError: true } , reply: EVIDENCE tool \| "${err}" } ; | | | check 'error result of Bash is not tool', blocked e && reason e .includes 'layer none' , e.out ; | | | const page = 'The hooks reference lists every event'; | | | const f = stop { entries: use 'w', 'WebFetch', { url: 'https://example.com' }, at 1 , res 'w', page, at 2 , reply: EVIDENCE web \| "${page}" } ; | | | check 'WebFetch result is web', blocked f , f.out ; | | | } | | | { | | | const sid = 'cap'; | | | const outs = ; | | | for let i = 0; i < 4; i++ outs.push stop { sid, noPrompt: i 0, reply: 'EVIDENCE read \| "this passage appears nowhere at all"' } .out ; | | | check 'three blocks, then the 4th in one prompt releases', outs.slice 0, 3 .every o = o.includes '"block"' && outs 3 === '', JSON.stringify outs ; | | | const again = stop { sid, reply: 'EVIDENCE read \| "this passage appears nowhere at all"' } ; | | | check 'a new prompt resets the count', blocked again , again.out ; | | | } | | | { | | | const a = stop { entries: readAt 1 , reply: 'Done. The build is green and the file is updated.' } ; | | | check 'a reply with no EVIDENCE lines passes', a.out === '', a.out ; | | | const b = stop { entries: , reply: 'UNVERIFIED: the deploy finished, I did not check the log.' } ; | | | check 'an UNVERIFIED sentence passes', b.out === '', b.out ; | | | } | | | { | | | const a = stop { entries: ...readAt 1 , human 'one more thing', at 3 , promptAt: at 3 , reply: EVIDENCE read \| "${RD}" } ; | | | check 'mid-turn user prompt moves the turn start state ', blocked a && reason a .includes 'found in no tool result of this turn' , a.out ; | | | const b = stop { noPrompt: true, entries: human 'first', at 0 , ...readAt 1 , human 'one more thing', at 3 , say 'ok', at 4 , reply: EVIDENCE read \| "${RD}" } ; | | | check 'mid-turn user prompt moves the turn start no state, from the transcript ', blocked b && reason b .includes 'found in no tool result of this turn' , b.out ; | | | const c = stop { entries: human 'go on', at 0 , ...readAt 1 , promptAt: at 0 , reply: EVIDENCE read \| "${RD}" } ; | | | check 'results after the prompt still count', blocked c , c.out ; | | | } | | | { | | | const late = 'result line that arrives late here'; | | | let calls = 0; | | | const onRetry = f = { calls++; fs.appendFileSync f, use 'l', 'Read', { file path: '/r/l' }, at 1 , res 'l', late, at 2 .map x = JSON.stringify x .join '\n' + '\n' ; }; | | | const a = stop { reply: EVIDENCE read \| "${late}" , opts: { retryMs: 5, onRetry } } ; | | | check 'transcript lag: a passage that lands during the retry passes', blocked a && calls === 1, ${a.out} calls ${calls} ; | | | } | | | { | | | const file = path.join ROOT, 'e2e.jsonl' ; fs.writeFileSync file, '\n' ; | | | const r = spawnSync process.execPath, HOOK , { input: JSON.stringify { hook event name: 'Stop', session id: 'e2e', transcript path: file, last assistant message: 'EVIDENCE read \| "nothing anywhere here"' } , env: { ...process.env, TMPDIR: ROOT }, encoding: 'utf8' } ; | | | let j = null; try { j = JSON.parse r.stdout ; } catch { / none / } | | | check 'stdin entry point prints block JSON and exits 0', r.status === 0 && j && j.decision === 'block' && j.reason.endsWith 'Fix each, or start the claim with UNVERIFIED:.' , ${r.status} ${r.stdout} ; | | | const bad = spawnSync process.execPath, HOOK , { input: 'not json', encoding: 'utf8' } ; | | | check 'a crash is a silent pass exit 0, no output ', bad.status === 0 && bad.stdout === '', ${bad.status} ${bad.stdout} ; | | | } | | | | | | { // 1: a read with no file mtime to check is stale after a later Bash call that is not an allowlisted read | | | const d = repo ; const folder = fs.mkdtempSync path.join ROOT, 'folder-' ; | | | const bashAt = id, cmd, out, k, cwd = d = use id, 'Bash', { command: cmd }, at k , res id, out, at k + 1 , { cwd } ; | | | const gs = 'On branch main, nothing to commit here'; const gl = 'src/b.ts listed by a glob'; const gp = 'notes.md:4: the folder grep hit line'; const mp = 'cat output of a path missing now'; | | | const a = stop { cwd: d, entries: ...bashAt 's1', 'git status', gs, 1 , ...bashAt 's2', 'touch x', '', 3 , reply: EVIDENCE read \| "${gs}" } , | | | stop { entries: use 'g1', 'Glob', { pattern: ' / .ts' }, at 1 , res 'g1', gl, at 2 , ...bashAt 's3', 'npm run build', 'built', 3 , reply: EVIDENCE read \| "${gl}" } , | | | stop { entries: use 'g2', 'Grep', { pattern: 'hit', path: folder }, at 1 , res 'g2', gp, at 2 , ...bashAt 's4', 'mkdir y', '', 3 , reply: EVIDENCE read \| "${gp}" } , | | | stop { entries: ...bashAt 's5', 'cat gone.md', mp, 1, folder , ...bashAt 's6', 'mv gone.md old.md', '', 3, folder , reply: EVIDENCE read \| "${mp}" } ; | | | check 'N1 staleness: git status, a Glob, a folder Grep or a Bash read of a missing path, then a non-read Bash call, is stale', | | | a.every o = blocked o && reason o .includes 'stale read' , a.map o = o.out \|\| 'passed' .join ' \| ' ; | | | fs.writeFileSync path.join folder, 'f' , 'x' ; fs.utimesSync path.join folder, 'f' , new Date old 0 , new Date old 0 ; const ls = 'f, notes listed by ls here'; | | | const b = stop { cwd: folder, entries: ...bashAt 'l1', 'ls', ls, 1, folder , ...bashAt 'l2', 'cat f', 'x', 3, folder , reply: EVIDENCE read \| "${ls}" } ; | | | check 'N1 staleness: ls, then cat f allowlisted reads only , keeps the ls result read', blocked b , b.out ; | | | } | | | { // 2: spilled tool outputs under <transcript folder /<session /tool-results/ can be read | | | const said = 'spilled output: 512 rows, all checked'; const sid = 'spill'; const spill = path.join TDIR, sid, 'tool-results' ; | | | const ok = 'Read', { file path: path.join spill, 'b1.txt' } , 'Grep', { pattern: 'rows', path: spill } , 'Bash', { command: grep -n rows ${path.join spill, 'b1.txt' } } ; | | | const a = ok.map name, input = stop { sid, entries: use 'x', name, input, at 1 , res 'x', said, at 2 , reply: EVIDENCE read \| "${said}" } ; | | | check 'N2 spill: a Read, Grep or Bash read under <session /tool-results/ is read', a.every o = blocked o , a.map o = o.out .join ' \| ' ; | | | const no = path.join TDIR, 'other', 'tool-results', 'x.txt' , path.join TDIR, sid, 'subagents', 'a.jsonl' , path.join TDIR, 'memory', 'MEMORY.md' , path.join TDIR, sid ; | | | const b = no.map p = stop { sid, entries: use 'x', 'Grep', { pattern: 'rows', path: p }, at 1 , res 'x', said, at 2 , reply: EVIDENCE read \| "${said}" } ; | | | check 'N2 spill: another session\'s tool-results, subagents, memory and the session folder stay not read', | | | b.every o = blocked o && reason o .includes 'none session transcript or its folder ' , b.map o, i = blocked o ? '' : no i .join ' ' ; | | | } | | | { // 3: core.fsmonitor true is git's built-in monitor | | | const g = 'abc1234 fix the build script'; | | | const a = stop { entries: use 'g', 'Bash', { command: 'git log --oneline -3' }, at 1 , res 'g', g, at 2 , { cwd: repo 'core.fsmonitor', 'true' } , reply: EVIDENCE read \| "${g}" } ; | | | check 'N3 git config: core.fsmonitor true keeps a git read read', blocked a , a.out ; | | | } | | | { // 4: line parsing | | | const a = stop { entries: readAt 1 , reply: EVIDENCE read \| "${RD}" \t } ; | | | check 'N4 parsing: trailing whitespace after a well-formed line is trimmed', blocked a , a.out ; | | | const b = stop { entries: readAt 1 , reply: 'EVIDENCE-based checks \| they matter here.\nEVIDENCE-based review is the goal.' } ; | | | check 'N4 parsing: EVIDENCE-based is not an EVIDENCE line', b.out === '', b.out ; | | | const c = stop { entries: readAt 1 , reply: 'The form:\n \nEVIDENCE <layer \| "<passage "\nEVIDENCE read \| "an example that is in no result"\n \nDone.' } ; | | | check 'N4 parsing: lines inside fences are skipped', c.out === '', c.out ; | | | } | | | { // 5: a cause for layer none | | | const sid = 'why'; const p = k = passage number ${k} for the none cause ; const gitDir = repo 'diff.external', 'x' ; | | | const entries = use 'a', 'Bash', { command: echo "${p 1 }" }, at 1 , res 'a', p 1 , at 2 , | | | use 'b', 'Bash', { command: 'cat nofile.md' }, at 3 , res 'b', p 2 , at 4 , { isError: true } , | | | use 'c', 'Write', { file path: '/r/w.md', content: p 3 }, at 5 , res 'c', 'File created successfully', at 6 , use 'd', 'Bash', { command: 'cat w.md' }, at 7 , res 'd', p 3 , at 8 , | | | use 'e', 'Read', { file path: path.join TDIR, ${sid}.jsonl }, at 9 , res 'e', p 4 , at 10 , | | | use 'f', 'Agent', { prompt: 'x', run in background: true }, at 11 , res 'f', p 5 , at 12 , { toolUseResult: { status: 'async launched' } } , | | | use 'g', 'Bash', { command: 'git log --oneline -3' }, at 13 , res 'g', p 6 , at 14 , { cwd: gitDir } ; | | | const a = stop { sid, entries, reply: 1, 2, 3, 4, 5, 6 .map k = EVIDENCE read \| "${p k }" .join '\n' } ; | | | const why = 'Bash command not on the read allowlist', 'Bash error', 'path written earlier this turn', 'session transcript or its folder', 'background agent launch notice', 'git config prints chosen text' ; | | | check 'N5 cause: the block message names why each record has layer none', blocked a && why.every w, i = reason a .includes none ${w} : EVIDENCE read \| "${p i + 1 }" , a.out ; | | | } | | | { // 6: inside "..." only $, backtick and backslash refuse | | | const kept = 'grep -n "TODO. fix" f', 'f' , 'cat "a b"', 'a b' , 'grep "a;b\|c&d" f', 'f' , 'cat "x 1 {y} z =w <q ? .md"', 'x 1 {y} z =w <q ? .md' ; | | | check 'N6 quotes: ? { } ; & \| < = are literal inside double quotes', kept.every x, w = JSON.stringify H.bashRead x === JSON.stringify w , | | | kept.filter x, w = JSON.stringify H.bashRead x == JSON.stringify w .map x = ${x} - ${JSON.stringify H.bashRead x } .join ' \| ' ; | | | const refused = 'grep "a$b" f', 'cat "a b "', 'cat "a\\b"' ; | | | check 'N6 quotes: $, backtick and backslash inside double quotes refuse', refused.every x = H.bashRead x === null , refused.filter x = H.bashRead x == null .join ' \| ' ; | | | } | | | { // 7: a missing transcript; the default wait | | | const r = spawnSync process.execPath, HOOK , { input: JSON.stringify { hook event name: 'Stop', session id: 'gone', transcript path: path.join ROOT, 'no-such.jsonl' , last assistant message: 'EVIDENCE read \| "nothing anywhere here"' } , encoding: 'utf8' } ; | | | check 'N7 a missing transcript passes silently exit 0, no output ', r.status === 0 && r.stdout === '' && r.stderr === '', ${r.status} ${r.stdout} ${r.stderr} ; | | | const t0 = Date.now ; const a = stop { entries: readAt 1 , reply: 'EVIDENCE read \| "found nowhere, with the default wait"', opts: { retryMs: undefined } } ; const ms = Date.now - t0; | | | check 'N7 with the default wait a passage found nowhere blocks after at least 1.4 s', blocked a && ms = 1400, ${ms} ms ${a.out} ; | | | } | | | | | | { // 8: lead and fences | | | const fake = 'EVIDENCE read \| "a passage that is in no tool result"'; | | | const r1 = stop { entries: readAt 1 , reply: 'EVIDENCE: read "the cache is cleared"' } ; | | | check 'R1 EVIDENCE: read "..." is malformed and blocks', blocked r1 && reason r1 .includes 'malformed: EVIDENCE: read "the cache is cleared"' , r1.out ; | | | const r2 = stop { entries: readAt 1 , reply: 'EVIDENCE: "the cache is cleared now"' } ; | | | check 'R2 EVIDENCE: "..." is malformed', blocked r2 && reason r2 .includes 'malformed: EVIDENCE: "the cache is cleared now"' , r2.out ; | | | const r3 = stop { entries: readAt 1 , reply: Run it:\n\ \ \ npm test\ \ \ \n${fake} } ; | | | check 'R3 a one-line npm test is not a fence: a later fake line blocks', blocked r3 && reason r3 .includes found in no tool result of this turn: ${fake} , r3.out ; | | | const r4 = stop { entries: readAt 1 , reply: Code:\n\ \ \ js\nconst x = 1;\n${fake} } ; | | | check 'R4 a fence never closed hides nothing: a later fake line blocks', blocked r4 && reason r4 .includes found in no tool result of this turn: ${fake} , r4.out ; | | | const a1 = stop { entries: readAt 1 , reply: The form:\n\ \ \ text\n${fake}\nEVIDENCE: read "an example"\n\ \ \ \nDone. } ; | | | check 'A1 lines between a matched opening and closing stay skipped', a1.out === '', a1.out ; | | | const a2 = stop { entries: readAt 1 , reply: 'Evidence: the build passed\nevidence: the cache is cleared' } ; | | | check 'A2 lowercase prose such as Evidence: the build passed stays ignored', a2.out === '', a2.out ; | | | const f = stop { entries: readAt 1 , reply: ~~~\n${fake}\n~~~\n\ \ \ \ md\n\ \ \ \n${fake}\n\ \ \ \n\ \ \ \ } ; | | | check 'N8 ~~~ fences and fences of 4 or more backticks are fences a shorter inner fence does not close ', f.out === '', f.out ; | | | } | | | | | | fs.rmSync ROOT, { recursive: true, force: true } ; | | | console.log TOTAL ${passed}/${total} ; | | | process.exit passed === total ? 0 : 1 ; |