| | #!/usr/bin/env node |
| | // Tests for evidence-hook.mjs. Run: node evidence-hook.test.mjs |
| | // Offline: fixture transcripts in the JSONL shape Claude Code writes, in a temp folder. |
| | import fs from 'node:fs'; |
| | import os from 'node:os'; |
| | import path from 'node:path'; |
| | import { spawnSync } from 'node:child_process'; |
| | import { fileURLToPath } from 'node:url'; |
| | |
| | const ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'evidence-hook-test-')); |
| | Object.assign(process.env, { TMPDIR: ROOT, TEMP: ROOT, TMP: ROOT }); // the hook keeps its state under os.tmpdir() |
| | delete process.env.CLAUDE_PROJECT_DIR; // the run layer cases set it themselves |
| | fs.writeFileSync(path.join(ROOT, 'gitconfig'), ''); // git sees only the test repos' own config |
| | Object.assign(process.env, { GIT_CONFIG_GLOBAL: path.join(ROOT, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' }); |
| | const TDIR = path.join(ROOT, 't'); fs.mkdirSync(TDIR); // transcripts, apart from the files the cases read |
| | const repo = (...kv) => { const d = fs.mkdtempSync(path.join(ROOT, 'repo-')); spawnSync('git', ['init', '-q', d]); for (const [k, v] of kv) spawnSync('git', ['config', k, v], { cwd: d }); return d; }; |
| | const DIR = path.dirname(fileURLToPath(import.meta.url)); |
| | const HOOK = path.join(DIR, 'evidence-hook.mjs'); |
| | const H = await import(HOOK); |
| | |
| | let passed = 0; let total = 0; |
| | function check(name, ok, detail = '') { total++; if (ok) passed++; console.log(${ok ? 'PASS' : 'FAIL'} ${name}${ok ? '' : ${detail}}); } |
| | |
| | // Transcript entries, in the shape Claude Code writes them. |
| | const P = Date.now(); |
| | const at = k => new Date(P + 1000 + k * 1000).toISOString(); |
| | const old = k => new Date(P - 3600e3 + k * 1000).toISOString(); |
| | const use = (id, name, input, t) => ({ type: 'assistant', timestamp: t, isSidechain: false, message: { role: 'assistant', content: [{ type: 'tool_use', id, name, input }] } }); |
| | const res = (id, text, t, extra = {}) => { |
| | const block = { type: 'tool_result', tool_use_id: id, content: extra.array ? [{ type: 'text', text }] : text }; |
| | if (extra.isError) block.is_error = true; |
| | const e = { type: 'user', timestamp: t, isSidechain: false, cwd: extra.cwd || '/r', message: { role: 'user', content: [block] } }; |
| | if (extra.toolUseResult) e.toolUseResult = extra.toolUseResult; |
| | return e; |
| | }; |
| | const human = (text, t) => ({ type: 'user', timestamp: t, isSidechain: false, message: { role: 'user', content: text }, origin: { kind: 'human' } }); |
| | const say = (text, t) => ({ type: 'assistant', timestamp: t, isSidechain: false, message: { role: 'assistant', content: [{ type: 'text', text }] } }); |
| | |
| | let n = 0; |
| | // One case: a user prompt (UserPromptSubmit), a transcript, then Stop. |
| | function stop({ entries = [], reply, sid, noPrompt, promptAt, cwd = '/r', opts = {} }) { |
| | sid = sid || case-${++n}; |
| | if (!noPrompt) H.run({ hook_event_name: 'UserPromptSubmit', session_id: sid, prompt: 'go on' }); |
| | if (promptAt) fs.writeFileSync(path.join(ROOT, 'evidence-hook', ${sid}.json), JSON.stringify({ promptAt, blocks: 0 })); |
| | const file = path.join(TDIR, ${sid}.jsonl); |
| | fs.writeFileSync(file, entries.map(e => JSON.stringify(e)).join('\n') + '\n'); |
| | const out = H.run({ hook_event_name: 'Stop', session_id: sid, transcript_path: file, cwd, last_assistant_message: reply }, { retryMs: 0, ...opts }); |
| | return { out, sid }; |
| | } |
| | const blocked = o => o.out.includes('"decision":"block"'); |
| | const reason = o => (blocked(o) ? JSON.parse(o.out).reason : ''); |
| | |
| | const RD = 'line 7: settings change only on a branch'; |
| | const readAt = k => [use(r${k}, 'Read', { file_path: '/r/a.md' }, at(k)), res( r${k}, RD, at(k + 1))]; |
| | |
| | { |
| | const a = stop({ entries: readAt(1), reply: Settings change on a branch.\nEVIDENCE read \| "${RD}" }); |
| | check('valid read via Read passes', !blocked(a), a.out); |
| | const b = stop({ entries: readAt(1), reply: 'EVIDENCE read | "this passage appears nowhere at all"' }); |
| | check('passage in no tool result blocks', blocked(b) && reason(b).includes('found in no tool result of this turn'), b.out); |
| | const c = stop({ entries: [use('o', 'Read', { file_path: '/r/a.md' }, old(1)), res('o', RD, old(2)), human('next', old(3))], reply: EVIDENCE read \| "${RD}" }); |
| | check('passage only in an earlier turn blocks', blocked(c) && reason(c).includes('found in no tool result of this turn'), c.out); |
| | const d = stop({ entries: readAt(1), reply: EVIDENCE rule \| "${RD}"\nEvidence read \| "${RD}"\nEVIDENCE read \| "too short" }); |
| | check('malformed lines block (unknown layer, wrong case, passage under 12 characters), with the form line', |
| | blocked(d) && reason(d).split('\n').filter(l => l.startsWith('malformed:')).length === 3 && reason(d).includes('Write: EVIDENCE <layer>'), d.out); |
| | const e = stop({ entries: readAt(1), reply: - EVIDENCE read \| "${RD}"\n> **EVIDENCE read \| "${RD}" }); |
| | check('markdown list and quote marks before a well-formed line are accepted', !blocked(e), e.out); |
| | const f = stop({ entries: readAt(1), reply: 'Evidence shows the bug is in X.\nEvidence is thin | so I read it again.\nEVIDENCES differ here.' }); |
| | check('prose led by Evidence is not an EVIDENCE line', f.out === '', f.out); |
| | const g = stop({ entries: readAt(1), reply: evidence: read \| "${RD}"\n**Evidence:** read \| "${RD}"\nEVIDENCE read "${RD}" }); |
| | check('evidence, an optional colon, one word and a | (any case), or the word EVIDENCE and a space, is malformed', |
| | blocked(g) && reason(g).split('\n').filter(l => l.startsWith('malformed:')).length === 3, g.out); |
| | } |
| | { |
| | const t = 'all suites green, 41 of 41 here'; |
| | const a = stop({ entries: [use('e', 'Bash', { command: echo "${t}" }, at(1)), res('e', t, at(2))], reply: EVIDENCE read \| "${t}" }); |
| | check('echo output is not read', blocked(a) && reason(a).includes('found only as layer none'), a.out); |
| | const g = 'abc1234 fix the build script'; |
| | const b = stop({ entries: [use('g', 'Bash', { command: 'git log --oneline -3' }, at(1)), res('g', g, at(2), { cwd: repo() })], reply: EVIDENCE read \| "${g}" }); |
| | check('allowlisted git log is read', !blocked(b), b.out); |
| | const bad = ['cat a; echo x', 'cat a | head', 'cat $HOME', 'cat x', 'cat a > b', 'git log --format x', 'git log --pretty=x', 'find .', 'stat f', 'git -c a=b log', 'node t.mjs']; |
| | check('chains, redirects, expansions, caller-text options and other programs are not read', bad.every(c => H.bashRead(c) === null), bad.filter(c => H.bashRead(c) !== null).join(' | ')); |
| | const body = 'model-written text, two lines long'; |
| | const c = stop({ entries: [use('w', 'Write', { file_path: '/r/new.md', content: body }, at(1)), res('w', 'File created successfully', at(2)), |
| | use('c', 'Bash', { command: 'cat new.md' }, at(3)), res('c', body, at(4))], reply: EVIDENCE read \| "${body}" }); |
| | check('a Bash read of a path written earlier this turn is not read', blocked(c) && reason(c).includes('found only as layer none'), c.out); |
| | const refused = ['cat fo*.txt', 'cat a?.md', 'cat [ab].md', 'cat a.md #x', 'cat a\ b.md', 'cat ~/a.md', 'ls ~', 'ls -D %s a', 'ls --time-style x a', |
| | 'git log --date x', 'git log --format x', 'git log --pretty x', 'git log --output x', 'git -C /x log', 'git -c x log', 'git diff --ext-diff', |
| | 'git show --textconv', 'git diff --no-index a b', 'grep --label x y a.md', 'tail -f a.md', 'head -n x a.md', 'git branch new']; |
| | check('allowlist: globs, escapes, ~ at word start and options off the tables are refused', refused.every(x => H.bashRead(x) === null), refused.filter(x => H.bashRead(x) !== null).join(' | ')); |
| | const kept = [['git log --oneline HEAD~1', ['HEAD~1']], ['cat "a b.md"', ['a b.md']], ['grep -n -e x a.md', ['a.md']], ['grep -rn x', ['.']], |
| | ['head -n 5 a.md', ['a.md']], ['tail -n +3 a.md', ['a.md']], ['git show -s --stat HEAD', ['HEAD']], ['git branch -l x', ['x']], ["grep 'ab' c.md", ['c.md']]]; |
| | check('allowlist: table options pass and give the path words (HEAD~1, quoted words kept)', kept.every(([x, w]) => JSON.stringify(H.bashRead(x)) === JSON.stringify(w)), |
| | kept.filter(([x, w]) => JSON.stringify(H.bashRead(x)) !== JSON.stringify(w)).map(([x]) => ${x} -> ${JSON.stringify(H.bashRead(x))}).join(' | ')); |
| | const g2 = 'model-written text, read back by a glob'; |
| | const d = stop({ entries: [use('w2', 'Write', { file_path: '/r/foo.txt', content: g2 }, at(1)), res('w2', 'File created successfully', at(2)), |
| | use('c2', 'Bash', { command: 'cat fo.txt' }, at(3)), res('c2', g2, at(4))], reply: EVIDENCE read \| "${g2}" }); |
| | check('a file written this turn read back through a glob is not read', blocked(d) && reason(d).includes('found only as layer none'), d.out); |
| | } |
| | { |
| | const proj = fs.mkdtempSync(path.join(ROOT, 'proj-')); |
| | const cfg = (dir, body, when = old(0)) => { const f = path.join(dir, '.claude', 'evidence-commands.json'); fs.mkdirSync(path.dirname(f), { recursive: true }); |
| | fs.writeFileSync(f, body); fs.utimesSync(f, new Date(when), new Date(when)); return dir; }; // an mtime from before the turn, unless given |
| | cfg(proj, JSON.stringify({ commands: ['npm test'] })); |
| | const out1 = 'Tests: 2 failed, 9 passed, 11 total'; const out2 = 'Tests: 11 passed, 11 total'; |
| | const runAt = (id, cmd, t, k, isError) => [use(id, 'Bash', { command: cmd }, at(k)), res(id, t, at(k + 1), { isError })]; |
| | process.env.CLAUDE_PROJECT_DIR = proj; |
| | const a = stop({ entries: runAt('t1', 'npm test', out1, 1, true), reply: EVIDENCE run \| "${out1}" }); |
| | check('run: a listed command (error result) is run, from CLAUDE_PROJECT_DIR', !blocked(a), a.out); |
| | const b = stop({ entries: [...runAt('t1', 'npm test', out1, 1, true), ...runAt('t2', 'npm test', out2, 3)], reply: EVIDENCE run \| "${out1}"\nEVIDENCE run \| "${out2}" }); |
| | check('run: only the newest run of a command is fresh', blocked(b) && reason(b).split('\n').filter(l => l.includes('older run')).length === 1 && !reason(b).includes(out2), b.out); |
| | const c = stop({ entries: [...runAt('t3', 'npm test -- x', out2, 1), ...runAt('t4', 'npm test', out1, 3)], reply: EVIDENCE run \| "${out2}"\nEVIDENCE read \| "${out1}" }); |
| | check('run: a command that only starts with an entry is not run, and a run is not read', blocked(c) && reason(c).includes('found only as layer none') && reason(c).includes('found only as layer run'), c.out); |
| | delete process.env.CLAUDE_PROJECT_DIR; |
| | const d = stop({ cwd: proj, entries: runAt('t1', 'npm test', out1, 1), reply: EVIDENCE run \| "${out1}" }); |
| | check('run: without CLAUDE_PROJECT_DIR the file is read from the input cwd', !blocked(d), d.out); |
| | const missing = fs.mkdtempSync(path.join(ROOT, 'none-')); const bad = cfg(fs.mkdtempSync(path.join(ROOT, 'bad-')), '{"commands": "npm test"'); |
| | const badShape = cfg(fs.mkdtempSync(path.join(ROOT, 'shape-')), JSON.stringify({ commands: ['npm test', 3] })); |
| | const e = [missing, bad, badShape].map(cwd => stop({ cwd, entries: runAt('t1', 'npm test', out1, 1), reply: EVIDENCE run \| "${out1}" })); |
| | check('run: a missing, unparsable or wrongly shaped file gives no run layer', e.every(o => blocked(o) && reason(o).includes('found only as layer none')), e.map(o => o.out).join(' | ')); |
| | const f = [at(0), at(2)].map(when => stop({ cwd: cfg(fs.mkdtempSync(path.join(ROOT, 'late-')), JSON.stringify({ commands: ['npm test'] }), when), promptAt: at(0), |
| | entries: runAt('t1', 'npm test', out1, 1), reply: EVIDENCE run \| "${out1}" })); |
| | check('A run list: a commands file with an mtime at or after the turn start gives no run layer', f.every(o => blocked(o) && reason(o).includes('found only as layer none')), f.map(o => o.out).join(' | ')); |
| | } |
| | { |
| | const g = 'abc1234 fix the build script'; |
| | const gitRead = cwd => [use('g', 'Bash', { command: 'git log --oneline -3' }, at(1)), res('g', g, at(2), { cwd })]; |
| | const bad = [['diff.external', 'x'], ['diff.x.textconv', 'x'], ['format.pretty', 'x'], ['log.date', 'format:%s'], ['core.fsmonitor', './watch.sh'], ['filter.x.clean', 'git-lfs clean -- %f']]; |
| | const a = bad.map(kv => stop({ entries: gitRead(repo(kv)), reply: EVIDENCE read \| "${g}" })); |
| | check('B git config: a git read where the cwd config sets an external diff, textconv, pretty format, format: date, fsmonitor or a non-lfs filter is not read', |
| | a.every(o => blocked(o) && reason(o).includes('found only as layer none')), a.map((o, i) => blocked(o) ? '' : bad[i][0]).join(' ')); |
| | const stock = repo(['filter.lfs.clean', 'git-lfs clean -- %f'], ['filter.lfs.smudge', 'git-lfs smudge -- %f'], ['filter.lfs.process', 'git-lfs filter-process'], |
| | ['log.date', 'iso'], ['core.fsmonitor', 'false'], ['log.showSignature', 'false']); |
| | const b = stop({ entries: gitRead(stock), reply: EVIDENCE read \| "${g}" }); |
| | check('B git config: stock git-lfs lines and harmless values keep a git read read', !blocked(b), b.out); |
| | } |
| | { |
| | const dir = fs.mkdtempSync(path.join(ROOT, 'files-')); const file = path.join(dir, 'c.md'); const body = 'line 3: the cache is cleared on start'; |
| | fs.writeFileSync(file, body); const touch = t => fs.utimesSync(file, new Date(t), new Date(t)); |
| | const cases = () => [[use('c1', 'Read', { file_path: file }, at(1)), res('c1', body, at(2))], [use('c2', 'Bash', { command: 'cat c.md' }, at(1)), res('c2', body, at(2), { cwd: dir })]] |
| | .map(entries => stop({ entries, reply: EVIDENCE read \| "${body}" })); |
| | touch(old(0)); const a = cases(); |
| | check('C mtime: a Read or Bash read of a file unchanged since its result is read', a.every(o => !blocked(o)), a.map(o => o.out).join(' | ')); |
| | touch(at(3)); const b = cases(); |
| | check('C mtime: a Read or Bash read of a file with an mtime after its result is stale', b.every(o => blocked(o) && reason(o).includes('stale read')), b.map(o => o.out).join(' | ')); |
| | } |
| | { |
| | const said = 'I ran the suite: 41 of 41 green'; |
| | const tr = sid => path.join(TDIR, ${sid}.jsonl); |
| | const cases = [['tr-read', 'Read', { file_path: tr('tr-read') }], ['tr-sub', 'Read', { file_path: path.join(TDIR, 'tr-sub', 'subagents', 'x.jsonl') }], |
| | ['tr-grep', 'Grep', { pattern: '41 of 41', path: TDIR }], ['tr-up', 'Grep', { pattern: '41 of 41', path: ROOT }], ['tr-bash', 'Bash', { command: grep -n green ${tr('tr-bash')} }]]; |
| | const a = cases.map(([sid, name, input]) => stop({ sid, entries: [use('x', name, input, at(1)), res('x', said, at(2))], reply: EVIDENCE read \| "${said}" })); |
| | check('D transcript: a Read, Grep or Bash read of the transcript, a file under its folder, or a folder above it is not read', |
| | a.every(o => blocked(o) && reason(o).includes('found only as layer none')), a.map((o, i) => blocked(o) ? '' : cases[i][0]).join(' ')); |
| | } |
| | { |
| | const miss = 'cat: TOTAL 41 of 41 passed: No such file or directory'; const p = 'TOTAL 41 of 41 passed'; |
| | const a = stop({ entries: [use('x', 'Bash', { command: cat "${p}" }, at(1)), res('x', miss, at(2), { isError: true })], reply: EVIDENCE read \| "${p}" }); |
| | check('E is_error: an allowlisted Bash read whose result is an error is not read', blocked(a) && reason(a).includes('found only as layer none'), a.out); |
| | } |
| | { |
| | const page = 'issue 42 is open, labeled bug, 3 comments'; |
| | const mcp = (isError) => [use('m1', 'mcp__tracker__get_issue', { id: 42 }, at(1)), res('m1', page, at(2), { isError })]; |
| | const a = stop({ entries: mcp(false), reply: EVIDENCE mcp \| "${page}" }); |
| | check('mcp: an error-free mcp__ tool result is mcp', !blocked(a), a.out); |
| | const b = stop({ entries: mcp(true), reply: EVIDENCE mcp \| "${page}" }); |
| | check('mcp: an mcp__ error result is tool, not mcp', blocked(b) && reason(b).includes('found only as layer tool'), b.out); |
| | const c = stop({ entries: mcp(false), reply: EVIDENCE read \| "${page}" }); |
| | check('mcp: an mcp__ result cited as read blocks', blocked(c) && reason(c).includes('found only as layer mcp'), c.out); |
| | } |
| | { // the gist's companion files, when they sit next to the tests |
| | const snip = path.join(DIR, 'settings-snippet.json'); const readme = path.join(DIR, 'README.md'); |
| | if (fs.existsSync(snip)) { |
| | const s = JSON.parse(fs.readFileSync(snip, 'utf8')); const h = s.hooks; const want = 'node "${CLAUDE_PROJECT_DIR}/.claude/hooks/evidence-hook.mjs"'; |
| | const cmds = ['UserPromptSubmit', 'Stop'].flatMap(k => (h[k] || []).flatMap(g => g.hooks.map(x => x.command))); |
| | check('settings snippet runs the hook by ${CLAUDE_PROJECT_DIR} on both events', cmds.length === 2 && cmds.every(x => x === want), JSON.stringify(cmds)); |
| | const deny = (s.permissions && s.permissions.deny) || []; |
| | check('A settings snippet denies Claude edits to the run list', deny.includes('Edit(/.claude/evidence-commands.json)'), JSON.stringify(deny)); |
| | } |
| | if (fs.existsSync(readme)) { |
| | const md = fs.readFileSync(readme, 'utf8'); const part = (a, b) => md.slice(md.indexOf(a), b ? md.indexOf(b) : undefined); |
| | const rule = part('## The rule', '## What it checks'); const checks = part('## What it checks', '## Known limits'); const limits = part('## Known limits'); |
| | const ok = H.LAYERS.every(l => rule.includes(- ${l}:) && checks.includes(- ${l}:)) && md.includes('evidence-commands.json') && !md.includes('relative to the project root') |
| | && ['last_assistant_message', 'then read with Read', 'echoes its input', 'script or test file', 'parsed whole', 'fires on each user message'].every(s => limits.includes(s)); |
| | check('README lists all six layers in the rule and the checks, the commands file and the limits', ok); |
| | const need = ['Edit(/.claude/evidence-commands.json)', 'all built-in tools that edit files', 'primary working directory', 'mtime', 'git config', 'transcript']; |
| | check('F README explains the run list deny rule and the new checks', need.every(x => md.includes(x)), need.filter(x => !md.includes(x)).join(' | ')); |
| | } |
| | } |
| | { |
| | const edit = k => [use(e${k}, 'Edit', { file_path: '/r/a.md', old_string: 'a', new_string: 'b' }, at(k)), res(e${k}, 'The file has been updated successfully.', at(k + 1))]; |
| | const a = stop({ entries: [...readAt(1), ...edit(3)], reply: EVIDENCE read \| "${RD}" }); |
| | check('Read then Edit of the same path is stale', blocked(a) && reason(a).includes('stale read'), a.out); |
| | const b = stop({ entries: [...edit(1), ...readAt(3)], reply: EVIDENCE read \| "${RD}" }); |
| | check('Edit then Read of the same path passes', !blocked(b), b.out); |
| | } |
| | { |
| | const rep = 'Builder reports: suite green, 12 of 12'; |
| | const a = stop({ entries: [use('ag', 'Agent', { prompt: 'x' }, at(1)), res('ag', rep, at(2), { array: true, toolUseResult: { status: 'completed' } })], reply: EVIDENCE report \| "${rep}" }); |
| | check('Agent result is report', !blocked(a), a.out); |
| | const b = stop({ entries: [use('ag', 'Agent', { prompt: 'x' }, at(1)), res('ag', rep, at(2), { array: true, toolUseResult: { status: 'completed' } })], reply: EVIDENCE read \| "${rep}" }); |
| | check('Agent result cited as read blocks', blocked(b) && reason(b).includes('found only as layer report'), b.out); |
| | const tn = { type: 'user', timestamp: at(3), isSidechain: false, message: { role: 'user', content: <task-notification><result>${rep}</result></task-notification> }, origin: { kind: 'task-notification' } }; |
| | const c = stop({ entries: [tn], reply: EVIDENCE report \| "${rep}" }); |
| | check('task-notification result is report', !blocked(c), c.out); |
| | const err = 'File does not exist at that path now'; |
| | const d = stop({ entries: [use('m', 'Read', { file_path: '/r/x' }, at(1)), res('m', err, at(2), { isError: true })], reply: EVIDENCE tool \| "${err}" }); |
| | check('error result of a non-Bash tool is tool', !blocked(d), d.out); |
| | const e = stop({ entries: [use('m', 'Bash', { command: 'ls x' }, at(1)), res('m', err, at(2), { isError: true })], reply: EVIDENCE tool \| "${err}" }); |
| | check('error result of Bash is not tool', blocked(e) && reason(e).includes('layer none'), e.out); |
| | const page = 'The hooks reference lists every event'; |
| | const f = stop({ entries: [use('w', 'WebFetch', { url: 'https://example.com' }, at(1)), res('w', page, at(2))], reply: EVIDENCE web \| "${page}" }); |
| | check('WebFetch result is web', !blocked(f), f.out); |
| | } |
| | { |
| | const sid = 'cap'; |
| | const outs = []; |
| | for (let i = 0; i < 4; i++) outs.push(stop({ sid, noPrompt: i > 0, reply: 'EVIDENCE read | "this passage appears nowhere at all"' }).out); |
| | check('three blocks, then the 4th in one prompt releases', outs.slice(0, 3).every(o => o.includes('"block"')) && outs[3] === '', JSON.stringify(outs)); |
| | const again = stop({ sid, reply: 'EVIDENCE read | "this passage appears nowhere at all"' }); |
| | check('a new prompt resets the count', blocked(again), again.out); |
| | } |
| | { |
| | const a = stop({ entries: readAt(1), reply: 'Done. The build is green and the file is updated.' }); |
| | check('a reply with no EVIDENCE lines passes', a.out === '', a.out); |
| | const b = stop({ entries: [], reply: 'UNVERIFIED: the deploy finished, I did not check the log.' }); |
| | check('an UNVERIFIED sentence passes', b.out === '', b.out); |
| | } |
| | { |
| | const a = stop({ entries: [...readAt(1), human('one more thing', at(3))], promptAt: at(3), reply: EVIDENCE read \| "${RD}" }); |
| | check('mid-turn user prompt moves the turn start (state)', blocked(a) && reason(a).includes('found in no tool result of this turn'), a.out); |
| | const b = stop({ noPrompt: true, entries: [human('first', at(0)), ...readAt(1), human('one more thing', at(3)), say('ok', at(4))], reply: EVIDENCE read \| "${RD}" }); |
| | check('mid-turn user prompt moves the turn start (no state, from the transcript)', blocked(b) && reason(b).includes('found in no tool result of this turn'), b.out); |
| | const c = stop({ entries: [human('go on', at(0)), ...readAt(1)], promptAt: at(0), reply: EVIDENCE read \| "${RD}" }); |
| | check('results after the prompt still count', !blocked(c), c.out); |
| | } |
| | { |
| | const late = 'result line that arrives late here'; |
| | let calls = 0; |
| | const onRetry = f => { calls++; fs.appendFileSync(f, [use('l', 'Read', { file_path: '/r/l' }, at(1)), res('l', late, at(2))].map(x => JSON.stringify(x)).join('\n') + '\n'); }; |
| | const a = stop({ reply: EVIDENCE read \| "${late}", opts: { retryMs: 5, onRetry } }); |
| | check('transcript lag: a passage that lands during the retry passes', !blocked(a) && calls === 1, ${a.out} calls ${calls}); |
| | } |
| | { |
| | const file = path.join(ROOT, 'e2e.jsonl'); fs.writeFileSync(file, '\n'); |
| | const r = spawnSync(process.execPath, [HOOK], { input: JSON.stringify({ hook_event_name: 'Stop', session_id: 'e2e', transcript_path: file, last_assistant_message: 'EVIDENCE read | "nothing anywhere here"' }), env: { ...process.env, TMPDIR: ROOT }, encoding: 'utf8' }); |
| | let j = null; try { j = JSON.parse(r.stdout); } catch { /* none / } |
| | check('stdin entry point prints block JSON and exits 0', r.status === 0 && j && j.decision === 'block' && j.reason.endsWith('Fix each, or start the claim with UNVERIFIED:.'), ${r.status} ${r.stdout}); |
| | const bad = spawnSync(process.execPath, [HOOK], { input: 'not json', encoding: 'utf8' }); |
| | check('a crash is a silent pass (exit 0, no output)', bad.status === 0 && bad.stdout === '', ${bad.status} ${bad.stdout}); |
| | } |
| | |
| | { // 1: a read with no file mtime to check is stale after a later Bash call that is not an allowlisted read |
| | const d = repo(); const folder = fs.mkdtempSync(path.join(ROOT, 'folder-')); |
| | const bashAt = (id, cmd, out, k, cwd = d) => [use(id, 'Bash', { command: cmd }, at(k)), res(id, out, at(k + 1), { cwd })]; |
| | const gs = 'On branch main, nothing to commit here'; const gl = 'src/b.ts listed by a glob'; const gp = 'notes.md:4: the folder grep hit line'; const mp = 'cat output of a path missing now'; |
| | const a = [stop({ cwd: d, entries: [...bashAt('s1', 'git status', gs, 1), ...bashAt('s2', 'touch x', '', 3)], reply: EVIDENCE read \| "${gs}" }), |
| | stop({ entries: [use('g1', 'Glob', { pattern: '**/.ts' }, at(1)), res('g1', gl, at(2)), ...bashAt('s3', 'npm run build', 'built', 3)], reply: EVIDENCE read \| "${gl}" }), |
| | stop({ entries: [use('g2', 'Grep', { pattern: 'hit', path: folder }, at(1)), res('g2', gp, at(2)), ...bashAt('s4', 'mkdir y', '', 3)], reply: EVIDENCE read \| "${gp}" }), |
| | stop({ entries: [...bashAt('s5', 'cat gone.md', mp, 1, folder), ...bashAt('s6', 'mv gone.md old.md', '', 3, folder)], reply: EVIDENCE read \| "${mp}" })]; |
| | check('N1 staleness: git status, a Glob, a folder Grep or a Bash read of a missing path, then a non-read Bash call, is stale', |
| | a.every(o => blocked(o) && reason(o).includes('stale read')), a.map(o => o.out || 'passed').join(' | ')); |
| | fs.writeFileSync(path.join(folder, 'f'), 'x'); fs.utimesSync(path.join(folder, 'f'), new Date(old(0)), new Date(old(0))); const ls = 'f, notes listed by ls here'; |
| | const b = stop({ cwd: folder, entries: [...bashAt('l1', 'ls', ls, 1, folder), ...bashAt('l2', 'cat f', 'x', 3, folder)], reply: EVIDENCE read \| "${ls}" }); |
| | check('N1 staleness: ls, then cat f (allowlisted reads only), keeps the ls result read', !blocked(b), b.out); |
| | } |
| | { // 2: spilled tool outputs under <transcript folder>/<session>/tool-results/ can be read |
| | const said = 'spilled output: 512 rows, all checked'; const sid = 'spill'; const spill = path.join(TDIR, sid, 'tool-results'); |
| | const ok = [['Read', { file_path: path.join(spill, 'b1.txt') }], ['Grep', { pattern: 'rows', path: spill }], ['Bash', { command: grep -n rows ${path.join(spill, 'b1.txt')} }]]; |
| | const a = ok.map(([name, input]) => stop({ sid, entries: [use('x', name, input, at(1)), res('x', said, at(2))], reply: EVIDENCE read \| "${said}" })); |
| | check('N2 spill: a Read, Grep or Bash read under <session>/tool-results/ is read', a.every(o => !blocked(o)), a.map(o => o.out).join(' | ')); |
| | const no = [path.join(TDIR, 'other', 'tool-results', 'x.txt'), path.join(TDIR, sid, 'subagents', 'a.jsonl'), path.join(TDIR, 'memory', 'MEMORY.md'), path.join(TDIR, sid)]; |
| | const b = no.map(p => stop({ sid, entries: [use('x', 'Grep', { pattern: 'rows', path: p }, at(1)), res('x', said, at(2))], reply: EVIDENCE read \| "${said}" })); |
| | check('N2 spill: another session's tool-results, subagents, memory and the session folder stay not read', |
| | b.every(o => blocked(o) && reason(o).includes('none (session transcript or its folder)')), b.map((o, i) => blocked(o) ? '' : no[i]).join(' ')); |
| | } |
| | { // 3: core.fsmonitor true is git's built-in monitor |
| | const g = 'abc1234 fix the build script'; |
| | const a = stop({ entries: [use('g', 'Bash', { command: 'git log --oneline -3' }, at(1)), res('g', g, at(2), { cwd: repo(['core.fsmonitor', 'true']) })], reply: EVIDENCE read \| "${g}" }); |
| | check('N3 git config: core.fsmonitor true keeps a git read read', !blocked(a), a.out); |
| | } |
| | { // 4: line parsing |
| | const a = stop({ entries: readAt(1), reply: EVIDENCE read \| "${RD}" \t }); |
| | check('N4 parsing: trailing whitespace after a well-formed line is trimmed', !blocked(a), a.out); |
| | const b = stop({ entries: readAt(1), reply: 'EVIDENCE-based checks | they matter here.\nEVIDENCE-based review is the goal.' }); |
| | check('N4 parsing: EVIDENCE-based is not an EVIDENCE line', b.out === '', b.out); |
| | const c = stop({ entries: readAt(1), reply: 'The form:\n\nEVIDENCE <layer> \| "<passage>"\nEVIDENCE read \| "an example that is in no result"\n\nDone.' }); |
| | check('N4 parsing: lines inside fences are skipped', c.out === '', c.out); | | | } | | | { // 5: a cause for layer none | | | const sid = 'why'; const p = k => `passage number ${k} for the none cause`; const gitDir = repo(['diff.external', 'x']); | | | const entries = [use('a', 'Bash', { command: `echo "${p(1)}"` }, at(1)), res('a', p(1), at(2)), | | | use('b', 'Bash', { command: 'cat nofile.md' }, at(3)), res('b', p(2), at(4), { isError: true }), | | | use('c', 'Write', { file_path: '/r/w.md', content: p(3) }, at(5)), res('c', 'File created successfully', at(6)), use('d', 'Bash', { command: 'cat w.md' }, at(7)), res('d', p(3), at(8)), | | | use('e', 'Read', { file_path: path.join(TDIR, `${sid}.jsonl`) }, at(9)), res('e', p(4), at(10)), | | | use('f', 'Agent', { prompt: 'x', run_in_background: true }, at(11)), res('f', p(5), at(12), { toolUseResult: { status: 'async_launched' } }), | | | use('g', 'Bash', { command: 'git log --oneline -3' }, at(13)), res('g', p(6), at(14), { cwd: gitDir })]; | | | const a = stop({ sid, entries, reply: [1, 2, 3, 4, 5, 6].map(k => `EVIDENCE read \| "${p(k)}"`).join('\n') }); | | | const why = ['Bash command not on the read allowlist', 'Bash error', 'path written earlier this turn', 'session transcript or its folder', 'background agent launch notice', 'git config prints chosen text']; | | | check('N5 cause: the block message names why each record has layer none', blocked(a) && why.every((w, i) => reason(a).includes(`none (${w}): EVIDENCE read \| "${p(i + 1)}"`)), a.out); | | | } | | | { // 6: inside "..." only $, backtick and backslash refuse | | | const kept = [['grep -n "TODO.*fix" f', ['f']], ['cat "a*b"', ['a*b']], ['grep "a;b\|c&d" f', ['f']], ['cat "x (1) {y} #z =w <q>?[].md"', ['x (1) {y} #z =w <q>?[].md']]]; | | | check('N6 quotes: * ? [ ] ( ) { } ; & \| < > # = are literal inside double quotes', kept.every(([x, w]) => JSON.stringify(H.bashRead(x)) === JSON.stringify(w)), | | | kept.filter(([x, w]) => JSON.stringify(H.bashRead(x)) !== JSON.stringify(w)).map(([x]) => `${x} -> ${JSON.stringify(H.bashRead(x))}`).join(' \| ')); | | | const refused = ['grep "a$b" f', 'cat "a`b`"', 'cat "a\\b"']; | | | check('N6 quotes: $, backtick and backslash inside double quotes refuse', refused.every(x => H.bashRead(x) === null), refused.filter(x => H.bashRead(x) !== null).join(' \| ')); | | | } | | | { // 7: a missing transcript; the default wait | | | const r = spawnSync(process.execPath, [HOOK], { input: JSON.stringify({ hook_event_name: 'Stop', session_id: 'gone', transcript_path: path.join(ROOT, 'no-such.jsonl'), last_assistant_message: 'EVIDENCE read \| "nothing anywhere here"' }), encoding: 'utf8' }); | | | check('N7 a missing transcript passes silently (exit 0, no output)', r.status === 0 && r.stdout === '' && r.stderr === '', `${r.status} ${r.stdout} ${r.stderr}`); | | | const t0 = Date.now(); const a = stop({ entries: readAt(1), reply: 'EVIDENCE read \| "found nowhere, with the default wait"', opts: { retryMs: undefined } }); const ms = Date.now() - t0; | | | check('N7 with the default wait a passage found nowhere blocks after at least 1.4 s', blocked(a) && ms >= 1400, `${ms} ms ${a.out}`); | | | } | | | | | | { // 8: lead and fences | | | const fake = 'EVIDENCE read \| "a passage that is in no tool result"'; | | | const r1 = stop({ entries: readAt(1), reply: 'EVIDENCE: read "the cache is cleared"' }); | | | check('R1 EVIDENCE: read "..." is malformed and blocks', blocked(r1) && reason(r1).includes('malformed: EVIDENCE: read "the cache is cleared"'), r1.out); | | | const r2 = stop({ entries: readAt(1), reply: 'EVIDENCE: "the cache is cleared now"' }); | | | check('R2 EVIDENCE: "..." is malformed', blocked(r2) && reason(r2).includes('malformed: EVIDENCE: "the cache is cleared now"'), r2.out); | | | const r3 = stop({ entries: readAt(1), reply: `Run it:\n\`\`\` npm test\`\`\`\n${fake}` }); | | | check('R3 a one-line npm testis not a fence: a later fake line blocks', blocked(r3) && reason(r3).includes(`found in no tool result of this turn: ${fake}`), r3.out); | | | const r4 = stop({ entries: readAt(1), reply: `Code:\n\`\`\` js\nconst x = 1;\n${fake}` }); | | | check('R4 a fence never closed hides nothing: a later fake line blocks', blocked(r4) && reason(r4).includes(`found in no tool result of this turn: ${fake}`), r4.out); | | | const a1 = stop({ entries: readAt(1), reply: `The form:\n\`\`\` text\n${fake}\nEVIDENCE: read "an example"\n\`\`\`\nDone.` }); | | | check('A1 lines between a matched opening and closing stay skipped', a1.out === '', a1.out); |
| | const a2 = stop({ entries: readAt(1), reply: 'Evidence: the build passed\nevidence: the cache is cleared' }); |
| | check('A2 lowercase prose such as Evidence: the build passed stays ignored', a2.out === '', a2.out); |
| | const f = stop({ entries: readAt(1), reply: ~~~\n${fake}\n~~~\n\``` md\n```\n${fake}\n```\n```` }); | | | check('N8 ~~~ fences and fences of 4 or more backticks are fences (a shorter inner fence does not close)', f.out === '', f.out); | | | } | | | | | | fs.rmSync(ROOT, { recursive: true, force: true }); | | | console.log(TOTAL ${passed}/${total}`); |
| | process.exit(passed === total ? 0 : 1); |
My coding agent said "done" 1,521 times. Half of those had nothing behind them.