| | # The nine questions used to triage 1,895 open GitHub issues | | | # (n8n, Supabase, Cal.com, Appwrite, Home Assistant) with Jev and Claude Sonnet. | | | # | | | # Both models got exactly this text: same questions, same one-line criteria. | | | # All nine are asked in one call per issue. The issue goes in as: |
| | # REPOSITORY: <repo> |
| | # TITLE: <title> |
| | # <body> | | | # | | | # Types: "choice" picks one option, "noul" is a yes/no probability, | | | # "score" places the issue on an ordered scale. | | | # | | | # Check: "kind" was compared with the labels maintainers put on the issues | | | # themselves (767 issues). Jev 96.1%, Sonnet 94.0%. |
| | QUESTIONS = { |
| | "kind": {"type": "choice", |
| | "instructions": "What kind of thing is this issue?", | | | "criteria": { | | | "bug": "Reports something broken or behaving wrongly.", | | | "feature": "Asks for new or changed capability.", | | | "docs": "Is about documentation being wrong, missing or unclear.", | | | "question": "Asks how to do something, with no defect claimed."}}, | | | "repro": {"type": "noul", | | | "instructions": "Does it contain steps another person could follow to reproduce the problem?", | | | "criteria": {"true": "Concrete steps, input, or a workflow another person could replay.", | | | "false": "Describes the symptom only, with no replayable steps."}}, | | | "version": {"type": "noul", | | | "instructions": "Does it state which version or environment this happened on?", | | | "criteria": {"true": "Names a version, build, deployment mode or environment.", | | | "false": "No version or environment stated anywhere."}}, | | | "expected": {"type": "noul", | | | "instructions": "Does it say what the author expected to happen instead?", | | | "criteria": {"true": "States the expected behaviour explicitly.", | | | "false": "Says what went wrong but never what should have happened."}}, | | | "actionable": {"type": "noul", | | | "instructions": "Could an engineer start work on this as written, without asking the author anything first?", | | | "criteria": {"true": "Everything needed to begin is already here.", | | | "false": "Someone would have to go back to the author before starting."}}, | | | "severity": {"type": "score", | | | "instructions": "How badly is this blocking the person who filed it?", | | | "criteria": ["cosmetic or a minor annoyance", | | | "works but is painful or needs a workaround", | | | "blocks their work entirely or loses data"]}, | | | "frustration": {"type": "noul", | | | "instructions": "Does the author sound frustrated enough to be at risk of leaving the product?", | | | "criteria": {"true": "Expresses frustration, lost time, or mentions leaving or switching.", |
| | "false": "Neutral or constructive in tone."}}, |
| | "data": {"type": "noul", |
| | "instructions": "Does this involve data being lost, corrupted or returned wrong?", | | | "criteria": {"true": "Data loss, corruption, duplication or incorrect results are involved.", |
| | "false": "No data integrity concern."}}, |
| | "workaround": {"type": "noul", |
| | "instructions": "Does the author already describe a workaround?", | | | "criteria": {"true": "A workaround or partial fix is described.", | | | "false": "No workaround is mentioned."}}, | | | } |