I ran Sol 6 vs. 6.1 on tasks from my own repo. It's cheaper and at least as good

…stet runs repair-ai-coverage --out /path/to/h2h-output --model-key model-one --c 292292

293293Use `--parse-retries` when saved `code_review` prompts exist but evaluator

294294responses failed JSON/schema parsing. The command rewrites validation artifacts

295−only after a successful repair and regenerates summaries/task details.

295+only after a successful repair and regenerates summaries/task details. Stored

296+prompts retain their measurement mode and truncation. Missing-prompt repair

297+requires the original task projection, matching artifact digests, persisted test

298+commands, and complete repository context; unavailable provenance stays

299+unresolved. Attempt counts and retry reasons distinguish recovery from an

300+initial successful grade; assertions and unrecoverable runtime errors are not

301+retried.

296302

297303Regrade additive custom or bundled graders from canonical task details:

298304

…const coverageValidationGolden = `{

843843 },

844844 "equiv_rescue_policy": "on",

845845 "equivalence": { 846+ "attempt_count": 1,

846847 "confidence": 0.95,

847848 "evaluator_model": "judge-a", 848849 "issue_type": "stylistic",

…func (p *repairAICoverageProducer) Produce(mctx *MutationContext) (*ProduceResul 274274 var mu sync.Mutex

275275 `runWithConcurrency(jobs, min(len(jobs), coverageRepairWorkers), func(job repairJob) {`

276276 `reporter.recordTaskStart(variant, job.TaskID, job.CurrentWork, time.Now().UTC())`

277− result, repairErr := validate.RepairCoverage(ctx, validate.CoverageRepairConfig{

277+ repairCfg := validate.CoverageRepairConfig{ 278278 Existing: job.Validation,

279279 AI: ai,

280280 AIModelID: resolvedAIModelID,

281281 RepairEquivalence: repairEquivalence && needsRepairCoverage(job.Validation, TaskGraderEquivalence, resolvedAIModelID),

282282 RepairCodeReview: repairCodeReview && needsRepairCoverage(job.Validation, TaskGraderCodeReview, resolvedAIModelID),

283283 ReviewParseRetries: cfg.ParseRetries,

284− }) 284+ }

285+ cleanup, reconstructionIssues := prepareCoverageRepairPrompts(&repairCfg, manifest, NewTaskArtifacts(outputRoot, variant, job.TaskID, ""))

286+ `if cleanup != nil {`

287+ `defer cleanup()`

288+ }

289+ result := validate.CoverageRepairResult{Validation: job.Validation} 290+ var repairErr error

291+ if repairCfg.RepairEquivalence || repairCfg.RepairCodeReview {

292+ result, repairErr = validate.RepairCoverage(ctx, repairCfg)

293+ }

294+ result.Unresolved = append(result.Unresolved, reconstructionIssues...)

285295 `if repairErr != nil {`

286296 `reporter.recordTaskEnd(variant, job.TaskID, job.CurrentWork, "failed", time.Now().UTC())`

287297 `mu.Lock()`

…func RunAdaptedReferenceTest(ctx context.Context, ai AIClient, promptCtx PromptC

292292 composedDiff, composeErr := composeAdaptedReferenceTestPatch(runCtx, inputs, referenceTestPatch, canonicalDiff)

293293 if composeErr != nil { 294294 result.Reason = "patch_compose_failed"

295− result.Summary = fmt.Sprintf("compose canonical reference test patch and adapted_diff: %v", composeErr)

295+ result.Summary = fmt.Sprintf("adapted_diff could not be composed with the canonical reference test patch: %v", composeErr)

296296 return result

297297 }

298298 composedPayload := payload …import (

99 "strings"

1010 "time"

1111

12+ "stet/internal/commandlist"

1213 "stet/internal/filecopy"

1314 "stet/internal/gitops"

1415 "stet/internal/harness"

…`func legacyCommandsThatNameAgentTests(commands, agentFiles []string) []string {`

192193 `files := normalizeAgentTestFiles(agentFiles)`

193194 `var out []string`

194195 `for _, command := range normalizeCommands(commands) {`

195− if commandHasPotentialNonPathTestSelection(command) || receiptBackedCommandHasShellControl(command) {

196+ if _, _, safe := commandlist.TestInvocation(command); !safe || commandHasPotentialNonPathTestSelection(command) || receiptBackedCommandHasShellControl(command) { 196197 continue

197198 }

198199 for _, file := range files { …import (

66 "fmt"

77 "os"

88 "path/filepath"

9+ "strconv"

910 "strings"

1011

12+ "stet/internal/commandlist"

1113 "stet/internal/shellcmd"

1214)

1315

…func replaceExactFinalTestPath(command, sourcePath, replacement string) (string,

198200}

199201

200202`func certifiedGoReplayCommand(command, sourcePath string) (string, bool) {`

201− `if !strings.HasSuffix(strings.ToLower(sourcePath), ".go") || !goCommandTargetsAgentTestFiles(command, []string{sourcePath}) {`

203+ `_, fields, safe := commandlist.TestInvocation(command)`

204+ `if !safe || len(fields) < 3 || fields[0].Value != "go" || fields[1].Value != "test" || !strings.HasSuffix(strings.ToLower(sourcePath), ".go") {`

202205 return "", false

203206 }

204− `replay, changed := removeGoRunSelector(command)`

205− `if !changed || hasRemainingGoTestSelector(replay) {`

207+ // Require one explicit package. A recursive selector or additional package

208+ // can credit tests outside the certified source package.

209+ `sourceDirectory := filepath.Dir(sourcePath)`

210+ `allFields, _ := shellcmd.FieldSpans(command)`

211+ // Go also consumes selectors from GOFLAGS. Recorded overrides are opaque

212+ // to this bounded rewrite, so they cannot certify an unfiltered replay.

213+ `for _, field := range allFields {`

214+ `if field.Start >= fields[0].Start {`

215+ break

216+ }

217+ if strings.HasPrefix(field.Value, "GOFLAGS=") { 218+ return "", false

219+ }

220+ }

221+ `if len(allFields) >= 3 && allFields[0].Value == "cd" {`

222+ `relative, err := filepath.Rel(allFields[1].Value, sourceDirectory)`

223+ `if err != nil || relative == ".." || strings.HasPrefix(relative, "../") {`

224+ return "", false

225+ }

226+ sourceDirectory = relative

227+ }

228+ `packagePath := "./" + filepath.ToSlash(sourceDirectory)`

229+ `if sourceDirectory == "." {`

230+ packagePath = "."

231+ }

232+ if strings.TrimSuffix(fields[2].Value, "/") != packagePath { 233+ return "", false

234+ }

235+ `start, end := -1, -1`

236+ `for i := 3; i < len(fields); i++ {`

237+ `value := fields[i].Value`

238+ `if value == "-run" || strings.HasPrefix(value, "-run=") {`

239+ `if start >= 0 {`

240+ return "", false

241+ }

242+ `start, end = fields[i].Start, fields[i].End`

243+ `if value == "-run" {`

244+ `if i+1 >= len(fields) || strings.HasPrefix(fields[i+1].Value, "-") {`

245+ return "", false

246+ }

247+ i++

248+ end = fields[i].End 249+ }

250+ continue

251+ }

252+ // Only flags with self-contained values may remain. Positional words,

253+ // passthrough args, and further selectors cannot be proven safe.

254+ `name := strings.SplitN(value, "=", 2)[0]`

255+ `if name == "-count" {`

256+ `countText, hasValue := strings.CutPrefix(value, "-count=")`

257+ `count, err := strconv.Atoi(countText)`

258+ `if !hasValue || err != nil || count <= 0 {`

259+ return "", false

260+ }

261+ }

262+ switch name {

263+ `case "-v", "-race", "-short", "-count", "-timeout", "-parallel", "-shuffle", "-tags", "-mod":`

264+ `if !strings.Contains(value, "=") && name != "-v" && name != "-race" && name != "-short" {`

265+ return "", false

266+ }

267+ default:

268+ return "", false

269+ }

270+ }

271+ if start < 0 {

206272 return "", false

207273 }

208− return replay, true

274+ return strings.TrimSpace(strings.TrimRight(command[:start], " \t") + command[end:]), true 209275}

210276

211277func normalizeReplayRelativePath(path string) string {

…func TestValidateAgentTestReplayPlanRejectsUnprovenCommand(t *testing.T) {

6161 t.Fatalf("validation error = %v", err) 6262 }

6363}

64+

65+func TestCertifiedGoReplayPreservesOnlyProvenInvocation(t *testing.T) {

66+ `command := "cd module && env MODE='test value' timeout 30s go test ./pkg -run 'TestHidden|TestOther' -count=1"`

67+ `got, ok := certifiedGoReplayCommand(command, "module/pkg/hidden_test.go")`

68+ `if !ok || got != "cd module && env MODE='test value' timeout 30s go test ./pkg -count=1" {`

69+ `t.Fatalf("replay = %q, %v", got, ok)`

70+ }

71+ for _, unsafe := range []string{ 72+ "echo go test ./pkg -run TestHidden", "go test ./pkg -run TestHidden; true",

73+ "go test ./pkg -run TestHidden && go test ./other", "go test ./... -run TestHidden",

74+ "go test ./pkg ./other -run TestHidden", "go test ./pkg -run TestHidden -skip TestOther",

75+ "go test ./pkg -run TestHidden -args payload", "go test ./pkg -run TestHidden -run TestOther",

76+ `"go test ./pkg -run TestHidden -count=0", "go test ./pkg -run TestHidden -count=-1",`

77+ `"go test ./pkg -run TestHidden -count=invalid", "go test ./pkg -run TestHidden -count 0",`

78+ `"go test ./pkg -run TestHidden -count -1",`

79+ `"GOFLAGS=-count=0 go test ./pkg -run TestHidden",`

80+ `"GOFLAGS='-run=Nothing' go test ./pkg -run TestHidden",`

81+ `"env GOFLAGS=-count=0 go test ./pkg -run TestHidden",`

82+ `"cd module && env GOFLAGS='-run=Nothing' go test ./pkg -run TestHidden",`

83+ `} {`

84+ `if replay, ok := certifiedGoReplayCommand(unsafe, "pkg/hidden_test.go"); ok {`

85+ t.Errorf("accepted %q as %q", unsafe, replay)

86+ }

87+ }

88+}

89+

90+func TestLegacyReplayRequiresAnActualTestRunner(t *testing.T) {

91+ commands := legacyCommandsThatNameAgentTests([]string{ 92+ "echo tests/agent_test.sh", "cat tests/agent_test.sh", "rm tests/agent_test.sh",

93+ "sh -c tests/agent_test.sh", "'CI=1' sh tests/agent_test.sh", "sh tests/agent_test.sh",

94+ `}, []string{"tests/agent_test.sh"})`

95+ `if len(commands) != 1 || commands[0] != "sh 'tests/agent_test.sh'" {`

96+ `t.Fatalf("legacy replay commands = %q", commands)`

97+ }

98+}

…func buildEquivalenceRetryPrompt(originalPrompt, rawOutput string, parseErr erro

1818 )

1919}

2020

21−func RunEquivalence(ctx context.Context, ai AIClient, promptCtx PromptContext) EquivalenceResult {

21+func RunEquivalence(ctx context.Context, ai AIClient, promptCtx PromptContext) (result EquivalenceResult) {

22+ mode := EquivalencePromptModeStandard

23+ if goldBlindEquivalenceEnabled() { 24+ mode = EquivalencePromptModeGoldBlind

25+ }

26+ history := newGradeAttemptClient(ai) 27+ ai = history

28+ defer func() { result.PromptMode = mode; history.annotateEquivalence(&result) }() 29+

2230 if goldBlindEquivalenceEnabled() { 2331 // No gold reaches the model, so gold truncation cannot have mutilated

2432 // the graded evidence: clear it before the truncation annotation runs.

…

11package validate

22

33import (

4+ "fmt"

45 "os"

56 "strconv"

67 "strings"

…notes: <short explanation tied to the task obligations>

127128// mode. In gold-blind mode the gold patch is dropped from the template data as

128129// well as from the template, so no gold content can reach the model.

129130func renderEquivalencePrompt(promptCtx PromptContext) (prompt string, goldBlind bool, err error) {

130− `if !goldBlindEquivalenceEnabled() {`

131− `rendered, renderErr := renderPrompt(equivalencePromptTemplate, promptCtx.asTemplateData())`

132− return rendered, false, renderErr

131+ mode := EquivalencePromptModeStandard

132+ if goldBlindEquivalenceEnabled() { 133+ mode = EquivalencePromptModeGoldBlind

134+ }

135+ rendered, err := RenderEquivalencePromptForMode(promptCtx, mode) 136+ return rendered, mode == EquivalencePromptModeGoldBlind, err

137+}

138+

139+// RenderEquivalencePromptForMode uses recorded measurement authority, never the process environment.

140+func RenderEquivalencePromptForMode(promptCtx PromptContext, mode EquivalencePromptMode) (string, error) {

141+ switch mode {

142+ case EquivalencePromptModeStandard:

143+ return renderPrompt(equivalencePromptTemplate, promptCtx.asTemplateData())

144+ case EquivalencePromptModeGoldBlind:

145+ `data := promptCtx.asTemplateData()`

146+ `delete(data, "GoldPatch")`

147+ return renderPrompt(equivalenceGoldBlindPromptTemplate, data)

148+ default:

149+ return "", fmt.Errorf("unknown equivalence prompt mode %q", mode)

133150 }

134− `data := promptCtx.asTemplateData()`

135− `delete(data, "GoldPatch")`

136− rendered, renderErr := renderPrompt(equivalenceGoldBlindPromptTemplate, data)

137− return rendered, true, renderErr

138151}

…func runPerCommandTests(

22022202 `cmdCfg.Commands = []string{cmd}`

22032203 `goldRun, err := RunGoldBaselineTests(infraCtx, inputs, cmdCfg)`

22042204 gr.EnvCache = goldRun.EnvCache

2205− `gr.Result = TestCommandResult{Command: cmd}`

2205+ `gr.Result = TestCommandResult{Command: cmd, AttemptCount: 1}`

22062206 `if err != nil {`

22072207 gr.Err = err

2208− `if mode := inferInfraFailureModeFromError(err); mode != "" {`

2208+ `if mode := retryableVerifierFailureMode(goldRun, err); mode != "" {`

22092209 gr.Result.GoldOutcome = TestOutcomeFail

22102210 gr.Result.GoldFailureMode = mode

22112211 gr.IsInfra = true

…func runPerCommandTests(

22182218 gr.Result.GoldOutcome = TestOutcomeUnknown

22192219 gr.Notes = append(gr.Notes, fmt.Sprintf("gold baseline failed for %q: %v", cmd, err)) 22202220 }

2221− return

2221+ if !isRetryableGoldInfrastructureFailure(gr.Result.GoldFailureMode) {

2222+ return

2223+ }

22222224 }

22232225 gr.EnvCache = goldRun.EnvCache

22242226 applyGoldRunResult(gr, goldRun)

2225− `if gr.Result.GoldOutcome == TestOutcomeFail && isRetryableGoldInfrastructureFailure(gr.Result.GoldFailureMode) && budgetCtx.Err() == nil {`

2227+ `if isUnsetFailureMode(gr.Result.GoldFailureMode) {`

2228+ gr.Result.GoldFailureMode = retryableVerifierFailureMode(goldRun, err)

2229+ }

2230+ if gr.Result.GoldOutcome != TestOutcomePass && isRetryableGoldInfrastructureFailure(gr.Result.GoldFailureMode) && budgetCtx.Err() == nil { 22262231 beginGoldRetry(&gr.Result, goldRun)

2232+ recordTestRetry(&gr.Result, gr.Result.GoldFailureMode)

22272233 `retryCfg := goldRetryAttemptConfig(overrideCfg, gr.Index, 2)`

22282234 `retryCfg.Commands = []string{cmd}`

22292235 retryRun, retryErr := RunGoldBaselineTests(infraCtx, inputs, retryCfg)

…func runPerCommandTests(

22442250 gr.Result.GoldRetried = true

22452251 finishGoldRetry(&gr.Result, retryRun)

22462252 if gr.Result.GoldOutcome == TestOutcomePass { 2253+ gr.Result.RecoveredAfterRetry = true

2254+ gr.Err = nil

2255+ gr.IsInfra = false

22472256 gr.Notes = append(gr.Notes, fmt.Sprintf("gold baseline for %q recovered on retry after transient dependency fetch", cmd))

22482257 } else { 22492258 gr.Notes = append(gr.Notes, fmt.Sprintf("gold baseline retry for %q did not recover", cmd))

…func runPerCommandTests(

24842493 `cmdCfg := overrideCfg`

24852494 `cmdCfg.Commands = []string{entry.cmd}`

24862495 or.Attempted = true

2496+ or.Result.AttemptCount++

24872497 overrideRun, err := RunOverrideTests(budgetCtx, inputs, cmdCfg) 24882498 or.EnvCache = overrideRun.EnvCache

24892499 `if err != nil {`

2490− `if mode := inferInfraFailureModeFromError(err); mode != "" {`

2500+ `if mode := retryableVerifierFailureMode(overrideRun, err); mode != "" {`

24912501 or.Result.AgentFailureMode = mode

24922502 `if mode == "timeout" {`

24932503 `or.Notes = append(or.Notes, fmt.Sprintf("agent retest timeout for %q", entry.cmd))`

…func runPerCommandTests(

24962506 or.Notes = append(or.Notes, fmt.Sprintf("agent retest failed for %q: %v", entry.cmd, err)) 24972507 or.Result.AgentOutcome = TestOutcomeUnknown

24982508 or.Err = err

2499− return

2509+ if !isRetryableGoldInfrastructureFailure(or.Result.AgentFailureMode) {

2510+ return

2511+ }

25002512 }

25012513 or.Result.AgentOutcome = testOutcomeFromTrial(overrideRun.Trial)

25022514 or.Result.AgentTrialName = overrideRun.Trial.TrialName

2503− or.Result.AgentFailureMode = overrideRun.Trial.FailureMode

2515+ or.Result.AgentFailureMode = retryableVerifierFailureMode(overrideRun, err)

25042516

2505− // Retry once if agent failed but gold passed.

2506− if or.Result.AgentOutcome == TestOutcomeFail && or.Result.GoldOutcome == TestOutcomePass && budgetCtx.Err() == nil { 2517+ // Retry only a typed transient infrastructure failure; assertions are terminal.

2518+ `if or.Result.AgentOutcome != TestOutcomePass && isRetryableGoldInfrastructureFailure(or.Result.AgentFailureMode) && or.Result.GoldOutcome == TestOutcomePass && budgetCtx.Err() == nil {`

25072519 `retryCfg := overrideCfg`

25082520 `retryCfg.Commands = []string{entry.cmd}`

2521+ recordTestRetry(&or.Result, or.Result.AgentFailureMode)

2522+ or.Result.AgentRetried = true

25092523 retryRun, retryErr := RunOverrideTests(budgetCtx, inputs, retryCfg) 25102524 or.RetryEnvCache = &retryRun.EnvCache

2525+ or.Result.AgentOutcome = testOutcomeFromTrial(retryRun.Trial)

2526+ or.Result.AgentTrialName = retryRun.Trial.TrialName

2527+ or.Result.AgentFailureMode = retryableVerifierFailureMode(retryRun, retryErr)

2528+ or.Err = retryErr

25112529 `if retryErr == nil {`

25122530 `retryOutcome := testOutcomeFromTrial(retryRun.Trial)`

25132531 `if retryOutcome == TestOutcomePass {`

…func runPerCommandTests(

25152533 or.Result.AgentTrialName = retryRun.Trial.TrialName

25162534 or.Result.AgentFailureMode = retryRun.Trial.FailureMode

25172535 or.Result.AgentRetried = true

2536+ or.Result.RecoveredAfterRetry = true

2537+ or.Err = nil

25182538 or.Notes = append(or.Notes, fmt.Sprintf("agent retest for %q succeeded on retry", entry.cmd))

25192539 }

25202540 }

…func runGoldBatchSlice(

347347 `cleanups := []func(){}`

348348 `for _, idx := range indexes {`

349349 `gr := &goldResults[idx]`

350+ `priorResult := gr.Result`

350351 priorRetried, priorEvidence := gr.Result.GoldRetried, gr.Result.GoldRetryEvidence

351352 gr.IsInfra = false

352353 gr.IsGlobalInfra = false

353354 gr.Err = nil

354355 `if ctx.Err() != nil {`

355356 `gr.Result = TestCommandResult{`

356− Command: gr.Command,

357− GoldOutcome: TestOutcomeUnknown,

358− GoldRetried: priorRetried,

359− GoldRetryEvidence: priorEvidence,

357+ Command: gr.Command,

358+ GoldOutcome: TestOutcomeUnknown,

359+ GoldRetried: priorRetried,

360+ GoldRetryEvidence: priorEvidence,

361+ AttemptCount: priorResult.AttemptCount,

362+ RetryCount: priorResult.RetryCount,

363+ RetryReasons: priorResult.RetryReasons,

364+ RecoveredAfterRetry: priorResult.RecoveredAfterRetry,

360365 }

361366 gr.Err = ctx.Err() 362367 continue

…func runGoldBatchSlice(

364369 stageRoot, cacheResult, cleanup, err := stageGoldBaselineTask(ctx, inputs, []string{commands[idx]}, false, overrideCfg.OutputDir) 365370 gr.EnvCache = cacheResult

366371 gr.Result = TestCommandResult{ 367− Command: gr.Command,

368− GoldRetried: priorRetried,

369− GoldRetryEvidence: priorEvidence,

372+ Command: gr.Command,

373+ GoldRetried: priorRetried,

374+ GoldRetryEvidence: priorEvidence,

375+ AttemptCount: priorResult.AttemptCount,

376+ RetryCount: priorResult.RetryCount,

377+ RetryReasons: priorResult.RetryReasons,

378+ RecoveredAfterRetry: priorResult.RecoveredAfterRetry,

370379 }

371380 if err != nil { 372381 gr.Err = err

…func runGoldBatchSlice(

407416 gr.Result.GoldRetried = true

408417 gr.Result.GoldRetryEvidence = &TestRetryEvidence{First: *gr.FirstRetryEvidence}

409418 }

419+ recordTestRetry(&gr.Result, FailureModeInfraDependencyFetch)

410420 gr.RetryDispatched = true

411421 }

412422 }

413423

424+ for _, item := range items { 425+ var idx int

426+ `fmt.Sscanf(item.ProjectionKey, "gold-%d", &idx)`

427+ `if attempt == 1 {`

428+ goldResults[idx].Result.AttemptCount++

429+ }

430+ }

414431 `results, err := batchRunner.RunBatch(ctx, harness.BatchRunConfig{`

415432 `OutputDir:           filepath.Join(filepath.Dir(items[0].SourceTaskDir), ".tb-runs"),`

416433 `RunID:               fmt.Sprintf("validate-gold-batch-%d", time.Now().UnixNano()),`

…func runGoldBatchSlice(

452469 evidence := retainBatchGoldRetryAttemptEvidence(projected, gr.Result.GoldFailureMode, overrideCfg, gr.Index, attempt)

453470 gr.FirstRetryEvidence = &evidence

454471 }

472+ if attempt > 1 && gr.Result.GoldOutcome == TestOutcomePass { 473+ gr.Result.RecoveredAfterRetry = true

474+ }

455475 if attempt > 1 && gr.Result.GoldRetryEvidence != nil {

456476 evidence := retainBatchGoldRetryAttemptEvidence(projected, gr.Result.GoldFailureMode, overrideCfg, gr.Index, attempt)

457477 gr.Result.GoldRetryEvidence.Retry = &evidence

…func runOverrideBatchSlice(

564584 return

565585 }

566586

587+ for _, item := range items { 588+ var idx int

589+ `fmt.Sscanf(item.ProjectionKey, "override-%d", &idx)`

590+ `overrideResults[idx-offset].Result.AttemptCount++`

591+ }

592+

567593 `results, err := batchRunner.RunBatch(ctx, harness.BatchRunConfig{`

568594 `OutputDir:           filepath.Join(filepath.Dir(items[0].SourceTaskDir), ".tb-runs"),`

569595 `RunID:               fmt.Sprintf("validate-override-batch-%d", time.Now().UnixNano()),`

…func runOverrideBatchSlice(

604630 projected := results[fmt.Sprintf("override-%d", offset+i)] 605631 or.Result.AgentOutcome = testOutcomeFromTrial(projected.Trial)

606632 or.Result.AgentTrialName = projected.Trial.TrialName

607− or.Result.AgentFailureMode = projected.Trial.FailureMode

608− if or.Result.AgentOutcome == TestOutcomeFail && or.Result.GoldOutcome == TestOutcomePass && ctx.Err() == nil { 633+ or.Result.AgentFailureMode = classifyBatchGoldFailureMode(projected)

634+ `if or.Result.AgentOutcome != TestOutcomePass && isRetryableGoldInfrastructureFailure(or.Result.AgentFailureMode) && or.Result.GoldOutcome == TestOutcomePass && ctx.Err() == nil {`

609635 `stageRoot, cacheResult, cleanup, err := stageOverrideTask(ctx, inputs, []string{or.Command}, false, overrideCfg.OutputDir)`

610636 `if err == nil {`

611637 or.RetryEnvCache = &cacheResult

…func runOverrideBatchSlice(

623649 if len(retryItems) == 0 { 624650 return

625651 }

652+ `for _, idx := range retryIndexes {`

653+ `or := &overrideResults[idx]`

654+ recordTestRetry(&or.Result, or.Result.AgentFailureMode)

655+ or.Result.AgentRetried = true

656+ }

626657 `retryResults, err := batchRunner.RunBatch(ctx, harness.BatchRunConfig{`

627658 `OutputDir:           filepath.Join(filepath.Dir(retryItems[0].SourceTaskDir), ".tb-runs"),`

628659 `RunID:               fmt.Sprintf("validate-override-retry-batch-%d", time.Now().UnixNano()),`

…func runOverrideBatchSlice(

634665 HarnessCLICacheMode: overrideCfg.HarnessCLICacheMode,

635666 Items: retryItems,

636667 `})`

637− `if err != nil {`

638− return

639− }

640668 `for _, idx := range retryIndexes {`

641669 `or := &overrideResults[idx]`

642670 `projected := retryResults[fmt.Sprintf("retry-%d", offset+idx)]`

643− `if testOutcomeFromTrial(projected.Trial) == TestOutcomePass {`

644− or.Result.AgentOutcome = TestOutcomePass

645− or.Result.AgentTrialName = projected.Trial.TrialName

646− or.Result.AgentFailureMode = projected.Trial.FailureMode

647− or.Result.AgentRetried = true

671+ or.Result.AgentOutcome = testOutcomeFromTrial(projected.Trial)

672+ or.Result.AgentTrialName = projected.Trial.TrialName

673+ or.Result.AgentFailureMode = classifyBatchGoldFailureMode(projected)

674+ or.Err = err

675+ `if err != nil {`

676+ `if isUnsetFailureMode(or.Result.AgentFailureMode) {`

677+ or.Result.AgentFailureMode = inferInfraFailureModeFromError(err)

678+ }

679+ or.Notes = append(or.Notes, fmt.Sprintf("agent retest retry for %q failed: %v", or.Command, err)) 680+ }

681+ if err == nil && or.Result.AgentOutcome == TestOutcomePass { 682+ or.Result.RecoveredAfterRetry = true

648683 or.Notes = append(or.Notes, fmt.Sprintf("agent retest for %q succeeded on retry", or.Command))

649684 }

650685 }

…go: golang.org/x/sys@v0.42.0: Get "https://proxy.golang.org/golang.org/x/sys/@v/

50215021 t.Fatalf("command results = %d, want 1", len(summary.CommandResults)) 50225022 }

50235023 result := summary.CommandResults[0] 5024+ if result.AttemptCount != 3 || result.RetryCount != 1 || !result.RecoveredAfterRetry || len(result.RetryReasons) != 1 || result.RetryReasons[0] != FailureModeInfraDependencyFetch {

5025+ t.Fatalf("retry metadata = %+v", result) 5026+ }

50245027 if !result.GoldRetried || result.GoldOutcome != TestOutcomePass || result.GoldTrialName != "gold-retry" {

50255028 t.Fatalf("gold retry result = %+v", result) 50265029 }

…go: golang.org/x/sys@v0.42.0: Get "https://proxy.golang.org/golang.org/x/sys/@v/

51115114 t.Fatalf("command results = %d, want 1", len(summary.CommandResults)) 51125115 }

51135116 result := summary.CommandResults[0] 5117+ if result.AttemptCount != 3 || result.RetryCount != 1 || !result.RecoveredAfterRetry || len(result.RetryReasons) != 1 || result.RetryReasons[0] != FailureModeInfraDependencyFetch {

5118+ t.Fatalf("retry metadata = %+v", result) 5119+ }

51145120 if !result.GoldRetried || result.GoldOutcome != TestOutcomePass || result.GoldTrialName != "gold-retry" {

51155121 t.Fatalf("gold retry result = %+v", result) 51165122 }

…func TestRunPerCommandTestsBatchedGoldTimeoutClassifiedAsTimeBudgetExhausted(t *

53825388 }

53835389}

53845390

5385−func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried(t *testing.T) {

5391+func TestRunPerCommandTestsBatchedRetriesDependencyFetchOverrideAndMarksRetried(t *testing.T) {

53865392 `t.Parallel()`

53875393 `repo, base, head := createRepoWithCommits(t)`

53885394 `dataset := t.TempDir()`

…func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried(t *testin

54185424 `runner := &batchValidateRunner{`

54195425 `results: map[string]TBTrialResult{`

54205426 `"gold-0":     {TaskID: taskID, IsResolved: &pass, TrialName: "gold-a"},`

5421− `"override-0": {TaskID: taskID, IsResolved: &fail, TrialName: "override-a"},`

5427+ `"override-0": {TaskID: taskID, IsResolved: &fail, TrialName: "override-a", FailureMode: FailureModeInfraDependencyFetch},`

54225428 `"retry-0":    {TaskID: taskID, IsResolved: &pass, TrialName: "retry-a"},`

54235429 },

54245430 }

…func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried(t *testin

54435449 t.Fatalf("expected 1 command result, got %d", len(result.Tests.CommandResults))

54445450 }

54455451 `cmd := result.Tests.CommandResults[0]`

5452+ `if cmd.AttemptCount != 3 || cmd.RetryCount != 1 || !cmd.RecoveredAfterRetry || len(cmd.RetryReasons) != 1 || cmd.RetryReasons[0] != FailureModeInfraDependencyFetch {`

5453+ `t.Fatalf("retry metadata = %+v", cmd)`

5454+ }

54465455 if !cmd.AgentRetried {

54475456 t.Fatal("expected AgentRetried=true") 54485457 }

…

11package validate

22

3−import "fmt"

3+import (

4+ "fmt"

5+ "path/filepath"

6+ "strings"

7+)

48

59// OfflinePromptInputs describes everything needed to reassemble the grader

610// PromptContext for a single already-evaluated cell without re-running the

…type OfflinePromptInputs struct {

1822 // TBResultsRaw is the raw TB results JSON for this cell, or "" when

1923 // unavailable (e.g. no-test-signal corpora). 2024 TBResultsRaw string

25+ // EffectiveTestCommands overrides dataset commands when non-nil. An empty

26+ // non-nil slice preserves a persisted grade with no verifier commands.

27+ EffectiveTestCommands []string 2128 // PromptPatchMaxBytes caps the gold/agent patch bytes embedded in the

2229 // prompt. Values <= 0 fall back to DefaultPromptPatchMaxBytes, matching the

2330 // orchestrator/config default.

…type OfflinePromptInputs struct {

2532 // RepoContextOptions controls repo tree/excerpt extraction. The zero value

2633 // inherits BuildRepoContext defaults, matching the live default path.

2734 RepoContextOptions RepoContextOptions

35+ // RequireRepoContext makes reconstruction fail when the source repository

36+ // cannot be extracted, instead of grading an unavailable-context marker.

37+ RequireRepoContext bool

38+ // ExpectedArtifacts binds reconstruction to the original graded files.

39+ // When supplied, every source path and digest must be present and match.

40+ ExpectedArtifacts *ArtifactManifest

2841}

2942

3043// BuildOfflinePromptContext reassembles the grader PromptContext for one cell

…type OfflinePromptInputs struct {

3447// that removes the extracted repo temp dir; callers should defer it.

3548//

3649// The assembly intentionally mirrors internal/validate/orchestrator.go so the

37−// resulting context is byte-for-byte identical to a live grade except for the

38−// template version: it reuses ReadTaskInputs, truncatePatch, and

39−// BuildRepoContext, and applies the same graceful repo-context fallback.

50+// assembly reuses ReadTaskInputs, truncatePatch, and BuildRepoContext. Callers

51+// can supply persisted effective commands and require usable repository context;

52+// otherwise it applies the live path's graceful repo-context fallback.

4053func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), error) { 4154 inputs, err := ReadTaskInputs(in.DatasetPath, in.TaskID, in.AgentPatchPath)

4255 `if err != nil {`

…`func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e`

4558 `if err := verifyRepoSourceForConsumption(inputs); err != nil {`

4659 `return PromptContext{}, nil, err`

4760 }

61+ `if in.ExpectedArtifacts != nil {`

62+ `current := BuildArtifactManifest(inputs)`

63+ `for _, pair := range []struct {`

64+ name string

65+ want, got ArtifactFileManifest

66+ }{ 67+ {"agent patch", in.ExpectedArtifacts.AgentPatch, current.AgentPatch},

68+ {"gold patch", in.ExpectedArtifacts.GoldPatch, current.GoldPatch},

69+ {"repository", in.ExpectedArtifacts.RepoTarball, current.RepoTarball},

70+ `} {`

71+ `if pair.want.Path == "" || pair.want.SHA256 == "" {`

72+ return PromptContext{}, nil, fmt.Errorf("missing original %s path or digest", pair.name)

73+ }

74+ if filepath.Clean(pair.want.Path) != filepath.Clean(pair.got.Path) || !strings.EqualFold(strings.TrimPrefix(pair.want.SHA256, "sha256:"), pair.got.SHA256) { 75+ return PromptContext{}, nil, fmt.Errorf("original %s provenance mismatch", pair.name)

76+ }

77+ }

78+ }

4879

4980 maxBytes := in.PromptPatchMaxBytes

5081 `if maxBytes <= 0 {`

…`func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e`

5283 }

5384 goldPatchPrompt, goldPatchTruncation := truncatePatch(inputs.GoldPatch, maxBytes)

5485 agentPatchPrompt, agentPatchTruncation := truncatePatch(inputs.AgentPatch, maxBytes)

86+ if in.ExpectedArtifacts != nil && (goldPatchTruncation.Truncated || agentPatchTruncation.Truncated) {

87+ return PromptContext{}, nil, fmt.Errorf("reconstructed prompt patches would be truncated; original context budgets are unavailable")

88+ }

89+ testCommands := inputs.TestCommands

90+ if in.EffectiveTestCommands != nil {

91+ testCommands = append([]string{}, in.EffectiveTestCommands...) 92+ }

5593

5694 // Mirror orchestrator.go: on repo-context failure, degrade RepoTree to an

5795 // "unavailable: <err>" marker and continue rather than failing the build.

…`func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e`

5997 `repoCtx := RepoContext{}`

6098 builtRepoCtx, repoCleanup, repoErr := BuildRepoContext(inputs.RepoTarballPath, in.RepoContextOptions)

6199 if repoErr != nil { 100+ if in.RequireRepoContext {

101+ return PromptContext{}, nil, fmt.Errorf("build required repo context: %w", repoErr) 102+ }

62103 `builtRepoCtx = RepoContext{`

63104 `RepoTree: fmt.Sprintf("unavailable: %v", repoErr),`

64105 }

65106 } else { 66107 cleanup = repoCleanup

108+ if in.ExpectedArtifacts != nil && (builtRepoCtx.RepoTreeTruncation.Truncated || builtRepoCtx.RepoExcerptsTruncation.Truncated) {

109+ `if cleanup != nil {`

110+ `cleanup()`

111+ }

112+ return PromptContext{}, nil, fmt.Errorf("reconstructed repository context would be truncated; original context budgets are unavailable")

113+ }

67114 }

68115 repoCtx = builtRepoCtx

69116

…func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e 71118 TaskYAML: inputs.TaskYAML,

72119 GoldPatch: goldPatchPrompt,

73120 AgentPatch: agentPatchPrompt,

74− TestCommands: inputs.TestCommands,

121+ TestCommands: testCommands,

75122 TBResults: in.TBResultsRaw,

76123 RepoPath: repoCtx.RepoPath,

77124 RepoTree: repoCtx.RepoTree,

…func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e 80127 GoldPatchTruncated: goldPatchTruncation.Truncated,

81128 }, cleanup, nil

82129}

130+

131+// RenderReviewPrompt renders the built-in review template from supplied evidence.

132+`func RenderReviewPrompt(promptCtx PromptContext) (string, error) {`

133+ `return renderPrompt(reviewPromptTemplate, promptCtx.asTemplateData())`

134+}

…func TestBuildOfflinePromptContextDegradesWhenTarballUnextractable(t *testing.T)

164164 t.Fatalf("expected agent patch to be populated, got %q", promptCtx.AgentPatch)

165165 }

166166}

167+

168+func TestBuildOfflinePromptContextRequiresUsableRepositoryWhenRequested(t *testing.T) {

169+ `datasetPath, taskID, agentPatchPath := writeOfflineTaskDir(t, false)`

170+ `_, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{`

171+ DatasetPath: datasetPath, TaskID: taskID, AgentPatchPath: agentPatchPath,

172+ RequireRepoContext: true,

173+ `})`

174+ `if cleanup != nil {`

175+ `defer cleanup()`

176+ }

177+ if err == nil || !strings.Contains(err.Error(), "build required repo context") { 178+ t.Fatalf("expected required repository extraction failure, got %v", err)

179+ }

180+}

181+

182+func TestBuildOfflinePromptContextUsesEffectiveVerifierCommands(t *testing.T) {

183+ `datasetPath, taskID, agentPatchPath := writeOfflineTaskDir(t, true)`

184+ `for _, commands := range [][]string{nil, {}, {"effective verifier"}} {`

185+ `ctx, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{`

186+ DatasetPath: datasetPath, TaskID: taskID, AgentPatchPath: agentPatchPath,

187+ EffectiveTestCommands: commands,

188+ `})`

189+ `if cleanup != nil {`

190+ `defer cleanup()`

191+ }

192+ `if err != nil {`

193+ `t.Fatal(err)`

194+ }

195+ `want := commands`

196+ `if commands == nil {`

197+ `want = []string{"pytest -q"}`

198+ }

199+ if !reflect.DeepEqual(ctx.TestCommands, want) { 200+ t.Fatalf("effective commands %v: got %v, want %v", commands, ctx.TestCommands, want)

201+ }

202+ }

203+}

204+

205+func TestBuildOfflinePromptContextRejectsMutatedOriginalArtifacts(t *testing.T) {

206+ `for _, artifact := range []string{"agent patch", "gold patch", "repository"} {`

207+ `t.Run(artifact, func(t *testing.T) {`

208+ `dataset, taskID, patch := writeOfflineTaskDir(t, true)`

209+ `inputs, err := ReadTaskInputs(dataset, taskID, patch)`

210+ `if err != nil {`

211+ `t.Fatal(err)`

212+ }

213+ `original := BuildArtifactManifest(inputs)`

214+ `path := patch`

215+ `if artifact == "gold patch" {`

216+ path = inputs.GoldPatchPath

217+ }

218+ if artifact == "repository" { 219+ path = inputs.RepoTarballPath

220+ }

221+ `if err := os.WriteFile(path, []byte("mutated evidence"), 0o644); err != nil {`

222+ `t.Fatal(err)`

223+ }

224+ _, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{ 225+ DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch,

226+ ExpectedArtifacts: &original, RequireRepoContext: true,

227+ `})`

228+ `if cleanup != nil {`

229+ `defer cleanup()`

230+ }

231+ if err == nil || !strings.Contains(err.Error(), artifact+" provenance mismatch") { 232+ t.Fatalf("expected original %s mismatch, got %v", artifact, err)

233+ }

234+ }) 235+ }

236+}

237+

238+func TestBuildOfflinePromptContextRejectsAbsentOriginalArtifactDigest(t *testing.T) {

239+ `dataset, taskID, patch := writeOfflineTaskDir(t, true)`

240+ `inputs, err := ReadTaskInputs(dataset, taskID, patch)`

241+ `if err != nil {`

242+ `t.Fatal(err)`

243+ }

244+ original := BuildArtifactManifest(inputs) 245+ original.AgentPatch.SHA256 = ""

246+ _, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{ 247+ DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch, ExpectedArtifacts: &original,

248+ `})`

249+ `if cleanup != nil {`

250+ `defer cleanup()`

251+ }

252+ if err == nil || !strings.Contains(err.Error(), "missing original agent patch") { 253+ t.Fatalf("expected absent original provenance failure, got %v", err)

254+ }

255+}

256+

257+func TestBuildOfflinePromptContextRejectsNewTruncationWithOriginalProvenance(t *testing.T) {

258+ `dataset, taskID, patch := writeOfflineTaskDir(t, true)`

259+ `if err := os.WriteFile(patch, []byte(largePatch(4_000)), 0o644); err != nil {`

260+ `t.Fatal(err)`

261+ }

262+ `inputs, err := ReadTaskInputs(dataset, taskID, patch)`

263+ `if err != nil {`

264+ `t.Fatal(err)`

265+ }

266+ `original := BuildArtifactManifest(inputs)`

267+ `_, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{`

268+ DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch, ExpectedArtifacts: &original,

269+ PromptPatchMaxBytes: 600,

270+ `})`

271+ `if cleanup != nil {`

272+ `defer cleanup()`

273+ }

274+ if err == nil || !strings.Contains(err.Error(), "would be truncated") { 275+ t.Fatalf("expected new truncation refusal, got %v", err)

276+ }

277+}

…const (

2828var coverageRepairRetryBackoffs = []time.Duration{time.Second, 2 * time.Second} 2929

3030type CoverageRepairConfig struct {

31+ EquivalenceAI AIClient

32+ EquivalencePrompt string

33+ EquivalencePromptContext *PromptContext

34+ EquivalencePromptMode EquivalencePromptMode

35+ ReviewAI AIClient

36+ ReviewPrompt string

37+ ReviewPromptContext *PromptContext

38+

3139 Existing ValidationResult

3240 AI AIClient

3341 AIModelID string

…type CoverageRepairConfig struct {

3745}

3846

3947type CoverageRepairIssue struct {

48+ AttemptCount int `json:"attempt_count,omitempty"`

49+ RetryCount int `json:"retry_count,omitempty"`

50+ RetryReasons []string `json:"retry_reasons,omitempty"`

51+

4052 Dimension string `json:"dimension"`

4153 Reason string `json:"reason"`

4254 Category string `json:"category,omitempty"`

…func RepairCoverage(ctx context.Context, cfg CoverageRepairConfig) (CoverageRepa

6981 updated := false 7082

7183 if cfg.RepairEquivalence {

72− equivalence, repaired, issue := repairEquivalenceFromPrompt(ctx, ai, cfg.Existing.Equivalence, cfg.AIModelID, cfg.ReviewParseRetries)

84+ source := cfg.Existing.Equivalence

85+ if cfg.EquivalencePrompt != "" { 86+ source.Prompt = cfg.EquivalencePrompt

87+ }

88+ mode := source.PromptMode 89+ if cfg.EquivalencePromptMode != "" {

90+ if mode != "" && mode != cfg.EquivalencePromptMode { 91+ return result, errors.New("equivalence repair prompt mode differs from original measurement")

92+ }

93+ mode = cfg.EquivalencePromptMode

94+ }

95+ if cfg.EquivalencePromptContext != nil {

96+ rendered, err := RenderEquivalencePromptForMode(*cfg.EquivalencePromptContext, mode)

97+ if err != nil { 98+ return result, err

99+ }

100+ if cfg.EquivalencePrompt != "" && cfg.EquivalencePrompt != rendered {

101+ return result, errors.New("equivalence repair prompt differs from supplied context")

102+ }

103+ source.Prompt = rendered

104+ }

105+ source.PromptMode = mode

106+ `dimensionAI := cfg.EquivalenceAI`

107+ `if dimensionAI == nil {`

108+ dimensionAI = ai

109+ }

110+ history := newGradeAttemptClient(dimensionAI) 111+ equivalence, repaired, issue := repairEquivalenceFromPrompt(ctx, history, source, cfg.AIModelID, cfg.ReviewParseRetries)

112+ if repaired {

113+ equivalence.PromptMode = mode

114+ if cfg.EquivalencePromptContext != nil {

115+ pc := *cfg.EquivalencePromptContext

116+ if mode == EquivalencePromptModeGoldBlind { 117+ pc.GoldPatchTruncated = false

118+ }

119+ equivalence = annotateEquivalenceTruncation(equivalence, pc)

120+ }

121+ history.annotateEquivalence(&equivalence)

122+ mergeEquivalenceRepairHistory(&equivalence, source)

123+ }

73124 if issue != nil { 125+ issue.AttemptCount = history.attempts

126+ `issue.RetryCount = max(0, history.attempts-1)`

127+ `issue.RetryReasons = append([]string(nil), history.reasons...)`

74128 result.Unresolved = append(result.Unresolved, *issue)

75129 } else { 76130 result.Validation.Equivalence = equivalence

…func RepairCoverage(ctx context.Context, cfg CoverageRepairConfig) (CoverageRepa

84138 if strings.TrimSpace(source.Prompt) == "" && cfg.Existing.Review != nil { 85139 source = *cfg.Existing.Review

86140 }

87− review, repaired, issue := repairReviewFromPrompt(ctx, ai, source, cfg.AIModelID, cfg.ReviewParseRetries)

141+ if cfg.ReviewPrompt != "" { 142+ source.Prompt = cfg.ReviewPrompt

143+ }

144+ if cfg.ReviewPromptContext != nil {

145+ `rendered, err := renderPrompt(reviewPromptTemplate, cfg.ReviewPromptContext.asTemplateData())`

146+ `if err != nil {`

147+ return result, err

148+ }

149+ if cfg.ReviewPrompt != "" && cfg.ReviewPrompt != rendered {

150+ return result, errors.New("review repair prompt differs from supplied context")

151+ }

152+ source.Prompt = rendered

153+ }

154+ if source.Prompt != resolvedRepairReview(cfg.Existing).Prompt { 155+ source.Output = ""

156+ }

157+ `dimensionAI := cfg.ReviewAI`

158+ `if dimensionAI == nil {`

159+ dimensionAI = ai

160+ }

161+ history := newGradeAttemptClient(dimensionAI) 162+ review, repaired, issue := repairReviewFromPromptContext(ctx, history, source, cfg.AIModelID, cfg.ReviewParseRetries, cfg.ReviewPromptContext)

163+ if repaired && history.attempts > 0 {

164+ history.annotateReview(&review) 165+ mergeReviewRepairHistory(&review, source)

166+ }

167+ if repaired && cfg.ReviewPromptContext != nil {

168+ review = annotateReviewTruncation(review, *cfg.ReviewPromptContext)

169+ }

88170 if issue != nil { 171+ issue.AttemptCount = history.attempts

172+ `issue.RetryCount = max(0, history.attempts-1)`

173+ `issue.RetryReasons = append([]string(nil), history.reasons...)`

89174 result.Unresolved = append(result.Unresolved, *issue)

90175 } else { 91176 result.Validation.CodeReview = review

…func RepairCoverage(ctx context.Context, cfg CoverageRepairConfig) (CoverageRepa

104189}

105190

106191func repairEquivalenceFromPrompt(ctx context.Context, ai AIClient, existing EquivalenceResult, overrideModelID string, parseRetries int) (EquivalenceResult, bool, *CoverageRepairIssue) {

107− `prompt := strings.TrimSpace(existing.Prompt)`

108− `if prompt == "" {`

192+ `prompt := existing.Prompt`

193+ `if strings.TrimSpace(prompt) == "" {`

109194 return existing, false, &CoverageRepairIssue{

110195 Dimension: coverageRepairDimensionEquivalence,

111196 Reason: "missing_prompt",

…func repairEquivalenceFromPrompt(ctx context.Context, ai AIClient, existing Equi

145230

146231 parsed.Status = AIStatusUsed

147232 parsed.EvaluatorModel = resolveRepairEvaluatorModel(existing.EvaluatorModel, overrideModelID)

233+ parsed.PromptMode = existing.PromptMode

234+ parsed.Truncated = existing.Truncated

235+ parsed.TruncationReason = existing.TruncationReason

148236 parsed.Prompt = prompt

149237 parsed.Output = output

150238 return parsed, true, nil

…func reviewVerdictIsGuardSuppressible(result ReviewResult) bool {

159247}

160248

161249func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewResult, overrideModelID string, parseRetries int) (ReviewResult, bool, *CoverageRepairIssue) {

162− `prompt := strings.TrimSpace(existing.Prompt)`

163− `if prompt == "" {`

250+ return repairReviewFromPromptContext(ctx, ai, existing, overrideModelID, parseRetries, nil)

251+}

252+

253+func repairReviewFromPromptContext(ctx context.Context, ai AIClient, existing ReviewResult, overrideModelID string, parseRetries int, promptCtx *PromptContext) (ReviewResult, bool, *CoverageRepairIssue) {

254+ `prompt := existing.Prompt`

255+ `if strings.TrimSpace(prompt) == "" {`

164256 return existing, false, &CoverageRepairIssue{

165257 Dimension: coverageRepairDimensionCodeReview,

166258 Reason: "missing_prompt",

…func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes

174266 // re-apply that guard, so such verdicts must be resampled, never recovered.

175267 `if existing.Status == AIStatusUnavailable && strings.TrimSpace(existing.Output) != "" {`

176268 `if reparsed, reparseErr := parseReviewOutput(existing.Output); reparseErr == nil &&`

177− `!reviewVerdictIsGuardSuppressible(reparsed) {`

269+ `((promptCtx == nil && !reviewVerdictIsGuardSuppressible(reparsed)) || (promptCtx != nil && validateReviewAgainstPromptContext(reparsed, *promptCtx) == nil)) {`

178270 reparsed.Status = AIStatusUsed

179271 reparsed.Prompt = prompt

180272 reparsed.Output = existing.Output

…func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes

184276 if reparsed.EvaluatorModel == "" { 185277 reparsed.EvaluatorModel = strings.TrimSpace(overrideModelID)

186278 }

279+ reparsed.AttemptCount = existing.AttemptCount

280+ reparsed.RetryCount = existing.RetryCount

281+ reparsed.RetryReasons = append([]string(nil), existing.RetryReasons...) 282+ reparsed.RecoveredAfterRetry = existing.RecoveredAfterRetry

187283 reparsed.Truncated = existing.Truncated

188284 reparsed.TruncationReason = existing.TruncationReason

189285 return reparsed, true, nil

…func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes

196292 }

197293

198294 `parsed, parseErr := parseReviewOutput(output)`

295+ `if parseErr == nil && promptCtx != nil {`

296+ parseErr = validateReviewAgainstPromptContext(parsed, *promptCtx)

297+ }

199298 `for attempt := 0; parseErr != nil && attempt < normalizedReviewParseRetries(parseRetries); attempt++ {`

200299 `retryPrompt := buildReviewRetryPrompt(prompt, output, parseErr)`

201300 `retryOutput, retryErr := runAIWithRetries(ctx, ai, retryPrompt)`

…func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes

203302 return existing, false, newReviewRuntimeFailureIssue(retryErr)

204303 }

205304 `parsed, parseErr = parseReviewOutput(retryOutput)`

305+ `if parseErr == nil && promptCtx != nil {`

306+ parseErr = validateReviewAgainstPromptContext(parsed, *promptCtx)

307+ }

206308 output = retryOutput

207309 }

208310 if parseErr != nil { 209311 return existing, false, classifyReviewParseIssue(parseErr)

210312 }

211313

314+ if promptCtx == nil && reviewVerdictIsGuardSuppressible(parsed) { 315+ return existing, false, classifyReviewParseIssue(errors.New("review repair requires original verifier context for speculative blocker verdict"))

316+ }

317+ parsed.Truncated = existing.Truncated

318+ parsed.TruncationReason = existing.TruncationReason

212319 parsed.Status = AIStatusUsed

213320 parsed.Prompt = prompt

214321 parsed.Output = output

…func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes

217324}

218325

219326func runAIWithRetries(ctx context.Context, ai AIClient, prompt string) (string, error) { 220− var lastErr error

221− `for attempt := 0; attempt <= len(coverageRepairRetryBackoffs); attempt++ {`

222− `output, err := ai.Run(ctx, prompt)`

223− `if err == nil {`

224− return output, nil

225− }

226− lastErr = err

227− if errors.Is(err, ErrAIDisabled) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { 228− break

229− }

230− if attempt >= len(coverageRepairRetryBackoffs) { 231− break

232− }

233− select {

234− `case <-ctx.Done():`

235− `return "", ctx.Err()`

236− `case <-time.After(coverageRepairRetryBackoffs[attempt]):`

237− }

327+ `if history, ok := ai.(*gradeAttemptClient); ok {`

328+ `return history.Run(ctx, prompt)`

238329 }

239− return "", lastErr

330+ return newGradeAttemptClient(ai).Run(ctx, prompt) 240331}

241332

242333func normalizedReviewParseRetries(parseRetries int) int {

…func appendReviewNote(existing, extra string) string { 5353 }

5454}

5555

56−func RunReview(ctx context.Context, ai AIClient, promptCtx PromptContext) ReviewResult {

56+func RunReview(ctx context.Context, ai AIClient, promptCtx PromptContext) (result ReviewResult) {

57+ `history := newGradeAttemptClient(ai)`

58+ `defer func() { history.annotateReview(&result) }()`

59+ ai = history

60+

5761 return annotateReviewTruncation(runReview(ctx, ai, promptCtx), promptCtx)

5862}

5963

…type TestVerifierEvidence struct {

712712}

713713

714714type TestCommandResult struct {

715− Command string `json:"command"`

716− GoldOutcome TestOutcome `json:"gold_outcome,omitempty"`

717− GoldTrialName string `json:"gold_tb_trial_name,omitempty"`

718− GoldFailureMode string `json:"gold_failure_mode,omitempty"`

719− GoldRetried bool `json:"gold_retried,omitempty"`

720− GoldRetryEvidence *TestRetryEvidence `json:"gold_retry_evidence,omitempty"`

721− AgentOutcome TestOutcome `json:"agent_outcome,omitempty"`

722− AgentTrialName string `json:"agent_tb_trial_name,omitempty"`

723− AgentFailureMode string `json:"agent_failure_mode,omitempty"`

724− AgentRetried bool `json:"agent_retried,omitempty"`

715+ // Counts cover dispatched gold and candidate verifier calls for this command.

716+ AttemptCount int `json:"attempt_count,omitempty"`

717+ RetryCount int `json:"retry_count,omitempty"`

718+ RetryReasons []string `json:"retry_reasons,omitempty"`

719+ RecoveredAfterRetry bool `json:"recovered_after_retry,omitempty"`

720+ Command string `json:"command"`

721+ GoldOutcome TestOutcome `json:"gold_outcome,omitempty"`

722+ GoldTrialName string `json:"gold_tb_trial_name,omitempty"`

723+ GoldFailureMode string `json:"gold_failure_mode,omitempty"`

724+ GoldRetried bool `json:"gold_retried,omitempty"`

725+ GoldRetryEvidence *TestRetryEvidence `json:"gold_retry_evidence,omitempty"`

726+ AgentOutcome TestOutcome `json:"agent_outcome,omitempty"`

727+ AgentTrialName string `json:"agent_tb_trial_name,omitempty"`

728+ AgentFailureMode string `json:"agent_failure_mode,omitempty"`

729+ AgentRetried bool `json:"agent_retried,omitempty"`

725730}

726731

727732// TestRetryEvidence records the two verifier attempts that authorized and

…type TestRetryAttemptEvidence struct {

742747}

743748

744749type EquivalenceResult struct {

750+ AttemptCount int `json:"attempt_count,omitempty"`

751+ RetryCount int `json:"retry_count,omitempty"`

752+ RetryReasons []string `json:"retry_reasons,omitempty"`

753+ RecoveredAfterRetry bool `json:"recovered_after_retry,omitempty"`

754+

745755 Outcome EquivalenceOutcome `json:"outcome"`

746756 Status AIStatus `json:"status"`

747757 // PromptMode records the exact built-in equivalence template variant that

…type EquivalenceObligation struct {

785795}

786796

787797type ReviewResult struct {

798+ AttemptCount int `json:"attempt_count,omitempty"`

799+ RetryCount int `json:"retry_count,omitempty"`

800+ RetryReasons []string `json:"retry_reasons,omitempty"`

801+ RecoveredAfterRetry bool `json:"recovered_after_retry,omitempty"`

802+

788803 Status AIStatus `json:"status"`

789804 SchemaVersion string `json:"schema_version,omitempty"`

790805 ProfileID string `json:"profile_id,omitempty"`

…Recovery rules:

793793 Those repair steps preserve compare-critical metadata and existing custom

794794 grader surfaces so the arm stays compare-compatible while recovering

795795 coverage. Add `--parse-retries N` when saved grader prompts failed

796− JSON/schema parsing. Keep the original custom rubric file path on the regrade

797− command.

796+ JSON/schema parsing. Repair preserves recorded prompt mode, truncation, and

797+ attempt history. Missing prompts require matching original task and artifact

798+ provenance plus complete repository context; ambiguous reconstruction remains

799+ unresolved. Runtime retries cover recognized transient failures only.

800+ Keep the original custom rubric file path on the regrade command.

798801- If `stet runs repair-ai-coverage --cr-only` still leaves `code_review` 799802 unresolved, use the summary's stable `unresolved[].reason` and optional

800803 `category`/` detail` fields to separate `model_output_shape`,

…

1+package commandlist

2+

3+import (

4+ "path"

5+ "regexp"

6+ "strings"

7+

8+ "stet/internal/shellcmd"

9+)

10+

11+``var assignment = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*=`)``

12+``var duration = regexp.MustCompile(`^[0-9]+(?:\.[0-9]+)?[smhd]?$`)``

13+

14+// TestInvocation identifies exactly one known test command. The original text

15+// is retained, including literal environment assignments and a relative cwd.

16+// Opaque scripts and shell evaluation are deliberately not interpreted.

17+`func TestInvocation(command string) (string, []shellcmd.FieldSpan, bool) {`

18+ `command = strings.TrimSpace(command)`

19+ ``if command == "" || strings.ContainsAny(command, "\n\r`$") {``

20+ return "", nil, false

21+ }

22+ `spans, ok := shellcmd.FieldSpans(command)`

23+ `if !ok || len(spans) == 0 {`

24+ return "", nil, false

25+ }

26+ `i := 0`

27+ `if spans[0].Value == "cd" {`

28+ `if len(spans) < 4 || spans[2].Value != "&&" || !relativeDirectory(spans[1].Value) {`

29+ return "", nil, false

30+ }

31+ if unsafeShellControl(command[:spans[2].Start]) { 32+ return "", nil, false

33+ }

34+ i = 3

35+ }

36+ if i >= len(spans) || unsafeShellControl(command[spans[i].Start:]) { 37+ return "", nil, false

38+ }

39+ // The sole supported control operator belongs to the proven cd prefix.

40+ `for _, span := range spans[i:] {`

41+ `if span.Value == "&&" || span.Value == "&" {`

42+ return "", nil, false

43+ }

44+ }

45+ `for i < len(spans) && assignment.MatchString(spans[i].Value) {`

46+ `if !assignment.MatchString(command[spans[i].Start:spans[i].End]) {`

47+ return "", nil, false

48+ }

49+ i++

50+ }

51+ if i < len(spans) && spans[i].Value == "env" { 52+ i++

53+ for i < len(spans) && assignment.MatchString(spans[i].Value) { 54+ i++

55+ }

56+ }

57+ if i < len(spans) && spans[i].Value == "timeout" { 58+ i++

59+ if i >= len(spans) || !duration.MatchString(spans[i].Value) { 60+ return "", nil, false

61+ }

62+ i++

63+ }

64+ if i >= len(spans) { 65+ return "", nil, false

66+ }

67+ `fields := spans[i:]`

68+ `value := func(n int) string {`

69+ `if n >= len(fields) {`

70+ return ""

71+ }

72+ return fields[n].Value 73+ }

74+ switch value(0) { 75+ case "go", "cargo":

76+ ok = value(1) == "test" 77+ case "sh", "bash":

78+ ok = len(fields) == 2 && strings.HasSuffix(value(1), ".sh") && relativeDirectory(value(1)) 79+ case "pytest", "pytest-3", "vitest", "jest", "phpunit":

80+ ok = true

81+ case "python", "python3":

82+ ok = value(1) == "-m" && value(2) == "pytest" 83+ case "npm", "pnpm", "yarn":

84+ ok = value(1) == "test" || (value(1) == "run" && value(2) == "test") 85+ case "npx":

86+ ok = value(1) == "vitest" || value(1) == "jest" 87+ default:

88+ ok = false

89+ }

90+ if !ok {

91+ return "", nil, false

92+ }

93+ return command, fields, true

94+}

95+

96+`func relativeDirectory(directory string) bool {`

97+ `if directory == "" || strings.HasPrefix(directory, "-") || strings.Contains(directory, "\\") || path.IsAbs(directory) {`

98+ return false

99+ }

100+ clean := path.Clean(directory) 101+ return clean != ".." && !strings.HasPrefix(clean, "../")

102+}

103+

104+func unsafeShellControl(command string) bool { 105+ var quote rune

106+ `escaped := false`

107+ `for _, r := range command {`

108+ if escaped {

109+ escaped = false

110+ continue

111+ }

112+ `if quote == '\'' {`

113+ `if r == '\'' {`

114+ quote = 0

115+ }

116+ continue

117+ }

118+ if r == '\\' { 119+ escaped = true

120+ continue

121+ }

122+ `if quote == '"' {`

123+ `if r == '"' {`

124+ quote = 0

125+ }

126+ continue

127+ }

128+ if r == '\'' || r == '"' { 129+ quote = r

130+ continue

131+ }

132+ if strings.ContainsRune(";|&<>#()", r) { 133+ return true

134+ }

135+ }

136+ return false

137+}

…

1+package commandlist

2+

3+import "testing"

4+

5+func TestTestInvocationPreservesProvenExecutionContext(t *testing.T) {

6+ `for _, command := range []string{`

7+ `"go test ./pkg -run 'TestOne|TestTwo' -count=1",`

8+ "cd packages/api && FOO='a b' env BAR=2 timeout 30s python3 -m pytest tests/test_api.py -q",

9+ `"npm run test -- --runInBand", "env CI=1 cargo test --locked", "pytest 'tests/test space.py'",`

10+ `} {`

11+ `got, fields, ok := TestInvocation(command)`

12+ `if !ok || got != command || len(fields) == 0 {`

13+ `t.Errorf("TestInvocation(%q) = %q, %v", command, got, ok)`

14+ }

15+ }

16+}

17+

18+func TestTestInvocationRejectsAmbiguousAndUnsafeWrappers(t *testing.T) {

19+ for _, command := range []string{ 20+ "echo go test ./pkg", "sh -c 'go test ./pkg'", "go test ./pkg && pytest tests",

21+ "go test ./pkg; true", "go test ./pkg || true", "cd ../outside && go test ./pkg",

22+ "env -i go test ./pkg", "timeout --signal=KILL 30 go test ./pkg", "FOO=$(touch sentinel) go test ./pkg", 23+ "go test ./pkg > result", "cd pkg&&go test ./pkg", "go test ./pkg &", "make test", "\"CI=1\" go test ./pkg", "F\"OO\"=x go test ./pkg",

24+ "cd pkg;touch && go test ./pkg", "echo tests/agent.sh", "cat tests/agent.sh", "rm tests/agent.sh", "sh -c tests/agent.sh",

25+ `} {`

26+ `if _, _, ok := TestInvocation(command); ok {`

27+ `t.Errorf("accepted %q", command)`

28+ }

29+ }

30+}

…

1+package h2h

2+

3+import (

4+ "fmt"

5+ "path/filepath"

6+ "strings"

7+

8+ "stet/internal/validate"

9+)

10+

11+// prepareCoverageRepairPrompts reconstructs only absent prompts. Stored prompts

12+// retain their original evidence; reconstructed prompts use the sealed task

13+// source and keep its extracted repository alive through grading.

14+`func prepareCoverageRepairPrompts(cfg *validate.CoverageRepairConfig, manifest *Manifest, artifacts TaskArtifacts) (func(), []validate.CoverageRepairIssue) {`

15+ `missingEq := cfg.RepairEquivalence && strings.TrimSpace(cfg.Existing.Equivalence.Prompt) == ""`

16+ `review := cfg.Existing.CodeReview`

17+ `if strings.TrimSpace(review.Prompt) == "" && cfg.Existing.Review != nil {`

18+ review = *cfg.Existing.Review

19+ }

20+ missingReview := cfg.RepairCodeReview && strings.TrimSpace(review.Prompt) == "" 21+ if !missingEq && !missingReview {

22+ return nil, nil

23+ }

24+ var issues []validate.CoverageRepairIssue

25+ `fail := func(dimension string, err error) {`

26+ `detail := "prompt reconstruction failed: " + err.Error()`

27+ `if len(detail) > 240 {`

28+ `detail = detail[:240]`

29+ }

30+ `issues = append(issues, validate.CoverageRepairIssue{Dimension: dimension, Reason: "missing_prompt", Detail: detail, Cause: err})`

31+ `if dimension == TaskGraderEquivalence {`

32+ cfg.RepairEquivalence = false

33+ } else { 34+ cfg.RepairCodeReview = false

35+ }

36+ }

37+ mode := cfg.Existing.Equivalence.PromptMode

38+ if missingEq && mode != validate.EquivalencePromptModeStandard && mode != validate.EquivalencePromptModeGoldBlind {

39+ fail(TaskGraderEquivalence, fmt.Errorf("missing or unsupported equivalence prompt mode %q", mode))

40+ missingEq = false

41+ }

42+ if !missingEq && !missingReview {

43+ return nil, issues

44+ }

45+ truncation := cfg.Existing.PromptTruncation

46+ if truncation.AgentPatch.Truncated || truncation.GoldPatch.Truncated || truncation.RepoTree.Truncated || truncation.RepoExcerpts.Truncated || cfg.Existing.Equivalence.Truncated || review.Truncated {

47+ err := fmt.Errorf("original prompt was truncated; original context budgets are unavailable") 48+ if missingEq {

49+ fail(TaskGraderEquivalence, err) 50+ }

51+ if missingReview {

52+ fail(TaskGraderCodeReview, err) 53+ }

54+ return nil, issues

55+ }

56+ dataset := manifest.Dataset 57+ goldPath := strings.TrimSpace(cfg.Existing.ArtifactsManifest.GoldPatch.Path)

58+ `if goldPath != "" {`

59+ `taskDir := filepath.Dir(goldPath)`

60+ `if filepath.Base(taskDir) != artifacts.TaskID {`

61+ err := fmt.Errorf("gold patch directory does not match task %q", artifacts.TaskID)

62+ if missingEq {

63+ fail(TaskGraderEquivalence, err) 64+ }

65+ if missingReview {

66+ fail(TaskGraderCodeReview, err) 67+ }

68+ return nil, issues

69+ }

70+ dataset = filepath.Dir(taskDir) 71+ }

72+ if err := verifyCoverageRepairTaskAuthority(manifest, artifacts, dataset); err != nil { 73+ if missingEq {

74+ fail(TaskGraderEquivalence, err) 75+ }

76+ if missingReview {

77+ fail(TaskGraderCodeReview, err) 78+ }

79+ return nil, issues

80+ }

81+ `resolved, _, _, err := resolveTaskArtifactsFromResults(artifacts)`

82+ `if err == nil {`

83+ artifacts = resolved

84+ }

85+ patchPath := strings.TrimSpace(cfg.Existing.ArtifactsManifest.AgentPatch.Path)

86+ `if patchPath == "" {`

87+ `patchPath = artifacts.ResolveAgentPatchPath()`

88+ }

89+ promptCtx, cleanup, err := validate.BuildOfflinePromptContext(validate.OfflinePromptInputs{

90+ DatasetPath: dataset, TaskID: artifacts.TaskID, AgentPatchPath: patchPath,

91+ `TBResultsRaw: validate.ReadTBResultsRaw(artifacts.ResultsPath(), artifacts.TaskID), RequireRepoContext: true,`

92+ `EffectiveTestCommands: append([]string{}, cfg.Existing.Tests.Commands...),`

93+ ExpectedArtifacts: &cfg.Existing.ArtifactsManifest,

94+ RepoContextOptions: validate.RepoContextOptions{ExcludePrefixes: cfg.Existing.ContextExcludePaths},

95+ `})`

96+ `if err != nil {`

97+ if missingEq {

98+ fail(TaskGraderEquivalence, err) 99+ }

100+ if missingReview {

101+ fail(TaskGraderCodeReview, err) 102+ }

103+ return nil, issues

104+ }

105+ if strings.TrimSpace(promptCtx.AgentPatch) == "" || strings.TrimSpace(promptCtx.TaskYAML) == "" { 106+ err := fmt.Errorf("task YAML and agent patch are required for prompt reconstruction")

107+ if missingEq {

108+ fail(TaskGraderEquivalence, err) 109+ }

110+ if missingReview {

111+ fail(TaskGraderCodeReview, err) 112+ }

113+ return cleanup, issues

114+ }

115+ `client := cfg.AI`

116+ `if shell, ok := client.(validate.ShellAIClient); ok {`

117+ shell.Dir = promptCtx.RepoPath

118+ client = shell

119+ }

120+ if missingEq {

121+ `prompt, err := validate.RenderEquivalencePromptForMode(promptCtx, mode)`

122+ `if err != nil {`

123+ `fail(TaskGraderEquivalence, err)`

124+ `} else {`

125+ cfg.EquivalencePrompt, cfg.EquivalencePromptMode = prompt, mode

126+ cfg.EquivalencePromptContext, cfg.EquivalenceAI = &promptCtx, client

127+ }

128+ }

129+ if missingReview {

130+ `prompt, err := validate.RenderReviewPrompt(promptCtx)`

131+ `if err != nil {`

132+ `fail(TaskGraderCodeReview, err)`

133+ `} else {`

134+ cfg.ReviewPrompt, cfg.ReviewPromptContext, cfg.ReviewAI = prompt, &promptCtx, client

135+ }

136+ }

137+ return cleanup, issues

138+}

139+

140+func verifyCoverageRepairTaskAuthority(manifest *Manifest, artifacts TaskArtifacts, dataset string) error {

141+ if manifest.TaskSelection == nil || manifest.TaskSelection.PromptProjectionReceipt == "" { 142+ return fmt.Errorf("missing original task prompt projection authority")

143+ }

144+ `receipt, err := LoadTaskPromptProjectionReceipt(artifacts.OutputRoot)`

145+ `if err != nil {`

146+ return err

147+ }

148+ if err := validateTaskSelectionProjection(manifest.TaskSelection, receipt); err != nil { 149+ return err

150+ }

151+ `digest, err := digestTaskContentTree(filepath.Join(dataset, artifacts.TaskID))`

152+ `if err != nil {`

153+ return err

154+ }

155+ `for _, row := range receipt.Tasks {`

156+ `if row.TaskID == artifacts.TaskID {`

157+ if row.TaskContentSHA256 != digest {

158+ return fmt.Errorf("original task content provenance mismatch for %q", artifacts.TaskID)

159+ }

160+ return nil

161+ }

162+ }

163+ return fmt.Errorf("missing original task content authority for %q", artifacts.TaskID)

164+}

…

1+package h2h

2+

3+import (

4+ "bytes"

5+ "context"

6+ "fmt"

7+ "os"

8+ "path/filepath"

9+ "strings"

10+ "testing"

11+ "time"

12+

13+ "stet/internal/validate"

14+)

15+

16+func TestPrepareCoverageRepairPromptsPreservesStoredPrompts(t *testing.T) {

17+ cfg := validate.CoverageRepairConfig{ 18+ Existing: validate.ValidationResult{

19+ `Equivalence: validate.EquivalenceResult{Prompt: "stored equivalence"},`

20+ `Review:      &validate.ReviewResult{Prompt: "legacy stored review"},`

21+ `}, RepairEquivalence: true, RepairCodeReview: true,`

22+ }

23+ `cleanup, issues := prepareCoverageRepairPrompts(&cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"})`

24+ `if cleanup != nil || len(issues) != 0 || cfg.EquivalencePromptContext != nil || cfg.ReviewPromptContext != nil {`

25+ t.Fatalf("stored prompts should require no reconstruction: %+v", cfg)

26+ }

27+}

28+

29+func TestPrepareCoverageRepairPromptsRejectsUnknownModeWithoutBlockingStoredReview(t *testing.T) {

30+ `cfg := validate.CoverageRepairConfig{`

31+ `Existing:          validate.ValidationResult{CodeReview: validate.ReviewResult{Prompt: "stored review"}},`

32+ RepairEquivalence: true, RepairCodeReview: true,

33+ }

34+ `_, issues := prepareCoverageRepairPrompts(&cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"})`

35+ `if len(issues) != 1 || issues[0].Dimension != TaskGraderEquivalence || cfg.RepairEquivalence || !cfg.RepairCodeReview {`

36+ t.Fatalf("unexpected dimension selection: %+v, issues %+v", cfg, issues)

37+ }

38+}

39+

40+func TestPrepareCoverageRepairPromptsRejectsUnknownOriginalTruncationBudget(t *testing.T) {

41+ cfg := validate.CoverageRepairConfig{RepairCodeReview: true} 42+ cfg.Existing.PromptTruncation.AgentPatch.Truncated = true

43+ `_, issues := prepareCoverageRepairPrompts(&cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"})`

44+ `if len(issues) != 1 || cfg.RepairCodeReview || !strings.Contains(issues[0].Detail, "original context budgets") {`

45+ t.Fatalf("unknown original truncation must prevent reconstruction: %+v", issues)

46+ }

47+}

48+

49+func TestPrepareCoverageRepairPromptsReconstructsPersistedModeAndVerifierContext(t *testing.T) {

50+ `t.Parallel()`

51+ `root := t.TempDir()`

52+ `taskDir := filepath.Join(root, "dataset", "task-a")`

53+ `mustWriteFile(t, filepath.Join(taskDir, "task.yaml"), "id: task-a\ninstruction: fix greeting\n")`

54+ mustWriteFile(t, filepath.Join(taskDir, "gold.patch"), "gold reference secret\n")

55+ `mustWriteFile(t, filepath.Join(taskDir, "tests", "test_outputs.py"), "COMMANDS = [\"dataset test\"]\n")`

56+ `patch := filepath.Join(root, "agent.patch")`

57+ `mustWriteFile(t, patch, "diff --git a/hello.txt b/hello.txt\n+hello\n")`

58+ `writeAnswerSimilarityTar(t, filepath.Join(taskDir, "repo.tar.gz"), map[string]string{"hello.txt": "old\n"}, nil)`

59+ `existing := validate.ValidationResult{`

60+ Equivalence: validate.EquivalenceResult{PromptMode: validate.EquivalencePromptModeGoldBlind},

61+ Tests: validate.TestResult{Commands: []string{"effective verifier"}}, 62+ }

63+ `inputs, err := validate.ReadTaskInputs(filepath.Dir(taskDir), "task-a", patch)`

64+ `if err != nil {`

65+ `t.Fatal(err)`

66+ }

67+ existing.ArtifactsManifest = validate.BuildArtifactManifest(inputs)

68+ `manifest := sealCoverageRepairTestTask(t, root, filepath.Dir(taskDir))`

69+ `cfg := validate.CoverageRepairConfig{Existing: existing, AI: validate.ShellAIClient{}, RepairEquivalence: true, RepairCodeReview: true}`

70+ `cleanup, issues := prepareCoverageRepairPrompts(&cfg, manifest, NewTaskArtifacts(root, VariantSpec{RunID: "run-a", ModelKey: "model-a"}, "task-a", ""))`

71+ `if cleanup != nil {`

72+ `defer cleanup()`

73+ }

74+ `if len(issues) != 0 {`

75+ `t.Fatalf("reconstruction failed: %+v", issues)`

76+ }

77+ if cfg.EquivalencePromptMode != validate.EquivalencePromptModeGoldBlind || strings.Contains(cfg.EquivalencePrompt, "gold reference secret") {

78+ t.Fatalf("gold-blind provenance was lost: %s", cfg.EquivalencePrompt) 79+ }

80+ if !strings.Contains(cfg.ReviewPrompt, "effective verifier") || strings.Contains(cfg.ReviewPrompt, "dataset test") {

81+ t.Fatalf("effective verifier context was lost: %s", cfg.ReviewPrompt)

82+ }

83+ `for _, client := range []validate.AIClient{cfg.EquivalenceAI, cfg.ReviewAI} {`

84+ `shell := client.(validate.ShellAIClient)`

85+ `if shell.Dir == "" || shell.Dir != cfg.ReviewPromptContext.RepoPath {`

86+ `t.Fatalf("missing grading repository: %+v", shell)`

87+ }

88+ `if _, err := os.Stat(filepath.Join(shell.Dir, "hello.txt")); err != nil {`

89+ `t.Fatal(err)`

90+ }

91+ }

92+}

93+

94+func TestPrepareCoverageRepairPromptsRejectsUnavailableRepository(t *testing.T) {

95+ `root := t.TempDir()`

96+ `taskDir := filepath.Join(root, "dataset", "task-a")`

97+ `mustWriteFile(t, filepath.Join(taskDir, "task.yaml"), "id: task-a\n")`

98+ `mustWriteFile(t, filepath.Join(taskDir, "gold.patch"), "gold\n")`

99+ `mustWriteFile(t, filepath.Join(taskDir, "repo.tar.gz"), "invalid gzip")`

100+ `patch := filepath.Join(root, "agent.patch")`

101+ mustWriteFile(t, patch, "agent\n")

102+ `manifest := sealCoverageRepairTestTask(t, root, filepath.Dir(taskDir))`

103+ `cfg := validate.CoverageRepairConfig{RepairCodeReview: true}`

104+ `inputs, err := validate.ReadTaskInputs(filepath.Dir(taskDir), "task-a", patch)`

105+ `if err != nil {`

106+ `t.Fatal(err)`

107+ }

108+ cfg.Existing.ArtifactsManifest = validate.BuildArtifactManifest(inputs)

109+ `cleanup, issues := prepareCoverageRepairPrompts(&cfg, manifest, NewTaskArtifacts(root, VariantSpec{RunID: "run-a"}, "task-a", ""))`

110+ `if cleanup != nil {`

111+ `defer cleanup()`

112+ }

113+ if len(issues) != 1 || cfg.RepairCodeReview || cfg.ReviewPrompt != "" || !strings.Contains(issues[0].Detail, "required repo context") { 114+ t.Fatalf("repository failure should prevent reconstructed grade: %+v, issues %+v", cfg, issues)

115+ }

116+}

117+

118+func sealCoverageRepairTestTask(t *testing.T, root, dataset string) *Manifest {

119+ `t.Helper()`

120+ `selection := &TaskSelectionRecord{RealizedTaskIDs: []string{"task-a"}}`

121+ `receipt, err := BuildTaskPromptProjectionReceipt(dataset, root, selection, nil, "dataset", map[string]string{"task-a": "fix greeting"}, time.Time{})`

122+ `if err != nil {`

123+ `t.Fatal(err)`

124+ }

125+ `if _, err := WriteTaskPromptProjectionReceipt(root, receipt); err != nil {`

126+ `t.Fatal(err)`

127+ }

128+ return &Manifest{Dataset: dataset, TaskSelection: ApplyTaskPromptProjectionReceipt(selection, receipt)} 129+}

130+

131+func TestPrepareCoverageRepairPromptsRejectsMutatedTaskYAML(t *testing.T) {

132+ `root := t.TempDir()`

133+ `dataset := filepath.Join(root, "dataset")`

134+ `mustWriteFile(t, filepath.Join(dataset, "task-a", "task.yaml"), "id: task-a\ninstruction: original\n")`

135+ `manifest := sealCoverageRepairTestTask(t, root, dataset)`

136+ `mustWriteFile(t, filepath.Join(dataset, "task-a", "task.yaml"), "id: task-a\ninstruction: mutated\n")`

137+ `cfg := validate.CoverageRepairConfig{RepairCodeReview: true}`

138+ `_, issues := prepareCoverageRepairPrompts(&cfg, manifest, NewTaskArtifacts(root, VariantSpec{RunID: "run-a"}, "task-a", ""))`

139+ `if len(issues) != 1 || cfg.RepairCodeReview || !strings.Contains(issues[0].Detail, "task content provenance mismatch") {`

140+ t.Fatalf("changed task must not reconstruct: %+v", issues)

141+ }

142+}

143+

144+func TestRunRepairAICoverageReconstructsBoundSourceAtSubprocessBoundary(t *testing.T) {

145+ `for _, mutate := range []bool{false, true} {`

146+ `t.Run(fmt.Sprintf("mutate_%t", mutate), func(t *testing.T) {`

147+ `root := t.TempDir()`

148+ `dataset := filepath.Join(root, "dataset")`

149+ `taskDir := filepath.Join(dataset, "task-a")`

150+ `mustWriteFile(t, filepath.Join(taskDir, "task.yaml"), "id: task-a\ninstruction: fix greeting\n")`

151+ mustWriteFile(t, filepath.Join(taskDir, "gold.patch"), "gold reference secret\n")

152+ `writeAnswerSimilarityTar(t, filepath.Join(taskDir, "repo.tar.gz"), map[string]string{"hello.txt": "old\n"}, nil)`

153+ `patch := filepath.Join(root, "agent.patch")`

154+ `mustWriteFile(t, patch, "diff --git a/hello.txt b/hello.txt\n--- a/hello.txt\n+++ b/hello.txt\n@@ -1 +1 @@\n-old\n+hello\n")`

155+ `manifest := sealCoverageRepairTestTask(t, root, dataset)`

156+ manifest.Version, manifest.OutputRoot = ManifestVersion, root

157+ `manifest.Models = []string{"model/a"}`

158+ `manifest.ModelKeys, manifest.RunIDs = map[string]string{"model/a": "model-a"}, map[string]string{"model/a": "run-a"}`

159+ `if err := SaveManifest(filepath.Join(root, "manifest.json"), manifest); err != nil {`

160+ `t.Fatal(err)`

161+ }

162+ `inputs, err := validate.ReadTaskInputs(dataset, "task-a", patch)`

163+ `if err != nil {`

164+ `t.Fatal(err)`

165+ }

166+ original := validate.ValidationResult{ 167+ TaskID: "task-a", PrimarySignal: validate.PrimarySignalTests, Status: validate.StatusPass, MatrixStatus: validate.MatrixPass,

168+ Guardrails: validate.GuardrailResult{Status: validate.StatusPass},

169+ Tests: validate.TestResult{Outcome: validate.TestOutcomePass, Strength: validate.TestStrengthStrong, Commands: []string{"effective verifier"}}, 170+ Equivalence: validate.EquivalenceResult{Status: validate.AIStatusUnavailable, Outcome: validate.EquivalenceUnknown, PromptMode: validate.EquivalencePromptModeGoldBlind},

171+ CodeReview: validate.ReviewResult{Status: validate.AIStatusUnavailable, Signal: validate.ReviewSignalUnsure},

172+ ArtifactsManifest: validate.BuildArtifactManifest(inputs),

173+ }

174+ validationRoot := filepath.Join(root, "validation", "model-a") 175+ mustWriteRepairValidation(t, validationRoot, "task-a", original)

176+ if mutate {

177+ mustWriteFile(t, patch, "mutated agent evidence\n")

178+ }

179+ `log := filepath.Join(root, "calls")`

180+ `script := filepath.Join(root, "judge.py")`

181+ mustWriteFile(t, script, `import json, pathlib, sys

182+`prompt = sys.stdin.read()`

183+`assert pathlib.Path('hello.txt').read_text() == 'old\n'`

184+`with open(sys.argv[1], 'a') as log: log.write('call\n')`

185+if 'You are reviewing' in prompt:

186+ assert 'effective verifier' in prompt

187+ scores = {name: {'score_0_4': 4, 'confidence_0_1': .95, 'rationale': 'ok', 'citations': [{'path':'hello.txt','side':'new','start_line':1,'end_line':1,'anchor_kind':'diff','head_sha':'abc1234'}]} for name in ['correctness','introduced_bug_risk','edge_case_handling','maintainability_idioms']}

188+ print(json.dumps({'schema_version':'stet.review.v1','dimension_scores':scores,'findings':[],'overall':{'confidence_overall':.95,'summary':'repair passed'}})) 189+else:

190+ assert 'gold reference secret' not in prompt

191+ print('obligations:\n- fix greeting => MET\n\nequivalent: YES\nconfidence: 0.95\nissue_type: stylistic\nnotes: repaired') 192+`)

193+ var stdout, stderr bytes.Buffer

194+ err = RunRepairAICoverage(context.Background(), []string{"--out", root, "--model-key", "model-a", "--ai-cmd", "python3 " + shellQuote(script) + " " + shellQuote(log), "--ai-model-id", "judge-a"}, &stdout, &stderr)

195+ if mutate {

196+ if err == nil { 197+ t.Fatal("mutated artifact repair must fail")

198+ }

199+ `if _, err := os.Stat(log); !os.IsNotExist(err) {`

200+ `t.Fatalf("mutated source invoked grader: %v", err)`

201+ }

202+ `if !strings.Contains(stdout.String(), "agent patch provenance mismatch") {`

203+ `t.Fatalf("missing mutation reason: %s", stdout.String())`

204+ }

205+ return

206+ }

207+ `if err != nil {`

208+ `t.Fatalf("repair failed: %v\nstdout %s\nstderr %s", err, stdout.String(), stderr.String())`

209+ }

210+ updated := mustReadRepairValidation(t, filepath.Join(validationRoot, "task-a", "validation.json")) 211+ if updated.Equivalence.PromptMode != validate.EquivalencePromptModeGoldBlind || updated.Equivalence.Status != validate.AIStatusUsed || updated.CodeReview.Status != validate.AIStatusUsed || updated.CodeReview.AttemptCount != 1 {

212+ t.Fatalf("repair lost mode or attempt metadata: %+v", updated)

213+ }

214+ if updated.ArtifactsManifest.AgentPatch.SHA256 != original.ArtifactsManifest.AgentPatch.SHA256 {

215+ t.Fatal("repair changed original source authority")

216+ }

217+ `calls, err := os.ReadFile(log)`

218+ `if err != nil || strings.Count(string(calls), "call") != 2 {`

219+ `t.Fatalf("unexpected calls: %s, %v", calls, err)`

220+ }

221+ }) 222+ }

223+}

…

1+package validate

2+

3+import (

4+ "context"

5+ "errors"

6+ "net"

7+ "strings"

8+ "time"

9+

10+ "stet/internal/runtimefailure"

11+)

12+

13+// gradeAttemptClient retains every dispatch, including runtime and parse retries.

14+// One instance belongs to one measurement dimension and must not be shared.

15+type gradeAttemptClient struct {

16+ ai AIClient

17+ attempts int

18+ reasons []string 19+ previousSucceeded bool

20+}

21+

22+func newGradeAttemptClient(ai AIClient) *gradeAttemptClient {

23+ `if ai == nil {`

24+ `ai = NoopAIClient{}`

25+ }

26+ return &gradeAttemptClient{ai: ai} 27+}

28+

29+func (a *gradeAttemptClient) Run(ctx context.Context, prompt string) (string, error) { 30+ if a.previousSucceeded {

31+ a.reasons = append(a.reasons, "model_output_shape") 32+ }

33+ a.previousSucceeded = false

34+ `for attempt := 0; ; attempt++ {`

35+ `if err := ctx.Err(); err != nil {`

36+ return "", err

37+ }

38+ a.attempts++

39+ `output, err := a.ai.Run(ctx, prompt)`

40+ `if err == nil {`

41+ a.previousSucceeded = true

42+ return output, nil

43+ }

44+ `reason := recoverableGradeFailure(err)`

45+ `if reason == "" || attempt >= len(coverageRepairRetryBackoffs) {`

46+ return "", err

47+ }

48+ select {

49+ `case <-ctx.Done():`

50+ `return "", ctx.Err()`

51+ `case <-time.After(coverageRepairRetryBackoffs[attempt]):`

52+ }

53+ a.reasons = append(a.re

source & further reading

stet.sh — original article On 25 Stet Tasks, Opus 4.8 Wrote Smaller Patches. Opus 5 Searched Wider. I Tested Six Ways to Save GPT-5.6 Tokens. The Typical Task and the Recorded Workload Disagreed Sonnet 5 vs Opus 4.8: how they behave and when to use each