I ran Sol 6 vs. 6.1 on tasks from my own repo. It's cheaper and at least as good

A developer's head-to-head test of Sol 6 versus Sol 6.1 on tasks from their own repository found the newer model cheaper and at least as capable. The comparison used the `stet runs repair-ai-coverage` command with `--model-key model-one`, and the accompanying code changes add `--parse-retries` handling for saved `code_review` prompts whose evaluator responses failed JSON/schema parsing, rewriting validation artifacts only after a successful repair. The patch also adds a `prepareCoverageRepairPrompts` step that requires the original task projection, matching artifact digests, persisted test commands, and complete repository context, leaving unavailable provenance unresolved.

… stet runs repair-ai-coverage --out /path/to/h2h-output --model-key model-one --c 292292 293293 Use --parse-retries when saved code review prompts exist but evaluator 294294 responses failed JSON/schema parsing. The command rewrites validation artifacts 295− only after a successful repair and regenerates summaries/task details. 295+ only after a successful repair and regenerates summaries/task details. Stored 296+ prompts retain their measurement mode and truncation. Missing-prompt repair 297+ requires the original task projection, matching artifact digests, persisted test 298+ commands, and complete repository context; unavailable provenance stays 299+ unresolved. Attempt counts and retry reasons distinguish recovery from an 300+ initial successful grade; assertions and unrecoverable runtime errors are not 301+ retried. 296302 297303 Regrade additive custom or bundled graders from canonical task details: 298304 … const coverageValidationGolden = { 843843 }, 844844 "equiv rescue policy": "on", 845845 "equivalence": { 846+ "attempt count": 1, 846847 "confidence": 0.95, 847848 "evaluator model": "judge-a", 848849 "issue type": "stylistic", … func p repairAICoverageProducer Produce mctx MutationContext ProduceResul 274274 var mu sync.Mutex 275275 runWithConcurrency jobs, min len jobs , coverageRepairWorkers , func job repairJob { 276276 reporter.recordTaskStart variant, job.TaskID, job.CurrentWork, time.Now .UTC 277− result, repairErr := validate.RepairCoverage ctx, validate.CoverageRepairConfig{ 277+ repairCfg := validate.CoverageRepairConfig{ 278278 Existing: job.Validation, 279279 AI: ai, 280280 AIModelID: resolvedAIModelID, 281281 RepairEquivalence: repairEquivalence && needsRepairCoverage job.Validation, TaskGraderEquivalence, resolvedAIModelID , 282282 RepairCodeReview: repairCodeReview && needsRepairCoverage job.Validation, TaskGraderCodeReview, resolvedAIModelID , 283283 ReviewParseRetries: cfg.ParseRetries, 284− } 284+ } 285+ cleanup, reconstructionIssues := prepareCoverageRepairPrompts &repairCfg, manifest, NewTaskArtifacts outputRoot, variant, job.TaskID, "" 286+ if cleanup = nil { 287+ defer cleanup 288+ } 289+ result := validate.CoverageRepairResult{Validation: job.Validation} 290+ var repairErr error 291+ if repairCfg.RepairEquivalence || repairCfg.RepairCodeReview { 292+ result, repairErr = validate.RepairCoverage ctx, repairCfg 293+ } 294+ result.Unresolved = append result.Unresolved, reconstructionIssues... 285295 if repairErr = nil { 286296 reporter.recordTaskEnd variant, job.TaskID, job.CurrentWork, "failed", time.Now .UTC 287297 mu.Lock … func RunAdaptedReferenceTest ctx context.Context, ai AIClient, promptCtx PromptC 292292 composedDiff, composeErr := composeAdaptedReferenceTestPatch runCtx, inputs, referenceTestPatch, canonicalDiff 293293 if composeErr = nil { 294294 result.Reason = "patch compose failed" 295− result.Summary = fmt.Sprintf "compose canonical reference test patch and adapted diff: %v", composeErr 295+ result.Summary = fmt.Sprintf "adapted diff could not be composed with the canonical reference test patch: %v", composeErr 296296 return result 297297 } 298298 composedPayload := payload … import 99 "strings" 1010 "time" 1111 12+ "stet/internal/commandlist" 1213 "stet/internal/filecopy" 1314 "stet/internal/gitops" 1415 "stet/internal/harness" … func legacyCommandsThatNameAgentTests commands, agentFiles string string { 192193 files := normalizeAgentTestFiles agentFiles 193194 var out string 194195 for , command := range normalizeCommands commands { 195− if commandHasPotentialNonPathTestSelection command || receiptBackedCommandHasShellControl command { 196+ if , , safe := commandlist.TestInvocation command ; safe || commandHasPotentialNonPathTestSelection command || receiptBackedCommandHasShellControl command { 196197 continue 197198 } 198199 for , file := range files { … import 66 "fmt" 77 "os" 88 "path/filepath" 9+ "strconv" 910 "strings" 1011 12+ "stet/internal/commandlist" 1113 "stet/internal/shellcmd" 1214 1315 … func replaceExactFinalTestPath command, sourcePath, replacement string string, 198200 } 199201 200202 func certifiedGoReplayCommand command, sourcePath string string, bool { 201− if strings.HasSuffix strings.ToLower sourcePath , ".go" || goCommandTargetsAgentTestFiles command, string{sourcePath} { 203+ , fields, safe := commandlist.TestInvocation command 204+ if safe || len fields < 3 || fields 0 .Value = "go" || fields 1 .Value = "test" || strings.HasSuffix strings.ToLower sourcePath , ".go" { 202205 return "", false 203206 } 204− replay, changed := removeGoRunSelector command 205− if changed || hasRemainingGoTestSelector replay { 207+ // Require one explicit package. A recursive selector or additional package 208+ // can credit tests outside the certified source package. 209+ sourceDirectory := filepath.Dir sourcePath 210+ allFields, := shellcmd.FieldSpans command 211+ // Go also consumes selectors from GOFLAGS. Recorded overrides are opaque 212+ // to this bounded rewrite, so they cannot certify an unfiltered replay. 213+ for , field := range allFields { 214+ if field.Start = fields 0 .Start { 215+ break 216+ } 217+ if strings.HasPrefix field.Value, "GOFLAGS=" { 218+ return "", false 219+ } 220+ } 221+ if len allFields = 3 && allFields 0 .Value == "cd" { 222+ relative, err := filepath.Rel allFields 1 .Value, sourceDirectory 223+ if err = nil || relative == ".." || strings.HasPrefix relative, "../" { 224+ return "", false 225+ } 226+ sourceDirectory = relative 227+ } 228+ packagePath := "./" + filepath.ToSlash sourceDirectory 229+ if sourceDirectory == "." { 230+ packagePath = "." 231+ } 232+ if strings.TrimSuffix fields 2 .Value, "/" = packagePath { 233+ return "", false 234+ } 235+ start, end := -1, -1 236+ for i := 3; i < len fields ; i++ { 237+ value := fields i .Value 238+ if value == "-run" || strings.HasPrefix value, "-run=" { 239+ if start = 0 { 240+ return "", false 241+ } 242+ start, end = fields i .Start, fields i .End 243+ if value == "-run" { 244+ if i+1 = len fields || strings.HasPrefix fields i+1 .Value, "-" { 245+ return "", false 246+ } 247+ i++ 248+ end = fields i .End 249+ } 250+ continue 251+ } 252+ // Only flags with self-contained values may remain. Positional words, 253+ // passthrough args, and further selectors cannot be proven safe. 254+ name := strings.SplitN value, "=", 2 0 255+ if name == "-count" { 256+ countText, hasValue := strings.CutPrefix value, "-count=" 257+ count, err := strconv.Atoi countText 258+ if hasValue || err = nil || count <= 0 { 259+ return "", false 260+ } 261+ } 262+ switch name { 263+ case "-v", "-race", "-short", "-count", "-timeout", "-parallel", "-shuffle", "-tags", "-mod": 264+ if strings.Contains value, "=" && name = "-v" && name = "-race" && name = "-short" { 265+ return "", false 266+ } 267+ default: 268+ return "", false 269+ } 270+ } 271+ if start < 0 { 206272 return "", false 207273 } 208− return replay, true 274+ return strings.TrimSpace strings.TrimRight command :start , " \t" + command end: , true 209275 } 210276 211277 func normalizeReplayRelativePath path string string { … func TestValidateAgentTestReplayPlanRejectsUnprovenCommand t testing.T { 6161 t.Fatalf "validation error = %v", err 6262 } 6363 } 64+ 65+ func TestCertifiedGoReplayPreservesOnlyProvenInvocation t testing.T { 66+ command := "cd module && env MODE='test value' timeout 30s go test ./pkg -run 'TestHidden|TestOther' -count=1" 67+ got, ok := certifiedGoReplayCommand command, "module/pkg/hidden test.go" 68+ if ok || got = "cd module && env MODE='test value' timeout 30s go test ./pkg -count=1" { 69+ t.Fatalf "replay = %q, %v", got, ok 70+ } 71+ for , unsafe := range string{ 72+ "echo go test ./pkg -run TestHidden", "go test ./pkg -run TestHidden; true", 73+ "go test ./pkg -run TestHidden && go test ./other", "go test ./... -run TestHidden", 74+ "go test ./pkg ./other -run TestHidden", "go test ./pkg -run TestHidden -skip TestOther", 75+ "go test ./pkg -run TestHidden -args payload", "go test ./pkg -run TestHidden -run TestOther", 76+ "go test ./pkg -run TestHidden -count=0", "go test ./pkg -run TestHidden -count=-1", 77+ "go test ./pkg -run TestHidden -count=invalid", "go test ./pkg -run TestHidden -count 0", 78+ "go test ./pkg -run TestHidden -count -1", 79+ "GOFLAGS=-count=0 go test ./pkg -run TestHidden", 80+ "GOFLAGS='-run=Nothing' go test ./pkg -run TestHidden", 81+ "env GOFLAGS=-count=0 go test ./pkg -run TestHidden", 82+ "cd module && env GOFLAGS='-run=Nothing' go test ./pkg -run TestHidden", 83+ } { 84+ if replay, ok := certifiedGoReplayCommand unsafe, "pkg/hidden test.go" ; ok { 85+ t.Errorf "accepted %q as %q", unsafe, replay 86+ } 87+ } 88+ } 89+ 90+ func TestLegacyReplayRequiresAnActualTestRunner t testing.T { 91+ commands := legacyCommandsThatNameAgentTests string{ 92+ "echo tests/agent test.sh", "cat tests/agent test.sh", "rm tests/agent test.sh", 93+ "sh -c tests/agent test.sh", "'CI=1' sh tests/agent test.sh", "sh tests/agent test.sh", 94+ }, string{"tests/agent test.sh"} 95+ if len commands = 1 || commands 0 = "sh 'tests/agent test.sh'" { 96+ t.Fatalf "legacy replay commands = %q", commands 97+ } 98+ } … func buildEquivalenceRetryPrompt originalPrompt, rawOutput string, parseErr erro 1818 1919 } 2020 21− func RunEquivalence ctx context.Context, ai AIClient, promptCtx PromptContext EquivalenceResult { 21+ func RunEquivalence ctx context.Context, ai AIClient, promptCtx PromptContext result EquivalenceResult { 22+ mode := EquivalencePromptModeStandard 23+ if goldBlindEquivalenceEnabled { 24+ mode = EquivalencePromptModeGoldBlind 25+ } 26+ history := newGradeAttemptClient ai 27+ ai = history 28+ defer func { result.PromptMode = mode; history.annotateEquivalence &result } 29+ 2230 if goldBlindEquivalenceEnabled { 2331 // No gold reaches the model, so gold truncation cannot have mutilated 2432 // the graded evidence: clear it before the truncation annotation runs. … 11 package validate 22 33 import 4+ "fmt" 45 "os" 56 "strconv" 67 "strings" … notes: <short explanation tied to the task obligations 127128 // mode. In gold-blind mode the gold patch is dropped from the template data as 128129 // well as from the template, so no gold content can reach the model. 129130 func renderEquivalencePrompt promptCtx PromptContext prompt string, goldBlind bool, err error { 130− if goldBlindEquivalenceEnabled { 131− rendered, renderErr := renderPrompt equivalencePromptTemplate, promptCtx.asTemplateData 132− return rendered, false, renderErr 131+ mode := EquivalencePromptModeStandard 132+ if goldBlindEquivalenceEnabled { 133+ mode = EquivalencePromptModeGoldBlind 134+ } 135+ rendered, err := RenderEquivalencePromptForMode promptCtx, mode 136+ return rendered, mode == EquivalencePromptModeGoldBlind, err 137+ } 138+ 139+ // RenderEquivalencePromptForMode uses recorded measurement authority, never the process environment. 140+ func RenderEquivalencePromptForMode promptCtx PromptContext, mode EquivalencePromptMode string, error { 141+ switch mode { 142+ case EquivalencePromptModeStandard: 143+ return renderPrompt equivalencePromptTemplate, promptCtx.asTemplateData 144+ case EquivalencePromptModeGoldBlind: 145+ data := promptCtx.asTemplateData 146+ delete data, "GoldPatch" 147+ return renderPrompt equivalenceGoldBlindPromptTemplate, data 148+ default: 149+ return "", fmt.Errorf "unknown equivalence prompt mode %q", mode 133150 } 134− data := promptCtx.asTemplateData 135− delete data, "GoldPatch" 136− rendered, renderErr := renderPrompt equivalenceGoldBlindPromptTemplate, data 137− return rendered, true, renderErr 138151 } … func runPerCommandTests 22022202 cmdCfg.Commands = string{cmd} 22032203 goldRun, err := RunGoldBaselineTests infraCtx, inputs, cmdCfg 22042204 gr.EnvCache = goldRun.EnvCache 2205− gr.Result = TestCommandResult{Command: cmd} 2205+ gr.Result = TestCommandResult{Command: cmd, AttemptCount: 1} 22062206 if err = nil { 22072207 gr.Err = err 2208− if mode := inferInfraFailureModeFromError err ; mode = "" { 2208+ if mode := retryableVerifierFailureMode goldRun, err ; mode = "" { 22092209 gr.Result.GoldOutcome = TestOutcomeFail 22102210 gr.Result.GoldFailureMode = mode 22112211 gr.IsInfra = true … func runPerCommandTests 22182218 gr.Result.GoldOutcome = TestOutcomeUnknown 22192219 gr.Notes = append gr.Notes, fmt.Sprintf "gold baseline failed for %q: %v", cmd, err 22202220 } 2221− return 2221+ if isRetryableGoldInfrastructureFailure gr.Result.GoldFailureMode { 2222+ return 2223+ } 22222224 } 22232225 gr.EnvCache = goldRun.EnvCache 22242226 applyGoldRunResult gr, goldRun 2225− if gr.Result.GoldOutcome == TestOutcomeFail && isRetryableGoldInfrastructureFailure gr.Result.GoldFailureMode && budgetCtx.Err == nil { 2227+ if isUnsetFailureMode gr.Result.GoldFailureMode { 2228+ gr.Result.GoldFailureMode = retryableVerifierFailureMode goldRun, err 2229+ } 2230+ if gr.Result.GoldOutcome = TestOutcomePass && isRetryableGoldInfrastructureFailure gr.Result.GoldFailureMode && budgetCtx.Err == nil { 22262231 beginGoldRetry &gr.Result, goldRun 2232+ recordTestRetry &gr.Result, gr.Result.GoldFailureMode 22272233 retryCfg := goldRetryAttemptConfig overrideCfg, gr.Index, 2 22282234 retryCfg.Commands = string{cmd} 22292235 retryRun, retryErr := RunGoldBaselineTests infraCtx, inputs, retryCfg … func runPerCommandTests 22442250 gr.Result.GoldRetried = true 22452251 finishGoldRetry &gr.Result, retryRun 22462252 if gr.Result.GoldOutcome == TestOutcomePass { 2253+ gr.Result.RecoveredAfterRetry = true 2254+ gr.Err = nil 2255+ gr.IsInfra = false 22472256 gr.Notes = append gr.Notes, fmt.Sprintf "gold baseline for %q recovered on retry after transient dependency fetch", cmd 22482257 } else { 22492258 gr.Notes = append gr.Notes, fmt.Sprintf "gold baseline retry for %q did not recover", cmd … func runPerCommandTests 24842493 cmdCfg := overrideCfg 24852494 cmdCfg.Commands = string{entry.cmd} 24862495 or.Attempted = true 2496+ or.Result.AttemptCount++ 24872497 overrideRun, err := RunOverrideTests budgetCtx, inputs, cmdCfg 24882498 or.EnvCache = overrideRun.EnvCache 24892499 if err = nil { 2490− if mode := inferInfraFailureModeFromError err ; mode = "" { 2500+ if mode := retryableVerifierFailureMode overrideRun, err ; mode = "" { 24912501 or.Result.AgentFailureMode = mode 24922502 if mode == "timeout" { 24932503 or.Notes = append or.Notes, fmt.Sprintf "agent retest timeout for %q", entry.cmd … func runPerCommandTests 24962506 or.Notes = append or.Notes, fmt.Sprintf "agent retest failed for %q: %v", entry.cmd, err 24972507 or.Result.AgentOutcome = TestOutcomeUnknown 24982508 or.Err = err 2499− return 2509+ if isRetryableGoldInfrastructureFailure or.Result.AgentFailureMode { 2510+ return 2511+ } 25002512 } 25012513 or.Result.AgentOutcome = testOutcomeFromTrial overrideRun.Trial 25022514 or.Result.AgentTrialName = overrideRun.Trial.TrialName 2503− or.Result.AgentFailureMode = overrideRun.Trial.FailureMode 2515+ or.Result.AgentFailureMode = retryableVerifierFailureMode overrideRun, err 25042516 2505− // Retry once if agent failed but gold passed. 2506− if or.Result.AgentOutcome == TestOutcomeFail && or.Result.GoldOutcome == TestOutcomePass && budgetCtx.Err == nil { 2517+ // Retry only a typed transient infrastructure failure; assertions are terminal. 2518+ if or.Result.AgentOutcome = TestOutcomePass && isRetryableGoldInfrastructureFailure or.Result.AgentFailureMode && or.Result.GoldOutcome == TestOutcomePass && budgetCtx.Err == nil { 25072519 retryCfg := overrideCfg 25082520 retryCfg.Commands = string{entry.cmd} 2521+ recordTestRetry &or.Result, or.Result.AgentFailureMode 2522+ or.Result.AgentRetried = true 25092523 retryRun, retryErr := RunOverrideTests budgetCtx, inputs, retryCfg 25102524 or.RetryEnvCache = &retryRun.EnvCache 2525+ or.Result.AgentOutcome = testOutcomeFromTrial retryRun.Trial 2526+ or.Result.AgentTrialName = retryRun.Trial.TrialName 2527+ or.Result.AgentFailureMode = retryableVerifierFailureMode retryRun, retryErr 2528+ or.Err = retryErr 25112529 if retryErr == nil { 25122530 retryOutcome := testOutcomeFromTrial retryRun.Trial 25132531 if retryOutcome == TestOutcomePass { … func runPerCommandTests 25152533 or.Result.AgentTrialName = retryRun.Trial.TrialName 25162534 or.Result.AgentFailureMode = retryRun.Trial.FailureMode 25172535 or.Result.AgentRetried = true 2536+ or.Result.RecoveredAfterRetry = true 2537+ or.Err = nil 25182538 or.Notes = append or.Notes, fmt.Sprintf "agent retest for %q succeeded on retry", entry.cmd 25192539 } 25202540 } … func runGoldBatchSlice 347347 cleanups := func {} 348348 for , idx := range indexes { 349349 gr := &goldResults idx 350+ priorResult := gr.Result 350351 priorRetried, priorEvidence := gr.Result.GoldRetried, gr.Result.GoldRetryEvidence 351352 gr.IsInfra = false 352353 gr.IsGlobalInfra = false 353354 gr.Err = nil 354355 if ctx.Err = nil { 355356 gr.Result = TestCommandResult{ 356− Command: gr.Command, 357− GoldOutcome: TestOutcomeUnknown, 358− GoldRetried: priorRetried, 359− GoldRetryEvidence: priorEvidence, 357+ Command: gr.Command, 358+ GoldOutcome: TestOutcomeUnknown, 359+ GoldRetried: priorRetried, 360+ GoldRetryEvidence: priorEvidence, 361+ AttemptCount: priorResult.AttemptCount, 362+ RetryCount: priorResult.RetryCount, 363+ RetryReasons: priorResult.RetryReasons, 364+ RecoveredAfterRetry: priorResult.RecoveredAfterRetry, 360365 } 361366 gr.Err = ctx.Err 362367 continue … func runGoldBatchSlice 364369 stageRoot, cacheResult, cleanup, err := stageGoldBaselineTask ctx, inputs, string{commands idx }, false, overrideCfg.OutputDir 365370 gr.EnvCache = cacheResult 366371 gr.Result = TestCommandResult{ 367− Command: gr.Command, 368− GoldRetried: priorRetried, 369− GoldRetryEvidence: priorEvidence, 372+ Command: gr.Command, 373+ GoldRetried: priorRetried, 374+ GoldRetryEvidence: priorEvidence, 375+ AttemptCount: priorResult.AttemptCount, 376+ RetryCount: priorResult.RetryCount, 377+ RetryReasons: priorResult.RetryReasons, 378+ RecoveredAfterRetry: priorResult.RecoveredAfterRetry, 370379 } 371380 if err = nil { 372381 gr.Err = err … func runGoldBatchSlice 407416 gr.Result.GoldRetried = true 408417 gr.Result.GoldRetryEvidence = &TestRetryEvidence{First: gr.FirstRetryEvidence} 409418 } 419+ recordTestRetry &gr.Result, FailureModeInfraDependencyFetch 410420 gr.RetryDispatched = true 411421 } 412422 } 413423 424+ for , item := range items { 425+ var idx int 426+ fmt.Sscanf item.ProjectionKey, "gold-%d", &idx 427+ if attempt == 1 { 428+ goldResults idx .Result.AttemptCount++ 429+ } 430+ } 414431 results, err := batchRunner.RunBatch ctx, harness.BatchRunConfig{ 415432 OutputDir: filepath.Join filepath.Dir items 0 .SourceTaskDir , ".tb-runs" , 416433 RunID: fmt.Sprintf "validate-gold-batch-%d", time.Now .UnixNano , … func runGoldBatchSlice 452469 evidence := retainBatchGoldRetryAttemptEvidence projected, gr.Result.GoldFailureMode, overrideCfg, gr.Index, attempt 453470 gr.FirstRetryEvidence = &evidence 454471 } 472+ if attempt 1 && gr.Result.GoldOutcome == TestOutcomePass { 473+ gr.Result.RecoveredAfterRetry = true 474+ } 455475 if attempt 1 && gr.Result.GoldRetryEvidence = nil { 456476 evidence := retainBatchGoldRetryAttemptEvidence projected, gr.Result.GoldFailureMode, overrideCfg, gr.Index, attempt 457477 gr.Result.GoldRetryEvidence.Retry = &evidence … func runOverrideBatchSlice 564584 return 565585 } 566586 587+ for , item := range items { 588+ var idx int 589+ fmt.Sscanf item.ProjectionKey, "override-%d", &idx 590+ overrideResults idx-offset .Result.AttemptCount++ 591+ } 592+ 567593 results, err := batchRunner.RunBatch ctx, harness.BatchRunConfig{ 568594 OutputDir: filepath.Join filepath.Dir items 0 .SourceTaskDir , ".tb-runs" , 569595 RunID: fmt.Sprintf "validate-override-batch-%d", time.Now .UnixNano , … func runOverrideBatchSlice 604630 projected := results fmt.Sprintf "override-%d", offset+i 605631 or.Result.AgentOutcome = testOutcomeFromTrial projected.Trial 606632 or.Result.AgentTrialName = projected.Trial.TrialName 607− or.Result.AgentFailureMode = projected.Trial.FailureMode 608− if or.Result.AgentOutcome == TestOutcomeFail && or.Result.GoldOutcome == TestOutcomePass && ctx.Err == nil { 633+ or.Result.AgentFailureMode = classifyBatchGoldFailureMode projected 634+ if or.Result.AgentOutcome = TestOutcomePass && isRetryableGoldInfrastructureFailure or.Result.AgentFailureMode && or.Result.GoldOutcome == TestOutcomePass && ctx.Err == nil { 609635 stageRoot, cacheResult, cleanup, err := stageOverrideTask ctx, inputs, string{or.Command}, false, overrideCfg.OutputDir 610636 if err == nil { 611637 or.RetryEnvCache = &cacheResult … func runOverrideBatchSlice 623649 if len retryItems == 0 { 624650 return 625651 } 652+ for , idx := range retryIndexes { 653+ or := &overrideResults idx 654+ recordTestRetry &or.Result, or.Result.AgentFailureMode 655+ or.Result.AgentRetried = true 656+ } 626657 retryResults, err := batchRunner.RunBatch ctx, harness.BatchRunConfig{ 627658 OutputDir: filepath.Join filepath.Dir retryItems 0 .SourceTaskDir , ".tb-runs" , 628659 RunID: fmt.Sprintf "validate-override-retry-batch-%d", time.Now .UnixNano , … func runOverrideBatchSlice 634665 HarnessCLICacheMode: overrideCfg.HarnessCLICacheMode, 635666 Items: retryItems, 636667 } 637− if err = nil { 638− return 639− } 640668 for , idx := range retryIndexes { 641669 or := &overrideResults idx 642670 projected := retryResults fmt.Sprintf "retry-%d", offset+idx 643− if testOutcomeFromTrial projected.Trial == TestOutcomePass { 644− or.Result.AgentOutcome = TestOutcomePass 645− or.Result.AgentTrialName = projected.Trial.TrialName 646− or.Result.AgentFailureMode = projected.Trial.FailureMode 647− or.Result.AgentRetried = true 671+ or.Result.AgentOutcome = testOutcomeFromTrial projected.Trial 672+ or.Result.AgentTrialName = projected.Trial.TrialName 673+ or.Result.AgentFailureMode = classifyBatchGoldFailureMode projected 674+ or.Err = err 675+ if err = nil { 676+ if isUnsetFailureMode or.Result.AgentFailureMode { 677+ or.Result.AgentFailureMode = inferInfraFailureModeFromError err 678+ } 679+ or.Notes = append or.Notes, fmt.Sprintf "agent retest retry for %q failed: %v", or.Command, err 680+ } 681+ if err == nil && or.Result.AgentOutcome == TestOutcomePass { 682+ or.Result.RecoveredAfterRetry = true 648683 or.Notes = append or.Notes, fmt.Sprintf "agent retest for %q succeeded on retry", or.Command 649684 } 650685 } … go: golang.org/x/sys@v0.42.0: Get "https://proxy.golang.org/golang.org/x/sys/@v/ 50215021 t.Fatalf "command results = %d, want 1", len summary.CommandResults 50225022 } 50235023 result := summary.CommandResults 0 5024+ if result.AttemptCount = 3 || result.RetryCount = 1 || result.RecoveredAfterRetry || len result.RetryReasons = 1 || result.RetryReasons 0 = FailureModeInfraDependencyFetch { 5025+ t.Fatalf "retry metadata = %+v", result 5026+ } 50245027 if result.GoldRetried || result.GoldOutcome = TestOutcomePass || result.GoldTrialName = "gold-retry" { 50255028 t.Fatalf "gold retry result = %+v", result 50265029 } … go: golang.org/x/sys@v0.42.0: Get "https://proxy.golang.org/golang.org/x/sys/@v/ 51115114 t.Fatalf "command results = %d, want 1", len summary.CommandResults 51125115 } 51135116 result := summary.CommandResults 0 5117+ if result.AttemptCount = 3 || result.RetryCount = 1 || result.RecoveredAfterRetry || len result.RetryReasons = 1 || result.RetryReasons 0 = FailureModeInfraDependencyFetch { 5118+ t.Fatalf "retry metadata = %+v", result 5119+ } 51145120 if result.GoldRetried || result.GoldOutcome = TestOutcomePass || result.GoldTrialName = "gold-retry" { 51155121 t.Fatalf "gold retry result = %+v", result 51165122 } … func TestRunPerCommandTestsBatchedGoldTimeoutClassifiedAsTimeBudgetExhausted t 53825388 } 53835389 } 53845390 5385− func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried t testing.T { 5391+ func TestRunPerCommandTestsBatchedRetriesDependencyFetchOverrideAndMarksRetried t testing.T { 53865392 t.Parallel 53875393 repo, base, head := createRepoWithCommits t 53885394 dataset := t.TempDir … func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried t testin 54185424 runner := &batchValidateRunner{ 54195425 results: map string TBTrialResult{ 54205426 "gold-0": {TaskID: taskID, IsResolved: &pass, TrialName: "gold-a"}, 5421− "override-0": {TaskID: taskID, IsResolved: &fail, TrialName: "override-a"}, 5427+ "override-0": {TaskID: taskID, IsResolved: &fail, TrialName: "override-a", FailureMode: FailureModeInfraDependencyFetch}, 54225428 "retry-0": {TaskID: taskID, IsResolved: &pass, TrialName: "retry-a"}, 54235429 }, 54245430 } … func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried t testin 54435449 t.Fatalf "expected 1 command result, got %d", len result.Tests.CommandResults 54445450 } 54455451 cmd := result.Tests.CommandResults 0 5452+ if cmd.AttemptCount = 3 || cmd.RetryCount = 1 || cmd.RecoveredAfterRetry || len cmd.RetryReasons = 1 || cmd.RetryReasons 0 = FailureModeInfraDependencyFetch { 5453+ t.Fatalf "retry metadata = %+v", cmd 5454+ } 54465455 if cmd.AgentRetried { 54475456 t.Fatal "expected AgentRetried=true" 54485457 } … 11 package validate 22 3− import "fmt" 3+ import 4+ "fmt" 5+ "path/filepath" 6+ "strings" 7+ 48 59 // OfflinePromptInputs describes everything needed to reassemble the grader 610 // PromptContext for a single already-evaluated cell without re-running the … type OfflinePromptInputs struct { 1822 // TBResultsRaw is the raw TB results JSON for this cell, or "" when 1923 // unavailable e.g. no-test-signal corpora . 2024 TBResultsRaw string 25+ // EffectiveTestCommands overrides dataset commands when non-nil. An empty 26+ // non-nil slice preserves a persisted grade with no verifier commands. 27+ EffectiveTestCommands string 2128 // PromptPatchMaxBytes caps the gold/agent patch bytes embedded in the 2229 // prompt. Values <= 0 fall back to DefaultPromptPatchMaxBytes, matching the 2330 // orchestrator/config default. … type OfflinePromptInputs struct { 2532 // RepoContextOptions controls repo tree/excerpt extraction. The zero value 2633 // inherits BuildRepoContext defaults, matching the live default path. 2734 RepoContextOptions RepoContextOptions 35+ // RequireRepoContext makes reconstruction fail when the source repository 36+ // cannot be extracted, instead of grading an unavailable-context marker. 37+ RequireRepoContext bool 38+ // ExpectedArtifacts binds reconstruction to the original graded files. 39+ // When supplied, every source path and digest must be present and match. 40+ ExpectedArtifacts ArtifactManifest 2841 } 2942 3043 // BuildOfflinePromptContext reassembles the grader PromptContext for one cell … type OfflinePromptInputs struct { 3447 // that removes the extracted repo temp dir; callers should defer it. 3548 // 3649 // The assembly intentionally mirrors internal/validate/orchestrator.go so the 37− // resulting context is byte-for-byte identical to a live grade except for the 38− // template version: it reuses ReadTaskInputs, truncatePatch, and 39− // BuildRepoContext, and applies the same graceful repo-context fallback. 50+ // assembly reuses ReadTaskInputs, truncatePatch, and BuildRepoContext. Callers 51+ // can supply persisted effective commands and require usable repository context; 52+ // otherwise it applies the live path's graceful repo-context fallback. 4053 func BuildOfflinePromptContext in OfflinePromptInputs PromptContext, func , error { 4154 inputs, err := ReadTaskInputs in.DatasetPath, in.TaskID, in.AgentPatchPath 4255 if err = nil { … func BuildOfflinePromptContext in OfflinePromptInputs PromptContext, func , e 4558 if err := verifyRepoSourceForConsumption inputs ; err = nil { 4659 return PromptContext{}, nil, err 4760 } 61+ if in.ExpectedArtifacts = nil { 62+ current := BuildArtifactManifest inputs 63+ for , pair := range struct { 64+ name string 65+ want, got ArtifactFileManifest 66+ }{ 67+ {"agent patch", in.ExpectedArtifacts.AgentPatch, current.AgentPatch}, 68+ {"gold patch", in.ExpectedArtifacts.GoldPatch, current.GoldPatch}, 69+ {"repository", in.ExpectedArtifacts.RepoTarball, current.RepoTarball}, 70+ } { 71+ if pair.want.Path == "" || pair.want.SHA256 == "" { 72+ return PromptContext{}, nil, fmt.Errorf "missing original %s path or digest", pair.name 73+ } 74+ if filepath.Clean pair.want.Path = filepath.Clean pair.got.Path || strings.EqualFold strings.TrimPrefix pair.want.SHA256, "sha256:" , pair.got.SHA256 { 75+ return PromptContext{}, nil, fmt.Errorf "original %s provenance mismatch", pair.name 76+ } 77+ } 78+ } 4879 4980 maxBytes := in.PromptPatchMaxBytes 5081 if maxBytes <= 0 { … func BuildOfflinePromptContext in OfflinePromptInputs PromptContext, func , e 5283 } 5384 goldPatchPrompt, goldPatchTruncation := truncatePatch inputs.GoldPatch, maxBytes 5485 agentPatchPrompt, agentPatchTruncation := truncatePatch inputs.AgentPatch, maxBytes 86+ if in.ExpectedArtifacts = nil && goldPatchTruncation.Truncated || agentPatchTruncation.Truncated { 87+ return PromptContext{}, nil, fmt.Errorf "reconstructed prompt patches would be truncated; original context budgets are unavailable" 88+ } 89+ testCommands := inputs.TestCommands 90+ if in.EffectiveTestCommands = nil { 91+ testCommands = append string{}, in.EffectiveTestCommands... 92+ } 5593 5694 // Mirror orchestrator.go: on repo-context failure, degrade RepoTree to an 5795 // "unavailable: <err " marker and continue rather than failing the build. … func BuildOfflinePromptContext in OfflinePromptInputs PromptContext, func , e 5997 repoCtx := RepoContext{} 6098 builtRepoCtx, repoCleanup, repoErr := BuildRepoContext inputs.RepoTarballPath, in.RepoContextOptions 6199 if repoErr = nil { 100+ if in.RequireRepoContext { 101+ return PromptContext{}, nil, fmt.Errorf "build required repo context: %w", repoErr 102+ } 62103 builtRepoCtx = RepoContext{ 63104 RepoTree: fmt.Sprintf "unavailable: %v", repoErr , 64105 } 65106 } else { 66107 cleanup = repoCleanup 108+ if in.ExpectedArtifacts = nil && builtRepoCtx.RepoTreeTruncation.Truncated || builtRepoCtx.RepoExcerptsTruncation.Truncated { 109+ if cleanup = nil { 110+ cleanup 111+ } 112+ return PromptContext{}, nil, fmt.Errorf "reconstructed repository context would be truncated; original context budgets are unavailable" 113+ } 67114 } 68115 repoCtx = builtRepoCtx 69116 … func BuildOfflinePromptContext in OfflinePromptInputs PromptContext, func , e 71118 TaskYAML: inputs.TaskYAML, 72119 GoldPatch: goldPatchPrompt, 73120 AgentPatch: agentPatchPrompt, 74− TestCommands: inputs.TestCommands, 121+ TestCommands: testCommands, 75122 TBResults: in.TBResultsRaw, 76123 RepoPath: repoCtx.RepoPath, 77124 RepoTree: repoCtx.RepoTree, … func BuildOfflinePromptContext in OfflinePromptInputs PromptContext, func , e 80127 GoldPatchTruncated: goldPatchTruncation.Truncated, 81128 }, cleanup, nil 82129 } 130+ 131+ // RenderReviewPrompt renders the built-in review template from supplied evidence. 132+ func RenderReviewPrompt promptCtx PromptContext string, error { 133+ return renderPrompt reviewPromptTemplate, promptCtx.asTemplateData 134+ } … func TestBuildOfflinePromptContextDegradesWhenTarballUnextractable t testing.T 164164 t.Fatalf "expected agent patch to be populated, got %q", promptCtx.AgentPatch 165165 } 166166 } 167+ 168+ func TestBuildOfflinePromptContextRequiresUsableRepositoryWhenRequested t testing.T { 169+ datasetPath, taskID, agentPatchPath := writeOfflineTaskDir t, false 170+ , cleanup, err := BuildOfflinePromptContext OfflinePromptInputs{ 171+ DatasetPath: datasetPath, TaskID: taskID, AgentPatchPath: agentPatchPath, 172+ RequireRepoContext: true, 173+ } 174+ if cleanup = nil { 175+ defer cleanup 176+ } 177+ if err == nil || strings.Contains err.Error , "build required repo context" { 178+ t.Fatalf "expected required repository extraction failure, got %v", err 179+ } 180+ } 181+ 182+ func TestBuildOfflinePromptContextUsesEffectiveVerifierCommands t testing.T { 183+ datasetPath, taskID, agentPatchPath := writeOfflineTaskDir t, true 184+ for , commands := range string{nil, {}, {"effective verifier"}} { 185+ ctx, cleanup, err := BuildOfflinePromptContext OfflinePromptInputs{ 186+ DatasetPath: datasetPath, TaskID: taskID, AgentPatchPath: agentPatchPath, 187+ EffectiveTestCommands: commands, 188+ } 189+ if cleanup = nil { 190+ defer cleanup 191+ } 192+ if err = nil { 193+ t.Fatal err 194+ } 195+ want := commands 196+ if commands == nil { 197+ want = string{"pytest -q"} 198+ } 199+ if reflect.DeepEqual ctx.TestCommands, want { 200+ t.Fatalf "effective commands %v: got %v, want %v", commands, ctx.TestCommands, want 201+ } 202+ } 203+ } 204+ 205+ func TestBuildOfflinePromptContextRejectsMutatedOriginalArtifacts t testing.T { 206+ for , artifact := range string{"agent patch", "gold patch", "repository"} { 207+ t.Run artifact, func t testing.T { 208+ dataset, taskID, patch := writeOfflineTaskDir t, true 209+ inputs, err := ReadTaskInputs dataset, taskID, patch 210+ if err = nil { 211+ t.Fatal err 212+ } 213+ original := BuildArtifactManifest inputs 214+ path := patch 215+ if artifact == "gold patch" { 216+ path = inputs.GoldPatchPath 217+ } 218+ if artifact == "repository" { 219+ path = inputs.RepoTarballPath 220+ } 221+ if err := os.WriteFile path, byte "mutated evidence" , 0o644 ; err = nil { 222+ t.Fatal err 223+ } 224+ , cleanup, err := BuildOfflinePromptContext OfflinePromptInputs{ 225+ DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch, 226+ ExpectedArtifacts: &original, RequireRepoContext: true, 227+ } 228+ if cleanup = nil { 229+ defer cleanup 230+ } 231+ if err == nil || strings.Contains err.Error , artifact+" provenance mismatch" { 232+ t.Fatalf "expected original %s mismatch, got %v", artifact, err 233+ } 234+ } 235+ } 236+ } 237+ 238+ func TestBuildOfflinePromptContextRejectsAbsentOriginalArtifactDigest t testing.T { 239+ dataset, taskID, patch := writeOfflineTaskDir t, true 240+ inputs, err := ReadTaskInputs dataset, taskID, patch 241+ if err = nil { 242+ t.Fatal err 243+ } 244+ original := BuildArtifactManifest inputs 245+ original.AgentPatch.SHA256 = "" 246+ , cleanup, err := BuildOfflinePromptContext OfflinePromptInputs{ 247+ DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch, ExpectedArtifacts: &original, 248+ } 249+ if cleanup = nil { 250+ defer cleanup 251+ } 252+ if err == nil || strings.Contains err.Error , "missing original agent patch" { 253+ t.Fatalf "expected absent original provenance failure, got %v", err 254+ } 255+ } 256+ 257+ func TestBuildOfflinePromptContextRejectsNewTruncationWithOriginalProvenance t testing.T { 258+ dataset, taskID, patch := writeOfflineTaskDir t, true 259+ if err := os.WriteFile patch, byte largePatch 4 000 , 0o644 ; err = nil { 260+ t.Fatal err 261+ } 262+ inputs, err := ReadTaskInputs dataset, taskID, patch 263+ if err = nil { 264+ t.Fatal err 265+ } 266+ original := BuildArtifactManifest inputs 267+ , cleanup, err := BuildOfflinePromptContext OfflinePromptInputs{ 268+ DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch, ExpectedArtifacts: &original, 269+ PromptPatchMaxBytes: 600, 270+ } 271+ if cleanup = nil { 272+ defer cleanup 273+ } 274+ if err == nil || strings.Contains err.Error , "would be truncated" { 275+ t.Fatalf "expected new truncation refusal, got %v", err 276+ } 277+ } … const 2828 var coverageRepairRetryBackoffs = time.Duration{time.Second, 2 time.Second} 2929 3030 type CoverageRepairConfig struct { 31+ EquivalenceAI AIClient 32+ EquivalencePrompt string 33+ EquivalencePromptContext PromptContext 34+ EquivalencePromptMode EquivalencePromptMode 35+ ReviewAI AIClient 36+ ReviewPrompt string 37+ ReviewPromptContext PromptContext 38+ 3139 Existing ValidationResult 3240 AI AIClient 3341 AIModelID string … type CoverageRepairConfig struct { 3745 } 3846 3947 type CoverageRepairIssue struct { 48+ AttemptCount int json:"attempt count,omitempty" 49+ RetryCount int json:"retry count,omitempty" 50+ RetryReasons string json:"retry reasons,omitempty" 51+ 4052 Dimension string json:"dimension" 4153 Reason string json:"reason" 4254 Category string json:"category,omitempty" … func RepairCoverage ctx context.Context, cfg CoverageRepairConfig CoverageRepa 6981 updated := false 7082 7183 if cfg.RepairEquivalence { 72− equivalence, repaired, issue := repairEquivalenceFromPrompt ctx, ai, cfg.Existing.Equivalence, cfg.AIModelID, cfg.ReviewParseRetries 84+ source := cfg.Existing.Equivalence 85+ if cfg.EquivalencePrompt = "" { 86+ source.Prompt = cfg.EquivalencePrompt 87+ } 88+ mode := source.PromptMode 89+ if cfg.EquivalencePromptMode = "" { 90+ if mode = "" && mode = cfg.EquivalencePromptMode { 91+ return result, errors.New "equivalence repair prompt mode differs from original measurement" 92+ } 93+ mode = cfg.EquivalencePromptMode 94+ } 95+ if cfg.EquivalencePromptContext = nil { 96+ rendered, err := RenderEquivalencePromptForMode cfg.EquivalencePromptContext, mode 97+ if err = nil { 98+ return result, err 99+ } 100+ if cfg.EquivalencePrompt = "" && cfg.EquivalencePrompt = rendered { 101+ return result, errors.New "equivalence repair prompt differs from supplied context" 102+ } 103+ source.Prompt = rendered 104+ } 105+ source.PromptMode = mode 106+ dimensionAI := cfg.EquivalenceAI 107+ if dimensionAI == nil { 108+ dimensionAI = ai 109+ } 110+ history := newGradeAttemptClient dimensionAI 111+ equivalence, repaired, issue := repairEquivalenceFromPrompt ctx, history, source, cfg.AIModelID, cfg.ReviewParseRetries 112+ if repaired { 113+ equivalence.PromptMode = mode 114+ if cfg.EquivalencePromptContext = nil { 115+ pc := cfg.EquivalencePromptContext 116+ if mode == EquivalencePromptModeGoldBlind { 117+ pc.GoldPatchTruncated = false 118+ } 119+ equivalence = annotateEquivalenceTruncation equivalence, pc 120+ } 121+ history.annotateEquivalence &equivalence 122+ mergeEquivalenceRepairHistory &equivalence, source 123+ } 73124 if issue = nil { 125+ issue.AttemptCount = history.attempts 126+ issue.RetryCount = max 0, history.attempts-1 127+ issue.RetryReasons = append string nil , history.reasons... 74128 result.Unresolved = append result.Unresolved, issue 75129 } else { 76130 result.Validation.Equivalence = equivalence … func RepairCoverage ctx context.Context, cfg CoverageRepairConfig CoverageRepa 84138 if strings.TrimSpace source.Prompt == "" && cfg.Existing.Review = nil { 85139 source = cfg.Existing.Review 86140 } 87− review, repaired, issue := repairReviewFromPrompt ctx, ai, source, cfg.AIModelID, cfg.ReviewParseRetries 141+ if cfg.ReviewPrompt = "" { 142+ source.Prompt = cfg.ReviewPrompt 143+ } 144+ if cfg.ReviewPromptContext = nil { 145+ rendered, err := renderPrompt reviewPromptTemplate, cfg.ReviewPromptContext.asTemplateData 146+ if err = nil { 147+ return result, err 148+ } 149+ if cfg.ReviewPrompt = "" && cfg.ReviewPrompt = rendered { 150+ return result, errors.New "review repair prompt differs from supplied context" 151+ } 152+ source.Prompt = rendered 153+ } 154+ if source.Prompt = resolvedRepairReview cfg.Existing .Prompt { 155+ source.Output = "" 156+ } 157+ dimensionAI := cfg.ReviewAI 158+ if dimensionAI == nil { 159+ dimensionAI = ai 160+ } 161+ history := newGradeAttemptClient dimensionAI 162+ review, repaired, issue := repairReviewFromPromptContext ctx, history, source, cfg.AIModelID, cfg.ReviewParseRetries, cfg.ReviewPromptContext 163+ if repaired && history.attempts 0 { 164+ history.annotateReview &review 165+ mergeReviewRepairHistory &review, source 166+ } 167+ if repaired && cfg.ReviewPromptContext = nil { 168+ review = annotateReviewTruncation review, cfg.ReviewPromptContext 169+ } 88170 if issue = nil { 171+ issue.AttemptCount = history.attempts 172+ issue.RetryCount = max 0, history.attempts-1 173+ issue.RetryReasons = append string nil , history.reasons... 89174 result.Unresolved = append result.Unresolved, issue 90175 } else { 91176 result.Validation.CodeReview = review … func RepairCoverage ctx context.Context, cfg CoverageRepairConfig CoverageRepa 104189 } 105190 106191 func repairEquivalenceFromPrompt ctx context.Context, ai AIClient, existing EquivalenceResult, overrideModelID string, parseRetries int EquivalenceResult, bool, CoverageRepairIssue { 107− prompt := strings.TrimSpace existing.Prompt 108− if prompt == "" { 192+ prompt := existing.Prompt 193+ if strings.TrimSpace prompt == "" { 109194 return existing, false, &CoverageRepairIssue{ 110195 Dimension: coverageRepairDimensionEquivalence, 111196 Reason: "missing prompt", … func repairEquivalenceFromPrompt ctx context.Context, ai AIClient, existing Equi 145230 146231 parsed.Status = AIStatusUsed 147232 parsed.EvaluatorModel = resolveRepairEvaluatorModel existing.EvaluatorModel, overrideModelID 233+ parsed.PromptMode = existing.PromptMode 234+ parsed.Truncated = existing.Truncated 235+ parsed.TruncationReason = existing.TruncationReason 148236 parsed.Prompt = prompt 149237 parsed.Output = output 150238 return parsed, true, nil … func reviewVerdictIsGuardSuppressible result ReviewResult bool { 159247 } 160248 161249 func repairReviewFromPrompt ctx context.Context, ai AIClient, existing ReviewResult, overrideModelID string, parseRetries int ReviewResult, bool, CoverageRepairIssue { 162− prompt := strings.TrimSpace existing.Prompt 163− if prompt == "" { 250+ return repairReviewFromPromptContext ctx, ai, existing, overrideModelID, parseRetries, nil 251+ } 252+ 253+ func repairReviewFromPromptContext ctx context.Context, ai AIClient, existing ReviewResult, overrideModelID string, parseRetries int, promptCtx PromptContext ReviewResult, bool, CoverageRepairIssue { 254+ prompt := existing.Prompt 255+ if strings.TrimSpace prompt == "" { 164256 return existing, false, &CoverageRepairIssue{ 165257 Dimension: coverageRepairDimensionCodeReview, 166258 Reason: "missing prompt", … func repairReviewFromPrompt ctx context.Context, ai AIClient, existing ReviewRes 174266 // re-apply that guard, so such verdicts must be resampled, never recovered. 175267 if existing.Status == AIStatusUnavailable && strings.TrimSpace existing.Output = "" { 176268 if reparsed, reparseErr := parseReviewOutput existing.Output ; reparseErr == nil && 177− reviewVerdictIsGuardSuppressible reparsed { 269+ promptCtx == nil && reviewVerdictIsGuardSuppressible reparsed || promptCtx = nil && validateReviewAgainstPromptContext reparsed, promptCtx == nil { 178270 reparsed.Status = AIStatusUsed 179271 reparsed.Prompt = prompt 180272 reparsed.Output = existing.Output … func repairReviewFromPrompt ctx context.Context, ai AIClient, existing ReviewRes 184276 if reparsed.EvaluatorModel == "" { 185277 reparsed.EvaluatorModel = strings.TrimSpace overrideModelID 186278 } 279+ reparsed.AttemptCount = existing.AttemptCount 280+ reparsed.RetryCount = existing.RetryCount 281+ reparsed.RetryReasons = append string nil , existing.RetryReasons... 282+ reparsed.RecoveredAfterRetry = existing.RecoveredAfterRetry 187283 reparsed.Truncated = existing.Truncated 188284 reparsed.TruncationReason = existing.TruncationReason 189285 return reparsed, true, nil … func repairReviewFromPrompt ctx context.Context, ai AIClient, existing ReviewRes 196292 } 197293 198294 parsed, parseErr := parseReviewOutput output 295+ if parseErr == nil && promptCtx = nil { 296+ parseErr = validateReviewAgainstPromptContext parsed, promptCtx 297+ } 199298 for attempt := 0; parseErr = nil && attempt < normalizedReviewParseRetries parseRetries ; attempt++ { 200299 retryPrompt := buildReviewRetryPrompt prompt, output, parseErr 201300 retryOutput, retryErr := runAIWithRetries ctx, ai, retryPrompt … func repairReviewFromPrompt ctx context.Context, ai AIClient, existing ReviewRes 203302 return existing, false, newReviewRuntimeFailureIssue retryErr 204303 } 205304 parsed, parseErr = parseReviewOutput retryOutput 305+ if parseErr == nil && promptCtx = nil { 306+ parseErr = validateReviewAgainstPromptContext parsed, promptCtx 307+ } 206308 output = retryOutput 207309 } 208310 if parseErr = nil { 209311 return existing, false, classifyReviewParseIssue parseErr 210312 } 211313 314+ if promptCtx == nil && reviewVerdictIsGuardSuppressible parsed { 315+ return existing, false, classifyReviewParseIssue errors.New "review repair requires original verifier context for speculative blocker verdict" 316+ } 317+ parsed.Truncated = existing.Truncated 318+ parsed.TruncationReason = existing.TruncationReason 212319 parsed.Status = AIStatusUsed 213320 parsed.Prompt = prompt 214321 parsed.Output = output … func repairReviewFromPrompt ctx context.Context, ai AIClient, existing ReviewRes 217324 } 218325 219326 func runAIWithRetries ctx context.Context, ai AIClient, prompt string string, error { 220− var lastErr error 221− for attempt := 0; attempt <= len coverageRepairRetryBackoffs ; attempt++ { 222− output, err := ai.Run ctx, prompt 223− if err == nil { 224− return output, nil 225− } 226− lastErr = err 227− if errors.Is err, ErrAIDisabled || errors.Is err, context.Canceled || errors.Is err, context.DeadlineExceeded { 228− break 229− } 230− if attempt = len coverageRepairRetryBackoffs { 231− break 232− } 233− select { 234− case <-ctx.Done : 235− return "", ctx.Err 236− case <-time.After coverageRepairRetryBackoffs attempt : 237− } 327+ if history, ok := ai. gradeAttemptClient ; ok { 328+ return history.Run ctx, prompt 238329 } 239− return "", lastErr 330+ return newGradeAttemptClient ai .Run ctx, prompt 240331 } 241332 242333 func normalizedReviewParseRetries parseRetries int int { … func appendReviewNote existing, extra string string { 5353 } 5454 } 5555 56− func RunReview ctx context.Context, ai AIClient, promptCtx PromptContext ReviewResult { 56+ func RunReview ctx context.Context, ai AIClient, promptCtx PromptContext result ReviewResult { 57+ history := newGradeAttemptClient ai 58+ defer func { history.annotateReview &result } 59+ ai = history 60+ 5761 return annotateReviewTruncation runReview ctx, ai, promptCtx , promptCtx 5862 } 5963 … type TestVerifierEvidence struct { 712712 } 713713 714714 type TestCommandResult struct { 715− Command string json:"command" 716− GoldOutcome TestOutcome json:"gold outcome,omitempty" 717− GoldTrialName string json:"gold tb trial name,omitempty" 718− GoldFailureMode string json:"gold failure mode,omitempty" 719− GoldRetried bool json:"gold retried,omitempty" 720− GoldRetryEvidence TestRetryEvidence json:"gold retry evidence,omitempty" 721− AgentOutcome TestOutcome json:"agent outcome,omitempty" 722− AgentTrialName string json:"agent tb trial name,omitempty" 723− AgentFailureMode string json:"agent failure mode,omitempty" 724− AgentRetried bool json:"agent retried,omitempty" 715+ // Counts cover dispatched gold and candidate verifier calls for this command. 716+ AttemptCount int json:"attempt count,omitempty" 717+ RetryCount int json:"retry count,omitempty" 718+ RetryReasons string json:"retry reasons,omitempty" 719+ RecoveredAfterRetry bool json:"recovered after retry,omitempty" 720+ Command string json:"command" 721+ GoldOutcome TestOutcome json:"gold outcome,omitempty" 722+ GoldTrialName string json:"gold tb trial name,omitempty" 723+ GoldFailureMode string json:"gold failure mode,omitempty" 724+ GoldRetried bool json:"gold retried,omitempty" 725+ GoldRetryEvidence TestRetryEvidence json:"gold retry evidence,omitempty" 726+ AgentOutcome TestOutcome json:"agent outcome,omitempty" 727+ AgentTrialName string json:"agent tb trial name,omitempty" 728+ AgentFailureMode string json:"agent failure mode,omitempty" 729+ AgentRetried bool json:"agent retried,omitempty" 725730 } 726731 727732 // TestRetryEvidence records the two verifier attempts that authorized and … type TestRetryAttemptEvidence struct { 742747 } 743748 744749 type EquivalenceResult struct { 750+ AttemptCount int json:"attempt count,omitempty" 751+ RetryCount int json:"retry count,omitempty" 752+ RetryReasons string json:"retry reasons,omitempty" 753+ RecoveredAfterRetry bool json:"recovered after retry,omitempty" 754+ 745755 Outcome EquivalenceOutcome json:"outcome" 746756 Status AIStatus json:"status" 747757 // PromptMode records the exact built-in equivalence template variant that … type EquivalenceObligation struct { 785795 } 786796 787797 type ReviewResult struct { 798+ AttemptCount int json:"attempt count,omitempty" 799+ RetryCount int json:"retry count,omitempty" 800+ RetryReasons string json:"retry reasons,omitempty" 801+ RecoveredAfterRetry bool json:"recovered after retry,omitempty" 802+ 788803 Status AIStatus json:"status" 789804 SchemaVersion string json:"schema version,omitempty" 790805 ProfileID string json:"profile id,omitempty" … Recovery rules: 793793 Those repair steps preserve compare-critical metadata and existing custom 794794 grader surfaces so the arm stays compare-compatible while recovering 795795 coverage. Add --parse-retries N when saved grader prompts failed 796− JSON/schema parsing. Keep the original custom rubric file path on the regrade 797− command. 796+ JSON/schema parsing. Repair preserves recorded prompt mode, truncation, and 797+ attempt history. Missing prompts require matching original task and artifact 798+ provenance plus complete repository context; ambiguous reconstruction remains 799+ unresolved. Runtime retries cover recognized transient failures only. 800+ Keep the original custom rubric file path on the regrade command. 798801 - If stet runs repair-ai-coverage --cr-only still leaves code review 799802 unresolved, use the summary's stable unresolved .reason and optional 800803 category / detail fields to separate model output shape , … 1+ package commandlist 2+ 3+ import 4+ "path" 5+ "regexp" 6+ "strings" 7+ 8+ "stet/internal/shellcmd" 9+ 10+ 11+ var assignment = regexp.MustCompile ^ A-Za-z A-Za-z0-9 = 12+ var duration = regexp.MustCompile ^ 0-9 + ?:\. 0-9 + ? smhd ?$ 13+ 14+ // TestInvocation identifies exactly one known test command. The original text 15+ // is retained, including literal environment assignments and a relative cwd. 16+ // Opaque scripts and shell evaluation are deliberately not interpreted. 17+ func TestInvocation command string string, shellcmd.FieldSpan, bool { 18+ command = strings.TrimSpace command 19+ if command == "" || strings.ContainsAny command, "\n\r $" { 20+ return "", nil, false 21+ } 22+ spans, ok := shellcmd.FieldSpans command 23+ if ok || len spans == 0 { 24+ return "", nil, false 25+ } 26+ i := 0 27+ if spans 0 .Value == "cd" { 28+ if len spans < 4 || spans 2 .Value = "&&" || relativeDirectory spans 1 .Value { 29+ return "", nil, false 30+ } 31+ if unsafeShellControl command :spans 2 .Start { 32+ return "", nil, false 33+ } 34+ i = 3 35+ } 36+ if i = len spans || unsafeShellControl command spans i .Start: { 37+ return "", nil, false 38+ } 39+ // The sole supported control operator belongs to the proven cd prefix. 40+ for , span := range spans i: { 41+ if span.Value == "&&" || span.Value == "&" { 42+ return "", nil, false 43+ } 44+ } 45+ for i < len spans && assignment.MatchString spans i .Value { 46+ if assignment.MatchString command spans i .Start:spans i .End { 47+ return "", nil, false 48+ } 49+ i++ 50+ } 51+ if i < len spans && spans i .Value == "env" { 52+ i++ 53+ for i < len spans && assignment.MatchString spans i .Value { 54+ i++ 55+ } 56+ } 57+ if i < len spans && spans i .Value == "timeout" { 58+ i++ 59+ if i = len spans || duration.MatchString spans i .Value { 60+ return "", nil, false 61+ } 62+ i++ 63+ } 64+ if i = len spans { 65+ return "", nil, false 66+ } 67+ fields := spans i: 68+ value := func n int string { 69+ if n = len fields { 70+ return "" 71+ } 72+ return fields n .Value 73+ } 74+ switch value 0 { 75+ case "go", "cargo": 76+ ok = value 1 == "test" 77+ case "sh", "bash": 78+ ok = len fields == 2 && strings.HasSuffix value 1 , ".sh" && relativeDirectory value 1 79+ case "pytest", "pytest-3", "vitest", "jest", "phpunit": 80+ ok = true 81+ case "python", "python3": 82+ ok = value 1 == "-m" && value 2 == "pytest" 83+ case "npm", "pnpm", "yarn": 84+ ok = value 1 == "test" || value 1 == "run" && value 2 == "test" 85+ case "npx": 86+ ok = value 1 == "vitest" || value 1 == "jest" 87+ default: 88+ ok = false 89+ } 90+ if ok { 91+ return "", nil, false 92+ } 93+ return command, fields, true 94+ } 95+ 96+ func relativeDirectory directory string bool { 97+ if directory == "" || strings.HasPrefix directory, "-" || strings.Contains directory, "\\" || path.IsAbs directory { 98+ return false 99+ } 100+ clean := path.Clean directory 101+ return clean = ".." && strings.HasPrefix clean, "../" 102+ } 103+ 104+ func unsafeShellControl command string bool { 105+ var quote rune 106+ escaped := false 107+ for , r := range command { 108+ if escaped { 109+ escaped = false 110+ continue 111+ } 112+ if quote == '\'' { 113+ if r == '\'' { 114+ quote = 0 115+ } 116+ continue 117+ } 118+ if r == '\\' { 119+ escaped = true 120+ continue 121+ } 122+ if quote == '"' { 123+ if r == '"' { 124+ quote = 0 125+ } 126+ continue 127+ } 128+ if r == '\'' || r == '"' { 129+ quote = r 130+ continue 131+ } 132+ if strings.ContainsRune ";|&< ", r { 133+ return true 134+ } 135+ } 136+ return false 137+ } … 1+ package commandlist 2+ 3+ import "testing" 4+ 5+ func TestTestInvocationPreservesProvenExecutionContext t testing.T { 6+ for , command := range string{ 7+ "go test ./pkg -run 'TestOne|TestTwo' -count=1", 8+ "cd packages/api && FOO='a b' env BAR=2 timeout 30s python3 -m pytest tests/test api.py -q", 9+ "npm run test -- --runInBand", "env CI=1 cargo test --locked", "pytest 'tests/test space.py'", 10+ } { 11+ got, fields, ok := TestInvocation command 12+ if ok || got = command || len fields == 0 { 13+ t.Errorf "TestInvocation %q = %q, %v", command, got, ok 14+ } 15+ } 16+ } 17+ 18+ func TestTestInvocationRejectsAmbiguousAndUnsafeWrappers t testing.T { 19+ for , command := range string{ 20+ "echo go test ./pkg", "sh -c 'go test ./pkg'", "go test ./pkg && pytest tests", 21+ "go test ./pkg; true", "go test ./pkg || true", "cd ../outside && go test ./pkg", 22+ "env -i go test ./pkg", "timeout --signal=KILL 30 go test ./pkg", "FOO=$ touch sentinel go test ./pkg", 23+ "go test ./pkg result", "cd pkg&&go test ./pkg", "go test ./pkg &", "make test", "\"CI=1\" go test ./pkg", "F\"OO\"=x go test ./pkg", 24+ "cd pkg;touch && go test ./pkg", "echo tests/agent.sh", "cat tests/agent.sh", "rm tests/agent.sh", "sh -c tests/agent.sh", 25+ } { 26+ if , , ok := TestInvocation command ; ok { 27+ t.Errorf "accepted %q", command 28+ } 29+ } 30+ } … 1+ package h2h 2+ 3+ import 4+ "fmt" 5+ "path/filepath" 6+ "strings" 7+ 8+ "stet/internal/validate" 9+ 10+ 11+ // prepareCoverageRepairPrompts reconstructs only absent prompts. Stored prompts 12+ // retain their original evidence; reconstructed prompts use the sealed task 13+ // source and keep its extracted repository alive through grading. 14+ func prepareCoverageRepairPrompts cfg validate.CoverageRepairConfig, manifest Manifest, artifacts TaskArtifacts func , validate.CoverageRepairIssue { 15+ missingEq := cfg.RepairEquivalence && strings.TrimSpace cfg.Existing.Equivalence.Prompt == "" 16+ review := cfg.Existing.CodeReview 17+ if strings.TrimSpace review.Prompt == "" && cfg.Existing.Review = nil { 18+ review = cfg.Existing.Review 19+ } 20+ missingReview := cfg.RepairCodeReview && strings.TrimSpace review.Prompt == "" 21+ if missingEq && missingReview { 22+ return nil, nil 23+ } 24+ var issues validate.CoverageRepairIssue 25+ fail := func dimension string, err error { 26+ detail := "prompt reconstruction failed: " + err.Error 27+ if len detail 240 { 28+ detail = detail :240 29+ } 30+ issues = append issues, validate.CoverageRepairIssue{Dimension: dimension, Reason: "missing prompt", Detail: detail, Cause: err} 31+ if dimension == TaskGraderEquivalence { 32+ cfg.RepairEquivalence = false 33+ } else { 34+ cfg.RepairCodeReview = false 35+ } 36+ } 37+ mode := cfg.Existing.Equivalence.PromptMode 38+ if missingEq && mode = validate.EquivalencePromptModeStandard && mode = validate.EquivalencePromptModeGoldBlind { 39+ fail TaskGraderEquivalence, fmt.Errorf "missing or unsupported equivalence prompt mode %q", mode 40+ missingEq = false 41+ } 42+ if missingEq && missingReview { 43+ return nil, issues 44+ } 45+ truncation := cfg.Existing.PromptTruncation 46+ if truncation.AgentPatch.Truncated || truncation.GoldPatch.Truncated || truncation.RepoTree.Truncated || truncation.RepoExcerpts.Truncated || cfg.Existing.Equivalence.Truncated || review.Truncated { 47+ err := fmt.Errorf "original prompt was truncated; original context budgets are unavailable" 48+ if missingEq { 49+ fail TaskGraderEquivalence, err 50+ } 51+ if missingReview { 52+ fail TaskGraderCodeReview, err 53+ } 54+ return nil, issues 55+ } 56+ dataset := manifest.Dataset 57+ goldPath := strings.TrimSpace cfg.Existing.ArtifactsManifest.GoldPatch.Path 58+ if goldPath = "" { 59+ taskDir := filepath.Dir goldPath 60+ if filepath.Base taskDir = artifacts.TaskID { 61+ err := fmt.Errorf "gold patch directory does not match task %q", artifacts.TaskID 62+ if missingEq { 63+ fail TaskGraderEquivalence, err 64+ } 65+ if missingReview { 66+ fail TaskGraderCodeReview, err 67+ } 68+ return nil, issues 69+ } 70+ dataset = filepath.Dir taskDir 71+ } 72+ if err := verifyCoverageRepairTaskAuthority manifest, artifacts, dataset ; err = nil { 73+ if missingEq { 74+ fail TaskGraderEquivalence, err 75+ } 76+ if missingReview { 77+ fail TaskGraderCodeReview, err 78+ } 79+ return nil, issues 80+ } 81+ resolved, , , err := resolveTaskArtifactsFromResults artifacts 82+ if err == nil { 83+ artifacts = resolved 84+ } 85+ patchPath := strings.TrimSpace cfg.Existing.ArtifactsManifest.AgentPatch.Path 86+ if patchPath == "" { 87+ patchPath = artifacts.ResolveAgentPatchPath 88+ } 89+ promptCtx, cleanup, err := validate.BuildOfflinePromptContext validate.OfflinePromptInputs{ 90+ DatasetPath: dataset, TaskID: artifacts.TaskID, AgentPatchPath: patchPath, 91+ TBResultsRaw: validate.ReadTBResultsRaw artifacts.ResultsPath , artifacts.TaskID , RequireRepoContext: true, 92+ EffectiveTestCommands: append string{}, cfg.Existing.Tests.Commands... , 93+ ExpectedArtifacts: &cfg.Existing.ArtifactsManifest, 94+ RepoContextOptions: validate.RepoContextOptions{ExcludePrefixes: cfg.Existing.ContextExcludePaths}, 95+ } 96+ if err = nil { 97+ if missingEq { 98+ fail TaskGraderEquivalence, err 99+ } 100+ if missingReview { 101+ fail TaskGraderCodeReview, err 102+ } 103+ return nil, issues 104+ } 105+ if strings.TrimSpace promptCtx.AgentPatch == "" || strings.TrimSpace promptCtx.TaskYAML == "" { 106+ err := fmt.Errorf "task YAML and agent patch are required for prompt reconstruction" 107+ if missingEq { 108+ fail TaskGraderEquivalence, err 109+ } 110+ if missingReview { 111+ fail TaskGraderCodeReview, err 112+ } 113+ return cleanup, issues 114+ } 115+ client := cfg.AI 116+ if shell, ok := client. validate.ShellAIClient ; ok { 117+ shell.Dir = promptCtx.RepoPath 118+ client = shell 119+ } 120+ if missingEq { 121+ prompt, err := validate.RenderEquivalencePromptForMode promptCtx, mode 122+ if err = nil { 123+ fail TaskGraderEquivalence, err 124+ } else { 125+ cfg.EquivalencePrompt, cfg.EquivalencePromptMode = prompt, mode 126+ cfg.EquivalencePromptContext, cfg.EquivalenceAI = &promptCtx, client 127+ } 128+ } 129+ if missingReview { 130+ prompt, err := validate.RenderReviewPrompt promptCtx 131+ if err = nil { 132+ fail TaskGraderCodeReview, err 133+ } else { 134+ cfg.ReviewPrompt, cfg.ReviewPromptContext, cfg.ReviewAI = prompt, &promptCtx, client 135+ } 136+ } 137+ return cleanup, issues 138+ } 139+ 140+ func verifyCoverageRepairTaskAuthority manifest Manifest, artifacts TaskArtifacts, dataset string error { 141+ if manifest.TaskSelection == nil || manifest.TaskSelection.PromptProjectionReceipt == "" { 142+ return fmt.Errorf "missing original task prompt projection authority" 143+ } 144+ receipt, err := LoadTaskPromptProjectionReceipt artifacts.OutputRoot 145+ if err = nil { 146+ return err 147+ } 148+ if err := validateTaskSelectionProjection manifest.TaskSelection, receipt ; err = nil { 149+ return err 150+ } 151+ digest, err := digestTaskContentTree filepath.Join dataset, artifacts.TaskID 152+ if err = nil { 153+ return err 154+ } 155+ for , row := range receipt.Tasks { 156+ if row.TaskID == artifacts.TaskID { 157+ if row.TaskContentSHA256 = digest { 158+ return fmt.Errorf "original task content provenance mismatch for %q", artifacts.TaskID 159+ } 160+ return nil 161+ } 162+ } 163+ return fmt.Errorf "missing original task content authority for %q", artifacts.TaskID 164+ } … 1+ package h2h 2+ 3+ import 4+ "bytes" 5+ "context" 6+ "fmt" 7+ "os" 8+ "path/filepath" 9+ "strings" 10+ "testing" 11+ "time" 12+ 13+ "stet/internal/validate" 14+ 15+ 16+ func TestPrepareCoverageRepairPromptsPreservesStoredPrompts t testing.T { 17+ cfg := validate.CoverageRepairConfig{ 18+ Existing: validate.ValidationResult{ 19+ Equivalence: validate.EquivalenceResult{Prompt: "stored equivalence"}, 20+ Review: &validate.ReviewResult{Prompt: "legacy stored review"}, 21+ }, RepairEquivalence: true, RepairCodeReview: true, 22+ } 23+ cleanup, issues := prepareCoverageRepairPrompts &cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"} 24+ if cleanup = nil || len issues = 0 || cfg.EquivalencePromptContext = nil || cfg.ReviewPromptContext = nil { 25+ t.Fatalf "stored prompts should require no reconstruction: %+v", cfg 26+ } 27+ } 28+ 29+ func TestPrepareCoverageRepairPromptsRejectsUnknownModeWithoutBlockingStoredReview t testing.T { 30+ cfg := validate.CoverageRepairConfig{ 31+ Existing: validate.ValidationResult{CodeReview: validate.ReviewResult{Prompt: "stored review"}}, 32+ RepairEquivalence: true, RepairCodeReview: true, 33+ } 34+ , issues := prepareCoverageRepairPrompts &cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"} 35+ if len issues = 1 || issues 0 .Dimension = TaskGraderEquivalence || cfg.RepairEquivalence || cfg.RepairCodeReview { 36+ t.Fatalf "unexpected dimension selection: %+v, issues %+v", cfg, issues 37+ } 38+ } 39+ 40+ func TestPrepareCoverageRepairPromptsRejectsUnknownOriginalTruncationBudget t testing.T { 41+ cfg := validate.CoverageRepairConfig{RepairCodeReview: true} 42+ cfg.Existing.PromptTruncation.AgentPatch.Truncated = true 43+ , issues := prepareCoverageRepairPrompts &cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"} 44+ if len issues = 1 || cfg.RepairCodeReview || strings.Contains issues 0 .Detail, "original context budgets" { 45+ t.Fatalf "unknown original truncation must prevent reconstruction: %+v", issues 46+ } 47+ } 48+ 49+ func TestPrepareCoverageRepairPromptsReconstructsPersistedModeAndVerifierContext t testing.T { 50+ t.Parallel 51+ root := t.TempDir 52+ taskDir := filepath.Join root, "dataset", "task-a" 53+ mustWriteFile t, filepath.Join taskDir, "task.yaml" , "id: task-a\ninstruction: fix greeting\n" 54+ mustWriteFile t, filepath.Join taskDir, "gold.patch" , "gold reference secret\n" 55+ mustWriteFile t, filepath.Join taskDir, "tests", "test outputs.py" , "COMMANDS = \"dataset test\" \n" 56+ patch := filepath.Join root, "agent.patch" 57+ mustWriteFile t, patch, "diff --git a/hello.txt b/hello.txt\n+hello\n" 58+ writeAnswerSimilarityTar t, filepath.Join taskDir, "repo.tar.gz" , map string string{"hello.txt": "old\n"}, nil 59+ existing := validate.ValidationResult{ 60+ Equivalence: validate.EquivalenceResult{PromptMode: validate.EquivalencePromptModeGoldBlind}, 61+ Tests: validate.TestResult{Commands: string{"effective verifier"}}, 62+ } 63+ inputs, err := validate.ReadTaskInputs filepath.Dir taskDir , "task-a", patch 64+ if err = nil { 65+ t.Fatal err 66+ } 67+ existing.ArtifactsManifest = validate.BuildArtifactManifest inputs 68+ manifest := sealCoverageRepairTestTask t, root, filepath.Dir taskDir 69+ cfg := validate.CoverageRepairConfig{Existing: existing, AI: validate.ShellAIClient{}, RepairEquivalence: true, RepairCodeReview: true} 70+ cleanup, issues := prepareCoverageRepairPrompts &cfg, manifest, NewTaskArtifacts root, VariantSpec{RunID: "run-a", ModelKey: "model-a"}, "task-a", "" 71+ if cleanup = nil { 72+ defer cleanup 73+ } 74+ if len issues = 0 { 75+ t.Fatalf "reconstruction failed: %+v", issues 76+ } 77+ if cfg.EquivalencePromptMode = validate.EquivalencePromptModeGoldBlind || strings.Contains cfg.EquivalencePrompt, "gold reference secret" { 78+ t.Fatalf "gold-blind provenance was lost: %s", cfg.EquivalencePrompt 79+ } 80+ if strings.Contains cfg.ReviewPrompt, "effective verifier" || strings.Contains cfg.ReviewPrompt, "dataset test" { 81+ t.Fatalf "effective verifier context was lost: %s", cfg.ReviewPrompt 82+ } 83+ for , client := range validate.AIClient{cfg.EquivalenceAI, cfg.ReviewAI} { 84+ shell := client. validate.ShellAIClient 85+ if shell.Dir == "" || shell.Dir = cfg.ReviewPromptContext.RepoPath { 86+ t.Fatalf "missing grading repository: %+v", shell 87+ } 88+ if , err := os.Stat filepath.Join shell.Dir, "hello.txt" ; err = nil { 89+ t.Fatal err 90+ } 91+ } 92+ } 93+ 94+ func TestPrepareCoverageRepairPromptsRejectsUnavailableRepository t testing.T { 95+ root := t.TempDir 96+ taskDir := filepath.Join root, "dataset", "task-a" 97+ mustWriteFile t, filepath.Join taskDir, "task.yaml" , "id: task-a\n" 98+ mustWriteFile t, filepath.Join taskDir, "gold.patch" , "gold\n" 99+ mustWriteFile t, filepath.Join taskDir, "repo.tar.gz" , "invalid gzip" 100+ patch := filepath.Join root, "agent.patch" 101+ mustWriteFile t, patch, "agent\n" 102+ manifest := sealCoverageRepairTestTask t, root, filepath.Dir taskDir 103+ cfg := validate.CoverageRepairConfig{RepairCodeReview: true} 104+ inputs, err := validate.ReadTaskInputs filepath.Dir taskDir , "task-a", patch 105+ if err = nil { 106+ t.Fatal err 107+ } 108+ cfg.Existing.ArtifactsManifest = validate.BuildArtifactManifest inputs 109+ cleanup, issues := prepareCoverageRepairPrompts &cfg, manifest, NewTaskArtifacts root, VariantSpec{RunID: "run-a"}, "task-a", "" 110+ if cleanup = nil { 111+ defer cleanup 112+ } 113+ if len issues = 1 || cfg.RepairCodeReview || cfg.ReviewPrompt = "" || strings.Contains issues 0 .Detail, "required repo context" { 114+ t.Fatalf "repository failure should prevent reconstructed grade: %+v, issues %+v", cfg, issues 115+ } 116+ } 117+ 118+ func sealCoverageRepairTestTask t testing.T, root, dataset string Manifest { 119+ t.Helper 120+ selection := &TaskSelectionRecord{RealizedTaskIDs: string{"task-a"}} 121+ receipt, err := BuildTaskPromptProjectionReceipt dataset, root, selection, nil, "dataset", map string string{"task-a": "fix greeting"}, time.Time{} 122+ if err = nil { 123+ t.Fatal err 124+ } 125+ if , err := WriteTaskPromptProjectionReceipt root, receipt ; err = nil { 126+ t.Fatal err 127+ } 128+ return &Manifest{Dataset: dataset, TaskSelection: ApplyTaskPromptProjectionReceipt selection, receipt } 129+ } 130+ 131+ func TestPrepareCoverageRepairPromptsRejectsMutatedTaskYAML t testing.T { 132+ root := t.TempDir 133+ dataset := filepath.Join root, "dataset" 134+ mustWriteFile t, filepath.Join dataset, "task-a", "task.yaml" , "id: task-a\ninstruction: original\n" 135+ manifest := sealCoverageRepairTestTask t, root, dataset 136+ mustWriteFile t, filepath.Join dataset, "task-a", "task.yaml" , "id: task-a\ninstruction: mutated\n" 137+ cfg := validate.CoverageRepairConfig{RepairCodeReview: true} 138+ , issues := prepareCoverageRepairPrompts &cfg, manifest, NewTaskArtifacts root, VariantSpec{RunID: "run-a"}, "task-a", "" 139+ if len issues = 1 || cfg.RepairCodeReview || strings.Contains issues 0 .Detail, "task content provenance mismatch" { 140+ t.Fatalf "changed task must not reconstruct: %+v", issues 141+ } 142+ } 143+ 144+ func TestRunRepairAICoverageReconstructsBoundSourceAtSubprocessBoundary t testing.T { 145+ for , mutate := range bool{false, true} { 146+ t.Run fmt.Sprintf "mutate %t", mutate , func t testing.T { 147+ root := t.TempDir 148+ dataset := filepath.Join root, "dataset" 149+ taskDir := filepath.Join dataset, "task-a" 150+ mustWriteFile t, filepath.Join taskDir, "task.yaml" , "id: task-a\ninstruction: fix greeting\n" 151+ mustWriteFile t, filepath.Join taskDir, "gold.patch" , "gold reference secret\n" 152+ writeAnswerSimilarityTar t, filepath.Join taskDir, "repo.tar.gz" , map string string{"hello.txt": "old\n"}, nil 153+ patch := filepath.Join root, "agent.patch" 154+ mustWriteFile t, patch, "diff --git a/hello.txt b/hello.txt\n--- a/hello.txt\n+++ b/hello.txt\n@@ -1 +1 @@\n-old\n+hello\n" 155+ manifest := sealCoverageRepairTestTask t, root, dataset 156+ manifest.Version, manifest.OutputRoot = ManifestVersion, root 157+ manifest.Models = string{"model/a"} 158+ manifest.ModelKeys, manifest.RunIDs = map string string{"model/a": "model-a"}, map string string{"model/a": "run-a"} 159+ if err := SaveManifest filepath.Join root, "manifest.json" , manifest ; err = nil { 160+ t.Fatal err 161+ } 162+ inputs, err := validate.ReadTaskInputs dataset, "task-a", patch 163+ if err = nil { 164+ t.Fatal err 165+ } 166+ original := validate.ValidationResult{ 167+ TaskID: "task-a", PrimarySignal: validate.PrimarySignalTests, Status: validate.StatusPass, MatrixStatus: validate.MatrixPass, 168+ Guardrails: validate.GuardrailResult{Status: validate.StatusPass}, 169+ Tests: validate.TestResult{Outcome: validate.TestOutcomePass, Strength: validate.TestStrengthStrong, Commands: string{"effective verifier"}}, 170+ Equivalence: validate.EquivalenceResult{Status: validate.AIStatusUnavailable, Outcome: validate.EquivalenceUnknown, PromptMode: validate.EquivalencePromptModeGoldBlind}, 171+ CodeReview: validate.ReviewResult{Status: validate.AIStatusUnavailable, Signal: validate.ReviewSignalUnsure}, 172+ ArtifactsManifest: validate.BuildArtifactManifest inputs , 173+ } 174+ validationRoot := filepath.Join root, "validation", "model-a" 175+ mustWriteRepairValidation t, validationRoot, "task-a", original 176+ if mutate { 177+ mustWriteFile t, patch, "mutated agent evidence\n" 178+ } 179+ log := filepath.Join root, "calls" 180+ script := filepath.Join root, "judge.py" 181+ mustWriteFile t, script, import json, pathlib, sys 182+ prompt = sys.stdin.read 183+ assert pathlib.Path 'hello.txt' .read text == 'old\n' 184+ with open sys.argv 1 , 'a' as log: log.write 'call\n' 185+ if 'You are reviewing' in prompt: 186+ assert 'effective verifier' in prompt 187+ scores = {name: {'score 0 4': 4, 'confidence 0 1': .95, 'rationale': 'ok', 'citations': {'path':'hello.txt','side':'new','start line':1,'end line':1,'anchor kind':'diff','head sha':'abc1234'} } for name in 'correctness','introduced bug risk','edge case handling','maintainability idioms' } 188+ print json.dumps {'schema version':'stet.review.v1','dimension scores':scores,'findings': ,'overall':{'confidence overall':.95,'summary':'repair passed'}} 189+ else: 190+ assert 'gold reference secret' not in prompt 191+ print 'obligations:\n- fix greeting = MET\n\nequivalent: YES\nconfidence: 0.95\nissue type: stylistic\nnotes: repaired' 192+ 193+ var stdout, stderr bytes.Buffer 194+ err = RunRepairAICoverage context.Background , string{"--out", root, "--model-key", "model-a", "--ai-cmd", "python3 " + shellQuote script + " " + shellQuote log , "--ai-model-id", "judge-a"}, &stdout, &stderr 195+ if mutate { 196+ if err == nil { 197+ t.Fatal "mutated artifact repair must fail" 198+ } 199+ if , err := os.Stat log ; os.IsNotExist err { 200+ t.Fatalf "mutated source invoked grader: %v", err 201+ } 202+ if strings.Contains stdout.String , "agent patch provenance mismatch" { 203+ t.Fatalf "missing mutation reason: %s", stdout.String 204+ } 205+ return 206+ } 207+ if err = nil { 208+ t.Fatalf "repair failed: %v\nstdout %s\nstderr %s", err, stdout.String , stderr.String 209+ } 210+ updated := mustReadRepairValidation t, filepath.Join validationRoot, "task-a", "validation.json" 211+ if updated.Equivalence.PromptMode = validate.EquivalencePromptModeGoldBlind || updated.Equivalence.Status = validate.AIStatusUsed || updated.CodeReview.Status = validate.AIStatusUsed || updated.CodeReview.AttemptCount = 1 { 212+ t.Fatalf "repair lost mode or attempt metadata: %+v", updated 213+ } 214+ if updated.ArtifactsManifest.AgentPatch.SHA256 = original.ArtifactsManifest.AgentPatch.SHA256 { 215+ t.Fatal "repair changed original source authority" 216+ } 217+ calls, err := os.ReadFile log 218+ if err = nil || strings.Count string calls , "call" = 2 { 219+ t.Fatalf "unexpected calls: %s, %v", calls, err 220+ } 221+ } 222+ } 223+ } … 1+ package validate 2+ 3+ import 4+ "context" 5+ "errors" 6+ "net" 7+ "strings" 8+ "time" 9+ 10+ "stet/internal/runtimefailure" 11+ 12+ 13+ // gradeAttemptClient retains every dispatch, including runtime and parse retries. 14+ // One instance belongs to one measurement dimension and must not be shared. 15+ type gradeAttemptClient struct { 16+ ai AIClient 17+ attempts int 18+ reasons string 19+ previousSucceeded bool 20+ } 21+ 22+ func newGradeAttemptClient ai AIClient gradeAttemptClient { 23+ if ai == nil { 24+ ai = NoopAIClient{} 25+ } 26+ return &gradeAttemptClient{ai: ai} 27+ } 28+ 29+ func a gradeAttemptClient Run ctx context.Context, prompt string string, error { 30+ if a.previousSucceeded { 31+ a.reasons = append a.reasons, "model output shape" 32+ } 33+ a.previousSucceeded = false 34+ for attempt := 0; ; attempt++ { 35+ if err := ctx.Err ; err = nil { 36+ return "", err 37+ } 38+ a.attempts++ 39+ output, err := a.ai.Run ctx, prompt 40+ if err == nil { 41+ a.previousSucceeded = true 42+ return output, nil 43+ } 44+ reason := recoverableGradeFailure err 45+ if reason == "" || attempt = len coverageRepairRetryBackoffs { 46+ return "", err 47+ } 48+ select { 49+ case <-ctx.Done : 50+ return "", ctx.Err 51+ case <-time.After coverageRepairRetryBackoffs attempt : 52+ } 53+ a.reasons = append a.re