# I ran Sol 6 vs. 6.1 on tasks from my own repo. It's cheaper and at least as good > Source: > Published: 2026-10-12 16:29:52+00:00 …`stet runs repair-ai-coverage --out /path/to/h2h-output --model-key model-one --c` 292292 293293``Use `--parse-retries` when saved `code_review` prompts exist but evaluator`` 294294`responses failed JSON/schema parsing. The command rewrites validation artifacts` 295−`only after a successful repair and regenerates summaries/task details.` 295+`only after a successful repair and regenerates summaries/task details. Stored` 296+`prompts retain their measurement mode and truncation. Missing-prompt repair` 297+`requires the original task projection, matching artifact digests, persisted test` 298+`commands, and complete repository context; unavailable provenance stays` 299+`unresolved. Attempt counts and retry reasons distinguish recovery from an` 300+`initial successful grade; assertions and unrecoverable runtime errors are not` 301+`retried.` 296302 297303`Regrade additive custom or bundled graders from canonical task details:` 298304 …``const coverageValidationGolden = `{`` 843843 `},` 844844 `"equiv_rescue_policy": "on",` 845845 `"equivalence": {` 846+ `"attempt_count": 1,` 846847 `"confidence": 0.95,` 847848 `"evaluator_model": "judge-a",` 848849 `"issue_type": "stylistic",` …`func (p *repairAICoverageProducer) Produce(mctx *MutationContext) (*ProduceResul` 274274 `var mu sync.Mutex` 275275 `runWithConcurrency(jobs, min(len(jobs), coverageRepairWorkers), func(job repairJob) {` 276276 `reporter.recordTaskStart(variant, job.TaskID, job.CurrentWork, time.Now().UTC())` 277− `result, repairErr := validate.RepairCoverage(ctx, validate.CoverageRepairConfig{` 277+ `repairCfg := validate.CoverageRepairConfig{` 278278 `Existing: job.Validation,` 279279 `AI: ai,` 280280 `AIModelID: resolvedAIModelID,` 281281 `RepairEquivalence: repairEquivalence && needsRepairCoverage(job.Validation, TaskGraderEquivalence, resolvedAIModelID),` 282282 `RepairCodeReview: repairCodeReview && needsRepairCoverage(job.Validation, TaskGraderCodeReview, resolvedAIModelID),` 283283 `ReviewParseRetries: cfg.ParseRetries,` 284− `})` 284+ `}` 285+ `cleanup, reconstructionIssues := prepareCoverageRepairPrompts(&repairCfg, manifest, NewTaskArtifacts(outputRoot, variant, job.TaskID, ""))` 286+ `if cleanup != nil {` 287+ `defer cleanup()` 288+ `}` 289+ `result := validate.CoverageRepairResult{Validation: job.Validation}` 290+ `var repairErr error` 291+ `if repairCfg.RepairEquivalence || repairCfg.RepairCodeReview {` 292+ `result, repairErr = validate.RepairCoverage(ctx, repairCfg)` 293+ `}` 294+ `result.Unresolved = append(result.Unresolved, reconstructionIssues...)` 285295 `if repairErr != nil {` 286296 `reporter.recordTaskEnd(variant, job.TaskID, job.CurrentWork, "failed", time.Now().UTC())` 287297 `mu.Lock()` …`func RunAdaptedReferenceTest(ctx context.Context, ai AIClient, promptCtx PromptC` 292292 `composedDiff, composeErr := composeAdaptedReferenceTestPatch(runCtx, inputs, referenceTestPatch, canonicalDiff)` 293293 `if composeErr != nil {` 294294 `result.Reason = "patch_compose_failed"` 295− `result.Summary = fmt.Sprintf("compose canonical reference test patch and adapted_diff: %v", composeErr)` 295+ `result.Summary = fmt.Sprintf("adapted_diff could not be composed with the canonical reference test patch: %v", composeErr)` 296296 `return result` 297297 `}` 298298 `composedPayload := payload` …`import (` 99 `"strings"` 1010 `"time"` 1111 12+ `"stet/internal/commandlist"` 1213 `"stet/internal/filecopy"` 1314 `"stet/internal/gitops"` 1415 `"stet/internal/harness"` …`func legacyCommandsThatNameAgentTests(commands, agentFiles []string) []string {` 192193 `files := normalizeAgentTestFiles(agentFiles)` 193194 `var out []string` 194195 `for _, command := range normalizeCommands(commands) {` 195− `if commandHasPotentialNonPathTestSelection(command) || receiptBackedCommandHasShellControl(command) {` 196+ `if _, _, safe := commandlist.TestInvocation(command); !safe || commandHasPotentialNonPathTestSelection(command) || receiptBackedCommandHasShellControl(command) {` 196197 `continue` 197198 `}` 198199 `for _, file := range files {` …`import (` 66 `"fmt"` 77 `"os"` 88 `"path/filepath"` 9+ `"strconv"` 910 `"strings"` 1011 12+ `"stet/internal/commandlist"` 1113 `"stet/internal/shellcmd"` 1214`)` 1315 …`func replaceExactFinalTestPath(command, sourcePath, replacement string) (string,` 198200`}` 199201 200202`func certifiedGoReplayCommand(command, sourcePath string) (string, bool) {` 201− `if !strings.HasSuffix(strings.ToLower(sourcePath), ".go") || !goCommandTargetsAgentTestFiles(command, []string{sourcePath}) {` 203+ `_, fields, safe := commandlist.TestInvocation(command)` 204+ `if !safe || len(fields) < 3 || fields[0].Value != "go" || fields[1].Value != "test" || !strings.HasSuffix(strings.ToLower(sourcePath), ".go") {` 202205 `return "", false` 203206 `}` 204− `replay, changed := removeGoRunSelector(command)` 205− `if !changed || hasRemainingGoTestSelector(replay) {` 207+ `// Require one explicit package. A recursive selector or additional package` 208+ `// can credit tests outside the certified source package.` 209+ `sourceDirectory := filepath.Dir(sourcePath)` 210+ `allFields, _ := shellcmd.FieldSpans(command)` 211+ `// Go also consumes selectors from GOFLAGS. Recorded overrides are opaque` 212+ `// to this bounded rewrite, so they cannot certify an unfiltered replay.` 213+ `for _, field := range allFields {` 214+ `if field.Start >= fields[0].Start {` 215+ `break` 216+ `}` 217+ `if strings.HasPrefix(field.Value, "GOFLAGS=") {` 218+ `return "", false` 219+ `}` 220+ `}` 221+ `if len(allFields) >= 3 && allFields[0].Value == "cd" {` 222+ `relative, err := filepath.Rel(allFields[1].Value, sourceDirectory)` 223+ `if err != nil || relative == ".." || strings.HasPrefix(relative, "../") {` 224+ `return "", false` 225+ `}` 226+ `sourceDirectory = relative` 227+ `}` 228+ `packagePath := "./" + filepath.ToSlash(sourceDirectory)` 229+ `if sourceDirectory == "." {` 230+ `packagePath = "."` 231+ `}` 232+ `if strings.TrimSuffix(fields[2].Value, "/") != packagePath {` 233+ `return "", false` 234+ `}` 235+ `start, end := -1, -1` 236+ `for i := 3; i < len(fields); i++ {` 237+ `value := fields[i].Value` 238+ `if value == "-run" || strings.HasPrefix(value, "-run=") {` 239+ `if start >= 0 {` 240+ `return "", false` 241+ `}` 242+ `start, end = fields[i].Start, fields[i].End` 243+ `if value == "-run" {` 244+ `if i+1 >= len(fields) || strings.HasPrefix(fields[i+1].Value, "-") {` 245+ `return "", false` 246+ `}` 247+ `i++` 248+ `end = fields[i].End` 249+ `}` 250+ `continue` 251+ `}` 252+ `// Only flags with self-contained values may remain. Positional words,` 253+ `// passthrough args, and further selectors cannot be proven safe.` 254+ `name := strings.SplitN(value, "=", 2)[0]` 255+ `if name == "-count" {` 256+ `countText, hasValue := strings.CutPrefix(value, "-count=")` 257+ `count, err := strconv.Atoi(countText)` 258+ `if !hasValue || err != nil || count <= 0 {` 259+ `return "", false` 260+ `}` 261+ `}` 262+ `switch name {` 263+ `case "-v", "-race", "-short", "-count", "-timeout", "-parallel", "-shuffle", "-tags", "-mod":` 264+ `if !strings.Contains(value, "=") && name != "-v" && name != "-race" && name != "-short" {` 265+ `return "", false` 266+ `}` 267+ `default:` 268+ `return "", false` 269+ `}` 270+ `}` 271+ `if start < 0 {` 206272 `return "", false` 207273 `}` 208− `return replay, true` 274+ `return strings.TrimSpace(strings.TrimRight(command[:start], " \t") + command[end:]), true` 209275`}` 210276 211277`func normalizeReplayRelativePath(path string) string {` …`func TestValidateAgentTestReplayPlanRejectsUnprovenCommand(t *testing.T) {` 6161 `t.Fatalf("validation error = %v", err)` 6262 `}` 6363`}` 64+ 65+`func TestCertifiedGoReplayPreservesOnlyProvenInvocation(t *testing.T) {` 66+ `command := "cd module && env MODE='test value' timeout 30s go test ./pkg -run 'TestHidden|TestOther' -count=1"` 67+ `got, ok := certifiedGoReplayCommand(command, "module/pkg/hidden_test.go")` 68+ `if !ok || got != "cd module && env MODE='test value' timeout 30s go test ./pkg -count=1" {` 69+ `t.Fatalf("replay = %q, %v", got, ok)` 70+ `}` 71+ `for _, unsafe := range []string{` 72+ `"echo go test ./pkg -run TestHidden", "go test ./pkg -run TestHidden; true",` 73+ `"go test ./pkg -run TestHidden && go test ./other", "go test ./... -run TestHidden",` 74+ `"go test ./pkg ./other -run TestHidden", "go test ./pkg -run TestHidden -skip TestOther",` 75+ `"go test ./pkg -run TestHidden -args payload", "go test ./pkg -run TestHidden -run TestOther",` 76+ `"go test ./pkg -run TestHidden -count=0", "go test ./pkg -run TestHidden -count=-1",` 77+ `"go test ./pkg -run TestHidden -count=invalid", "go test ./pkg -run TestHidden -count 0",` 78+ `"go test ./pkg -run TestHidden -count -1",` 79+ `"GOFLAGS=-count=0 go test ./pkg -run TestHidden",` 80+ `"GOFLAGS='-run=Nothing' go test ./pkg -run TestHidden",` 81+ `"env GOFLAGS=-count=0 go test ./pkg -run TestHidden",` 82+ `"cd module && env GOFLAGS='-run=Nothing' go test ./pkg -run TestHidden",` 83+ `} {` 84+ `if replay, ok := certifiedGoReplayCommand(unsafe, "pkg/hidden_test.go"); ok {` 85+ `t.Errorf("accepted %q as %q", unsafe, replay)` 86+ `}` 87+ `}` 88+`}` 89+ 90+`func TestLegacyReplayRequiresAnActualTestRunner(t *testing.T) {` 91+ `commands := legacyCommandsThatNameAgentTests([]string{` 92+ `"echo tests/agent_test.sh", "cat tests/agent_test.sh", "rm tests/agent_test.sh",` 93+ `"sh -c tests/agent_test.sh", "'CI=1' sh tests/agent_test.sh", "sh tests/agent_test.sh",` 94+ `}, []string{"tests/agent_test.sh"})` 95+ `if len(commands) != 1 || commands[0] != "sh 'tests/agent_test.sh'" {` 96+ `t.Fatalf("legacy replay commands = %q", commands)` 97+ `}` 98+`}` …`func buildEquivalenceRetryPrompt(originalPrompt, rawOutput string, parseErr erro` 1818 `)` 1919`}` 2020 21−`func RunEquivalence(ctx context.Context, ai AIClient, promptCtx PromptContext) EquivalenceResult {` 21+`func RunEquivalence(ctx context.Context, ai AIClient, promptCtx PromptContext) (result EquivalenceResult) {` 22+ `mode := EquivalencePromptModeStandard` 23+ `if goldBlindEquivalenceEnabled() {` 24+ `mode = EquivalencePromptModeGoldBlind` 25+ `}` 26+ `history := newGradeAttemptClient(ai)` 27+ `ai = history` 28+ `defer func() { result.PromptMode = mode; history.annotateEquivalence(&result) }()` 29+ 2230 `if goldBlindEquivalenceEnabled() {` 2331 `// No gold reaches the model, so gold truncation cannot have mutilated` 2432 `// the graded evidence: clear it before the truncation annotation runs.` … 11`package validate` 22 33`import (` 4+ `"fmt"` 45 `"os"` 56 `"strconv"` 67 `"strings"` …`notes: ` 127128`// mode. In gold-blind mode the gold patch is dropped from the template data as` 128129`// well as from the template, so no gold content can reach the model.` 129130`func renderEquivalencePrompt(promptCtx PromptContext) (prompt string, goldBlind bool, err error) {` 130− `if !goldBlindEquivalenceEnabled() {` 131− `rendered, renderErr := renderPrompt(equivalencePromptTemplate, promptCtx.asTemplateData())` 132− `return rendered, false, renderErr` 131+ `mode := EquivalencePromptModeStandard` 132+ `if goldBlindEquivalenceEnabled() {` 133+ `mode = EquivalencePromptModeGoldBlind` 134+ `}` 135+ `rendered, err := RenderEquivalencePromptForMode(promptCtx, mode)` 136+ `return rendered, mode == EquivalencePromptModeGoldBlind, err` 137+`}` 138+ 139+`// RenderEquivalencePromptForMode uses recorded measurement authority, never the process environment.` 140+`func RenderEquivalencePromptForMode(promptCtx PromptContext, mode EquivalencePromptMode) (string, error) {` 141+ `switch mode {` 142+ `case EquivalencePromptModeStandard:` 143+ `return renderPrompt(equivalencePromptTemplate, promptCtx.asTemplateData())` 144+ `case EquivalencePromptModeGoldBlind:` 145+ `data := promptCtx.asTemplateData()` 146+ `delete(data, "GoldPatch")` 147+ `return renderPrompt(equivalenceGoldBlindPromptTemplate, data)` 148+ `default:` 149+ `return "", fmt.Errorf("unknown equivalence prompt mode %q", mode)` 133150 `}` 134− `data := promptCtx.asTemplateData()` 135− `delete(data, "GoldPatch")` 136− `rendered, renderErr := renderPrompt(equivalenceGoldBlindPromptTemplate, data)` 137− `return rendered, true, renderErr` 138151`}` …`func runPerCommandTests(` 22022202 `cmdCfg.Commands = []string{cmd}` 22032203 `goldRun, err := RunGoldBaselineTests(infraCtx, inputs, cmdCfg)` 22042204 `gr.EnvCache = goldRun.EnvCache` 2205− `gr.Result = TestCommandResult{Command: cmd}` 2205+ `gr.Result = TestCommandResult{Command: cmd, AttemptCount: 1}` 22062206 `if err != nil {` 22072207 `gr.Err = err` 2208− `if mode := inferInfraFailureModeFromError(err); mode != "" {` 2208+ `if mode := retryableVerifierFailureMode(goldRun, err); mode != "" {` 22092209 `gr.Result.GoldOutcome = TestOutcomeFail` 22102210 `gr.Result.GoldFailureMode = mode` 22112211 `gr.IsInfra = true` …`func runPerCommandTests(` 22182218 `gr.Result.GoldOutcome = TestOutcomeUnknown` 22192219 `gr.Notes = append(gr.Notes, fmt.Sprintf("gold baseline failed for %q: %v", cmd, err))` 22202220 `}` 2221− `return` 2221+ `if !isRetryableGoldInfrastructureFailure(gr.Result.GoldFailureMode) {` 2222+ `return` 2223+ `}` 22222224 `}` 22232225 `gr.EnvCache = goldRun.EnvCache` 22242226 `applyGoldRunResult(gr, goldRun)` 2225− `if gr.Result.GoldOutcome == TestOutcomeFail && isRetryableGoldInfrastructureFailure(gr.Result.GoldFailureMode) && budgetCtx.Err() == nil {` 2227+ `if isUnsetFailureMode(gr.Result.GoldFailureMode) {` 2228+ `gr.Result.GoldFailureMode = retryableVerifierFailureMode(goldRun, err)` 2229+ `}` 2230+ `if gr.Result.GoldOutcome != TestOutcomePass && isRetryableGoldInfrastructureFailure(gr.Result.GoldFailureMode) && budgetCtx.Err() == nil {` 22262231 `beginGoldRetry(&gr.Result, goldRun)` 2232+ `recordTestRetry(&gr.Result, gr.Result.GoldFailureMode)` 22272233 `retryCfg := goldRetryAttemptConfig(overrideCfg, gr.Index, 2)` 22282234 `retryCfg.Commands = []string{cmd}` 22292235 `retryRun, retryErr := RunGoldBaselineTests(infraCtx, inputs, retryCfg)` …`func runPerCommandTests(` 22442250 `gr.Result.GoldRetried = true` 22452251 `finishGoldRetry(&gr.Result, retryRun)` 22462252 `if gr.Result.GoldOutcome == TestOutcomePass {` 2253+ `gr.Result.RecoveredAfterRetry = true` 2254+ `gr.Err = nil` 2255+ `gr.IsInfra = false` 22472256 `gr.Notes = append(gr.Notes, fmt.Sprintf("gold baseline for %q recovered on retry after transient dependency fetch", cmd))` 22482257 `} else {` 22492258 `gr.Notes = append(gr.Notes, fmt.Sprintf("gold baseline retry for %q did not recover", cmd))` …`func runPerCommandTests(` 24842493 `cmdCfg := overrideCfg` 24852494 `cmdCfg.Commands = []string{entry.cmd}` 24862495 `or.Attempted = true` 2496+ `or.Result.AttemptCount++` 24872497 `overrideRun, err := RunOverrideTests(budgetCtx, inputs, cmdCfg)` 24882498 `or.EnvCache = overrideRun.EnvCache` 24892499 `if err != nil {` 2490− `if mode := inferInfraFailureModeFromError(err); mode != "" {` 2500+ `if mode := retryableVerifierFailureMode(overrideRun, err); mode != "" {` 24912501 `or.Result.AgentFailureMode = mode` 24922502 `if mode == "timeout" {` 24932503 `or.Notes = append(or.Notes, fmt.Sprintf("agent retest timeout for %q", entry.cmd))` …`func runPerCommandTests(` 24962506 `or.Notes = append(or.Notes, fmt.Sprintf("agent retest failed for %q: %v", entry.cmd, err))` 24972507 `or.Result.AgentOutcome = TestOutcomeUnknown` 24982508 `or.Err = err` 2499− `return` 2509+ `if !isRetryableGoldInfrastructureFailure(or.Result.AgentFailureMode) {` 2510+ `return` 2511+ `}` 25002512 `}` 25012513 `or.Result.AgentOutcome = testOutcomeFromTrial(overrideRun.Trial)` 25022514 `or.Result.AgentTrialName = overrideRun.Trial.TrialName` 2503− `or.Result.AgentFailureMode = overrideRun.Trial.FailureMode` 2515+ `or.Result.AgentFailureMode = retryableVerifierFailureMode(overrideRun, err)` 25042516 2505− `// Retry once if agent failed but gold passed.` 2506− `if or.Result.AgentOutcome == TestOutcomeFail && or.Result.GoldOutcome == TestOutcomePass && budgetCtx.Err() == nil {` 2517+ `// Retry only a typed transient infrastructure failure; assertions are terminal.` 2518+ `if or.Result.AgentOutcome != TestOutcomePass && isRetryableGoldInfrastructureFailure(or.Result.AgentFailureMode) && or.Result.GoldOutcome == TestOutcomePass && budgetCtx.Err() == nil {` 25072519 `retryCfg := overrideCfg` 25082520 `retryCfg.Commands = []string{entry.cmd}` 2521+ `recordTestRetry(&or.Result, or.Result.AgentFailureMode)` 2522+ `or.Result.AgentRetried = true` 25092523 `retryRun, retryErr := RunOverrideTests(budgetCtx, inputs, retryCfg)` 25102524 `or.RetryEnvCache = &retryRun.EnvCache` 2525+ `or.Result.AgentOutcome = testOutcomeFromTrial(retryRun.Trial)` 2526+ `or.Result.AgentTrialName = retryRun.Trial.TrialName` 2527+ `or.Result.AgentFailureMode = retryableVerifierFailureMode(retryRun, retryErr)` 2528+ `or.Err = retryErr` 25112529 `if retryErr == nil {` 25122530 `retryOutcome := testOutcomeFromTrial(retryRun.Trial)` 25132531 `if retryOutcome == TestOutcomePass {` …`func runPerCommandTests(` 25152533 `or.Result.AgentTrialName = retryRun.Trial.TrialName` 25162534 `or.Result.AgentFailureMode = retryRun.Trial.FailureMode` 25172535 `or.Result.AgentRetried = true` 2536+ `or.Result.RecoveredAfterRetry = true` 2537+ `or.Err = nil` 25182538 `or.Notes = append(or.Notes, fmt.Sprintf("agent retest for %q succeeded on retry", entry.cmd))` 25192539 `}` 25202540 `}` …`func runGoldBatchSlice(` 347347 `cleanups := []func(){}` 348348 `for _, idx := range indexes {` 349349 `gr := &goldResults[idx]` 350+ `priorResult := gr.Result` 350351 `priorRetried, priorEvidence := gr.Result.GoldRetried, gr.Result.GoldRetryEvidence` 351352 `gr.IsInfra = false` 352353 `gr.IsGlobalInfra = false` 353354 `gr.Err = nil` 354355 `if ctx.Err() != nil {` 355356 `gr.Result = TestCommandResult{` 356− `Command: gr.Command,` 357− `GoldOutcome: TestOutcomeUnknown,` 358− `GoldRetried: priorRetried,` 359− `GoldRetryEvidence: priorEvidence,` 357+ `Command: gr.Command,` 358+ `GoldOutcome: TestOutcomeUnknown,` 359+ `GoldRetried: priorRetried,` 360+ `GoldRetryEvidence: priorEvidence,` 361+ `AttemptCount: priorResult.AttemptCount,` 362+ `RetryCount: priorResult.RetryCount,` 363+ `RetryReasons: priorResult.RetryReasons,` 364+ `RecoveredAfterRetry: priorResult.RecoveredAfterRetry,` 360365 `}` 361366 `gr.Err = ctx.Err()` 362367 `continue` …`func runGoldBatchSlice(` 364369 `stageRoot, cacheResult, cleanup, err := stageGoldBaselineTask(ctx, inputs, []string{commands[idx]}, false, overrideCfg.OutputDir)` 365370 `gr.EnvCache = cacheResult` 366371 `gr.Result = TestCommandResult{` 367− `Command: gr.Command,` 368− `GoldRetried: priorRetried,` 369− `GoldRetryEvidence: priorEvidence,` 372+ `Command: gr.Command,` 373+ `GoldRetried: priorRetried,` 374+ `GoldRetryEvidence: priorEvidence,` 375+ `AttemptCount: priorResult.AttemptCount,` 376+ `RetryCount: priorResult.RetryCount,` 377+ `RetryReasons: priorResult.RetryReasons,` 378+ `RecoveredAfterRetry: priorResult.RecoveredAfterRetry,` 370379 `}` 371380 `if err != nil {` 372381 `gr.Err = err` …`func runGoldBatchSlice(` 407416 `gr.Result.GoldRetried = true` 408417 `gr.Result.GoldRetryEvidence = &TestRetryEvidence{First: *gr.FirstRetryEvidence}` 409418 `}` 419+ `recordTestRetry(&gr.Result, FailureModeInfraDependencyFetch)` 410420 `gr.RetryDispatched = true` 411421 `}` 412422 `}` 413423 424+ `for _, item := range items {` 425+ `var idx int` 426+ `fmt.Sscanf(item.ProjectionKey, "gold-%d", &idx)` 427+ `if attempt == 1 {` 428+ `goldResults[idx].Result.AttemptCount++` 429+ `}` 430+ `}` 414431 `results, err := batchRunner.RunBatch(ctx, harness.BatchRunConfig{` 415432 `OutputDir: filepath.Join(filepath.Dir(items[0].SourceTaskDir), ".tb-runs"),` 416433 `RunID: fmt.Sprintf("validate-gold-batch-%d", time.Now().UnixNano()),` …`func runGoldBatchSlice(` 452469 `evidence := retainBatchGoldRetryAttemptEvidence(projected, gr.Result.GoldFailureMode, overrideCfg, gr.Index, attempt)` 453470 `gr.FirstRetryEvidence = &evidence` 454471 `}` 472+ `if attempt > 1 && gr.Result.GoldOutcome == TestOutcomePass {` 473+ `gr.Result.RecoveredAfterRetry = true` 474+ `}` 455475 `if attempt > 1 && gr.Result.GoldRetryEvidence != nil {` 456476 `evidence := retainBatchGoldRetryAttemptEvidence(projected, gr.Result.GoldFailureMode, overrideCfg, gr.Index, attempt)` 457477 `gr.Result.GoldRetryEvidence.Retry = &evidence` …`func runOverrideBatchSlice(` 564584 `return` 565585 `}` 566586 587+ `for _, item := range items {` 588+ `var idx int` 589+ `fmt.Sscanf(item.ProjectionKey, "override-%d", &idx)` 590+ `overrideResults[idx-offset].Result.AttemptCount++` 591+ `}` 592+ 567593 `results, err := batchRunner.RunBatch(ctx, harness.BatchRunConfig{` 568594 `OutputDir: filepath.Join(filepath.Dir(items[0].SourceTaskDir), ".tb-runs"),` 569595 `RunID: fmt.Sprintf("validate-override-batch-%d", time.Now().UnixNano()),` …`func runOverrideBatchSlice(` 604630 `projected := results[fmt.Sprintf("override-%d", offset+i)]` 605631 `or.Result.AgentOutcome = testOutcomeFromTrial(projected.Trial)` 606632 `or.Result.AgentTrialName = projected.Trial.TrialName` 607− `or.Result.AgentFailureMode = projected.Trial.FailureMode` 608− `if or.Result.AgentOutcome == TestOutcomeFail && or.Result.GoldOutcome == TestOutcomePass && ctx.Err() == nil {` 633+ `or.Result.AgentFailureMode = classifyBatchGoldFailureMode(projected)` 634+ `if or.Result.AgentOutcome != TestOutcomePass && isRetryableGoldInfrastructureFailure(or.Result.AgentFailureMode) && or.Result.GoldOutcome == TestOutcomePass && ctx.Err() == nil {` 609635 `stageRoot, cacheResult, cleanup, err := stageOverrideTask(ctx, inputs, []string{or.Command}, false, overrideCfg.OutputDir)` 610636 `if err == nil {` 611637 `or.RetryEnvCache = &cacheResult` …`func runOverrideBatchSlice(` 623649 `if len(retryItems) == 0 {` 624650 `return` 625651 `}` 652+ `for _, idx := range retryIndexes {` 653+ `or := &overrideResults[idx]` 654+ `recordTestRetry(&or.Result, or.Result.AgentFailureMode)` 655+ `or.Result.AgentRetried = true` 656+ `}` 626657 `retryResults, err := batchRunner.RunBatch(ctx, harness.BatchRunConfig{` 627658 `OutputDir: filepath.Join(filepath.Dir(retryItems[0].SourceTaskDir), ".tb-runs"),` 628659 `RunID: fmt.Sprintf("validate-override-retry-batch-%d", time.Now().UnixNano()),` …`func runOverrideBatchSlice(` 634665 `HarnessCLICacheMode: overrideCfg.HarnessCLICacheMode,` 635666 `Items: retryItems,` 636667 `})` 637− `if err != nil {` 638− `return` 639− `}` 640668 `for _, idx := range retryIndexes {` 641669 `or := &overrideResults[idx]` 642670 `projected := retryResults[fmt.Sprintf("retry-%d", offset+idx)]` 643− `if testOutcomeFromTrial(projected.Trial) == TestOutcomePass {` 644− `or.Result.AgentOutcome = TestOutcomePass` 645− `or.Result.AgentTrialName = projected.Trial.TrialName` 646− `or.Result.AgentFailureMode = projected.Trial.FailureMode` 647− `or.Result.AgentRetried = true` 671+ `or.Result.AgentOutcome = testOutcomeFromTrial(projected.Trial)` 672+ `or.Result.AgentTrialName = projected.Trial.TrialName` 673+ `or.Result.AgentFailureMode = classifyBatchGoldFailureMode(projected)` 674+ `or.Err = err` 675+ `if err != nil {` 676+ `if isUnsetFailureMode(or.Result.AgentFailureMode) {` 677+ `or.Result.AgentFailureMode = inferInfraFailureModeFromError(err)` 678+ `}` 679+ `or.Notes = append(or.Notes, fmt.Sprintf("agent retest retry for %q failed: %v", or.Command, err))` 680+ `}` 681+ `if err == nil && or.Result.AgentOutcome == TestOutcomePass {` 682+ `or.Result.RecoveredAfterRetry = true` 648683 `or.Notes = append(or.Notes, fmt.Sprintf("agent retest for %q succeeded on retry", or.Command))` 649684 `}` 650685 `}` …`go: golang.org/x/sys@v0.42.0: Get "https://proxy.golang.org/golang.org/x/sys/@v/` 50215021 `t.Fatalf("command results = %d, want 1", len(summary.CommandResults))` 50225022 `}` 50235023 `result := summary.CommandResults[0]` 5024+ `if result.AttemptCount != 3 || result.RetryCount != 1 || !result.RecoveredAfterRetry || len(result.RetryReasons) != 1 || result.RetryReasons[0] != FailureModeInfraDependencyFetch {` 5025+ `t.Fatalf("retry metadata = %+v", result)` 5026+ `}` 50245027 `if !result.GoldRetried || result.GoldOutcome != TestOutcomePass || result.GoldTrialName != "gold-retry" {` 50255028 `t.Fatalf("gold retry result = %+v", result)` 50265029 `}` …`go: golang.org/x/sys@v0.42.0: Get "https://proxy.golang.org/golang.org/x/sys/@v/` 51115114 `t.Fatalf("command results = %d, want 1", len(summary.CommandResults))` 51125115 `}` 51135116 `result := summary.CommandResults[0]` 5117+ `if result.AttemptCount != 3 || result.RetryCount != 1 || !result.RecoveredAfterRetry || len(result.RetryReasons) != 1 || result.RetryReasons[0] != FailureModeInfraDependencyFetch {` 5118+ `t.Fatalf("retry metadata = %+v", result)` 5119+ `}` 51145120 `if !result.GoldRetried || result.GoldOutcome != TestOutcomePass || result.GoldTrialName != "gold-retry" {` 51155121 `t.Fatalf("gold retry result = %+v", result)` 51165122 `}` …`func TestRunPerCommandTestsBatchedGoldTimeoutClassifiedAsTimeBudgetExhausted(t *` 53825388 `}` 53835389`}` 53845390 5385−`func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried(t *testing.T) {` 5391+`func TestRunPerCommandTestsBatchedRetriesDependencyFetchOverrideAndMarksRetried(t *testing.T) {` 53865392 `t.Parallel()` 53875393 `repo, base, head := createRepoWithCommits(t)` 53885394 `dataset := t.TempDir()` …`func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried(t *testin` 54185424 `runner := &batchValidateRunner{` 54195425 `results: map[string]TBTrialResult{` 54205426 `"gold-0": {TaskID: taskID, IsResolved: &pass, TrialName: "gold-a"},` 5421− `"override-0": {TaskID: taskID, IsResolved: &fail, TrialName: "override-a"},` 5427+ `"override-0": {TaskID: taskID, IsResolved: &fail, TrialName: "override-a", FailureMode: FailureModeInfraDependencyFetch},` 54225428 `"retry-0": {TaskID: taskID, IsResolved: &pass, TrialName: "retry-a"},` 54235429 `},` 54245430 `}` …`func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried(t *testin` 54435449 `t.Fatalf("expected 1 command result, got %d", len(result.Tests.CommandResults))` 54445450 `}` 54455451 `cmd := result.Tests.CommandResults[0]` 5452+ `if cmd.AttemptCount != 3 || cmd.RetryCount != 1 || !cmd.RecoveredAfterRetry || len(cmd.RetryReasons) != 1 || cmd.RetryReasons[0] != FailureModeInfraDependencyFetch {` 5453+ `t.Fatalf("retry metadata = %+v", cmd)` 5454+ `}` 54465455 `if !cmd.AgentRetried {` 54475456 `t.Fatal("expected AgentRetried=true")` 54485457 `}` … 11`package validate` 22 3−`import "fmt"` 3+`import (` 4+ `"fmt"` 5+ `"path/filepath"` 6+ `"strings"` 7+`)` 48 59`// OfflinePromptInputs describes everything needed to reassemble the grader` 610`// PromptContext for a single already-evaluated cell without re-running the` …`type OfflinePromptInputs struct {` 1822 `// TBResultsRaw is the raw TB results JSON for this cell, or "" when` 1923 `// unavailable (e.g. no-test-signal corpora).` 2024 `TBResultsRaw string` 25+ `// EffectiveTestCommands overrides dataset commands when non-nil. An empty` 26+ `// non-nil slice preserves a persisted grade with no verifier commands.` 27+ `EffectiveTestCommands []string` 2128 `// PromptPatchMaxBytes caps the gold/agent patch bytes embedded in the` 2229 `// prompt. Values <= 0 fall back to DefaultPromptPatchMaxBytes, matching the` 2330 `// orchestrator/config default.` …`type OfflinePromptInputs struct {` 2532 `// RepoContextOptions controls repo tree/excerpt extraction. The zero value` 2633 `// inherits BuildRepoContext defaults, matching the live default path.` 2734 `RepoContextOptions RepoContextOptions` 35+ `// RequireRepoContext makes reconstruction fail when the source repository` 36+ `// cannot be extracted, instead of grading an unavailable-context marker.` 37+ `RequireRepoContext bool` 38+ `// ExpectedArtifacts binds reconstruction to the original graded files.` 39+ `// When supplied, every source path and digest must be present and match.` 40+ `ExpectedArtifacts *ArtifactManifest` 2841`}` 2942 3043`// BuildOfflinePromptContext reassembles the grader PromptContext for one cell` …`type OfflinePromptInputs struct {` 3447`// that removes the extracted repo temp dir; callers should defer it.` 3548`//` 3649`// The assembly intentionally mirrors internal/validate/orchestrator.go so the` 37−`// resulting context is byte-for-byte identical to a live grade except for the` 38−`// template version: it reuses ReadTaskInputs, truncatePatch, and` 39−`// BuildRepoContext, and applies the same graceful repo-context fallback.` 50+`// assembly reuses ReadTaskInputs, truncatePatch, and BuildRepoContext. Callers` 51+`// can supply persisted effective commands and require usable repository context;` 52+`// otherwise it applies the live path's graceful repo-context fallback.` 4053`func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), error) {` 4154 `inputs, err := ReadTaskInputs(in.DatasetPath, in.TaskID, in.AgentPatchPath)` 4255 `if err != nil {` …`func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e` 4558 `if err := verifyRepoSourceForConsumption(inputs); err != nil {` 4659 `return PromptContext{}, nil, err` 4760 `}` 61+ `if in.ExpectedArtifacts != nil {` 62+ `current := BuildArtifactManifest(inputs)` 63+ `for _, pair := range []struct {` 64+ `name string` 65+ `want, got ArtifactFileManifest` 66+ `}{` 67+ `{"agent patch", in.ExpectedArtifacts.AgentPatch, current.AgentPatch},` 68+ `{"gold patch", in.ExpectedArtifacts.GoldPatch, current.GoldPatch},` 69+ `{"repository", in.ExpectedArtifacts.RepoTarball, current.RepoTarball},` 70+ `} {` 71+ `if pair.want.Path == "" || pair.want.SHA256 == "" {` 72+ `return PromptContext{}, nil, fmt.Errorf("missing original %s path or digest", pair.name)` 73+ `}` 74+ `if filepath.Clean(pair.want.Path) != filepath.Clean(pair.got.Path) || !strings.EqualFold(strings.TrimPrefix(pair.want.SHA256, "sha256:"), pair.got.SHA256) {` 75+ `return PromptContext{}, nil, fmt.Errorf("original %s provenance mismatch", pair.name)` 76+ `}` 77+ `}` 78+ `}` 4879 4980 `maxBytes := in.PromptPatchMaxBytes` 5081 `if maxBytes <= 0 {` …`func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e` 5283 `}` 5384 `goldPatchPrompt, goldPatchTruncation := truncatePatch(inputs.GoldPatch, maxBytes)` 5485 `agentPatchPrompt, agentPatchTruncation := truncatePatch(inputs.AgentPatch, maxBytes)` 86+ `if in.ExpectedArtifacts != nil && (goldPatchTruncation.Truncated || agentPatchTruncation.Truncated) {` 87+ `return PromptContext{}, nil, fmt.Errorf("reconstructed prompt patches would be truncated; original context budgets are unavailable")` 88+ `}` 89+ `testCommands := inputs.TestCommands` 90+ `if in.EffectiveTestCommands != nil {` 91+ `testCommands = append([]string{}, in.EffectiveTestCommands...)` 92+ `}` 5593 5694 `// Mirror orchestrator.go: on repo-context failure, degrade RepoTree to an` 5795 `// "unavailable: " marker and continue rather than failing the build.` …`func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e` 5997 `repoCtx := RepoContext{}` 6098 `builtRepoCtx, repoCleanup, repoErr := BuildRepoContext(inputs.RepoTarballPath, in.RepoContextOptions)` 6199 `if repoErr != nil {` 100+ `if in.RequireRepoContext {` 101+ `return PromptContext{}, nil, fmt.Errorf("build required repo context: %w", repoErr)` 102+ `}` 62103 `builtRepoCtx = RepoContext{` 63104 `RepoTree: fmt.Sprintf("unavailable: %v", repoErr),` 64105 `}` 65106 `} else {` 66107 `cleanup = repoCleanup` 108+ `if in.ExpectedArtifacts != nil && (builtRepoCtx.RepoTreeTruncation.Truncated || builtRepoCtx.RepoExcerptsTruncation.Truncated) {` 109+ `if cleanup != nil {` 110+ `cleanup()` 111+ `}` 112+ `return PromptContext{}, nil, fmt.Errorf("reconstructed repository context would be truncated; original context budgets are unavailable")` 113+ `}` 67114 `}` 68115 `repoCtx = builtRepoCtx` 69116 …`func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e` 71118 `TaskYAML: inputs.TaskYAML,` 72119 `GoldPatch: goldPatchPrompt,` 73120 `AgentPatch: agentPatchPrompt,` 74− `TestCommands: inputs.TestCommands,` 121+ `TestCommands: testCommands,` 75122 `TBResults: in.TBResultsRaw,` 76123 `RepoPath: repoCtx.RepoPath,` 77124 `RepoTree: repoCtx.RepoTree,` …`func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e` 80127 `GoldPatchTruncated: goldPatchTruncation.Truncated,` 81128 `}, cleanup, nil` 82129`}` 130+ 131+`// RenderReviewPrompt renders the built-in review template from supplied evidence.` 132+`func RenderReviewPrompt(promptCtx PromptContext) (string, error) {` 133+ `return renderPrompt(reviewPromptTemplate, promptCtx.asTemplateData())` 134+`}` …`func TestBuildOfflinePromptContextDegradesWhenTarballUnextractable(t *testing.T)` 164164 `t.Fatalf("expected agent patch to be populated, got %q", promptCtx.AgentPatch)` 165165 `}` 166166`}` 167+ 168+`func TestBuildOfflinePromptContextRequiresUsableRepositoryWhenRequested(t *testing.T) {` 169+ `datasetPath, taskID, agentPatchPath := writeOfflineTaskDir(t, false)` 170+ `_, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{` 171+ `DatasetPath: datasetPath, TaskID: taskID, AgentPatchPath: agentPatchPath,` 172+ `RequireRepoContext: true,` 173+ `})` 174+ `if cleanup != nil {` 175+ `defer cleanup()` 176+ `}` 177+ `if err == nil || !strings.Contains(err.Error(), "build required repo context") {` 178+ `t.Fatalf("expected required repository extraction failure, got %v", err)` 179+ `}` 180+`}` 181+ 182+`func TestBuildOfflinePromptContextUsesEffectiveVerifierCommands(t *testing.T) {` 183+ `datasetPath, taskID, agentPatchPath := writeOfflineTaskDir(t, true)` 184+ `for _, commands := range [][]string{nil, {}, {"effective verifier"}} {` 185+ `ctx, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{` 186+ `DatasetPath: datasetPath, TaskID: taskID, AgentPatchPath: agentPatchPath,` 187+ `EffectiveTestCommands: commands,` 188+ `})` 189+ `if cleanup != nil {` 190+ `defer cleanup()` 191+ `}` 192+ `if err != nil {` 193+ `t.Fatal(err)` 194+ `}` 195+ `want := commands` 196+ `if commands == nil {` 197+ `want = []string{"pytest -q"}` 198+ `}` 199+ `if !reflect.DeepEqual(ctx.TestCommands, want) {` 200+ `t.Fatalf("effective commands %v: got %v, want %v", commands, ctx.TestCommands, want)` 201+ `}` 202+ `}` 203+`}` 204+ 205+`func TestBuildOfflinePromptContextRejectsMutatedOriginalArtifacts(t *testing.T) {` 206+ `for _, artifact := range []string{"agent patch", "gold patch", "repository"} {` 207+ `t.Run(artifact, func(t *testing.T) {` 208+ `dataset, taskID, patch := writeOfflineTaskDir(t, true)` 209+ `inputs, err := ReadTaskInputs(dataset, taskID, patch)` 210+ `if err != nil {` 211+ `t.Fatal(err)` 212+ `}` 213+ `original := BuildArtifactManifest(inputs)` 214+ `path := patch` 215+ `if artifact == "gold patch" {` 216+ `path = inputs.GoldPatchPath` 217+ `}` 218+ `if artifact == "repository" {` 219+ `path = inputs.RepoTarballPath` 220+ `}` 221+ `if err := os.WriteFile(path, []byte("mutated evidence"), 0o644); err != nil {` 222+ `t.Fatal(err)` 223+ `}` 224+ `_, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{` 225+ `DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch,` 226+ `ExpectedArtifacts: &original, RequireRepoContext: true,` 227+ `})` 228+ `if cleanup != nil {` 229+ `defer cleanup()` 230+ `}` 231+ `if err == nil || !strings.Contains(err.Error(), artifact+" provenance mismatch") {` 232+ `t.Fatalf("expected original %s mismatch, got %v", artifact, err)` 233+ `}` 234+ `})` 235+ `}` 236+`}` 237+ 238+`func TestBuildOfflinePromptContextRejectsAbsentOriginalArtifactDigest(t *testing.T) {` 239+ `dataset, taskID, patch := writeOfflineTaskDir(t, true)` 240+ `inputs, err := ReadTaskInputs(dataset, taskID, patch)` 241+ `if err != nil {` 242+ `t.Fatal(err)` 243+ `}` 244+ `original := BuildArtifactManifest(inputs)` 245+ `original.AgentPatch.SHA256 = ""` 246+ `_, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{` 247+ `DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch, ExpectedArtifacts: &original,` 248+ `})` 249+ `if cleanup != nil {` 250+ `defer cleanup()` 251+ `}` 252+ `if err == nil || !strings.Contains(err.Error(), "missing original agent patch") {` 253+ `t.Fatalf("expected absent original provenance failure, got %v", err)` 254+ `}` 255+`}` 256+ 257+`func TestBuildOfflinePromptContextRejectsNewTruncationWithOriginalProvenance(t *testing.T) {` 258+ `dataset, taskID, patch := writeOfflineTaskDir(t, true)` 259+ `if err := os.WriteFile(patch, []byte(largePatch(4_000)), 0o644); err != nil {` 260+ `t.Fatal(err)` 261+ `}` 262+ `inputs, err := ReadTaskInputs(dataset, taskID, patch)` 263+ `if err != nil {` 264+ `t.Fatal(err)` 265+ `}` 266+ `original := BuildArtifactManifest(inputs)` 267+ `_, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{` 268+ `DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch, ExpectedArtifacts: &original,` 269+ `PromptPatchMaxBytes: 600,` 270+ `})` 271+ `if cleanup != nil {` 272+ `defer cleanup()` 273+ `}` 274+ `if err == nil || !strings.Contains(err.Error(), "would be truncated") {` 275+ `t.Fatalf("expected new truncation refusal, got %v", err)` 276+ `}` 277+`}` …`const (` 2828`var coverageRepairRetryBackoffs = []time.Duration{time.Second, 2 * time.Second}` 2929 3030`type CoverageRepairConfig struct {` 31+ `EquivalenceAI AIClient` 32+ `EquivalencePrompt string` 33+ `EquivalencePromptContext *PromptContext` 34+ `EquivalencePromptMode EquivalencePromptMode` 35+ `ReviewAI AIClient` 36+ `ReviewPrompt string` 37+ `ReviewPromptContext *PromptContext` 38+ 3139 `Existing ValidationResult` 3240 `AI AIClient` 3341 `AIModelID string` …`type CoverageRepairConfig struct {` 3745`}` 3846 3947`type CoverageRepairIssue struct {` 48+ `` AttemptCount int `json:"attempt_count,omitempty"` `` 49+ `` RetryCount int `json:"retry_count,omitempty"` `` 50+ `` RetryReasons []string `json:"retry_reasons,omitempty"` `` 51+ 4052 `` Dimension string `json:"dimension"` `` 4153 `` Reason string `json:"reason"` `` 4254 `` Category string `json:"category,omitempty"` `` …`func RepairCoverage(ctx context.Context, cfg CoverageRepairConfig) (CoverageRepa` 6981 `updated := false` 7082 7183 `if cfg.RepairEquivalence {` 72− `equivalence, repaired, issue := repairEquivalenceFromPrompt(ctx, ai, cfg.Existing.Equivalence, cfg.AIModelID, cfg.ReviewParseRetries)` 84+ `source := cfg.Existing.Equivalence` 85+ `if cfg.EquivalencePrompt != "" {` 86+ `source.Prompt = cfg.EquivalencePrompt` 87+ `}` 88+ `mode := source.PromptMode` 89+ `if cfg.EquivalencePromptMode != "" {` 90+ `if mode != "" && mode != cfg.EquivalencePromptMode {` 91+ `return result, errors.New("equivalence repair prompt mode differs from original measurement")` 92+ `}` 93+ `mode = cfg.EquivalencePromptMode` 94+ `}` 95+ `if cfg.EquivalencePromptContext != nil {` 96+ `rendered, err := RenderEquivalencePromptForMode(*cfg.EquivalencePromptContext, mode)` 97+ `if err != nil {` 98+ `return result, err` 99+ `}` 100+ `if cfg.EquivalencePrompt != "" && cfg.EquivalencePrompt != rendered {` 101+ `return result, errors.New("equivalence repair prompt differs from supplied context")` 102+ `}` 103+ `source.Prompt = rendered` 104+ `}` 105+ `source.PromptMode = mode` 106+ `dimensionAI := cfg.EquivalenceAI` 107+ `if dimensionAI == nil {` 108+ `dimensionAI = ai` 109+ `}` 110+ `history := newGradeAttemptClient(dimensionAI)` 111+ `equivalence, repaired, issue := repairEquivalenceFromPrompt(ctx, history, source, cfg.AIModelID, cfg.ReviewParseRetries)` 112+ `if repaired {` 113+ `equivalence.PromptMode = mode` 114+ `if cfg.EquivalencePromptContext != nil {` 115+ `pc := *cfg.EquivalencePromptContext` 116+ `if mode == EquivalencePromptModeGoldBlind {` 117+ `pc.GoldPatchTruncated = false` 118+ `}` 119+ `equivalence = annotateEquivalenceTruncation(equivalence, pc)` 120+ `}` 121+ `history.annotateEquivalence(&equivalence)` 122+ `mergeEquivalenceRepairHistory(&equivalence, source)` 123+ `}` 73124 `if issue != nil {` 125+ `issue.AttemptCount = history.attempts` 126+ `issue.RetryCount = max(0, history.attempts-1)` 127+ `issue.RetryReasons = append([]string(nil), history.reasons...)` 74128 `result.Unresolved = append(result.Unresolved, *issue)` 75129 `} else {` 76130 `result.Validation.Equivalence = equivalence` …`func RepairCoverage(ctx context.Context, cfg CoverageRepairConfig) (CoverageRepa` 84138 `if strings.TrimSpace(source.Prompt) == "" && cfg.Existing.Review != nil {` 85139 `source = *cfg.Existing.Review` 86140 `}` 87− `review, repaired, issue := repairReviewFromPrompt(ctx, ai, source, cfg.AIModelID, cfg.ReviewParseRetries)` 141+ `if cfg.ReviewPrompt != "" {` 142+ `source.Prompt = cfg.ReviewPrompt` 143+ `}` 144+ `if cfg.ReviewPromptContext != nil {` 145+ `rendered, err := renderPrompt(reviewPromptTemplate, cfg.ReviewPromptContext.asTemplateData())` 146+ `if err != nil {` 147+ `return result, err` 148+ `}` 149+ `if cfg.ReviewPrompt != "" && cfg.ReviewPrompt != rendered {` 150+ `return result, errors.New("review repair prompt differs from supplied context")` 151+ `}` 152+ `source.Prompt = rendered` 153+ `}` 154+ `if source.Prompt != resolvedRepairReview(cfg.Existing).Prompt {` 155+ `source.Output = ""` 156+ `}` 157+ `dimensionAI := cfg.ReviewAI` 158+ `if dimensionAI == nil {` 159+ `dimensionAI = ai` 160+ `}` 161+ `history := newGradeAttemptClient(dimensionAI)` 162+ `review, repaired, issue := repairReviewFromPromptContext(ctx, history, source, cfg.AIModelID, cfg.ReviewParseRetries, cfg.ReviewPromptContext)` 163+ `if repaired && history.attempts > 0 {` 164+ `history.annotateReview(&review)` 165+ `mergeReviewRepairHistory(&review, source)` 166+ `}` 167+ `if repaired && cfg.ReviewPromptContext != nil {` 168+ `review = annotateReviewTruncation(review, *cfg.ReviewPromptContext)` 169+ `}` 88170 `if issue != nil {` 171+ `issue.AttemptCount = history.attempts` 172+ `issue.RetryCount = max(0, history.attempts-1)` 173+ `issue.RetryReasons = append([]string(nil), history.reasons...)` 89174 `result.Unresolved = append(result.Unresolved, *issue)` 90175 `} else {` 91176 `result.Validation.CodeReview = review` …`func RepairCoverage(ctx context.Context, cfg CoverageRepairConfig) (CoverageRepa` 104189`}` 105190 106191`func repairEquivalenceFromPrompt(ctx context.Context, ai AIClient, existing EquivalenceResult, overrideModelID string, parseRetries int) (EquivalenceResult, bool, *CoverageRepairIssue) {` 107− `prompt := strings.TrimSpace(existing.Prompt)` 108− `if prompt == "" {` 192+ `prompt := existing.Prompt` 193+ `if strings.TrimSpace(prompt) == "" {` 109194 `return existing, false, &CoverageRepairIssue{` 110195 `Dimension: coverageRepairDimensionEquivalence,` 111196 `Reason: "missing_prompt",` …`func repairEquivalenceFromPrompt(ctx context.Context, ai AIClient, existing Equi` 145230 146231 `parsed.Status = AIStatusUsed` 147232 `parsed.EvaluatorModel = resolveRepairEvaluatorModel(existing.EvaluatorModel, overrideModelID)` 233+ `parsed.PromptMode = existing.PromptMode` 234+ `parsed.Truncated = existing.Truncated` 235+ `parsed.TruncationReason = existing.TruncationReason` 148236 `parsed.Prompt = prompt` 149237 `parsed.Output = output` 150238 `return parsed, true, nil` …`func reviewVerdictIsGuardSuppressible(result ReviewResult) bool {` 159247`}` 160248 161249`func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewResult, overrideModelID string, parseRetries int) (ReviewResult, bool, *CoverageRepairIssue) {` 162− `prompt := strings.TrimSpace(existing.Prompt)` 163− `if prompt == "" {` 250+ `return repairReviewFromPromptContext(ctx, ai, existing, overrideModelID, parseRetries, nil)` 251+`}` 252+ 253+`func repairReviewFromPromptContext(ctx context.Context, ai AIClient, existing ReviewResult, overrideModelID string, parseRetries int, promptCtx *PromptContext) (ReviewResult, bool, *CoverageRepairIssue) {` 254+ `prompt := existing.Prompt` 255+ `if strings.TrimSpace(prompt) == "" {` 164256 `return existing, false, &CoverageRepairIssue{` 165257 `Dimension: coverageRepairDimensionCodeReview,` 166258 `Reason: "missing_prompt",` …`func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes` 174266 `// re-apply that guard, so such verdicts must be resampled, never recovered.` 175267 `if existing.Status == AIStatusUnavailable && strings.TrimSpace(existing.Output) != "" {` 176268 `if reparsed, reparseErr := parseReviewOutput(existing.Output); reparseErr == nil &&` 177− `!reviewVerdictIsGuardSuppressible(reparsed) {` 269+ `((promptCtx == nil && !reviewVerdictIsGuardSuppressible(reparsed)) || (promptCtx != nil && validateReviewAgainstPromptContext(reparsed, *promptCtx) == nil)) {` 178270 `reparsed.Status = AIStatusUsed` 179271 `reparsed.Prompt = prompt` 180272 `reparsed.Output = existing.Output` …`func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes` 184276 `if reparsed.EvaluatorModel == "" {` 185277 `reparsed.EvaluatorModel = strings.TrimSpace(overrideModelID)` 186278 `}` 279+ `reparsed.AttemptCount = existing.AttemptCount` 280+ `reparsed.RetryCount = existing.RetryCount` 281+ `reparsed.RetryReasons = append([]string(nil), existing.RetryReasons...)` 282+ `reparsed.RecoveredAfterRetry = existing.RecoveredAfterRetry` 187283 `reparsed.Truncated = existing.Truncated` 188284 `reparsed.TruncationReason = existing.TruncationReason` 189285 `return reparsed, true, nil` …`func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes` 196292 `}` 197293 198294 `parsed, parseErr := parseReviewOutput(output)` 295+ `if parseErr == nil && promptCtx != nil {` 296+ `parseErr = validateReviewAgainstPromptContext(parsed, *promptCtx)` 297+ `}` 199298 `for attempt := 0; parseErr != nil && attempt < normalizedReviewParseRetries(parseRetries); attempt++ {` 200299 `retryPrompt := buildReviewRetryPrompt(prompt, output, parseErr)` 201300 `retryOutput, retryErr := runAIWithRetries(ctx, ai, retryPrompt)` …`func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes` 203302 `return existing, false, newReviewRuntimeFailureIssue(retryErr)` 204303 `}` 205304 `parsed, parseErr = parseReviewOutput(retryOutput)` 305+ `if parseErr == nil && promptCtx != nil {` 306+ `parseErr = validateReviewAgainstPromptContext(parsed, *promptCtx)` 307+ `}` 206308 `output = retryOutput` 207309 `}` 208310 `if parseErr != nil {` 209311 `return existing, false, classifyReviewParseIssue(parseErr)` 210312 `}` 211313 314+ `if promptCtx == nil && reviewVerdictIsGuardSuppressible(parsed) {` 315+ `return existing, false, classifyReviewParseIssue(errors.New("review repair requires original verifier context for speculative blocker verdict"))` 316+ `}` 317+ `parsed.Truncated = existing.Truncated` 318+ `parsed.TruncationReason = existing.TruncationReason` 212319 `parsed.Status = AIStatusUsed` 213320 `parsed.Prompt = prompt` 214321 `parsed.Output = output` …`func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes` 217324`}` 218325 219326`func runAIWithRetries(ctx context.Context, ai AIClient, prompt string) (string, error) {` 220− `var lastErr error` 221− `for attempt := 0; attempt <= len(coverageRepairRetryBackoffs); attempt++ {` 222− `output, err := ai.Run(ctx, prompt)` 223− `if err == nil {` 224− `return output, nil` 225− `}` 226− `lastErr = err` 227− `if errors.Is(err, ErrAIDisabled) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {` 228− `break` 229− `}` 230− `if attempt >= len(coverageRepairRetryBackoffs) {` 231− `break` 232− `}` 233− `select {` 234− `case <-ctx.Done():` 235− `return "", ctx.Err()` 236− `case <-time.After(coverageRepairRetryBackoffs[attempt]):` 237− `}` 327+ `if history, ok := ai.(*gradeAttemptClient); ok {` 328+ `return history.Run(ctx, prompt)` 238329 `}` 239− `return "", lastErr` 330+ `return newGradeAttemptClient(ai).Run(ctx, prompt)` 240331`}` 241332 242333`func normalizedReviewParseRetries(parseRetries int) int {` …`func appendReviewNote(existing, extra string) string {` 5353 `}` 5454`}` 5555 56−`func RunReview(ctx context.Context, ai AIClient, promptCtx PromptContext) ReviewResult {` 56+`func RunReview(ctx context.Context, ai AIClient, promptCtx PromptContext) (result ReviewResult) {` 57+ `history := newGradeAttemptClient(ai)` 58+ `defer func() { history.annotateReview(&result) }()` 59+ `ai = history` 60+ 5761 `return annotateReviewTruncation(runReview(ctx, ai, promptCtx), promptCtx)` 5862`}` 5963 …`type TestVerifierEvidence struct {` 712712`}` 713713 714714`type TestCommandResult struct {` 715− `` Command string `json:"command"` `` 716− `` GoldOutcome TestOutcome `json:"gold_outcome,omitempty"` `` 717− `` GoldTrialName string `json:"gold_tb_trial_name,omitempty"` `` 718− `` GoldFailureMode string `json:"gold_failure_mode,omitempty"` `` 719− `` GoldRetried bool `json:"gold_retried,omitempty"` `` 720− `` GoldRetryEvidence *TestRetryEvidence `json:"gold_retry_evidence,omitempty"` `` 721− `` AgentOutcome TestOutcome `json:"agent_outcome,omitempty"` `` 722− `` AgentTrialName string `json:"agent_tb_trial_name,omitempty"` `` 723− `` AgentFailureMode string `json:"agent_failure_mode,omitempty"` `` 724− `` AgentRetried bool `json:"agent_retried,omitempty"` `` 715+ `// Counts cover dispatched gold and candidate verifier calls for this command.` 716+ `` AttemptCount int `json:"attempt_count,omitempty"` `` 717+ `` RetryCount int `json:"retry_count,omitempty"` `` 718+ `` RetryReasons []string `json:"retry_reasons,omitempty"` `` 719+ `` RecoveredAfterRetry bool `json:"recovered_after_retry,omitempty"` `` 720+ `` Command string `json:"command"` `` 721+ `` GoldOutcome TestOutcome `json:"gold_outcome,omitempty"` `` 722+ `` GoldTrialName string `json:"gold_tb_trial_name,omitempty"` `` 723+ `` GoldFailureMode string `json:"gold_failure_mode,omitempty"` `` 724+ `` GoldRetried bool `json:"gold_retried,omitempty"` `` 725+ `` GoldRetryEvidence *TestRetryEvidence `json:"gold_retry_evidence,omitempty"` `` 726+ `` AgentOutcome TestOutcome `json:"agent_outcome,omitempty"` `` 727+ `` AgentTrialName string `json:"agent_tb_trial_name,omitempty"` `` 728+ `` AgentFailureMode string `json:"agent_failure_mode,omitempty"` `` 729+ `` AgentRetried bool `json:"agent_retried,omitempty"` `` 725730`}` 726731 727732`// TestRetryEvidence records the two verifier attempts that authorized and` …`type TestRetryAttemptEvidence struct {` 742747`}` 743748 744749`type EquivalenceResult struct {` 750+ `` AttemptCount int `json:"attempt_count,omitempty"` `` 751+ `` RetryCount int `json:"retry_count,omitempty"` `` 752+ `` RetryReasons []string `json:"retry_reasons,omitempty"` `` 753+ `` RecoveredAfterRetry bool `json:"recovered_after_retry,omitempty"` `` 754+ 745755 `` Outcome EquivalenceOutcome `json:"outcome"` `` 746756 `` Status AIStatus `json:"status"` `` 747757 `// PromptMode records the exact built-in equivalence template variant that` …`type EquivalenceObligation struct {` 785795`}` 786796 787797`type ReviewResult struct {` 798+ `` AttemptCount int `json:"attempt_count,omitempty"` `` 799+ `` RetryCount int `json:"retry_count,omitempty"` `` 800+ `` RetryReasons []string `json:"retry_reasons,omitempty"` `` 801+ `` RecoveredAfterRetry bool `json:"recovered_after_retry,omitempty"` `` 802+ 788803 `` Status AIStatus `json:"status"` `` 789804 `` SchemaVersion string `json:"schema_version,omitempty"` `` 790805 `` ProfileID string `json:"profile_id,omitempty"` `` …`Recovery rules:` 793793 `Those repair steps preserve compare-critical metadata and existing custom` 794794 `grader surfaces so the arm stays compare-compatible while recovering` 795795 ``coverage. Add `--parse-retries N` when saved grader prompts failed`` 796− `JSON/schema parsing. Keep the original custom rubric file path on the regrade` 797− `command.` 796+ `JSON/schema parsing. Repair preserves recorded prompt mode, truncation, and` 797+ `attempt history. Missing prompts require matching original task and artifact` 798+ `provenance plus complete repository context; ambiguous reconstruction remains` 799+ `unresolved. Runtime retries cover recognized transient failures only.` 800+ `Keep the original custom rubric file path on the regrade command.` 798801`` - If `stet runs repair-ai-coverage --cr-only` still leaves `code_review` `` 799802 ``unresolved, use the summary's stable `unresolved[].reason` and optional`` 800803 `` `category`/` detail` fields to separate `model_output_shape`, `` … 1+`package commandlist` 2+ 3+`import (` 4+ `"path"` 5+ `"regexp"` 6+ `"strings"` 7+ 8+ `"stet/internal/shellcmd"` 9+`)` 10+ 11+``var assignment = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*=`)`` 12+``var duration = regexp.MustCompile(`^[0-9]+(?:\.[0-9]+)?[smhd]?$`)`` 13+ 14+`// TestInvocation identifies exactly one known test command. The original text` 15+`// is retained, including literal environment assignments and a relative cwd.` 16+`// Opaque scripts and shell evaluation are deliberately not interpreted.` 17+`func TestInvocation(command string) (string, []shellcmd.FieldSpan, bool) {` 18+ `command = strings.TrimSpace(command)` 19+ ``if command == "" || strings.ContainsAny(command, "\n\r`$") {`` 20+ `return "", nil, false` 21+ `}` 22+ `spans, ok := shellcmd.FieldSpans(command)` 23+ `if !ok || len(spans) == 0 {` 24+ `return "", nil, false` 25+ `}` 26+ `i := 0` 27+ `if spans[0].Value == "cd" {` 28+ `if len(spans) < 4 || spans[2].Value != "&&" || !relativeDirectory(spans[1].Value) {` 29+ `return "", nil, false` 30+ `}` 31+ `if unsafeShellControl(command[:spans[2].Start]) {` 32+ `return "", nil, false` 33+ `}` 34+ `i = 3` 35+ `}` 36+ `if i >= len(spans) || unsafeShellControl(command[spans[i].Start:]) {` 37+ `return "", nil, false` 38+ `}` 39+ `// The sole supported control operator belongs to the proven cd prefix.` 40+ `for _, span := range spans[i:] {` 41+ `if span.Value == "&&" || span.Value == "&" {` 42+ `return "", nil, false` 43+ `}` 44+ `}` 45+ `for i < len(spans) && assignment.MatchString(spans[i].Value) {` 46+ `if !assignment.MatchString(command[spans[i].Start:spans[i].End]) {` 47+ `return "", nil, false` 48+ `}` 49+ `i++` 50+ `}` 51+ `if i < len(spans) && spans[i].Value == "env" {` 52+ `i++` 53+ `for i < len(spans) && assignment.MatchString(spans[i].Value) {` 54+ `i++` 55+ `}` 56+ `}` 57+ `if i < len(spans) && spans[i].Value == "timeout" {` 58+ `i++` 59+ `if i >= len(spans) || !duration.MatchString(spans[i].Value) {` 60+ `return "", nil, false` 61+ `}` 62+ `i++` 63+ `}` 64+ `if i >= len(spans) {` 65+ `return "", nil, false` 66+ `}` 67+ `fields := spans[i:]` 68+ `value := func(n int) string {` 69+ `if n >= len(fields) {` 70+ `return ""` 71+ `}` 72+ `return fields[n].Value` 73+ `}` 74+ `switch value(0) {` 75+ `case "go", "cargo":` 76+ `ok = value(1) == "test"` 77+ `case "sh", "bash":` 78+ `ok = len(fields) == 2 && strings.HasSuffix(value(1), ".sh") && relativeDirectory(value(1))` 79+ `case "pytest", "pytest-3", "vitest", "jest", "phpunit":` 80+ `ok = true` 81+ `case "python", "python3":` 82+ `ok = value(1) == "-m" && value(2) == "pytest"` 83+ `case "npm", "pnpm", "yarn":` 84+ `ok = value(1) == "test" || (value(1) == "run" && value(2) == "test")` 85+ `case "npx":` 86+ `ok = value(1) == "vitest" || value(1) == "jest"` 87+ `default:` 88+ `ok = false` 89+ `}` 90+ `if !ok {` 91+ `return "", nil, false` 92+ `}` 93+ `return command, fields, true` 94+`}` 95+ 96+`func relativeDirectory(directory string) bool {` 97+ `if directory == "" || strings.HasPrefix(directory, "-") || strings.Contains(directory, "\\") || path.IsAbs(directory) {` 98+ `return false` 99+ `}` 100+ `clean := path.Clean(directory)` 101+ `return clean != ".." && !strings.HasPrefix(clean, "../")` 102+`}` 103+ 104+`func unsafeShellControl(command string) bool {` 105+ `var quote rune` 106+ `escaped := false` 107+ `for _, r := range command {` 108+ `if escaped {` 109+ `escaped = false` 110+ `continue` 111+ `}` 112+ `if quote == '\'' {` 113+ `if r == '\'' {` 114+ `quote = 0` 115+ `}` 116+ `continue` 117+ `}` 118+ `if r == '\\' {` 119+ `escaped = true` 120+ `continue` 121+ `}` 122+ `if quote == '"' {` 123+ `if r == '"' {` 124+ `quote = 0` 125+ `}` 126+ `continue` 127+ `}` 128+ `if r == '\'' || r == '"' {` 129+ `quote = r` 130+ `continue` 131+ `}` 132+ `if strings.ContainsRune(";|&<>#()", r) {` 133+ `return true` 134+ `}` 135+ `}` 136+ `return false` 137+`}` … 1+`package commandlist` 2+ 3+`import "testing"` 4+ 5+`func TestTestInvocationPreservesProvenExecutionContext(t *testing.T) {` 6+ `for _, command := range []string{` 7+ `"go test ./pkg -run 'TestOne|TestTwo' -count=1",` 8+ `"cd packages/api && FOO='a b' env BAR=2 timeout 30s python3 -m pytest tests/test_api.py -q",` 9+ `"npm run test -- --runInBand", "env CI=1 cargo test --locked", "pytest 'tests/test space.py'",` 10+ `} {` 11+ `got, fields, ok := TestInvocation(command)` 12+ `if !ok || got != command || len(fields) == 0 {` 13+ `t.Errorf("TestInvocation(%q) = %q, %v", command, got, ok)` 14+ `}` 15+ `}` 16+`}` 17+ 18+`func TestTestInvocationRejectsAmbiguousAndUnsafeWrappers(t *testing.T) {` 19+ `for _, command := range []string{` 20+ `"echo go test ./pkg", "sh -c 'go test ./pkg'", "go test ./pkg && pytest tests",` 21+ `"go test ./pkg; true", "go test ./pkg || true", "cd ../outside && go test ./pkg",` 22+ `"env -i go test ./pkg", "timeout --signal=KILL 30 go test ./pkg", "FOO=$(touch sentinel) go test ./pkg",` 23+ `"go test ./pkg > result", "cd pkg&&go test ./pkg", "go test ./pkg &", "make test", "\"CI=1\" go test ./pkg", "F\"OO\"=x go test ./pkg",` 24+ `"cd pkg;touch && go test ./pkg", "echo tests/agent.sh", "cat tests/agent.sh", "rm tests/agent.sh", "sh -c tests/agent.sh",` 25+ `} {` 26+ `if _, _, ok := TestInvocation(command); ok {` 27+ `t.Errorf("accepted %q", command)` 28+ `}` 29+ `}` 30+`}` … 1+`package h2h` 2+ 3+`import (` 4+ `"fmt"` 5+ `"path/filepath"` 6+ `"strings"` 7+ 8+ `"stet/internal/validate"` 9+`)` 10+ 11+`// prepareCoverageRepairPrompts reconstructs only absent prompts. Stored prompts` 12+`// retain their original evidence; reconstructed prompts use the sealed task` 13+`// source and keep its extracted repository alive through grading.` 14+`func prepareCoverageRepairPrompts(cfg *validate.CoverageRepairConfig, manifest *Manifest, artifacts TaskArtifacts) (func(), []validate.CoverageRepairIssue) {` 15+ `missingEq := cfg.RepairEquivalence && strings.TrimSpace(cfg.Existing.Equivalence.Prompt) == ""` 16+ `review := cfg.Existing.CodeReview` 17+ `if strings.TrimSpace(review.Prompt) == "" && cfg.Existing.Review != nil {` 18+ `review = *cfg.Existing.Review` 19+ `}` 20+ `missingReview := cfg.RepairCodeReview && strings.TrimSpace(review.Prompt) == ""` 21+ `if !missingEq && !missingReview {` 22+ `return nil, nil` 23+ `}` 24+ `var issues []validate.CoverageRepairIssue` 25+ `fail := func(dimension string, err error) {` 26+ `detail := "prompt reconstruction failed: " + err.Error()` 27+ `if len(detail) > 240 {` 28+ `detail = detail[:240]` 29+ `}` 30+ `issues = append(issues, validate.CoverageRepairIssue{Dimension: dimension, Reason: "missing_prompt", Detail: detail, Cause: err})` 31+ `if dimension == TaskGraderEquivalence {` 32+ `cfg.RepairEquivalence = false` 33+ `} else {` 34+ `cfg.RepairCodeReview = false` 35+ `}` 36+ `}` 37+ `mode := cfg.Existing.Equivalence.PromptMode` 38+ `if missingEq && mode != validate.EquivalencePromptModeStandard && mode != validate.EquivalencePromptModeGoldBlind {` 39+ `fail(TaskGraderEquivalence, fmt.Errorf("missing or unsupported equivalence prompt mode %q", mode))` 40+ `missingEq = false` 41+ `}` 42+ `if !missingEq && !missingReview {` 43+ `return nil, issues` 44+ `}` 45+ `truncation := cfg.Existing.PromptTruncation` 46+ `if truncation.AgentPatch.Truncated || truncation.GoldPatch.Truncated || truncation.RepoTree.Truncated || truncation.RepoExcerpts.Truncated || cfg.Existing.Equivalence.Truncated || review.Truncated {` 47+ `err := fmt.Errorf("original prompt was truncated; original context budgets are unavailable")` 48+ `if missingEq {` 49+ `fail(TaskGraderEquivalence, err)` 50+ `}` 51+ `if missingReview {` 52+ `fail(TaskGraderCodeReview, err)` 53+ `}` 54+ `return nil, issues` 55+ `}` 56+ `dataset := manifest.Dataset` 57+ `goldPath := strings.TrimSpace(cfg.Existing.ArtifactsManifest.GoldPatch.Path)` 58+ `if goldPath != "" {` 59+ `taskDir := filepath.Dir(goldPath)` 60+ `if filepath.Base(taskDir) != artifacts.TaskID {` 61+ `err := fmt.Errorf("gold patch directory does not match task %q", artifacts.TaskID)` 62+ `if missingEq {` 63+ `fail(TaskGraderEquivalence, err)` 64+ `}` 65+ `if missingReview {` 66+ `fail(TaskGraderCodeReview, err)` 67+ `}` 68+ `return nil, issues` 69+ `}` 70+ `dataset = filepath.Dir(taskDir)` 71+ `}` 72+ `if err := verifyCoverageRepairTaskAuthority(manifest, artifacts, dataset); err != nil {` 73+ `if missingEq {` 74+ `fail(TaskGraderEquivalence, err)` 75+ `}` 76+ `if missingReview {` 77+ `fail(TaskGraderCodeReview, err)` 78+ `}` 79+ `return nil, issues` 80+ `}` 81+ `resolved, _, _, err := resolveTaskArtifactsFromResults(artifacts)` 82+ `if err == nil {` 83+ `artifacts = resolved` 84+ `}` 85+ `patchPath := strings.TrimSpace(cfg.Existing.ArtifactsManifest.AgentPatch.Path)` 86+ `if patchPath == "" {` 87+ `patchPath = artifacts.ResolveAgentPatchPath()` 88+ `}` 89+ `promptCtx, cleanup, err := validate.BuildOfflinePromptContext(validate.OfflinePromptInputs{` 90+ `DatasetPath: dataset, TaskID: artifacts.TaskID, AgentPatchPath: patchPath,` 91+ `TBResultsRaw: validate.ReadTBResultsRaw(artifacts.ResultsPath(), artifacts.TaskID), RequireRepoContext: true,` 92+ `EffectiveTestCommands: append([]string{}, cfg.Existing.Tests.Commands...),` 93+ `ExpectedArtifacts: &cfg.Existing.ArtifactsManifest,` 94+ `RepoContextOptions: validate.RepoContextOptions{ExcludePrefixes: cfg.Existing.ContextExcludePaths},` 95+ `})` 96+ `if err != nil {` 97+ `if missingEq {` 98+ `fail(TaskGraderEquivalence, err)` 99+ `}` 100+ `if missingReview {` 101+ `fail(TaskGraderCodeReview, err)` 102+ `}` 103+ `return nil, issues` 104+ `}` 105+ `if strings.TrimSpace(promptCtx.AgentPatch) == "" || strings.TrimSpace(promptCtx.TaskYAML) == "" {` 106+ `err := fmt.Errorf("task YAML and agent patch are required for prompt reconstruction")` 107+ `if missingEq {` 108+ `fail(TaskGraderEquivalence, err)` 109+ `}` 110+ `if missingReview {` 111+ `fail(TaskGraderCodeReview, err)` 112+ `}` 113+ `return cleanup, issues` 114+ `}` 115+ `client := cfg.AI` 116+ `if shell, ok := client.(validate.ShellAIClient); ok {` 117+ `shell.Dir = promptCtx.RepoPath` 118+ `client = shell` 119+ `}` 120+ `if missingEq {` 121+ `prompt, err := validate.RenderEquivalencePromptForMode(promptCtx, mode)` 122+ `if err != nil {` 123+ `fail(TaskGraderEquivalence, err)` 124+ `} else {` 125+ `cfg.EquivalencePrompt, cfg.EquivalencePromptMode = prompt, mode` 126+ `cfg.EquivalencePromptContext, cfg.EquivalenceAI = &promptCtx, client` 127+ `}` 128+ `}` 129+ `if missingReview {` 130+ `prompt, err := validate.RenderReviewPrompt(promptCtx)` 131+ `if err != nil {` 132+ `fail(TaskGraderCodeReview, err)` 133+ `} else {` 134+ `cfg.ReviewPrompt, cfg.ReviewPromptContext, cfg.ReviewAI = prompt, &promptCtx, client` 135+ `}` 136+ `}` 137+ `return cleanup, issues` 138+`}` 139+ 140+`func verifyCoverageRepairTaskAuthority(manifest *Manifest, artifacts TaskArtifacts, dataset string) error {` 141+ `if manifest.TaskSelection == nil || manifest.TaskSelection.PromptProjectionReceipt == "" {` 142+ `return fmt.Errorf("missing original task prompt projection authority")` 143+ `}` 144+ `receipt, err := LoadTaskPromptProjectionReceipt(artifacts.OutputRoot)` 145+ `if err != nil {` 146+ `return err` 147+ `}` 148+ `if err := validateTaskSelectionProjection(manifest.TaskSelection, receipt); err != nil {` 149+ `return err` 150+ `}` 151+ `digest, err := digestTaskContentTree(filepath.Join(dataset, artifacts.TaskID))` 152+ `if err != nil {` 153+ `return err` 154+ `}` 155+ `for _, row := range receipt.Tasks {` 156+ `if row.TaskID == artifacts.TaskID {` 157+ `if row.TaskContentSHA256 != digest {` 158+ `return fmt.Errorf("original task content provenance mismatch for %q", artifacts.TaskID)` 159+ `}` 160+ `return nil` 161+ `}` 162+ `}` 163+ `return fmt.Errorf("missing original task content authority for %q", artifacts.TaskID)` 164+`}` … 1+`package h2h` 2+ 3+`import (` 4+ `"bytes"` 5+ `"context"` 6+ `"fmt"` 7+ `"os"` 8+ `"path/filepath"` 9+ `"strings"` 10+ `"testing"` 11+ `"time"` 12+ 13+ `"stet/internal/validate"` 14+`)` 15+ 16+`func TestPrepareCoverageRepairPromptsPreservesStoredPrompts(t *testing.T) {` 17+ `cfg := validate.CoverageRepairConfig{` 18+ `Existing: validate.ValidationResult{` 19+ `Equivalence: validate.EquivalenceResult{Prompt: "stored equivalence"},` 20+ `Review: &validate.ReviewResult{Prompt: "legacy stored review"},` 21+ `}, RepairEquivalence: true, RepairCodeReview: true,` 22+ `}` 23+ `cleanup, issues := prepareCoverageRepairPrompts(&cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"})` 24+ `if cleanup != nil || len(issues) != 0 || cfg.EquivalencePromptContext != nil || cfg.ReviewPromptContext != nil {` 25+ `t.Fatalf("stored prompts should require no reconstruction: %+v", cfg)` 26+ `}` 27+`}` 28+ 29+`func TestPrepareCoverageRepairPromptsRejectsUnknownModeWithoutBlockingStoredReview(t *testing.T) {` 30+ `cfg := validate.CoverageRepairConfig{` 31+ `Existing: validate.ValidationResult{CodeReview: validate.ReviewResult{Prompt: "stored review"}},` 32+ `RepairEquivalence: true, RepairCodeReview: true,` 33+ `}` 34+ `_, issues := prepareCoverageRepairPrompts(&cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"})` 35+ `if len(issues) != 1 || issues[0].Dimension != TaskGraderEquivalence || cfg.RepairEquivalence || !cfg.RepairCodeReview {` 36+ `t.Fatalf("unexpected dimension selection: %+v, issues %+v", cfg, issues)` 37+ `}` 38+`}` 39+ 40+`func TestPrepareCoverageRepairPromptsRejectsUnknownOriginalTruncationBudget(t *testing.T) {` 41+ `cfg := validate.CoverageRepairConfig{RepairCodeReview: true}` 42+ `cfg.Existing.PromptTruncation.AgentPatch.Truncated = true` 43+ `_, issues := prepareCoverageRepairPrompts(&cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"})` 44+ `if len(issues) != 1 || cfg.RepairCodeReview || !strings.Contains(issues[0].Detail, "original context budgets") {` 45+ `t.Fatalf("unknown original truncation must prevent reconstruction: %+v", issues)` 46+ `}` 47+`}` 48+ 49+`func TestPrepareCoverageRepairPromptsReconstructsPersistedModeAndVerifierContext(t *testing.T) {` 50+ `t.Parallel()` 51+ `root := t.TempDir()` 52+ `taskDir := filepath.Join(root, "dataset", "task-a")` 53+ `mustWriteFile(t, filepath.Join(taskDir, "task.yaml"), "id: task-a\ninstruction: fix greeting\n")` 54+ `mustWriteFile(t, filepath.Join(taskDir, "gold.patch"), "gold reference secret\n")` 55+ `mustWriteFile(t, filepath.Join(taskDir, "tests", "test_outputs.py"), "COMMANDS = [\"dataset test\"]\n")` 56+ `patch := filepath.Join(root, "agent.patch")` 57+ `mustWriteFile(t, patch, "diff --git a/hello.txt b/hello.txt\n+hello\n")` 58+ `writeAnswerSimilarityTar(t, filepath.Join(taskDir, "repo.tar.gz"), map[string]string{"hello.txt": "old\n"}, nil)` 59+ `existing := validate.ValidationResult{` 60+ `Equivalence: validate.EquivalenceResult{PromptMode: validate.EquivalencePromptModeGoldBlind},` 61+ `Tests: validate.TestResult{Commands: []string{"effective verifier"}},` 62+ `}` 63+ `inputs, err := validate.ReadTaskInputs(filepath.Dir(taskDir), "task-a", patch)` 64+ `if err != nil {` 65+ `t.Fatal(err)` 66+ `}` 67+ `existing.ArtifactsManifest = validate.BuildArtifactManifest(inputs)` 68+ `manifest := sealCoverageRepairTestTask(t, root, filepath.Dir(taskDir))` 69+ `cfg := validate.CoverageRepairConfig{Existing: existing, AI: validate.ShellAIClient{}, RepairEquivalence: true, RepairCodeReview: true}` 70+ `cleanup, issues := prepareCoverageRepairPrompts(&cfg, manifest, NewTaskArtifacts(root, VariantSpec{RunID: "run-a", ModelKey: "model-a"}, "task-a", ""))` 71+ `if cleanup != nil {` 72+ `defer cleanup()` 73+ `}` 74+ `if len(issues) != 0 {` 75+ `t.Fatalf("reconstruction failed: %+v", issues)` 76+ `}` 77+ `if cfg.EquivalencePromptMode != validate.EquivalencePromptModeGoldBlind || strings.Contains(cfg.EquivalencePrompt, "gold reference secret") {` 78+ `t.Fatalf("gold-blind provenance was lost: %s", cfg.EquivalencePrompt)` 79+ `}` 80+ `if !strings.Contains(cfg.ReviewPrompt, "effective verifier") || strings.Contains(cfg.ReviewPrompt, "dataset test") {` 81+ `t.Fatalf("effective verifier context was lost: %s", cfg.ReviewPrompt)` 82+ `}` 83+ `for _, client := range []validate.AIClient{cfg.EquivalenceAI, cfg.ReviewAI} {` 84+ `shell := client.(validate.ShellAIClient)` 85+ `if shell.Dir == "" || shell.Dir != cfg.ReviewPromptContext.RepoPath {` 86+ `t.Fatalf("missing grading repository: %+v", shell)` 87+ `}` 88+ `if _, err := os.Stat(filepath.Join(shell.Dir, "hello.txt")); err != nil {` 89+ `t.Fatal(err)` 90+ `}` 91+ `}` 92+`}` 93+ 94+`func TestPrepareCoverageRepairPromptsRejectsUnavailableRepository(t *testing.T) {` 95+ `root := t.TempDir()` 96+ `taskDir := filepath.Join(root, "dataset", "task-a")` 97+ `mustWriteFile(t, filepath.Join(taskDir, "task.yaml"), "id: task-a\n")` 98+ `mustWriteFile(t, filepath.Join(taskDir, "gold.patch"), "gold\n")` 99+ `mustWriteFile(t, filepath.Join(taskDir, "repo.tar.gz"), "invalid gzip")` 100+ `patch := filepath.Join(root, "agent.patch")` 101+ `mustWriteFile(t, patch, "agent\n")` 102+ `manifest := sealCoverageRepairTestTask(t, root, filepath.Dir(taskDir))` 103+ `cfg := validate.CoverageRepairConfig{RepairCodeReview: true}` 104+ `inputs, err := validate.ReadTaskInputs(filepath.Dir(taskDir), "task-a", patch)` 105+ `if err != nil {` 106+ `t.Fatal(err)` 107+ `}` 108+ `cfg.Existing.ArtifactsManifest = validate.BuildArtifactManifest(inputs)` 109+ `cleanup, issues := prepareCoverageRepairPrompts(&cfg, manifest, NewTaskArtifacts(root, VariantSpec{RunID: "run-a"}, "task-a", ""))` 110+ `if cleanup != nil {` 111+ `defer cleanup()` 112+ `}` 113+ `if len(issues) != 1 || cfg.RepairCodeReview || cfg.ReviewPrompt != "" || !strings.Contains(issues[0].Detail, "required repo context") {` 114+ `t.Fatalf("repository failure should prevent reconstructed grade: %+v, issues %+v", cfg, issues)` 115+ `}` 116+`}` 117+ 118+`func sealCoverageRepairTestTask(t *testing.T, root, dataset string) *Manifest {` 119+ `t.Helper()` 120+ `selection := &TaskSelectionRecord{RealizedTaskIDs: []string{"task-a"}}` 121+ `receipt, err := BuildTaskPromptProjectionReceipt(dataset, root, selection, nil, "dataset", map[string]string{"task-a": "fix greeting"}, time.Time{})` 122+ `if err != nil {` 123+ `t.Fatal(err)` 124+ `}` 125+ `if _, err := WriteTaskPromptProjectionReceipt(root, receipt); err != nil {` 126+ `t.Fatal(err)` 127+ `}` 128+ `return &Manifest{Dataset: dataset, TaskSelection: ApplyTaskPromptProjectionReceipt(selection, receipt)}` 129+`}` 130+ 131+`func TestPrepareCoverageRepairPromptsRejectsMutatedTaskYAML(t *testing.T) {` 132+ `root := t.TempDir()` 133+ `dataset := filepath.Join(root, "dataset")` 134+ `mustWriteFile(t, filepath.Join(dataset, "task-a", "task.yaml"), "id: task-a\ninstruction: original\n")` 135+ `manifest := sealCoverageRepairTestTask(t, root, dataset)` 136+ `mustWriteFile(t, filepath.Join(dataset, "task-a", "task.yaml"), "id: task-a\ninstruction: mutated\n")` 137+ `cfg := validate.CoverageRepairConfig{RepairCodeReview: true}` 138+ `_, issues := prepareCoverageRepairPrompts(&cfg, manifest, NewTaskArtifacts(root, VariantSpec{RunID: "run-a"}, "task-a", ""))` 139+ `if len(issues) != 1 || cfg.RepairCodeReview || !strings.Contains(issues[0].Detail, "task content provenance mismatch") {` 140+ `t.Fatalf("changed task must not reconstruct: %+v", issues)` 141+ `}` 142+`}` 143+ 144+`func TestRunRepairAICoverageReconstructsBoundSourceAtSubprocessBoundary(t *testing.T) {` 145+ `for _, mutate := range []bool{false, true} {` 146+ `t.Run(fmt.Sprintf("mutate_%t", mutate), func(t *testing.T) {` 147+ `root := t.TempDir()` 148+ `dataset := filepath.Join(root, "dataset")` 149+ `taskDir := filepath.Join(dataset, "task-a")` 150+ `mustWriteFile(t, filepath.Join(taskDir, "task.yaml"), "id: task-a\ninstruction: fix greeting\n")` 151+ `mustWriteFile(t, filepath.Join(taskDir, "gold.patch"), "gold reference secret\n")` 152+ `writeAnswerSimilarityTar(t, filepath.Join(taskDir, "repo.tar.gz"), map[string]string{"hello.txt": "old\n"}, nil)` 153+ `patch := filepath.Join(root, "agent.patch")` 154+ `mustWriteFile(t, patch, "diff --git a/hello.txt b/hello.txt\n--- a/hello.txt\n+++ b/hello.txt\n@@ -1 +1 @@\n-old\n+hello\n")` 155+ `manifest := sealCoverageRepairTestTask(t, root, dataset)` 156+ `manifest.Version, manifest.OutputRoot = ManifestVersion, root` 157+ `manifest.Models = []string{"model/a"}` 158+ `manifest.ModelKeys, manifest.RunIDs = map[string]string{"model/a": "model-a"}, map[string]string{"model/a": "run-a"}` 159+ `if err := SaveManifest(filepath.Join(root, "manifest.json"), manifest); err != nil {` 160+ `t.Fatal(err)` 161+ `}` 162+ `inputs, err := validate.ReadTaskInputs(dataset, "task-a", patch)` 163+ `if err != nil {` 164+ `t.Fatal(err)` 165+ `}` 166+ `original := validate.ValidationResult{` 167+ `TaskID: "task-a", PrimarySignal: validate.PrimarySignalTests, Status: validate.StatusPass, MatrixStatus: validate.MatrixPass,` 168+ `Guardrails: validate.GuardrailResult{Status: validate.StatusPass},` 169+ `Tests: validate.TestResult{Outcome: validate.TestOutcomePass, Strength: validate.TestStrengthStrong, Commands: []string{"effective verifier"}},` 170+ `Equivalence: validate.EquivalenceResult{Status: validate.AIStatusUnavailable, Outcome: validate.EquivalenceUnknown, PromptMode: validate.EquivalencePromptModeGoldBlind},` 171+ `CodeReview: validate.ReviewResult{Status: validate.AIStatusUnavailable, Signal: validate.ReviewSignalUnsure},` 172+ `ArtifactsManifest: validate.BuildArtifactManifest(inputs),` 173+ `}` 174+ `validationRoot := filepath.Join(root, "validation", "model-a")` 175+ `mustWriteRepairValidation(t, validationRoot, "task-a", original)` 176+ `if mutate {` 177+ `mustWriteFile(t, patch, "mutated agent evidence\n")` 178+ `}` 179+ `log := filepath.Join(root, "calls")` 180+ `script := filepath.Join(root, "judge.py")` 181+ ``mustWriteFile(t, script, `import json, pathlib, sys`` 182+`prompt = sys.stdin.read()` 183+`assert pathlib.Path('hello.txt').read_text() == 'old\n'` 184+`with open(sys.argv[1], 'a') as log: log.write('call\n')` 185+`if 'You are reviewing' in prompt:` 186+ `assert 'effective verifier' in prompt` 187+ `scores = {name: {'score_0_4': 4, 'confidence_0_1': .95, 'rationale': 'ok', 'citations': [{'path':'hello.txt','side':'new','start_line':1,'end_line':1,'anchor_kind':'diff','head_sha':'abc1234'}]} for name in ['correctness','introduced_bug_risk','edge_case_handling','maintainability_idioms']}` 188+ `print(json.dumps({'schema_version':'stet.review.v1','dimension_scores':scores,'findings':[],'overall':{'confidence_overall':.95,'summary':'repair passed'}}))` 189+`else:` 190+ `assert 'gold reference secret' not in prompt` 191+ `print('obligations:\n- fix greeting => MET\n\nequivalent: YES\nconfidence: 0.95\nissue_type: stylistic\nnotes: repaired')` 192+`` `) `` 193+ `var stdout, stderr bytes.Buffer` 194+ `err = RunRepairAICoverage(context.Background(), []string{"--out", root, "--model-key", "model-a", "--ai-cmd", "python3 " + shellQuote(script) + " " + shellQuote(log), "--ai-model-id", "judge-a"}, &stdout, &stderr)` 195+ `if mutate {` 196+ `if err == nil {` 197+ `t.Fatal("mutated artifact repair must fail")` 198+ `}` 199+ `if _, err := os.Stat(log); !os.IsNotExist(err) {` 200+ `t.Fatalf("mutated source invoked grader: %v", err)` 201+ `}` 202+ `if !strings.Contains(stdout.String(), "agent patch provenance mismatch") {` 203+ `t.Fatalf("missing mutation reason: %s", stdout.String())` 204+ `}` 205+ `return` 206+ `}` 207+ `if err != nil {` 208+ `t.Fatalf("repair failed: %v\nstdout %s\nstderr %s", err, stdout.String(), stderr.String())` 209+ `}` 210+ `updated := mustReadRepairValidation(t, filepath.Join(validationRoot, "task-a", "validation.json"))` 211+ `if updated.Equivalence.PromptMode != validate.EquivalencePromptModeGoldBlind || updated.Equivalence.Status != validate.AIStatusUsed || updated.CodeReview.Status != validate.AIStatusUsed || updated.CodeReview.AttemptCount != 1 {` 212+ `t.Fatalf("repair lost mode or attempt metadata: %+v", updated)` 213+ `}` 214+ `if updated.ArtifactsManifest.AgentPatch.SHA256 != original.ArtifactsManifest.AgentPatch.SHA256 {` 215+ `t.Fatal("repair changed original source authority")` 216+ `}` 217+ `calls, err := os.ReadFile(log)` 218+ `if err != nil || strings.Count(string(calls), "call") != 2 {` 219+ `t.Fatalf("unexpected calls: %s, %v", calls, err)` 220+ `}` 221+ `})` 222+ `}` 223+`}` … 1+`package validate` 2+ 3+`import (` 4+ `"context"` 5+ `"errors"` 6+ `"net"` 7+ `"strings"` 8+ `"time"` 9+ 10+ `"stet/internal/runtimefailure"` 11+`)` 12+ 13+`// gradeAttemptClient retains every dispatch, including runtime and parse retries.` 14+`// One instance belongs to one measurement dimension and must not be shared.` 15+`type gradeAttemptClient struct {` 16+ `ai AIClient` 17+ `attempts int` 18+ `reasons []string` 19+ `previousSucceeded bool` 20+`}` 21+ 22+`func newGradeAttemptClient(ai AIClient) *gradeAttemptClient {` 23+ `if ai == nil {` 24+ `ai = NoopAIClient{}` 25+ `}` 26+ `return &gradeAttemptClient{ai: ai}` 27+`}` 28+ 29+`func (a *gradeAttemptClient) Run(ctx context.Context, prompt string) (string, error) {` 30+ `if a.previousSucceeded {` 31+ `a.reasons = append(a.reasons, "model_output_shape")` 32+ `}` 33+ `a.previousSucceeded = false` 34+ `for attempt := 0; ; attempt++ {` 35+ `if err := ctx.Err(); err != nil {` 36+ `return "", err` 37+ `}` 38+ `a.attempts++` 39+ `output, err := a.ai.Run(ctx, prompt)` 40+ `if err == nil {` 41+ `a.previousSucceeded = true` 42+ `return output, nil` 43+ `}` 44+ `reason := recoverableGradeFailure(err)` 45+ `if reason == "" || attempt >= len(coverageRepairRetryBackoffs) {` 46+ `return "", err` 47+ `}` 48+ `select {` 49+ `case <-ctx.Done():` 50+ `return "", ctx.Err()` 51+ `case <-time.After(coverageRepairRetryBackoffs[attempt]):` 52+ `}` 53+ `a.reasons = append(a.re`