feat: require validation failure history in review reports (Closes #396)
Reviewer runs that observe a validation failure and later rerun to green must report the failure and its investigation; a later pass never erases the earlier failure: - New reviewer_validation_failure_history.py verifier: reports must carry a Validation failure history section naming every observed failure with command, failing test or error, suspected cause, reproduced status, baseline master status, and PR-caused evidence; rerun-pass claims must explain what changed between runs, whether environmental state (such as stale /tmp files) was cleaned, and why the pass is trustworthy; an unexplained transient failure cannot be reported as a bare pass and must use the status 'passed after transient failure investigation' (exported as TRANSIENT_INVESTIGATION_STATUS). - Re-export from review_proofs and wire into build_final_report as a downgrade check. - Review-merge final-report schema gains the Validation failure history template section; contract test locks the markers and an export test locks the re-export. - 12 tests covering the issue's scenarios: failing-then-passing rerun, baseline-equivalent failure, /tmp state contamination, unexplained transient failure, and a report omitting an earlier failure. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
@@ -1735,6 +1735,11 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
|
||||
if report_text
|
||||
else {"proven": True, "block": False, "reasons": []}
|
||||
)
|
||||
validation_failure_history = (
|
||||
assess_validation_failure_history_report(report_text)
|
||||
if report_text
|
||||
else {"proven": True, "block": False, "reasons": []}
|
||||
)
|
||||
if baseline_validation is None and report_text:
|
||||
baseline_validation = assess_reviewer_baseline_validation_proof(
|
||||
report_text=report_text,
|
||||
@@ -1932,6 +1937,11 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
|
||||
"inventory pagination or worktree fields inconsistent (#293)"
|
||||
)
|
||||
downgrade_reasons.extend(inventory_worktree.get("reasons", []))
|
||||
if not validation_failure_history.get("proven"):
|
||||
downgrade_reasons.append(
|
||||
"validation failure history incomplete or omitted (#396)"
|
||||
)
|
||||
downgrade_reasons.extend(validation_failure_history.get("reasons", []))
|
||||
if not baseline_validation.get("proven"):
|
||||
downgrade_reasons.append(
|
||||
"reviewer baseline validation proof missing or failed (#325)"
|
||||
@@ -5425,6 +5435,15 @@ def assess_inventory_worktree_report(report_text, **kwargs):
|
||||
return _assess(report_text, **kwargs)
|
||||
|
||||
|
||||
def assess_validation_failure_history_report(report_text, **kwargs):
|
||||
"""#396: observed validation failures must be reported with investigation."""
|
||||
from reviewer_validation_failure_history import (
|
||||
assess_validation_failure_history_report as _assess,
|
||||
)
|
||||
|
||||
return _assess(report_text, **kwargs)
|
||||
|
||||
|
||||
def assess_infra_stop_handoff_report(report_text, **kwargs):
|
||||
"""#289: repair handoff purity when infra_stop blocks reviewer capability."""
|
||||
from reviewer_infra_stop_handoff import assess_infra_stop_handoff_report as _assess
|
||||
|
||||
Reference in New Issue
Block a user