feat: require validation failure history in review reports (Closes #396)

Reviewer runs that observe a validation failure and later rerun to green must report the failure and its investigation; a later pass never erases the earlier failure: - New reviewer_validation_failure_history.py verifier: reports must carry a Validation failure history section naming every observed failure with command, failing test or error, suspected cause, reproduced status, baseline master status, and PR-caused evidence; rerun-pass claims must explain what changed between runs, whether environmental state (such as stale /tmp files) was cleaned, and why the pass is trustworthy; an unexplained transient failure cannot be reported as a bare pass and must use the status 'passed after transient failure investigation' (exported as TRANSIENT_INVESTIGATION_STATUS). - Re-export from review_proofs and wire into build_final_report as a downgrade check. - Review-merge final-report schema gains the Validation failure history template section; contract test locks the markers and an export test locks the re-export. - 12 tests covering the issue's scenarios: failing-then-passing rerun, baseline-equivalent failure, /tmp state contamination, unexplained transient failure, and a report omitting an earlier failure. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
2026-07-07 11:26:40 -04:00
parent fcb9944378
commit f20e0fc338
5 changed files with 367 additions and 1 deletions
@@ -74,6 +74,14 @@ def test_review_merge_final_report_schema_exists():
    assert "Candidate head SHA:" in text
    assert "Terminal review mutation:" in text
    assert "Workspace mutations" in text
+    assert "### Validation failure history (#396)" in text
+    assert "passed after transient failure investigation" in text
+
+
+def test_validation_failure_history_verifier_exported():
+    from review_proofs import assess_validation_failure_history_report
+
+    assert callable(assess_validation_failure_history_report)


 def test_reconcile_landed_workflow_contract():
@@ -0,0 +1,207 @@
+"""Tests for validation failure history reporting (#396)."""
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from reviewer_validation_failure_history import (  # noqa: E402
+    TRANSIENT_INVESTIGATION_STATUS,
+    assess_validation_failure_history_report,
+)
+
+_OBSERVED = [
+    {
+        "command": "venv/bin/pytest tests/ -q",
+        "failure": "tests/test_worktrees.py::TestWorktreeStart::test_rejects_untraceable_branches",
+    }
+]
+
+
+def _history_block(**overrides):
+    lines = {
+        "heading": "## Validation failure history",
+        "command": "- Command: venv/bin/pytest tests/ -q in branches/review-pr381",
+        "failure": (
+            "- Failing test: tests/test_worktrees.py::TestWorktreeStart::"
+            "test_rejects_untraceable_branches"
+        ),
+        "cause": "- Suspected cause: stale /tmp/gitea_issue_lock.json from a concurrent session",
+        "reproduced": "- Reproduced: no (single occurrence, not reproducible in isolation)",
+        "baseline": "- Baseline master: does not fail on clean baseline",
+        "evidence": (
+            "- PR-caused evidence: not PR-caused; failure touches issue-lock state "
+            "outside the PR diff"
+        ),
+        "rerun": (
+            "- Rerun: passed on rerun; what changed between runs: stale "
+            "/tmp/gitea_issue_lock.json environmental state was cleaned; the pass is "
+            "trustworthy because the failing test passes in isolation and in the full "
+            "suite after cleanup"
+        ),
+    }
+    lines.update(overrides)
+    return "\n".join(v for v in lines.values() if v)
+
+
+class TestOmittedFailures(unittest.TestCase):
+    def test_report_omitting_earlier_failure_fails(self):
+        report = "## Final Report\n- Validation: passed (1497 passed, 6 skipped)\n"
+        result = assess_validation_failure_history_report(
+            report, observed_failures=_OBSERVED
+        )
+        self.assertFalse(result["proven"])
+        self.assertTrue(
+            any("history" in r.lower() for r in result["reasons"]),
+            result["reasons"],
+        )
+
+    def test_history_section_missing_observed_test_fails(self):
+        report = (
+            "## Validation failure history\n"
+            "- Command: pytest tests/test_other.py\n"
+            "- Failing test: tests/test_other.py::test_something_else\n"
+        )
+        result = assess_validation_failure_history_report(
+            report, observed_failures=_OBSERVED
+        )
+        self.assertFalse(result["proven"])
+        self.assertTrue(
+            any("test_rejects_untraceable_branches" in r for r in result["reasons"]),
+            result["reasons"],
+        )
+
+    def test_no_observed_failures_and_no_claims_passes(self):
+        report = "## Final Report\n- Validation: passed (1501 passed)\n"
+        result = assess_validation_failure_history_report(report)
+        self.assertTrue(result["proven"], result["reasons"])
+
+
+class TestFailureEntryFields(unittest.TestCase):
+    def test_full_history_entry_passes(self):
+        report = _history_block() + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS
+        result = assess_validation_failure_history_report(
+            report, observed_failures=_OBSERVED
+        )
+        self.assertTrue(result["proven"], result["reasons"])
+
+    def test_missing_suspected_cause_fails(self):
+        report = _history_block(cause="") + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS
+        result = assess_validation_failure_history_report(
+            report, observed_failures=_OBSERVED
+        )
+        self.assertFalse(result["proven"])
+        self.assertTrue(
+            any("suspected cause" in r.lower() for r in result["reasons"]),
+            result["reasons"],
+        )
+
+    def test_missing_baseline_status_fails(self):
+        report = _history_block(baseline="") + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS
+        result = assess_validation_failure_history_report(
+            report, observed_failures=_OBSERVED
+        )
+        self.assertFalse(result["proven"])
+        self.assertTrue(
+            any("baseline" in r.lower() for r in result["reasons"]),
+            result["reasons"],
+        )
+
+    def test_baseline_equivalent_failure_passes(self):
+        report = (
+            _history_block(
+                baseline="- Baseline master: fails identically on clean baseline worktree",
+                evidence=(
+                    "- PR-caused evidence: not PR-caused; identical failure signature "
+                    "on baseline master"
+                ),
+                rerun="- Rerun: not rerun; baseline-equivalent failure documented",
+            )
+            + "\n\nValidation status: failed (baseline-equivalent, not PR-caused)"
+        )
+        result = assess_validation_failure_history_report(
+            report, observed_failures=_OBSERVED
+        )
+        self.assertTrue(result["proven"], result["reasons"])
+
+
+class TestRerunPassClaims(unittest.TestCase):
+    def test_rerun_pass_without_what_changed_fails(self):
+        report = (
+            _history_block(rerun="- Rerun: passed on rerun")
+            + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS
+        )
+        result = assess_validation_failure_history_report(
+            report, observed_failures=_OBSERVED
+        )
+        self.assertFalse(result["proven"])
+        self.assertTrue(
+            any("what changed" in r.lower() for r in result["reasons"]),
+            result["reasons"],
+        )
+
+    def test_tmp_state_contamination_documented_passes(self):
+        result = assess_validation_failure_history_report(
+            _history_block() + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS,
+            observed_failures=_OBSERVED,
+        )
+        self.assertTrue(result["proven"], result["reasons"])
+
+
+class TestStatusVocabulary(unittest.TestCase):
+    def test_unexplained_transient_with_bare_passed_fails(self):
+        report = (
+            _history_block(
+                cause="- Suspected cause: unknown",
+                rerun=(
+                    "- Rerun: passed on rerun; what changed between runs: nothing "
+                    "identified; environmental state: none cleaned; the pass is "
+                    "trustworthy because three consecutive reruns pass"
+                ),
+            )
+            + "\n\nValidation status: passed"
+        )
+        result = assess_validation_failure_history_report(
+            report, observed_failures=_OBSERVED
+        )
+        self.assertFalse(result["proven"])
+        self.assertTrue(
+            any(TRANSIENT_INVESTIGATION_STATUS in r for r in result["reasons"]),
+            result["reasons"],
+        )
+
+    def test_unexplained_transient_with_investigation_status_passes(self):
+        report = (
+            _history_block(
+                cause="- Suspected cause: unknown",
+                rerun=(
+                    "- Rerun: passed on rerun; what changed between runs: nothing "
+                    "identified; environmental state: none cleaned; the pass is "
+                    "trustworthy because three consecutive reruns pass"
+                ),
+            )
+            + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS
+        )
+        result = assess_validation_failure_history_report(
+            report, observed_failures=_OBSERVED
+        )
+        self.assertTrue(result["proven"], result["reasons"])
+
+
+class TestExports(unittest.TestCase):
+    def test_review_proofs_reexport(self):
+        from review_proofs import (
+            assess_validation_failure_history_report as exported,
+        )
+
+        self.assertTrue(callable(exported))
+
+    def test_status_constant_value(self):
+        self.assertEqual(
+            TRANSIENT_INVESTIGATION_STATUS,
+            "passed after transient failure investigation",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()