feat: require validation failure history in review reports (Closes #396)

Reviewer runs that observe a validation failure and later rerun to green must report the failure and its investigation; a later pass never erases the earlier failure: - New reviewer_validation_failure_history.py verifier: reports must carry a Validation failure history section naming every observed failure with command, failing test or error, suspected cause, reproduced status, baseline master status, and PR-caused evidence; rerun-pass claims must explain what changed between runs, whether environmental state (such as stale /tmp files) was cleaned, and why the pass is trustworthy; an unexplained transient failure cannot be reported as a bare pass and must use the status 'passed after transient failure investigation' (exported as TRANSIENT_INVESTIGATION_STATUS). - Re-export from review_proofs and wire into build_final_report as a downgrade check. - Review-merge final-report schema gains the Validation failure history template section; contract test locks the markers and an export test locks the re-export. - 12 tests covering the issue's scenarios: failing-then-passing rerun, baseline-equivalent failure, /tmp state contamination, unexplained transient failure, and a report omitting an earlier failure. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
2026-07-07 11:26:40 -04:00
parent fcb9944378
commit f20e0fc338
5 changed files with 367 additions and 1 deletions
@@ -0,0 +1,109 @@
+"""Validation failure history verifier for reviewer reports (#396).
+
+A reviewer run that observes a validation failure and later reruns to green
+must report the failure, its investigation, and why the eventual pass is
+trustworthy. A later pass never erases the duty to report the earlier
+failure. This verifier fails closed when known failures are omitted, when
+failure entries lack the required investigation fields, when a rerun-pass
+claim lacks a what-changed explanation, or when an unexplained transient
+failure is reported as a bare pass.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+# Issue #396 criterion 4: required status wording when the cause is unknown.
+TRANSIENT_INVESTIGATION_STATUS = "passed after transient failure investigation"
+
+_HISTORY_SECTION_RE = re.compile(
+    r"(?:^#+\s*validation failure history\b|validation failure history\s*:)",
+    re.IGNORECASE | re.MULTILINE,
+)
+_FAILURE_ENTRY_FIELDS = (
+    ("command", re.compile(r"command\s*:", re.IGNORECASE)),
+    ("failing test or error", re.compile(r"(?:failing test|error)\s*:", re.IGNORECASE)),
+    ("suspected cause", re.compile(r"suspected cause\s*:", re.IGNORECASE)),
+    ("reproduced status", re.compile(r"reproduced\s*:", re.IGNORECASE)),
+    ("baseline master status", re.compile(r"baseline(?:\s+master)?\s*:", re.IGNORECASE)),
+    ("PR-caused evidence", re.compile(r"(?:pr-caused|not pr-caused|evidence)", re.IGNORECASE)),
+)
+_RERUN_PASS_RE = re.compile(
+    r"(?:passed on rerun|rerun[^.\n]*passed|later (?:run|rerun) passed)",
+    re.IGNORECASE,
+)
+_WHAT_CHANGED_RE = re.compile(r"what changed", re.IGNORECASE)
+_ENV_STATE_RE = re.compile(
+    r"(?:environmental state|/tmp/|state (?:was )?cleaned|none cleaned)",
+    re.IGNORECASE,
+)
+_TRUST_RE = re.compile(r"(?:trustworthy|trust the pass|why the pass)", re.IGNORECASE)
+_UNKNOWN_CAUSE_RE = re.compile(r"suspected cause\s*:\s*unknown", re.IGNORECASE)
+_BARE_PASSED_STATUS_RE = re.compile(
+    r"validation status\s*:\s*passed\s*$",
+    re.IGNORECASE | re.MULTILINE,
+)
+
+
+def assess_validation_failure_history_report(
+    report_text: str,
+    *,
+    observed_failures: list[dict] | None = None,
+) -> dict[str, Any]:
+    """Require complete failure-history reporting for observed failures (#396)."""
+    text = report_text or ""
+    reasons: list[str] = []
+    has_history = bool(_HISTORY_SECTION_RE.search(text))
+
+    if observed_failures and not has_history:
+        reasons.append(
+            "validation failures were observed this session but the report has "
+            "no Validation failure history section"
+        )
+
+    if observed_failures and has_history:
+        for entry in observed_failures:
+            failure = str(entry.get("failure") or "").strip()
+            if failure and failure not in text:
+                reasons.append(
+                    f"observed validation failure omitted from history: {failure}"
+                )
+
+        missing_fields = [
+            name for name, pattern in _FAILURE_ENTRY_FIELDS if not pattern.search(text)
+        ]
+        for name in missing_fields:
+            reasons.append(f"failure history entry missing required field: {name}")
+
+    if has_history and _RERUN_PASS_RE.search(text):
+        if not _WHAT_CHANGED_RE.search(text):
+            reasons.append(
+                "rerun-pass claim does not explain what changed between runs"
+            )
+        if not _ENV_STATE_RE.search(text):
+            reasons.append(
+                "rerun-pass claim does not state whether environmental state "
+                "was cleaned"
+            )
+        if not _TRUST_RE.search(text):
+            reasons.append(
+                "rerun-pass claim does not explain why the pass is trustworthy"
+            )
+
+    if (
+        observed_failures
+        and _UNKNOWN_CAUSE_RE.search(text)
+        and _BARE_PASSED_STATUS_RE.search(text)
+    ):
+        reasons.append(
+            "unexplained transient failure cannot be reported as a bare pass; "
+            f"use '{TRANSIENT_INVESTIGATION_STATUS}'"
+        )
+
+    return {
+        "proven": not reasons,
+        "block": bool(reasons),
+        "reasons": reasons,
+        "history_section_present": has_history,
+    }