"""Validation failure history verifier for reviewer reports (#396). A reviewer run that observes a validation failure and later reruns to green must report the failure, its investigation, and why the eventual pass is trustworthy. A later pass never erases the duty to report the earlier failure. This verifier fails closed when known failures are omitted, when failure entries lack the required investigation fields, when a rerun-pass claim lacks a what-changed explanation, or when an unexplained transient failure is reported as a bare pass. """ from __future__ import annotations import re from typing import Any # Issue #396 criterion 4: required status wording when the cause is unknown. TRANSIENT_INVESTIGATION_STATUS = "passed after transient failure investigation" _HISTORY_SECTION_RE = re.compile( r"(?:^#+\s*validation failure history\b|validation failure history\s*:)", re.IGNORECASE | re.MULTILINE, ) _FAILURE_ENTRY_FIELDS = ( ("command", re.compile(r"command\s*:", re.IGNORECASE)), ("failing test or error", re.compile(r"(?:failing test|error)\s*:", re.IGNORECASE)), ("suspected cause", re.compile(r"suspected cause\s*:", re.IGNORECASE)), ("reproduced status", re.compile(r"reproduced\s*:", re.IGNORECASE)), ("baseline master status", re.compile(r"baseline(?:\s+master)?\s*:", re.IGNORECASE)), ("PR-caused evidence", re.compile(r"(?:pr-caused|not pr-caused|evidence)", re.IGNORECASE)), ) _RERUN_PASS_RE = re.compile( r"(?:passed on rerun|rerun[^.\n]*passed|later (?:run|rerun) passed)", re.IGNORECASE, ) _WHAT_CHANGED_RE = re.compile(r"what changed", re.IGNORECASE) _ENV_STATE_RE = re.compile( r"(?:environmental state|/tmp/|state (?:was )?cleaned|none cleaned)", re.IGNORECASE, ) _TRUST_RE = re.compile(r"(?:trustworthy|trust the pass|why the pass)", re.IGNORECASE) _UNKNOWN_CAUSE_RE = re.compile(r"suspected cause\s*:\s*unknown", re.IGNORECASE) _BARE_PASSED_STATUS_RE = re.compile( r"validation status\s*:\s*passed\s*$", re.IGNORECASE | re.MULTILINE, ) def assess_validation_failure_history_report( report_text: str, *, observed_failures: list[dict] | None = None, ) -> dict[str, Any]: """Require complete failure-history reporting for observed failures (#396).""" text = report_text or "" reasons: list[str] = [] has_history = bool(_HISTORY_SECTION_RE.search(text)) if observed_failures and not has_history: reasons.append( "validation failures were observed this session but the report has " "no Validation failure history section" ) if observed_failures and has_history: for entry in observed_failures: failure = str(entry.get("failure") or "").strip() if failure and failure not in text: reasons.append( f"observed validation failure omitted from history: {failure}" ) missing_fields = [ name for name, pattern in _FAILURE_ENTRY_FIELDS if not pattern.search(text) ] for name in missing_fields: reasons.append(f"failure history entry missing required field: {name}") if has_history and _RERUN_PASS_RE.search(text): if not _WHAT_CHANGED_RE.search(text): reasons.append( "rerun-pass claim does not explain what changed between runs" ) if not _ENV_STATE_RE.search(text): reasons.append( "rerun-pass claim does not state whether environmental state " "was cleaned" ) if not _TRUST_RE.search(text): reasons.append( "rerun-pass claim does not explain why the pass is trustworthy" ) if ( observed_failures and _UNKNOWN_CAUSE_RE.search(text) and _BARE_PASSED_STATUS_RE.search(text) ): reasons.append( "unexplained transient failure cannot be reported as a bare pass; " f"use '{TRANSIENT_INVESTIGATION_STATUS}'" ) return { "proven": not reasons, "block": bool(reasons), "reasons": reasons, "history_section_present": has_history, }