"""Transient validation failure history verifier (#396). Reviewer sessions may observe validation failures that later pass on rerun. Final reports must document every failure observed during the session, not only the last passing result. """ from __future__ import annotations import re from typing import Any _SECTION_RE = re.compile( r"validation failure history", re.IGNORECASE, ) _FAILURE_ENTRY_RE = re.compile( r"(?:failure\s*(?:#|entry)?\s*\d+|validation failure)\s*:", re.IGNORECASE, ) _COMMAND_RE = re.compile( r"(?:command|validation command)\s*:\s*(.+)", re.IGNORECASE, ) _FAILING_TEST_RE = re.compile( r"(?:failing test|failure|error)\s*:\s*(.+)", re.IGNORECASE, ) _CAUSE_RE = re.compile( r"(?:suspected cause|cause)\s*:\s*(.+)", re.IGNORECASE, ) _REPRODUCED_RE = re.compile( r"(?:reproduced|reproduces)\s*:\s*(yes|no|unknown)", re.IGNORECASE, ) _BASELINE_RE = re.compile( r"(?:on baseline master|baseline master|exists on baseline)\s*:\s*(yes|no|unknown|not checked)", re.IGNORECASE, ) _PR_CAUSED_RE = re.compile( r"(?:pr[- ]caused|pr caused)\s*:\s*(yes|no|unknown|not proven)", re.IGNORECASE, ) _TRANSIENT_STATUS_RE = re.compile( r"(?:passed after transient failure investigation|transient failure investigation)", re.IGNORECASE, ) _PLAIN_PASS_RE = re.compile( r"validation\s*:\s*(?:pass|passed|strong|ok|green)\b", re.IGNORECASE, ) _ENV_CLEANUP_RE = re.compile( r"(?:environmental cleanup|state cleaned|what changed between runs)\s*:", re.IGNORECASE, ) _UNKNOWN_CAUSE_RE = re.compile( r"(?:suspected cause|cause)\s*:\s*(?:unknown|unexplained|not determined)", re.IGNORECASE, ) def _failure_documented_in_text(text: str, failure: dict[str, Any]) -> bool: """Return True when *failure* appears documented in free-form report text.""" command = (failure.get("command") or "").strip() failing = (failure.get("failing_test") or failure.get("error") or "").strip() if command and command not in text: return False if failing and failing not in text: return False return bool(command or failing) def _section_has_structured_fields(text: str) -> bool: if not _SECTION_RE.search(text): return False section_start = _SECTION_RE.search(text).start() section = text[section_start:] has_command = bool(_COMMAND_RE.search(section)) has_failure = bool(_FAILING_TEST_RE.search(section)) return has_command and has_failure def assess_validation_failure_history_report( report_text: str, *, validation_session: dict | None = None, ) -> dict[str, Any]: """Require final reports to account for transient validation failures (#396).""" text = report_text or "" session = dict(validation_session or {}) reasons: list[str] = [] violations: list[str] = [] observed = list(session.get("observed_failures") or []) final_status = (session.get("final_validation_status") or "").strip().lower() cause_unknown = any( (f.get("suspected_cause") or "").strip().lower() in { "unknown", "unexplained", "not determined", "" } and not (f.get("pr_caused") or "").strip().lower() in {"yes", "no"} for f in observed ) or bool(session.get("cause_unknown")) if not observed: return { "proven": True, "block": False, "reasons": [], "violations": [], "observed_failure_count": 0, "safe_next_action": "proceed", } documented = _section_has_structured_fields(text) if not documented: for failure in observed: if _failure_documented_in_text(text, failure): documented = True break if not documented: violations.append( "session observed validation failure(s) but final report omits " "Validation failure history" ) reasons.append( "every validation failure observed during the session must appear " "in a Validation failure history section" ) if documented and not _SECTION_RE.search(text): reasons.append( "failure details must appear under an explicit " "'Validation failure history' heading" ) for idx, failure in enumerate(observed, start=1): prefix = f"failure #{idx}" if not _failure_documented_in_text(text, failure) and documented: reasons.append( f"{prefix}: command and failing test/error not documented in report" ) command = (failure.get("command") or "").strip() failing = (failure.get("failing_test") or failure.get("error") or "").strip() if not command: reasons.append(f"{prefix}: missing command in session failure record") if not failing: reasons.append( f"{prefix}: missing failing test or error in session failure record" ) plain_pass = bool(_PLAIN_PASS_RE.search(text)) transient_wording = bool(_TRANSIENT_STATUS_RE.search(text)) final_passed = final_status in { "passed", "pass", "passed_after_transient_failure_investigation", } if (final_passed or plain_pass) and observed: if cause_unknown and not transient_wording: violations.append( "unknown transient failure cause cannot be erased as plain 'passed'" ) reasons.append( "when cause is unknown, use status " "'passed after transient failure investigation'" ) elif not transient_wording and final_status != "passed_after_transient_failure_investigation": if not _ENV_CLEANUP_RE.search(text): reasons.append( "later passing rerun must document what changed between runs " "or environmental cleanup performed" ) if session.get("environmental_contamination") and not _ENV_CLEANUP_RE.search(text): reasons.append( "environmental /tmp state contamination must document cleanup or " "what changed before the passing rerun" ) proven = not reasons and not violations return { "proven": proven, "block": bool(violations) or not proven, "reasons": reasons, "violations": violations, "observed_failure_count": len(observed), "documented": documented, "safe_next_action": ( "add Validation failure history with command, failing test, cause, " "reproduction, baseline comparison, and PR-caused evidence; use " "'passed after transient failure investigation' when appropriate" if not proven else "proceed" ), }