"""Pre-merge baseline proof verifier for non-zero validation exits (#533). A controller/reviewer may reproduce a failing test on *current* master and describe it as proof of a "known baseline failure". Reproducing a failure on post-merge master only proves the failure exists on current master — it does NOT prove the failure pre-existed the PR under review. A PR can introduce or preserve a regression, then once merged the failure appears on master and gets mislabeled "baseline", weakening validation gates. This module distinguishes four canonical validation outcomes: - ``CLEAN_PASS`` — the validation command exited zero. - ``CURRENT_MASTER_FAILURE_REPRODUCED`` — the same failure reproduces on current (post-merge) master. Not sufficient as baseline proof. - ``PREMERGE_BASELINE_PROVEN_FAILURE`` — the failure is proven on the PR's pre-merge base commit, or by a documented known-failure record that predates the PR. - ``UNRESOLVED_REGRESSION_RISK`` — a non-zero exit that is neither a clean pass nor proven pre-existing. A report may only call a non-zero validation exit a "baseline"/"pre-existing" failure when it carries pre-merge proof. """ from __future__ import annotations import re from typing import Any CLEAN_PASS = "clean pass" CURRENT_MASTER_FAILURE_REPRODUCED = "current-master failure reproduced" PREMERGE_BASELINE_PROVEN_FAILURE = "pre-merge baseline-proven failure" UNRESOLVED_REGRESSION_RISK = "unresolved regression risk" # A non-zero validation exit is claimed somewhere in the report. _NONZERO_EXIT_RE = re.compile( r"(?:exit(?:\s*(?:status|code))?\s*[:=]?\s*(?:[1-9]\d*)" r"|\b\d+\s+failed\b" r"|\bnon[- ]zero\s+(?:exit|validation))", re.IGNORECASE, ) # The report claims the failure is pre-existing / a baseline failure. _BASELINE_CLAIM_RE = re.compile( r"(?:pre[- ]?existing(?:\s+(?:baseline|failure))?" r"|baseline(?:[- ]proven)?\s+failure" r"|known[- ]baseline" r"|failure\s+is\s+baseline" r"|already\s+fail(?:s|ing)\s+on\s+master)", re.IGNORECASE, ) # Only-current-master reproduction language (insufficient on its own). _CURRENT_MASTER_RE = re.compile( r"(?:current[- ]master\s+(?:reproduc|failure)" r"|reproduc\w*\s+on\s+(?:current\s+)?master" r"|post[- ]merge\s+master" r"|fails\s+on\s+(?:current\s+)?master\s+(?:now|too|as\s+well))", re.IGNORECASE, ) # Pre-merge base proof: a base commit tested before the PR landed. _PREMERGE_BASE_RE = re.compile( r"(?:pre[- ]merge\s+base(?:\s+commit)?" r"|pr\s+base\s+commit" r"|merge[- ]base(?:\s+commit)?" r"|base\s+commit\s+before\s+(?:the\s+)?pr)", re.IGNORECASE, ) # Documented known-failure record predating the PR. _KNOWN_FAILURE_RECORD_RE = re.compile( r"known[- ]failure\s+(?:record|reference|ledger)\b.{0,80}?" r"(?:predat|before\s+(?:the\s+)?pr|pre[- ]existing|prior\s+to\s+(?:the\s+)?pr)", re.IGNORECASE | re.DOTALL, ) # Required proof fields for a pre-merge baseline claim. _FIELD_PATTERNS: dict[str, re.Pattern[str]] = { "base commit": re.compile( r"(?:pre[- ]merge\s+)?base\s+commit\s*[:=]\s*([0-9a-f]{7,40})", re.IGNORECASE ), "tested commit": re.compile( r"tested\s+commit\s*[:=]\s*([0-9a-f]{7,40})", re.IGNORECASE ), "command": re.compile(r"command\s*[:=]\s*\S", re.IGNORECASE), "exit status": re.compile(r"exit\s*(?:status|code)?\s*[:=]\s*\d+", re.IGNORECASE), "failure signature": re.compile( r"failure\s+signature\s*[:=]\s*\S", re.IGNORECASE ), } def assess_premerge_baseline_proof(report_text: str) -> dict[str, Any]: """Assess whether a claimed baseline failure carries pre-merge proof. Returns a dict with ``proven``, ``block``, ``label``, ``reasons``, ``skipped`` and ``safe_next_action``. Only blocks when the report claims a non-zero validation exit is baseline/pre-existing without valid pre-merge proof. """ text = report_text or "" has_failure = bool(_NONZERO_EXIT_RE.search(text)) claims_baseline = bool(_BASELINE_CLAIM_RE.search(text)) # No failure claimed, or no baseline assertion — nothing to enforce here. if not has_failure and not claims_baseline: return { "proven": True, "block": False, "label": CLEAN_PASS, "reasons": [], "skipped": True, "safe_next_action": "", } if not claims_baseline: # A non-zero exit not asserted as baseline — surface as regression risk, # but do not block (the report never claimed a clean/baseline pass). return { "proven": True, "block": False, "label": UNRESOLVED_REGRESSION_RISK, "reasons": [], "skipped": False, "safe_next_action": "", } has_premerge_base = bool(_PREMERGE_BASE_RE.search(text)) has_known_record = bool(_KNOWN_FAILURE_RECORD_RE.search(text)) only_current_master = bool(_CURRENT_MASTER_RE.search(text)) reasons: list[str] = [] if not (has_premerge_base or has_known_record): if only_current_master: reasons.append( "baseline failure claimed from current-master reproduction only; " "that proves the failure on current master, not that it pre-existed " "the PR — provide pre-merge base proof or a known-failure record" ) else: reasons.append( "baseline/pre-existing failure claimed without pre-merge proof " "(no pre-merge base commit and no documented known-failure record " "predating the PR)" ) # Require the concrete proof fields regardless of proof source. missing = [name for name, pat in _FIELD_PATTERNS.items() if not pat.search(text)] if missing: reasons.append( "pre-merge baseline claim missing required proof field(s): " + ", ".join(missing) ) if reasons: return { "proven": False, "block": True, "label": UNRESOLVED_REGRESSION_RISK, "reasons": reasons, "skipped": False, "safe_next_action": ( "prove the failure on the PR pre-merge base commit (or cite a " "known-failure record predating the PR) and state base commit, " "tested commit, command, exit status, and failure signature; " "current-master reproduction alone is not baseline proof" ), } return { "proven": True, "block": False, "label": PREMERGE_BASELINE_PROVEN_FAILURE, "reasons": [], "skipped": False, "safe_next_action": "", }