Gitea-Tools/premerge_baseline_proof.py

"""Pre-merge baseline proof verifier for non-zero validation exits (#533).

A controller/reviewer may reproduce a failing test on *current* master and
describe it as proof of a "known baseline failure". Reproducing a failure on
post-merge master only proves the failure exists on current master — it does
NOT prove the failure pre-existed the PR under review. A PR can introduce or
preserve a regression, then once merged the failure appears on master and gets
mislabeled "baseline", weakening validation gates.

This module distinguishes four canonical validation outcomes:

- ``CLEAN_PASS`` — the validation command exited zero.
- ``CURRENT_MASTER_FAILURE_REPRODUCED`` — the same failure reproduces on
  current (post-merge) master. Not sufficient as baseline proof.
- ``PREMERGE_BASELINE_PROVEN_FAILURE`` — the failure is proven on the PR's
  pre-merge base commit, or by a documented known-failure record that predates
  the PR.
- ``UNRESOLVED_REGRESSION_RISK`` — a non-zero exit that is neither a clean pass
  nor proven pre-existing.

A report may only call a non-zero validation exit a "baseline"/"pre-existing"
failure when it carries pre-merge proof.
"""

from __future__ import annotations

import re
from typing import Any

CLEAN_PASS = "clean pass"
CURRENT_MASTER_FAILURE_REPRODUCED = "current-master failure reproduced"
PREMERGE_BASELINE_PROVEN_FAILURE = "pre-merge baseline-proven failure"
UNRESOLVED_REGRESSION_RISK = "unresolved regression risk"

# A non-zero validation exit is claimed somewhere in the report.
_NONZERO_EXIT_RE = re.compile(
    r"(?:exit(?:\s*(?:status|code))?\s*[:=]?\s*(?:[1-9]\d*)"
    r"|\b\d+\s+failed\b"
    r"|\bnon[- ]zero\s+(?:exit|validation))",
    re.IGNORECASE,
)
# The report claims the failure is pre-existing / a baseline failure.
_BASELINE_CLAIM_RE = re.compile(
    r"(?:pre[- ]?existing(?:\s+(?:baseline|failure))?"
    r"|baseline(?:[- ]proven)?\s+failure"
    r"|known[- ]baseline"
    r"|failure\s+is\s+baseline"
    r"|already\s+fail(?:s|ing)\s+on\s+master)",
    re.IGNORECASE,
)
# Only-current-master reproduction language (insufficient on its own).
_CURRENT_MASTER_RE = re.compile(
    r"(?:current[- ]master\s+(?:reproduc|failure)"
    r"|reproduc\w*\s+on\s+(?:current\s+)?master"
    r"|post[- ]merge\s+master"
    r"|fails\s+on\s+(?:current\s+)?master\s+(?:now|too|as\s+well))",
    re.IGNORECASE,
)
# Pre-merge base proof: a base commit tested before the PR landed.
_PREMERGE_BASE_RE = re.compile(
    r"(?:pre[- ]merge\s+base(?:\s+commit)?"
    r"|pr\s+base\s+commit"
    r"|merge[- ]base(?:\s+commit)?"
    r"|base\s+commit\s+before\s+(?:the\s+)?pr)",
    re.IGNORECASE,
)
# Documented known-failure record predating the PR.
_KNOWN_FAILURE_RECORD_RE = re.compile(
    r"known[- ]failure\s+(?:record|reference|ledger)\b.{0,80}?"
    r"(?:predat|before\s+(?:the\s+)?pr|pre[- ]existing|prior\s+to\s+(?:the\s+)?pr)",
    re.IGNORECASE | re.DOTALL,
)

# Required proof fields for a pre-merge baseline claim.
_FIELD_PATTERNS: dict[str, re.Pattern[str]] = {
    "base commit": re.compile(
        r"(?:pre[- ]merge\s+)?base\s+commit\s*[:=]\s*([0-9a-f]{7,40})", re.IGNORECASE
    ),
    "tested commit": re.compile(
        r"tested\s+commit\s*[:=]\s*([0-9a-f]{7,40})", re.IGNORECASE
    ),
    "command": re.compile(r"command\s*[:=]\s*\S", re.IGNORECASE),
    "exit status": re.compile(r"exit\s*(?:status|code)?\s*[:=]\s*\d+", re.IGNORECASE),
    "failure signature": re.compile(
        r"failure\s+signature\s*[:=]\s*\S", re.IGNORECASE
    ),
}


def assess_premerge_baseline_proof(report_text: str) -> dict[str, Any]:
    """Assess whether a claimed baseline failure carries pre-merge proof.

    Returns a dict with ``proven``, ``block``, ``label``, ``reasons``,
    ``skipped`` and ``safe_next_action``. Only blocks when the report claims a
    non-zero validation exit is baseline/pre-existing without valid pre-merge
    proof.
    """
    text = report_text or ""

    has_failure = bool(_NONZERO_EXIT_RE.search(text))
    claims_baseline = bool(_BASELINE_CLAIM_RE.search(text))

    # No failure claimed, or no baseline assertion — nothing to enforce here.
    if not has_failure and not claims_baseline:
        return {
            "proven": True,
            "block": False,
            "label": CLEAN_PASS,
            "reasons": [],
            "skipped": True,
            "safe_next_action": "",
        }

    if not claims_baseline:
        # A non-zero exit not asserted as baseline — surface as regression risk,
        # but do not block (the report never claimed a clean/baseline pass).
        return {
            "proven": True,
            "block": False,
            "label": UNRESOLVED_REGRESSION_RISK,
            "reasons": [],
            "skipped": False,
            "safe_next_action": "",
        }

    has_premerge_base = bool(_PREMERGE_BASE_RE.search(text))
    has_known_record = bool(_KNOWN_FAILURE_RECORD_RE.search(text))
    only_current_master = bool(_CURRENT_MASTER_RE.search(text))

    reasons: list[str] = []

    if not (has_premerge_base or has_known_record):
        if only_current_master:
            reasons.append(
                "baseline failure claimed from current-master reproduction only; "
                "that proves the failure on current master, not that it pre-existed "
                "the PR — provide pre-merge base proof or a known-failure record"
            )
        else:
            reasons.append(
                "baseline/pre-existing failure claimed without pre-merge proof "
                "(no pre-merge base commit and no documented known-failure record "
                "predating the PR)"
            )

    # Require the concrete proof fields regardless of proof source.
    missing = [name for name, pat in _FIELD_PATTERNS.items() if not pat.search(text)]
    if missing:
        reasons.append(
            "pre-merge baseline claim missing required proof field(s): "
            + ", ".join(missing)
        )

    if reasons:
        return {
            "proven": False,
            "block": True,
            "label": UNRESOLVED_REGRESSION_RISK,
            "reasons": reasons,
            "skipped": False,
            "safe_next_action": (
                "prove the failure on the PR pre-merge base commit (or cite a "
                "known-failure record predating the PR) and state base commit, "
                "tested commit, command, exit status, and failure signature; "
                "current-master reproduction alone is not baseline proof"
            ),
        }

    return {
        "proven": True,
        "block": False,
        "label": PREMERGE_BASELINE_PROVEN_FAILURE,
        "reasons": [],
        "skipped": False,
        "safe_next_action": "",
    }