Gitea-Tools/reviewer_validation_failure_history.py

"""Transient validation failure history verifier (#396).

Reviewer sessions may observe validation failures that later pass on rerun.
Final reports must document every failure observed during the session, not
only the last passing result.
"""

from __future__ import annotations

import re
from typing import Any

_SECTION_RE = re.compile(
    r"validation failure history",
    re.IGNORECASE,
)
_FAILURE_ENTRY_RE = re.compile(
    r"(?:failure\s*(?:#|entry)?\s*\d+|validation failure)\s*:",
    re.IGNORECASE,
)
_COMMAND_RE = re.compile(
    r"(?:command|validation command)\s*:\s*(.+)",
    re.IGNORECASE,
)
_FAILING_TEST_RE = re.compile(
    r"(?:failing test|failure|error)\s*:\s*(.+)",
    re.IGNORECASE,
)
_CAUSE_RE = re.compile(
    r"(?:suspected cause|cause)\s*:\s*(.+)",
    re.IGNORECASE,
)
_REPRODUCED_RE = re.compile(
    r"(?:reproduced|reproduces)\s*:\s*(yes|no|unknown)",
    re.IGNORECASE,
)
_BASELINE_RE = re.compile(
    r"(?:on baseline master|baseline master|exists on baseline)\s*:\s*(yes|no|unknown|not checked)",
    re.IGNORECASE,
)
_PR_CAUSED_RE = re.compile(
    r"(?:pr[- ]caused|pr caused)\s*:\s*(yes|no|unknown|not proven)",
    re.IGNORECASE,
)
_TRANSIENT_STATUS_RE = re.compile(
    r"(?:passed after transient failure investigation|transient failure investigation)",
    re.IGNORECASE,
)
_PLAIN_PASS_RE = re.compile(
    r"validation\s*:\s*(?:pass|passed|strong|ok|green)\b",
    re.IGNORECASE,
)
_ENV_CLEANUP_RE = re.compile(
    r"(?:environmental cleanup|state cleaned|what changed between runs)\s*:",
    re.IGNORECASE,
)
_UNKNOWN_CAUSE_RE = re.compile(
    r"(?:suspected cause|cause)\s*:\s*(?:unknown|unexplained|not determined)",
    re.IGNORECASE,
)


def _failure_documented_in_text(text: str, failure: dict[str, Any]) -> bool:
    """Return True when *failure* appears documented in free-form report text."""
    command = (failure.get("command") or "").strip()
    failing = (failure.get("failing_test") or failure.get("error") or "").strip()
    if command and command not in text:
        return False
    if failing and failing not in text:
        return False
    return bool(command or failing)


def _section_has_structured_fields(text: str) -> bool:
    if not _SECTION_RE.search(text):
        return False
    section_start = _SECTION_RE.search(text).start()
    section = text[section_start:]
    has_command = bool(_COMMAND_RE.search(section))
    has_failure = bool(_FAILING_TEST_RE.search(section))
    return has_command and has_failure


def assess_validation_failure_history_report(
    report_text: str,
    *,
    validation_session: dict | None = None,
) -> dict[str, Any]:
    """Require final reports to account for transient validation failures (#396)."""
    text = report_text or ""
    session = dict(validation_session or {})
    reasons: list[str] = []
    violations: list[str] = []

    observed = list(session.get("observed_failures") or [])
    final_status = (session.get("final_validation_status") or "").strip().lower()
    cause_unknown = any(
        (f.get("suspected_cause") or "").strip().lower() in {
            "unknown", "unexplained", "not determined", ""
        }
        and not (f.get("pr_caused") or "").strip().lower() in {"yes", "no"}
        for f in observed
    ) or bool(session.get("cause_unknown"))

    if not observed:
        return {
            "proven": True,
            "block": False,
            "reasons": [],
            "violations": [],
            "observed_failure_count": 0,
            "safe_next_action": "proceed",
        }

    documented = _section_has_structured_fields(text)
    if not documented:
        for failure in observed:
            if _failure_documented_in_text(text, failure):
                documented = True
                break

    if not documented:
        violations.append(
            "session observed validation failure(s) but final report omits "
            "Validation failure history"
        )
        reasons.append(
            "every validation failure observed during the session must appear "
            "in a Validation failure history section"
        )

    if documented and not _SECTION_RE.search(text):
        reasons.append(
            "failure details must appear under an explicit "
            "'Validation failure history' heading"
        )

    for idx, failure in enumerate(observed, start=1):
        prefix = f"failure #{idx}"
        if not _failure_documented_in_text(text, failure) and documented:
            reasons.append(
                f"{prefix}: command and failing test/error not documented in report"
            )
        command = (failure.get("command") or "").strip()
        failing = (failure.get("failing_test") or failure.get("error") or "").strip()
        if not command:
            reasons.append(f"{prefix}: missing command in session failure record")
        if not failing:
            reasons.append(
                f"{prefix}: missing failing test or error in session failure record"
            )

    plain_pass = bool(_PLAIN_PASS_RE.search(text))
    transient_wording = bool(_TRANSIENT_STATUS_RE.search(text))
    final_passed = final_status in {
        "passed",
        "pass",
        "passed_after_transient_failure_investigation",
    }

    if (final_passed or plain_pass) and observed:
        if cause_unknown and not transient_wording:
            violations.append(
                "unknown transient failure cause cannot be erased as plain 'passed'"
            )
            reasons.append(
                "when cause is unknown, use status "
                "'passed after transient failure investigation'"
            )
        elif not transient_wording and final_status != "passed_after_transient_failure_investigation":
            if not _ENV_CLEANUP_RE.search(text):
                reasons.append(
                    "later passing rerun must document what changed between runs "
                    "or environmental cleanup performed"
                )

    if session.get("environmental_contamination") and not _ENV_CLEANUP_RE.search(text):
        reasons.append(
            "environmental /tmp state contamination must document cleanup or "
            "what changed before the passing rerun"
        )

    proven = not reasons and not violations
    return {
        "proven": proven,
        "block": bool(violations) or not proven,
        "reasons": reasons,
        "violations": violations,
        "observed_failure_count": len(observed),
        "documented": documented,
        "safe_next_action": (
            "add Validation failure history with command, failing test, cause, "
            "reproduction, baseline comparison, and PR-caused evidence; use "
            "'passed after transient failure investigation' when appropriate"
            if not proven
            else "proceed"
        ),
    }