Gitea-Tools/reviewer_validation_integrity.py

"""PR-head vs diagnostic validation integrity verifier (#316)."""

from __future__ import annotations

import re
from typing import Any

_DIAGNOSTIC_LABEL_RE = re.compile(
    r"diagnostic local experiment|not pr-head validation|diagnostic-only validation",
    re.IGNORECASE,
)
_OFFICIAL_VALIDATION_RE = re.compile(
    r"(?:official validation|pr-head validation|validation integrity)",
    re.IGNORECASE,
)
_OFFICIAL_COMMAND_RE = re.compile(
    r"(?:official validation command|pr-head validation command|validation command)",
    re.IGNORECASE,
)
_OFFICIAL_RESULT_RE = re.compile(
    r"(?:official validation result|pr-head validation result|validation result|validation integrity status)",
    re.IGNORECASE,
)
_DIRTY_AFTER_RE = re.compile(
    r"(?:worktree dirty after (?:official )?validation|dirty (?:state )?after (?:official )?validation|"
    r"validation worktree dirty after)",
    re.IGNORECASE,
)
_DIAGNOSTIC_EDIT_RE = re.compile(
    r"(?:diagnostic edit(?:ed)? path|file edits by reviewer|diagnostic local edit)",
    re.IGNORECASE,
)
_DIAGNOSTIC_COMMAND_RE = re.compile(
    r"(?:diagnostic (?:validation )?command|diagnostic test run|diagnostic result)",
    re.IGNORECASE,
)
_SUGGESTED_FIX_RE = re.compile(
    r"(?:diagnostic results? (?:were )?used only for suggested fix|suggested fix only|"
    r"not used as official validation)",
    re.IGNORECASE,
)
_INTEGRITY_STATUS_RE = re.compile(
    r"validation integrity status\s*:\s*(passed|failed|not run|contaminated)",
    re.IGNORECASE,
)
_POST_EDIT_AS_OFFICIAL_RE = re.compile(
    r"(?:official validation(?:\s+result)?\s*:\s*pass|validation passed after (?:local )?edit|"
    r"full suite passed after diagnostic edit)",
    re.IGNORECASE,
)


def _performed_edits(action_log: list[dict] | None) -> list[str]:
    paths: list[str] = []
    for entry in action_log or []:
        if entry.get("gated_rejected") or entry.get("performed") is False:
            continue
        path = entry.get("path")
        if path and entry.get("kind", "file_edit") in {"file_edit", "edit", "write"}:
            paths.append(str(path))
    return paths


def assess_validation_integrity_report(
    report_text: str,
    *,
    validation_session: dict | None = None,
    action_log: list[dict] | None = None,
) -> dict[str, Any]:
    """Separate official PR-head validation from diagnostic edited-worktree tests (#316)."""
    text = report_text or ""
    session = dict(validation_session or {})
    reasons: list[str] = []

    diagnostic_edits = list(session.get("diagnostic_edits") or [])
    if not diagnostic_edits:
        diagnostic_edits = _performed_edits(action_log)
    diagnostic_ran = bool(
        session.get("diagnostic_validation_ran")
        or session.get("diagnostic_runs")
    )
    official_ran = bool(session.get("official_validation_ran", True))
    official_result = (session.get("official_result") or "").strip().lower()
    diagnostic_result = (session.get("diagnostic_result") or "").strip().lower()
    dirty_after = session.get("worktree_dirty_after_official")
    contaminated = bool(session.get("contaminated"))

    if official_ran:
        if not _OFFICIAL_VALIDATION_RE.search(text) and not _OFFICIAL_COMMAND_RE.search(text):
            reasons.append("report missing official PR-head validation section")
        if not _OFFICIAL_COMMAND_RE.search(text) and "validation:" not in text.lower():
            reasons.append("report missing official validation command")
        if not _OFFICIAL_RESULT_RE.search(text):
            reasons.append("report missing official validation result or integrity status")
        if dirty_after is not None and not _DIRTY_AFTER_RE.search(text):
            reasons.append(
                "report missing worktree dirty state after official validation"
            )
        elif dirty_after is None and official_ran and not _DIRTY_AFTER_RE.search(text):
            reasons.append(
                "report missing worktree dirty state after official validation"
            )

    if diagnostic_edits or diagnostic_ran:
        if not _DIAGNOSTIC_LABEL_RE.search(text):
            reasons.append(
                "post-edit diagnostic runs must be labeled "
                "'Diagnostic local experiment — not PR-head validation'"
            )
        if diagnostic_edits and not _DIAGNOSTIC_EDIT_RE.search(text):
            reasons.append("report missing diagnostic edit path")
        if diagnostic_ran and not _DIAGNOSTIC_COMMAND_RE.search(text):
            reasons.append("report missing diagnostic command/result")
        if not _SUGGESTED_FIX_RE.search(text):
            reasons.append(
                "report must state diagnostic results were used only for suggested fix"
            )
        if _POST_EDIT_AS_OFFICIAL_RE.search(text):
            reasons.append(
                "post-edit diagnostic results cannot be reported as official PR-head validation"
            )
        if diagnostic_result == "pass" and official_result == "fail":
            if "request_changes" not in text.lower() and "failed" not in text.lower():
                reasons.append(
                    "request-changes must cite unmodified PR-head failure, not diagnostic pass"
                )

    if contaminated:
        status = _INTEGRITY_STATUS_RE.search(text)
        if not status or "contaminated" not in status.group(1).lower():
            reasons.append(
                "contaminated validation must report integrity status "
                "'contaminated — recovery required'"
            )

    proven = not reasons
    return {
        "proven": proven,
        "block": not proven,
        "reasons": reasons,
        "diagnostic_edits": diagnostic_edits,
        "safe_next_action": (
            "separate official PR-head validation from diagnostic experiments; "
            "report dirty-after-validation state"
            if reasons
            else "proceed"
        ),
    }