Gitea-Tools/tests/test_reviewer_validation_failure_history.py

"""Tests for validation failure history reporting (#396)."""
import sys
import unittest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from reviewer_validation_failure_history import (  # noqa: E402
    TRANSIENT_INVESTIGATION_STATUS,
    assess_validation_failure_history_report,
)

_OBSERVED = [
    {
        "command": "venv/bin/pytest tests/ -q",
        "failure": "tests/test_worktrees.py::TestWorktreeStart::test_rejects_untraceable_branches",
    }
]


def _history_block(**overrides):
    lines = {
        "heading": "## Validation failure history",
        "command": "- Command: venv/bin/pytest tests/ -q in branches/review-pr381",
        "failure": (
            "- Failing test: tests/test_worktrees.py::TestWorktreeStart::"
            "test_rejects_untraceable_branches"
        ),
        "cause": "- Suspected cause: stale /tmp/gitea_issue_lock.json from a concurrent session",
        "reproduced": "- Reproduced: no (single occurrence, not reproducible in isolation)",
        "baseline": "- Baseline master: does not fail on clean baseline",
        "evidence": (
            "- PR-caused evidence: not PR-caused; failure touches issue-lock state "
            "outside the PR diff"
        ),
        "rerun": (
            "- Rerun: passed on rerun; what changed between runs: stale "
            "/tmp/gitea_issue_lock.json environmental state was cleaned; the pass is "
            "trustworthy because the failing test passes in isolation and in the full "
            "suite after cleanup"
        ),
    }
    lines.update(overrides)
    return "\n".join(v for v in lines.values() if v)


class TestOmittedFailures(unittest.TestCase):
    def test_report_omitting_earlier_failure_fails(self):
        report = "## Final Report\n- Validation: passed (1497 passed, 6 skipped)\n"
        result = assess_validation_failure_history_report(
            report, observed_failures=_OBSERVED
        )
        self.assertFalse(result["proven"])
        self.assertTrue(
            any("history" in r.lower() for r in result["reasons"]),
            result["reasons"],
        )

    def test_history_section_missing_observed_test_fails(self):
        report = (
            "## Validation failure history\n"
            "- Command: pytest tests/test_other.py\n"
            "- Failing test: tests/test_other.py::test_something_else\n"
        )
        result = assess_validation_failure_history_report(
            report, observed_failures=_OBSERVED
        )
        self.assertFalse(result["proven"])
        self.assertTrue(
            any("test_rejects_untraceable_branches" in r for r in result["reasons"]),
            result["reasons"],
        )

    def test_no_observed_failures_and_no_claims_passes(self):
        report = "## Final Report\n- Validation: passed (1501 passed)\n"
        result = assess_validation_failure_history_report(report)
        self.assertTrue(result["proven"], result["reasons"])


class TestFailureEntryFields(unittest.TestCase):
    def test_full_history_entry_passes(self):
        report = _history_block() + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS
        result = assess_validation_failure_history_report(
            report, observed_failures=_OBSERVED
        )
        self.assertTrue(result["proven"], result["reasons"])

    def test_missing_suspected_cause_fails(self):
        report = _history_block(cause="") + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS
        result = assess_validation_failure_history_report(
            report, observed_failures=_OBSERVED
        )
        self.assertFalse(result["proven"])
        self.assertTrue(
            any("suspected cause" in r.lower() for r in result["reasons"]),
            result["reasons"],
        )

    def test_missing_baseline_status_fails(self):
        report = _history_block(baseline="") + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS
        result = assess_validation_failure_history_report(
            report, observed_failures=_OBSERVED
        )
        self.assertFalse(result["proven"])
        self.assertTrue(
            any("baseline" in r.lower() for r in result["reasons"]),
            result["reasons"],
        )

    def test_baseline_equivalent_failure_passes(self):
        report = (
            _history_block(
                baseline="- Baseline master: fails identically on clean baseline worktree",
                evidence=(
                    "- PR-caused evidence: not PR-caused; identical failure signature "
                    "on baseline master"
                ),
                rerun="- Rerun: not rerun; baseline-equivalent failure documented",
            )
            + "\n\nValidation status: failed (baseline-equivalent, not PR-caused)"
        )
        result = assess_validation_failure_history_report(
            report, observed_failures=_OBSERVED
        )
        self.assertTrue(result["proven"], result["reasons"])


class TestRerunPassClaims(unittest.TestCase):
    def test_rerun_pass_without_what_changed_fails(self):
        report = (
            _history_block(rerun="- Rerun: passed on rerun")
            + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS
        )
        result = assess_validation_failure_history_report(
            report, observed_failures=_OBSERVED
        )
        self.assertFalse(result["proven"])
        self.assertTrue(
            any("what changed" in r.lower() for r in result["reasons"]),
            result["reasons"],
        )

    def test_tmp_state_contamination_documented_passes(self):
        result = assess_validation_failure_history_report(
            _history_block() + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS,
            observed_failures=_OBSERVED,
        )
        self.assertTrue(result["proven"], result["reasons"])


class TestStatusVocabulary(unittest.TestCase):
    def test_unexplained_transient_with_bare_passed_fails(self):
        report = (
            _history_block(
                cause="- Suspected cause: unknown",
                rerun=(
                    "- Rerun: passed on rerun; what changed between runs: nothing "
                    "identified; environmental state: none cleaned; the pass is "
                    "trustworthy because three consecutive reruns pass"
                ),
            )
            + "\n\nValidation status: passed"
        )
        result = assess_validation_failure_history_report(
            report, observed_failures=_OBSERVED
        )
        self.assertFalse(result["proven"])
        self.assertTrue(
            any(TRANSIENT_INVESTIGATION_STATUS in r for r in result["reasons"]),
            result["reasons"],
        )

    def test_unexplained_transient_with_investigation_status_passes(self):
        report = (
            _history_block(
                cause="- Suspected cause: unknown",
                rerun=(
                    "- Rerun: passed on rerun; what changed between runs: nothing "
                    "identified; environmental state: none cleaned; the pass is "
                    "trustworthy because three consecutive reruns pass"
                ),
            )
            + "\n\nValidation status: " + TRANSIENT_INVESTIGATION_STATUS
        )
        result = assess_validation_failure_history_report(
            report, observed_failures=_OBSERVED
        )
        self.assertTrue(result["proven"], result["reasons"])


class TestExports(unittest.TestCase):
    def test_review_proofs_reexport(self):
        from review_proofs import (
            assess_validation_failure_history_report as exported,
        )

        self.assertTrue(callable(exported))

    def test_status_constant_value(self):
        self.assertEqual(
            TRANSIENT_INVESTIGATION_STATUS,
            "passed after transient failure investigation",
        )


if __name__ == "__main__":
    unittest.main()