"""Tests for transient validation failure history verifier (#396).""" import sys import unittest from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from final_report_validator import assess_final_report_validator # noqa: E402 from reviewer_validation_failure_history import ( # noqa: E402 assess_validation_failure_history_report, ) def _full_history_report() -> str: return "\n".join([ "Validation failure history", "Failure #1:", "Command: venv/bin/python -m pytest tests/test_worktrees.py -q", "Failing test: tests/test_worktrees.py::TestWorktreeStart::test_rejects_untraceable_branches", "Suspected cause: stale /tmp/gitea_issue_lock.json from prior session", "Reproduced: no", "On baseline master: no", "PR-caused: no", "What changed between runs: removed /tmp/gitea_issue_lock.json", "Environmental cleanup: lock file removed before rerun", "Validation: passed after transient failure investigation", "Final validation command: venv/bin/python -m pytest tests/ -q", "Final result: 1497 passed, 6 skipped", ]) class TestValidationFailureHistory(unittest.TestCase): def test_no_failures_observed_passes(self): result = assess_validation_failure_history_report( "Validation: passed", validation_session={"observed_failures": []}, ) self.assertTrue(result["proven"]) def test_omitted_failure_blocks(self): result = assess_validation_failure_history_report( "Validation: passed\n1497 passed, 6 skipped", validation_session={ "observed_failures": [{ "command": "pytest tests/test_worktrees.py -q", "failing_test": ( "tests/test_worktrees.py::TestWorktreeStart::" "test_rejects_untraceable_branches" ), }], "final_validation_status": "passed", }, ) self.assertFalse(result["proven"]) self.assertTrue(result["block"]) def test_full_history_with_transient_investigation_passes(self): result = assess_validation_failure_history_report( _full_history_report(), validation_session={ "observed_failures": [{ "command": ( "venv/bin/python -m pytest tests/test_worktrees.py -q" ), "failing_test": ( "tests/test_worktrees.py::TestWorktreeStart::" "test_rejects_untraceable_branches" ), "suspected_cause": "stale /tmp/gitea_issue_lock.json", "reproduced": "no", "on_baseline_master": "no", "pr_caused": "no", }], "final_validation_status": "passed_after_transient_failure_investigation", "environmental_contamination": True, }, ) self.assertTrue(result["proven"], result["reasons"]) def test_baseline_equivalent_failure_documented(self): report = "\n".join([ "Validation failure history", "Command: pytest tests/test_example.py -q", "Failing test: tests/test_example.py::test_flaky", "Suspected cause: pre-existing master failure", "On baseline master: yes", "PR-caused: no", "What changed between runs: none; failure matches baseline", "Validation: passed after transient failure investigation", ]) result = assess_validation_failure_history_report( report, validation_session={ "observed_failures": [{ "command": "pytest tests/test_example.py -q", "failing_test": "tests/test_example.py::test_flaky", "on_baseline_master": "yes", "pr_caused": "no", }], "final_validation_status": "passed_after_transient_failure_investigation", }, ) self.assertTrue(result["proven"], result["reasons"]) def test_unknown_cause_plain_pass_blocks(self): result = assess_validation_failure_history_report( "Validation: passed", validation_session={ "observed_failures": [{ "command": "pytest tests/ -q", "failing_test": "tests/test_foo.py::test_bar", "suspected_cause": "unknown", }], "final_validation_status": "passed", "cause_unknown": True, }, ) self.assertFalse(result["proven"]) self.assertTrue(any("transient" in r.lower() for r in result["reasons"])) def test_final_report_validator_rejects_omitted_failure(self): result = assess_final_report_validator( "Validation: passed", "review_pr", validation_session={ "observed_failures": [{ "command": "pytest tests/ -q", "failing_test": "tests/test_foo.py::test_bar", }], }, ) self.assertTrue(result["blocked"] or result["downgraded"]) self.assertTrue( any( f.get("rule_id") == "reviewer.validation_failure_history" for f in result.get("findings") or [] ) ) def test_exported_from_review_proofs(self): from review_proofs import assess_validation_failure_history_report as exported self.assertTrue(callable(exported)) if __name__ == "__main__": unittest.main()