"""Tests for validation status vocabulary (#406).""" import sys import unittest from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from final_report_validator import assess_final_report_validator # noqa: E402 from validation_status_vocabulary import ( # noqa: E402 STATUS_BASELINE_EQUIVALENT, STATUS_FAILED, STATUS_MERGE_SIM_RESOLVED, STATUS_PASSED, STATUS_TRANSIENT_PASS, assess_validation_status_vocabulary, ) def _handoff(**extra): fields = { "Task": "review PR #386", "Validation status": STATUS_PASSED, "Raw PR-head validation result": "passed", "Merge simulation result": "not run", "Baseline worktree used": "none", } fields.update(extra) lines = ["## Controller Handoff", ""] lines.extend(f"- {key}: {value}" for key, value in fields.items()) return "\n".join(lines) class TestValidationStatusVocabulary(unittest.TestCase): def test_raw_head_pass_status(self): report = _handoff() result = assess_validation_status_vocabulary(report) self.assertFalse(result["block"]) self.assertEqual(result["status_claimed"], STATUS_PASSED) def test_raw_head_failure_with_baseline_match(self): report = _handoff( **{ "Validation status": STATUS_BASELINE_EQUIVALENT, "Raw PR-head validation result": "failed", "Baseline worktree used": "branches/baseline-master-pr386", "Baseline target SHA": "a" * 40, "Baseline failures": "test_foo failed", "PR failures": "test_foo failed", "Failure signatures match": "true", } ) result = assess_validation_status_vocabulary(report) self.assertFalse(result["block"]) self.assertTrue(result["baseline_proof_complete"]) def test_baseline_equivalent_without_baseline_proof_blocked(self): report = _handoff( **{ "Validation status": STATUS_BASELINE_EQUIVALENT, "Raw PR-head validation result": "failed", } ) result = assess_validation_status_vocabulary(report) self.assertTrue(result["block"]) self.assertIn("baseline-equivalent", result["reasons"][0]) def test_merge_simulation_resolution_passes(self): report = "\n".join([ _handoff( **{ "Validation status": STATUS_MERGE_SIM_RESOLVED, "Raw PR-head validation result": "failed", "Merge simulation result": "passed", } ), "Worktree/index mutations: merge simulation in branches/review-pr386", "Worktree path: branches/review-pr386", "Pre-simulation clean status: clean", "Merge result: clean merge", "Abort command: git merge --abort", "Post-abort clean status: clean", ]) command_log = [ {"command": "git merge --no-commit prgs/master"}, {"command": "git merge --abort"}, ] result = assess_validation_status_vocabulary( report, command_log=command_log ) self.assertFalse(result["block"]) self.assertTrue(result["merge_simulation_passed"]) def test_merge_simulation_failure_stays_failed(self): report = _handoff( **{ "Validation status": STATUS_FAILED, "Raw PR-head validation result": "failed", "Merge simulation result": "failed", } ) result = assess_validation_status_vocabulary(report) self.assertFalse(result["block"]) def test_failed_status_with_passing_merge_sim_blocked(self): report = "\n".join([ _handoff( **{ "Validation status": STATUS_FAILED, "Raw PR-head validation result": "failed", "Merge simulation result": "passed", } ), "Worktree/index mutations: merge simulation", "Worktree path: branches/review-pr386", "Pre-simulation clean status: clean", "Merge result: clean", "Abort command: git merge --abort", "Post-abort clean status: clean", ]) result = assess_validation_status_vocabulary( report, command_log=[{"command": "git merge --no-commit prgs/master"}], ) self.assertTrue(result["block"]) def test_transient_failure_then_pass(self): report = _handoff( **{ "Validation status": STATUS_TRANSIENT_PASS, "Raw PR-head validation result": "passed", "Transient validation failure history": ( "first run failed with infra flake; rerun passed" ), } ) result = assess_validation_status_vocabulary(report) self.assertFalse(result["block"]) def test_transient_pass_without_history_blocked(self): report = _handoff( **{ "Validation status": STATUS_TRANSIENT_PASS, "Raw PR-head validation result": "passed", } ) result = assess_validation_status_vocabulary(report) self.assertTrue(result["block"]) def test_bare_passed_after_raw_failure_blocked(self): report = _handoff( **{ "Validation status": STATUS_PASSED, "Raw PR-head validation result": "failed", } ) result = assess_validation_status_vocabulary(report) self.assertTrue(result["block"]) def test_wrong_baseline_label_when_merge_sim_used_blocked(self): report = "\n".join([ _handoff( **{ "Validation status": STATUS_BASELINE_EQUIVALENT, "Raw PR-head validation result": "failed", "Merge simulation result": "passed", } ), "Worktree/index mutations: merge simulation", "Worktree path: branches/review-pr386", "Pre-simulation clean status: clean", "Merge result: clean", "Abort command: git merge --abort", "Post-abort clean status: clean", ]) result = assess_validation_status_vocabulary( report, command_log=[{"command": "git merge --no-commit prgs/master"}], ) self.assertTrue(result["block"]) joined = " ".join(result["reasons"]).lower() self.assertTrue( "misleading" in joined or "baseline-equivalent" in joined ) def test_final_report_validator_integration_blocks_misleading_label(self): report = _handoff( **{ "Validation status": STATUS_BASELINE_EQUIVALENT, "Raw PR-head validation result": "failed", } ) result = assess_final_report_validator(report, "review_pr") blocked_ids = {f["rule_id"] for f in result["findings"]} self.assertIn("reviewer.validation_status_vocabulary", blocked_ids) if __name__ == "__main__": unittest.main()