sysadminandClaude Opus 4.8
4ea618e8b4
Separate official PR-head validation from post-edit diagnostic runs in reviewer reports. Block treating diagnostic passes as official validation and require contaminated sessions to report recovery-required status. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
131 lines
4.9 KiB
Python
131 lines
4.9 KiB
Python
"""Tests for PR-head vs diagnostic validation integrity verifier (#316)."""
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from reviewer_validation_integrity import assess_validation_integrity_report # noqa: E402
|
|
|
|
|
|
def _official_only_report() -> str:
|
|
return "\n".join([
|
|
"Official PR-head validation on unmodified worktree.",
|
|
"Official validation command: pytest tests/ -q",
|
|
"Official validation result: failed (3 failed)",
|
|
"Validation integrity status: failed",
|
|
"Worktree dirty after official validation: clean",
|
|
"Review decision: request_changes",
|
|
])
|
|
|
|
|
|
def _diagnostic_report() -> str:
|
|
return "\n".join([
|
|
_official_only_report(),
|
|
"Diagnostic local experiment — not PR-head validation",
|
|
"Diagnostic edit path: tests/test_example.py",
|
|
"File edits by reviewer: tests/test_example.py",
|
|
"Diagnostic command: pytest tests/test_example.py -q",
|
|
"Diagnostic result: passed after local fix",
|
|
"Diagnostic results used only for suggested fix; blocker remains PR-head failure.",
|
|
])
|
|
|
|
|
|
class TestValidationIntegrity(unittest.TestCase):
|
|
def test_official_only_passes(self):
|
|
result = assess_validation_integrity_report(
|
|
_official_only_report(),
|
|
validation_session={
|
|
"official_validation_ran": True,
|
|
"official_result": "fail",
|
|
"worktree_dirty_after_official": False,
|
|
},
|
|
)
|
|
self.assertTrue(result["proven"], result["reasons"])
|
|
|
|
def test_failing_pr_head_with_diagnostic_pass_passes(self):
|
|
result = assess_validation_integrity_report(
|
|
_diagnostic_report(),
|
|
validation_session={
|
|
"official_validation_ran": True,
|
|
"official_result": "fail",
|
|
"diagnostic_validation_ran": True,
|
|
"diagnostic_result": "pass",
|
|
"diagnostic_edits": ["tests/test_example.py"],
|
|
"worktree_dirty_after_official": False,
|
|
},
|
|
)
|
|
self.assertTrue(result["proven"], result["reasons"])
|
|
|
|
def test_post_edit_tests_as_official_blocks(self):
|
|
report = "\n".join([
|
|
"Official validation result: passed after diagnostic edit",
|
|
"pytest passed after local fix in validation worktree",
|
|
])
|
|
result = assess_validation_integrity_report(
|
|
report,
|
|
validation_session={
|
|
"official_validation_ran": True,
|
|
"diagnostic_edits": ["tests/test_example.py"],
|
|
"diagnostic_validation_ran": True,
|
|
},
|
|
)
|
|
self.assertFalse(result["proven"])
|
|
self.assertTrue(result["block"])
|
|
|
|
def test_missing_dirty_after_validation_blocks(self):
|
|
report = _official_only_report().replace(
|
|
"Worktree dirty after official validation: clean",
|
|
"",
|
|
)
|
|
result = assess_validation_integrity_report(
|
|
report,
|
|
validation_session={"official_validation_ran": True, "official_result": "fail"},
|
|
)
|
|
self.assertFalse(result["proven"])
|
|
self.assertTrue(any("dirty" in r.lower() for r in result["reasons"]))
|
|
|
|
def test_diagnostic_without_label_blocks(self):
|
|
report = _diagnostic_report().replace(
|
|
"Diagnostic local experiment — not PR-head validation",
|
|
"",
|
|
)
|
|
result = assess_validation_integrity_report(
|
|
report,
|
|
validation_session={
|
|
"official_validation_ran": True,
|
|
"diagnostic_edits": ["tests/test_example.py"],
|
|
"diagnostic_validation_ran": True,
|
|
},
|
|
)
|
|
self.assertFalse(result["proven"])
|
|
self.assertTrue(any("diagnostic" in r.lower() for r in result["reasons"]))
|
|
|
|
def test_contaminated_requires_integrity_status(self):
|
|
report = "Validation worktree was edited before official validation completed."
|
|
result = assess_validation_integrity_report(
|
|
report,
|
|
validation_session={"contaminated": True, "official_validation_ran": True},
|
|
)
|
|
self.assertFalse(result["proven"])
|
|
self.assertTrue(any("contaminated" in r.lower() for r in result["reasons"]))
|
|
|
|
def test_action_log_edits_trigger_diagnostic_rules(self):
|
|
result = assess_validation_integrity_report(
|
|
_official_only_report(),
|
|
validation_session={"official_validation_ran": True, "official_result": "fail"},
|
|
action_log=[{"path": "tests/test_example.py", "kind": "file_edit", "performed": True}],
|
|
)
|
|
self.assertFalse(result["proven"])
|
|
|
|
|
|
class TestExport(unittest.TestCase):
|
|
def test_review_proofs_reexport(self):
|
|
from review_proofs import assess_validation_integrity_report as exported
|
|
|
|
self.assertTrue(callable(exported))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|