feat: add validation ledger and final-report verifier (Closes #271)
Introduce validation_ledger with command ledger format, full-suite claim checks, and structured final-report verification so sessions cannot overclaim validation or misreport review/merge state. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
@@ -0,0 +1,155 @@
|
||||
"""Tests for validation ledger and final-report verifier (#271)."""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from validation_ledger import ( # noqa: E402
|
||||
assess_full_suite_claim,
|
||||
assess_ledger_entry,
|
||||
new_ledger_entry,
|
||||
verify_final_report,
|
||||
)
|
||||
|
||||
|
||||
class TestLedgerEntry(unittest.TestCase):
|
||||
def test_valid_entry_is_claimable(self):
|
||||
entry = new_ledger_entry(
|
||||
command="pytest tests/ -q",
|
||||
cwd="/repo/branches/feat-issue-271",
|
||||
exit_code=0,
|
||||
output_summary="994 passed, 6 skipped",
|
||||
)
|
||||
result = assess_ledger_entry(entry)
|
||||
self.assertTrue(result["claimable"])
|
||||
self.assertEqual(result["verdict"], "strong")
|
||||
|
||||
def test_missing_command_output_is_invalid(self):
|
||||
entry = new_ledger_entry(
|
||||
command="pytest tests/ -q",
|
||||
cwd="/repo/branches/task",
|
||||
exit_code=0,
|
||||
output_summary="",
|
||||
)
|
||||
result = assess_ledger_entry(entry)
|
||||
self.assertFalse(result["claimable"])
|
||||
self.assertEqual(result["verdict"], "invalid")
|
||||
|
||||
|
||||
class TestFullSuiteClaim(unittest.TestCase):
|
||||
def _entry(self, **kwargs):
|
||||
base = new_ledger_entry(
|
||||
command="pytest tests/ -q",
|
||||
cwd="/repo/branches/task",
|
||||
exit_code=0,
|
||||
output_summary="10 passed",
|
||||
)
|
||||
base.update(kwargs)
|
||||
return base
|
||||
|
||||
def test_full_suite_overclaim_with_ignored_paths_fails(self):
|
||||
entry = self._entry(
|
||||
ignored_paths=[{"path": "tests/integration/", "justification": ""}],
|
||||
)
|
||||
result = assess_full_suite_claim(
|
||||
[entry],
|
||||
report_text="Validation: full suite passed.",
|
||||
)
|
||||
self.assertFalse(result["claimable"])
|
||||
self.assertTrue(
|
||||
any("full suite" in r.lower() for r in result["reasons"])
|
||||
)
|
||||
|
||||
def test_full_suite_except_note_allows_ignored_paths(self):
|
||||
entry = self._entry(
|
||||
ignored_paths=[{
|
||||
"path": "tests/integration/",
|
||||
"justification": "network tests require GITEA_INTEGRATION=1",
|
||||
}],
|
||||
)
|
||||
result = assess_full_suite_claim(
|
||||
[entry],
|
||||
report_text="Validation: full suite except integration tests.",
|
||||
)
|
||||
self.assertTrue(result["claimable"])
|
||||
|
||||
|
||||
class TestFinalReportVerifier(unittest.TestCase):
|
||||
def _base_report(self, **overrides):
|
||||
report = {
|
||||
"identity": "jcwalker3",
|
||||
"profile": "prgs-author",
|
||||
"capability_resolved": True,
|
||||
"worktree_path": "/repo/branches/feat-issue-271",
|
||||
"changed_files": ["validation_ledger.py"],
|
||||
"validation_ledger": [
|
||||
new_ledger_entry(
|
||||
command="pytest tests/test_validation_ledger.py -q",
|
||||
cwd="/repo/branches/feat-issue-271",
|
||||
exit_code=0,
|
||||
output_summary="12 passed",
|
||||
)
|
||||
],
|
||||
"validation_claim": "passed",
|
||||
"review_submitted": False,
|
||||
"review_visible_approved": False,
|
||||
"review_pending": False,
|
||||
"merge_performed": False,
|
||||
"merge_blocker": "not a reviewer task",
|
||||
"workspace_clean": True,
|
||||
}
|
||||
report.update(overrides)
|
||||
return report
|
||||
|
||||
def test_complete_report_is_proven(self):
|
||||
result = verify_final_report(self._base_report())
|
||||
self.assertTrue(result["proven"])
|
||||
self.assertFalse(result["block"])
|
||||
|
||||
def test_validation_overclaim_fails(self):
|
||||
report = self._base_report(
|
||||
validation_claim="passed",
|
||||
validation_ledger=[
|
||||
new_ledger_entry(
|
||||
command="pytest tests/ -q",
|
||||
cwd="/repo/branches/task",
|
||||
exit_code=1,
|
||||
output_summary="3 failed",
|
||||
)
|
||||
],
|
||||
)
|
||||
result = verify_final_report(report)
|
||||
self.assertFalse(result["proven"])
|
||||
self.assertTrue(result["contradictions"])
|
||||
|
||||
def test_pending_approval_misreported_as_approved_fails(self):
|
||||
report = self._base_report(
|
||||
claims_approved_review=True,
|
||||
review_visible_approved=False,
|
||||
review_pending=True,
|
||||
)
|
||||
result = verify_final_report(report)
|
||||
self.assertFalse(result["proven"])
|
||||
self.assertTrue(
|
||||
any("approved" in c.lower() for c in result["contradictions"])
|
||||
)
|
||||
|
||||
def test_missing_command_output_in_ledger_fails(self):
|
||||
report = self._base_report(
|
||||
validation_ledger=[
|
||||
new_ledger_entry(
|
||||
command="pytest tests/ -q",
|
||||
cwd="/repo/branches/task",
|
||||
exit_code=0,
|
||||
output_summary="",
|
||||
)
|
||||
],
|
||||
)
|
||||
result = verify_final_report(report)
|
||||
self.assertFalse(result["proven"])
|
||||
self.assertTrue(result["reasons"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user