feat(review-workflow): raise A-bar with capability, sweep, live-state, and role-boundary proofs (#179)
Extends review_proofs.py with the four #179 proofs, the successor set to the #173 checkout/inventory proofs: - assess_capability_evidence: a capability claim (review_pr, merge_pr, ...) counts only with exact evidence citing gitea_resolve_task_capability output or equivalent runtime context; no claims at all fails closed. - assess_sweep_evidence: secret/provenance sweeps must state the exact command/script/pattern/named method, the scope scanned, and a boolean result; vague summaries are downgraded and a missing sweep fails closed. - assess_live_state_recheck: an explicit pre-mutation recheck must prove the PR is still open, the live head equals the pinned head (full 40-hex), the base branch is unchanged, and blocking review state was checked and absent; not performing it fails closed. - assess_role_boundary: a reviewer run using an author namespace (or vice versa) is clean only with an explicit justification; unreported namespace usage fails closed. build_final_report now takes the four proofs as keyword arguments: any missing or failed proof downgrades the grade, merge_allowed additionally requires the proven live-state recheck, and a merge performed without it is a blocked violation. Existing #173 semantics are unchanged otherwise; gates only get stricter. tests/test_review_proofs.py adds 29 tests covering the issue's harness assertions: capability claims without evidence downgraded, vague sweeps downgraded, missing/stale live-state recheck downgrades and blocks merge (violation when a merge is claimed anyway), unjustified author-namespace use downgraded, and the #173 positive baseline preserved. SKILL.md sections F/G and the review-pr/merge-pr templates now require the capability evidence, exact sweep, pre-verdict and pre-merge live-state rechecks, and reviewer-namespace discipline. Closes #179 Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
@@ -21,10 +21,13 @@ import unittest
|
||||
sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent.parent))
|
||||
|
||||
from review_proofs import ( # noqa: E402
|
||||
assess_capability_evidence,
|
||||
assess_controller_handoff,
|
||||
assess_inventory_completeness,
|
||||
assess_live_state_recheck,
|
||||
assess_role_boundary,
|
||||
assess_self_review_contamination,
|
||||
assess_sweep_evidence,
|
||||
assess_validation_report,
|
||||
build_final_report,
|
||||
pr_inventory_trust_gate,
|
||||
@@ -116,6 +119,63 @@ def _good_role_boundary():
|
||||
)
|
||||
|
||||
|
||||
def _good_capability_evidence():
|
||||
return assess_capability_evidence([
|
||||
{
|
||||
"task": "review_pr",
|
||||
"allowed": True,
|
||||
"evidence_source": (
|
||||
"gitea_resolve_task_capability(review_pr) output: "
|
||||
"allowed_in_current_session=true, profile prgs-reviewer"
|
||||
),
|
||||
},
|
||||
{
|
||||
"task": "merge_pr",
|
||||
"allowed": True,
|
||||
"evidence_source": (
|
||||
"gitea_resolve_task_capability(merge_pr) output: "
|
||||
"allowed_in_current_session=true, profile prgs-reviewer"
|
||||
),
|
||||
},
|
||||
])
|
||||
|
||||
|
||||
def _good_sweep(**overrides):
|
||||
sweep = {
|
||||
"command": (
|
||||
"git diff prgs/master...HEAD | grep -inE "
|
||||
"'password|token|secret|api[_-]?key|authorization|bearer|https?://'"
|
||||
),
|
||||
"scope": "full PR diff against prgs/master",
|
||||
"clean": True,
|
||||
}
|
||||
sweep.update(overrides)
|
||||
return assess_sweep_evidence(sweep)
|
||||
|
||||
|
||||
def _good_live_state(**overrides):
|
||||
recheck = {
|
||||
"pr_state": "open",
|
||||
"pinned_head_sha": PINNED,
|
||||
"live_head_sha": PINNED,
|
||||
"pinned_base_ref": "master",
|
||||
"live_base_ref": "master",
|
||||
"blocking_change_requests": False,
|
||||
}
|
||||
recheck.update(overrides)
|
||||
return assess_live_state_recheck(recheck)
|
||||
|
||||
|
||||
def _good_role_boundary_179(**overrides):
|
||||
kwargs = {
|
||||
"task_role": "reviewer",
|
||||
"namespaces_used": ["gitea-reviewer"],
|
||||
"justification": None,
|
||||
}
|
||||
kwargs.update(overrides)
|
||||
return assess_role_boundary(**kwargs)
|
||||
|
||||
|
||||
class TestCheckoutProof(unittest.TestCase):
|
||||
"""Required behavior 1 + 2: prove HEAD == pinned PR head or stop."""
|
||||
|
||||
@@ -466,6 +526,9 @@ class TestFinalReport(unittest.TestCase):
|
||||
"identity_eligible": True,
|
||||
"merge_performed": False,
|
||||
"issue_status_verified": True,
|
||||
"capability_evidence": _good_capability_evidence(),
|
||||
"sweep": _good_sweep(),
|
||||
"live_state": _good_live_state(),
|
||||
"role_boundary": _good_role_boundary(),
|
||||
}
|
||||
kwargs.update(overrides)
|
||||
@@ -879,5 +942,209 @@ class TestPRInventoryTrustGate(unittest.TestCase):
|
||||
self.assertTrue(res["corroborated"])
|
||||
|
||||
|
||||
class TestCapabilityEvidence(unittest.TestCase):
|
||||
"""#179 gap 1: capability claims need exact evidence."""
|
||||
|
||||
def test_evidence_backed_claims_are_proven(self):
|
||||
result = _good_capability_evidence()
|
||||
self.assertTrue(result["proven"])
|
||||
self.assertEqual(result["reasons"], [])
|
||||
|
||||
def test_claim_without_evidence_source_is_not_proven(self):
|
||||
result = assess_capability_evidence([
|
||||
{"task": "review_pr", "allowed": True, "evidence_source": ""},
|
||||
])
|
||||
self.assertFalse(result["proven"])
|
||||
self.assertTrue(any("evidence" in r.lower() for r in result["reasons"]))
|
||||
|
||||
def test_no_claims_at_all_fails_closed(self):
|
||||
result = assess_capability_evidence([])
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
def test_disallowed_task_is_not_proven(self):
|
||||
result = assess_capability_evidence([
|
||||
{
|
||||
"task": "merge_pr",
|
||||
"allowed": False,
|
||||
"evidence_source": "gitea_resolve_task_capability output",
|
||||
},
|
||||
])
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
|
||||
class TestSweepEvidence(unittest.TestCase):
|
||||
"""#179 gap 2: secret/provenance sweep must be exact."""
|
||||
|
||||
def test_exact_sweep_is_proven(self):
|
||||
result = _good_sweep()
|
||||
self.assertEqual(result["verdict"], "exact")
|
||||
self.assertTrue(result["proven"])
|
||||
|
||||
def test_vague_sweep_without_command_is_downgraded(self):
|
||||
result = _good_sweep(command="")
|
||||
self.assertEqual(result["verdict"], "vague")
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
def test_sweep_without_scope_is_downgraded(self):
|
||||
result = _good_sweep(scope="")
|
||||
self.assertEqual(result["verdict"], "vague")
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
def test_missing_sweep_fails_closed(self):
|
||||
result = assess_sweep_evidence(None)
|
||||
self.assertEqual(result["verdict"], "missing")
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
def test_unstated_result_is_downgraded(self):
|
||||
result = _good_sweep(clean=None)
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
|
||||
class TestLiveStateRecheck(unittest.TestCase):
|
||||
"""#179 gap 3: explicit pre-mutation live-state recheck."""
|
||||
|
||||
def test_clean_recheck_is_proven(self):
|
||||
result = _good_live_state()
|
||||
self.assertTrue(result["proven"])
|
||||
self.assertFalse(result["block"])
|
||||
|
||||
def test_missing_recheck_fails_closed(self):
|
||||
result = assess_live_state_recheck(None)
|
||||
self.assertFalse(result["proven"])
|
||||
self.assertTrue(result["block"])
|
||||
|
||||
def test_closed_pr_blocks(self):
|
||||
result = _good_live_state(pr_state="closed")
|
||||
self.assertFalse(result["proven"])
|
||||
self.assertTrue(result["block"])
|
||||
|
||||
def test_moved_head_blocks(self):
|
||||
result = _good_live_state(live_head_sha=OTHER)
|
||||
self.assertFalse(result["proven"])
|
||||
self.assertTrue(any("head" in r.lower() for r in result["reasons"]))
|
||||
|
||||
def test_changed_base_blocks(self):
|
||||
result = _good_live_state(live_base_ref="develop")
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
def test_unresolved_blocking_reviews_block(self):
|
||||
result = _good_live_state(blocking_change_requests=True)
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
def test_unchecked_blocking_state_fails_closed(self):
|
||||
result = _good_live_state(blocking_change_requests=None)
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
|
||||
class TestRoleBoundary179(unittest.TestCase):
|
||||
"""#179 gap 4: reviewer flows avoid unjustified author-namespace use."""
|
||||
|
||||
def test_native_namespace_only_is_clean(self):
|
||||
result = _good_role_boundary_179()
|
||||
self.assertTrue(result["proven"])
|
||||
|
||||
def test_foreign_namespace_without_justification_is_downgraded(self):
|
||||
result = _good_role_boundary_179(
|
||||
namespaces_used=["gitea-reviewer", "gitea-author"]
|
||||
)
|
||||
self.assertFalse(result["proven"])
|
||||
self.assertTrue(any("justif" in r.lower() for r in result["reasons"]))
|
||||
|
||||
def test_foreign_namespace_with_justification_is_clean(self):
|
||||
result = _good_role_boundary_179(
|
||||
namespaces_used=["gitea-reviewer", "gitea-author"],
|
||||
justification=(
|
||||
"author namespace read-only whoami used to evidence "
|
||||
"self-review contamination status"
|
||||
),
|
||||
)
|
||||
self.assertTrue(result["proven"])
|
||||
|
||||
def test_unreported_namespaces_fail_closed(self):
|
||||
result = _good_role_boundary_179(namespaces_used=None)
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
|
||||
class TestFinalReport179Bar(unittest.TestCase):
|
||||
"""#179 acceptance adds capability, sweep, live-state, and role proofs."""
|
||||
|
||||
def _report(self, **overrides):
|
||||
kwargs = {
|
||||
"checkout_proof": _good_checkout(),
|
||||
"inventory": _good_inventory(),
|
||||
"validation": _good_validation(),
|
||||
"contamination": _good_contamination(),
|
||||
"identity_eligible": True,
|
||||
"merge_performed": False,
|
||||
"issue_status_verified": True,
|
||||
"capability_evidence": _good_capability_evidence(),
|
||||
"sweep": _good_sweep(),
|
||||
"live_state": _good_live_state(),
|
||||
"role_boundary": _good_role_boundary(),
|
||||
}
|
||||
kwargs.update(overrides)
|
||||
return build_final_report(**kwargs)
|
||||
|
||||
def test_all_179_proofs_present_is_grade_a(self):
|
||||
report = self._report()
|
||||
self.assertEqual(report["grade"], "A")
|
||||
self.assertTrue(report["capability_evidence_proven"])
|
||||
self.assertEqual(report["sweep_verdict"], "exact")
|
||||
self.assertTrue(report["live_state_recheck_proven"])
|
||||
self.assertTrue(report["role_boundary_clean"])
|
||||
|
||||
def test_missing_capability_evidence_downgrades(self):
|
||||
report = self._report(capability_evidence=None)
|
||||
self.assertNotEqual(report["grade"], "A")
|
||||
self.assertFalse(report["capability_evidence_proven"])
|
||||
|
||||
def test_unevidenced_capability_claim_downgrades(self):
|
||||
report = self._report(
|
||||
capability_evidence=assess_capability_evidence([
|
||||
{"task": "review_pr", "allowed": True, "evidence_source": ""},
|
||||
])
|
||||
)
|
||||
self.assertNotEqual(report["grade"], "A")
|
||||
|
||||
def test_vague_sweep_downgrades(self):
|
||||
report = self._report(sweep=_good_sweep(command=""))
|
||||
self.assertNotEqual(report["grade"], "A")
|
||||
self.assertEqual(report["sweep_verdict"], "vague")
|
||||
|
||||
def test_missing_sweep_downgrades(self):
|
||||
report = self._report(sweep=None)
|
||||
self.assertNotEqual(report["grade"], "A")
|
||||
|
||||
def test_missing_live_state_recheck_downgrades_and_blocks_merge(self):
|
||||
report = self._report(live_state=None)
|
||||
self.assertNotEqual(report["grade"], "A")
|
||||
self.assertFalse(report["merge_allowed"])
|
||||
self.assertFalse(report["live_state_recheck_proven"])
|
||||
|
||||
def test_stale_live_state_blocks_merge(self):
|
||||
report = self._report(live_state=_good_live_state(live_head_sha=OTHER))
|
||||
self.assertNotEqual(report["grade"], "A")
|
||||
self.assertFalse(report["merge_allowed"])
|
||||
|
||||
def test_merge_claim_without_live_recheck_is_a_violation(self):
|
||||
report = self._report(live_state=None, merge_performed=True)
|
||||
self.assertEqual(report["grade"], "blocked")
|
||||
self.assertTrue(report["violations"])
|
||||
|
||||
def test_unjustified_author_namespace_downgrades(self):
|
||||
report = self._report(
|
||||
role_boundary=_good_role_boundary_179(
|
||||
namespaces_used=["gitea-reviewer", "gitea-author"]
|
||||
)
|
||||
)
|
||||
self.assertNotEqual(report["grade"], "A")
|
||||
self.assertFalse(report["role_boundary_clean"])
|
||||
|
||||
def test_positive_baseline_from_173_still_holds(self):
|
||||
report = self._report()
|
||||
self.assertTrue(report["inventory_complete"])
|
||||
self.assertTrue(report["validated_on_pinned_head"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user