feat: require proof-backed PR review handoff claims (Closes #395)
Reviewer final reports must not claim proof-sensitive workflow steps unless each claim carries explicit command/tool evidence or structured MCP result metadata: - New reviewer_proof_backed_claims.py verifier covering the five claim families from the issue: inventory completeness (explicit pagination metadata required; page-size assumptions rejected), earlier-PR skips (per-PR command/tool or structured mergeability/blocker evidence), baseline validation (worktree path, target SHA, dirty-before, exact command, exact result, dirty-after), master integration (exact merge/rebase/simulation command), and cleanup (final git worktree list proof). - PROOF_PROVENANCE_CLASSES constant distinguishes command actually run, MCP metadata, inherited from previous blocker state, and not checked. - Re-export from review_proofs and wire into build_final_report as a downgrade check with proof_backed_claims_proven/violations fields. - New workflow section 30A documents the rule; contract test locks the marker and an export test locks the re-export. - 13 tests in tests/test_reviewer_proof_backed_claims.py covering the required scenarios: baseline claim without command, skipped-PR claim without evidence, inventory-complete from page-size assumption only, cleanup without final worktree proof, and a fully proof-backed report passing. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,132 @@
|
||||
"""Proof-backed reviewer handoff claim verifier (#395).
|
||||
|
||||
Reviewer final reports must not claim proof-sensitive workflow steps unless
|
||||
each claim is backed by explicit command/tool evidence in the transcript or
|
||||
structured MCP result metadata. This verifier scans report text for the five
|
||||
proof-sensitive claim families from issue #395 (inventory completeness,
|
||||
earlier-PR skips, baseline validation, master integration, cleanup) and fails
|
||||
closed when a claim appears without its required evidence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
# Issue #395 criterion 6: reports must distinguish these proof provenances.
|
||||
PROOF_PROVENANCE_CLASSES = (
|
||||
"command actually run",
|
||||
"MCP metadata",
|
||||
"inherited from previous blocker state",
|
||||
"not checked",
|
||||
)
|
||||
|
||||
_INVENTORY_CLAIM_RE = re.compile(
|
||||
r"inventory\s+(?:complete|completeness\s*:\s*complete)", re.IGNORECASE
|
||||
)
|
||||
_PAGINATION_METADATA_RE = re.compile(
|
||||
r"(?:has_more\s*=?\s*false|is_final_page\s*=?\s*true|"
|
||||
r"inventory_complete\s*=?\s*true|total[_ ]count\s*[=:]\s*\d+|"
|
||||
r"no next page|final page|pagination complete)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_PAGE_SIZE_ASSUMPTION_RE = re.compile(
|
||||
r"(?:less|fewer)\s+than\s+(?:the\s+)?(?:default\s+)?page\s*size|"
|
||||
r"under\s+the\s+(?:default\s+)?page\s*size|"
|
||||
r"below\s+(?:the\s+)?(?:default\s+)?page\s*size",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_SKIPPED_PR_RE = re.compile(r"skipped\s+PR\s*#?(\d+)", re.IGNORECASE)
|
||||
_SKIP_EVIDENCE_RE = re.compile(
|
||||
r"(?:mergeable\s*:\s*false|gitea_view_pr|gitea_get_pr_review_feedback|"
|
||||
r"gitea_check_pr_eligibility|git\s+merge-tree|merge simulation|"
|
||||
r"conflicting files?\s*:|REQUEST_CHANGES\s+at\s+(?:current\s+)?head|"
|
||||
r"prior blocker reused|blocker revalidated live)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_BASELINE_CLAIM_RE = re.compile(r"\bbaseline\b", re.IGNORECASE)
|
||||
_BASELINE_REQUIRED = (
|
||||
("baseline worktree path", re.compile(r"branches/baseline-[\w./-]+", re.IGNORECASE)),
|
||||
("baseline target SHA", re.compile(r"target\s+SHA\s*:?\s*[0-9a-f]{7,40}", re.IGNORECASE)),
|
||||
("baseline dirty-before status", re.compile(r"dirty\s+before\s*:?\s*\w+", re.IGNORECASE)),
|
||||
("exact baseline validation command", re.compile(r"(?:command\s*:|pytest|python\s+-m)", re.IGNORECASE)),
|
||||
("exact baseline result", re.compile(r"(?:result\s*:|passed|failed)", re.IGNORECASE)),
|
||||
("baseline dirty-after status", re.compile(r"dirty\s+after\s*:?\s*\w+", re.IGNORECASE)),
|
||||
)
|
||||
|
||||
_INTEGRATION_CLAIM_RE = re.compile(
|
||||
r"(?:master\s+(?:was\s+)?(?:merged|rebased|integrated)|"
|
||||
r"merged?\s+master\s+into|master[- ]integration|rebased\s+onto\s+master)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_INTEGRATION_EVIDENCE_RE = re.compile(
|
||||
r"(?:git\s+merge(?:\s+--no-commit|\s+--no-ff)?|git\s+rebase|git\s+merge-tree)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_CLEANUP_CLAIM_RE = re.compile(
|
||||
r"worktrees?\s+(?:were\s+)?(?:removed|cleaned|deleted)|"
|
||||
r"cleanup\s*:\s*(?!none)\S",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_CLEANUP_EVIDENCE_RE = re.compile(r"git\s+worktree\s+list", re.IGNORECASE)
|
||||
|
||||
|
||||
def assess_proof_backed_handoff_report(report_text: str) -> dict[str, Any]:
|
||||
"""Fail closed on proof-sensitive claims that lack explicit evidence (#395)."""
|
||||
text = report_text or ""
|
||||
reasons: list[str] = []
|
||||
|
||||
if _INVENTORY_CLAIM_RE.search(text):
|
||||
if _PAGE_SIZE_ASSUMPTION_RE.search(text):
|
||||
reasons.append(
|
||||
"inventory completeness claimed from a page size assumption; "
|
||||
"explicit pagination metadata (has_more=false / is_final_page=true / "
|
||||
"inventory_complete=true / total count) is required"
|
||||
)
|
||||
elif not _PAGINATION_METADATA_RE.search(text):
|
||||
reasons.append(
|
||||
"inventory completeness claimed without explicit pagination "
|
||||
"metadata proof (has_more=false, is_final_page=true, "
|
||||
"inventory_complete=true, or total count)"
|
||||
)
|
||||
|
||||
for match in _SKIPPED_PR_RE.finditer(text):
|
||||
pr_number = match.group(1)
|
||||
window = text[match.start():match.start() + 400]
|
||||
if not _SKIP_EVIDENCE_RE.search(window):
|
||||
reasons.append(
|
||||
f"skipped PR #{pr_number} claim has no explicit command/tool "
|
||||
"proof or structured mergeability/blocker evidence"
|
||||
)
|
||||
|
||||
if _BASELINE_CLAIM_RE.search(text):
|
||||
missing = [
|
||||
name for name, pattern in _BASELINE_REQUIRED if not pattern.search(text)
|
||||
]
|
||||
if missing:
|
||||
reasons.append(
|
||||
"baseline validation claim missing required proof fields: "
|
||||
+ ", ".join(missing)
|
||||
)
|
||||
|
||||
if _INTEGRATION_CLAIM_RE.search(text) and not _INTEGRATION_EVIDENCE_RE.search(text):
|
||||
reasons.append(
|
||||
"master-integration claim lacks the exact merge/rebase/simulation "
|
||||
"command and result"
|
||||
)
|
||||
|
||||
if _CLEANUP_CLAIM_RE.search(text) and not _CLEANUP_EVIDENCE_RE.search(text):
|
||||
reasons.append(
|
||||
"cleanup claim lacks final `git worktree list` (or equivalent) proof "
|
||||
"that session-owned worktrees were removed"
|
||||
)
|
||||
|
||||
return {
|
||||
"proven": not reasons,
|
||||
"block": bool(reasons),
|
||||
"reasons": reasons,
|
||||
"provenance_classes": PROOF_PROVENANCE_CLASSES,
|
||||
}
|
||||
Reference in New Issue
Block a user