From da9c0ba54cc6de37e114ebcac789b764952c2269 Mon Sep 17 00:00:00 2001 From: Jason Walker <913443@dadeschools.net> Date: Tue, 7 Jul 2026 05:08:42 -0400 Subject: [PATCH] feat: enforce validation environment proof in reviewer reports (Closes #311) Add assess_validation_environment_proof requiring exact validation command, working directory, result evidence, and bare-pytest executable/version/venv proof before reviewer reports can claim validation success. Co-Authored-By: Claude Opus 4.8 (1M context) --- review_proofs.py | 216 +++++++++++++++++++++++++++++++++++- tests/test_review_proofs.py | 103 +++++++++++++++++ 2 files changed, 318 insertions(+), 1 deletion(-) diff --git a/review_proofs.py b/review_proofs.py index 9cd7e8c..3962e41 100644 --- a/review_proofs.py +++ b/review_proofs.py @@ -1231,7 +1231,8 @@ def build_final_report(checkout_proof, inventory, validation, contamination, role_boundary=None, review_mutation=None, report_text=None, review_decision_lock=None, controller_handoff=None, capability_proof=None, - sweep_proof=None, worktree_proof=None): + sweep_proof=None, worktree_proof=None, + validation_environment=None): """Required behavior 6 + acceptance criteria: one report, distinct proofs. Combines the individual proof verdicts into the final-report fields the @@ -1258,6 +1259,22 @@ def build_final_report(checkout_proof, inventory, validation, contamination, ) empty_queue_report = assess_empty_queue_report(report_text) + if validation_environment is None and report_text: + validation_env_proof = assess_validation_environment_proof( + report_text=report_text, + ) + elif validation_environment is not None: + validation_env_proof = assess_validation_environment_proof( + report_text=report_text, + validation_environment=validation_environment, + ) + else: + validation_env_proof = { + "proven": True, + "block": False, + "reasons": [], + "violations": [], + } contamination_status = contamination.get("status", "unknown") checkout_proven = bool(checkout_proof.get("proven")) @@ -1387,6 +1404,11 @@ def build_final_report(checkout_proof, inventory, validation, contamination, "empty-queue report missing or failed trust-gate proof (#198)" ) downgrade_reasons.extend(empty_queue_report.get("reasons", [])) + if not validation_env_proof.get("proven"): + downgrade_reasons.append( + "validation environment proof missing or failed (#311)" + ) + downgrade_reasons.extend(validation_env_proof.get("reasons", [])) merge_allowed = ( identity_eligible @@ -1398,9 +1420,11 @@ def build_final_report(checkout_proof, inventory, validation, contamination, # #179: no merge without a proven final live-state recheck. and live_state_proven and worktree_proven + and validation_env_proof.get("proven") ) violations = [] + violations.extend(validation_env_proof.get("violations", [])) if merge_performed and not merge_allowed: violations.append( "merge was performed/claimed although the proofs did not allow " @@ -1452,6 +1476,12 @@ def build_final_report(checkout_proof, inventory, validation, contamination, else True ), "empty_queue_trust_gate_status": empty_queue_report.get("status"), + "validation_environment_proven": bool( + validation_env_proof.get("proven") + ), + "validation_environment_violations": list( + validation_env_proof.get("violations") or [] + ), } @@ -3020,6 +3050,190 @@ def build_review_mutation_proof(run_log: list[dict]) -> dict: } +# --------------------------------------------------------------------------- +# Validation environment proof (#311) +# --------------------------------------------------------------------------- + +_VALIDATION_CLAIM_RE = re.compile( + r"(?:validation\s+command|pytest\s+(?:executed|passed|failed)|" + r"\d+\s+passed|\d+\s+failed|tests?\s+passed)", + re.IGNORECASE, +) + +_VALIDATION_CMD_LINE_RE = re.compile( + r"validation\s+command\s*:\s*(.+)", + re.IGNORECASE, +) + +_VALIDATION_CWD_LINE_RE = re.compile( + r"working\s+directory\s*:\s*(\S+)", + re.IGNORECASE, +) + +_WHICH_PYTEST_LINE_RE = re.compile( + r"which\s+pytest\s*:\s*(.+)", + re.IGNORECASE, +) + +_PYTEST_VERSION_LINE_RE = re.compile( + r"pytest\s+--version\s*:\s*(.+)", + re.IGNORECASE, +) + +_BARE_PYTEST_CMD_RE = re.compile( + r"^(?:python\s+-m\s+)?pytest(?:\s|$)", + re.IGNORECASE, +) + +_VAGUE_PYTEST_CLAIM_RE = re.compile( + r"pytest\s+(?:executed|passed|failed)\b", + re.IGNORECASE, +) + + +def _parse_validation_command(report_text: str) -> str: + for line in (report_text or "").splitlines(): + match = _VALIDATION_CMD_LINE_RE.search(line) + if match: + return match.group(1).strip().rstrip(".") + return "" + + +def _parse_validation_cwd(report_text: str) -> str: + for line in (report_text or "").splitlines(): + match = _VALIDATION_CWD_LINE_RE.search(line) + if match: + return match.group(1).strip().rstrip(",.;") + return "" + + +def _is_bare_pytest_command(command: str) -> bool: + text = (command or "").strip() + if not text: + return False + if "/" in text or "\\" in text: + return False + return bool(_BARE_PYTEST_CMD_RE.match(text)) + + +def assess_validation_environment_proof( + *, + report_text: str | None = None, + validation_environment: dict | None = None, +) -> dict: + """#311: reviewer reports must include exact validation command/environment.""" + text = report_text or "" + lower = text.lower() + env = validation_environment or {} + reasons: list[str] = [] + violations: list[str] = [] + + claims_validation = bool(_VALIDATION_CLAIM_RE.search(text)) + if not claims_validation and not env.get("command"): + return { + "proven": True, + "block": False, + "claims_validation": False, + "reasons": [], + "violations": [], + "safe_next_action": "proceed", + } + + command = (env.get("command") or _parse_validation_command(text)).strip() + working_directory = ( + env.get("working_directory") or env.get("cwd") + or _parse_validation_cwd(text) + ).strip() + + if not command: + reasons.append( + "validation result claimed without exact validation command (#311)" + ) + elif _VAGUE_PYTEST_CLAIM_RE.search(text) and command == "": + violations.append( + "vague pytest summary without exact validation command (#311)" + ) + + if not working_directory: + reasons.append( + "validation result claimed without working directory (#311)" + ) + + has_result = any( + token in lower + for token in ("passed", "failed", "skipped", "error") + ) or any( + env.get(key) is not None + for key in ("passed", "failed", "skipped", "result") + ) + if not has_result: + reasons.append( + "validation result claimed without pass/fail result evidence (#311)" + ) + + if command and _is_bare_pytest_command(command): + which_pytest = ( + (env.get("which_pytest") or env.get("executable_path") or "") + .strip() + ) + if not which_pytest: + for line in text.splitlines(): + match = _WHICH_PYTEST_LINE_RE.search(line) + if match: + which_pytest = match.group(1).strip() + break + if not which_pytest: + reasons.append( + "bare pytest command requires which-pytest/executable proof (#311)" + ) + + pytest_version = (env.get("pytest_version") or "").strip() + if not pytest_version: + for line in text.splitlines(): + match = _PYTEST_VERSION_LINE_RE.search(line) + if match: + pytest_version = match.group(1).strip() + break + if not pytest_version: + reasons.append( + "bare pytest command requires pytest --version proof (#311)" + ) + + venv_resolved = env.get("venv_resolved") + if venv_resolved is None: + venv_resolved = any( + marker in lower + for marker in ( + "project venv", + "venv/bin/pytest", + "resolves to the project venv", + "venv resolved: yes", + ) + ) + if not venv_resolved and which_pytest and "venv" not in which_pytest: + reasons.append( + "bare pytest command requires proof whether executable " + "resolves to the project venv (#311)" + ) + + proven = not reasons and not violations + return { + "proven": proven, + "block": bool(violations), + "claims_validation": True, + "command": command or None, + "working_directory": working_directory or None, + "reasons": reasons, + "violations": violations, + "safe_next_action": ( + "report exact validation command, working directory, result, and " + "for bare pytest also which-pytest, pytest --version, and venv proof" + if not proven + else "proceed" + ), + } + + # --------------------------------------------------------------------------- # Identity disclosure (#305) # --------------------------------------------------------------------------- diff --git a/tests/test_review_proofs.py b/tests/test_review_proofs.py index b4683dc..10fd0d3 100644 --- a/tests/test_review_proofs.py +++ b/tests/test_review_proofs.py @@ -24,6 +24,7 @@ from review_proofs import ( # noqa: E402 ISSUE_SELECTION_CONTINUATION_EXPLICIT, ISSUE_SELECTION_REPRESENTED_BY_OPEN_PR, assess_author_pr_report, + assess_validation_environment_proof, assess_capability_evidence, assess_capability_proof, assess_contradictory_no_pr_claim, @@ -2160,5 +2161,107 @@ class TestCreateIssueFinalReport(unittest.TestCase): self.assertTrue(result["downgraded"]) +class TestValidationEnvironmentProof(unittest.TestCase): + """Issue #311: exact validation command and execution environment.""" + + ROOT = "/repo/Gitea-Tools/branches/review-pr-280" + + def test_venv_pytest_with_full_report_passes(self): + result = assess_validation_environment_proof( + report_text=( + "Validation command: venv/bin/pytest tests/ --ignore=branches\n" + f"Working directory: {self.ROOT}\n" + "Result: 1014 passed, 6 skipped" + ), + ) + self.assertTrue(result["proven"]) + + def test_bare_pytest_without_path_proof_fails(self): + result = assess_validation_environment_proof( + report_text=( + "Validation command: pytest\n" + f"Working directory: {self.ROOT}\n" + "1014 passed, 6 skipped" + ), + ) + self.assertFalse(result["proven"]) + + def test_bare_pytest_with_path_and_version_proof_passes(self): + result = assess_validation_environment_proof( + report_text=( + "Validation command: pytest tests/\n" + f"Working directory: {self.ROOT}\n" + "which pytest: /repo/Gitea-Tools/venv/bin/pytest\n" + "pytest --version: pytest 8.3.4\n" + "Resolves to the project venv: yes\n" + "1014 passed, 6 skipped" + ), + ) + self.assertTrue(result["proven"]) + + def test_vague_pytest_summary_without_command_fails(self): + result = assess_validation_environment_proof( + report_text=( + f"Working directory: {self.ROOT}\n" + "pytest executed inside the worktree: 1014 passed, 6 skipped" + ), + ) + self.assertFalse(result["proven"]) + + def test_structured_environment_proof_passes(self): + result = assess_validation_environment_proof( + validation_environment={ + "command": "pytest tests/", + "working_directory": self.ROOT, + "which_pytest": "/repo/Gitea-Tools/venv/bin/pytest", + "pytest_version": "pytest 8.3.4", + "venv_resolved": True, + "passed": 1014, + "skipped": 6, + }, + ) + self.assertTrue(result["proven"]) + + def test_missing_working_directory_fails(self): + result = assess_validation_environment_proof( + report_text=( + "Validation command: venv/bin/pytest tests/\n" + "1014 passed, 6 skipped" + ), + ) + self.assertFalse(result["proven"]) + + def test_build_final_report_downgrades_missing_environment_proof(self): + report = build_final_report( + checkout_proof=_good_checkout(), + inventory=_good_inventory(), + validation=_good_validation(), + contamination=_good_contamination(), + identity_eligible=True, + merge_performed=False, + issue_status_verified=True, + capability_evidence=_good_capability_evidence(), + sweep=_good_sweep(), + live_state=_good_live_state(), + role_boundary=_good_role_boundary(), + review_mutation=_good_review_mutation(), + controller_handoff=_good_handoff(), + capability_proof=_good_capability_proof(), + sweep_proof=_good_secret_sweep(), + worktree_proof={ + "worktree_path": "/repo/branches/review-feat-issue-224", + "porcelain_status": "", + "pr_scope_files": ["docs/wiki/Repositories.md"], + "scratch_used": True, + }, + report_text=( + f"Working directory: {self.ROOT}\n" + "pytest executed: 1014 passed, 6 skipped" + ), + ) + self.assertNotEqual(report["grade"], "A") + self.assertFalse(report["validation_environment_proven"]) + + if __name__ == "__main__": unittest.main() -- 2.43.7