Scaled-Tech-Consulting/Gitea-Tools
2
0
Fork 0
Code Issues 73 Pull Requests 9 Actions Packages Projects Releases Wiki Activity

feat: enforce validation environment proof in reviewer reports (Closes #311)

Browse Source 
Add assess_validation_environment_proof requiring exact validation command,
working directory, result evidence, and bare-pytest executable/version/venv
proof before reviewer reports can claim validation success.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
sysadmin
2026-07-07 05:08:42 -04:00
co-authored by Claude Opus 4.8
parent d7d2f92286
commit da9c0ba54c
2 changed files with 318 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
review_proofs.py
+215 -1
View File
@@ -1231,7 +1231,8 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
role_boundary=None, review_mutation=None,
report_text=None, review_decision_lock=None,
controller_handoff=None, capability_proof=None,
sweep_proof=None, worktree_proof=None):
sweep_proof=None, worktree_proof=None,
validation_environment=None):
"""Required behavior 6 + acceptance criteria: one report, distinct proofs.
Combines the individual proof verdicts into the final-report fields the
@@ -1258,6 +1259,22 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
)
empty_queue_report = assess_empty_queue_report(report_text)
if validation_environment is None and report_text:
validation_env_proof = assess_validation_environment_proof(
report_text=report_text,
)
elif validation_environment is not None:
validation_env_proof = assess_validation_environment_proof(
report_text=report_text,
validation_environment=validation_environment,
)
else:
validation_env_proof = {
"proven": True,
"block": False,
"reasons": [],
"violations": [],
}
contamination_status = contamination.get("status", "unknown")
checkout_proven = bool(checkout_proof.get("proven"))
@@ -1387,6 +1404,11 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
"empty-queue report missing or failed trust-gate proof (#198)"
)
downgrade_reasons.extend(empty_queue_report.get("reasons", []))
if not validation_env_proof.get("proven"):
downgrade_reasons.append(
"validation environment proof missing or failed (#311)"
)
downgrade_reasons.extend(validation_env_proof.get("reasons", []))
merge_allowed = (
identity_eligible
@@ -1398,9 +1420,11 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
# #179: no merge without a proven final live-state recheck.
and live_state_proven
and worktree_proven
and validation_env_proof.get("proven")
)
violations = []
violations.extend(validation_env_proof.get("violations", []))
if merge_performed and not merge_allowed:
violations.append(
"merge was performed/claimed although the proofs did not allow "
@@ -1452,6 +1476,12 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
else True
),
"empty_queue_trust_gate_status": empty_queue_report.get("status"),
"validation_environment_proven": bool(
validation_env_proof.get("proven")
),
"validation_environment_violations": list(
validation_env_proof.get("violations") or []
),
}
@@ -3020,6 +3050,190 @@ def build_review_mutation_proof(run_log: list[dict]) -> dict:
}
# ---------------------------------------------------------------------------
# Validation environment proof (#311)
# ---------------------------------------------------------------------------
_VALIDATION_CLAIM_RE = re.compile(
r"(?:validation\s+command|pytest\s+(?:executed|passed|failed)|"
r"\d+\s+passed|\d+\s+failed|tests?\s+passed)",
re.IGNORECASE,
)
_VALIDATION_CMD_LINE_RE = re.compile(
r"validation\s+command\s*:\s*(.+)",
re.IGNORECASE,
)
_VALIDATION_CWD_LINE_RE = re.compile(
r"working\s+directory\s*:\s*(\S+)",
re.IGNORECASE,
)
_WHICH_PYTEST_LINE_RE = re.compile(
r"which\s+pytest\s*:\s*(.+)",
re.IGNORECASE,
)
_PYTEST_VERSION_LINE_RE = re.compile(
r"pytest\s+--version\s*:\s*(.+)",
re.IGNORECASE,
)
_BARE_PYTEST_CMD_RE = re.compile(
r"^(?:python\s+-m\s+)?pytest(?:\s|$)",
re.IGNORECASE,
)
_VAGUE_PYTEST_CLAIM_RE = re.compile(
r"pytest\s+(?:executed|passed|failed)\b",
re.IGNORECASE,
)
def _parse_validation_command(report_text: str) -> str:
for line in (report_text or "").splitlines():
match = _VALIDATION_CMD_LINE_RE.search(line)
if match:
return match.group(1).strip().rstrip(".")
return ""
def _parse_validation_cwd(report_text: str) -> str:
for line in (report_text or "").splitlines():
match = _VALIDATION_CWD_LINE_RE.search(line)
if match:
return match.group(1).strip().rstrip(",.;")
return ""
def _is_bare_pytest_command(command: str) -> bool:
text = (command or "").strip()
if not text:
return False
if "/" in text or "\\" in text:
return False
return bool(_BARE_PYTEST_CMD_RE.match(text))
def assess_validation_environment_proof(
*,
report_text: str | None = None,
validation_environment: dict | None = None,
) -> dict:
"""#311: reviewer reports must include exact validation command/environment."""
text = report_text or ""
lower = text.lower()
env = validation_environment or {}
reasons: list[str] = []
violations: list[str] = []
claims_validation = bool(_VALIDATION_CLAIM_RE.search(text))
if not claims_validation and not env.get("command"):
return {
"proven": True,
"block": False,
"claims_validation": False,
"reasons": [],
"violations": [],
"safe_next_action": "proceed",
}
command = (env.get("command") or _parse_validation_command(text)).strip()
working_directory = (
env.get("working_directory") or env.get("cwd")
or _parse_validation_cwd(text)
).strip()
if not command:
reasons.append(
"validation result claimed without exact validation command (#311)"
)
elif _VAGUE_PYTEST_CLAIM_RE.search(text) and command == "":
violations.append(
"vague pytest summary without exact validation command (#311)"
)
if not working_directory:
reasons.append(
"validation result claimed without working directory (#311)"
)
has_result = any(
token in lower
for token in ("passed", "failed", "skipped", "error")
) or any(
env.get(key) is not None
for key in ("passed", "failed", "skipped", "result")
)
if not has_result:
reasons.append(
"validation result claimed without pass/fail result evidence (#311)"
)
if command and _is_bare_pytest_command(command):
which_pytest = (
(env.get("which_pytest") or env.get("executable_path") or "")
.strip()
)
if not which_pytest:
for line in text.splitlines():
match = _WHICH_PYTEST_LINE_RE.search(line)
if match:
which_pytest = match.group(1).strip()
break
if not which_pytest:
reasons.append(
"bare pytest command requires which-pytest/executable proof (#311)"
)
pytest_version = (env.get("pytest_version") or "").strip()
if not pytest_version:
for line in text.splitlines():
match = _PYTEST_VERSION_LINE_RE.search(line)
if match:
pytest_version = match.group(1).strip()
break
if not pytest_version:
reasons.append(
"bare pytest command requires pytest --version proof (#311)"
)
venv_resolved = env.get("venv_resolved")
if venv_resolved is None:
venv_resolved = any(
marker in lower
for marker in (
"project venv",
"venv/bin/pytest",
"resolves to the project venv",
"venv resolved: yes",
)
)
if not venv_resolved and which_pytest and "venv" not in which_pytest:
reasons.append(
"bare pytest command requires proof whether executable "
"resolves to the project venv (#311)"
)
proven = not reasons and not violations
return {
"proven": proven,
"block": bool(violations),
"claims_validation": True,
"command": command or None,
"working_directory": working_directory or None,
"reasons": reasons,
"violations": violations,
"safe_next_action": (
"report exact validation command, working directory, result, and "
"for bare pytest also which-pytest, pytest --version, and venv proof"
if not proven
else "proceed"
),
}
# ---------------------------------------------------------------------------
# Identity disclosure (#305)
# ---------------------------------------------------------------------------
tests/test_review_proofs.py
+103
View File
@@ -24,6 +24,7 @@ from review_proofs import ( # noqa: E402
ISSUE_SELECTION_CONTINUATION_EXPLICIT,
ISSUE_SELECTION_REPRESENTED_BY_OPEN_PR,
assess_author_pr_report,
assess_validation_environment_proof,
assess_capability_evidence,
assess_capability_proof,
assess_contradictory_no_pr_claim,
@@ -2160,5 +2161,107 @@ class TestCreateIssueFinalReport(unittest.TestCase):
self.assertTrue(result["downgraded"])
class TestValidationEnvironmentProof(unittest.TestCase):
"""Issue #311: exact validation command and execution environment."""
ROOT = "/repo/Gitea-Tools/branches/review-pr-280"
def test_venv_pytest_with_full_report_passes(self):
result = assess_validation_environment_proof(
report_text=(
"Validation command: venv/bin/pytest tests/ --ignore=branches\n"
f"Working directory: {self.ROOT}\n"
"Result: 1014 passed, 6 skipped"
),
)
self.assertTrue(result["proven"])
def test_bare_pytest_without_path_proof_fails(self):
result = assess_validation_environment_proof(
report_text=(
"Validation command: pytest\n"
f"Working directory: {self.ROOT}\n"
"1014 passed, 6 skipped"
),
)
self.assertFalse(result["proven"])
def test_bare_pytest_with_path_and_version_proof_passes(self):
result = assess_validation_environment_proof(
report_text=(
"Validation command: pytest tests/\n"
f"Working directory: {self.ROOT}\n"
"which pytest: /repo/Gitea-Tools/venv/bin/pytest\n"
"pytest --version: pytest 8.3.4\n"
"Resolves to the project venv: yes\n"
"1014 passed, 6 skipped"
),
)
self.assertTrue(result["proven"])
def test_vague_pytest_summary_without_command_fails(self):
result = assess_validation_environment_proof(
report_text=(
f"Working directory: {self.ROOT}\n"
"pytest executed inside the worktree: 1014 passed, 6 skipped"
),
)
self.assertFalse(result["proven"])
def test_structured_environment_proof_passes(self):
result = assess_validation_environment_proof(
validation_environment={
"command": "pytest tests/",
"working_directory": self.ROOT,
"which_pytest": "/repo/Gitea-Tools/venv/bin/pytest",
"pytest_version": "pytest 8.3.4",
"venv_resolved": True,
"passed": 1014,
"skipped": 6,
},
)
self.assertTrue(result["proven"])
def test_missing_working_directory_fails(self):
result = assess_validation_environment_proof(
report_text=(
"Validation command: venv/bin/pytest tests/\n"
"1014 passed, 6 skipped"
),
)
self.assertFalse(result["proven"])
def test_build_final_report_downgrades_missing_environment_proof(self):
report = build_final_report(
checkout_proof=_good_checkout(),
inventory=_good_inventory(),
validation=_good_validation(),
contamination=_good_contamination(),
identity_eligible=True,
merge_performed=False,
issue_status_verified=True,
capability_evidence=_good_capability_evidence(),
sweep=_good_sweep(),
live_state=_good_live_state(),
role_boundary=_good_role_boundary(),
review_mutation=_good_review_mutation(),
controller_handoff=_good_handoff(),
capability_proof=_good_capability_proof(),
sweep_proof=_good_secret_sweep(),
worktree_proof={
"worktree_path": "/repo/branches/review-feat-issue-224",
"porcelain_status": "",
"pr_scope_files": ["docs/wiki/Repositories.md"],
"scratch_used": True,
},
report_text=(
f"Working directory: {self.ROOT}\n"
"pytest executed: 1014 passed, 6 skipped"
),
)
self.assertNotEqual(report["grade"], "A")
self.assertFalse(report["validation_environment_proven"])
if __name__ == "__main__":
unittest.main()