feat: enforce validation environment proof in reviewer reports (Closes #311)
Add assess_validation_environment_proof requiring exact validation command, working directory, result evidence, and bare-pytest executable/version/venv proof before reviewer reports can claim validation success. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Claude Opus 4.8
+215
-1
@@ -1231,7 +1231,8 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
|
||||
role_boundary=None, review_mutation=None,
|
||||
report_text=None, review_decision_lock=None,
|
||||
controller_handoff=None, capability_proof=None,
|
||||
sweep_proof=None, worktree_proof=None):
|
||||
sweep_proof=None, worktree_proof=None,
|
||||
validation_environment=None):
|
||||
"""Required behavior 6 + acceptance criteria: one report, distinct proofs.
|
||||
|
||||
Combines the individual proof verdicts into the final-report fields the
|
||||
@@ -1258,6 +1259,22 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
|
||||
)
|
||||
|
||||
empty_queue_report = assess_empty_queue_report(report_text)
|
||||
if validation_environment is None and report_text:
|
||||
validation_env_proof = assess_validation_environment_proof(
|
||||
report_text=report_text,
|
||||
)
|
||||
elif validation_environment is not None:
|
||||
validation_env_proof = assess_validation_environment_proof(
|
||||
report_text=report_text,
|
||||
validation_environment=validation_environment,
|
||||
)
|
||||
else:
|
||||
validation_env_proof = {
|
||||
"proven": True,
|
||||
"block": False,
|
||||
"reasons": [],
|
||||
"violations": [],
|
||||
}
|
||||
|
||||
contamination_status = contamination.get("status", "unknown")
|
||||
checkout_proven = bool(checkout_proof.get("proven"))
|
||||
@@ -1387,6 +1404,11 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
|
||||
"empty-queue report missing or failed trust-gate proof (#198)"
|
||||
)
|
||||
downgrade_reasons.extend(empty_queue_report.get("reasons", []))
|
||||
if not validation_env_proof.get("proven"):
|
||||
downgrade_reasons.append(
|
||||
"validation environment proof missing or failed (#311)"
|
||||
)
|
||||
downgrade_reasons.extend(validation_env_proof.get("reasons", []))
|
||||
|
||||
merge_allowed = (
|
||||
identity_eligible
|
||||
@@ -1398,9 +1420,11 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
|
||||
# #179: no merge without a proven final live-state recheck.
|
||||
and live_state_proven
|
||||
and worktree_proven
|
||||
and validation_env_proof.get("proven")
|
||||
)
|
||||
|
||||
violations = []
|
||||
violations.extend(validation_env_proof.get("violations", []))
|
||||
if merge_performed and not merge_allowed:
|
||||
violations.append(
|
||||
"merge was performed/claimed although the proofs did not allow "
|
||||
@@ -1452,6 +1476,12 @@ def build_final_report(checkout_proof, inventory, validation, contamination,
|
||||
else True
|
||||
),
|
||||
"empty_queue_trust_gate_status": empty_queue_report.get("status"),
|
||||
"validation_environment_proven": bool(
|
||||
validation_env_proof.get("proven")
|
||||
),
|
||||
"validation_environment_violations": list(
|
||||
validation_env_proof.get("violations") or []
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -3020,6 +3050,190 @@ def build_review_mutation_proof(run_log: list[dict]) -> dict:
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation environment proof (#311)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_VALIDATION_CLAIM_RE = re.compile(
|
||||
r"(?:validation\s+command|pytest\s+(?:executed|passed|failed)|"
|
||||
r"\d+\s+passed|\d+\s+failed|tests?\s+passed)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_VALIDATION_CMD_LINE_RE = re.compile(
|
||||
r"validation\s+command\s*:\s*(.+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_VALIDATION_CWD_LINE_RE = re.compile(
|
||||
r"working\s+directory\s*:\s*(\S+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_WHICH_PYTEST_LINE_RE = re.compile(
|
||||
r"which\s+pytest\s*:\s*(.+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_PYTEST_VERSION_LINE_RE = re.compile(
|
||||
r"pytest\s+--version\s*:\s*(.+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_BARE_PYTEST_CMD_RE = re.compile(
|
||||
r"^(?:python\s+-m\s+)?pytest(?:\s|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_VAGUE_PYTEST_CLAIM_RE = re.compile(
|
||||
r"pytest\s+(?:executed|passed|failed)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _parse_validation_command(report_text: str) -> str:
|
||||
for line in (report_text or "").splitlines():
|
||||
match = _VALIDATION_CMD_LINE_RE.search(line)
|
||||
if match:
|
||||
return match.group(1).strip().rstrip(".")
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_validation_cwd(report_text: str) -> str:
|
||||
for line in (report_text or "").splitlines():
|
||||
match = _VALIDATION_CWD_LINE_RE.search(line)
|
||||
if match:
|
||||
return match.group(1).strip().rstrip(",.;")
|
||||
return ""
|
||||
|
||||
|
||||
def _is_bare_pytest_command(command: str) -> bool:
|
||||
text = (command or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
if "/" in text or "\\" in text:
|
||||
return False
|
||||
return bool(_BARE_PYTEST_CMD_RE.match(text))
|
||||
|
||||
|
||||
def assess_validation_environment_proof(
|
||||
*,
|
||||
report_text: str | None = None,
|
||||
validation_environment: dict | None = None,
|
||||
) -> dict:
|
||||
"""#311: reviewer reports must include exact validation command/environment."""
|
||||
text = report_text or ""
|
||||
lower = text.lower()
|
||||
env = validation_environment or {}
|
||||
reasons: list[str] = []
|
||||
violations: list[str] = []
|
||||
|
||||
claims_validation = bool(_VALIDATION_CLAIM_RE.search(text))
|
||||
if not claims_validation and not env.get("command"):
|
||||
return {
|
||||
"proven": True,
|
||||
"block": False,
|
||||
"claims_validation": False,
|
||||
"reasons": [],
|
||||
"violations": [],
|
||||
"safe_next_action": "proceed",
|
||||
}
|
||||
|
||||
command = (env.get("command") or _parse_validation_command(text)).strip()
|
||||
working_directory = (
|
||||
env.get("working_directory") or env.get("cwd")
|
||||
or _parse_validation_cwd(text)
|
||||
).strip()
|
||||
|
||||
if not command:
|
||||
reasons.append(
|
||||
"validation result claimed without exact validation command (#311)"
|
||||
)
|
||||
elif _VAGUE_PYTEST_CLAIM_RE.search(text) and command == "":
|
||||
violations.append(
|
||||
"vague pytest summary without exact validation command (#311)"
|
||||
)
|
||||
|
||||
if not working_directory:
|
||||
reasons.append(
|
||||
"validation result claimed without working directory (#311)"
|
||||
)
|
||||
|
||||
has_result = any(
|
||||
token in lower
|
||||
for token in ("passed", "failed", "skipped", "error")
|
||||
) or any(
|
||||
env.get(key) is not None
|
||||
for key in ("passed", "failed", "skipped", "result")
|
||||
)
|
||||
if not has_result:
|
||||
reasons.append(
|
||||
"validation result claimed without pass/fail result evidence (#311)"
|
||||
)
|
||||
|
||||
if command and _is_bare_pytest_command(command):
|
||||
which_pytest = (
|
||||
(env.get("which_pytest") or env.get("executable_path") or "")
|
||||
.strip()
|
||||
)
|
||||
if not which_pytest:
|
||||
for line in text.splitlines():
|
||||
match = _WHICH_PYTEST_LINE_RE.search(line)
|
||||
if match:
|
||||
which_pytest = match.group(1).strip()
|
||||
break
|
||||
if not which_pytest:
|
||||
reasons.append(
|
||||
"bare pytest command requires which-pytest/executable proof (#311)"
|
||||
)
|
||||
|
||||
pytest_version = (env.get("pytest_version") or "").strip()
|
||||
if not pytest_version:
|
||||
for line in text.splitlines():
|
||||
match = _PYTEST_VERSION_LINE_RE.search(line)
|
||||
if match:
|
||||
pytest_version = match.group(1).strip()
|
||||
break
|
||||
if not pytest_version:
|
||||
reasons.append(
|
||||
"bare pytest command requires pytest --version proof (#311)"
|
||||
)
|
||||
|
||||
venv_resolved = env.get("venv_resolved")
|
||||
if venv_resolved is None:
|
||||
venv_resolved = any(
|
||||
marker in lower
|
||||
for marker in (
|
||||
"project venv",
|
||||
"venv/bin/pytest",
|
||||
"resolves to the project venv",
|
||||
"venv resolved: yes",
|
||||
)
|
||||
)
|
||||
if not venv_resolved and which_pytest and "venv" not in which_pytest:
|
||||
reasons.append(
|
||||
"bare pytest command requires proof whether executable "
|
||||
"resolves to the project venv (#311)"
|
||||
)
|
||||
|
||||
proven = not reasons and not violations
|
||||
return {
|
||||
"proven": proven,
|
||||
"block": bool(violations),
|
||||
"claims_validation": True,
|
||||
"command": command or None,
|
||||
"working_directory": working_directory or None,
|
||||
"reasons": reasons,
|
||||
"violations": violations,
|
||||
"safe_next_action": (
|
||||
"report exact validation command, working directory, result, and "
|
||||
"for bare pytest also which-pytest, pytest --version, and venv proof"
|
||||
if not proven
|
||||
else "proceed"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Identity disclosure (#305)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user