feat: enforce validation environment proof in reviewer reports (Closes #311)
Add assess_validation_environment_proof requiring exact validation command, working directory, result evidence, and bare-pytest executable/version/venv proof before reviewer reports can claim validation success. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
@@ -24,6 +24,7 @@ from review_proofs import ( # noqa: E402
|
||||
ISSUE_SELECTION_CONTINUATION_EXPLICIT,
|
||||
ISSUE_SELECTION_REPRESENTED_BY_OPEN_PR,
|
||||
assess_author_pr_report,
|
||||
assess_validation_environment_proof,
|
||||
assess_capability_evidence,
|
||||
assess_capability_proof,
|
||||
assess_contradictory_no_pr_claim,
|
||||
@@ -2160,5 +2161,107 @@ class TestCreateIssueFinalReport(unittest.TestCase):
|
||||
self.assertTrue(result["downgraded"])
|
||||
|
||||
|
||||
class TestValidationEnvironmentProof(unittest.TestCase):
|
||||
"""Issue #311: exact validation command and execution environment."""
|
||||
|
||||
ROOT = "/repo/Gitea-Tools/branches/review-pr-280"
|
||||
|
||||
def test_venv_pytest_with_full_report_passes(self):
|
||||
result = assess_validation_environment_proof(
|
||||
report_text=(
|
||||
"Validation command: venv/bin/pytest tests/ --ignore=branches\n"
|
||||
f"Working directory: {self.ROOT}\n"
|
||||
"Result: 1014 passed, 6 skipped"
|
||||
),
|
||||
)
|
||||
self.assertTrue(result["proven"])
|
||||
|
||||
def test_bare_pytest_without_path_proof_fails(self):
|
||||
result = assess_validation_environment_proof(
|
||||
report_text=(
|
||||
"Validation command: pytest\n"
|
||||
f"Working directory: {self.ROOT}\n"
|
||||
"1014 passed, 6 skipped"
|
||||
),
|
||||
)
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
def test_bare_pytest_with_path_and_version_proof_passes(self):
|
||||
result = assess_validation_environment_proof(
|
||||
report_text=(
|
||||
"Validation command: pytest tests/\n"
|
||||
f"Working directory: {self.ROOT}\n"
|
||||
"which pytest: /repo/Gitea-Tools/venv/bin/pytest\n"
|
||||
"pytest --version: pytest 8.3.4\n"
|
||||
"Resolves to the project venv: yes\n"
|
||||
"1014 passed, 6 skipped"
|
||||
),
|
||||
)
|
||||
self.assertTrue(result["proven"])
|
||||
|
||||
def test_vague_pytest_summary_without_command_fails(self):
|
||||
result = assess_validation_environment_proof(
|
||||
report_text=(
|
||||
f"Working directory: {self.ROOT}\n"
|
||||
"pytest executed inside the worktree: 1014 passed, 6 skipped"
|
||||
),
|
||||
)
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
def test_structured_environment_proof_passes(self):
|
||||
result = assess_validation_environment_proof(
|
||||
validation_environment={
|
||||
"command": "pytest tests/",
|
||||
"working_directory": self.ROOT,
|
||||
"which_pytest": "/repo/Gitea-Tools/venv/bin/pytest",
|
||||
"pytest_version": "pytest 8.3.4",
|
||||
"venv_resolved": True,
|
||||
"passed": 1014,
|
||||
"skipped": 6,
|
||||
},
|
||||
)
|
||||
self.assertTrue(result["proven"])
|
||||
|
||||
def test_missing_working_directory_fails(self):
|
||||
result = assess_validation_environment_proof(
|
||||
report_text=(
|
||||
"Validation command: venv/bin/pytest tests/\n"
|
||||
"1014 passed, 6 skipped"
|
||||
),
|
||||
)
|
||||
self.assertFalse(result["proven"])
|
||||
|
||||
def test_build_final_report_downgrades_missing_environment_proof(self):
|
||||
report = build_final_report(
|
||||
checkout_proof=_good_checkout(),
|
||||
inventory=_good_inventory(),
|
||||
validation=_good_validation(),
|
||||
contamination=_good_contamination(),
|
||||
identity_eligible=True,
|
||||
merge_performed=False,
|
||||
issue_status_verified=True,
|
||||
capability_evidence=_good_capability_evidence(),
|
||||
sweep=_good_sweep(),
|
||||
live_state=_good_live_state(),
|
||||
role_boundary=_good_role_boundary(),
|
||||
review_mutation=_good_review_mutation(),
|
||||
controller_handoff=_good_handoff(),
|
||||
capability_proof=_good_capability_proof(),
|
||||
sweep_proof=_good_secret_sweep(),
|
||||
worktree_proof={
|
||||
"worktree_path": "/repo/branches/review-feat-issue-224",
|
||||
"porcelain_status": "",
|
||||
"pr_scope_files": ["docs/wiki/Repositories.md"],
|
||||
"scratch_used": True,
|
||||
},
|
||||
report_text=(
|
||||
f"Working directory: {self.ROOT}\n"
|
||||
"pytest executed: 1014 passed, 6 skipped"
|
||||
),
|
||||
)
|
||||
self.assertNotEqual(report["grade"], "A")
|
||||
self.assertFalse(report["validation_environment_proven"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user