sysadminandClaude Opus 4.8
7fb813446f
Add premerge_baseline_proof.py distinguishing four validation outcomes (clean pass / current-master failure reproduced / pre-merge baseline-proven failure / unresolved regression risk). A non-zero exit may only be labeled "baseline"/"pre-existing" with pre-merge proof: the failure reproduced on the PR's pre-merge base commit, or a documented known-failure record predating the PR. Current-master (post-merge) reproduction is explicitly rejected as sufficient baseline proof. - premerge_baseline_proof.py: assess_premerge_baseline_proof + labels - final_report_validator.py: reviewer.premerge_baseline_proof rule on review_pr and reconcile_already_landed tasks - skills/llm-project-workflow/templates/review-pr.md: baseline proof section requiring base commit, tested commit, command, exit status, failure signature - tests/test_premerge_baseline_proof.py: reject current-master-only proof, accept pre-merge base proof and documented known-failure records Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
113 lines
4.2 KiB
Python
113 lines
4.2 KiB
Python
import unittest
|
|
|
|
from premerge_baseline_proof import (
|
|
CLEAN_PASS,
|
|
PREMERGE_BASELINE_PROVEN_FAILURE,
|
|
UNRESOLVED_REGRESSION_RISK,
|
|
assess_premerge_baseline_proof,
|
|
)
|
|
from final_report_validator import assess_final_report_validator
|
|
|
|
_FIELDS = (
|
|
"Pre-merge base commit: 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b\n"
|
|
"Tested commit: 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b\n"
|
|
"Command: venv/bin/python -m pytest tests/test_foo.py -q\n"
|
|
"Exit status: 1\n"
|
|
"Failure signature: AssertionError: None is not true\n"
|
|
)
|
|
|
|
# Current-master-only reproduction carries no pre-merge base proof (that is the
|
|
# whole defect #533 guards against): a command/exit/signature but no base commit.
|
|
_CURRENT_ONLY_FIELDS = (
|
|
"Command: venv/bin/python -m pytest tests/test_foo.py -q\n"
|
|
"Exit status: 1\n"
|
|
"Failure signature: AssertionError: None is not true\n"
|
|
)
|
|
|
|
|
|
class TestPremergeBaselineProof(unittest.TestCase):
|
|
def test_clean_pass_not_blocked(self):
|
|
result = assess_premerge_baseline_proof(
|
|
"Validation: full suite passed, exit status 0. Clean pass."
|
|
)
|
|
self.assertFalse(result["block"])
|
|
self.assertTrue(result["skipped"])
|
|
self.assertEqual(result["label"], CLEAN_PASS)
|
|
|
|
def test_nonzero_exit_not_claimed_baseline_not_blocked(self):
|
|
result = assess_premerge_baseline_proof(
|
|
"Full suite: 1 failed. Investigating the regression; not yet classified."
|
|
)
|
|
self.assertFalse(result["block"])
|
|
self.assertEqual(result["label"], UNRESOLVED_REGRESSION_RISK)
|
|
|
|
def test_current_master_only_reproduction_rejected(self):
|
|
report = (
|
|
"Full suite: 1 failed. This is a pre-existing baseline failure — "
|
|
"reproduced on current master after the PR merged.\n" + _CURRENT_ONLY_FIELDS
|
|
)
|
|
result = assess_premerge_baseline_proof(report)
|
|
self.assertTrue(result["block"])
|
|
self.assertTrue(
|
|
any("current-master reproduction only" in r for r in result["reasons"])
|
|
)
|
|
|
|
def test_baseline_claim_missing_fields_blocked(self):
|
|
report = (
|
|
"Full suite: 1 failed. This failure is baseline / pre-existing. "
|
|
"Verified against the pre-merge base commit."
|
|
)
|
|
result = assess_premerge_baseline_proof(report)
|
|
self.assertTrue(result["block"])
|
|
self.assertTrue(
|
|
any("missing required proof field" in r for r in result["reasons"])
|
|
)
|
|
|
|
def test_premerge_base_proof_accepted(self):
|
|
report = (
|
|
"Full suite: 1 failed. This is a pre-existing baseline failure, proven "
|
|
"on the PR pre-merge base commit.\n" + _FIELDS
|
|
)
|
|
result = assess_premerge_baseline_proof(report)
|
|
self.assertFalse(result["block"])
|
|
self.assertTrue(result["proven"])
|
|
self.assertEqual(result["label"], PREMERGE_BASELINE_PROVEN_FAILURE)
|
|
|
|
def test_documented_known_failure_record_accepted(self):
|
|
report = (
|
|
"Full suite: 1 failed. Baseline failure: cited known-failure record #529 "
|
|
"which predates the PR.\n" + _FIELDS
|
|
)
|
|
result = assess_premerge_baseline_proof(report)
|
|
self.assertFalse(result["block"])
|
|
self.assertEqual(result["label"], PREMERGE_BASELINE_PROVEN_FAILURE)
|
|
|
|
|
|
class TestPremergeBaselineProofValidatorIntegration(unittest.TestCase):
|
|
def _has_rule(self, report):
|
|
result = assess_final_report_validator(report, "review_pr")
|
|
return any(
|
|
f["rule_id"] == "reviewer.premerge_baseline_proof"
|
|
for f in result["findings"]
|
|
)
|
|
|
|
def test_review_report_current_master_only_flagged(self):
|
|
report = (
|
|
"## Reviewer final report\n"
|
|
"Full suite: 1 failed. Pre-existing baseline failure — reproduced on "
|
|
"post-merge master.\n" + _CURRENT_ONLY_FIELDS
|
|
)
|
|
self.assertTrue(self._has_rule(report))
|
|
|
|
def test_review_report_premerge_proof_clean(self):
|
|
report = (
|
|
"## Reviewer final report\n"
|
|
"Full suite: 1 failed. Pre-existing baseline failure proven on the "
|
|
"pre-merge base commit.\n" + _FIELDS
|
|
)
|
|
self.assertFalse(self._has_rule(report))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|