Merge pull request 'feat: require true pre-merge baseline proof for non-zero validation exits (Closes #533)' (#574) from feat/issue-533-premerge-baseline-proof into master

2026-07-08 22:21:30 -05:00
parent f29df56a99 7fb813446f
commit 066eec8477
4 changed files with 319 additions and 0 deletions
@@ -0,0 +1,112 @@
+import unittest
+
+from premerge_baseline_proof import (
+    CLEAN_PASS,
+    PREMERGE_BASELINE_PROVEN_FAILURE,
+    UNRESOLVED_REGRESSION_RISK,
+    assess_premerge_baseline_proof,
+)
+from final_report_validator import assess_final_report_validator
+
+_FIELDS = (
+    "Pre-merge base commit: 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b\n"
+    "Tested commit: 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b\n"
+    "Command: venv/bin/python -m pytest tests/test_foo.py -q\n"
+    "Exit status: 1\n"
+    "Failure signature: AssertionError: None is not true\n"
+)
+
+# Current-master-only reproduction carries no pre-merge base proof (that is the
+# whole defect #533 guards against): a command/exit/signature but no base commit.
+_CURRENT_ONLY_FIELDS = (
+    "Command: venv/bin/python -m pytest tests/test_foo.py -q\n"
+    "Exit status: 1\n"
+    "Failure signature: AssertionError: None is not true\n"
+)
+
+
+class TestPremergeBaselineProof(unittest.TestCase):
+    def test_clean_pass_not_blocked(self):
+        result = assess_premerge_baseline_proof(
+            "Validation: full suite passed, exit status 0. Clean pass."
+        )
+        self.assertFalse(result["block"])
+        self.assertTrue(result["skipped"])
+        self.assertEqual(result["label"], CLEAN_PASS)
+
+    def test_nonzero_exit_not_claimed_baseline_not_blocked(self):
+        result = assess_premerge_baseline_proof(
+            "Full suite: 1 failed. Investigating the regression; not yet classified."
+        )
+        self.assertFalse(result["block"])
+        self.assertEqual(result["label"], UNRESOLVED_REGRESSION_RISK)
+
+    def test_current_master_only_reproduction_rejected(self):
+        report = (
+            "Full suite: 1 failed. This is a pre-existing baseline failure — "
+            "reproduced on current master after the PR merged.\n" + _CURRENT_ONLY_FIELDS
+        )
+        result = assess_premerge_baseline_proof(report)
+        self.assertTrue(result["block"])
+        self.assertTrue(
+            any("current-master reproduction only" in r for r in result["reasons"])
+        )
+
+    def test_baseline_claim_missing_fields_blocked(self):
+        report = (
+            "Full suite: 1 failed. This failure is baseline / pre-existing. "
+            "Verified against the pre-merge base commit."
+        )
+        result = assess_premerge_baseline_proof(report)
+        self.assertTrue(result["block"])
+        self.assertTrue(
+            any("missing required proof field" in r for r in result["reasons"])
+        )
+
+    def test_premerge_base_proof_accepted(self):
+        report = (
+            "Full suite: 1 failed. This is a pre-existing baseline failure, proven "
+            "on the PR pre-merge base commit.\n" + _FIELDS
+        )
+        result = assess_premerge_baseline_proof(report)
+        self.assertFalse(result["block"])
+        self.assertTrue(result["proven"])
+        self.assertEqual(result["label"], PREMERGE_BASELINE_PROVEN_FAILURE)
+
+    def test_documented_known_failure_record_accepted(self):
+        report = (
+            "Full suite: 1 failed. Baseline failure: cited known-failure record #529 "
+            "which predates the PR.\n" + _FIELDS
+        )
+        result = assess_premerge_baseline_proof(report)
+        self.assertFalse(result["block"])
+        self.assertEqual(result["label"], PREMERGE_BASELINE_PROVEN_FAILURE)
+
+
+class TestPremergeBaselineProofValidatorIntegration(unittest.TestCase):
+    def _has_rule(self, report):
+        result = assess_final_report_validator(report, "review_pr")
+        return any(
+            f["rule_id"] == "reviewer.premerge_baseline_proof"
+            for f in result["findings"]
+        )
+
+    def test_review_report_current_master_only_flagged(self):
+        report = (
+            "## Reviewer final report\n"
+            "Full suite: 1 failed. Pre-existing baseline failure — reproduced on "
+            "post-merge master.\n" + _CURRENT_ONLY_FIELDS
+        )
+        self.assertTrue(self._has_rule(report))
+
+    def test_review_report_premerge_proof_clean(self):
+        report = (
+            "## Reviewer final report\n"
+            "Full suite: 1 failed. Pre-existing baseline failure proven on the "
+            "pre-merge base commit.\n" + _FIELDS
+        )
+        self.assertFalse(self._has_rule(report))
+
+
+if __name__ == "__main__":
+    unittest.main()