sysadminandClaude Opus 4.8
eddb68818e
Delivers #529 acceptance criterion 6: controller issue closure must be
blocked when the closure report calls a non-zero test-suite exit an
"expected pre-existing"/baseline failure without pre-merge proof.
- controller_closure_baseline_proof.py: assess_controller_closure_baseline_proof
reuses the #533 pre-merge baseline verifier; empty report -> skipped/allowed,
clean pass -> allowed, proven baseline -> allowed, unproven baseline -> block.
- gitea_close_issue: optional closure_report param; fails closed (no state PATCH)
when the gate blocks.
- final_report_validator: new controller_close task kind (alias close_issue)
wired to the pre-merge baseline proof rule so a closure can be pre-checked.
- Tests: tests/test_controller_closure_baseline_proof.py plus two
gitea_close_issue gate tests.
Scope: criteria 2/3 (non-zero exit not a clean pass; baseline claims need
proof) are already enforced on master via premerge_baseline_proof (#533) and
the reviewer schema rules. Criteria 1/4/5 (post-merge moot canonical wording
and validation:* process-state labels) remain follow-up work.
Validation: full suite 2365 passed, 6 skipped; the 9 remaining failures
(tests/test_config.py TestAuthIntegration, tests/test_credentials.py
TestGetCredentials, test_issue_540 reviewer-role) are baseline-proven —
identical failures on clean prgs/master 6913ac9 (keychain/env dependent),
unrelated to this change.
Refs #529
Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
114 lines
4.3 KiB
Python
114 lines
4.3 KiB
Python
"""Controller-closure baseline-proof gate tests (#529, criterion 6)."""
|
|
|
|
import unittest
|
|
|
|
from controller_closure_baseline_proof import (
|
|
assess_controller_closure_baseline_proof,
|
|
)
|
|
from premerge_baseline_proof import (
|
|
CLEAN_PASS,
|
|
PREMERGE_BASELINE_PROVEN_FAILURE,
|
|
UNRESOLVED_REGRESSION_RISK,
|
|
)
|
|
from final_report_validator import assess_final_report_validator
|
|
|
|
# Complete pre-merge proof fields for a baseline claim (see #533).
|
|
_PROOF_FIELDS = (
|
|
"Pre-merge base commit: 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b\n"
|
|
"Tested commit: 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b\n"
|
|
"Command: venv/bin/python -m pytest tests/test_foo.py -q\n"
|
|
"Exit status: 1\n"
|
|
"Failure signature: AssertionError: None is not true\n"
|
|
)
|
|
|
|
|
|
class TestControllerClosureBaselineProof(unittest.TestCase):
|
|
def test_empty_report_skipped_and_allowed(self):
|
|
result = assess_controller_closure_baseline_proof("")
|
|
self.assertFalse(result["block"])
|
|
self.assertTrue(result["skipped"])
|
|
self.assertEqual(result["label"], CLEAN_PASS)
|
|
|
|
def test_none_report_allowed(self):
|
|
result = assess_controller_closure_baseline_proof(None)
|
|
self.assertFalse(result["block"])
|
|
self.assertTrue(result["skipped"])
|
|
|
|
def test_clean_pass_closure_allowed(self):
|
|
result = assess_controller_closure_baseline_proof(
|
|
"Closing #529. Validation: full suite passed, exit status 0. Clean pass."
|
|
)
|
|
self.assertFalse(result["block"])
|
|
self.assertEqual(result["label"], CLEAN_PASS)
|
|
|
|
def test_expected_preexisting_failure_without_proof_blocked(self):
|
|
report = (
|
|
"Closing #529. Full suite: 1 failed. This is an expected "
|
|
"pre-existing failure, safe to close."
|
|
)
|
|
result = assess_controller_closure_baseline_proof(report)
|
|
self.assertTrue(result["block"])
|
|
self.assertFalse(result["proven"])
|
|
self.assertEqual(result["label"], UNRESOLVED_REGRESSION_RISK)
|
|
self.assertTrue(result["reasons"])
|
|
self.assertTrue(result["safe_next_action"])
|
|
|
|
def test_current_master_reproduction_only_blocked(self):
|
|
report = (
|
|
"Closing #529. Full suite: 1 failed. Pre-existing baseline failure — "
|
|
"reproduced on current master after merge.\n"
|
|
"Command: venv/bin/python -m pytest tests/test_foo.py -q\n"
|
|
"Exit status: 1\n"
|
|
"Failure signature: AssertionError: None is not true\n"
|
|
)
|
|
result = assess_controller_closure_baseline_proof(report)
|
|
self.assertTrue(result["block"])
|
|
|
|
def test_proven_baseline_closure_allowed(self):
|
|
report = (
|
|
"Closing #529. Full suite: 1 failed, an expected pre-existing "
|
|
"baseline failure proven on the PR pre-merge base commit.\n"
|
|
+ _PROOF_FIELDS
|
|
)
|
|
result = assess_controller_closure_baseline_proof(report)
|
|
self.assertFalse(result["block"])
|
|
self.assertEqual(result["label"], PREMERGE_BASELINE_PROVEN_FAILURE)
|
|
|
|
|
|
class TestControllerCloseTaskKind(unittest.TestCase):
|
|
"""The composable validator must cover the controller_close task kind."""
|
|
|
|
def test_controller_close_blocks_unproven_baseline(self):
|
|
report = (
|
|
"Full suite: 1 failed. Expected pre-existing failure, closing the "
|
|
"tracking issue."
|
|
)
|
|
result = assess_final_report_validator(report, "controller_close")
|
|
self.assertTrue(result["blocked"])
|
|
self.assertTrue(
|
|
any(
|
|
"premerge_baseline_proof" in f["rule_id"]
|
|
for f in result["findings"]
|
|
)
|
|
)
|
|
|
|
def test_controller_close_alias_close_issue(self):
|
|
report = (
|
|
"Full suite: 1 failed. Expected pre-existing failure, closing the "
|
|
"tracking issue."
|
|
)
|
|
result = assess_final_report_validator(report, "close_issue")
|
|
self.assertTrue(result["blocked"])
|
|
|
|
def test_controller_close_allows_proven_baseline(self):
|
|
report = (
|
|
"Full suite: 1 failed, expected pre-existing baseline failure proven "
|
|
"on the PR pre-merge base commit.\n" + _PROOF_FIELDS
|
|
)
|
|
result = assess_final_report_validator(report, "controller_close")
|
|
self.assertFalse(result["blocked"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|