sysadminandClaude Opus 4.8
f20e0fc338
Reviewer runs that observe a validation failure and later rerun to green must report the failure and its investigation; a later pass never erases the earlier failure: - New reviewer_validation_failure_history.py verifier: reports must carry a Validation failure history section naming every observed failure with command, failing test or error, suspected cause, reproduced status, baseline master status, and PR-caused evidence; rerun-pass claims must explain what changed between runs, whether environmental state (such as stale /tmp files) was cleaned, and why the pass is trustworthy; an unexplained transient failure cannot be reported as a bare pass and must use the status 'passed after transient failure investigation' (exported as TRANSIENT_INVESTIGATION_STATUS). - Re-export from review_proofs and wire into build_final_report as a downgrade check. - Review-merge final-report schema gains the Validation failure history template section; contract test locks the markers and an export test locks the re-export. - 12 tests covering the issue's scenarios: failing-then-passing rerun, baseline-equivalent failure, /tmp state contamination, unexplained transient failure, and a report omitting an earlier failure. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
110 lines
4.1 KiB
Python
110 lines
4.1 KiB
Python
"""Validation failure history verifier for reviewer reports (#396).
|
|
|
|
A reviewer run that observes a validation failure and later reruns to green
|
|
must report the failure, its investigation, and why the eventual pass is
|
|
trustworthy. A later pass never erases the duty to report the earlier
|
|
failure. This verifier fails closed when known failures are omitted, when
|
|
failure entries lack the required investigation fields, when a rerun-pass
|
|
claim lacks a what-changed explanation, or when an unexplained transient
|
|
failure is reported as a bare pass.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any
|
|
|
|
# Issue #396 criterion 4: required status wording when the cause is unknown.
|
|
TRANSIENT_INVESTIGATION_STATUS = "passed after transient failure investigation"
|
|
|
|
_HISTORY_SECTION_RE = re.compile(
|
|
r"(?:^#+\s*validation failure history\b|validation failure history\s*:)",
|
|
re.IGNORECASE | re.MULTILINE,
|
|
)
|
|
_FAILURE_ENTRY_FIELDS = (
|
|
("command", re.compile(r"command\s*:", re.IGNORECASE)),
|
|
("failing test or error", re.compile(r"(?:failing test|error)\s*:", re.IGNORECASE)),
|
|
("suspected cause", re.compile(r"suspected cause\s*:", re.IGNORECASE)),
|
|
("reproduced status", re.compile(r"reproduced\s*:", re.IGNORECASE)),
|
|
("baseline master status", re.compile(r"baseline(?:\s+master)?\s*:", re.IGNORECASE)),
|
|
("PR-caused evidence", re.compile(r"(?:pr-caused|not pr-caused|evidence)", re.IGNORECASE)),
|
|
)
|
|
_RERUN_PASS_RE = re.compile(
|
|
r"(?:passed on rerun|rerun[^.\n]*passed|later (?:run|rerun) passed)",
|
|
re.IGNORECASE,
|
|
)
|
|
_WHAT_CHANGED_RE = re.compile(r"what changed", re.IGNORECASE)
|
|
_ENV_STATE_RE = re.compile(
|
|
r"(?:environmental state|/tmp/|state (?:was )?cleaned|none cleaned)",
|
|
re.IGNORECASE,
|
|
)
|
|
_TRUST_RE = re.compile(r"(?:trustworthy|trust the pass|why the pass)", re.IGNORECASE)
|
|
_UNKNOWN_CAUSE_RE = re.compile(r"suspected cause\s*:\s*unknown", re.IGNORECASE)
|
|
_BARE_PASSED_STATUS_RE = re.compile(
|
|
r"validation status\s*:\s*passed\s*$",
|
|
re.IGNORECASE | re.MULTILINE,
|
|
)
|
|
|
|
|
|
def assess_validation_failure_history_report(
|
|
report_text: str,
|
|
*,
|
|
observed_failures: list[dict] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Require complete failure-history reporting for observed failures (#396)."""
|
|
text = report_text or ""
|
|
reasons: list[str] = []
|
|
has_history = bool(_HISTORY_SECTION_RE.search(text))
|
|
|
|
if observed_failures and not has_history:
|
|
reasons.append(
|
|
"validation failures were observed this session but the report has "
|
|
"no Validation failure history section"
|
|
)
|
|
|
|
if observed_failures and has_history:
|
|
for entry in observed_failures:
|
|
failure = str(entry.get("failure") or "").strip()
|
|
if failure and failure not in text:
|
|
reasons.append(
|
|
f"observed validation failure omitted from history: {failure}"
|
|
)
|
|
|
|
missing_fields = [
|
|
name for name, pattern in _FAILURE_ENTRY_FIELDS if not pattern.search(text)
|
|
]
|
|
for name in missing_fields:
|
|
reasons.append(f"failure history entry missing required field: {name}")
|
|
|
|
if has_history and _RERUN_PASS_RE.search(text):
|
|
if not _WHAT_CHANGED_RE.search(text):
|
|
reasons.append(
|
|
"rerun-pass claim does not explain what changed between runs"
|
|
)
|
|
if not _ENV_STATE_RE.search(text):
|
|
reasons.append(
|
|
"rerun-pass claim does not state whether environmental state "
|
|
"was cleaned"
|
|
)
|
|
if not _TRUST_RE.search(text):
|
|
reasons.append(
|
|
"rerun-pass claim does not explain why the pass is trustworthy"
|
|
)
|
|
|
|
if (
|
|
observed_failures
|
|
and _UNKNOWN_CAUSE_RE.search(text)
|
|
and _BARE_PASSED_STATUS_RE.search(text)
|
|
):
|
|
reasons.append(
|
|
"unexplained transient failure cannot be reported as a bare pass; "
|
|
f"use '{TRANSIENT_INVESTIGATION_STATUS}'"
|
|
)
|
|
|
|
return {
|
|
"proven": not reasons,
|
|
"block": bool(reasons),
|
|
"reasons": reasons,
|
|
"history_section_present": has_history,
|
|
}
|