feat: require transient validation failure history in reviewer reports (Closes #396)
Add assess_validation_failure_history_report so reviewer final reports must document every validation failure observed during a session, not only the final passing result. Wire the verifier into final_report_validator, gitea_validate_review_final_report, and the review-merge workflow template. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,198 @@
|
||||
"""Transient validation failure history verifier (#396).
|
||||
|
||||
Reviewer sessions may observe validation failures that later pass on rerun.
|
||||
Final reports must document every failure observed during the session, not
|
||||
only the last passing result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
_SECTION_RE = re.compile(
|
||||
r"validation failure history",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_FAILURE_ENTRY_RE = re.compile(
|
||||
r"(?:failure\s*(?:#|entry)?\s*\d+|validation failure)\s*:",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_COMMAND_RE = re.compile(
|
||||
r"(?:command|validation command)\s*:\s*(.+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_FAILING_TEST_RE = re.compile(
|
||||
r"(?:failing test|failure|error)\s*:\s*(.+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_CAUSE_RE = re.compile(
|
||||
r"(?:suspected cause|cause)\s*:\s*(.+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_REPRODUCED_RE = re.compile(
|
||||
r"(?:reproduced|reproduces)\s*:\s*(yes|no|unknown)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_BASELINE_RE = re.compile(
|
||||
r"(?:on baseline master|baseline master|exists on baseline)\s*:\s*(yes|no|unknown|not checked)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_PR_CAUSED_RE = re.compile(
|
||||
r"(?:pr[- ]caused|pr caused)\s*:\s*(yes|no|unknown|not proven)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TRANSIENT_STATUS_RE = re.compile(
|
||||
r"(?:passed after transient failure investigation|transient failure investigation)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_PLAIN_PASS_RE = re.compile(
|
||||
r"validation\s*:\s*(?:pass|passed|strong|ok|green)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_ENV_CLEANUP_RE = re.compile(
|
||||
r"(?:environmental cleanup|state cleaned|what changed between runs)\s*:",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_UNKNOWN_CAUSE_RE = re.compile(
|
||||
r"(?:suspected cause|cause)\s*:\s*(?:unknown|unexplained|not determined)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _failure_documented_in_text(text: str, failure: dict[str, Any]) -> bool:
|
||||
"""Return True when *failure* appears documented in free-form report text."""
|
||||
command = (failure.get("command") or "").strip()
|
||||
failing = (failure.get("failing_test") or failure.get("error") or "").strip()
|
||||
if command and command not in text:
|
||||
return False
|
||||
if failing and failing not in text:
|
||||
return False
|
||||
return bool(command or failing)
|
||||
|
||||
|
||||
def _section_has_structured_fields(text: str) -> bool:
|
||||
if not _SECTION_RE.search(text):
|
||||
return False
|
||||
section_start = _SECTION_RE.search(text).start()
|
||||
section = text[section_start:]
|
||||
has_command = bool(_COMMAND_RE.search(section))
|
||||
has_failure = bool(_FAILING_TEST_RE.search(section))
|
||||
return has_command and has_failure
|
||||
|
||||
|
||||
def assess_validation_failure_history_report(
|
||||
report_text: str,
|
||||
*,
|
||||
validation_session: dict | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Require final reports to account for transient validation failures (#396)."""
|
||||
text = report_text or ""
|
||||
session = dict(validation_session or {})
|
||||
reasons: list[str] = []
|
||||
violations: list[str] = []
|
||||
|
||||
observed = list(session.get("observed_failures") or [])
|
||||
final_status = (session.get("final_validation_status") or "").strip().lower()
|
||||
cause_unknown = any(
|
||||
(f.get("suspected_cause") or "").strip().lower() in {
|
||||
"unknown", "unexplained", "not determined", ""
|
||||
}
|
||||
and not (f.get("pr_caused") or "").strip().lower() in {"yes", "no"}
|
||||
for f in observed
|
||||
) or bool(session.get("cause_unknown"))
|
||||
|
||||
if not observed:
|
||||
return {
|
||||
"proven": True,
|
||||
"block": False,
|
||||
"reasons": [],
|
||||
"violations": [],
|
||||
"observed_failure_count": 0,
|
||||
"safe_next_action": "proceed",
|
||||
}
|
||||
|
||||
documented = _section_has_structured_fields(text)
|
||||
if not documented:
|
||||
for failure in observed:
|
||||
if _failure_documented_in_text(text, failure):
|
||||
documented = True
|
||||
break
|
||||
|
||||
if not documented:
|
||||
violations.append(
|
||||
"session observed validation failure(s) but final report omits "
|
||||
"Validation failure history"
|
||||
)
|
||||
reasons.append(
|
||||
"every validation failure observed during the session must appear "
|
||||
"in a Validation failure history section"
|
||||
)
|
||||
|
||||
if documented and not _SECTION_RE.search(text):
|
||||
reasons.append(
|
||||
"failure details must appear under an explicit "
|
||||
"'Validation failure history' heading"
|
||||
)
|
||||
|
||||
for idx, failure in enumerate(observed, start=1):
|
||||
prefix = f"failure #{idx}"
|
||||
if not _failure_documented_in_text(text, failure) and documented:
|
||||
reasons.append(
|
||||
f"{prefix}: command and failing test/error not documented in report"
|
||||
)
|
||||
command = (failure.get("command") or "").strip()
|
||||
failing = (failure.get("failing_test") or failure.get("error") or "").strip()
|
||||
if not command:
|
||||
reasons.append(f"{prefix}: missing command in session failure record")
|
||||
if not failing:
|
||||
reasons.append(
|
||||
f"{prefix}: missing failing test or error in session failure record"
|
||||
)
|
||||
|
||||
plain_pass = bool(_PLAIN_PASS_RE.search(text))
|
||||
transient_wording = bool(_TRANSIENT_STATUS_RE.search(text))
|
||||
final_passed = final_status in {
|
||||
"passed",
|
||||
"pass",
|
||||
"passed_after_transient_failure_investigation",
|
||||
}
|
||||
|
||||
if (final_passed or plain_pass) and observed:
|
||||
if cause_unknown and not transient_wording:
|
||||
violations.append(
|
||||
"unknown transient failure cause cannot be erased as plain 'passed'"
|
||||
)
|
||||
reasons.append(
|
||||
"when cause is unknown, use status "
|
||||
"'passed after transient failure investigation'"
|
||||
)
|
||||
elif not transient_wording and final_status != "passed_after_transient_failure_investigation":
|
||||
if not _ENV_CLEANUP_RE.search(text):
|
||||
reasons.append(
|
||||
"later passing rerun must document what changed between runs "
|
||||
"or environmental cleanup performed"
|
||||
)
|
||||
|
||||
if session.get("environmental_contamination") and not _ENV_CLEANUP_RE.search(text):
|
||||
reasons.append(
|
||||
"environmental /tmp state contamination must document cleanup or "
|
||||
"what changed before the passing rerun"
|
||||
)
|
||||
|
||||
proven = not reasons and not violations
|
||||
return {
|
||||
"proven": proven,
|
||||
"block": bool(violations) or not proven,
|
||||
"reasons": reasons,
|
||||
"violations": violations,
|
||||
"observed_failure_count": len(observed),
|
||||
"documented": documented,
|
||||
"safe_next_action": (
|
||||
"add Validation failure history with command, failing test, cause, "
|
||||
"reproduction, baseline comparison, and PR-caused evidence; use "
|
||||
"'passed after transient failure investigation' when appropriate"
|
||||
if not proven
|
||||
else "proceed"
|
||||
),
|
||||
}
|
||||
Reference in New Issue
Block a user