fix(validator): align final-report validator with canonical schema (Closes #698)
Fixes the final-report validator defects from Issue #698 (original lead plus the independent reproduction recorded during the PR #703 formal review, comment 11246): - Legacy fields: the review/merger required-field tables no longer demand 'Pinned reviewed head', 'Scratch worktree used', 'Worktree path', 'Worktree dirty', 'Mutations', 'Next', 'Issue/PR', 'Branch/SHA', or 'Files changed' — names the canonical review-merge-final-report schema forbids or replaces. The canonical names ('Reviewed head SHA', 'Review worktree path/dirty', 'Safe next action', mutation categories) are required instead, with backward-compatible aliases where the schema permits them. - Structured proof: workflow-load helper results are recognized in colon, key=value, and JSON renderings; validation pass proof is accepted anywhere in the Validation field value (for example 'focused 50 passed; full 2665 passed'). - Mutation inference: review mutations are inferred only from authoritative evidence (performed=true and not gated); read-only diagnostics and pre-API rejections no longer count as mutations. - Lease release vs cleanup: canonical reviewer lease release (release tool call or terminal phase=released marker) is lease lifecycle, not post-merge cleanup, and no longer triggers the branch/worktree cleanup checklist; genuine delete/remove claims still require full proof. - Blocked reports: a legitimately blocked run that states an explicit 'Reviewed/Candidate head SHA: none' with no verdict, merge, or started validation owes no head proofs; approval-time and merge-time live-head proofs are demanded only once the corresponding phase begins, and a report that states no head at all still fails closed. - action_log robustness: malformed (non-dict) entries and non-list logs are reported as clear sanitized findings (position and type only, no content echo) instead of crashing with AttributeError; a defective validator rule now fails closed with a sanitized block finding rather than raising a secondary exception. 27 new regression tests, including canonical fixtures modeled on the PR #703 formal-review handoff and the blocked preflight report from the prior #698 reproductions. Two legacy-field test fixtures updated to the canonical schema. Full suite: 2663 passed, 6 skipped, 161 subtests. Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
Claude Fable 5
@@ -134,16 +134,22 @@ _TARGET_BRANCH_SHA_RE = re.compile(
|
||||
r"target branch sha\s*:\s*[0-9a-f]{40}",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
# #698: structured proof is rendered in several equivalent shapes —
|
||||
# `workflow_hash: abc...`, `workflow_hash=abc...`, or JSON
|
||||
# `"workflow_hash": "abc..."`. Recognize all of them; demanding one exact
|
||||
# punctuation style rejects legitimate structured workflow-load proof.
|
||||
_WORKFLOW_LOAD_HELPER_RE = re.compile(
|
||||
r"workflow[- ]load helper result\s*:",
|
||||
r"workflow[-_ ]load[-_ ]helper[-_ ]result\s*[:=]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_WORKFLOW_LOAD_HASH_RE = re.compile(
|
||||
r"workflow[- ]load helper result[\s\S]{0,400}?workflow[_ ]hash\s*:\s*[0-9a-f]{12}",
|
||||
r"workflow[-_ ]load[-_ ]helper[-_ ]result[\s\S]{0,400}?"
|
||||
r"workflow[_ ]hash\"?\s*[:=]\s*\"?[0-9a-f]{12}",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_WORKFLOW_LOAD_BOUNDARY_RE = re.compile(
|
||||
r"workflow[- ]load helper result[\s\S]{0,400}?boundary[_ ]status\s*:\s*(?:clean|violation)",
|
||||
r"workflow[-_ ]load[-_ ]helper[-_ ]result[\s\S]{0,400}?"
|
||||
r"boundary[_ ]status\"?\s*[:=]\s*\"?(?:clean|violation)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_WORKFLOW_FILE_VIEW_NARRATIVE_RE = re.compile(
|
||||
@@ -244,6 +250,59 @@ def _normalize_task_kind(task_kind: str | None) -> str:
|
||||
return _TASK_KIND_ALIASES.get(raw, raw)
|
||||
|
||||
|
||||
def _iter_action_entries(action_log: list | None) -> list[dict]:
|
||||
"""Yield only structured (dict) action-log entries (#698).
|
||||
|
||||
Callers must never crash on malformed entries (strings, numbers, null)
|
||||
that reach the validator from LLM-composed or partially parsed logs;
|
||||
:func:`sanitize_action_log` reports them separately.
|
||||
"""
|
||||
return [e for e in (action_log or []) if isinstance(e, dict)]
|
||||
|
||||
|
||||
def sanitize_action_log(
|
||||
action_log: list | None,
|
||||
) -> tuple[list[dict], list[dict[str, str]]]:
|
||||
"""Split an action log into structured entries and sanitized findings (#698).
|
||||
|
||||
Malformed entries become clear, sanitized ``warning`` findings — the
|
||||
offending value's content is never echoed back (only its position and
|
||||
type), so secrets or garbage in a broken log cannot leak into validation
|
||||
errors, and validation itself proceeds without secondary exceptions.
|
||||
"""
|
||||
if action_log is None:
|
||||
return [], []
|
||||
if not isinstance(action_log, (list, tuple)):
|
||||
return [], [
|
||||
validator_finding(
|
||||
"shared.action_log_malformed",
|
||||
"downgrade",
|
||||
"Action log",
|
||||
"action_log is not a list of structured entries "
|
||||
f"(got {type(action_log).__name__}); it was ignored",
|
||||
"pass action_log as a list of dict entries",
|
||||
)
|
||||
]
|
||||
entries: list[dict] = []
|
||||
findings: list[dict[str, str]] = []
|
||||
for index, entry in enumerate(action_log):
|
||||
if isinstance(entry, dict):
|
||||
entries.append(entry)
|
||||
continue
|
||||
findings.append(
|
||||
validator_finding(
|
||||
"shared.action_log_malformed",
|
||||
"downgrade",
|
||||
"Action log",
|
||||
f"action_log entry {index} is not a structured mapping "
|
||||
f"(got {type(entry).__name__}); the entry was ignored",
|
||||
"repair the malformed action_log entry or drop it before "
|
||||
"revalidating",
|
||||
)
|
||||
)
|
||||
return entries, findings
|
||||
|
||||
|
||||
def validator_finding(
|
||||
rule_id: str,
|
||||
severity: str,
|
||||
@@ -421,7 +480,7 @@ def _rule_shared_canonical_comment_post_claim(
|
||||
rejected_in_report = bool(_CANONICAL_VALIDATION_REJECTED_RE.search(text))
|
||||
rejected_in_log = False
|
||||
if action_log:
|
||||
for entry in action_log:
|
||||
for entry in _iter_action_entries(action_log):
|
||||
validation = entry.get("canonical_comment_validation") or {}
|
||||
if validation.get("allowed") is False:
|
||||
rejected_in_log = True
|
||||
@@ -475,9 +534,13 @@ def _rule_reviewer_vague_mutations_none(
|
||||
action_log: list[dict] | None = None,
|
||||
mutations_observed: bool = False,
|
||||
) -> list[dict[str, str]]:
|
||||
# #698: infer review mutations only from authoritative evidence — an
|
||||
# entry proves a mutation only when it affirmatively records
|
||||
# performed=true and was not gated. Read-only diagnostics and pre-API
|
||||
# rejections (entries without a performed flag) are not mutations.
|
||||
performed = any(
|
||||
e.get("performed") is not False and not e.get("gated_rejected")
|
||||
for e in (action_log or [])
|
||||
e.get("performed") is True and not e.get("gated_rejected")
|
||||
for e in _iter_action_entries(action_log)
|
||||
)
|
||||
if not (mutations_observed or performed):
|
||||
return []
|
||||
@@ -536,7 +599,7 @@ def _rule_reviewer_git_fetch_readonly(
|
||||
text = report_text or ""
|
||||
fetch_observed = any(
|
||||
_GIT_FETCH_RE.search(str(e.get("command") or e.get("action") or ""))
|
||||
for e in (action_log or [])
|
||||
for e in _iter_action_entries(action_log)
|
||||
) or _GIT_FETCH_RE.search(text)
|
||||
if not fetch_observed:
|
||||
return []
|
||||
@@ -977,7 +1040,7 @@ def _rule_reviewer_target_branch_freshness(
|
||||
fields = _handoff_fields(text)
|
||||
fetch_reported = bool(_GIT_FETCH_RE.search(text)) or any(
|
||||
_GIT_FETCH_RE.search(str(e.get("command") or e.get("action") or ""))
|
||||
for e in (action_log or [])
|
||||
for e in _iter_action_entries(action_log)
|
||||
)
|
||||
target_sha_reported = bool(_TARGET_BRANCH_SHA_RE.search(text)) or any(
|
||||
"target branch" in key and "sha" in key and _FULL_SHA_RE.search(value)
|
||||
@@ -1735,6 +1798,12 @@ def assess_final_report_validator(
|
||||
checks: dict[str, Any] = {}
|
||||
findings: list[dict[str, str]] = []
|
||||
|
||||
# #698: malformed action_log data must never crash validation with a
|
||||
# secondary exception; malformed entries surface as sanitized findings.
|
||||
sanitized_action_log, action_log_findings = sanitize_action_log(action_log)
|
||||
action_log = sanitized_action_log
|
||||
findings.extend(action_log_findings)
|
||||
|
||||
if normalized_kind == "issue_filing" and issue_filing_lock is not None:
|
||||
checks["issue_filing"] = assess_issue_filing_final_report(
|
||||
report_text,
|
||||
@@ -1763,7 +1832,23 @@ def assess_final_report_validator(
|
||||
}
|
||||
|
||||
for rule in _RULES_BY_TASK.get(normalized_kind, ()):
|
||||
findings.extend(_call_rule(rule, report_text, normalized_kind, rule_kwargs))
|
||||
try:
|
||||
findings.extend(
|
||||
_call_rule(rule, report_text, normalized_kind, rule_kwargs)
|
||||
)
|
||||
except Exception as exc: # #698: fail closed with a sanitized error
|
||||
findings.append(
|
||||
validator_finding(
|
||||
"shared.validator_rule_error",
|
||||
"block",
|
||||
"Validator",
|
||||
f"validator rule '{getattr(rule, '__name__', 'unknown')}' "
|
||||
f"failed with {type(exc).__name__} (details withheld; "
|
||||
"sanitized)",
|
||||
"file a validator defect with the rule name; do not "
|
||||
"bypass final-report validation",
|
||||
)
|
||||
)
|
||||
|
||||
grade, blocked, downgraded = _aggregate_grade(findings)
|
||||
reasons = [f"{f['rule_id']}: {f['reason']}" for f in findings]
|
||||
|
||||
Reference in New Issue
Block a user