fix(drain): fail closed on missing or unproven acknowledgement evidence (#661)
The acks_or_timeout check treated an absent `acks` key as proof that no
session needed to acknowledge: `drain_state.get("acks") or {}` collapsed
absent, None, and empty into the same value, and the resulting empty mapping
satisfied `no_sessions_to_ack`. The impact report's counts.sessions_live_other
was never consulted, so absence of evidence was read as evidence of absence.
Reproduced at head 1cbbde0089: with
sessions_live_other = 3 and the acknowledgement key absent, acks_or_timeout
passed with detail "no other live sessions required to acknowledge", the proof
minted clean, and gate_apply_restart returned verdict allow — a restart
authorized against three live sessions with zero acknowledgement evidence, and
the resulting artifact carried a valid signature.
Whether acknowledgement is required is now derived from the impact report,
never from the shape of the drain state:
- _live_session_count() reads counts.sessions_live_other and returns None for a
missing, malformed, negative, or bool value, so an unreadable report fails
closed instead of reading as "nobody was live".
- Absent, None, non-mapping, empty, partially-covering, and unparseable or
stale acknowledgement data all fail closed while live sessions require
acknowledgement.
- _is_acknowledged() no longer coerces with str(); only an explicit
"ack"/"acked"/"acknowledged" string counts, so None, timestamps, and
"pending"/"stale" markers are never read as an acknowledgement.
- Present-but-unacknowledged entries fail closed even when the report claims
zero live sessions: that contradiction is not safe to resolve in favour of
the restart.
- ack_timeout_policy_applied stays strict (`value is True`), so an absent, null,
or non-boolean value cannot open the gate on its own.
The genuine no-other-live-sessions case still passes, now justified by the
report proving sessions_live_other == 0 rather than by the absence of data.
Adds AcknowledgementFailClosedTests: 14 cases / 26 subtests covering missing,
null, empty, malformed, stale, partial-coverage, and unproven-count inputs,
the valid-acknowledgement and zero-live-session paths, timeout-policy
strictness, and that a failed check blocks proof.clean, verification, and the
restart gate.
Restart-surface suite: 132 passed, 38 subtests (branch baseline 118 passed,
12 subtests; +14 new tests, no regressions).
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01VEaP3TohHLFWkp3Z2mmuZw
This commit is contained in:
+121
-16
@@ -71,6 +71,11 @@ CHECK_HANDOFFS_OK = "handoffs_ok"
|
||||
CHECK_LEASES_HANDLED = "leases_handled"
|
||||
CHECK_ACKS_OR_TIMEOUT = "acks_or_timeout"
|
||||
|
||||
# The only values accepted as an explicit acknowledgement from a live session.
|
||||
# Anything else — including a missing entry, a null, or a "pending"/"stale"
|
||||
# marker — leaves that session unacknowledged.
|
||||
_ACK_TOKENS = frozenset({"ack", "acked", "acknowledged"})
|
||||
|
||||
REQUIRED_CHECKS: tuple[str, ...] = (
|
||||
CHECK_NO_INFLIGHT_MUTATIONS,
|
||||
CHECK_ASSIGNMENTS_STOPPED,
|
||||
@@ -291,6 +296,38 @@ def _bool_input(value: Any) -> bool:
|
||||
return value is True
|
||||
|
||||
|
||||
def _live_session_count(impact_report: Mapping[str, Any]) -> int | None:
|
||||
"""Other-live-session count from the report, or ``None`` when unproven.
|
||||
|
||||
Only a real, non-negative integer counts. A missing ``counts`` block, a
|
||||
malformed one, a non-integer, a bool (``True`` is an ``int`` in Python), or
|
||||
a negative value all return ``None`` so the caller fails closed rather than
|
||||
treating an unreadable report as "nobody was live".
|
||||
"""
|
||||
|
||||
counts = impact_report.get("counts")
|
||||
if not isinstance(counts, Mapping):
|
||||
return None
|
||||
value = counts.get("sessions_live_other")
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _is_acknowledged(value: Any) -> bool:
|
||||
"""True only for an explicit acknowledgement token.
|
||||
|
||||
Deliberately strict: the value must already be a string carrying one of the
|
||||
recognised tokens. Non-strings are not coerced, so ``None``, timestamps,
|
||||
objects, and states such as "pending" or "stale" are never read as an
|
||||
acknowledgement.
|
||||
"""
|
||||
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
return value.strip().lower() in _ACK_TOKENS
|
||||
|
||||
|
||||
def _evaluate_checks(
|
||||
impact_report: Mapping[str, Any],
|
||||
drain_state: Mapping[str, Any],
|
||||
@@ -406,24 +443,92 @@ def _evaluate_checks(
|
||||
)
|
||||
|
||||
# 6. Acknowledgements received, or an explicit timeout policy was applied.
|
||||
acks = drain_state.get("acks") or {}
|
||||
ack_values = list(acks.values()) if isinstance(acks, Mapping) else []
|
||||
all_acked = bool(ack_values) and all(
|
||||
str(v).strip().lower() in {"ack", "acked", "acknowledged"}
|
||||
for v in ack_values
|
||||
)
|
||||
no_sessions_to_ack = isinstance(acks, Mapping) and len(ack_values) == 0
|
||||
#
|
||||
# Whether acknowledgements are *required* is answered by the impact report,
|
||||
# never inferred from the shape of the drain state. Previously an absent
|
||||
# ``acks`` key collapsed to ``{}`` and was read as "no session needed to
|
||||
# acknowledge", so a proof minted clean — and the restart gate allowed —
|
||||
# while the report still showed other live sessions. Absence of evidence is
|
||||
# not evidence of absence: missing, malformed, stale, or otherwise unproven
|
||||
# acknowledgement data fails closed whenever live sessions require
|
||||
# acknowledgement, and only explicitly verified acknowledgement evidence
|
||||
# may permit the operation.
|
||||
sessions_live_other = _live_session_count(impact_report)
|
||||
live_count_known = sessions_live_other is not None
|
||||
# An unknown or malformed count fails closed: it cannot prove nobody had to
|
||||
# acknowledge, so acknowledgement stays required.
|
||||
acks_required = (not live_count_known) or sessions_live_other > 0
|
||||
|
||||
raw_acks = drain_state.get("acks")
|
||||
acks_present = isinstance(raw_acks, Mapping)
|
||||
ack_values = list(raw_acks.values()) if acks_present else []
|
||||
acked_count = sum(1 for value in ack_values if _is_acknowledged(value))
|
||||
every_entry_acked = bool(ack_values) and acked_count == len(ack_values)
|
||||
covers_live_sessions = live_count_known and acked_count >= sessions_live_other
|
||||
# Strict by construction: only an explicit ``True`` counts, so an absent,
|
||||
# null, or non-boolean ``ack_timeout_policy_applied`` can never open this
|
||||
# gate on its own.
|
||||
timeout_policy = _bool_input(drain_state.get("ack_timeout_policy_applied"))
|
||||
acks_ok = all_acked or no_sessions_to_ack or timeout_policy
|
||||
if acks_ok:
|
||||
if timeout_policy and not all_acked:
|
||||
detail = "explicit ack timeout policy applied"
|
||||
elif no_sessions_to_ack:
|
||||
detail = "no other live sessions required to acknowledge"
|
||||
else:
|
||||
detail = "all affected sessions acknowledged"
|
||||
|
||||
# Present-but-unacknowledged entries always fail closed, even when the
|
||||
# report claims nobody was live: the drain state naming an outstanding
|
||||
# session contradicts that claim, and the safe reading of a contradiction
|
||||
# is that an acknowledgement is still owed.
|
||||
outstanding_entries = acks_present and bool(ack_values) and not every_entry_acked
|
||||
|
||||
if timeout_policy and (acks_required or outstanding_entries):
|
||||
acks_ok = True
|
||||
detail = "explicit ack timeout policy applied and recorded"
|
||||
elif outstanding_entries:
|
||||
acks_ok = False
|
||||
detail = (
|
||||
f"{len(ack_values) - acked_count} of {len(ack_values)} "
|
||||
"acknowledgement entries are unparseable, stale, or not "
|
||||
"acknowledged (fail closed)"
|
||||
)
|
||||
elif not acks_required:
|
||||
acks_ok = True
|
||||
detail = (
|
||||
"impact report proves no other live sessions required to "
|
||||
"acknowledge (sessions_live_other=0)"
|
||||
)
|
||||
elif every_entry_acked and covers_live_sessions:
|
||||
acks_ok = True
|
||||
detail = (
|
||||
f"{acked_count} acknowledgement entr"
|
||||
f"{'y' if acked_count == 1 else 'ies'} verified; covers "
|
||||
f"sessions_live_other={sessions_live_other}"
|
||||
)
|
||||
else:
|
||||
detail = "outstanding acks with no timeout policy (fail closed)"
|
||||
acks_ok = False
|
||||
if not live_count_known:
|
||||
detail = (
|
||||
"impact report does not prove the live-session count; "
|
||||
"acknowledgement required and unproven (fail closed)"
|
||||
)
|
||||
elif raw_acks is None:
|
||||
detail = (
|
||||
"acknowledgement evidence absent while "
|
||||
f"sessions_live_other={sessions_live_other} require "
|
||||
"acknowledgement (fail closed)"
|
||||
)
|
||||
elif not acks_present:
|
||||
detail = (
|
||||
"acknowledgement evidence malformed: expected a mapping, got "
|
||||
f"{type(raw_acks).__name__} (fail closed)"
|
||||
)
|
||||
elif not ack_values:
|
||||
detail = (
|
||||
"acknowledgement mapping empty while "
|
||||
f"sessions_live_other={sessions_live_other} require "
|
||||
"acknowledgement (fail closed)"
|
||||
)
|
||||
else:
|
||||
detail = (
|
||||
f"acknowledgements cover only {acked_count} session(s) but the "
|
||||
f"report shows sessions_live_other={sessions_live_other} "
|
||||
"(fail closed)"
|
||||
)
|
||||
checks.append(DrainCheck(CHECK_ACKS_OR_TIMEOUT, acks_ok, detail))
|
||||
|
||||
return checks
|
||||
|
||||
Reference in New Issue
Block a user