feat(fleet): add CAS-protected stale worker retirement capability
Adds a sanctioned controller/reconciler capability that retires conclusively stale MCP worker-registry rows through a dry-run-first, compare-and-swap protected workflow (#980). The #978 fleet snapshot is read-only, and its registry_revision is seeded with snapshot_at at second precision, so a CAS gated on it can never pass. That token is left unchanged; retirement gets its own stable registry_fingerprint derived exclusively from canonical retirement-relevant registry content, plus a candidate_fingerprint pinning the approved set. Identical contents observed at different times produce the same token; row order never affects it; any retirement-relevant change moves it. WorkerRegistry.retire_stale_workers performs the whole decision inside one BEGIN IMMEDIATE transaction: re-read rows, recompute the fingerprint from those rows, recompute the eligibility plan from those rows (re-reading active workflow leases), compare the candidate fingerprint, revalidate every target, then retire each survivor with a guarded UPDATE asserting its status, heartbeat, generation, session, fencing epoch, and pid are unchanged. Drift retires zero workers and reports registry_revision_moved or candidate_set_moved; any exception rolls back and reports transaction_failed, so a partial write is never reported as success. Retirement requires a full conjunction: active status, complete registry fields, parsable heartbeat, not live, pid_alive false, expired heartbeat, stale ownership, canonical repository binding, no identity evidence shared with a live or unprobeable worker, and no active workflow-lease ownership. Everything else is preserved with a structured reason code. Retired rows become historical rather than stale and keep their history; nothing is deleted. Retirement does not repair untrusted live identity: live workers on legacy instance identities are preserved and keep their legacy_incomplete_identity blockers, so the result never claims the fleet became safe. Exposed as gitea_plan_stale_worker_retirement and gitea_apply_stale_worker_retirement, restricted to controller/reconciler role kinds. The Gitea operation gate stays gitea.read because the mutation lands in the local control-plane registry, matching the #601 lease lifecycle; no new Gitea write permission is introduced and no author permission is broadened. Tests: tests/test_issue_980_stale_worker_retirement.py (40 passed, 23 subtests), including the regression test proving the old snapshot_at derivation moved the token one second apart while the new registry CAS token does not. Full suite from the branch worktree matches the master baseline exactly: 28 failed / 6206 passed vs 28 failed / 6166 passed, identical failure set. Closes #980 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Claude Opus 4.8
+279
-1
@@ -39,7 +39,7 @@ import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Iterator
|
||||
from typing import Any, Callable, Iterator, Sequence
|
||||
|
||||
|
||||
# --- Provenance verdicts -------------------------------------------------
|
||||
@@ -224,6 +224,13 @@ def _heartbeat_expectation_drift(
|
||||
STATUS_ACTIVE = "active"
|
||||
STATUS_SUPERSEDED = "superseded"
|
||||
STATUS_RELEASED = "released"
|
||||
#: #980 terminal state for a registration whose owning process is conclusively
|
||||
#: gone. Distinct from ``released`` (the worker said goodbye) and
|
||||
#: ``superseded`` (a newer generation took over): ``retired`` records that the
|
||||
#: *control plane* concluded the row was a stale orphan and retired it under a
|
||||
#: compare-and-swap. Like every non-active status it is not live, so a retired
|
||||
#: row counts as historical rather than stale in the #978 fleet snapshot.
|
||||
STATUS_RETIRED = "retired"
|
||||
|
||||
_TRUE_VALUES = frozenset({"1", "true", "yes", "client_managed"})
|
||||
_FALSE_VALUES = frozenset({"0", "false", "no", "manual", "manual_launch"})
|
||||
@@ -315,6 +322,12 @@ _SCHEMA_OPTIONAL_COLUMNS: tuple[tuple[str, str], ...] = (
|
||||
("parity_revision", "TEXT"),
|
||||
("live_revision", "TEXT"),
|
||||
("instance_id_provenance", "TEXT"),
|
||||
# #980 retirement bookkeeping. Deliberately outside the CAS fingerprint
|
||||
# field set: they record *that* a retirement happened, and are written only
|
||||
# by the retirement transaction itself.
|
||||
("retired_at", "TEXT"),
|
||||
("retired_by", "TEXT"),
|
||||
("retirement_reason", "TEXT"),
|
||||
)
|
||||
|
||||
|
||||
@@ -1217,6 +1230,271 @@ class WorkerRegistry:
|
||||
"reasons": [],
|
||||
}
|
||||
|
||||
def retire_stale_workers(
|
||||
self,
|
||||
*,
|
||||
expected_registry_fingerprint: str,
|
||||
expected_candidate_fingerprint: str,
|
||||
worker_identities: Sequence[str],
|
||||
fingerprint_fn: Callable[[list[dict[str, Any]]], str],
|
||||
plan_fn: Callable[[list[dict[str, Any]]], dict[str, Any]],
|
||||
retired_by: str | None = None,
|
||||
retirement_reason: str = "",
|
||||
now: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Compare-and-swap retirement of conclusively stale registrations (#980).
|
||||
|
||||
The whole decision happens inside one ``BEGIN IMMEDIATE`` transaction:
|
||||
the authoritative rows are re-read, the stable registry fingerprint is
|
||||
recomputed from *those* rows, the eligibility plan is recomputed from
|
||||
*those* rows, and only then are the approved targets retired — each
|
||||
with a per-row guarded ``UPDATE`` that also asserts the row's identity,
|
||||
liveness, and ownership columns are byte-identical to what the
|
||||
revalidation just read. There is no window in which a safety check and
|
||||
its matching write are separated by another statement, so a worker that
|
||||
comes back to life, changes ownership, or is retired concurrently
|
||||
cannot be deleted on the strength of a stale observation.
|
||||
|
||||
``fingerprint_fn`` and ``plan_fn`` are injected rather than imported so
|
||||
the storage layer never depends on the decision layer; production wires
|
||||
in :func:`mcp_fleet_retirement.registry_fingerprint` and
|
||||
:func:`mcp_fleet_retirement.plan_stale_worker_retirement`, which is
|
||||
exactly what the plan surface used.
|
||||
|
||||
Any exception — including a failure to commit — rolls the transaction
|
||||
back and is reported as ``transaction_failed`` with zero retirements
|
||||
and ``mutation_performed`` false, so a partial write can never be
|
||||
reported as success.
|
||||
"""
|
||||
requested = [str(w) for w in (worker_identities or []) if str(w).strip()]
|
||||
stamp = _ts(now or _utc_now())
|
||||
result: dict[str, Any] | None = None
|
||||
|
||||
def _base(outcome: str) -> dict[str, Any]:
|
||||
return {
|
||||
"success": True,
|
||||
"outcome": outcome,
|
||||
"mutation_performed": False,
|
||||
"retired": [],
|
||||
"retired_count": 0,
|
||||
"preserved": [],
|
||||
"preserved_count": 0,
|
||||
"requested_count": len(requested),
|
||||
"expected_registry_fingerprint": expected_registry_fingerprint,
|
||||
"expected_candidate_fingerprint": expected_candidate_fingerprint,
|
||||
"acting_identity": retired_by,
|
||||
"reasons": [],
|
||||
}
|
||||
|
||||
if not requested:
|
||||
outcome = _base("nothing_requested")
|
||||
outcome["reasons"] = [
|
||||
"no worker identities were supplied; nothing to retire"
|
||||
]
|
||||
return outcome
|
||||
|
||||
try:
|
||||
with self._tx() as conn:
|
||||
rows = [
|
||||
self._row_to_record(r)
|
||||
for r in conn.execute(
|
||||
"SELECT * FROM worker_registrations"
|
||||
).fetchall()
|
||||
]
|
||||
by_identity = {
|
||||
str(row.get("worker_identity")): row for row in rows
|
||||
}
|
||||
current_registry_fingerprint = fingerprint_fn(rows)
|
||||
|
||||
if current_registry_fingerprint != expected_registry_fingerprint:
|
||||
already = [
|
||||
wid
|
||||
for wid in requested
|
||||
if str(
|
||||
(by_identity.get(wid) or {}).get("status") or ""
|
||||
)
|
||||
== STATUS_RETIRED
|
||||
]
|
||||
idempotent = len(already) == len(requested)
|
||||
result = _base(
|
||||
"already_retired" if idempotent else "registry_revision_moved"
|
||||
)
|
||||
result["idempotent"] = idempotent
|
||||
result["current_registry_fingerprint"] = (
|
||||
current_registry_fingerprint
|
||||
)
|
||||
result["reasons"] = [
|
||||
"the worker registry changed between plan and apply; "
|
||||
"retiring zero workers"
|
||||
if not idempotent
|
||||
else "every requested registration is already retired; "
|
||||
"safe no-op"
|
||||
]
|
||||
result["preserved"] = [
|
||||
{
|
||||
"worker_identity": wid,
|
||||
"reason_code": "registry_revision_moved",
|
||||
"detail": (
|
||||
"aborted before any retirement: registry "
|
||||
"fingerprint moved"
|
||||
),
|
||||
}
|
||||
for wid in requested
|
||||
]
|
||||
result["preserved_count"] = len(requested)
|
||||
return result
|
||||
|
||||
fresh_plan = plan_fn(rows)
|
||||
current_candidate_fingerprint = fresh_plan.get(
|
||||
"candidate_fingerprint"
|
||||
)
|
||||
if current_candidate_fingerprint != expected_candidate_fingerprint:
|
||||
result = _base("candidate_set_moved")
|
||||
result["current_registry_fingerprint"] = (
|
||||
current_registry_fingerprint
|
||||
)
|
||||
result["current_candidate_fingerprint"] = (
|
||||
current_candidate_fingerprint
|
||||
)
|
||||
result["reasons"] = [
|
||||
"the retirement candidate set changed between plan and "
|
||||
"apply; retiring zero workers"
|
||||
]
|
||||
result["preserved"] = [
|
||||
{
|
||||
"worker_identity": wid,
|
||||
"reason_code": "candidate_set_moved",
|
||||
"detail": (
|
||||
"aborted before any retirement: candidate "
|
||||
"fingerprint moved"
|
||||
),
|
||||
}
|
||||
for wid in requested
|
||||
]
|
||||
result["preserved_count"] = len(requested)
|
||||
return result
|
||||
|
||||
eligible = {
|
||||
str(c.get("worker_identity")): c
|
||||
for c in fresh_plan.get("candidates") or []
|
||||
}
|
||||
preserved_index = {
|
||||
str(p.get("worker_identity")): p
|
||||
for p in fresh_plan.get("preserved") or []
|
||||
}
|
||||
|
||||
retired: list[dict[str, Any]] = []
|
||||
preserved: list[dict[str, Any]] = []
|
||||
for wid in requested:
|
||||
row = by_identity.get(wid)
|
||||
if row is None:
|
||||
preserved.append(
|
||||
{
|
||||
"worker_identity": wid,
|
||||
"reason_code": "registration_missing",
|
||||
"detail": "no registration row with this identity",
|
||||
}
|
||||
)
|
||||
continue
|
||||
if wid not in eligible:
|
||||
blocked = preserved_index.get(wid) or {}
|
||||
preserved.append(
|
||||
{
|
||||
"worker_identity": wid,
|
||||
"reason_code": blocked.get("reason_code")
|
||||
or "not_in_current_plan",
|
||||
"detail": blocked.get("detail")
|
||||
or (
|
||||
"revalidation immediately before retirement "
|
||||
"no longer finds this worker eligible"
|
||||
),
|
||||
"evidence": blocked.get("evidence"),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
cursor = conn.execute(
|
||||
"UPDATE worker_registrations "
|
||||
"SET status = ?, retired_at = ?, retired_by = ?, "
|
||||
" retirement_reason = ? "
|
||||
"WHERE worker_identity = ? "
|
||||
" AND status = ? "
|
||||
" AND last_heartbeat_at = ? "
|
||||
" AND generation_id = ? "
|
||||
" AND session_id = ? "
|
||||
" AND fencing_epoch = ? "
|
||||
" AND IFNULL(pid, -1) = IFNULL(?, -1)",
|
||||
(
|
||||
STATUS_RETIRED,
|
||||
stamp,
|
||||
retired_by,
|
||||
retirement_reason
|
||||
or (eligible[wid].get("reason_code") or ""),
|
||||
wid,
|
||||
STATUS_ACTIVE,
|
||||
row.get("last_heartbeat_at"),
|
||||
row.get("generation_id"),
|
||||
row.get("session_id"),
|
||||
row.get("fencing_epoch"),
|
||||
row.get("pid"),
|
||||
),
|
||||
)
|
||||
if cursor.rowcount == 1:
|
||||
retired.append(
|
||||
{
|
||||
"worker_identity": wid,
|
||||
"reason_code": eligible[wid].get("reason_code"),
|
||||
"retired_at": stamp,
|
||||
"retired_by": retired_by,
|
||||
"evidence": eligible[wid].get("evidence"),
|
||||
}
|
||||
)
|
||||
else:
|
||||
preserved.append(
|
||||
{
|
||||
"worker_identity": wid,
|
||||
"reason_code": "row_changed_since_plan",
|
||||
"detail": (
|
||||
"guarded update matched no row; the "
|
||||
"registration changed inside the retirement "
|
||||
"transaction"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
result = _base("applied")
|
||||
result["mutation_performed"] = bool(retired)
|
||||
result["retired"] = retired
|
||||
result["retired_count"] = len(retired)
|
||||
result["preserved"] = preserved
|
||||
result["preserved_count"] = len(preserved)
|
||||
result["current_registry_fingerprint"] = (
|
||||
current_registry_fingerprint
|
||||
)
|
||||
result["current_candidate_fingerprint"] = (
|
||||
current_candidate_fingerprint
|
||||
)
|
||||
result["retired_at"] = stamp if retired else None
|
||||
except Exception as exc: # rolled back by _tx; report, never half-claim
|
||||
failure = _base("transaction_failed")
|
||||
failure["success"] = False
|
||||
failure["reasons"] = [
|
||||
"retirement transaction failed and was rolled back; zero "
|
||||
f"registrations were retired: {type(exc).__name__}: {exc}"
|
||||
]
|
||||
failure["preserved"] = [
|
||||
{
|
||||
"worker_identity": wid,
|
||||
"reason_code": "transaction_failed",
|
||||
"detail": "transaction rolled back before any commit",
|
||||
}
|
||||
for wid in requested
|
||||
]
|
||||
failure["preserved_count"] = len(requested)
|
||||
return failure
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _public_record(record: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Registry row minus anything that should not travel to an LLM surface."""
|
||||
|
||||
Reference in New Issue
Block a user