sysadminandClaude Opus 4.8
c0c6d14b73
Adds a sanctioned controller/reconciler capability that retires conclusively stale MCP worker-registry rows through a dry-run-first, compare-and-swap protected workflow (#980). The #978 fleet snapshot is read-only, and its registry_revision is seeded with snapshot_at at second precision, so a CAS gated on it can never pass. That token is left unchanged; retirement gets its own stable registry_fingerprint derived exclusively from canonical retirement-relevant registry content, plus a candidate_fingerprint pinning the approved set. Identical contents observed at different times produce the same token; row order never affects it; any retirement-relevant change moves it. WorkerRegistry.retire_stale_workers performs the whole decision inside one BEGIN IMMEDIATE transaction: re-read rows, recompute the fingerprint from those rows, recompute the eligibility plan from those rows (re-reading active workflow leases), compare the candidate fingerprint, revalidate every target, then retire each survivor with a guarded UPDATE asserting its status, heartbeat, generation, session, fencing epoch, and pid are unchanged. Drift retires zero workers and reports registry_revision_moved or candidate_set_moved; any exception rolls back and reports transaction_failed, so a partial write is never reported as success. Retirement requires a full conjunction: active status, complete registry fields, parsable heartbeat, not live, pid_alive false, expired heartbeat, stale ownership, canonical repository binding, no identity evidence shared with a live or unprobeable worker, and no active workflow-lease ownership. Everything else is preserved with a structured reason code. Retired rows become historical rather than stale and keep their history; nothing is deleted. Retirement does not repair untrusted live identity: live workers on legacy instance identities are preserved and keep their legacy_incomplete_identity blockers, so the result never claims the fleet became safe. Exposed as gitea_plan_stale_worker_retirement and gitea_apply_stale_worker_retirement, restricted to controller/reconciler role kinds. The Gitea operation gate stays gitea.read because the mutation lands in the local control-plane registry, matching the #601 lease lifecycle; no new Gitea write permission is introduced and no author permission is broadened. Tests: tests/test_issue_980_stale_worker_retirement.py (40 passed, 23 subtests), including the regression test proving the old snapshot_at derivation moved the token one second apart while the new registry CAS token does not. Full suite from the branch worktree matches the master baseline exactly: 28 failed / 6206 passed vs 28 failed / 6166 passed, identical failure set. Closes #980 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
807 lines
32 KiB
Python
807 lines
32 KiB
Python
"""CAS-protected stale worker retirement (#980).
|
|
|
|
Covers the acceptance criteria: the registry CAS token is stable across time
|
|
and row order but moves on any retirement-relevant change, plan mutates
|
|
nothing, apply fails closed on drift, every target is revalidated inside the
|
|
retirement transaction, and live / ambiguous / incomplete rows are preserved.
|
|
|
|
The regression test that matters most is
|
|
``test_snapshot_at_would_have_moved_the_token``: it demonstrates the exact
|
|
defect the R3-C review found — ``mcp_fleet_snapshot._consistency_token``
|
|
changes when only the observation time changed — and proves the new token does
|
|
not.
|
|
|
|
All identifiers are synthetic.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any
|
|
|
|
import mcp_fleet_retirement as retire
|
|
import mcp_fleet_snapshot as fleet
|
|
import mcp_worker_identity as mwi
|
|
|
|
|
|
NOW = datetime(2026, 7, 30, 7, 0, 0, tzinfo=timezone.utc)
|
|
TTL = 900.0
|
|
REPO = "/synthetic/repo/Gitea-Tools"
|
|
|
|
_ROW_COLUMNS = (
|
|
"worker_identity",
|
|
"client_name",
|
|
"client_instance_id",
|
|
"session_id",
|
|
"generation_id",
|
|
"role",
|
|
"profile",
|
|
"namespace",
|
|
"remote",
|
|
"repository_binding",
|
|
"pid",
|
|
"process_identity",
|
|
"transport",
|
|
"token_fingerprint",
|
|
"started_at",
|
|
"last_heartbeat_at",
|
|
"heartbeat_ttl_seconds",
|
|
"fencing_epoch",
|
|
"status",
|
|
"fleet_run_id",
|
|
"authenticated_account",
|
|
"instance_id_provenance",
|
|
)
|
|
|
|
|
|
def _registry() -> mwi.WorkerRegistry:
|
|
handle, path = tempfile.mkstemp(suffix=".sqlite3")
|
|
os.close(handle)
|
|
os.unlink(path)
|
|
return mwi.WorkerRegistry(path)
|
|
|
|
|
|
def _row(
|
|
*,
|
|
identity: str,
|
|
pid: int | None,
|
|
instance: str = "inst-codex-20260730T070000Z-0123456789ab",
|
|
session: str = "sess-a",
|
|
generation: str = "gen-a",
|
|
namespace: str = "author",
|
|
heartbeat: datetime | None = None,
|
|
status: str = mwi.STATUS_ACTIVE,
|
|
repository_binding: str | None = REPO,
|
|
ttl: float = TTL,
|
|
**overrides: Any,
|
|
) -> dict[str, Any]:
|
|
"""One synthetic ``worker_registrations`` row."""
|
|
record = {
|
|
"worker_identity": identity,
|
|
"client_name": "codex",
|
|
"client_instance_id": instance,
|
|
"session_id": session,
|
|
"generation_id": generation,
|
|
"role": namespace,
|
|
"profile": f"prgs-{namespace}",
|
|
"namespace": namespace,
|
|
"remote": "prgs",
|
|
"repository_binding": repository_binding,
|
|
"pid": pid,
|
|
"process_identity": f"pid-{pid}" if pid is not None else None,
|
|
"transport": "stdio",
|
|
"token_fingerprint": None,
|
|
"started_at": "2026-07-30T06:00:00Z",
|
|
"last_heartbeat_at": mwi._ts(heartbeat or (NOW - timedelta(seconds=7200))),
|
|
"heartbeat_ttl_seconds": ttl,
|
|
"fencing_epoch": 1,
|
|
"status": status,
|
|
"fleet_run_id": "run-canary",
|
|
"authenticated_account": "synthetic-user",
|
|
"instance_id_provenance": fleet.INSTANCE_ID_PROVENANCE_TRUSTED,
|
|
}
|
|
record.update(overrides)
|
|
return record
|
|
|
|
|
|
def _dead(_pid: int | None) -> bool:
|
|
return False
|
|
|
|
|
|
def _alive(_pid: int | None) -> bool:
|
|
return True
|
|
|
|
|
|
def _selective(alive_pids: set[int]):
|
|
def probe(pid: int | None) -> bool:
|
|
return pid in alive_pids
|
|
|
|
return probe
|
|
|
|
|
|
def _plan(rows, *, probe=_dead, now=NOW, **kwargs):
|
|
return retire.plan_stale_worker_retirement(
|
|
rows,
|
|
now=now,
|
|
pid_alive_probe=probe,
|
|
canonical_repository=REPO,
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
def _register(registry: mwi.WorkerRegistry, row: dict[str, Any]) -> None:
|
|
"""Insert a synthetic row directly, bypassing register()'s live-now stamps."""
|
|
columns = [name for name in _ROW_COLUMNS if name in row]
|
|
placeholders = ", ".join("?" for _ in columns)
|
|
with registry._tx() as conn:
|
|
conn.execute(
|
|
f"INSERT INTO worker_registrations ({', '.join(columns)}) "
|
|
f"VALUES ({placeholders})",
|
|
[row[name] for name in columns],
|
|
)
|
|
|
|
|
|
class RegistryFingerprintStabilityTests(unittest.TestCase):
|
|
"""AC: identical contents observed at different times produce one token."""
|
|
|
|
def test_same_contents_different_observation_times_same_token(self):
|
|
rows = [_row(identity="w-1", pid=101), _row(identity="w-2", pid=102)]
|
|
self.assertEqual(
|
|
retire.registry_fingerprint(rows), retire.registry_fingerprint(rows)
|
|
)
|
|
# Recompute after the clock has moved a full hour: the token is derived
|
|
# from content only, so it cannot notice.
|
|
plan_early = _plan(rows, now=NOW)
|
|
plan_late = _plan(rows, now=NOW + timedelta(hours=1))
|
|
self.assertEqual(
|
|
plan_early["registry_fingerprint"], plan_late["registry_fingerprint"]
|
|
)
|
|
|
|
def test_snapshot_at_would_have_moved_the_token(self):
|
|
"""Regression: the old time-seeded derivation moved, the new one does not."""
|
|
rows = [_row(identity="w-1", pid=101)]
|
|
old_early = fleet._consistency_token(rows, "2026-07-30T07:00:00Z")
|
|
old_late = fleet._consistency_token(rows, "2026-07-30T07:00:01Z")
|
|
self.assertNotEqual(
|
|
old_early,
|
|
old_late,
|
|
"the #980 defect: one second of observation drift changed the token",
|
|
)
|
|
self.assertEqual(
|
|
retire.registry_fingerprint(rows), retire.registry_fingerprint(rows)
|
|
)
|
|
|
|
def test_row_order_does_not_change_the_token(self):
|
|
rows = [
|
|
_row(identity="w-1", pid=101),
|
|
_row(identity="w-2", pid=102),
|
|
_row(identity="w-3", pid=103),
|
|
]
|
|
self.assertEqual(
|
|
retire.registry_fingerprint(rows),
|
|
retire.registry_fingerprint(list(reversed(rows))),
|
|
)
|
|
|
|
def test_numeric_typing_does_not_change_the_token(self):
|
|
as_float = [_row(identity="w-1", pid=101, heartbeat_ttl_seconds=900.0)]
|
|
as_int = [_row(identity="w-1", pid=101, heartbeat_ttl_seconds=900)]
|
|
self.assertEqual(
|
|
retire.registry_fingerprint(as_float),
|
|
retire.registry_fingerprint(as_int),
|
|
)
|
|
|
|
def test_retirement_relevant_changes_move_the_token(self):
|
|
base = [_row(identity="w-1", pid=101)]
|
|
baseline = retire.registry_fingerprint(base)
|
|
mutations = {
|
|
"row added": base + [_row(identity="w-2", pid=102)],
|
|
"row removed": [],
|
|
"status changed": [_row(identity="w-1", pid=101, status="released")],
|
|
"identity changed": [
|
|
_row(
|
|
identity="w-1",
|
|
pid=101,
|
|
instance="inst-codex-20260730T070000Z-ffffffffffff",
|
|
)
|
|
],
|
|
"ownership changed": [_row(identity="w-1", pid=101, session="sess-other")],
|
|
"generation changed": [
|
|
_row(identity="w-1", pid=101, generation="gen-other")
|
|
],
|
|
"pid changed": [_row(identity="w-1", pid=999)],
|
|
"repository binding changed": [
|
|
_row(identity="w-1", pid=101, repository_binding="/elsewhere")
|
|
],
|
|
}
|
|
for label, rows in mutations.items():
|
|
with self.subTest(change=label):
|
|
self.assertNotEqual(baseline, retire.registry_fingerprint(rows))
|
|
|
|
def test_heartbeat_change_moves_the_token(self):
|
|
base = [_row(identity="w-1", pid=101)]
|
|
moved = [_row(identity="w-1", pid=101, heartbeat=NOW - timedelta(seconds=30))]
|
|
self.assertNotEqual(
|
|
retire.registry_fingerprint(base), retire.registry_fingerprint(moved)
|
|
)
|
|
|
|
def test_ttl_change_moves_the_token(self):
|
|
base = [_row(identity="w-1", pid=101)]
|
|
moved = [_row(identity="w-1", pid=101, ttl=60.0)]
|
|
self.assertNotEqual(
|
|
retire.registry_fingerprint(base), retire.registry_fingerprint(moved)
|
|
)
|
|
|
|
|
|
class PlanEligibilityTests(unittest.TestCase):
|
|
"""AC: only conclusively stale orphans are selected; everything else stays."""
|
|
|
|
def test_plan_selects_dead_stale_orphan(self):
|
|
plan = _plan([_row(identity="w-1", pid=101)])
|
|
self.assertEqual(plan["candidate_count"], 1)
|
|
self.assertEqual(plan["candidate_worker_identities"], ["w-1"])
|
|
self.assertEqual(plan["candidates"][0]["reason_code"], retire.REASON_ELIGIBLE)
|
|
self.assertFalse(plan["mutation_performed"])
|
|
self.assertTrue(plan["read_only"])
|
|
|
|
def test_plan_performs_no_mutation(self):
|
|
registry = _registry()
|
|
_register(registry, _row(identity="w-1", pid=101))
|
|
before = registry.list_workers(status=None)
|
|
plan = _plan(before)
|
|
after = registry.list_workers(status=None)
|
|
self.assertEqual(plan["candidate_count"], 1)
|
|
self.assertEqual(before, after)
|
|
self.assertEqual([r["status"] for r in after], [mwi.STATUS_ACTIVE])
|
|
self.assertFalse(plan["mutation_performed"])
|
|
|
|
def test_live_worker_is_preserved(self):
|
|
row = _row(identity="w-live", pid=101, heartbeat=NOW - timedelta(seconds=10))
|
|
plan = _plan([row], probe=_alive)
|
|
self.assertEqual(plan["candidate_count"], 0)
|
|
self.assertEqual(
|
|
plan["preserved"][0]["reason_code"], retire.REASON_WORKER_LIVE
|
|
)
|
|
|
|
def test_fresh_heartbeat_with_dead_pid_still_fails_closed(self):
|
|
"""A dead pid withdraws liveness; the unexpired heartbeat still preserves."""
|
|
row = _row(identity="w-fresh", pid=101, heartbeat=NOW - timedelta(seconds=10))
|
|
plan = _plan([row], probe=_dead)
|
|
self.assertEqual(plan["candidate_count"], 0)
|
|
self.assertEqual(
|
|
plan["preserved"][0]["reason_code"], retire.REASON_HEARTBEAT_FRESH
|
|
)
|
|
|
|
def test_unprobeable_pid_is_preserved(self):
|
|
plan = _plan([_row(identity="w-1", pid=101)], probe=lambda _pid: None)
|
|
self.assertEqual(plan["candidate_count"], 0)
|
|
self.assertEqual(
|
|
plan["preserved"][0]["reason_code"], retire.REASON_PID_UNKNOWN
|
|
)
|
|
|
|
def test_incomplete_legacy_identity_is_preserved(self):
|
|
"""AC: incomplete legacy identities remain fail-closed."""
|
|
rows = [
|
|
_row(identity="w-nopid", pid=None, instance="legacy-pid-27833"),
|
|
_row(identity="w-nosession", pid=102, session=""),
|
|
]
|
|
plan = _plan(rows)
|
|
self.assertEqual(plan["candidate_count"], 0)
|
|
self.assertEqual(
|
|
{p["reason_code"] for p in plan["preserved"]},
|
|
{retire.REASON_INCOMPLETE_IDENTITY},
|
|
)
|
|
detail = next(
|
|
p["detail"] for p in plan["preserved"] if p["worker_identity"] == "w-nopid"
|
|
)
|
|
self.assertIn("pid", detail)
|
|
|
|
def test_unparsable_heartbeat_is_preserved(self):
|
|
rows = [_row(identity="w-1", pid=101, last_heartbeat_at="not-a-stamp")]
|
|
plan = _plan(rows)
|
|
self.assertEqual(plan["candidate_count"], 0)
|
|
self.assertEqual(
|
|
plan["preserved"][0]["reason_code"], retire.REASON_UNPARSABLE_HEARTBEAT
|
|
)
|
|
|
|
def test_identity_shared_with_live_worker_is_preserved(self):
|
|
"""Two rows, one live: the dead one shares session evidence, so it stays."""
|
|
rows = [
|
|
_row(
|
|
identity="w-live",
|
|
pid=101,
|
|
session="sess-shared",
|
|
heartbeat=NOW - timedelta(seconds=5),
|
|
),
|
|
_row(identity="w-dead", pid=102, session="sess-shared"),
|
|
]
|
|
plan = _plan(rows, probe=_selective({101}))
|
|
self.assertEqual(plan["candidate_count"], 0)
|
|
codes = {p["reason_code"] for p in plan["preserved"]}
|
|
self.assertIn(retire.REASON_CONFLICTING_IDENTITY, codes)
|
|
|
|
def test_foreign_or_missing_repository_binding_is_preserved(self):
|
|
rows = [
|
|
_row(identity="w-foreign", pid=101, repository_binding="/other/repo"),
|
|
_row(identity="w-unbound", pid=102, repository_binding=None),
|
|
]
|
|
plan = _plan(rows)
|
|
self.assertEqual(plan["candidate_count"], 0)
|
|
self.assertEqual(
|
|
{p["reason_code"] for p in plan["preserved"]},
|
|
{retire.REASON_FOREIGN_REPOSITORY},
|
|
)
|
|
|
|
def test_active_workflow_owner_is_preserved(self):
|
|
rows = [_row(identity="w-1", pid=101, session="sess-leased")]
|
|
plan = _plan(rows, protected_session_ids=["sess-leased"])
|
|
self.assertEqual(plan["candidate_count"], 0)
|
|
self.assertEqual(
|
|
plan["preserved"][0]["reason_code"], retire.REASON_PROTECTED_OWNER
|
|
)
|
|
|
|
def test_terminal_rows_are_not_retired_again(self):
|
|
rows = [_row(identity="w-1", pid=101, status=mwi.STATUS_RELEASED)]
|
|
plan = _plan(rows)
|
|
self.assertEqual(plan["candidate_count"], 0)
|
|
self.assertEqual(
|
|
plan["preserved"][0]["reason_code"], retire.REASON_ALREADY_TERMINAL
|
|
)
|
|
|
|
def test_mixed_fleet_produces_correct_per_worker_outcomes(self):
|
|
rows = [
|
|
_row(identity="w-stale-1", pid=101, session="s1", generation="g1"),
|
|
_row(identity="w-stale-2", pid=102, session="s2", generation="g2"),
|
|
_row(
|
|
identity="w-live",
|
|
pid=103,
|
|
session="s3",
|
|
generation="g3",
|
|
heartbeat=NOW - timedelta(seconds=5),
|
|
),
|
|
_row(identity="w-nopid", pid=None, session="s4", generation="g4"),
|
|
_row(
|
|
identity="w-foreign",
|
|
pid=105,
|
|
session="s5",
|
|
generation="g5",
|
|
repository_binding="/other",
|
|
),
|
|
_row(
|
|
identity="w-terminal",
|
|
pid=106,
|
|
session="s6",
|
|
generation="g6",
|
|
status=mwi.STATUS_SUPERSEDED,
|
|
),
|
|
]
|
|
plan = _plan(rows, probe=_selective({103}))
|
|
self.assertEqual(
|
|
sorted(plan["candidate_worker_identities"]), ["w-stale-1", "w-stale-2"]
|
|
)
|
|
by_identity = {
|
|
p["worker_identity"]: p["reason_code"] for p in plan["preserved"]
|
|
}
|
|
self.assertEqual(by_identity["w-live"], retire.REASON_WORKER_LIVE)
|
|
self.assertEqual(by_identity["w-nopid"], retire.REASON_INCOMPLETE_IDENTITY)
|
|
self.assertEqual(by_identity["w-foreign"], retire.REASON_FOREIGN_REPOSITORY)
|
|
self.assertEqual(by_identity["w-terminal"], retire.REASON_ALREADY_TERMINAL)
|
|
self.assertEqual(plan["assessed_count"], 6)
|
|
self.assertEqual(plan["preserved_count"], 4)
|
|
|
|
def test_plan_is_deterministic_for_identical_contents(self):
|
|
rows = [_row(identity="w-1", pid=101), _row(identity="w-2", pid=102)]
|
|
first = _plan(rows)
|
|
second = _plan(list(reversed(rows)), now=NOW + timedelta(minutes=5))
|
|
self.assertEqual(first["registry_fingerprint"], second["registry_fingerprint"])
|
|
self.assertEqual(
|
|
first["candidate_fingerprint"], second["candidate_fingerprint"]
|
|
)
|
|
self.assertEqual(
|
|
sorted(first["candidate_worker_identities"]),
|
|
sorted(second["candidate_worker_identities"]),
|
|
)
|
|
|
|
|
|
class ApplyCasTests(unittest.TestCase):
|
|
"""AC: apply is compare-and-swap protected and revalidates every target."""
|
|
|
|
def setUp(self) -> None:
|
|
self.registry = _registry()
|
|
self.probe = _dead
|
|
|
|
def _rows(self):
|
|
return self.registry.list_workers(status=None)
|
|
|
|
def _apply(self, plan, *, probe=None, identities=None, **kwargs):
|
|
chosen = probe or self.probe
|
|
return self.registry.retire_stale_workers(
|
|
expected_registry_fingerprint=plan["registry_fingerprint"],
|
|
expected_candidate_fingerprint=plan["candidate_fingerprint"],
|
|
worker_identities=(
|
|
identities
|
|
if identities is not None
|
|
else plan["candidate_worker_identities"]
|
|
),
|
|
fingerprint_fn=retire.registry_fingerprint,
|
|
plan_fn=lambda rows: _plan(rows, probe=chosen, **kwargs),
|
|
retired_by="synthetic-user/prgs-reconciler",
|
|
now=NOW,
|
|
)
|
|
|
|
def test_matching_token_retires_the_planned_set(self):
|
|
_register(self.registry, _row(identity="w-1", pid=101, session="s1"))
|
|
_register(self.registry, _row(identity="w-2", pid=102, session="s2"))
|
|
plan = _plan(self._rows(), probe=self.probe)
|
|
result = self._apply(plan)
|
|
self.assertEqual(result["outcome"], "applied")
|
|
self.assertTrue(result["mutation_performed"])
|
|
self.assertEqual(result["retired_count"], 2)
|
|
statuses = {r["worker_identity"]: r["status"] for r in self._rows()}
|
|
self.assertEqual(
|
|
statuses, {"w-1": mwi.STATUS_RETIRED, "w-2": mwi.STATUS_RETIRED}
|
|
)
|
|
retired_row = self.registry.get("w-1")
|
|
self.assertEqual(retired_row["retired_at"], mwi._ts(NOW))
|
|
self.assertEqual(retired_row["retired_by"], "synthetic-user/prgs-reconciler")
|
|
|
|
def test_moved_registry_token_retires_zero_workers(self):
|
|
_register(self.registry, _row(identity="w-1", pid=101))
|
|
plan = _plan(self._rows(), probe=self.probe)
|
|
# An unrelated registration lands between plan and apply.
|
|
_register(self.registry, _row(identity="w-2", pid=102, session="s2"))
|
|
result = self._apply(plan)
|
|
self.assertEqual(result["outcome"], retire.OUTCOME_REGISTRY_MOVED)
|
|
self.assertEqual(result["retired_count"], 0)
|
|
self.assertFalse(result["mutation_performed"])
|
|
self.assertTrue(all(r["status"] == mwi.STATUS_ACTIVE for r in self._rows()))
|
|
|
|
def test_moved_candidate_set_retires_zero_workers(self):
|
|
"""Registry unchanged, but the eligibility verdict is no longer the same."""
|
|
_register(self.registry, _row(identity="w-1", pid=101))
|
|
plan = _plan(self._rows(), probe=self.probe)
|
|
# The row did not change; the process came back (pid probes alive), so
|
|
# revalidation inside the transaction finds no candidates at all.
|
|
result = self._apply(plan, probe=_alive)
|
|
self.assertEqual(result["outcome"], retire.OUTCOME_CANDIDATES_MOVED)
|
|
self.assertEqual(result["retired_count"], 0)
|
|
self.assertFalse(result["mutation_performed"])
|
|
self.assertEqual(self.registry.get("w-1")["status"], mwi.STATUS_ACTIVE)
|
|
|
|
def test_worker_that_becomes_live_between_plan_and_apply_is_preserved(self):
|
|
_register(self.registry, _row(identity="w-dead", pid=101, session="s1"))
|
|
_register(self.registry, _row(identity="w-back", pid=102, session="s2"))
|
|
plan = _plan(self._rows(), probe=self.probe)
|
|
self.assertEqual(len(plan["candidate_worker_identities"]), 2)
|
|
# w-back's process is alive by the time apply runs, so the candidate
|
|
# fingerprint moves and nothing at all is retired.
|
|
result = self._apply(plan, probe=_selective({102}))
|
|
self.assertEqual(result["outcome"], retire.OUTCOME_CANDIDATES_MOVED)
|
|
self.assertEqual(result["retired_count"], 0)
|
|
self.assertEqual(self.registry.get("w-back")["status"], mwi.STATUS_ACTIVE)
|
|
self.assertEqual(self.registry.get("w-dead")["status"], mwi.STATUS_ACTIVE)
|
|
|
|
def test_worker_that_becomes_ambiguous_between_plan_and_apply_is_preserved(self):
|
|
_register(self.registry, _row(identity="w-1", pid=101, session="s1"))
|
|
plan = _plan(self._rows(), probe=self.probe)
|
|
result = self._apply(plan, probe=lambda _pid: None)
|
|
self.assertEqual(result["outcome"], retire.OUTCOME_CANDIDATES_MOVED)
|
|
self.assertEqual(result["retired_count"], 0)
|
|
self.assertEqual(self.registry.get("w-1")["status"], mwi.STATUS_ACTIVE)
|
|
|
|
def test_apply_revalidates_and_will_not_narrow_the_approved_set(self):
|
|
"""A protection appearing after plan moves the CAS, so nothing is retired."""
|
|
_register(self.registry, _row(identity="w-1", pid=101, session="s1"))
|
|
_register(
|
|
self.registry, _row(identity="w-protected", pid=102, session="sess-leased")
|
|
)
|
|
unprotected_plan = _plan(self._rows(), probe=self.probe)
|
|
self.assertEqual(unprotected_plan["candidate_count"], 2)
|
|
result = self._apply(
|
|
unprotected_plan, protected_session_ids=["sess-leased"]
|
|
)
|
|
self.assertEqual(result["outcome"], retire.OUTCOME_CANDIDATES_MOVED)
|
|
self.assertEqual(result["retired_count"], 0)
|
|
self.assertEqual(
|
|
self.registry.get("w-protected")["status"], mwi.STATUS_ACTIVE
|
|
)
|
|
|
|
def test_target_outside_the_current_plan_is_preserved(self):
|
|
_register(
|
|
self.registry,
|
|
_row(identity="w-1", pid=101, session="s1", generation="g1"),
|
|
)
|
|
_register(
|
|
self.registry,
|
|
_row(
|
|
identity="w-live",
|
|
pid=102,
|
|
session="s2",
|
|
generation="g2",
|
|
heartbeat=NOW - timedelta(seconds=5),
|
|
),
|
|
)
|
|
probe = _selective({102})
|
|
plan = _plan(self._rows(), probe=probe)
|
|
self.assertEqual(plan["candidate_worker_identities"], ["w-1"])
|
|
# A caller that appends a live identity to the approved list gets it
|
|
# preserved with the live reason code; only the real candidate retires.
|
|
result = self._apply(plan, probe=probe, identities=["w-1", "w-live"])
|
|
self.assertEqual(result["outcome"], "applied")
|
|
self.assertEqual(result["retired_count"], 1)
|
|
self.assertEqual(result["retired"][0]["worker_identity"], "w-1")
|
|
preserved = {
|
|
p["worker_identity"]: p["reason_code"] for p in result["preserved"]
|
|
}
|
|
self.assertEqual(preserved["w-live"], retire.REASON_WORKER_LIVE)
|
|
self.assertEqual(self.registry.get("w-live")["status"], mwi.STATUS_ACTIVE)
|
|
|
|
def test_missing_registration_is_reported_not_invented(self):
|
|
_register(self.registry, _row(identity="w-1", pid=101))
|
|
plan = _plan(self._rows(), probe=self.probe)
|
|
result = self._apply(plan, identities=["w-1", "w-ghost"])
|
|
self.assertEqual(result["retired_count"], 1)
|
|
preserved = {
|
|
p["worker_identity"]: p["reason_code"] for p in result["preserved"]
|
|
}
|
|
self.assertEqual(preserved["w-ghost"], "registration_missing")
|
|
|
|
def test_reapplying_a_completed_plan_is_a_safe_no_op(self):
|
|
_register(self.registry, _row(identity="w-1", pid=101))
|
|
plan = _plan(self._rows(), probe=self.probe)
|
|
first = self._apply(plan)
|
|
self.assertEqual(first["retired_count"], 1)
|
|
second = self._apply(plan)
|
|
self.assertEqual(second["outcome"], retire.OUTCOME_ALREADY_RETIRED)
|
|
self.assertTrue(second["idempotent"])
|
|
self.assertEqual(second["retired_count"], 0)
|
|
self.assertFalse(second["mutation_performed"])
|
|
self.assertEqual(self.registry.get("w-1")["status"], mwi.STATUS_RETIRED)
|
|
|
|
def test_empty_target_list_reports_nothing_requested(self):
|
|
_register(self.registry, _row(identity="w-1", pid=101))
|
|
plan = _plan(self._rows(), probe=self.probe)
|
|
result = self._apply(plan, identities=[])
|
|
self.assertEqual(result["outcome"], retire.OUTCOME_NOTHING_REQUESTED)
|
|
self.assertFalse(result["mutation_performed"])
|
|
self.assertEqual(self.registry.get("w-1")["status"], mwi.STATUS_ACTIVE)
|
|
|
|
def test_transaction_failure_cannot_report_partial_success(self):
|
|
_register(self.registry, _row(identity="w-1", pid=101, session="s1"))
|
|
_register(self.registry, _row(identity="w-2", pid=102, session="s2"))
|
|
plan = _plan(self._rows(), probe=self.probe)
|
|
calls = {"n": 0}
|
|
|
|
def exploding_plan(rows):
|
|
calls["n"] += 1
|
|
raise RuntimeError("synthetic revalidation failure")
|
|
|
|
result = self.registry.retire_stale_workers(
|
|
expected_registry_fingerprint=plan["registry_fingerprint"],
|
|
expected_candidate_fingerprint=plan["candidate_fingerprint"],
|
|
worker_identities=plan["candidate_worker_identities"],
|
|
fingerprint_fn=retire.registry_fingerprint,
|
|
plan_fn=exploding_plan,
|
|
retired_by="synthetic-user/prgs-reconciler",
|
|
now=NOW,
|
|
)
|
|
self.assertEqual(calls["n"], 1)
|
|
self.assertFalse(result["success"])
|
|
self.assertEqual(result["outcome"], "transaction_failed")
|
|
self.assertEqual(result["retired_count"], 0)
|
|
self.assertFalse(result["mutation_performed"])
|
|
self.assertTrue(
|
|
all(r["status"] == mwi.STATUS_ACTIVE for r in self._rows()),
|
|
"a failed transaction must roll back every retirement",
|
|
)
|
|
|
|
def test_structured_result_fields_are_accurate(self):
|
|
_register(self.registry, _row(identity="w-1", pid=101))
|
|
plan = _plan(self._rows(), probe=self.probe)
|
|
blocked = self._apply(plan, probe=_alive)
|
|
self.assertFalse(blocked["mutation_performed"])
|
|
self.assertEqual(blocked["retired"], [])
|
|
self.assertEqual(blocked["requested_count"], 1)
|
|
self.assertEqual(
|
|
blocked["expected_registry_fingerprint"], plan["registry_fingerprint"]
|
|
)
|
|
applied = self._apply(plan)
|
|
self.assertTrue(applied["mutation_performed"])
|
|
self.assertEqual(applied["acting_identity"], "synthetic-user/prgs-reconciler")
|
|
self.assertEqual(applied["retired"][0]["reason_code"], retire.REASON_ELIGIBLE)
|
|
self.assertIn("evidence", applied["retired"][0])
|
|
|
|
|
|
class PostRetirementFleetCompatibilityTests(unittest.TestCase):
|
|
"""AC: retired rows leave the stale count and never fake fleet safety."""
|
|
|
|
def test_retired_rows_are_historical_not_stale(self):
|
|
registry = _registry()
|
|
_register(
|
|
registry, _row(identity="w-1", pid=101, session="s1", generation="g1")
|
|
)
|
|
_register(
|
|
registry,
|
|
_row(
|
|
identity="w-live",
|
|
pid=102,
|
|
session="s2",
|
|
generation="g2",
|
|
instance="legacy-pid-27833",
|
|
instance_id_provenance=fleet.INSTANCE_ID_PROVENANCE_LEGACY,
|
|
heartbeat=NOW - timedelta(seconds=5),
|
|
),
|
|
)
|
|
probe = _selective({102})
|
|
before = fleet.snapshot_instance_fleet(
|
|
registry.list_workers(status=None),
|
|
now=NOW,
|
|
pid_alive_probe=probe,
|
|
canonical_repository=REPO,
|
|
)
|
|
self.assertEqual(before["stale_worker_count"], 1)
|
|
|
|
plan = _plan(registry.list_workers(status=None), probe=probe)
|
|
result = registry.retire_stale_workers(
|
|
expected_registry_fingerprint=plan["registry_fingerprint"],
|
|
expected_candidate_fingerprint=plan["candidate_fingerprint"],
|
|
worker_identities=plan["candidate_worker_identities"],
|
|
fingerprint_fn=retire.registry_fingerprint,
|
|
plan_fn=lambda rows: _plan(rows, probe=probe),
|
|
retired_by="synthetic-user/prgs-reconciler",
|
|
now=NOW,
|
|
)
|
|
self.assertEqual(result["retired_count"], 1)
|
|
|
|
after = fleet.snapshot_instance_fleet(
|
|
registry.list_workers(status=None),
|
|
now=NOW,
|
|
pid_alive_probe=probe,
|
|
canonical_repository=REPO,
|
|
)
|
|
self.assertEqual(after["stale_worker_count"], 0)
|
|
self.assertEqual(after["live_worker_count"], 1)
|
|
self.assertEqual(after["historical_worker_count"], 1)
|
|
# The surviving live worker still carries a legacy instance identity, so
|
|
# the fleet must not be declared safe.
|
|
self.assertFalse(after["live_fleet_safe"])
|
|
self.assertIn(
|
|
fleet.CLASS_LEGACY_INCOMPLETE,
|
|
{f["classification"] for f in after["active_blockers"]},
|
|
)
|
|
|
|
def test_existing_snapshot_shape_is_unchanged(self):
|
|
rows = [_row(identity="w-1", pid=101)]
|
|
snapshot = fleet.snapshot_instance_fleet(
|
|
rows, now=NOW, pid_alive_probe=_dead, canonical_repository=REPO
|
|
)
|
|
for key in (
|
|
"consistency_token",
|
|
"registry_revision",
|
|
"snapshot_at",
|
|
"live_workers",
|
|
"stale_workers",
|
|
"historical_workers",
|
|
"findings",
|
|
"live_fleet_safe",
|
|
):
|
|
self.assertIn(key, snapshot)
|
|
self.assertTrue(snapshot["registry_revision"].startswith("fleetrev-"))
|
|
self.assertTrue(snapshot["read_only"])
|
|
self.assertFalse(snapshot.get("mutation_performed", False))
|
|
|
|
|
|
class CapabilityExposureTests(unittest.TestCase):
|
|
"""AC: only controller/reconciler reach the surface; author policy unchanged."""
|
|
|
|
def test_capability_map_declares_controller_role(self):
|
|
import task_capability_map as tcm
|
|
|
|
for task in (
|
|
"plan_stale_worker_retirement",
|
|
"gitea_plan_stale_worker_retirement",
|
|
"apply_stale_worker_retirement",
|
|
"gitea_apply_stale_worker_retirement",
|
|
):
|
|
with self.subTest(task=task):
|
|
self.assertEqual(tcm.required_permission(task), "gitea.read")
|
|
self.assertEqual(tcm.required_role(task), "controller")
|
|
|
|
def test_ordinary_author_capability_resolution_is_unaffected(self):
|
|
import task_capability_map as tcm
|
|
|
|
self.assertEqual(tcm.required_permission("work_issue"), "gitea.pr.create")
|
|
self.assertEqual(tcm.required_role("work_issue"), "author")
|
|
self.assertEqual(tcm.required_permission("create_pr"), "gitea.pr.create")
|
|
self.assertEqual(tcm.required_role("create_pr"), "author")
|
|
self.assertEqual(tcm.required_permission("commit_files"), "gitea.repo.commit")
|
|
self.assertEqual(tcm.required_permission("push_branch"), "gitea.branch.push")
|
|
self.assertEqual(
|
|
tcm.required_permission("snapshot_instance_fleet"), "gitea.read"
|
|
)
|
|
|
|
def test_no_new_gitea_write_permission_is_introduced(self):
|
|
import task_capability_map as tcm
|
|
|
|
for task in ("plan_stale_worker_retirement", "apply_stale_worker_retirement"):
|
|
with self.subTest(task=task):
|
|
self.assertNotIn(
|
|
tcm.required_permission(task),
|
|
{
|
|
"gitea.pr.approve",
|
|
"gitea.pr.merge",
|
|
"gitea.pr.review",
|
|
"gitea.branch.push",
|
|
"gitea.repo.commit",
|
|
},
|
|
)
|
|
|
|
|
|
class SurfaceRegistrationTests(unittest.TestCase):
|
|
"""AC: the dry-run and apply modes are exposed as sanctioned native tools."""
|
|
|
|
def test_tools_are_registered_and_role_restricted(self):
|
|
import inspect
|
|
|
|
import gitea_mcp_server as server
|
|
|
|
for name in (
|
|
"gitea_plan_stale_worker_retirement",
|
|
"gitea_apply_stale_worker_retirement",
|
|
):
|
|
with self.subTest(tool=name):
|
|
tool = getattr(server, name)
|
|
self.assertTrue(callable(tool))
|
|
source = inspect.getsource(tool)
|
|
# Both tools route their role check through the shared guard.
|
|
self.assertIn(f'_retirement_role_block("{name}")', source)
|
|
|
|
guard = inspect.getsource(server._retirement_role_block)
|
|
self.assertIn('{"controller", "reconciler"}', guard)
|
|
self.assertIn("required_roles", guard)
|
|
|
|
apply_source = inspect.getsource(server.gitea_apply_stale_worker_retirement)
|
|
self.assertIn("registry_fingerprint", apply_source)
|
|
self.assertIn("candidate_fingerprint", apply_source)
|
|
self.assertIn("_retirement_runtime_block", apply_source)
|
|
self.assertIn("classify_cohort", apply_source)
|
|
# Revalidation inside the transaction re-reads lease protection rather
|
|
# than reusing the pre-transaction snapshot.
|
|
self.assertIn("_retirement_revalidation_plan", apply_source)
|
|
revalidation = inspect.getsource(server._retirement_revalidation_plan)
|
|
self.assertIn("_retirement_protected_owners", revalidation)
|
|
self.assertIn("raise RuntimeError", revalidation)
|
|
|
|
def test_plan_tool_declares_itself_read_only(self):
|
|
import inspect
|
|
|
|
import gitea_mcp_server as server
|
|
|
|
source = inspect.getsource(server.gitea_plan_stale_worker_retirement)
|
|
self.assertIn("read_only", source)
|
|
self.assertNotIn("retire_stale_workers", source)
|
|
|
|
def test_docs_exist(self):
|
|
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
path = os.path.join(root, "docs", "stale-worker-retirement.md")
|
|
self.assertTrue(os.path.isfile(path))
|
|
with open(path, encoding="utf-8") as handle:
|
|
text = handle.read()
|
|
for token in (
|
|
"registry_fingerprint",
|
|
"candidate_fingerprint",
|
|
"gitea_plan_stale_worker_retirement",
|
|
"gitea_apply_stale_worker_retirement",
|
|
"eligible_stale_orphan",
|
|
"registry_revision_moved",
|
|
"does not repair untrusted live identity",
|
|
):
|
|
with self.subTest(token=token):
|
|
self.assertIn(token, text)
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover
|
|
unittest.main()
|