sysadminandClaude Opus 4.8
4a1cc63e94
Add a pure fleet snapshot assessor and a read-only controller/reconciler MCP tool so multi-instance fleets can be enumerated by client_instance_id without treating shared client_type as duplication. Trusted launcher instance IDs, classification (missing/unmanifested/collisions/historical), registry schema extensions, docs, and regression tests for #978. Closes #978 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1055 lines
36 KiB
Python
1055 lines
36 KiB
Python
"""Instance-level fleet identity and health snapshots (#978).
|
||
|
||
Covers the acceptance criteria for multi-instance fleets: two Codex launches
|
||
remain distinct, ten workers attribute correctly, same client_type is not a
|
||
duplicate, collisions fail closed, historical rows stay non-blocking, and the
|
||
controller/reconciler read surface grants no mutation authority.
|
||
|
||
All identifiers are synthetic.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import inspect
|
||
import json
|
||
import os
|
||
import tempfile
|
||
import unittest
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Any
|
||
from unittest import mock
|
||
|
||
import mcp_fleet_snapshot as fleet
|
||
import mcp_worker_identity as mwi
|
||
|
||
|
||
NOW = datetime(2026, 7, 30, 7, 0, 0, tzinfo=timezone.utc)
|
||
TTL = 900.0
|
||
NAMESPACES = ("author", "reviewer", "merger", "controller", "reconciler")
|
||
|
||
|
||
def _registry() -> mwi.WorkerRegistry:
|
||
handle, path = tempfile.mkstemp(suffix=".sqlite3")
|
||
os.close(handle)
|
||
os.unlink(path)
|
||
return mwi.WorkerRegistry(path)
|
||
|
||
|
||
def _alive(_pid: int | None) -> bool:
|
||
return True
|
||
|
||
|
||
def _dead(_pid: int | None) -> bool:
|
||
return False
|
||
|
||
|
||
def _register(
|
||
registry: mwi.WorkerRegistry,
|
||
*,
|
||
client: str,
|
||
instance: str,
|
||
session: str,
|
||
generation: str,
|
||
namespace: str,
|
||
pid: int,
|
||
now: datetime = NOW,
|
||
ttl: float = TTL,
|
||
fleet_run_id: str | None = "run-canary",
|
||
status_after: str | None = None,
|
||
worker_identity: str | None = None,
|
||
process_identity: str | None = None,
|
||
repository_binding: str = "/repo/Gitea-Tools",
|
||
startup_revision: str = "rev-live",
|
||
instance_id_provenance: str = fleet.INSTANCE_ID_PROVENANCE_TRUSTED,
|
||
role: str | None = None,
|
||
) -> dict[str, Any]:
|
||
identity = worker_identity or mwi.generate_worker_identity(
|
||
client, f"{session}-{namespace}", now=now, nonce=f"{pid}-{namespace}"
|
||
)
|
||
outcome = registry.register(
|
||
worker_identity=identity,
|
||
client_name=client,
|
||
client_instance_id=instance,
|
||
session_id=session,
|
||
generation_id=generation,
|
||
role=role or namespace,
|
||
profile=f"prgs-{namespace}",
|
||
namespace=namespace,
|
||
remote="prgs",
|
||
repository_binding=repository_binding,
|
||
pid=pid,
|
||
heartbeat_ttl_seconds=ttl,
|
||
now=now,
|
||
fleet_run_id=fleet_run_id,
|
||
process_identity=process_identity or f"pid-{pid}",
|
||
startup_revision=startup_revision,
|
||
loaded_revision=startup_revision,
|
||
parity_revision=startup_revision,
|
||
live_revision=startup_revision,
|
||
instance_id_provenance=instance_id_provenance,
|
||
pid_alive_probe=_alive,
|
||
)
|
||
outcome["identity"] = identity
|
||
if status_after and outcome.get("registered"):
|
||
if status_after == mwi.STATUS_RELEASED:
|
||
registry.release(worker_identity=identity, now=now)
|
||
elif status_after == mwi.STATUS_SUPERSEDED:
|
||
# Direct status write for historical rows.
|
||
with registry._tx() as conn: # noqa: SLF001 — test-only fixture
|
||
conn.execute(
|
||
"UPDATE worker_registrations SET status = ? WHERE worker_identity = ?",
|
||
(mwi.STATUS_SUPERSEDED, identity),
|
||
)
|
||
return outcome
|
||
|
||
|
||
def _five_workers(
|
||
registry: mwi.WorkerRegistry,
|
||
*,
|
||
client: str,
|
||
instance: str,
|
||
session_prefix: str,
|
||
gen_prefix: str,
|
||
pid_base: int,
|
||
now: datetime = NOW,
|
||
) -> list[dict[str, Any]]:
|
||
out = []
|
||
for i, ns in enumerate(NAMESPACES):
|
||
out.append(
|
||
_register(
|
||
registry,
|
||
client=client,
|
||
instance=instance,
|
||
session=f"{session_prefix}-{ns}",
|
||
generation=f"{gen_prefix}-{ns}",
|
||
namespace=ns,
|
||
pid=pid_base + i,
|
||
now=now,
|
||
)
|
||
)
|
||
return out
|
||
|
||
|
||
class InstanceIdentityTests(unittest.TestCase):
|
||
def test_generate_distinct_instance_ids(self):
|
||
a = fleet.generate_client_instance_id("codex", launch_nonce="a", now=NOW)
|
||
b = fleet.generate_client_instance_id("codex", launch_nonce="b", now=NOW)
|
||
self.assertNotEqual(a, b)
|
||
self.assertTrue(a.startswith("inst-codex-"))
|
||
self.assertTrue(fleet.assess_instance_identity(a)["trusted"])
|
||
|
||
def test_legacy_pid_fallback_not_trusted(self):
|
||
a = fleet.assess_instance_identity("pid-1234")
|
||
self.assertFalse(a["trusted"])
|
||
self.assertEqual(a["provenance"], fleet.INSTANCE_ID_PROVENANCE_LEGACY)
|
||
|
||
def test_missing_instance_id(self):
|
||
a = fleet.assess_instance_identity(None)
|
||
self.assertFalse(a["complete"])
|
||
self.assertEqual(a["provenance"], fleet.INSTANCE_ID_PROVENANCE_MISSING)
|
||
|
||
|
||
class TwoCodexInstancesTests(unittest.TestCase):
|
||
"""AC1–AC3: two Codex instances, ten workers, same type not duplicate."""
|
||
|
||
def test_two_codex_instances_ten_workers(self):
|
||
registry = _registry()
|
||
inst_a = fleet.generate_client_instance_id("codex", launch_nonce="A", now=NOW)
|
||
inst_b = fleet.generate_client_instance_id("codex", launch_nonce="B", now=NOW)
|
||
_five_workers(
|
||
registry,
|
||
client="codex",
|
||
instance=inst_a,
|
||
session_prefix="sessA",
|
||
gen_prefix="genA",
|
||
pid_base=1000,
|
||
)
|
||
_five_workers(
|
||
registry,
|
||
client="codex",
|
||
instance=inst_b,
|
||
session_prefix="sessB",
|
||
gen_prefix="genB",
|
||
pid_base=2000,
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(status=None),
|
||
now=NOW + timedelta(seconds=10),
|
||
pid_alive_probe=_alive,
|
||
canonical_repository="/repo/Gitea-Tools",
|
||
expected_live_revision="rev-live",
|
||
)
|
||
self.assertEqual(snap["live_worker_count"], 10)
|
||
self.assertEqual(snap["instance_count"], 2)
|
||
self.assertIn("codex", snap["multi_instance_same_client_type"])
|
||
self.assertEqual(len(snap["multi_instance_same_client_type"]["codex"]), 2)
|
||
self.assertTrue(snap["same_client_type_not_duplicate"])
|
||
# No finding that treats same client_type alone as a duplicate.
|
||
for f in snap["active_blockers"]:
|
||
self.assertNotIn("client_type alone", f["detail"].lower())
|
||
by_inst = {i["client_instance_id"]: i for i in snap["instances"]}
|
||
self.assertEqual(by_inst[inst_a]["worker_count"], 5)
|
||
self.assertEqual(by_inst[inst_b]["worker_count"], 5)
|
||
self.assertEqual(set(by_inst[inst_a]["namespaces"]), set(NAMESPACES))
|
||
self.assertEqual(set(by_inst[inst_b]["namespaces"]), set(NAMESPACES))
|
||
self.assertTrue(snap["live_fleet_safe"])
|
||
|
||
def test_mixed_client_types(self):
|
||
registry = _registry()
|
||
_five_workers(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-codex-1",
|
||
session_prefix="c",
|
||
gen_prefix="gc",
|
||
pid_base=10,
|
||
)
|
||
_five_workers(
|
||
registry,
|
||
client="claude_code",
|
||
instance="inst-claude-1",
|
||
session_prefix="l",
|
||
gen_prefix="gl",
|
||
pid_base=20,
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=5),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
self.assertEqual(snap["instance_count"], 2)
|
||
self.assertTrue(snap["live_fleet_safe"])
|
||
|
||
|
||
class CollisionTests(unittest.TestCase):
|
||
def test_live_reuse_of_instance_id_different_client_types(self):
|
||
registry = _registry()
|
||
shared = "inst-shared-collision"
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance=shared,
|
||
session="s1",
|
||
generation="g1",
|
||
namespace="author",
|
||
pid=1,
|
||
)
|
||
# Second client type forced onto same instance id via direct SQL to
|
||
# bypass register's same-namespace guard and exercise classification.
|
||
with registry._tx() as conn: # noqa: SLF001
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO worker_registrations (
|
||
worker_identity, client_name, client_instance_id, session_id,
|
||
generation_id, role, profile, namespace, remote,
|
||
repository_binding, pid, transport, token_fingerprint,
|
||
started_at, last_heartbeat_at, heartbeat_ttl_seconds,
|
||
fencing_epoch, status, process_identity, instance_id_provenance
|
||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||
""",
|
||
(
|
||
"claude_code-20260730T070000Z-aaaaaaaaaaaa",
|
||
"claude_code",
|
||
shared,
|
||
"s2",
|
||
"g2",
|
||
"author",
|
||
"prgs-author",
|
||
"reviewer",
|
||
"prgs",
|
||
"/repo/Gitea-Tools",
|
||
2,
|
||
None,
|
||
None,
|
||
mwi._ts(NOW), # noqa: SLF001
|
||
mwi._ts(NOW), # noqa: SLF001
|
||
TTL,
|
||
1,
|
||
mwi.STATUS_ACTIVE,
|
||
"pid-2",
|
||
fleet.INSTANCE_ID_PROVENANCE_TRUSTED,
|
||
),
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertIn(fleet.CLASS_INSTANCE_ID_COLLISION, classes)
|
||
self.assertFalse(snap["live_fleet_safe"])
|
||
|
||
def test_duplicate_namespace_within_instance(self):
|
||
registry = _registry()
|
||
inst = "inst-dup-ns"
|
||
first = _register(
|
||
registry,
|
||
client="codex",
|
||
instance=inst,
|
||
session="s-a",
|
||
generation="g-a",
|
||
namespace="author",
|
||
pid=11,
|
||
)
|
||
self.assertTrue(first["registered"], first)
|
||
# Direct second author under same instance (bypass register guard).
|
||
with registry._tx() as conn: # noqa: SLF001
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO worker_registrations (
|
||
worker_identity, client_name, client_instance_id, session_id,
|
||
generation_id, role, profile, namespace, remote,
|
||
repository_binding, pid, transport, token_fingerprint,
|
||
started_at, last_heartbeat_at, heartbeat_ttl_seconds,
|
||
fencing_epoch, status, process_identity, instance_id_provenance
|
||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||
""",
|
||
(
|
||
"codex-20260730T070000Z-bbbbbbbbbbbb",
|
||
"codex",
|
||
inst,
|
||
"s-b",
|
||
"g-b",
|
||
"author",
|
||
"prgs-author",
|
||
"author",
|
||
"prgs",
|
||
"/repo/Gitea-Tools",
|
||
12,
|
||
None,
|
||
None,
|
||
mwi._ts(NOW), # noqa: SLF001
|
||
mwi._ts(NOW), # noqa: SLF001
|
||
TTL,
|
||
2,
|
||
mwi.STATUS_ACTIVE,
|
||
"pid-12",
|
||
fleet.INSTANCE_ID_PROVENANCE_TRUSTED,
|
||
),
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertIn(fleet.CLASS_DUPLICATE_NAMESPACE, classes)
|
||
|
||
def test_register_refuses_duplicate_namespace(self):
|
||
registry = _registry()
|
||
inst = "inst-reg-dup"
|
||
a = _register(
|
||
registry,
|
||
client="codex",
|
||
instance=inst,
|
||
session="s1",
|
||
generation="g1",
|
||
namespace="author",
|
||
pid=21,
|
||
)
|
||
self.assertTrue(a["registered"])
|
||
b = _register(
|
||
registry,
|
||
client="codex",
|
||
instance=inst,
|
||
session="s2",
|
||
generation="g2",
|
||
namespace="author",
|
||
pid=22,
|
||
)
|
||
self.assertFalse(b["registered"])
|
||
self.assertEqual(b.get("collision_kind"), "duplicate_namespace_worker")
|
||
|
||
def test_reused_worker_identity(self):
|
||
registry = _registry()
|
||
identity = mwi.generate_worker_identity("codex", "sess", now=NOW, nonce="fixed")
|
||
first = _register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-1",
|
||
session="s1",
|
||
generation="g1",
|
||
namespace="author",
|
||
pid=31,
|
||
worker_identity=identity,
|
||
)
|
||
self.assertTrue(first["registered"])
|
||
second = registry.register(
|
||
worker_identity=identity,
|
||
client_name="codex",
|
||
client_instance_id="inst-2",
|
||
session_id="s2",
|
||
generation_id="g2",
|
||
namespace="author",
|
||
pid=32,
|
||
now=NOW,
|
||
pid_alive_probe=_alive,
|
||
)
|
||
self.assertFalse(second["registered"])
|
||
self.assertTrue(second["collision"])
|
||
|
||
def test_reused_session_identity(self):
|
||
registry = _registry()
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-1",
|
||
session="shared-session",
|
||
generation="g1",
|
||
namespace="author",
|
||
pid=41,
|
||
)
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-2",
|
||
session="shared-session",
|
||
generation="g2",
|
||
namespace="reviewer",
|
||
pid=42,
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertIn(fleet.CLASS_SESSION_COLLISION, classes)
|
||
|
||
def test_reused_generation_across_instances(self):
|
||
registry = _registry()
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-1",
|
||
session="s1",
|
||
generation="gen-shared",
|
||
namespace="author",
|
||
pid=51,
|
||
)
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-2",
|
||
session="s2",
|
||
generation="gen-shared",
|
||
namespace="author",
|
||
pid=52,
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertIn(fleet.CLASS_GENERATION_COLLISION, classes)
|
||
|
||
def test_reused_pid_and_process_identity(self):
|
||
registry = _registry()
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-1",
|
||
session="s1",
|
||
generation="g1",
|
||
namespace="author",
|
||
pid=61,
|
||
process_identity="proc-same",
|
||
)
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-2",
|
||
session="s2",
|
||
generation="g2",
|
||
namespace="reviewer",
|
||
pid=61,
|
||
process_identity="proc-same",
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertIn(fleet.CLASS_PID_COLLISION, classes)
|
||
self.assertIn(fleet.CLASS_PROCESS_COLLISION, classes)
|
||
|
||
def test_ownership_fencing_collision(self):
|
||
registry = _registry()
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-1",
|
||
session="s1",
|
||
generation="gen-fence",
|
||
namespace="author",
|
||
pid=71,
|
||
)
|
||
# Second instance forced onto same generation+epoch via SQL.
|
||
with registry._tx() as conn: # noqa: SLF001
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO worker_registrations (
|
||
worker_identity, client_name, client_instance_id, session_id,
|
||
generation_id, role, profile, namespace, remote,
|
||
repository_binding, pid, transport, token_fingerprint,
|
||
started_at, last_heartbeat_at, heartbeat_ttl_seconds,
|
||
fencing_epoch, status, process_identity, instance_id_provenance
|
||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||
""",
|
||
(
|
||
"codex-20260730T070000Z-cccccccccccc",
|
||
"codex",
|
||
"inst-2",
|
||
"s2",
|
||
"gen-fence",
|
||
"reviewer",
|
||
"prgs-reviewer",
|
||
"reviewer",
|
||
"prgs",
|
||
"/repo/Gitea-Tools",
|
||
72,
|
||
None,
|
||
None,
|
||
mwi._ts(NOW), # noqa: SLF001
|
||
mwi._ts(NOW), # noqa: SLF001
|
||
TTL,
|
||
1, # same epoch as first (first got epoch 1)
|
||
mwi.STATUS_ACTIVE,
|
||
"pid-72",
|
||
fleet.INSTANCE_ID_PROVENANCE_TRUSTED,
|
||
),
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertTrue(
|
||
{fleet.CLASS_OWNERSHIP_COLLISION, fleet.CLASS_GENERATION_COLLISION}
|
||
& classes
|
||
)
|
||
|
||
|
||
class ManifestAndEdgeTests(unittest.TestCase):
|
||
def test_missing_and_unmanifested(self):
|
||
registry = _registry()
|
||
live = "inst-live"
|
||
expected = "inst-expected-missing"
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance=live,
|
||
session="s",
|
||
generation="g",
|
||
namespace="author",
|
||
pid=81,
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
expected_manifest=[
|
||
{"client_instance_id": expected, "client_type": "codex"},
|
||
],
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
self.assertEqual(snap["missing_expected_instance_ids"], [expected])
|
||
self.assertEqual(snap["unmanifested_instance_ids"], [live])
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertIn(fleet.CLASS_MISSING, classes)
|
||
self.assertIn(fleet.CLASS_UNMANIFESTED, classes)
|
||
|
||
def test_unknown_client(self):
|
||
registry = _registry()
|
||
with registry._tx() as conn: # noqa: SLF001
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO worker_registrations (
|
||
worker_identity, client_name, client_instance_id, session_id,
|
||
generation_id, role, profile, namespace, remote,
|
||
repository_binding, pid, transport, token_fingerprint,
|
||
started_at, last_heartbeat_at, heartbeat_ttl_seconds,
|
||
fencing_epoch, status, process_identity, instance_id_provenance
|
||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||
""",
|
||
(
|
||
"unknown_client-20260730T070000Z-dddddddddddd",
|
||
"unknown_client",
|
||
"inst-unk",
|
||
"s",
|
||
"g",
|
||
"author",
|
||
"prgs-author",
|
||
"author",
|
||
"prgs",
|
||
"/repo/Gitea-Tools",
|
||
91,
|
||
None,
|
||
None,
|
||
mwi._ts(NOW), # noqa: SLF001
|
||
mwi._ts(NOW), # noqa: SLF001
|
||
TTL,
|
||
1,
|
||
mwi.STATUS_ACTIVE,
|
||
"pid-91",
|
||
fleet.INSTANCE_ID_PROVENANCE_TRUSTED,
|
||
),
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertIn(fleet.CLASS_UNKNOWN_CLIENT, classes)
|
||
|
||
def test_foreign_repository(self):
|
||
registry = _registry()
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-fr",
|
||
session="s",
|
||
generation="g",
|
||
namespace="author",
|
||
pid=101,
|
||
repository_binding="/other/repo",
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
canonical_repository="/repo/Gitea-Tools",
|
||
)
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertIn(fleet.CLASS_FOREIGN_REPOSITORY, classes)
|
||
|
||
def test_old_revision(self):
|
||
registry = _registry()
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-old",
|
||
session="s",
|
||
generation="g",
|
||
namespace="author",
|
||
pid=111,
|
||
startup_revision="rev-old",
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
expected_live_revision="rev-live",
|
||
)
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertIn(fleet.CLASS_OLD_REVISION, classes)
|
||
|
||
def test_stale_worker(self):
|
||
registry = _registry()
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-stale",
|
||
session="s",
|
||
generation="g",
|
||
namespace="author",
|
||
pid=121,
|
||
now=NOW,
|
||
ttl=60.0,
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=120),
|
||
pid_alive_probe=_dead,
|
||
)
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertIn(fleet.CLASS_STALE_WORKER, classes)
|
||
|
||
def test_historical_not_active_blocker(self):
|
||
registry = _registry()
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-hist",
|
||
session="s",
|
||
generation="g",
|
||
namespace="author",
|
||
pid=131,
|
||
status_after=mwi.STATUS_RELEASED,
|
||
)
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-live",
|
||
session="s2",
|
||
generation="g2",
|
||
namespace="author",
|
||
pid=132,
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(status=None),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
self.assertEqual(snap["historical_worker_count"], 1)
|
||
self.assertTrue(
|
||
all(f["classification"] == fleet.CLASS_HISTORICAL for f in snap["historical_findings"])
|
||
)
|
||
# Only historical finding for the dead row — live fleet otherwise safe.
|
||
hist_blockers = [
|
||
f
|
||
for f in snap["active_blockers"]
|
||
if f["classification"] == fleet.CLASS_HISTORICAL
|
||
]
|
||
self.assertEqual(hist_blockers, [])
|
||
self.assertTrue(snap["live_fleet_safe"])
|
||
|
||
def test_legacy_incomplete_identity(self):
|
||
registry = _registry()
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="pid-999",
|
||
session="s",
|
||
generation="g",
|
||
namespace="author",
|
||
pid=141,
|
||
instance_id_provenance=fleet.INSTANCE_ID_PROVENANCE_LEGACY,
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
classes = {f["classification"] for f in snap["active_blockers"]}
|
||
self.assertIn(fleet.CLASS_LEGACY_INCOMPLETE, classes)
|
||
self.assertFalse(snap["mutation_safe"])
|
||
|
||
|
||
class LifecycleTests(unittest.TestCase):
|
||
def test_heartbeat_continuity_across_snapshots(self):
|
||
registry = _registry()
|
||
outcome = _register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-hb",
|
||
session="s",
|
||
generation="g",
|
||
namespace="author",
|
||
pid=151,
|
||
)
|
||
identity = outcome["identity"]
|
||
earlier = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
later_time = NOW + timedelta(seconds=30)
|
||
registry.heartbeat(
|
||
worker_identity=identity,
|
||
fencing_epoch=outcome["fencing_epoch"],
|
||
now=later_time,
|
||
expected_session_id="s",
|
||
expected_generation_id="g",
|
||
expected_client_name="codex",
|
||
expected_pid=151,
|
||
)
|
||
later = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=later_time + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
cmp = fleet.compare_snapshot_heartbeats(earlier, later)
|
||
self.assertTrue(cmp["stable_ownership"])
|
||
self.assertEqual(cmp["shared_live_workers"], [identity])
|
||
self.assertTrue(cmp["continuity"][0]["heartbeat_non_decreasing"])
|
||
|
||
def test_worker_reconnect_preserves_instance(self):
|
||
"""Reconnect = heartbeat renew; same instance/worker/session."""
|
||
registry = _registry()
|
||
o = _register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-rc",
|
||
session="s-rc",
|
||
generation="g-rc",
|
||
namespace="author",
|
||
pid=161,
|
||
)
|
||
before = registry.get(o["identity"])
|
||
registry.heartbeat(
|
||
worker_identity=o["identity"],
|
||
fencing_epoch=o["fencing_epoch"],
|
||
now=NOW + timedelta(seconds=20),
|
||
expected_session_id="s-rc",
|
||
expected_generation_id="g-rc",
|
||
expected_client_name="codex",
|
||
expected_pid=161,
|
||
)
|
||
after = registry.get(o["identity"])
|
||
self.assertEqual(before["client_instance_id"], after["client_instance_id"])
|
||
self.assertEqual(before["session_id"], after["session_id"])
|
||
self.assertEqual(before["worker_identity"], after["worker_identity"])
|
||
|
||
def test_worker_restart_new_worker_same_instance(self):
|
||
registry = _registry()
|
||
inst = "inst-restart"
|
||
old = _register(
|
||
registry,
|
||
client="codex",
|
||
instance=inst,
|
||
session="s-old",
|
||
generation="g-old",
|
||
namespace="author",
|
||
pid=171,
|
||
)
|
||
registry.release(worker_identity=old["identity"], now=NOW + timedelta(seconds=5))
|
||
new = _register(
|
||
registry,
|
||
client="codex",
|
||
instance=inst,
|
||
session="s-new",
|
||
generation="g-new",
|
||
namespace="author",
|
||
pid=172,
|
||
now=NOW + timedelta(seconds=10),
|
||
)
|
||
self.assertTrue(new["registered"])
|
||
self.assertNotEqual(old["identity"], new["identity"])
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(status=None),
|
||
now=NOW + timedelta(seconds=15),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
live = [w for w in snap["live_workers"] if w["client_instance_id"] == inst]
|
||
self.assertEqual(len(live), 1)
|
||
self.assertEqual(live[0]["worker_identity"], new["identity"])
|
||
self.assertEqual(snap["historical_worker_count"], 1)
|
||
|
||
def test_full_application_restart_new_instance(self):
|
||
registry = _registry()
|
||
old_inst = fleet.generate_client_instance_id("codex", launch_nonce="old", now=NOW)
|
||
new_inst = fleet.generate_client_instance_id(
|
||
"codex", launch_nonce="new", now=NOW + timedelta(hours=1)
|
||
)
|
||
for ns, pid in zip(NAMESPACES, range(181, 186)):
|
||
o = _register(
|
||
registry,
|
||
client="codex",
|
||
instance=old_inst,
|
||
session=f"old-{ns}",
|
||
generation=f"oldg-{ns}",
|
||
namespace=ns,
|
||
pid=pid,
|
||
)
|
||
registry.release(worker_identity=o["identity"], now=NOW + timedelta(minutes=1))
|
||
for ns, pid in zip(NAMESPACES, range(191, 196)):
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance=new_inst,
|
||
session=f"new-{ns}",
|
||
generation=f"newg-{ns}",
|
||
namespace=ns,
|
||
pid=pid,
|
||
now=NOW + timedelta(minutes=2),
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(status=None),
|
||
now=NOW + timedelta(minutes=3),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
live_ids = {i["client_instance_id"] for i in snap["instances"]}
|
||
self.assertEqual(live_ids, {new_inst})
|
||
self.assertEqual(snap["historical_worker_count"], 5)
|
||
|
||
|
||
class PermissionAndSingleClientTests(unittest.TestCase):
|
||
def test_single_client_still_supported(self):
|
||
registry = _registry()
|
||
_five_workers(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-single",
|
||
session_prefix="solo",
|
||
gen_prefix="solo-g",
|
||
pid_base=300,
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
self.assertEqual(snap["instance_count"], 1)
|
||
self.assertEqual(snap["live_worker_count"], 5)
|
||
self.assertTrue(snap["live_fleet_safe"])
|
||
self.assertTrue(snap["mutation_safe"])
|
||
|
||
def test_snapshot_has_consistency_token(self):
|
||
registry = _registry()
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-tok",
|
||
session="s",
|
||
generation="g",
|
||
namespace="author",
|
||
pid=400,
|
||
)
|
||
snap = fleet.snapshot_instance_fleet(
|
||
registry.list_workers(),
|
||
now=NOW + timedelta(seconds=1),
|
||
pid_alive_probe=_alive,
|
||
)
|
||
self.assertTrue(snap["consistency_token"].startswith("fleetrev-"))
|
||
self.assertEqual(snap["consistency_token"], snap["registry_revision"])
|
||
self.assertIn("snapshot_at", snap)
|
||
|
||
def test_tool_role_gate_denies_author(self):
|
||
import gitea_mcp_server as server
|
||
|
||
with mock.patch.object(server, "_profile_operation_gate", return_value=[]):
|
||
with mock.patch.object(
|
||
server,
|
||
"get_profile",
|
||
return_value={
|
||
"profile_name": "prgs-author",
|
||
"role": "author",
|
||
"allowed_operations": ["gitea.read"],
|
||
"forbidden_operations": [],
|
||
},
|
||
):
|
||
result = server.gitea_snapshot_instance_fleet(remote="prgs")
|
||
self.assertFalse(result.get("success"))
|
||
self.assertEqual(result.get("denied_role"), "author")
|
||
self.assertTrue(result.get("read_only"))
|
||
self.assertFalse(result.get("mutation_performed", True))
|
||
|
||
def test_tool_allows_controller(self):
|
||
import gitea_mcp_server as server
|
||
|
||
registry = _registry()
|
||
_register(
|
||
registry,
|
||
client="codex",
|
||
instance="inst-ctrl",
|
||
session="s",
|
||
generation="g",
|
||
namespace="controller",
|
||
pid=410,
|
||
)
|
||
with mock.patch.object(server, "_profile_operation_gate", return_value=[]):
|
||
with mock.patch.object(
|
||
server,
|
||
"get_profile",
|
||
return_value={
|
||
"profile_name": "prgs-controller",
|
||
"role": "controller",
|
||
"allowed_operations": ["gitea.read"],
|
||
"forbidden_operations": [],
|
||
},
|
||
):
|
||
with mock.patch.object(server, "_worker_registry", return_value=registry):
|
||
with mock.patch.object(
|
||
server, "_current_master_parity", return_value={}
|
||
):
|
||
result = server.gitea_snapshot_instance_fleet(remote="prgs")
|
||
self.assertTrue(result.get("success"), result)
|
||
self.assertTrue(result.get("read_only"))
|
||
self.assertTrue(result["permission_scope"]["denied_unrelated_mutations"])
|
||
self.assertEqual(result["permission_scope"]["granted_operations"], ["gitea.read"])
|
||
|
||
def test_tool_allows_reconciler(self):
|
||
import gitea_mcp_server as server
|
||
|
||
registry = _registry()
|
||
with mock.patch.object(server, "_profile_operation_gate", return_value=[]):
|
||
with mock.patch.object(
|
||
server,
|
||
"get_profile",
|
||
return_value={
|
||
"profile_name": "prgs-reconciler",
|
||
"role": "reconciler",
|
||
"allowed_operations": ["gitea.read"],
|
||
"forbidden_operations": [],
|
||
},
|
||
):
|
||
with mock.patch.object(server, "_worker_registry", return_value=registry):
|
||
with mock.patch.object(
|
||
server, "_current_master_parity", return_value={}
|
||
):
|
||
result = server.gitea_snapshot_instance_fleet(remote="prgs")
|
||
self.assertTrue(result.get("success"), result)
|
||
|
||
def test_capability_map_entry(self):
|
||
import task_capability_map as tcm
|
||
|
||
entry = tcm.TASK_CAPABILITY_MAP["snapshot_instance_fleet"]
|
||
self.assertEqual(entry["permission"], "gitea.read")
|
||
self.assertEqual(entry["role"], "controller")
|
||
self.assertIn("gitea_snapshot_instance_fleet", tcm.TASK_CAPABILITY_MAP)
|
||
|
||
def test_fleet_run_env_allowlisted(self):
|
||
import gitea_config
|
||
|
||
self.assertIn("GITEA_MCP_FLEET_RUN_ID", gitea_config.RECOGNIZED_GITEA_ENV_KEYS)
|
||
|
||
def test_docs_exist(self):
|
||
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
path = os.path.join(root, "docs", "instance-fleet-identity.md")
|
||
self.assertTrue(os.path.isfile(path))
|
||
with open(path, encoding="utf-8") as fh:
|
||
text = fh.read()
|
||
self.assertIn("client_instance_id", text)
|
||
self.assertIn("exactly_one_per_profile", text)
|
||
self.assertIn("gitea_snapshot_instance_fleet", text)
|
||
|
||
def test_tool_is_registered(self):
|
||
import gitea_mcp_server as server
|
||
|
||
self.assertTrue(callable(server.gitea_snapshot_instance_fleet))
|
||
src = inspect.getsource(server.gitea_snapshot_instance_fleet)
|
||
self.assertIn("controller", src)
|
||
self.assertIn("reconciler", src)
|
||
|
||
|
||
class ClientHintsTests(unittest.TestCase):
|
||
def test_client_hints_legacy_when_unset(self):
|
||
import gitea_mcp_server as server
|
||
|
||
env = {"GITEA_MCP_CLIENT": "codex"}
|
||
with mock.patch.dict(os.environ, env, clear=False):
|
||
# Ensure instance key is absent.
|
||
os.environ.pop("GITEA_MCP_CLIENT_INSTANCE", None)
|
||
hints = server._client_identity_hints()
|
||
self.assertFalse(hints["instance_identity_trusted"])
|
||
self.assertTrue(
|
||
str(hints["client_instance_id"]).startswith("legacy-pid-")
|
||
or hints["instance_id_provenance"] == fleet.INSTANCE_ID_PROVENANCE_LEGACY
|
||
)
|
||
|
||
def test_client_hints_trusted_when_set(self):
|
||
import gitea_mcp_server as server
|
||
|
||
inst = fleet.generate_client_instance_id("codex", launch_nonce="t", now=NOW)
|
||
with mock.patch.dict(
|
||
os.environ,
|
||
{
|
||
"GITEA_MCP_CLIENT": "codex",
|
||
"GITEA_MCP_CLIENT_INSTANCE": inst,
|
||
"GITEA_MCP_FLEET_RUN_ID": "run-1",
|
||
},
|
||
clear=False,
|
||
):
|
||
hints = server._client_identity_hints()
|
||
self.assertTrue(hints["instance_identity_trusted"])
|
||
self.assertEqual(hints["client_instance_id"], inst)
|
||
self.assertEqual(hints["fleet_run_id"], "run-1")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|