feat(controller): expose instance-level fleet identity and health snapshots
Add a pure fleet snapshot assessor and a read-only controller/reconciler MCP tool so multi-instance fleets can be enumerated by client_instance_id without treating shared client_type as duplication. Trusted launcher instance IDs, classification (missing/unmanifested/collisions/historical), registry schema extensions, docs, and regression tests for #978. Closes #978 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Claude Opus 4.8
+97
-2
@@ -301,8 +301,22 @@ CREATE INDEX IF NOT EXISTS idx_worker_session
|
||||
ON worker_registrations(session_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_worker_profile
|
||||
ON worker_registrations(profile, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_worker_instance
|
||||
ON worker_registrations(client_instance_id, status);
|
||||
"""
|
||||
|
||||
#: #978 optional columns added without rewriting historical rows.
|
||||
_SCHEMA_OPTIONAL_COLUMNS: tuple[tuple[str, str], ...] = (
|
||||
("fleet_run_id", "TEXT"),
|
||||
("authenticated_account", "TEXT"),
|
||||
("process_identity", "TEXT"),
|
||||
("startup_revision", "TEXT"),
|
||||
("loaded_revision", "TEXT"),
|
||||
("parity_revision", "TEXT"),
|
||||
("live_revision", "TEXT"),
|
||||
("instance_id_provenance", "TEXT"),
|
||||
)
|
||||
|
||||
|
||||
class WorkerRegistryError(RuntimeError):
|
||||
"""Raised for registry misuse that is a programming error, not a refusal."""
|
||||
@@ -634,6 +648,17 @@ class WorkerRegistry:
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.executescript(_SCHEMA_SQL)
|
||||
existing = {
|
||||
row[1]
|
||||
for row in conn.execute(
|
||||
"PRAGMA table_info(worker_registrations)"
|
||||
).fetchall()
|
||||
}
|
||||
for name, decl in _SCHEMA_OPTIONAL_COLUMNS:
|
||||
if name not in existing:
|
||||
conn.execute(
|
||||
f"ALTER TABLE worker_registrations ADD COLUMN {name} {decl}"
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -758,6 +783,14 @@ class WorkerRegistry:
|
||||
heartbeat_ttl_seconds: float = DEFAULT_HEARTBEAT_TTL_SECONDS,
|
||||
now: datetime | None = None,
|
||||
pid_alive_probe=None,
|
||||
fleet_run_id: str | None = None,
|
||||
authenticated_account: str | None = None,
|
||||
process_identity: str | None = None,
|
||||
startup_revision: str | None = None,
|
||||
loaded_revision: str | None = None,
|
||||
parity_revision: str | None = None,
|
||||
live_revision: str | None = None,
|
||||
instance_id_provenance: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically register one worker identity.
|
||||
|
||||
@@ -766,6 +799,10 @@ class WorkerRegistry:
|
||||
to mint a different identity and register that instead (AC32). The
|
||||
existing registration is returned untouched so the caller can see what
|
||||
it collided with.
|
||||
|
||||
#978 also fails closed when a *live* registration already holds the same
|
||||
``client_instance_id`` for the same namespace: two workers of one
|
||||
namespace cannot share one instance.
|
||||
"""
|
||||
parsed = parse_worker_identity(worker_identity)
|
||||
if not parsed["valid"]:
|
||||
@@ -783,6 +820,9 @@ class WorkerRegistry:
|
||||
}
|
||||
|
||||
stamp = _ts(now or _utc_now())
|
||||
proc_id = process_identity or (
|
||||
f"pid-{int(pid)}" if pid is not None else None
|
||||
)
|
||||
with self._tx() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT * FROM worker_registrations WHERE worker_identity = ?",
|
||||
@@ -819,6 +859,41 @@ class WorkerRegistry:
|
||||
),
|
||||
}
|
||||
|
||||
# #978: refuse a second live worker for the same (instance, namespace).
|
||||
if namespace and client_instance_id:
|
||||
peers = conn.execute(
|
||||
"SELECT * FROM worker_registrations "
|
||||
"WHERE client_instance_id = ? AND namespace = ? AND status = ?",
|
||||
(client_instance_id, namespace, STATUS_ACTIVE),
|
||||
).fetchall()
|
||||
for peer_row in peers:
|
||||
peer = self._row_to_record(peer_row)
|
||||
pid_alive = (
|
||||
pid_alive_probe(peer.get("pid"))
|
||||
if pid_alive_probe is not None
|
||||
else None
|
||||
)
|
||||
if self.is_live(peer, now=now, pid_alive=pid_alive)["live"]:
|
||||
return {
|
||||
"success": False,
|
||||
"registered": False,
|
||||
"mutation_performed": False,
|
||||
"blocker_kind": BLOCKER_CONFLICTING_SESSIONS,
|
||||
"collision": True,
|
||||
"collision_kind": "duplicate_namespace_worker",
|
||||
"existing_registration": _public_record(peer),
|
||||
"reasons": [
|
||||
f"client_instance_id {client_instance_id!r} already "
|
||||
f"has a live worker for namespace {namespace!r} "
|
||||
f"({peer.get('worker_identity')!r}); #978 fails closed "
|
||||
"rather than registering a second worker"
|
||||
],
|
||||
"exact_next_action": (
|
||||
"Stop the extra namespace worker, or use a distinct "
|
||||
"client_instance_id for a separate application launch."
|
||||
),
|
||||
}
|
||||
|
||||
epoch = self._next_epoch(conn, generation_id)
|
||||
conn.execute(
|
||||
"""
|
||||
@@ -827,8 +902,11 @@ class WorkerRegistry:
|
||||
generation_id, role, profile, namespace, remote,
|
||||
repository_binding, pid, transport, token_fingerprint,
|
||||
started_at, last_heartbeat_at, heartbeat_ttl_seconds,
|
||||
fencing_epoch, status
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
fencing_epoch, status,
|
||||
fleet_run_id, authenticated_account, process_identity,
|
||||
startup_revision, loaded_revision, parity_revision,
|
||||
live_revision, instance_id_provenance
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
""",
|
||||
(
|
||||
worker_identity,
|
||||
@@ -849,6 +927,14 @@ class WorkerRegistry:
|
||||
float(heartbeat_ttl_seconds),
|
||||
epoch,
|
||||
STATUS_ACTIVE,
|
||||
(fleet_run_id or "").strip() or None,
|
||||
(authenticated_account or "").strip() or None,
|
||||
proc_id,
|
||||
(startup_revision or "").strip() or None,
|
||||
(loaded_revision or "").strip() or None,
|
||||
(parity_revision or "").strip() or None,
|
||||
(live_revision or "").strip() or None,
|
||||
(instance_id_provenance or "").strip() or None,
|
||||
),
|
||||
)
|
||||
row = conn.execute(
|
||||
@@ -1149,8 +1235,17 @@ def _public_record(record: dict[str, Any]) -> dict[str, Any]:
|
||||
"transport": record.get("transport"),
|
||||
"started_at": record.get("started_at"),
|
||||
"last_heartbeat_at": record.get("last_heartbeat_at"),
|
||||
"heartbeat_ttl_seconds": record.get("heartbeat_ttl_seconds"),
|
||||
"fencing_epoch": record.get("fencing_epoch"),
|
||||
"status": record.get("status"),
|
||||
"fleet_run_id": record.get("fleet_run_id"),
|
||||
"authenticated_account": record.get("authenticated_account"),
|
||||
"process_identity": record.get("process_identity"),
|
||||
"startup_revision": record.get("startup_revision"),
|
||||
"loaded_revision": record.get("loaded_revision"),
|
||||
"parity_revision": record.get("parity_revision"),
|
||||
"live_revision": record.get("live_revision"),
|
||||
"instance_id_provenance": record.get("instance_id_provenance"),
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user