"""Authoritative, read-only PRGS MCP fleet inventory (#949). Every pre-existing runtime surface is *per process*. ``gitea_get_runtime_context`` and ``gitea_assess_master_parity`` describe only the server answering the call. ``gitea_assess_mcp_namespace_health`` accepts ``process``, ``probe_result`` and ``registered_tools`` **from the caller**, so it cannot constrain the caller. The control-plane ``sessions`` table records allocator *task* sessions, not server processes. Five independent self-reports of the same revision therefore never proved that exactly five processes exist, that no sixth exists, or that all five belong to one client cohort. Evidence model -------------- Two independent sources must agree before a fleet member counts as running: ``control-plane runtime registry`` A row each server writes **about itself** at native transport bind (:func:`build_process_runtime_record`). No caller can supply it. It is authoritative for identity: namespace, profile, role, repository binding, cohort, and the revision the process started at. ``server-side process observation`` A process listing performed by the server answering the inventory call (:func:`scan_mcp_server_processes`), never by the caller. It is authoritative for existence and liveness, and it is the only source that can show a process the registry does not know about. A member is ``live`` only when a registry row has a matching, still-running process whose start time precedes the registration (so a recycled PID cannot impersonate a dead server). Anything the two sources cannot jointly establish is reported as unknown and fails the mutation gate closed — configuration alone never counts as a running member, and matching Git revisions never establish a single cohort. This module performs no restart, reconnect, lease mutation, issue mutation, or process termination. The only signal it ever sends is ``signal 0`` liveness probing, which delivers nothing to the target process. """ from __future__ import annotations import os import secrets import subprocess from datetime import datetime, timezone from typing import Any, Iterable, Mapping, Sequence # ── expected fleet ──────────────────────────────────────────────────────────── # The configured PRGS fleet. Each entry is one expected member; the roster is # the definition of "expected" for missing/unexpected classification. EXPECTED_PRGS_FLEET: tuple[dict[str, str], ...] = ( {"namespace": "gitea-author", "profile": "prgs-author", "role": "author"}, { "namespace": "gitea-controller", "profile": "prgs-controller", "role": "controller", }, {"namespace": "gitea-reviewer", "profile": "prgs-reviewer", "role": "reviewer"}, {"namespace": "gitea-merger", "profile": "prgs-merger", "role": "merger"}, { "namespace": "gitea-reconciler", "profile": "prgs-reconciler", "role": "reconciler", }, ) # Cohort identity supplied by a client that manages the whole fleet. COHORT_ID_ENV = "GITEA_MCP_CLIENT_COHORT_ID" # Registry rows older than this are pruned at *registration* time (a startup # write), never on the read path. Dead rows inside the window are still reported # as stale evidence rather than silently dropped. RUNTIME_RETENTION_SECONDS = 7 * 24 * 3600 # Liveness classifications for a registry row. LIVENESS_LIVE = "live" LIVENESS_DEAD = "dead" LIVENESS_PID_RECYCLED = "pid_recycled" LIVENESS_UNOBSERVED = "unobserved" LIVENESS_UNKNOWN = "unknown" # Per-member health classifications. HEALTH_RUNNING = "running" HEALTH_MISSING = "missing" HEALTH_DUPLICATE = "duplicate" HEALTH_UNEXPECTED = "unexpected" HEALTH_STALE = "stale" HEALTH_UNKNOWN = "unknown" EVIDENCE_AUTHORITY = "control_plane_runtime_registry+server_process_observation" _MCP_PROCESS_MARKER = "mcp_server.py" _LSTART_FORMAT = "%a %b %d %H:%M:%S %Y" _ISO_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # ── time helpers ────────────────────────────────────────────────────────────── def _utcnow() -> datetime: return datetime.now(timezone.utc) def iso_now() -> str: return _utcnow().strftime(_ISO_FORMAT) def _parse_iso(value: Any) -> datetime | None: text = (str(value) if value is not None else "").strip() if not text: return None if text.endswith("Z"): text = text[:-1] + "+00:00" try: parsed = datetime.fromisoformat(text) except ValueError: return None if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=timezone.utc) return parsed.astimezone(timezone.utc) def _int_or_none(value: Any) -> int | None: try: return int(value) except (TypeError, ValueError): return None def _clean(value: Any) -> str | None: text = (str(value) if value is not None else "").strip() return text or None # ── process-level evidence (server side only) ───────────────────────────────── def probe_pid_alive(pid: int | None) -> bool | None: """Return whether *pid* exists. ``None`` when it cannot be determined. Uses ``signal 0``, which performs a permission/existence check and delivers nothing to the target. This module never sends a terminating signal. """ resolved = _int_or_none(pid) if resolved is None or resolved <= 0: return None try: os.kill(resolved, 0) except ProcessLookupError: return False except PermissionError: # The process exists but belongs to another user. return True except OSError: return None return True def scan_mcp_server_processes(*, runner=subprocess.run) -> dict[str, Any]: """Observe running Gitea MCP server processes from this server process. This is deliberately performed by the answering server, never by the caller: a caller-supplied process list is exactly the input a fleet gate must not trust. When the listing cannot be obtained the result reports ``available=False`` so the inventory fails closed instead of assuming that no unregistered process exists. """ try: proc = runner( ["ps", "-o", "pid,lstart,command", "-ax"], capture_output=True, text=True, check=True, ) except Exception as exc: # noqa: BLE001 - any failure means unknown evidence return { "available": False, "processes": [], "reason": f"process listing unavailable: {exc}", } processes: list[dict[str, Any]] = [] for raw_line in (proc.stdout or "").splitlines()[1:]: line = raw_line.strip() if not line or _MCP_PROCESS_MARKER not in line: continue parts = line.split(None, 6) if len(parts) < 7: continue pid = _int_or_none(parts[0]) if pid is None: continue try: naive = datetime.strptime(" ".join(parts[1:6]), _LSTART_FORMAT) started_at = naive.astimezone(timezone.utc) except (ValueError, OSError): started_at = None processes.append( { "pid": pid, "started_at": started_at.strftime(_ISO_FORMAT) if started_at else None, "command": parts[6], } ) processes.sort(key=lambda item: item["pid"]) return {"available": True, "processes": processes, "reason": None} # ── cohort / registration record ────────────────────────────────────────────── def namespace_for_profile(profile: str | None, *, default: str | None = None) -> str | None: """Map a configured profile to its fleet namespace. Resolved from the expected roster rather than from name-shape heuristics. ``role_namespace_gate.infer_mcp_namespace`` only recognises author and reviewer, so it returns the *profile* name for controller, merger, and reconciler — which would label three of the five members with a namespace that does not exist. Widening that helper is controller role-metadata work and belongs to #950; the roster already carries the mapping this inventory needs, so it is read from there. A profile outside the roster falls back to *default* (typically the caller's existing inference), so an unexpected member is still described rather than dropped. """ cleaned = _clean(profile) for entry in EXPECTED_PRGS_FLEET: if entry["profile"] == cleaned: return entry["namespace"] return default if default is not None else cleaned def derive_cohort_identity(env: Mapping[str, str] | None = None) -> dict[str, Any]: """Derive the client/cohort identity of this server process. A cohort is the set of servers a single client launched together. The parent process is the durable expression of that: an IDE/CLI client spawns every namespace as its own child. A process whose parent has gone away (reparented to init) cannot prove which cohort it belongs to, and says so rather than guessing. Revisions are deliberately not consulted here. Two servers built from the same commit are not thereby one cohort (#949 AC7). """ source_env = os.environ if env is None else env explicit = _clean(source_env.get(COHORT_ID_ENV)) if explicit: return {"cohort_id": explicit, "cohort_source": "explicit_env"} try: ppid = os.getppid() except OSError: ppid = 0 if ppid and ppid > 1: return {"cohort_id": f"ppid:{ppid}", "cohort_source": "parent_process"} return { "cohort_id": None, "cohort_source": "unknown", "cohort_reason": ( "parent process is unavailable or reparented to init; this server " "cannot prove which client cohort launched it" ), } def build_process_runtime_record( *, namespace: str, profile: str | None, role: str | None, remote: str | None = None, org: str | None = None, repo: str | None = None, repository_root: str | None = None, pid: int | None = None, startup_head: str | None = None, daemon_start_head: str | None = None, transport: str | None = None, client_provenance: str | None = None, env: Mapping[str, str] | None = None, boot_id: str | None = None, registered_at: str | None = None, ) -> dict[str, Any]: """Build the row a server writes about itself at native transport bind. Every field describes the *calling* process. Nothing here is caller-supplied in the MCP sense: the only code that reaches this function is the official entrypoint of the process being described. """ cohort = derive_cohort_identity(env) resolved_pid = _int_or_none(pid) if resolved_pid is None: resolved_pid = os.getpid() token = boot_id or secrets.token_hex(8) return { "runtime_id": f"{namespace}:{resolved_pid}:{token}", "namespace": namespace, "profile": _clean(profile), "role": _clean(role), "remote": _clean(remote), "org": _clean(org), "repo": _clean(repo), "repository_root": _clean(repository_root), "pid": resolved_pid, "cohort_id": cohort["cohort_id"], "cohort_source": cohort["cohort_source"], "client_provenance": _clean(client_provenance) or "unknown", "boot_id": token, "startup_head": _clean(startup_head), "daemon_start_head": _clean(daemon_start_head), "transport": _clean(transport), "registered_at": registered_at or iso_now(), "status": "running", } # ── classification ──────────────────────────────────────────────────────────── def _expected_index( expected_fleet: Sequence[Mapping[str, str]], ) -> dict[str, dict[str, str]]: index: dict[str, dict[str, str]] = {} for entry in expected_fleet: profile = _clean(entry.get("profile")) if profile: index[profile] = dict(entry) return index def _sort_key(member: Mapping[str, Any]) -> tuple: return ( str(member.get("namespace") or ""), str(member.get("profile") or ""), _int_or_none(member.get("pid")) or 0, str(member.get("runtime_id") or ""), ) def _normalize_row( row: Mapping[str, Any], *, observed_by_pid: Mapping[int, Mapping[str, Any]], process_scan_available: bool, now: datetime, ) -> dict[str, Any]: pid = _int_or_none(row.get("pid")) member: dict[str, Any] = { "runtime_id": _clean(row.get("runtime_id")), "namespace": _clean(row.get("namespace")), "profile": _clean(row.get("profile")), "role": _clean(row.get("role")), "remote": _clean(row.get("remote")), "org": _clean(row.get("org")), "repo": _clean(row.get("repo")), "repository_root": _clean(row.get("repository_root")), "pid": pid, "cohort_id": _clean(row.get("cohort_id")), "cohort_source": _clean(row.get("cohort_source")) or "unknown", "client_provenance": _clean(row.get("client_provenance")) or "unknown", "boot_id": _clean(row.get("boot_id")), "startup_head": _clean(row.get("startup_head")), "daemon_start_head": _clean(row.get("daemon_start_head")), "transport": _clean(row.get("transport")), "registered_at": _clean(row.get("registered_at")), "last_heartbeat_at": _clean(row.get("last_heartbeat_at")), "recorded_status": _clean(row.get("status")) or "unknown", } pid_alive = probe_pid_alive(pid) member["pid_alive"] = pid_alive observed = observed_by_pid.get(pid) if pid is not None else None member["process_observed"] = bool(observed) if process_scan_available else None if not process_scan_available: # Existence cannot be corroborated; never upgrade to live on the # registry's word alone. member["liveness"] = LIVENESS_UNKNOWN member["liveness_reason"] = ( "process observation unavailable; registry rows cannot be corroborated" ) elif pid_alive is False: member["liveness"] = LIVENESS_DEAD member["liveness_reason"] = "recorded PID is not running" elif pid_alive is None: member["liveness"] = LIVENESS_UNKNOWN member["liveness_reason"] = "PID liveness could not be determined" elif observed is None: member["liveness"] = LIVENESS_UNOBSERVED member["liveness_reason"] = ( "recorded PID is not a running Gitea MCP server process" ) else: started_at = _parse_iso(observed.get("started_at")) registered_at = _parse_iso(member["registered_at"]) if started_at and registered_at and started_at > registered_at: member["liveness"] = LIVENESS_PID_RECYCLED member["liveness_reason"] = ( "the process now holding this PID started after the registry row " "was written; the registered server is gone" ) else: member["liveness"] = LIVENESS_LIVE member["liveness_reason"] = None heartbeat = _parse_iso(member["last_heartbeat_at"]) member["heartbeat_age_seconds"] = ( int((now - heartbeat).total_seconds()) if heartbeat else None ) return member def _binding_matches( member: Mapping[str, Any], expected_binding: Mapping[str, Any] | None ) -> bool | None: if not expected_binding: return None for field in ("remote", "org", "repo"): expected = _clean(expected_binding.get(field)) if expected is None: continue actual = _clean(member.get(field)) if actual is None: return None if actual != expected: return False return True def classify_fleet_inventory( *, runtime_rows: Iterable[Mapping[str, Any]], process_scan: Mapping[str, Any] | None = None, expected_fleet: Sequence[Mapping[str, str]] = EXPECTED_PRGS_FLEET, expected_binding: Mapping[str, Any] | None = None, registry_available: bool = True, registry_error: str | None = None, now: datetime | None = None, answering_namespace: str | None = None, ) -> dict[str, Any]: """Classify a fleet snapshot. Pure: identical input yields identical output. The verdict never depends on which namespace asked, so controller and reconciler agree by construction; ``answering_namespace`` is reported as metadata only. """ moment = now or _utcnow() scan = dict(process_scan or {"available": False, "processes": [], "reason": None}) scan_available = bool(scan.get("available")) observed_processes = list(scan.get("processes") or []) observed_by_pid: dict[int, Mapping[str, Any]] = {} for proc in observed_processes: observed_pid = _int_or_none(proc.get("pid")) if observed_pid is not None: observed_by_pid[observed_pid] = proc expected_index = _expected_index(expected_fleet) members = [ _normalize_row( row, observed_by_pid=observed_by_pid, process_scan_available=scan_available, now=moment, ) for row in (runtime_rows or []) ] live = [m for m in members if m["liveness"] == LIVENESS_LIVE] not_live = [m for m in members if m["liveness"] != LIVENESS_LIVE] live_by_profile: dict[str, list[dict[str, Any]]] = {} for member in live: live_by_profile.setdefault(member["profile"] or "", []).append(member) running_members: list[dict[str, Any]] = [] missing_members: list[dict[str, Any]] = [] duplicate_members: list[dict[str, Any]] = [] unexpected_members: list[dict[str, Any]] = [] binding_mismatches: list[dict[str, Any]] = [] role_mismatches: list[dict[str, Any]] = [] configured_members: list[dict[str, Any]] = [] for entry in expected_fleet: profile = _clean(entry.get("profile")) or "" instances = sorted(live_by_profile.get(profile, []), key=_sort_key) configured_members.append( { "namespace": _clean(entry.get("namespace")), "profile": profile, "role": _clean(entry.get("role")), "instance_count": len(instances), "health": ( HEALTH_MISSING if not instances else HEALTH_RUNNING if len(instances) == 1 else HEALTH_DUPLICATE ), } ) if not instances: missing_members.append( { "namespace": _clean(entry.get("namespace")), "profile": profile, "role": _clean(entry.get("role")), "health": HEALTH_MISSING, "reason": ( "no live registry row corroborated by a running server " "process" ), } ) continue for instance in instances: instance["health"] = ( HEALTH_RUNNING if len(instances) == 1 else HEALTH_DUPLICATE ) instance["expected"] = True running_members.append(instance) if len(instances) > 1: duplicate_members.append( { "namespace": _clean(entry.get("namespace")), "profile": profile, "role": _clean(entry.get("role")), "health": HEALTH_DUPLICATE, "instance_count": len(instances), "pids": sorted( i["pid"] for i in instances if i["pid"] is not None ), "instances": instances, "reason": ( "more than one live server is registered for this profile" ), } ) for member in sorted(live, key=_sort_key): profile = member["profile"] or "" expected_entry = expected_index.get(profile) if expected_entry is not None: expected_role = _clean(expected_entry.get("role")) actual_role = member["role"] if expected_role and actual_role and actual_role != expected_role: role_mismatches.append( { "namespace": member["namespace"], "profile": profile, "pid": member["pid"], "expected_role": expected_role, "declared_role": actual_role, "reason": "declared role does not match the configured profile", } ) expected_namespace = _clean(expected_entry.get("namespace")) if ( expected_namespace and member["namespace"] and member["namespace"] != expected_namespace ): role_mismatches.append( { "namespace": member["namespace"], "profile": profile, "pid": member["pid"], "expected_namespace": expected_namespace, "declared_role": member["role"], "reason": ( "profile is served from a namespace it is not " "configured for" ), } ) else: member["health"] = HEALTH_UNEXPECTED member["expected"] = False unexpected_members.append(member) match = _binding_matches(member, expected_binding) member["repository_binding_matches"] = match if match is False: binding_mismatches.append( { "namespace": member["namespace"], "profile": profile, "pid": member["pid"], "remote": member["remote"], "org": member["org"], "repo": member["repo"], "expected": dict(expected_binding or {}), "reason": "member is bound to a different repository", } ) elif match is None and expected_binding: binding_mismatches.append( { "namespace": member["namespace"], "profile": profile, "pid": member["pid"], "remote": member["remote"], "org": member["org"], "repo": member["repo"], "expected": dict(expected_binding or {}), "reason": "member did not record a complete repository binding", } ) stale_members: list[dict[str, Any]] = [] for member in sorted(not_live, key=_sort_key): member["health"] = ( HEALTH_UNKNOWN if member["liveness"] == LIVENESS_UNKNOWN else HEALTH_STALE ) stale_members.append(member) registered_pids = {m["pid"] for m in live if m["pid"] is not None} unregistered_processes: list[dict[str, Any]] = [] if scan_available: for proc in observed_processes: observed_pid = _int_or_none(proc.get("pid")) if observed_pid is None or observed_pid in registered_pids: continue unregistered_processes.append( {"pid": observed_pid, "started_at": proc.get("started_at")} ) unregistered_processes.sort(key=lambda item: item["pid"]) # Cohort. Derived only from recorded cohort identity — never from revisions. cohort_ids = {m["cohort_id"] for m in live} cohort_unknown = any(cohort_id is None for cohort_id in cohort_ids) known_cohorts = sorted(c for c in cohort_ids if c is not None) if not live or cohort_unknown: single_cohort: bool | None = None mixed_cohort: bool | None = None else: single_cohort = len(known_cohorts) == 1 mixed_cohort = len(known_cohorts) > 1 # Revision spread. Reported independently of cohort; never used to infer it. heads = {m["startup_head"] for m in live} head_unknown = any(head is None for head in heads) known_heads = sorted(h for h in heads if h is not None) mixed_revision = (len(known_heads) > 1) if known_heads else None exactly_one_per_profile = not missing_members and not duplicate_members no_unexpected_members = not unexpected_members incomplete_reasons: list[str] = [] if not registry_available: incomplete_reasons.append( registry_error or "the control-plane runtime registry could not be read" ) if not scan_available: incomplete_reasons.append( str(scan.get("reason") or "server-side process observation unavailable") ) if unregistered_processes: pids = ", ".join(str(p["pid"]) for p in unregistered_processes) incomplete_reasons.append( f"running Gitea MCP server process(es) with no runtime registry row " f"(PIDs: {pids}); the fleet contains members this inventory cannot " f"describe" ) if live and cohort_unknown: incomplete_reasons.append( "one or more live members did not record a client cohort identity; " "matching revisions do not establish a single cohort" ) if live and head_unknown: incomplete_reasons.append( "one or more live members did not record a startup revision" ) if any(m["liveness"] == LIVENESS_UNKNOWN for m in members): incomplete_reasons.append( "liveness of one or more registry rows could not be determined" ) inventory_complete = not incomplete_reasons blocked_reasons: list[str] = list(incomplete_reasons) if missing_members: names = ", ".join(sorted(m["profile"] for m in missing_members)) blocked_reasons.append(f"expected fleet member(s) not running: {names}") if duplicate_members: names = ", ".join(sorted(d["profile"] for d in duplicate_members)) blocked_reasons.append( f"duplicate server(s) registered for profile(s): {names}" ) if unexpected_members: names = ", ".join( sorted( str(m["profile"] or m["namespace"] or "?") for m in unexpected_members ) ) blocked_reasons.append(f"unexpected fleet member(s) running: {names}") if mixed_cohort: blocked_reasons.append( "live members span more than one client cohort: " + ", ".join(known_cohorts) ) if mixed_revision: blocked_reasons.append( "live members started at different revisions: " + ", ".join(known_heads) ) if binding_mismatches: blocked_reasons.append( "one or more live members are not bound to the expected repository" ) if role_mismatches: blocked_reasons.append( "one or more live members declare a role or namespace that does not " "match the configured profile" ) mutation_gate_satisfied = bool( inventory_complete and exactly_one_per_profile and no_unexpected_members and single_cohort is True and mixed_revision is False and not binding_mismatches and not role_mismatches ) if not mutation_gate_satisfied and not blocked_reasons: blocked_reasons.append( "the fleet snapshot did not establish the exact-one-instance-per-" "profile, single-cohort invariant" ) return { "success": True, "read_only": True, "evidence_authority": EVIDENCE_AUTHORITY, "answering_namespace": _clean(answering_namespace), "generated_at": moment.strftime(_ISO_FORMAT), "inventory_complete": inventory_complete, "incomplete_reasons": incomplete_reasons, "configured_members": configured_members, "running_members": sorted(running_members, key=_sort_key), "missing_members": sorted(missing_members, key=_sort_key), "duplicate_members": sorted(duplicate_members, key=_sort_key), "unexpected_members": sorted(unexpected_members, key=_sort_key), "stale_members": stale_members, "unregistered_processes": unregistered_processes, "repository_binding_mismatches": sorted(binding_mismatches, key=_sort_key), "role_mismatches": sorted(role_mismatches, key=_sort_key), "cohort_ids": known_cohorts, "single_cohort": single_cohort, "mixed_cohort": mixed_cohort, "startup_revisions": known_heads, "mixed_revision": mixed_revision, "exactly_one_per_profile": exactly_one_per_profile, "no_unexpected_members": no_unexpected_members, "expected_member_count": len(expected_fleet), "running_member_count": len(running_members), "mutation_gate_satisfied": mutation_gate_satisfied, "blocked_reason": blocked_reasons[0] if blocked_reasons else None, "blocked_reasons": blocked_reasons, "process_observation": { "available": scan_available, "observed_process_count": len(observed_processes), "reason": scan.get("reason"), }, "registry": { "available": registry_available, "row_count": len(members), "error": registry_error, }, "mutations_performed": [], } def summarize(result: Mapping[str, Any]) -> str: """One-line human summary of a classification result.""" if result.get("mutation_gate_satisfied"): return ( f"fleet healthy: {result.get('running_member_count')} of " f"{result.get('expected_member_count')} members running in a single " f"cohort at one revision" ) return f"fleet not provable: {result.get('blocked_reason')}"