sysadminandClaude Opus 5
92615f474b
Every pre-existing runtime surface is per-process. gitea_get_runtime_context and gitea_assess_master_parity describe only the server answering the call, and gitea_assess_mcp_namespace_health accepts process, probe_result and registered_tools from the caller, so it cannot constrain the caller. The control-plane sessions table records allocator task sessions, not server processes. Five self-reports of one revision therefore never proved that five processes exist, that no sixth exists, or that all five share one client cohort. Add gitea_assess_fleet_inventory, a strictly read-only capability that takes no evidence parameters. It combines two sources that must agree: - a control-plane runtime registry row each server writes about itself, from the official entrypoint, immediately after the native transport bind; - a process observation the answering server performs, never the caller. A member is running only when both agree and the observed process predates its registration, so configuration alone never counts and a recycled PID cannot impersonate an exited server. Classification is a pure function of the snapshot, so gitea-controller and gitea-reconciler return the same verdict. Missing, duplicate, unexpected, stale and unregistered members are reported in separate fields rather than collapsed into one error, because each needs a different operator action. single_cohort is derived only from recorded cohort identity and stays null when unknown, so matching revisions never establish a cohort. Any unreadable registry, unavailable listing, unregistered process, unknown cohort or unknown revision sets inventory_complete and mutation_gate_satisfied false with a specific blocked_reason. The capability performs no restart, reconnect, drain, lease mutation, issue mutation or process termination, never terminates a duplicate, and never manufactures restart evidence. The only signal sent is signal 0. Also resolves the fleet namespace from the expected roster: role_namespace_gate.infer_mcp_namespace recognises only author and reviewer and echoes the profile name otherwise, which would have mislabelled the controller, merger and reconciler members. Widening that shared helper is controller role-metadata work owned by #950 and is deliberately not done here. Schema v5 to v6 is additive and idempotent: one new table, no existing table, tool signature or result field changed. Two control-plane tests that pinned the schema version to the literal 5 now assert against SCHEMA_VERSION. #950 (controller role metadata), #951 (restart receipts) and #952 (stale-lease consistency) remain separate and untouched. Tests: 118 new across tests/test_mcp_fleet_inventory.py, tests/test_control_plane_db_server_runtimes.py and tests/test_fleet_inventory_tool.py, covering every acceptance criterion including the multi-LLM duplicate-server regression that motivated the issue. Closes #949 Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01NBaQKcRRKeKbZuhk72MhEf
789 lines
31 KiB
Python
789 lines
31 KiB
Python
"""Authoritative fleet inventory classification (#949).
|
|
|
|
One test group per acceptance criterion. The classifier is pure, so every
|
|
scenario is expressed as a snapshot: registry rows plus a process observation.
|
|
PID liveness is the one impure input, so it is patched per test rather than
|
|
depending on whatever happens to be running on the machine.
|
|
"""
|
|
|
|
import os
|
|
import unittest
|
|
from datetime import datetime, timedelta, timezone
|
|
from unittest import mock
|
|
|
|
import mcp_fleet_inventory as mfi
|
|
|
|
|
|
NOW = datetime(2026, 7, 28, 3, 0, 0, tzinfo=timezone.utc)
|
|
HEAD_A = "82d71b77028a7abd4f8ab4a4e4d89658a187f73d"
|
|
HEAD_B = "35ed8a2fcb11134a37c862ca6eaca26e3028902a"
|
|
COHORT_A = "ppid:40990"
|
|
COHORT_B = "ppid:51022"
|
|
BINDING = {"remote": "prgs", "org": "Scaled-Tech-Consulting", "repo": "Gitea-Tools"}
|
|
|
|
_BASE_PID = 41000
|
|
|
|
|
|
def row(
|
|
namespace,
|
|
profile,
|
|
role,
|
|
pid,
|
|
*,
|
|
cohort_id=COHORT_A,
|
|
startup_head=HEAD_A,
|
|
registered_at="2026-07-28T02:00:00Z",
|
|
**overrides,
|
|
):
|
|
"""One control-plane runtime registry row."""
|
|
record = {
|
|
"runtime_id": f"{namespace}:{pid}:boot{pid}",
|
|
"namespace": namespace,
|
|
"profile": profile,
|
|
"role": role,
|
|
"remote": BINDING["remote"],
|
|
"org": BINDING["org"],
|
|
"repo": BINDING["repo"],
|
|
"repository_root": "/checkout/Gitea-Tools",
|
|
"pid": pid,
|
|
"cohort_id": cohort_id,
|
|
"cohort_source": "parent_process",
|
|
"client_provenance": "client_managed",
|
|
"boot_id": f"boot{pid}",
|
|
"startup_head": startup_head,
|
|
"daemon_start_head": startup_head,
|
|
"transport": "stdio",
|
|
"registered_at": registered_at,
|
|
"last_heartbeat_at": registered_at,
|
|
"status": "running",
|
|
}
|
|
record.update(overrides)
|
|
return record
|
|
|
|
|
|
def healthy_rows():
|
|
"""One live row per expected member, all one cohort, all one revision."""
|
|
return [
|
|
row(entry["namespace"], entry["profile"], entry["role"], _BASE_PID + index)
|
|
for index, entry in enumerate(mfi.EXPECTED_PRGS_FLEET)
|
|
]
|
|
|
|
|
|
def scan_for(rows, *, available=True, extra_pids=(), started_at="2026-07-28T01:00:00Z"):
|
|
"""A process observation that corroborates *rows* (plus any extra PIDs)."""
|
|
processes = [
|
|
{"pid": r["pid"], "started_at": started_at, "command": "python mcp_server.py"}
|
|
for r in rows
|
|
]
|
|
processes.extend(
|
|
{"pid": pid, "started_at": started_at, "command": "python mcp_server.py"}
|
|
for pid in extra_pids
|
|
)
|
|
return {"available": available, "processes": processes, "reason": None}
|
|
|
|
|
|
def classify(rows, scan=None, **kwargs):
|
|
kwargs.setdefault("expected_binding", BINDING)
|
|
kwargs.setdefault("now", NOW)
|
|
return mfi.classify_fleet_inventory(
|
|
runtime_rows=rows,
|
|
process_scan=scan if scan is not None else scan_for(rows),
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
class _AliveMixin:
|
|
"""Treat every PID as alive unless a test declares it dead or unknown."""
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.dead_pids = set()
|
|
self.unknown_pids = set()
|
|
|
|
def probe(pid):
|
|
if pid in self.unknown_pids:
|
|
return None
|
|
return pid not in self.dead_pids
|
|
|
|
patcher = mock.patch.object(mfi, "probe_pid_alive", side_effect=probe)
|
|
patcher.start()
|
|
self.addCleanup(patcher.stop)
|
|
|
|
|
|
class TestHealthyFleet(_AliveMixin, unittest.TestCase):
|
|
"""AC1: a healthy five-server fleet reports all five members exactly once."""
|
|
|
|
def test_five_members_each_reported_once(self):
|
|
result = classify(healthy_rows())
|
|
self.assertEqual(len(result["configured_members"]), 5)
|
|
self.assertEqual(len(result["running_members"]), 5)
|
|
self.assertEqual(result["running_member_count"], 5)
|
|
self.assertEqual(result["missing_members"], [])
|
|
self.assertEqual(result["duplicate_members"], [])
|
|
self.assertEqual(result["unexpected_members"], [])
|
|
self.assertTrue(result["exactly_one_per_profile"])
|
|
for member in result["configured_members"]:
|
|
self.assertEqual(member["instance_count"], 1)
|
|
self.assertEqual(member["health"], mfi.HEALTH_RUNNING)
|
|
|
|
def test_healthy_fleet_satisfies_the_mutation_gate(self):
|
|
result = classify(healthy_rows())
|
|
self.assertTrue(result["inventory_complete"])
|
|
self.assertTrue(result["mutation_gate_satisfied"])
|
|
self.assertIsNone(result["blocked_reason"])
|
|
self.assertTrue(result["single_cohort"])
|
|
self.assertFalse(result["mixed_cohort"])
|
|
self.assertFalse(result["mixed_revision"])
|
|
|
|
def test_every_expected_namespace_and_profile_is_present(self):
|
|
result = classify(healthy_rows())
|
|
self.assertEqual(
|
|
sorted(m["profile"] for m in result["configured_members"]),
|
|
[
|
|
"prgs-author",
|
|
"prgs-controller",
|
|
"prgs-merger",
|
|
"prgs-reconciler",
|
|
"prgs-reviewer",
|
|
],
|
|
)
|
|
|
|
|
|
class TestDuplicateMember(_AliveMixin, unittest.TestCase):
|
|
"""AC2: two processes on one profile are a duplicate and fail the gate."""
|
|
|
|
def test_duplicate_is_reported_with_every_pid(self):
|
|
rows = healthy_rows()
|
|
rows.append(row("gitea-author", "prgs-author", "author", 49999))
|
|
result = classify(rows)
|
|
self.assertEqual(len(result["duplicate_members"]), 1)
|
|
duplicate = result["duplicate_members"][0]
|
|
self.assertEqual(duplicate["profile"], "prgs-author")
|
|
self.assertEqual(duplicate["instance_count"], 2)
|
|
self.assertEqual(duplicate["pids"], [_BASE_PID, 49999])
|
|
|
|
def test_duplicate_fails_the_mutation_gate(self):
|
|
rows = healthy_rows()
|
|
rows.append(row("gitea-author", "prgs-author", "author", 49999))
|
|
result = classify(rows)
|
|
self.assertFalse(result["exactly_one_per_profile"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
self.assertIn("duplicate", result["blocked_reason"])
|
|
|
|
def test_duplicate_is_not_reported_as_missing_or_unexpected(self):
|
|
rows = healthy_rows()
|
|
rows.append(row("gitea-author", "prgs-author", "author", 49999))
|
|
result = classify(rows)
|
|
self.assertEqual(result["missing_members"], [])
|
|
self.assertEqual(result["unexpected_members"], [])
|
|
|
|
|
|
class TestMissingMember(_AliveMixin, unittest.TestCase):
|
|
"""AC3: a missing expected server is identified and fails the gate."""
|
|
|
|
def test_missing_member_is_named(self):
|
|
rows = [r for r in healthy_rows() if r["profile"] != "prgs-merger"]
|
|
result = classify(rows)
|
|
self.assertEqual(len(result["missing_members"]), 1)
|
|
self.assertEqual(result["missing_members"][0]["profile"], "prgs-merger")
|
|
self.assertEqual(result["missing_members"][0]["health"], mfi.HEALTH_MISSING)
|
|
|
|
def test_missing_member_fails_the_mutation_gate(self):
|
|
rows = [r for r in healthy_rows() if r["profile"] != "prgs-merger"]
|
|
result = classify(rows)
|
|
self.assertFalse(result["exactly_one_per_profile"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
self.assertIn("prgs-merger", result["blocked_reason"])
|
|
|
|
def test_missing_member_still_lists_all_configured_members(self):
|
|
rows = [r for r in healthy_rows() if r["profile"] != "prgs-merger"]
|
|
result = classify(rows)
|
|
self.assertEqual(len(result["configured_members"]), 5)
|
|
merger = [
|
|
m for m in result["configured_members"] if m["profile"] == "prgs-merger"
|
|
][0]
|
|
self.assertEqual(merger["instance_count"], 0)
|
|
self.assertEqual(merger["health"], mfi.HEALTH_MISSING)
|
|
|
|
|
|
class TestUnexpectedMember(_AliveMixin, unittest.TestCase):
|
|
"""AC4: an unexpected PRGS server is reported explicitly."""
|
|
|
|
def test_unexpected_member_is_its_own_category(self):
|
|
rows = healthy_rows()
|
|
rows.append(row("gitea-shadow", "prgs-shadow", "author", 47777))
|
|
result = classify(rows)
|
|
self.assertEqual(len(result["unexpected_members"]), 1)
|
|
self.assertEqual(result["unexpected_members"][0]["profile"], "prgs-shadow")
|
|
self.assertEqual(
|
|
result["unexpected_members"][0]["health"], mfi.HEALTH_UNEXPECTED
|
|
)
|
|
|
|
def test_unexpected_member_is_not_collapsed_into_duplicates_or_missing(self):
|
|
rows = healthy_rows()
|
|
rows.append(row("gitea-shadow", "prgs-shadow", "author", 47777))
|
|
result = classify(rows)
|
|
self.assertEqual(result["duplicate_members"], [])
|
|
self.assertEqual(result["missing_members"], [])
|
|
self.assertTrue(result["exactly_one_per_profile"])
|
|
self.assertFalse(result["no_unexpected_members"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
def test_unexpected_member_blocks_with_its_own_reason(self):
|
|
rows = healthy_rows()
|
|
rows.append(row("gitea-shadow", "prgs-shadow", "author", 47777))
|
|
result = classify(rows)
|
|
self.assertTrue(
|
|
any("unexpected" in reason for reason in result["blocked_reasons"])
|
|
)
|
|
|
|
|
|
class TestMixedCohort(_AliveMixin, unittest.TestCase):
|
|
"""AC5: members from different client cohorts are detected."""
|
|
|
|
def test_two_cohorts_are_detected(self):
|
|
rows = healthy_rows()
|
|
rows[0]["cohort_id"] = COHORT_B
|
|
result = classify(rows)
|
|
self.assertTrue(result["mixed_cohort"])
|
|
self.assertFalse(result["single_cohort"])
|
|
self.assertEqual(result["cohort_ids"], sorted([COHORT_A, COHORT_B]))
|
|
|
|
def test_mixed_cohort_fails_the_mutation_gate(self):
|
|
rows = healthy_rows()
|
|
rows[0]["cohort_id"] = COHORT_B
|
|
result = classify(rows)
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
self.assertTrue(any("cohort" in reason for reason in result["blocked_reasons"]))
|
|
|
|
def test_mixed_cohort_is_not_a_duplicate_or_missing_report(self):
|
|
rows = healthy_rows()
|
|
rows[0]["cohort_id"] = COHORT_B
|
|
result = classify(rows)
|
|
self.assertEqual(result["duplicate_members"], [])
|
|
self.assertEqual(result["missing_members"], [])
|
|
|
|
|
|
class TestMixedRevision(_AliveMixin, unittest.TestCase):
|
|
"""AC6: members running different startup revisions are detected."""
|
|
|
|
def test_two_revisions_are_detected(self):
|
|
rows = healthy_rows()
|
|
rows[0]["startup_head"] = HEAD_B
|
|
result = classify(rows)
|
|
self.assertTrue(result["mixed_revision"])
|
|
self.assertEqual(result["startup_revisions"], sorted([HEAD_A, HEAD_B]))
|
|
|
|
def test_mixed_revision_fails_the_mutation_gate(self):
|
|
rows = healthy_rows()
|
|
rows[0]["startup_head"] = HEAD_B
|
|
result = classify(rows)
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
self.assertTrue(
|
|
any("revision" in reason for reason in result["blocked_reasons"])
|
|
)
|
|
|
|
def test_mixed_revision_does_not_by_itself_imply_mixed_cohort(self):
|
|
rows = healthy_rows()
|
|
rows[0]["startup_head"] = HEAD_B
|
|
result = classify(rows)
|
|
self.assertTrue(result["single_cohort"])
|
|
self.assertFalse(result["mixed_cohort"])
|
|
|
|
|
|
class TestRevisionIsNotCohort(_AliveMixin, unittest.TestCase):
|
|
"""AC7: matching Git revisions alone do not establish a single cohort."""
|
|
|
|
def test_identical_revisions_with_unknown_cohort_stay_unknown(self):
|
|
rows = healthy_rows()
|
|
for r in rows:
|
|
r["cohort_id"] = None
|
|
result = classify(rows)
|
|
self.assertEqual(len({r["startup_head"] for r in rows}), 1)
|
|
self.assertIsNone(result["single_cohort"])
|
|
self.assertIsNone(result["mixed_cohort"])
|
|
|
|
def test_identical_revisions_with_unknown_cohort_fail_closed(self):
|
|
rows = healthy_rows()
|
|
for r in rows:
|
|
r["cohort_id"] = None
|
|
result = classify(rows)
|
|
self.assertFalse(result["inventory_complete"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
self.assertTrue(
|
|
any("cohort" in reason for reason in result["incomplete_reasons"])
|
|
)
|
|
|
|
def test_one_unknown_cohort_among_known_ones_still_fails_closed(self):
|
|
rows = healthy_rows()
|
|
rows[0]["cohort_id"] = None
|
|
result = classify(rows)
|
|
self.assertIsNone(result["single_cohort"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
def test_cohort_derivation_never_consults_revisions(self):
|
|
"""The cohort helper takes no revision input at all."""
|
|
identity = mfi.derive_cohort_identity({mfi.COHORT_ID_ENV: "cohort-x"})
|
|
self.assertEqual(identity["cohort_id"], "cohort-x")
|
|
self.assertEqual(identity["cohort_source"], "explicit_env")
|
|
|
|
def test_orphaned_process_reports_unknown_cohort(self):
|
|
with mock.patch("os.getppid", return_value=1):
|
|
identity = mfi.derive_cohort_identity({})
|
|
self.assertIsNone(identity["cohort_id"])
|
|
self.assertEqual(identity["cohort_source"], "unknown")
|
|
|
|
def test_parent_process_is_the_cohort_when_no_env_is_set(self):
|
|
with mock.patch("os.getppid", return_value=40990):
|
|
identity = mfi.derive_cohort_identity({})
|
|
self.assertEqual(identity["cohort_id"], COHORT_A)
|
|
self.assertEqual(identity["cohort_source"], "parent_process")
|
|
|
|
|
|
class TestConfigurationIsNotRunning(_AliveMixin, unittest.TestCase):
|
|
"""AC8: configuration without a live worker is not a running member."""
|
|
|
|
def test_no_registry_rows_means_every_member_is_missing(self):
|
|
result = classify([], scan=scan_for([]))
|
|
self.assertEqual(len(result["missing_members"]), 5)
|
|
self.assertEqual(result["running_members"], [])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
def test_dead_pid_is_stale_not_running(self):
|
|
rows = healthy_rows()
|
|
self.dead_pids = {rows[0]["pid"]}
|
|
result = classify(rows)
|
|
self.assertEqual(len(result["running_members"]), 4)
|
|
self.assertEqual(len(result["stale_members"]), 1)
|
|
self.assertEqual(result["stale_members"][0]["liveness"], mfi.LIVENESS_DEAD)
|
|
self.assertEqual(result["stale_members"][0]["health"], mfi.HEALTH_STALE)
|
|
self.assertEqual(len(result["missing_members"]), 1)
|
|
|
|
def test_registry_row_without_a_matching_process_is_not_running(self):
|
|
rows = healthy_rows()
|
|
scan = scan_for(rows[1:]) # first member's process is absent
|
|
result = classify(rows, scan=scan)
|
|
self.assertEqual(len(result["running_members"]), 4)
|
|
self.assertEqual(
|
|
result["stale_members"][0]["liveness"], mfi.LIVENESS_UNOBSERVED
|
|
)
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
def test_recycled_pid_does_not_impersonate_a_dead_server(self):
|
|
rows = healthy_rows()
|
|
scan = scan_for(rows)
|
|
# The process now holding the first PID started *after* registration.
|
|
scan["processes"][0]["started_at"] = "2026-07-28T02:30:00Z"
|
|
result = classify(rows, scan=scan)
|
|
stale = [
|
|
m
|
|
for m in result["stale_members"]
|
|
if m["liveness"] == mfi.LIVENESS_PID_RECYCLED
|
|
]
|
|
self.assertEqual(len(stale), 1)
|
|
self.assertEqual(len(result["running_members"]), 4)
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
|
|
class TestIncompleteEvidenceFailsClosed(_AliveMixin, unittest.TestCase):
|
|
"""AC9: unknown or unavailable evidence produces a fail-closed result."""
|
|
|
|
def test_unavailable_process_listing_fails_closed(self):
|
|
rows = healthy_rows()
|
|
result = classify(
|
|
rows,
|
|
scan={"available": False, "processes": [], "reason": "ps unavailable"},
|
|
)
|
|
self.assertFalse(result["inventory_complete"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
self.assertIn("ps unavailable", result["incomplete_reasons"])
|
|
self.assertEqual(result["running_members"], [])
|
|
|
|
def test_unreadable_registry_fails_closed(self):
|
|
result = classify(
|
|
[],
|
|
scan=scan_for([]),
|
|
registry_available=False,
|
|
registry_error="registry unreadable",
|
|
)
|
|
self.assertFalse(result["inventory_complete"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
self.assertIn("registry unreadable", result["incomplete_reasons"])
|
|
|
|
def test_unregistered_running_process_fails_closed(self):
|
|
rows = healthy_rows()
|
|
result = classify(rows, scan=scan_for(rows, extra_pids=[59999]))
|
|
self.assertEqual([p["pid"] for p in result["unregistered_processes"]], [59999])
|
|
self.assertFalse(result["inventory_complete"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
self.assertTrue(
|
|
any("59999" in reason for reason in result["incomplete_reasons"])
|
|
)
|
|
|
|
def test_undeterminable_pid_liveness_is_unknown_not_healthy(self):
|
|
rows = healthy_rows()
|
|
self.unknown_pids = {rows[0]["pid"]}
|
|
result = classify(rows)
|
|
self.assertEqual(result["stale_members"][0]["liveness"], mfi.LIVENESS_UNKNOWN)
|
|
self.assertEqual(result["stale_members"][0]["health"], mfi.HEALTH_UNKNOWN)
|
|
self.assertFalse(result["inventory_complete"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
def test_unknown_startup_revision_fails_closed(self):
|
|
rows = healthy_rows()
|
|
rows[0]["startup_head"] = None
|
|
result = classify(rows)
|
|
self.assertFalse(result["inventory_complete"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
self.assertTrue(
|
|
any("startup revision" in r for r in result["incomplete_reasons"])
|
|
)
|
|
|
|
def test_unknown_is_distinguished_from_healthy(self):
|
|
rows = healthy_rows()
|
|
for r in rows:
|
|
r["cohort_id"] = None
|
|
result = classify(rows)
|
|
# Not "unhealthy" in the sense of a named defect: nothing is missing,
|
|
# duplicated or unexpected. It is *unknown*, and that still fails closed.
|
|
self.assertEqual(result["missing_members"], [])
|
|
self.assertEqual(result["duplicate_members"], [])
|
|
self.assertEqual(result["unexpected_members"], [])
|
|
self.assertIsNone(result["single_cohort"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
|
|
class TestBindingAndRoleConsistency(_AliveMixin, unittest.TestCase):
|
|
"""Repository-binding and role/profile mismatches fail closed."""
|
|
|
|
def test_repository_binding_mismatch_is_reported(self):
|
|
rows = healthy_rows()
|
|
rows[0]["repo"] = "Some-Other-Repo"
|
|
result = classify(rows)
|
|
self.assertEqual(len(result["repository_binding_mismatches"]), 1)
|
|
self.assertEqual(
|
|
result["repository_binding_mismatches"][0]["repo"], "Some-Other-Repo"
|
|
)
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
def test_incomplete_repository_binding_is_reported(self):
|
|
rows = healthy_rows()
|
|
rows[0]["org"] = None
|
|
result = classify(rows)
|
|
self.assertEqual(len(result["repository_binding_mismatches"]), 1)
|
|
self.assertIn("complete", result["repository_binding_mismatches"][0]["reason"])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
def test_role_mismatch_is_reported(self):
|
|
rows = healthy_rows()
|
|
rows[0]["role"] = "merger" # prgs-author is configured as author
|
|
result = classify(rows)
|
|
self.assertEqual(len(result["role_mismatches"]), 1)
|
|
self.assertEqual(result["role_mismatches"][0]["expected_role"], "author")
|
|
self.assertEqual(result["role_mismatches"][0]["declared_role"], "merger")
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
def test_profile_served_from_the_wrong_namespace_is_reported(self):
|
|
rows = healthy_rows()
|
|
rows[0]["namespace"] = "gitea-reviewer"
|
|
result = classify(rows)
|
|
self.assertTrue(
|
|
any(
|
|
m.get("expected_namespace") == "gitea-author"
|
|
for m in result["role_mismatches"]
|
|
)
|
|
)
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
def test_matching_binding_produces_no_mismatch(self):
|
|
result = classify(healthy_rows())
|
|
self.assertEqual(result["repository_binding_mismatches"], [])
|
|
self.assertEqual(result["role_mismatches"], [])
|
|
|
|
|
|
class TestDeterminism(_AliveMixin, unittest.TestCase):
|
|
"""AC10 / stable ordering and deterministic structured output."""
|
|
|
|
def test_identical_snapshots_produce_identical_results(self):
|
|
rows = healthy_rows()
|
|
first = classify(rows, scan=scan_for(rows))
|
|
second = classify(healthy_rows(), scan=scan_for(healthy_rows()))
|
|
self.assertEqual(first, second)
|
|
|
|
def test_row_order_does_not_change_the_result(self):
|
|
rows = healthy_rows()
|
|
shuffled = list(reversed(healthy_rows()))
|
|
forward = classify(rows, scan=scan_for(rows))
|
|
backward = classify(shuffled, scan=scan_for(shuffled))
|
|
self.assertEqual(forward["running_members"], backward["running_members"])
|
|
self.assertEqual(forward["configured_members"], backward["configured_members"])
|
|
self.assertEqual(
|
|
forward["mutation_gate_satisfied"], backward["mutation_gate_satisfied"]
|
|
)
|
|
|
|
def test_members_are_sorted_by_namespace_then_profile_then_pid(self):
|
|
rows = healthy_rows()
|
|
rows.append(row("gitea-author", "prgs-author", "author", 40001))
|
|
result = classify(rows)
|
|
keys = [
|
|
(m["namespace"], m["profile"], m["pid"]) for m in result["running_members"]
|
|
]
|
|
self.assertEqual(keys, sorted(keys))
|
|
|
|
def test_answering_namespace_does_not_change_the_verdict(self):
|
|
"""AC10: controller and reconciler agree for one fleet snapshot."""
|
|
rows = healthy_rows()
|
|
controller = classify(
|
|
rows, scan=scan_for(rows), answering_namespace="gitea-controller"
|
|
)
|
|
reconciler = classify(
|
|
healthy_rows(),
|
|
scan=scan_for(healthy_rows()),
|
|
answering_namespace="gitea-reconciler",
|
|
)
|
|
self.assertEqual(controller["answering_namespace"], "gitea-controller")
|
|
self.assertEqual(reconciler["answering_namespace"], "gitea-reconciler")
|
|
for key in sorted(set(controller) - {"answering_namespace"}):
|
|
self.assertEqual(controller[key], reconciler[key], f"{key} disagreed")
|
|
|
|
def test_controller_and_reconciler_agree_on_an_unhealthy_fleet(self):
|
|
rows = [r for r in healthy_rows() if r["profile"] != "prgs-merger"]
|
|
controller = classify(
|
|
rows, scan=scan_for(rows), answering_namespace="gitea-controller"
|
|
)
|
|
reconciler = classify(
|
|
rows, scan=scan_for(rows), answering_namespace="gitea-reconciler"
|
|
)
|
|
self.assertEqual(controller["blocked_reason"], reconciler["blocked_reason"])
|
|
self.assertEqual(
|
|
controller["mutation_gate_satisfied"],
|
|
reconciler["mutation_gate_satisfied"],
|
|
)
|
|
|
|
|
|
class TestReadOnly(_AliveMixin, unittest.TestCase):
|
|
"""AC11: the capability mutates nothing."""
|
|
|
|
def test_classification_reports_no_mutations(self):
|
|
result = classify(healthy_rows())
|
|
self.assertEqual(result["mutations_performed"], [])
|
|
self.assertTrue(result["read_only"])
|
|
|
|
def test_classification_does_not_write_the_input_rows_back(self):
|
|
rows = healthy_rows()
|
|
snapshot = [dict(r) for r in rows]
|
|
classify(rows)
|
|
self.assertEqual(rows, snapshot)
|
|
|
|
|
|
class TestLivenessProbe(unittest.TestCase):
|
|
"""AC11: the real probe never sends a terminating signal.
|
|
|
|
Deliberately *not* using ``_AliveMixin`` — these tests exercise
|
|
``probe_pid_alive`` itself, which the mixin replaces.
|
|
"""
|
|
|
|
def test_only_signal_zero_is_ever_sent(self):
|
|
with mock.patch("os.kill") as killer:
|
|
mfi.probe_pid_alive(4242)
|
|
killer.assert_called_once_with(4242, 0)
|
|
|
|
def test_classifying_a_duplicate_never_terminates_it(self):
|
|
rows = healthy_rows()
|
|
rows.append(row("gitea-author", "prgs-author", "author", 49999))
|
|
with mock.patch("os.kill") as killer:
|
|
result = mfi.classify_fleet_inventory(
|
|
runtime_rows=rows,
|
|
process_scan=scan_for(rows),
|
|
expected_binding=BINDING,
|
|
now=NOW,
|
|
)
|
|
self.assertTrue(killer.call_args_list, "liveness must actually be probed")
|
|
for call in killer.call_args_list:
|
|
self.assertEqual(call.args[1], 0, "only signal 0 may ever be sent")
|
|
self.assertEqual(result["mutations_performed"], [])
|
|
|
|
def test_liveness_probe_tolerates_a_missing_process(self):
|
|
with mock.patch("os.kill", side_effect=ProcessLookupError):
|
|
self.assertFalse(mfi.probe_pid_alive(4242))
|
|
|
|
def test_liveness_probe_treats_permission_error_as_alive(self):
|
|
with mock.patch("os.kill", side_effect=PermissionError):
|
|
self.assertTrue(mfi.probe_pid_alive(4242))
|
|
|
|
def test_liveness_probe_returns_unknown_on_other_os_errors(self):
|
|
with mock.patch("os.kill", side_effect=OSError):
|
|
self.assertIsNone(mfi.probe_pid_alive(4242))
|
|
|
|
def test_invalid_pid_is_unknown_and_probes_nothing(self):
|
|
with mock.patch("os.kill") as killer:
|
|
self.assertIsNone(mfi.probe_pid_alive(None))
|
|
self.assertIsNone(mfi.probe_pid_alive(0))
|
|
killer.assert_not_called()
|
|
|
|
|
|
class TestMultiClientRegression(_AliveMixin, unittest.TestCase):
|
|
"""The multi-LLM duplicate-server scenario that motivated #949."""
|
|
|
|
@staticmethod
|
|
def _two_client_rows():
|
|
first = healthy_rows()
|
|
second = [
|
|
row(
|
|
entry["namespace"],
|
|
entry["profile"],
|
|
entry["role"],
|
|
50000 + index,
|
|
cohort_id=COHORT_B,
|
|
)
|
|
for index, entry in enumerate(mfi.EXPECTED_PRGS_FLEET)
|
|
]
|
|
return first + second
|
|
|
|
def test_second_client_running_the_same_five_profiles_is_caught(self):
|
|
"""Two clients, ten servers, same profiles, same revision.
|
|
|
|
Every member self-reports the same parity, which is exactly why the old
|
|
per-process surfaces reported success. The fleet inventory must report
|
|
five duplicates, two cohorts, and a closed gate.
|
|
"""
|
|
rows = self._two_client_rows()
|
|
result = classify(rows, scan=scan_for(rows))
|
|
|
|
self.assertEqual(len(result["duplicate_members"]), 5)
|
|
self.assertFalse(result["exactly_one_per_profile"])
|
|
self.assertTrue(result["mixed_cohort"])
|
|
self.assertFalse(result["single_cohort"])
|
|
self.assertFalse(result["mixed_revision"], "both clients share a revision")
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
self.assertEqual(result["missing_members"], [])
|
|
|
|
def test_identical_parity_across_ten_servers_is_not_health(self):
|
|
rows = self._two_client_rows()
|
|
result = classify(rows, scan=scan_for(rows))
|
|
self.assertEqual(result["startup_revisions"], [HEAD_A])
|
|
self.assertFalse(result["mutation_gate_satisfied"])
|
|
|
|
def test_every_duplicate_pid_is_named_for_the_operator(self):
|
|
rows = self._two_client_rows()
|
|
result = classify(rows, scan=scan_for(rows))
|
|
for duplicate in result["duplicate_members"]:
|
|
self.assertEqual(len(duplicate["pids"]), 2)
|
|
|
|
|
|
class TestProcessScan(unittest.TestCase):
|
|
"""The process observation is server-side and fails closed."""
|
|
|
|
def test_scan_parses_mcp_server_processes(self):
|
|
stdout = (
|
|
" PID STARTED COMMAND\n"
|
|
" 41000 Mon Jul 27 20:00:00 2026 python /path/mcp_server.py\n"
|
|
" 41001 Mon Jul 27 20:00:01 2026 python /path/other_server.py\n"
|
|
)
|
|
result = mfi.scan_mcp_server_processes(
|
|
runner=lambda *a, **k: mock.Mock(stdout=stdout)
|
|
)
|
|
self.assertTrue(result["available"])
|
|
self.assertEqual([p["pid"] for p in result["processes"]], [41000])
|
|
|
|
def test_scan_failure_reports_unavailable_rather_than_empty(self):
|
|
def boom(*args, **kwargs):
|
|
raise OSError("ps missing")
|
|
|
|
result = mfi.scan_mcp_server_processes(runner=boom)
|
|
self.assertFalse(result["available"])
|
|
self.assertEqual(result["processes"], [])
|
|
self.assertIn("ps missing", result["reason"])
|
|
|
|
def test_scan_results_are_sorted_by_pid(self):
|
|
stdout = (
|
|
" PID STARTED COMMAND\n"
|
|
" 41005 Mon Jul 27 20:00:00 2026 python /path/mcp_server.py\n"
|
|
" 41001 Mon Jul 27 20:00:01 2026 python /path/mcp_server.py\n"
|
|
)
|
|
result = mfi.scan_mcp_server_processes(
|
|
runner=lambda *a, **k: mock.Mock(stdout=stdout)
|
|
)
|
|
self.assertEqual([p["pid"] for p in result["processes"]], [41001, 41005])
|
|
|
|
|
|
class TestRuntimeRecord(unittest.TestCase):
|
|
"""The row a server writes about itself."""
|
|
|
|
def test_record_describes_the_calling_process(self):
|
|
record = mfi.build_process_runtime_record(
|
|
namespace="gitea-controller",
|
|
profile="prgs-controller",
|
|
role="controller",
|
|
remote="prgs",
|
|
org=BINDING["org"],
|
|
repo=BINDING["repo"],
|
|
pid=41022,
|
|
startup_head=HEAD_A,
|
|
transport="stdio",
|
|
client_provenance="client_managed",
|
|
env={mfi.COHORT_ID_ENV: COHORT_A},
|
|
boot_id="deadbeefcafe0001",
|
|
)
|
|
self.assertEqual(
|
|
record["runtime_id"], "gitea-controller:41022:deadbeefcafe0001"
|
|
)
|
|
self.assertEqual(record["namespace"], "gitea-controller")
|
|
self.assertEqual(record["cohort_id"], COHORT_A)
|
|
self.assertEqual(record["cohort_source"], "explicit_env")
|
|
self.assertEqual(record["startup_head"], HEAD_A)
|
|
self.assertEqual(record["status"], "running")
|
|
|
|
def test_record_defaults_pid_to_the_current_process(self):
|
|
record = mfi.build_process_runtime_record(
|
|
namespace="gitea-author", profile="prgs-author", role="author"
|
|
)
|
|
self.assertEqual(record["pid"], os.getpid())
|
|
|
|
def test_each_boot_gets_a_distinct_runtime_id(self):
|
|
first = mfi.build_process_runtime_record(
|
|
namespace="gitea-author", profile="prgs-author", role="author", pid=1
|
|
)
|
|
second = mfi.build_process_runtime_record(
|
|
namespace="gitea-author", profile="prgs-author", role="author", pid=1
|
|
)
|
|
self.assertNotEqual(first["runtime_id"], second["runtime_id"])
|
|
|
|
def test_registered_at_uses_the_control_plane_timestamp_format(self):
|
|
record = mfi.build_process_runtime_record(
|
|
namespace="gitea-author", profile="prgs-author", role="author", pid=1
|
|
)
|
|
datetime.strptime(record["registered_at"], "%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
class TestSummary(_AliveMixin, unittest.TestCase):
|
|
def test_healthy_summary_names_the_counts(self):
|
|
result = classify(healthy_rows())
|
|
self.assertIn("5 of 5", mfi.summarize(result))
|
|
|
|
def test_blocked_summary_repeats_the_blocked_reason(self):
|
|
rows = [r for r in healthy_rows() if r["profile"] != "prgs-merger"]
|
|
result = classify(rows)
|
|
self.assertIn(result["blocked_reason"], mfi.summarize(result))
|
|
|
|
|
|
class TestHeartbeatAge(_AliveMixin, unittest.TestCase):
|
|
def test_heartbeat_age_is_reported_for_diagnosis(self):
|
|
rows = healthy_rows()
|
|
stamp = (NOW - timedelta(minutes=30)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
rows[0]["last_heartbeat_at"] = stamp
|
|
result = classify(rows)
|
|
member = [m for m in result["running_members"] if m["pid"] == rows[0]["pid"]][0]
|
|
self.assertEqual(member["heartbeat_age_seconds"], 1800)
|
|
|
|
def test_missing_heartbeat_is_reported_as_unknown_age(self):
|
|
rows = healthy_rows()
|
|
rows[0]["last_heartbeat_at"] = None
|
|
result = classify(rows)
|
|
member = [m for m in result["running_members"] if m["pid"] == rows[0]["pid"]][0]
|
|
self.assertIsNone(member["heartbeat_age_seconds"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|