From 5494696227b84148b374412e9b38d58c1eccfca5 Mon Sep 17 00:00:00 2001 From: jcwalker3 Date: Wed, 22 Jul 2026 15:59:30 -0500 Subject: [PATCH] feat(webui): read-only system-health API (Closes #634) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `GET /api/v1/system/health`, a structured read-only health surface for automated readiness checks, and keeps `/health` as the cheap liveness probe. webui/system_health.py composes a DTO from fail-soft dependency probes: the control-plane database, the local checkout, and — opt-in via `?deep=1` — live Gitea reachability, each carrying status, reason, and probe latency. Required probes drive readiness; the optional Gitea probe can only degrade overall status, because local inventory stays serveable when the remote is unreachable. A probe that did not run leaves readiness incomplete rather than silently passing. Read-only throughout: the control-plane database is opened through a `mode=ro` URI because `ControlPlaneDB.__init__` creates directories and runs migrations, which a health check must never do. No restart or reload control is exposed; those are Phase 2 and #630 forbids process-kill recovery. No unproven claims: `stale_runtime.mutation_safe` is true only when the runtime, checkout, and remote commits are all known and equal, and MCP namespaces always report `unproven` because a web process cannot exercise the IDE-managed client path (#543). Probe details are redacted at the browser boundary — URLs lose userinfo and query strings, credential-shaped text is masked. `/health` is expanded additively: every MVP key is retained, plus `started_at`, `uptime_seconds`, and a pointer to the versioned API. The versioned route returns 503 when not ready so automation can branch on the status code alone. Verified at master 9eb0f29: focused file 40 passed / 11 subtests; `-k "webui or health"` 230 passed / 159 subtests; full suite 4358 passed with the 11 pre-existing master-drift failures unchanged from the clean-master baseline. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/webui-local-dev.md | 82 +++- tests/test_webui_system_health.py | 499 ++++++++++++++++++++++ webui/app.py | 34 ++ webui/system_health.py | 682 ++++++++++++++++++++++++++++++ 4 files changed, 1296 insertions(+), 1 deletion(-) create mode 100644 tests/test_webui_system_health.py create mode 100644 webui/system_health.py diff --git a/docs/webui-local-dev.md b/docs/webui-local-dev.md index a92e509..eacc739 100644 --- a/docs/webui-local-dev.md +++ b/docs/webui-local-dev.md @@ -48,7 +48,8 @@ that govern when a write path may open (#632, epic #631). | Path | Description | |------|-------------| | `/` | Home / operator overview | -| `/health` | JSON liveness (`status`, `service`, `mode`, `timestamp`) | +| `/health` | JSON liveness (`status`, `service`, `mode`, `timestamp`, `uptime_seconds`) | +| `/api/v1/system/health` | Structured read-only system health (#634) | | `/queue` | Live PR and issue queue dashboard (#429) | | `/api/queue` | JSON queue export with pagination metadata | | `/projects` | Project registry list (#427) | @@ -72,6 +73,85 @@ Most routes are GET-only. POST/PUT/PATCH/DELETE return `405` with `read-only-mvp`, except `/audit` and `/api/audit` which accept POST for local validator preview only (no Gitea mutations, no server-side storage). +## System health API (#634) + +`GET /api/v1/system/health` is the structured, read-only health surface for +automated readiness checks. It is the first console API under the `/api/v1` +prefix; the unversioned MVP exports remain as compatibility aliases. + +`/health` is unchanged for existing consumers — every MVP key is still present +— and now also carries `started_at`, `uptime_seconds`, and a +`system_health_api` pointer. It stays deliberately cheap and runs no dependency +probe, because answering readiness costs real work. + +**Status codes.** `200` when ready, `503` when a required dependency failed or +was never probed. Automation can branch on the code without parsing the body. + +**Query flags.** The Gitea check is a network call, so it is opt-in: +`GET /api/v1/system/health?deep=1` runs it and caches the result for +`WEBUI_HEALTH_PROBE_TTL_SECONDS` (default 15s) so dashboard polling does not +amplify into remote load. Without the flag that probe reports `skipped`. + +**Dependencies.** `control_plane_db` and `repository` are required and drive +readiness. `gitea` is optional: when it fails the overall `status` degrades but +`readiness.ready` stays true, because local inventory is still serveable. Each +entry carries `status`, `detail`, `required`, and `latency_ms`. + +Two honesty rules are worth knowing before reading the payload: + +* `stale_runtime.mutation_safe` is true only when the runtime, checkout, and + remote-tracking commits are all known and equal. An unfetched remote is + reported as indeterminate, never as safe. +* `mcp_namespaces` entries are always `unproven`. A web process runs outside + the IDE-managed MCP client and cannot invoke a namespace tool, so per #543 + only a `client_namespace` probe can prove that path. + +Sample response (abridged, healthy): + +```json +{ + "status": "ok", + "service": "mcp-control-plane-webui", + "mode": "read-only", + "api": "/api/v1/system/health", + "timestamp": "2026-07-22T11:04:18.512034+00:00", + "readiness": { "ready": true, "complete": true, "reasons": [] }, + "version": { + "git_sha": "620ed6e9a9550b8da2ceb82d9ab8744e8920490f", + "git_describe": "v1.1.0-898-g620ed6e", + "control_plane_schema_version": 4, + "python_version": "3.14.5", + "known": true + }, + "process": { "started_at": "2026-07-22T10:58:02.114+00:00", "uptime_seconds": 376.4 }, + "deep_probes_requested": false, + "dependencies": [ + { + "name": "control_plane_db", + "kind": "sqlite", + "status": "ok", + "detail": "schema v4 readable", + "required": true, + "healthy": true, + "latency_ms": 1.482, + "metadata": { "schema_version": 4, "active_leases": 3 } + }, + { "name": "repository", "kind": "git", "status": "ok", "required": true, "healthy": true }, + { "name": "gitea", "kind": "http", "status": "skipped", "required": false, "healthy": false } + ], + "mcp_namespaces": [ + { "namespace": "gitea-author", "required_tool": "gitea_whoami", "status": "unproven" } + ], + "stale_runtime": { "stale": false, "determinable": true, "mutation_safe": true, "reasons": [] }, + "probe_errors": [] +} +``` + +No restart, reload, or process-kill control is exposed here: those are Phase 2 +at the earliest, and #630 forbids process-kill recovery outright. Every probe +opens its subject read-only — the control-plane database is opened through a +`mode=ro` URI so a health check can never create or migrate a schema. + ## Report audit (#431) Paste an LLM final report at `/audit` or POST JSON to `/api/audit`. The UI diff --git a/tests/test_webui_system_health.py b/tests/test_webui_system_health.py new file mode 100644 index 0000000..69cd1dd --- /dev/null +++ b/tests/test_webui_system_health.py @@ -0,0 +1,499 @@ +"""Tests for the read-only system-health API (#634). + +Covers the acceptance criteria directly: a structured payload with readiness +and a dependency list (AC1), version and uptime when knowable (AC2), stale +runtime reported without a false mutation-safe claim (AC3), and the healthy / +degraded-dependency / redaction cases (AC4). +""" +import json +import os +import sqlite3 +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from starlette.testclient import TestClient + +import control_plane_db +from webui.app import create_app +from webui.deployment_boundary import scan_text_for_client_secrets +from webui.system_health import ( + API_PATH, + STATUS_DEGRADED, + STATUS_DOWN, + STATUS_OK, + STATUS_SKIPPED, + DependencyProbe, + StaleRuntime, + assess_stale_runtime, + clear_probe_cache, + load_system_health, + namespace_summaries, + probe_control_plane_db, + probe_gitea, + process_uptime, + redact, + redact_url, + snapshot_to_dict, +) + + +def _probe(name, status, *, required=True, detail="detail", kind="test"): + return DependencyProbe( + name=name, + kind=kind, + status=status, + detail=detail, + required=required, + latency_ms=1.5, + metadata={}, + ) + + +_ALL_HEALTHY = ( + _probe("control_plane_db", STATUS_OK, kind="sqlite"), + _probe("repository", STATUS_OK, kind="git"), + _probe("gitea", STATUS_OK, required=False, kind="http"), +) + +_CLEAN_PARITY = StaleRuntime( + daemon_head="abc123", + checkout_head="abc123", + remote_head="abc123", + stale=False, + determinable=True, + mutation_safe=True, + reasons=(), +) + + +class CleanParityMixin: + """Pin parity for tests about aggregation rather than staleness. + + Without this the assertions depend on the real checkout: a worktree whose + branch is ahead of its upstream is genuinely stale, which would degrade the + overall status and make these cases fail for an unrelated reason. + """ + + def setUp(self): + super().setUp() + patcher = mock.patch( + "webui.system_health.assess_stale_runtime", + return_value=_CLEAN_PARITY, + ) + patcher.start() + self.addCleanup(patcher.stop) + + +class TestDependencyAggregation(CleanParityMixin, unittest.TestCase): + """AC1 — readiness and dependency list derived from probe results.""" + + def test_all_healthy_is_ok_and_ready(self): + snapshot = load_system_health(probes=_ALL_HEALTHY, daemon_head="abc123") + self.assertEqual(snapshot.status, STATUS_OK) + self.assertTrue(snapshot.ready) + self.assertTrue(snapshot.readiness_complete) + self.assertEqual(snapshot.readiness_reasons, ()) + self.assertEqual(len(snapshot.dependencies), 3) + + def test_required_dependency_down_blocks_readiness(self): + probes = ( + _probe("control_plane_db", STATUS_DOWN, detail="file missing", kind="sqlite"), + _probe("repository", STATUS_OK, kind="git"), + _probe("gitea", STATUS_OK, required=False, kind="http"), + ) + snapshot = load_system_health(probes=probes, daemon_head="abc123") + self.assertEqual(snapshot.status, STATUS_DOWN) + self.assertFalse(snapshot.ready) + self.assertTrue( + any("control_plane_db" in reason for reason in snapshot.readiness_reasons) + ) + + def test_optional_dependency_down_degrades_but_stays_ready(self): + """A failing optional probe must not claim the process itself is unready.""" + probes = ( + _probe("control_plane_db", STATUS_OK, kind="sqlite"), + _probe("repository", STATUS_OK, kind="git"), + _probe("gitea", STATUS_DOWN, required=False, detail="timeout", kind="http"), + ) + snapshot = load_system_health(probes=probes, daemon_head="abc123") + self.assertEqual(snapshot.status, STATUS_DEGRADED) + self.assertTrue(snapshot.ready) + self.assertTrue(any("gitea" in reason for reason in snapshot.readiness_reasons)) + + def test_unrun_required_probe_leaves_readiness_incomplete(self): + """Not probed is not the same as passing.""" + probes = ( + _probe("control_plane_db", STATUS_OK, kind="sqlite"), + _probe("repository", STATUS_SKIPPED, detail="offline", kind="git"), + ) + snapshot = load_system_health(probes=probes, daemon_head="abc123") + self.assertFalse(snapshot.ready) + self.assertFalse(snapshot.readiness_complete) + self.assertEqual(snapshot.status, STATUS_DEGRADED) + + def test_skipped_optional_probe_does_not_block_readiness(self): + probes = ( + _probe("control_plane_db", STATUS_OK, kind="sqlite"), + _probe("repository", STATUS_OK, kind="git"), + _probe("gitea", STATUS_SKIPPED, required=False, kind="http"), + ) + snapshot = load_system_health(probes=probes, daemon_head="abc123") + self.assertTrue(snapshot.ready) + self.assertTrue(snapshot.readiness_complete) + + +class TestVersionAndUptime(CleanParityMixin, unittest.TestCase): + """AC2 — version and uptime present when knowable.""" + + def test_uptime_and_start_time_present(self): + snapshot = load_system_health(probes=_ALL_HEALTHY, daemon_head="abc123") + self.assertGreaterEqual(snapshot.uptime_seconds, 0.0) + self.assertIn("T", snapshot.started_at) + + def test_process_uptime_helper_matches_shape(self): + started_at, uptime = process_uptime() + self.assertIn("T", started_at) + self.assertGreaterEqual(uptime, 0.0) + + def test_version_reports_python_and_schema_version(self): + probes = ( + DependencyProbe( + name="control_plane_db", + kind="sqlite", + status=STATUS_OK, + detail="ok", + required=True, + latency_ms=1.0, + metadata={"schema_version": control_plane_db.SCHEMA_VERSION}, + ), + _probe("repository", STATUS_OK, kind="git"), + ) + snapshot = load_system_health(probes=probes, daemon_head="abc123") + self.assertEqual( + snapshot.version.control_plane_schema_version, + control_plane_db.SCHEMA_VERSION, + ) + self.assertTrue(snapshot.version.python_version) + + def test_version_known_flag_false_when_sha_unavailable(self): + with mock.patch("webui.system_health._git", return_value=None): + snapshot = load_system_health(probes=_ALL_HEALTHY, daemon_head="abc") + self.assertIsNone(snapshot.version.git_sha) + self.assertFalse(snapshot.version.known) + + +class TestStaleRuntime(unittest.TestCase): + """AC3 — stale runtime reflected without a false mutation-safe claim.""" + + def test_matching_commits_are_mutation_safe(self): + assessment = assess_stale_runtime( + Path("/tmp"), + daemon_head="aaa", + git_reader=lambda *args: "aaa", + ) + self.assertFalse(assessment.stale) + self.assertTrue(assessment.determinable) + self.assertTrue(assessment.mutation_safe) + + def test_diverged_commits_are_stale_and_not_mutation_safe(self): + reads = {"HEAD": "aaa", "@{upstream}": "bbb"} + assessment = assess_stale_runtime( + Path("/tmp"), + daemon_head="aaa", + git_reader=lambda *args: reads.get(args[-1]), + ) + self.assertTrue(assessment.stale) + self.assertFalse(assessment.mutation_safe) + self.assertTrue(assessment.reasons) + + def test_unknown_remote_is_not_mutation_safe(self): + """Indeterminate must never read as safe.""" + reads = {"HEAD": "aaa", "@{upstream}": None} + assessment = assess_stale_runtime( + Path("/tmp"), + daemon_head="aaa", + git_reader=lambda *args: reads.get(args[-1]), + ) + self.assertFalse(assessment.determinable) + self.assertFalse(assessment.mutation_safe) + self.assertFalse(assessment.stale) + self.assertTrue( + any("indeterminate" in reason for reason in assessment.reasons) + ) + + def test_unobservable_daemon_head_is_disclosed(self): + assessment = assess_stale_runtime( + Path("/tmp"), + git_reader=lambda *args: "aaa", + ) + self.assertTrue( + any("not observable" in reason for reason in assessment.reasons) + ) + + def test_stale_runtime_degrades_overall_status(self): + reads = {"HEAD": "aaa", "@{upstream}": "bbb"} + # Pinned rather than inherited: this path uses the default git reader, + # so the assertion must hold whether or not the suite runs offline. + with mock.patch.dict(os.environ, {"WEBUI_TEST_OFFLINE": ""}), mock.patch( + "webui.system_health._git", + side_effect=lambda repo, *args: reads.get(args[-1]), + ): + snapshot = load_system_health(probes=_ALL_HEALTHY, daemon_head="aaa") + self.assertTrue(snapshot.stale_runtime.stale) + self.assertFalse(snapshot.stale_runtime.mutation_safe) + self.assertEqual(snapshot.status, STATUS_DEGRADED) + + +class TestControlPlaneDbProbe(unittest.TestCase): + """The required local dependency, probed read-only.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.db_path = str(Path(self.tmp.name) / "control-plane.db") + + def _build_db(self, schema_version): + conn = sqlite3.connect(self.db_path) + conn.execute("CREATE TABLE schema_meta (key TEXT PRIMARY KEY, value TEXT)") + conn.execute("CREATE TABLE leases (lease_id TEXT PRIMARY KEY, status TEXT)") + conn.execute( + "INSERT INTO schema_meta(key, value) VALUES ('schema_version', ?)", + (str(schema_version),), + ) + conn.execute("INSERT INTO leases(lease_id, status) VALUES ('l1', 'active')") + conn.commit() + conn.close() + + def test_missing_database_is_down(self): + probe = probe_control_plane_db(str(Path(self.tmp.name) / "absent.db")) + self.assertEqual(probe.status, STATUS_DOWN) + self.assertTrue(probe.required) + self.assertIsNotNone(probe.latency_ms) + + def test_matching_schema_is_ok(self): + self._build_db(control_plane_db.SCHEMA_VERSION) + probe = probe_control_plane_db(self.db_path) + self.assertEqual(probe.status, STATUS_OK) + self.assertEqual( + probe.metadata["schema_version"], control_plane_db.SCHEMA_VERSION + ) + self.assertEqual(probe.metadata["active_leases"], 1) + + def test_mismatched_schema_is_degraded(self): + self._build_db(control_plane_db.SCHEMA_VERSION + 99) + probe = probe_control_plane_db(self.db_path) + self.assertEqual(probe.status, STATUS_DEGRADED) + + def test_probe_does_not_create_a_database(self): + """A health check must never initialise the substrate it inspects.""" + absent = str(Path(self.tmp.name) / "never-created.db") + probe_control_plane_db(absent) + self.assertFalse(Path(absent).exists()) + + def test_unreadable_database_is_down_not_raised(self): + Path(self.db_path).write_text("this is not a sqlite database") + probe = probe_control_plane_db(self.db_path) + self.assertEqual(probe.status, STATUS_DOWN) + + +class TestRedaction(unittest.TestCase): + """AC4 — redaction. No credential-shaped text crosses the boundary.""" + + def test_redacts_token_assignment(self): + cleaned = redact("failed with token=ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345") + self.assertNotIn("ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345", cleaned) + self.assertIn("[redacted]", cleaned) + + def test_redacts_authorization_header_text(self): + cleaned = redact("Authorization: Bearer abcdefghijklmnopqrstuvwxyz123456") + self.assertNotIn("abcdefghijklmnopqrstuvwxyz123456", cleaned) + + def test_redacts_long_opaque_strings(self): + cleaned = redact("value 0123456789abcdef0123456789abcdef here") + self.assertNotIn("0123456789abcdef0123456789abcdef", cleaned) + + def test_url_userinfo_and_query_are_stripped(self): + cleaned = redact_url("https://user:secretpass@gitea.example.com/api/v1?token=xyz") + self.assertNotIn("secretpass", cleaned) + self.assertNotIn("token=xyz", cleaned) + self.assertEqual(cleaned, "https://gitea.example.com/api/v1") + + def test_url_inside_free_text_is_redacted(self): + cleaned = redact("GET https://u:p@host.example.com/x?token=abc failed") + self.assertNotIn("u:p@", cleaned) + self.assertNotIn("token=abc", cleaned) + + def test_gitea_probe_failure_detail_is_redacted(self): + boom = RuntimeError( + "connection refused for https://user:hunter2@gitea.example.com/api/v1/version" + ) + with mock.patch("webui.system_health.get_auth_header", return_value="token x"), \ + mock.patch("webui.system_health.api_request", side_effect=boom): + probe = probe_gitea("gitea.example.com") + self.assertEqual(probe.status, STATUS_DOWN) + self.assertNotIn("hunter2", probe.detail) + self.assertEqual(scan_text_for_client_secrets(probe.detail), []) + + def test_credential_guard_refusal_is_a_status_not_a_crash(self): + with mock.patch( + "webui.system_health.get_auth_header", + side_effect=RuntimeError("daemon guard refused"), + ): + probe = probe_gitea("gitea.example.com") + self.assertEqual(probe.status, STATUS_DEGRADED) + self.assertFalse(probe.required) + + +class TestNamespaceSummaries(unittest.TestCase): + """A web process cannot prove IDE namespace health, and must not claim to.""" + + def test_every_namespace_reports_unproven(self): + rows = namespace_summaries() + self.assertTrue(rows) + for row in rows: + with self.subTest(namespace=row["namespace"]): + self.assertEqual(row["status"], "unproven") + self.assertFalse(row["ide_namespace_proven"]) + self.assertIn("client_namespace", row["reason"]) + + +class TestSystemHealthRoutes(CleanParityMixin, unittest.TestCase): + """The HTTP surface: versioned path, status codes, read-only guard.""" + + def setUp(self): + super().setUp() + clear_probe_cache() + self.addCleanup(clear_probe_cache) + self.client = TestClient(create_app()) + + def _patch_snapshot(self, probes, daemon_head="abc123"): + snapshot = load_system_health(probes=probes, daemon_head=daemon_head) + patcher = mock.patch( + "webui.app.load_system_health", + return_value=snapshot, + ) + patcher.start() + self.addCleanup(patcher.stop) + return snapshot + + def test_versioned_route_is_registered(self): + self.assertEqual(API_PATH, "/api/v1/system/health") + self._patch_snapshot(_ALL_HEALTHY) + response = self.client.get(API_PATH) + self.assertEqual(response.status_code, 200) + + def test_healthy_payload_shape(self): + self._patch_snapshot(_ALL_HEALTHY) + data = self.client.get(API_PATH).json() + self.assertEqual(data["status"], STATUS_OK) + self.assertTrue(data["readiness"]["ready"]) + self.assertTrue(data["readiness"]["complete"]) + self.assertEqual(data["api"], API_PATH) + self.assertEqual(len(data["dependencies"]), 3) + for key in ("version", "process", "stale_runtime", "mcp_namespaces"): + self.assertIn(key, data) + self.assertIn("uptime_seconds", data["process"]) + self.assertIn("mutation_safe", data["stale_runtime"]) + + def test_degraded_dependency_returns_503(self): + probes = ( + _probe("control_plane_db", STATUS_DOWN, detail="missing", kind="sqlite"), + _probe("repository", STATUS_OK, kind="git"), + ) + self._patch_snapshot(probes) + response = self.client.get(API_PATH) + self.assertEqual(response.status_code, 503) + data = response.json() + self.assertFalse(data["readiness"]["ready"]) + self.assertTrue(data["readiness"]["reasons"]) + + def test_dependency_entries_expose_status_and_latency(self): + self._patch_snapshot(_ALL_HEALTHY) + data = self.client.get(API_PATH).json() + names = {entry["name"] for entry in data["dependencies"]} + self.assertEqual(names, {"control_plane_db", "repository", "gitea"}) + for entry in data["dependencies"]: + with self.subTest(dependency=entry["name"]): + self.assertIn("status", entry) + self.assertIn("required", entry) + self.assertIn("latency_ms", entry) + + def test_response_body_carries_no_client_secrets(self): + self._patch_snapshot(_ALL_HEALTHY) + body = self.client.get(API_PATH).text + self.assertEqual(scan_text_for_client_secrets(body), []) + + def test_deep_flag_is_forwarded(self): + snapshot = load_system_health(probes=_ALL_HEALTHY, daemon_head="abc") + with mock.patch( + "webui.app.load_system_health", return_value=snapshot + ) as loader: + self.client.get(f"{API_PATH}?deep=1") + loader.assert_called_once_with(deep=True) + + def test_shallow_is_the_default(self): + snapshot = load_system_health(probes=_ALL_HEALTHY, daemon_head="abc") + with mock.patch( + "webui.app.load_system_health", return_value=snapshot + ) as loader: + self.client.get(API_PATH) + loader.assert_called_once_with(deep=False) + + def test_route_rejects_mutation_methods(self): + for method in ("POST", "PUT", "PATCH", "DELETE"): + with self.subTest(method=method): + response = self.client.request(method, API_PATH) + self.assertEqual(response.status_code, 405) + self.assertEqual(response.json()["error"], "read-only-mvp") + + def test_default_shallow_call_skips_the_network_probe(self): + """The expensive probe must not run unless it was asked for.""" + with mock.patch("webui.system_health.probe_gitea") as probe: + snapshot = load_system_health(deep=False) + probe.assert_not_called() + gitea = next(p for p in snapshot.dependencies if p.name == "gitea") + self.assertEqual(gitea.status, STATUS_SKIPPED) + + +class TestHealthRouteBackwardCompatibility(unittest.TestCase): + """`/health` is expanded additively; MVP consumers must keep working.""" + + def setUp(self): + self.client = TestClient(create_app()) + + def test_mvp_keys_are_unchanged(self): + data = self.client.get("/health").json() + self.assertEqual(data["status"], "ok") + self.assertEqual(data["service"], "mcp-control-plane-webui") + self.assertEqual(data["mode"], "read-only-mvp") + self.assertIn("timestamp", data) + self.assertEqual(data["deployment"]["mode"], "internal-operator-console") + + def test_health_points_at_the_versioned_api(self): + data = self.client.get("/health").json() + self.assertEqual(data["system_health_api"], API_PATH) + self.assertIn("uptime_seconds", data) + self.assertIn("started_at", data) + + def test_health_runs_no_dependency_probe(self): + """Liveness must stay cheap: no probe, no snapshot assembly.""" + with mock.patch("webui.app.load_system_health") as loader: + response = self.client.get("/health") + self.assertEqual(response.status_code, 200) + loader.assert_not_called() + + +class TestSnapshotSerialisation(CleanParityMixin, unittest.TestCase): + def test_snapshot_dict_is_json_serialisable(self): + snapshot = load_system_health(probes=_ALL_HEALTHY, daemon_head="abc123") + encoded = json.dumps(snapshot_to_dict(snapshot)) + self.assertIn("readiness", encoded) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/app.py b/webui/app.py index d0f832b..021b0db 100644 --- a/webui/app.py +++ b/webui/app.py @@ -29,6 +29,12 @@ from webui.worktree_scanner import load_hygiene_snapshot, snapshot_to_dict as wo from webui.worktree_views import render_worktrees_page from webui.runtime_health import load_runtime_snapshot, snapshot_to_dict as runtime_snapshot_to_dict from webui.runtime_views import render_runtime_page +from webui.system_health import ( + API_PATH as SYSTEM_HEALTH_API_PATH, + load_system_health, + process_uptime, + snapshot_to_dict as system_health_to_dict, +) _READ_ONLY_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) _AUDIT_MUTATION_PATHS = frozenset({"/audit", "/api/audit"}) @@ -62,16 +68,43 @@ async def home(_request: Request) -> HTMLResponse: async def health(_request: Request) -> JSONResponse: + """Liveness only — deliberately cheap, runs no dependency probe (#634). + + Every MVP key is retained so existing pollers keep working; the additions + are a pointer to the structured API and the in-memory process uptime. + Readiness lives at that API because answering it costs real probes. + """ bind_host = _request.app.state.webui_bind_host + started_at, uptime_seconds = process_uptime() return JSONResponse({ "status": "ok", "service": "mcp-control-plane-webui", "mode": "read-only-mvp", "timestamp": datetime.now(timezone.utc).isoformat(), "deployment": deployment_snapshot(bind_host=bind_host), + "started_at": started_at, + "uptime_seconds": uptime_seconds, + "system_health_api": SYSTEM_HEALTH_API_PATH, }) +def _truthy_flag(value: str | None) -> bool: + return (value or "").strip().lower() in {"1", "true", "yes", "on"} + + +async def api_system_health(request: Request) -> JSONResponse: + """Structured read-only system health (#634). + + `?deep=1` opts into the expensive network probe. The response status code + reflects readiness so automated checks can branch on it without parsing the + body: 200 when ready, 503 when a required dependency failed or never ran. + """ + deep = _truthy_flag(request.query_params.get("deep")) + snapshot = load_system_health(deep=deep) + payload = system_health_to_dict(snapshot) + return JSONResponse(payload, status_code=200 if snapshot.ready else 503) + + async def queue(_request: Request) -> HTMLResponse: snapshot = load_queue_snapshot() return HTMLResponse(render_page(title="Queue", body_html=render_queue_page(snapshot))) @@ -263,6 +296,7 @@ def create_app(*, bind_host: str | None = None) -> Starlette: routes=[ Route("/", home, methods=["GET"]), Route("/health", health, methods=["GET"]), + Route(SYSTEM_HEALTH_API_PATH, api_system_health, methods=["GET"]), Route("/queue", queue, methods=["GET"]), Route("/api/queue", api_queue, methods=["GET"]), Route("/projects", projects, methods=["GET"]), diff --git a/webui/system_health.py b/webui/system_health.py new file mode 100644 index 0000000..a36438b --- /dev/null +++ b/webui/system_health.py @@ -0,0 +1,682 @@ +"""Read-only system-health model for the operator console API (#634). + +`/health` answers liveness only. Operators automating readiness checks need a +structured view of *why* the control plane is or is not usable: which +dependencies answered, how long they took, what version of the code is running, +and whether the runtime is stale relative to its remote. + +Three rules shape this module. + +* **Read-only.** Every probe opens its subject read-only. The control-plane + database is opened through a ``mode=ro`` URI so a health check can never + create or migrate a schema, and no probe writes, restarts, or reloads + anything — restart controls are Phase 2, and #630 forbids process-kill + recovery outright. +* **Fail-soft.** A dependency that is unreachable is a *status*, not an + exception. Probes catch their own failures and report them as a degraded or + down entry carrying a reason. +* **Never claim more than was proven.** Readiness is derived only from probes + that actually ran, ``mutation_safe`` stays false unless the parity commits are + known and equal, and an MCP namespace is reported unproven because a web + process cannot exercise the IDE-managed client path (#543). +""" + +from __future__ import annotations + +import os +import re +import sqlite3 +import subprocess +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable +from urllib.parse import urlsplit, urlunsplit + +import control_plane_db +import mcp_namespace_health +from gitea_auth import api_request, get_auth_header, gitea_url + +from webui.project_registry import load_registry + +SERVICE_NAME = "mcp-control-plane-webui" +API_PATH = "/api/v1/system/health" + +STATUS_OK = "ok" +STATUS_DEGRADED = "degraded" +STATUS_DOWN = "down" +STATUS_SKIPPED = "skipped" +STATUS_UNPROVEN = "unproven" + +# Statuses that count as a healthy answer from a probe. +_HEALTHY_STATUSES = frozenset({STATUS_OK}) +# Statuses meaning "this probe did not run", as opposed to "it ran and failed". +_NOT_RUN_STATUSES = frozenset({STATUS_SKIPPED}) + +_DEEP_PROBE_TTL_ENV = "WEBUI_HEALTH_PROBE_TTL_SECONDS" +_DEFAULT_DEEP_PROBE_TTL = 15.0 +_GITEA_PROBE_TIMEOUT_SECONDS = 5.0 + +_OFFLINE_ENV = "WEBUI_TEST_OFFLINE" + +# Credential-shaped material that must never reach the browser, mirroring the +# forbidden client patterns in webui/deployment_boundary.py. +_SECRET_RE = re.compile( + r"(?i)\b(token|password|passwd|secret|authorization|bearer)\b\s*[:=]?\s*\S+" +) +_LONG_OPAQUE_RE = re.compile(r"\b[A-Za-z0-9_\-]{32,}\b") + +# Captured once at import so uptime measures this process, not the request. +_STARTED_AT = datetime.now(timezone.utc) +_STARTED_MONOTONIC = time.monotonic() + +# TTL cache for the expensive (network) probe only. +_deep_cache: dict[str, tuple[float, "DependencyProbe"]] = {} + + +@dataclass(frozen=True) +class DependencyProbe: + """One dependency check, fail-soft, with its own latency.""" + + name: str + kind: str + status: str + detail: str + required: bool + latency_ms: float | None = None + metadata: dict[str, Any] | None = None + + @property + def healthy(self) -> bool: + return self.status in _HEALTHY_STATUSES + + @property + def ran(self) -> bool: + return self.status not in _NOT_RUN_STATUSES + + +@dataclass(frozen=True) +class VersionInfo: + git_sha: str | None + git_describe: str | None + control_plane_schema_version: int | None + python_version: str + known: bool + + +@dataclass(frozen=True) +class StaleRuntime: + """Parity between the running code, the checkout, and the remote. + + ``mutation_safe`` is deliberately conservative: unknown is not safe. + """ + + daemon_head: str | None + checkout_head: str | None + remote_head: str | None + stale: bool + determinable: bool + mutation_safe: bool + reasons: tuple[str, ...] + + +@dataclass(frozen=True) +class SystemHealthSnapshot: + status: str + ready: bool + readiness_complete: bool + readiness_reasons: tuple[str, ...] + service: str + mode: str + version: VersionInfo + started_at: str + uptime_seconds: float + timestamp: str + deep_probes_requested: bool + dependencies: tuple[DependencyProbe, ...] + mcp_namespaces: tuple[dict[str, Any], ...] + stale_runtime: StaleRuntime + probe_errors: tuple[str, ...] = () + + +def process_uptime() -> tuple[str, float]: + """Process start timestamp and uptime — in-memory, safe for `/health`.""" + return _STARTED_AT.isoformat(), round(time.monotonic() - _STARTED_MONOTONIC, 3) + + +def _offline() -> bool: + return (os.environ.get(_OFFLINE_ENV) or "").strip().lower() in {"1", "true", "yes"} + + +def _repo_root() -> Path: + override = (os.environ.get("WEBUI_REPO_ROOT") or "").strip() + if override: + return Path(override).resolve() + return Path(__file__).resolve().parent.parent + + +def _deep_probe_ttl() -> float: + raw = (os.environ.get(_DEEP_PROBE_TTL_ENV) or "").strip() + if not raw: + return _DEFAULT_DEEP_PROBE_TTL + try: + value = float(raw) + except ValueError: + return _DEFAULT_DEEP_PROBE_TTL + return value if value >= 0 else _DEFAULT_DEEP_PROBE_TTL + + +def redact(text: str) -> str: + """Strip credential-shaped material from operator-visible probe text. + + Probe details carry exception strings, and an exception raised by an HTTP + client can quote the request that failed. Redaction happens here, at the + boundary where those strings become part of a browser-bound payload. + """ + if not text: + return "" + cleaned = _redact_urls(text) + cleaned = _SECRET_RE.sub(lambda m: f"{m.group(1)}=[redacted]", cleaned) + return _LONG_OPAQUE_RE.sub("[redacted]", cleaned) + + +def _redact_urls(text: str) -> str: + return re.sub(r"https?://\S+", lambda m: redact_url(m.group(0)), text) + + +def redact_url(url: str) -> str: + """Reduce a URL to scheme://host/path — no userinfo, no query, no fragment.""" + try: + parts = urlsplit(url) + except ValueError: + return "[redacted-url]" + if not parts.scheme or not parts.hostname: + return "[redacted-url]" + netloc = parts.hostname + if parts.port: + netloc = f"{netloc}:{parts.port}" + return urlunsplit((parts.scheme, netloc, parts.path, "", "")) + + +def _git(repo: Path, *args: str) -> str | None: + try: + completed = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, + text=True, + check=False, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return None + if completed.returncode != 0: + return None + return (completed.stdout or "").strip() or None + + +def _load_version(repo: Path, *, schema_version: int | None) -> VersionInfo: + import platform + + git_sha = None if _offline() else _git(repo, "rev-parse", "HEAD") + describe = None if _offline() else _git(repo, "describe", "--tags", "--always") + return VersionInfo( + git_sha=git_sha, + git_describe=describe, + control_plane_schema_version=schema_version, + python_version=platform.python_version(), + known=bool(git_sha), + ) + + +# --------------------------------------------------------------------------- +# Dependency probes +# --------------------------------------------------------------------------- + + +def _elapsed_ms(started: float) -> float: + return round((time.monotonic() - started) * 1000, 3) + + +def probe_control_plane_db(db_path: str | None = None) -> DependencyProbe: + """Read-only reachability check for the control-plane SQLite substrate. + + Opened through a ``mode=ro`` URI on purpose: ``ControlPlaneDB.__init__`` + creates directories and runs schema migrations, which a health check must + never do. + """ + path = (db_path or control_plane_db.default_db_path()).strip() + started = time.monotonic() + metadata: dict[str, Any] = {"path": path} + + def _result(status: str, detail: str) -> DependencyProbe: + return DependencyProbe( + name="control_plane_db", + kind="sqlite", + status=status, + detail=detail, + required=True, + latency_ms=_elapsed_ms(started), + metadata=metadata, + ) + + if not path or not os.path.exists(path): + return _result(STATUS_DOWN, "control-plane database file does not exist yet") + try: + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5) + try: + row = conn.execute( + "SELECT value FROM schema_meta WHERE key = 'schema_version'" + ).fetchone() + leases = conn.execute( + "SELECT COUNT(*) FROM leases WHERE status = 'active'" + ).fetchone() + finally: + conn.close() + except sqlite3.Error as exc: + return _result(STATUS_DOWN, redact(f"control-plane database unreadable: {exc}")) + + schema_version = int(row[0]) if row and str(row[0]).isdigit() else None + metadata["schema_version"] = schema_version + metadata["active_leases"] = int(leases[0]) if leases else None + if schema_version is None: + return _result( + STATUS_DEGRADED, "control-plane database has no recorded schema version" + ) + if schema_version != control_plane_db.SCHEMA_VERSION: + return _result( + STATUS_DEGRADED, + f"control-plane schema version {schema_version} does not match the " + f"version this code expects ({control_plane_db.SCHEMA_VERSION})", + ) + return _result(STATUS_OK, f"schema v{schema_version} readable") + + +def probe_repository(repo: Path) -> DependencyProbe: + """Local checkout reachability — required, cheap, no network.""" + started = time.monotonic() + metadata: dict[str, Any] = {"repo_root": str(repo)} + if _offline(): + return DependencyProbe( + name="repository", + kind="git", + status=STATUS_SKIPPED, + detail=f"{_OFFLINE_ENV} is set; git probe skipped", + required=True, + latency_ms=_elapsed_ms(started), + metadata=metadata, + ) + head = _git(repo, "rev-parse", "HEAD") + if not head: + return DependencyProbe( + name="repository", + kind="git", + status=STATUS_DOWN, + detail=f"HEAD could not be read at {repo}", + required=True, + latency_ms=_elapsed_ms(started), + metadata=metadata, + ) + branch = _git(repo, "rev-parse", "--abbrev-ref", "HEAD") + metadata["head"] = head + metadata["branch"] = branch + return DependencyProbe( + name="repository", + kind="git", + status=STATUS_OK, + detail=f"checkout readable at {branch or 'detached HEAD'}", + required=True, + latency_ms=_elapsed_ms(started), + metadata=metadata, + ) + + +def probe_gitea(host: str) -> DependencyProbe: + """Live Gitea reachability. Expensive (network), so opt-in via ``deep``. + + Optional by design: the console stays useful for local inventory when the + remote is unreachable, so a failure here degrades status without claiming + the process itself is unready. + """ + started = time.monotonic() + metadata: dict[str, Any] = {"host": host} + + def _failure(status: str, detail: str) -> DependencyProbe: + return DependencyProbe( + name="gitea", + kind="http", + status=status, + detail=detail, + required=False, + latency_ms=_elapsed_ms(started), + metadata=metadata, + ) + + if not host: + return _failure(STATUS_DEGRADED, "no Gitea host is configured in the registry") + try: + auth = get_auth_header(host) + except Exception as exc: # noqa: BLE001 — credential guards are a status here + return _failure(STATUS_DEGRADED, redact(f"credential lookup refused: {exc}")) + if not auth: + return _failure(STATUS_DEGRADED, f"no credentials available for {host}") + + url = gitea_url(host, "/api/v1/version") + metadata["endpoint"] = redact_url(url) + try: + data = api_request("GET", url, auth, timeout=_GITEA_PROBE_TIMEOUT_SECONDS) + except Exception as exc: # noqa: BLE001 — a down dependency is a status + return _failure(STATUS_DOWN, redact(f"Gitea probe failed: {exc}")) + + if isinstance(data, dict) and data.get("version"): + metadata["gitea_version"] = str(data["version"]) + return DependencyProbe( + name="gitea", + kind="http", + status=STATUS_OK, + detail=f"{host} reachable", + required=False, + latency_ms=_elapsed_ms(started), + metadata=metadata, + ) + + +def _skipped_gitea(host: str) -> DependencyProbe: + return DependencyProbe( + name="gitea", + kind="http", + status=STATUS_SKIPPED, + detail="network probe not requested; call with ?deep=1 to run it", + required=False, + latency_ms=None, + metadata={"host": host}, + ) + + +def namespace_summaries() -> tuple[dict[str, Any], ...]: + """Declared MCP namespaces, each honestly reported as unproven. + + The web process runs outside the IDE-managed MCP client, so it cannot + invoke a namespace tool. Per #543 only a ``client_namespace`` probe proves + that path, and inventing a healthy verdict here is exactly the false claim + the mutation gates exist to prevent. + """ + rows: list[dict[str, Any]] = [] + for namespace, required_tool in sorted( + mcp_namespace_health.REQUIRED_NAMESPACE_TOOLS.items() + ): + classification = mcp_namespace_health.classify_namespace_probe( + namespace, + required_tool=required_tool, + probe_result=None, + probe_source=mcp_namespace_health.PROBE_SOURCE_UNKNOWN, + ) + rows.append( + { + "namespace": namespace, + "required_tool": required_tool, + "status": STATUS_UNPROVEN, + "ide_namespace_proven": bool(classification.get("ide_namespace_proven")), + "reason": ( + "the web console cannot invoke the IDE-managed MCP client; " + "namespace health must be proven with a client_namespace " + "probe (#543)" + ), + "error_type": classification.get("error_type"), + } + ) + return tuple(rows) + + +def assess_stale_runtime( + repo: Path, + *, + daemon_head: str | None = None, + git_reader: Callable[..., str | None] | None = None, +) -> StaleRuntime: + """Three-way parity view: running code, local checkout, remote-tracking ref. + + ``mutation_safe`` requires all three to be known and equal. Anything less — + including "the remote ref was never fetched" — is reported as not safe with + a reason, so an operator never reads an unproven green. + """ + reader = git_reader or (lambda *args: _git(repo, *args)) + reasons: list[str] = [] + # The offline switch suppresses real subprocess calls; an explicitly + # injected reader is already a substitute for them and is always used. + offline = _offline() and git_reader is None + + checkout_head = None if offline else reader("rev-parse", "HEAD") + remote_head = None if offline else reader("rev-parse", "@{upstream}") + if offline: + reasons.append(f"{_OFFLINE_ENV} is set; parity commits were not read") + else: + if checkout_head is None: + reasons.append("local checkout HEAD could not be read") + if remote_head is None: + reasons.append( + "no remote-tracking commit is known for the current branch; " + "remote staleness is indeterminate (no fetch is performed here)" + ) + + effective_daemon = daemon_head if daemon_head is not None else checkout_head + if daemon_head is None: + reasons.append( + "the running MCP daemon's startup commit is not observable from the " + "web process; the checkout commit is reported in its place" + ) + + determinable = bool(checkout_head and remote_head and effective_daemon) + stale = bool( + determinable and len({checkout_head, remote_head, effective_daemon}) > 1 + ) + if stale: + reasons.append( + "runtime, checkout, and remote commits disagree; restart the MCP " + "server after updating the checkout before trusting capability gates" + ) + + return StaleRuntime( + daemon_head=effective_daemon, + checkout_head=checkout_head, + remote_head=remote_head, + stale=stale, + determinable=determinable, + mutation_safe=bool(determinable and not stale), + reasons=tuple(reasons), + ) + + +# --------------------------------------------------------------------------- +# Snapshot assembly +# --------------------------------------------------------------------------- + + +def _default_host() -> str: + registry = load_registry() + if not registry.projects: + return "" + raw = registry.projects[0].remote_host + parts = urlsplit(raw.strip()) + return parts.netloc or raw.strip().rstrip("/") + + +def _aggregate( + probes: tuple[DependencyProbe, ...], +) -> tuple[str, bool, bool, tuple[str, ...]]: + """Fold probe results into overall status and readiness. + + Required probes drive readiness; optional probes can only degrade status. + A probe that did not run leaves readiness incomplete rather than passing. + """ + reasons: list[str] = [] + required = [probe for probe in probes if probe.required] + unrun_required = [probe for probe in required if not probe.ran] + failed_required = [probe for probe in required if probe.ran and not probe.healthy] + failed_optional = [ + probe + for probe in probes + if not probe.required and probe.ran and not probe.healthy + ] + + for probe in unrun_required: + reasons.append( + f"required dependency '{probe.name}' was not probed: {probe.detail}" + ) + for probe in failed_required: + reasons.append( + f"required dependency '{probe.name}' is {probe.status}: {probe.detail}" + ) + for probe in failed_optional: + reasons.append( + f"optional dependency '{probe.name}' is {probe.status}: {probe.detail}" + ) + + readiness_complete = not unrun_required + ready = readiness_complete and not failed_required + + if any(probe.status == STATUS_DOWN for probe in failed_required): + status = STATUS_DOWN + elif failed_required or failed_optional or unrun_required: + status = STATUS_DEGRADED + else: + status = STATUS_OK + return status, ready, readiness_complete, tuple(reasons) + + +def load_system_health( + *, + deep: bool = False, + host: str | None = None, + probes: tuple[DependencyProbe, ...] | None = None, + daemon_head: str | None = None, + use_cache: bool = True, +) -> SystemHealthSnapshot: + """Assemble the read-only system-health snapshot. + + ``deep=True`` adds the network probe against Gitea; its result is cached for + a short TTL so repeated dashboard polls do not amplify into remote load. + """ + repo = _repo_root() + probe_errors: list[str] = [] + + if probes is None: + collected: list[DependencyProbe] = [] + for probe_fn in ( + lambda: probe_control_plane_db(), + lambda: probe_repository(repo), + ): + try: + collected.append(probe_fn()) + except Exception as exc: # noqa: BLE001 — a probe must not 500 the API + probe_errors.append(redact(f"probe raised: {exc}")) + resolved_host = host if host is not None else _default_host() + if deep and not _offline(): + collected.append(_cached_gitea_probe(resolved_host, use_cache=use_cache)) + else: + collected.append(_skipped_gitea(resolved_host)) + probes = tuple(collected) + + status, ready, readiness_complete, reasons = _aggregate(probes) + stale = assess_stale_runtime(repo, daemon_head=daemon_head) + if stale.stale: + if status == STATUS_OK: + status = STATUS_DEGRADED + reasons = reasons + ( + "runtime is stale relative to its remote-tracking commit", + ) + + db_probe = next((p for p in probes if p.name == "control_plane_db"), None) + schema_version = None + if db_probe and db_probe.metadata: + schema_version = db_probe.metadata.get("schema_version") + + return SystemHealthSnapshot( + status=status, + ready=ready, + readiness_complete=readiness_complete, + readiness_reasons=reasons, + service=SERVICE_NAME, + mode="read-only", + version=_load_version(repo, schema_version=schema_version), + started_at=_STARTED_AT.isoformat(), + uptime_seconds=round(time.monotonic() - _STARTED_MONOTONIC, 3), + timestamp=datetime.now(timezone.utc).isoformat(), + deep_probes_requested=deep, + dependencies=probes, + mcp_namespaces=namespace_summaries(), + stale_runtime=stale, + probe_errors=tuple(probe_errors), + ) + + +def _cached_gitea_probe(host: str, *, use_cache: bool = True) -> DependencyProbe: + ttl = _deep_probe_ttl() + now = time.monotonic() + if use_cache and ttl > 0: + cached = _deep_cache.get(host) + if cached and (now - cached[0]) < ttl: + return cached[1] + probe = probe_gitea(host) + if use_cache and ttl > 0: + _deep_cache[host] = (now, probe) + return probe + + +def clear_probe_cache() -> None: + """Drop cached deep-probe results (tests and operator-forced refresh).""" + _deep_cache.clear() + + +def probe_to_dict(probe: DependencyProbe) -> dict[str, Any]: + return { + "name": probe.name, + "kind": probe.kind, + "status": probe.status, + "detail": probe.detail, + "required": probe.required, + "healthy": probe.healthy, + "latency_ms": probe.latency_ms, + "metadata": dict(probe.metadata or {}), + } + + +def snapshot_to_dict(snapshot: SystemHealthSnapshot) -> dict[str, Any]: + return { + "status": snapshot.status, + "service": snapshot.service, + "mode": snapshot.mode, + "api": API_PATH, + "timestamp": snapshot.timestamp, + "readiness": { + "ready": snapshot.ready, + "complete": snapshot.readiness_complete, + "reasons": list(snapshot.readiness_reasons), + }, + "version": { + "git_sha": snapshot.version.git_sha, + "git_describe": snapshot.version.git_describe, + "control_plane_schema_version": ( + snapshot.version.control_plane_schema_version + ), + "python_version": snapshot.version.python_version, + "known": snapshot.version.known, + }, + "process": { + "started_at": snapshot.started_at, + "uptime_seconds": snapshot.uptime_seconds, + }, + "deep_probes_requested": snapshot.deep_probes_requested, + "dependencies": [probe_to_dict(probe) for probe in snapshot.dependencies], + "mcp_namespaces": [dict(row) for row in snapshot.mcp_namespaces], + "stale_runtime": { + "daemon_head": snapshot.stale_runtime.daemon_head, + "checkout_head": snapshot.stale_runtime.checkout_head, + "remote_head": snapshot.stale_runtime.remote_head, + "stale": snapshot.stale_runtime.stale, + "determinable": snapshot.stale_runtime.determinable, + "mutation_safe": snapshot.stale_runtime.mutation_safe, + "reasons": list(snapshot.stale_runtime.reasons), + }, + "probe_errors": list(snapshot.probe_errors), + }