From 1c88b87ec5030256fe5573bdc711792bf5bd5d37 Mon Sep 17 00:00:00 2001 From: Jason Walker <913443@dadeschools.net> Date: Sat, 25 Jul 2026 08:19:06 -0400 Subject: [PATCH] feat(webui): implement Phase 2 recovery controls & playbooks (#644) --- docs/sanctioned-recovery-playbooks.md | 62 +++ docs/webui-authz-audit.md | 3 + task_capability_map.py | 13 + tests/test_webui_console_recovery.py | 182 ++++++++ webui/app.py | 83 ++++ webui/console_authz.py | 34 ++ webui/console_recovery.py | 573 ++++++++++++++++++++++++++ webui/gated_actions.py | 7 + webui/system_health_views.py | 61 ++- 9 files changed, 998 insertions(+), 20 deletions(-) create mode 100644 docs/sanctioned-recovery-playbooks.md create mode 100644 tests/test_webui_console_recovery.py create mode 100644 webui/console_recovery.py diff --git a/docs/sanctioned-recovery-playbooks.md b/docs/sanctioned-recovery-playbooks.md new file mode 100644 index 0000000..819de2c --- /dev/null +++ b/docs/sanctioned-recovery-playbooks.md @@ -0,0 +1,62 @@ +# Sanctioned Recovery Playbooks & Controls (Phase 2 #644) + +## Overview + +Stale runtimes, worktree binding mismatches, and un-reconciled merged branches previously required expert manual shell recovery. Manual process kills (`pkill -f mcp_server.py`) are strictly forbidden and classified as runtime contamination ([#630](file:///Users/jasonwalker/Development/Gitea-Tools/docs/sanctioned-restart-controls.md)). + +Phase 2 introduces **sanctioned recovery playbooks and controls** into the Web Console: +- **Diagnose**: Surface stale runtimes, worktree binding errors, contamination markers, and worktree anomalies via health & inventory APIs. +- **Preview**: Render mutation ledgers and exact confirmation phrases for recovery playbooks. +- **Confirm & Apply**: Execute sanctioned recovery actions through gated, audited paths. +- **Verify**: Revalidate control-plane state post-recovery before claiming clean status. + +--- + +## Recovery Playbook Taxonomy + +| Playbook ID | Action ID | Minimum Role | Target / Scope | Description | +|---|---|---|---|---| +| `clear_stale_binding` | `system.clear_stale_binding` | Operator | Active worktree binding | Clear provably missing or superseded `GITEA_ACTIVE_WORKTREE` binding ([#702](file:///Users/jasonwalker/Development/Gitea-Tools/stale_binding_recovery.py)). | +| `rebind_session_worktree` | `system.rebind_session_worktree` | Operator | Session worktree | Rebind or synchronize session worktree to verified lease worktree ([#864](file:///Users/jasonwalker/Development/Gitea-Tools/dirty_same_claimant_session_rebind.py)). | +| `reconcile_cleanups` | `system.reconcile_cleanups` | Controller | Worktree hygiene | Execute reconciler cleanup preview and apply for merged/superseded PR branches. | +| `sanctioned_restart` | `system.restart_namespace` | Admin | MCP Namespace | Restart MCP daemon gracefully via host supervisor ([#642](file:///Users/jasonwalker/Development/Gitea-Tools/docs/sanctioned-restart-controls.md)). | + +--- + +## Wizard Workflow (Diagnose → Preview → Confirm → Verify) + +### 1. Diagnose (`GET /api/v1/system/recovery/diagnose`) +Runs control-plane diagnostics: +- **Stale Runtime**: Mismatch between running daemon HEAD, local checkout HEAD, and remote-tracking HEAD. +- **Worktree Binding**: Missing path (`provably_stale_missing_path`), unverified inherited binding (`unverified_inherited`), or superseded binding (`superseded_by_session_lease`). +- **Contamination**: Checks for live contamination markers from unmanaged process kills. +- **Worktree Anomalies**: Scans `branches/` directory for un-reconciled cleanups or missing preserved worktrees. + +Returns `RecoveryDiagnosis` with eligible playbooks. + +### 2. Preview (`POST /api/v1/system/recovery/preview`) +Takes `playbook_id` and optional `target`/`params`. +Returns: +- **Mutation Ledger**: Step-by-step sequence of actions. +- **Confirmation Phrase**: Exact phrase required to authorize execution (e.g., `confirm clear_stale_binding`). +- **Authorization Decision**: RBAC check against the operator's principal. + +### 3. Apply (`POST /api/v1/system/recovery/apply`) +Requires `playbook_id` and matching `confirmation` phrase. +- Validates RBAC permissions (`console_authz`). +- Verifies confirmation phrase (`confirmation_matches`). +- Enforces contamination rules ([#630](file:///Users/jasonwalker/Development/Gitea-Tools/docs/sanctioned-restart-controls.md)): A contaminated runtime must be cleared through reconciler cleanup before other playbooks run. +- Enforces master parity ([#610](file:///Users/jasonwalker/Development/Gitea-Tools/master_parity_gate.py)). +- Applies sanctioned recovery logic. +- Logs audit record in `console_audit`. + +### 4. Verify (`POST /api/v1/system/recovery/verify`) +Re-evaluates control-plane diagnostics post-recovery. Asserts `clean: true` before transitioning out of recovery mode. + +--- + +## Safety & Governance Principles + +1. **No Manual `pkill`**: Direct process killing remains forbidden and is recorded as contamination. +2. **Auditability**: Every recovery preview and execution is logged in the console audit trail. +3. **Master Parity & Dual Control**: High-privilege recovery actions require controller/admin roles and explicit confirmation phrases. diff --git a/docs/webui-authz-audit.md b/docs/webui-authz-audit.md index 2dcffac..9b9575e 100644 --- a/docs/webui-authz-audit.md +++ b/docs/webui-authz-audit.md @@ -94,6 +94,9 @@ already define, and a regression test asserts each mapping matches. | `record_analytics_usage` | operator | gated_write | `runtime.record_analytics_usage` | Yes | No | No | 2 | | `system.reload_namespace` | controller | privileged | `runtime.reload_namespace` | Yes | No | No | 2 | | `system.restart_namespace` | admin | destructive | `runtime.restart_namespace` | Yes | **Yes** | **Yes** | 2 | +| `system.clear_stale_binding` | operator | gated_write | `gitea.read` | Yes | No | No | 2 | +| `system.rebind_session_worktree` | operator | gated_write | `gitea.read` | Yes | No | No | 2 | +| `system.reconcile_cleanups` | controller | privileged | `gitea.pr.close` | Yes | No | No | 2 | **Dual control** means the acting principal may not be the sole authority: a second distinct principal must confirm. **Break-glass** means the action is diff --git a/task_capability_map.py b/task_capability_map.py index 878cf0a..3a8efd1 100644 --- a/task_capability_map.py +++ b/task_capability_map.py @@ -142,6 +142,19 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = { "permission": "gitea.read", "role": "author", }, + # #644: Phase 2 Web Console recovery tasks. + "clear_stale_binding": { + "permission": "gitea.read", + "role": "author", + }, + "rebind_session_worktree": { + "permission": "gitea.read", + "role": "author", + }, + "reconcile_cleanups": { + "permission": "gitea.pr.close", + "role": "reconciler", + }, # PR synchronization lifecycle: assess is read-only (any role with gitea.read); # update-by-merge is author-only and mutates the PR head via Gitea API. "assess_pr_sync_status": { diff --git a/tests/test_webui_console_recovery.py b/tests/test_webui_console_recovery.py new file mode 100644 index 0000000..343a61b --- /dev/null +++ b/tests/test_webui_console_recovery.py @@ -0,0 +1,182 @@ +"""Unit and integration tests for Phase 2 Web Console recovery controls (#644).""" + +from __future__ import annotations + +import json +import os +import unittest +from unittest.mock import patch + +from starlette.testclient import TestClient + +from webui import console_audit, console_authz, console_recovery +from webui.app import create_app + + +class TestConsoleRecovery(unittest.TestCase): + + def test_diagnose_recovery_healthy(self) -> None: + diag = console_recovery.diagnose_recovery() + self.assertIn(diag.status, {console_recovery.STATUS_HEALTHY, console_recovery.STATUS_ACTION_REQUIRED}) + self.assertIsInstance(diag.playbooks, tuple) + self.assertGreaterEqual(len(diag.playbooks), 4) + + playbook_ids = {pb.playbook_id for pb in diag.playbooks} + self.assertIn(console_recovery.PLAYBOOK_CLEAR_STALE_BINDING, playbook_ids) + self.assertIn(console_recovery.PLAYBOOK_REBIND_SESSION, playbook_ids) + self.assertIn(console_recovery.PLAYBOOK_RECONCILE_CLEANUPS, playbook_ids) + self.assertIn(console_recovery.PLAYBOOK_SANCTIONED_RESTART, playbook_ids) + + def test_confirmation_phrase_generation_and_matching(self) -> None: + phrase = console_recovery.confirmation_phrase("clear_stale_binding") + self.assertEqual(phrase, "confirm clear_stale_binding") + self.assertTrue(console_recovery.confirmation_matches("clear_stale_binding", "confirm clear_stale_binding")) + self.assertFalse(console_recovery.confirmation_matches("clear_stale_binding", "wrong phrase")) + + phrase_target = console_recovery.confirmation_phrase("sanctioned_restart", "gitea-author") + self.assertEqual(phrase_target, "confirm sanctioned_restart gitea-author") + self.assertTrue(console_recovery.confirmation_matches("sanctioned_restart", "confirm sanctioned_restart gitea-author", "gitea-author")) + + def test_build_recovery_preview(self) -> None: + principal = console_authz.Principal("dev@example.com", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True) + preview = console_recovery.build_recovery_preview( + playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING, + target="test-worktree", + principal=principal, + ) + self.assertEqual(preview["playbook_id"], console_recovery.PLAYBOOK_CLEAR_STALE_BINDING) + self.assertEqual(preview["action_id"], console_recovery.ACTION_CLEAR_STALE_BINDING) + self.assertEqual(preview["confirmation_phrase"], "confirm clear_stale_binding test-worktree") + self.assertTrue(len(preview["mutation_ledger"]) >= 3) + self.assertTrue(preview["authorization"]["allowed"]) + + def test_build_recovery_preview_unknown_playbook(self) -> None: + preview = console_recovery.build_recovery_preview("unknown_playbook") + self.assertFalse(preview.get("allowed")) + self.assertEqual(preview.get("error"), "unknown_playbook") + + def test_execute_recovery_playbook_confirmation_mismatch(self) -> None: + principal = console_authz.Principal("dev@example.com", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True) + result = console_recovery.execute_recovery_playbook( + playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING, + confirmation="invalid confirmation", + principal=principal, + ) + self.assertFalse(result["success"]) + self.assertFalse(result["allowed"]) + self.assertEqual(result["error"], "confirmation_mismatch") + + def test_execute_recovery_playbook_unauthorized(self) -> None: + # Anonymous principal has viewer role -> should be denied + result = console_recovery.execute_recovery_playbook( + playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING, + confirmation="confirm clear_stale_binding", + principal=console_authz.ANONYMOUS, + ) + self.assertFalse(result["success"]) + self.assertFalse(result["allowed"]) + self.assertEqual(result["error"], console_authz.DENY_UNAUTHENTICATED) + + def test_execute_recovery_playbook_clear_stale_binding_success(self) -> None: + principal = console_authz.Principal("dev@example.com", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True) + phrase = console_recovery.confirmation_phrase(console_recovery.PLAYBOOK_CLEAR_STALE_BINDING) + + result = console_recovery.execute_recovery_playbook( + playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING, + confirmation=phrase, + principal=principal, + ) + self.assertTrue(result["allowed"]) + self.assertIn("applied_result", result) + self.assertIn("post_recovery_verification", result) + + self.assertIn("audit", result) + self.assertEqual(result["audit"]["event"]["action"], console_recovery.ACTION_CLEAR_STALE_BINDING) + + def test_execute_recovery_playbook_rebind_session_success(self) -> None: + principal = console_authz.Principal("dev@example.com", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True) + phrase = console_recovery.confirmation_phrase(console_recovery.PLAYBOOK_REBIND_SESSION, "branches/feat-issue-644") + + result = console_recovery.execute_recovery_playbook( + playbook_id=console_recovery.PLAYBOOK_REBIND_SESSION, + confirmation=phrase, + target="branches/feat-issue-644", + principal=principal, + ) + self.assertTrue(result["allowed"]) + self.assertTrue(result["success"]) + self.assertEqual(result["applied_result"]["rebound_worktree"], "branches/feat-issue-644") + + def test_verify_post_recovery(self) -> None: + verification = console_recovery.verify_post_recovery() + self.assertIn("clean", verification) + self.assertIn("status", verification) + self.assertIn("reasons", verification) + + +class TestConsoleRecoveryApi(unittest.TestCase): + def setUp(self) -> None: + self.app = create_app() + self.client = TestClient(self.app) + + def test_api_recovery_diagnose(self) -> None: + res = self.client.get("/api/v1/system/recovery/diagnose") + self.assertEqual(res.status_code, 200) + data = res.json() + self.assertIn("status", data) + self.assertIn("clean", data) + self.assertIn("playbooks", data) + self.assertTrue(len(data["playbooks"]) >= 4) + + def test_api_recovery_preview(self) -> None: + res = self.client.post( + "/api/v1/system/recovery/preview", + json={"playbook_id": "clear_stale_binding", "target": "active"}, + ) + self.assertEqual(res.status_code, 200) + data = res.json() + self.assertEqual(data["playbook_id"], "clear_stale_binding") + self.assertEqual(data["confirmation_phrase"], "confirm clear_stale_binding active") + self.assertIn("mutation_ledger", data) + + def test_api_recovery_apply_denied_without_auth(self) -> None: + res = self.client.post( + "/api/v1/system/recovery/apply", + json={"playbook_id": "clear_stale_binding", "confirmation": "confirm clear_stale_binding"}, + ) + self.assertEqual(res.status_code, 400) + data = res.json() + self.assertFalse(data["success"]) + self.assertFalse(data["allowed"]) + + def test_api_recovery_apply_with_dev_auth(self) -> None: + env = { + "WEBUI_AUTH_MODE": "local_dev", + "WEBUI_DEV_SUBJECT": "dev@example.com", + "WEBUI_DEV_ROLE": "operator", + } + with patch.dict(os.environ, env): + res = self.client.post( + "/api/v1/system/recovery/apply", + json={ + "playbook_id": "rebind_session_worktree", + "target": "branches/feat-issue-644", + "confirmation": "confirm rebind_session_worktree branches/feat-issue-644", + }, + ) + self.assertEqual(res.status_code, 200) + data = res.json() + self.assertTrue(data["success"]) + self.assertTrue(data["allowed"]) + self.assertEqual(data["playbook_id"], "rebind_session_worktree") + + def test_api_recovery_verify(self) -> None: + res = self.client.get("/api/v1/system/recovery/verify") + self.assertEqual(res.status_code, 200) + data = res.json() + self.assertIn("clean", data) + self.assertIn("status", data) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/app.py b/webui/app.py index 560ec75..6fece41 100644 --- a/webui/app.py +++ b/webui/app.py @@ -200,6 +200,84 @@ async def system_health(request: Request) -> HTMLResponse: ) +async def api_recovery_diagnose(_request: Request) -> JSONResponse: + from webui.console_recovery import diagnose_recovery + diag = diagnose_recovery() + return JSONResponse({ + "status": diag.status, + "clean": diag.clean, + "stale_runtime": diag.stale_runtime, + "master_parity": diag.master_parity, + "stale_binding": diag.stale_binding, + "contamination": diag.contamination, + "worktree_anomalies": list(diag.worktree_anomalies), + "reasons": list(diag.reasons), + "playbooks": [ + { + "playbook_id": pb.playbook_id, + "label": pb.label, + "action_id": pb.action_id, + "description": pb.description, + "eligible": pb.eligible, + "requires_confirmation": pb.requires_confirmation, + "reason": pb.reason, + "params_schema": pb.params_schema, + } + for pb in diag.playbooks + ], + }) + + +async def api_recovery_preview(request: Request) -> JSONResponse: + from webui.console_recovery import build_recovery_preview + body = {} + try: + body = await request.json() + except Exception: + pass + playbook_id = body.get("playbook_id") or request.query_params.get("playbook_id") or "" + target = body.get("target") or request.query_params.get("target") + principal = resolve_principal(request.headers) + preview = build_recovery_preview( + playbook_id=playbook_id, + target=target, + params=body, + principal=principal, + ) + status = 200 if preview.get("playbook_id") else 400 + return JSONResponse(preview, status_code=status) + + +async def api_recovery_apply(request: Request) -> JSONResponse: + from webui.console_recovery import execute_recovery_playbook + body = {} + try: + body = await request.json() + except Exception: + pass + playbook_id = body.get("playbook_id", "") + confirmation = body.get("confirmation", "") + target = body.get("target") + principal = resolve_principal(request.headers) + request_id = getattr(request.state, "request_id", None) + result = execute_recovery_playbook( + playbook_id=playbook_id, + confirmation=confirmation, + target=target, + params=body, + principal=principal, + request_id=request_id, + ) + status_code = 200 if result.get("success") else 400 + return JSONResponse(result, status_code=status_code) + + +async def api_recovery_verify(_request: Request) -> JSONResponse: + from webui.console_recovery import verify_post_recovery + verification = verify_post_recovery() + return JSONResponse(verification, status_code=200) + + async def queue(_request: Request) -> HTMLResponse: snapshot = load_queue_snapshot() return HTMLResponse(render_page(title="Queue", body_html=render_queue_page(snapshot))) @@ -818,6 +896,11 @@ def create_app(*, bind_host: str | None = None) -> Starlette: api_console_security_model, methods=["GET"], ), + # #644 Phase 2 Recovery API routes + Route("/api/v1/system/recovery/diagnose", api_recovery_diagnose, methods=["GET"]), + Route("/api/v1/system/recovery/preview", api_recovery_preview, methods=["POST", "GET"]), + Route("/api/v1/system/recovery/apply", api_recovery_apply, methods=["POST"]), + Route("/api/v1/system/recovery/verify", api_recovery_verify, methods=["POST", "GET"]), *[ Route(path, phase_stub, methods=["GET"]) for path in STUB_PAGES diff --git a/webui/console_authz.py b/webui/console_authz.py index 0c51037..e8de56b 100644 --- a/webui/console_authz.py +++ b/webui/console_authz.py @@ -277,6 +277,40 @@ _ACTION_SPECS: tuple[ConsoleAction, ...] = ( phase=2, summary="Restart one MCP namespace via the host supervisor.", ), + # #644: Phase 2 recovery controls & playbooks. + ConsoleAction( + action_id="system.clear_stale_binding", + task_key="clear_stale_binding", + action_class=CLASS_WRITE, + minimum_role=OPERATOR, + requires_confirmation=True, + dual_control=False, + break_glass=False, + phase=2, + summary="Clear provably stale or superseded GITEA_ACTIVE_WORKTREE binding.", + ), + ConsoleAction( + action_id="system.rebind_session_worktree", + task_key="rebind_session_worktree", + action_class=CLASS_WRITE, + minimum_role=OPERATOR, + requires_confirmation=True, + dual_control=False, + break_glass=False, + phase=2, + summary="Rebind session worktree context to verified lease worktree.", + ), + ConsoleAction( + action_id="system.reconcile_cleanups", + task_key="reconcile_cleanups", + action_class=CLASS_PRIVILEGED, + minimum_role=CONTROLLER, + requires_confirmation=True, + dual_control=False, + break_glass=False, + phase=2, + summary="Run reconciler cleanup for merged or superseded PR branches.", + ), ) ACTIONS: dict[str, ConsoleAction] = {a.action_id: a for a in _ACTION_SPECS} diff --git a/webui/console_recovery.py b/webui/console_recovery.py new file mode 100644 index 0000000..61783b0 --- /dev/null +++ b/webui/console_recovery.py @@ -0,0 +1,573 @@ +"""Web Console Phase 2 Recovery Controls & Playbooks (#644). + +Provides canonical recovery controls for the web console: +1. Diagnosis: Surfaces stale runtimes, worktree binding errors, contamination markers, + and un-reconciled cleanups. +2. Gated Actions & Playbooks: Guided recovery (rebind session worktree, clear stale + binding, trigger reconciler cleanups, sanctioned restart). +3. RBAC, Contamination (#630), and Master Parity (#610) integration. +4. Audit trail via ``console_audit`` and mandatory post-recovery revalidation. +""" + +from __future__ import annotations + +import os +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +import master_parity_gate +import merged_cleanup_reconcile +import runtime_recovery_guard +import stale_binding_recovery +from webui import console_audit, console_authz, sanctioned_restart, system_health, worktree_scanner + +# --- Recovery Statuses ------------------------------------------------------ +STATUS_HEALTHY = "healthy" +STATUS_ACTION_REQUIRED = "action_required" +STATUS_BLOCKED_CONTAMINATION = "blocked_contamination" +STATUS_RECONNECT_REQUIRED = "reconnect_required" + +# --- Playbook Identifiers --------------------------------------------------- +PLAYBOOK_CLEAR_STALE_BINDING = "clear_stale_binding" +PLAYBOOK_REBIND_SESSION = "rebind_session_worktree" +PLAYBOOK_RECONCILE_CLEANUPS = "reconcile_cleanups" +PLAYBOOK_SANCTIONED_RESTART = "sanctioned_restart" + +KNOWN_PLAYBOOKS: tuple[str, ...] = ( + PLAYBOOK_CLEAR_STALE_BINDING, + PLAYBOOK_REBIND_SESSION, + PLAYBOOK_RECONCILE_CLEANUPS, + PLAYBOOK_SANCTIONED_RESTART, +) + +# --- Console Action Mapping ------------------------------------------------- +ACTION_CLEAR_STALE_BINDING = "system.clear_stale_binding" +ACTION_REBIND_SESSION = "system.rebind_session_worktree" +ACTION_RECONCILE_CLEANUPS = "system.reconcile_cleanups" + +PLAYBOOK_ACTIONS: dict[str, str] = { + PLAYBOOK_CLEAR_STALE_BINDING: ACTION_CLEAR_STALE_BINDING, + PLAYBOOK_REBIND_SESSION: ACTION_REBIND_SESSION, + PLAYBOOK_RECONCILE_CLEANUPS: ACTION_RECONCILE_CLEANUPS, + PLAYBOOK_SANCTIONED_RESTART: sanctioned_restart.ACTION_RESTART_NAMESPACE, +} + + +@dataclass(frozen=True) +class RecoveryLedgerEntry: + """One planned recovery step displayed before execution.""" + + sequence: int + step: str + summary: str + executes_process_kill: bool = False + + +@dataclass(frozen=True) +class PlaybookDescriptor: + """Structured recovery playbook option returned during diagnosis.""" + + playbook_id: str + label: str + action_id: str + description: str + eligible: bool + requires_confirmation: bool + reason: str + params_schema: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class RecoveryDiagnosis: + """Complete diagnostic snapshot of control-plane recovery needs.""" + + status: str + clean: bool + stale_runtime: dict[str, Any] + master_parity: dict[str, Any] + stale_binding: dict[str, Any] + contamination: dict[str, Any] + worktree_anomalies: tuple[str, ...] + playbooks: tuple[PlaybookDescriptor, ...] + reasons: tuple[str, ...] + + +def _repo_root(custom_path: Path | str | None = None) -> Path: + if custom_path: + return Path(custom_path).resolve() + override = (os.environ.get("WEBUI_REPO_ROOT") or "").strip() + if override: + return Path(override).resolve() + return Path(__file__).resolve().parent.parent + + +def confirmation_phrase(playbook_id: str, target: str | None = None) -> str: + """Construct exact confirmation phrase required for a recovery playbook.""" + clean_target = (target or "").strip() + if clean_target: + return f"confirm {playbook_id} {clean_target}" + return f"confirm {playbook_id}" + + +def confirmation_matches( + playbook_id: str, confirmation: str | None, target: str | None = None +) -> bool: + expected = confirmation_phrase(playbook_id, target) + return str(confirmation or "").strip() == expected + + +def _build_ledger( + playbook_id: str, target: str | None = None +) -> tuple[RecoveryLedgerEntry, ...]: + if playbook_id == PLAYBOOK_CLEAR_STALE_BINDING: + return ( + RecoveryLedgerEntry(1, "quiesce", "Stop admitting new gated mutations."), + RecoveryLedgerEntry( + 2, + "clear_env", + f"Remove stale env binding GITEA_ACTIVE_WORKTREE ({target or 'active'}).", + ), + RecoveryLedgerEntry( + 3, "audit", "Record clear_stale_binding event in console audit log." + ), + RecoveryLedgerEntry( + 4, "revalidate", "Re-run diagnosis to verify clean binding state." + ), + ) + if playbook_id == PLAYBOOK_REBIND_SESSION: + return ( + RecoveryLedgerEntry(1, "quiesce", "Stop admitting new gated mutations."), + RecoveryLedgerEntry( + 2, + "rebind_worktree", + f"Rebind session worktree context safely to {target or 'target worktree'}.", + ), + RecoveryLedgerEntry( + 3, "audit", "Record rebind_session_worktree event in console audit log." + ), + RecoveryLedgerEntry( + 4, "revalidate", "Re-run diagnosis to verify worktree binding state." + ), + ) + if playbook_id == PLAYBOOK_RECONCILE_CLEANUPS: + return ( + RecoveryLedgerEntry(1, "quiesce", "Stop admitting new gated mutations."), + RecoveryLedgerEntry( + 2, + "reconcile_cleanups", + "Execute sanctioned reconciler cleanup for merged or superseded PRs.", + ), + RecoveryLedgerEntry( + 3, "audit", "Record reconcile_cleanups event in console audit log." + ), + RecoveryLedgerEntry( + 4, "revalidate", "Re-run worktree scanner to verify clean tree." + ), + ) + if playbook_id == PLAYBOOK_SANCTIONED_RESTART: + restart_ledger = sanctioned_restart._mutation_ledger( + target or "gitea-author", sanctioned_restart.MODE_RESTART + ) + return tuple( + RecoveryLedgerEntry( + sequence=e.sequence, + step=e.step, + summary=e.summary, + executes_process_kill=e.executes_process_kill, + ) + for e in restart_ledger + ) + return ( + RecoveryLedgerEntry(1, "unspecified", f"Execute recovery playbook {playbook_id}."), + ) + + +def diagnose_recovery( + repo_path: Path | str | None = None, + env: dict[str, str] | None = None, + active_worktree_val: str | None = None, + session_lease_wt: str | None = None, + role_kind: str | None = None, +) -> RecoveryDiagnosis: + """Run full control-plane diagnostics to determine recovery needs and options.""" + root = _repo_root(repo_path) + source_env = dict(env) if env is not None else dict(os.environ) + reasons: list[str] = [] + + # 1. Stale runtime assessment + stale_runtime_obj = system_health.assess_stale_runtime(root) + stale_runtime_dict = { + "daemon_head": stale_runtime_obj.daemon_head, + "checkout_head": stale_runtime_obj.checkout_head, + "remote_head": stale_runtime_obj.remote_head, + "stale": stale_runtime_obj.stale, + "determinable": stale_runtime_obj.determinable, + "mutation_safe": stale_runtime_obj.mutation_safe, + "reasons": list(stale_runtime_obj.reasons), + } + if stale_runtime_obj.stale: + reasons.append("Runtime HEAD disagrees with checkout/remote HEAD.") + + # 2. Master parity assessment + checkout_head = stale_runtime_obj.checkout_head + startup_dict = master_parity_gate.capture_startup_parity(str(root), head=checkout_head) + parity_dict = master_parity_gate.assess_master_parity(startup_dict, checkout_head) + if not parity_dict.get("in_parity", True): + reasons.append("Repository is not in master parity.") + + # 3. Worktree binding classification + boot_bindings = stale_binding_recovery.snapshot_boot_bindings(source_env) + active_val = ( + active_worktree_val + if active_worktree_val is not None + else source_env.get(stale_binding_recovery.ACTIVE_WORKTREE_ENV) + ) + boot_inherited = bool(boot_bindings.get("active_worktree") and active_val == boot_bindings.get("active_worktree")) + + path_exists = None + if active_val: + path_exists = os.path.exists(os.path.realpath(active_val)) + + binding_class = stale_binding_recovery.classify_active_worktree_binding( + active_value=active_val, + session_lease_worktree=session_lease_wt, + boot_inherited=boot_inherited, + path_exists=path_exists, + role_kind=role_kind, + ) + + if binding_class.get("clear_eligible"): + reasons.append( + f"Active worktree binding is stale ({binding_class.get('classification')})." + ) + elif binding_class.get("classification") == stale_binding_recovery.CLASSIFICATION_UNVERIFIED_INHERITED: + reasons.append("Inherited worktree binding is unverified.") + + # 4. Contamination assessment (#630) + contamination_dict = runtime_recovery_guard.assess_contamination_gate( + marker=None, task=None, actual_role=role_kind + ) + if contamination_dict.get("block"): + reasons.append("Runtime is contaminated by manual process kill (#630).") + + # 5. Worktree scanner hygiene & anomalies + hygiene = worktree_scanner.load_hygiene_snapshot(project_root=str(root)) + worktree_anomalies = hygiene.anomalies + + # Determine status & eligible playbooks + playbooks: list[PlaybookDescriptor] = [] + + # Playbook 1: Clear Stale Binding + clear_eligible = bool(binding_class.get("clear_eligible")) + playbooks.append( + PlaybookDescriptor( + playbook_id=PLAYBOOK_CLEAR_STALE_BINDING, + label="Clear Stale Worktree Binding", + action_id=ACTION_CLEAR_STALE_BINDING, + description="Clear provably stale or superseded GITEA_ACTIVE_WORKTREE environment binding.", + eligible=clear_eligible, + requires_confirmation=True, + reason=( + f"Binding classified as {binding_class.get('classification')}; clear is authorized." + if clear_eligible + else "Active worktree binding is clean, corroborated, or absent." + ), + ) + ) + + # Playbook 2: Rebind Session Worktree + rebind_eligible = bool( + active_val + or binding_class.get("classification") == stale_binding_recovery.CLASSIFICATION_UNVERIFIED_INHERITED + ) + playbooks.append( + PlaybookDescriptor( + playbook_id=PLAYBOOK_REBIND_SESSION, + label="Rebind Session Worktree", + action_id=ACTION_REBIND_SESSION, + description="Rebind or synchronize session worktree binding safely with active lease.", + eligible=rebind_eligible, + requires_confirmation=True, + reason=( + "Session worktree binding can be rebound to verified lease worktree." + if rebind_eligible + else "Session worktree is properly bound." + ), + params_schema={"target_worktree": "string"}, + ) + ) + + # Playbook 3: Reconcile Cleanups + reconcile_eligible = bool(hygiene.anomalies or any(e.classification in {"stale-clean", "detached-review"} for e in hygiene.entries)) + playbooks.append( + PlaybookDescriptor( + playbook_id=PLAYBOOK_RECONCILE_CLEANUPS, + label="Trigger Reconciler Cleanups", + action_id=ACTION_RECONCILE_CLEANUPS, + description="Run sanctioned reconciler cleanup preview and apply for merged/superseded PR branches.", + eligible=reconcile_eligible, + requires_confirmation=True, + reason=( + f"Worktree hygiene scanner detected {len(hygiene.anomalies)} anomalies and cleanups needed." + if reconcile_eligible + else "No reconciler cleanups pending." + ), + ) + ) + + # Playbook 4: Sanctioned Restart + restart_eligible = bool(stale_runtime_obj.stale or contamination_dict.get("contaminated")) + playbooks.append( + PlaybookDescriptor( + playbook_id=PLAYBOOK_SANCTIONED_RESTART, + label="Sanctioned MCP Restart", + action_id=sanctioned_restart.ACTION_RESTART_NAMESPACE, + description="Restart MCP daemon via configured host supervisor without manual process kill.", + eligible=restart_eligible, + requires_confirmation=True, + reason=( + "Stale runtime or contamination detected; host supervisor restart available." + if restart_eligible + else "Runtime is healthy and clean." + ), + params_schema={"namespace": "string", "mode": "restart|reload"}, + ) + ) + + clean = not reasons and not contamination_dict.get("contaminated") + if contamination_dict.get("contaminated"): + status = STATUS_BLOCKED_CONTAMINATION + elif reasons: + status = STATUS_ACTION_REQUIRED + else: + status = STATUS_HEALTHY + + return RecoveryDiagnosis( + status=status, + clean=clean, + stale_runtime=stale_runtime_dict, + master_parity=parity_dict, + stale_binding=binding_class, + contamination=contamination_dict, + worktree_anomalies=tuple(worktree_anomalies), + playbooks=tuple(playbooks), + reasons=tuple(reasons), + ) + + +def build_recovery_preview( + playbook_id: str, + target: str | None = None, + params: dict[str, Any] | None = None, + principal: console_authz.Principal | None = None, + env: dict[str, str] | None = None, +) -> dict[str, Any]: + """Generate dry-run preview & mutation ledger for a recovery playbook.""" + if playbook_id not in KNOWN_PLAYBOOKS: + return { + "allowed": False, + "error": "unknown_playbook", + "detail": f"Playbook {playbook_id!r} is not a registered recovery playbook.", + } + + action_id = PLAYBOOK_ACTIONS[playbook_id] + action = console_authz.get_action(action_id) + decision = console_authz.authorize(action_id, principal) + phrase = confirmation_phrase(playbook_id, target) + ledger = _build_ledger(playbook_id, target) + + return { + "playbook_id": playbook_id, + "action_id": action_id, + "target": target, + "required_role": action.minimum_role if action else console_authz.OPERATOR, + "required_permission": action.mcp_permission if action else "gitea.read", + "requires_confirmation": True, + "confirmation_phrase": phrase, + "mutation_ledger": [asdict(entry) for entry in ledger], + "authorization": decision.to_dict(), + "params": dict(params or {}), + "execution_enabled": False, + } + + +def execute_recovery_playbook( + playbook_id: str, + confirmation: str | None = None, + target: str | None = None, + params: dict[str, Any] | None = None, + principal: console_authz.Principal | None = None, + env: dict[str, str] | None = None, + request_id: str | None = None, + session_id: str | None = None, +) -> dict[str, Any]: + """Gated execution of a recovery playbook with audit logging and revalidation.""" + if playbook_id not in KNOWN_PLAYBOOKS: + return { + "success": False, + "allowed": False, + "error": "unknown_playbook", + "detail": f"Playbook {playbook_id!r} is not known.", + } + + action_id = PLAYBOOK_ACTIONS[playbook_id] + source_env = dict(env) if env is not None else dict(os.environ) + + # 1. Authorization check + decision = console_authz.authorize(action_id, principal) + if not decision.allowed: + console_audit.record_event( + action_id=action_id, + result=console_audit.RESULT_DENIED, + principal=principal, + target={"playbook_id": playbook_id, "target": target}, + reason_code=decision.reason_code, + detail=decision.detail, + request_id=request_id, + session_id=session_id, + ) + return { + "success": False, + "allowed": False, + "error": decision.reason_code, + "detail": decision.detail, + } + + # 2. Confirmation phrase check + if not confirmation_matches(playbook_id, confirmation, target): + expected = confirmation_phrase(playbook_id, target) + detail = f"Confirmation phrase mismatch. Expected: {expected!r}" + console_audit.record_event( + action_id=action_id, + result=console_audit.RESULT_DENIED, + principal=principal, + target={"playbook_id": playbook_id, "target": target}, + reason_code="confirmation_mismatch", + detail=detail, + request_id=request_id, + session_id=session_id, + ) + return { + "success": False, + "allowed": False, + "error": "confirmation_mismatch", + "detail": detail, + "expected_confirmation_phrase": expected, + } + + # 3. Contamination rule (#630) check + role_str = principal.role if principal else None + contam = runtime_recovery_guard.assess_contamination_gate(marker=None, task=action_id, actual_role=role_str) + if contam.get("block"): + if playbook_id != PLAYBOOK_RECONCILE_CLEANUPS: + detail = "Runtime is contaminated by a manual process kill (#630). Run reconciler cleanup playbook first." + console_audit.record_event( + action_id=action_id, + result=console_audit.RESULT_DENIED, + principal=principal, + target={"playbook_id": playbook_id, "target": target}, + reason_code="contaminated_runtime", + detail=detail, + request_id=request_id, + session_id=session_id, + ) + return { + "success": False, + "allowed": False, + "error": "contaminated_runtime", + "detail": detail, + } + + # 4. Execute playbook action + applied_result: dict[str, Any] = {"performed": False} + if playbook_id == PLAYBOOK_CLEAR_STALE_BINDING: + diagnosis = diagnose_recovery(env=source_env) + plan = stale_binding_recovery.plan_recovery(diagnosis.stale_binding) + applied_result = stale_binding_recovery.apply_recovery(plan, env=source_env) + elif playbook_id == PLAYBOOK_REBIND_SESSION: + target_wt = target or (params or {}).get("target_worktree") + if target_wt: + source_env[stale_binding_recovery.ACTIVE_WORKTREE_ENV] = target_wt + applied_result = { + "performed": True, + "rebound_worktree": target_wt, + "cleared_stale": True, + } + else: + applied_result = { + "performed": False, + "reason": "No target_worktree specified for rebind.", + } + elif playbook_id == PLAYBOOK_RECONCILE_CLEANUPS: + try: + snapshot = merged_cleanup_reconcile.reconcile_merged_cleanups( + apply=True, project_root=str(_repo_root()) + ) + applied_result = { + "performed": True, + "reconciled_count": len(snapshot.get("reconciled") or []), + "snapshot": snapshot, + } + except Exception as exc: + applied_result = { + "performed": False, + "error": str(exc), + } + elif playbook_id == PLAYBOOK_SANCTIONED_RESTART: + ns = target or (params or {}).get("namespace", "gitea-author") + md = (params or {}).get("mode", sanctioned_restart.MODE_RESTART) + restart_res = sanctioned_restart.execute_restart( + namespace=ns, + mode=md, + principal=principal, + confirmation=f"{md} {ns}", + env=source_env, + request_id=request_id, + session_id=session_id, + ) + applied_result = restart_res + + performed = bool(applied_result.get("performed") or applied_result.get("allowed")) + + # 5. Record Audit Log + audit_record = console_audit.record_event( + action_id=action_id, + result=console_audit.RESULT_ALLOWED if performed else console_audit.RESULT_DENIED, + principal=principal, + target={"playbook_id": playbook_id, "target": target}, + reason_code="recovery_executed" if performed else "recovery_failed", + detail=f"Executed recovery playbook {playbook_id}", + request_id=request_id, + session_id=session_id, + metadata={"applied_result": applied_result}, + ) + + # 6. Post-recovery verification recheck + post_verification = verify_post_recovery(env=source_env) + + return { + "success": performed, + "allowed": True, + "playbook_id": playbook_id, + "action_id": action_id, + "applied_result": applied_result, + "audit": audit_record, + "post_recovery_verification": post_verification, + } + + +def verify_post_recovery( + repo_path: Path | str | None = None, env: dict[str, str] | None = None +) -> dict[str, Any]: + """Revalidate control-plane state post-recovery before clean status.""" + diag = diagnose_recovery(repo_path, env) + return { + "clean": diag.clean, + "status": diag.status, + "stale_runtime_clean": not diag.stale_runtime.get("stale"), + "binding_clean": not diag.stale_binding.get("clear_eligible"), + "contamination_clean": not diag.contamination.get("contaminated"), + "anomalies_count": len(diag.worktree_anomalies), + "reasons": list(diag.reasons), + } diff --git a/webui/gated_actions.py b/webui/gated_actions.py index d914874..7e42861 100644 --- a/webui/gated_actions.py +++ b/webui/gated_actions.py @@ -178,6 +178,13 @@ def build_action_registry() -> ActionRegistry: ("system.restart_namespace", "Restart MCP namespace", "restart_namespace", "host.supervisor_restart", "Restart one MCP namespace via the host supervisor."), + # #644: Phase 2 recovery playbooks & controls. + ("system.clear_stale_binding", "Clear stale binding", "clear_stale_binding", + "console.clear_stale_binding", "Clear provably stale or superseded env binding."), + ("system.rebind_session_worktree", "Rebind session worktree", "rebind_session_worktree", + "console.rebind_session_worktree", "Rebind session worktree to verified lease."), + ("system.reconcile_cleanups", "Reconcile cleanups", "reconcile_cleanups", + "console.reconcile_cleanups", "Run reconciler cleanup for merged or superseded PRs."), ) actions = tuple( GatedAction( diff --git a/webui/system_health_views.py b/webui/system_health_views.py index c0474c7..980e92a 100644 --- a/webui/system_health_views.py +++ b/webui/system_health_views.py @@ -270,26 +270,47 @@ def _probe_error_card(snapshot: SystemHealthSnapshot) -> str: def _recovery_card() -> str: - """Sanctioned recovery pointers only — never a manual process kill (#630).""" - return ( - "
" - "

Recovery

" - "

This dashboard is read-only. Restart and reload " - "controls arrive in Phase 2 (#642); until then recovery runs through " - "the sanctioned client reconnect / operator restart path.

" - "" - "
" - ) + """Sanctioned recovery controls & playbooks (#644, Phase 2).""" + try: + from webui import console_recovery + diag = console_recovery.diagnose_recovery() + status_badge = f"{diag.status}" + playbook_lis = "" + for pb in diag.playbooks: + elig = "eligible" if pb.eligible else "disabled" + playbook_lis += ( + f"
  • {pb.label} ({pb.playbook_id}) — " + f"{elig}: {pb.description} " + f"({pb.reason})
    • " + ) + reasons_html = "" + if diag.reasons: + items = "".join(f"
  • {r}
    • " for r in diag.reasons) + reasons_html = f"" + else: + reasons_html = "

    No recovery actions currently required. Control plane is healthy.

    " + + return ( + "
    " + f"

    Sanctioned Recovery Controls (Phase 2 #644) {status_badge}

    " + "

    Guided recovery wizard: Diagnose → Preview → Confirm → Verify. " + "Reconnect the MCP client from the IDE, then re-run the blocked cycle. " + "Never kill the daemon process manually: unmanaged kills are recorded as runtime contamination (#630).

    " + f"{reasons_html}" + "

    Available Recovery Playbooks

    " + f"" + "

    APIs: /api/v1/system/recovery/diagnose, " + "/api/v1/system/recovery/preview, /api/v1/system/recovery/apply, " + "/api/v1/system/recovery/verify.

    " + "
    " + ) + except Exception as exc: + return ( + "
    " + "

    Sanctioned Recovery Controls (Phase 2 #644)

    " + f"

    Recovery diagnostics unavailable: {exc}

    " + "
    " + ) def render_system_health_page(snapshot: SystemHealthSnapshot) -> str: