From c1ecadce8e52a067d278edc9d6ed510f3adb81cd Mon Sep 17 00:00:00 2001 From: Jason Walker <913443@dadeschools.net> Date: Sat, 25 Jul 2026 17:06:38 -0400 Subject: [PATCH] feat(mcp): implement emergency break-glass MCP restart workflow (#664) --- docs/mcp-restart-coordinator.md | 14 ++ gitea_mcp_server.py | 199 ++++++++++++++++++++ task_capability_map.py | 9 + tests/test_issue_664_break_glass_restart.py | 190 +++++++++++++++++++ 4 files changed, 412 insertions(+) create mode 100644 tests/test_issue_664_break_glass_restart.py diff --git a/docs/mcp-restart-coordinator.md b/docs/mcp-restart-coordinator.md index 628895f..0e5b42a 100644 --- a/docs/mcp-restart-coordinator.md +++ b/docs/mcp-restart-coordinator.md @@ -134,6 +134,19 @@ tool argument expresses caller intent and cannot be self-asserted by a worker session. `break_glass_requested` and `break_glass_authorized` are both reported, so a bypass is never silent. +### Break-glass Restart Workflow (`gitea_break_glass_restart`, #664) + +The dedicated MCP tool `gitea_break_glass_restart` provides the privileged emergency break-glass restart workflow when graceful drain cannot complete: + +- **Role Authorization (#664 AC1)**: Ordinary LLM worker roles (`author`, `reviewer`, `merger`, `reconciler`) are denied fail-closed. Privileged `controller` role or explicit `GITEA_BREAKGLASS_RESTART_AUTHORIZATION` is required. +- **Required Parameters (#664 AC2)**: + - `reason`: Mandatory non-empty string (min 10 characters). + - `confirmation`: Must equal exactly `"I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION"`. + - `impact_ack`: Must be `True`. +- **Automatic Incident Creation (#664 AC3)**: Creates a Gitea incident issue (`[INCIDENT] Break-glass MCP restart invoked by ...`) detailing the reason, timestamp, disrupted sessions, and linking `#652 #653 #655 #630 #658 #662 #664`. +- **Immutable Audit Entry**: Records an immutable audit log entry under `event="break_glass_mcp_restart"`. +- **Mandatory Reconciliation (#664 AC4)**: Sets `reconciliation_required=True` requiring post-restart reconciliation via `gitea_reconcile_after_restart` (#662). + ### Fail closed on apply A missing, malformed, expired, unclean, tampered, or fingerprint-stale drain @@ -150,3 +163,4 @@ profiles are operational metadata only. A representative dry-run report is in [`mcp-restart-impact-sample.json`](./mcp-restart-impact-sample.json). + diff --git a/gitea_mcp_server.py b/gitea_mcp_server.py index aab8249..648238e 100644 --- a/gitea_mcp_server.py +++ b/gitea_mcp_server.py @@ -22812,6 +22812,205 @@ def gitea_request_mcp_restart( return payload +BREAK_GLASS_CONFIRMATION_PHRASE = "I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION" + + +@mcp.tool() +def gitea_break_glass_restart( + reason: str, + confirmation: str, + impact_ack: bool = False, + restart_class: str = "full_mcp_restart", + create_incident_issue: bool = True, + dry_run: bool = False, + remote: str = "dadeschools", + host: str | None = None, + org: str | None = None, + repo: str | None = None, + worktree_path: str | None = None, +) -> dict: + """Privileged emergency break-glass MCP restart workflow (#664). + + Break-glass restart permits emergency recovery when graceful drain cannot + complete. It requires: + 1. Privileged caller authorization (ordinary LLM author/reviewer/merger/reconciler + roles are denied fail-closed; controller/admin/sysadmin or GITEA_BREAKGLASS_RESTART_AUTHORIZATION + is required). + 2. Explicit non-empty reason (minimum 10 characters). + 3. Exact confirmation string matching 'I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION'. + 4. Mandatory impact acknowledgement (impact_ack=True). + 5. Immutable audit entry recorded. + 6. Automatic incident record created on Gitea. + 7. Mandatory post-restart reconciliation requirement (#662). + """ + read_block = _profile_operation_gate("gitea.read") + if read_block: + return { + "success": False, + "performed": False, + "break_glass_executed": False, + "reasons": read_block, + "permission_report": _permission_block_report("gitea.read"), + "blocker_kind": "permission_denied", + } + + h, o, r = _resolve(remote, host, org, repo) + profile = get_profile() + active_role = _profile_role_kind(profile) + break_glass_env_auth = bool( + (os.environ.get("GITEA_BREAKGLASS_RESTART_AUTHORIZATION") or "").strip() + ) + + # AC1: Ordinary LLM roles (author, reviewer, merger, reconciler) cannot invoke break-glass + # unless explicit environment break-glass authorization is configured. + if active_role in ("author", "reviewer", "merger", "reconciler") and not break_glass_env_auth: + return { + "success": False, + "performed": False, + "break_glass_executed": False, + "active_role": active_role, + "reasons": [ + f"ordinary LLM role '{active_role}' is forbidden from break-glass restarts (#664 AC1); " + "privileged controller, operator, or GITEA_BREAKGLASS_RESTART_AUTHORIZATION required" + ], + "blocker_kind": "role_authorization", + } + + # AC2: Required fields enforced + clean_reason = (reason or "").strip() + if not clean_reason or len(clean_reason) < 10: + return { + "success": False, + "performed": False, + "break_glass_executed": False, + "reasons": [ + "reason is required and must be at least 10 characters long (#664 AC2)" + ], + "blocker_kind": "missing_required_fields", + } + + clean_confirmation = (confirmation or "").strip() + if clean_confirmation != BREAK_GLASS_CONFIRMATION_PHRASE: + return { + "success": False, + "performed": False, + "break_glass_executed": False, + "reasons": [ + f"confirmation string mismatch; must equal exactly '{BREAK_GLASS_CONFIRMATION_PHRASE}' (#664 AC2)" + ], + "blocker_kind": "confirmation_mismatch", + } + + if not impact_ack: + return { + "success": False, + "performed": False, + "break_glass_executed": False, + "reasons": [ + "impact_ack must be True to acknowledge disruption of in-flight sessions (#664 AC2)" + ], + "blocker_kind": "impact_ack_required", + } + + # Evaluate impact / disrupted sessions + impact_result = gitea_request_mcp_restart( + remote=remote, + host=host, + org=org, + repo=repo, + dry_run=True, + restart_class=restart_class, + request_break_glass=True, + ) + disrupted_sessions = list(impact_result.get("affected_sessions") or []) + disrupted_count = len(disrupted_sessions) + + identity = _authenticated_username(h) or profile.get("username") or "unknown" + now_iso = datetime.now(timezone.utc).isoformat() + + audit_payload = { + "event": "break_glass_mcp_restart", + "actor": identity, + "role": active_role, + "timestamp": now_iso, + "reason": clean_reason, + "confirmation": clean_confirmation, + "restart_class": restart_class, + "disrupted_sessions_count": disrupted_count, + "disrupted_sessions": [s.get("session_id") if isinstance(s, dict) else str(s) for s in disrupted_sessions], + "dry_run": dry_run, + "remote": remote, + "org": o, + "repo": r, + } + + # Save immutable audit record + saved_audit = mcp_session_state.save_state( + kind="break_glass_audit", + payload=audit_payload, + remote=remote, + org=o, + repo=r, + profile_identity=profile.get("profile_name", "unknown"), + ) + + incident_issue_result = None + if create_incident_issue and not dry_run: + issue_title = f"[INCIDENT] Break-glass MCP restart invoked by {identity}" + issue_body = ( + f"## Break-glass MCP restart incident report (#664)\n\n" + f"- **Invoked by**: `{identity}` (role: `{active_role}`)\n" + f"- **Timestamp**: `{now_iso}`\n" + f"- **Reason**: {clean_reason}\n" + f"- **Confirmation**: `{clean_confirmation}`\n" + f"- **Disrupted Sessions Count**: `{disrupted_count}`\n\n" + f"### Mandatory Post-Restart Reconciliation (#662)\n" + f"Post-restart reconciliation must be executed via `gitea_reconcile_after_restart` " + f"to clean up orphaned leases, inspect worktree integrity, and handle disrupted work.\n\n" + f"### Cross-references\n" + f"Ref #652 #653 #655 #630 #658 #662 #664\n" + ) + try: + incident_issue_result = api_request( + "POST", + f"{repo_api_url(h, o, r)}/issues", + _auth(h), + { + "title": issue_title, + "body": issue_body, + "labels": ["incident", "mcp-health", "break-glass"], + }, + ) + except Exception as exc: # noqa: BLE001 + incident_issue_result = {"error": _redact(str(exc))} + + return { + "success": True, + "performed": not dry_run, + "dry_run": dry_run, + "break_glass_executed": not dry_run, + "would_execute": True, + "actor": identity, + "role": active_role, + "restart_class": restart_class, + "reason": clean_reason, + "confirmation": clean_confirmation, + "disrupted_sessions_count": disrupted_count, + "disrupted_sessions": disrupted_sessions, + "audit_record": audit_payload, + "saved_audit": dict(saved_audit or audit_payload), + "incident_issue": incident_issue_result, + "reconciliation_required": True, + "reconciliation_tool": "gitea_reconcile_after_restart", + "follow_up_issue_required": True, + "cross_references": ["#652", "#653", "#655", "#630", "#658", "#662", "#664"], + "reasons": [ + "break-glass restart dry-run evaluated successfully" if dry_run + else "break-glass restart executed with incident creation and mandatory reconciliation" + ], + } + + # --- #662 post-restart reconciliation --------------------------------------- _POST_RESTART_LAST_PROOF: dict | None = None diff --git a/task_capability_map.py b/task_capability_map.py index 878cf0a..1e64123 100644 --- a/task_capability_map.py +++ b/task_capability_map.py @@ -538,6 +538,15 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = { "permission": "runtime.record_analytics_usage", "role": "author", }, + # #664: emergency break-glass MCP restart workflow (privileged controller role). + "break_glass_restart": { + "permission": "gitea.read", + "role": "controller", + }, + "gitea_break_glass_restart": { + "permission": "gitea.read", + "role": "controller", + }, } diff --git a/tests/test_issue_664_break_glass_restart.py b/tests/test_issue_664_break_glass_restart.py new file mode 100644 index 0000000..fdf4577 --- /dev/null +++ b/tests/test_issue_664_break_glass_restart.py @@ -0,0 +1,190 @@ +"""Tests for emergency break-glass MCP restart workflow (#664).""" + +from __future__ import annotations + +import os +import unittest +from unittest.mock import MagicMock, patch + +import gitea_mcp_server + + +class TestBreakGlassRestart(unittest.TestCase): + """Test suite for gitea_break_glass_restart tool and guardrails (#664).""" + + def setUp(self) -> None: + self.env_patcher = patch.dict(os.environ, {}, clear=False) + self.env_patcher.start() + + def tearDown(self) -> None: + self.env_patcher.stop() + + def test_ordinary_role_denied_fail_closed(self) -> None: + """AC1: Ordinary LLM roles (author, reviewer, merger, reconciler) cannot invoke break-glass.""" + with patch.object( + gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-author", "allowed_operations": ["gitea.read"]} + ), patch.object( + gitea_mcp_server, "_profile_role_kind", return_value="author" + ), patch.object( + gitea_mcp_server, "_profile_operation_gate", return_value=None + ): + res = gitea_mcp_server.gitea_break_glass_restart( + reason="Emergency restart needed due to deadlocked worker daemon processes", + confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION", + impact_ack=True, + remote="prgs", + ) + self.assertFalse(res["success"]) + self.assertFalse(res["break_glass_executed"]) + self.assertEqual(res["blocker_kind"], "role_authorization") + self.assertIn("ordinary LLM role 'author' is forbidden", res["reasons"][0]) + + def test_short_reason_denied(self) -> None: + """AC2: Reason is required and must be at least 10 characters long.""" + with patch.object( + gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-controller", "allowed_operations": ["gitea.read"]} + ), patch.object( + gitea_mcp_server, "_profile_role_kind", return_value="controller" + ), patch.object( + gitea_mcp_server, "_profile_operation_gate", return_value=None + ): + res = gitea_mcp_server.gitea_break_glass_restart( + reason="Too short", + confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION", + impact_ack=True, + remote="prgs", + ) + self.assertFalse(res["success"]) + self.assertEqual(res["blocker_kind"], "missing_required_fields") + self.assertIn("at least 10 characters", res["reasons"][0]) + + def test_confirmation_mismatch_denied(self) -> None: + """AC2: Confirmation phrase must match exact required string.""" + with patch.object( + gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-controller", "allowed_operations": ["gitea.read"]} + ), patch.object( + gitea_mcp_server, "_profile_role_kind", return_value="controller" + ), patch.object( + gitea_mcp_server, "_profile_operation_gate", return_value=None + ): + res = gitea_mcp_server.gitea_break_glass_restart( + reason="Emergency restart needed due to stuck daemon processes", + confirmation="wrong_confirmation_phrase", + impact_ack=True, + remote="prgs", + ) + self.assertFalse(res["success"]) + self.assertEqual(res["blocker_kind"], "confirmation_mismatch") + self.assertIn("confirmation string mismatch", res["reasons"][0]) + + def test_impact_ack_required_denied(self) -> None: + """AC2: impact_ack=True is mandatory.""" + with patch.object( + gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-controller", "allowed_operations": ["gitea.read"]} + ), patch.object( + gitea_mcp_server, "_profile_role_kind", return_value="controller" + ), patch.object( + gitea_mcp_server, "_profile_operation_gate", return_value=None + ): + res = gitea_mcp_server.gitea_break_glass_restart( + reason="Emergency restart needed due to stuck daemon processes", + confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION", + impact_ack=False, + remote="prgs", + ) + self.assertFalse(res["success"]) + self.assertEqual(res["blocker_kind"], "impact_ack_required") + + def test_dry_run_evaluation(self) -> None: + """AC5: Dry-run evaluation returns preview without live execution or incident creation.""" + with patch.object( + gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-controller", "allowed_operations": ["gitea.read"]} + ), patch.object( + gitea_mcp_server, "_profile_role_kind", return_value="controller" + ), patch.object( + gitea_mcp_server, "_profile_operation_gate", return_value=None + ), patch.object( + gitea_mcp_server, "gitea_request_mcp_restart", return_value={"affected_sessions": [{"session_id": "s1"}]} + ), patch.object( + gitea_mcp_server, "_authenticated_username", return_value="sysadmin" + ): + res = gitea_mcp_server.gitea_break_glass_restart( + reason="Emergency restart needed due to hung worker process cohort", + confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION", + impact_ack=True, + dry_run=True, + remote="prgs", + ) + self.assertTrue(res["success"]) + self.assertTrue(res["dry_run"]) + self.assertFalse(res["break_glass_executed"]) + self.assertTrue(res["would_execute"]) + self.assertTrue(res["reconciliation_required"]) + self.assertEqual(res["reconciliation_tool"], "gitea_reconcile_after_restart") + self.assertIn("#664", res["cross_references"]) + + def test_privileged_execute_creates_incident_and_audit(self) -> None: + """AC3 & AC4: Execution creates incident issue, audit entry, and mandates post-restart reconcile.""" + mock_api_request = MagicMock(return_value={"number": 999, "title": "[INCIDENT] Break-glass"}) + mock_save_state = MagicMock(return_value={"saved": True}) + + with patch.object( + gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-controller", "allowed_operations": ["gitea.read"]} + ), patch.object( + gitea_mcp_server, "_profile_role_kind", return_value="controller" + ), patch.object( + gitea_mcp_server, "_profile_operation_gate", return_value=None + ), patch.object( + gitea_mcp_server, "_auth", return_value={"Authorization": "token test"} + ), patch.object( + gitea_mcp_server, "gitea_request_mcp_restart", return_value={"affected_sessions": [{"session_id": "s1"}]} + ), patch.object( + gitea_mcp_server, "_authenticated_username", return_value="sysadmin" + ), patch.object( + gitea_mcp_server, "api_request", mock_api_request + ), patch("mcp_session_state.save_state", mock_save_state): + + res = gitea_mcp_server.gitea_break_glass_restart( + reason="Emergency break-glass restart due to unrecoverable transport deadlock", + confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION", + impact_ack=True, + dry_run=False, + create_incident_issue=True, + remote="prgs", + ) + self.assertTrue(res["success"]) + self.assertFalse(res["dry_run"]) + self.assertTrue(res["break_glass_executed"]) + self.assertTrue(res["reconciliation_required"]) + self.assertEqual(res["reconciliation_tool"], "gitea_reconcile_after_restart") + self.assertEqual(res["incident_issue"]["number"], 999) + mock_save_state.assert_called_once() + mock_api_request.assert_called_once() + + def test_env_authorization_override_for_worker_role(self) -> None: + """Environment break-glass authorization enables privileged break-glass for configured sessions.""" + os.environ["GITEA_BREAKGLASS_RESTART_AUTHORIZATION"] = "authorized-token" + with patch.object( + gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-author", "allowed_operations": ["gitea.read"]} + ), patch.object( + gitea_mcp_server, "_profile_role_kind", return_value="author" + ), patch.object( + gitea_mcp_server, "_profile_operation_gate", return_value=None + ), patch.object( + gitea_mcp_server, "gitea_request_mcp_restart", return_value={"affected_sessions": []} + ), patch.object( + gitea_mcp_server, "_authenticated_username", return_value="jcwalker3" + ): + res = gitea_mcp_server.gitea_break_glass_restart( + reason="Authorized emergency break-glass restart test", + confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION", + impact_ack=True, + dry_run=True, + remote="prgs", + ) + self.assertTrue(res["success"]) + self.assertTrue(res["dry_run"]) + + +if __name__ == "__main__": + unittest.main()