feat(mcp): implement emergency break-glass MCP restart workflow (#664)

docs/mcp-restart-coordinator.md

@@ -134,6 +134,19 @@ tool argument expresses caller intent and cannot be self-asserted by a worker
 session. `break_glass_requested` and `break_glass_authorized` are both reported,
 so a bypass is never silent.
 
+### Break-glass Restart Workflow (`gitea_break_glass_restart`, #664)
+
+The dedicated MCP tool `gitea_break_glass_restart` provides the privileged emergency break-glass restart workflow when graceful drain cannot complete:
+
+- **Role Authorization (#664 AC1)**: Ordinary LLM worker roles (`author`, `reviewer`, `merger`, `reconciler`) are denied fail-closed. Privileged `controller` role or explicit `GITEA_BREAKGLASS_RESTART_AUTHORIZATION` is required.
+- **Required Parameters (#664 AC2)**:
+  - `reason`: Mandatory non-empty string (min 10 characters).
+  - `confirmation`: Must equal exactly `"I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION"`.
+  - `impact_ack`: Must be `True`.
+- **Automatic Incident Creation (#664 AC3)**: Creates a Gitea incident issue (`[INCIDENT] Break-glass MCP restart invoked by ...`) detailing the reason, timestamp, disrupted sessions, and linking `#652 #653 #655 #630 #658 #662 #664`.
+- **Immutable Audit Entry**: Records an immutable audit log entry under `event="break_glass_mcp_restart"`.
+- **Mandatory Reconciliation (#664 AC4)**: Sets `reconciliation_required=True` requiring post-restart reconciliation via `gitea_reconcile_after_restart` (#662).
+
 ### Fail closed on apply
 
 A missing, malformed, expired, unclean, tampered, or fingerprint-stale drain
@@ -150,3 +163,4 @@ profiles are operational metadata only.
 
 A representative dry-run report is in
 [`mcp-restart-impact-sample.json`](./mcp-restart-impact-sample.json).
+

gitea_mcp_server.py

@@ -22812,6 +22812,205 @@ def gitea_request_mcp_restart(
     return payload
 
 
+BREAK_GLASS_CONFIRMATION_PHRASE = "I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION"
+
+
+@mcp.tool()
+def gitea_break_glass_restart(
+    reason: str,
+    confirmation: str,
+    impact_ack: bool = False,
+    restart_class: str = "full_mcp_restart",
+    create_incident_issue: bool = True,
+    dry_run: bool = False,
+    remote: str = "dadeschools",
+    host: str | None = None,
+    org: str | None = None,
+    repo: str | None = None,
+    worktree_path: str | None = None,
+) -> dict:
+    """Privileged emergency break-glass MCP restart workflow (#664).
+
+    Break-glass restart permits emergency recovery when graceful drain cannot
+    complete. It requires:
+    1. Privileged caller authorization (ordinary LLM author/reviewer/merger/reconciler
+       roles are denied fail-closed; controller/admin/sysadmin or GITEA_BREAKGLASS_RESTART_AUTHORIZATION
+       is required).
+    2. Explicit non-empty reason (minimum 10 characters).
+    3. Exact confirmation string matching 'I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION'.
+    4. Mandatory impact acknowledgement (impact_ack=True).
+    5. Immutable audit entry recorded.
+    6. Automatic incident record created on Gitea.
+    7. Mandatory post-restart reconciliation requirement (#662).
+    """
+    read_block = _profile_operation_gate("gitea.read")
+    if read_block:
+        return {
+            "success": False,
+            "performed": False,
+            "break_glass_executed": False,
+            "reasons": read_block,
+            "permission_report": _permission_block_report("gitea.read"),
+            "blocker_kind": "permission_denied",
+        }
+
+    h, o, r = _resolve(remote, host, org, repo)
+    profile = get_profile()
+    active_role = _profile_role_kind(profile)
+    break_glass_env_auth = bool(
+        (os.environ.get("GITEA_BREAKGLASS_RESTART_AUTHORIZATION") or "").strip()
+    )
+
+    # AC1: Ordinary LLM roles (author, reviewer, merger, reconciler) cannot invoke break-glass
+    # unless explicit environment break-glass authorization is configured.
+    if active_role in ("author", "reviewer", "merger", "reconciler") and not break_glass_env_auth:
+        return {
+            "success": False,
+            "performed": False,
+            "break_glass_executed": False,
+            "active_role": active_role,
+            "reasons": [
+                f"ordinary LLM role '{active_role}' is forbidden from break-glass restarts (#664 AC1); "
+                "privileged controller, operator, or GITEA_BREAKGLASS_RESTART_AUTHORIZATION required"
+            ],
+            "blocker_kind": "role_authorization",
+        }
+
+    # AC2: Required fields enforced
+    clean_reason = (reason or "").strip()
+    if not clean_reason or len(clean_reason) < 10:
+        return {
+            "success": False,
+            "performed": False,
+            "break_glass_executed": False,
+            "reasons": [
+                "reason is required and must be at least 10 characters long (#664 AC2)"
+            ],
+            "blocker_kind": "missing_required_fields",
+        }
+
+    clean_confirmation = (confirmation or "").strip()
+    if clean_confirmation != BREAK_GLASS_CONFIRMATION_PHRASE:
+        return {
+            "success": False,
+            "performed": False,
+            "break_glass_executed": False,
+            "reasons": [
+                f"confirmation string mismatch; must equal exactly '{BREAK_GLASS_CONFIRMATION_PHRASE}' (#664 AC2)"
+            ],
+            "blocker_kind": "confirmation_mismatch",
+        }
+
+    if not impact_ack:
+        return {
+            "success": False,
+            "performed": False,
+            "break_glass_executed": False,
+            "reasons": [
+                "impact_ack must be True to acknowledge disruption of in-flight sessions (#664 AC2)"
+            ],
+            "blocker_kind": "impact_ack_required",
+        }
+
+    # Evaluate impact / disrupted sessions
+    impact_result = gitea_request_mcp_restart(
+        remote=remote,
+        host=host,
+        org=org,
+        repo=repo,
+        dry_run=True,
+        restart_class=restart_class,
+        request_break_glass=True,
+    )
+    disrupted_sessions = list(impact_result.get("affected_sessions") or [])
+    disrupted_count = len(disrupted_sessions)
+
+    identity = _authenticated_username(h) or profile.get("username") or "unknown"
+    now_iso = datetime.now(timezone.utc).isoformat()
+
+    audit_payload = {
+        "event": "break_glass_mcp_restart",
+        "actor": identity,
+        "role": active_role,
+        "timestamp": now_iso,
+        "reason": clean_reason,
+        "confirmation": clean_confirmation,
+        "restart_class": restart_class,
+        "disrupted_sessions_count": disrupted_count,
+        "disrupted_sessions": [s.get("session_id") if isinstance(s, dict) else str(s) for s in disrupted_sessions],
+        "dry_run": dry_run,
+        "remote": remote,
+        "org": o,
+        "repo": r,
+    }
+
+    # Save immutable audit record
+    saved_audit = mcp_session_state.save_state(
+        kind="break_glass_audit",
+        payload=audit_payload,
+        remote=remote,
+        org=o,
+        repo=r,
+        profile_identity=profile.get("profile_name", "unknown"),
+    )
+
+    incident_issue_result = None
+    if create_incident_issue and not dry_run:
+        issue_title = f"[INCIDENT] Break-glass MCP restart invoked by {identity}"
+        issue_body = (
+            f"## Break-glass MCP restart incident report (#664)\n\n"
+            f"- **Invoked by**: `{identity}` (role: `{active_role}`)\n"
+            f"- **Timestamp**: `{now_iso}`\n"
+            f"- **Reason**: {clean_reason}\n"
+            f"- **Confirmation**: `{clean_confirmation}`\n"
+            f"- **Disrupted Sessions Count**: `{disrupted_count}`\n\n"
+            f"### Mandatory Post-Restart Reconciliation (#662)\n"
+            f"Post-restart reconciliation must be executed via `gitea_reconcile_after_restart` "
+            f"to clean up orphaned leases, inspect worktree integrity, and handle disrupted work.\n\n"
+            f"### Cross-references\n"
+            f"Ref #652 #653 #655 #630 #658 #662 #664\n"
+        )
+        try:
+            incident_issue_result = api_request(
+                "POST",
+                f"{repo_api_url(h, o, r)}/issues",
+                _auth(h),
+                {
+                    "title": issue_title,
+                    "body": issue_body,
+                    "labels": ["incident", "mcp-health", "break-glass"],
+                },
+            )
+        except Exception as exc:  # noqa: BLE001
+            incident_issue_result = {"error": _redact(str(exc))}
+
+    return {
+        "success": True,
+        "performed": not dry_run,
+        "dry_run": dry_run,
+        "break_glass_executed": not dry_run,
+        "would_execute": True,
+        "actor": identity,
+        "role": active_role,
+        "restart_class": restart_class,
+        "reason": clean_reason,
+        "confirmation": clean_confirmation,
+        "disrupted_sessions_count": disrupted_count,
+        "disrupted_sessions": disrupted_sessions,
+        "audit_record": audit_payload,
+        "saved_audit": dict(saved_audit or audit_payload),
+        "incident_issue": incident_issue_result,
+        "reconciliation_required": True,
+        "reconciliation_tool": "gitea_reconcile_after_restart",
+        "follow_up_issue_required": True,
+        "cross_references": ["#652", "#653", "#655", "#630", "#658", "#662", "#664"],
+        "reasons": [
+            "break-glass restart dry-run evaluated successfully" if dry_run
+            else "break-glass restart executed with incident creation and mandatory reconciliation"
+        ],
+    }
+
+
 # --- #662 post-restart reconciliation ---------------------------------------
 
 _POST_RESTART_LAST_PROOF: dict | None = None

task_capability_map.py

@@ -538,6 +538,15 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = {
         "permission": "runtime.record_analytics_usage",
         "role": "author",
     },
+    # #664: emergency break-glass MCP restart workflow (privileged controller role).
+    "break_glass_restart": {
+        "permission": "gitea.read",
+        "role": "controller",
+    },
+    "gitea_break_glass_restart": {
+        "permission": "gitea.read",
+        "role": "controller",
+    },
 }
 
 

tests/test_issue_664_break_glass_restart.py

@@ -0,0 +1,190 @@
+"""Tests for emergency break-glass MCP restart workflow (#664)."""
+
+from __future__ import annotations
+
+import os
+import unittest
+from unittest.mock import MagicMock, patch
+
+import gitea_mcp_server
+
+
+class TestBreakGlassRestart(unittest.TestCase):
+    """Test suite for gitea_break_glass_restart tool and guardrails (#664)."""
+
+    def setUp(self) -> None:
+        self.env_patcher = patch.dict(os.environ, {}, clear=False)
+        self.env_patcher.start()
+
+    def tearDown(self) -> None:
+        self.env_patcher.stop()
+
+    def test_ordinary_role_denied_fail_closed(self) -> None:
+        """AC1: Ordinary LLM roles (author, reviewer, merger, reconciler) cannot invoke break-glass."""
+        with patch.object(
+            gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-author", "allowed_operations": ["gitea.read"]}
+        ), patch.object(
+            gitea_mcp_server, "_profile_role_kind", return_value="author"
+        ), patch.object(
+            gitea_mcp_server, "_profile_operation_gate", return_value=None
+        ):
+            res = gitea_mcp_server.gitea_break_glass_restart(
+                reason="Emergency restart needed due to deadlocked worker daemon processes",
+                confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION",
+                impact_ack=True,
+                remote="prgs",
+            )
+            self.assertFalse(res["success"])
+            self.assertFalse(res["break_glass_executed"])
+            self.assertEqual(res["blocker_kind"], "role_authorization")
+            self.assertIn("ordinary LLM role 'author' is forbidden", res["reasons"][0])
+
+    def test_short_reason_denied(self) -> None:
+        """AC2: Reason is required and must be at least 10 characters long."""
+        with patch.object(
+            gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-controller", "allowed_operations": ["gitea.read"]}
+        ), patch.object(
+            gitea_mcp_server, "_profile_role_kind", return_value="controller"
+        ), patch.object(
+            gitea_mcp_server, "_profile_operation_gate", return_value=None
+        ):
+            res = gitea_mcp_server.gitea_break_glass_restart(
+                reason="Too short",
+                confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION",
+                impact_ack=True,
+                remote="prgs",
+            )
+            self.assertFalse(res["success"])
+            self.assertEqual(res["blocker_kind"], "missing_required_fields")
+            self.assertIn("at least 10 characters", res["reasons"][0])
+
+    def test_confirmation_mismatch_denied(self) -> None:
+        """AC2: Confirmation phrase must match exact required string."""
+        with patch.object(
+            gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-controller", "allowed_operations": ["gitea.read"]}
+        ), patch.object(
+            gitea_mcp_server, "_profile_role_kind", return_value="controller"
+        ), patch.object(
+            gitea_mcp_server, "_profile_operation_gate", return_value=None
+        ):
+            res = gitea_mcp_server.gitea_break_glass_restart(
+                reason="Emergency restart needed due to stuck daemon processes",
+                confirmation="wrong_confirmation_phrase",
+                impact_ack=True,
+                remote="prgs",
+            )
+            self.assertFalse(res["success"])
+            self.assertEqual(res["blocker_kind"], "confirmation_mismatch")
+            self.assertIn("confirmation string mismatch", res["reasons"][0])
+
+    def test_impact_ack_required_denied(self) -> None:
+        """AC2: impact_ack=True is mandatory."""
+        with patch.object(
+            gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-controller", "allowed_operations": ["gitea.read"]}
+        ), patch.object(
+            gitea_mcp_server, "_profile_role_kind", return_value="controller"
+        ), patch.object(
+            gitea_mcp_server, "_profile_operation_gate", return_value=None
+        ):
+            res = gitea_mcp_server.gitea_break_glass_restart(
+                reason="Emergency restart needed due to stuck daemon processes",
+                confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION",
+                impact_ack=False,
+                remote="prgs",
+            )
+            self.assertFalse(res["success"])
+            self.assertEqual(res["blocker_kind"], "impact_ack_required")
+
+    def test_dry_run_evaluation(self) -> None:
+        """AC5: Dry-run evaluation returns preview without live execution or incident creation."""
+        with patch.object(
+            gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-controller", "allowed_operations": ["gitea.read"]}
+        ), patch.object(
+            gitea_mcp_server, "_profile_role_kind", return_value="controller"
+        ), patch.object(
+            gitea_mcp_server, "_profile_operation_gate", return_value=None
+        ), patch.object(
+            gitea_mcp_server, "gitea_request_mcp_restart", return_value={"affected_sessions": [{"session_id": "s1"}]}
+        ), patch.object(
+            gitea_mcp_server, "_authenticated_username", return_value="sysadmin"
+        ):
+            res = gitea_mcp_server.gitea_break_glass_restart(
+                reason="Emergency restart needed due to hung worker process cohort",
+                confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION",
+                impact_ack=True,
+                dry_run=True,
+                remote="prgs",
+            )
+            self.assertTrue(res["success"])
+            self.assertTrue(res["dry_run"])
+            self.assertFalse(res["break_glass_executed"])
+            self.assertTrue(res["would_execute"])
+            self.assertTrue(res["reconciliation_required"])
+            self.assertEqual(res["reconciliation_tool"], "gitea_reconcile_after_restart")
+            self.assertIn("#664", res["cross_references"])
+
+    def test_privileged_execute_creates_incident_and_audit(self) -> None:
+        """AC3 & AC4: Execution creates incident issue, audit entry, and mandates post-restart reconcile."""
+        mock_api_request = MagicMock(return_value={"number": 999, "title": "[INCIDENT] Break-glass"})
+        mock_save_state = MagicMock(return_value={"saved": True})
+
+        with patch.object(
+            gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-controller", "allowed_operations": ["gitea.read"]}
+        ), patch.object(
+            gitea_mcp_server, "_profile_role_kind", return_value="controller"
+        ), patch.object(
+            gitea_mcp_server, "_profile_operation_gate", return_value=None
+        ), patch.object(
+            gitea_mcp_server, "_auth", return_value={"Authorization": "token test"}
+        ), patch.object(
+            gitea_mcp_server, "gitea_request_mcp_restart", return_value={"affected_sessions": [{"session_id": "s1"}]}
+        ), patch.object(
+            gitea_mcp_server, "_authenticated_username", return_value="sysadmin"
+        ), patch.object(
+            gitea_mcp_server, "api_request", mock_api_request
+        ), patch("mcp_session_state.save_state", mock_save_state):
+
+            res = gitea_mcp_server.gitea_break_glass_restart(
+                reason="Emergency break-glass restart due to unrecoverable transport deadlock",
+                confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION",
+                impact_ack=True,
+                dry_run=False,
+                create_incident_issue=True,
+                remote="prgs",
+            )
+            self.assertTrue(res["success"])
+            self.assertFalse(res["dry_run"])
+            self.assertTrue(res["break_glass_executed"])
+            self.assertTrue(res["reconciliation_required"])
+            self.assertEqual(res["reconciliation_tool"], "gitea_reconcile_after_restart")
+            self.assertEqual(res["incident_issue"]["number"], 999)
+            mock_save_state.assert_called_once()
+            mock_api_request.assert_called_once()
+
+    def test_env_authorization_override_for_worker_role(self) -> None:
+        """Environment break-glass authorization enables privileged break-glass for configured sessions."""
+        os.environ["GITEA_BREAKGLASS_RESTART_AUTHORIZATION"] = "authorized-token"
+        with patch.object(
+            gitea_mcp_server, "get_profile", return_value={"profile_name": "prgs-author", "allowed_operations": ["gitea.read"]}
+        ), patch.object(
+            gitea_mcp_server, "_profile_role_kind", return_value="author"
+        ), patch.object(
+            gitea_mcp_server, "_profile_operation_gate", return_value=None
+        ), patch.object(
+            gitea_mcp_server, "gitea_request_mcp_restart", return_value={"affected_sessions": []}
+        ), patch.object(
+            gitea_mcp_server, "_authenticated_username", return_value="jcwalker3"
+        ):
+            res = gitea_mcp_server.gitea_break_glass_restart(
+                reason="Authorized emergency break-glass restart test",
+                confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION",
+                impact_ack=True,
+                dry_run=True,
+                remote="prgs",
+            )
+            self.assertTrue(res["success"])
+            self.assertTrue(res["dry_run"])
+
+
+if __name__ == "__main__":
+    unittest.main()