feat(mcp): implement emergency break-glass MCP restart workflow (#664)
This commit is contained in:
@@ -22812,6 +22812,205 @@ def gitea_request_mcp_restart(
|
||||
return payload
|
||||
|
||||
|
||||
BREAK_GLASS_CONFIRMATION_PHRASE = "I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION"
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def gitea_break_glass_restart(
|
||||
reason: str,
|
||||
confirmation: str,
|
||||
impact_ack: bool = False,
|
||||
restart_class: str = "full_mcp_restart",
|
||||
create_incident_issue: bool = True,
|
||||
dry_run: bool = False,
|
||||
remote: str = "dadeschools",
|
||||
host: str | None = None,
|
||||
org: str | None = None,
|
||||
repo: str | None = None,
|
||||
worktree_path: str | None = None,
|
||||
) -> dict:
|
||||
"""Privileged emergency break-glass MCP restart workflow (#664).
|
||||
|
||||
Break-glass restart permits emergency recovery when graceful drain cannot
|
||||
complete. It requires:
|
||||
1. Privileged caller authorization (ordinary LLM author/reviewer/merger/reconciler
|
||||
roles are denied fail-closed; controller/admin/sysadmin or GITEA_BREAKGLASS_RESTART_AUTHORIZATION
|
||||
is required).
|
||||
2. Explicit non-empty reason (minimum 10 characters).
|
||||
3. Exact confirmation string matching 'I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION'.
|
||||
4. Mandatory impact acknowledgement (impact_ack=True).
|
||||
5. Immutable audit entry recorded.
|
||||
6. Automatic incident record created on Gitea.
|
||||
7. Mandatory post-restart reconciliation requirement (#662).
|
||||
"""
|
||||
read_block = _profile_operation_gate("gitea.read")
|
||||
if read_block:
|
||||
return {
|
||||
"success": False,
|
||||
"performed": False,
|
||||
"break_glass_executed": False,
|
||||
"reasons": read_block,
|
||||
"permission_report": _permission_block_report("gitea.read"),
|
||||
"blocker_kind": "permission_denied",
|
||||
}
|
||||
|
||||
h, o, r = _resolve(remote, host, org, repo)
|
||||
profile = get_profile()
|
||||
active_role = _profile_role_kind(profile)
|
||||
break_glass_env_auth = bool(
|
||||
(os.environ.get("GITEA_BREAKGLASS_RESTART_AUTHORIZATION") or "").strip()
|
||||
)
|
||||
|
||||
# AC1: Ordinary LLM roles (author, reviewer, merger, reconciler) cannot invoke break-glass
|
||||
# unless explicit environment break-glass authorization is configured.
|
||||
if active_role in ("author", "reviewer", "merger", "reconciler") and not break_glass_env_auth:
|
||||
return {
|
||||
"success": False,
|
||||
"performed": False,
|
||||
"break_glass_executed": False,
|
||||
"active_role": active_role,
|
||||
"reasons": [
|
||||
f"ordinary LLM role '{active_role}' is forbidden from break-glass restarts (#664 AC1); "
|
||||
"privileged controller, operator, or GITEA_BREAKGLASS_RESTART_AUTHORIZATION required"
|
||||
],
|
||||
"blocker_kind": "role_authorization",
|
||||
}
|
||||
|
||||
# AC2: Required fields enforced
|
||||
clean_reason = (reason or "").strip()
|
||||
if not clean_reason or len(clean_reason) < 10:
|
||||
return {
|
||||
"success": False,
|
||||
"performed": False,
|
||||
"break_glass_executed": False,
|
||||
"reasons": [
|
||||
"reason is required and must be at least 10 characters long (#664 AC2)"
|
||||
],
|
||||
"blocker_kind": "missing_required_fields",
|
||||
}
|
||||
|
||||
clean_confirmation = (confirmation or "").strip()
|
||||
if clean_confirmation != BREAK_GLASS_CONFIRMATION_PHRASE:
|
||||
return {
|
||||
"success": False,
|
||||
"performed": False,
|
||||
"break_glass_executed": False,
|
||||
"reasons": [
|
||||
f"confirmation string mismatch; must equal exactly '{BREAK_GLASS_CONFIRMATION_PHRASE}' (#664 AC2)"
|
||||
],
|
||||
"blocker_kind": "confirmation_mismatch",
|
||||
}
|
||||
|
||||
if not impact_ack:
|
||||
return {
|
||||
"success": False,
|
||||
"performed": False,
|
||||
"break_glass_executed": False,
|
||||
"reasons": [
|
||||
"impact_ack must be True to acknowledge disruption of in-flight sessions (#664 AC2)"
|
||||
],
|
||||
"blocker_kind": "impact_ack_required",
|
||||
}
|
||||
|
||||
# Evaluate impact / disrupted sessions
|
||||
impact_result = gitea_request_mcp_restart(
|
||||
remote=remote,
|
||||
host=host,
|
||||
org=org,
|
||||
repo=repo,
|
||||
dry_run=True,
|
||||
restart_class=restart_class,
|
||||
request_break_glass=True,
|
||||
)
|
||||
disrupted_sessions = list(impact_result.get("affected_sessions") or [])
|
||||
disrupted_count = len(disrupted_sessions)
|
||||
|
||||
identity = _authenticated_username(h) or profile.get("username") or "unknown"
|
||||
now_iso = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
audit_payload = {
|
||||
"event": "break_glass_mcp_restart",
|
||||
"actor": identity,
|
||||
"role": active_role,
|
||||
"timestamp": now_iso,
|
||||
"reason": clean_reason,
|
||||
"confirmation": clean_confirmation,
|
||||
"restart_class": restart_class,
|
||||
"disrupted_sessions_count": disrupted_count,
|
||||
"disrupted_sessions": [s.get("session_id") if isinstance(s, dict) else str(s) for s in disrupted_sessions],
|
||||
"dry_run": dry_run,
|
||||
"remote": remote,
|
||||
"org": o,
|
||||
"repo": r,
|
||||
}
|
||||
|
||||
# Save immutable audit record
|
||||
saved_audit = mcp_session_state.save_state(
|
||||
kind="break_glass_audit",
|
||||
payload=audit_payload,
|
||||
remote=remote,
|
||||
org=o,
|
||||
repo=r,
|
||||
profile_identity=profile.get("profile_name", "unknown"),
|
||||
)
|
||||
|
||||
incident_issue_result = None
|
||||
if create_incident_issue and not dry_run:
|
||||
issue_title = f"[INCIDENT] Break-glass MCP restart invoked by {identity}"
|
||||
issue_body = (
|
||||
f"## Break-glass MCP restart incident report (#664)\n\n"
|
||||
f"- **Invoked by**: `{identity}` (role: `{active_role}`)\n"
|
||||
f"- **Timestamp**: `{now_iso}`\n"
|
||||
f"- **Reason**: {clean_reason}\n"
|
||||
f"- **Confirmation**: `{clean_confirmation}`\n"
|
||||
f"- **Disrupted Sessions Count**: `{disrupted_count}`\n\n"
|
||||
f"### Mandatory Post-Restart Reconciliation (#662)\n"
|
||||
f"Post-restart reconciliation must be executed via `gitea_reconcile_after_restart` "
|
||||
f"to clean up orphaned leases, inspect worktree integrity, and handle disrupted work.\n\n"
|
||||
f"### Cross-references\n"
|
||||
f"Ref #652 #653 #655 #630 #658 #662 #664\n"
|
||||
)
|
||||
try:
|
||||
incident_issue_result = api_request(
|
||||
"POST",
|
||||
f"{repo_api_url(h, o, r)}/issues",
|
||||
_auth(h),
|
||||
{
|
||||
"title": issue_title,
|
||||
"body": issue_body,
|
||||
"labels": ["incident", "mcp-health", "break-glass"],
|
||||
},
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
incident_issue_result = {"error": _redact(str(exc))}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"performed": not dry_run,
|
||||
"dry_run": dry_run,
|
||||
"break_glass_executed": not dry_run,
|
||||
"would_execute": True,
|
||||
"actor": identity,
|
||||
"role": active_role,
|
||||
"restart_class": restart_class,
|
||||
"reason": clean_reason,
|
||||
"confirmation": clean_confirmation,
|
||||
"disrupted_sessions_count": disrupted_count,
|
||||
"disrupted_sessions": disrupted_sessions,
|
||||
"audit_record": audit_payload,
|
||||
"saved_audit": dict(saved_audit or audit_payload),
|
||||
"incident_issue": incident_issue_result,
|
||||
"reconciliation_required": True,
|
||||
"reconciliation_tool": "gitea_reconcile_after_restart",
|
||||
"follow_up_issue_required": True,
|
||||
"cross_references": ["#652", "#653", "#655", "#630", "#658", "#662", "#664"],
|
||||
"reasons": [
|
||||
"break-glass restart dry-run evaluated successfully" if dry_run
|
||||
else "break-glass restart executed with incident creation and mandatory reconciliation"
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# --- #662 post-restart reconciliation ---------------------------------------
|
||||
|
||||
_POST_RESTART_LAST_PROOF: dict | None = None
|
||||
|
||||
Reference in New Issue
Block a user