fix(#664): remediate break-glass restart workflow blockers B1, B6/B11, B9, B10, B8, and B12

This commit is contained in:
2026-07-28 22:44:43 -04:00
parent 75794609d1
commit 4463a300ba
5 changed files with 500 additions and 87 deletions
+3 -3
View File
@@ -148,13 +148,13 @@ so a bypass is never silent.
The dedicated MCP tool `gitea_break_glass_restart` provides the privileged emergency break-glass restart workflow when graceful drain cannot complete: The dedicated MCP tool `gitea_break_glass_restart` provides the privileged emergency break-glass restart workflow when graceful drain cannot complete:
- **Role Authorization (#664 AC1)**: Ordinary LLM worker roles (`author`, `reviewer`, `merger`, `reconciler`) are denied fail-closed. Privileged `controller` role or explicit `GITEA_BREAKGLASS_RESTART_AUTHORIZATION` is required. - **Role Authorization (#664 AC1)**: Ordinary LLM worker roles (`author`, `reviewer`, `merger`, `reconciler`) are denied fail-closed. Privileged `controller` role is required and enforced via the canonical `_profile_role_kind` resolver and `runtime.break_glass_restart` capability. `GITEA_BREAKGLASS_RESTART_AUTHORIZATION` is a fail-closed feature/configuration gate and does not provide identity, role, capability, confirmation, or permission.
- **Required Parameters (#664 AC2)**: - **Required Parameters (#664 AC2)**:
- `reason`: Mandatory non-empty string (min 10 characters). - `reason`: Mandatory non-empty string (min 10 characters).
- `confirmation`: Must equal exactly `"I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION"`. - `confirmation`: Must equal exactly `"I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION"`.
- `impact_ack`: Must be `True`. - `impact_ack`: Must be `True`.
- **Automatic Incident Creation (#664 AC3)**: Creates a Gitea incident issue (`[INCIDENT] Break-glass MCP restart invoked by ...`) detailing the reason, timestamp, disrupted sessions, and linking `#652 #653 #655 #630 #658 #662 #664`. - **Automatic Incident Creation (#664 AC3)**: Creates a Gitea incident issue (`[INCIDENT] [REQUESTED] Break-glass MCP restart invoked by ...`) detailing the reason, timestamp, disrupted sessions, and linking `#652 #653 #655 #630 #658 #662 #664`.
- **Immutable Audit Entry**: Records an immutable audit log entry under `event="break_glass_mcp_restart"`. - **Immutable Append-Only Audit Entry**: Records immutable pre-execution (REQUESTED) and post-execution (SUCCEEDED/FAILED) audit log entries with correlation identifiers.
- **Mandatory Reconciliation (#664 AC4)**: Sets `reconciliation_required=True` requiring post-restart reconciliation via `gitea_reconcile_after_restart` (#662). - **Mandatory Reconciliation (#664 AC4)**: Sets `reconciliation_required=True` requiring post-restart reconciliation via `gitea_reconcile_after_restart` (#662).
### Fail closed on apply ### Fail closed on apply
+26 -4
View File
@@ -27,13 +27,32 @@ ALLOWED = "allowed"
BLOCKED = "blocked" BLOCKED = "blocked"
FAILED = "failed" FAILED = "failed"
SUCCEEDED = "succeeded" SUCCEEDED = "succeeded"
REQUESTED = "requested"
ACCEPTED = "accepted"
PENDING = "pending"
REDACTED = "[REDACTED]" REDACTED = "[REDACTED]"
# A dict key containing any of these (case-insensitive) has its value redacted. # A dict key containing any of these (case-insensitive) has its value redacted.
_SECRET_KEY_HINTS = ("token", "password", "secret", "authorization", "auth") _SECRET_KEY_HINTS = ("token", "password", "secret", "authorization", "auth")
# A string value starting with one of these has the following run redacted. # A string value starting with one of these has the following run redacted.
_SECRET_VALUE_PREFIXES = ("token ", "Basic ", "Bearer ") _SECRET_VALUE_PREFIXES = ("token ", "Basic ", "Bearer ", "token:", "bearer:", "basic:")
_BARE_SECRET_PATTERN = re.compile(
r'(?i)\b(?:'
r'ghp_[A-Za-z0-9_]{16,}'
r'|gho_[A-Za-z0-9_]{16,}'
r'|ghu_[A-Za-z0-9_]{16,}'
r'|ghs_[A-Za-z0-9_]{16,}'
r'|ghr_[A-Za-z0-9_]{16,}'
r'|sk-live-[A-Za-z0-9_-]{16,}'
r'|sk-proj-[A-Za-z0-9_-]{16,}'
r'|sk-[A-Za-z0-9_-]{20,}'
r'|glpat-[A-Za-z0-9_-]{16,}'
r'|sec-[A-Za-z0-9_-]{16,}'
r')\b'
)
# Known synthetic test-only domains/hostnames to preserve # Known synthetic test-only domains/hostnames to preserve
_SYNTHETIC_HOSTS = { _SYNTHETIC_HOSTS = {
@@ -115,20 +134,23 @@ def redact_urls(text: str) -> str:
def _redact_str(text): def _redact_str(text):
"""Redact anything that looks like an Authorization credential or raw URL in *text*.""" """Redact anything that looks like an Authorization credential, bare secret token, or raw URL in *text*."""
if not isinstance(text, str) or not text: if not isinstance(text, str) or not text:
return text return text
out = text out = _BARE_SECRET_PATTERN.sub(REDACTED, text)
out_lower = out.lower()
for prefix in _SECRET_VALUE_PREFIXES: for prefix in _SECRET_VALUE_PREFIXES:
prefix_lower = prefix.lower()
idx = 0 idx = 0
while True: while True:
i = out.find(prefix, idx) i = out_lower.find(prefix_lower, idx)
if i == -1: if i == -1:
break break
j = i + len(prefix) j = i + len(prefix)
while j < len(out) and not out[j].isspace(): while j < len(out) and not out[j].isspace():
j += 1 j += 1
out = out[:i] + prefix + REDACTED + out[j:] out = out[:i] + prefix + REDACTED + out[j:]
out_lower = out.lower()
idx = i + len(prefix) + len(REDACTED) idx = i + len(prefix) + len(REDACTED)
return redact_urls(out) return redact_urls(out)
+304 -66
View File
@@ -236,16 +236,15 @@ def _effective_workspace_role() -> str:
def _profile_role_kind(profile: dict) -> str: def _profile_role_kind(profile: dict) -> str:
"""Resolve a profile's declared role before inferring from permissions. """Resolve a profile's declared role before inferring from permissions.
Declared ``role`` / ``role_kind`` always wins so a controller profile is Declared ``role`` / ``role_kind`` or profile_name containing 'controller'
never reclassified as reconciler from permission inference (#840). always resolves to 'controller' (#840/#664).
""" """
role = (profile.get("role") or profile.get("role_kind") or "").strip().lower()
if role:
# Normalize aliases / case.
if "control" in role:
return "controller"
return role
profile_name = (profile.get("profile_name") or "").strip().lower() profile_name = (profile.get("profile_name") or "").strip().lower()
role = (profile.get("role") or profile.get("role_kind") or "").strip().lower()
if "controller" in profile_name or "control" in role or role == "controller":
return "controller"
if role:
return role
for candidate in ( for candidate in (
"controller", "controller",
"reconciler", "reconciler",
@@ -23923,7 +23922,7 @@ def gitea_request_mcp_restart(
BREAK_GLASS_CONFIRMATION_PHRASE = "I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION" BREAK_GLASS_CONFIRMATION_PHRASE = "I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION"
PRIVILEGED_BREAK_GLASS_ROLES = frozenset({"controller", "operator", "admin", "sysadmin"}) PRIVILEGED_BREAK_GLASS_ROLES = frozenset({"controller"})
@mcp.tool() @mcp.tool()
@@ -23944,25 +23943,25 @@ def gitea_break_glass_restart(
Break-glass restart permits emergency recovery when graceful drain cannot Break-glass restart permits emergency recovery when graceful drain cannot
complete. It requires: complete. It requires:
1. Privileged caller authorization (explicit allowlist: controller, operator, 1. Privileged caller authorization (explicit allowlist: controller;
admin, sysadmin; ordinary roles and unknown/malformed roles fail closed). ordinary roles and unknown/malformed roles fail closed).
2. Explicit non-empty reason (minimum 10 characters). 2. Explicit non-empty reason (minimum 10 characters).
3. Exact confirmation string matching 'I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION'. 3. Exact confirmation string matching 'I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION'.
4. Mandatory impact acknowledgement (impact_ack=True). 4. Mandatory impact acknowledgement (impact_ack=True).
5. Immutable append-only audit entry recorded prior to execution. 5. Immutable append-only audit entry recorded prior to execution and after terminal completion.
6. Automatic incident record created on Gitea prior to execution. 6. Automatic incident record created on Gitea prior to execution.
7. Truthful execution reporting and mandatory post-restart reconciliation (#662). 7. Truthful execution reporting and mandatory post-restart reconciliation (#662).
""" """
read_block = _profile_operation_gate("gitea.read") read_block = _profile_operation_gate("runtime.break_glass_restart")
if read_block: if read_block:
return { return gitea_audit.redact({
"success": False, "success": False,
"performed": False, "performed": False,
"break_glass_executed": False, "break_glass_executed": False,
"reasons": read_block, "reasons": read_block,
"permission_report": _permission_block_report("gitea.read"), "permission_report": _permission_block_report("runtime.break_glass_restart"),
"blocker_kind": "permission_denied", "blocker_kind": "permission_denied",
} })
h, o, r = _resolve(remote, host, org, repo) h, o, r = _resolve(remote, host, org, repo)
profile = get_profile() profile = get_profile()
@@ -23971,25 +23970,25 @@ def gitea_break_glass_restart(
(os.environ.get("GITEA_BREAKGLASS_RESTART_AUTHORIZATION") or "").strip() (os.environ.get("GITEA_BREAKGLASS_RESTART_AUTHORIZATION") or "").strip()
) )
# 1. Privileged role authorization (explicit allowlist - Finding 1 & 5) # 1. Privileged role authorization (explicit allowlist: controller only - B1)
if active_role not in PRIVILEGED_BREAK_GLASS_ROLES: if active_role not in PRIVILEGED_BREAK_GLASS_ROLES:
return { return gitea_audit.redact({
"success": False, "success": False,
"performed": False, "performed": False,
"break_glass_executed": False, "break_glass_executed": False,
"active_role": active_role, "active_role": active_role,
"reasons": [ "reasons": [
f"role '{active_role}' is not in privileged break-glass allowlist " f"role '{active_role}' is not in privileged break-glass allowlist "
f"({sorted(PRIVILEGED_BREAK_GLASS_ROLES)}); break-glass restart requires a privileged role (#664 AC1)" f"({sorted(PRIVILEGED_BREAK_GLASS_ROLES)}); break-glass restart requires a privileged controller role (#664 AC1)"
], ],
"blocker_kind": "role_authorization", "blocker_kind": "role_authorization",
} })
# 2. Required fields and redaction (Finding 3) # 2. Required fields and redaction (B8)
raw_reason = (reason or "").strip() raw_reason = (reason or "").strip()
clean_reason = _redact(raw_reason) clean_reason = _redact(raw_reason)
if not raw_reason or len(raw_reason) < 10: if not raw_reason or len(raw_reason) < 10:
return { return gitea_audit.redact({
"success": False, "success": False,
"performed": False, "performed": False,
"break_glass_executed": False, "break_glass_executed": False,
@@ -23997,11 +23996,11 @@ def gitea_break_glass_restart(
"reason is required and must be at least 10 characters long (#664 AC2)" "reason is required and must be at least 10 characters long (#664 AC2)"
], ],
"blocker_kind": "missing_required_fields", "blocker_kind": "missing_required_fields",
} })
clean_confirmation = (confirmation or "").strip() clean_confirmation = (confirmation or "").strip()
if clean_confirmation != BREAK_GLASS_CONFIRMATION_PHRASE: if clean_confirmation != BREAK_GLASS_CONFIRMATION_PHRASE:
return { return gitea_audit.redact({
"success": False, "success": False,
"performed": False, "performed": False,
"break_glass_executed": False, "break_glass_executed": False,
@@ -24009,10 +24008,10 @@ def gitea_break_glass_restart(
f"confirmation string mismatch; must equal exactly '{BREAK_GLASS_CONFIRMATION_PHRASE}' (#664 AC2)" f"confirmation string mismatch; must equal exactly '{BREAK_GLASS_CONFIRMATION_PHRASE}' (#664 AC2)"
], ],
"blocker_kind": "confirmation_mismatch", "blocker_kind": "confirmation_mismatch",
} })
if not impact_ack: if not impact_ack:
return { return gitea_audit.redact({
"success": False, "success": False,
"performed": False, "performed": False,
"break_glass_executed": False, "break_glass_executed": False,
@@ -24020,20 +24019,20 @@ def gitea_break_glass_restart(
"impact_ack must be True to acknowledge disruption of in-flight sessions (#664 AC2)" "impact_ack must be True to acknowledge disruption of in-flight sessions (#664 AC2)"
], ],
"blocker_kind": "impact_ack_required", "blocker_kind": "impact_ack_required",
} })
# Gate incident issue creation on gitea.issue.create permission (Finding 3, B3) # Gate incident issue creation on gitea.issue.create permission (B3)
if create_incident_issue: if create_incident_issue:
create_block = _profile_operation_gate("gitea.issue.create") create_block = _profile_operation_gate("gitea.issue.create")
if create_block: if create_block:
return { return gitea_audit.redact({
"success": False, "success": False,
"performed": False, "performed": False,
"break_glass_executed": False, "break_glass_executed": False,
"reasons": create_block, "reasons": create_block,
"permission_report": _permission_block_report("gitea.issue.create"), "permission_report": _permission_block_report("gitea.issue.create"),
"blocker_kind": "permission_denied", "blocker_kind": "permission_denied",
} })
# Evaluate impact / disrupted sessions # Evaluate impact / disrupted sessions
impact_result = gitea_request_mcp_restart( impact_result = gitea_request_mcp_restart(
@@ -24050,11 +24049,16 @@ def gitea_break_glass_restart(
identity = _authenticated_username(h) or profile.get("username") or "unknown" identity = _authenticated_username(h) or profile.get("username") or "unknown"
now_iso = datetime.now(timezone.utc).isoformat() now_iso = datetime.now(timezone.utc).isoformat()
ns_ctx = _resolve_namespace_mutation_context(worktree_path)
mcp_namespace = ns_ctx.get("mcp_namespace") or profile.get("profile_name") or "gitea-controller"
correlation_id = f"bg-{uuid.uuid4().hex[:12]}"
audit_payload = { audit_payload = gitea_audit.redact({
"event": "break_glass_mcp_restart", "event": "break_glass_mcp_restart",
"correlation_id": correlation_id,
"actor": identity, "actor": identity,
"role": active_role, "role": active_role,
"mcp_namespace": mcp_namespace,
"timestamp": now_iso, "timestamp": now_iso,
"reason": clean_reason, "reason": clean_reason,
"confirmation": clean_confirmation, "confirmation": clean_confirmation,
@@ -24069,18 +24073,20 @@ def gitea_break_glass_restart(
"org": o, "org": o,
"repo": r, "repo": r,
"env_auth_present": break_glass_env_auth, "env_auth_present": break_glass_env_auth,
} })
# 4. Dry-run handling (Finding 4, B7) # Dry-run handling (B7: no durable mutation)
if dry_run: if dry_run:
return { return gitea_audit.redact({
"success": True, "success": True,
"performed": False, "performed": False,
"dry_run": True, "dry_run": True,
"break_glass_executed": False, "break_glass_executed": False,
"would_execute": True, "would_execute": True,
"correlation_id": correlation_id,
"actor": identity, "actor": identity,
"role": active_role, "role": active_role,
"mcp_namespace": mcp_namespace,
"restart_class": restart_class, "restart_class": restart_class,
"reason": clean_reason, "reason": clean_reason,
"confirmation": clean_confirmation, "confirmation": clean_confirmation,
@@ -24094,11 +24100,11 @@ def gitea_break_glass_restart(
"follow_up_issue_required": True, "follow_up_issue_required": True,
"cross_references": ["#652", "#653", "#655", "#630", "#658", "#662", "#664"], "cross_references": ["#652", "#653", "#655", "#630", "#658", "#662", "#664"],
"reasons": ["break-glass restart dry-run evaluated successfully"], "reasons": ["break-glass restart dry-run evaluated successfully"],
} })
# Fail closed if create_incident_issue is False on real execution (Finding 2, B5) # Fail closed if create_incident_issue is False on real execution (B5)
if not create_incident_issue: if not create_incident_issue:
return { return gitea_audit.redact({
"success": False, "success": False,
"performed": False, "performed": False,
"break_glass_executed": False, "break_glass_executed": False,
@@ -24107,12 +24113,24 @@ def gitea_break_glass_restart(
"pre-execution incident creation is mandatory (#664 AC3)" "pre-execution incident creation is mandatory (#664 AC3)"
], ],
"blocker_kind": "incident_creation_required", "blocker_kind": "incident_creation_required",
} })
# Pre-execution recording: Audit record (Finding 2, B4) # B9: Fail closed if audit backend is disabled
audit_event = gitea_audit.build_event( if not gitea_audit.audit_enabled():
action="break_glass_mcp_restart", return gitea_audit.redact({
result=gitea_audit.SUCCEEDED, "success": False,
"performed": False,
"break_glass_executed": False,
"reasons": [
"audit recording is disabled or unavailable; break-glass restart requires an enabled audit backend (#664 AC3)"
],
"blocker_kind": "audit_recording_failed",
})
# Pre-execution recording: Audit record in REQUESTED state (B4, B10)
pre_audit_event = gitea_audit.build_event(
action="break_glass_mcp_restart_requested",
result=gitea_audit.REQUESTED,
remote=remote, remote=remote,
server=(gitea_url(h, "").rstrip("/") if h else None), server=(gitea_url(h, "").rstrip("/") if h else None),
repository=r, repository=r,
@@ -24120,11 +24138,12 @@ def gitea_break_glass_restart(
audit_label=profile.get("audit_label", "unknown"), audit_label=profile.get("audit_label", "unknown"),
authenticated_username=identity, authenticated_username=identity,
reason=clean_reason, reason=clean_reason,
mcp_namespace="gitea-author", mcp_namespace=mcp_namespace,
task_role=active_role, task_role=active_role,
operation="break_glass_mcp_restart", operation="break_glass_mcp_restart",
now=now_iso, now=now_iso,
request_metadata={ request_metadata={
"correlation_id": correlation_id,
"confirmation": clean_confirmation, "confirmation": clean_confirmation,
"restart_class": restart_class, "restart_class": restart_class,
"disrupted_sessions_count": disrupted_count, "disrupted_sessions_count": disrupted_count,
@@ -24134,12 +24153,10 @@ def gitea_break_glass_restart(
], ],
}, },
) )
audit_write_success = True audit_write_success = gitea_audit.write_event(pre_audit_event)
if gitea_audit.audit_enabled():
audit_write_success = gitea_audit.write_event(audit_event)
if not audit_write_success: if not audit_write_success:
return { return gitea_audit.redact({
"success": False, "success": False,
"performed": False, "performed": False,
"break_glass_executed": False, "break_glass_executed": False,
@@ -24147,13 +24164,14 @@ def gitea_break_glass_restart(
"failed to persist required pre-execution audit event (#664 AC3)" "failed to persist required pre-execution audit event (#664 AC3)"
], ],
"blocker_kind": "audit_recording_failed", "blocker_kind": "audit_recording_failed",
} })
# Pre-execution recording: Gitea Incident Issue (Finding 2 & 3, B5) # Pre-execution recording: Gitea Incident Issue (B5, B8)
issue_title = f"[INCIDENT] Break-glass MCP restart invoked by {identity}" issue_title = _redact(f"[INCIDENT] [REQUESTED] Break-glass MCP restart invoked by {identity} ({correlation_id})")
issue_body = ( issue_body = _redact(
f"## Break-glass MCP restart incident report (#664)\n\n" f"## Break-glass MCP restart incident report (#664)\n\n"
f"- **Invoked by**: `{identity}` (role: `{active_role}`)\n" f"- **Correlation ID**: `{correlation_id}`\n"
f"- **Invoked by**: `{identity}` (role: `{active_role}`, namespace: `{mcp_namespace}`)\n"
f"- **Timestamp**: `{now_iso}`\n" f"- **Timestamp**: `{now_iso}`\n"
f"- **Reason**: {clean_reason}\n" f"- **Reason**: {clean_reason}\n"
f"- **Confirmation**: `{clean_confirmation}`\n" f"- **Confirmation**: `{clean_confirmation}`\n"
@@ -24177,18 +24195,18 @@ def gitea_break_glass_restart(
}, },
) )
if not isinstance(incident_issue_result, dict) or "number" not in incident_issue_result: if not isinstance(incident_issue_result, dict) or "number" not in incident_issue_result:
return { return gitea_audit.redact({
"success": False, "success": False,
"performed": False, "performed": False,
"break_glass_executed": False, "break_glass_executed": False,
"reasons": [ "reasons": [
f"incident issue creation failed (#664 AC3): {incident_issue_result}" f"incident issue creation failed (#664 AC3): {_redact(str(incident_issue_result))}"
], ],
"blocker_kind": "incident_creation_failed", "blocker_kind": "incident_creation_failed",
"incident_issue": incident_issue_result, "incident_issue": incident_issue_result,
} })
except Exception as exc: except Exception as exc:
return { return gitea_audit.redact({
"success": False, "success": False,
"performed": False, "performed": False,
"break_glass_executed": False, "break_glass_executed": False,
@@ -24197,9 +24215,11 @@ def gitea_break_glass_restart(
], ],
"blocker_kind": "incident_creation_failed", "blocker_kind": "incident_creation_failed",
"incident_issue": {"error": _redact(str(exc))}, "incident_issue": {"error": _redact(str(exc))},
} })
# Execute or delegate canonical non-dry-run restart (Finding 4) incident_number = incident_issue_result.get("number")
# Execute or delegate canonical non-dry-run restart (B6/B11)
restart_exec_result = gitea_request_mcp_restart( restart_exec_result = gitea_request_mcp_restart(
remote=remote, remote=remote,
host=host, host=host,
@@ -24210,13 +24230,107 @@ def gitea_break_glass_restart(
request_break_glass=True, request_break_glass=True,
) )
if not restart_exec_result.get("apply_authorized") or not restart_exec_result.get("success"): apply_supported = bool(restart_exec_result.get("apply_supported", False))
return { apply_authorized = bool(restart_exec_result.get("apply_authorized", False))
exec_success = bool(restart_exec_result.get("success", False))
restart_performed = bool(
restart_exec_result.get("restart_performed", False)
or restart_exec_result.get("break_glass_executed", False)
)
if not apply_supported:
term_audit = gitea_audit.build_event(
action="break_glass_mcp_restart_terminal",
result=gitea_audit.FAILED,
remote=remote,
server=(gitea_url(h, "").rstrip("/") if h else None),
repository=r,
profile_name=profile.get("profile_name", "unknown"),
audit_label=profile.get("audit_label", "unknown"),
authenticated_username=identity,
reason="apply_unsupported",
mcp_namespace=mcp_namespace,
task_role=active_role,
operation="break_glass_mcp_restart",
now=datetime.now(timezone.utc).isoformat(),
request_metadata={
"correlation_id": correlation_id,
"incident_number": incident_number,
"break_glass_executed": False,
"blocker_kind": "apply_unsupported",
},
)
gitea_audit.write_event(term_audit)
if incident_number:
try:
api_request(
"POST",
f"{repo_api_url(h, o, r)}/issues/{incident_number}/comments",
_auth(h),
{"body": _redact(f"**Terminal Status**: `apply_unsupported` - Restart coordinator does not support apply execution. No restart was performed. ({correlation_id})")},
)
except Exception:
pass
return gitea_audit.redact({
"success": False, "success": False,
"performed": False, "performed": False,
"break_glass_executed": False, "break_glass_executed": False,
"actor": identity, "actor": identity,
"role": active_role, "role": active_role,
"mcp_namespace": mcp_namespace,
"restart_class": restart_class,
"reasons": [
"break-glass apply is unsupported by restart coordinator (apply_supported=False) (#664)",
*(restart_exec_result.get("reasons") or []),
],
"blocker_kind": "apply_unsupported",
"restart_result": restart_exec_result,
"incident_issue": incident_issue_result,
"audit_record": pre_audit_event,
})
if not apply_authorized or not exec_success or not restart_performed:
term_audit = gitea_audit.build_event(
action="break_glass_mcp_restart_terminal",
result=gitea_audit.FAILED,
remote=remote,
server=(gitea_url(h, "").rstrip("/") if h else None),
repository=r,
profile_name=profile.get("profile_name", "unknown"),
audit_label=profile.get("audit_label", "unknown"),
authenticated_username=identity,
reason="restart_delegation_failed",
mcp_namespace=mcp_namespace,
task_role=active_role,
operation="break_glass_mcp_restart",
now=datetime.now(timezone.utc).isoformat(),
request_metadata={
"correlation_id": correlation_id,
"incident_number": incident_number,
"break_glass_executed": False,
"blocker_kind": "restart_delegation_failed",
},
)
gitea_audit.write_event(term_audit)
if incident_number:
try:
api_request(
"POST",
f"{repo_api_url(h, o, r)}/issues/{incident_number}/comments",
_auth(h),
{"body": _redact(f"**Terminal Status**: `restart_delegation_failed` - Restart execution failed or was denied. No restart was performed. ({correlation_id})")},
)
except Exception:
pass
return gitea_audit.redact({
"success": False,
"performed": False,
"break_glass_executed": False,
"actor": identity,
"role": active_role,
"mcp_namespace": mcp_namespace,
"restart_class": restart_class, "restart_class": restart_class,
"reasons": [ "reasons": [
"delegated restart execution failed or was denied by coordinator (#664)", "delegated restart execution failed or was denied by coordinator (#664)",
@@ -24225,25 +24339,149 @@ def gitea_break_glass_restart(
"blocker_kind": "restart_delegation_failed", "blocker_kind": "restart_delegation_failed",
"restart_result": restart_exec_result, "restart_result": restart_exec_result,
"incident_issue": incident_issue_result, "incident_issue": incident_issue_result,
"audit_record": audit_event, "audit_record": pre_audit_event,
} })
return { # Restart occurred! Perform mandatory post-restart reconciliation (B10)
recon_result = None
try:
recon_result = gitea_reconcile_after_restart(
remote=remote,
host=host,
org=org,
repo=repo,
)
except Exception as exc:
recon_result = {"success": False, "error": _redact(str(exc))}
recon_success = bool(recon_result and isinstance(recon_result, dict) and recon_result.get("success", False))
if not recon_success:
term_audit = gitea_audit.build_event(
action="break_glass_mcp_restart_terminal",
result=gitea_audit.FAILED,
remote=remote,
server=(gitea_url(h, "").rstrip("/") if h else None),
repository=r,
profile_name=profile.get("profile_name", "unknown"),
audit_label=profile.get("audit_label", "unknown"),
authenticated_username=identity,
reason="reconciliation_failed",
mcp_namespace=mcp_namespace,
task_role=active_role,
operation="break_glass_mcp_restart",
now=datetime.now(timezone.utc).isoformat(),
request_metadata={
"correlation_id": correlation_id,
"incident_number": incident_number,
"break_glass_executed": True,
"reconciliation_success": False,
"blocker_kind": "reconciliation_failed",
},
)
gitea_audit.write_event(term_audit)
if incident_number:
try:
api_request(
"POST",
f"{repo_api_url(h, o, r)}/issues/{incident_number}/comments",
_auth(h),
{"body": _redact(f"**Terminal Status**: `reconciliation_failed` - Restart was executed but post-restart reconciliation failed. ({correlation_id})")},
)
except Exception:
pass
return gitea_audit.redact({
"success": False,
"performed": True,
"break_glass_executed": True,
"actor": identity,
"role": active_role,
"mcp_namespace": mcp_namespace,
"restart_class": restart_class,
"reasons": [
"break-glass restart executed but post-restart reconciliation failed (#664/#662)"
],
"blocker_kind": "reconciliation_failed",
"restart_result": restart_exec_result,
"reconciliation_result": recon_result,
"incident_issue": incident_issue_result,
"audit_record": pre_audit_event,
})
# Terminal audit append for successful execution + reconciliation
term_audit = gitea_audit.build_event(
action="break_glass_mcp_restart_terminal",
result=gitea_audit.SUCCEEDED,
remote=remote,
server=(gitea_url(h, "").rstrip("/") if h else None),
repository=r,
profile_name=profile.get("profile_name", "unknown"),
audit_label=profile.get("audit_label", "unknown"),
authenticated_username=identity,
reason=clean_reason,
mcp_namespace=mcp_namespace,
task_role=active_role,
operation="break_glass_mcp_restart",
now=datetime.now(timezone.utc).isoformat(),
request_metadata={
"correlation_id": correlation_id,
"incident_number": incident_number,
"break_glass_executed": True,
"reconciliation_success": True,
},
)
term_write_success = gitea_audit.write_event(term_audit)
if incident_number:
try:
api_request(
"POST",
f"{repo_api_url(h, o, r)}/issues/{incident_number}/comments",
_auth(h),
{"body": _redact(f"**Terminal Status**: `succeeded` - Break-glass restart executed and reconciled successfully ({correlation_id}).")},
)
except Exception:
pass
if not term_write_success:
return gitea_audit.redact({
"success": False,
"performed": True,
"break_glass_executed": True,
"actor": identity,
"role": active_role,
"mcp_namespace": mcp_namespace,
"restart_class": restart_class,
"reasons": [
"break-glass restart executed and reconciled but terminal audit recording failed (#664)"
],
"blocker_kind": "terminal_audit_failed",
"restart_result": restart_exec_result,
"reconciliation_result": recon_result,
"incident_issue": incident_issue_result,
"audit_record": pre_audit_event,
})
return gitea_audit.redact({
"success": True, "success": True,
"performed": True, "performed": True,
"dry_run": False, "dry_run": False,
"break_glass_executed": True, "break_glass_executed": True,
"would_execute": True, "would_execute": True,
"correlation_id": correlation_id,
"actor": identity, "actor": identity,
"role": active_role, "role": active_role,
"mcp_namespace": mcp_namespace,
"restart_class": restart_class, "restart_class": restart_class,
"reason": clean_reason, "reason": clean_reason,
"confirmation": clean_confirmation, "confirmation": clean_confirmation,
"disrupted_sessions_count": disrupted_count, "disrupted_sessions_count": disrupted_count,
"disrupted_sessions": disrupted_sessions, "disrupted_sessions": disrupted_sessions,
"audit_record": audit_event, "audit_record": term_audit,
"saved_audit": audit_event, "saved_audit": term_audit,
"incident_issue": incident_issue_result, "incident_issue": incident_issue_result,
"reconciliation_result": recon_result,
"reconciliation_required": True, "reconciliation_required": True,
"reconciliation_tool": "gitea_reconcile_after_restart", "reconciliation_tool": "gitea_reconcile_after_restart",
"follow_up_issue_required": True, "follow_up_issue_required": True,
@@ -24251,7 +24489,7 @@ def gitea_break_glass_restart(
"reasons": [ "reasons": [
"break-glass restart executed with incident creation and mandatory reconciliation" "break-glass restart executed with incident creation and mandatory reconciliation"
], ],
} })
+2 -2
View File
@@ -578,11 +578,11 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = {
}, },
# #664: emergency break-glass MCP restart workflow (privileged controller role). # #664: emergency break-glass MCP restart workflow (privileged controller role).
"break_glass_restart": { "break_glass_restart": {
"permission": "gitea.read", "permission": "runtime.break_glass_restart",
"role": "controller", "role": "controller",
}, },
"gitea_break_glass_restart": { "gitea_break_glass_restart": {
"permission": "gitea.read", "permission": "runtime.break_glass_restart",
"role": "controller", "role": "controller",
}, },
} }
+165 -12
View File
@@ -23,14 +23,12 @@ class TestBreakGlassRestart(unittest.TestCase):
def test_role_authorization_matrix_with_real_resolver(self) -> None: def test_role_authorization_matrix_with_real_resolver(self) -> None:
"""AC1: Explicit allowlist enforcement using production _profile_role_kind resolver. """AC1: Explicit allowlist enforcement using production _profile_role_kind resolver.
Privileged roles (controller, operator, admin, sysadmin) pass role check. Privileged controller role passes role check.
Ordinary LLM roles (author, reviewer, merger, reconciler) and unknown/malformed roles fail closed. Synthetic roles (operator, admin, sysadmin), ordinary LLM roles (author, reviewer, merger, reconciler), and unknown/malformed roles fail closed.
""" """
allowed_profiles = [ allowed_profiles = [
{"role": "controller", "allowed_operations": ["gitea.read", "gitea.issue.create"]}, {"role": "controller", "allowed_operations": ["gitea.read", "gitea.issue.create", "runtime.break_glass_restart"]},
{"role": "operator", "allowed_operations": ["gitea.read", "gitea.issue.create"]}, {"profile_name": "prgs-controller", "allowed_operations": ["gitea.read", "gitea.issue.create", "runtime.break_glass_restart"]},
{"role": "admin", "allowed_operations": ["gitea.read", "gitea.issue.create"]},
{"role": "sysadmin", "allowed_operations": ["gitea.read", "gitea.issue.create"]},
] ]
for prof in allowed_profiles: for prof in allowed_profiles:
@@ -48,10 +46,13 @@ class TestBreakGlassRestart(unittest.TestCase):
) )
self.assertTrue( self.assertTrue(
res["success"], res["success"],
f"Profile with role '{prof.get('role')}' should pass role check", f"Profile {prof} should pass role check",
) )
denied_profiles = [ denied_profiles = [
{"role": "operator", "allowed_operations": ["gitea.read", "gitea.issue.create"]},
{"role": "admin", "allowed_operations": ["gitea.read", "gitea.issue.create"]},
{"role": "sysadmin", "allowed_operations": ["gitea.read", "gitea.issue.create"]},
{"role": "author", "allowed_operations": ["gitea.read", "gitea.issue.create"]}, {"role": "author", "allowed_operations": ["gitea.read", "gitea.issue.create"]},
{"role": "reviewer", "allowed_operations": ["gitea.read"]}, {"role": "reviewer", "allowed_operations": ["gitea.read"]},
{"role": "merger", "allowed_operations": ["gitea.read"]}, {"role": "merger", "allowed_operations": ["gitea.read"]},
@@ -174,7 +175,8 @@ class TestBreakGlassRestart(unittest.TestCase):
raw_reason = "Emergency restart: token ghp_secretToken12345 and url https://user:[email protected]/api" raw_reason = "Emergency restart: token ghp_secretToken12345 and url https://user:[email protected]/api"
mock_api_request = MagicMock(return_value={"number": 101, "title": "[INCIDENT]"}) mock_api_request = MagicMock(return_value={"number": 101, "title": "[INCIDENT]"})
mock_restart_exec = {"success": True, "apply_authorized": True} mock_restart_exec = {"success": True, "apply_supported": True, "apply_authorized": True, "restart_performed": True}
mock_recon = {"success": True}
with patch.object(gitea_mcp_server, "get_profile", return_value=controller_prof), patch.object( with patch.object(gitea_mcp_server, "get_profile", return_value=controller_prof), patch.object(
gitea_mcp_server, "_profile_operation_gate", return_value=None gitea_mcp_server, "_profile_operation_gate", return_value=None
@@ -186,6 +188,10 @@ class TestBreakGlassRestart(unittest.TestCase):
gitea_mcp_server, "api_request", mock_api_request gitea_mcp_server, "api_request", mock_api_request
), patch.object( ), patch.object(
gitea_mcp_server, "gitea_request_mcp_restart", return_value=mock_restart_exec gitea_mcp_server, "gitea_request_mcp_restart", return_value=mock_restart_exec
), patch.object(
gitea_mcp_server, "gitea_reconcile_after_restart", return_value=mock_recon
), patch.object(
gitea_audit, "audit_enabled", return_value=True
), patch.object( ), patch.object(
gitea_audit, "write_event", return_value=True gitea_audit, "write_event", return_value=True
): ):
@@ -253,6 +259,8 @@ class TestBreakGlassRestart(unittest.TestCase):
gitea_mcp_server, "_authenticated_username", return_value="sysadmin" gitea_mcp_server, "_authenticated_username", return_value="sysadmin"
), patch.object( ), patch.object(
gitea_mcp_server, "gitea_request_mcp_restart", side_effect=[{"affected_sessions": []}, mock_restart_exec] gitea_mcp_server, "gitea_request_mcp_restart", side_effect=[{"affected_sessions": []}, mock_restart_exec]
), patch.object(
gitea_audit, "audit_enabled", return_value=True
), patch.object( ), patch.object(
gitea_audit, "write_event", return_value=True gitea_audit, "write_event", return_value=True
), patch.object( ), patch.object(
@@ -326,9 +334,10 @@ class TestBreakGlassRestart(unittest.TestCase):
def test_successful_real_execution(self) -> None: def test_successful_real_execution(self) -> None:
"""Requirement 4: Real execution calls canonical restart path and reports execution truthfully.""" """Requirement 4: Real execution calls canonical restart path and reports execution truthfully."""
controller_prof = {"role": "controller", "allowed_operations": ["gitea.read", "gitea.issue.create"]} controller_prof = {"profile_name": "prgs-controller", "allowed_operations": ["gitea.read", "gitea.issue.create", "runtime.break_glass_restart"]}
mock_api_request = MagicMock(return_value={"number": 555, "title": "[INCIDENT]"}) mock_api_request = MagicMock(return_value={"number": 555, "title": "[INCIDENT]"})
mock_restart_exec = {"success": True, "apply_authorized": True, "affected_sessions": []} mock_restart_exec = {"success": True, "apply_supported": True, "apply_authorized": True, "restart_performed": True, "affected_sessions": []}
mock_recon = {"success": True, "reconciled": True}
with patch.object(gitea_mcp_server, "get_profile", return_value=controller_prof), patch.object( with patch.object(gitea_mcp_server, "get_profile", return_value=controller_prof), patch.object(
gitea_mcp_server, "_profile_operation_gate", return_value=None gitea_mcp_server, "_profile_operation_gate", return_value=None
@@ -338,6 +347,10 @@ class TestBreakGlassRestart(unittest.TestCase):
gitea_mcp_server, "_authenticated_username", return_value="sysadmin" gitea_mcp_server, "_authenticated_username", return_value="sysadmin"
), patch.object( ), patch.object(
gitea_mcp_server, "gitea_request_mcp_restart", return_value=mock_restart_exec gitea_mcp_server, "gitea_request_mcp_restart", return_value=mock_restart_exec
), patch.object(
gitea_mcp_server, "gitea_reconcile_after_restart", return_value=mock_recon
), patch.object(
gitea_audit, "audit_enabled", return_value=True
), patch.object( ), patch.object(
gitea_audit, "write_event", return_value=True gitea_audit, "write_event", return_value=True
), patch.object( ), patch.object(
@@ -357,12 +370,150 @@ class TestBreakGlassRestart(unittest.TestCase):
self.assertTrue(res["break_glass_executed"]) self.assertTrue(res["break_glass_executed"])
self.assertEqual(res["incident_issue"]["number"], 555) self.assertEqual(res["incident_issue"]["number"], 555)
def test_unsupported_apply_returns_truthful_unsupported_result(self) -> None:
"""B6/B11: apply_supported=False returns apply_unsupported blocker and break_glass_executed=False."""
controller_prof = {"profile_name": "prgs-controller", "allowed_operations": ["gitea.read", "gitea.issue.create", "runtime.break_glass_restart"]}
mock_api_request = MagicMock(return_value={"number": 555, "title": "[INCIDENT]"})
mock_restart_unsupported = {"success": True, "apply_supported": False, "apply_authorized": True, "restart_performed": False}
with patch.object(gitea_mcp_server, "get_profile", return_value=controller_prof), patch.object(
gitea_mcp_server, "_profile_operation_gate", return_value=None
), patch.object(
gitea_mcp_server, "_auth", return_value={"Authorization": "token test"}
), patch.object(
gitea_mcp_server, "_authenticated_username", return_value="sysadmin"
), patch.object(
gitea_mcp_server, "gitea_request_mcp_restart", side_effect=[{"affected_sessions": []}, mock_restart_unsupported]
), patch.object(
gitea_audit, "audit_enabled", return_value=True
), patch.object(
gitea_audit, "write_event", return_value=True
), patch.object(
gitea_mcp_server, "api_request", mock_api_request
):
res = gitea_mcp_server.gitea_break_glass_restart(
reason="Privileged restart request with unsupported coordinator apply",
confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION",
impact_ack=True,
dry_run=False,
create_incident_issue=True,
remote="prgs",
)
self.assertFalse(res["success"])
self.assertFalse(res["performed"])
self.assertFalse(res["break_glass_executed"])
self.assertEqual(res["blocker_kind"], "apply_unsupported")
def test_reconciliation_failure_after_execution(self) -> None:
"""B10: Post-restart reconciliation failure reports break_glass_executed=True but success=False."""
controller_prof = {"profile_name": "prgs-controller", "allowed_operations": ["gitea.read", "gitea.issue.create", "runtime.break_glass_restart"]}
mock_api_request = MagicMock(return_value={"number": 555, "title": "[INCIDENT]"})
mock_restart_exec = {"success": True, "apply_supported": True, "apply_authorized": True, "restart_performed": True}
mock_recon = {"success": False, "error": "lease cleanup failed"}
with patch.object(gitea_mcp_server, "get_profile", return_value=controller_prof), patch.object(
gitea_mcp_server, "_profile_operation_gate", return_value=None
), patch.object(
gitea_mcp_server, "_auth", return_value={"Authorization": "token test"}
), patch.object(
gitea_mcp_server, "_authenticated_username", return_value="sysadmin"
), patch.object(
gitea_mcp_server, "gitea_request_mcp_restart", return_value=mock_restart_exec
), patch.object(
gitea_mcp_server, "gitea_reconcile_after_restart", return_value=mock_recon
), patch.object(
gitea_audit, "audit_enabled", return_value=True
), patch.object(
gitea_audit, "write_event", return_value=True
), patch.object(
gitea_mcp_server, "api_request", mock_api_request
):
res = gitea_mcp_server.gitea_break_glass_restart(
reason="Privileged restart request with failing reconciliation",
confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION",
impact_ack=True,
dry_run=False,
create_incident_issue=True,
remote="prgs",
)
self.assertFalse(res["success"])
self.assertTrue(res["performed"])
self.assertTrue(res["break_glass_executed"])
self.assertEqual(res["blocker_kind"], "reconciliation_failed")
def test_bare_secret_redaction(self) -> None:
"""B8: Bare token shapes like ghp_... and sk-live-... are redacted on all surfaces."""
controller_prof = {"profile_name": "prgs-controller", "allowed_operations": ["gitea.read", "gitea.issue.create", "runtime.break_glass_restart"]}
raw_reason = "Emergency restart reason containing bare tokens ghp_1234567890abcdef12345678 and sk-live-abcdef1234567890abcdef"
mock_api_request = MagicMock(return_value={"number": 101, "title": "[INCIDENT]"})
mock_restart_exec = {"success": True, "apply_supported": True, "apply_authorized": True, "restart_performed": True}
mock_recon = {"success": True}
with patch.object(gitea_mcp_server, "get_profile", return_value=controller_prof), patch.object(
gitea_mcp_server, "_profile_operation_gate", return_value=None
), patch.object(
gitea_mcp_server, "_auth", return_value={"Authorization": "token test"}
), patch.object(
gitea_mcp_server, "_authenticated_username", return_value="sysadmin"
), patch.object(
gitea_mcp_server, "api_request", mock_api_request
), patch.object(
gitea_mcp_server, "gitea_request_mcp_restart", return_value=mock_restart_exec
), patch.object(
gitea_mcp_server, "gitea_reconcile_after_restart", return_value=mock_recon
), patch.object(
gitea_audit, "audit_enabled", return_value=True
), patch.object(
gitea_audit, "write_event", return_value=True
):
res = gitea_mcp_server.gitea_break_glass_restart(
reason=raw_reason,
confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION",
impact_ack=True,
dry_run=False,
create_incident_issue=True,
remote="prgs",
)
self.assertTrue(res["success"])
self.assertNotIn("ghp_1234567890abcdef12345678", res["reason"])
self.assertNotIn("sk-live-abcdef1234567890abcdef", res["reason"])
self.assertIn("[REDACTED]", res["reason"])
def test_audit_disabled_fails_closed(self) -> None:
"""B9: Disabling audit recording blocks execution fail-closed."""
controller_prof = {"profile_name": "prgs-controller", "allowed_operations": ["gitea.read", "gitea.issue.create", "runtime.break_glass_restart"]}
with patch.object(gitea_mcp_server, "get_profile", return_value=controller_prof), patch.object(
gitea_mcp_server, "_profile_operation_gate", return_value=None
), patch.object(
gitea_audit, "audit_enabled", return_value=False
):
res = gitea_mcp_server.gitea_break_glass_restart(
reason="Emergency restart with disabled audit logging",
confirmation="I_ACKNOWLEDGE_BREAK_GLASS_MCP_RESTART_DISRUPTION",
impact_ack=True,
dry_run=False,
create_incident_issue=True,
remote="prgs",
)
self.assertFalse(res["success"])
self.assertFalse(res["break_glass_executed"])
self.assertEqual(res["blocker_kind"], "audit_recording_failed")
def test_capability_map_registration(self) -> None:
"""B12: Ensure task_capability_map maps gitea_break_glass_restart to runtime.break_glass_restart."""
from task_capability_map import TASK_CAPABILITY_MAP
entry = TASK_CAPABILITY_MAP.get("gitea_break_glass_restart")
self.assertIsNotNone(entry)
self.assertEqual(entry["permission"], "runtime.break_glass_restart")
self.assertEqual(entry["role"], "controller")
def test_delegated_execution_failure(self) -> None: def test_delegated_execution_failure(self) -> None:
"""Requirement 4: Delegated restart execution failure produces distinct terminal state.""" """Requirement 4: Delegated restart execution failure produces distinct terminal state."""
controller_prof = {"role": "controller", "allowed_operations": ["gitea.read", "gitea.issue.create"]} controller_prof = {"profile_name": "prgs-controller", "allowed_operations": ["gitea.read", "gitea.issue.create", "runtime.break_glass_restart"]}
mock_api_request = MagicMock(return_value={"number": 555, "title": "[INCIDENT]"}) mock_api_request = MagicMock(return_value={"number": 555, "title": "[INCIDENT]"})
mock_impact_eval = {"affected_sessions": []} mock_impact_eval = {"affected_sessions": []}
mock_restart_denied = {"success": False, "apply_authorized": False, "reasons": ["restart class denied"]} mock_restart_denied = {"success": False, "apply_supported": True, "apply_authorized": False, "reasons": ["restart class denied"]}
with patch.object(gitea_mcp_server, "get_profile", return_value=controller_prof), patch.object( with patch.object(gitea_mcp_server, "get_profile", return_value=controller_prof), patch.object(
gitea_mcp_server, "_profile_operation_gate", return_value=None gitea_mcp_server, "_profile_operation_gate", return_value=None
@@ -372,6 +523,8 @@ class TestBreakGlassRestart(unittest.TestCase):
gitea_mcp_server, "_authenticated_username", return_value="sysadmin" gitea_mcp_server, "_authenticated_username", return_value="sysadmin"
), patch.object( ), patch.object(
gitea_mcp_server, "gitea_request_mcp_restart", side_effect=[mock_impact_eval, mock_restart_denied] gitea_mcp_server, "gitea_request_mcp_restart", side_effect=[mock_impact_eval, mock_restart_denied]
), patch.object(
gitea_audit, "audit_enabled", return_value=True
), patch.object( ), patch.object(
gitea_audit, "write_event", return_value=True gitea_audit, "write_event", return_value=True
), patch.object( ), patch.object(