"""MCP restart lifecycle audit events and incident materialization (#665). Restarts and recovery attempts must leave a forensic trail: impact previews, drain enter/exit, drain-proof results, apply gate verdicts, break-glass, and post-restart reconcile outcomes. Failed drains and break-glass must also raise durable Gitea incident issues so they cannot be silently repeated. This module is the pure + sink layer for that trail: * **Schema** — ``mcp.restart.*`` event names and a redacted payload builder. * **Emission** — append-only via :mod:`gitea_audit` (off when ``GITEA_AUDIT_LOG`` is unset; privileged apply can still *require* a successful write). * **Incidents** — descriptors for failed drain / break-glass / unguarded restart, plus an optional materializer that creates a Gitea issue through an injected ``create_issue_fn`` (keeps this module free of network I/O in tests). Design rules: * **No secrets.** All free text is redacted before write or issue body assembly. * **Never raises from emission.** ``emit_restart_event`` returns False on sink failure so callers can decide fail-closed policy for privileged restarts. * **Does not restart.** Audit never executes a process restart. * **Drain proof stays #661.** This module records what the gate decided; it does not re-verify proofs. """ from __future__ import annotations import uuid from datetime import datetime, timezone from typing import Any, Callable, Mapping, Sequence import gitea_audit # ── Event vocabulary (stable identifiers for operators + tests) ─────────────── EVENT_IMPACT_PREVIEW = "mcp.restart.impact_preview" EVENT_DRAIN_ENTER = "mcp.restart.drain_enter" EVENT_DRAIN_EXIT = "mcp.restart.drain_exit" EVENT_DRAIN_PROOF = "mcp.restart.drain_proof" EVENT_APPLY_GATE = "mcp.restart.apply_gate" EVENT_BREAK_GLASS = "mcp.restart.break_glass" EVENT_POST_RESTART_RECONCILE = "mcp.restart.post_restart_reconcile" EVENT_NARROWER_RECOVERY = "mcp.restart.narrower_recovery" EVENT_UNGUARDED_DETECTED = "mcp.restart.unguarded_detected" RESTART_EVENT_TYPES: frozenset[str] = frozenset( { EVENT_IMPACT_PREVIEW, EVENT_DRAIN_ENTER, EVENT_DRAIN_EXIT, EVENT_DRAIN_PROOF, EVENT_APPLY_GATE, EVENT_BREAK_GLASS, EVENT_POST_RESTART_RECONCILE, EVENT_NARROWER_RECOVERY, EVENT_UNGUARDED_DETECTED, } ) # Incident kinds (durable Gitea issues). INCIDENT_FAILED_DRAIN = "restart_failed_drain" INCIDENT_BREAK_GLASS = "restart_break_glass" INCIDENT_RECONCILE_UNRESOLVED = "restart_reconcile_unresolved" INCIDENT_UNGUARDED = "restart_unguarded_detected" DEFAULT_INCIDENT_LABELS: tuple[str, ...] = ( "mcp-health", "safety", "observability", "status:ready", "type:bug", "workflow-hardening", ) CreateIssueFn = Callable[..., dict[str, Any]] def _utc_now_iso() -> str: return datetime.now(timezone.utc).isoformat() def new_correlation_id() -> str: """Mint a short correlation id shared across a restart lifecycle.""" return f"rst-{uuid.uuid4().hex[:16]}" def build_restart_event( *, event_type: str, outcome: str, correlation_id: str | None = None, remote: str | None = None, org: str | None = None, repo: str | None = None, requesting_session_id: str | None = None, restart_class: str | None = None, profile_name: str | None = None, authenticated_username: str | None = None, reasons: Sequence[str] | None = None, details: Mapping[str, Any] | None = None, now: str | None = None, ) -> dict[str, Any]: """Build a redacted ``mcp.restart.*`` audit event. Raises ``ValueError`` on unknown event types so a typo cannot silently land under a free-form action name. """ name = str(event_type or "").strip() if name not in RESTART_EVENT_TYPES: raise ValueError( f"unknown restart audit event_type {name!r}; expected one of " f"{sorted(RESTART_EVENT_TYPES)}" ) redacted_reasons = [ gitea_audit.redact(str(r)) for r in (reasons or []) if str(r).strip() ] redacted_details = gitea_audit.redact(dict(details or {})) if not isinstance(redacted_details, dict): redacted_details = {"value": redacted_details} event = gitea_audit.build_event( action=name, result=str(outcome or "unknown"), remote=remote, repository=f"{org}/{repo}" if org and repo else None, profile_name=profile_name, authenticated_username=authenticated_username, reason="; ".join(redacted_reasons) if redacted_reasons else None, request_metadata={ "event_family": "mcp.restart", "correlation_id": correlation_id or new_correlation_id(), "restart_class": restart_class, "requesting_session_id": requesting_session_id, "org": org, "repo": repo, "details": redacted_details, "reasons": redacted_reasons, }, now=now or _utc_now_iso(), operation=name, ) event["action_type"] = "restart_lifecycle" event["event_type"] = name event["correlation_id"] = (event.get("request_metadata") or {}).get( "correlation_id" ) return event def emit_restart_event(event: Mapping[str, Any], *, path: str | None = None) -> bool: """Append *event* to the audit sink. Never raises. Returns write success.""" try: return bool(gitea_audit.write_event(dict(event), path=path)) except Exception: return False def require_audit_or_deny( *, privileged: bool, written: bool, audit_enabled: bool | None = None, ) -> list[str]: """Return deny reasons when a privileged restart path fails to audit. When audit is not configured (``GITEA_AUDIT_LOG`` unset), privileged apply still proceeds under the rollout policy "enable audit before enforcing deny-on-audit-fail" — but *if* audit is enabled and the write fails, privileged apply is denied (fail closed). """ enabled = ( gitea_audit.audit_enabled() if audit_enabled is None else bool(audit_enabled) ) if not privileged: return [] if not enabled: return [] if written: return [] return [ "privileged restart path requires a successful audit write; " "audit sink failed (fail closed, #665)" ] # ── Incident descriptors ────────────────────────────────────────────────────── def build_incident_descriptor( *, kind: str, reasons: Sequence[str], correlation_id: str | None = None, requesting_session_id: str | None = None, restart_class: str | None = None, remote: str | None = None, org: str | None = None, repo: str | None = None, proof_id: str | None = None, at: str | None = None, ) -> dict[str, Any]: """Build a durable incident descriptor (no network).""" titles = { INCIDENT_FAILED_DRAIN: "Restart denied: drain proof failed the hard gate", INCIDENT_BREAK_GLASS: "Break-glass MCP restart authorized", INCIDENT_RECONCILE_UNRESOLVED: "Post-restart reconcile left unresolved work", INCIDENT_UNGUARDED: "Unguarded MCP restart attempt detected", } title = titles.get(kind, f"MCP restart incident ({kind})") redacted_reasons = [ gitea_audit.redact(str(r)) for r in reasons if str(r).strip() ] return { "kind": kind, "title": title, "labels": list(DEFAULT_INCIDENT_LABELS), "reasons": redacted_reasons, "correlation_id": correlation_id, "requesting_session_id": requesting_session_id, "restart_class": restart_class, "remote": remote, "org": org, "repo": repo, "proof_id": proof_id, "at": at or _utc_now_iso(), "source": "restart_audit#665", } def incident_body(descriptor: Mapping[str, Any]) -> str: """Render a redacted markdown body for a Gitea incident issue.""" reasons = descriptor.get("reasons") or [] reason_lines = "\n".join(f"- {gitea_audit.redact(str(r))}" for r in reasons) or ( "- (no reasons recorded)" ) return "\n".join( [ "", f"## MCP restart incident (`{descriptor.get('kind')}`)", "", f"**Correlation:** `{descriptor.get('correlation_id') or 'none'}`", f"**Session:** `{descriptor.get('requesting_session_id') or 'none'}`", f"**Class:** `{descriptor.get('restart_class') or 'none'}`", f"**Scope:** `{descriptor.get('remote')}/{descriptor.get('org')}/" f"{descriptor.get('repo')}`", f"**At:** `{descriptor.get('at')}`", f"**Proof id:** `{descriptor.get('proof_id') or 'none'}`", "", "### Reasons", reason_lines, "", "### Operator next steps", "- Treat this as durable follow-up work under the restart-governance umbrella (#655).", "- Do not invent a second restart path; use sanctioned coordinator tools only.", "- Raw secrets must never appear in this issue (already redacted).", "", f"_Source: {descriptor.get('source')}_", ] ) def materialize_incident( descriptor: Mapping[str, Any], *, create_issue_fn: CreateIssueFn | None, dry_run: bool = False, ) -> dict[str, Any]: """Create a Gitea issue from *descriptor* when *create_issue_fn* is provided. Returns a result dict with ``created`` / ``issue_number`` / ``dry_run`` / ``reasons``. Never raises. """ base: dict[str, Any] = { "created": False, "dry_run": bool(dry_run), "issue_number": None, "kind": descriptor.get("kind"), "reasons": [], "descriptor": dict(descriptor), } if dry_run: base["reasons"] = ["dry-run only; no Gitea issue created"] return base if create_issue_fn is None: base["reasons"] = [ "create_issue_fn not provided; incident descriptor retained only" ] return base try: result = create_issue_fn( title=str(descriptor.get("title") or "MCP restart incident"), body=incident_body(descriptor), labels=list(descriptor.get("labels") or DEFAULT_INCIDENT_LABELS), org=descriptor.get("org"), repo=descriptor.get("repo"), ) number = None if isinstance(result, dict): number = result.get("number") or result.get("issue_number") if number is not None: base["created"] = True base["issue_number"] = int(number) base["reasons"] = [f"created incident issue #{int(number)}"] else: base["reasons"] = ["create_issue_fn returned no issue number"] except Exception as exc: # noqa: BLE001 — never break restart path here base["reasons"] = [ f"incident issue creation failed: {gitea_audit.redact(str(exc))}" ] return base def record_restart_lifecycle( *, event_type: str, outcome: str, correlation_id: str, remote: str | None = None, org: str | None = None, repo: str | None = None, requesting_session_id: str | None = None, restart_class: str | None = None, profile_name: str | None = None, authenticated_username: str | None = None, reasons: Sequence[str] | None = None, details: Mapping[str, Any] | None = None, privileged: bool = False, create_incident: Mapping[str, Any] | None = None, create_issue_fn: CreateIssueFn | None = None, dry_run_incident: bool = False, audit_path: str | None = None, ) -> dict[str, Any]: """Emit one restart audit event and optionally materialize an incident. Returns ``{event, audit_written, deny_reasons, incident_result}``. """ event = build_restart_event( event_type=event_type, outcome=outcome, correlation_id=correlation_id, remote=remote, org=org, repo=repo, requesting_session_id=requesting_session_id, restart_class=restart_class, profile_name=profile_name, authenticated_username=authenticated_username, reasons=reasons, details=details, ) written = emit_restart_event(event, path=audit_path) deny = require_audit_or_deny(privileged=privileged, written=written) incident_result = None if create_incident is not None: incident_result = materialize_incident( create_incident, create_issue_fn=create_issue_fn, dry_run=dry_run_incident, ) return { "event": event, "audit_written": written, "deny_reasons": deny, "incident_result": incident_result, "correlation_id": correlation_id, } def incident_from_apply_gate( *, gate_payload: Mapping[str, Any], break_glass: bool, correlation_id: str, requesting_session_id: str | None, restart_class: str | None, remote: str | None, org: str | None, repo: str | None, ) -> dict[str, Any] | None: """Choose an incident descriptor from an apply-gate payload, if required.""" reasons = list(gate_payload.get("reasons") or []) proof_id = gate_payload.get("proof_id") if break_glass: return build_incident_descriptor( kind=INCIDENT_BREAK_GLASS, reasons=reasons or ["break-glass restart path used; durable incident required (#665)"], correlation_id=correlation_id, requesting_session_id=requesting_session_id, restart_class=restart_class, remote=remote, org=org, repo=repo, proof_id=proof_id if isinstance(proof_id, str) else None, ) # Failed drain / deny path. incident = gate_payload.get("incident") if isinstance(incident, Mapping) and incident: # Normalize gate-provided descriptor into our schema. return build_incident_descriptor( kind=INCIDENT_FAILED_DRAIN, reasons=list(incident.get("reasons") or reasons), correlation_id=correlation_id, requesting_session_id=requesting_session_id or incident.get("requesting_session_id"), restart_class=restart_class, remote=remote, org=org, repo=repo, proof_id=incident.get("proof_id") or proof_id, at=incident.get("at"), ) if not gate_payload.get("allow") and not gate_payload.get("drain_gate_allow", True): return build_incident_descriptor( kind=INCIDENT_FAILED_DRAIN, reasons=reasons or ["restart apply denied"], correlation_id=correlation_id, requesting_session_id=requesting_session_id, restart_class=restart_class, remote=remote, org=org, repo=repo, proof_id=proof_id if isinstance(proof_id, str) else None, ) return None