feat(restart): audit lifecycle events and durable incidents (#665)
Add restart_audit with mcp.restart.* event schema, redacted emission via gitea_audit, correlation ids, and incident materialization for failed drain and break-glass. Wire gitea_request_mcp_restart to always audit impact previews and fail closed on privileged apply when the audit sink is enabled but write fails. Closes #665
This commit is contained in:
@@ -0,0 +1,426 @@
|
||||
"""MCP restart lifecycle audit events and incident materialization (#665).
|
||||
|
||||
Restarts and recovery attempts must leave a forensic trail: impact previews,
|
||||
drain enter/exit, drain-proof results, apply gate verdicts, break-glass, and
|
||||
post-restart reconcile outcomes. Failed drains and break-glass must also raise
|
||||
durable Gitea incident issues so they cannot be silently repeated.
|
||||
|
||||
This module is the pure + sink layer for that trail:
|
||||
|
||||
* **Schema** — ``mcp.restart.*`` event names and a redacted payload builder.
|
||||
* **Emission** — append-only via :mod:`gitea_audit` (off when ``GITEA_AUDIT_LOG``
|
||||
is unset; privileged apply can still *require* a successful write).
|
||||
* **Incidents** — descriptors for failed drain / break-glass / unguarded restart,
|
||||
plus an optional materializer that creates a Gitea issue through an injected
|
||||
``create_issue_fn`` (keeps this module free of network I/O in tests).
|
||||
|
||||
Design rules:
|
||||
|
||||
* **No secrets.** All free text is redacted before write or issue body assembly.
|
||||
* **Never raises from emission.** ``emit_restart_event`` returns False on sink
|
||||
failure so callers can decide fail-closed policy for privileged restarts.
|
||||
* **Does not restart.** Audit never executes a process restart.
|
||||
* **Drain proof stays #661.** This module records what the gate decided; it
|
||||
does not re-verify proofs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Callable, Mapping, Sequence
|
||||
|
||||
import gitea_audit
|
||||
|
||||
# ── Event vocabulary (stable identifiers for operators + tests) ───────────────
|
||||
|
||||
EVENT_IMPACT_PREVIEW = "mcp.restart.impact_preview"
|
||||
EVENT_DRAIN_ENTER = "mcp.restart.drain_enter"
|
||||
EVENT_DRAIN_EXIT = "mcp.restart.drain_exit"
|
||||
EVENT_DRAIN_PROOF = "mcp.restart.drain_proof"
|
||||
EVENT_APPLY_GATE = "mcp.restart.apply_gate"
|
||||
EVENT_BREAK_GLASS = "mcp.restart.break_glass"
|
||||
EVENT_POST_RESTART_RECONCILE = "mcp.restart.post_restart_reconcile"
|
||||
EVENT_NARROWER_RECOVERY = "mcp.restart.narrower_recovery"
|
||||
EVENT_UNGUARDED_DETECTED = "mcp.restart.unguarded_detected"
|
||||
|
||||
RESTART_EVENT_TYPES: frozenset[str] = frozenset(
|
||||
{
|
||||
EVENT_IMPACT_PREVIEW,
|
||||
EVENT_DRAIN_ENTER,
|
||||
EVENT_DRAIN_EXIT,
|
||||
EVENT_DRAIN_PROOF,
|
||||
EVENT_APPLY_GATE,
|
||||
EVENT_BREAK_GLASS,
|
||||
EVENT_POST_RESTART_RECONCILE,
|
||||
EVENT_NARROWER_RECOVERY,
|
||||
EVENT_UNGUARDED_DETECTED,
|
||||
}
|
||||
)
|
||||
|
||||
# Incident kinds (durable Gitea issues).
|
||||
INCIDENT_FAILED_DRAIN = "restart_failed_drain"
|
||||
INCIDENT_BREAK_GLASS = "restart_break_glass"
|
||||
INCIDENT_RECONCILE_UNRESOLVED = "restart_reconcile_unresolved"
|
||||
INCIDENT_UNGUARDED = "restart_unguarded_detected"
|
||||
|
||||
DEFAULT_INCIDENT_LABELS: tuple[str, ...] = (
|
||||
"mcp-health",
|
||||
"safety",
|
||||
"observability",
|
||||
"status:ready",
|
||||
"type:bug",
|
||||
"workflow-hardening",
|
||||
)
|
||||
|
||||
CreateIssueFn = Callable[..., dict[str, Any]]
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def new_correlation_id() -> str:
|
||||
"""Mint a short correlation id shared across a restart lifecycle."""
|
||||
return f"rst-{uuid.uuid4().hex[:16]}"
|
||||
|
||||
|
||||
def build_restart_event(
|
||||
*,
|
||||
event_type: str,
|
||||
outcome: str,
|
||||
correlation_id: str | None = None,
|
||||
remote: str | None = None,
|
||||
org: str | None = None,
|
||||
repo: str | None = None,
|
||||
requesting_session_id: str | None = None,
|
||||
restart_class: str | None = None,
|
||||
profile_name: str | None = None,
|
||||
authenticated_username: str | None = None,
|
||||
reasons: Sequence[str] | None = None,
|
||||
details: Mapping[str, Any] | None = None,
|
||||
now: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a redacted ``mcp.restart.*`` audit event.
|
||||
|
||||
Raises ``ValueError`` on unknown event types so a typo cannot silently land
|
||||
under a free-form action name.
|
||||
"""
|
||||
name = str(event_type or "").strip()
|
||||
if name not in RESTART_EVENT_TYPES:
|
||||
raise ValueError(
|
||||
f"unknown restart audit event_type {name!r}; expected one of "
|
||||
f"{sorted(RESTART_EVENT_TYPES)}"
|
||||
)
|
||||
redacted_reasons = [
|
||||
gitea_audit.redact(str(r)) for r in (reasons or []) if str(r).strip()
|
||||
]
|
||||
redacted_details = gitea_audit.redact(dict(details or {}))
|
||||
if not isinstance(redacted_details, dict):
|
||||
redacted_details = {"value": redacted_details}
|
||||
|
||||
event = gitea_audit.build_event(
|
||||
action=name,
|
||||
result=str(outcome or "unknown"),
|
||||
remote=remote,
|
||||
repository=f"{org}/{repo}" if org and repo else None,
|
||||
profile_name=profile_name,
|
||||
authenticated_username=authenticated_username,
|
||||
reason="; ".join(redacted_reasons) if redacted_reasons else None,
|
||||
request_metadata={
|
||||
"event_family": "mcp.restart",
|
||||
"correlation_id": correlation_id or new_correlation_id(),
|
||||
"restart_class": restart_class,
|
||||
"requesting_session_id": requesting_session_id,
|
||||
"org": org,
|
||||
"repo": repo,
|
||||
"details": redacted_details,
|
||||
"reasons": redacted_reasons,
|
||||
},
|
||||
now=now or _utc_now_iso(),
|
||||
operation=name,
|
||||
)
|
||||
event["action_type"] = "restart_lifecycle"
|
||||
event["event_type"] = name
|
||||
event["correlation_id"] = (event.get("request_metadata") or {}).get(
|
||||
"correlation_id"
|
||||
)
|
||||
return event
|
||||
|
||||
|
||||
def emit_restart_event(event: Mapping[str, Any], *, path: str | None = None) -> bool:
|
||||
"""Append *event* to the audit sink. Never raises. Returns write success."""
|
||||
try:
|
||||
return bool(gitea_audit.write_event(dict(event), path=path))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def require_audit_or_deny(
|
||||
*,
|
||||
privileged: bool,
|
||||
written: bool,
|
||||
audit_enabled: bool | None = None,
|
||||
) -> list[str]:
|
||||
"""Return deny reasons when a privileged restart path fails to audit.
|
||||
|
||||
When audit is not configured (``GITEA_AUDIT_LOG`` unset), privileged apply
|
||||
still proceeds under the rollout policy "enable audit before enforcing
|
||||
deny-on-audit-fail" — but *if* audit is enabled and the write fails,
|
||||
privileged apply is denied (fail closed).
|
||||
"""
|
||||
enabled = (
|
||||
gitea_audit.audit_enabled() if audit_enabled is None else bool(audit_enabled)
|
||||
)
|
||||
if not privileged:
|
||||
return []
|
||||
if not enabled:
|
||||
return []
|
||||
if written:
|
||||
return []
|
||||
return [
|
||||
"privileged restart path requires a successful audit write; "
|
||||
"audit sink failed (fail closed, #665)"
|
||||
]
|
||||
|
||||
|
||||
# ── Incident descriptors ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def build_incident_descriptor(
|
||||
*,
|
||||
kind: str,
|
||||
reasons: Sequence[str],
|
||||
correlation_id: str | None = None,
|
||||
requesting_session_id: str | None = None,
|
||||
restart_class: str | None = None,
|
||||
remote: str | None = None,
|
||||
org: str | None = None,
|
||||
repo: str | None = None,
|
||||
proof_id: str | None = None,
|
||||
at: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a durable incident descriptor (no network)."""
|
||||
titles = {
|
||||
INCIDENT_FAILED_DRAIN: "Restart denied: drain proof failed the hard gate",
|
||||
INCIDENT_BREAK_GLASS: "Break-glass MCP restart authorized",
|
||||
INCIDENT_RECONCILE_UNRESOLVED: "Post-restart reconcile left unresolved work",
|
||||
INCIDENT_UNGUARDED: "Unguarded MCP restart attempt detected",
|
||||
}
|
||||
title = titles.get(kind, f"MCP restart incident ({kind})")
|
||||
redacted_reasons = [
|
||||
gitea_audit.redact(str(r)) for r in reasons if str(r).strip()
|
||||
]
|
||||
return {
|
||||
"kind": kind,
|
||||
"title": title,
|
||||
"labels": list(DEFAULT_INCIDENT_LABELS),
|
||||
"reasons": redacted_reasons,
|
||||
"correlation_id": correlation_id,
|
||||
"requesting_session_id": requesting_session_id,
|
||||
"restart_class": restart_class,
|
||||
"remote": remote,
|
||||
"org": org,
|
||||
"repo": repo,
|
||||
"proof_id": proof_id,
|
||||
"at": at or _utc_now_iso(),
|
||||
"source": "restart_audit#665",
|
||||
}
|
||||
|
||||
|
||||
def incident_body(descriptor: Mapping[str, Any]) -> str:
|
||||
"""Render a redacted markdown body for a Gitea incident issue."""
|
||||
reasons = descriptor.get("reasons") or []
|
||||
reason_lines = "\n".join(f"- {gitea_audit.redact(str(r))}" for r in reasons) or (
|
||||
"- (no reasons recorded)"
|
||||
)
|
||||
return "\n".join(
|
||||
[
|
||||
"<!-- mcp-restart-incident:v1 -->",
|
||||
f"## MCP restart incident (`{descriptor.get('kind')}`)",
|
||||
"",
|
||||
f"**Correlation:** `{descriptor.get('correlation_id') or 'none'}`",
|
||||
f"**Session:** `{descriptor.get('requesting_session_id') or 'none'}`",
|
||||
f"**Class:** `{descriptor.get('restart_class') or 'none'}`",
|
||||
f"**Scope:** `{descriptor.get('remote')}/{descriptor.get('org')}/"
|
||||
f"{descriptor.get('repo')}`",
|
||||
f"**At:** `{descriptor.get('at')}`",
|
||||
f"**Proof id:** `{descriptor.get('proof_id') or 'none'}`",
|
||||
"",
|
||||
"### Reasons",
|
||||
reason_lines,
|
||||
"",
|
||||
"### Operator next steps",
|
||||
"- Treat this as durable follow-up work under the restart-governance umbrella (#655).",
|
||||
"- Do not invent a second restart path; use sanctioned coordinator tools only.",
|
||||
"- Raw secrets must never appear in this issue (already redacted).",
|
||||
"",
|
||||
f"_Source: {descriptor.get('source')}_",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def materialize_incident(
|
||||
descriptor: Mapping[str, Any],
|
||||
*,
|
||||
create_issue_fn: CreateIssueFn | None,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a Gitea issue from *descriptor* when *create_issue_fn* is provided.
|
||||
|
||||
Returns a result dict with ``created`` / ``issue_number`` / ``dry_run`` /
|
||||
``reasons``. Never raises.
|
||||
"""
|
||||
base: dict[str, Any] = {
|
||||
"created": False,
|
||||
"dry_run": bool(dry_run),
|
||||
"issue_number": None,
|
||||
"kind": descriptor.get("kind"),
|
||||
"reasons": [],
|
||||
"descriptor": dict(descriptor),
|
||||
}
|
||||
if dry_run:
|
||||
base["reasons"] = ["dry-run only; no Gitea issue created"]
|
||||
return base
|
||||
if create_issue_fn is None:
|
||||
base["reasons"] = [
|
||||
"create_issue_fn not provided; incident descriptor retained only"
|
||||
]
|
||||
return base
|
||||
try:
|
||||
result = create_issue_fn(
|
||||
title=str(descriptor.get("title") or "MCP restart incident"),
|
||||
body=incident_body(descriptor),
|
||||
labels=list(descriptor.get("labels") or DEFAULT_INCIDENT_LABELS),
|
||||
org=descriptor.get("org"),
|
||||
repo=descriptor.get("repo"),
|
||||
)
|
||||
number = None
|
||||
if isinstance(result, dict):
|
||||
number = result.get("number") or result.get("issue_number")
|
||||
if number is not None:
|
||||
base["created"] = True
|
||||
base["issue_number"] = int(number)
|
||||
base["reasons"] = [f"created incident issue #{int(number)}"]
|
||||
else:
|
||||
base["reasons"] = ["create_issue_fn returned no issue number"]
|
||||
except Exception as exc: # noqa: BLE001 — never break restart path here
|
||||
base["reasons"] = [
|
||||
f"incident issue creation failed: {gitea_audit.redact(str(exc))}"
|
||||
]
|
||||
return base
|
||||
|
||||
|
||||
def record_restart_lifecycle(
|
||||
*,
|
||||
event_type: str,
|
||||
outcome: str,
|
||||
correlation_id: str,
|
||||
remote: str | None = None,
|
||||
org: str | None = None,
|
||||
repo: str | None = None,
|
||||
requesting_session_id: str | None = None,
|
||||
restart_class: str | None = None,
|
||||
profile_name: str | None = None,
|
||||
authenticated_username: str | None = None,
|
||||
reasons: Sequence[str] | None = None,
|
||||
details: Mapping[str, Any] | None = None,
|
||||
privileged: bool = False,
|
||||
create_incident: Mapping[str, Any] | None = None,
|
||||
create_issue_fn: CreateIssueFn | None = None,
|
||||
dry_run_incident: bool = False,
|
||||
audit_path: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Emit one restart audit event and optionally materialize an incident.
|
||||
|
||||
Returns ``{event, audit_written, deny_reasons, incident_result}``.
|
||||
"""
|
||||
event = build_restart_event(
|
||||
event_type=event_type,
|
||||
outcome=outcome,
|
||||
correlation_id=correlation_id,
|
||||
remote=remote,
|
||||
org=org,
|
||||
repo=repo,
|
||||
requesting_session_id=requesting_session_id,
|
||||
restart_class=restart_class,
|
||||
profile_name=profile_name,
|
||||
authenticated_username=authenticated_username,
|
||||
reasons=reasons,
|
||||
details=details,
|
||||
)
|
||||
written = emit_restart_event(event, path=audit_path)
|
||||
deny = require_audit_or_deny(privileged=privileged, written=written)
|
||||
incident_result = None
|
||||
if create_incident is not None:
|
||||
incident_result = materialize_incident(
|
||||
create_incident,
|
||||
create_issue_fn=create_issue_fn,
|
||||
dry_run=dry_run_incident,
|
||||
)
|
||||
return {
|
||||
"event": event,
|
||||
"audit_written": written,
|
||||
"deny_reasons": deny,
|
||||
"incident_result": incident_result,
|
||||
"correlation_id": correlation_id,
|
||||
}
|
||||
|
||||
|
||||
def incident_from_apply_gate(
|
||||
*,
|
||||
gate_payload: Mapping[str, Any],
|
||||
break_glass: bool,
|
||||
correlation_id: str,
|
||||
requesting_session_id: str | None,
|
||||
restart_class: str | None,
|
||||
remote: str | None,
|
||||
org: str | None,
|
||||
repo: str | None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Choose an incident descriptor from an apply-gate payload, if required."""
|
||||
reasons = list(gate_payload.get("reasons") or [])
|
||||
proof_id = gate_payload.get("proof_id")
|
||||
if break_glass:
|
||||
return build_incident_descriptor(
|
||||
kind=INCIDENT_BREAK_GLASS,
|
||||
reasons=reasons
|
||||
or ["break-glass restart path used; durable incident required (#665)"],
|
||||
correlation_id=correlation_id,
|
||||
requesting_session_id=requesting_session_id,
|
||||
restart_class=restart_class,
|
||||
remote=remote,
|
||||
org=org,
|
||||
repo=repo,
|
||||
proof_id=proof_id if isinstance(proof_id, str) else None,
|
||||
)
|
||||
# Failed drain / deny path.
|
||||
incident = gate_payload.get("incident")
|
||||
if isinstance(incident, Mapping) and incident:
|
||||
# Normalize gate-provided descriptor into our schema.
|
||||
return build_incident_descriptor(
|
||||
kind=INCIDENT_FAILED_DRAIN,
|
||||
reasons=list(incident.get("reasons") or reasons),
|
||||
correlation_id=correlation_id,
|
||||
requesting_session_id=requesting_session_id
|
||||
or incident.get("requesting_session_id"),
|
||||
restart_class=restart_class,
|
||||
remote=remote,
|
||||
org=org,
|
||||
repo=repo,
|
||||
proof_id=incident.get("proof_id") or proof_id,
|
||||
at=incident.get("at"),
|
||||
)
|
||||
if not gate_payload.get("allow") and not gate_payload.get("drain_gate_allow", True):
|
||||
return build_incident_descriptor(
|
||||
kind=INCIDENT_FAILED_DRAIN,
|
||||
reasons=reasons or ["restart apply denied"],
|
||||
correlation_id=correlation_id,
|
||||
requesting_session_id=requesting_session_id,
|
||||
restart_class=restart_class,
|
||||
remote=remote,
|
||||
org=org,
|
||||
repo=repo,
|
||||
proof_id=proof_id if isinstance(proof_id, str) else None,
|
||||
)
|
||||
return None
|
||||
Reference in New Issue
Block a user