sysadmin
22e0a41bd5
Add restart_audit with mcp.restart.* event schema, redacted emission via gitea_audit, correlation ids, and incident materialization for failed drain and break-glass. Wire gitea_request_mcp_restart to always audit impact previews and fail closed on privileged apply when the audit sink is enabled but write fails. Closes #665
427 lines
15 KiB
Python
427 lines
15 KiB
Python
"""MCP restart lifecycle audit events and incident materialization (#665).
|
|
|
|
Restarts and recovery attempts must leave a forensic trail: impact previews,
|
|
drain enter/exit, drain-proof results, apply gate verdicts, break-glass, and
|
|
post-restart reconcile outcomes. Failed drains and break-glass must also raise
|
|
durable Gitea incident issues so they cannot be silently repeated.
|
|
|
|
This module is the pure + sink layer for that trail:
|
|
|
|
* **Schema** — ``mcp.restart.*`` event names and a redacted payload builder.
|
|
* **Emission** — append-only via :mod:`gitea_audit` (off when ``GITEA_AUDIT_LOG``
|
|
is unset; privileged apply can still *require* a successful write).
|
|
* **Incidents** — descriptors for failed drain / break-glass / unguarded restart,
|
|
plus an optional materializer that creates a Gitea issue through an injected
|
|
``create_issue_fn`` (keeps this module free of network I/O in tests).
|
|
|
|
Design rules:
|
|
|
|
* **No secrets.** All free text is redacted before write or issue body assembly.
|
|
* **Never raises from emission.** ``emit_restart_event`` returns False on sink
|
|
failure so callers can decide fail-closed policy for privileged restarts.
|
|
* **Does not restart.** Audit never executes a process restart.
|
|
* **Drain proof stays #661.** This module records what the gate decided; it
|
|
does not re-verify proofs.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Callable, Mapping, Sequence
|
|
|
|
import gitea_audit
|
|
|
|
# ── Event vocabulary (stable identifiers for operators + tests) ───────────────
|
|
|
|
EVENT_IMPACT_PREVIEW = "mcp.restart.impact_preview"
|
|
EVENT_DRAIN_ENTER = "mcp.restart.drain_enter"
|
|
EVENT_DRAIN_EXIT = "mcp.restart.drain_exit"
|
|
EVENT_DRAIN_PROOF = "mcp.restart.drain_proof"
|
|
EVENT_APPLY_GATE = "mcp.restart.apply_gate"
|
|
EVENT_BREAK_GLASS = "mcp.restart.break_glass"
|
|
EVENT_POST_RESTART_RECONCILE = "mcp.restart.post_restart_reconcile"
|
|
EVENT_NARROWER_RECOVERY = "mcp.restart.narrower_recovery"
|
|
EVENT_UNGUARDED_DETECTED = "mcp.restart.unguarded_detected"
|
|
|
|
RESTART_EVENT_TYPES: frozenset[str] = frozenset(
|
|
{
|
|
EVENT_IMPACT_PREVIEW,
|
|
EVENT_DRAIN_ENTER,
|
|
EVENT_DRAIN_EXIT,
|
|
EVENT_DRAIN_PROOF,
|
|
EVENT_APPLY_GATE,
|
|
EVENT_BREAK_GLASS,
|
|
EVENT_POST_RESTART_RECONCILE,
|
|
EVENT_NARROWER_RECOVERY,
|
|
EVENT_UNGUARDED_DETECTED,
|
|
}
|
|
)
|
|
|
|
# Incident kinds (durable Gitea issues).
|
|
INCIDENT_FAILED_DRAIN = "restart_failed_drain"
|
|
INCIDENT_BREAK_GLASS = "restart_break_glass"
|
|
INCIDENT_RECONCILE_UNRESOLVED = "restart_reconcile_unresolved"
|
|
INCIDENT_UNGUARDED = "restart_unguarded_detected"
|
|
|
|
DEFAULT_INCIDENT_LABELS: tuple[str, ...] = (
|
|
"mcp-health",
|
|
"safety",
|
|
"observability",
|
|
"status:ready",
|
|
"type:bug",
|
|
"workflow-hardening",
|
|
)
|
|
|
|
CreateIssueFn = Callable[..., dict[str, Any]]
|
|
|
|
|
|
def _utc_now_iso() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def new_correlation_id() -> str:
|
|
"""Mint a short correlation id shared across a restart lifecycle."""
|
|
return f"rst-{uuid.uuid4().hex[:16]}"
|
|
|
|
|
|
def build_restart_event(
|
|
*,
|
|
event_type: str,
|
|
outcome: str,
|
|
correlation_id: str | None = None,
|
|
remote: str | None = None,
|
|
org: str | None = None,
|
|
repo: str | None = None,
|
|
requesting_session_id: str | None = None,
|
|
restart_class: str | None = None,
|
|
profile_name: str | None = None,
|
|
authenticated_username: str | None = None,
|
|
reasons: Sequence[str] | None = None,
|
|
details: Mapping[str, Any] | None = None,
|
|
now: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Build a redacted ``mcp.restart.*`` audit event.
|
|
|
|
Raises ``ValueError`` on unknown event types so a typo cannot silently land
|
|
under a free-form action name.
|
|
"""
|
|
name = str(event_type or "").strip()
|
|
if name not in RESTART_EVENT_TYPES:
|
|
raise ValueError(
|
|
f"unknown restart audit event_type {name!r}; expected one of "
|
|
f"{sorted(RESTART_EVENT_TYPES)}"
|
|
)
|
|
redacted_reasons = [
|
|
gitea_audit.redact(str(r)) for r in (reasons or []) if str(r).strip()
|
|
]
|
|
redacted_details = gitea_audit.redact(dict(details or {}))
|
|
if not isinstance(redacted_details, dict):
|
|
redacted_details = {"value": redacted_details}
|
|
|
|
event = gitea_audit.build_event(
|
|
action=name,
|
|
result=str(outcome or "unknown"),
|
|
remote=remote,
|
|
repository=f"{org}/{repo}" if org and repo else None,
|
|
profile_name=profile_name,
|
|
authenticated_username=authenticated_username,
|
|
reason="; ".join(redacted_reasons) if redacted_reasons else None,
|
|
request_metadata={
|
|
"event_family": "mcp.restart",
|
|
"correlation_id": correlation_id or new_correlation_id(),
|
|
"restart_class": restart_class,
|
|
"requesting_session_id": requesting_session_id,
|
|
"org": org,
|
|
"repo": repo,
|
|
"details": redacted_details,
|
|
"reasons": redacted_reasons,
|
|
},
|
|
now=now or _utc_now_iso(),
|
|
operation=name,
|
|
)
|
|
event["action_type"] = "restart_lifecycle"
|
|
event["event_type"] = name
|
|
event["correlation_id"] = (event.get("request_metadata") or {}).get(
|
|
"correlation_id"
|
|
)
|
|
return event
|
|
|
|
|
|
def emit_restart_event(event: Mapping[str, Any], *, path: str | None = None) -> bool:
|
|
"""Append *event* to the audit sink. Never raises. Returns write success."""
|
|
try:
|
|
return bool(gitea_audit.write_event(dict(event), path=path))
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def require_audit_or_deny(
|
|
*,
|
|
privileged: bool,
|
|
written: bool,
|
|
audit_enabled: bool | None = None,
|
|
) -> list[str]:
|
|
"""Return deny reasons when a privileged restart path fails to audit.
|
|
|
|
When audit is not configured (``GITEA_AUDIT_LOG`` unset), privileged apply
|
|
still proceeds under the rollout policy "enable audit before enforcing
|
|
deny-on-audit-fail" — but *if* audit is enabled and the write fails,
|
|
privileged apply is denied (fail closed).
|
|
"""
|
|
enabled = (
|
|
gitea_audit.audit_enabled() if audit_enabled is None else bool(audit_enabled)
|
|
)
|
|
if not privileged:
|
|
return []
|
|
if not enabled:
|
|
return []
|
|
if written:
|
|
return []
|
|
return [
|
|
"privileged restart path requires a successful audit write; "
|
|
"audit sink failed (fail closed, #665)"
|
|
]
|
|
|
|
|
|
# ── Incident descriptors ──────────────────────────────────────────────────────
|
|
|
|
|
|
def build_incident_descriptor(
|
|
*,
|
|
kind: str,
|
|
reasons: Sequence[str],
|
|
correlation_id: str | None = None,
|
|
requesting_session_id: str | None = None,
|
|
restart_class: str | None = None,
|
|
remote: str | None = None,
|
|
org: str | None = None,
|
|
repo: str | None = None,
|
|
proof_id: str | None = None,
|
|
at: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Build a durable incident descriptor (no network)."""
|
|
titles = {
|
|
INCIDENT_FAILED_DRAIN: "Restart denied: drain proof failed the hard gate",
|
|
INCIDENT_BREAK_GLASS: "Break-glass MCP restart authorized",
|
|
INCIDENT_RECONCILE_UNRESOLVED: "Post-restart reconcile left unresolved work",
|
|
INCIDENT_UNGUARDED: "Unguarded MCP restart attempt detected",
|
|
}
|
|
title = titles.get(kind, f"MCP restart incident ({kind})")
|
|
redacted_reasons = [
|
|
gitea_audit.redact(str(r)) for r in reasons if str(r).strip()
|
|
]
|
|
return {
|
|
"kind": kind,
|
|
"title": title,
|
|
"labels": list(DEFAULT_INCIDENT_LABELS),
|
|
"reasons": redacted_reasons,
|
|
"correlation_id": correlation_id,
|
|
"requesting_session_id": requesting_session_id,
|
|
"restart_class": restart_class,
|
|
"remote": remote,
|
|
"org": org,
|
|
"repo": repo,
|
|
"proof_id": proof_id,
|
|
"at": at or _utc_now_iso(),
|
|
"source": "restart_audit#665",
|
|
}
|
|
|
|
|
|
def incident_body(descriptor: Mapping[str, Any]) -> str:
|
|
"""Render a redacted markdown body for a Gitea incident issue."""
|
|
reasons = descriptor.get("reasons") or []
|
|
reason_lines = "\n".join(f"- {gitea_audit.redact(str(r))}" for r in reasons) or (
|
|
"- (no reasons recorded)"
|
|
)
|
|
return "\n".join(
|
|
[
|
|
"<!-- mcp-restart-incident:v1 -->",
|
|
f"## MCP restart incident (`{descriptor.get('kind')}`)",
|
|
"",
|
|
f"**Correlation:** `{descriptor.get('correlation_id') or 'none'}`",
|
|
f"**Session:** `{descriptor.get('requesting_session_id') or 'none'}`",
|
|
f"**Class:** `{descriptor.get('restart_class') or 'none'}`",
|
|
f"**Scope:** `{descriptor.get('remote')}/{descriptor.get('org')}/"
|
|
f"{descriptor.get('repo')}`",
|
|
f"**At:** `{descriptor.get('at')}`",
|
|
f"**Proof id:** `{descriptor.get('proof_id') or 'none'}`",
|
|
"",
|
|
"### Reasons",
|
|
reason_lines,
|
|
"",
|
|
"### Operator next steps",
|
|
"- Treat this as durable follow-up work under the restart-governance umbrella (#655).",
|
|
"- Do not invent a second restart path; use sanctioned coordinator tools only.",
|
|
"- Raw secrets must never appear in this issue (already redacted).",
|
|
"",
|
|
f"_Source: {descriptor.get('source')}_",
|
|
]
|
|
)
|
|
|
|
|
|
def materialize_incident(
|
|
descriptor: Mapping[str, Any],
|
|
*,
|
|
create_issue_fn: CreateIssueFn | None,
|
|
dry_run: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Create a Gitea issue from *descriptor* when *create_issue_fn* is provided.
|
|
|
|
Returns a result dict with ``created`` / ``issue_number`` / ``dry_run`` /
|
|
``reasons``. Never raises.
|
|
"""
|
|
base: dict[str, Any] = {
|
|
"created": False,
|
|
"dry_run": bool(dry_run),
|
|
"issue_number": None,
|
|
"kind": descriptor.get("kind"),
|
|
"reasons": [],
|
|
"descriptor": dict(descriptor),
|
|
}
|
|
if dry_run:
|
|
base["reasons"] = ["dry-run only; no Gitea issue created"]
|
|
return base
|
|
if create_issue_fn is None:
|
|
base["reasons"] = [
|
|
"create_issue_fn not provided; incident descriptor retained only"
|
|
]
|
|
return base
|
|
try:
|
|
result = create_issue_fn(
|
|
title=str(descriptor.get("title") or "MCP restart incident"),
|
|
body=incident_body(descriptor),
|
|
labels=list(descriptor.get("labels") or DEFAULT_INCIDENT_LABELS),
|
|
org=descriptor.get("org"),
|
|
repo=descriptor.get("repo"),
|
|
)
|
|
number = None
|
|
if isinstance(result, dict):
|
|
number = result.get("number") or result.get("issue_number")
|
|
if number is not None:
|
|
base["created"] = True
|
|
base["issue_number"] = int(number)
|
|
base["reasons"] = [f"created incident issue #{int(number)}"]
|
|
else:
|
|
base["reasons"] = ["create_issue_fn returned no issue number"]
|
|
except Exception as exc: # noqa: BLE001 — never break restart path here
|
|
base["reasons"] = [
|
|
f"incident issue creation failed: {gitea_audit.redact(str(exc))}"
|
|
]
|
|
return base
|
|
|
|
|
|
def record_restart_lifecycle(
|
|
*,
|
|
event_type: str,
|
|
outcome: str,
|
|
correlation_id: str,
|
|
remote: str | None = None,
|
|
org: str | None = None,
|
|
repo: str | None = None,
|
|
requesting_session_id: str | None = None,
|
|
restart_class: str | None = None,
|
|
profile_name: str | None = None,
|
|
authenticated_username: str | None = None,
|
|
reasons: Sequence[str] | None = None,
|
|
details: Mapping[str, Any] | None = None,
|
|
privileged: bool = False,
|
|
create_incident: Mapping[str, Any] | None = None,
|
|
create_issue_fn: CreateIssueFn | None = None,
|
|
dry_run_incident: bool = False,
|
|
audit_path: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Emit one restart audit event and optionally materialize an incident.
|
|
|
|
Returns ``{event, audit_written, deny_reasons, incident_result}``.
|
|
"""
|
|
event = build_restart_event(
|
|
event_type=event_type,
|
|
outcome=outcome,
|
|
correlation_id=correlation_id,
|
|
remote=remote,
|
|
org=org,
|
|
repo=repo,
|
|
requesting_session_id=requesting_session_id,
|
|
restart_class=restart_class,
|
|
profile_name=profile_name,
|
|
authenticated_username=authenticated_username,
|
|
reasons=reasons,
|
|
details=details,
|
|
)
|
|
written = emit_restart_event(event, path=audit_path)
|
|
deny = require_audit_or_deny(privileged=privileged, written=written)
|
|
incident_result = None
|
|
if create_incident is not None:
|
|
incident_result = materialize_incident(
|
|
create_incident,
|
|
create_issue_fn=create_issue_fn,
|
|
dry_run=dry_run_incident,
|
|
)
|
|
return {
|
|
"event": event,
|
|
"audit_written": written,
|
|
"deny_reasons": deny,
|
|
"incident_result": incident_result,
|
|
"correlation_id": correlation_id,
|
|
}
|
|
|
|
|
|
def incident_from_apply_gate(
|
|
*,
|
|
gate_payload: Mapping[str, Any],
|
|
break_glass: bool,
|
|
correlation_id: str,
|
|
requesting_session_id: str | None,
|
|
restart_class: str | None,
|
|
remote: str | None,
|
|
org: str | None,
|
|
repo: str | None,
|
|
) -> dict[str, Any] | None:
|
|
"""Choose an incident descriptor from an apply-gate payload, if required."""
|
|
reasons = list(gate_payload.get("reasons") or [])
|
|
proof_id = gate_payload.get("proof_id")
|
|
if break_glass:
|
|
return build_incident_descriptor(
|
|
kind=INCIDENT_BREAK_GLASS,
|
|
reasons=reasons
|
|
or ["break-glass restart path used; durable incident required (#665)"],
|
|
correlation_id=correlation_id,
|
|
requesting_session_id=requesting_session_id,
|
|
restart_class=restart_class,
|
|
remote=remote,
|
|
org=org,
|
|
repo=repo,
|
|
proof_id=proof_id if isinstance(proof_id, str) else None,
|
|
)
|
|
# Failed drain / deny path.
|
|
incident = gate_payload.get("incident")
|
|
if isinstance(incident, Mapping) and incident:
|
|
# Normalize gate-provided descriptor into our schema.
|
|
return build_incident_descriptor(
|
|
kind=INCIDENT_FAILED_DRAIN,
|
|
reasons=list(incident.get("reasons") or reasons),
|
|
correlation_id=correlation_id,
|
|
requesting_session_id=requesting_session_id
|
|
or incident.get("requesting_session_id"),
|
|
restart_class=restart_class,
|
|
remote=remote,
|
|
org=org,
|
|
repo=repo,
|
|
proof_id=incident.get("proof_id") or proof_id,
|
|
at=incident.get("at"),
|
|
)
|
|
if not gate_payload.get("allow") and not gate_payload.get("drain_gate_allow", True):
|
|
return build_incident_descriptor(
|
|
kind=INCIDENT_FAILED_DRAIN,
|
|
reasons=reasons or ["restart apply denied"],
|
|
correlation_id=correlation_id,
|
|
requesting_session_id=requesting_session_id,
|
|
restart_class=restart_class,
|
|
remote=remote,
|
|
org=org,
|
|
repo=repo,
|
|
proof_id=proof_id if isinstance(proof_id, str) else None,
|
|
)
|
|
return None
|