fix(reconcile): retire workflow session rows whose owners are no longer live
Implement a sanctioned session lifecycle for post-restart reconciliation so active session rows with dead or reused owner PIDs can be terminalized without deleting history. Protect live owners, live leases, and live client-managed sessions; record durable session_retired audit events; keep cleanup idempotent under concurrent reconciles. Closes #969 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,703 @@
|
||||
"""Safe lifecycle for workflow session rows with dead or reused owners (#969).
|
||||
|
||||
Post-restart reconciliation previously left hundreds of ``active`` session rows
|
||||
with dead owner PIDs permanently unresolved. A PID existence check alone is not
|
||||
enough: operating systems reuse PIDs, so an unrelated live process can appear to
|
||||
own a historical session.
|
||||
|
||||
This module is the pure classification + apply core for session retirement:
|
||||
|
||||
* distinguish live, disconnected, stale, protected, and terminal records
|
||||
* refuse retirement when a live lease or live verified owner remains
|
||||
* protect live client-managed sessions
|
||||
* detect PID reuse via process start time vs session start / heartbeat
|
||||
* terminalize confirmed-stale rows idempotently with durable audit events
|
||||
* stay safe under concurrent reconciles (CAS on status)
|
||||
|
||||
Design mirrors ``lease_lifecycle`` / ``post_restart_reconcile``:
|
||||
|
||||
* Pure classification accepts injectable checkers so unit tests never touch
|
||||
real processes.
|
||||
* Apply mutations go only through :meth:`ControlPlaneDB.retire_session`.
|
||||
* Historical rows are never deleted; status moves to a terminal value and an
|
||||
events-row records the action + reason.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Callable, Mapping, Sequence
|
||||
|
||||
import control_plane_db as cpd
|
||||
import lease_lifecycle
|
||||
|
||||
# Session status vocabulary.
|
||||
SESSION_STATUS_ACTIVE = "active"
|
||||
SESSION_STATUS_RETIRED = "retired"
|
||||
SESSION_STATUS_ENDED = "ended"
|
||||
SESSION_STATUS_TERMINAL = "terminal"
|
||||
|
||||
TERMINAL_SESSION_STATUSES = frozenset(
|
||||
{
|
||||
SESSION_STATUS_RETIRED,
|
||||
SESSION_STATUS_ENDED,
|
||||
SESSION_STATUS_TERMINAL,
|
||||
"dead",
|
||||
"stale",
|
||||
"orphaned",
|
||||
}
|
||||
)
|
||||
|
||||
# Classification outcomes for one session row.
|
||||
CLASS_LIVE = "live"
|
||||
CLASS_DISCONNECTED = "disconnected"
|
||||
CLASS_STALE = "stale"
|
||||
CLASS_PROTECTED = "protected"
|
||||
CLASS_TERMINAL = "terminal"
|
||||
|
||||
# Stable reason codes (audit + tests).
|
||||
REASON_ALREADY_TERMINAL = "already_terminal"
|
||||
REASON_LIVE_OWNER = "live_owner"
|
||||
REASON_LIVE_LEASE = "live_lease"
|
||||
REASON_CLIENT_MANAGED_LIVE = "client_managed_live"
|
||||
REASON_DEAD_OWNER = "dead_owner"
|
||||
REASON_PID_REUSE = "pid_reuse"
|
||||
REASON_HEARTBEAT_STALE_DEAD = "heartbeat_stale_dead_owner"
|
||||
REASON_MISSING_PID = "missing_pid_no_lease"
|
||||
|
||||
# Event type written to control-plane events table.
|
||||
EVENT_SESSION_RETIRED = "session_retired"
|
||||
|
||||
# Default heartbeat window before a still-alive PID is treated as disconnected
|
||||
# rather than live (does not alone authorize retirement).
|
||||
DEFAULT_HEARTBEAT_STALE_SECONDS = 900
|
||||
|
||||
# PID reuse: process start must be strictly later than session started_at by
|
||||
# more than this skew (ps lstart is second-resolution; clocks can lag).
|
||||
PID_REUSE_SKEW = timedelta(seconds=2)
|
||||
|
||||
# Live lease freshness values that block retirement.
|
||||
_LIVE_LEASE_FRESHNESS = frozenset({"active", "live"})
|
||||
|
||||
|
||||
class SessionLifecycleError(RuntimeError):
|
||||
"""Fail-closed session lifecycle policy error."""
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _parse_ts(value: str | datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
return cpd._parse_ts(str(value))
|
||||
|
||||
|
||||
def _ts(dt: datetime | None = None) -> str:
|
||||
return cpd._ts(dt)
|
||||
|
||||
|
||||
def process_start_time(pid: int | None) -> datetime | None:
|
||||
"""Return the OS start time of *pid*, or None when it cannot be resolved.
|
||||
|
||||
Uses ``ps -o lstart=`` (POSIX). Failures return None rather than inventing
|
||||
evidence — missing start time never authorizes retirement of a live PID.
|
||||
"""
|
||||
if pid is None:
|
||||
return None
|
||||
try:
|
||||
pid_i = int(pid)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if pid_i <= 0:
|
||||
return None
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["ps", "-o", "lstart=", "-p", str(pid_i)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=2,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
text = (proc.stdout or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
# Example: "Wed Jul 29 09:14:36 2026"
|
||||
naive = datetime.strptime(text, "%a %b %d %H:%M:%S %Y")
|
||||
return naive.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _lease_freshness_label(lease: Mapping[str, Any]) -> str:
|
||||
fr = lease.get("freshness")
|
||||
if isinstance(fr, Mapping):
|
||||
return str(fr.get("freshness") or "").strip().lower()
|
||||
if fr:
|
||||
return str(fr).strip().lower()
|
||||
# Fall back to classifying a raw lease row.
|
||||
try:
|
||||
return str(
|
||||
lease_lifecycle.classify_lease_freshness(lease).get("freshness") or ""
|
||||
).strip().lower()
|
||||
except Exception: # noqa: BLE001 — pure classifier must not raise on bad rows
|
||||
status = str(lease.get("status") or "").strip().lower()
|
||||
return status or "unknown"
|
||||
|
||||
|
||||
def live_lease_session_ids(
|
||||
leases: Sequence[Mapping[str, Any]] | None,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||
) -> set[str]:
|
||||
"""Session ids that still hold a live (or ambiguous-active) workflow lease."""
|
||||
live: set[str] = set()
|
||||
moment = now or _utc_now()
|
||||
for lease in leases or ():
|
||||
if not isinstance(lease, Mapping):
|
||||
continue
|
||||
status = str(lease.get("status") or "").strip().lower()
|
||||
if status and status not in {
|
||||
lease_lifecycle.LEASE_STATUS_ACTIVE,
|
||||
"",
|
||||
}:
|
||||
# Explicit terminal lease statuses never protect a session.
|
||||
if status in {
|
||||
lease_lifecycle.LEASE_STATUS_RELEASED,
|
||||
lease_lifecycle.LEASE_STATUS_EXPIRED,
|
||||
lease_lifecycle.LEASE_STATUS_ABANDONED,
|
||||
}:
|
||||
continue
|
||||
freshness = _lease_freshness_label(lease)
|
||||
if freshness in _LIVE_LEASE_FRESHNESS or freshness in {"", "unknown"}:
|
||||
# Ambiguous active rows: re-check with authoritative classifier.
|
||||
try:
|
||||
fr = lease_lifecycle.classify_lease_freshness(
|
||||
lease, now=moment, pid_checker=pid_checker
|
||||
)
|
||||
freshness = str(fr.get("freshness") or "").strip().lower()
|
||||
except Exception: # noqa: BLE001
|
||||
freshness = "unknown"
|
||||
if freshness in _LIVE_LEASE_FRESHNESS:
|
||||
sid = str(lease.get("session_id") or "").strip()
|
||||
if sid:
|
||||
live.add(sid)
|
||||
elif freshness == "unknown" and status in {
|
||||
lease_lifecycle.LEASE_STATUS_ACTIVE,
|
||||
"",
|
||||
}:
|
||||
# Fail closed: active lease with unknown freshness blocks retirement.
|
||||
sid = str(lease.get("session_id") or "").strip()
|
||||
if sid:
|
||||
live.add(sid)
|
||||
return live
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SessionClassification:
|
||||
"""Classification of one workflow session row."""
|
||||
|
||||
session_id: str
|
||||
classification: str
|
||||
reason: str
|
||||
retireable: bool
|
||||
status: str | None
|
||||
pid: int | None
|
||||
pid_alive: bool | None
|
||||
pid_reused: bool
|
||||
heartbeat_stale: bool
|
||||
has_live_lease: bool
|
||||
client_managed: bool
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"session_id": self.session_id,
|
||||
"classification": self.classification,
|
||||
"reason": self.reason,
|
||||
"retireable": self.retireable,
|
||||
"status": self.status,
|
||||
"pid": self.pid,
|
||||
"pid_alive": self.pid_alive,
|
||||
"pid_reused": self.pid_reused,
|
||||
"heartbeat_stale": self.heartbeat_stale,
|
||||
"has_live_lease": self.has_live_lease,
|
||||
"client_managed": self.client_managed,
|
||||
"details": dict(self.details),
|
||||
}
|
||||
|
||||
|
||||
def classify_session(
|
||||
row: Mapping[str, Any],
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
||||
live_lease_sessions: set[str] | frozenset[str] | None = None,
|
||||
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
||||
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
||||
) -> SessionClassification:
|
||||
"""Classify one session row for retirement decisions (#969).
|
||||
|
||||
Rules (first match wins where noted):
|
||||
|
||||
1. Non-active / already-terminal status → ``terminal`` (not retireable).
|
||||
2. Session holds a live lease → ``protected`` (never retire).
|
||||
3. PID missing and no live lease → ``stale`` (retireable: missing owner).
|
||||
4. PID alive + process start after session start → ``stale`` (PID reuse).
|
||||
5. PID alive + client-managed → ``live`` protected (never retire).
|
||||
6. PID alive + fresh heartbeat → ``live``.
|
||||
7. PID alive + stale heartbeat → ``disconnected`` (not retireable alone).
|
||||
8. PID dead → ``stale`` (retireable).
|
||||
"""
|
||||
moment = now or _utc_now()
|
||||
session_id = str(row.get("session_id") or "").strip()
|
||||
status = str(row.get("status") or "").strip().lower() or None
|
||||
raw_pid = row.get("pid")
|
||||
try:
|
||||
pid = int(raw_pid) if raw_pid is not None else None
|
||||
except (TypeError, ValueError):
|
||||
pid = None
|
||||
|
||||
client_managed = bool(
|
||||
row.get("client_managed")
|
||||
or row.get("is_client_managed")
|
||||
or (
|
||||
client_managed_sessions is not None
|
||||
and session_id in client_managed_sessions
|
||||
)
|
||||
)
|
||||
has_live_lease = bool(
|
||||
live_lease_sessions is not None and session_id in live_lease_sessions
|
||||
)
|
||||
|
||||
hb = _parse_ts(row.get("last_heartbeat_at"))
|
||||
heartbeat_stale = bool(
|
||||
hb is not None
|
||||
and (moment - hb).total_seconds() > max(0, int(heartbeat_stale_seconds))
|
||||
)
|
||||
if hb is None and status == SESSION_STATUS_ACTIVE:
|
||||
# No heartbeat evidence: treat as stale for liveness bookkeeping only.
|
||||
heartbeat_stale = True
|
||||
|
||||
started = _parse_ts(row.get("started_at"))
|
||||
recorded_proc_start = _parse_ts(row.get("owner_process_started_at"))
|
||||
|
||||
if status in TERMINAL_SESSION_STATUSES:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_TERMINAL,
|
||||
reason=REASON_ALREADY_TERMINAL,
|
||||
retireable=False,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=None,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=has_live_lease,
|
||||
client_managed=client_managed,
|
||||
)
|
||||
|
||||
if has_live_lease:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_PROTECTED,
|
||||
reason=REASON_LIVE_LEASE,
|
||||
retireable=False,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=pid_checker(pid) if pid is not None else None,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=True,
|
||||
client_managed=client_managed,
|
||||
details={"blocker": "live_workflow_lease"},
|
||||
)
|
||||
|
||||
if pid is None:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_STALE,
|
||||
reason=REASON_MISSING_PID,
|
||||
retireable=True,
|
||||
status=status,
|
||||
pid=None,
|
||||
pid_alive=False,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=False,
|
||||
client_managed=client_managed,
|
||||
)
|
||||
|
||||
pid_alive = bool(pid_checker(pid))
|
||||
pid_reused = False
|
||||
proc_start: datetime | None = None
|
||||
|
||||
if pid_alive:
|
||||
proc_start = recorded_proc_start or process_start_probe(pid)
|
||||
# PID reuse: live process started after the session row itself was
|
||||
# created. Anchor on started_at only — last_heartbeat alone is not a
|
||||
# safe bound (synthetic inventories and long-lived processes would
|
||||
# false-positive against a live ``ps`` probe).
|
||||
if proc_start is not None and started is not None:
|
||||
if proc_start > (started + PID_REUSE_SKEW):
|
||||
pid_reused = True
|
||||
|
||||
if pid_reused:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_STALE,
|
||||
reason=REASON_PID_REUSE,
|
||||
retireable=True,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=True,
|
||||
pid_reused=True,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=False,
|
||||
client_managed=client_managed,
|
||||
details={
|
||||
"process_started_at": _ts(proc_start) if proc_start else None,
|
||||
"session_started_at": row.get("started_at"),
|
||||
"last_heartbeat_at": row.get("last_heartbeat_at"),
|
||||
},
|
||||
)
|
||||
|
||||
if pid_alive and client_managed:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_LIVE,
|
||||
reason=REASON_CLIENT_MANAGED_LIVE,
|
||||
retireable=False,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=True,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=False,
|
||||
client_managed=True,
|
||||
)
|
||||
|
||||
if pid_alive and not heartbeat_stale:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_LIVE,
|
||||
reason=REASON_LIVE_OWNER,
|
||||
retireable=False,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=True,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=False,
|
||||
has_live_lease=False,
|
||||
client_managed=client_managed,
|
||||
)
|
||||
|
||||
if pid_alive and heartbeat_stale:
|
||||
# Process still exists but has not heartbeated — disconnected, not
|
||||
# confirmed stale. Do not retire; operator/reconnect owns next step.
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_DISCONNECTED,
|
||||
reason=REASON_LIVE_OWNER,
|
||||
retireable=False,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=True,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=True,
|
||||
has_live_lease=False,
|
||||
client_managed=client_managed,
|
||||
details={"note": "alive_pid_stale_heartbeat_not_retired"},
|
||||
)
|
||||
|
||||
# PID dead (or checker said not alive).
|
||||
reason = REASON_DEAD_OWNER
|
||||
if heartbeat_stale:
|
||||
reason = REASON_HEARTBEAT_STALE_DEAD
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_STALE,
|
||||
reason=reason,
|
||||
retireable=True,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=False,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=False,
|
||||
client_managed=client_managed,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SessionFleetReport:
|
||||
"""Fleet-wide classification summary for reconcile + apply."""
|
||||
|
||||
classifications: tuple[SessionClassification, ...]
|
||||
live_count: int
|
||||
disconnected_count: int
|
||||
stale_count: int
|
||||
protected_count: int
|
||||
terminal_count: int
|
||||
retireable: tuple[SessionClassification, ...]
|
||||
counts_by_reason: dict[str, int]
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"live_count": self.live_count,
|
||||
"disconnected_count": self.disconnected_count,
|
||||
"stale_count": self.stale_count,
|
||||
"protected_count": self.protected_count,
|
||||
"terminal_count": self.terminal_count,
|
||||
"retireable_count": len(self.retireable),
|
||||
"retireable_session_ids": [c.session_id for c in self.retireable],
|
||||
"counts_by_reason": dict(self.counts_by_reason),
|
||||
"classifications": [c.as_dict() for c in self.classifications],
|
||||
"sessions_dimension_resolved": len(self.retireable) == 0,
|
||||
}
|
||||
|
||||
|
||||
def classify_sessions(
|
||||
sessions: Sequence[Mapping[str, Any]] | None,
|
||||
*,
|
||||
leases: Sequence[Mapping[str, Any]] | None = None,
|
||||
now: datetime | None = None,
|
||||
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
||||
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
||||
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
||||
) -> SessionFleetReport:
|
||||
"""Classify a fleet of session rows against live leases (#969)."""
|
||||
moment = now or _utc_now()
|
||||
live_leases = live_lease_session_ids(
|
||||
leases, now=moment, pid_checker=pid_checker
|
||||
)
|
||||
results: list[SessionClassification] = []
|
||||
for row in sessions or ():
|
||||
if not isinstance(row, Mapping):
|
||||
continue
|
||||
results.append(
|
||||
classify_session(
|
||||
row,
|
||||
now=moment,
|
||||
pid_checker=pid_checker,
|
||||
process_start_probe=process_start_probe,
|
||||
live_lease_sessions=live_leases,
|
||||
client_managed_sessions=client_managed_sessions,
|
||||
heartbeat_stale_seconds=heartbeat_stale_seconds,
|
||||
)
|
||||
)
|
||||
|
||||
live_count = sum(1 for c in results if c.classification == CLASS_LIVE)
|
||||
disconnected_count = sum(
|
||||
1 for c in results if c.classification == CLASS_DISCONNECTED
|
||||
)
|
||||
stale_count = sum(1 for c in results if c.classification == CLASS_STALE)
|
||||
protected_count = sum(1 for c in results if c.classification == CLASS_PROTECTED)
|
||||
terminal_count = sum(1 for c in results if c.classification == CLASS_TERMINAL)
|
||||
retireable = tuple(c for c in results if c.retireable)
|
||||
by_reason: dict[str, int] = {}
|
||||
for c in results:
|
||||
by_reason[c.reason] = by_reason.get(c.reason, 0) + 1
|
||||
|
||||
return SessionFleetReport(
|
||||
classifications=tuple(results),
|
||||
live_count=live_count,
|
||||
disconnected_count=disconnected_count,
|
||||
stale_count=stale_count,
|
||||
protected_count=protected_count,
|
||||
terminal_count=terminal_count,
|
||||
retireable=retireable,
|
||||
counts_by_reason=by_reason,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RetirementResult:
|
||||
"""Outcome of one session retirement attempt."""
|
||||
|
||||
session_id: str
|
||||
outcome: str # retired | already_terminal | skipped | blocked | missing
|
||||
reason: str
|
||||
prior_status: str | None = None
|
||||
new_status: str | None = None
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"session_id": self.session_id,
|
||||
"outcome": self.outcome,
|
||||
"reason": self.reason,
|
||||
"prior_status": self.prior_status,
|
||||
"new_status": self.new_status,
|
||||
"details": dict(self.details),
|
||||
}
|
||||
|
||||
|
||||
def apply_session_retirements(
|
||||
db: cpd.ControlPlaneDB,
|
||||
report: SessionFleetReport,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
actor_session_id: str | None = None,
|
||||
now: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Terminalize every retireable session in *report* (idempotent).
|
||||
|
||||
Concurrent reconciles are safe: each retirement is a CAS on
|
||||
``status='active'`` (or other non-terminal). A second pass that sees the
|
||||
same session already retired records ``already_terminal`` rather than
|
||||
duplicating audit noise beyond a single no-op outcome.
|
||||
"""
|
||||
moment = now or _utc_now()
|
||||
results: list[RetirementResult] = []
|
||||
retired = 0
|
||||
already = 0
|
||||
blocked = 0
|
||||
missing = 0
|
||||
|
||||
for classification in report.retireable:
|
||||
if not classification.retireable:
|
||||
continue
|
||||
if dry_run:
|
||||
results.append(
|
||||
RetirementResult(
|
||||
session_id=classification.session_id,
|
||||
outcome="skipped",
|
||||
reason=classification.reason,
|
||||
prior_status=classification.status,
|
||||
new_status=SESSION_STATUS_RETIRED,
|
||||
details={"dry_run": True, **classification.details},
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
applied = db.retire_session(
|
||||
session_id=classification.session_id,
|
||||
reason=classification.reason,
|
||||
actor_session_id=actor_session_id,
|
||||
details={
|
||||
"classification": classification.classification,
|
||||
"pid": classification.pid,
|
||||
"pid_alive": classification.pid_alive,
|
||||
"pid_reused": classification.pid_reused,
|
||||
"client_managed": classification.client_managed,
|
||||
**classification.details,
|
||||
},
|
||||
now=moment,
|
||||
)
|
||||
outcome = str(applied.get("outcome") or "missing")
|
||||
results.append(
|
||||
RetirementResult(
|
||||
session_id=classification.session_id,
|
||||
outcome=outcome,
|
||||
reason=str(applied.get("reason") or classification.reason),
|
||||
prior_status=applied.get("prior_status"),
|
||||
new_status=applied.get("new_status"),
|
||||
details=dict(applied.get("details") or {}),
|
||||
)
|
||||
)
|
||||
if outcome == "retired":
|
||||
retired += 1
|
||||
elif outcome == "already_terminal":
|
||||
already += 1
|
||||
elif outcome == "blocked":
|
||||
blocked += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
# Non-retireable classifications are recorded for audit completeness when
|
||||
# dry-run lists the fleet, but apply only mutates retireable rows.
|
||||
return {
|
||||
"success": True,
|
||||
"dry_run": dry_run,
|
||||
"retired_count": retired,
|
||||
"already_terminal_count": already,
|
||||
"blocked_count": blocked,
|
||||
"missing_count": missing,
|
||||
"planned_count": len(report.retireable),
|
||||
"results": [r.as_dict() for r in results],
|
||||
"audit_action": EVENT_SESSION_RETIRED,
|
||||
"actor_session_id": actor_session_id,
|
||||
"recorded_at": _ts(moment),
|
||||
}
|
||||
|
||||
|
||||
def retire_stale_sessions(
|
||||
db: cpd.ControlPlaneDB,
|
||||
*,
|
||||
sessions: Sequence[Mapping[str, Any]] | None = None,
|
||||
leases: Sequence[Mapping[str, Any]] | None = None,
|
||||
dry_run: bool = False,
|
||||
actor_session_id: str | None = None,
|
||||
now: datetime | None = None,
|
||||
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
||||
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
||||
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
||||
session_limit: int = 500,
|
||||
) -> dict[str, Any]:
|
||||
"""End-to-end plan + apply for stale session retirement (#969).
|
||||
|
||||
When *sessions* is omitted the control-plane DB is inventoried (active
|
||||
rows only). Callers that already gathered inventory should pass it.
|
||||
"""
|
||||
moment = now or _utc_now()
|
||||
if sessions is None:
|
||||
sessions = db.list_sessions(
|
||||
statuses=(SESSION_STATUS_ACTIVE,),
|
||||
limit=max(1, int(session_limit)),
|
||||
)
|
||||
if leases is None:
|
||||
try:
|
||||
leases = db.list_leases(statuses=("active",), limit=max(1, int(session_limit)))
|
||||
except Exception: # noqa: BLE001
|
||||
leases = []
|
||||
|
||||
report = classify_sessions(
|
||||
sessions,
|
||||
leases=leases,
|
||||
now=moment,
|
||||
pid_checker=pid_checker,
|
||||
process_start_probe=process_start_probe,
|
||||
client_managed_sessions=client_managed_sessions,
|
||||
heartbeat_stale_seconds=heartbeat_stale_seconds,
|
||||
)
|
||||
apply_result = apply_session_retirements(
|
||||
db,
|
||||
report,
|
||||
dry_run=dry_run,
|
||||
actor_session_id=actor_session_id,
|
||||
now=moment,
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"fleet": report.as_dict(),
|
||||
"apply": apply_result,
|
||||
"sessions_dimension_resolved": (
|
||||
report.as_dict()["sessions_dimension_resolved"]
|
||||
if dry_run
|
||||
else apply_result["planned_count"]
|
||||
== (
|
||||
apply_result["retired_count"]
|
||||
+ apply_result["already_terminal_count"]
|
||||
)
|
||||
and apply_result["blocked_count"] == 0
|
||||
),
|
||||
}
|
||||
Reference in New Issue
Block a user