sysadminandClaude Opus 4.8
e8bae606cb
Implement a sanctioned session lifecycle for post-restart reconciliation so active session rows with dead or reused owner PIDs can be terminalized without deleting history. Protect live owners, live leases, and live client-managed sessions; record durable session_retired audit events; keep cleanup idempotent under concurrent reconciles. Closes #969 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
704 lines
24 KiB
Python
704 lines
24 KiB
Python
"""Safe lifecycle for workflow session rows with dead or reused owners (#969).
|
|
|
|
Post-restart reconciliation previously left hundreds of ``active`` session rows
|
|
with dead owner PIDs permanently unresolved. A PID existence check alone is not
|
|
enough: operating systems reuse PIDs, so an unrelated live process can appear to
|
|
own a historical session.
|
|
|
|
This module is the pure classification + apply core for session retirement:
|
|
|
|
* distinguish live, disconnected, stale, protected, and terminal records
|
|
* refuse retirement when a live lease or live verified owner remains
|
|
* protect live client-managed sessions
|
|
* detect PID reuse via process start time vs session start / heartbeat
|
|
* terminalize confirmed-stale rows idempotently with durable audit events
|
|
* stay safe under concurrent reconciles (CAS on status)
|
|
|
|
Design mirrors ``lease_lifecycle`` / ``post_restart_reconcile``:
|
|
|
|
* Pure classification accepts injectable checkers so unit tests never touch
|
|
real processes.
|
|
* Apply mutations go only through :meth:`ControlPlaneDB.retire_session`.
|
|
* Historical rows are never deleted; status moves to a terminal value and an
|
|
events-row records the action + reason.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any, Callable, Mapping, Sequence
|
|
|
|
import control_plane_db as cpd
|
|
import lease_lifecycle
|
|
|
|
# Session status vocabulary.
|
|
SESSION_STATUS_ACTIVE = "active"
|
|
SESSION_STATUS_RETIRED = "retired"
|
|
SESSION_STATUS_ENDED = "ended"
|
|
SESSION_STATUS_TERMINAL = "terminal"
|
|
|
|
TERMINAL_SESSION_STATUSES = frozenset(
|
|
{
|
|
SESSION_STATUS_RETIRED,
|
|
SESSION_STATUS_ENDED,
|
|
SESSION_STATUS_TERMINAL,
|
|
"dead",
|
|
"stale",
|
|
"orphaned",
|
|
}
|
|
)
|
|
|
|
# Classification outcomes for one session row.
|
|
CLASS_LIVE = "live"
|
|
CLASS_DISCONNECTED = "disconnected"
|
|
CLASS_STALE = "stale"
|
|
CLASS_PROTECTED = "protected"
|
|
CLASS_TERMINAL = "terminal"
|
|
|
|
# Stable reason codes (audit + tests).
|
|
REASON_ALREADY_TERMINAL = "already_terminal"
|
|
REASON_LIVE_OWNER = "live_owner"
|
|
REASON_LIVE_LEASE = "live_lease"
|
|
REASON_CLIENT_MANAGED_LIVE = "client_managed_live"
|
|
REASON_DEAD_OWNER = "dead_owner"
|
|
REASON_PID_REUSE = "pid_reuse"
|
|
REASON_HEARTBEAT_STALE_DEAD = "heartbeat_stale_dead_owner"
|
|
REASON_MISSING_PID = "missing_pid_no_lease"
|
|
|
|
# Event type written to control-plane events table.
|
|
EVENT_SESSION_RETIRED = "session_retired"
|
|
|
|
# Default heartbeat window before a still-alive PID is treated as disconnected
|
|
# rather than live (does not alone authorize retirement).
|
|
DEFAULT_HEARTBEAT_STALE_SECONDS = 900
|
|
|
|
# PID reuse: process start must be strictly later than session started_at by
|
|
# more than this skew (ps lstart is second-resolution; clocks can lag).
|
|
PID_REUSE_SKEW = timedelta(seconds=2)
|
|
|
|
# Live lease freshness values that block retirement.
|
|
_LIVE_LEASE_FRESHNESS = frozenset({"active", "live"})
|
|
|
|
|
|
class SessionLifecycleError(RuntimeError):
|
|
"""Fail-closed session lifecycle policy error."""
|
|
|
|
|
|
def _utc_now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def _parse_ts(value: str | datetime | None) -> datetime | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, datetime):
|
|
if value.tzinfo is None:
|
|
return value.replace(tzinfo=timezone.utc)
|
|
return value.astimezone(timezone.utc)
|
|
return cpd._parse_ts(str(value))
|
|
|
|
|
|
def _ts(dt: datetime | None = None) -> str:
|
|
return cpd._ts(dt)
|
|
|
|
|
|
def process_start_time(pid: int | None) -> datetime | None:
|
|
"""Return the OS start time of *pid*, or None when it cannot be resolved.
|
|
|
|
Uses ``ps -o lstart=`` (POSIX). Failures return None rather than inventing
|
|
evidence — missing start time never authorizes retirement of a live PID.
|
|
"""
|
|
if pid is None:
|
|
return None
|
|
try:
|
|
pid_i = int(pid)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if pid_i <= 0:
|
|
return None
|
|
try:
|
|
proc = subprocess.run(
|
|
["ps", "-o", "lstart=", "-p", str(pid_i)],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
timeout=2,
|
|
)
|
|
except (OSError, subprocess.SubprocessError):
|
|
return None
|
|
if proc.returncode != 0:
|
|
return None
|
|
text = (proc.stdout or "").strip()
|
|
if not text:
|
|
return None
|
|
try:
|
|
# Example: "Wed Jul 29 09:14:36 2026"
|
|
naive = datetime.strptime(text, "%a %b %d %H:%M:%S %Y")
|
|
return naive.replace(tzinfo=timezone.utc)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _lease_freshness_label(lease: Mapping[str, Any]) -> str:
|
|
fr = lease.get("freshness")
|
|
if isinstance(fr, Mapping):
|
|
return str(fr.get("freshness") or "").strip().lower()
|
|
if fr:
|
|
return str(fr).strip().lower()
|
|
# Fall back to classifying a raw lease row.
|
|
try:
|
|
return str(
|
|
lease_lifecycle.classify_lease_freshness(lease).get("freshness") or ""
|
|
).strip().lower()
|
|
except Exception: # noqa: BLE001 — pure classifier must not raise on bad rows
|
|
status = str(lease.get("status") or "").strip().lower()
|
|
return status or "unknown"
|
|
|
|
|
|
def live_lease_session_ids(
|
|
leases: Sequence[Mapping[str, Any]] | None,
|
|
*,
|
|
now: datetime | None = None,
|
|
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
|
) -> set[str]:
|
|
"""Session ids that still hold a live (or ambiguous-active) workflow lease."""
|
|
live: set[str] = set()
|
|
moment = now or _utc_now()
|
|
for lease in leases or ():
|
|
if not isinstance(lease, Mapping):
|
|
continue
|
|
status = str(lease.get("status") or "").strip().lower()
|
|
if status and status not in {
|
|
lease_lifecycle.LEASE_STATUS_ACTIVE,
|
|
"",
|
|
}:
|
|
# Explicit terminal lease statuses never protect a session.
|
|
if status in {
|
|
lease_lifecycle.LEASE_STATUS_RELEASED,
|
|
lease_lifecycle.LEASE_STATUS_EXPIRED,
|
|
lease_lifecycle.LEASE_STATUS_ABANDONED,
|
|
}:
|
|
continue
|
|
freshness = _lease_freshness_label(lease)
|
|
if freshness in _LIVE_LEASE_FRESHNESS or freshness in {"", "unknown"}:
|
|
# Ambiguous active rows: re-check with authoritative classifier.
|
|
try:
|
|
fr = lease_lifecycle.classify_lease_freshness(
|
|
lease, now=moment, pid_checker=pid_checker
|
|
)
|
|
freshness = str(fr.get("freshness") or "").strip().lower()
|
|
except Exception: # noqa: BLE001
|
|
freshness = "unknown"
|
|
if freshness in _LIVE_LEASE_FRESHNESS:
|
|
sid = str(lease.get("session_id") or "").strip()
|
|
if sid:
|
|
live.add(sid)
|
|
elif freshness == "unknown" and status in {
|
|
lease_lifecycle.LEASE_STATUS_ACTIVE,
|
|
"",
|
|
}:
|
|
# Fail closed: active lease with unknown freshness blocks retirement.
|
|
sid = str(lease.get("session_id") or "").strip()
|
|
if sid:
|
|
live.add(sid)
|
|
return live
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SessionClassification:
|
|
"""Classification of one workflow session row."""
|
|
|
|
session_id: str
|
|
classification: str
|
|
reason: str
|
|
retireable: bool
|
|
status: str | None
|
|
pid: int | None
|
|
pid_alive: bool | None
|
|
pid_reused: bool
|
|
heartbeat_stale: bool
|
|
has_live_lease: bool
|
|
client_managed: bool
|
|
details: dict[str, Any] = field(default_factory=dict)
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"session_id": self.session_id,
|
|
"classification": self.classification,
|
|
"reason": self.reason,
|
|
"retireable": self.retireable,
|
|
"status": self.status,
|
|
"pid": self.pid,
|
|
"pid_alive": self.pid_alive,
|
|
"pid_reused": self.pid_reused,
|
|
"heartbeat_stale": self.heartbeat_stale,
|
|
"has_live_lease": self.has_live_lease,
|
|
"client_managed": self.client_managed,
|
|
"details": dict(self.details),
|
|
}
|
|
|
|
|
|
def classify_session(
|
|
row: Mapping[str, Any],
|
|
*,
|
|
now: datetime | None = None,
|
|
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
|
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
|
live_lease_sessions: set[str] | frozenset[str] | None = None,
|
|
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
|
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
|
) -> SessionClassification:
|
|
"""Classify one session row for retirement decisions (#969).
|
|
|
|
Rules (first match wins where noted):
|
|
|
|
1. Non-active / already-terminal status → ``terminal`` (not retireable).
|
|
2. Session holds a live lease → ``protected`` (never retire).
|
|
3. PID missing and no live lease → ``stale`` (retireable: missing owner).
|
|
4. PID alive + process start after session start → ``stale`` (PID reuse).
|
|
5. PID alive + client-managed → ``live`` protected (never retire).
|
|
6. PID alive + fresh heartbeat → ``live``.
|
|
7. PID alive + stale heartbeat → ``disconnected`` (not retireable alone).
|
|
8. PID dead → ``stale`` (retireable).
|
|
"""
|
|
moment = now or _utc_now()
|
|
session_id = str(row.get("session_id") or "").strip()
|
|
status = str(row.get("status") or "").strip().lower() or None
|
|
raw_pid = row.get("pid")
|
|
try:
|
|
pid = int(raw_pid) if raw_pid is not None else None
|
|
except (TypeError, ValueError):
|
|
pid = None
|
|
|
|
client_managed = bool(
|
|
row.get("client_managed")
|
|
or row.get("is_client_managed")
|
|
or (
|
|
client_managed_sessions is not None
|
|
and session_id in client_managed_sessions
|
|
)
|
|
)
|
|
has_live_lease = bool(
|
|
live_lease_sessions is not None and session_id in live_lease_sessions
|
|
)
|
|
|
|
hb = _parse_ts(row.get("last_heartbeat_at"))
|
|
heartbeat_stale = bool(
|
|
hb is not None
|
|
and (moment - hb).total_seconds() > max(0, int(heartbeat_stale_seconds))
|
|
)
|
|
if hb is None and status == SESSION_STATUS_ACTIVE:
|
|
# No heartbeat evidence: treat as stale for liveness bookkeeping only.
|
|
heartbeat_stale = True
|
|
|
|
started = _parse_ts(row.get("started_at"))
|
|
recorded_proc_start = _parse_ts(row.get("owner_process_started_at"))
|
|
|
|
if status in TERMINAL_SESSION_STATUSES:
|
|
return SessionClassification(
|
|
session_id=session_id,
|
|
classification=CLASS_TERMINAL,
|
|
reason=REASON_ALREADY_TERMINAL,
|
|
retireable=False,
|
|
status=status,
|
|
pid=pid,
|
|
pid_alive=None,
|
|
pid_reused=False,
|
|
heartbeat_stale=heartbeat_stale,
|
|
has_live_lease=has_live_lease,
|
|
client_managed=client_managed,
|
|
)
|
|
|
|
if has_live_lease:
|
|
return SessionClassification(
|
|
session_id=session_id,
|
|
classification=CLASS_PROTECTED,
|
|
reason=REASON_LIVE_LEASE,
|
|
retireable=False,
|
|
status=status,
|
|
pid=pid,
|
|
pid_alive=pid_checker(pid) if pid is not None else None,
|
|
pid_reused=False,
|
|
heartbeat_stale=heartbeat_stale,
|
|
has_live_lease=True,
|
|
client_managed=client_managed,
|
|
details={"blocker": "live_workflow_lease"},
|
|
)
|
|
|
|
if pid is None:
|
|
return SessionClassification(
|
|
session_id=session_id,
|
|
classification=CLASS_STALE,
|
|
reason=REASON_MISSING_PID,
|
|
retireable=True,
|
|
status=status,
|
|
pid=None,
|
|
pid_alive=False,
|
|
pid_reused=False,
|
|
heartbeat_stale=heartbeat_stale,
|
|
has_live_lease=False,
|
|
client_managed=client_managed,
|
|
)
|
|
|
|
pid_alive = bool(pid_checker(pid))
|
|
pid_reused = False
|
|
proc_start: datetime | None = None
|
|
|
|
if pid_alive:
|
|
proc_start = recorded_proc_start or process_start_probe(pid)
|
|
# PID reuse: live process started after the session row itself was
|
|
# created. Anchor on started_at only — last_heartbeat alone is not a
|
|
# safe bound (synthetic inventories and long-lived processes would
|
|
# false-positive against a live ``ps`` probe).
|
|
if proc_start is not None and started is not None:
|
|
if proc_start > (started + PID_REUSE_SKEW):
|
|
pid_reused = True
|
|
|
|
if pid_reused:
|
|
return SessionClassification(
|
|
session_id=session_id,
|
|
classification=CLASS_STALE,
|
|
reason=REASON_PID_REUSE,
|
|
retireable=True,
|
|
status=status,
|
|
pid=pid,
|
|
pid_alive=True,
|
|
pid_reused=True,
|
|
heartbeat_stale=heartbeat_stale,
|
|
has_live_lease=False,
|
|
client_managed=client_managed,
|
|
details={
|
|
"process_started_at": _ts(proc_start) if proc_start else None,
|
|
"session_started_at": row.get("started_at"),
|
|
"last_heartbeat_at": row.get("last_heartbeat_at"),
|
|
},
|
|
)
|
|
|
|
if pid_alive and client_managed:
|
|
return SessionClassification(
|
|
session_id=session_id,
|
|
classification=CLASS_LIVE,
|
|
reason=REASON_CLIENT_MANAGED_LIVE,
|
|
retireable=False,
|
|
status=status,
|
|
pid=pid,
|
|
pid_alive=True,
|
|
pid_reused=False,
|
|
heartbeat_stale=heartbeat_stale,
|
|
has_live_lease=False,
|
|
client_managed=True,
|
|
)
|
|
|
|
if pid_alive and not heartbeat_stale:
|
|
return SessionClassification(
|
|
session_id=session_id,
|
|
classification=CLASS_LIVE,
|
|
reason=REASON_LIVE_OWNER,
|
|
retireable=False,
|
|
status=status,
|
|
pid=pid,
|
|
pid_alive=True,
|
|
pid_reused=False,
|
|
heartbeat_stale=False,
|
|
has_live_lease=False,
|
|
client_managed=client_managed,
|
|
)
|
|
|
|
if pid_alive and heartbeat_stale:
|
|
# Process still exists but has not heartbeated — disconnected, not
|
|
# confirmed stale. Do not retire; operator/reconnect owns next step.
|
|
return SessionClassification(
|
|
session_id=session_id,
|
|
classification=CLASS_DISCONNECTED,
|
|
reason=REASON_LIVE_OWNER,
|
|
retireable=False,
|
|
status=status,
|
|
pid=pid,
|
|
pid_alive=True,
|
|
pid_reused=False,
|
|
heartbeat_stale=True,
|
|
has_live_lease=False,
|
|
client_managed=client_managed,
|
|
details={"note": "alive_pid_stale_heartbeat_not_retired"},
|
|
)
|
|
|
|
# PID dead (or checker said not alive).
|
|
reason = REASON_DEAD_OWNER
|
|
if heartbeat_stale:
|
|
reason = REASON_HEARTBEAT_STALE_DEAD
|
|
return SessionClassification(
|
|
session_id=session_id,
|
|
classification=CLASS_STALE,
|
|
reason=reason,
|
|
retireable=True,
|
|
status=status,
|
|
pid=pid,
|
|
pid_alive=False,
|
|
pid_reused=False,
|
|
heartbeat_stale=heartbeat_stale,
|
|
has_live_lease=False,
|
|
client_managed=client_managed,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SessionFleetReport:
|
|
"""Fleet-wide classification summary for reconcile + apply."""
|
|
|
|
classifications: tuple[SessionClassification, ...]
|
|
live_count: int
|
|
disconnected_count: int
|
|
stale_count: int
|
|
protected_count: int
|
|
terminal_count: int
|
|
retireable: tuple[SessionClassification, ...]
|
|
counts_by_reason: dict[str, int]
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"live_count": self.live_count,
|
|
"disconnected_count": self.disconnected_count,
|
|
"stale_count": self.stale_count,
|
|
"protected_count": self.protected_count,
|
|
"terminal_count": self.terminal_count,
|
|
"retireable_count": len(self.retireable),
|
|
"retireable_session_ids": [c.session_id for c in self.retireable],
|
|
"counts_by_reason": dict(self.counts_by_reason),
|
|
"classifications": [c.as_dict() for c in self.classifications],
|
|
"sessions_dimension_resolved": len(self.retireable) == 0,
|
|
}
|
|
|
|
|
|
def classify_sessions(
|
|
sessions: Sequence[Mapping[str, Any]] | None,
|
|
*,
|
|
leases: Sequence[Mapping[str, Any]] | None = None,
|
|
now: datetime | None = None,
|
|
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
|
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
|
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
|
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
|
) -> SessionFleetReport:
|
|
"""Classify a fleet of session rows against live leases (#969)."""
|
|
moment = now or _utc_now()
|
|
live_leases = live_lease_session_ids(
|
|
leases, now=moment, pid_checker=pid_checker
|
|
)
|
|
results: list[SessionClassification] = []
|
|
for row in sessions or ():
|
|
if not isinstance(row, Mapping):
|
|
continue
|
|
results.append(
|
|
classify_session(
|
|
row,
|
|
now=moment,
|
|
pid_checker=pid_checker,
|
|
process_start_probe=process_start_probe,
|
|
live_lease_sessions=live_leases,
|
|
client_managed_sessions=client_managed_sessions,
|
|
heartbeat_stale_seconds=heartbeat_stale_seconds,
|
|
)
|
|
)
|
|
|
|
live_count = sum(1 for c in results if c.classification == CLASS_LIVE)
|
|
disconnected_count = sum(
|
|
1 for c in results if c.classification == CLASS_DISCONNECTED
|
|
)
|
|
stale_count = sum(1 for c in results if c.classification == CLASS_STALE)
|
|
protected_count = sum(1 for c in results if c.classification == CLASS_PROTECTED)
|
|
terminal_count = sum(1 for c in results if c.classification == CLASS_TERMINAL)
|
|
retireable = tuple(c for c in results if c.retireable)
|
|
by_reason: dict[str, int] = {}
|
|
for c in results:
|
|
by_reason[c.reason] = by_reason.get(c.reason, 0) + 1
|
|
|
|
return SessionFleetReport(
|
|
classifications=tuple(results),
|
|
live_count=live_count,
|
|
disconnected_count=disconnected_count,
|
|
stale_count=stale_count,
|
|
protected_count=protected_count,
|
|
terminal_count=terminal_count,
|
|
retireable=retireable,
|
|
counts_by_reason=by_reason,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RetirementResult:
|
|
"""Outcome of one session retirement attempt."""
|
|
|
|
session_id: str
|
|
outcome: str # retired | already_terminal | skipped | blocked | missing
|
|
reason: str
|
|
prior_status: str | None = None
|
|
new_status: str | None = None
|
|
details: dict[str, Any] = field(default_factory=dict)
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"session_id": self.session_id,
|
|
"outcome": self.outcome,
|
|
"reason": self.reason,
|
|
"prior_status": self.prior_status,
|
|
"new_status": self.new_status,
|
|
"details": dict(self.details),
|
|
}
|
|
|
|
|
|
def apply_session_retirements(
|
|
db: cpd.ControlPlaneDB,
|
|
report: SessionFleetReport,
|
|
*,
|
|
dry_run: bool = False,
|
|
actor_session_id: str | None = None,
|
|
now: datetime | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Terminalize every retireable session in *report* (idempotent).
|
|
|
|
Concurrent reconciles are safe: each retirement is a CAS on
|
|
``status='active'`` (or other non-terminal). A second pass that sees the
|
|
same session already retired records ``already_terminal`` rather than
|
|
duplicating audit noise beyond a single no-op outcome.
|
|
"""
|
|
moment = now or _utc_now()
|
|
results: list[RetirementResult] = []
|
|
retired = 0
|
|
already = 0
|
|
blocked = 0
|
|
missing = 0
|
|
|
|
for classification in report.retireable:
|
|
if not classification.retireable:
|
|
continue
|
|
if dry_run:
|
|
results.append(
|
|
RetirementResult(
|
|
session_id=classification.session_id,
|
|
outcome="skipped",
|
|
reason=classification.reason,
|
|
prior_status=classification.status,
|
|
new_status=SESSION_STATUS_RETIRED,
|
|
details={"dry_run": True, **classification.details},
|
|
)
|
|
)
|
|
continue
|
|
|
|
applied = db.retire_session(
|
|
session_id=classification.session_id,
|
|
reason=classification.reason,
|
|
actor_session_id=actor_session_id,
|
|
details={
|
|
"classification": classification.classification,
|
|
"pid": classification.pid,
|
|
"pid_alive": classification.pid_alive,
|
|
"pid_reused": classification.pid_reused,
|
|
"client_managed": classification.client_managed,
|
|
**classification.details,
|
|
},
|
|
now=moment,
|
|
)
|
|
outcome = str(applied.get("outcome") or "missing")
|
|
results.append(
|
|
RetirementResult(
|
|
session_id=classification.session_id,
|
|
outcome=outcome,
|
|
reason=str(applied.get("reason") or classification.reason),
|
|
prior_status=applied.get("prior_status"),
|
|
new_status=applied.get("new_status"),
|
|
details=dict(applied.get("details") or {}),
|
|
)
|
|
)
|
|
if outcome == "retired":
|
|
retired += 1
|
|
elif outcome == "already_terminal":
|
|
already += 1
|
|
elif outcome == "blocked":
|
|
blocked += 1
|
|
else:
|
|
missing += 1
|
|
|
|
# Non-retireable classifications are recorded for audit completeness when
|
|
# dry-run lists the fleet, but apply only mutates retireable rows.
|
|
return {
|
|
"success": True,
|
|
"dry_run": dry_run,
|
|
"retired_count": retired,
|
|
"already_terminal_count": already,
|
|
"blocked_count": blocked,
|
|
"missing_count": missing,
|
|
"planned_count": len(report.retireable),
|
|
"results": [r.as_dict() for r in results],
|
|
"audit_action": EVENT_SESSION_RETIRED,
|
|
"actor_session_id": actor_session_id,
|
|
"recorded_at": _ts(moment),
|
|
}
|
|
|
|
|
|
def retire_stale_sessions(
|
|
db: cpd.ControlPlaneDB,
|
|
*,
|
|
sessions: Sequence[Mapping[str, Any]] | None = None,
|
|
leases: Sequence[Mapping[str, Any]] | None = None,
|
|
dry_run: bool = False,
|
|
actor_session_id: str | None = None,
|
|
now: datetime | None = None,
|
|
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
|
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
|
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
|
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
|
session_limit: int = 500,
|
|
) -> dict[str, Any]:
|
|
"""End-to-end plan + apply for stale session retirement (#969).
|
|
|
|
When *sessions* is omitted the control-plane DB is inventoried (active
|
|
rows only). Callers that already gathered inventory should pass it.
|
|
"""
|
|
moment = now or _utc_now()
|
|
if sessions is None:
|
|
sessions = db.list_sessions(
|
|
statuses=(SESSION_STATUS_ACTIVE,),
|
|
limit=max(1, int(session_limit)),
|
|
)
|
|
if leases is None:
|
|
try:
|
|
leases = db.list_leases(statuses=("active",), limit=max(1, int(session_limit)))
|
|
except Exception: # noqa: BLE001
|
|
leases = []
|
|
|
|
report = classify_sessions(
|
|
sessions,
|
|
leases=leases,
|
|
now=moment,
|
|
pid_checker=pid_checker,
|
|
process_start_probe=process_start_probe,
|
|
client_managed_sessions=client_managed_sessions,
|
|
heartbeat_stale_seconds=heartbeat_stale_seconds,
|
|
)
|
|
apply_result = apply_session_retirements(
|
|
db,
|
|
report,
|
|
dry_run=dry_run,
|
|
actor_session_id=actor_session_id,
|
|
now=moment,
|
|
)
|
|
return {
|
|
"success": True,
|
|
"fleet": report.as_dict(),
|
|
"apply": apply_result,
|
|
"sessions_dimension_resolved": (
|
|
report.as_dict()["sessions_dimension_resolved"]
|
|
if dry_run
|
|
else apply_result["planned_count"]
|
|
== (
|
|
apply_result["retired_count"]
|
|
+ apply_result["already_terminal_count"]
|
|
)
|
|
and apply_result["blocked_count"] == 0
|
|
),
|
|
}
|