"""Safe lifecycle for workflow session rows with dead or reused owners (#969). Post-restart reconciliation previously left hundreds of ``active`` session rows with dead owner PIDs permanently unresolved. A PID existence check alone is not enough: operating systems reuse PIDs, so an unrelated live process can appear to own a historical session. This module is the pure classification + apply core for session retirement: * distinguish live, disconnected, stale, protected, and terminal records * refuse retirement when a live lease or live verified owner remains * protect live client-managed sessions * detect PID reuse via process start time vs session start / heartbeat * terminalize confirmed-stale rows idempotently with durable audit events * stay safe under concurrent reconciles (CAS on status) Design mirrors ``lease_lifecycle`` / ``post_restart_reconcile``: * Pure classification accepts injectable checkers so unit tests never touch real processes. * Apply mutations go only through :meth:`ControlPlaneDB.retire_session`. * Historical rows are never deleted; status moves to a terminal value and an events-row records the action + reason. """ from __future__ import annotations import json import os import subprocess from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any, Callable, Mapping, Sequence import control_plane_db as cpd import lease_lifecycle # Session status vocabulary. SESSION_STATUS_ACTIVE = "active" SESSION_STATUS_RETIRED = "retired" SESSION_STATUS_ENDED = "ended" SESSION_STATUS_TERMINAL = "terminal" TERMINAL_SESSION_STATUSES = frozenset( { SESSION_STATUS_RETIRED, SESSION_STATUS_ENDED, SESSION_STATUS_TERMINAL, "dead", "stale", "orphaned", } ) # Classification outcomes for one session row. CLASS_LIVE = "live" CLASS_DISCONNECTED = "disconnected" CLASS_STALE = "stale" CLASS_PROTECTED = "protected" CLASS_TERMINAL = "terminal" # Stable reason codes (audit + tests). REASON_ALREADY_TERMINAL = "already_terminal" REASON_LIVE_OWNER = "live_owner" REASON_LIVE_LEASE = "live_lease" REASON_CLIENT_MANAGED_LIVE = "client_managed_live" REASON_DEAD_OWNER = "dead_owner" REASON_PID_REUSE = "pid_reuse" REASON_HEARTBEAT_STALE_DEAD = "heartbeat_stale_dead_owner" REASON_MISSING_PID = "missing_pid_no_lease" # Event type written to control-plane events table. EVENT_SESSION_RETIRED = "session_retired" # Default heartbeat window before a still-alive PID is treated as disconnected # rather than live (does not alone authorize retirement). DEFAULT_HEARTBEAT_STALE_SECONDS = 900 # PID reuse: process start must be strictly later than session started_at by # more than this skew (ps lstart is second-resolution; clocks can lag). PID_REUSE_SKEW = timedelta(seconds=2) # Live lease freshness values that block retirement. _LIVE_LEASE_FRESHNESS = frozenset({"active", "live"}) class SessionLifecycleError(RuntimeError): """Fail-closed session lifecycle policy error.""" def _utc_now() -> datetime: return datetime.now(timezone.utc) def _parse_ts(value: str | datetime | None) -> datetime | None: if value is None: return None if isinstance(value, datetime): if value.tzinfo is None: return value.replace(tzinfo=timezone.utc) return value.astimezone(timezone.utc) return cpd._parse_ts(str(value)) def _ts(dt: datetime | None = None) -> str: return cpd._ts(dt) def process_start_time(pid: int | None) -> datetime | None: """Return the OS start time of *pid*, or None when it cannot be resolved. Uses ``ps -o lstart=`` (POSIX). Failures return None rather than inventing evidence — missing start time never authorizes retirement of a live PID. """ if pid is None: return None try: pid_i = int(pid) except (TypeError, ValueError): return None if pid_i <= 0: return None try: proc = subprocess.run( ["ps", "-o", "lstart=", "-p", str(pid_i)], capture_output=True, text=True, check=False, timeout=2, ) except (OSError, subprocess.SubprocessError): return None if proc.returncode != 0: return None text = (proc.stdout or "").strip() if not text: return None try: # Example: "Wed Jul 29 09:14:36 2026" naive = datetime.strptime(text, "%a %b %d %H:%M:%S %Y") return naive.replace(tzinfo=timezone.utc) except ValueError: return None def _lease_freshness_label(lease: Mapping[str, Any]) -> str: fr = lease.get("freshness") if isinstance(fr, Mapping): return str(fr.get("freshness") or "").strip().lower() if fr: return str(fr).strip().lower() # Fall back to classifying a raw lease row. try: return str( lease_lifecycle.classify_lease_freshness(lease).get("freshness") or "" ).strip().lower() except Exception: # noqa: BLE001 — pure classifier must not raise on bad rows status = str(lease.get("status") or "").strip().lower() return status or "unknown" def live_lease_session_ids( leases: Sequence[Mapping[str, Any]] | None, *, now: datetime | None = None, pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive, ) -> set[str]: """Session ids that still hold a live (or ambiguous-active) workflow lease.""" live: set[str] = set() moment = now or _utc_now() for lease in leases or (): if not isinstance(lease, Mapping): continue status = str(lease.get("status") or "").strip().lower() if status and status not in { lease_lifecycle.LEASE_STATUS_ACTIVE, "", }: # Explicit terminal lease statuses never protect a session. if status in { lease_lifecycle.LEASE_STATUS_RELEASED, lease_lifecycle.LEASE_STATUS_EXPIRED, lease_lifecycle.LEASE_STATUS_ABANDONED, }: continue freshness = _lease_freshness_label(lease) if freshness in _LIVE_LEASE_FRESHNESS or freshness in {"", "unknown"}: # Ambiguous active rows: re-check with authoritative classifier. try: fr = lease_lifecycle.classify_lease_freshness( lease, now=moment, pid_checker=pid_checker ) freshness = str(fr.get("freshness") or "").strip().lower() except Exception: # noqa: BLE001 freshness = "unknown" if freshness in _LIVE_LEASE_FRESHNESS: sid = str(lease.get("session_id") or "").strip() if sid: live.add(sid) elif freshness == "unknown" and status in { lease_lifecycle.LEASE_STATUS_ACTIVE, "", }: # Fail closed: active lease with unknown freshness blocks retirement. sid = str(lease.get("session_id") or "").strip() if sid: live.add(sid) return live @dataclass(frozen=True) class SessionClassification: """Classification of one workflow session row.""" session_id: str classification: str reason: str retireable: bool status: str | None pid: int | None pid_alive: bool | None pid_reused: bool heartbeat_stale: bool has_live_lease: bool client_managed: bool details: dict[str, Any] = field(default_factory=dict) def as_dict(self) -> dict[str, Any]: return { "session_id": self.session_id, "classification": self.classification, "reason": self.reason, "retireable": self.retireable, "status": self.status, "pid": self.pid, "pid_alive": self.pid_alive, "pid_reused": self.pid_reused, "heartbeat_stale": self.heartbeat_stale, "has_live_lease": self.has_live_lease, "client_managed": self.client_managed, "details": dict(self.details), } def classify_session( row: Mapping[str, Any], *, now: datetime | None = None, pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive, process_start_probe: Callable[[int | None], datetime | None] = process_start_time, live_lease_sessions: set[str] | frozenset[str] | None = None, client_managed_sessions: set[str] | frozenset[str] | None = None, heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS, ) -> SessionClassification: """Classify one session row for retirement decisions (#969). Rules (first match wins where noted): 1. Non-active / already-terminal status → ``terminal`` (not retireable). 2. Session holds a live lease → ``protected`` (never retire). 3. PID missing and no live lease → ``stale`` (retireable: missing owner). 4. PID alive + process start after session start → ``stale`` (PID reuse). 5. PID alive + client-managed → ``live`` protected (never retire). 6. PID alive + fresh heartbeat → ``live``. 7. PID alive + stale heartbeat → ``disconnected`` (not retireable alone). 8. PID dead → ``stale`` (retireable). """ moment = now or _utc_now() session_id = str(row.get("session_id") or "").strip() status = str(row.get("status") or "").strip().lower() or None raw_pid = row.get("pid") try: pid = int(raw_pid) if raw_pid is not None else None except (TypeError, ValueError): pid = None client_managed = bool( row.get("client_managed") or row.get("is_client_managed") or ( client_managed_sessions is not None and session_id in client_managed_sessions ) ) has_live_lease = bool( live_lease_sessions is not None and session_id in live_lease_sessions ) hb = _parse_ts(row.get("last_heartbeat_at")) heartbeat_stale = bool( hb is not None and (moment - hb).total_seconds() > max(0, int(heartbeat_stale_seconds)) ) if hb is None and status == SESSION_STATUS_ACTIVE: # No heartbeat evidence: treat as stale for liveness bookkeeping only. heartbeat_stale = True started = _parse_ts(row.get("started_at")) recorded_proc_start = _parse_ts(row.get("owner_process_started_at")) if status in TERMINAL_SESSION_STATUSES: return SessionClassification( session_id=session_id, classification=CLASS_TERMINAL, reason=REASON_ALREADY_TERMINAL, retireable=False, status=status, pid=pid, pid_alive=None, pid_reused=False, heartbeat_stale=heartbeat_stale, has_live_lease=has_live_lease, client_managed=client_managed, ) if has_live_lease: return SessionClassification( session_id=session_id, classification=CLASS_PROTECTED, reason=REASON_LIVE_LEASE, retireable=False, status=status, pid=pid, pid_alive=pid_checker(pid) if pid is not None else None, pid_reused=False, heartbeat_stale=heartbeat_stale, has_live_lease=True, client_managed=client_managed, details={"blocker": "live_workflow_lease"}, ) if pid is None: return SessionClassification( session_id=session_id, classification=CLASS_STALE, reason=REASON_MISSING_PID, retireable=True, status=status, pid=None, pid_alive=False, pid_reused=False, heartbeat_stale=heartbeat_stale, has_live_lease=False, client_managed=client_managed, ) pid_alive = bool(pid_checker(pid)) pid_reused = False proc_start: datetime | None = None if pid_alive: proc_start = recorded_proc_start or process_start_probe(pid) # PID reuse: live process started after the session row itself was # created. Anchor on started_at only — last_heartbeat alone is not a # safe bound (synthetic inventories and long-lived processes would # false-positive against a live ``ps`` probe). if proc_start is not None and started is not None: if proc_start > (started + PID_REUSE_SKEW): pid_reused = True if pid_reused: return SessionClassification( session_id=session_id, classification=CLASS_STALE, reason=REASON_PID_REUSE, retireable=True, status=status, pid=pid, pid_alive=True, pid_reused=True, heartbeat_stale=heartbeat_stale, has_live_lease=False, client_managed=client_managed, details={ "process_started_at": _ts(proc_start) if proc_start else None, "session_started_at": row.get("started_at"), "last_heartbeat_at": row.get("last_heartbeat_at"), }, ) if pid_alive and client_managed: return SessionClassification( session_id=session_id, classification=CLASS_LIVE, reason=REASON_CLIENT_MANAGED_LIVE, retireable=False, status=status, pid=pid, pid_alive=True, pid_reused=False, heartbeat_stale=heartbeat_stale, has_live_lease=False, client_managed=True, ) if pid_alive and not heartbeat_stale: return SessionClassification( session_id=session_id, classification=CLASS_LIVE, reason=REASON_LIVE_OWNER, retireable=False, status=status, pid=pid, pid_alive=True, pid_reused=False, heartbeat_stale=False, has_live_lease=False, client_managed=client_managed, ) if pid_alive and heartbeat_stale: # Process still exists but has not heartbeated — disconnected, not # confirmed stale. Do not retire; operator/reconnect owns next step. return SessionClassification( session_id=session_id, classification=CLASS_DISCONNECTED, reason=REASON_LIVE_OWNER, retireable=False, status=status, pid=pid, pid_alive=True, pid_reused=False, heartbeat_stale=True, has_live_lease=False, client_managed=client_managed, details={"note": "alive_pid_stale_heartbeat_not_retired"}, ) # PID dead (or checker said not alive). reason = REASON_DEAD_OWNER if heartbeat_stale: reason = REASON_HEARTBEAT_STALE_DEAD return SessionClassification( session_id=session_id, classification=CLASS_STALE, reason=reason, retireable=True, status=status, pid=pid, pid_alive=False, pid_reused=False, heartbeat_stale=heartbeat_stale, has_live_lease=False, client_managed=client_managed, ) @dataclass(frozen=True) class SessionFleetReport: """Fleet-wide classification summary for reconcile + apply.""" classifications: tuple[SessionClassification, ...] live_count: int disconnected_count: int stale_count: int protected_count: int terminal_count: int retireable: tuple[SessionClassification, ...] counts_by_reason: dict[str, int] def as_dict(self) -> dict[str, Any]: return { "live_count": self.live_count, "disconnected_count": self.disconnected_count, "stale_count": self.stale_count, "protected_count": self.protected_count, "terminal_count": self.terminal_count, "retireable_count": len(self.retireable), "retireable_session_ids": [c.session_id for c in self.retireable], "counts_by_reason": dict(self.counts_by_reason), "classifications": [c.as_dict() for c in self.classifications], "sessions_dimension_resolved": len(self.retireable) == 0, } def classify_sessions( sessions: Sequence[Mapping[str, Any]] | None, *, leases: Sequence[Mapping[str, Any]] | None = None, now: datetime | None = None, pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive, process_start_probe: Callable[[int | None], datetime | None] = process_start_time, client_managed_sessions: set[str] | frozenset[str] | None = None, heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS, ) -> SessionFleetReport: """Classify a fleet of session rows against live leases (#969).""" moment = now or _utc_now() live_leases = live_lease_session_ids( leases, now=moment, pid_checker=pid_checker ) results: list[SessionClassification] = [] for row in sessions or (): if not isinstance(row, Mapping): continue results.append( classify_session( row, now=moment, pid_checker=pid_checker, process_start_probe=process_start_probe, live_lease_sessions=live_leases, client_managed_sessions=client_managed_sessions, heartbeat_stale_seconds=heartbeat_stale_seconds, ) ) live_count = sum(1 for c in results if c.classification == CLASS_LIVE) disconnected_count = sum( 1 for c in results if c.classification == CLASS_DISCONNECTED ) stale_count = sum(1 for c in results if c.classification == CLASS_STALE) protected_count = sum(1 for c in results if c.classification == CLASS_PROTECTED) terminal_count = sum(1 for c in results if c.classification == CLASS_TERMINAL) retireable = tuple(c for c in results if c.retireable) by_reason: dict[str, int] = {} for c in results: by_reason[c.reason] = by_reason.get(c.reason, 0) + 1 return SessionFleetReport( classifications=tuple(results), live_count=live_count, disconnected_count=disconnected_count, stale_count=stale_count, protected_count=protected_count, terminal_count=terminal_count, retireable=retireable, counts_by_reason=by_reason, ) @dataclass(frozen=True) class RetirementResult: """Outcome of one session retirement attempt.""" session_id: str outcome: str # retired | already_terminal | skipped | blocked | missing reason: str prior_status: str | None = None new_status: str | None = None details: dict[str, Any] = field(default_factory=dict) def as_dict(self) -> dict[str, Any]: return { "session_id": self.session_id, "outcome": self.outcome, "reason": self.reason, "prior_status": self.prior_status, "new_status": self.new_status, "details": dict(self.details), } def apply_session_retirements( db: cpd.ControlPlaneDB, report: SessionFleetReport, *, dry_run: bool = False, actor_session_id: str | None = None, now: datetime | None = None, ) -> dict[str, Any]: """Terminalize every retireable session in *report* (idempotent). Concurrent reconciles are safe: each retirement is a CAS on ``status='active'`` (or other non-terminal). A second pass that sees the same session already retired records ``already_terminal`` rather than duplicating audit noise beyond a single no-op outcome. """ moment = now or _utc_now() results: list[RetirementResult] = [] retired = 0 already = 0 blocked = 0 missing = 0 for classification in report.retireable: if not classification.retireable: continue if dry_run: results.append( RetirementResult( session_id=classification.session_id, outcome="skipped", reason=classification.reason, prior_status=classification.status, new_status=SESSION_STATUS_RETIRED, details={"dry_run": True, **classification.details}, ) ) continue applied = db.retire_session( session_id=classification.session_id, reason=classification.reason, actor_session_id=actor_session_id, details={ "classification": classification.classification, "pid": classification.pid, "pid_alive": classification.pid_alive, "pid_reused": classification.pid_reused, "client_managed": classification.client_managed, **classification.details, }, now=moment, ) outcome = str(applied.get("outcome") or "missing") results.append( RetirementResult( session_id=classification.session_id, outcome=outcome, reason=str(applied.get("reason") or classification.reason), prior_status=applied.get("prior_status"), new_status=applied.get("new_status"), details=dict(applied.get("details") or {}), ) ) if outcome == "retired": retired += 1 elif outcome == "already_terminal": already += 1 elif outcome == "blocked": blocked += 1 else: missing += 1 # Non-retireable classifications are recorded for audit completeness when # dry-run lists the fleet, but apply only mutates retireable rows. return { "success": True, "dry_run": dry_run, "retired_count": retired, "already_terminal_count": already, "blocked_count": blocked, "missing_count": missing, "planned_count": len(report.retireable), "results": [r.as_dict() for r in results], "audit_action": EVENT_SESSION_RETIRED, "actor_session_id": actor_session_id, "recorded_at": _ts(moment), } def retire_stale_sessions( db: cpd.ControlPlaneDB, *, sessions: Sequence[Mapping[str, Any]] | None = None, leases: Sequence[Mapping[str, Any]] | None = None, dry_run: bool = False, actor_session_id: str | None = None, now: datetime | None = None, pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive, process_start_probe: Callable[[int | None], datetime | None] = process_start_time, client_managed_sessions: set[str] | frozenset[str] | None = None, heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS, session_limit: int = 500, ) -> dict[str, Any]: """End-to-end plan + apply for stale session retirement (#969). When *sessions* is omitted the control-plane DB is inventoried (active rows only). Callers that already gathered inventory should pass it. """ moment = now or _utc_now() if sessions is None: sessions = db.list_sessions( statuses=(SESSION_STATUS_ACTIVE,), limit=max(1, int(session_limit)), ) if leases is None: try: leases = db.list_leases(statuses=("active",), limit=max(1, int(session_limit))) except Exception: # noqa: BLE001 leases = [] report = classify_sessions( sessions, leases=leases, now=moment, pid_checker=pid_checker, process_start_probe=process_start_probe, client_managed_sessions=client_managed_sessions, heartbeat_stale_seconds=heartbeat_stale_seconds, ) apply_result = apply_session_retirements( db, report, dry_run=dry_run, actor_session_id=actor_session_id, now=moment, ) return { "success": True, "fleet": report.as_dict(), "apply": apply_result, "sessions_dimension_resolved": ( report.as_dict()["sessions_dimension_resolved"] if dry_run else apply_result["planned_count"] == ( apply_result["retired_count"] + apply_result["already_terminal_count"] ) and apply_result["blocked_count"] == 0 ), }