fix(reconcile): retire workflow session rows whose owners are no longer live
Implement a sanctioned session lifecycle for post-restart reconciliation so active session rows with dead or reused owner PIDs can be terminalized without deleting history. Protect live owners, live leases, and live client-managed sessions; record durable session_retired audit events; keep cleanup idempotent under concurrent reconciles. Closes #969 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
+57
-12
@@ -475,24 +475,65 @@ def reconcile_after_restart(
|
||||
)
|
||||
)
|
||||
|
||||
# --- sessions -------------------------------------------------------
|
||||
# --- sessions (#969: dead-owner / PID-reuse lifecycle) ---------------
|
||||
# Prefer a precomputed fleet report from the gather/apply path when present;
|
||||
# otherwise classify pure from inventory (injectable checkers stay default).
|
||||
sessions = [s for s in (inventory.get("sessions") or []) if isinstance(s, Mapping)]
|
||||
orphan_sessions = [
|
||||
s
|
||||
for s in sessions
|
||||
if str(s.get("status") or "").lower() == "active"
|
||||
and s.get("pid") is not None
|
||||
and not lease_lifecycle.is_process_alive(s.get("pid"))
|
||||
leases_for_sessions = [
|
||||
L for L in (inventory.get("leases") or []) if isinstance(L, Mapping)
|
||||
]
|
||||
if orphan_sessions:
|
||||
fleet_report = inventory.get("session_fleet")
|
||||
if isinstance(fleet_report, Mapping) and "retireable_session_ids" in fleet_report:
|
||||
fleet_details = dict(fleet_report)
|
||||
retireable_ids = list(fleet_details.get("retireable_session_ids") or [])
|
||||
resolved = bool(fleet_details.get("sessions_dimension_resolved", not retireable_ids))
|
||||
else:
|
||||
try:
|
||||
import session_lifecycle as _sl
|
||||
|
||||
client_managed = inventory.get("client_managed_session_ids")
|
||||
cm_set = None
|
||||
if isinstance(client_managed, (list, tuple, set, frozenset)):
|
||||
cm_set = {str(x) for x in client_managed}
|
||||
fleet = _sl.classify_sessions(
|
||||
sessions,
|
||||
leases=leases_for_sessions,
|
||||
now=started,
|
||||
client_managed_sessions=cm_set,
|
||||
)
|
||||
fleet_details = fleet.as_dict()
|
||||
retireable_ids = list(fleet_details.get("retireable_session_ids") or [])
|
||||
resolved = bool(fleet_details.get("sessions_dimension_resolved"))
|
||||
except Exception as exc: # noqa: BLE001 — fail closed to legacy signal
|
||||
# Legacy fallback: dead-pid active rows only (pre-#969 behaviour).
|
||||
orphan_sessions = [
|
||||
s
|
||||
for s in sessions
|
||||
if str(s.get("status") or "").lower() == "active"
|
||||
and s.get("pid") is not None
|
||||
and not lease_lifecycle.is_process_alive(s.get("pid"))
|
||||
]
|
||||
retireable_ids = [s.get("session_id") for s in orphan_sessions]
|
||||
resolved = not orphan_sessions
|
||||
fleet_details = {
|
||||
"total_sessions": len(sessions),
|
||||
"retireable_session_ids": retireable_ids,
|
||||
"legacy_fallback": True,
|
||||
"fallback_error": str(exc),
|
||||
}
|
||||
|
||||
if not resolved and retireable_ids:
|
||||
items.append(
|
||||
_item(
|
||||
DIM_SESSIONS,
|
||||
ITEM_UNRESOLVED,
|
||||
f"{len(orphan_sessions)} active session row(s) with dead owner pid",
|
||||
f"{len(retireable_ids)} session row(s) with dead/reused owner "
|
||||
f"await retirement",
|
||||
details={
|
||||
"orphan_session_ids": [s.get("session_id") for s in orphan_sessions],
|
||||
"orphan_session_ids": retireable_ids,
|
||||
"retireable_session_ids": retireable_ids,
|
||||
"total_sessions": len(sessions),
|
||||
"fleet": fleet_details,
|
||||
},
|
||||
follow_up=True,
|
||||
)
|
||||
@@ -502,8 +543,12 @@ def reconcile_after_restart(
|
||||
_item(
|
||||
DIM_SESSIONS,
|
||||
ITEM_RESOLVED,
|
||||
f"{len(sessions)} session row(s) reconciled (no dead-pid orphans)",
|
||||
details={"total_sessions": len(sessions)},
|
||||
f"{len(sessions)} session row(s) reconciled "
|
||||
f"(no retireable dead/reused owners)",
|
||||
details={
|
||||
"total_sessions": len(sessions),
|
||||
"fleet": fleet_details,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user