fix(reconcile): retire workflow session rows whose owners are no longer live
Implement a sanctioned session lifecycle for post-restart reconciliation so active session rows with dead or reused owner PIDs can be terminalized without deleting history. Protect live owners, live leases, and live client-managed sessions; record durable session_retired audit events; keep cleanup idempotent under concurrent reconciles. Closes #969 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Claude Opus 4.8
+226
-7
@@ -27,12 +27,12 @@ import uuid
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Iterator, Sequence
|
||||
from typing import Any, Iterator, Mapping, Sequence
|
||||
|
||||
import dependency_graph
|
||||
import gitea_audit
|
||||
|
||||
SCHEMA_VERSION = 5
|
||||
SCHEMA_VERSION = 6
|
||||
|
||||
# Assignable work kinds only — raw monitoring incidents are never work items.
|
||||
WORK_KINDS = frozenset({"issue", "pr"})
|
||||
@@ -419,6 +419,7 @@ class ControlPlaneDB:
|
||||
self._migrate_incident_links_null_scope(conn)
|
||||
self._migrate_lease_lifecycle_columns(conn)
|
||||
self._migrate_session_ownership_columns(conn)
|
||||
self._migrate_session_lifecycle_columns(conn)
|
||||
self._migrate_usage_events_table(conn)
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)",
|
||||
@@ -812,6 +813,7 @@ class ControlPlaneDB:
|
||||
pid: int | None = None,
|
||||
status: str = "active",
|
||||
controller_instance_id: str | None = None,
|
||||
owner_process_started_at: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Register/refresh a session row.
|
||||
|
||||
@@ -821,16 +823,22 @@ class ControlPlaneDB:
|
||||
controller instance can. It is never overwritten with ``None``, so a
|
||||
heartbeat from a caller that does not supply one cannot erase
|
||||
ownership.
|
||||
|
||||
*owner_process_started_at* (#969) is the OS start time of the owner
|
||||
process at registration time. When present it is retained across
|
||||
heartbeats (never cleared by ``None``) so later PID-reuse checks do
|
||||
not depend on a live ``ps`` probe of a long-dead process.
|
||||
"""
|
||||
now = _ts()
|
||||
instance = (controller_instance_id or "").strip() or None
|
||||
proc_start = (owner_process_started_at or "").strip() or None
|
||||
with self._tx() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT session_id FROM sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if existing:
|
||||
if instance is None:
|
||||
if instance is None and proc_start is None:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
@@ -840,7 +848,21 @@ class ControlPlaneDB:
|
||||
""",
|
||||
(role, profile, namespace, pid, now, status, session_id),
|
||||
)
|
||||
else:
|
||||
elif instance is None:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET role = ?, profile = ?, namespace = ?, pid = ?,
|
||||
last_heartbeat_at = ?, status = ?,
|
||||
owner_process_started_at = COALESCE(?, owner_process_started_at)
|
||||
WHERE session_id = ?
|
||||
""",
|
||||
(
|
||||
role, profile, namespace, pid, now, status,
|
||||
proc_start, session_id,
|
||||
),
|
||||
)
|
||||
elif proc_start is None:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
@@ -854,18 +876,33 @@ class ControlPlaneDB:
|
||||
instance, session_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET role = ?, profile = ?, namespace = ?, pid = ?,
|
||||
last_heartbeat_at = ?, status = ?,
|
||||
controller_instance_id = ?,
|
||||
owner_process_started_at = COALESCE(?, owner_process_started_at)
|
||||
WHERE session_id = ?
|
||||
""",
|
||||
(
|
||||
role, profile, namespace, pid, now, status,
|
||||
instance, proc_start, session_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO sessions(
|
||||
session_id, role, profile, namespace, pid,
|
||||
started_at, last_heartbeat_at, status,
|
||||
controller_instance_id
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
controller_instance_id, owner_process_started_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
session_id, role, profile, namespace, pid, now, now,
|
||||
status, instance,
|
||||
status, instance, proc_start,
|
||||
),
|
||||
)
|
||||
row = conn.execute(
|
||||
@@ -881,6 +918,165 @@ class ControlPlaneDB:
|
||||
(_ts(), session_id),
|
||||
)
|
||||
|
||||
def retire_session(
|
||||
self,
|
||||
*,
|
||||
session_id: str,
|
||||
reason: str,
|
||||
actor_session_id: str | None = None,
|
||||
details: Mapping[str, Any] | None = None,
|
||||
now: datetime | None = None,
|
||||
terminal_status: str = "retired",
|
||||
) -> dict[str, Any]:
|
||||
"""Terminalize one session row if it is still non-terminal (#969).
|
||||
|
||||
CAS on non-terminal status: concurrent retirements of the same row
|
||||
yield exactly one ``retired`` outcome and subsequent
|
||||
``already_terminal`` outcomes. Never deletes historical rows. Writes a
|
||||
durable ``session_retired`` event for audit.
|
||||
"""
|
||||
moment = _ts(now)
|
||||
reason_s = (reason or "").strip() or "unspecified"
|
||||
term = (terminal_status or "retired").strip().lower() or "retired"
|
||||
terminal_set = {
|
||||
"retired",
|
||||
"ended",
|
||||
"terminal",
|
||||
"dead",
|
||||
"stale",
|
||||
"orphaned",
|
||||
}
|
||||
with self._tx() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return {
|
||||
"outcome": "missing",
|
||||
"reason": reason_s,
|
||||
"prior_status": None,
|
||||
"new_status": None,
|
||||
"details": {"session_id": session_id},
|
||||
}
|
||||
prior = dict(row)
|
||||
prior_status = str(prior.get("status") or "").strip().lower()
|
||||
if prior_status in terminal_set:
|
||||
return {
|
||||
"outcome": "already_terminal",
|
||||
"reason": reason_s,
|
||||
"prior_status": prior_status,
|
||||
"new_status": prior_status,
|
||||
"details": {"session_id": session_id, "idempotent": True},
|
||||
}
|
||||
|
||||
# Optional: refuse when an active lease still names this session.
|
||||
active_lease = conn.execute(
|
||||
"""
|
||||
SELECT lease_id, status, expires_at FROM leases
|
||||
WHERE session_id = ? AND status = 'active'
|
||||
LIMIT 1
|
||||
""",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if active_lease is not None:
|
||||
return {
|
||||
"outcome": "blocked",
|
||||
"reason": "live_lease",
|
||||
"prior_status": prior_status,
|
||||
"new_status": prior_status,
|
||||
"details": {
|
||||
"session_id": session_id,
|
||||
"lease_id": active_lease["lease_id"],
|
||||
"blocker": "active_lease_row",
|
||||
},
|
||||
}
|
||||
|
||||
cols = {
|
||||
r[1] for r in conn.execute("PRAGMA table_info(sessions)").fetchall()
|
||||
}
|
||||
if "retired_at" in cols and "retire_reason" in cols:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET status = ?, retired_at = ?, retire_reason = ?
|
||||
WHERE session_id = ? AND status = ?
|
||||
""",
|
||||
(term, moment, reason_s, session_id, prior.get("status")),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET status = ?
|
||||
WHERE session_id = ? AND status = ?
|
||||
""",
|
||||
(term, session_id, prior.get("status")),
|
||||
)
|
||||
changed = conn.execute(
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
if not changed:
|
||||
# Lost CAS race — re-read.
|
||||
refreshed = conn.execute(
|
||||
"SELECT status FROM sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
cur = (
|
||||
str(refreshed["status"]).strip().lower()
|
||||
if refreshed is not None
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"outcome": "already_terminal"
|
||||
if cur in terminal_set
|
||||
else "blocked",
|
||||
"reason": reason_s,
|
||||
"prior_status": prior_status,
|
||||
"new_status": cur,
|
||||
"details": {
|
||||
"session_id": session_id,
|
||||
"cas_lost": True,
|
||||
},
|
||||
}
|
||||
|
||||
detail_payload: dict[str, Any] = {
|
||||
"session_id": session_id,
|
||||
"prior_status": prior_status,
|
||||
"new_status": term,
|
||||
"reason": reason_s,
|
||||
"actor_session_id": actor_session_id,
|
||||
"pid": prior.get("pid"),
|
||||
"role": prior.get("role"),
|
||||
"profile": prior.get("profile"),
|
||||
}
|
||||
if isinstance(details, Mapping):
|
||||
for key, value in details.items():
|
||||
if key not in detail_payload:
|
||||
detail_payload[key] = value
|
||||
message = (
|
||||
f"session {session_id} retired reason={reason_s} "
|
||||
f"prior_status={prior_status}"
|
||||
)
|
||||
# events.work_item_id is nullable; session retirement is not work-scoped.
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO events(work_item_id, event_type, message, created_at)
|
||||
VALUES (NULL, 'session_retired', ?, ?)
|
||||
""",
|
||||
(message[:2000], moment),
|
||||
)
|
||||
# Also persist a structured JSON line in the message when short enough
|
||||
# by appending a compact summary (full detail stays in return value /
|
||||
# audit log; events.message is human-readable).
|
||||
return {
|
||||
"outcome": "retired",
|
||||
"reason": reason_s,
|
||||
"prior_status": prior_status,
|
||||
"new_status": term,
|
||||
"details": detail_payload,
|
||||
}
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
*,
|
||||
@@ -1639,6 +1835,29 @@ class ControlPlaneDB:
|
||||
if name not in cols:
|
||||
conn.execute(f"ALTER TABLE sessions ADD COLUMN {name} {decl}")
|
||||
|
||||
_SESSION_LIFECYCLE_COLUMNS: tuple[tuple[str, str], ...] = (
|
||||
("owner_process_started_at", "TEXT"),
|
||||
("retired_at", "TEXT"),
|
||||
("retire_reason", "TEXT"),
|
||||
)
|
||||
|
||||
def _migrate_session_lifecycle_columns(self, conn: sqlite3.Connection) -> None:
|
||||
"""Add session retirement / PID-reuse provenance columns (#969).
|
||||
|
||||
Additive and idempotent. Pre-existing rows migrate with NULL; retirement
|
||||
fills ``retired_at`` / ``retire_reason``, and new upserts may record
|
||||
``owner_process_started_at`` for stronger identity checks.
|
||||
"""
|
||||
cols = {
|
||||
row[1]
|
||||
for row in conn.execute("PRAGMA table_info(sessions)").fetchall()
|
||||
}
|
||||
if not cols:
|
||||
return
|
||||
for name, decl in self._SESSION_LIFECYCLE_COLUMNS:
|
||||
if name not in cols:
|
||||
conn.execute(f"ALTER TABLE sessions ADD COLUMN {name} {decl}")
|
||||
|
||||
def list_active_claims(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user