fix(reconcile): retire workflow session rows whose owners are no longer live

Implement a sanctioned session lifecycle for post-restart reconciliation so
active session rows with dead or reused owner PIDs can be terminalized
without deleting history. Protect live owners, live leases, and live
client-managed sessions; record durable session_retired audit events; keep
cleanup idempotent under concurrent reconciles.

Closes #969

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
2026-07-29 05:27:42 -04:00
co-authored by Claude Opus 4.8
parent 956fa15fe3
commit e8bae606cb
8 changed files with 1630 additions and 30 deletions
+145 -6
View File
@@ -24432,10 +24432,18 @@ def _run_post_restart_reconcile(
repo: str | None = None,
mode: str | None = None,
limit: int = 200,
apply_session_cleanup: bool = False,
) -> dict:
"""Gather + classify post-restart state; cache the latest proof (#662)."""
"""Gather + classify post-restart state; cache the latest proof (#662 / #969).
When *apply_session_cleanup* is true, confirmed-stale session rows (dead
owner / PID reuse, no live lease) are terminalized through
``session_lifecycle`` before re-classification so the sessions dimension
can resolve. Default remains dry (read-only) to preserve #662 rollout.
"""
global _POST_RESTART_LAST_PROOF, _POST_RESTART_BOOT_RAN
import post_restart_reconcile as prr
import session_lifecycle as sl
try:
_h, o, r = _resolve(remote, None, org, repo)
@@ -24449,13 +24457,48 @@ def _run_post_restart_reconcile(
inventory = _gather_post_restart_inventory(
remote=remote, org=o, repo=r, limit=limit
)
session_cleanup: dict | None = None
if apply_session_cleanup:
db, db_errs = _control_plane_db_or_error()
if db is None:
session_cleanup = {
"success": False,
"reasons": db_errs
or ["control-plane DB unavailable; cannot retire sessions"],
}
else:
profile = get_profile()
profile_name = (profile.get("profile_name") or "").strip() or "session"
actor = f"{profile_name}-{os.getpid()}"
session_cleanup = sl.retire_stale_sessions(
db,
sessions=inventory.get("sessions") or [],
leases=inventory.get("leases") or [],
dry_run=False,
actor_session_id=actor,
session_limit=max(1, int(limit)),
)
# Re-gather active sessions after mutation so classification sees
# the post-retirement fleet (retired rows drop out of active list).
try:
inventory["sessions"] = db.list_sessions(
statuses=("active",), limit=max(1, int(limit))
)
except Exception as exc: # noqa: BLE001
inventory["incomplete_reasons"] = list(
inventory.get("incomplete_reasons") or []
) + [f"post-retirement session re-list failed: {_redact(str(exc))}"]
fleet = (session_cleanup.get("fleet") or {}) if session_cleanup else {}
inventory["session_fleet"] = fleet
proof = prr.reconcile_after_restart(
inventory,
mode=mode or _post_restart_reconcile_mode(),
)
payload = proof.as_dict()
payload["success"] = True
payload["read_only"] = True
payload["read_only"] = not apply_session_cleanup
payload["remote"] = remote
payload["org"] = o
payload["repo"] = r
@@ -24464,6 +24507,9 @@ def _run_post_restart_reconcile(
"proposed_follow_ups lists durable issues the apply path may create; "
"this tool never creates them (log-only by default, #662 rollout)"
)
if session_cleanup is not None:
payload["session_cleanup"] = session_cleanup
payload["apply_session_cleanup"] = bool(apply_session_cleanup)
_POST_RESTART_LAST_PROOF = payload
_POST_RESTART_BOOT_RAN = True
return payload
@@ -24490,8 +24536,9 @@ def gitea_reconcile_after_restart(
repo: str | None = None,
mode: str | None = None,
limit: int = 200,
apply_session_cleanup: bool = False,
) -> dict:
"""Run post-restart MCP reconciliation and return a completion proof (#662).
"""Run post-restart MCP reconciliation and return a completion proof (#662 / #969).
Gathers live control-plane sessions, leases, worktree bindings, and
master-parity evidence, then classifies them with the pure
@@ -24499,12 +24546,17 @@ def gitea_reconcile_after_restart(
machine-readable completion proof listing resolved / unresolved dimensions
and proposed durable follow-up issues.
Read-only by design: never restarts MCP, never auto-resumes write
mutations, and never creates Gitea issues (those are a separate apply
path). Default mode is ``log_only``; set
Never restarts MCP, never auto-resumes write mutations, and never creates
Gitea issues. Default mode is ``log_only``; set
``GITEA_POST_RESTART_RECONCILE_MODE=enforce`` (or pass ``mode='enforce'``)
to set ``mutation_hold`` when anything remains unresolved.
*apply_session_cleanup* (#969): when true, confirmed-stale workflow session
rows (dead owner PID / PID reuse, no live lease, not a live client-managed
owner) are terminalized through the sanctioned ``session_lifecycle`` path
before re-classification. Default false preserves the historical read-only
gather+classify behaviour. Does not delete historical rows.
Soft-depends on #660 for session checkpoints: when the checkpoint schema
module is absent the checkpoints dimension is ``skipped`` with an explicit
reason rather than inventing a schema.
@@ -24524,9 +24576,96 @@ def gitea_reconcile_after_restart(
repo=repo,
mode=mode,
limit=limit,
apply_session_cleanup=bool(apply_session_cleanup),
)
@mcp.tool()
def gitea_retire_stale_workflow_sessions(
remote: str = "dadeschools",
host: str | None = None,
org: str | None = None,
repo: str | None = None,
apply: bool = False,
limit: int = 500,
) -> dict:
"""Retire workflow session rows whose owners are no longer live (#969).
Plans (and optionally applies) terminalization of control-plane session
rows that are confirmed stale:
* owner PID absent / dead
* PID reused by an unrelated process (process start after session start)
* no live workflow lease
* not a live client-managed owner
Default ``apply=false`` is dry-run only. ``apply=true`` performs CAS
status updates to ``retired`` and writes durable ``session_retired``
events. Idempotent under concurrent reconciliation. Never deletes rows
and never touches sessions protected by a live lease or live owner.
"""
read_block = _profile_operation_gate("gitea.read")
if read_block:
return {
"success": False,
"read_only": not apply,
"reasons": read_block,
"permission_report": _permission_block_report("gitea.read"),
}
try:
_h, o, r = _resolve(remote, None, org, repo)
except ValueError as exc:
return {"success": False, "reasons": [str(exc)], "read_only": not apply}
db, db_errs = _control_plane_db_or_error()
if db is None:
return {
"success": False,
"read_only": not apply,
"reasons": db_errs or ["control-plane DB unavailable"],
}
import session_lifecycle as sl
profile = get_profile()
profile_name = (profile.get("profile_name") or "").strip() or "session"
actor = f"{profile_name}-{os.getpid()}"
# Prefer lease inventory scoped to the requested repo when available.
leases: list[dict] = []
try:
lease_result = lease_lifecycle.list_active_leases(
db,
remote=remote if remote in REMOTES else remote,
org=o,
repo=r,
role=None,
include_non_active=True,
limit=max(1, int(limit)),
)
leases = list(lease_result.get("leases") or [])
except Exception: # noqa: BLE001
try:
leases = db.list_leases(statuses=("active",), limit=max(1, int(limit)))
except Exception: # noqa: BLE001
leases = []
result = sl.retire_stale_sessions(
db,
leases=leases,
dry_run=not bool(apply),
actor_session_id=actor,
session_limit=max(1, int(limit)),
)
result["read_only"] = not bool(apply)
result["apply"] = bool(apply)
result["remote"] = remote
result["org"] = o
result["repo"] = r
return result
@mcp.tool()
def gitea_inspect_workflow_lease(
lease_id: str,