fix(reconcile): retire workflow session rows whose owners are no longer live
Implement a sanctioned session lifecycle for post-restart reconciliation so active session rows with dead or reused owner PIDs can be terminalized without deleting history. Protect live owners, live leases, and live client-managed sessions; record durable session_retired audit events; keep cleanup idempotent under concurrent reconciles. Closes #969 Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
+145
-6
@@ -24432,10 +24432,18 @@ def _run_post_restart_reconcile(
|
||||
repo: str | None = None,
|
||||
mode: str | None = None,
|
||||
limit: int = 200,
|
||||
apply_session_cleanup: bool = False,
|
||||
) -> dict:
|
||||
"""Gather + classify post-restart state; cache the latest proof (#662)."""
|
||||
"""Gather + classify post-restart state; cache the latest proof (#662 / #969).
|
||||
|
||||
When *apply_session_cleanup* is true, confirmed-stale session rows (dead
|
||||
owner / PID reuse, no live lease) are terminalized through
|
||||
``session_lifecycle`` before re-classification so the sessions dimension
|
||||
can resolve. Default remains dry (read-only) to preserve #662 rollout.
|
||||
"""
|
||||
global _POST_RESTART_LAST_PROOF, _POST_RESTART_BOOT_RAN
|
||||
import post_restart_reconcile as prr
|
||||
import session_lifecycle as sl
|
||||
|
||||
try:
|
||||
_h, o, r = _resolve(remote, None, org, repo)
|
||||
@@ -24449,13 +24457,48 @@ def _run_post_restart_reconcile(
|
||||
inventory = _gather_post_restart_inventory(
|
||||
remote=remote, org=o, repo=r, limit=limit
|
||||
)
|
||||
|
||||
session_cleanup: dict | None = None
|
||||
if apply_session_cleanup:
|
||||
db, db_errs = _control_plane_db_or_error()
|
||||
if db is None:
|
||||
session_cleanup = {
|
||||
"success": False,
|
||||
"reasons": db_errs
|
||||
or ["control-plane DB unavailable; cannot retire sessions"],
|
||||
}
|
||||
else:
|
||||
profile = get_profile()
|
||||
profile_name = (profile.get("profile_name") or "").strip() or "session"
|
||||
actor = f"{profile_name}-{os.getpid()}"
|
||||
session_cleanup = sl.retire_stale_sessions(
|
||||
db,
|
||||
sessions=inventory.get("sessions") or [],
|
||||
leases=inventory.get("leases") or [],
|
||||
dry_run=False,
|
||||
actor_session_id=actor,
|
||||
session_limit=max(1, int(limit)),
|
||||
)
|
||||
# Re-gather active sessions after mutation so classification sees
|
||||
# the post-retirement fleet (retired rows drop out of active list).
|
||||
try:
|
||||
inventory["sessions"] = db.list_sessions(
|
||||
statuses=("active",), limit=max(1, int(limit))
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
inventory["incomplete_reasons"] = list(
|
||||
inventory.get("incomplete_reasons") or []
|
||||
) + [f"post-retirement session re-list failed: {_redact(str(exc))}"]
|
||||
fleet = (session_cleanup.get("fleet") or {}) if session_cleanup else {}
|
||||
inventory["session_fleet"] = fleet
|
||||
|
||||
proof = prr.reconcile_after_restart(
|
||||
inventory,
|
||||
mode=mode or _post_restart_reconcile_mode(),
|
||||
)
|
||||
payload = proof.as_dict()
|
||||
payload["success"] = True
|
||||
payload["read_only"] = True
|
||||
payload["read_only"] = not apply_session_cleanup
|
||||
payload["remote"] = remote
|
||||
payload["org"] = o
|
||||
payload["repo"] = r
|
||||
@@ -24464,6 +24507,9 @@ def _run_post_restart_reconcile(
|
||||
"proposed_follow_ups lists durable issues the apply path may create; "
|
||||
"this tool never creates them (log-only by default, #662 rollout)"
|
||||
)
|
||||
if session_cleanup is not None:
|
||||
payload["session_cleanup"] = session_cleanup
|
||||
payload["apply_session_cleanup"] = bool(apply_session_cleanup)
|
||||
_POST_RESTART_LAST_PROOF = payload
|
||||
_POST_RESTART_BOOT_RAN = True
|
||||
return payload
|
||||
@@ -24490,8 +24536,9 @@ def gitea_reconcile_after_restart(
|
||||
repo: str | None = None,
|
||||
mode: str | None = None,
|
||||
limit: int = 200,
|
||||
apply_session_cleanup: bool = False,
|
||||
) -> dict:
|
||||
"""Run post-restart MCP reconciliation and return a completion proof (#662).
|
||||
"""Run post-restart MCP reconciliation and return a completion proof (#662 / #969).
|
||||
|
||||
Gathers live control-plane sessions, leases, worktree bindings, and
|
||||
master-parity evidence, then classifies them with the pure
|
||||
@@ -24499,12 +24546,17 @@ def gitea_reconcile_after_restart(
|
||||
machine-readable completion proof listing resolved / unresolved dimensions
|
||||
and proposed durable follow-up issues.
|
||||
|
||||
Read-only by design: never restarts MCP, never auto-resumes write
|
||||
mutations, and never creates Gitea issues (those are a separate apply
|
||||
path). Default mode is ``log_only``; set
|
||||
Never restarts MCP, never auto-resumes write mutations, and never creates
|
||||
Gitea issues. Default mode is ``log_only``; set
|
||||
``GITEA_POST_RESTART_RECONCILE_MODE=enforce`` (or pass ``mode='enforce'``)
|
||||
to set ``mutation_hold`` when anything remains unresolved.
|
||||
|
||||
*apply_session_cleanup* (#969): when true, confirmed-stale workflow session
|
||||
rows (dead owner PID / PID reuse, no live lease, not a live client-managed
|
||||
owner) are terminalized through the sanctioned ``session_lifecycle`` path
|
||||
before re-classification. Default false preserves the historical read-only
|
||||
gather+classify behaviour. Does not delete historical rows.
|
||||
|
||||
Soft-depends on #660 for session checkpoints: when the checkpoint schema
|
||||
module is absent the checkpoints dimension is ``skipped`` with an explicit
|
||||
reason rather than inventing a schema.
|
||||
@@ -24524,9 +24576,96 @@ def gitea_reconcile_after_restart(
|
||||
repo=repo,
|
||||
mode=mode,
|
||||
limit=limit,
|
||||
apply_session_cleanup=bool(apply_session_cleanup),
|
||||
)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def gitea_retire_stale_workflow_sessions(
|
||||
remote: str = "dadeschools",
|
||||
host: str | None = None,
|
||||
org: str | None = None,
|
||||
repo: str | None = None,
|
||||
apply: bool = False,
|
||||
limit: int = 500,
|
||||
) -> dict:
|
||||
"""Retire workflow session rows whose owners are no longer live (#969).
|
||||
|
||||
Plans (and optionally applies) terminalization of control-plane session
|
||||
rows that are confirmed stale:
|
||||
|
||||
* owner PID absent / dead
|
||||
* PID reused by an unrelated process (process start after session start)
|
||||
* no live workflow lease
|
||||
* not a live client-managed owner
|
||||
|
||||
Default ``apply=false`` is dry-run only. ``apply=true`` performs CAS
|
||||
status updates to ``retired`` and writes durable ``session_retired``
|
||||
events. Idempotent under concurrent reconciliation. Never deletes rows
|
||||
and never touches sessions protected by a live lease or live owner.
|
||||
"""
|
||||
read_block = _profile_operation_gate("gitea.read")
|
||||
if read_block:
|
||||
return {
|
||||
"success": False,
|
||||
"read_only": not apply,
|
||||
"reasons": read_block,
|
||||
"permission_report": _permission_block_report("gitea.read"),
|
||||
}
|
||||
|
||||
try:
|
||||
_h, o, r = _resolve(remote, None, org, repo)
|
||||
except ValueError as exc:
|
||||
return {"success": False, "reasons": [str(exc)], "read_only": not apply}
|
||||
|
||||
db, db_errs = _control_plane_db_or_error()
|
||||
if db is None:
|
||||
return {
|
||||
"success": False,
|
||||
"read_only": not apply,
|
||||
"reasons": db_errs or ["control-plane DB unavailable"],
|
||||
}
|
||||
|
||||
import session_lifecycle as sl
|
||||
|
||||
profile = get_profile()
|
||||
profile_name = (profile.get("profile_name") or "").strip() or "session"
|
||||
actor = f"{profile_name}-{os.getpid()}"
|
||||
|
||||
# Prefer lease inventory scoped to the requested repo when available.
|
||||
leases: list[dict] = []
|
||||
try:
|
||||
lease_result = lease_lifecycle.list_active_leases(
|
||||
db,
|
||||
remote=remote if remote in REMOTES else remote,
|
||||
org=o,
|
||||
repo=r,
|
||||
role=None,
|
||||
include_non_active=True,
|
||||
limit=max(1, int(limit)),
|
||||
)
|
||||
leases = list(lease_result.get("leases") or [])
|
||||
except Exception: # noqa: BLE001
|
||||
try:
|
||||
leases = db.list_leases(statuses=("active",), limit=max(1, int(limit)))
|
||||
except Exception: # noqa: BLE001
|
||||
leases = []
|
||||
|
||||
result = sl.retire_stale_sessions(
|
||||
db,
|
||||
leases=leases,
|
||||
dry_run=not bool(apply),
|
||||
actor_session_id=actor,
|
||||
session_limit=max(1, int(limit)),
|
||||
)
|
||||
result["read_only"] = not bool(apply)
|
||||
result["apply"] = bool(apply)
|
||||
result["remote"] = remote
|
||||
result["org"] = o
|
||||
result["repo"] = r
|
||||
return result
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def gitea_inspect_workflow_lease(
|
||||
lease_id: str,
|
||||
|
||||
Reference in New Issue
Block a user