feat(recovery): stale worktree binding demotion and crash-orphan lease recovery (Closes #702)
An unhandled daemon crash skips teardown: the durable reviewer lease comment survives, the in-memory session lease dies, and the auto-reconnected daemon inherits a stale GITEA_ACTIVE_WORKTREE from its parent environment (PR #701 / branches/review-pr-654 incident). AC2 — safe clear/re-bind of stale bindings: - new stale_binding_recovery.py: classifies the active env binding (corroborated / unverified_inherited / provably_stale_missing_path / superseded_by_session_lease) and produces a fail-closed recovery plan; only provably stale bindings are clear-eligible - gitea_resolve_task_capability and daemon boot run the sanctioned recovery (durable audit via session state); runtime context surfaces the classification read-only - namespace_workspace_binding.resolve_namespace_workspace(verify_paths=…) demotes env-bound worktrees whose path no longer exists; mutation and runtime-context resolution always verify AC3 — recoverable lease cleanup after unexpected exit: - durable session-lease shadow (KIND_REVIEWER_SESSION_LEASE) written on sanctioned record/heartbeat, removed on sanctioned clear; a crash leaves provable orphan evidence (owner pid + session id) - assess_crashed_session_lease_orphan derives tri-state owner_process_alive: only a dead recorded owner pid proves exit; PID liveness is never ownership proof - diagnose/cleanup wrappers consume the shadow instead of passing owner_process_alive=None - new classification orphaned_expired_superseded_head: an expired superseded-head lease with no terminal review stops being a permanent ambiguous/wait; post-expiry it unlocks fresh acquisition, and guarded cleanup becomes eligible only with owner-exit evidence + clean/absent worktree + controller authorization + confirmation. Pre-expiry behavior is unchanged (fail-closed wait, no steal). Validation: tests/test_issue_702_stale_binding_lease_recovery.py (29 tests covering lost in-session leases, old-head leases, runtime/worktree mismatch, managed reconnect, safe expiry); full suite 2665 passed, 6 skipped, 161 subtests. Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
Claude Fable 5
@@ -407,18 +407,150 @@ def record_session_lease(
|
||||
if lease_provenance:
|
||||
stored["lease_provenance"] = dict(lease_provenance)
|
||||
_SESSION_LEASE = stored
|
||||
_persist_session_lease_shadow(stored)
|
||||
return dict(_SESSION_LEASE)
|
||||
|
||||
|
||||
def clear_session_lease() -> None:
|
||||
global _SESSION_LEASE
|
||||
_SESSION_LEASE = None
|
||||
_persist_session_lease_shadow(None)
|
||||
|
||||
|
||||
def get_session_lease() -> dict[str, Any] | None:
|
||||
return dict(_SESSION_LEASE) if _SESSION_LEASE else None
|
||||
|
||||
|
||||
def _persist_session_lease_shadow(lease: dict[str, Any] | None) -> None:
|
||||
"""Best-effort durable shadow of the in-session lease (#702).
|
||||
|
||||
A daemon that dies without teardown leaves this record behind, giving a
|
||||
later process provable orphan evidence (owner pid + session id) for the
|
||||
guarded cleanup path. Observability only — mutation gates never read it,
|
||||
and failures here must never block lease operations.
|
||||
"""
|
||||
try:
|
||||
import mcp_session_state as mss
|
||||
|
||||
if lease is None:
|
||||
mss.clear_state(kind=mss.KIND_REVIEWER_SESSION_LEASE)
|
||||
return
|
||||
mss.save_state(
|
||||
kind=mss.KIND_REVIEWER_SESSION_LEASE,
|
||||
payload={
|
||||
"pr_number": lease.get("pr_number"),
|
||||
"session_id": lease.get("session_id"),
|
||||
"comment_id": lease.get("comment_id"),
|
||||
"candidate_head": lease.get("candidate_head"),
|
||||
"worktree": lease.get("worktree"),
|
||||
"phase": lease.get("phase"),
|
||||
"reviewer_identity": lease.get("reviewer_identity"),
|
||||
"profile": lease.get("profile"),
|
||||
"lease_repo": lease.get("repo"),
|
||||
"owner_pid": os.getpid(),
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def load_session_lease_shadow() -> dict[str, Any] | None:
|
||||
"""Load the durable session-lease shadow left by this profile identity."""
|
||||
try:
|
||||
import mcp_session_state as mss
|
||||
|
||||
return mss.load_state(kind=mss.KIND_REVIEWER_SESSION_LEASE)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _pid_alive(pid: int) -> bool:
|
||||
"""Probe process existence; fail closed toward 'alive' when unsure."""
|
||||
try:
|
||||
os.kill(int(pid), 0)
|
||||
return True
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
|
||||
def assess_crashed_session_lease_orphan(
|
||||
shadow: dict[str, Any] | None,
|
||||
*,
|
||||
lease: dict[str, Any] | None,
|
||||
current_pid: int | None = None,
|
||||
pid_alive: bool | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Derive owner-process evidence for a durable lease from the shadow (#702).
|
||||
|
||||
Returns ``owner_process_alive`` tri-state. Only a provably dead recorded
|
||||
owner pid yields ``False`` (a dead pid cannot still be running the owning
|
||||
daemon); an alive pid is *not* ownership proof and stays ``None`` unless
|
||||
it is this very process. Session ids must match exactly — the shadow of a
|
||||
different lease proves nothing.
|
||||
"""
|
||||
reasons: list[str] = []
|
||||
if not shadow or not lease:
|
||||
return {
|
||||
"orphan_evidence": False,
|
||||
"owner_process_alive": None,
|
||||
"reasons": ["no durable session-lease shadow evidence available"],
|
||||
}
|
||||
shadow_sid = (shadow.get("session_id") or "").strip()
|
||||
lease_sid = (lease.get("session_id") or "").strip()
|
||||
if not shadow_sid or not lease_sid or shadow_sid != lease_sid:
|
||||
return {
|
||||
"orphan_evidence": False,
|
||||
"owner_process_alive": None,
|
||||
"reasons": [
|
||||
"session-lease shadow session_id does not match the durable "
|
||||
f"lease (shadow={shadow_sid or None}, lease={lease_sid or None})"
|
||||
],
|
||||
}
|
||||
owner_pid = shadow.get("owner_pid") or shadow.get("writer_pid")
|
||||
try:
|
||||
owner_pid = int(owner_pid)
|
||||
except (TypeError, ValueError):
|
||||
return {
|
||||
"orphan_evidence": False,
|
||||
"owner_process_alive": None,
|
||||
"reasons": ["session-lease shadow has no usable owner pid (fail closed)"],
|
||||
}
|
||||
this_pid = current_pid if current_pid is not None else os.getpid()
|
||||
if owner_pid == this_pid:
|
||||
return {
|
||||
"orphan_evidence": False,
|
||||
"owner_process_alive": True,
|
||||
"owner_pid": owner_pid,
|
||||
"reasons": ["shadow owner pid is the current process"],
|
||||
}
|
||||
alive = pid_alive if pid_alive is not None else _pid_alive(owner_pid)
|
||||
if alive:
|
||||
reasons.append(
|
||||
f"shadow owner pid {owner_pid} observed alive; PID liveness is not "
|
||||
"ownership proof — owner evidence stays unknown (fail closed)"
|
||||
)
|
||||
return {
|
||||
"orphan_evidence": False,
|
||||
"owner_process_alive": None,
|
||||
"owner_pid": owner_pid,
|
||||
"reasons": reasons,
|
||||
}
|
||||
reasons.append(
|
||||
f"shadow owner pid {owner_pid} is dead: the daemon that recorded the "
|
||||
"matching session lease exited without teardown (crash orphan, #702)"
|
||||
)
|
||||
return {
|
||||
"orphan_evidence": True,
|
||||
"owner_process_alive": False,
|
||||
"owner_pid": owner_pid,
|
||||
"reasons": reasons,
|
||||
}
|
||||
|
||||
|
||||
def assess_mutation_lease_gate(
|
||||
*,
|
||||
pr_number: int,
|
||||
@@ -557,6 +689,7 @@ _HANDOFF_CLASSIFICATIONS = frozenset({
|
||||
"foreign_completed_superseded_head",
|
||||
"foreign_expired_superseded_head",
|
||||
"orphaned_owner_missing",
|
||||
"orphaned_expired_superseded_head",
|
||||
"ambiguous_conflicting_evidence",
|
||||
"instructed_lease_missing_with_replacement",
|
||||
"worktree_binding_mismatch",
|
||||
@@ -850,6 +983,29 @@ def assess_obsolete_reviewer_comment_lease_cleanup(
|
||||
classification = "foreign_expired_superseded_head"
|
||||
elif head_superseded and has_terminal and not past_expiry:
|
||||
classification = "foreign_completed_superseded_head"
|
||||
elif head_superseded and not has_terminal and past_expiry:
|
||||
# Crash-orphan shape (#702): the lease outlived its head with no
|
||||
# formal terminal review and has now passed canonical expiry, so
|
||||
# it no longer gates fresh acquisition. Guarded cleanup of the
|
||||
# ledger marker still demands provable owner-process-exit
|
||||
# evidence and a safe worktree — never inferred.
|
||||
classification = "orphaned_expired_superseded_head"
|
||||
if owner_process_alive is True:
|
||||
fail_closed_reasons.append(
|
||||
"lease past expiry on superseded head but owner process "
|
||||
"still alive; cleanup denied"
|
||||
)
|
||||
elif owner_process_alive is None:
|
||||
fail_closed_reasons.append(
|
||||
"expired superseded-head lease with no terminal review: "
|
||||
"cleanup requires provable owner-process-exit evidence "
|
||||
"(owner_process_alive=false); the expired lease no longer "
|
||||
"gates fresh acquisition"
|
||||
)
|
||||
elif worktree_clean is False:
|
||||
fail_closed_reasons.append(
|
||||
"owner process absent but worktree dirty; cleanup denied"
|
||||
)
|
||||
elif head_superseded and not has_terminal:
|
||||
classification = "ambiguous_conflicting_evidence"
|
||||
fail_closed_reasons.append(
|
||||
@@ -891,6 +1047,7 @@ def assess_obsolete_reviewer_comment_lease_cleanup(
|
||||
"foreign_expired_superseded_head",
|
||||
"foreign_expired_current_head",
|
||||
"orphaned_owner_missing",
|
||||
"orphaned_expired_superseded_head",
|
||||
}:
|
||||
fail_closed_reasons.append(
|
||||
"controller/recovery capability required "
|
||||
@@ -909,6 +1066,7 @@ def assess_obsolete_reviewer_comment_lease_cleanup(
|
||||
"foreign_completed_superseded_head",
|
||||
"foreign_expired_superseded_head",
|
||||
"orphaned_owner_missing",
|
||||
"orphaned_expired_superseded_head",
|
||||
"foreign_expired_current_head",
|
||||
}
|
||||
# foreign_expired_current_head needs orphan/clean worktree + authority.
|
||||
@@ -1181,6 +1339,30 @@ def diagnose_reviewer_pr_lease_handoff(
|
||||
"foreign lease pinned to superseded head with completed formal "
|
||||
"review; use guarded obsolete-lease cleanup (do not wait indefinitely)"
|
||||
)
|
||||
elif not is_own and head_superseded and not terminal and past_expiry:
|
||||
# Crash-orphan shape after canonical expiry (#702): the expired
|
||||
# marker no longer gates acquisition, so a fresh reviewer may
|
||||
# acquire at the current head. Guarded ledger cleanup becomes
|
||||
# available only with provable owner-exit evidence.
|
||||
classification = "orphaned_expired_superseded_head"
|
||||
if owner_process_alive is False and (
|
||||
worktree_clean is True or worktree_exists is False
|
||||
):
|
||||
next_action = NEXT_ACTION_CLEANUP_OBSOLETE_LEASE
|
||||
reasons.append(
|
||||
"expired superseded-head lease with no terminal review and "
|
||||
"provable owner-process exit (crash orphan, #702); guarded "
|
||||
"obsolete-lease cleanup eligible"
|
||||
)
|
||||
else:
|
||||
next_action = NEXT_ACTION_ACQUIRE
|
||||
reasons.append(
|
||||
"lease head superseded and lease past canonical expiry with "
|
||||
"no formal terminal review (crash orphan, #702); expired "
|
||||
"marker no longer gates acquisition — acquire a fresh lease "
|
||||
"at the current head; guarded cleanup needs "
|
||||
"owner_process_alive=false and a clean/absent worktree"
|
||||
)
|
||||
elif not is_own and head_superseded and not terminal:
|
||||
classification = "ambiguous_conflicting_evidence"
|
||||
next_action = NEXT_ACTION_WAIT
|
||||
|
||||
Reference in New Issue
Block a user