"""Sanctioned reconciliation for worktree bindings whose paths are missing (#970). When control-plane leases, session checkpoints, or issue locks hold a ``worktree_path`` pointing to a filesystem path that no longer exists on disk, this module provides a fail-closed, auditable classification and resolution workflow. Core principles: 1. **Multi-dimensional Correlation**: Correlates the binding with repository, host, branch, issue/PR, session, and lease state. 2. **Safety-first Distinction**: - Distinguishes a deleted worktree from a moved path, unavailable host/mount, live lease, active session, or temporary filesystem problem. - Host / mount failure (e.g. repo root or branches/ directory inaccessible) blocks retirement (``unavailable_host_or_mount``). - Moved worktree (found under another path in ``git worktree list``) blocks retirement and reports the new location (``moved_worktree_path``). - Live lease or active session/process blocks retirement (``live_lease_protected``, ``live_session_protected``). 3. **Atomic Pre-Mutation Re-Validation**: Immediately before committing a mutation, re-evaluates filesystem existence, git worktree list, host health, and lease status to prevent recreation races. 4. **Targeted Retirement**: Retires only the exact confirmed stale binding (clearing ``worktree_path`` on the lease/checkpoint/lock row and recording durable provenance) while preserving unrelated worktrees, leases, branches, and git metadata. 5. **Dry-Run & Operator Authorization**: Supports dry-run inspection and requires explicit cleanup authorization or reconciler workflow for mutation. 6. **Idempotency & Durable Proof**: Repeated execution on an already-retired binding is safe, idempotent, and produces durable audit evidence. """ from __future__ import annotations import json import os import subprocess from typing import Any, Mapping, Sequence import control_plane_db as cpd import issue_lock_store import lease_lifecycle from merged_cleanup_reconcile import list_local_worktrees # Classification vocabulary (#970 AC1-AC4). CLASS_CONFIRMED_STALE_DELETED = "confirmed_stale_deleted_worktree" CLASS_MOVED_WORKTREE = "moved_worktree_path" CLASS_UNAVAILABLE_HOST_MOUNT = "unavailable_host_or_mount" CLASS_LIVE_LEASE_PROTECTED = "live_lease_protected" CLASS_LIVE_SESSION_PROTECTED = "live_session_protected" CLASS_PRESENT_VALID = "present_valid_worktree" CLASS_ALREADY_RETIRED = "already_retired_binding" ALL_CLASSIFICATIONS = frozenset({ CLASS_CONFIRMED_STALE_DELETED, CLASS_MOVED_WORKTREE, CLASS_UNAVAILABLE_HOST_MOUNT, CLASS_LIVE_LEASE_PROTECTED, CLASS_LIVE_SESSION_PROTECTED, CLASS_PRESENT_VALID, CLASS_ALREADY_RETIRED, }) RETIRE_ELIGIBLE_CLASSES = frozenset({ CLASS_CONFIRMED_STALE_DELETED, }) def _norm_path(p: str | None) -> str: if not p or not str(p).strip(): return "" try: return os.path.realpath(os.path.abspath(str(p).strip())) except Exception: return str(p).strip() def check_host_mount_health(project_root: str) -> dict[str, Any]: """Check if the repository root and branches/ mount are healthy and accessible.""" root = _norm_path(project_root) if not root or not os.path.isdir(root): return { "healthy": False, "reason": f"project_root '{project_root}' does not exist or is not a directory", } # Verify project_root is inside a valid git repository try: res = subprocess.run( ["git", "-C", root, "rev-parse", "--git-dir"], capture_output=True, text=True, check=False, ) if res.returncode != 0: return { "healthy": False, "reason": f"project_root '{root}' is not a valid git repository: {res.stderr.strip()}", } except Exception as exc: return { "healthy": False, "reason": f"git execution failed at project_root '{root}': {exc}", } branches_dir = os.path.join(root, "branches") if os.path.exists(branches_dir) and not os.path.isdir(branches_dir): return { "healthy": False, "reason": f"branches path '{branches_dir}' exists but is not a directory", } return {"healthy": True, "project_root": root, "branches_dir": branches_dir} def find_moved_worktree_path( project_root: str, recorded_path: str, branch: str | None = None, ) -> str | None: """Check if a missing recorded worktree path is actually present at another location. Returns the new path if found, or None if truly absent. """ norm_rec = _norm_path(recorded_path) if not norm_rec: return None wt_list = [] try: wt_list = list_local_worktrees(project_root) except Exception: wt_list = [] for entry in wt_list: p = _norm_path(entry.get("path")) b = entry.get("branch") if not p: continue # If git worktree list shows a worktree matching the branch at a different path if branch and b and b.strip() == branch.strip() and p != norm_rec: if os.path.exists(p): return p # If the basename matches and path exists if os.path.basename(p) == os.path.basename(norm_rec) and p != norm_rec: if os.path.exists(p): return p # Check potential filesystem candidates under branches/ branches_dir = os.path.join(_norm_path(project_root), "branches") if os.path.isdir(branches_dir): candidates_to_check: list[str] = [] if branch: candidates_to_check.append(os.path.join(branches_dir, branch.replace("/", "-"))) candidates_to_check.append(os.path.join(branches_dir, branch)) if recorded_path: candidates_to_check.append(os.path.join(branches_dir, os.path.basename(recorded_path))) for cand in candidates_to_check: norm_cand = _norm_path(cand) if norm_cand and norm_cand != norm_rec and os.path.isdir(norm_cand): return norm_cand return None def classify_worktree_binding( *, recorded_path: str, lease_status: str | None = None, lease_phase: str | None = None, owner_pid: int | None = None, session_id: str | None = None, session_active: bool = False, branch: str | None = None, project_root: str, host_health: dict[str, Any] | None = None, path_exists_override: bool | None = None, ) -> dict[str, Any]: """Classify a single worktree binding record for missing-path safety. Fail-closed: returns detailed reasons and classification. """ norm_p = _norm_path(recorded_path) reasons: list[str] = [] if not norm_p: return { "classification": CLASS_ALREADY_RETIRED, "retire_eligible": False, "recorded_path": "", "reasons": ["worktree_path is empty (already retired/unbound)"], } # Check 1: Host / mount health hh = host_health or check_host_mount_health(project_root) if not hh.get("healthy"): reasons.append( f"host or filesystem mount unavailable: {hh.get('reason')}; " "refusing to classify missing path as deleted (fail closed)" ) return { "classification": CLASS_UNAVAILABLE_HOST_MOUNT, "retire_eligible": False, "recorded_path": norm_p, "reasons": reasons, "host_health": hh, } # Check 2: Path existence exists = ( path_exists_override if path_exists_override is not None else os.path.exists(norm_p) ) if exists: return { "classification": CLASS_PRESENT_VALID, "retire_eligible": False, "recorded_path": norm_p, "reasons": [f"worktree path '{norm_p}' exists on disk"], } # Path is missing on disk. Run safety checks before declaring confirmed stale. # Check 3: Moved path moved_path = find_moved_worktree_path(project_root, norm_p, branch=branch) if moved_path: reasons.append( f"recorded path '{norm_p}' is missing, but worktree for branch '{branch}' " f"was found moved to '{moved_path}'" ) return { "classification": CLASS_MOVED_WORKTREE, "retire_eligible": False, "recorded_path": norm_p, "moved_to_path": moved_path, "reasons": reasons, } # Check 4: Live lease protection st = (lease_status or "").strip().lower() pid_alive = lease_lifecycle.is_process_alive(owner_pid) if owner_pid else False if st == "active" and (pid_alive or owner_pid is None): reasons.append( f"lease is status='active' (owner_pid={owner_pid}, alive={pid_alive}); " "active lease protects worktree binding" ) return { "classification": CLASS_LIVE_LEASE_PROTECTED, "retire_eligible": False, "recorded_path": norm_p, "reasons": reasons, } # Check 5: Live session / process protection if session_active and pid_alive: reasons.append( f"session '{session_id}' is active with live process PID {owner_pid}; " "live session protects worktree binding" ) return { "classification": CLASS_LIVE_SESSION_PROTECTED, "retire_eligible": False, "recorded_path": norm_p, "reasons": reasons, } # Confirmed stale deleted reasons.append( f"worktree path '{norm_p}' does not exist on disk, host is healthy, " "no moved path was found, and no active lease or live process references it" ) return { "classification": CLASS_CONFIRMED_STALE_DELETED, "retire_eligible": True, "recorded_path": norm_p, "reasons": reasons, } def audit_missing_worktree_bindings( db: cpd.ControlPlaneDB | None = None, *, project_root: str | None = None, remote: str = "prgs", org: str | None = None, repo: str | None = None, host: str | None = None, ) -> dict[str, Any]: """Audit all recorded worktree bindings and classify missing-path entries. Read-only / side-effect free. """ root = _norm_path(project_root or os.getcwd()) hh = check_host_mount_health(root) cp_db = db or cpd.ControlPlaneDB() bindings: list[dict[str, Any]] = [] # 1. Audit control-plane leases try: leases_res = lease_lifecycle.list_active_leases( cp_db, remote=remote, org=org, repo=repo, include_non_active=True, limit=500, ) for L in leases_res.get("leases") or []: wt = (L.get("worktree_path") or "").strip() if not wt: continue owner_pid = L.get("owner_pid") or L.get("session_pid") sess_id = L.get("session_id") br = L.get("branch") if not br and isinstance(L.get("provenance"), dict): br = L["provenance"].get("branch") cls_info = classify_worktree_binding( recorded_path=wt, lease_status=L.get("status"), lease_phase=L.get("phase"), owner_pid=owner_pid, session_id=sess_id, branch=br, project_root=root, host_health=hh, ) bindings.append({ "source": "lease", "lease_id": L.get("lease_id"), "session_id": sess_id, "work_kind": L.get("work_kind"), "work_number": L.get("work_number"), "branch": br, "worktree_path": wt, "lease_status": L.get("status"), "lease_phase": L.get("phase"), "owner_pid": owner_pid, "remote": L.get("remote") or remote, "org": L.get("org") or org, "repo": L.get("repo") or repo, "host": L.get("host") or host, **cls_info, }) except Exception as exc: # noqa: BLE001 bindings.append({ "source": "lease_query_error", "error": str(exc), }) # 2. Audit session checkpoints try: checkpoints = cp_db.list_session_checkpoints(limit=500) for cp in checkpoints: wt = (cp.get("worktree_path") or "").strip() if not wt: continue sess_id = cp.get("session_id") cls_info = classify_worktree_binding( recorded_path=wt, session_id=sess_id, branch=cp.get("branch"), project_root=root, host_health=hh, ) bindings.append({ "source": "session_checkpoint", "checkpoint_id": cp.get("checkpoint_id"), "session_id": sess_id, "work_kind": cp.get("work_kind"), "work_number": cp.get("work_number"), "branch": cp.get("branch"), "worktree_path": wt, "status": cp.get("status"), "remote": cp.get("remote") or remote, "org": cp.get("org") or org, "repo": cp.get("repo") or repo, **cls_info, }) except Exception: # noqa: BLE001 pass missing_bindings = [ b for b in bindings if b.get("classification") != CLASS_PRESENT_VALID ] confirmed_stale = [ b for b in missing_bindings if b.get("classification") == CLASS_CONFIRMED_STALE_DELETED ] moved = [ b for b in missing_bindings if b.get("classification") == CLASS_MOVED_WORKTREE ] live_protected = [ b for b in missing_bindings if b.get("classification") in (CLASS_LIVE_LEASE_PROTECTED, CLASS_LIVE_SESSION_PROTECTED) ] unavailable_host = [ b for b in missing_bindings if b.get("classification") == CLASS_UNAVAILABLE_HOST_MOUNT ] return { "project_root": root, "host_mount_healthy": hh.get("healthy", False), "host_health_reason": hh.get("reason"), "total_bindings_audited": len(bindings), "present_count": len(bindings) - len(missing_bindings), "missing_count": len(missing_bindings), "confirmed_stale_count": len(confirmed_stale), "moved_count": len(moved), "live_protected_count": len(live_protected), "unavailable_host_count": len(unavailable_host), "missing_bindings": missing_bindings, "confirmed_stale_candidates": confirmed_stale, "worktrees_dimension_resolved": len(missing_bindings) == 0, } def resolve_missing_worktree_binding( db: cpd.ControlPlaneDB | None = None, *, binding: dict[str, Any], dry_run: bool = True, operator_authorized: bool = False, workflow_authorized: bool = False, project_root: str | None = None, ) -> dict[str, Any]: """Safely resolve (retire) one confirmed stale missing worktree binding (#970). Pre-mutation re-validation runs immediately before mutation to prevent races. """ root = _norm_path(project_root or os.getcwd()) cp_db = db or cpd.ControlPlaneDB() rec_path = binding.get("worktree_path") or binding.get("recorded_path") or "" if not rec_path: return { "success": True, "performed": False, "outcome": "already_retired", "message": "worktree_path is already empty (no action needed)", } # 1. Authorization check authorized = operator_authorized or workflow_authorized if not authorized and not dry_run: return { "success": False, "performed": False, "reason": "authorization_required", "message": "Mutation refused: operator_authorized or workflow_authorized is required to retire a stale binding", } # 2. Immediate pre-mutation re-validation reval = classify_worktree_binding( recorded_path=rec_path, lease_status=binding.get("lease_status"), lease_phase=binding.get("lease_phase"), owner_pid=binding.get("owner_pid"), session_id=binding.get("session_id"), branch=binding.get("branch"), project_root=root, ) if reval.get("classification") != CLASS_CONFIRMED_STALE_DELETED: return { "success": False, "performed": False, "reason": "revalidation_failed", "revalidation": reval, "message": f"Pre-mutation re-validation failed: binding classified as '{reval.get('classification')}' instead of '{CLASS_CONFIRMED_STALE_DELETED}'", } source = binding.get("source", "lease") lease_id = binding.get("lease_id") session_id = binding.get("session_id") checkpoint_id = binding.get("checkpoint_id") planned_action = { "source": source, "lease_id": lease_id, "session_id": session_id, "checkpoint_id": checkpoint_id, "retired_worktree_path": rec_path, "classification": CLASS_CONFIRMED_STALE_DELETED, } if dry_run: return { "success": True, "performed": False, "dry_run": True, "action": "retire_stale_binding", "planned_action": planned_action, "message": f"Dry-run: would retire stale missing worktree_path '{rec_path}' for {source} (lease={lease_id})", } # 3. Apply mode: execute targeted retirement audit_proof: dict[str, Any] = { "retired_worktree_path": rec_path, "source": source, "lease_id": lease_id, "session_id": session_id, "checkpoint_id": checkpoint_id, "classification": CLASS_CONFIRMED_STALE_DELETED, } if source == "lease" and lease_id: ret_proof = cp_db.retire_lease_worktree_path( lease_id, expected_path=rec_path, reason="missing_worktree_path_retired_by_reconciler", ) audit_proof["db_lease_update"] = ret_proof if source == "session_checkpoint" and (checkpoint_id or session_id): ret_cp = cp_db.retire_session_checkpoint_worktree_path( session_id=session_id or "", checkpoint_id=checkpoint_id, expected_path=rec_path, reason="missing_worktree_path_retired_by_reconciler", ) audit_proof["db_checkpoint_update"] = ret_cp # Also clean up any matching lease row if checkpoint was supplied or vice versa if lease_id and source != "lease": try: cp_db.retire_lease_worktree_path(lease_id, expected_path=rec_path) except Exception: # noqa: BLE001 pass return { "success": True, "performed": True, "dry_run": False, "action": "retired_stale_binding", "audit_proof": audit_proof, "message": f"Successfully retired stale missing worktree_path '{rec_path}' for lease {lease_id}", } def reconcile_missing_worktree_bindings( db: cpd.ControlPlaneDB | None = None, *, project_root: str | None = None, remote: str = "prgs", org: str | None = None, repo: str | None = None, host: str | None = None, dry_run: bool = True, operator_authorized: bool = False, workflow_authorized: bool = False, ) -> dict[str, Any]: """Audit and safely reconcile missing worktree path bindings (#970). Audits all recorded bindings, identifies confirmed stale missing paths, retires eligible bindings (when dry_run=False and authorized), and returns durable before/after audit evidence. """ cp_db = db or cpd.ControlPlaneDB() root = _norm_path(project_root or os.getcwd()) initial_audit = audit_missing_worktree_bindings( cp_db, project_root=root, remote=remote, org=org, repo=repo, host=host, ) candidates = initial_audit.get("confirmed_stale_candidates") or [] resolutions: list[dict[str, Any]] = [] for candidate in candidates: res = resolve_missing_worktree_binding( cp_db, binding=candidate, dry_run=dry_run, operator_authorized=operator_authorized, workflow_authorized=workflow_authorized, project_root=root, ) resolutions.append(res) post_audit = audit_missing_worktree_bindings( cp_db, project_root=root, remote=remote, org=org, repo=repo, host=host, ) return { "success": True, "dry_run": dry_run, "operator_authorized": operator_authorized, "workflow_authorized": workflow_authorized, "before_audit": initial_audit, "resolutions": resolutions, "after_audit": post_audit, "worktrees_dimension_resolved": post_audit.get("worktrees_dimension_resolved", False), "stale_candidates_count": len(candidates), "resolved_count": sum(1 for r in resolutions if r.get("performed") or (dry_run and r.get("success"))), }