612 lines
21 KiB
Python
612 lines
21 KiB
Python
"""Sanctioned reconciliation for worktree bindings whose paths are missing (#970).
|
|
|
|
When control-plane leases, session checkpoints, or issue locks hold a
|
|
``worktree_path`` pointing to a filesystem path that no longer exists on disk,
|
|
this module provides a fail-closed, auditable classification and resolution
|
|
workflow.
|
|
|
|
Core principles:
|
|
1. **Multi-dimensional Correlation**:
|
|
Correlates the binding with repository, host, branch, issue/PR, session,
|
|
and lease state.
|
|
2. **Safety-first Distinction**:
|
|
- Distinguishes a deleted worktree from a moved path, unavailable host/mount,
|
|
live lease, active session, or temporary filesystem problem.
|
|
- Host / mount failure (e.g. repo root or branches/ directory inaccessible)
|
|
blocks retirement (``unavailable_host_or_mount``).
|
|
- Moved worktree (found under another path in ``git worktree list``)
|
|
blocks retirement and reports the new location (``moved_worktree_path``).
|
|
- Live lease or active session/process blocks retirement (``live_lease_protected``,
|
|
``live_session_protected``).
|
|
3. **Atomic Pre-Mutation Re-Validation**:
|
|
Immediately before committing a mutation, re-evaluates filesystem existence,
|
|
git worktree list, host health, and lease status to prevent recreation races.
|
|
4. **Targeted Retirement**:
|
|
Retires only the exact confirmed stale binding (clearing ``worktree_path`` on
|
|
the lease/checkpoint/lock row and recording durable provenance) while
|
|
preserving unrelated worktrees, leases, branches, and git metadata.
|
|
5. **Dry-Run & Operator Authorization**:
|
|
Supports dry-run inspection and requires explicit cleanup authorization or
|
|
reconciler workflow for mutation.
|
|
6. **Idempotency & Durable Proof**:
|
|
Repeated execution on an already-retired binding is safe, idempotent, and
|
|
produces durable audit evidence.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
from typing import Any, Mapping, Sequence
|
|
|
|
import control_plane_db as cpd
|
|
import issue_lock_store
|
|
import lease_lifecycle
|
|
from merged_cleanup_reconcile import list_local_worktrees
|
|
|
|
# Classification vocabulary (#970 AC1-AC4).
|
|
CLASS_CONFIRMED_STALE_DELETED = "confirmed_stale_deleted_worktree"
|
|
CLASS_MOVED_WORKTREE = "moved_worktree_path"
|
|
CLASS_UNAVAILABLE_HOST_MOUNT = "unavailable_host_or_mount"
|
|
CLASS_LIVE_LEASE_PROTECTED = "live_lease_protected"
|
|
CLASS_LIVE_SESSION_PROTECTED = "live_session_protected"
|
|
CLASS_PRESENT_VALID = "present_valid_worktree"
|
|
CLASS_ALREADY_RETIRED = "already_retired_binding"
|
|
|
|
ALL_CLASSIFICATIONS = frozenset({
|
|
CLASS_CONFIRMED_STALE_DELETED,
|
|
CLASS_MOVED_WORKTREE,
|
|
CLASS_UNAVAILABLE_HOST_MOUNT,
|
|
CLASS_LIVE_LEASE_PROTECTED,
|
|
CLASS_LIVE_SESSION_PROTECTED,
|
|
CLASS_PRESENT_VALID,
|
|
CLASS_ALREADY_RETIRED,
|
|
})
|
|
|
|
RETIRE_ELIGIBLE_CLASSES = frozenset({
|
|
CLASS_CONFIRMED_STALE_DELETED,
|
|
})
|
|
|
|
|
|
def _norm_path(p: str | None) -> str:
|
|
if not p or not str(p).strip():
|
|
return ""
|
|
try:
|
|
return os.path.realpath(os.path.abspath(str(p).strip()))
|
|
except Exception:
|
|
return str(p).strip()
|
|
|
|
|
|
def check_host_mount_health(project_root: str) -> dict[str, Any]:
|
|
"""Check if the repository root and branches/ mount are healthy and accessible."""
|
|
root = _norm_path(project_root)
|
|
if not root or not os.path.isdir(root):
|
|
return {
|
|
"healthy": False,
|
|
"reason": f"project_root '{project_root}' does not exist or is not a directory",
|
|
}
|
|
|
|
# Verify project_root is inside a valid git repository
|
|
try:
|
|
res = subprocess.run(
|
|
["git", "-C", root, "rev-parse", "--git-dir"],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
if res.returncode != 0:
|
|
return {
|
|
"healthy": False,
|
|
"reason": f"project_root '{root}' is not a valid git repository: {res.stderr.strip()}",
|
|
}
|
|
except Exception as exc:
|
|
return {
|
|
"healthy": False,
|
|
"reason": f"git execution failed at project_root '{root}': {exc}",
|
|
}
|
|
|
|
branches_dir = os.path.join(root, "branches")
|
|
if os.path.exists(branches_dir) and not os.path.isdir(branches_dir):
|
|
return {
|
|
"healthy": False,
|
|
"reason": f"branches path '{branches_dir}' exists but is not a directory",
|
|
}
|
|
|
|
return {"healthy": True, "project_root": root, "branches_dir": branches_dir}
|
|
|
|
|
|
def find_moved_worktree_path(
|
|
project_root: str,
|
|
recorded_path: str,
|
|
branch: str | None = None,
|
|
) -> str | None:
|
|
"""Check if a missing recorded worktree path is actually present at another location.
|
|
|
|
Returns the new path if found, or None if truly absent.
|
|
"""
|
|
norm_rec = _norm_path(recorded_path)
|
|
if not norm_rec:
|
|
return None
|
|
|
|
wt_list = []
|
|
try:
|
|
wt_list = list_local_worktrees(project_root)
|
|
except Exception:
|
|
wt_list = []
|
|
|
|
for entry in wt_list:
|
|
p = _norm_path(entry.get("path"))
|
|
b = entry.get("branch")
|
|
if not p:
|
|
continue
|
|
# If git worktree list shows a worktree matching the branch at a different path
|
|
if branch and b and b.strip() == branch.strip() and p != norm_rec:
|
|
if os.path.exists(p):
|
|
return p
|
|
# If the basename matches and path exists
|
|
if os.path.basename(p) == os.path.basename(norm_rec) and p != norm_rec:
|
|
if os.path.exists(p):
|
|
return p
|
|
|
|
# Check potential filesystem candidates under branches/
|
|
branches_dir = os.path.join(_norm_path(project_root), "branches")
|
|
if os.path.isdir(branches_dir):
|
|
candidates_to_check: list[str] = []
|
|
if branch:
|
|
candidates_to_check.append(os.path.join(branches_dir, branch.replace("/", "-")))
|
|
candidates_to_check.append(os.path.join(branches_dir, branch))
|
|
if recorded_path:
|
|
candidates_to_check.append(os.path.join(branches_dir, os.path.basename(recorded_path)))
|
|
|
|
for cand in candidates_to_check:
|
|
norm_cand = _norm_path(cand)
|
|
if norm_cand and norm_cand != norm_rec and os.path.isdir(norm_cand):
|
|
return norm_cand
|
|
|
|
return None
|
|
|
|
|
|
def classify_worktree_binding(
|
|
*,
|
|
recorded_path: str,
|
|
lease_status: str | None = None,
|
|
lease_phase: str | None = None,
|
|
owner_pid: int | None = None,
|
|
session_id: str | None = None,
|
|
session_active: bool = False,
|
|
branch: str | None = None,
|
|
project_root: str,
|
|
host_health: dict[str, Any] | None = None,
|
|
path_exists_override: bool | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Classify a single worktree binding record for missing-path safety.
|
|
|
|
Fail-closed: returns detailed reasons and classification.
|
|
"""
|
|
norm_p = _norm_path(recorded_path)
|
|
reasons: list[str] = []
|
|
|
|
if not norm_p:
|
|
return {
|
|
"classification": CLASS_ALREADY_RETIRED,
|
|
"retire_eligible": False,
|
|
"recorded_path": "",
|
|
"reasons": ["worktree_path is empty (already retired/unbound)"],
|
|
}
|
|
|
|
# Check 1: Host / mount health
|
|
hh = host_health or check_host_mount_health(project_root)
|
|
if not hh.get("healthy"):
|
|
reasons.append(
|
|
f"host or filesystem mount unavailable: {hh.get('reason')}; "
|
|
"refusing to classify missing path as deleted (fail closed)"
|
|
)
|
|
return {
|
|
"classification": CLASS_UNAVAILABLE_HOST_MOUNT,
|
|
"retire_eligible": False,
|
|
"recorded_path": norm_p,
|
|
"reasons": reasons,
|
|
"host_health": hh,
|
|
}
|
|
|
|
# Check 2: Path existence
|
|
exists = (
|
|
path_exists_override
|
|
if path_exists_override is not None
|
|
else os.path.exists(norm_p)
|
|
)
|
|
|
|
if exists:
|
|
return {
|
|
"classification": CLASS_PRESENT_VALID,
|
|
"retire_eligible": False,
|
|
"recorded_path": norm_p,
|
|
"reasons": [f"worktree path '{norm_p}' exists on disk"],
|
|
}
|
|
|
|
# Path is missing on disk. Run safety checks before declaring confirmed stale.
|
|
|
|
# Check 3: Moved path
|
|
moved_path = find_moved_worktree_path(project_root, norm_p, branch=branch)
|
|
if moved_path:
|
|
reasons.append(
|
|
f"recorded path '{norm_p}' is missing, but worktree for branch '{branch}' "
|
|
f"was found moved to '{moved_path}'"
|
|
)
|
|
return {
|
|
"classification": CLASS_MOVED_WORKTREE,
|
|
"retire_eligible": False,
|
|
"recorded_path": norm_p,
|
|
"moved_to_path": moved_path,
|
|
"reasons": reasons,
|
|
}
|
|
|
|
# Check 4: Live lease protection
|
|
st = (lease_status or "").strip().lower()
|
|
pid_alive = lease_lifecycle.is_process_alive(owner_pid) if owner_pid else False
|
|
if st == "active" and (pid_alive or owner_pid is None):
|
|
reasons.append(
|
|
f"lease is status='active' (owner_pid={owner_pid}, alive={pid_alive}); "
|
|
"active lease protects worktree binding"
|
|
)
|
|
return {
|
|
"classification": CLASS_LIVE_LEASE_PROTECTED,
|
|
"retire_eligible": False,
|
|
"recorded_path": norm_p,
|
|
"reasons": reasons,
|
|
}
|
|
|
|
# Check 5: Live session / process protection
|
|
if session_active and pid_alive:
|
|
reasons.append(
|
|
f"session '{session_id}' is active with live process PID {owner_pid}; "
|
|
"live session protects worktree binding"
|
|
)
|
|
return {
|
|
"classification": CLASS_LIVE_SESSION_PROTECTED,
|
|
"retire_eligible": False,
|
|
"recorded_path": norm_p,
|
|
"reasons": reasons,
|
|
}
|
|
|
|
# Confirmed stale deleted
|
|
reasons.append(
|
|
f"worktree path '{norm_p}' does not exist on disk, host is healthy, "
|
|
"no moved path was found, and no active lease or live process references it"
|
|
)
|
|
return {
|
|
"classification": CLASS_CONFIRMED_STALE_DELETED,
|
|
"retire_eligible": True,
|
|
"recorded_path": norm_p,
|
|
"reasons": reasons,
|
|
}
|
|
|
|
|
|
def audit_missing_worktree_bindings(
|
|
db: cpd.ControlPlaneDB | None = None,
|
|
*,
|
|
project_root: str | None = None,
|
|
remote: str = "prgs",
|
|
org: str | None = None,
|
|
repo: str | None = None,
|
|
host: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Audit all recorded worktree bindings and classify missing-path entries.
|
|
|
|
Read-only / side-effect free.
|
|
"""
|
|
root = _norm_path(project_root or os.getcwd())
|
|
hh = check_host_mount_health(root)
|
|
cp_db = db or cpd.ControlPlaneDB()
|
|
|
|
bindings: list[dict[str, Any]] = []
|
|
|
|
# 1. Audit control-plane leases
|
|
try:
|
|
leases_res = lease_lifecycle.list_active_leases(
|
|
cp_db,
|
|
remote=remote,
|
|
org=org,
|
|
repo=repo,
|
|
include_non_active=True,
|
|
limit=500,
|
|
)
|
|
for L in leases_res.get("leases") or []:
|
|
wt = (L.get("worktree_path") or "").strip()
|
|
if not wt:
|
|
continue
|
|
owner_pid = L.get("owner_pid") or L.get("session_pid")
|
|
sess_id = L.get("session_id")
|
|
br = L.get("branch")
|
|
if not br and isinstance(L.get("provenance"), dict):
|
|
br = L["provenance"].get("branch")
|
|
|
|
cls_info = classify_worktree_binding(
|
|
recorded_path=wt,
|
|
lease_status=L.get("status"),
|
|
lease_phase=L.get("phase"),
|
|
owner_pid=owner_pid,
|
|
session_id=sess_id,
|
|
branch=br,
|
|
project_root=root,
|
|
host_health=hh,
|
|
)
|
|
|
|
bindings.append({
|
|
"source": "lease",
|
|
"lease_id": L.get("lease_id"),
|
|
"session_id": sess_id,
|
|
"work_kind": L.get("work_kind"),
|
|
"work_number": L.get("work_number"),
|
|
"branch": br,
|
|
"worktree_path": wt,
|
|
"lease_status": L.get("status"),
|
|
"lease_phase": L.get("phase"),
|
|
"owner_pid": owner_pid,
|
|
"remote": L.get("remote") or remote,
|
|
"org": L.get("org") or org,
|
|
"repo": L.get("repo") or repo,
|
|
"host": L.get("host") or host,
|
|
**cls_info,
|
|
})
|
|
except Exception as exc: # noqa: BLE001
|
|
bindings.append({
|
|
"source": "lease_query_error",
|
|
"error": str(exc),
|
|
})
|
|
|
|
# 2. Audit session checkpoints
|
|
try:
|
|
checkpoints = cp_db.list_session_checkpoints(limit=500)
|
|
for cp in checkpoints:
|
|
wt = (cp.get("worktree_path") or "").strip()
|
|
if not wt:
|
|
continue
|
|
sess_id = cp.get("session_id")
|
|
cls_info = classify_worktree_binding(
|
|
recorded_path=wt,
|
|
session_id=sess_id,
|
|
branch=cp.get("branch"),
|
|
project_root=root,
|
|
host_health=hh,
|
|
)
|
|
bindings.append({
|
|
"source": "session_checkpoint",
|
|
"checkpoint_id": cp.get("checkpoint_id"),
|
|
"session_id": sess_id,
|
|
"work_kind": cp.get("work_kind"),
|
|
"work_number": cp.get("work_number"),
|
|
"branch": cp.get("branch"),
|
|
"worktree_path": wt,
|
|
"status": cp.get("status"),
|
|
"remote": cp.get("remote") or remote,
|
|
"org": cp.get("org") or org,
|
|
"repo": cp.get("repo") or repo,
|
|
**cls_info,
|
|
})
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
|
|
missing_bindings = [
|
|
b for b in bindings if b.get("classification") != CLASS_PRESENT_VALID
|
|
]
|
|
confirmed_stale = [
|
|
b for b in missing_bindings if b.get("classification") == CLASS_CONFIRMED_STALE_DELETED
|
|
]
|
|
moved = [
|
|
b for b in missing_bindings if b.get("classification") == CLASS_MOVED_WORKTREE
|
|
]
|
|
live_protected = [
|
|
b for b in missing_bindings if b.get("classification") in (CLASS_LIVE_LEASE_PROTECTED, CLASS_LIVE_SESSION_PROTECTED)
|
|
]
|
|
unavailable_host = [
|
|
b for b in missing_bindings if b.get("classification") == CLASS_UNAVAILABLE_HOST_MOUNT
|
|
]
|
|
|
|
return {
|
|
"project_root": root,
|
|
"host_mount_healthy": hh.get("healthy", False),
|
|
"host_health_reason": hh.get("reason"),
|
|
"total_bindings_audited": len(bindings),
|
|
"present_count": len(bindings) - len(missing_bindings),
|
|
"missing_count": len(missing_bindings),
|
|
"confirmed_stale_count": len(confirmed_stale),
|
|
"moved_count": len(moved),
|
|
"live_protected_count": len(live_protected),
|
|
"unavailable_host_count": len(unavailable_host),
|
|
"missing_bindings": missing_bindings,
|
|
"confirmed_stale_candidates": confirmed_stale,
|
|
"worktrees_dimension_resolved": len(missing_bindings) == 0,
|
|
}
|
|
|
|
|
|
def resolve_missing_worktree_binding(
|
|
db: cpd.ControlPlaneDB | None = None,
|
|
*,
|
|
binding: dict[str, Any],
|
|
dry_run: bool = True,
|
|
operator_authorized: bool = False,
|
|
workflow_authorized: bool = False,
|
|
project_root: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Safely resolve (retire) one confirmed stale missing worktree binding (#970).
|
|
|
|
Pre-mutation re-validation runs immediately before mutation to prevent races.
|
|
"""
|
|
root = _norm_path(project_root or os.getcwd())
|
|
cp_db = db or cpd.ControlPlaneDB()
|
|
|
|
rec_path = binding.get("worktree_path") or binding.get("recorded_path") or ""
|
|
if not rec_path:
|
|
return {
|
|
"success": True,
|
|
"performed": False,
|
|
"outcome": "already_retired",
|
|
"message": "worktree_path is already empty (no action needed)",
|
|
}
|
|
|
|
# 1. Authorization check
|
|
authorized = operator_authorized or workflow_authorized
|
|
if not authorized and not dry_run:
|
|
return {
|
|
"success": False,
|
|
"performed": False,
|
|
"reason": "authorization_required",
|
|
"message": "Mutation refused: operator_authorized or workflow_authorized is required to retire a stale binding",
|
|
}
|
|
|
|
# 2. Immediate pre-mutation re-validation
|
|
reval = classify_worktree_binding(
|
|
recorded_path=rec_path,
|
|
lease_status=binding.get("lease_status"),
|
|
lease_phase=binding.get("lease_phase"),
|
|
owner_pid=binding.get("owner_pid"),
|
|
session_id=binding.get("session_id"),
|
|
branch=binding.get("branch"),
|
|
project_root=root,
|
|
)
|
|
|
|
if reval.get("classification") != CLASS_CONFIRMED_STALE_DELETED:
|
|
return {
|
|
"success": False,
|
|
"performed": False,
|
|
"reason": "revalidation_failed",
|
|
"revalidation": reval,
|
|
"message": f"Pre-mutation re-validation failed: binding classified as '{reval.get('classification')}' instead of '{CLASS_CONFIRMED_STALE_DELETED}'",
|
|
}
|
|
|
|
source = binding.get("source", "lease")
|
|
lease_id = binding.get("lease_id")
|
|
session_id = binding.get("session_id")
|
|
checkpoint_id = binding.get("checkpoint_id")
|
|
|
|
planned_action = {
|
|
"source": source,
|
|
"lease_id": lease_id,
|
|
"session_id": session_id,
|
|
"checkpoint_id": checkpoint_id,
|
|
"retired_worktree_path": rec_path,
|
|
"classification": CLASS_CONFIRMED_STALE_DELETED,
|
|
}
|
|
|
|
if dry_run:
|
|
return {
|
|
"success": True,
|
|
"performed": False,
|
|
"dry_run": True,
|
|
"action": "retire_stale_binding",
|
|
"planned_action": planned_action,
|
|
"message": f"Dry-run: would retire stale missing worktree_path '{rec_path}' for {source} (lease={lease_id})",
|
|
}
|
|
|
|
# 3. Apply mode: execute targeted retirement
|
|
audit_proof: dict[str, Any] = {
|
|
"retired_worktree_path": rec_path,
|
|
"source": source,
|
|
"lease_id": lease_id,
|
|
"session_id": session_id,
|
|
"checkpoint_id": checkpoint_id,
|
|
"classification": CLASS_CONFIRMED_STALE_DELETED,
|
|
}
|
|
|
|
if source == "lease" and lease_id:
|
|
ret_proof = cp_db.retire_lease_worktree_path(
|
|
lease_id,
|
|
expected_path=rec_path,
|
|
reason="missing_worktree_path_retired_by_reconciler",
|
|
)
|
|
audit_proof["db_lease_update"] = ret_proof
|
|
|
|
if source == "session_checkpoint" and (checkpoint_id or session_id):
|
|
ret_cp = cp_db.retire_session_checkpoint_worktree_path(
|
|
session_id=session_id or "",
|
|
checkpoint_id=checkpoint_id,
|
|
expected_path=rec_path,
|
|
reason="missing_worktree_path_retired_by_reconciler",
|
|
)
|
|
audit_proof["db_checkpoint_update"] = ret_cp
|
|
|
|
# Also clean up any matching lease row if checkpoint was supplied or vice versa
|
|
if lease_id and source != "lease":
|
|
try:
|
|
cp_db.retire_lease_worktree_path(lease_id, expected_path=rec_path)
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
|
|
return {
|
|
"success": True,
|
|
"performed": True,
|
|
"dry_run": False,
|
|
"action": "retired_stale_binding",
|
|
"audit_proof": audit_proof,
|
|
"message": f"Successfully retired stale missing worktree_path '{rec_path}' for lease {lease_id}",
|
|
}
|
|
|
|
|
|
def reconcile_missing_worktree_bindings(
|
|
db: cpd.ControlPlaneDB | None = None,
|
|
*,
|
|
project_root: str | None = None,
|
|
remote: str = "prgs",
|
|
org: str | None = None,
|
|
repo: str | None = None,
|
|
host: str | None = None,
|
|
dry_run: bool = True,
|
|
operator_authorized: bool = False,
|
|
workflow_authorized: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Audit and safely reconcile missing worktree path bindings (#970).
|
|
|
|
Audits all recorded bindings, identifies confirmed stale missing paths,
|
|
retires eligible bindings (when dry_run=False and authorized), and returns
|
|
durable before/after audit evidence.
|
|
"""
|
|
cp_db = db or cpd.ControlPlaneDB()
|
|
root = _norm_path(project_root or os.getcwd())
|
|
|
|
initial_audit = audit_missing_worktree_bindings(
|
|
cp_db,
|
|
project_root=root,
|
|
remote=remote,
|
|
org=org,
|
|
repo=repo,
|
|
host=host,
|
|
)
|
|
|
|
candidates = initial_audit.get("confirmed_stale_candidates") or []
|
|
resolutions: list[dict[str, Any]] = []
|
|
|
|
for candidate in candidates:
|
|
res = resolve_missing_worktree_binding(
|
|
cp_db,
|
|
binding=candidate,
|
|
dry_run=dry_run,
|
|
operator_authorized=operator_authorized,
|
|
workflow_authorized=workflow_authorized,
|
|
project_root=root,
|
|
)
|
|
resolutions.append(res)
|
|
|
|
post_audit = audit_missing_worktree_bindings(
|
|
cp_db,
|
|
project_root=root,
|
|
remote=remote,
|
|
org=org,
|
|
repo=repo,
|
|
host=host,
|
|
)
|
|
|
|
return {
|
|
"success": True,
|
|
"dry_run": dry_run,
|
|
"operator_authorized": operator_authorized,
|
|
"workflow_authorized": workflow_authorized,
|
|
"before_audit": initial_audit,
|
|
"resolutions": resolutions,
|
|
"after_audit": post_audit,
|
|
"worktrees_dimension_resolved": post_audit.get("worktrees_dimension_resolved", False),
|
|
"stale_candidates_count": len(candidates),
|
|
"resolved_count": sum(1 for r in resolutions if r.get("performed") or (dry_run and r.get("success"))),
|
|
}
|