diff --git a/branch_cleanup_guard.py b/branch_cleanup_guard.py index 64db84a..e91ea02 100644 --- a/branch_cleanup_guard.py +++ b/branch_cleanup_guard.py @@ -525,6 +525,90 @@ def assess_ownership_record_activity(record: dict[str, Any]) -> dict[str, Any]: } +# Reviewer-lease reclaim is only reachable from a non-live (expired/stale) lease. +_RECLAIMABLE_REVIEWER_STATUSES = _EXPIRED_STATUSES | _STALE_STATUSES + + +def is_active_ownership_status(status: str | None) -> bool: + """True when *status* denotes live/active ownership of a branch (#855). + + Used to decide whether a *competing* active claimant still uses a branch + when weighing an expired reviewer lease for reclaim. Expired, stale, + released, and terminal statuses are not active. + """ + return _norm_str(status).lower() in _ACTIVE_OWNERSHIP_STATUSES + + +def assess_expired_reviewer_lease_reclaim( + *, + role: str, + status: str, + pr_merged: bool | None, + owner_pid_alive: bool | None, + competing_active_claimant: bool | None, +) -> dict[str, Any]: + """Decide, explicitly and fail-closed, whether an expired reviewer lease + may stop protecting an already-merged branch (#855 AC4). + + An expired reviewer lease should not protect a merged branch forever once + its work is done and no live claimant remains. Reclaim is permitted only + when **every** condition below is provably satisfied; any unknown + (``None``) or contrary value keeps the lease protective: + + - the lease is a ``reviewer`` lease (author/merger/controller/reconciler + leases are out of scope and always keep protecting); + - its status is expired or stale (never an active/live lease); + - the PR is proven merged (``pr_merged is True``); + - the lease owner process is proven dead (``owner_pid_alive is False``); + - no competing active claimant uses the branch + (``competing_active_claimant is False``). + + Returns a decision dict with ``reclaim_allowed`` and, when refused, the + fail-closed ``reasons``. The reasons never contain secrets — only the + role, the status, and which condition was unproven. + """ + reasons: list[str] = [] + normalized_role = _norm_str(role).lower() + normalized_status = _norm_str(status).lower() + + if normalized_role != "reviewer": + reasons.append( + f"lease role '{normalized_role or 'unknown'}' is not a reviewer " + "lease; expired-reviewer reclaim does not apply" + ) + if normalized_status not in _RECLAIMABLE_REVIEWER_STATUSES: + reasons.append( + f"lease status '{normalized_status or 'unknown'}' is not expired " + "or stale; only a non-live reviewer lease may be reclaimed" + ) + if pr_merged is not True: + reasons.append( + "PR merged state is not proven true; reclaim requires an " + "already-merged PR (fail closed)" + ) + if owner_pid_alive is not False: + reasons.append( + "lease owner process liveness is not proven dead; a live owner " + "still protects the branch (fail closed)" + ) + if competing_active_claimant is not False: + reasons.append( + "a competing active claimant may still use the branch; reclaim " + "requires no other active ownership (fail closed)" + ) + + allowed = not reasons + return { + "reclaim_allowed": allowed, + "role": normalized_role, + "status": normalized_status, + "decision": ( + "reclaim_expired_reviewer_lease" if allowed else "keep_protecting" + ), + "reasons": [] if allowed else reasons, + } + + def assess_active_branch_ownership( *, remote: str, diff --git a/control_plane_db.py b/control_plane_db.py index 75af7c9..4616cb3 100644 --- a/control_plane_db.py +++ b/control_plane_db.py @@ -599,6 +599,35 @@ class ControlPlaneDB: (_ts(), session_id), ) + def list_sessions( + self, + *, + statuses: Sequence[str] | None = None, + limit: int = 500, + ) -> list[dict[str, Any]]: + """List session rows for restart / impact analysis (#658). + + Read-only. Sessions are the process-level unit an MCP restart + disrupts, so the restart coordinator inventories them to compute blast + radius. Optional ``statuses`` filter (e.g. ``('active',)``) narrows to + live rows. Never returns secrets — only operational metadata. + """ + clauses: list[str] = [] + params: list[Any] = [] + if statuses: + placeholders = ", ".join("?" for _ in statuses) + clauses.append(f"status IN ({placeholders})") + params.extend(statuses) + where = ("WHERE " + " AND ".join(clauses)) if clauses else "" + sql = ( + f"SELECT * FROM sessions {where} " + "ORDER BY last_heartbeat_at DESC LIMIT ?" + ) + params.append(max(1, int(limit))) + with self._tx(immediate=False) as conn: + rows = conn.execute(sql, params).fetchall() + return [dict(r) for r in rows] + # ── work items ──────────────────────────────────────────────────────── def upsert_work_item( diff --git a/dirty_orphan_worktree_recovery.py b/dirty_orphan_worktree_recovery.py new file mode 100644 index 0000000..17db425 --- /dev/null +++ b/dirty_orphan_worktree_recovery.py @@ -0,0 +1,1080 @@ +"""Dirty orphaned author-issue worktree recovery (#860). + +Self-hosting deadlock class observed against #850 / PR #853 and #855: + +* same-claimant durable issue lock (``jcwalker3`` / ``prgs-author``) +* registered dirty worktree under ``branches/`` +* lock missing PID / session PID / expiry / heartbeat (malformed) +* existing recovery (#753/#768/#772) and renewal require a *clean* worktree + and/or a determinable owner PID +* no sanctioned dirty-preserving rebind + sync to a newer remote PR head + +This module is the pure evidence assessor and crash-safe recovery orchestrator +for that one class. It is an **explicit** recovery operation — it does not +silently widen ``gitea_lock_issue``. + +Safety model +------------ +Eligibility requires *all* of: + +* same claimant identity and profile as the durable lock +* exact issue / repository / branch / registered source worktree agreement +* registered worktree under the canonical branches root (resolved-path ancestry) +* no active original process (when PID is present) +* no active competing author session or workflow lease +* sufficient corroborating evidence when PID fields are absent (caller pins) +* explicit caller-provided local head, remote/PR head, and dirty fingerprints +* no foreign, duplicate, or ambiguous ownership + +A PID-less lock is **never** considered live merely because expiration fields +are absent (see also ``issue_lock_store.assess_lock_freshness``). + +Dirty preservation +------------------ +The source worktree is frozen: never cleaned, reset, overwritten, or deleted. +Recovery prefers a separately prepared recovery worktree checked out at the +pinned remote PR head. Dirty bytes are re-applied with path-level conflict +detection when upstream also changed a dirty path. + +Crash safety +------------ +A durable journal is written *before* filesystem or ownership mutation. +Retries resume or fail closed without stealing ownership, duplicating +worktrees, or losing dirty bytes. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import stat +import subprocess +from dataclasses import dataclass +from typing import Any, Mapping, Sequence + +from issue_lock_store import is_process_alive +from reviewer_worktree import parse_dirty_tracked_files + +# Outcomes +ELIGIBLE = "ELIGIBLE" +REFUSED = "REFUSED" +NO_CANDIDATE = "NO_CANDIDATE" +RECOVERY_COMPLETED = "RECOVERY_COMPLETED" +RECOVERY_RESUMED = "RECOVERY_RESUMED" +CONFLICTS_PRESENT = "CONFLICTS_PRESENT" + +# Journal phases (ordered) +PHASE_ELIGIBILITY = "1_eligibility_proven" +PHASE_JOURNAL_PERSISTED = "2_journal_persisted" +PHASE_RECOVERY_WORKTREE = "3_recovery_worktree_prepared" +PHASE_DIRTY_APPLIED = "4_dirty_applied" +PHASE_BINDING = "5_session_bound" +PHASE_COMPLETE = "6_complete" + +JOURNAL_DIR_NAME = "dirty-orphan-recovery-journals" +REQUIRED_LOCK_FIELDS = ("issue_number", "branch_name", "worktree_path") + +# Marker directory written into recovery worktrees for governed conflicts. +CONFLICT_STATE_DIR = ".gitea-recovery" +CONFLICT_STATE_FILE = "conflicts.json" + + +def _text(value: Any) -> str: + return str(value or "").strip() + + +def _same_realpath(left: str | None, right: str | None) -> bool: + if not left or not right: + return False + try: + return os.path.realpath(left) == os.path.realpath(right) + except OSError: + return left == right + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: str) -> str: + with open(path, "rb") as fh: + return sha256_bytes(fh.read()) + + +def get_journal_dir(override: str | None = None) -> str: + if override: + path = override + elif os.environ.get("GITEA_DIRTY_ORPHAN_RECOVERY_JOURNAL_DIR"): + path = os.environ["GITEA_DIRTY_ORPHAN_RECOVERY_JOURNAL_DIR"] + else: + path = os.path.join( + os.path.expanduser("~/.cache/gitea-tools"), JOURNAL_DIR_NAME + ) + os.makedirs(path, mode=0o700, exist_ok=True) + return path + + +def _journal_path(idempotency_key: str, journal_dir: str | None = None) -> str: + safe = "".join( + c if c.isalnum() or c in ("-", "_", ".") else "_" + for c in idempotency_key + ) + return os.path.join(get_journal_dir(journal_dir), f"{safe}.json") + + +def load_journal( + idempotency_key: str, journal_dir: str | None = None +) -> dict[str, Any] | None: + path = _journal_path(idempotency_key, journal_dir=journal_dir) + if not os.path.isfile(path): + return None + if os.path.islink(path): + raise ValueError(f"refusing journal path that is a symlink: {path}") + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + if not isinstance(data, dict): + raise ValueError("corrupt recovery journal: not an object") + return data + + +def save_journal( + journal: Mapping[str, Any], journal_dir: str | None = None +) -> str: + key = _text(journal.get("idempotency_key")) + if not key: + raise ValueError("journal requires idempotency_key") + path = _journal_path(key, journal_dir=journal_dir) + if os.path.islink(path): + raise ValueError(f"refusing to write journal through symlink: {path}") + tmp = f"{path}.tmp.{os.getpid()}" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(dict(journal), fh, indent=2, sort_keys=True) + fh.flush() + os.fsync(fh.fileno()) + os.replace(tmp, path) + return path + + +def derive_idempotency_key( + *, + remote: str, + org: str, + repo: str, + issue_number: int, + source_worktree: str, + expected_local_head: str, + expected_remote_head: str, +) -> str: + raw = "|".join( + [ + "dirty-orphan-recovery", + _text(remote), + _text(org), + _text(repo), + f"issue-{int(issue_number)}", + os.path.realpath(_text(source_worktree)), + _text(expected_local_head)[:40], + _text(expected_remote_head)[:40], + ] + ) + return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:32] + + +def _lock_claimant(lock: Mapping[str, Any]) -> dict[str, str]: + claimant = lock.get("claimant") + if not isinstance(claimant, Mapping): + lease = lock.get("work_lease") + claimant = lease.get("claimant") if isinstance(lease, Mapping) else None + if not isinstance(claimant, Mapping): + return {} + return { + "username": _text(claimant.get("username")), + "profile": _text(claimant.get("profile")), + } + + +def _recorded_pid(lock: Mapping[str, Any]) -> Any: + pid = lock.get("session_pid") + if pid is None: + pid = lock.get("pid") + return pid + + +def _malformed_pid_fields(lock: Mapping[str, Any]) -> list[str]: + """Return reasons the lock's PID fields are unusable (not automatically live).""" + missing: list[str] = [] + pid = _recorded_pid(lock) + if pid is None or _text(pid) == "": + missing.append("session_pid/pid") + return missing + try: + if int(pid) <= 0: + missing.append("session_pid/pid") + except (TypeError, ValueError): + missing.append("session_pid/pid") + return missing + + +def is_path_under_canonical_branches( + path: str, + *, + canonical_repo_root: str, + branches_dirname: str = "branches", +) -> tuple[bool, list[str]]: + """Resolved-path ancestry under ``{canonical_repo_root}/branches``. + + Rejects string-substring tricks, traversal, and symlink escapes outside + the canonical branches root. + """ + reasons: list[str] = [] + if not path or not canonical_repo_root: + return False, ["path and canonical_repo_root are required"] + try: + repo_root = os.path.realpath(canonical_repo_root) + branches_root = os.path.realpath(os.path.join(repo_root, branches_dirname)) + # realpath on a non-existent path still normalizes; prefer it so + # assessment can run without the directory existing yet. + target = os.path.realpath(path) + except OSError as exc: + return False, [f"path resolution failed: {exc}"] + + if not target.startswith(branches_root + os.sep) and target != branches_root: + reasons.append( + f"path '{path}' is not under canonical branches root '{branches_root}'" + ) + return False, reasons + try: + rel = os.path.relpath(target, branches_root) + except ValueError: + return False, ["path not relative to branches root"] + if rel.startswith(".."): + return False, ["path escapes branches root via relative traversal"] + return True, [] + + +def _result( + outcome: str, + *, + eligible: bool, + reasons: list[str], + evidence: dict[str, Any], +) -> dict[str, Any]: + return { + "outcome": outcome, + "eligible": eligible, + "recovery_eligible": eligible, + "reasons": list(reasons), + "evidence": evidence, + } + + +def assess_dirty_orphan_recovery( + existing_lock: Mapping[str, Any] | None, + *, + issue_number: int, + branch_name: str, + source_worktree_path: str, + remote: str, + org: str, + repo: str, + identity: str, + profile: str, + expected_local_head: str, + expected_remote_head: str, + expected_dirty_fingerprints: Mapping[str, str], + current_branch: str | None, + porcelain_status: str, + observed_local_head: str | None, + observed_remote_head: str | None, + observed_dirty_fingerprints: Mapping[str, str] | None, + competing_live_locks: Sequence[Mapping[str, Any]] | None = None, + competing_live_sessions: Sequence[Mapping[str, Any]] | None = None, + workflow_lease_active: bool | None = None, + workflow_lease_expired: bool | None = None, + canonical_repo_root: str, + worktree_registered: bool, + current_pid: int | None = None, + owner_process_alive_override: bool | None = None, +) -> dict[str, Any]: + """Assess whether dirty-orphan recovery is eligible. Pure; no I/O mutation.""" + evidence: dict[str, Any] = { + "issue_number": issue_number, + "branch_name": branch_name, + "source_worktree_path": source_worktree_path, + "remote": remote, + "org": org, + "repo": repo, + "identity": identity, + "profile": profile, + "expected_local_head": _text(expected_local_head), + "expected_remote_head": _text(expected_remote_head), + "expected_dirty_paths": sorted(expected_dirty_fingerprints or {}), + } + reasons: list[str] = [] + + if not existing_lock: + return _result( + NO_CANDIDATE, + eligible=False, + reasons=["no existing durable lock for this issue"], + evidence=evidence, + ) + + lock = dict(existing_lock) + if lock.get("issue_number") != issue_number: + return _result( + NO_CANDIDATE, + eligible=False, + reasons=[ + f"existing lock targets issue #{lock.get('issue_number')}, " + f"not #{issue_number}" + ], + evidence=evidence, + ) + + for field in REQUIRED_LOCK_FIELDS: + if not _text(lock.get(field)): + reasons.append(f"durable lock missing required field '{field}'") + + # Repository / branch / worktree agreement + for field, expected in (("remote", remote), ("org", org), ("repo", repo)): + actual = _text(lock.get(field)) + if actual and actual != _text(expected): + reasons.append( + f"lock {field} '{actual}' does not match requested '{_text(expected)}'" + ) + elif not actual: + # Some legacy locks omit remote/org/repo; require explicit pin match + # via caller still supplying them and branch/worktree agreement. + evidence[f"lock_{field}_absent"] = True + + locked_branch = _text(lock.get("branch_name")) + if locked_branch != _text(branch_name): + reasons.append( + f"lock branch '{locked_branch}' does not match requested " + f"'{_text(branch_name)}'" + ) + evidence["locked_branch"] = locked_branch + + locked_wt = _text(lock.get("worktree_path")) + if not _same_realpath(locked_wt, source_worktree_path): + reasons.append( + f"lock worktree '{locked_wt}' does not match declared source " + f"'{_text(source_worktree_path)}'" + ) + evidence["locked_worktree_path"] = locked_wt + + under, under_reasons = is_path_under_canonical_branches( + source_worktree_path, canonical_repo_root=canonical_repo_root + ) + if not under: + reasons.extend(under_reasons) + if not worktree_registered: + reasons.append("source worktree is not registered in git worktree list") + + # Claimant identity + claimant = _lock_claimant(lock) + evidence["lock_claimant"] = claimant + if claimant.get("username") != _text(identity): + reasons.append( + f"foreign claimant identity '{claimant.get('username')}' " + f"(caller '{_text(identity)}')" + ) + if claimant.get("profile") != _text(profile): + reasons.append( + f"foreign claimant profile '{claimant.get('profile')}' " + f"(caller '{_text(profile)}')" + ) + + # PID / liveness + pid_missing = _malformed_pid_fields(lock) + recorded_pid = _recorded_pid(lock) + evidence["recorded_pid"] = recorded_pid + evidence["pid_fields_missing"] = pid_missing + if pid_missing: + evidence["pid_less_malformed"] = True + # PID-less is never live by missing expiry alone. Eligibility continues + # only with full corroborating pins (already required below). + else: + alive = ( + owner_process_alive_override + if owner_process_alive_override is not None + else is_process_alive(int(recorded_pid)) + ) + evidence["owner_process_alive"] = alive + if alive: + reasons.append( + f"original owner process pid {recorded_pid} is still alive; " + "recovery refused" + ) + + # Competing ownership + for entry in competing_live_locks or (): + if not isinstance(entry, Mapping): + continue + if entry.get("issue_number") == issue_number: + reasons.append( + "competing live lock observed for the same issue; recovery refused" + ) + for entry in competing_live_sessions or (): + if not isinstance(entry, Mapping): + continue + sess_issue = entry.get("issue_number") + sess_identity = _text(entry.get("identity") or entry.get("username")) + if sess_issue == issue_number and sess_identity and sess_identity != _text(identity): + reasons.append( + f"competing live author session by '{sess_identity}' on issue " + f"#{issue_number}" + ) + elif sess_issue == issue_number and entry.get("active"): + # same claimant active elsewhere still blocks ambiguous ownership + if entry.get("session_pid") not in (None, current_pid, os.getpid()): + reasons.append( + "ambiguous competing same-issue author session still active" + ) + + if workflow_lease_active is True and workflow_lease_expired is not True: + reasons.append( + "active competing workflow lease still live; recovery refused" + ) + evidence["workflow_lease_active"] = workflow_lease_active + evidence["workflow_lease_expired"] = workflow_lease_expired + + # Dirty requirement (this recovery class is *for* dirty trees) + dirty_files = parse_dirty_tracked_files(porcelain_status) + if not dirty_files and not expected_dirty_fingerprints: + reasons.append( + "worktree is clean and no dirty fingerprints were provided; " + "use clean-worktree recovery (#753/#772) instead" + ) + evidence["observed_dirty_files"] = dirty_files + + # Explicit pins + if not _text(expected_local_head) or len(_text(expected_local_head)) < 40: + reasons.append("expected_local_head pin missing or not a full SHA") + if not _text(expected_remote_head) or len(_text(expected_remote_head)) < 40: + reasons.append("expected_remote_head pin missing or not a full SHA") + if not expected_dirty_fingerprints: + reasons.append("expected_dirty_fingerprints pin is required") + + if _text(observed_local_head) and _text(observed_local_head) != _text( + expected_local_head + ): + reasons.append( + f"local head mismatch: observed {_text(observed_local_head)} != " + f"pinned {_text(expected_local_head)}" + ) + if _text(observed_remote_head) and _text(observed_remote_head) != _text( + expected_remote_head + ): + reasons.append( + f"remote/PR head mismatch: observed {_text(observed_remote_head)} != " + f"pinned {_text(expected_remote_head)}" + ) + if _text(expected_local_head) == _text(expected_remote_head): + # Divergence is the motivating case; equal heads are allowed only when + # dirty files still need rebinding, so do not refuse equality. + evidence["heads_equal"] = True + else: + evidence["heads_diverged"] = True + + if current_branch and _text(current_branch) != locked_branch: + reasons.append( + f"source worktree is on branch '{_text(current_branch)}', not " + f"locked branch '{locked_branch}'" + ) + + # Fingerprint verification + observed_fps = dict(observed_dirty_fingerprints or {}) + for path, expected_fp in (expected_dirty_fingerprints or {}).items(): + rel = _text(path) + if not rel or rel.startswith("/") or ".." in rel.split("/"): + reasons.append(f"unsafe dirty path pin refused: {path!r}") + continue + obs = _text(observed_fps.get(rel)) + if not obs: + reasons.append(f"missing observed fingerprint for dirty path '{rel}'") + elif obs != _text(expected_fp): + reasons.append( + f"dirty fingerprint mismatch for '{rel}': " + f"observed {obs} != pinned {_text(expected_fp)}" + ) + + # PID-less corroboration: all explicit pins must already have passed. + if pid_missing and reasons: + reasons.append( + "PID-less malformed lock additionally requires full pin corroboration; " + "one or more corroborating checks failed" + ) + + if reasons: + return _result(REFUSED, eligible=False, reasons=reasons, evidence=evidence) + + evidence["eligibility"] = ELIGIBLE + return _result(ELIGIBLE, eligible=True, reasons=[], evidence=evidence) + + +def detect_path_conflicts( + *, + dirty_paths: Sequence[str], + local_head_contents: Mapping[str, bytes | None], + remote_head_contents: Mapping[str, bytes | None], + dirty_contents: Mapping[str, bytes], +) -> list[dict[str, Any]]: + """Path-level conflicts: upstream and dirty patch both changed the path.""" + conflicts: list[dict[str, Any]] = [] + for path in dirty_paths: + local_b = local_head_contents.get(path) + remote_b = remote_head_contents.get(path) + dirty_b = dirty_contents.get(path) + if dirty_b is None: + continue + # Upstream changed relative to the local head version of the path. + upstream_changed = (local_b or b"") != (remote_b or b"") + dirty_differs_from_remote = dirty_b != (remote_b or b"") + if upstream_changed and dirty_differs_from_remote: + conflicts.append( + { + "path": path, + "local_head_sha256": sha256_bytes(local_b) if local_b is not None else None, + "remote_head_sha256": sha256_bytes(remote_b) if remote_b is not None else None, + "dirty_sha256": sha256_bytes(dirty_b), + "reason": ( + "upstream and preserved dirty patch both changed this path" + ), + } + ) + return conflicts + + +def _safe_open_lockfile(path: str): + """Open a lock file refusing symlinks (O_NOFOLLOW when available).""" + if os.path.islink(path): + raise ValueError(f"refusing lock file that is a symlink: {path}") + flags = os.O_RDWR | os.O_CREAT + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + fd = os.open(path, flags, 0o600) + try: + st = os.fstat(fd) + if stat.S_ISLNK(st.st_mode): + os.close(fd) + raise ValueError(f"refusing lock file that is a symlink: {path}") + except Exception: + try: + os.close(fd) + except OSError: + pass + raise + return fd + + +def build_recovery_lock_record( + *, + existing_lock: Mapping[str, Any], + issue_number: int, + branch_name: str, + recovery_worktree_path: str, + remote: str, + org: str, + repo: str, + identity: str, + profile: str, + expected_remote_head: str, + source_worktree_path: str, + conflicts: Sequence[Mapping[str, Any]], + session_pid: int, +) -> dict[str, Any]: + """Construct a new durable lock bound to the recovery worktree + live PID.""" + from datetime import datetime, timedelta, timezone + + now = datetime.now(timezone.utc) + expires = now + timedelta(hours=4) + record = { + "remote": remote, + "org": org, + "repo": repo, + "issue_number": issue_number, + "branch_name": branch_name, + "worktree_path": recovery_worktree_path, + "pid": session_pid, + "session_pid": session_pid, + "claimant": {"username": identity, "profile": profile}, + "work_lease": { + "operation_type": "author_issue_work", + "issue_number": issue_number, + "pr_number": existing_lock.get("work_lease", {}).get("pr_number") + if isinstance(existing_lock.get("work_lease"), Mapping) + else existing_lock.get("pr_number"), + "branch": branch_name, + "worktree_path": recovery_worktree_path, + "claimant": {"username": identity, "profile": profile}, + "created_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "expires_at": expires.strftime("%Y-%m-%dT%H:%M:%SZ"), + "last_heartbeat_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + }, + "dirty_orphan_recovery": { + "recovered": True, + "source_worktree_path": source_worktree_path, + "recovery_worktree_path": recovery_worktree_path, + "accepted_head": expected_remote_head, + "conflicts": list(conflicts), + "source_frozen": True, + }, + "lock_provenance": { + "source": "gitea_recover_dirty_orphaned_issue_worktree", + "written_by_tool": "gitea_recover_dirty_orphaned_issue_worktree", + "written_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "claimant": {"username": identity, "profile": profile}, + }, + } + # Preserve prior assignment/lease ids when present (non-authoritative). + for key in ("assignment_id", "lease_id", "owner_session", "expected_base_sha"): + if key in existing_lock: + record[key] = existing_lock[key] + return record + + +def write_conflict_state( + recovery_worktree: str, conflicts: Sequence[Mapping[str, Any]] +) -> str: + state_dir = os.path.join(recovery_worktree, CONFLICT_STATE_DIR) + os.makedirs(state_dir, mode=0o700, exist_ok=True) + path = os.path.join(state_dir, CONFLICT_STATE_FILE) + payload = { + "conflicts": list(conflicts), + "resolution": "author_edit_required" if conflicts else "none", + } + with open(path, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2, sort_keys=True) + return path + + +def apply_dirty_bytes( + *, + recovery_worktree: str, + dirty_contents: Mapping[str, bytes], + conflict_paths: set[str], +) -> list[str]: + """Write non-conflicting dirty bytes into the recovery worktree. + + Conflicting paths are written as ``*.recovered-dirty`` siblings so the + original dirty bytes remain recoverable without overwriting upstream. + """ + written: list[str] = [] + for rel, data in dirty_contents.items(): + rel = _text(rel) + if not rel or rel.startswith("/") or ".." in rel.split("/"): + raise ValueError(f"unsafe relative path: {rel!r}") + dest = os.path.join(recovery_worktree, rel) + parent = os.path.dirname(dest) + if parent: + os.makedirs(parent, exist_ok=True) + if rel in conflict_paths: + sidecar = dest + ".recovered-dirty" + with open(sidecar, "wb") as fh: + fh.write(data) + written.append(rel + ".recovered-dirty") + else: + with open(dest, "wb") as fh: + fh.write(data) + written.append(rel) + return written + + +@dataclass +class GitOps: + """Injectable git operations for tests.""" + + def run(self, args: list[str], *, cwd: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + args, + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def prepare_recovery_worktree( + *, + canonical_repo_root: str, + recovery_worktree_path: str, + branch_name: str, + remote_head: str, + git_ops: GitOps | None = None, +) -> dict[str, Any]: + """Create or resume a recovery worktree at the pinned remote head. + + Leaves the source worktree untouched. Uses ``git worktree add`` only when + the recovery path does not already exist (idempotent resume). + """ + git = git_ops or GitOps() + under, reasons = is_path_under_canonical_branches( + recovery_worktree_path, canonical_repo_root=canonical_repo_root + ) + if not under: + return {"success": False, "reasons": reasons, "created": False} + + if os.path.isdir(recovery_worktree_path): + # Resume: verify HEAD matches pin. + probe = git.run( + ["git", "rev-parse", "HEAD"], cwd=recovery_worktree_path + ) + head = (probe.stdout or "").strip() + if probe.returncode != 0 or head != remote_head: + return { + "success": False, + "created": False, + "resumed": True, + "head": head, + "reasons": [ + f"resume recovery worktree HEAD '{head}' does not match expected remote HEAD '{remote_head}'" + ], + } + return { + "success": True, + "created": False, + "resumed": True, + "head": head, + "reasons": [], + } + + # Create detached-at-head worktree. Do not run git checkout -B because + # the source worktree holds the branch name (Git exit 128). + add = git.run( + [ + "git", + "worktree", + "add", + "--detach", + recovery_worktree_path, + remote_head, + ], + cwd=canonical_repo_root, + ) + if add.returncode != 0: + return { + "success": False, + "created": False, + "reasons": [ + f"git worktree add failed: {(add.stderr or add.stdout or '').strip()}" + ], + } + return { + "success": True, + "created": True, + "resumed": False, + "head": remote_head, + "reasons": [], + } + + +def run_dirty_orphan_recovery( + *, + assessment: Mapping[str, Any], + existing_lock: Mapping[str, Any], + issue_number: int, + branch_name: str, + source_worktree_path: str, + recovery_worktree_path: str, + remote: str, + org: str, + repo: str, + identity: str, + profile: str, + expected_local_head: str, + expected_remote_head: str, + expected_dirty_fingerprints: Mapping[str, str], + dirty_contents: Mapping[str, bytes], + local_head_contents: Mapping[str, bytes | None], + remote_head_contents: Mapping[str, bytes | None], + canonical_repo_root: str, + bind_lock: bool, + lock_writer: Any | None = None, + git_ops: GitOps | None = None, + journal_dir: str | None = None, + session_pid: int | None = None, + interrupt_after_phase: str | None = None, +) -> dict[str, Any]: + """Execute recovery with crash-journal phases. Idempotent on retry.""" + if not assessment.get("eligible"): + return { + "success": False, + "performed": False, + "outcome": assessment.get("outcome") or REFUSED, + "reasons": list(assessment.get("reasons") or ["not eligible"]), + "evidence": dict(assessment.get("evidence") or {}), + } + + # Verify dirty bytes match pins before any mutation. + for path, expected_fp in expected_dirty_fingerprints.items(): + data = dirty_contents.get(path) + if data is None: + return { + "success": False, + "performed": False, + "outcome": REFUSED, + "reasons": [f"dirty content missing for pinned path '{path}'"], + "evidence": {}, + } + if sha256_bytes(data) != _text(expected_fp): + return { + "success": False, + "performed": False, + "outcome": REFUSED, + "reasons": [f"dirty content fingerprint drift for '{path}'"], + "evidence": {}, + } + + idem = derive_idempotency_key( + remote=remote, + org=org, + repo=repo, + issue_number=issue_number, + source_worktree=source_worktree_path, + expected_local_head=expected_local_head, + expected_remote_head=expected_remote_head, + ) + journal = load_journal(idem, journal_dir=journal_dir) or { + "idempotency_key": idem, + "issue_number": issue_number, + "branch_name": branch_name, + "source_worktree_path": os.path.realpath(source_worktree_path), + "recovery_worktree_path": recovery_worktree_path, + "expected_local_head": expected_local_head, + "expected_remote_head": expected_remote_head, + "expected_dirty_fingerprints": dict(expected_dirty_fingerprints), + "phase": None, + "artifacts_created": { + "journal": False, + "recovery_worktree": False, + "dirty_applied": False, + "binding": False, + }, + "conflicts": [], + "complete": False, + } + + if journal.get("complete"): + return { + "success": True, + "performed": False, + "outcome": RECOVERY_RESUMED, + "reasons": ["recovery already complete; idempotent no-op"], + "evidence": { + "journal": journal, + "recovery_worktree_path": journal.get("recovery_worktree_path"), + }, + "journal": journal, + } + + # Phase 1: eligibility already proven by caller assessment. + journal["phase"] = PHASE_ELIGIBILITY + if interrupt_after_phase == PHASE_ELIGIBILITY: + return { + "success": False, + "performed": False, + "outcome": "INTERRUPTED", + "reasons": ["interrupted after eligibility (test harness)"], + "journal": journal, + "evidence": {"phase": PHASE_ELIGIBILITY}, + } + + # Phase 2: persist journal BEFORE filesystem/ownership mutation. + journal["phase"] = PHASE_JOURNAL_PERSISTED + journal["artifacts_created"]["journal"] = True + save_journal(journal, journal_dir=journal_dir) + if interrupt_after_phase == PHASE_JOURNAL_PERSISTED: + return { + "success": False, + "performed": True, + "outcome": "INTERRUPTED", + "reasons": ["interrupted after journal persistence (test harness)"], + "journal": journal, + "evidence": {"phase": PHASE_JOURNAL_PERSISTED}, + } + + # Phase 3: recovery worktree at remote head. + prep = prepare_recovery_worktree( + canonical_repo_root=canonical_repo_root, + recovery_worktree_path=recovery_worktree_path, + branch_name=branch_name, + remote_head=expected_remote_head, + git_ops=git_ops, + ) + if not prep.get("success"): + journal["phase"] = PHASE_RECOVERY_WORKTREE + journal["last_error"] = prep.get("reasons") + save_journal(journal, journal_dir=journal_dir) + return { + "success": False, + "performed": True, + "outcome": REFUSED, + "reasons": list(prep.get("reasons") or ["recovery worktree failed"]), + "journal": journal, + "evidence": prep, + } + if prep.get("created"): + journal["artifacts_created"]["recovery_worktree"] = True + journal["phase"] = PHASE_RECOVERY_WORKTREE + save_journal(journal, journal_dir=journal_dir) + if interrupt_after_phase == PHASE_RECOVERY_WORKTREE: + return { + "success": False, + "performed": True, + "outcome": "INTERRUPTED", + "reasons": ["interrupted after recovery worktree creation (test harness)"], + "journal": journal, + "evidence": prep, + } + + # Phase 4: conflict detection + dirty apply (source frozen). + conflicts = detect_path_conflicts( + dirty_paths=list(expected_dirty_fingerprints.keys()), + local_head_contents=local_head_contents, + remote_head_contents=remote_head_contents, + dirty_contents=dirty_contents, + ) + conflict_paths = {c["path"] for c in conflicts} + written = apply_dirty_bytes( + recovery_worktree=recovery_worktree_path, + dirty_contents=dirty_contents, + conflict_paths=conflict_paths, + ) + conflict_state_path = write_conflict_state(recovery_worktree_path, conflicts) + journal["conflicts"] = list(conflicts) + journal["written_paths"] = written + journal["conflict_state_path"] = conflict_state_path + journal["artifacts_created"]["dirty_applied"] = True + journal["phase"] = PHASE_DIRTY_APPLIED + # Prove source worktree still exists and was not deleted. + journal["source_still_present"] = os.path.isdir(source_worktree_path) + save_journal(journal, journal_dir=journal_dir) + + # #860 F4: Do NOT finalize session binding if conflicts remain. + if conflicts: + return { + "success": False, + "performed": True, + "outcome": CONFLICTS_PRESENT, + "reasons": [ + "conflicts present: manual resolution required before session binding (fail closed)" + ], + "conflicts": list(conflicts), + "recovery_worktree_path": recovery_worktree_path, + "source_worktree_path": source_worktree_path, + "source_frozen": True, + "journal": journal, + "evidence": { + "written_paths": written, + "conflict_state_path": conflict_state_path, + "accepted_head": expected_remote_head, + }, + } + + # Phase 5: bind session (optional for pure assessor tests). + pid = session_pid if session_pid is not None else os.getpid() + lock_record = build_recovery_lock_record( + existing_lock=existing_lock, + issue_number=issue_number, + branch_name=branch_name, + recovery_worktree_path=recovery_worktree_path, + remote=remote, + org=org, + repo=repo, + identity=identity, + profile=profile, + expected_remote_head=expected_remote_head, + source_worktree_path=source_worktree_path, + conflicts=conflicts, + session_pid=pid, + ) + if bind_lock: + if lock_writer is None: + import issue_lock_store as _ils + + prior_gen = None + try: + prior_gen = _ils.lock_generation(dict(existing_lock)) + except Exception: + prior_gen = None + _ils.bind_session_lock( + lock_record, + expected_generation=prior_gen, + recovery_sanctioned=True, + ) + else: + lock_writer(lock_record) + journal["artifacts_created"]["binding"] = True + journal["phase"] = PHASE_BINDING + save_journal(journal, journal_dir=journal_dir) + if interrupt_after_phase == PHASE_BINDING: + return { + "success": False, + "performed": True, + "outcome": "INTERRUPTED", + "reasons": ["interrupted after binding (test harness)"], + "journal": journal, + "evidence": {"lock_record": lock_record}, + } + + journal["phase"] = PHASE_COMPLETE + journal["complete"] = True + save_journal(journal, journal_dir=journal_dir) + + return { + "success": True, + "performed": True, + "outcome": RECOVERY_COMPLETED, + "reasons": [], + "conflicts": [], + "recovery_worktree_path": recovery_worktree_path, + "source_worktree_path": source_worktree_path, + "source_frozen": True, + "lock_record": lock_record, + "journal": journal, + "evidence": { + "written_paths": written, + "conflict_state_path": conflict_state_path, + "accepted_head": expected_remote_head, + "recovery_provenance": "dirty_orphan_recovery", + }, + } + + +def preflight_recognizes_recovered_provenance( + lock: Mapping[str, Any] | None, +) -> dict[str, Any]: + """Whether commit/publication preflights should accept recovered provenance.""" + if not lock: + return {"recognized": False, "reasons": ["no lock"]} + rec = lock.get("dirty_orphan_recovery") + if not isinstance(rec, Mapping) or not rec.get("recovered"): + return {"recognized": False, "reasons": ["no dirty_orphan_recovery record"]} + conflicts = rec.get("conflicts") or [] + if conflicts: + return { + "recognized": False, + "reasons": [ + "recovery conflicts remain; author must resolve before " + "commit/publication preflight" + ], + "conflicts": list(conflicts), + } + if not _text(lock.get("worktree_path")): + return {"recognized": False, "reasons": ["recovered lock missing worktree"]} + if _recorded_pid(lock) is None: + return { + "recognized": False, + "reasons": ["recovered lock still PID-less; binding incomplete"], + } + return { + "recognized": True, + "reasons": [], + "recovery_worktree_path": rec.get("recovery_worktree_path"), + "source_worktree_path": rec.get("source_worktree_path"), + "accepted_head": rec.get("accepted_head"), + } diff --git a/docs/architecture/mcp-restart-governance.md b/docs/architecture/mcp-restart-governance.md new file mode 100644 index 0000000..2931d10 --- /dev/null +++ b/docs/architecture/mcp-restart-governance.md @@ -0,0 +1,223 @@ +# ADR: MCP restart governance and authorization policy + +- **Status:** Accepted (policy effective immediately for LLM and operator sessions; enforcement tooling may lag) +- **Date:** 2026-07-23 +- **Tracking issue:** [#656](https://gitea.prgs.cc/Scaled-Tech-Consulting/Gitea-Tools/issues/656) +- **Policy version:** `restart-governance/v1` +- **Related:** + - Umbrella: [#655](https://gitea.prgs.cc/Scaled-Tech-Consulting/Gitea-Tools/issues/655) — governed MCP restart coordination and zero-disruption recovery + - Vision: [#652](https://gitea.prgs.cc/Scaled-Tech-Consulting/Gitea-Tools/issues/652) — MCP Control Plane Web Console product vision (§A system health and process control) + - Roadmap: [#653](https://gitea.prgs.cc/Scaled-Tech-Consulting/Gitea-Tools/issues/653) — Control Plane Web Console phased delivery (Phase 2 restart controls) + - Contamination guard: [#630](https://gitea.prgs.cc/Scaled-Tech-Consulting/Gitea-Tools/issues/630) — blocks manual process-kill recovery + - Console restart UX: [#642](https://gitea.prgs.cc/Scaled-Tech-Consulting/Gitea-Tools/issues/642) — sanctioned restart and graceful reload + - Existing restart / reconnect paths to inventory: [#591](https://gitea.prgs.cc/Scaled-Tech-Consulting/Gitea-Tools/issues/591) — auto-restart on master advance (closed); [#584](https://gitea.prgs.cc/Scaled-Tech-Consulting/Gitea-Tools/issues/584) — host auto-reconnect on transport flap + - Stable-control runtime split: `docs/architecture/mcp-stable-control-runtime-policy-adr.md` (#615) + - Client-namespace health: `docs/mcp-namespace-health.md` (#543) + - Reconnect-only EOF recovery: `docs/mcp-namespace-eof-recovery.md` + +## 1. Context + +The Gitea MCP server is the **control plane** for real issue and PR mutations +(create, comment, lock, review, merge, reconcile). The same process serves every +role namespace (`gitea-author`, `gitea-reviewer`, `gitea-merger`, +`gitea-reconciler`, `gitea-controller`) and holds the in-memory capability-gate +code loaded at startup. + +Restarting that process is destructive to concurrent work: + +- It resets every session's identity, preflight, and capability-lease binding. +- It can interrupt a mutation mid-critical-section (a lock acquire, a review + submit, a merge), leaving durable state half-written. +- Relaunching from the wrong checkout or worktree silently changes which code + the control plane runs, defeating master-parity gates (#420 / #615). + +Today there is **no durable written policy** stating who may restart MCP, under +what conditions, that restart is a last resort, and how controller approval, +automated safety gates, and break-glass interact. Operators and LLM sessions +therefore invent restart behavior ad hoc, which makes concurrent multi-role work +unsafe. #630 and #642 need this policy as their backbone. + +This ADR defines that policy. It does **not** implement coordinator code or HA +multi-instance restart (those are later children of #655). + +## 2. Decision + +### 2.1 v1 decision (recorded) + +**Restart authority in v1 is `controller approval + automated safety gates`.** + +A restart of the stable control runtime is authorized only when **both** hold: + +1. A **controller** role explicitly approves the restart, recording an audit + entry (who, why, scope, affected sessions), **and** +2. The **automated safety gates** pass: a completed drain acknowledgement (no + affected session is mid-critical-section) or a declared break-glass incident + (§2.5). + +Quorum among multiple controllers is **not** required day-one. It is deferred +unless a later investigation (tracked under #653) proves single-controller +approval is insufficient. This ADR records the v1 decision so enforcement code +(#630) has a fixed target; changing it requires a superseding ADR. + +### 2.2 Restart is a last resort — the recovery ladder + +Restart is the **last** rung. Before any restart, exhaust the narrower +recoveries, in order: + +1. **Reconnect** the IDE/client MCP namespace (transport EOF, `client is + closing: EOF`, transient `#584` flap). No process change. See + `docs/mcp-namespace-eof-recovery.md`. +2. **Refresh / rebind** the session workspace: re-run `gitea_whoami`, + `gitea_resolve_task_capability`, and pass an explicit validated + `worktree_path`. Fixes stale session context without touching the process. +3. **Scoped restart** of a single misbehaving namespace/service (where the + deployment supports per-service restart) rather than the whole control plane. +4. **Full restart** of the stable control runtime process — operator-owned, + controller-approved, drained. +5. **Host / infrastructure restart** — the broadest action; same authorization + as a full restart plus infrastructure ownership. + +A session **must** try rungs 1–2 and record why they were insufficient before +requesting a restart at rung 3 or above. Skipping straight to restart is a +policy violation. + +### 2.3 Authorization matrix + +| Role | Reconnect (1) | Refresh/rebind (2) | Scoped restart (3) | Full restart (4) | Host restart (5) | +|---|---|---|---|---|---| +| **author** | self | self | request only | **forbidden** | forbidden | +| **reviewer** | self | self | request only | **forbidden** | forbidden | +| **merger** | self | self | request only | **forbidden** | forbidden | +| **reconciler** | self | self | request only | **forbidden** | forbidden | +| **controller** | self | self | **approve** (+gates) | **approve** (+gates) | request to operator | +| **operator** | self | self | execute (controller-approved) | execute (controller-approved) | execute (controller-approved) | +| **admin** | self | self | execute | execute | execute (break-glass) | + +Legend: *self* = may perform for its own client session; *request only* = may +raise a restart request but not authorize or execute it; *approve* = may +authorize under §2.1 gates; *execute* = may perform the process action after the +authorization is recorded. + +Key invariants: + +- **No LLM worker role (author/reviewer/merger/reconciler) may perform or + authorize a full or host restart.** They may only reconnect/rebind their own + client and file a restart request. +- **Controller approval authorizes; operator/admin executes.** The approving + controller and the executing operator may be the same human, but both the + approval and the execution are audited. +- Privileged process actions (full restart, host restart) are reserved to + **operator/admin**, never to an automated worker. + +### 2.4 Approved conditions + +A restart at rung 3+ is approved only under one of these recorded conditions: + +- **No affected sessions:** the control plane has no live session that would be + interrupted (verified, not assumed). +- **Full drain acknowledged:** every affected session has drained + (no open critical section — no held mutation lease mid-write) and the drain is + acknowledged in the audit record. +- **Controller + gates:** controller approval plus passing automated safety + gates (§2.1), the standard v1 path. +- **Quorum:** not required in v1; reserved for a future superseding ADR. +- **Break-glass:** an incident-backed emergency exception (§2.5). + +Restart **never** bypasses mutation gates mid-critical-section. Drain before +restart is mandatory except under break-glass with a declared incident. + +### 2.5 Break-glass + +Break-glass is a **separate, narrower** authorization path for emergencies where +the normal drain-and-approve path cannot complete (e.g. the control plane is +wedged and cannot drain). + +Break-glass conditions: + +- A declared incident record exists (id, timestamp, declarer) **before** the + action. +- The action is taken by **operator or admin** authority only — never by an LLM + worker role, and never unilaterally by an operator with active peers when a + controller is reachable. +- The scope is the minimum necessary rung of the ladder. +- A **mandatory post-hoc audit** entry is filed: what was restarted, why the + normal path was impossible, which sessions were affected, and the incident id. + +Break-glass suspends the drain requirement, not the audit requirement. + +### 2.6 Explicit prohibitions + +- **A unilateral LLM or operator full restart while active peer sessions + exist is forbidden.** An LLM worker role must not kill, restart, or relaunch + the MCP process; a lone operator must not full-restart over live peer work + without controller approval or a break-glass incident. +- Process-kill recovery is forbidden as a routine tool (#630). This ADR does not + introduce a kill path. +- Ambiguous policy state **denies** restart (§4). + +## 3. Security requirements + +- Full restart and host restart are **privileged**; only operator/admin execute + them, only after a controller approval or break-glass incident is recorded. +- Break-glass is a distinct authorization path with its own audit mandate; it is + never the default and never silent. +- **Every approval and every restart action is audited** (who approved, who + executed, scope, affected sessions, condition, policy version). No restart is + authorized without a durable audit entry. + +## 4. Failure behavior + +**Ambiguous policy → deny restart.** If it cannot be established that a +restart is authorized under §2 — unknown affected-session state, missing +controller approval, absent break-glass incident, or an unclassifiable request — +the safe action is to **refuse** the restart and stop with a recovery report, +never to restart on assumption. + +## 5. Policy IDs (for enforcement code) + +Enforcement code — the restart coordinator (a later child of #655), the #630 +contamination guard, and the #642 console restart UX — binds to these stable +policy identifiers rather than to prose: + +| Policy ID | Statement | +|---|---| +| `RG-01` | Restart is last resort; rungs 1–2 must be tried and recorded first (§2.2). | +| `RG-02` | v1 authority = controller approval + automated safety gates (§2.1). | +| `RG-03` | No LLM worker role performs or authorizes full/host restart (§2.3). | +| `RG-04` | Full/host restart executed by operator/admin only, post approval (§2.3). | +| `RG-05` | Drain before restart is mandatory except break-glass with incident (§2.4). | +| `RG-06` | Break-glass requires a pre-declared incident and post-hoc audit (§2.5). | +| `RG-07` | Unilateral LLM/operator full restart with active peers is forbidden (§2.6). | +| `RG-08` | Ambiguous policy state denies restart (§4). | + +The `restart-governance/v1` **policy version** field is emitted on future +restart audit events so approvals can be reconciled against the policy revision +in force. + +## 6. Dogfooding + +Gitea-Tools governs its own MCP control plane by this policy. Author, reviewer, +merger, and reconciler sessions operating on this repository use the recovery +ladder (§2.2) — reconnect and rebind, never self-restart — and any real restart +of the Gitea-Tools stable control runtime follows the controller-approval + +drain path defined here. + +## 7. Acceptance and cross-links + +This ADR is the authoritative restart-governance policy. It **must** stay +cross-linked from the safety model and the web-console deployment boundary: + +- `docs/safety-model.md` § Process restart governance references this ADR. +- `docs/webui-deployment.md` references this ADR for restart/reload disposition. + +It is linked to its issue lineage — umbrella **#655**, vision **#652**, roadmap +**#653**, contamination guard **#630**, and console restart UX **#642** — in +§ Related above. + +## 8. Non-goals + +- Implementing the restart coordinator or approval state machine (#630, later + children of #655). +- Implementing HA multi-instance restart or quorum machinery. +- Introducing any process-kill or auto-restart tool; existing auto-restart + behavior must be inventoried before any new restart tool is enabled. diff --git a/docs/mcp-restart-coordinator.md b/docs/mcp-restart-coordinator.md new file mode 100644 index 0000000..b1368c0 --- /dev/null +++ b/docs/mcp-restart-coordinator.md @@ -0,0 +1,95 @@ +# MCP restart coordinator and impact analysis (#658) + +Before any sanctioned MCP restart, a central coordinator evaluates the live +control-plane state and produces an **impact preview** so operators and the web +console (#642 / #652) can see the blast radius *before* concurrent LLM work is +disrupted. Uncoordinated restarts destroy in-flight author/reviewer/merger work +and give operators no way to see what they are about to break. + +This lands the coordinator + impact DTO + a dry-run MCP tool. It is the single +sanctioned entry point for restart evaluation post-#657 (which inventoried the +restart/reload/kill paths). The **mutative apply** path — actually performing a +restart — is a later child gated by a drain proof and is explicitly out of +scope here. + +## Components + +| Piece | Where | Responsibility | +|-------|-------|----------------| +| `restart_coordinator.evaluate_restart_impact` | `restart_coordinator.py` | Pure classification: inventory → impact report DTO. No I/O, no restart. | +| `RestartImpactReport` / `SessionImpact` / `LeaseImpact` | `restart_coordinator.py` | Console-facing DTO (`.as_dict()` is JSON-serializable). | +| `ControlPlaneDB.list_sessions` | `control_plane_db.py` | Read-only session inventory (the process-level unit a restart kills). | +| `gitea_request_mcp_restart` | `gitea_mcp_server.py` | MCP tool: gathers inventory from the #613 DB, calls the coordinator, returns the report. Dry-run only. | + +## Dimensions evaluated + +The coordinator classifies the inventory across the dimensions #658 requires: + +- **Sessions** — every active MCP session; a restart terminates all of them. + Liveness = `status == active` **and** the owner pid is alive **and** the + heartbeat is fresh (default window 15 min). Dead/stale sessions do not count + toward blast radius. +- **Leases / locks** — control-plane leases joined with work items and their + freshness (`lease_lifecycle.classify_lease_freshness`). Only `active` (live + owner) leases are *disruptive*; expired / released / dead-process leases never + withhold a restart. +- **Issue / PR work** — the issues and PRs behind disruptive leases. +- **Mutations / critical sections** — a live lease carrying an author worktree + or a mutating phase (`implementing`, `publishing`, `merging`, …) is a + critical section a restart must not sever. +- **Terminal (merge) lock** — an active terminal lock always makes a restart + unsafe. +- **Prior recovery attempts** — narrower recovery already tried (e.g. sanctioned + client reconnects) is echoed so the operator sees the escalation history. + +## Verdict + +Exactly three verdicts, matching the acceptance criteria: + +| Verdict | `allow_restart` | Meaning | +|---------|-----------------|---------| +| `safe` | `true` | No other live sessions, no live leases, no terminal lock. | +| `unsafe` | `false` | Live work would be disrupted and no operator override is present — **or** the inventory could not be completed (fail closed). | +| `override` | `true` | Live work present, but an operator override accepts the blast radius. | + +`override_would_allow` tells the console whether an override path exists for the +current state. `blast_radius` is a `none` / `low` / `medium` / `high` severity +band derived from the affected session and work counts. + +### Fail closed + +If the control-plane inventory cannot be completed (DB unavailable, a listing +failed), `inventory_complete` is `false` and the verdict is `unsafe` / deny. An +incomplete evaluation must never green-light a restart. + +### Operator override authority + +Override authority is read from the environment variable +`GITEA_OPERATOR_RESTART_OVERRIDE_AUTHORIZATION` and **never** from a tool +argument. A worker session cannot set an environment variable on an +already-running daemon, so override cannot be self-asserted (same pattern as the +#630 daemon-maintenance authorization). The `request_override` tool argument only +expresses caller intent; it takes effect solely when the environment +authorization is present. + +## The tool + +```text +gitea_request_mcp_restart(remote, host, org, repo, + dry_run=True, request_override=False, + session_id=None, limit=200) +``` + +Read-only, dry-run, and it **never restarts anything**. `apply_supported` is +always `false`; passing `dry_run=False` performs no restart and reports that +apply is gated by a drain proof (a separate child). + +## Audit + +Every evaluation carries an `audit_record` (event, coordinator version, verdict, +allow decision, blast radius, counts, timestamp) so restart decisions are +auditable. No secrets flow through the coordinator — session ids, pids, and +profiles are operational metadata only. + +A representative dry-run report is in +[`mcp-restart-impact-sample.json`](./mcp-restart-impact-sample.json). diff --git a/docs/mcp-restart-impact-sample.json b/docs/mcp-restart-impact-sample.json new file mode 100644 index 0000000..e0136ee --- /dev/null +++ b/docs/mcp-restart-impact-sample.json @@ -0,0 +1,148 @@ +{ + "coordinator_version": "1.0.0-issue-658", + "evaluated_at": "2026-07-24T06:00:00+00:00", + "dry_run": true, + "restart_performed": false, + "inventory_complete": true, + "incomplete_reasons": [], + "verdict": "unsafe", + "allow_restart": false, + "override_would_allow": true, + "operator_override": false, + "blast_radius": "high", + "reasons": [ + "live work would be disrupted; restart denied without operator override", + "1 critical section(s) in flight (active lease with a live owner)" + ], + "affected_sessions": [ + { + "session_id": "prgs-author-30988-d6f43c25", + "role": "author", + "profile": "prgs-author", + "pid": 1, + "status": "active", + "alive": true, + "heartbeat_stale": false, + "is_requester": false, + "live": true + }, + { + "session_id": "prgs-reviewer-4157-0ce9", + "role": "reviewer", + "profile": "prgs-reviewer", + "pid": 1, + "status": "active", + "alive": true, + "heartbeat_stale": false, + "is_requester": true, + "live": true + } + ], + "affected_leases": [ + { + "lease_id": "lease-abc", + "session_id": "prgs-author-30988-d6f43c25", + "role": "author", + "phase": "implementing", + "freshness": "active", + "work_kind": "issue", + "work_number": 658, + "worktree_path": "/repo/branches/feat-issue-658", + "disruptive": true, + "is_mutation": true, + "is_critical_section": true + }, + { + "lease_id": "lease-dead", + "session_id": "prgs-author-91485", + "role": "author", + "phase": "allocated", + "freshness": "stale_dead_process", + "work_kind": "issue", + "work_number": 651, + "worktree_path": null, + "disruptive": false, + "is_mutation": false, + "is_critical_section": false + } + ], + "critical_sections": [ + { + "lease_id": "lease-abc", + "session_id": "prgs-author-30988-d6f43c25", + "role": "author", + "phase": "implementing", + "freshness": "active", + "work_kind": "issue", + "work_number": 658, + "worktree_path": "/repo/branches/feat-issue-658", + "disruptive": true, + "is_mutation": true, + "is_critical_section": true + } + ], + "affected_issues": [ + 658 + ], + "affected_prs": [], + "mutations": [ + { + "lease_id": "lease-abc", + "session_id": "prgs-author-30988-d6f43c25", + "role": "author", + "phase": "implementing", + "freshness": "active", + "work_kind": "issue", + "work_number": 658, + "worktree_path": "/repo/branches/feat-issue-658", + "disruptive": true, + "is_mutation": true, + "is_critical_section": true + } + ], + "terminal_lock": null, + "ack_state": { + "prgs-author-30988-d6f43c25": "pending" + }, + "prior_recovery_attempts": [ + { + "kind": "client_reconnect", + "at": "2026-07-24T06:00:00+00:00", + "outcome": "insufficient" + } + ], + "counts": { + "sessions_total": 2, + "sessions_live_other": 1, + "leases_total": 2, + "leases_disruptive": 1, + "critical_sections": 1, + "mutations": 1, + "affected_issues": 1, + "affected_prs": 0, + "prior_recovery_attempts": 1 + }, + "audit_record": { + "event": "restart_impact_evaluated", + "coordinator_version": "1.0.0-issue-658", + "evaluated_at": "2026-07-24T06:00:00+00:00", + "dry_run": true, + "operator_override": false, + "requesting_session_id": "prgs-reviewer-4157-0ce9", + "inventory_complete": true, + "verdict": "unsafe", + "allow_restart": false, + "blast_radius": "high", + "counts": { + "sessions_total": 2, + "sessions_live_other": 1, + "leases_total": 2, + "leases_disruptive": 1, + "critical_sections": 1, + "mutations": 1, + "affected_issues": 1, + "affected_prs": 0, + "prior_recovery_attempts": 1 + } + } +} diff --git a/docs/mcp-restart-path-inventory.md b/docs/mcp-restart-path-inventory.md new file mode 100644 index 0000000..c880269 --- /dev/null +++ b/docs/mcp-restart-path-inventory.md @@ -0,0 +1,90 @@ +# MCP restart / reload / kill path inventory (#657) + +Complete inventory of every code, script, and host path that can **restart, +reload, reconnect, kill, or force-recreate** an MCP process in this project, +with each path classified and linked to the guard that constrains it. + +This document is the human-readable companion to the machine-readable registry +in [`mcp_restart_paths.py`](../mcp_restart_paths.py). The two are kept in +lock-step by [`tests/test_mcp_restart_paths.py`](../tests/test_mcp_restart_paths.py): +every `path_id` below must appear in this file, and the source guards are run +against the live tree. + +Roadmap linkage: this inventory is the enumeration step of the restart +governance work — parent **#655**, restart-governance ADR **#656**, vision +**#652**, roadmap **#653**. Related detection/guard work: master-advance +staleness **#591**/**#420**, side-effect-free resolver **#685**, transport flap +**#584**, manual-kill contamination **#630**. + +## Classifications + +| Classification | Meaning | +|---|---| +| `sanctioned_narrow_recovery` | One-shot, safe-by-construction recovery that never targets the running daemon. | +| `guarded_fail_closed` | Detects a restart-requiring condition, then fails mutations closed and emits reconnect guidance. Never self-restarts. | +| `forbidden` | A workflow-safety violation; where an LLM tool could invoke it, it is marked contamination. | +| `removed` | A previously-existing unguarded restart primitive that has been deleted; a regression guard keeps it absent. | +| `host_residual` | Behavior owned by the host/IDE, outside this process's control. Documented, not code-guarded here. | + +## The rule + +**No component may perform an unguarded full restart of the MCP daemon.** The +in-process daemon (`gitea_mcp_server.py`, `mcp_server.py`, +`role_session_router.py`) must never replace or terminate its own process: +replacing the process after the host has wired up the stdio pipes desyncs the +JSON-RPC transport (observed with Antigravity/Cascade hosts). Recovery is owned +by the host/operator via a client reconnect — the daemon only ever *detects* +and *fails closed*. + +## Inventory + +| path_id | Classification | Mechanism | Guard | Refs | +|---|---|---|---|---| +| `cli_venv_bootstrap_execv` | sanctioned_narrow_recovery | CLI wrapper scripts re-exec into `venv/bin/python3` via `os.execv`, guarded by `sys.executable != venv_python`. | One-shot pre-import bootstrap; runs before any MCP transport exists and only when not already on the venv interpreter; idempotent guard prevents a re-exec loop. | #657 | +| `daemon_self_replacement` | forbidden | The daemon replacing/terminating its own process (`os.execv`/`os.kill`/`os._exit`) to reload code. | Forbidden by design; enforced against the source tree by `assert_no_daemon_self_replacement()`. | #657, #584 | +| `legacy_auto_restart_helper` | removed | A helper (`_trigger_mcp_auto_restart`) that actively restarted the server from the read-only resolver path. | Removed in #685; kept absent by `assert_auto_restart_helper_absent()`. | #685, #657 | +| `config_touch_reload` | removed | Touching (utime) the MCP client config to make the host reload the server. | Removed from the resolver in #685: stale detection is report-only, never mutating config, spawning threads, or calling `os._exit`. | #685, #657 | +| `master_advance_auto_restart` | guarded_fail_closed | On-disk master advancing past the running code. | `master_parity_gate` captures startup parity and blocks mutations while stale, emitting restart guidance; the process never self-restarts. | #420, #591, #657 | +| `stale_runtime_resolver_reconnect` | guarded_fail_closed | The capability resolver detecting a stale serving process. | Report-only (#685): returns `restart_required`/`stop_required` and an exact reconnect action; no restart, thread, config touch, or `os._exit`. | #685, #657 | +| `manual_daemon_kill` | forbidden | Shell kills of the daemon: `pkill -f mcp_server.py`, `killall`, broad `pkill -f python` sweeps, or `kill ` of a daemon pid. | Forbidden (#630): `runtime_recovery_guard` classifies these as contamination and `gitea_record_daemon_process_kill_attempt` writes a durable marker that fails later mutations closed. Operator maintenance authorization is read only from the environment. | #630, #657 | +| `conflict_marker_infra_stop` | guarded_fail_closed | The daemon entrypoint scans for unresolved merge-conflict markers at startup and stops (`sys.exit(1)`). | Fail-closed startup stop, not a restart: the process exits and waits for the operator to resolve conflicts and relaunch; never loops. | #657 | +| `ide_client_reconnect` | host_residual | A manual `/mcp reconnect` (or equivalent host action) that recreates the MCP client connection. | Outside this process's control; the sanctioned recovery the gates point operators toward. No in-process code initiates it. | #584, #656, #657 | +| `profile_switch_runtime` | sanctioned_narrow_recovery | Switching the active execution profile at runtime (dynamic-profile mode). | In-process and restart-free: `runtime_switching_supported` is true, so a switch rebinds capability without recreating the process. | #656, #657 | + +## Guards enforced in CI + +`tests/test_mcp_restart_paths.py` asserts, against the live source tree: + +1. **Registry well-formedness** — every path has a valid classification, a + non-empty guard description, references, and locations; ids are unique; all + five classifications are represented. +2. **Unknown restart attempts fail closed** — + `assert_restart_attempt_registered()` raises `UnknownRestartPathError` for + any path id not in this inventory, so a novel/unnamed restart primitive + cannot slip through silently. +3. **Daemon never self-replaces** — `assert_no_daemon_self_replacement()` scans + the daemon modules for `os.execv`/`os.kill`/`os._exit`/`os.abort` calls + (comment/docstring mentions are ignored) and finds none. +4. **Legacy helper stays removed** — `assert_auto_restart_helper_absent()` + confirms `_trigger_mcp_auto_restart` has not returned. +5. **pkill stays forbidden** — a daemon `pkill` command still classifies as + contamination via `runtime_recovery_guard`. + +## Residual host behaviors (outside process control) + +* `/mcp reconnect` in the IDE/host — the sanctioned recovery for stale-runtime, + transport-flap (#584), and worktree-binding conditions. The daemon can only + emit guidance toward it. +* Host-level process management (the operator relaunching the daemon after a + fail-closed stop, or after resolving merge conflicts). + +These are documented rather than code-guarded because the process cannot +observe or gate them from inside itself. + +## Rollout + +Per #657, guards are introduced flag-free as **regression assertions** (they +codify invariants that already hold) before any hard runtime block is layered +on. When the restart coordinator (#655/#656) lands, registered paths gain a +coordinator token/capability check; unregistered attempts already fail closed +today via `assert_restart_attempt_registered()`. diff --git a/docs/mcp-tool-inventory.md b/docs/mcp-tool-inventory.md index 8cb865b..b1aed12 100644 --- a/docs/mcp-tool-inventory.md +++ b/docs/mcp-tool-inventory.md @@ -135,6 +135,7 @@ that gates each call, not which tools exist. - `gitea_release_merger_pr_lease` - `gitea_release_reviewer_pr_lease` - `gitea_release_workflow_lease` +- `gitea_request_mcp_restart` - `gitea_resolve_task_capability` - `gitea_resume_review_draft` - `gitea_review_pr` diff --git a/docs/safety-model.md b/docs/safety-model.md index 31c740a..bbab241 100644 --- a/docs/safety-model.md +++ b/docs/safety-model.md @@ -46,3 +46,17 @@ If shell helpers are unavailable and MCP commit cannot run, stop with a recovery report (restart session, clear hung terminals, use MCP-native commit). See [`llm-workflow-runbooks.md`](llm-workflow-runbooks.md) § MCP-native commit path (#260) and agent temp artifact cleanup (#261). + +## 7. Process restart governance + +Restarting the MCP control-plane process is destructive to concurrent multi-role +work and is governed by a dedicated policy. Restart is a **last resort** behind +narrower recoveries (reconnect, rebind), full/host restart is reserved to +operator/admin under **controller approval + automated safety gates**, a +unilateral LLM or operator full restart with active peers is **forbidden**, and +ambiguous policy state **denies** restart. Break-glass is a separate, +incident-backed path with a mandatory audit. + +See [`architecture/mcp-restart-governance.md`](architecture/mcp-restart-governance.md) +(#656) for the authorization matrix, the recovery ladder, break-glass +conditions, and the `RG-01`–`RG-08` policy IDs. diff --git a/docs/webui-deployment.md b/docs/webui-deployment.md index fa754ad..2ad46ab 100644 --- a/docs/webui-deployment.md +++ b/docs/webui-deployment.md @@ -55,6 +55,15 @@ shipped to the browser. assumption paths, and the client-secret policy. Use it to verify an instance is configured for internal-only operation. +## Process restart / reload disposition + +The console never exposes a restart or reload control; process restart of the +MCP control-plane runtime is governed separately. Restart is a last resort behind +reconnect/rebind, full restart is operator/admin-only under controller approval +plus safety gates, and break-glass is an incident-backed path. See +[`architecture/mcp-restart-governance.md`](architecture/mcp-restart-governance.md) +(#656). + ## Non-goals (MVP) - Full SSO or session login in the UI diff --git a/docs/webui-local-dev.md b/docs/webui-local-dev.md index b37d53a..a41aa10 100644 --- a/docs/webui-local-dev.md +++ b/docs/webui-local-dev.md @@ -54,6 +54,7 @@ status, onboarding checklist state, and the fail-closed error payloads (#635). | `/` | Home / operator overview | | `/health` | JSON liveness (`status`, `service`, `mode`, `timestamp`, `uptime_seconds`) | | `/api/v1/system/health` | Structured read-only system health (#634) | +| `/system-health` | System-health dashboard — readiness, version/uptime, dependencies, MCP namespaces, stale-runtime parity (#639) | | `/queue` | Live PR and issue queue dashboard (#429) | | `/api/queue` | JSON queue export with pagination metadata | | `/projects` | Project registry list with status and onboarding progress (#427, #635) | @@ -258,6 +259,37 @@ Not-yet-implemented surfaces (`/sessions`, `/inventory`, `/timeline`, surfaces are backed by #636). Mutating methods on stub routes still fail closed with `read-only-mvp`. +## System-health dashboard (#639) + +`/system-health` renders the same snapshot the `/api/v1/system/health` API +returns, so the page and the API can never disagree. Cards: overall readiness, +stale-runtime parity, version and uptime, dependency probes, MCP namespaces, +probe errors (only when present), and recovery pointers. `?deep=1` opts into +the network probe exactly as the API does; the plain page load stays cheap. + +Field authority and honesty rules: + +* `ready` and `readiness_complete` are shown separately. A snapshot whose + required probes never ran is not the same as one that ran them and passed, + and the page never collapses the two into an unproven green. +* A probe that did not run appears under **Not probed**, never as healthy. +* `stale_runtime.mutation_safe` is displayed verbatim from the API. When the + runtime is stale, or when parity is indeterminate, the page warns and does + not claim mutation safety. +* MCP namespaces are reported `unproven`: the web process runs outside the + IDE-managed MCP client and cannot prove that path (#543). + +Redaction is split by field kind. Free text — probe details, readiness and +parity reasons, probe errors — passes through `system_health.redact`. +Structured fields — commit SHAs, probe names, statuses, timestamps — are +HTML-escaped only, because `redact`'s opaque-token rule matches any run of 32 +or more characters and would otherwise blank every 40-character git SHA, which +is precisely the evidence the parity view exists to show. + +The dashboard is read-only: no restart, reload, or process-kill control. Those +arrive in Phase 2 (#642). Recovery guidance points at the sanctioned client +reconnect / operator restart path — never a manual daemon kill (#630). + ## Deployment boundary (#435) MVP serves on loopback by default. Binding `0.0.0.0` or `::` is **refused** diff --git a/gitea_mcp_server.py b/gitea_mcp_server.py index 8e1661a..0199910 100644 --- a/gitea_mcp_server.py +++ b/gitea_mcp_server.py @@ -1469,7 +1469,7 @@ def verify_preflight_purity( dirty_files = sorted( _parse_porcelain_entries(_get_workspace_porcelain(workspace)) ) - if dirty_files: + if dirty_files and task != "commit_files": raise RuntimeError( nwb.format_namespace_workspace_binding_error( role_kind=role, @@ -2040,6 +2040,7 @@ import dependency_graph # noqa: E402 # #784 durable dependency edges import control_plane_db # noqa: E402 import lease_lifecycle # noqa: E402 import workflow_dashboard # noqa: E402 # #605 live queue/lease dashboard +import restart_coordinator # noqa: E402 # #658 MCP restart coordinator/impact import incident_bridge # noqa: E402 import sentry_observability # noqa: E402 (#606 optional Sentry observability) import sentry_incident_bridge # noqa: E402 (#607 Sentry→Gitea incident bridge) @@ -2050,6 +2051,7 @@ import issue_lock_store # noqa: E402 import issue_lock_adoption # noqa: E402 import issue_lock_recovery # noqa: E402 import issue_lock_renewal # noqa: E402 +import dirty_orphan_worktree_recovery # noqa: E402 # #860 dirty orphan recovery import dirty_same_claimant_session_rebind # noqa: E402 # #864 import stacked_pr_support # noqa: E402 import merge_approval_gate # noqa: E402 @@ -2445,6 +2447,20 @@ def _evaluate_issue_lock_recovery( descendant_sha=local_head, ) + # #871: the inverse of the #768 descendant relation — the *remote* head may + # have advanced past the local/recorded head via a sanctioned merge-based + # branch sync (``gitea_update_pr_branch_by_merge``) while the local worktree + # stayed put. Observe that provenance server-side so the assessor can prove + # it and nothing else. Probed only when the heads differ; never from any + # caller-supplied value. + sync_provenance: dict | None = None + if remote_head and local_head and remote_head != local_head: + sync_provenance = issue_lock_worktree.read_merge_sync_provenance( + worktree_path, + prior_head_sha=local_head, + synced_head_sha=remote_head, + ) + # #772: with no remote branch there is no head to measure against, so the # base the branch was cut from is observed instead. Probed only in that # case, so the published path's evidence is untouched (#772 AC8). @@ -2487,6 +2503,7 @@ def _evaluate_issue_lock_recovery( remote_branch_exists=remote_branch_exists, recorded_base_sha=recorded_base, base_ancestry=base_ancestry, + sync_provenance=sync_provenance, ) @@ -4362,6 +4379,280 @@ def gitea_lock_issue( return result +@mcp.tool() +@mcp.tool() +def gitea_recover_dirty_orphaned_issue_worktree( + issue_number: int, + branch_name: str, + source_worktree_path: str, + expected_local_head: str, + expected_remote_head: str, + expected_dirty_fingerprints: dict, + remote: str = "dadeschools", + host: str | None = None, + org: str | None = None, + repo: str | None = None, + recovery_worktree_path: str | None = None, + dry_run: bool = False, +) -> dict: + """Recover a dirty orphaned same-claimant author issue worktree (#860). + + Explicit recovery operation — does **not** silently widen ``gitea_lock_issue``. + + Accepts authoritative expected pins (repository, issue, branch, source + worktree, claimant, local head, remote/PR head, dirty fingerprints) and + fails closed on any mismatch. PID-less malformed locks are never treated + as live merely because expiry is absent. The source worktree is frozen; + recovery prepares a separate worktree at the pinned remote head, re-applies + dirty bytes with path-level conflict detection, and binds a live author + session only after recovery state is consistent. + + Args: + issue_number: Issue whose durable claim is being recovered. + branch_name: Locked branch ``(fix|feat|docs|chore)/issue-N-…``. + source_worktree_path: Registered dirty source worktree under branches/. + expected_local_head: Full 40-char SHA of the source worktree HEAD. + expected_remote_head: Full 40-char SHA of the remote/PR head to sync to. + expected_dirty_fingerprints: ``{relative_path: sha256}`` of dirty bytes. + remote/host/org/repo: Repository binding. + recovery_worktree_path: Optional recovery worktree path under branches/. + dry_run: Assess eligibility only; no filesystem or lock mutation. + + Returns: + dict with success, outcome, conflicts, recovery_worktree_path, reasons, + evidence, and journal metadata. + """ + task = "recover_dirty_orphaned_issue_worktree" + ok, block_reasons = role_session_router.check_author_mutation_after_reviewer_stop( + task + ) + if not ok: + return { + "success": False, + "performed": False, + "outcome": "REFUSED", + "reasons": block_reasons, + } + blocked = _namespace_mutation_block(task, remote=remote) + if blocked: + return blocked + blocked = _profile_permission_block( + task_capability_map.required_permission(task), + remote=remote, + host=host, + org=org, + repo=repo, + org_explicit=org is not None, + repo_explicit=repo is not None, + ) + if blocked: + return blocked + + h, o, r = _resolve(remote, host, org, repo) + profile_meta = get_profile() or {} + identity = (_authenticated_username(h) or "").strip() + profile = (profile_meta.get("profile_name") or "").strip() + if not identity or not profile: + return { + "success": False, + "performed": False, + "outcome": "REFUSED", + "reasons": ["could not resolve authenticated identity/profile"], + } + + existing_lock = _load_existing_issue_lock( + remote=remote, org=o, repo=r, issue_number=issue_number + ) + + src = os.path.realpath(source_worktree_path) + git_state = issue_lock_worktree.read_worktree_git_state(src) + observed_local = (git_state.get("head_sha") or "").strip() + porcelain = git_state.get("porcelain_status") or "" + current_branch = git_state.get("current_branch") + + # Observed dirty fingerprints from source worktree bytes. + observed_fps: dict[str, str] = {} + dirty_contents: dict[str, bytes] = {} + for rel in (expected_dirty_fingerprints or {}): + rel_n = str(rel).strip() + fpath = os.path.join(src, rel_n) + if not os.path.isfile(fpath): + continue + with open(fpath, "rb") as fh: + data = fh.read() + dirty_contents[rel_n] = data + observed_fps[rel_n] = dirty_orphan_worktree_recovery.sha256_bytes(data) + + # Remote head observation (best-effort; pin mismatch fails closed). + observed_remote = "" + try: + probe = subprocess.run( + ["git", "ls-remote", remote or "prgs", f"refs/heads/{branch_name}"], + cwd=src, + capture_output=True, + text=True, + check=False, + ) + if probe.returncode == 0 and (probe.stdout or "").strip(): + observed_remote = (probe.stdout or "").strip().split()[0] + except Exception: + observed_remote = "" + + registered = False + try: + listing = subprocess.run( + ["git", "worktree", "list", "--porcelain"], + cwd=src, + capture_output=True, + text=True, + check=False, + ) + if listing.returncode == 0: + registered = src in (listing.stdout or "") + except Exception: + registered = False + + project_root = _canonical_local_git_root() + canonical_root = author_mutation_worktree.resolve_canonical_repo_root( + src, project_root + ) + + competing_locks: list[dict] = [] + try: + all_live = issue_lock_store.list_live_locks() + for l in all_live: + if l.get("issue_number") == issue_number: + wt = l.get("worktree_path") + if not wt or not issue_lock_store._same_realpath(wt, src): + competing_locks.append(l) + except Exception: + competing_locks = [] + + wf_active = False + wf_expired = True + try: + db, _ = _control_plane_db_or_error() + if db is not None: + active_leases_data = lease_lifecycle.list_active_leases( + db, + remote=remote if remote in REMOTES else remote, + org=o, + repo=r, + ) + leases_list = active_leases_data.get("leases") or [] + for l in leases_list: + if l.get("work_number") == issue_number and l.get("work_kind") == "issue": + fresh = l.get("freshness") or {} + if fresh.get("status") == "active": + wf_active = True + wf_expired = False + elif fresh.get("status") in ("expired", "stale_dead_process"): + wf_active = False + wf_expired = True + except Exception: + pass + + assessment = dirty_orphan_worktree_recovery.assess_dirty_orphan_recovery( + existing_lock, + issue_number=issue_number, + branch_name=branch_name, + source_worktree_path=src, + remote=remote if remote else "prgs", + org=o, + repo=r, + identity=identity, + profile=profile, + expected_local_head=expected_local_head, + expected_remote_head=expected_remote_head, + expected_dirty_fingerprints=expected_dirty_fingerprints or {}, + current_branch=current_branch, + porcelain_status=porcelain, + observed_local_head=observed_local, + observed_remote_head=observed_remote, + observed_dirty_fingerprints=observed_fps, + competing_live_locks=competing_locks, + competing_live_sessions=[], + workflow_lease_active=wf_active, + workflow_lease_expired=wf_expired, + canonical_repo_root=canonical_root, + worktree_registered=registered, + current_pid=os.getpid(), + ) + if dry_run or not assessment.get("eligible"): + return { + "success": bool(assessment.get("eligible")), + "performed": False, + "dry_run": dry_run, + "outcome": assessment.get("outcome"), + "reasons": list(assessment.get("reasons") or []), + "evidence": dict(assessment.get("evidence") or {}), + "eligible": bool(assessment.get("eligible")), + } + + if not recovery_worktree_path: + recovery_worktree_path = os.path.join( + canonical_root, + "branches", + f"recovery-issue-{issue_number}-dirty-orphan", + ) + + # Load blob contents at local/remote heads for conflict detection. + def _blob_at(head: str, rel: str) -> bytes | None: + try: + proc = subprocess.run( + ["git", "show", f"{head}:{rel}"], + cwd=src, + capture_output=True, + check=False, + ) + if proc.returncode != 0: + return None + return proc.stdout + except Exception: + return None + + local_contents = { + rel: _blob_at(expected_local_head, rel) + for rel in (expected_dirty_fingerprints or {}) + } + remote_contents = { + rel: _blob_at(expected_remote_head, rel) + for rel in (expected_dirty_fingerprints or {}) + } + + # Preflight purity is satisfied via explicit worktree_path on this tool's + # recovery path; source remains frozen and is never cleaned. + result = dirty_orphan_worktree_recovery.run_dirty_orphan_recovery( + assessment=assessment, + existing_lock=existing_lock or {}, + issue_number=issue_number, + branch_name=branch_name, + source_worktree_path=src, + recovery_worktree_path=recovery_worktree_path, + remote=remote if remote else "prgs", + org=o, + repo=r, + identity=identity, + profile=profile, + expected_local_head=expected_local_head, + expected_remote_head=expected_remote_head, + expected_dirty_fingerprints=expected_dirty_fingerprints or {}, + dirty_contents=dirty_contents, + local_head_contents=local_contents, + remote_head_contents=remote_contents, + canonical_repo_root=canonical_root, + bind_lock=True, + session_pid=os.getpid(), + ) + # Surface preflight recognition for recovered provenance. + if result.get("success") and result.get("lock_record"): + result["preflight_provenance"] = ( + dirty_orphan_worktree_recovery.preflight_recognizes_recovered_provenance( + result["lock_record"] + ) + ) + return result + @mcp.tool() def gitea_rebind_dirty_same_claimant_author_session( issue_number: int, @@ -4618,6 +4909,8 @@ def gitea_rebind_dirty_same_claimant_author_session( } return result + return result + @mcp.tool() def gitea_assess_work_issue_duplicate( @@ -11116,6 +11409,9 @@ def _collect_branch_ownership_records( """ records: list[dict] = [] inventory_error = False + # #855 AC4: expired/stale reviewer-lease records eligible for an explicit + # reclaim decision, evaluated after the full ownership inventory is built. + reviewer_reclaim_candidates: list[tuple[dict, bool | None]] = [] target_branch = (branch or "").strip() if not target_branch: return {"records": records, "inventory_error": False} @@ -11264,15 +11560,28 @@ def _collect_branch_ownership_records( else: status = freshness_status reclaim_allowed = False - records.append( - _base_rec( - category=category, - status=status, - reclaim_allowed=reclaim_allowed, - role=role, - host=lease_host or host_n or host, - ) + rec = _base_rec( + category=category, + status=status, + reclaim_allowed=reclaim_allowed, + role=role, + host=lease_host or host_n or host, ) + records.append(rec) + # #855 AC4: a reviewer lease that is expired/stale (its owner + # gone) becomes a candidate for an explicit, fail-closed + # reclaim decision made once the full inventory is known. + if ( + role == "reviewer" + and status + in branch_cleanup_guard._RECLAIMABLE_REVIEWER_STATUSES + ): + owner_alive = ( + fr.get("owner_pid_alive") if isinstance(fr, dict) else None + ) + reviewer_reclaim_candidates.append( + (rec, owner_alive if isinstance(owner_alive, bool) else None) + ) except Exception: # O1: fail closed on control-plane inventory errors. inventory_error = True @@ -11343,6 +11652,44 @@ def _collect_branch_ownership_records( ) ) + # #855 AC4: decide, explicitly and fail-closed, whether any expired/stale + # reviewer lease may stop protecting an already-merged branch. This runs + # only after the full ownership inventory is built, so a competing active + # claimant (an active lease, author session, worktree binding, or active + # reviewer comment lease) is visible. An inventory failure keeps every + # reclaim candidate protective (reclaim_allowed stays False). + if reviewer_reclaim_candidates and not inventory_error: + pr_merged_state: bool | None = None + if pr_number is not None and auth and base_api: + try: + pr_live = api_request( + "GET", f"{base_api}/pulls/{int(pr_number)}", auth + ) + if isinstance(pr_live, dict) and pr_live: + pr_merged_state = bool( + pr_live.get("merged") or pr_live.get("merged_at") + ) + except Exception: + # Unknown merged state fails closed (candidate stays protective). + pr_merged_state = None + for cand_rec, owner_alive in reviewer_reclaim_candidates: + competing = any( + other is not cand_rec + and branch_cleanup_guard.is_active_ownership_status( + other.get("status") + ) + for other in records + ) + decision = branch_cleanup_guard.assess_expired_reviewer_lease_reclaim( + role=str(cand_rec.get("role")), + status=str(cand_rec.get("status")), + pr_merged=pr_merged_state, + owner_pid_alive=owner_alive, + competing_active_claimant=competing, + ) + cand_rec["reclaim_allowed"] = decision["reclaim_allowed"] + cand_rec["reclaim_decision"] = decision["decision"] + return {"records": records, "inventory_error": inventory_error} @@ -11405,6 +11752,7 @@ def gitea_reconcile_merged_cleanups( dry_run: bool = True, execute_confirmed: bool = False, limit: int = 50, + pr_number: int | None = None, remote: str = "dadeschools", host: str | None = None, org: str | None = None, @@ -11415,7 +11763,11 @@ def gitea_reconcile_merged_cleanups( Args: dry_run: Defaults to True. When True, only builds the reconciliation report. execute_confirmed: Must be True when dry_run=False. - limit: Max number of closed PRs to inspect. + limit: Max number of closed PRs to inspect (batch mode only; ignored when + ``pr_number`` is set). + pr_number: Optional exact merged PR selector (#855). When set, only that + PR is assessed/acted on (fail closed if missing, unmerged, or + ambiguous). When omitted, existing batch behaviour is preserved. remote: Known Gitea instance ('dadeschools' or 'prgs'). host: Override the Gitea host. org: Override the owner/organization. @@ -11450,11 +11802,120 @@ def gitea_reconcile_merged_cleanups( "audit_phase": audit_reconciliation_mode.current_phase(), } + # #855: optional exact PR pin. Fail closed before any inventory mutation. + exact_pr: int | None = None + if pr_number is not None: + try: + exact_pr = int(pr_number) + except (TypeError, ValueError): + return { + "success": False, + "performed": False, + "executed": False, + "dry_run": bool(dry_run), + "selection_mode": "exact_pr", + "selected_pr_number": pr_number, + "reasons": [ + f"pr_number={pr_number!r} is not a valid integer " + "(fail closed; no mutation)" + ], + "blocker_kind": "invalid_pr_number", + } + if exact_pr <= 0: + return { + "success": False, + "performed": False, + "executed": False, + "dry_run": bool(dry_run), + "selection_mode": "exact_pr", + "selected_pr_number": exact_pr, + "reasons": [ + f"pr_number={exact_pr} must be a positive integer " + "(fail closed; no mutation)" + ], + "blocker_kind": "invalid_pr_number", + } + h, o, r = _resolve(remote, host, org, repo) auth = _auth(h) base = repo_api_url(h, o, r) - closed_prs = api_get_all(f"{base}/pulls?state=closed", auth, limit=limit) - open_prs = api_get_all(f"{base}/pulls?state=open", auth) + + selection_mode = "batch" + closed_prs: list[dict] = [] + open_prs: list[dict] = [] + if exact_pr is not None: + selection_mode = "exact_pr" + try: + pr_live = api_request("GET", f"{base}/pulls/{exact_pr}", auth) + except Exception as exc: + return { + "success": False, + "performed": False, + "executed": False, + "dry_run": bool(dry_run), + "selection_mode": selection_mode, + "selected_pr_number": exact_pr, + "reasons": [ + f"PR #{exact_pr} could not be uniquely resolved " + f"(fail closed; no mutation): {_redact(str(exc))}" + ], + "blocker_kind": "pr_unresolvable", + } + if not isinstance(pr_live, dict) or not pr_live: + return { + "success": False, + "performed": False, + "executed": False, + "dry_run": bool(dry_run), + "selection_mode": selection_mode, + "selected_pr_number": exact_pr, + "reasons": [ + f"PR #{exact_pr} could not be uniquely resolved " + "(empty response; fail closed; no mutation)" + ], + "blocker_kind": "pr_unresolvable", + } + live_number = pr_live.get("number") + try: + live_number_int = int(live_number) if live_number is not None else None + except (TypeError, ValueError): + live_number_int = None + if live_number_int != exact_pr: + return { + "success": False, + "performed": False, + "executed": False, + "dry_run": bool(dry_run), + "selection_mode": selection_mode, + "selected_pr_number": exact_pr, + "reasons": [ + f"PR #{exact_pr} resolution is ambiguous or mismatched " + f"(live number={live_number!r}; fail closed; no mutation)" + ], + "blocker_kind": "pr_ambiguous", + } + if not (pr_live.get("merged") or pr_live.get("merged_at")): + return { + "success": False, + "performed": False, + "executed": False, + "dry_run": bool(dry_run), + "selection_mode": selection_mode, + "selected_pr_number": exact_pr, + "reasons": [ + f"PR #{exact_pr} is not merged " + "(exact-target cleanup requires a merged PR; " + "fail closed; no mutation)" + ], + "blocker_kind": "pr_not_merged", + } + closed_prs = [pr_live] + # Exact mode still needs open heads for remote-delete safety gates. + open_prs = api_get_all(f"{base}/pulls?state=open", auth) + else: + # Preserve historical call order (closed then open) for batch callers/tests. + closed_prs = api_get_all(f"{base}/pulls?state=closed", auth, limit=limit) + open_prs = api_get_all(f"{base}/pulls?state=open", auth) merged_closed: list[dict] = [] remote_branch_exists: dict[str, bool] = {} @@ -11481,6 +11942,13 @@ def gitea_reconcile_merged_cleanups( scratch_candidates = merged_cleanup_reconcile.discover_reviewer_scratch_worktrees( _canonical_local_git_root() ) + # #855: exact-target never inventories or mutates foreign PR scratch trees. + if exact_pr is not None: + scratch_candidates = [ + s + for s in scratch_candidates + if int(s.get("pr_number") or 0) == int(exact_pr) + ] active_reviewer_leases: dict[int, bool] = {} pr_states: dict[int, dict] = {} for scratch in scratch_candidates: @@ -11515,6 +11983,33 @@ def gitea_reconcile_merged_cleanups( active_reviewer_leases=active_reviewer_leases, pr_states=pr_states, ) + report["selection_mode"] = selection_mode + if exact_pr is not None: + report["selected_pr_number"] = exact_pr + # Fail closed if exact pin somehow produced other or zero entries. + entries = list(report.get("entries") or []) + entry_numbers = [] + for entry in entries: + try: + entry_numbers.append(int(entry.get("pr_number"))) + except (TypeError, ValueError): + entry_numbers.append(entry.get("pr_number")) + if entry_numbers != [exact_pr]: + return { + "success": False, + "performed": False, + "executed": False, + "dry_run": bool(dry_run), + "selection_mode": selection_mode, + "selected_pr_number": exact_pr, + "reasons": [ + f"exact PR #{exact_pr} selection produced unexpected " + f"candidate set {entry_numbers!r} " + "(fail closed; no mutation)" + ], + "blocker_kind": "exact_selection_mismatch", + "entries": entries, + } if dry_run: report["dry_run"] = True @@ -18889,10 +19384,59 @@ def gitea_update_pr_branch_by_merge( prepared_verdict_head_sha=live_pr_head, ) - return { - "success": True, + # #871: the remote head is now advanced; the durable linked-issue lock must + # be advanced with it, or a later dead-session recovery can never prove + # ownership at the new head. This runs AFTER the successful remote update, so + # a failure here is a *partial* lifecycle failure — the remote moved but the + # durable state did not — and must never be reported as a full success. + claimant = _work_lease_claimant(h) + matched_issue = ownership.get("matched_issue") + synced_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + lock_refresh: dict = { + "refreshed": False, + "reasons": ["durable lock head refresh was not attempted"], + } + if new_head and matched_issue and source_branch and (wt or None): + try: + lock_refresh = issue_lock_store.apply_durable_lock_head_refresh( + remote=remote, + org=o, + repo=r, + issue_number=int(matched_issue), + branch_name=source_branch, + worktree_path=wt, + pr_number=pr_number, + identity=claimant.get("username"), + profile=claimant.get("profile"), + current_pid=os.getpid(), + expected_old_head=live_pr_head, + new_head=new_head, + synced_at=synced_at, + base_head=live_base_head, + ) + except Exception as exc: + lock_refresh = { + "refreshed": False, + "reasons": [ + f"durable lock head refresh raised (fail closed): {_redact(str(exc))}" + ], + } + else: + lock_refresh = { + "refreshed": False, + "reasons": [ + "durable lock head refresh could not run: missing new head, " + "linked issue, source branch, or worktree binding" + ], + } + + durable_refreshed = bool(lock_refresh.get("refreshed")) + base_result = { "performed": True, "mutation_allowed": True, + "durable_lock_refreshed": durable_refreshed, + "durable_lock_refresh": lock_refresh, + "fully_synchronized": durable_refreshed, "style": "merge", "force_push": False, "rebase": False, @@ -18914,19 +19458,37 @@ def gitea_update_pr_branch_by_merge( "prepared_verdict_invalidated": transition.get( "prepared_verdict_invalidated" ), - "recommended_next_action": transition.get( - "recommended_next_action", - pr_sync_status.ACTION_FRESH_REVIEW_REQUIRED, - ), "transition": transition, "role_kind": role, "profile_name": profile.get("profile_name"), "worktree_path": wt or None, - "reasons": list(transition.get("reasons") or []) + [ - "update-by-merge completed via native Gitea API (style=merge only)" - ], } + if durable_refreshed: + base_result["success"] = True + base_result["recommended_next_action"] = transition.get( + "recommended_next_action", + pr_sync_status.ACTION_FRESH_REVIEW_REQUIRED, + ) + base_result["reasons"] = list(transition.get("reasons") or []) + [ + "update-by-merge completed via native Gitea API (style=merge only)", + f"durable linked-issue lock #{matched_issue} head refreshed to " + f"{new_head} (verified by read-after-write)", + ] + return base_result + + # Partial lifecycle failure: the remote advanced but the durable lock did + # not. Do NOT report a fully successful synchronization (#871). + base_result["success"] = False + base_result["partial_lifecycle_failure"] = True + base_result["recommended_next_action"] = pr_sync_status.ACTION_BLOCKED + base_result["reasons"] = list(lock_refresh.get("reasons") or []) + [ + f"PARTIAL LIFECYCLE FAILURE: PR #{pr_number} remote head advanced to " + f"{new_head} but the durable linked-issue lock head was not refreshed; " + "the synchronization is NOT complete", + ] + return base_result + @mcp.tool() def gitea_assess_conflict_fix_push( @@ -21489,6 +22051,156 @@ def gitea_workflow_dashboard( return payload +@mcp.tool() +def gitea_request_mcp_restart( + remote: str = "dadeschools", + host: str | None = None, + org: str | None = None, + repo: str | None = None, + dry_run: bool = True, + request_override: bool = False, + session_id: str | None = None, + limit: int = 200, +) -> dict: + """Evaluate a proposed MCP restart and return an impact preview (#658). + + Central restart coordinator: gathers live control-plane state (sessions, + leases/locks, in-flight issue/PR work, mutations, worktrees) and returns a + blast-radius impact report with a ``safe`` / ``unsafe`` / ``override`` + verdict, so the console (#642/#652) and operators can see what a restart + would disrupt *before* any concurrent LLM work is destroyed. + + This tool is **dry-run and never restarts anything.** The mutative apply + path is a separate child gated by a drain proof (non-goal here); calling + with ``dry_run=False`` still performs no restart and reports that apply is + not yet available. + + Operator override authority is read from the process environment + (``GITEA_OPERATOR_RESTART_OVERRIDE_AUTHORIZATION``), never self-asserted by + the requesting session: ``request_override`` only expresses caller intent + and takes effect solely when that environment authorization is present. + + Fails closed: if the control-plane inventory cannot be completed, the + verdict is ``unsafe`` / deny (an incomplete evaluation must never green-light + a restart). + """ + read_block = _profile_operation_gate("gitea.read") + if read_block: + return { + "success": False, + "read_only": True, + "dry_run": True, + "restart_performed": False, + "reasons": read_block, + "permission_report": _permission_block_report("gitea.read"), + } + + try: + h, o, r = _resolve(remote, host, org, repo) + except ValueError as exc: + return { + "success": False, + "read_only": True, + "dry_run": True, + "restart_performed": False, + "reasons": [str(exc)], + } + + inventory_complete = True + incomplete_reasons: list[str] = [] + sessions: list[dict] = [] + leases: list[dict] = [] + terminal_lock: dict | None = None + + db, db_errs = _control_plane_db_or_error() + if db is None: + inventory_complete = False + incomplete_reasons.extend( + db_errs or ["control-plane DB unavailable; cannot evaluate restart"] + ) + else: + try: + sessions = db.list_sessions(statuses=("active",), limit=max(1, int(limit))) + except Exception as exc: # noqa: BLE001 + inventory_complete = False + incomplete_reasons.append( + f"session inventory failed: {_redact(str(exc))}" + ) + try: + lease_result = lease_lifecycle.list_active_leases( + db, + remote=remote if remote in REMOTES else remote, + org=o, + repo=r, + role=None, + include_non_active=False, + limit=max(1, int(limit)), + ) + leases = list(lease_result.get("leases") or []) + except Exception as exc: # noqa: BLE001 + inventory_complete = False + incomplete_reasons.append( + f"lease inventory failed: {_redact(str(exc))}" + ) + try: + terminal = db.get_active_terminal_lock( + remote=remote if remote in REMOTES else remote, + org=o, + repo=r, + ) + if terminal: + terminal_lock = dict(terminal) + except Exception as exc: # noqa: BLE001 + inventory_complete = False + incomplete_reasons.append( + f"terminal lock lookup failed: {_redact(str(exc))}" + ) + + profile = get_profile() + profile_name = (profile.get("profile_name") or "").strip() or "session" + sid = (session_id or "").strip() or f"{profile_name}-{os.getpid()}" + + # Override authority is read from the environment only — a worker session + # cannot set an env var for an already-running daemon, so it cannot be + # self-asserted the way a tool argument could (#630/#710 F1 pattern). + operator_authorized = bool( + (os.environ.get("GITEA_OPERATOR_RESTART_OVERRIDE_AUTHORIZATION") or "").strip() + ) + operator_override = bool(request_override and operator_authorized) + + inventory = { + "sessions": sessions, + "leases": leases, + "terminal_lock": terminal_lock, + "inventory_complete": inventory_complete, + "incomplete_reasons": incomplete_reasons, + } + + report = restart_coordinator.evaluate_restart_impact( + inventory, + operator_override=operator_override, + requesting_session_id=sid, + dry_run=True, # coordinator is always analysis-only (#658) + ) + + payload = report.as_dict() + payload["success"] = True + payload["read_only"] = True + payload["remote"] = remote + payload["org"] = o + payload["repo"] = r + payload["requesting_session_id"] = sid + payload["operator_override_requested"] = bool(request_override) + payload["operator_override_authorized"] = operator_authorized + payload["apply_supported"] = False + if not dry_run: + payload["reasons"] = list(payload.get("reasons") or []) + [ + "apply requested but not supported: sanctioned restart apply is " + "gated by a drain proof (separate child); no restart performed (#658)" + ] + return payload + + @mcp.tool() def gitea_inspect_workflow_lease( lease_id: str, diff --git a/issue_lock_provenance.py b/issue_lock_provenance.py index 544e017..b79d8ad 100644 --- a/issue_lock_provenance.py +++ b/issue_lock_provenance.py @@ -16,6 +16,7 @@ ISSUE_LOCK_FILE = os.environ.get("GITEA_ISSUE_LOCK_FILE", "/tmp/gitea_issue_lock SOURCE_LOCK_ISSUE = "gitea_lock_issue" SOURCE_LOCK_ADOPTION = "gitea_lock_issue_adoption" SOURCE_OPERATOR_OVERRIDE = "operator_override" +SOURCE_RECOVER_DIRTY_ORPHANED = "gitea_recover_dirty_orphaned_issue_worktree" # #864: dirty-preserving same-claimant author-session rebind (dead owner PID). SOURCE_DIRTY_SAME_CLAIMANT_REBIND = ( "gitea_rebind_dirty_same_claimant_author_session" @@ -25,6 +26,7 @@ SANCTIONED_LOCK_SOURCES = frozenset({ SOURCE_LOCK_ISSUE, SOURCE_LOCK_ADOPTION, SOURCE_OPERATOR_OVERRIDE, + SOURCE_RECOVER_DIRTY_ORPHANED, SOURCE_DIRTY_SAME_CLAIMANT_REBIND, }) diff --git a/issue_lock_recovery.py b/issue_lock_recovery.py index ecc99c6..ae0cc35 100644 --- a/issue_lock_recovery.py +++ b/issue_lock_recovery.py @@ -85,6 +85,12 @@ HEAD_RELATION_STRICT_DESCENDANT = "strict_descendant" # #772: an unpublished claim has no recorded head to compare against at all, so # its head is measured against the base the branch was cut from instead. HEAD_RELATION_DESCENDS_FROM_BASE = "descends_from_recorded_base" +# #871: the remote/PR head advanced *past* the recorded head via a sanctioned +# merge-based branch synchronization (``gitea_update_pr_branch_by_merge``) while +# the local worktree stayed at the recorded head. This is the inverse of the +# #768 descendant relation — here the *remote* strictly descends the local head, +# and only because a base was merged into the branch, proven server-side. +HEAD_RELATION_REMOTE_MERGE_SYNCED = "remote_merge_synced" # Which body of evidence a recovery was decided on (#772 AC10). These are not # interchangeable: a published claim proves ownership against a remote/PR head, @@ -266,6 +272,70 @@ def _assess_base_descendancy( ] +def _assess_remote_merge_synced( + sync_provenance: Mapping[str, Any] | None, + *, + recorded_head: str, + remote_head: str, +) -> tuple[bool, list[str]]: + """Did ``remote_head`` advance past ``recorded_head`` via a sanctioned + merge-based branch sync (#871)? + + ``sync_provenance`` is the server-side git observation from + ``issue_lock_worktree.read_merge_sync_provenance``. Its own + ``prior_head_sha`` / ``synced_head_sha`` are re-checked against the heads + this assessment is actually reasoning about, so an observation taken for some + other pair of commits — stale, mismatched, or hand-built — can never + authorize recovery. This is the inverse of ``_assess_strict_descendant``: the + recorded head is the ancestor and the *remote* head is the descendant, and it + is accepted only because the remote head is a base-into-branch merge that + preserved the branch mainline back to the recorded head. + + Returns ``(proven, notes)``. Notes name the exact missing element so a + refused caller sees why, never a bare "unproven". + """ + if not isinstance(sync_provenance, Mapping): + return False, [ + "no server-derived merge-sync provenance observation was available; a " + "remote head ahead of the recorded head cannot be accepted" + ] + + probe_prior = _text(sync_provenance.get("prior_head_sha")) + probe_synced = _text(sync_provenance.get("synced_head_sha")) + if probe_prior != recorded_head or probe_synced != remote_head: + return False, [ + f"merge-sync observation covers {probe_prior or 'unknown'} -> " + f"{probe_synced or 'unknown'}, not the heads under assessment " + f"({recorded_head} -> {remote_head})" + ] + if not sync_provenance.get("probe_ok"): + return False, ( + list(sync_provenance.get("reasons") or []) + or ["merge-sync provenance probe did not complete; provenance unproven"] + ) + if not sync_provenance.get("prior_is_ancestor"): + return False, [ + f"recorded head {recorded_head} is not an ancestor of remote head " + f"{remote_head}; a rewritten or force-moved head cannot be recovered" + ] + if not sync_provenance.get("is_merge_sync"): + return False, ( + list(sync_provenance.get("reasons") or []) + or [ + f"remote head {remote_head} is not a sanctioned merge-based sync " + f"of the base into the branch above {recorded_head}" + ] + ) + + proof = _text(sync_provenance.get("proof")) or ( + f"{remote_head} merged the base into the branch above {recorded_head}" + ) + return True, [ + f"remote head {remote_head} advanced past recorded head {recorded_head} " + f"via a sanctioned merge-based branch sync ({proof})" + ] + + def assess_dead_session_lock_recovery( existing_lock: Mapping[str, Any] | None, *, @@ -290,6 +360,7 @@ def assess_dead_session_lock_recovery( remote_branch_exists: bool | None = None, recorded_base_sha: str | None = None, base_ancestry: Mapping[str, Any] | None = None, + sync_provenance: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Decide whether a dead-session author lock may be natively recovered. @@ -469,19 +540,39 @@ def assess_dead_session_lock_recovery( head_relation = HEAD_RELATION_STRICT_DESCENDANT ancestry_proof = notes[0] if notes else None else: - reasons.append( - f"local head {local_head} does not match remote branch head " - f"{remote_head}" + # #871: the reverse relation — the remote head advanced past + # the recorded/local head via a sanctioned merge-based branch + # sync while the local worktree stayed put. Accepted only on + # server-proven merge-sync provenance, never a caller claim. + synced, sync_notes = _assess_remote_merge_synced( + sync_provenance, + recorded_head=local_head, + remote_head=remote_head, ) - reasons.extend(notes) + if synced: + head_relation = HEAD_RELATION_REMOTE_MERGE_SYNCED + ancestry_proof = sync_notes[0] if sync_notes else None + else: + reasons.append( + f"local head {local_head} does not match remote branch " + f"head {remote_head}" + ) + reasons.extend(notes) + reasons.extend(sync_notes) evidence["recorded_base"] = recorded_base or None evidence["local_head"] = local_head or None evidence["remote_head"] = remote_head or None # ``recorded_head`` is the head recovery is being measured against; # ``accepted_head`` is the head this recovery actually adopts. They differ # only in the descendant case, and downstream gates need both (#768 AC2/AC7). + # #871: in the merge-sync case the branch/PR already carries the synced + # remote head, so that is the head recovery adopts; the local worktree stays + # at the ancestor recorded head. evidence["recorded_head"] = remote_head or None - evidence["accepted_head"] = local_head or None + if head_relation == HEAD_RELATION_REMOTE_MERGE_SYNCED: + evidence["accepted_head"] = remote_head or None + else: + evidence["accepted_head"] = local_head or None evidence["head_relation"] = head_relation evidence["ancestry_proof"] = ancestry_proof @@ -493,12 +584,17 @@ def assess_dead_session_lock_recovery( # contradictory; re-stating it as a head mismatch would only obscure why. if not unpublished and local_head and pr_head != local_head: # A descendant recovery has not been published yet, so the open PR - # legitimately still points at the recorded head. Any other - # disagreement is a real mismatch. + # legitimately still points at the recorded head. A merge-sync + # recovery's PR legitimately sits at the advanced remote head. Any + # other disagreement is a real mismatch. if not ( head_relation == HEAD_RELATION_STRICT_DESCENDANT and remote_head and pr_head == remote_head + ) and not ( + head_relation == HEAD_RELATION_REMOTE_MERGE_SYNCED + and remote_head + and pr_head == remote_head ): reasons.append( f"open PR #{pr_number} head {pr_head} does not match local head " @@ -619,7 +715,11 @@ def assess_dead_session_lock_recovery( ) if ( head_relation - in (HEAD_RELATION_STRICT_DESCENDANT, HEAD_RELATION_DESCENDS_FROM_BASE) + in ( + HEAD_RELATION_STRICT_DESCENDANT, + HEAD_RELATION_DESCENDS_FROM_BASE, + HEAD_RELATION_REMOTE_MERGE_SYNCED, + ) and ancestry_proof ): proof.append(ancestry_proof) @@ -697,6 +797,16 @@ def owning_pr_recovery_evidence( return None if accepted_head != local_head: return None + elif relation == HEAD_RELATION_REMOTE_MERGE_SYNCED: + # #871: the PR already sits at the advanced remote head; the local + # worktree is the ancestor the merge preserved. The head the open PR + # shows and the head recovery adopts are both the synced remote head. + if not remote_head or pr_head != remote_head: + return None + if accepted_head != remote_head: + return None + if not local_head or local_head == remote_head: + return None else: return None try: @@ -765,6 +875,18 @@ def recovered_owning_pr_from_lock( return None if not accepted_head or accepted_head == recorded_head: return None + elif relation == HEAD_RELATION_REMOTE_MERGE_SYNCED: + # #871: PR sits at the advanced remote head, which is both the recorded + # measured-against head and the adopted head; the local worktree is the + # ancestor the merge preserved. + remote_head = _text(record.get("remote_head")) + local_head = _text(record.get("local_head")) + if not remote_head or pr_head != remote_head: + return None + if accepted_head and accepted_head != remote_head: + return None + if not local_head or local_head == remote_head: + return None else: return None try: diff --git a/issue_lock_store.py b/issue_lock_store.py index 713fa2a..d011c1c 100644 --- a/issue_lock_store.py +++ b/issue_lock_store.py @@ -169,6 +169,7 @@ def bind_session_lock( *, expected_generation: int | None = None, renewal_sanctioned: bool = False, + recovery_sanctioned: bool = False, ) -> str: """Persist a keyed lock and bind it to the current process session. @@ -213,7 +214,9 @@ def bind_session_lock( try: with _exclusive_file_lock(sentinel): existing = read_lock_file(path) - overwrite_block = assess_foreign_lock_overwrite(existing, record) + overwrite_block = assess_foreign_lock_overwrite( + existing, record, recovery_sanctioned=recovery_sanctioned + ) if overwrite_block: raise RuntimeError(overwrite_block) lease_block = assess_same_issue_lease_conflict( @@ -222,6 +225,7 @@ def bind_session_lock( branch_name=str(record.get("branch_name") or ""), worktree_path=str(record.get("worktree_path") or ""), renewal_sanctioned=renewal_sanctioned, + recovery_sanctioned=recovery_sanctioned, ) if lease_block: raise RuntimeError(lease_block) @@ -380,7 +384,18 @@ def assess_lock_freshness( pid = lock_data.get("session_pid") if pid is None: pid = lock_data.get("pid") - pid_alive = is_process_alive(pid) if pid is not None else False + if pid is None: + pid = lock_data.get("owner_pid") + pid_missing = pid is None or str(pid).strip() == "" + try: + pid_int = int(pid) if not pid_missing else None + if pid_int is not None and pid_int <= 0: + pid_missing = True + pid_int = None + except (TypeError, ValueError): + pid_missing = True + pid_int = None + pid_alive = is_process_alive(pid_int) if pid_int is not None else False if expires_at and expires_at <= current: return { @@ -389,15 +404,36 @@ def assess_lock_freshness( "stale": True, "reason": f"lease expired at {expires_at.isoformat()}", "pid_alive": pid_alive, + "pid_missing": pid_missing, } - if pid is not None and not pid_alive: + # #860: a PID-less lock must never be considered live merely because + # expiration / heartbeat fields are absent. Missing PID is insufficient + # evidence of a live owner; treat as malformed/stale so recovery routes + # can evaluate corroborating pins instead of blocking on a false live flag. + if pid_missing: + return { + "status": "malformed", + "live": False, + "stale": True, + "reason": ( + "lock has no usable session pid; cannot prove live ownership " + "(PID-less locks are never live by missing expiry alone)" + ), + "pid_alive": False, + "pid_missing": True, + "heartbeat_at": heartbeat_at.isoformat() if heartbeat_at else None, + "expires_at": expires_at.isoformat() if expires_at else None, + } + + if pid_int is not None and not pid_alive: return { "status": "stale", "live": False, "stale": True, - "reason": f"owner pid {pid} is not alive", + "reason": f"owner pid {pid_int} is not alive", "pid_alive": False, + "pid_missing": False, } return { @@ -406,6 +442,7 @@ def assess_lock_freshness( "stale": False, "reason": "lock heartbeat and lease are fresh", "pid_alive": pid_alive, + "pid_missing": False, "heartbeat_at": heartbeat_at.isoformat() if heartbeat_at else None, "expires_at": expires_at.isoformat() if expires_at else None, } @@ -486,6 +523,7 @@ def assess_same_issue_lease_conflict( worktree_path: str, operation_type: str = AUTHOR_ISSUE_WORK_LEASE, renewal_sanctioned: bool = False, + recovery_sanctioned: bool = False, now: datetime | None = None, ) -> str | None: """Return a fail-closed error when a competing live lease blocks acquisition. @@ -517,6 +555,8 @@ def assess_same_issue_lease_conflict( existing_branch == branch_name and _same_realpath(str(existing_worktree or ""), worktree_path) ) + if recovery_sanctioned and existing_issue == issue_number and existing_branch == branch_name: + return None if is_lease_expired(existing_lock, now=now): # #760 AC1/AC2: exact-owner renewal is a different disposition from # foreign takeover and is evaluated first. Before this, both branches @@ -547,10 +587,26 @@ def assess_same_issue_lease_conflict( ) +def _lock_claimant(lock: dict[str, Any] | None) -> dict[str, str]: + if not isinstance(lock, dict): + return {} + claimant = lock.get("claimant") + if not isinstance(claimant, dict): + lease = lock.get("work_lease") + claimant = lease.get("claimant") if isinstance(lease, dict) else None + if not isinstance(claimant, dict): + return {} + return { + "username": str(claimant.get("username") or ""), + "profile": str(claimant.get("profile") or ""), + } + + def assess_foreign_lock_overwrite( existing_lock: dict[str, Any] | None, incoming_lock: dict[str, Any], *, + recovery_sanctioned: bool = False, now: datetime | None = None, ) -> str | None: """Block writes that would clobber an unrelated live lease on the same key.""" @@ -565,8 +621,31 @@ def assess_foreign_lock_overwrite( ) if same_issue and same_branch and same_worktree: return None - if not is_lease_live(existing_lock, now=now): + + existing_claimant = _lock_claimant(existing_lock) + incoming_claimant = _lock_claimant(incoming_lock) + same_claimant = ( + bool(existing_claimant.get("username")) + and existing_claimant.get("username") == incoming_claimant.get("username") + and existing_claimant.get("profile") == incoming_claimant.get("profile") + ) + + if recovery_sanctioned and same_issue and same_branch and same_claimant: return None + + if not is_lease_live(existing_lock, now=now): + # #860 F8: A non-live or PID-less lock still blocks foreign overwrite + # unless same claimant or sanctioned reclaim is proven. + if not same_claimant and same_issue: + reclaim = assess_expired_lock_reclaim(existing_lock, now=now) + if not reclaim.get("reclaim_allowed"): + return ( + "Refusing foreign overwrite of non-live issue lock " + f"(issue #{existing_lock.get('issue_number')}, owner '{existing_claimant.get('username')}') " + "without sanctioned reclaim proof (fail closed)" + ) + return None + return ( "Refusing to overwrite a live foreign issue lock " f"(issue #{existing_lock.get('issue_number')}, " @@ -737,4 +816,308 @@ def format_lock_proof( parts.append("lock released") elif released is False: parts.append("lock retained") - return "; ".join(parts) \ No newline at end of file + return "; ".join(parts) + + +# ── #871: durable linked-issue lock head refresh after branch synchronization ── +_FULL_SHA_RE = re.compile(r"^[0-9a-f]{40}$", re.IGNORECASE) + +# Provenance recorded on the lock when the head is refreshed by a sanctioned +# merge-based branch synchronization (``gitea_update_pr_branch_by_merge``). +LOCK_HEAD_REFRESH_PROVENANCE_MERGE_SYNC = "gitea_update_pr_branch_by_merge" + + +def _norm_sha(value: Any) -> str | None: + text = str(value or "").strip().lower() + return text if _FULL_SHA_RE.match(text) else None + + +def _lock_claimant_view(lock: dict[str, Any] | None) -> dict[str, Any]: + if not isinstance(lock, dict): + return {} + claimant = lock.get("claimant") + if not isinstance(claimant, dict): + lease = lock.get("work_lease") + claimant = lease.get("claimant") if isinstance(lease, dict) else None + return dict(claimant) if isinstance(claimant, dict) else {} + + +def assess_durable_lock_head_refresh( + existing_lock: dict[str, Any] | None, + *, + remote: str, + org: str, + repo: str, + issue_number: int, + branch_name: str, + worktree_path: str, + pr_number: int | None, + identity: str | None, + profile: str | None, + current_pid: int | None, + expected_old_head: str | None, + new_head: str | None, + base_head: str | None = None, +) -> dict[str, Any]: + """Fail-closed assessment for refreshing a durable lock's recorded head (#871). + + A successful ``gitea_update_pr_branch_by_merge`` advances the *remote* PR head + but must also advance the durable linked-issue lock so a later dead-session + recovery can prove ownership. This decides whether that refresh is permitted; + it mutates nothing. + + Every element of durable ownership is re-verified against the persisted lock — + repository, issue, branch, worktree, claimant identity/profile, and the live + owning session — and the recorded head is compare-and-swapped: the lock's + currently recorded synced head (if any) must equal ``expected_old_head``, so a + lock whose head or provenance changed concurrently is never overwritten. + """ + reasons: list[str] = [] + old = _norm_sha(expected_old_head) + new = _norm_sha(new_head) + evidence: dict[str, Any] = { + "issue_number": issue_number, + "branch_name": branch_name, + "worktree_path": worktree_path, + "pr_number": pr_number, + "expected_old_head": old, + "new_head": new, + "base_head": _norm_sha(base_head), + } + + if not isinstance(existing_lock, dict) or not existing_lock: + reasons.append("no durable lock exists for this issue; nothing to refresh") + return {"allowed": False, "reasons": reasons, "evidence": evidence, + "expected_generation": 0} + + lock = dict(existing_lock) + evidence["current_generation"] = lock_generation(lock) + + if lock.get("issue_number") != issue_number: + reasons.append( + f"durable lock targets issue #{lock.get('issue_number')}, not " + f"#{issue_number}; refusing head refresh" + ) + + for field, expected in (("remote", remote), ("org", org), ("repo", repo)): + actual = str(lock.get(field) or "").strip() + if actual != str(expected or "").strip(): + reasons.append( + f"lock {field} '{actual}' does not match requested " + f"'{str(expected or '').strip()}'" + ) + + locked_branch = str(lock.get("branch_name") or "").strip() + if locked_branch != str(branch_name or "").strip(): + reasons.append( + f"lock branch '{locked_branch}' does not match requested " + f"'{str(branch_name or '').strip()}'" + ) + + locked_worktree = str(lock.get("worktree_path") or "").strip() + try: + same_wt = bool(locked_worktree) and bool(worktree_path) and ( + os.path.realpath(locked_worktree) == os.path.realpath(worktree_path) + ) + except OSError: + same_wt = locked_worktree == (worktree_path or "") + if not same_wt: + reasons.append( + f"lock worktree '{locked_worktree}' does not match declared " + f"'{str(worktree_path or '').strip()}'" + ) + + claimant = _lock_claimant_view(lock) + locked_identity = str(claimant.get("username") or "").strip() + locked_profile = str(claimant.get("profile") or "").strip() + if not locked_identity or not locked_profile: + reasons.append( + "durable lock does not record a claimant identity/profile; " + "ownership could not be proven for head refresh" + ) + if not str(identity or "").strip() or not str(profile or "").strip(): + reasons.append( + "active session identity/profile is unknown; ownership could not be " + "proven for head refresh" + ) + if locked_identity and str(identity or "").strip() and locked_identity != str(identity).strip(): + reasons.append( + f"lock claimant '{locked_identity}' does not match active identity " + f"'{str(identity).strip()}'" + ) + if locked_profile and str(profile or "").strip() and locked_profile != str(profile).strip(): + reasons.append( + f"lock profile '{locked_profile}' does not match active profile " + f"'{str(profile).strip()}'" + ) + + # The refresh is written by the LIVE owning author session. A refresh is not + # a recovery: the current process must be the recorded owner. + recorded_pid = lock.get("session_pid") + if recorded_pid is None: + recorded_pid = lock.get("pid") + evidence["recorded_pid"] = recorded_pid + evidence["current_pid"] = current_pid + if current_pid is None: + reasons.append("current session pid is unknown; cannot prove live ownership") + else: + try: + if recorded_pid is None or int(recorded_pid) != int(current_pid): + reasons.append( + f"durable lock is owned by pid {recorded_pid}, not the current " + f"session pid {current_pid}; head refresh requires the live owner" + ) + except (TypeError, ValueError): + reasons.append( + "durable lock owner pid is malformed; cannot prove live ownership" + ) + + if not old: + reasons.append("expected_old_head is not a full 40-char hex SHA (fail closed)") + if not new: + reasons.append("new_head is not a full 40-char hex SHA (fail closed)") + if old and new and old == new: + reasons.append( + "new head equals the expected old head; a sync must advance the head" + ) + + # Compare-and-swap on the recorded head: if the lock already records a synced + # head it must be exactly the expected old head, else another sync moved it. + recorded_synced = _norm_sha(lock.get("synced_pr_head")) + evidence["recorded_synced_pr_head"] = recorded_synced + if recorded_synced is not None and old is not None and recorded_synced != old: + reasons.append( + f"durable lock already records synced head {recorded_synced}, not the " + f"expected old head {old}; a concurrent sync changed it (CAS fail closed)" + ) + + if reasons: + return {"allowed": False, "reasons": reasons, "evidence": evidence, + "expected_generation": lock_generation(lock)} + + return { + "allowed": True, + "reasons": [ + f"durable lock for issue #{issue_number} branch '{locked_branch}' is " + f"owned by the live session; refresh recorded head {old} -> {new}" + ], + "evidence": evidence, + "expected_generation": lock_generation(lock), + } + + +def apply_durable_lock_head_refresh( + *, + remote: str, + org: str, + repo: str, + issue_number: int, + branch_name: str, + worktree_path: str, + pr_number: int | None, + identity: str | None, + profile: str | None, + current_pid: int | None, + expected_old_head: str | None, + new_head: str | None, + synced_at: str, + base_head: str | None = None, + provenance: str = LOCK_HEAD_REFRESH_PROVENANCE_MERGE_SYNC, + lock_dir: str | None = None, +) -> dict[str, Any]: + """CAS-refresh the durable lock's recorded head after a branch sync (#871). + + Reads the durable lock from disk, re-asserts ownership via + ``assess_durable_lock_head_refresh``, and — only when permitted — writes the + new synced head through ``bind_session_lock`` with a generation compare-and- + swap. Then re-reads the lock and proves it records the complete new head + (read-after-write). Any failure at any step returns ``refreshed=False`` with + reasons; the caller must treat that as a partial lifecycle failure and never + report a fully successful synchronization. + """ + existing = load_issue_lock( + remote=remote, org=org, repo=repo, issue_number=issue_number, lock_dir=lock_dir + ) + assessment = assess_durable_lock_head_refresh( + existing, + remote=remote, + org=org, + repo=repo, + issue_number=issue_number, + branch_name=branch_name, + worktree_path=worktree_path, + pr_number=pr_number, + identity=identity, + profile=profile, + current_pid=current_pid, + expected_old_head=expected_old_head, + new_head=new_head, + base_head=base_head, + ) + result: dict[str, Any] = { + "refreshed": False, + "read_after_write_ok": False, + "prior_head": _norm_sha(expected_old_head), + "new_head": _norm_sha(new_head), + "reasons": list(assessment.get("reasons") or []), + "evidence": assessment.get("evidence"), + } + if not assessment.get("allowed"): + return result + + new = _norm_sha(new_head) + old = _norm_sha(expected_old_head) + record = dict(existing or {}) + sync_block = { + "last_synced_pr_head": new, + "prior_pr_head": old, + "base_head": _norm_sha(base_head), + "pr_number": pr_number, + "provenance": provenance, + "synced_at": synced_at, + "synced_by_pid": current_pid, + "synced_by": { + "username": str(identity or "").strip() or None, + "profile": str(profile or "").strip() or None, + }, + } + record["synced_pr_head"] = new + record["branch_sync"] = sync_block + history = record.get("branch_sync_history") + if not isinstance(history, list): + history = [] + history = list(history) + history.append(sync_block) + record["branch_sync_history"] = history + + try: + bind_session_lock( + record, + lock_dir=lock_dir, + expected_generation=assessment.get("expected_generation"), + renewal_sanctioned=True, + ) + except Exception as exc: # CAS miss or write failure — partial lifecycle failure + result["reasons"].append( + f"durable lock head refresh write failed (fail closed): {exc}" + ) + return result + + after = load_issue_lock( + remote=remote, org=org, repo=repo, issue_number=issue_number, lock_dir=lock_dir + ) + after_head = _norm_sha((after or {}).get("synced_pr_head")) + result["lock_generation_after"] = lock_generation(after) + if after_head == new and new is not None: + result["refreshed"] = True + result["read_after_write_ok"] = True + result["reasons"].append( + f"durable lock recorded head refreshed to {new} and verified by " + "read-after-write" + ) + else: + result["reasons"].append( + "read-after-write verification failed: durable lock does not record " + f"the new head {new} (found {after_head}); partial lifecycle failure" + ) + return result \ No newline at end of file diff --git a/issue_lock_worktree.py b/issue_lock_worktree.py index de52ff7..8188536 100644 --- a/issue_lock_worktree.py +++ b/issue_lock_worktree.py @@ -145,6 +145,175 @@ def read_head_ancestry( return result +def read_merge_sync_provenance( + worktree_path: str, + *, + prior_head_sha: str | None, + synced_head_sha: str | None, +) -> dict: + """Observe whether ``synced_head_sha`` is a sanctioned merge-based branch sync + that advanced the PR branch past ``prior_head_sha`` (#871). + + ``gitea_update_pr_branch_by_merge`` advances a PR branch by merging the base + branch *into* the branch (``POST /pulls/{n}/update?style=merge``). The result + is a merge commit ``M`` on the branch whose **first** parent is the prior + branch head and whose second parent is the base tip. When the owning session + then dies without the durable lock's recorded head being refreshed, the local + worktree still sits at ``prior_head_sha`` while the live PR head is ``M``. + + Recovering that drift safely requires proving the remote head is *exactly* + such a merge-sync — not a rewrite, rebase, force-push, or an unrelated + commit. This is that server-side observation. It reports facts only; the + disposition lives in ``issue_lock_recovery``. Every field is read from git in + the declared worktree — nothing is supplied by, or reachable from, an MCP + caller (#871). + + Provenance is proven only when ALL hold: + + * both commits are present (a rewritten/force-moved prior head leaves the + object graph and fails closed); + * ``prior_head_sha`` is a strict ancestor of ``synced_head_sha`` (the branch + history is preserved, never replaced); + * ``synced_head_sha`` is a merge commit (two or more parents), i.e. a base + merged in — a plain fast-forward of new direct commits is not a sync; + * ``prior_head_sha`` is an ancestor of the merge's **first** parent, so the + branch mainline (first-parent lineage) still reaches the prior head — a + rebase/force-push that re-authored the branch side fails this. + """ + path = (worktree_path or "").strip() + prior = (prior_head_sha or "").strip() + synced = (synced_head_sha or "").strip() + result: dict = { + "prior_head_sha": prior or None, + "synced_head_sha": synced or None, + "probe_ok": False, + "prior_present": False, + "synced_present": False, + "prior_is_ancestor": False, + "synced_is_merge": False, + "first_parent_reaches_prior": False, + "is_merge_sync": False, + "first_parent_sha": None, + "parent_count": None, + "proof": None, + "reasons": [], + } + if not path or not prior or not synced: + result["reasons"].append( + "merge-sync provenance probe requires a worktree path and both " + "commit SHAs" + ) + return result + if prior == synced: + result["reasons"].append( + "prior and synced heads are identical; no branch sync occurred" + ) + return result + + def _present(sha: str) -> bool: + res = subprocess.run( + ["git", "-C", path, "rev-parse", "--verify", "--quiet", f"{sha}^{{commit}}"], + capture_output=True, + text=True, + check=False, + ) + return res.returncode == 0 + + def _is_ancestor(ancestor: str, descendant: str) -> bool | None: + res = subprocess.run( + ["git", "-C", path, "merge-base", "--is-ancestor", ancestor, descendant], + capture_output=True, + text=True, + check=False, + ) + if res.returncode == 0: + return True + if res.returncode == 1: + return False + return None # failed probe — never a silent "no" + + try: + result["prior_present"] = _present(prior) + result["synced_present"] = _present(synced) + except OSError as exc: # git unavailable — fail closed, never assume + result["reasons"].append(f"merge-sync provenance probe could not run: {exc}") + return result + + if not result["prior_present"]: + result["reasons"].append( + f"prior head {prior} is not reachable in '{path}'; history may have " + "been rewritten or force-moved" + ) + if not result["synced_present"]: + result["reasons"].append( + f"synced head {synced} is not reachable in '{path}'" + ) + if not (result["prior_present"] and result["synced_present"]): + return result + + ancestor = _is_ancestor(prior, synced) + if ancestor is None: + result["reasons"].append( + "ancestry probe failed; merge-sync provenance unproven" + ) + return result + result["prior_is_ancestor"] = bool(ancestor) + if not ancestor: + result["reasons"].append( + f"prior head {prior} is not an ancestor of synced head {synced}; " + "the branch history was not preserved (not a merge-based sync)" + ) + return result + + parents_res = subprocess.run( + ["git", "-C", path, "rev-list", "--parents", "-n", "1", synced], + capture_output=True, + text=True, + check=False, + ) + if parents_res.returncode != 0: + result["reasons"].append( + f"could not read parents of {synced}; merge-sync provenance unproven" + ) + return result + tokens = (parents_res.stdout or "").split() + # tokens[0] is the commit itself; the rest are its parents. + parents = tokens[1:] + result["parent_count"] = len(parents) + result["synced_is_merge"] = len(parents) >= 2 + if not result["synced_is_merge"]: + result["probe_ok"] = True + result["reasons"].append( + f"synced head {synced} has {len(parents)} parent(s); a merge-based " + "branch sync produces a merge commit (two or more parents)" + ) + return result + first_parent = parents[0] + result["first_parent_sha"] = first_parent + + fp_reaches = _is_ancestor(prior, first_parent) if prior != first_parent else True + if fp_reaches is None: + result["reasons"].append( + "first-parent ancestry probe failed; merge-sync provenance unproven" + ) + return result + result["first_parent_reaches_prior"] = bool(fp_reaches) + result["probe_ok"] = True + if not fp_reaches: + result["reasons"].append( + f"merge first parent {first_parent} does not reach prior head " + f"{prior}; the branch mainline was re-authored (not a sanctioned sync)" + ) + return result + + result["is_merge_sync"] = True + result["proof"] = ( + f"synced head {synced} is a merge commit (parents={len(parents)}) whose " + f"first-parent lineage reaches prior head {prior}; base merged into branch" + ) + return result + + def read_recorded_base( worktree_path: str, *, diff --git a/mcp_restart_paths.py b/mcp_restart_paths.py new file mode 100644 index 0000000..087fe72 --- /dev/null +++ b/mcp_restart_paths.py @@ -0,0 +1,475 @@ +"""Inventory and fail-closed guards for MCP restart/reload/kill paths (#657). + +Single source of truth enumerating every code/script/doc path that can +restart, reload, reconnect, kill, or force-recreate an MCP process. Each path +is classified and linked to the guard that constrains it. The companion +human-readable inventory lives in ``docs/mcp-restart-path-inventory.md`` and is +kept in lock-step with this module by ``tests/test_mcp_restart_paths.py``. + +Design intent (aligns with #655 restart-coordinator roadmap): + +* **No unguarded full restart.** The in-process MCP daemon + (``gitea_mcp_server.py`` / ``mcp_server.py`` / ``role_session_router.py``) + must never replace or kill its own process — replacing the process after the + host wired up the stdio pipes desyncs the JSON-RPC transport (observed with + Antigravity/Cascade hosts). ``assert_no_daemon_self_replacement`` enforces + this against the live source tree. +* **No legacy auto-restart helper.** ``_trigger_mcp_auto_restart`` was removed + when the stale-runtime resolver became side-effect free (#685); + ``assert_auto_restart_helper_absent`` keeps it removed. +* **Unknown restart attempts fail closed.** LLM tools must route any restart + intent through a *registered* path. ``assert_restart_attempt_registered`` + raises ``UnknownRestartPathError`` for anything not in this inventory. +* **pkill stays forbidden (#630).** Manual daemon kills are classified as + contamination by :mod:`runtime_recovery_guard`; this module records that path + and the test asserts the classification still holds. + +This module performs no restarts, spawns no threads, and touches no config or +process state. It is pure inventory + read-only source assertions. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +# --- Classifications ------------------------------------------------------- + +#: A narrow, one-shot recovery that is safe by construction (e.g. a CLI wrapper +#: re-execing into the venv interpreter before importing anything, or an +#: in-process profile switch). Never targets the running MCP daemon process. +CLASS_SANCTIONED_NARROW = "sanctioned_narrow_recovery" + +#: The path detects a condition that would require a restart, then *fails +#: closed* on mutations and emits restart/reconnect guidance. It never restarts +#: the process itself (recovery is owned by the host/operator). +CLASS_GUARDED_FAIL_CLOSED = "guarded_fail_closed" + +#: The path is forbidden. Attempting it is a workflow-safety violation and, +#: where an LLM tool could invoke it, is marked as contamination. +CLASS_FORBIDDEN = "forbidden" + +#: A previously-existing unguarded restart primitive that has been deleted. A +#: regression guard keeps it absent. +CLASS_REMOVED = "removed" + +#: Behavior that lives in the host/IDE and is outside this process's control +#: (e.g. a manual ``/mcp reconnect``). Documented, not code-guarded here. +CLASS_HOST_RESIDUAL = "host_residual" + +VALID_CLASSIFICATIONS = frozenset( + { + CLASS_SANCTIONED_NARROW, + CLASS_GUARDED_FAIL_CLOSED, + CLASS_FORBIDDEN, + CLASS_REMOVED, + CLASS_HOST_RESIDUAL, + } +) + +#: The in-process MCP daemon modules. These must never self-replace/self-kill. +DAEMON_MODULES = ( + "gitea_mcp_server.py", + "mcp_server.py", + "role_session_router.py", +) + +#: The legacy auto-restart helper removed in #685. Must stay removed. +LEGACY_AUTO_RESTART_HELPER = "_trigger_mcp_auto_restart" + +#: Call patterns that would let the daemon replace or terminate its own +#: process. Matched as calls (trailing ``(``) so prose/docstring mentions such +#: as "we do NOT os.execv() here" or "never calls ``os._exit``" do not trip the +#: scanner (comment lines are stripped first regardless). +DAEMON_SELF_REPLACEMENT_PRIMITIVES = ( + "os.execv(", + "os.execve(", + "os.execvp(", + "os.execvpe(", + "os.kill(", + "os.killpg(", + "os._exit(", + "os.abort(", +) + + +@dataclass(frozen=True) +class RestartPath: + """One classified restart/reload/kill path in the inventory.""" + + path_id: str + title: str + mechanism: str + classification: str + guard: str + locations: tuple[str, ...] + references: tuple[str, ...] + residual_host: bool = False + notes: str = "" + + +class UnknownRestartPathError(RuntimeError): + """Raised when a restart attempt is not a registered, classified path.""" + + +# --- The inventory --------------------------------------------------------- + +_RESTART_PATHS: tuple[RestartPath, ...] = ( + RestartPath( + path_id="cli_venv_bootstrap_execv", + title="CLI wrapper venv re-exec", + mechanism=( + "Standalone CLI scripts re-exec into venv/bin/python3 via os.execv " + "at import top, guarded by `sys.executable != venv_python`." + ), + classification=CLASS_SANCTIONED_NARROW, + guard=( + "One-shot, pre-import bootstrap; runs before any MCP transport " + "exists and only when not already on the venv interpreter, so it " + "cannot desync a live daemon. Idempotent guard condition prevents " + "a re-exec loop." + ), + locations=( + "create_pr.py", + "create_issue.py", + "close_issue.py", + "merge_pr.py", + "review_pr.py", + "edit_pr.py", + "delete_branch.py", + "mark_issue.py", + "manage_labels.py", + "list_issues.py", + "list_prs.py", + ), + references=("#657",), + ), + RestartPath( + path_id="daemon_self_replacement", + title="MCP daemon self-replacement", + mechanism=( + "The in-process MCP daemon replacing/terminating its own process " + "(os.execv/os.kill/os._exit) to reload code." + ), + classification=CLASS_FORBIDDEN, + guard=( + "Forbidden by design: replacing the process after the host wired " + "up stdio desyncs JSON-RPC (Antigravity/Cascade). Enforced against " + "the source tree by assert_no_daemon_self_replacement()." + ), + locations=("gitea_mcp_server.py:~155 (decision comment)",) + DAEMON_MODULES, + references=("#657", "#584"), + ), + RestartPath( + path_id="legacy_auto_restart_helper", + title="Legacy _trigger_mcp_auto_restart helper", + mechanism=( + "A helper that actively restarted the MCP server from the " + "read-only resolver path." + ), + classification=CLASS_REMOVED, + guard=( + "Removed in #685 when the resolver became side-effect free. Kept " + "absent by assert_auto_restart_helper_absent()." + ), + locations=("gitea_mcp_server.py", "mcp_server.py"), + references=("#685", "#657"), + ), + RestartPath( + path_id="config_touch_reload", + title="MCP client config-touch reload", + mechanism=( + "Touching (utime) the MCP client config file to make the host " + "reload/recreate the server process." + ), + classification=CLASS_REMOVED, + guard=( + "Removed from the resolver in #685: stale-runtime detection is " + "report-only and never mutates client config, spawns threads, or " + "calls os._exit." + ), + locations=("gitea_mcp_server.py (resolve_task_capability)",), + references=("#685", "#657"), + ), + RestartPath( + path_id="master_advance_auto_restart", + title="Master-advance staleness gate", + mechanism=( + "On-disk master advancing past the running code. The master-parity " + "gate detects it and fails mutations closed with restart guidance." + ), + classification=CLASS_GUARDED_FAIL_CLOSED, + guard=( + "Detect + fail closed only; the process never self-restarts. " + "master_parity_gate captures startup parity and blocks mutations " + "while stale, emitting restart/reconnect guidance." + ), + locations=( + "master_parity_gate.py", + "gitea_mcp_server.py (gitea_assess_master_parity)", + ), + references=("#420", "#591", "#657"), + ), + RestartPath( + path_id="stale_runtime_resolver_reconnect", + title="Stale-runtime resolver reconnect guidance", + mechanism=( + "The capability resolver detecting a stale serving process and " + "reporting restart_required/stop_required for a client reconnect." + ), + classification=CLASS_GUARDED_FAIL_CLOSED, + guard=( + "Report-only (#685): returns restart_required/stop_required and an " + "exact_safe_next_action pointing at IDE/client reconnect; performs " + "no restart, thread spawn, config touch, or os._exit." + ), + locations=("gitea_mcp_server.py (gitea_resolve_task_capability)",), + references=("#685", "#657"), + ), + RestartPath( + path_id="manual_daemon_kill", + title="Manual daemon kill (pkill/killall/kill)", + mechanism=( + "Shell kills of the MCP daemon: `pkill -f mcp_server.py`, " + "`killall`, broad `pkill -f python` sweeps, or `kill ` of a " + "daemon pid." + ), + classification=CLASS_FORBIDDEN, + guard=( + "Forbidden (#630): runtime_recovery_guard classifies these as " + "contamination and gitea_record_daemon_process_kill_attempt writes " + "a durable marker that fails subsequent mutations closed. Operator " + "maintenance authorization is read only from the environment, not " + "from a tool argument." + ), + locations=( + "runtime_recovery_guard.py", + "gitea_mcp_server.py (gitea_record_daemon_process_kill_attempt)", + ), + references=("#630", "#657"), + ), + RestartPath( + path_id="conflict_marker_infra_stop", + title="Startup conflict-marker infra stop", + mechanism=( + "The daemon entrypoint scans for unresolved merge-conflict markers " + "at startup and stops (sys.exit(1)) if found." + ), + classification=CLASS_GUARDED_FAIL_CLOSED, + guard=( + "Fail-closed startup stop, not a restart: the process exits and " + "waits for the operator to resolve conflicts and relaunch. Never " + "self-restarts or loops." + ), + locations=("mcp_server.py (check_conflict_markers)",), + references=("#657",), + ), + RestartPath( + path_id="ide_client_reconnect", + title="Host/IDE MCP reconnect", + mechanism=( + "A manual `/mcp reconnect` (or equivalent host action) that the " + "IDE performs to recreate the MCP client connection." + ), + classification=CLASS_HOST_RESIDUAL, + guard=( + "Outside this process's control. It is the sanctioned recovery the " + "gates point operators toward; documented as residual host " + "behavior. No in-process code initiates it." + ), + locations=("host/IDE",), + references=("#584", "#656", "#657"), + residual_host=True, + ), + RestartPath( + path_id="profile_switch_runtime", + title="Runtime profile switch", + mechanism=( + "Switching the active execution profile at runtime " + "(dynamic-profile mode)." + ), + classification=CLASS_SANCTIONED_NARROW, + guard=( + "In-process and restart-free: runtime_switching_supported is true, " + "so a profile switch rebinds capability without recreating the " + "process. No restart primitive is invoked." + ), + locations=("gitea_mcp_server.py (gitea_activate_profile)",), + references=("#656", "#657"), + ), +) + +_BY_ID: dict[str, RestartPath] = {p.path_id: p for p in _RESTART_PATHS} + + +# --- Read-only accessors --------------------------------------------------- + + +def iter_restart_paths() -> tuple[RestartPath, ...]: + """Return the full inventory as an immutable tuple.""" + + return _RESTART_PATHS + + +def restart_path_ids() -> frozenset[str]: + """Return the set of registered path ids.""" + + return frozenset(_BY_ID) + + +def get_restart_path(path_id: str) -> RestartPath: + """Return the registered path, or raise :class:`UnknownRestartPathError`.""" + + try: + return _BY_ID[path_id] + except KeyError as exc: + raise UnknownRestartPathError( + f"unknown restart path id {path_id!r}; not in the #657 inventory" + ) from exc + + +def paths_by_classification(classification: str) -> tuple[RestartPath, ...]: + """Return all registered paths with the given classification.""" + + if classification not in VALID_CLASSIFICATIONS: + raise ValueError(f"unknown classification {classification!r}") + return tuple(p for p in _RESTART_PATHS if p.classification == classification) + + +def assert_restart_attempt_registered(path_id: str) -> RestartPath: + """Fail closed unless ``path_id`` is a registered, classified restart path. + + LLM tools that intend to trigger any restart/reload/reconnect must name a + registered path so an unknown/novel restart primitive cannot slip through + silently. Forbidden and removed paths are registered too — this only + asserts the attempt is *known*, not that it is *permitted*; callers must + still honor the classification. + """ + + return get_restart_path(path_id) + + +def assert_registry_wellformed() -> None: + """Validate the inventory's own invariants (fail closed on drift).""" + + seen: set[str] = set() + for path in _RESTART_PATHS: + if path.path_id in seen: + raise ValueError(f"duplicate restart path id {path.path_id!r}") + seen.add(path.path_id) + if path.classification not in VALID_CLASSIFICATIONS: + raise ValueError( + f"{path.path_id!r} has invalid classification " + f"{path.classification!r}" + ) + if not path.guard.strip(): + raise ValueError(f"{path.path_id!r} is missing a guard description") + if not path.references: + raise ValueError(f"{path.path_id!r} is missing references") + if not path.locations: + raise ValueError(f"{path.path_id!r} is missing locations") + if path.classification == CLASS_HOST_RESIDUAL and not path.residual_host: + raise ValueError( + f"{path.path_id!r} is host_residual but residual_host is False" + ) + + +# --- Source-tree guards ---------------------------------------------------- + + +def _repo_root(root: str | os.PathLike[str] | None = None) -> Path: + if root is not None: + return Path(root) + return Path(__file__).resolve().parent + + +def _iter_code_lines(text: str) -> Iterable[tuple[int, str]]: + """Yield (1-based lineno, line) for lines that are not full-line comments.""" + + for lineno, line in enumerate(text.splitlines(), start=1): + if line.lstrip().startswith("#"): + continue + yield lineno, line + + +def scan_daemon_self_replacement( + root: str | os.PathLike[str] | None = None, +) -> list[dict[str, object]]: + """Return violations where a daemon module could self-replace/self-kill. + + Scans :data:`DAEMON_MODULES` for calls in + :data:`DAEMON_SELF_REPLACEMENT_PRIMITIVES`. Full-line comments are ignored, + and only call forms (with a trailing ``(``) match, so decision comments and + docstrings that merely mention the primitives do not produce false hits. + """ + + repo = _repo_root(root) + violations: list[dict[str, object]] = [] + for module in DAEMON_MODULES: + path = repo / module + if not path.exists(): + continue + text = path.read_text(encoding="utf-8", errors="replace") + for lineno, line in _iter_code_lines(text): + for primitive in DAEMON_SELF_REPLACEMENT_PRIMITIVES: + if primitive in line: + violations.append( + { + "module": module, + "line": lineno, + "primitive": primitive, + "text": line.strip(), + } + ) + return violations + + +def assert_no_daemon_self_replacement( + root: str | os.PathLike[str] | None = None, +) -> None: + """Fail closed if any daemon module can restart/kill its own process.""" + + violations = scan_daemon_self_replacement(root) + if violations: + rendered = "; ".join( + f"{v['module']}:{v['line']} {v['primitive']}" for v in violations + ) + raise AssertionError( + "MCP daemon must never self-replace/self-kill (#657); found: " + f"{rendered}" + ) + + +def scan_auto_restart_helper( + root: str | os.PathLike[str] | None = None, +) -> list[dict[str, object]]: + """Return occurrences of a *definition* of the legacy auto-restart helper.""" + + repo = _repo_root(root) + needle = f"def {LEGACY_AUTO_RESTART_HELPER}" + hits: list[dict[str, object]] = [] + for module in DAEMON_MODULES: + path = repo / module + if not path.exists(): + continue + text = path.read_text(encoding="utf-8", errors="replace") + for lineno, line in _iter_code_lines(text): + if needle in line: + hits.append({"module": module, "line": lineno}) + return hits + + +def assert_auto_restart_helper_absent( + root: str | os.PathLike[str] | None = None, +) -> None: + """Fail closed if the removed ``_trigger_mcp_auto_restart`` reappears.""" + + hits = scan_auto_restart_helper(root) + if hits: + rendered = "; ".join(f"{h['module']}:{h['line']}" for h in hits) + raise AssertionError( + f"{LEGACY_AUTO_RESTART_HELPER} was removed in #685 and must not " + f"return (#657); found definition at: {rendered}" + ) diff --git a/pr_work_lease.py b/pr_work_lease.py index 2fb6876..b877f10 100644 --- a/pr_work_lease.py +++ b/pr_work_lease.py @@ -228,25 +228,74 @@ def find_active_reviewer_lease( return None +def _conflict_fix_chain_key(lease: dict) -> tuple | None: + """Identity of the lease chain a conflict-fix marker belongs to (#842). + + Keyed by PR number, profile, head_before, and branch. Returns None when any + required component (pr_number, profile, head_before) is missing or malformed. + """ + raw = lease.get("raw_fields") or {} + pr_number = lease.get("pr_number") + profile = (lease.get("profile") or "").strip().lower() + head_before = lease.get("head_before") + branch = (lease.get("branch") or raw.get("branch") or "").strip() + if not (pr_number and profile and head_before): + return None + return (pr_number, profile, head_before, branch) + + +def _conflict_fix_chain_matches(key1: tuple, key2: tuple) -> bool: + """True when two conflict-fix chain keys refer to the same lease chain.""" + pr1, profile1, head1, branch1 = key1 + pr2, profile2, head2, branch2 = key2 + if pr1 != pr2 or profile1 != profile2 or head1 != head2: + return False + if branch1 and branch2 and branch1 != branch2: + return False + return True + + +def _conflict_fix_chain_terminated_after(entries: list[dict], index: int) -> bool: + """True when a later marker terminates the conflict-fix chain of ``entries[index]``. + + Append-only newest-wins: a terminal marker (phase=released/blocked/done) + ends only its matching claim chain (#842). + """ + key = _conflict_fix_chain_key(entries[index]) + if key is None: + return False + for later in entries[index + 1:]: + phase = (later.get("phase") or "").strip().lower() + if phase not in _TERMINAL_CONFLICT_FIX_PHASES: + continue + later_key = _conflict_fix_chain_key(later) + if later_key and _conflict_fix_chain_matches(key, later_key): + return True + return False + + def find_active_conflict_fix_lease( comments: list[dict], *, pr_number: int, now: datetime | None = None, ) -> dict[str, Any] | None: - """Return the newest unexpired conflict-fix lease for *pr_number*, if any.""" + """Return the newest unexpired, non-terminated conflict-fix lease for *pr_number*, if any.""" now = now or datetime.now(timezone.utc) candidates = [ entry for entry in _comment_entries(comments, pr_number=pr_number) if entry.get("lease_kind") == "conflict_fix" ] - for lease in reversed(candidates): + for index in range(len(candidates) - 1, -1, -1): + lease = candidates[index] if _lease_expired(lease, now=now): continue phase = (lease.get("phase") or "").strip().lower() if phase in _TERMINAL_CONFLICT_FIX_PHASES: continue if phase in _ACTIVE_CONFLICT_FIX_PHASES or phase: + if _conflict_fix_chain_terminated_after(candidates, index): + continue return lease return None diff --git a/restart_coordinator.py b/restart_coordinator.py new file mode 100644 index 0000000..5db7de0 --- /dev/null +++ b/restart_coordinator.py @@ -0,0 +1,451 @@ +"""MCP restart coordinator and impact analysis (#658). + +Before any sanctioned MCP restart, a central coordinator must evaluate the +live control-plane state — active sessions, leases/locks, in-flight issue/PR +work, mutations, worktrees, and recovery history — and produce an *impact +preview* so operators (and the web console, #642/#652) can see the blast +radius **before** concurrent LLM work is disrupted. + +Design rules (mirrors the read-only posture of ``workflow_dashboard`` / +``lease_lifecycle``): + +* **Pure classification.** :func:`evaluate_restart_impact` takes an already + gathered inventory and returns a structured report. It never touches the + network, the filesystem, or a live process, so multi-session fixtures can + drive every branch in unit tests. The coordinator *never restarts anything*; + a mutative apply path is a later child gated by a drain proof (non-goal here). +* **Fail closed.** If the inventory is not explicitly complete, the verdict is + ``unsafe`` / deny — an incomplete evaluation must never green-light a restart. +* **No secrets.** Session ids, pids, and profiles are operational metadata, not + credentials; nothing secret flows through this module. + +The single sanctioned entry point post-#657 is the MCP tool +``gitea_request_mcp_restart`` (dry-run by default), which gathers the inventory +from the #613 control-plane DB and calls :func:`evaluate_restart_impact`. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Mapping, Sequence + +import lease_lifecycle + +COORDINATOR_VERSION = "1.0.0-issue-658" + +# Restart verdicts. Exactly the three the acceptance criteria name. +VERDICT_SAFE = "safe" +VERDICT_UNSAFE = "unsafe" +VERDICT_OVERRIDE = "override" + +# Blast-radius severity bands. +BLAST_NONE = "none" +BLAST_LOW = "low" +BLAST_MEDIUM = "medium" +BLAST_HIGH = "high" + +# A live lease with a live owner process is treated as active in-flight work. +LEASE_FRESHNESS_LIVE = "active" + +# Default staleness window for a session heartbeat (seconds). A session whose +# last heartbeat is older than this is not counted as live even if its row is +# still marked ``active`` — it is assumed dead/detached. +DEFAULT_SESSION_HEARTBEAT_STALE_SECONDS = 900 + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _parse_ts(value: str | None) -> datetime | None: + return lease_lifecycle._parse_ts(value) + + +@dataclass(frozen=True) +class SessionImpact: + """One MCP session a restart would terminate.""" + + session_id: str + role: str | None + profile: str | None + pid: int | None + status: str | None + alive: bool | None + heartbeat_stale: bool + is_requester: bool + live: bool + + def as_dict(self) -> dict[str, Any]: + return { + "session_id": self.session_id, + "role": self.role, + "profile": self.profile, + "pid": self.pid, + "status": self.status, + "alive": self.alive, + "heartbeat_stale": self.heartbeat_stale, + "is_requester": self.is_requester, + "live": self.live, + } + + +@dataclass(frozen=True) +class LeaseImpact: + """One control-plane lease a restart would disrupt.""" + + lease_id: str | None + session_id: str | None + role: str | None + phase: str | None + freshness: str | None + work_kind: str | None + work_number: int | None + worktree_path: str | None + disruptive: bool + is_mutation: bool + is_critical_section: bool + + def as_dict(self) -> dict[str, Any]: + return { + "lease_id": self.lease_id, + "session_id": self.session_id, + "role": self.role, + "phase": self.phase, + "freshness": self.freshness, + "work_kind": self.work_kind, + "work_number": self.work_number, + "worktree_path": self.worktree_path, + "disruptive": self.disruptive, + "is_mutation": self.is_mutation, + "is_critical_section": self.is_critical_section, + } + + +@dataclass(frozen=True) +class RestartImpactReport: + """Impact preview DTO returned to the console / operator (#642/#652).""" + + coordinator_version: str + evaluated_at: str + dry_run: bool + restart_performed: bool + inventory_complete: bool + verdict: str + allow_restart: bool + override_would_allow: bool + operator_override: bool + blast_radius: str + reasons: list[str] + affected_sessions: list[SessionImpact] + affected_leases: list[LeaseImpact] + critical_sections: list[LeaseImpact] + affected_issues: list[int] + affected_prs: list[int] + mutations: list[LeaseImpact] + terminal_lock: dict[str, Any] | None + ack_state: dict[str, str] + prior_recovery_attempts: list[dict[str, Any]] + counts: dict[str, int] + audit_record: dict[str, Any] + incomplete_reasons: list[str] = field(default_factory=list) + + def as_dict(self) -> dict[str, Any]: + return { + "coordinator_version": self.coordinator_version, + "evaluated_at": self.evaluated_at, + "dry_run": self.dry_run, + "restart_performed": self.restart_performed, + "inventory_complete": self.inventory_complete, + "incomplete_reasons": list(self.incomplete_reasons), + "verdict": self.verdict, + "allow_restart": self.allow_restart, + "override_would_allow": self.override_would_allow, + "operator_override": self.operator_override, + "blast_radius": self.blast_radius, + "reasons": list(self.reasons), + "affected_sessions": [s.as_dict() for s in self.affected_sessions], + "affected_leases": [l.as_dict() for l in self.affected_leases], + "critical_sections": [l.as_dict() for l in self.critical_sections], + "affected_issues": list(self.affected_issues), + "affected_prs": list(self.affected_prs), + "mutations": [l.as_dict() for l in self.mutations], + "terminal_lock": self.terminal_lock, + "ack_state": dict(self.ack_state), + "prior_recovery_attempts": list(self.prior_recovery_attempts), + "counts": dict(self.counts), + "audit_record": dict(self.audit_record), + } + + +def _classify_session( + row: Mapping[str, Any], + *, + now: datetime, + requesting_session_id: str | None, + heartbeat_stale_seconds: int, +) -> SessionImpact: + session_id = str(row.get("session_id") or "") + pid = row.get("pid") + status = (row.get("status") or "").strip().lower() or None + alive = lease_lifecycle.is_process_alive(pid) if pid is not None else None + hb = _parse_ts(row.get("last_heartbeat_at")) + heartbeat_stale = bool( + hb is not None and (now - hb).total_seconds() > heartbeat_stale_seconds + ) + live = bool(status == "active" and alive is not False and not heartbeat_stale) + return SessionImpact( + session_id=session_id, + role=row.get("role"), + profile=row.get("profile"), + pid=pid, + status=status, + alive=alive, + heartbeat_stale=heartbeat_stale, + is_requester=bool( + requesting_session_id and session_id == requesting_session_id + ), + live=live, + ) + + +# Lease phases that represent an active mutation in flight (as opposed to a +# mere allocation/claim with no work committed yet). An active lease in any of +# these phases is a critical section a restart must not sever. +_MUTATING_PHASES = frozenset( + { + "implementing", + "publishing", + "pushing", + "committing", + "reviewing", + "merging", + "reconciling", + "conflict_fix", + } +) + + +def _classify_lease(row: Mapping[str, Any]) -> LeaseImpact: + freshness_obj = row.get("freshness") + if isinstance(freshness_obj, Mapping): + freshness = str(freshness_obj.get("freshness") or "").strip().lower() or None + else: + freshness = str(freshness_obj or "").strip().lower() or None + phase = (row.get("phase") or "").strip().lower() or None + worktree = row.get("worktree_path") + disruptive = freshness == LEASE_FRESHNESS_LIVE + # A live lease is a mutation-in-flight if it carries an author worktree or + # its phase names a mutating step. All disruptive leases are critical + # sections a restart would sever regardless. + is_mutation = bool( + disruptive and (bool(worktree) or (phase in _MUTATING_PHASES)) + ) + number = row.get("work_number") + try: + number = int(number) if number is not None else None + except (TypeError, ValueError): + number = None + return LeaseImpact( + lease_id=row.get("lease_id"), + session_id=row.get("session_id"), + role=row.get("role"), + phase=phase, + freshness=freshness, + work_kind=(str(row.get("work_kind") or "").strip().lower() or None), + work_number=number, + worktree_path=worktree, + disruptive=disruptive, + is_mutation=is_mutation, + is_critical_section=disruptive, + ) + + +def _blast_radius(*, session_count: int, work_count: int, mutation_count: int) -> str: + if mutation_count > 0 or work_count >= 3 or session_count >= 3: + return BLAST_HIGH + if work_count > 0 or session_count == 2: + return BLAST_MEDIUM + if session_count == 1: + return BLAST_LOW + return BLAST_NONE + + +def evaluate_restart_impact( + inventory: Mapping[str, Any], + *, + now: datetime | None = None, + operator_override: bool = False, + requesting_session_id: str | None = None, + dry_run: bool = True, + session_heartbeat_stale_seconds: int = DEFAULT_SESSION_HEARTBEAT_STALE_SECONDS, +) -> RestartImpactReport: + """Evaluate a proposed MCP restart and return an impact preview. + + ``inventory`` is a mapping with: + + * ``sessions`` — session rows (session_id, role, profile, pid, status, + last_heartbeat_at). + * ``leases`` — control-plane lease rows, each ideally carrying an enriched + ``freshness`` dict (as :func:`lease_lifecycle.list_active_leases` returns); + a bare string freshness is also accepted. + * ``terminal_lock`` — the active terminal (merge) lock, if any. + * ``prior_recovery_attempts`` — narrower recovery attempts already tried + (e.g. sanctioned reconnects) so the operator sees escalation history. + * ``inventory_complete`` — bool. **Must** be explicitly True; a missing or + falsy value forces a deny (fail closed). + * ``incomplete_reasons`` — optional reasons the inventory is incomplete. + + The coordinator never restarts anything: ``restart_performed`` is always + False and the mutative apply path is a later drain-gated child. + """ + moment = now or _utc_now() + reasons: list[str] = [] + + inventory_complete = bool(inventory.get("inventory_complete", False)) + incomplete_reasons = [str(r) for r in (inventory.get("incomplete_reasons") or [])] + + sessions_raw: Sequence[Mapping[str, Any]] = inventory.get("sessions") or [] + leases_raw: Sequence[Mapping[str, Any]] = inventory.get("leases") or [] + terminal_lock = inventory.get("terminal_lock") or None + prior_recovery_attempts = [ + dict(a) for a in (inventory.get("prior_recovery_attempts") or []) + ] + + session_impacts = [ + _classify_session( + s, + now=moment, + requesting_session_id=requesting_session_id, + heartbeat_stale_seconds=session_heartbeat_stale_seconds, + ) + for s in sessions_raw + ] + lease_impacts = [_classify_lease(l) for l in leases_raw] + + # Only *other* live sessions and live leases constitute blast radius: a + # restart that would kill only the requesting session with no other work in + # flight is safe. + other_live_sessions = [ + s for s in session_impacts if s.live and not s.is_requester + ] + disruptive_leases = [l for l in lease_impacts if l.disruptive] + critical_sections = [l for l in lease_impacts if l.is_critical_section] + mutations = [l for l in lease_impacts if l.is_mutation] + + affected_issues = sorted( + { + l.work_number + for l in disruptive_leases + if l.work_kind == "issue" and l.work_number is not None + } + ) + affected_prs = sorted( + { + l.work_number + for l in disruptive_leases + if l.work_kind == "pr" and l.work_number is not None + } + ) + + disruptive = bool(disruptive_leases or other_live_sessions or terminal_lock) + + if not inventory_complete: + verdict = VERDICT_UNSAFE + allow_restart = False + reasons.append( + "inventory incomplete: restart evaluation cannot confirm blast " + "radius — deny (fail closed, #658)" + ) + reasons.extend(incomplete_reasons) + elif not disruptive: + verdict = VERDICT_SAFE + allow_restart = True + reasons.append("no other live sessions, live leases, or terminal lock") + elif operator_override: + verdict = VERDICT_OVERRIDE + allow_restart = True + reasons.append( + "live work present; operator override accepts the blast radius" + ) + else: + verdict = VERDICT_UNSAFE + allow_restart = False + reasons.append( + "live work would be disrupted; restart denied without operator " + "override" + ) + + if critical_sections and inventory_complete: + reasons.append( + f"{len(critical_sections)} critical section(s) in flight " + "(active lease with a live owner)" + ) + if terminal_lock: + reasons.append("active terminal (merge) lock present") + + override_would_allow = bool(inventory_complete and disruptive) + + blast_radius = _blast_radius( + session_count=len(other_live_sessions), + work_count=len(affected_issues) + len(affected_prs), + mutation_count=len(mutations), + ) + + # Acknowledgement is a later child (drain protocol); expose per-session + # placeholders so the console can render the ack column now. + ack_state = {s.session_id: "pending" for s in other_live_sessions} + + counts = { + "sessions_total": len(session_impacts), + "sessions_live_other": len(other_live_sessions), + "leases_total": len(lease_impacts), + "leases_disruptive": len(disruptive_leases), + "critical_sections": len(critical_sections), + "mutations": len(mutations), + "affected_issues": len(affected_issues), + "affected_prs": len(affected_prs), + "prior_recovery_attempts": len(prior_recovery_attempts), + } + + audit_record = { + "event": "restart_impact_evaluated", + "coordinator_version": COORDINATOR_VERSION, + "evaluated_at": moment.isoformat(), + "dry_run": dry_run, + "operator_override": bool(operator_override), + "requesting_session_id": requesting_session_id, + "inventory_complete": inventory_complete, + "verdict": verdict, + "allow_restart": allow_restart, + "blast_radius": blast_radius, + "counts": counts, + } + + return RestartImpactReport( + coordinator_version=COORDINATOR_VERSION, + evaluated_at=moment.isoformat(), + dry_run=dry_run, + restart_performed=False, + inventory_complete=inventory_complete, + verdict=verdict, + allow_restart=allow_restart, + override_would_allow=override_would_allow, + operator_override=bool(operator_override), + blast_radius=blast_radius, + reasons=reasons, + affected_sessions=session_impacts, + affected_leases=lease_impacts, + critical_sections=critical_sections, + affected_issues=affected_issues, + affected_prs=affected_prs, + mutations=mutations, + terminal_lock=dict(terminal_lock) + if isinstance(terminal_lock, Mapping) + else terminal_lock, + ack_state=ack_state, + prior_recovery_attempts=prior_recovery_attempts, + counts=counts, + audit_record=audit_record, + incomplete_reasons=incomplete_reasons, + ) diff --git a/scripts/worktree-start b/scripts/worktree-start index a189164..a79e908 100755 --- a/scripts/worktree-start +++ b/scripts/worktree-start @@ -43,19 +43,21 @@ repo_root="$(cd "$script_dir/.." && pwd)" # Enforce issue-linked, traceable branch names (issue → branch → worktree → PR). if [[ "$allow_unlinked" -eq 0 ]]; then - locked_branch=$(python3 -c " + if [[ "$dry_run" -eq 0 ]] && [[ ! "$branch" =~ ^review/pr-[0-9]+-.+ ]]; then + locked_branch=$(python3 -c " import sys sys.path.insert(0, '$repo_root') import issue_lock_store print(issue_lock_store.resolve_locked_branch_for_session('$branch')) ") - if [[ -z "$locked_branch" ]]; then - echo "Error: No session issue lock is bound. Call gitea_lock_issue before branch creation (fail closed)." >&2 - exit 2 - fi - if [[ "$branch" != "$locked_branch" ]]; then - echo "Error: Requested branch '$branch' does not match locked branch '$locked_branch' (fail closed)." >&2 - exit 2 + if [[ -z "$locked_branch" ]]; then + echo "Error: No session issue lock is bound. Call gitea_lock_issue before branch creation (fail closed)." >&2 + exit 2 + fi + if [[ "$branch" != "$locked_branch" ]]; then + echo "Error: Requested branch '$branch' does not match locked branch '$locked_branch' (fail closed)." >&2 + exit 2 + fi fi if [[ "$branch" =~ ^(fix|feat|docs|chore)/issue-[0-9]+-.+ ]] \ diff --git a/task_capability_map.py b/task_capability_map.py index 7d7b76b..213e701 100644 --- a/task_capability_map.py +++ b/task_capability_map.py @@ -32,6 +32,15 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = { "permission": "gitea.issue.comment", "role": "author", }, + # #860: dirty orphaned same-claimant worktree recovery (explicit operation). + "recover_dirty_orphaned_issue_worktree": { + "permission": "gitea.issue.comment", + "role": "author", + }, + "gitea_recover_dirty_orphaned_issue_worktree": { + "permission": "gitea.issue.comment", + "role": "author", + }, # #864: dirty-preserving same-claimant author-session rebind (dead owner PID). # Author MCP tool path. Reconciler execute is gated inside the tool via # authorize_reconciler_execute + role_kind checks (not this map entry). @@ -488,6 +497,11 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = { # merger lease (#763). _PREFLIGHT_TASK_TRANSITIONS = frozenset({ ("review_pr", "acquire_reviewer_pr_lease"), + ("work_issue", "lock_issue"), + ("work_issue", "recover_dirty_orphaned_issue_worktree"), + ("work_issue", "gitea_recover_dirty_orphaned_issue_worktree"), + ("work_issue", "commit_files"), + ("work_issue", "gitea_commit_files"), }) diff --git a/tests/test_branch_cleanup_guard.py b/tests/test_branch_cleanup_guard.py index 0bd1fda..fddb534 100644 --- a/tests/test_branch_cleanup_guard.py +++ b/tests/test_branch_cleanup_guard.py @@ -1639,6 +1639,358 @@ class TestSecondRemediationIntegration(unittest.TestCase): self.assertTrue(ownership_calls) +class TestIssue855ExactPrSelector(unittest.TestCase): + """#855: exact pr_number pin for reconcile_merged_cleanups (#851 lifecycle).""" + + def setUp(self): + self._remotes = patch.dict( + mcp_server.REMOTES, + { + "prgs": { + "host": "gitea.example.com", + "org": "Scaled-Tech-Consulting", + "repo": "Gitea-Tools", + } + }, + ) + self._remotes.start() + patch("gitea_audit.audit_enabled", return_value=False).start() + self.mock_api = patch("mcp_server.api_request").start() + self.mock_all = patch("mcp_server.api_get_all", return_value=[]).start() + patch("mcp_server.get_auth_header", return_value=FAKE_AUTH).start() + patch( + "mcp_server.merged_cleanup_reconcile.is_head_ancestor_of_ref", + return_value=True, + ).start() + patch( + "mcp_server.get_profile", + return_value=dict(RECONCILER_WITH_DELETE), + ).start() + patch( + "mcp_server._profile_operation_gate", + return_value=[], + ).start() + patch( + "mcp_server._collect_branch_ownership_records", + return_value={"records": [], "inventory_error": False}, + ).start() + patch( + "mcp_server.merged_cleanup_reconcile.discover_reviewer_scratch_worktrees", + return_value=[], + ).start() + patch("mcp_server.verify_preflight_purity", return_value=None).start() + patch( + "mcp_server.audit_reconciliation_mode.check_cleanup_execution_allowed", + return_value=(True, []), + ).start() + + def tearDown(self): + patch.stopall() + + def _merged_pr(self, number, branch, sha="c" * 40): + return { + "number": number, + "title": f"PR {number}", + "body": f"Closes #{number - 4}", + "merged": True, + "merged_at": "2026-07-23T12:00:00Z", + "merge_commit_sha": "f" * 40, + "state": "closed", + "head": {"ref": branch, "sha": sha}, + "base": {"ref": "master"}, + } + + def test_exact_pr_848_ignores_newer_852_in_batch_queue(self): + """pr_number=848 selects only #848 even when #852 is newer/first.""" + from mcp_server import gitea_reconcile_merged_cleanups + + pr_848 = self._merged_pr( + 848, "fix/issue-844-exclude-epic-containers", sha="c3f282ba" + "0" * 32 + ) + # Closed list would rank #852 first in batch mode; exact pin must ignore it. + closed_batch = [ + self._merged_pr(852, "fix/issue-851-cleanup-worktree-before-remote-delete"), + pr_848, + self._merged_pr(849, "fix/issue-849-other"), + self._merged_pr(846, "fix/issue-846-other"), + self._merged_pr(845, "fix/issue-845-other"), + ] + batch_fetch_calls = [] + + def fake_api(method, url, *args, **kwargs): + if method == "GET" and url.rstrip("/").endswith("/pulls/848"): + return dict(pr_848) + if method == "GET" and "/pulls/" in url: + raise AssertionError(f"unexpected PR fetch: {url}") + if method == "GET" and "/branches/" in url: + return {"name": "present"} + return {} + + def fake_all(url, auth, limit=None): + batch_fetch_calls.append((url, limit)) + if "state=open" in url: + return [] + if "state=closed" in url: + # Exact mode must not use the closed batch list. + raise AssertionError( + "exact pr_number mode must not page closed PRs: " + url + ) + return [] + + self.mock_api.side_effect = fake_api + self.mock_all.side_effect = fake_all + patch( + "mcp_server._remote_branch_exists", + return_value=True, + ).start() + patch( + "mcp_server.merged_cleanup_reconcile.build_reconciliation_report", + side_effect=lambda **kwargs: { + "entries": [ + { + "pr_number": int(pr["number"]), + "head_branch": (pr.get("head") or {}).get("ref"), + "issue_number": 844, + "remote_branch": { + "safe_to_delete_remote": True, + "head_branch": (pr.get("head") or {}).get("ref"), + }, + "local_worktree": { + "safe_to_remove_worktree": True, + "worktree_path": ( + "/tmp/branches/fix-issue-844-exclude-epic-containers" + ), + }, + "planned_execution_order": ( + mcp_server.merged_cleanup_reconcile.plan_cleanup_execution_order( + remote_assessment={"safe_to_delete_remote": True}, + local_assessment={"safe_to_remove_worktree": True}, + ) + ), + } + for pr in kwargs.get("closed_prs") or [] + if pr.get("merged_at") or pr.get("merged") + ], + "reviewer_scratch_entries": [], + "merged_pr_count": len(kwargs.get("closed_prs") or []), + }, + ).start() + + res = gitea_reconcile_merged_cleanups( + dry_run=True, + pr_number=848, + remote="prgs", + org="Scaled-Tech-Consulting", + repo="Gitea-Tools", + ) + self.assertTrue(res.get("success")) + self.assertFalse(res.get("performed")) + self.assertEqual(res.get("selection_mode"), "exact_pr") + self.assertEqual(res.get("selected_pr_number"), 848) + entries = res.get("entries") or [] + self.assertEqual(len(entries), 1, entries) + self.assertEqual(entries[0].get("pr_number"), 848) + self.assertEqual( + entries[0].get("head_branch"), + "fix/issue-844-exclude-epic-containers", + ) + # No other PR appears in plan. + self.assertEqual(list((res.get("planned_execution_orders") or {}).keys()), ["848"]) + plan = (res.get("planned_execution_orders") or {}).get("848") or [] + actions = [s.get("action") for s in plan] + self.assertEqual( + actions, + [ + "remove_local_worktree", + "reassess_branch_ownership", + "delete_remote_branch", + ], + ) + # Prove we never scanned the multi-PR closed batch. + self.assertFalse(any("state=closed" in (u or "") for u, _ in batch_fetch_calls)) + # closed_batch fixture must remain unused (sanity). + self.assertEqual(closed_batch[0]["number"], 852) + + def test_exact_pr_execute_only_mutates_selected_pr(self): + """Execute with pr_number must never touch #845/#846/#849/#852.""" + from mcp_server import gitea_reconcile_merged_cleanups + + pr_848 = self._merged_pr(848, "fix/issue-844-exclude-epic-containers") + worktree_path = "/tmp/branches/fix-issue-844-exclude-epic-containers" + remove_calls = [] + delete_api_calls = [] + ownership_branches = [] + + def fake_api(method, url, *args, **kwargs): + if method == "GET" and url.rstrip("/").endswith("/pulls/848"): + return dict(pr_848) + if method == "DELETE": + delete_api_calls.append(url) + # Forbid foreign PR branch deletion by URL content. + for forbidden in ("845", "846", "849", "852"): + self.assertNotIn(forbidden, url) + return {} + + def fake_remove(project_root, branch, worktree_path=None): + remove_calls.append({"branch": branch, "worktree_path": worktree_path}) + return { + "success": True, + "performed": True, + "message": f"removed {worktree_path}", + "worktree_path": worktree_path, + } + + def fake_collect(**kwargs): + ownership_branches.append(kwargs.get("branch")) + return {"records": [], "inventory_error": False} + + def fake_probe(h, o, r, auth, br): + return guard.classify_branch_readback_http_status( + 404, not_found_scope=guard.NOT_FOUND_SCOPE_BRANCH + ) + + self.mock_api.side_effect = fake_api + self.mock_all.side_effect = lambda url, auth, limit=None: [] + patch("mcp_server._remote_branch_exists", return_value=True).start() + patch( + "mcp_server.merged_cleanup_reconcile.build_reconciliation_report", + return_value={ + "entries": [ + { + "pr_number": 848, + "head_branch": "fix/issue-844-exclude-epic-containers", + "remote_branch": {"safe_to_delete_remote": True}, + "local_worktree": { + "safe_to_remove_worktree": True, + "worktree_path": worktree_path, + }, + "planned_execution_order": [ + {"action": "remove_local_worktree", "phase": 1}, + {"action": "reassess_branch_ownership", "phase": 2}, + {"action": "delete_remote_branch", "phase": 3}, + ], + } + ], + "reviewer_scratch_entries": [ + # Foreign scratch must be filtered before report execute loop; + # if present here it would still be a test failure if acted on. + ], + "merged_pr_count": 1, + }, + ).start() + patch( + "mcp_server.merged_cleanup_reconcile.remove_local_worktree", + side_effect=fake_remove, + ).start() + patch( + "mcp_server._collect_branch_ownership_records", + side_effect=fake_collect, + ).start() + patch("mcp_server._probe_remote_branch", side_effect=fake_probe).start() + + res = gitea_reconcile_merged_cleanups( + dry_run=False, + execute_confirmed=True, + pr_number=848, + remote="prgs", + org="Scaled-Tech-Consulting", + repo="Gitea-Tools", + ) + self.assertTrue(res.get("performed") or res.get("executed")) + self.assertEqual(res.get("selection_mode"), "exact_pr") + self.assertEqual(res.get("selected_pr_number"), 848) + actions = res.get("actions") or [] + pr_numbers_touched = { + a.get("pr_number") for a in actions if a.get("pr_number") is not None + } + self.assertTrue(pr_numbers_touched.issubset({None, 848}) or not pr_numbers_touched) + removes = [a for a in actions if a.get("action") == "remove_local_worktree"] + deletes = [a for a in actions if a.get("action") == "delete_remote_branch"] + self.assertEqual(len(removes), 1) + self.assertEqual(remove_calls[0]["branch"], "fix/issue-844-exclude-epic-containers") + self.assertEqual(len(deletes), 1) + self.assertTrue(deletes[0].get("success")) + self.assertTrue(deletes[0].get("after_worktree_removal")) + self.assertEqual(len(delete_api_calls), 1) + self.assertEqual( + ownership_branches, ["fix/issue-844-exclude-epic-containers"] + ) + + def test_exact_pr_unknown_fails_closed_without_mutation(self): + from mcp_server import gitea_reconcile_merged_cleanups + + def fake_api(method, url, *args, **kwargs): + if method == "GET" and "/pulls/99999" in url: + raise RuntimeError("HTTP 404 Not Found") + raise AssertionError(f"unexpected API call {method} {url}") + + self.mock_api.side_effect = fake_api + res = gitea_reconcile_merged_cleanups( + dry_run=True, + pr_number=99999, + remote="prgs", + ) + self.assertFalse(res.get("success")) + self.assertFalse(res.get("performed")) + self.assertEqual(res.get("blocker_kind"), "pr_unresolvable") + self.assertIn("99999", " ".join(res.get("reasons") or [])) + + def test_exact_pr_not_merged_fails_closed(self): + from mcp_server import gitea_reconcile_merged_cleanups + + def fake_api(method, url, *args, **kwargs): + if method == "GET" and url.rstrip("/").endswith("/pulls/900"): + return { + "number": 900, + "merged": False, + "merged_at": None, + "state": "open", + "head": {"ref": "feat/x", "sha": "a" * 40}, + } + raise AssertionError(f"unexpected {method} {url}") + + self.mock_api.side_effect = fake_api + res = gitea_reconcile_merged_cleanups( + dry_run=False, + execute_confirmed=True, + pr_number=900, + remote="prgs", + ) + self.assertFalse(res.get("success")) + self.assertFalse(res.get("performed")) + self.assertEqual(res.get("blocker_kind"), "pr_not_merged") + + def test_exact_pr_invalid_number_fails_closed(self): + from mcp_server import gitea_reconcile_merged_cleanups + + res = gitea_reconcile_merged_cleanups( + dry_run=True, + pr_number=0, + remote="prgs", + ) + self.assertFalse(res.get("success")) + self.assertEqual(res.get("blocker_kind"), "invalid_pr_number") + self.mock_api.assert_not_called() + + def test_batch_mode_still_works_without_pr_number(self): + """Unfiltered batch path remains backward compatible.""" + from mcp_server import gitea_reconcile_merged_cleanups + + self.mock_all.side_effect = lambda url, auth, limit=None: [] + self.mock_api.side_effect = lambda *a, **k: {} + patch( + "mcp_server.merged_cleanup_reconcile.build_reconciliation_report", + return_value={ + "entries": [], + "reviewer_scratch_entries": [], + "merged_pr_count": 0, + }, + ).start() + res = gitea_reconcile_merged_cleanups(dry_run=True, remote="prgs", limit=10) + self.assertTrue(res.get("success")) + self.assertEqual(res.get("selection_mode"), "batch") + self.assertIsNone(res.get("selected_pr_number")) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_dirty_orphan_worktree_recovery.py b/tests/test_dirty_orphan_worktree_recovery.py new file mode 100644 index 0000000..4e58b4a --- /dev/null +++ b/tests/test_dirty_orphan_worktree_recovery.py @@ -0,0 +1,483 @@ +"""Synthetic regression coverage for dirty orphaned worktree recovery (#860). + +Modeled on the #850 / #855 shape without mutating their real state. +""" + +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import unittest +from unittest import mock + +import dirty_orphan_worktree_recovery as dorec +import issue_lock_store + + +DEAD_PID = 999_999_999 +LIVE_PID = os.getpid() +BRANCH = "fix/issue-901-dirty-orphan" +SOURCE_WT = "/repo/branches/issue-901-dirty-orphan" +RECOVERY_WT_NAME = "recovery-issue-901-dirty-orphan" +LOCAL_HEAD = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +REMOTE_HEAD = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +OTHER_HEAD = "cccccccccccccccccccccccccccccccccccccccc" +FP_A = dorec.sha256_bytes(b"dirty-a") +FP_B = dorec.sha256_bytes(b"dirty-b") +FP_C = dorec.sha256_bytes(b"dirty-c-conflict") + + +def durable_lock(**overrides): + """#850-shaped PID-less malformed same-claimant lock.""" + lock = { + "issue_number": 901, + "branch_name": BRANCH, + "worktree_path": SOURCE_WT, + "remote": "prgs", + "org": "Example-Org", + "repo": "Example-Repo", + # intentionally no pid / session_pid / work_lease expiry + "claimant": {"username": "author-user", "profile": "prgs-author"}, + } + lock.update(overrides) + return lock + + +def base_kwargs(**overrides): + kwargs = { + "issue_number": 901, + "branch_name": BRANCH, + "source_worktree_path": SOURCE_WT, + "remote": "prgs", + "org": "Example-Org", + "repo": "Example-Repo", + "identity": "author-user", + "profile": "prgs-author", + "expected_local_head": LOCAL_HEAD, + "expected_remote_head": REMOTE_HEAD, + "expected_dirty_fingerprints": {"a.py": FP_A, "b.py": FP_B}, + "current_branch": BRANCH, + "porcelain_status": " M a.py\n M b.py\n", + "observed_local_head": LOCAL_HEAD, + "observed_remote_head": REMOTE_HEAD, + "observed_dirty_fingerprints": {"a.py": FP_A, "b.py": FP_B}, + "competing_live_locks": [], + "competing_live_sessions": [], + "workflow_lease_active": False, + "workflow_lease_expired": True, + "canonical_repo_root": "/repo", + "worktree_registered": True, + "current_pid": LIVE_PID, + } + kwargs.update(overrides) + return kwargs + + +def assess(lock=None, **overrides): + return dorec.assess_dirty_orphan_recovery( + durable_lock() if lock is None else lock, **base_kwargs(**overrides) + ) + + +class FreshnessPidLess(unittest.TestCase): + def test_pid_less_lock_is_not_live(self): + freshness = issue_lock_store.assess_lock_freshness(durable_lock()) + self.assertFalse(freshness["live"]) + self.assertTrue(freshness.get("pid_missing")) + self.assertEqual(freshness["status"], "malformed") + + def test_pid_less_with_far_future_expiry_still_not_live(self): + lock = durable_lock( + work_lease={ + "operation_type": "author_issue_work", + "expires_at": "2999-01-01T00:00:00Z", + "last_heartbeat_at": "2999-01-01T00:00:00Z", + } + ) + freshness = issue_lock_store.assess_lock_freshness(lock) + self.assertFalse(freshness["live"]) + self.assertTrue(freshness.get("pid_missing")) + + +class EligibilityGranted(unittest.TestCase): + def test_dead_same_claimant_pid_less_dirty(self): + result = assess() + self.assertEqual(result["outcome"], dorec.ELIGIBLE) + self.assertTrue(result["eligible"]) + + def test_expired_workflow_lease_corroboration(self): + result = assess(workflow_lease_active=False, workflow_lease_expired=True) + self.assertTrue(result["eligible"]) + + def test_older_local_newer_remote_heads(self): + result = assess() + self.assertTrue(result["evidence"].get("heads_diverged")) + self.assertTrue(result["eligible"]) + + +class EligibilityRefused(unittest.TestCase): + def test_active_owner_with_pid(self): + lock = durable_lock(pid=LIVE_PID, session_pid=LIVE_PID) + result = assess(lock=lock, owner_process_alive_override=True) + self.assertEqual(result["outcome"], dorec.REFUSED) + self.assertFalse(result["eligible"]) + self.assertTrue(any("alive" in r for r in result["reasons"])) + + def test_foreign_claimant(self): + result = assess(identity="other-user") + self.assertEqual(result["outcome"], dorec.REFUSED) + self.assertTrue(any("foreign claimant identity" in r for r in result["reasons"])) + + def test_foreign_profile(self): + result = assess(profile="prgs-reviewer") + self.assertEqual(result["outcome"], dorec.REFUSED) + + def test_fingerprint_mismatch(self): + result = assess(observed_dirty_fingerprints={"a.py": "0" * 64, "b.py": FP_B}) + self.assertEqual(result["outcome"], dorec.REFUSED) + self.assertTrue(any("fingerprint mismatch" in r for r in result["reasons"])) + + def test_head_mismatch(self): + result = assess(observed_local_head=OTHER_HEAD) + self.assertEqual(result["outcome"], dorec.REFUSED) + + def test_remote_head_mismatch(self): + result = assess(observed_remote_head=OTHER_HEAD) + self.assertEqual(result["outcome"], dorec.REFUSED) + + def test_path_not_under_branches(self): + result = assess( + source_worktree_path="/tmp/branches/evil", + # lock path also changed so worktree agreement holds + lock=durable_lock(worktree_path="/tmp/branches/evil"), + ) + self.assertEqual(result["outcome"], dorec.REFUSED) + self.assertTrue(any("canonical branches" in r for r in result["reasons"])) + + def test_unregistered_worktree(self): + result = assess(worktree_registered=False) + self.assertEqual(result["outcome"], dorec.REFUSED) + + def test_active_workflow_lease(self): + result = assess(workflow_lease_active=True, workflow_lease_expired=False) + self.assertEqual(result["outcome"], dorec.REFUSED) + + def test_unsafe_dirty_path_pin(self): + result = assess( + expected_dirty_fingerprints={"../etc/passwd": FP_A}, + observed_dirty_fingerprints={"../etc/passwd": FP_A}, + ) + self.assertEqual(result["outcome"], dorec.REFUSED) + + def test_symlink_escape_rejected_by_ancestry(self): + ok, reasons = dorec.is_path_under_canonical_branches( + "/tmp/branches/evil", canonical_repo_root="/repo" + ) + self.assertFalse(ok) + self.assertTrue(reasons) + + +class ConflictDetection(unittest.TestCase): + def test_overlapping_upstream_change(self): + conflicts = dorec.detect_path_conflicts( + dirty_paths=["c.py"], + local_head_contents={"c.py": b"local-base"}, + remote_head_contents={"c.py": b"remote-changed"}, + dirty_contents={"c.py": b"dirty-c-conflict"}, + ) + self.assertEqual(len(conflicts), 1) + self.assertEqual(conflicts[0]["path"], "c.py") + + def test_unchanged_upstream_no_conflict(self): + conflicts = dorec.detect_path_conflicts( + dirty_paths=["a.py"], + local_head_contents={"a.py": b"same"}, + remote_head_contents={"a.py": b"same"}, + dirty_contents={"a.py": b"dirty-a"}, + ) + self.assertEqual(conflicts, []) + + +class CrashSafeRecovery(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="dirty-orphan-") + self.repo = os.path.join(self.tmp, "repo") + self.branches = os.path.join(self.repo, "branches") + self.source = os.path.join(self.branches, "issue-901-dirty-orphan") + self.recovery = os.path.join(self.branches, RECOVERY_WT_NAME) + os.makedirs(self.source, exist_ok=True) + os.makedirs(self.branches, exist_ok=True) + # seed dirty files in source + with open(os.path.join(self.source, "a.py"), "wb") as fh: + fh.write(b"dirty-a") + with open(os.path.join(self.source, "b.py"), "wb") as fh: + fh.write(b"dirty-b") + self.journal_dir = os.path.join(self.tmp, "journals") + self.lock = durable_lock(worktree_path=self.source) + self.assessment = dorec.assess_dirty_orphan_recovery( + self.lock, + **base_kwargs( + source_worktree_path=self.source, + canonical_repo_root=self.repo, + ), + ) + + class FakeGit(dorec.GitOps): + def __init__(self, recovery_path, head): + self.recovery_path = recovery_path + self.head = head + self.calls = [] + + def run(self, args, *, cwd): + self.calls.append((args, cwd)) + if args[:3] == ["git", "worktree", "add"]: + os.makedirs(self.recovery_path, exist_ok=True) + return mock.Mock(returncode=0, stdout="", stderr="") + if args[:2] == ["git", "checkout"]: + return mock.Mock(returncode=0, stdout="", stderr="") + if args[:2] == ["git", "rev-parse"]: + return mock.Mock(returncode=0, stdout=self.head + "\n", stderr="") + return mock.Mock(returncode=0, stdout="", stderr="") + + self.git = FakeGit(self.recovery, REMOTE_HEAD) + self.written_locks = [] + + def lock_writer(record): + self.written_locks.append(record) + + self.lock_writer = lock_writer + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def _run(self, **overrides): + kwargs = { + "assessment": self.assessment, + "existing_lock": self.lock, + "issue_number": 901, + "branch_name": BRANCH, + "source_worktree_path": self.source, + "recovery_worktree_path": self.recovery, + "remote": "prgs", + "org": "Example-Org", + "repo": "Example-Repo", + "identity": "author-user", + "profile": "prgs-author", + "expected_local_head": LOCAL_HEAD, + "expected_remote_head": REMOTE_HEAD, + "expected_dirty_fingerprints": {"a.py": FP_A, "b.py": FP_B}, + "dirty_contents": {"a.py": b"dirty-a", "b.py": b"dirty-b"}, + "local_head_contents": {"a.py": b"base-a", "b.py": b"base-b"}, + "remote_head_contents": {"a.py": b"base-a", "b.py": b"base-b"}, + "canonical_repo_root": self.repo, + "bind_lock": True, + "lock_writer": self.lock_writer, + "git_ops": self.git, + "journal_dir": self.journal_dir, + "session_pid": LIVE_PID, + } + kwargs.update(overrides) + return dorec.run_dirty_orphan_recovery(**kwargs) + + def test_success_preserves_dirty_bytes_and_source(self): + result = self._run() + self.assertTrue(result["success"]) + self.assertEqual(result["outcome"], dorec.RECOVERY_COMPLETED) + self.assertTrue(os.path.isdir(self.source)) + with open(os.path.join(self.source, "a.py"), "rb") as fh: + self.assertEqual(fh.read(), b"dirty-a") + with open(os.path.join(self.recovery, "a.py"), "rb") as fh: + self.assertEqual(fh.read(), b"dirty-a") + with open(os.path.join(self.recovery, "b.py"), "rb") as fh: + self.assertEqual(fh.read(), b"dirty-b") + self.assertEqual(len(self.written_locks), 1) + rec = self.written_locks[0] + self.assertEqual(rec["session_pid"], LIVE_PID) + self.assertTrue(rec["dirty_orphan_recovery"]["recovered"]) + self.assertTrue(rec["dirty_orphan_recovery"]["source_frozen"]) + + def test_conflict_leaves_governed_state(self): + result = self._run( + expected_dirty_fingerprints={"c.py": FP_C}, + dirty_contents={"c.py": b"dirty-c-conflict"}, + local_head_contents={"c.py": b"local-base"}, + remote_head_contents={"c.py": b"remote-changed"}, + ) + # #860 F4: session binding is NOT finalized while conflicts remain + self.assertFalse(result["success"]) + self.assertEqual(result["outcome"], dorec.CONFLICTS_PRESENT) + sidecar = os.path.join(self.recovery, "c.py.recovered-dirty") + self.assertTrue(os.path.isfile(sidecar)) + state = os.path.join( + self.recovery, dorec.CONFLICT_STATE_DIR, dorec.CONFLICT_STATE_FILE + ) + self.assertTrue(os.path.isfile(state)) + with open(state, "r", encoding="utf-8") as fh: + payload = json.load(fh) + self.assertEqual(payload["resolution"], "author_edit_required") + + def test_interrupt_before_journal_no_artifacts(self): + result = self._run(interrupt_after_phase=dorec.PHASE_ELIGIBILITY) + self.assertFalse(result["success"]) + self.assertEqual(result["outcome"], "INTERRUPTED") + self.assertFalse(os.path.isdir(self.recovery)) + + def test_interrupt_after_journal_then_retry_idempotent(self): + first = self._run(interrupt_after_phase=dorec.PHASE_JOURNAL_PERSISTED) + self.assertEqual(first["outcome"], "INTERRUPTED") + self.assertTrue(first["journal"]["artifacts_created"]["journal"]) + second = self._run() + self.assertTrue(second["success"]) + # source still recoverable + with open(os.path.join(self.source, "a.py"), "rb") as fh: + self.assertEqual(fh.read(), b"dirty-a") + + def test_interrupt_after_worktree_then_retry(self): + first = self._run(interrupt_after_phase=dorec.PHASE_RECOVERY_WORKTREE) + self.assertEqual(first["outcome"], "INTERRUPTED") + self.assertTrue(os.path.isdir(self.recovery)) + second = self._run() + self.assertTrue(second["success"]) + + def test_interrupt_after_binding_then_retry_complete(self): + first = self._run(interrupt_after_phase=dorec.PHASE_BINDING) + self.assertEqual(first["outcome"], "INTERRUPTED") + second = self._run() + self.assertTrue(second["success"]) + # completed journal makes further retries no-ops + third = self._run() + self.assertEqual(third["outcome"], dorec.RECOVERY_RESUMED) + + def test_source_worktree_never_deleted(self): + self._run() + self.assertTrue(os.path.isdir(self.source)) + self.assertTrue(os.path.isfile(os.path.join(self.source, "a.py"))) + + def test_fingerprint_drift_refuses_without_mutation(self): + result = self._run(dirty_contents={"a.py": b"CHANGED", "b.py": b"dirty-b"}) + self.assertFalse(result["success"]) + self.assertFalse(os.path.isdir(self.recovery)) + + +class SessionBindingPreflight(unittest.TestCase): + def test_canonical_session_binding_recognized(self): + lock = { + "worktree_path": "/repo/branches/recovery", + "session_pid": LIVE_PID, + "dirty_orphan_recovery": { + "recovered": True, + "conflicts": [], + "recovery_worktree_path": "/repo/branches/recovery", + "source_worktree_path": SOURCE_WT, + "accepted_head": REMOTE_HEAD, + }, + } + result = dorec.preflight_recognizes_recovered_provenance(lock) + self.assertTrue(result["recognized"]) + + def test_conflicts_block_commit_preflight(self): + lock = { + "worktree_path": "/repo/branches/recovery", + "session_pid": LIVE_PID, + "dirty_orphan_recovery": { + "recovered": True, + "conflicts": [{"path": "c.py"}], + }, + } + result = dorec.preflight_recognizes_recovered_provenance(lock) + self.assertFalse(result["recognized"]) + + def test_active_foreign_does_not_mutate(self): + # assess-only path: foreign refused before run + result = assess(identity="intruder") + self.assertFalse(result["eligible"]) + + +class JournalSymlinkRefusal(unittest.TestCase): + def test_symlink_journal_path_refused_on_load(self): + tmp = tempfile.mkdtemp() + try: + real = os.path.join(tmp, "real.json") + with open(real, "w", encoding="utf-8") as fh: + fh.write("{}") + link = os.path.join(tmp, "link.json") + os.symlink(real, link) + key = "symlink-test" + jdir = tmp + path = dorec._journal_path(key, journal_dir=jdir) + with open(path, "w", encoding="utf-8") as fh: + json.dump({"idempotency_key": key}, fh) + os.remove(path) + os.symlink(real, path) + with self.assertRaises(ValueError): + dorec.load_journal(key, journal_dir=jdir) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +class RealGitMultiWorktreeIntegration(unittest.TestCase): + def setUp(self): + import subprocess + self.tmp = tempfile.mkdtemp(prefix="git-integration-") + self.repo = os.path.join(self.tmp, "repo") + os.makedirs(self.repo, exist_ok=True) + subprocess.run(["git", "init"], cwd=self.repo, check=True, capture_output=True) + subprocess.run(["git", "config", "user.name", "Test User"], cwd=self.repo, check=True) + subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=self.repo, check=True) + with open(os.path.join(self.repo, "init.txt"), "w") as fh: + fh.write("init") + subprocess.run(["git", "add", "."], cwd=self.repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=self.repo, check=True) + branch = "fix/issue-999-test" + subprocess.run(["git", "branch", branch], cwd=self.repo, check=True) + self.branches = os.path.join(self.repo, "branches") + self.source = os.path.join(self.branches, "issue-999-test") + subprocess.run(["git", "worktree", "add", self.source, branch], cwd=self.repo, check=True) + self.dirty_path = os.path.join(self.source, "dirty.txt") + with open(self.dirty_path, "w") as fh: + fh.write("dirty-data") + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_prepare_recovery_worktree_detached_no_exit_128(self): + import subprocess + head_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.repo, text=True).strip() + rec_wt = os.path.join(self.branches, "recovery-issue-999-test") + res = dorec.prepare_recovery_worktree( + canonical_repo_root=self.repo, + recovery_worktree_path=rec_wt, + branch_name="fix/issue-999-test", + remote_head=head_sha, + ) + self.assertTrue(res["success"], res.get("reasons")) + self.assertTrue(os.path.isdir(rec_wt)) + + def test_real_lock_rebind_recovery_sanctioned(self): + lock_dir = os.path.join(self.tmp, "locks") + rec_wt = os.path.join(self.branches, "recovery-issue-999-test") + os.makedirs(rec_wt, exist_ok=True) + record = { + "remote": "prgs", + "org": "Example-Org", + "repo": "Example-Repo", + "issue_number": 999, + "branch_name": "fix/issue-999-test", + "worktree_path": rec_wt, + "claimant": {"username": "author-user", "profile": "prgs-author"}, + } + record_src = dict(record) + record_src["worktree_path"] = self.source + issue_lock_store.bind_session_lock(record_src, lock_dir=lock_dir) + path = issue_lock_store.bind_session_lock( + record, + lock_dir=lock_dir, + recovery_sanctioned=True, + ) + self.assertTrue(os.path.isfile(path)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_issue_855_expired_reviewer_reclaim.py b/tests/test_issue_855_expired_reviewer_reclaim.py new file mode 100644 index 0000000..288b7f7 --- /dev/null +++ b/tests/test_issue_855_expired_reviewer_reclaim.py @@ -0,0 +1,244 @@ +"""#855 AC4: an expired reviewer lease must not indefinitely protect an +already-merged branch when no live claimant exists. + +Two layers are covered: + +* ``branch_cleanup_guard.assess_expired_reviewer_lease_reclaim`` — the pure, + fail-closed reclaim decision. Every condition must be provably satisfied or + the lease keeps protecting the branch. +* ``gitea_mcp_server._collect_branch_ownership_records`` — the wiring that + supplies authoritative evidence (PR merged state, owner-process liveness, + competing ownership) to that decision, and flips an expired reviewer lease + to reclaimable only under the full policy. + +All inputs are fabricated; no real repository, lease, or credential is used. +""" + +import importlib +import unittest +from unittest.mock import patch + +import branch_cleanup_guard + +mcp_server = importlib.import_module("gitea_mcp_server") + +FAKE_AUTH = "token fake" +REMOTE = "prgs" +ORG = "Scaled-Tech-Consulting" +REPO = "Gitea-Tools" +HOST = "gitea.prgs.cc" +BRANCH = "feat/issue-638-webui-app-shell-phase1" +PR_NUMBER = 818 + + +class TestAssessExpiredReviewerLeaseReclaim(unittest.TestCase): + """Pure fail-closed reclaim decision (#855 AC4).""" + + def _call(self, **overrides): + base = dict( + role="reviewer", + status="expired", + pr_merged=True, + owner_pid_alive=False, + competing_active_claimant=False, + ) + base.update(overrides) + return branch_cleanup_guard.assess_expired_reviewer_lease_reclaim(**base) + + def test_full_policy_satisfied_allows_reclaim(self): + out = self._call() + self.assertTrue(out["reclaim_allowed"]) + self.assertEqual(out["reasons"], []) + self.assertEqual(out["decision"], "reclaim_expired_reviewer_lease") + + def test_stale_dead_process_reviewer_also_reclaimable(self): + out = self._call(status="stale_dead_process") + self.assertTrue(out["reclaim_allowed"]) + + def test_non_reviewer_role_never_reclaims(self): + for role in ("author", "merger", "controller", "reconciler", "unknown"): + with self.subTest(role=role): + out = self._call(role=role) + self.assertFalse(out["reclaim_allowed"]) + self.assertTrue(out["reasons"]) + self.assertEqual(out["decision"], "keep_protecting") + + def test_active_status_never_reclaims(self): + out = self._call(status="active") + self.assertFalse(out["reclaim_allowed"]) + + def test_pr_not_merged_blocks_reclaim(self): + out = self._call(pr_merged=False) + self.assertFalse(out["reclaim_allowed"]) + + def test_pr_merged_unknown_fails_closed(self): + out = self._call(pr_merged=None) + self.assertFalse(out["reclaim_allowed"]) + + def test_owner_process_alive_blocks_reclaim(self): + out = self._call(owner_pid_alive=True) + self.assertFalse(out["reclaim_allowed"]) + + def test_owner_liveness_unknown_fails_closed(self): + out = self._call(owner_pid_alive=None) + self.assertFalse(out["reclaim_allowed"]) + + def test_competing_active_claimant_blocks_reclaim(self): + out = self._call(competing_active_claimant=True) + self.assertFalse(out["reclaim_allowed"]) + + def test_competing_claimant_unknown_fails_closed(self): + out = self._call(competing_active_claimant=None) + self.assertFalse(out["reclaim_allowed"]) + + def test_reasons_never_leak_secrets(self): + out = self._call(role="author") + blob = " ".join(out["reasons"]).lower() + self.assertNotIn("token", blob) + self.assertNotIn("password", blob) + + +class _FakeLease(dict): + pass + + +class TestCollectorExpiredReviewerReclaimWiring(unittest.TestCase): + """`_collect_branch_ownership_records` supplies authoritative evidence and + flips an expired reviewer lease to reclaimable only under the full policy.""" + + def _run( + self, + *, + lease_role="reviewer", + lease_freshness="stale_dead_process", + owner_pid_alive=False, + pr_merged=True, + extra_leases=None, + worktree_on_branch=False, + ): + lease = _FakeLease( + role=lease_role, + work_kind="pr", + work_number=PR_NUMBER, + branch=BRANCH, + status="active", + owner_pid=999999, + remote=REMOTE, + org=ORG, + repo=REPO, + host=HOST, + freshness={ + "freshness": lease_freshness, + "owner_pid": 999999, + "owner_pid_alive": owner_pid_alive, + "expired_by_time": lease_freshness == "expired", + }, + ) + leases = [lease] + list(extra_leases or []) + + pr_payload = { + "number": PR_NUMBER, + "merged": pr_merged, + "merged_at": "2026-07-23T00:00:00Z" if pr_merged else None, + "head": {"ref": BRANCH}, + } + + def fake_api_request(method, url, *a, **k): + if method == "GET" and f"/pulls/{PR_NUMBER}" in url: + return pr_payload + raise AssertionError(f"unexpected api_request {method} {url}") + + wt_entries = [] + if worktree_on_branch: + wt_entries = [{"branch": BRANCH, "path": f"/x/branches/{BRANCH}"}] + + with patch.object( + mcp_server.lease_lifecycle, + "list_active_leases", + return_value={"leases": leases}, + ), patch.object( + mcp_server.control_plane_db, "get_db", return_value=object(), create=True + ), patch.object( + mcp_server.issue_lock_store, "iter_lock_files", return_value=[] + ), patch.object( + mcp_server.worktree_cleanup_audit, + "list_worktrees", + return_value=wt_entries, + ), patch.object( + mcp_server, "api_get_all", return_value=[] + ), patch.object( + mcp_server, "api_request", side_effect=fake_api_request + ): + return mcp_server._collect_branch_ownership_records( + remote=REMOTE, + host=HOST, + org=ORG, + repo=REPO, + branch=BRANCH, + pr_number=PR_NUMBER, + project_root="/x", + auth=FAKE_AUTH, + base_api="https://gitea.prgs.cc/api/v1/repos/x/y", + ) + + def _reviewer_records(self, bundle): + return [ + rec + for rec in bundle["records"] + if rec.get("category") + == branch_cleanup_guard.OWNERSHIP_CATEGORY_REVIEWER_LEASE + ] + + def test_merged_dead_uncontested_reviewer_lease_is_reclaimable(self): + bundle = self._run() + self.assertFalse(bundle["inventory_error"]) + recs = self._reviewer_records(bundle) + self.assertEqual(len(recs), 1) + self.assertTrue(recs[0]["reclaim_allowed"]) + # And the guard consequently does not block deletion on it. + ownership = branch_cleanup_guard.assess_active_branch_ownership( + remote=REMOTE, org=ORG, repo=REPO, branch=BRANCH, host=HOST, + records=bundle["records"], + ) + self.assertFalse(ownership["block"]) + + def test_unmerged_pr_keeps_reviewer_lease_protective(self): + bundle = self._run(pr_merged=False) + recs = self._reviewer_records(bundle) + self.assertEqual(len(recs), 1) + self.assertFalse(recs[0]["reclaim_allowed"]) + ownership = branch_cleanup_guard.assess_active_branch_ownership( + remote=REMOTE, org=ORG, repo=REPO, branch=BRANCH, host=HOST, + records=bundle["records"], + ) + self.assertTrue(ownership["block"]) + + def test_owner_process_alive_keeps_reviewer_lease_protective(self): + bundle = self._run(owner_pid_alive=True, lease_freshness="expired") + recs = self._reviewer_records(bundle) + self.assertFalse(recs[0]["reclaim_allowed"]) + + def test_competing_worktree_binding_keeps_reviewer_lease_protective(self): + bundle = self._run(worktree_on_branch=True) + recs = self._reviewer_records(bundle) + self.assertFalse(recs[0]["reclaim_allowed"]) + ownership = branch_cleanup_guard.assess_active_branch_ownership( + remote=REMOTE, org=ORG, repo=REPO, branch=BRANCH, host=HOST, + records=bundle["records"], + ) + self.assertTrue(ownership["block"]) + + def test_expired_author_lease_never_reclaimed_by_reviewer_policy(self): + bundle = self._run(lease_role="author") + author_recs = [ + rec + for rec in bundle["records"] + if rec.get("category") + == branch_cleanup_guard.OWNERSHIP_CATEGORY_AUTHOR_LEASE + ] + self.assertEqual(len(author_recs), 1) + self.assertFalse(author_recs[0]["reclaim_allowed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_issue_871_durable_lock_head_refresh.py b/tests/test_issue_871_durable_lock_head_refresh.py new file mode 100644 index 0000000..37851b1 --- /dev/null +++ b/tests/test_issue_871_durable_lock_head_refresh.py @@ -0,0 +1,630 @@ +"""Durable linked-issue lock head refresh + merge-sync dead-session recovery (#871). + +``gitea_update_pr_branch_by_merge`` advances a PR's *remote* head but historically +never advanced the linked durable issue lock's recorded head. After the owning +session died the drifted lock became unrecoverable and no further synchronization +was possible (PR #866 / issue #855). + +Two halves are covered: + +* the write-side refresh (``issue_lock_store.assess/apply_durable_lock_head_refresh``) + that records the new synced head under compare-and-swap with read-after-write; and +* the read-side recovery relation (``issue_lock_recovery`` + + ``issue_lock_worktree.read_merge_sync_provenance``) that lets a dead-session lock + whose recorded head is a merge-sync *ancestor* of the live PR head be recovered — + and nothing else. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import issue_lock_recovery # noqa: E402 +import issue_lock_store # noqa: E402 +import issue_lock_worktree # noqa: E402 + +ISSUE = 8710 +PR_NUMBER = 8711 +BRANCH = f"fix/issue-{ISSUE}-durable-lock-head-refresh" +IDENTITY = "example-user" +PROFILE = "example-author" +OLD = "a" * 40 +NEW1 = "b" * 40 +NEW2 = "c" * 40 +BASE = "d" * 40 +REMOTE = "prgs" +ORG = "ExampleOrg" +REPO = "ExampleRepo" + + +def dead_pid() -> int: + proc = subprocess.Popen([sys.executable, "-c", "pass"]) + proc.wait() + return proc.pid + + +def future_ts(hours: int = 4) -> str: + return ( + (datetime.now(timezone.utc) + timedelta(hours=hours)) + .isoformat() + .replace("+00:00", "Z") + ) + + +def _git(cwd, *args): + return subprocess.run( + ["git", "-C", cwd, *args], + capture_output=True, + text=True, + check=True, + ) + + +def _rev(cwd, ref="HEAD") -> str: + return _git(cwd, "rev-parse", ref).stdout.strip() + + +def build_merge_sync_repo(tmp: str) -> dict: + """Build a repo where a feature branch was synced by merging master in. + + Returns a dict with the prior (branch) head, the synced merge-commit head, + the master tip, plus a rebase-style linear descendant and an unrelated head. + """ + _git(tmp, "init", "-q", "-b", "master") + _git(tmp, "config", "user.email", "t@example.com") + _git(tmp, "config", "user.name", "T") + Path(tmp, "base.txt").write_text("base\n") + _git(tmp, "add", "-A") + _git(tmp, "commit", "-q", "-m", "root") + + # Feature branch cut from root, one commit — this is the PRIOR/recorded head. + _git(tmp, "checkout", "-q", "-b", BRANCH) + Path(tmp, "feature.txt").write_text("feature\n") + _git(tmp, "add", "-A") + _git(tmp, "commit", "-q", "-m", "feature work") + prior = _rev(tmp) + + # Master advances (the base the sync will merge in). + _git(tmp, "checkout", "-q", "master") + Path(tmp, "base.txt").write_text("base\nmore\n") + _git(tmp, "add", "-A") + _git(tmp, "commit", "-q", "-m", "master advance") + master_tip = _rev(tmp) + + # Sync: merge master INTO the feature branch → merge commit, first parent = prior. + _git(tmp, "checkout", "-q", BRANCH) + _git(tmp, "merge", "-q", "--no-ff", "-m", "Merge master into feature", "master") + synced = _rev(tmp) + + # A plain linear descendant of prior (NOT a merge) — a rebase/extra-commit shape. + _git(tmp, "checkout", "-q", "-b", "linear-branch", prior) + Path(tmp, "extra.txt").write_text("extra\n") + _git(tmp, "add", "-A") + _git(tmp, "commit", "-q", "-m", "extra linear commit") + linear = _rev(tmp) + + # An unrelated root (force-push / rewritten history shape). + unrelated_dir = tempfile.mkdtemp() + _git(unrelated_dir, "init", "-q", "-b", "x") + _git(unrelated_dir, "config", "user.email", "t@example.com") + _git(unrelated_dir, "config", "user.name", "T") + Path(unrelated_dir, "z.txt").write_text("z\n") + _git(unrelated_dir, "add", "-A") + _git(unrelated_dir, "commit", "-q", "-m", "unrelated") + unrelated = _rev(unrelated_dir) + + # Leave the worktree checked out on the feature branch at the PRIOR head, as + # a dead author session that never advanced would have left it. + _git(tmp, "checkout", "-q", BRANCH) + _git(tmp, "reset", "-q", "--hard", prior) + + return { + "prior": prior, + "master_tip": master_tip, + "synced": synced, + "linear": linear, + "unrelated": unrelated, + } + + +# ─────────────────────────── write-side refresh ─────────────────────────── + + +class TestDurableLockHeadRefresh(unittest.TestCase): + def setUp(self): + self.lock_dir = tempfile.mkdtemp() + self.wt = tempfile.mkdtemp() + lock_data = { + "issue_number": ISSUE, + "branch_name": BRANCH, + "worktree_path": self.wt, + "remote": REMOTE, + "org": ORG, + "repo": REPO, + "claimant": {"username": IDENTITY, "profile": PROFILE}, + "work_lease": { + "operation_type": issue_lock_store.AUTHOR_ISSUE_WORK_LEASE, + "issue_number": ISSUE, + "branch": BRANCH, + "worktree_path": self.wt, + "claimant": {"username": IDENTITY, "profile": PROFILE}, + "expires_at": future_ts(), + }, + } + issue_lock_store.bind_session_lock(lock_data, lock_dir=self.lock_dir) + + def _apply(self, **over): + kw = dict( + remote=REMOTE, org=ORG, repo=REPO, issue_number=ISSUE, + branch_name=BRANCH, worktree_path=self.wt, pr_number=PR_NUMBER, + identity=IDENTITY, profile=PROFILE, current_pid=os.getpid(), + expected_old_head=OLD, new_head=NEW1, synced_at=future_ts(0), + base_head=BASE, lock_dir=self.lock_dir, + ) + kw.update(over) + return issue_lock_store.apply_durable_lock_head_refresh(**kw) + + def _load(self): + return issue_lock_store.load_issue_lock( + remote=REMOTE, org=ORG, repo=REPO, issue_number=ISSUE, + lock_dir=self.lock_dir, + ) + + def test_first_sync_updates_recorded_head(self): + """AC1: first base sync writes the resulting head to the durable lock.""" + res = self._apply() + self.assertTrue(res["refreshed"], res["reasons"]) + self.assertTrue(res["read_after_write_ok"]) + self.assertEqual(self._load().get("synced_pr_head"), NEW1) + + def test_second_sync_after_master_advance(self): + """AC2: a later master advance permits a second sanctioned sync.""" + self.assertTrue(self._apply()["refreshed"]) + res2 = self._apply(expected_old_head=NEW1, new_head=NEW2) + self.assertTrue(res2["refreshed"], res2["reasons"]) + self.assertEqual(self._load().get("synced_pr_head"), NEW2) + history = self._load().get("branch_sync_history") + self.assertEqual(len(history), 2) + self.assertEqual(history[0]["last_synced_pr_head"], NEW1) + self.assertEqual(history[1]["prior_pr_head"], NEW1) + + def test_cas_detects_concurrent_head_change(self): + """AC6: CAS refuses when the recorded synced head is not the old head.""" + self.assertTrue(self._apply()["refreshed"]) # recorded head now NEW1 + # A second sync claiming the old head is still OLD must fail closed. + res = self._apply(expected_old_head=OLD, new_head=NEW2) + self.assertFalse(res["refreshed"]) + self.assertTrue(any("CAS" in r or "concurrent" in r for r in res["reasons"])) + self.assertEqual(self._load().get("synced_pr_head"), NEW1) + + def test_wrong_issue_fails_closed(self): + res = self._apply(issue_number=999999) + self.assertFalse(res["refreshed"]) + + def test_wrong_branch_fails_closed(self): + res = self._apply(branch_name="fix/issue-8710-wrong") + self.assertFalse(res["refreshed"]) + + def test_wrong_repo_fails_closed(self): + res = self._apply(repo="OtherRepo") + self.assertFalse(res["refreshed"]) + + def test_wrong_identity_fails_closed(self): + res = self._apply(identity="intruder") + self.assertFalse(res["refreshed"]) + + def test_wrong_profile_fails_closed(self): + res = self._apply(profile="prgs-reviewer") + self.assertFalse(res["refreshed"]) + + def test_foreign_session_fails_closed(self): + """A refresh is not a recovery: the current process must own the lock.""" + path = issue_lock_store.lock_file_path( + remote=REMOTE, org=ORG, repo=REPO, issue_number=ISSUE, + lock_dir=self.lock_dir, + ) + rec = issue_lock_store.read_lock_file(path) + rec["session_pid"] = dead_pid() + rec["pid"] = rec["session_pid"] + issue_lock_store.save_lock_file(path, rec) + res = self._apply() + self.assertFalse(res["refreshed"]) + self.assertTrue(any("current session" in r or "live owner" in r for r in res["reasons"])) + + def test_new_equals_old_fails_closed(self): + res = self._apply(expected_old_head=OLD, new_head=OLD) + self.assertFalse(res["refreshed"]) + + def test_non_full_sha_fails_closed(self): + self.assertFalse(self._apply(new_head="deadbeef")["refreshed"]) + self.assertFalse(self._apply(expected_old_head="xyz")["refreshed"]) + + def test_no_lock_fails_closed(self): + assessment = issue_lock_store.assess_durable_lock_head_refresh( + None, remote=REMOTE, org=ORG, repo=REPO, issue_number=ISSUE, + branch_name=BRANCH, worktree_path=self.wt, pr_number=PR_NUMBER, + identity=IDENTITY, profile=PROFILE, current_pid=os.getpid(), + expected_old_head=OLD, new_head=NEW1, + ) + self.assertFalse(assessment["allowed"]) + + +# ─────────────────────── merge-sync provenance (real git) ─────────────────── + + +class TestMergeSyncProvenanceObservation(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.shas = build_merge_sync_repo(self.tmp) + + def test_merge_sync_is_recognized(self): + obs = issue_lock_worktree.read_merge_sync_provenance( + self.tmp, prior_head_sha=self.shas["prior"], + synced_head_sha=self.shas["synced"], + ) + self.assertTrue(obs["is_merge_sync"], obs["reasons"]) + self.assertTrue(obs["prior_is_ancestor"]) + self.assertTrue(obs["synced_is_merge"]) + self.assertTrue(obs["first_parent_reaches_prior"]) + + def test_linear_descendant_is_not_a_merge_sync(self): + """A plain non-merge descendant (rebase/extra commit) is not a sync.""" + obs = issue_lock_worktree.read_merge_sync_provenance( + self.tmp, prior_head_sha=self.shas["prior"], + synced_head_sha=self.shas["linear"], + ) + self.assertTrue(obs["probe_ok"]) + self.assertFalse(obs["is_merge_sync"]) + self.assertFalse(obs["synced_is_merge"]) + + def test_unrelated_history_fails_closed(self): + """A rewritten/force-pushed head where prior is unreachable fails closed.""" + obs = issue_lock_worktree.read_merge_sync_provenance( + self.tmp, prior_head_sha=self.shas["prior"], + synced_head_sha=self.shas["unrelated"], + ) + self.assertFalse(obs["is_merge_sync"]) + + def test_missing_args_fail_closed(self): + obs = issue_lock_worktree.read_merge_sync_provenance( + self.tmp, prior_head_sha=None, synced_head_sha=self.shas["synced"], + ) + self.assertFalse(obs["is_merge_sync"]) + + +# ──────────────────── merge-sync dead-session recovery ────────────────────── + + +def make_dead_lock(worktree, **over): + pid = dead_pid() + lock = { + "issue_number": ISSUE, + "branch_name": BRANCH, + "worktree_path": worktree, + "remote": REMOTE, + "org": ORG, + "repo": REPO, + "session_pid": pid, + "pid": pid, + "claimant": {"username": IDENTITY, "profile": PROFILE}, + "work_lease": { + "operation_type": issue_lock_store.AUTHOR_ISSUE_WORK_LEASE, + "issue_number": ISSUE, + "branch": BRANCH, + "worktree_path": worktree, + "claimant": {"username": IDENTITY, "profile": PROFILE}, + "expires_at": future_ts(), + }, + } + lock.update(over) + return lock + + +def sync_prov(prior, synced, **over): + d = { + "prior_head_sha": prior, + "synced_head_sha": synced, + "probe_ok": True, + "prior_present": True, + "synced_present": True, + "prior_is_ancestor": True, + "synced_is_merge": True, + "first_parent_reaches_prior": True, + "is_merge_sync": True, + "first_parent_sha": prior, + "parent_count": 2, + "proof": f"{synced} merged base into branch above {prior}", + "reasons": [], + } + d.update(over) + return d + + +class TestMergeSyncRecovery(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.shas = build_merge_sync_repo(self.tmp) + self.prior = self.shas["prior"] + self.synced = self.shas["synced"] + + def _assess(self, **over): + lock = over.pop("_lock", None) or make_dead_lock(self.tmp) + kw = dict( + issue_number=ISSUE, branch_name=BRANCH, worktree_path=self.tmp, + remote=REMOTE, org=ORG, repo=REPO, identity=IDENTITY, profile=PROFILE, + current_branch=BRANCH, porcelain_status="", + head_sha=self.prior, remote_head_sha=self.synced, + pr_head_sha=self.synced, pr_number=PR_NUMBER, + competing_live_locks=[], candidate_branches=[BRANCH], + current_pid=os.getpid(), + remote_branch_exists=True, + sync_provenance=sync_prov(self.prior, self.synced), + ) + kw.update(over) + return issue_lock_recovery.assess_dead_session_lock_recovery(lock, **kw) + + def test_merge_sync_drift_is_recoverable(self): + """AC3/AC4: dead session, recorded head is a merge-sync ancestor of PR head.""" + res = self._assess() + self.assertEqual(res["outcome"], issue_lock_recovery.RECOVERY_SANCTIONED, res["reasons"]) + self.assertEqual( + res["evidence"]["head_relation"], + issue_lock_recovery.HEAD_RELATION_REMOTE_MERGE_SYNCED, + ) + self.assertEqual(res["evidence"]["accepted_head"], self.synced) + + def test_missing_provenance_fails_closed(self): + """No server-derived provenance → cannot accept a remote ahead of local.""" + res = self._assess(sync_provenance=None) + self.assertEqual(res["outcome"], issue_lock_recovery.REFUSED) + + def test_non_ancestor_recorded_head_fails_closed(self): + """AC7: provenance that does not prove ancestry is rejected.""" + res = self._assess( + sync_provenance=sync_prov( + self.prior, self.synced, prior_is_ancestor=False, is_merge_sync=False, + reasons=["prior head is not an ancestor"], + ) + ) + self.assertEqual(res["outcome"], issue_lock_recovery.REFUSED) + + def test_force_pushed_history_fails_closed(self): + """AC8: a rewritten head (not a merge sync) stays protected.""" + res = self._assess( + sync_provenance=sync_prov( + self.prior, self.synced, is_merge_sync=False, synced_is_merge=False, + reasons=["not a merge-based sync"], + ) + ) + self.assertEqual(res["outcome"], issue_lock_recovery.REFUSED) + + def test_provenance_for_other_commits_fails_closed(self): + """Provenance whose endpoints differ from the heads under assessment is rejected.""" + res = self._assess( + sync_provenance=sync_prov("f" * 40, self.synced), + ) + self.assertEqual(res["outcome"], issue_lock_recovery.REFUSED) + + def test_dirty_worktree_fails_closed(self): + """AC11: dirty worktrees remain protected.""" + res = self._assess(porcelain_status=" M feature.txt\n") + self.assertEqual(res["outcome"], issue_lock_recovery.REFUSED) + + def test_live_owner_fails_closed(self): + """AC10: a live recorded owner is not a dead-session recovery.""" + lock = make_dead_lock(self.tmp, session_pid=os.getpid(), pid=os.getpid()) + res = self._assess(_lock=lock) + self.assertEqual(res["outcome"], issue_lock_recovery.REFUSED) + + def test_competing_claimant_fails_closed(self): + """AC13: a competing live lock blocks recovery.""" + res = self._assess( + competing_live_locks=[{ + "issue_number": ISSUE, "branch_name": BRANCH, + "worktree_path": "/some/other/wt", "pid": os.getpid(), + }] + ) + self.assertEqual(res["outcome"], issue_lock_recovery.REFUSED) + + def test_wrong_branch_fails_closed(self): + """AC9: worktree on a different branch fails closed.""" + res = self._assess(current_branch="fix/issue-8710-other") + self.assertEqual(res["outcome"], issue_lock_recovery.REFUSED) + + def test_wrong_identity_fails_closed(self): + res = self._assess(identity="intruder") + self.assertEqual(res["outcome"], issue_lock_recovery.REFUSED) + + def test_pr_head_mismatch_fails_closed(self): + """The open PR must sit at the synced remote head.""" + res = self._assess(pr_head_sha="e" * 40) + self.assertEqual(res["outcome"], issue_lock_recovery.REFUSED) + + def test_owning_pr_evidence_for_merge_sync(self): + res = self._assess() + ev = issue_lock_recovery.owning_pr_recovery_evidence(res) + self.assertIsNotNone(ev) + self.assertEqual(ev["pr_number"], PR_NUMBER) + self.assertEqual(ev["head_sha"], self.synced) + self.assertEqual( + ev["head_relation"], + issue_lock_recovery.HEAD_RELATION_REMOTE_MERGE_SYNCED, + ) + + def test_recovered_owning_pr_from_persisted_record(self): + res = self._assess() + record = issue_lock_recovery.build_recovery_record(res, recovered_at=future_ts(0)) + lock = {"issue_number": ISSUE, "branch_name": BRANCH, + "dead_session_recovery": record} + rebuilt = issue_lock_recovery.recovered_owning_pr_from_lock(lock) + self.assertIsNotNone(rebuilt) + self.assertEqual(rebuilt["head_sha"], self.synced) + self.assertEqual( + rebuilt["head_relation"], + issue_lock_recovery.HEAD_RELATION_REMOTE_MERGE_SYNCED, + ) + + +class TestExistingRelationsUnchanged(unittest.TestCase): + """AC14/AC15: equal-head recovery still works; merge-sync did not weaken it.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.shas = build_merge_sync_repo(self.tmp) + + def test_equal_head_recovery_still_sanctioned(self): + # Worktree at prior head; remote also at prior head → the #753 equal case. + prior = self.shas["prior"] + lock = make_dead_lock(self.tmp) + res = issue_lock_recovery.assess_dead_session_lock_recovery( + lock, issue_number=ISSUE, branch_name=BRANCH, worktree_path=self.tmp, + remote=REMOTE, org=ORG, repo=REPO, identity=IDENTITY, profile=PROFILE, + current_branch=BRANCH, porcelain_status="", + head_sha=prior, remote_head_sha=prior, + pr_head_sha=prior, pr_number=PR_NUMBER, + competing_live_locks=[], candidate_branches=[BRANCH], + current_pid=os.getpid(), remote_branch_exists=True, + ) + self.assertEqual(res["outcome"], issue_lock_recovery.RECOVERY_SANCTIONED, res["reasons"]) + self.assertEqual( + res["evidence"]["head_relation"], issue_lock_recovery.HEAD_RELATION_EQUAL, + ) + + +class TestUpdatePrWrapperPartialFailure(unittest.TestCase): + """AC5/AC16: the tool advances the remote head then refreshes the durable lock. + + When the durable refresh fails after the remote advance, the tool must report a + partial lifecycle failure and NOT a fully successful synchronization. Exact PR- + head / base-head pinning is preserved (delegated to the real preflight, stubbed + here only to isolate the post-update lifecycle branch). + """ + + def setUp(self): + import gitea_mcp_server as gms # noqa: E402 + self.gms = gms + self._orig = {} + + def _patch(name, value): + self._orig[name] = getattr(gms, name) + setattr(gms, name, value) + + _patch("get_profile", lambda *a, **k: { + "allowed_operations": ["gitea.branch.push"], + "forbidden_operations": [], + "profile_name": "prgs-author", + }) + _patch("_role_kind", lambda *a, **k: "author") + _patch("_profile_operation_gate", lambda *a, **k: None) + _patch("_permission_block_report", lambda *a, **k: {}) + _patch("_resolve", lambda *a, **k: ("gitea.prgs.cc", ORG, REPO)) + _patch("_verify_role_mutation_workspace", lambda *a, **k: None) + _patch("_get_workspace_porcelain", lambda *a, **k: "") + _patch("_canonical_local_git_root", lambda *a, **k: "/x") + _patch("_master_parity_block", lambda *a, **k: None) + _patch("_auth", lambda *a, **k: {"token": "x"}) + _patch("repo_api_url", lambda *a, **k: "http://api") + _patch("_redact", lambda s: s) + _patch("_work_lease_claimant", lambda *a, **k: { + "username": IDENTITY, "profile": PROFILE, + }) + _patch("_prove_author_ownership_for_pr", lambda *a, **k: { + "has_author_lock": True, "matched_issue": ISSUE, + "matched_via": "branch", "linked_issues": [ISSUE], + "recovered_owning_pr": None, "reasons": [], + }) + + # Real preflight is unit-tested elsewhere; stub it to isolate the + # post-update durable-lock lifecycle branch under test. + orig_pf = gms.pr_sync_status.assess_update_pr_branch_preflight + self._orig_pf = orig_pf + gms.pr_sync_status.assess_update_pr_branch_preflight = ( + lambda *a, **k: {"mutation_allowed": True, "reasons": [], "performed": False} + ) + + # Sequence the two GET /pulls calls: OLD before update, NEW after. + self._pull_calls = {"n": 0} + + def fake_api_request(method, url, auth, *a, **k): + m = method.upper() + if m == "GET" and url.endswith(f"/pulls/{PR_NUMBER}"): + self._pull_calls["n"] += 1 + head = OLD if self._pull_calls["n"] == 1 else NEW1 + return { + "state": "open", + "head": {"sha": head, "ref": BRANCH}, + "base": {"sha": BASE, "ref": "master"}, + "mergeable": True, "title": "t", "body": "b", + } + if m == "GET" and "/branches/" in url: + return {"commit": {"id": BASE}} + if m == "POST" and "/update" in url: + return {} + return {} + + _patch("api_request", fake_api_request) + + def tearDown(self): + for name, value in self._orig.items(): + setattr(self.gms, name, value) + self.gms.pr_sync_status.assess_update_pr_branch_preflight = self._orig_pf + + def _run(self): + return self.gms.gitea_update_pr_branch_by_merge( + pr_number=PR_NUMBER, + expected_pr_head_sha=OLD, + expected_base_head_sha=BASE, + remote=REMOTE, + worktree_path="/tmp/branches/wt-871", + ) + + def test_partial_failure_when_refresh_fails(self): + self._orig["apply_durable_lock_head_refresh"] = ( + self.gms.issue_lock_store.apply_durable_lock_head_refresh + ) + self.gms.issue_lock_store.apply_durable_lock_head_refresh = ( + lambda **k: {"refreshed": False, "reasons": ["forced refresh failure"]} + ) + try: + res = self._run() + finally: + self.gms.issue_lock_store.apply_durable_lock_head_refresh = ( + self._orig["apply_durable_lock_head_refresh"] + ) + self.assertTrue(res["performed"]) + self.assertEqual(res["new_pr_head_sha"], NEW1) + self.assertFalse(res["success"]) + self.assertTrue(res["partial_lifecycle_failure"]) + self.assertFalse(res["durable_lock_refreshed"]) + + def test_full_success_when_refresh_succeeds(self): + self._orig["apply_durable_lock_head_refresh"] = ( + self.gms.issue_lock_store.apply_durable_lock_head_refresh + ) + self.gms.issue_lock_store.apply_durable_lock_head_refresh = ( + lambda **k: {"refreshed": True, "read_after_write_ok": True, + "new_head": NEW1, "reasons": ["ok"]} + ) + try: + res = self._run() + finally: + self.gms.issue_lock_store.apply_durable_lock_head_refresh = ( + self._orig["apply_durable_lock_head_refresh"] + ) + self.assertTrue(res["success"]) + self.assertTrue(res["performed"]) + self.assertTrue(res["durable_lock_refreshed"]) + self.assertTrue(res["fully_synchronized"]) + self.assertEqual(res["new_pr_head_sha"], NEW1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_issue_lock_store.py b/tests/test_issue_lock_store.py index 9c41907..a8a4a7b 100644 --- a/tests/test_issue_lock_store.py +++ b/tests/test_issue_lock_store.py @@ -24,6 +24,8 @@ def _lease(expires_at: str) -> dict: def _lock_record(**overrides) -> dict: + # #860: live locks require a usable session pid; PID-less records are never + # classified live merely because expiry/heartbeat fields are present. record = { "issue_number": 420, "branch_name": "feat/issue-420-server-code-parity", @@ -31,6 +33,8 @@ def _lock_record(**overrides) -> dict: "org": "Scaled-Tech-Consulting", "repo": "Gitea-Tools", "worktree_path": "/tmp/wt-420", + "session_pid": os.getpid(), + "pid": os.getpid(), "work_lease": _lease("2999-01-01T00:00:00Z"), } record.update(overrides) @@ -88,6 +92,8 @@ class TestIssueLockStore(unittest.TestCase): existing = _lock_record( branch_name="feat/issue-420-other", worktree_path="/tmp/other", + session_pid=os.getpid(), + pid=os.getpid(), work_lease=_lease("2999-01-01T00:00:00Z"), ) path = ils.lock_file_path( diff --git a/tests/test_mcp_restart_governance_docs.py b/tests/test_mcp_restart_governance_docs.py new file mode 100644 index 0000000..5b12e50 --- /dev/null +++ b/tests/test_mcp_restart_governance_docs.py @@ -0,0 +1,107 @@ +"""Documentation acceptance for the MCP restart governance ADR (#656). + +Enforces issue #656 acceptance criteria: + +* AC1 — policy document exists with an authorization matrix and the recorded + v1 decision (controller approval + automated safety gates). +* AC2 — restart is stated as a last resort with enumerated narrower recoveries. +* AC3 — a unilateral LLM full restart with affected sessions is forbidden. +* AC4 — break-glass conditions are listed. +* AC5 — the ADR is linked to #655, #652, #653, #630, #642, and is cross-linked + from the safety model and the web-console deployment boundary docs. +""" +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +ADR = REPO_ROOT / "docs" / "architecture" / "mcp-restart-governance.md" +ADR_BASENAME = "mcp-restart-governance.md" + +CROSS_LINK_DOCS = ( + REPO_ROOT / "docs" / "safety-model.md", + REPO_ROOT / "docs" / "webui-deployment.md", +) + +LINKED_ISSUES = ("#655", "#652", "#653", "#630", "#642") +POLICY_IDS = ("RG-01", "RG-02", "RG-03", "RG-04", "RG-05", "RG-06", "RG-07", "RG-08") + + +def _read(path: Path) -> str: + assert path.is_file(), f"missing {path.relative_to(REPO_ROOT)}" + return path.read_text(encoding="utf-8") + + +def test_ac1_adr_exists_with_matrix_and_v1_decision(): + text = _read(ADR) + lower = text.lower() + assert text.lstrip().startswith("#"), "ADR lacks a title" + assert "#656" in text + assert "authorization matrix" in lower + # The matrix is a real table with the worker and privileged roles. + for role in ("author", "reviewer", "merger", "reconciler", "controller", + "operator", "admin"): + assert role in lower, f"authorization matrix missing role {role!r}" + # Recorded v1 decision. + assert "restart-governance/v1" in text + assert "controller approval" in lower and "automated safety gates" in lower + + +def test_ac2_restart_is_last_resort_with_narrower_recoveries(): + text = _read(ADR) + lower = text.lower() + assert "last resort" in lower + # Enumerated narrower recoveries precede full restart on the ladder. + for rung in ("reconnect", "rebind", "scoped restart", "full restart", + "host"): + assert rung in lower, f"recovery ladder missing rung {rung!r}" + + +def test_ac3_forbids_unilateral_llm_full_restart_with_affected_sessions(): + text = _read(ADR) + lower = text.lower() + assert "forbidden" in lower + assert "llm" in lower and "restart" in lower + assert "unilateral" in lower + # A worker role must not perform or authorize full/host restart. + assert "must not" in lower + + +def test_ac4_break_glass_conditions_listed(): + text = _read(ADR) + lower = text.lower() + assert "break-glass" in lower + assert "incident" in lower + assert "audit" in lower + + +def test_ac5_adr_links_issue_lineage(): + text = _read(ADR) + for issue in LINKED_ISSUES: + assert issue in text, f"ADR must link issue {issue}" + + +def test_ac5_safety_model_and_deployment_cross_link_adr(): + for path in CROSS_LINK_DOCS: + text = _read(path) + assert ADR_BASENAME in text, ( + f"{path.relative_to(REPO_ROOT)} must cross-link {ADR_BASENAME} " + f"(issue #656 acceptance criterion 5)" + ) + + +def test_policy_ids_present_for_enforcement_code(): + text = _read(ADR) + for pid in POLICY_IDS: + assert pid in text, f"policy id {pid} missing from ADR" + + +def test_failure_behavior_denies_on_ambiguity(): + text = _read(ADR) + lower = text.lower() + assert "ambiguous" in lower and "deny" in lower + + +def test_cross_links_do_not_embed_secrets(): + for path in (ADR,) + CROSS_LINK_DOCS: + text = _read(path) + for marker in ("ghp_", "BEGIN PRIVATE KEY", "Authorization: Bearer"): + assert marker not in text, f"{path} contains {marker!r}" diff --git a/tests/test_mcp_restart_paths.py b/tests/test_mcp_restart_paths.py new file mode 100644 index 0000000..cf0c5eb --- /dev/null +++ b/tests/test_mcp_restart_paths.py @@ -0,0 +1,146 @@ +"""Tests for the MCP restart-path inventory and guards (#657). + +Covers: +* the registry is well-formed and every path is classified; +* unknown restart attempts fail closed (AC "fail closed on unknown restart"); +* the previously-unguarded full-restart primitives stay guarded/absent + against the real source tree (AC "tests for at least one previously + unguarded path"); +* pkill of the daemon is still classified as contamination (#630, AC3); +* the inventory doc and module stay in lock-step. +""" + +import os +import tempfile +import unittest +from pathlib import Path + +import mcp_restart_paths as rp +import runtime_recovery_guard + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DOC_PATH = os.path.join(REPO_ROOT, "docs", "mcp-restart-path-inventory.md") + + +class TestRegistryWellformed(unittest.TestCase): + def test_registry_is_wellformed(self): + # Must not raise. + rp.assert_registry_wellformed() + + def test_every_path_has_valid_classification(self): + for path in rp.iter_restart_paths(): + self.assertIn(path.classification, rp.VALID_CLASSIFICATIONS) + self.assertTrue(path.guard.strip(), path.path_id) + self.assertTrue(path.references, path.path_id) + self.assertTrue(path.locations, path.path_id) + + def test_ids_are_unique(self): + ids = [p.path_id for p in rp.iter_restart_paths()] + self.assertEqual(len(ids), len(set(ids))) + + def test_covers_every_classification(self): + present = {p.classification for p in rp.iter_restart_paths()} + self.assertEqual(present, set(rp.VALID_CLASSIFICATIONS)) + + +class TestUnknownAttemptFailsClosed(unittest.TestCase): + def test_unknown_path_raises(self): + with self.assertRaises(rp.UnknownRestartPathError): + rp.assert_restart_attempt_registered("totally_novel_restart_hack") + + def test_get_unknown_raises(self): + with self.assertRaises(rp.UnknownRestartPathError): + rp.get_restart_path("nope") + + def test_registered_attempt_returns_path(self): + path = rp.assert_restart_attempt_registered("manual_daemon_kill") + self.assertEqual(path.classification, rp.CLASS_FORBIDDEN) + + +class TestDaemonNeverSelfReplaces(unittest.TestCase): + """Previously-unguarded full-restart primitive: daemon self-replacement.""" + + def test_no_self_replacement_in_source(self): + # The live daemon modules must contain no os.execv/os.kill/os._exit + # self-restart call. Must not raise. + rp.assert_no_daemon_self_replacement(REPO_ROOT) + + def test_scanner_flags_injected_violation(self): + # Guard the guard: prove the scanner catches a real self-replace call. + with tempfile.TemporaryDirectory() as tmp: + bad = Path(tmp) / "gitea_mcp_server.py" + bad.write_text( + "import os\n" + "def restart():\n" + " os.execv('/usr/bin/python', ['python'])\n", + encoding="utf-8", + ) + found = rp.scan_daemon_self_replacement(tmp) + self.assertTrue(found) + with self.assertRaises(AssertionError): + rp.assert_no_daemon_self_replacement(tmp) + + def test_scanner_ignores_comment_and_docstring_mentions(self): + with tempfile.TemporaryDirectory() as tmp: + ok = Path(tmp) / "gitea_mcp_server.py" + ok.write_text( + "import os\n" + "# NOT os.execv() to re-point the interpreter here.\n" + '"""Never calls os._exit to restart."""\n' + "value = 1\n", + encoding="utf-8", + ) + self.assertEqual(rp.scan_daemon_self_replacement(tmp), []) + + +class TestLegacyAutoRestartHelperRemoved(unittest.TestCase): + """Previously-unguarded full-restart path: _trigger_mcp_auto_restart.""" + + def test_helper_absent_in_source(self): + # Must not raise: helper was removed in #685. + rp.assert_auto_restart_helper_absent(REPO_ROOT) + + def test_scanner_flags_reintroduced_helper(self): + with tempfile.TemporaryDirectory() as tmp: + bad = Path(tmp) / "mcp_server.py" + bad.write_text( + "def _trigger_mcp_auto_restart():\n return True\n", + encoding="utf-8", + ) + with self.assertRaises(AssertionError): + rp.assert_auto_restart_helper_absent(tmp) + + +class TestPkillStaysForbidden(unittest.TestCase): + """AC3: pkill of the daemon remains forbidden/contaminating (#630).""" + + def test_manual_daemon_kill_registered_as_forbidden(self): + path = rp.get_restart_path("manual_daemon_kill") + self.assertEqual(path.classification, rp.CLASS_FORBIDDEN) + + def test_pkill_classified_as_contamination(self): + assessment = runtime_recovery_guard.assess_recovery_command( + "pkill -f mcp_server.py" + ) + self.assertTrue(assessment["contaminated"]) + + def test_read_only_probe_not_contamination(self): + assessment = runtime_recovery_guard.assess_recovery_command( + "ps aux | grep mcp_server" + ) + self.assertFalse(assessment["contaminated"]) + + +class TestInventoryDocInSync(unittest.TestCase): + def test_doc_exists(self): + self.assertTrue(os.path.exists(DOC_PATH), DOC_PATH) + + def test_doc_mentions_every_path_id(self): + with open(DOC_PATH, encoding="utf-8") as handle: + doc = handle.read() + for path in rp.iter_restart_paths(): + self.assertIn(path.path_id, doc, f"doc missing {path.path_id}") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pr_ownership_issue_pr_mismatch.py b/tests/test_pr_ownership_issue_pr_mismatch.py index 2286dae..99b500c 100644 --- a/tests/test_pr_ownership_issue_pr_mismatch.py +++ b/tests/test_pr_ownership_issue_pr_mismatch.py @@ -37,6 +37,7 @@ def _live_lock( "operation_type": issue_lock_store.AUTHOR_ISSUE_WORK_LEASE, "acquired_at": now.isoformat(), "expires_at": (now + timedelta(hours=2)).isoformat(), + "session_pid": os.getpid(), "owner_pid": os.getpid(), "status": "active", } @@ -177,11 +178,24 @@ class TestAuthorOwnershipIssuePrMismatch(unittest.TestCase): self.assertFalse(result["proven"], result) self.assertTrue(any("branch" in r for r in result["reasons"])) - def test_no_lock_fail_closed(self): + def test_pidless_durable_lock_rejected(self): + """A lock without any PID identity must be classified as malformed/non-live and fail closed.""" + lock = _live_lock(issue_number=727) + lock.pop("session_pid", None) + lock.pop("owner_pid", None) + lock.pop("pid", None) + path = issue_lock_store.lock_file_path( + remote="prgs", + org="Scaled-Tech-Consulting", + repo="Gitea-Tools", + issue_number=727, + lock_dir=self.lock_dir, + ) + issue_lock_store.save_lock_file(path, lock) result = mcp._prove_author_ownership_for_pr( pr_number=728, pr_title="feat: pr sync", - pr_body="Closes #727", + pr_body="Fixes #727", source_branch="feat/issue-727-pr-sync-status", remote="prgs", host=None, diff --git a/tests/test_pr_work_lease.py b/tests/test_pr_work_lease.py index 8dbcb33..ddbf2cc 100644 --- a/tests/test_pr_work_lease.py +++ b/tests/test_pr_work_lease.py @@ -19,6 +19,7 @@ from pr_work_lease import ( # noqa: E402 assess_reviewer_mutation_blocked, assess_reviewer_stale_head_final_report, format_conflict_fix_lease_body, + find_active_conflict_fix_lease, parse_conflict_fix_lease_comment, parse_reviewer_lease_comment, ) @@ -203,5 +204,157 @@ class TestFormatLease(unittest.TestCase): self.assertEqual(parsed["pr_number"], 376) +class TestConflictFixLeaseLifecycle(unittest.TestCase): + def test_claim_followed_by_matching_release(self): + claim_body = _conflict_fix_body(phase="claimed", worktree="branches/fix-376") + expires = (NOW + timedelta(minutes=60)).isoformat().replace("+00:00", "Z") + release_body = "\n".join([ + CONFLICT_FIX_LEASE_MARKER, + "pr: #376", + "branch: feat/fix-376", + "worktree: branches/fix-376", + "profile: prgs-author", + "phase: released", + f"head_before: {HEAD_A}", + f"head_after: {HEAD_B}", + f"expires_at: {expires}", + ]) + comments = [{"body": claim_body}, {"body": release_body}] + lease = find_active_conflict_fix_lease(comments, pr_number=376, now=NOW) + self.assertIsNone(lease) + + def test_expired_claim_without_release(self): + past_expires = (NOW - timedelta(minutes=10)).isoformat().replace("+00:00", "Z") + claim_body = "\n".join([ + CONFLICT_FIX_LEASE_MARKER, + "pr: #376", + "phase: claimed", + f"head_before: {HEAD_A}", + f"expires_at: {past_expires}", + "profile: prgs-author", + ]) + comments = [{"body": claim_body}] + lease = find_active_conflict_fix_lease(comments, pr_number=376, now=NOW) + self.assertIsNone(lease) + + def test_mismatched_release_different_head(self): + claim_body = _conflict_fix_body(phase="claimed", worktree="branches/fix-376") + expires = (NOW + timedelta(minutes=60)).isoformat().replace("+00:00", "Z") + release_body = "\n".join([ + CONFLICT_FIX_LEASE_MARKER, + "pr: #376", + "profile: prgs-author", + "phase: released", + f"head_before: {HEAD_B}", + f"expires_at: {expires}", + ]) + comments = [{"body": claim_body}, {"body": release_body}] + lease = find_active_conflict_fix_lease(comments, pr_number=376, now=NOW) + self.assertIsNotNone(lease) + self.assertEqual(lease["phase"], "claimed") + + def test_mismatched_release_different_branch(self): + claim_body = "\n".join([ + CONFLICT_FIX_LEASE_MARKER, + "pr: #376", + "branch: feat/branch-A", + "phase: claimed", + f"head_before: {HEAD_A}", + f"expires_at: {(NOW + timedelta(minutes=60)).isoformat().replace('+00:00', 'Z')}", + "profile: prgs-author", + ]) + release_body = "\n".join([ + CONFLICT_FIX_LEASE_MARKER, + "pr: #376", + "branch: feat/branch-B", + "phase: released", + f"head_before: {HEAD_A}", + f"expires_at: {(NOW + timedelta(minutes=60)).isoformat().replace('+00:00', 'Z')}", + "profile: prgs-author", + ]) + comments = [{"body": claim_body}, {"body": release_body}] + lease = find_active_conflict_fix_lease(comments, pr_number=376, now=NOW) + self.assertIsNotNone(lease) + self.assertEqual(lease["phase"], "claimed") + + def test_release_followed_by_newer_claim(self): + claim_1 = _conflict_fix_body(phase="claimed", worktree="branches/fix-376") + expires = (NOW + timedelta(minutes=60)).isoformat().replace("+00:00", "Z") + release_1 = "\n".join([ + CONFLICT_FIX_LEASE_MARKER, + "pr: #376", + "profile: prgs-author", + "phase: released", + f"head_before: {HEAD_A}", + f"head_after: {HEAD_B}", + f"expires_at: {expires}", + ]) + claim_2 = "\n".join([ + CONFLICT_FIX_LEASE_MARKER, + "pr: #376", + "profile: prgs-author", + "phase: claimed", + f"head_before: {HEAD_B}", + f"expires_at: {expires}", + ]) + comments = [{"body": claim_1}, {"body": release_1}, {"body": claim_2}] + lease = find_active_conflict_fix_lease(comments, pr_number=376, now=NOW) + self.assertIsNotNone(lease) + self.assertEqual(lease["head_before"], HEAD_B) + + def test_malformed_or_ambiguous_markers(self): + malformed_release = "\n".join([ + CONFLICT_FIX_LEASE_MARKER, + "pr: #376", + "phase: released", + # missing head_before and profile + ]) + claim_body = _conflict_fix_body(phase="claimed") + comments = [{"body": claim_body}, {"body": malformed_release}] + lease = find_active_conflict_fix_lease(comments, pr_number=376, now=NOW) + self.assertIsNotNone(lease) + + def test_pr818_historical_sequence(self): + comment_14696 = "\n".join([ + "", + "pr: #818", + "branch: feat/issue-638-webui-app-shell-phase1", + "worktree: /Users/jasonwalker/Development/Gitea-Tools/branches/issue-638-webui-app-shell-phase1", + "profile: prgs-author", + "session_id: unknown", + "phase: claimed", + "head_before: 08061b7b8aebdd099a37d1abf5dafcf38e4fd3fb", + "expires_at: 2026-07-23T07:12:13Z", + "reviewer_active: no", + ]) + comment_14730 = "\n".join([ + "", + "pr: #818", + "branch: feat/issue-638-webui-app-shell-phase1", + "worktree: /Users/jasonwalker/Development/Gitea-Tools/branches/issue-638-webui-app-shell-phase1", + "profile: prgs-author", + "session_id: prgs-author-61241-e5129c60", + "phase: released", + "head_before: 08061b7b8aebdd099a37d1abf5dafcf38e4fd3fb", + "head_after: 64b6eb5d5402663098de5ded3b0617cc3b3df98f", + "expires_at: 2026-07-23T06:05:00Z", + "reviewer_active: no", + ]) + comments = [{"body": comment_14696}, {"body": comment_14730}] + check_now = datetime(2026, 7, 23, 6, 30, tzinfo=timezone.utc) + lease = find_active_conflict_fix_lease(comments, pr_number=818, now=check_now) + self.assertIsNone(lease) + + reviewer_gate = assess_reviewer_mutation_blocked( + pr_number=818, + comments=comments, + reviewed_head_sha="64b6eb5d5402663098de5ded3b0617cc3b3df98f", + live_head_sha="64b6eb5d5402663098de5ded3b0617cc3b3df98f", + mutation="approve", + now=check_now, + ) + self.assertTrue(reviewer_gate["mutation_allowed"]) + + if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/tests/test_restart_coordinator.py b/tests/test_restart_coordinator.py new file mode 100644 index 0000000..558aaa0 --- /dev/null +++ b/tests/test_restart_coordinator.py @@ -0,0 +1,340 @@ +"""Tests for the MCP restart coordinator and impact analysis (#658). + +Multi-session fixtures exercise every verdict branch: safe, unsafe (live work), +override, and the fail-closed deny on incomplete inventory. Also covers the +critical-section deny path and the new ``ControlPlaneDB.list_sessions``. +""" + +from __future__ import annotations + +import os +import tempfile +import unittest +from datetime import datetime, timedelta, timezone + +import restart_coordinator as rc +from control_plane_db import ControlPlaneDB + + +NOW = datetime(2026, 7, 24, 6, 0, 0, tzinfo=timezone.utc) + + +def _ts(dt: datetime) -> str: + return dt.isoformat() + + +def _live_pid() -> int: + return os.getpid() + + +def _dead_pid() -> int: + # A pid that is essentially never alive. os.kill(0) on it raises + # ProcessLookupError → is_process_alive False. + return 2_000_000_000 + + +def _session(session_id, *, pid, status="active", heartbeat=None, role="author"): + return { + "session_id": session_id, + "role": role, + "profile": "prgs-author", + "pid": pid, + "status": status, + "last_heartbeat_at": _ts(heartbeat or NOW), + } + + +def _lease( + lease_id, + *, + session_id, + freshness, + kind="issue", + number=658, + phase="allocated", + worktree=None, + role="author", +): + return { + "lease_id": lease_id, + "session_id": session_id, + "role": role, + "phase": phase, + "work_kind": kind, + "work_number": number, + "worktree_path": worktree, + "freshness": {"freshness": freshness}, + } + + +class EvaluateRestartImpactTest(unittest.TestCase): + def test_incomplete_inventory_denies_fail_closed(self) -> None: + report = rc.evaluate_restart_impact( + {"inventory_complete": False, "incomplete_reasons": ["db down"]}, + now=NOW, + ) + self.assertEqual(report.verdict, rc.VERDICT_UNSAFE) + self.assertFalse(report.allow_restart) + self.assertFalse(report.restart_performed) + self.assertIn("db down", report.incomplete_reasons) + self.assertTrue( + any("fail closed" in reasoning for reasoning in report.reasons) + ) + + def test_missing_completeness_flag_denies(self) -> None: + # No inventory_complete key at all → treated as incomplete. + report = rc.evaluate_restart_impact({}, now=NOW) + self.assertEqual(report.verdict, rc.VERDICT_UNSAFE) + self.assertFalse(report.allow_restart) + + def test_no_other_work_is_safe(self) -> None: + report = rc.evaluate_restart_impact( + { + "inventory_complete": True, + "sessions": [_session("requester", pid=_live_pid())], + "leases": [], + }, + now=NOW, + requesting_session_id="requester", + ) + self.assertEqual(report.verdict, rc.VERDICT_SAFE) + self.assertTrue(report.allow_restart) + self.assertEqual(report.blast_radius, rc.BLAST_NONE) + self.assertEqual(report.affected_issues, []) + + def test_dead_foreign_session_and_lease_are_not_disruptive(self) -> None: + report = rc.evaluate_restart_impact( + { + "inventory_complete": True, + "sessions": [ + _session("requester", pid=_live_pid()), + _session("dead", pid=_dead_pid()), + ], + "leases": [ + _lease("l-dead", session_id="dead", freshness="stale_dead_process") + ], + }, + now=NOW, + requesting_session_id="requester", + ) + self.assertEqual(report.verdict, rc.VERDICT_SAFE) + self.assertTrue(report.allow_restart) + self.assertEqual(report.counts["leases_disruptive"], 0) + self.assertEqual(report.counts["sessions_live_other"], 0) + + def test_live_foreign_lease_denies_without_override(self) -> None: + report = rc.evaluate_restart_impact( + { + "inventory_complete": True, + "sessions": [ + _session("requester", pid=_live_pid()), + _session("worker", pid=_live_pid()), + ], + "leases": [ + _lease( + "l1", + session_id="worker", + freshness="active", + worktree="/tmp/wt-658", + phase="implementing", + ) + ], + }, + now=NOW, + requesting_session_id="requester", + ) + self.assertEqual(report.verdict, rc.VERDICT_UNSAFE) + self.assertFalse(report.allow_restart) + # Critical section detected: active lease with a live owner. + self.assertEqual(len(report.critical_sections), 1) + self.assertEqual(report.affected_issues, [658]) + self.assertEqual(report.counts["mutations"], 1) + self.assertTrue(report.override_would_allow) + self.assertEqual(report.blast_radius, rc.BLAST_HIGH) + # Placeholder ack state for the affected session. + self.assertEqual(report.ack_state.get("worker"), "pending") + + def test_operator_override_allows_despite_live_work(self) -> None: + inv = { + "inventory_complete": True, + "sessions": [ + _session("requester", pid=_live_pid()), + _session("worker", pid=_live_pid()), + ], + "leases": [_lease("l1", session_id="worker", freshness="active")], + } + report = rc.evaluate_restart_impact( + inv, + now=NOW, + requesting_session_id="requester", + operator_override=True, + ) + self.assertEqual(report.verdict, rc.VERDICT_OVERRIDE) + self.assertTrue(report.allow_restart) + self.assertFalse(report.restart_performed) + + def test_deny_when_critical_section_open(self) -> None: + # A single live author lease in a mutating phase is a critical section + # that must deny an un-overridden restart. + report = rc.evaluate_restart_impact( + { + "inventory_complete": True, + "sessions": [_session("worker", pid=_live_pid())], + "leases": [ + _lease( + "l1", + session_id="worker", + freshness="active", + phase="merging", + kind="pr", + number=900, + ) + ], + }, + now=NOW, + requesting_session_id="requester", + ) + self.assertEqual(report.verdict, rc.VERDICT_UNSAFE) + self.assertFalse(report.allow_restart) + self.assertEqual(report.affected_prs, [900]) + self.assertEqual(len(report.critical_sections), 1) + + def test_terminal_lock_makes_restart_unsafe(self) -> None: + report = rc.evaluate_restart_impact( + { + "inventory_complete": True, + "sessions": [_session("requester", pid=_live_pid())], + "leases": [], + "terminal_lock": {"terminal_pr": 812}, + }, + now=NOW, + requesting_session_id="requester", + ) + self.assertEqual(report.verdict, rc.VERDICT_UNSAFE) + self.assertFalse(report.allow_restart) + self.assertIsNotNone(report.terminal_lock) + self.assertTrue( + any("terminal" in reasoning for reasoning in report.reasons) + ) + + def test_other_live_session_without_lease_is_disruptive(self) -> None: + report = rc.evaluate_restart_impact( + { + "inventory_complete": True, + "sessions": [ + _session("requester", pid=_live_pid()), + _session("idle-but-live", pid=_live_pid()), + ], + "leases": [], + }, + now=NOW, + requesting_session_id="requester", + ) + self.assertEqual(report.verdict, rc.VERDICT_UNSAFE) + self.assertEqual(report.counts["sessions_live_other"], 1) + + def test_stale_heartbeat_session_not_counted_live(self) -> None: + stale = NOW - timedelta(hours=2) + report = rc.evaluate_restart_impact( + { + "inventory_complete": True, + "sessions": [ + _session("requester", pid=_live_pid()), + _session("stale", pid=_live_pid(), heartbeat=stale), + ], + "leases": [], + }, + now=NOW, + requesting_session_id="requester", + ) + self.assertEqual(report.verdict, rc.VERDICT_SAFE) + self.assertEqual(report.counts["sessions_live_other"], 0) + + def test_prior_recovery_attempts_echoed(self) -> None: + report = rc.evaluate_restart_impact( + { + "inventory_complete": True, + "sessions": [_session("requester", pid=_live_pid())], + "leases": [], + "prior_recovery_attempts": [ + {"kind": "client_reconnect", "at": _ts(NOW)} + ], + }, + now=NOW, + requesting_session_id="requester", + ) + self.assertEqual(len(report.prior_recovery_attempts), 1) + self.assertEqual(report.counts["prior_recovery_attempts"], 1) + + def test_bare_string_freshness_accepted(self) -> None: + lease = _lease("l1", session_id="worker", freshness="active") + lease["freshness"] = "active" # bare string, not a dict + report = rc.evaluate_restart_impact( + { + "inventory_complete": True, + "sessions": [_session("worker", pid=_live_pid())], + "leases": [lease], + }, + now=NOW, + requesting_session_id="requester", + ) + self.assertEqual(report.counts["leases_disruptive"], 1) + + def test_as_dict_is_serializable_dto(self) -> None: + import json + + report = rc.evaluate_restart_impact( + { + "inventory_complete": True, + "sessions": [_session("requester", pid=_live_pid())], + "leases": [], + }, + now=NOW, + requesting_session_id="requester", + ) + payload = report.as_dict() + # Round-trips through JSON — safe for the console DTO. + encoded = json.dumps(payload) + decoded = json.loads(encoded) + self.assertEqual(decoded["verdict"], rc.VERDICT_SAFE) + self.assertIn("audit_record", decoded) + self.assertEqual(decoded["audit_record"]["event"], "restart_impact_evaluated") + self.assertFalse(decoded["restart_performed"]) + self.assertIn("coordinator_version", decoded) + + +class ListSessionsTest(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.db = ControlPlaneDB(os.path.join(self._tmp.name, "cp.sqlite3")) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def test_list_sessions_filters_by_status(self) -> None: + self.db.upsert_session(session_id="a", role="author", pid=1, status="active") + self.db.upsert_session(session_id="b", role="author", pid=2, status="ended") + active = self.db.list_sessions(statuses=("active",)) + ids = {row["session_id"] for row in active} + self.assertEqual(ids, {"a"}) + every = self.db.list_sessions() + self.assertEqual({row["session_id"] for row in every}, {"a", "b"}) + + def test_list_sessions_feeds_coordinator(self) -> None: + self.db.upsert_session( + session_id="requester", role="author", pid=os.getpid(), status="active" + ) + report = rc.evaluate_restart_impact( + { + "inventory_complete": True, + "sessions": self.db.list_sessions(statuses=("active",)), + "leases": [], + }, + now=NOW, + requesting_session_id="requester", + ) + self.assertEqual(report.counts["sessions_total"], 1) + + +if __name__ == "__main__": # pragma: no cover + unittest.main() diff --git a/tests/test_webui_system_health_dashboard.py b/tests/test_webui_system_health_dashboard.py new file mode 100644 index 0000000..417e4ae --- /dev/null +++ b/tests/test_webui_system_health_dashboard.py @@ -0,0 +1,345 @@ +"""Tests for the system-health dashboard view (#639). + +Covers the acceptance criteria directly: the page renders the health DTO +fields (AC1), degraded dependencies are visible (AC2), stale runtime is warned +prominently and never rendered as mutation-safe (AC3), healthy and degraded +fixtures both render (AC4), and the shell carries a nav entry (AC5). +""" +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from starlette.testclient import TestClient + +from webui.app import create_app +from webui.deployment_boundary import scan_text_for_client_secrets +from webui.layout import render_page +from webui.nav import iter_nav_items +from webui.system_health import ( + STATUS_DEGRADED, + STATUS_DOWN, + STATUS_OK, + STATUS_SKIPPED, + STATUS_UNPROVEN, + DependencyProbe, + StaleRuntime, + SystemHealthSnapshot, + VersionInfo, +) +from webui.system_health_views import render_system_health_page + +DASHBOARD_PATH = "/system-health" + + +def _version(*, known: bool = True) -> VersionInfo: + return VersionInfo( + git_sha="1c455b6ec0f9cb761fe6248de68c17e061fb5ecd" if known else None, + git_describe="v0.4.1-12-g1c455b6" if known else None, + control_plane_schema_version=4 if known else None, + python_version="3.13.1", + known=known, + ) + + +def _parity(*, stale: bool = False, determinable: bool = True) -> StaleRuntime: + if stale: + return StaleRuntime( + daemon_head="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + checkout_head="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + remote_head="cccccccccccccccccccccccccccccccccccccccc", + stale=True, + determinable=True, + mutation_safe=False, + reasons=("runtime, checkout, and remote commits disagree",), + ) + if not determinable: + return StaleRuntime( + daemon_head=None, + checkout_head=None, + remote_head=None, + stale=False, + determinable=False, + mutation_safe=False, + reasons=("local checkout HEAD could not be read",), + ) + return StaleRuntime( + daemon_head="1c455b6ec0f9cb761fe6248de68c17e061fb5ecd", + checkout_head="1c455b6ec0f9cb761fe6248de68c17e061fb5ecd", + remote_head="1c455b6ec0f9cb761fe6248de68c17e061fb5ecd", + stale=False, + determinable=True, + mutation_safe=True, + reasons=(), + ) + + +def _snapshot( + *, + status: str = STATUS_OK, + ready: bool = True, + readiness_complete: bool = True, + readiness_reasons: tuple[str, ...] = (), + dependencies: tuple[DependencyProbe, ...] | None = None, + parity: StaleRuntime | None = None, + namespaces: tuple[dict, ...] = (), + probe_errors: tuple[str, ...] = (), + version_known: bool = True, +) -> SystemHealthSnapshot: + if dependencies is None: + dependencies = ( + DependencyProbe( + name="control_plane_db", + kind="sqlite", + status=STATUS_OK, + detail="schema version 4", + required=True, + latency_ms=1.25, + metadata={"schema_version": 4}, + ), + ) + return SystemHealthSnapshot( + status=status, + ready=ready, + readiness_complete=readiness_complete, + readiness_reasons=readiness_reasons, + service="mcp-control-plane-webui", + mode="read-only", + version=_version(known=version_known), + started_at="2026-07-23T19:50:47+00:00", + uptime_seconds=3661.5, + timestamp="2026-07-23T20:51:48+00:00", + deep_probes_requested=False, + dependencies=dependencies, + mcp_namespaces=namespaces, + stale_runtime=parity if parity is not None else _parity(), + probe_errors=probe_errors, + ) + + +class TestHealthyRender(unittest.TestCase): + """AC1 / AC4 — every health DTO field reaches the page.""" + + def setUp(self): + self.html = render_system_health_page(_snapshot()) + + def test_readiness_fields_render(self): + self.assertIn("System health", self.html) + self.assertIn("Ready", self.html) + self.assertIn("mcp-control-plane-webui", self.html) + self.assertIn("read-only", self.html) + self.assertIn("2026-07-23T20:51:48+00:00", self.html) + + def test_version_and_uptime_render(self): + self.assertIn("1c455b6ec0f9cb761fe6248de68c17e061fb5ecd", self.html) + self.assertIn("v0.4.1-12-g1c455b6", self.html) + self.assertIn("3.13.1", self.html) + self.assertIn("3661.500s", self.html) + self.assertIn("1.02h", self.html) + + def test_dependency_row_renders_with_latency(self): + self.assertIn("control_plane_db", self.html) + self.assertIn("sqlite", self.html) + self.assertIn("schema version 4", self.html) + self.assertIn("1.2 ms", self.html) + + def test_healthy_page_shows_no_stale_warning(self): + self.assertNotIn("Stale runtime:", self.html) + self.assertNotIn("Staleness", self.html) + + def test_unknown_version_is_labelled_not_faked(self): + html = render_system_health_page(_snapshot(version_known=False)) + self.assertIn("unknown", html) + self.assertIn("unresolved", html) + + +class TestDegradedRender(unittest.TestCase): + """AC2 — a degraded or unrun dependency is visible, not swallowed.""" + + def setUp(self): + self.deps = ( + DependencyProbe( + name="control_plane_db", + kind="sqlite", + status=STATUS_OK, + detail="schema version 4", + required=True, + latency_ms=0.9, + ), + DependencyProbe( + name="repository", + kind="git", + status=STATUS_DOWN, + detail="repository root is not a git checkout", + required=True, + latency_ms=4.0, + ), + DependencyProbe( + name="gitea", + kind="http", + status=STATUS_SKIPPED, + detail="deep probe not requested", + required=False, + ), + ) + self.html = render_system_health_page( + _snapshot( + status=STATUS_DEGRADED, + ready=False, + readiness_complete=False, + readiness_reasons=("required dependency 'repository' is down",), + dependencies=self.deps, + ) + ) + + def test_degraded_banner_names_the_dependency(self): + self.assertIn("Degraded dependencies:", self.html) + self.assertIn("repository", self.html) + + def test_not_run_probe_is_reported_separately(self): + self.assertIn("Not probed:", self.html) + self.assertIn("gitea", self.html) + self.assertIn("not counted", self.html) + + def test_not_ready_headline_and_reason(self): + self.assertIn("Not ready", self.html) + self.assertIn("required dependency 'repository' is down", self.html) + + def test_degraded_status_badge_present(self): + self.assertIn("badge-health-degraded", self.html) + self.assertIn("badge-health-down", self.html) + + def test_ready_but_incomplete_is_not_shown_as_plain_ready(self): + html = render_system_health_page( + _snapshot(ready=True, readiness_complete=False) + ) + self.assertIn("Ready (incomplete evidence)", html) + + +class TestStaleRuntimeWarning(unittest.TestCase): + """AC3 — staleness is prominent and never claims mutation safety.""" + + def test_stale_runtime_warns_and_denies_mutation_safety(self): + html = render_system_health_page(_snapshot(parity=_parity(stale=True))) + self.assertIn("Stale runtime:", html) + self.assertIn("do not treat this runtime as mutation-safe", html) + self.assertIn("Mutation safeFalse", html) + + def test_indeterminate_parity_is_not_reported_safe(self): + html = render_system_health_page( + _snapshot(parity=_parity(determinable=False)) + ) + self.assertIn("Staleness", html) + self.assertIn("Mutation safeFalse", html) + self.assertIn("DeterminableFalse", html) + + def test_healthy_parity_reports_mutation_safe_true(self): + html = render_system_health_page(_snapshot()) + self.assertIn("Mutation safeTrue", html) + + +class TestNamespacesAndErrors(unittest.TestCase): + def test_unproven_namespace_rows_render(self): + html = render_system_health_page( + _snapshot( + namespaces=( + { + "namespace": "gitea-author", + "required_tool": "gitea_lock_issue", + "status": STATUS_UNPROVEN, + "ide_namespace_proven": False, + "reason": "the web console cannot invoke the IDE-managed MCP client", + }, + ) + ) + ) + self.assertIn("gitea-author", html) + self.assertIn("gitea_lock_issue", html) + self.assertIn("badge-health-unproven", html) + + def test_no_namespaces_degrades_gracefully(self): + html = render_system_health_page(_snapshot(namespaces=())) + self.assertIn("No MCP namespaces are declared.", html) + + def test_probe_errors_render_when_present(self): + html = render_system_health_page( + _snapshot(probe_errors=("probe raised: disk offline",)) + ) + self.assertIn("Probe errors", html) + self.assertIn("disk offline", html) + + def test_probe_error_card_absent_when_clean(self): + self.assertNotIn("Probe errors", render_system_health_page(_snapshot())) + + +class TestReadOnlyAndRedaction(unittest.TestCase): + def test_no_restart_or_kill_controls(self): + html = render_system_health_page(_snapshot()) + self.assertNotIn("", html) + self.assertIn("<script>", html) + + +class TestNavAndRoute(unittest.TestCase): + """AC5 — the shell links the dashboard, and the route serves it.""" + + def setUp(self): + self.client = TestClient(create_app()) + + def test_nav_contains_system_health(self): + self.assertIn( + (DASHBOARD_PATH, "System health"), + [(item.href, item.label) for item in iter_nav_items()], + ) + + def test_rendered_shell_links_dashboard(self): + page = render_page(title="Home", body_html="

x

") + self.assertIn(f'href="{DASHBOARD_PATH}"', page) + + def test_route_renders_dashboard(self): + response = self.client.get(DASHBOARD_PATH) + self.assertEqual(response.status_code, 200) + self.assertIn("System health", response.text) + self.assertIn("Stale-runtime parity", response.text) + + def test_route_is_read_only(self): + self.assertEqual(self.client.post(DASHBOARD_PATH).status_code, 405) + + def test_live_page_leaks_no_client_secret(self): + findings = scan_text_for_client_secrets(self.client.get(DASHBOARD_PATH).text) + self.assertEqual(findings, []) + + +if __name__ == "__main__": # pragma: no cover + unittest.main() diff --git a/webui/app.py b/webui/app.py index 5fbb787..f5147b5 100644 --- a/webui/app.py +++ b/webui/app.py @@ -53,6 +53,7 @@ from webui.system_health import ( process_uptime, snapshot_to_dict as system_health_to_dict, ) +from webui.system_health_views import render_system_health_page _READ_ONLY_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) _AUDIT_MUTATION_PATHS = frozenset({"/audit", "/api/audit"}) @@ -161,6 +162,24 @@ async def api_system_health(request: Request) -> JSONResponse: return JSONResponse(payload, status_code=200 if snapshot.ready else 503) +async def system_health(request: Request) -> HTMLResponse: + """Read-only system-health dashboard (#639). + + Shares the #634 snapshot loader with the JSON API so the page can never + disagree with it. `?deep=1` opts into the network probe exactly as the API + does; the default page load stays cheap. The response is always 200: this + is an operator view that must render the degraded state, not withhold it. + """ + deep = _truthy_flag(request.query_params.get("deep")) + snapshot = load_system_health(deep=deep) + return HTMLResponse( + render_page( + title="System health", + body_html=render_system_health_page(snapshot), + ) + ) + + async def queue(_request: Request) -> HTMLResponse: snapshot = load_queue_snapshot() return HTMLResponse(render_page(title="Queue", body_html=render_queue_page(snapshot))) @@ -571,6 +590,7 @@ def create_app(*, bind_host: str | None = None) -> Starlette: Route("/", home, methods=["GET"]), Route("/health", health, methods=["GET"]), Route(SYSTEM_HEALTH_API_PATH, api_system_health, methods=["GET"]), + Route("/system-health", system_health, methods=["GET"]), Route("/queue", queue, methods=["GET"]), Route("/api/queue", api_queue, methods=["GET"]), Route("/projects", projects, methods=["GET"]), diff --git a/webui/layout.py b/webui/layout.py index 4d62eca..6d8226c 100644 --- a/webui/layout.py +++ b/webui/layout.py @@ -236,6 +236,25 @@ def render_page(*, title: str, body_html: str, extra_head: str = "") -> str: .badge-in-review {{ color: #9ec8f0; border-color: #3d5f7a; }} .badge-duplicate {{ color: #e0c27a; border-color: #6b5730; }} .badge-stale {{ color: #c9b8e8; border-color: #5a4a78; }} + .badge-health-ok {{ color: #8fd19e; border-color: #3d6b4a; }} + .badge-health-degraded {{ color: #e0c27a; border-color: #6b5730; }} + .badge-health-down {{ color: #f0a8a8; border-color: #7a3b3b; }} + .badge-health-skipped {{ color: var(--muted); }} + .badge-health-unproven {{ color: #c9b8e8; border-color: #5a4a78; }} + .health-card {{ + margin: 1.25rem 0; + padding: 0.85rem 1rem 1rem; + border: 1px solid var(--border); + border-radius: 8px; + background: var(--surface); + }} + .health-card h3 {{ margin: 0 0 0.5rem; font-size: 1.05rem; }} + .health-card h4 {{ margin: 1rem 0 0.35rem; font-size: 0.92rem; color: var(--muted); }} + .health-headline {{ color: var(--text); font-size: 1rem; margin: 0 0 0.5rem; }} + .health-degraded {{ border-left-color: #e0c27a; }} + .health-stale {{ border-left-color: #f0a8a8; }} + ul.reasons {{ margin: 0.35rem 0; padding-left: 1.15rem; color: var(--muted); font-size: 0.9rem; }} + ul.reasons li {{ margin-bottom: 0.3rem; }} {extra_head} diff --git a/webui/nav.py b/webui/nav.py index edb128c..c24643d 100644 --- a/webui/nav.py +++ b/webui/nav.py @@ -38,6 +38,7 @@ class NavGroup: NAV_GROUPS: tuple[NavGroup, ...] = ( NavGroup("Health", ( NavItem("/health", "Liveness"), + NavItem("/system-health", "System health"), )), NavGroup("Traffic", ( NavItem("/queue", "Queue"), diff --git a/webui/system_health_views.py b/webui/system_health_views.py new file mode 100644 index 0000000..e7ba492 --- /dev/null +++ b/webui/system_health_views.py @@ -0,0 +1,307 @@ +"""HTML views for the system-health dashboard (#639). + +Renders the read-only :class:`~webui.system_health.SystemHealthSnapshot` +produced by the Phase 1 system-health API (#634). The page offers no restart, +reload, or process-kill control: those are Phase 2 work, and manual process +kills are the contamination path #630 exists to prevent. + +Every free-text field passes through :func:`webui.system_health.redact` before +it reaches HTML, so a probe detail that captured a token or a credentialed URL +cannot leak through the dashboard even though the API redacts it already. +""" + +from __future__ import annotations + +import html + +from webui.system_health import ( + STATUS_DEGRADED, + STATUS_DOWN, + STATUS_OK, + STATUS_SKIPPED, + STATUS_UNPROVEN, + DependencyProbe, + SystemHealthSnapshot, + redact, +) + +_STATUS_BADGE_CLASS = { + STATUS_OK: "badge-health-ok", + STATUS_DEGRADED: "badge-health-degraded", + STATUS_DOWN: "badge-health-down", + STATUS_SKIPPED: "badge-health-skipped", + STATUS_UNPROVEN: "badge-health-unproven", +} + + +def _safe(value: object) -> str: + """Escape free text for HTML after redacting anything secret-shaped. + + Use this for every value that can carry arbitrary text — probe details, + reasons, probe errors — because those are where a credential could ride + along. + """ + return html.escape(redact(str(value))) + + +def _esc(value: object) -> str: + """Escape a structured field for HTML without redacting it. + + Commit SHAs, probe names, statuses, and timestamps are enumerated or + machine-generated, never credential-bearing. They must not go through + :func:`redact`: its opaque-token rule matches any 32-plus-character run, + so a 40-character git SHA would render as ``[redacted]`` and the parity + view — the one thing an operator reads this page for — would be blank. + """ + return html.escape(str(value)) + + +def _status_badge(status: str) -> str: + css = _STATUS_BADGE_CLASS.get(status, "badge-health-unproven") + return f'{_esc(status)}' + + +def _reason_list(reasons: tuple[str, ...], *, empty: str) -> str: + if not reasons: + return f"

{html.escape(empty)}

" + items = "".join(f"
  • {_safe(reason)}
    • " for reason in reasons) + return f"
      {items}
    " + + +def _readiness_card(snapshot: SystemHealthSnapshot) -> str: + """Overall readiness. + + ``ready`` and ``readiness_complete`` are shown separately on purpose: a + snapshot whose required probes never ran is not the same as one that ran + them and passed, and collapsing the two would render an unproven green. + """ + if snapshot.ready and snapshot.readiness_complete: + headline = "Ready" + elif snapshot.ready: + headline = "Ready (incomplete evidence)" + else: + headline = "Not ready" + + return ( + "
    " + f"

    Readiness {_status_badge(snapshot.status)}

    " + f"

    {html.escape(headline)}

    " + "" + f"" + f"" + f"" + "" + f"" + "" + f"" + f"" + "
    Service{_esc(snapshot.service)}
    Mode{_esc(snapshot.mode)}
    Ready{_esc(snapshot.ready)}
    Readiness evidence complete{_esc(snapshot.readiness_complete)}
    Deep probes requested{_esc(snapshot.deep_probes_requested)}
    Observed at{_esc(snapshot.timestamp)}
    " + "

    Readiness reasons

    " + f"{_reason_list(snapshot.readiness_reasons, empty='No readiness objections recorded.')}" + "
    " + ) + + +def _version_card(snapshot: SystemHealthSnapshot) -> str: + version = snapshot.version + uptime_hours = snapshot.uptime_seconds / 3600.0 + known = ( + "resolved" + if version.known + else "unresolved — version fields could not be read from the checkout" + ) + schema = version.control_plane_schema_version + return ( + "
    " + "

    Version and uptime

    " + "" + f"" + "" + f"" + "" + f"" + f"" + f"" + f"" + "" + f"" + "
    Git SHA{_esc(version.git_sha or 'unknown')}
    Git describe{_esc(version.git_describe or 'unknown')}
    Control-plane schema{_esc(schema if schema is not None else 'unknown')}
    Python{_esc(version.python_version)}
    Version status{html.escape(known)}
    Started at{_esc(snapshot.started_at)}
    Uptime{snapshot.uptime_seconds:.3f}s ({uptime_hours:.2f}h)
    " + "
    " + ) + + +def _dependency_rows(probes: tuple[DependencyProbe, ...]) -> str: + if not probes: + return "

    No dependency probes were reported.

    " + rows = [] + for probe in probes: + latency = ( + f"{probe.latency_ms:.1f} ms" if probe.latency_ms is not None else "n/a" + ) + rows.append( + "" + f"{_esc(probe.name)}" + f"{_esc(probe.kind)}" + f"{_status_badge(probe.status)}" + f"{_esc('required' if probe.required else 'optional')}" + f"{html.escape(latency)}" + f"{_safe(probe.detail)}" + "" + ) + return ( + "" + "" + "" + "" + f"{''.join(rows)}
    DependencyKindStatusRequirementLatencyDetail
    " + ) + + +def _dependency_card(snapshot: SystemHealthSnapshot) -> str: + degraded = [probe for probe in snapshot.dependencies if probe.ran and not probe.healthy] + not_run = [probe for probe in snapshot.dependencies if not probe.ran] + + banner = "" + if degraded: + names = ", ".join(sorted(probe.name for probe in degraded)) + banner += ( + "

    Degraded dependencies: " + f"{_esc(names)}

    " + ) + if not_run: + names = ", ".join(sorted(probe.name for probe in not_run)) + banner += ( + "

    Not probed: " + f"{_esc(names)} — these contribute no evidence and are not counted " + "as healthy.

    " + ) + + return ( + "
    " + "

    Dependencies

    " + f"{banner}" + f"{_dependency_rows(snapshot.dependencies)}" + "

    Details are redacted at the API boundary and again " + "before rendering; credentials are never displayed.

    " + "
    " + ) + + +def _namespace_card(snapshot: SystemHealthSnapshot) -> str: + if not snapshot.mcp_namespaces: + body = "

    No MCP namespaces are declared.

    " + else: + rows = [] + for entry in snapshot.mcp_namespaces: + rows.append( + "" + f"{_esc(entry.get('namespace'))}" + f"{_esc(entry.get('required_tool'))}" + f"{_status_badge(str(entry.get('status') or STATUS_UNPROVEN))}" + f"{_esc(entry.get('ide_namespace_proven'))}" + f"{_safe(entry.get('reason'))}" + "" + ) + body = ( + "" + "" + "" + "" + f"{''.join(rows)}
    NamespaceRequired toolStatusIDE-provenReason
    " + ) + return ( + "
    " + "

    MCP namespaces

    " + f"{body}" + "

    The web process runs outside the IDE-managed MCP " + "client, so namespace health is reported as unproven rather than " + "guessed (#543).

    " + "
    " + ) + + +def _stale_runtime_card(snapshot: SystemHealthSnapshot) -> str: + stale = snapshot.stale_runtime + if stale.stale: + warning = ( + "

    Stale runtime: " + "the running code, the checkout, and the remote-tracking commit " + "disagree. Capability gates may be evaluating obsolete code — " + "do not treat this runtime as mutation-safe.

    " + ) + elif not stale.determinable: + warning = ( + "

    Staleness " + "indeterminate: parity could not be proven, so this " + "runtime is not reported as mutation-safe.

    " + ) + else: + warning = "" + + return ( + "
    " + "

    Stale-runtime parity

    " + f"{warning}" + "" + "" + f"" + "" + f"" + "" + f"" + f"" + f"" + f"" + "
    Daemon head{_esc(stale.daemon_head or 'unknown')}
    Checkout head{_esc(stale.checkout_head or 'unknown')}
    Remote head{_esc(stale.remote_head or 'unknown')}
    Stale{_esc(stale.stale)}
    Determinable{_esc(stale.determinable)}
    Mutation safe{_esc(stale.mutation_safe)}
    " + f"{_reason_list(stale.reasons, empty='Runtime, checkout, and remote agree.')}" + "
    " + ) + + +def _probe_error_card(snapshot: SystemHealthSnapshot) -> str: + if not snapshot.probe_errors: + return "" + return ( + "
    " + "

    Probe errors

    " + f"{_reason_list(snapshot.probe_errors, empty='')}" + "
    " + ) + + +def _recovery_card() -> str: + """Sanctioned recovery pointers only — never a manual process kill (#630).""" + return ( + "
    " + "

    Recovery

    " + "

    This dashboard is read-only. Restart and reload " + "controls arrive in Phase 2 (#642); until then recovery runs through " + "the sanctioned client reconnect / operator restart path.

    " + "
      " + "
    • Runtime and session view — active profile, " + "workflow hashes, and shell health.
      • " + "
    • Reconnect the MCP client from the IDE, then re-run the blocked " + "cycle. Never kill the daemon process manually: unmanaged kills are " + "recorded as runtime contamination (#630).
      • " + "
    • See docs/webui-local-dev.md for the documented " + "recovery sequence.
      • " + "
    " + "
    " + ) + + +def render_system_health_page(snapshot: SystemHealthSnapshot) -> str: + """Render the full system-health dashboard body.""" + return ( + "

    System health

    " + "

    Read-only view of the Phase 1 system-health API " + "(/api/v1/system/health). Reload this page to refresh; " + "nothing here polls or mutates on your behalf.

    " + f"{_readiness_card(snapshot)}" + f"{_stale_runtime_card(snapshot)}" + f"{_version_card(snapshot)}" + f"{_dependency_card(snapshot)}" + f"{_namespace_card(snapshot)}" + f"{_probe_error_card(snapshot)}" + f"{_recovery_card()}" + )