fix(cleanup): harden readback scope and ownership fail-closed (#687)
Require branch-scoped post-delete not-found (not generic/repo/host 404), emit consistent top-level cleanup fields, fail closed on ownership inventory errors, never auto-reclaim expired control-plane leases, include active comment-backed reviewer leases, apply the same gates to reconcile_merged_cleanups, and match ownership with normalized host identity.
This commit is contained in:
+199
-95
@@ -126,6 +126,13 @@ READBACK_AUTHENTICATION = "authentication_error"
|
||||
READBACK_AUTHORIZATION = "authorization_error"
|
||||
READBACK_TRANSPORT = "transport_error"
|
||||
READBACK_UNEXPECTED = "unexpected_response"
|
||||
READBACK_AMBIGUOUS_404 = "ambiguous_not_found"
|
||||
|
||||
# Scope of a 404: only branch-scoped absence may set verified_absent.
|
||||
NOT_FOUND_SCOPE_BRANCH = "branch"
|
||||
NOT_FOUND_SCOPE_REPOSITORY = "repository"
|
||||
NOT_FOUND_SCOPE_HOST = "host"
|
||||
NOT_FOUND_SCOPE_UNKNOWN = "unknown"
|
||||
|
||||
ERROR_CLASS_AUTHENTICATION = "authentication"
|
||||
ERROR_CLASS_AUTHORIZATION = "authorization"
|
||||
@@ -139,6 +146,7 @@ OWNERSHIP_CATEGORY_MERGER_LEASE = "merger_lease"
|
||||
OWNERSHIP_CATEGORY_CONTROLLER_LEASE = "controller_lease"
|
||||
OWNERSHIP_CATEGORY_RECONCILER_LEASE = "reconciler_lease"
|
||||
OWNERSHIP_CATEGORY_WORKTREE_BINDING = "worktree_binding"
|
||||
OWNERSHIP_CATEGORY_INVENTORY_ERROR = "ownership_inventory_error"
|
||||
|
||||
_ROLE_TO_OWNERSHIP_CATEGORY = {
|
||||
"author": OWNERSHIP_CATEGORY_AUTHOR_LEASE,
|
||||
@@ -162,21 +170,66 @@ def _norm_str(value: Any) -> str:
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
def normalize_host(host: str | None) -> str:
|
||||
"""Normalize a host identity for ownership matching (no credentials)."""
|
||||
text = _norm_str(host).lower()
|
||||
for prefix in ("https://", "http://"):
|
||||
if text.startswith(prefix):
|
||||
text = text[len(prefix) :]
|
||||
# Drop path/query if a full URL slipped through.
|
||||
text = text.split("/", 1)[0]
|
||||
text = text.split("?", 1)[0]
|
||||
return text.rstrip(".")
|
||||
|
||||
|
||||
def ownership_category_for_role(role: str | None) -> str:
|
||||
"""Map a role kind to a non-secret ownership category label."""
|
||||
key = _norm_str(role).lower()
|
||||
return _ROLE_TO_OWNERSHIP_CATEGORY.get(key, f"{key or 'unknown'}_lease")
|
||||
|
||||
|
||||
def classify_branch_readback_http_status(status_code: int | None) -> dict[str, Any]:
|
||||
"""Classify a GET-branch HTTP status into a secret-free readback result."""
|
||||
def classify_branch_readback_http_status(
|
||||
status_code: int | None,
|
||||
*,
|
||||
not_found_scope: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Classify a GET-branch HTTP status into a secret-free readback result.
|
||||
|
||||
R1: A bare/generic/repository/wrong-host 404 never yields
|
||||
``verified_absent=True``. Only an authoritative *branch-scoped* not-found
|
||||
(``not_found_scope='branch'``) may verify deletion.
|
||||
"""
|
||||
if status_code == 404:
|
||||
scope = _norm_str(not_found_scope).lower() or NOT_FOUND_SCOPE_UNKNOWN
|
||||
if scope == NOT_FOUND_SCOPE_BRANCH:
|
||||
return {
|
||||
"status": READBACK_NOT_FOUND,
|
||||
"error_class": None,
|
||||
"verified_absent": True,
|
||||
"branch_present": False,
|
||||
"not_found_scope": NOT_FOUND_SCOPE_BRANCH,
|
||||
"reasons": [],
|
||||
}
|
||||
# repository / host / unknown / generic 404 — not verified absence
|
||||
reason = {
|
||||
NOT_FOUND_SCOPE_REPOSITORY: (
|
||||
"post-delete readback 404 is repository-scoped, not branch absence"
|
||||
),
|
||||
NOT_FOUND_SCOPE_HOST: (
|
||||
"post-delete readback 404 is host-scoped, not branch absence"
|
||||
),
|
||||
}.get(
|
||||
scope,
|
||||
"post-delete readback 404 is ambiguous (not branch-scoped); "
|
||||
"cannot verify absence",
|
||||
)
|
||||
return {
|
||||
"status": READBACK_NOT_FOUND,
|
||||
"error_class": None,
|
||||
"verified_absent": True,
|
||||
"branch_present": False,
|
||||
"reasons": [],
|
||||
"status": READBACK_AMBIGUOUS_404,
|
||||
"error_class": ERROR_CLASS_UNEXPECTED,
|
||||
"verified_absent": False,
|
||||
"branch_present": None,
|
||||
"not_found_scope": scope,
|
||||
"reasons": [reason],
|
||||
}
|
||||
if status_code in (401, 407):
|
||||
return {
|
||||
@@ -220,70 +273,73 @@ def classify_branch_readback_http_status(status_code: int | None) -> dict[str, A
|
||||
}
|
||||
|
||||
|
||||
def classify_branch_readback_exception(exc: BaseException) -> dict[str, Any]:
|
||||
"""Classify a GET-branch exception without leaking credentials or bodies.
|
||||
|
||||
Prefer typed HTTP status when present on the exception chain; fall back to
|
||||
a narrow ``HTTP <code>`` prefix parse. Never includes response bodies,
|
||||
tokens, or raw exception text in the returned payload.
|
||||
"""
|
||||
status_code: int | None = None
|
||||
def _extract_http_status(exc: BaseException) -> int | None:
|
||||
"""Extract an HTTP status code from an exception chain (secret-free)."""
|
||||
seen: set[int] = set()
|
||||
current: BaseException | None = exc
|
||||
while current is not None and id(current) not in seen:
|
||||
seen.add(id(current))
|
||||
code = getattr(current, "code", None)
|
||||
if isinstance(code, int):
|
||||
status_code = code
|
||||
break
|
||||
status = getattr(current, "status", None)
|
||||
if isinstance(status, int):
|
||||
status_code = status
|
||||
break
|
||||
status_code_attr = getattr(current, "status_code", None)
|
||||
if isinstance(status_code_attr, int):
|
||||
status_code = status_code_attr
|
||||
break
|
||||
current = current.__cause__ or current.__context__
|
||||
|
||||
if status_code is None:
|
||||
# Narrow, non-secret parse of our own RuntimeError shape: "HTTP 404: ..."
|
||||
text = str(exc) if exc is not None else ""
|
||||
for attr in ("code", "status", "status_code"):
|
||||
value = getattr(current, attr, None)
|
||||
if isinstance(value, int) and 100 <= value <= 599:
|
||||
return value
|
||||
text = str(current) if current is not None else ""
|
||||
match = re.match(r"HTTP\s+(\d{3})\b", text)
|
||||
if match:
|
||||
status_code = int(match.group(1))
|
||||
else:
|
||||
lower = text.lower()
|
||||
if "not found" in lower or "404" in lower:
|
||||
status_code = 404
|
||||
elif "unauthorized" in lower or "401" in lower:
|
||||
status_code = 401
|
||||
elif "forbidden" in lower or "403" in lower:
|
||||
status_code = 403
|
||||
elif any(
|
||||
token in lower
|
||||
for token in (
|
||||
"timed out",
|
||||
"timeout",
|
||||
"connection",
|
||||
"network",
|
||||
"temporarily unavailable",
|
||||
"name or service not known",
|
||||
"nodename nor servname",
|
||||
)
|
||||
):
|
||||
return {
|
||||
"status": READBACK_TRANSPORT,
|
||||
"error_class": ERROR_CLASS_TRANSPORT,
|
||||
"verified_absent": False,
|
||||
"branch_present": None,
|
||||
"reasons": [
|
||||
"post-delete branch readback transport/upstream failure"
|
||||
],
|
||||
}
|
||||
return int(match.group(1))
|
||||
current = current.__cause__ or current.__context__
|
||||
return None
|
||||
|
||||
|
||||
def classify_branch_readback_exception(
|
||||
exc: BaseException,
|
||||
*,
|
||||
not_found_scope: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Classify a GET-branch exception without leaking credentials or bodies.
|
||||
|
||||
R1: substring ``404`` / ``not found`` alone never becomes verified_absent.
|
||||
Callers must pass ``not_found_scope='branch'`` only after authoritative
|
||||
proof that the repository/host is still reachable and the 404 is branch-level.
|
||||
"""
|
||||
status_code = _extract_http_status(exc)
|
||||
if status_code is None:
|
||||
lower = str(exc).lower() if exc is not None else ""
|
||||
if any(
|
||||
token in lower
|
||||
for token in (
|
||||
"timed out",
|
||||
"timeout",
|
||||
"connection",
|
||||
"network",
|
||||
"temporarily unavailable",
|
||||
"name or service not known",
|
||||
"nodename nor servname",
|
||||
)
|
||||
):
|
||||
return {
|
||||
"status": READBACK_TRANSPORT,
|
||||
"error_class": ERROR_CLASS_TRANSPORT,
|
||||
"verified_absent": False,
|
||||
"branch_present": None,
|
||||
"reasons": [
|
||||
"post-delete branch readback transport/upstream failure"
|
||||
],
|
||||
}
|
||||
if "unauthorized" in lower:
|
||||
status_code = 401
|
||||
elif "forbidden" in lower:
|
||||
status_code = 403
|
||||
elif "404" in lower or "not found" in lower:
|
||||
# Ambiguous: do NOT treat as branch absence without scope proof.
|
||||
status_code = 404
|
||||
if not_found_scope is None:
|
||||
not_found_scope = NOT_FOUND_SCOPE_UNKNOWN
|
||||
|
||||
if status_code is not None:
|
||||
return classify_branch_readback_http_status(status_code)
|
||||
return classify_branch_readback_http_status(
|
||||
status_code, not_found_scope=not_found_scope
|
||||
)
|
||||
|
||||
return {
|
||||
"status": READBACK_UNEXPECTED,
|
||||
@@ -297,21 +353,26 @@ def classify_branch_readback_exception(exc: BaseException) -> dict[str, Any]:
|
||||
def assess_post_delete_readback(readback: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Decide whether DELETE success may be reported after branch readback.
|
||||
|
||||
Success requires an authoritative not-found readback. DELETE HTTP success
|
||||
alone is never enough.
|
||||
Success requires authoritative branch-scoped not-found
|
||||
(``verified_absent=True``). DELETE HTTP success alone is never enough.
|
||||
Generic/repository/host 404 cannot verify absence (R1).
|
||||
"""
|
||||
payload = dict(readback or {})
|
||||
status = _norm_str(payload.get("status")) or READBACK_UNEXPECTED
|
||||
verified = bool(payload.get("verified_absent")) or status == READBACK_NOT_FOUND
|
||||
if status == READBACK_NOT_FOUND and verified:
|
||||
# Only explicit verified_absent flag counts — never infer from status alone
|
||||
# when status is a bare not_found without branch scope proof.
|
||||
verified = bool(payload.get("verified_absent")) is True
|
||||
if verified and status == READBACK_NOT_FOUND:
|
||||
return {
|
||||
"ok": True,
|
||||
"success": True,
|
||||
"verified_absent": True,
|
||||
"readback": {
|
||||
"status": READBACK_NOT_FOUND,
|
||||
"verified_absent": True,
|
||||
"branch_present": False,
|
||||
"error_class": None,
|
||||
"not_found_scope": NOT_FOUND_SCOPE_BRANCH,
|
||||
},
|
||||
"reasons": [],
|
||||
}
|
||||
@@ -326,29 +387,72 @@ def assess_post_delete_readback(readback: dict[str, Any] | None) -> dict[str, An
|
||||
reasons = ["post-delete branch readback authorization failed"]
|
||||
elif status == READBACK_TRANSPORT:
|
||||
reasons = ["post-delete branch readback transport/upstream failure"]
|
||||
elif status == READBACK_AMBIGUOUS_404:
|
||||
reasons = [
|
||||
"post-delete readback 404 is not branch-scoped; "
|
||||
"cannot verify absence"
|
||||
]
|
||||
else:
|
||||
reasons = ["post-delete branch readback could not verify deletion"]
|
||||
|
||||
return {
|
||||
"ok": False,
|
||||
"success": False,
|
||||
"verified_absent": False,
|
||||
"readback": {
|
||||
"status": status,
|
||||
"verified_absent": False,
|
||||
"branch_present": payload.get("branch_present"),
|
||||
"error_class": payload.get("error_class"),
|
||||
"not_found_scope": payload.get("not_found_scope"),
|
||||
},
|
||||
"reasons": reasons,
|
||||
"blocker_kind": "post_delete_readback_failed",
|
||||
}
|
||||
|
||||
|
||||
def _repo_matches(record: dict[str, Any], *, remote: str, org: str, repo: str) -> bool:
|
||||
return (
|
||||
_norm_str(record.get("remote")).lower() == _norm_str(remote).lower()
|
||||
and _norm_str(record.get("org")).lower() == _norm_str(org).lower()
|
||||
and _norm_str(record.get("repo")).lower() == _norm_str(repo).lower()
|
||||
)
|
||||
def cleanup_result_envelope(
|
||||
*,
|
||||
success: bool,
|
||||
performed: bool,
|
||||
delete_acknowledged: bool,
|
||||
verified_absent: bool,
|
||||
**extra: Any,
|
||||
) -> dict[str, Any]:
|
||||
"""R2: consistent top-level cleanup result fields on every return path."""
|
||||
out: dict[str, Any] = {
|
||||
"success": bool(success),
|
||||
"performed": bool(performed),
|
||||
"delete_acknowledged": bool(delete_acknowledged),
|
||||
"verified_absent": bool(verified_absent),
|
||||
}
|
||||
out.update(extra)
|
||||
return out
|
||||
|
||||
|
||||
def _repo_matches(
|
||||
record: dict[str, Any],
|
||||
*,
|
||||
remote: str,
|
||||
org: str,
|
||||
repo: str,
|
||||
host: str | None = None,
|
||||
) -> bool:
|
||||
"""Match ownership record to target remote/org/repo and normalized host."""
|
||||
if (
|
||||
_norm_str(record.get("remote")).lower() != _norm_str(remote).lower()
|
||||
or _norm_str(record.get("org")).lower() != _norm_str(org).lower()
|
||||
or _norm_str(record.get("repo")).lower() != _norm_str(repo).lower()
|
||||
):
|
||||
return False
|
||||
expected_host = normalize_host(host)
|
||||
record_host = normalize_host(record.get("host") or record.get("host_name"))
|
||||
# When both sides declare a host, they must agree after normalization.
|
||||
# A record host that disagrees with the expected host is out of scope.
|
||||
# Legacy records without host still match when remote/org/repo agree.
|
||||
if expected_host and record_host and expected_host != record_host:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _branch_matches(record: dict[str, Any], branch: str) -> bool:
|
||||
@@ -359,13 +463,21 @@ def assess_ownership_record_activity(record: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Classify one ownership record as blocking or non-blocking.
|
||||
|
||||
Distinguishes active ownership from expired/released/terminal/stale records.
|
||||
Stale/expired records block only when reclaim is not allowed (sticky foreign
|
||||
ownership with live owner + present worktree).
|
||||
Expired/stale records block unless reclaim_allowed is *explicitly* True
|
||||
(O2: never treat missing/unknown reclaim as auto-allowed).
|
||||
"""
|
||||
status = _norm_str(record.get("status")).lower()
|
||||
category = _norm_str(record.get("category")) or "unknown"
|
||||
reclaim_allowed = record.get("reclaim_allowed")
|
||||
|
||||
if category == OWNERSHIP_CATEGORY_INVENTORY_ERROR:
|
||||
return {
|
||||
"blocks": True,
|
||||
"status": status or "unknown",
|
||||
"category": category,
|
||||
"reason": "ownership inventory failed closed",
|
||||
}
|
||||
|
||||
if status in _TERMINAL_OWNERSHIP_STATUSES:
|
||||
return {
|
||||
"blocks": False,
|
||||
@@ -381,8 +493,7 @@ def assess_ownership_record_activity(record: dict[str, Any]) -> dict[str, Any]:
|
||||
"reason": f"active {category} ownership still uses the target branch",
|
||||
}
|
||||
if status in _EXPIRED_STATUSES or status in _STALE_STATUSES:
|
||||
# Canonical recovery policy (#601): reclaimable expired/stale does not
|
||||
# block; sticky expired/stale (live pid + present worktree) does.
|
||||
# O2: only explicit reclaim_allowed=True skips the block.
|
||||
if reclaim_allowed is True:
|
||||
return {
|
||||
"blocks": False,
|
||||
@@ -393,24 +504,13 @@ def assess_ownership_record_activity(record: dict[str, Any]) -> dict[str, Any]:
|
||||
"does not block deletion"
|
||||
),
|
||||
}
|
||||
if reclaim_allowed is False:
|
||||
return {
|
||||
"blocks": True,
|
||||
"status": status,
|
||||
"category": category,
|
||||
"reason": (
|
||||
f"sticky {status} {category} ownership still protects the "
|
||||
"target branch (recovery review required)"
|
||||
),
|
||||
}
|
||||
# Unknown reclaimability → fail closed
|
||||
return {
|
||||
"blocks": True,
|
||||
"status": status,
|
||||
"category": category,
|
||||
"reason": (
|
||||
f"{status} {category} ownership reclaimability unknown; "
|
||||
"fail closed"
|
||||
f"{status} {category} ownership still protects the target "
|
||||
"branch (reclaim not proven; fail closed)"
|
||||
),
|
||||
}
|
||||
# Unknown status → fail closed
|
||||
@@ -431,15 +531,16 @@ def assess_active_branch_ownership(
|
||||
org: str,
|
||||
repo: str,
|
||||
branch: str,
|
||||
host: str | None = None,
|
||||
records: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Assess whether any active ownership still uses *branch* in *repo*.
|
||||
|
||||
Only records matching the exact remote/org/repo/branch quadruple are
|
||||
considered. Other repositories or branches never produce a false block.
|
||||
Returned denial identifies ownership category only — never secrets.
|
||||
Matching requires remote/org/repo/branch and, when provided, normalized
|
||||
host identity. Other repositories, hosts, or branches never false-block.
|
||||
"""
|
||||
target_branch = _norm_str(branch)
|
||||
expected_host = normalize_host(host)
|
||||
considered: list[dict[str, Any]] = []
|
||||
blocking: list[dict[str, Any]] = []
|
||||
ignored: list[dict[str, Any]] = []
|
||||
@@ -447,11 +548,13 @@ def assess_active_branch_ownership(
|
||||
for raw in records or []:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
if not _repo_matches(raw, remote=remote, org=org, repo=repo):
|
||||
if not _repo_matches(
|
||||
raw, remote=remote, org=org, repo=repo, host=expected_host or None
|
||||
):
|
||||
ignored.append(
|
||||
{
|
||||
"category": _norm_str(raw.get("category")) or "unknown",
|
||||
"reason": "different repository scope",
|
||||
"reason": "different repository or host scope",
|
||||
}
|
||||
)
|
||||
continue
|
||||
@@ -488,6 +591,7 @@ def assess_active_branch_ownership(
|
||||
"remote": remote,
|
||||
"org": org,
|
||||
"repo": repo,
|
||||
"host": expected_host or None,
|
||||
"branch": target_branch,
|
||||
"blocking_categories": categories,
|
||||
"blocking": blocking,
|
||||
|
||||
Reference in New Issue
Block a user