Scaled-Tech-Consulting/Gitea-Tools
2
0
Fork 0
Code Issues 67 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: exclude foreign-claimed work from allocation instead of blockading the queue (Closes #765)

Browse Source 
Recovered preserved candidate d06198b onto current master 0c2f45a via clean
cherry-pick. Active foreign leases are excluded before allocator ranking;
controller_instance_id ownership is persisted; dashboard matches allocator
exclusion. Stable patch-id bacafc5f… preserved.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
jcwalker3
2026-07-20 13:52:39 -05:00
co-authored by Claude Opus 4.8
parent 0c2f45abb7
commit ad13d872df
5 changed files with 735 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
allocator_service.py
+187 -5
View File
@@ -21,7 +21,7 @@ from __future__ import annotations
import os
import uuid
from dataclasses import dataclass, field
from typing import Any, Sequence
from typing import Any, Mapping, Sequence
from control_plane_db import (
ControlPlaneDB,
@@ -40,6 +40,16 @@ OUTCOME_NEEDS_CONTROLLER = "needs_controller"
OUTCOME_NO_SAFE = "no_safe_work"
OUTCOME_ROLE_INELIGIBLE = "role_ineligible"
OUTCOME_PREVIEW = "preview" # dry-run only (apply=false)
# #765: ownership could not be established for every remaining candidate.
OUTCOME_OWNERSHIP_DEFECT = "allocator_ownership_defect"
# #765 skip reason code for work already claimed by a different controller.
SKIP_CLAIMED_BY_OTHER_SESSION = "claimed_by_other_session"
# Ownership verdicts for a live claim on a candidate (#765).
OWNERSHIP_OWN = "own"
OWNERSHIP_FOREIGN = "foreign"
OWNERSHIP_UNKNOWN = "unknown"
# Human-readable statement of how a winner is chosen (#758 AC10). Reported
# alongside allocator results so the flat status:ready tier and its
@@ -144,9 +154,76 @@ class SkipRecord:
kind: str
number: int
reason: str
reason_code: str | None = None
def as_dict(self) -> dict[str, Any]:
return {"kind": self.kind, "number": self.number, "reason": self.reason}
return {
"kind": self.kind,
"number": self.number,
"reason": self.reason,
"reason_code": self.reason_code,
}
CONTROLLER_INSTANCE_ENV = "GITEA_CONTROLLER_INSTANCE_ID"
def resolve_controller_instance_id(
env: Mapping[str, str] | None = None,
) -> str | None:
"""Return this controller's stable identity, or ``None`` if undeclared.
Deliberately has no derived fallback. The obvious candidates are unsafe:
``session_id`` is regenerated per invocation, and the MCP process pid is
shared by every controller attached to the same daemon — two independent
controllers really do report the same pid and profile. Guessing from either
would let one controller adopt another's lease, which is the failure #765
exists to prevent. When this returns ``None``, live claims are treated as
unidentified: they are excluded from selection and reported as ownership
defects rather than adopted.
"""
source = env if env is not None else os.environ
return (source.get(CONTROLLER_INSTANCE_ENV) or "").strip() or None
def classify_claim_ownership(
claim: dict[str, Any] | None,
*,
session_id: str | None,
controller_instance_id: str | None,
) -> str | None:
"""Classify a live claim as own / foreign / unknown ownership (#765).
Returns ``None`` when the candidate carries no live claim.
Session ids are regenerated per allocator invocation, so they only prove
ownership positively (an exact match is certainly this session). The
durable signal is ``controller_instance_id``. When either side lacks one,
ownership is *unknown*: the allocator must not assume that a lease sharing
the same profile belongs to this controller, so unknown is treated as
not-ours for selection purposes and reported as an ownership defect.
"""
if not claim:
return None
claim_session = str(claim.get("session_id") or "").strip()
claim_instance = str(claim.get("controller_instance_id") or "").strip()
own_session = str(session_id or "").strip()
own_instance = str(controller_instance_id or "").strip()
if claim_session and own_session and claim_session == own_session:
return OWNERSHIP_OWN
if claim_instance and own_instance:
return (
OWNERSHIP_OWN if claim_instance == own_instance else OWNERSHIP_FOREIGN
)
if not claim_instance and not own_instance:
# Neither side declares a controller identity. The session ids differ
# (an exact match returned OWN above), so this is simply someone
# else's lease: foreign, and we wait rather than adopt.
return OWNERSHIP_FOREIGN
# Exactly one side is identified, so the two cannot be compared: this may
# or may not be our own task under a different session id. Never guess.
return OWNERSHIP_UNKNOWN
def normalize_role(role: str | None, *, profile_name: str | None = None) -> str:
@@ -203,8 +280,16 @@ def classify_skip(
*,
role: str,
terminal_pr: int | None,
claim_ownership: str | None = None,
) -> str | None:
"""Return skip reason, or None if candidate is selectable for *role*."""
"""Return skip reason, or None if candidate is selectable for *role*.
*claim_ownership* (#765) is the verdict from
:func:`classify_claim_ownership` for this candidate's live claim. Foreign
and unknown claims are excluded so one session's in-progress task can never
blockade the queue for a different controller; ``own`` stays selectable so
a controller can resume its own work.
"""
if c.state in ("merged", "closed"):
return f"{c.kind}#{c.number} is {c.state}; never assign"
if c.blocked or "status:blocked" in c.labels:
@@ -214,6 +299,16 @@ def classify_skip(
c.dependency_reason
or f"{c.kind}#{c.number} has unmet dependencies"
)
if claim_ownership in (OWNERSHIP_FOREIGN, OWNERSHIP_UNKNOWN):
detail = (
"owned by another controller instance"
if claim_ownership == OWNERSHIP_FOREIGN
else "owner could not be identified; never adopt on a guess"
)
return (
f"{c.kind}#{c.number} {SKIP_CLAIMED_BY_OTHER_SESSION}: "
f"active lease {detail}"
)
if c.already_claimed_elsewhere:
return f"{c.kind}#{c.number} already claimed elsewhere"
if c.kind == "pr" and not (c.head_sha or "").strip():
@@ -287,6 +382,8 @@ def allocate_next_work(
profile_name: str | None = None,
username: str | None = None,
lease_ttl_seconds: int | None = None,
controller_instance_id: str | None = None,
claims: Mapping[tuple[str, int], dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""Select and optionally reserve the next work unit via control-plane DB.
@@ -328,6 +425,7 @@ def allocate_next_work(
role=role_norm,
profile=profile_name,
pid=os.getpid(),
controller_instance_id=controller_instance_id,
)
except Exception as exc: # noqa: BLE001 — surface structured
return {
@@ -369,25 +467,99 @@ def allocate_next_work(
}
terminal_pr = int(terminal["terminal_pr"]) if terminal else None
# #765: live claims exclude work owned by a *different* controller before
# ranking, so one session's in-progress task cannot blockade the queue.
if claims is None:
try:
claims = db.list_active_claims(remote=remote, org=org, repo=repo)
except Exception as exc: # noqa: BLE001
return {
"success": False,
"outcome": OUTCOME_NO_SAFE,
"reasons": [
f"active claim lookup failed: {exc} (fail closed, #765)"
],
"skipped": [],
"assignment": None,
"substrate": "control_plane_db",
}
skipped: list[SkipRecord] = []
claims_excluded: list[dict[str, Any]] = []
ownership_defects: list[dict[str, Any]] = []
ordered = sort_candidates(list(candidates))
selected: WorkCandidate | None = None
for c in ordered:
reason = classify_skip(c, role=role_norm, terminal_pr=terminal_pr)
claim = claims.get((c.kind, int(c.number))) if claims else None
ownership = classify_claim_ownership(
claim,
session_id=session_id,
controller_instance_id=controller_instance_id,
)
reason = classify_skip(
c,
role=role_norm,
terminal_pr=terminal_pr,
claim_ownership=ownership,
)
if reason:
skipped.append(SkipRecord(c.kind, c.number, reason))
is_claim_skip = SKIP_CLAIMED_BY_OTHER_SESSION in reason
skipped.append(
SkipRecord(
c.kind,
c.number,
reason,
SKIP_CLAIMED_BY_OTHER_SESSION if is_claim_skip else None,
)
)
if is_claim_skip and claim:
record = {
"kind": c.kind,
"number": c.number,
"ownership": ownership,
"lease_id": claim.get("lease_id"),
"owner_session_id": claim.get("session_id"),
"owner_controller_instance_id": claim.get(
"controller_instance_id"
),
"expires_at": claim.get("expires_at"),
}
claims_excluded.append(record)
if ownership == OWNERSHIP_UNKNOWN:
ownership_defects.append(record)
continue
selected = c
break
if selected is None:
# If terminal lock blocks all review work, surface that explicitly.
owner_session_id: str | None = None
if terminal_pr is not None and role_norm in (ROLE_REVIEWER, ROLE_MERGER):
outcome = OUTCOME_BLOCKED_TERMINAL
reasons = [
f"no safe work for role '{role_norm}': active terminal-review "
f"lock on PR #{terminal_pr} (resolve terminal path first, #332/#600)"
]
elif ownership_defects:
# #765: every remaining candidate is claimed and at least one owner
# could not be identified. Report the defect; never adopt.
outcome = OUTCOME_OWNERSHIP_DEFECT
reasons = [
f"no safe assignable work for role '{role_norm}': "
f"{len(ownership_defects)} candidate(s) carry an active lease "
"whose controller ownership could not be established. Record a "
"controller_instance_id on those sessions; the allocator will "
"not assume a shared profile means shared ownership (#765)."
]
elif claims_excluded:
outcome = OUTCOME_WAIT
reasons = [
f"no unclaimed work for role '{role_norm}': "
f"{len(claims_excluded)} candidate(s) are actively claimed by "
"another controller. Waiting; their leases are not adopted (#765)."
]
# Preserve the pre-#765 wait contract: name the blocking owner.
owner_session_id = claims_excluded[0].get("owner_session_id")
else:
outcome = OUTCOME_NO_SAFE
reasons = [
@@ -414,6 +586,10 @@ def allocate_next_work(
"substrate": "control_plane_db",
"file_lock_only": False,
"comment_lease_only": False,
"controller_instance_id": controller_instance_id,
"claims_excluded": list(claims_excluded),
"ownership_defects": list(ownership_defects),
"owner_session_id": owner_session_id,
"downstream_note": (
"#612 incident bridge remains downstream of #600; "
"allocator never assigns raw monitoring incidents"
@@ -460,6 +636,9 @@ def allocate_next_work(
"substrate": "control_plane_db",
"file_lock_only": False,
"comment_lease_only": False,
"controller_instance_id": controller_instance_id,
"claims_excluded": list(claims_excluded),
"ownership_defects": list(ownership_defects),
"downstream_note": (
"#612 incident bridge remains downstream of #600; "
"allocator never assigns raw monitoring incidents"
@@ -583,6 +762,9 @@ def allocate_next_work(
"substrate": "control_plane_db",
"file_lock_only": False,
"comment_lease_only": False,
"controller_instance_id": controller_instance_id,
"claims_excluded": list(claims_excluded),
"ownership_defects": list(ownership_defects),
"downstream_note": (
"#612 incident bridge remains downstream of #600; "
"allocator never assigns raw monitoring incidents"