Compare commits

...
Author SHA1 Message Date
sysadmin 1be4600261 fix: address #625 REQUEST_CHANGES for lease reclaim test and EOF blank line
- Update MCP expired-lock recovery test for sticky expired ownership
  (live pid + present worktree still fail closed under #601 reclaim rules)
- Remove trailing blank line at EOF in control_plane_db.py
2026-07-10 11:54:05 -04:00
sysadmin 780ef0b1be feat: first-class control-plane lease lifecycle (Closes #601)
Make active leases queryable workflow state with canonical adopt, release,
expire/reclaim, and abandon operations on the #613 control-plane DB substrate.

- lease_lifecycle.py: policy, proof gates, safe_next_action vocabulary
- control_plane_db: schema v3 provenance columns + list/inspect/adopt/abandon
- MCP tools: gitea_*_workflow_lease(s) for list/inspect/adopt/release/expire/abandon/reclaim
- issue_lock_store: sanctioned reclaim of expired author locks (dead pid / missing wt)
- Tests cover lifecycle, foreign steal refusal, allocator compatibility, merger handoff
2026-07-10 11:30:53 -04:00
sysadmin a654cda058 fix: import Any from typing to resolve IDE/compiler diagnostic warnings 2026-07-10 10:27:21 -04:00
sysadmin ab1c2d7973 Merge pull request 'feat: Sentry/GlitchTip incident bridge via incident_links (Closes #612)' (#623) from feat/issue-612-incident-bridge into master 2026-07-10 08:36:47 -05:00
sysadmin 6a179aeb85 feat: Sentry/GlitchTip incident bridge via incident_links (Closes #612)
Add phase-1 observability bridge that turns provider observations into
normal Gitea issues and control-plane incident_links rows. Dry-run is
default; apply creates/links through sanctioned Gitea issue paths only.
Raw incidents remain non-assignable; the #600 allocator sees bridge work
only as ordinary Gitea issues. Builds on #613 substrate and #600 allocator.

Closes #612
2026-07-10 03:18:54 -04:00
sysadmin f20c8436aa Merge pull request 'feat: controller-owned allocator API on control-plane DB (Closes #600)' (#622) from feat/issue-600-controller-allocator-api into master 2026-07-10 02:11:59 -05:00
sysadmin db5d14184f feat: controller-owned allocator API on control-plane DB (Closes #600)
Add gitea_allocate_next_work and allocator_service routing policy on top of
the #613 ControlPlaneDB substrate. Workers get atomic assign+lease results
(or wait/no_safe_work/terminal-path outcomes) without self-selecting work
via file locks or comment-only leases. #612 remains downstream.

Closes #600
2026-07-10 02:54:20 -04:00
sysadmin 10228e1c06 Merge pull request 'feat: control-plane DB substrate for atomic assign/lease (Closes #613)' (#619) from feat/issue-613-allocator-db-substrate into master 2026-07-10 01:44:19 -05:00
sysadmin d8b2b8f1a7 Merge pull request 'fix: head-scope durable #332 review-decision locks (Closes #620)' (#621) from fix/issue-620-head-scoped-review-locks into master
Reviewed-on: #621
2026-07-10 01:12:57 -05:00
sysadmin 9e1d5c5649 fix: head-scope durable #332 review-decision locks (#620)
Prior REQUEST_CHANGES on head A no longer blocks formal re-review on
head B of the same open PR. Locks store reviewed head SHA, hard-stop and
mark_ready/submit gates allow a fresh decision boundary when the live
head differs, and same-head duplicates remain fail-closed. Open-PR lock
cleanup stays forbidden (#594); assessment reports locked vs current
head and stale_by_head / fresh_review_on_current_head_allowed.

Closes #620
2026-07-10 01:04:37 -04:00
16 changed files with 5617 additions and 85 deletions
+610
View File
@@ -0,0 +1,610 @@
"""Controller-owned work allocator policy (#600).
Builds on the #613 control-plane DB substrate (``ControlPlaneDB.assign_and_lease``).
Workers must not self-select exclusive work under the standard multi-LLM
workflow. They call ``gitea_allocate_next_work`` which:
1. Inspects candidate Gitea issues/PRs (never raw monitoring incidents).
2. Applies ADR routing policy (role, terminal path, leases, blocked, deps).
3. Atomically assigns + leases the selected item in one DB transaction.
This module is pure selection + substrate orchestration. Gitea I/O for live
inventory lives in the MCP tool wrapper so tests can inject candidates.
#612 remains downstream: bridge-created Gitea issues become candidates only
after they exist as normal issues; this module never assigns incidents.
"""
from __future__ import annotations
import os
import uuid
from dataclasses import dataclass, field
from typing import Any, Sequence
from control_plane_db import (
ControlPlaneDB,
ControlPlaneError,
InvalidWorkKindError,
LeaseRequiredError,
WORK_KINDS,
)
# Outcomes required by #600 / ADR §5.
OUTCOME_ASSIGNED = "assigned_work"
OUTCOME_WAIT = "wait"
OUTCOME_BLOCKED_TERMINAL = "blocked_by_terminal_path"
OUTCOME_BLOCKED_LEASE = "blocked_by_active_lease"
OUTCOME_NEEDS_CONTROLLER = "needs_controller"
OUTCOME_NO_SAFE = "no_safe_work"
OUTCOME_ROLE_INELIGIBLE = "role_ineligible"
OUTCOME_PREVIEW = "preview" # dry-run only (apply=false)
ROLE_AUTHOR = "author"
ROLE_REVIEWER = "reviewer"
ROLE_MERGER = "merger"
ROLE_RECONCILER = "reconciler"
ROLE_CONTROLLER = "controller"
VALID_ROLES = frozenset(
{ROLE_AUTHOR, ROLE_REVIEWER, ROLE_MERGER, ROLE_RECONCILER, ROLE_CONTROLLER}
)
# Default action matrices by role (mutation gate will re-check).
ROLE_ACTIONS: dict[str, tuple[tuple[str, ...], tuple[str, ...]]] = {
ROLE_AUTHOR: (
("implement", "comment", "push", "create_pr"),
("approve", "merge", "request_changes", "self_select_without_assignment"),
),
ROLE_REVIEWER: (
("review", "comment", "approve", "request_changes"),
("merge", "push", "create_pr", "self_select_without_assignment"),
),
ROLE_MERGER: (
("merge", "comment"),
("approve", "request_changes", "push", "create_pr", "self_select_without_assignment"),
),
ROLE_RECONCILER: (
("comment", "diagnose", "cleanup"),
("approve", "merge", "push", "create_pr", "self_select_without_assignment"),
),
ROLE_CONTROLLER: (
("comment", "diagnose", "allocate"),
("approve", "merge", "push", "create_pr"),
),
}
@dataclass
class WorkCandidate:
"""One assignable Gitea issue or PR presented to the allocator."""
kind: str # issue | pr
number: int
state: str = "open"
labels: tuple[str, ...] = ()
title: str = ""
priority: int = 0
head_sha: str | None = None
# Routing signals (callers derive from Gitea / review feedback).
request_changes_current_head: bool = False
approval_on_current_head: bool = False
approval_stale: bool = False
approval_contaminated: bool = False
mergeable: bool = False
blocked: bool = False
dependency_unmet: bool = False
dependency_reason: str | None = None
already_claimed_elsewhere: bool = False
def __post_init__(self) -> None:
self.kind = (self.kind or "").strip().lower()
self.state = (self.state or "open").strip().lower()
self.labels = tuple(
str(x).strip().lower() for x in (self.labels or ()) if str(x).strip()
)
if self.kind not in WORK_KINDS:
raise InvalidWorkKindError(
f"candidate kind '{self.kind}' is not assignable; only "
f"{sorted(WORK_KINDS)} (never raw incidents)"
)
def as_dict(self) -> dict[str, Any]:
return {
"kind": self.kind,
"number": self.number,
"state": self.state,
"labels": list(self.labels),
"title": self.title,
"priority": self.priority,
"head_sha": self.head_sha,
"request_changes_current_head": self.request_changes_current_head,
"approval_on_current_head": self.approval_on_current_head,
"approval_stale": self.approval_stale,
"approval_contaminated": self.approval_contaminated,
"mergeable": self.mergeable,
"blocked": self.blocked,
"dependency_unmet": self.dependency_unmet,
"dependency_reason": self.dependency_reason,
}
@dataclass
class SkipRecord:
kind: str
number: int
reason: str
def as_dict(self) -> dict[str, Any]:
return {"kind": self.kind, "number": self.number, "reason": self.reason}
def normalize_role(role: str | None, *, profile_name: str | None = None) -> str:
"""Map profile/role strings to a canonical allocator role."""
raw = (role or "").strip().lower()
if raw in VALID_ROLES:
return raw
prof = (profile_name or "").strip().lower()
for token in VALID_ROLES:
if token in prof or prof.endswith(f"-{token}"):
return token
if "author" in raw:
return ROLE_AUTHOR
if "review" in raw:
return ROLE_REVIEWER
if "merg" in raw:
return ROLE_MERGER
if "reconcil" in raw:
return ROLE_RECONCILER
if "control" in raw:
return ROLE_CONTROLLER
raise ControlPlaneError(
f"unknown allocator role '{role}' (profile={profile_name!r}); "
f"expected one of {sorted(VALID_ROLES)}"
)
def expected_role_for_candidate(c: WorkCandidate) -> str:
"""ADR §5.3 routing: which role should take this work next."""
if c.kind == "pr":
if c.approval_contaminated:
return ROLE_RECONCILER
if c.request_changes_current_head:
return ROLE_AUTHOR
if c.approval_stale:
return ROLE_REVIEWER
if c.approval_on_current_head and c.mergeable:
return ROLE_MERGER
# Open PR without terminal verdict → reviewer
return ROLE_REVIEWER
# Issues: ready work → author by default; blocked stays controller/none
labels = set(c.labels)
if "status:blocked" in labels or c.blocked:
return ROLE_CONTROLLER
return ROLE_AUTHOR
def role_actions(role: str) -> tuple[tuple[str, ...], tuple[str, ...]]:
return ROLE_ACTIONS.get(role, ROLE_ACTIONS[ROLE_AUTHOR])
def classify_skip(
c: WorkCandidate,
*,
role: str,
terminal_pr: int | None,
) -> str | None:
"""Return skip reason, or None if candidate is selectable for *role*."""
if c.state in ("merged", "closed"):
return f"{c.kind}#{c.number} is {c.state}; never assign"
if c.blocked or "status:blocked" in c.labels:
return f"{c.kind}#{c.number} is blocked"
if c.dependency_unmet:
return (
c.dependency_reason
or f"{c.kind}#{c.number} has unmet dependencies"
)
if c.already_claimed_elsewhere:
return f"{c.kind}#{c.number} already claimed elsewhere"
if c.kind == "pr" and not (c.head_sha or "").strip():
return f"pr#{c.number} missing head_sha pin"
# Terminal path first: when an active terminal PR exists, only that PR
# (or controller diagnosis) is assignable for review-path roles.
if terminal_pr is not None and c.kind == "pr" and c.number != terminal_pr:
if role in (ROLE_REVIEWER, ROLE_MERGER):
return (
f"pr#{c.number} skipped: active terminal-review lock on "
f"PR #{terminal_pr} must be resolved first"
)
expected = expected_role_for_candidate(c)
if role == ROLE_CONTROLLER:
# Controller may inspect anything but only assigns diagnosis targets
# when contaminated / blocked.
if expected == ROLE_RECONCILER or c.blocked:
return None
return f"{c.kind}#{c.number} does not require controller (expected {expected})"
if role != expected:
return (
f"{c.kind}#{c.number} expects role '{expected}', active role is '{role}'"
)
# Ready-gate for issues: prefer status:ready when labels present.
if c.kind == "issue" and c.labels:
if "status:ready" not in c.labels and "status:in-progress" not in c.labels:
# Allow unlabeled open issues; only skip explicit non-ready states.
if any(l.startswith("status:") for l in c.labels):
return f"issue#{c.number} not status:ready ({','.join(c.labels)})"
return None
def sort_candidates(candidates: Sequence[WorkCandidate]) -> list[WorkCandidate]:
"""Higher priority first; then lower number (older issues) for stability."""
return sorted(
candidates,
key=lambda c: (-int(c.priority), c.kind != "pr", int(c.number)),
)
def allocate_next_work(
db: ControlPlaneDB,
*,
session_id: str,
role: str,
remote: str,
org: str,
repo: str,
candidates: Sequence[WorkCandidate],
apply: bool = False,
profile_name: str | None = None,
username: str | None = None,
lease_ttl_seconds: int | None = None,
) -> dict[str, Any]:
"""Select and optionally reserve the next work unit via control-plane DB.
*apply=False* (default): dry-run selection only — no lease/assignment.
*apply=True*: atomic ``assign_and_lease`` for the selected candidate.
Never uses file locks or comment-only leases as the assignment source.
"""
if db is None:
return {
"success": False,
"outcome": OUTCOME_NO_SAFE,
"reasons": [
"control-plane DB substrate unavailable (fail closed, #600/#613)"
],
"skipped": [],
"assignment": None,
"substrate": "control_plane_db",
"file_lock_only": False,
"comment_lease_only": False,
}
try:
role_norm = normalize_role(role, profile_name=profile_name)
except ControlPlaneError as exc:
return {
"success": False,
"outcome": OUTCOME_ROLE_INELIGIBLE,
"reasons": [str(exc)],
"skipped": [],
"assignment": None,
"substrate": "control_plane_db",
}
session_id = (session_id or "").strip() or f"alloc-{uuid.uuid4().hex[:12]}"
try:
db.upsert_session(
session_id=session_id,
role=role_norm,
profile=profile_name,
pid=os.getpid(),
)
except Exception as exc: # noqa: BLE001 — surface structured
return {
"success": False,
"outcome": OUTCOME_NO_SAFE,
"reasons": [
f"failed to register session in control-plane DB: {exc} "
"(fail closed, #613)"
],
"skipped": [],
"assignment": None,
"substrate": "control_plane_db",
}
# Expire stale leases globally before selection.
try:
db.expire_stale_leases()
except Exception as exc: # noqa: BLE001
return {
"success": False,
"outcome": OUTCOME_NO_SAFE,
"reasons": [f"lease expiry failed: {exc} (fail closed)"],
"skipped": [],
"assignment": None,
"substrate": "control_plane_db",
}
terminal = None
try:
terminal = db.get_active_terminal_lock(remote=remote, org=org, repo=repo)
except Exception as exc: # noqa: BLE001
return {
"success": False,
"outcome": OUTCOME_NO_SAFE,
"reasons": [f"terminal lock lookup failed: {exc} (fail closed)"],
"skipped": [],
"assignment": None,
"substrate": "control_plane_db",
}
terminal_pr = int(terminal["terminal_pr"]) if terminal else None
skipped: list[SkipRecord] = []
ordered = sort_candidates(list(candidates))
selected: WorkCandidate | None = None
for c in ordered:
reason = classify_skip(c, role=role_norm, terminal_pr=terminal_pr)
if reason:
skipped.append(SkipRecord(c.kind, c.number, reason))
continue
selected = c
break
if selected is None:
# If terminal lock blocks all review work, surface that explicitly.
if terminal_pr is not None and role_norm in (ROLE_REVIEWER, ROLE_MERGER):
outcome = OUTCOME_BLOCKED_TERMINAL
reasons = [
f"no safe work for role '{role_norm}': active terminal-review "
f"lock on PR #{terminal_pr} (resolve terminal path first, #332/#600)"
]
else:
outcome = OUTCOME_NO_SAFE
reasons = [
f"no safe assignable work for role '{role_norm}' "
f"among {len(ordered)} candidates"
]
return {
"success": True,
"outcome": outcome,
"apply": bool(apply),
"role": role_norm,
"profile_name": profile_name,
"username": username,
"session_id": session_id,
"remote": remote,
"org": org,
"repo": repo,
"selected": None,
"expected_role_next": None,
"reasons": reasons,
"skipped": [s.as_dict() for s in skipped],
"terminal_pr": terminal_pr,
"assignment": None,
"substrate": "control_plane_db",
"file_lock_only": False,
"comment_lease_only": False,
"downstream_note": (
"#612 incident bridge remains downstream of #600; "
"allocator never assigns raw monitoring incidents"
),
}
expected_role = expected_role_for_candidate(selected)
allowed, forbidden = role_actions(role_norm)
selection = {
"kind": selected.kind,
"number": selected.number,
"title": selected.title,
"labels": list(selected.labels),
"head_sha": selected.head_sha,
"priority": selected.priority,
"expected_role_next": expected_role,
"reason_selected": (
f"highest-priority candidate for role '{role_norm}' "
f"(expected_role={expected_role})"
),
}
if not apply:
return {
"success": True,
"outcome": OUTCOME_PREVIEW,
"apply": False,
"role": role_norm,
"profile_name": profile_name,
"username": username,
"session_id": session_id,
"remote": remote,
"org": org,
"repo": repo,
"selected": selection,
"expected_role_next": expected_role,
"reasons": [
"dry-run only (apply=false); no assignment/lease created — "
"call again with apply=true to reserve via control-plane DB"
],
"skipped": [s.as_dict() for s in skipped],
"terminal_pr": terminal_pr,
"assignment": None,
"substrate": "control_plane_db",
"file_lock_only": False,
"comment_lease_only": False,
"downstream_note": (
"#612 incident bridge remains downstream of #600; "
"allocator never assigns raw monitoring incidents"
),
}
# Atomic reserve via #613 substrate.
ttl = lease_ttl_seconds if lease_ttl_seconds is not None else None
try:
kwargs: dict[str, Any] = {
"session_id": session_id,
"role": role_norm,
"remote": remote,
"org": org,
"repo": repo,
"kind": selected.kind,
"number": selected.number,
"expected_head_sha": selected.head_sha,
"allowed_actions": allowed,
"forbidden_actions": forbidden,
"phase": "allocated",
}
if ttl is not None:
kwargs["lease_ttl_seconds"] = int(ttl)
result = db.assign_and_lease(**kwargs)
except (InvalidWorkKindError, LeaseRequiredError, ControlPlaneError) as exc:
return {
"success": False,
"outcome": OUTCOME_NO_SAFE,
"apply": True,
"role": role_norm,
"session_id": session_id,
"remote": remote,
"org": org,
"repo": repo,
"selected": selection,
"expected_role_next": expected_role,
"reasons": [f"atomic assign+lease failed: {exc} (fail closed, #613)"],
"skipped": [s.as_dict() for s in skipped],
"terminal_pr": terminal_pr,
"assignment": None,
"substrate": "control_plane_db",
"file_lock_only": False,
"comment_lease_only": False,
}
if result.outcome == "wait":
return {
"success": True,
"outcome": OUTCOME_WAIT,
"apply": True,
"role": role_norm,
"session_id": session_id,
"remote": remote,
"org": org,
"repo": repo,
"selected": selection,
"expected_role_next": expected_role,
"reasons": [result.reason or "foreign active lease"],
"skipped": [s.as_dict() for s in skipped],
"terminal_pr": terminal_pr,
"assignment": result.as_dict(),
"owner_session_id": result.owner_session_id,
"substrate": "control_plane_db",
"file_lock_only": False,
"comment_lease_only": False,
}
if result.outcome == "no_safe_work":
return {
"success": True,
"outcome": OUTCOME_NO_SAFE,
"apply": True,
"role": role_norm,
"session_id": session_id,
"remote": remote,
"org": org,
"repo": repo,
"selected": selection,
"expected_role_next": expected_role,
"reasons": [result.reason or "no_safe_work"],
"skipped": [s.as_dict() for s in skipped],
"terminal_pr": terminal_pr,
"assignment": result.as_dict(),
"substrate": "control_plane_db",
"file_lock_only": False,
"comment_lease_only": False,
}
# assigned
return {
"success": True,
"outcome": OUTCOME_ASSIGNED,
"apply": True,
"role": role_norm,
"profile_name": profile_name,
"username": username,
"session_id": session_id,
"remote": remote,
"org": org,
"repo": repo,
"selected": selection,
"expected_role_next": expected_role,
"reasons": [
selection["reason_selected"],
result.reason or "atomic assign+lease created",
],
"skipped": [s.as_dict() for s in skipped],
"terminal_pr": terminal_pr,
"assignment": result.as_dict(),
"lease_proof": {
"assignment_id": result.assignment_id,
"lease_id": result.lease_id,
"expires_at": result.expires_at,
"expected_head_sha": result.expected_head_sha,
"allowed_actions": list(result.allowed_actions),
"forbidden_actions": list(result.forbidden_actions),
"source": "control_plane_db.assign_and_lease",
},
"next_valid_command": _next_command(role_norm, selected),
"substrate": "control_plane_db",
"file_lock_only": False,
"comment_lease_only": False,
"downstream_note": (
"#612 incident bridge remains downstream of #600; "
"allocator never assigns raw monitoring incidents"
),
}
def _next_command(role: str, c: WorkCandidate) -> str:
if role == ROLE_AUTHOR and c.kind == "issue":
return f"implement issue #{c.number} under a branches/ worktree; open PR when ready"
if role == ROLE_AUTHOR and c.kind == "pr":
return (
f"address REQUEST_CHANGES on PR #{c.number} at head "
f"{(c.head_sha or '')[:12]} and push fixes"
)
if role == ROLE_REVIEWER:
return (
f"review PR #{c.number} pinned at head {(c.head_sha or '')[:12]} "
"via full reviewer workflow"
)
if role == ROLE_MERGER:
return (
f"merge PR #{c.number} only with explicit operator MERGE "
f"authorization at head {(c.head_sha or '')[:12]}"
)
if role == ROLE_RECONCILER:
return f"diagnose contested state for {c.kind}#{c.number}"
return f"proceed on {c.kind}#{c.number} under role {role}"
def candidate_from_dict(data: dict[str, Any]) -> WorkCandidate:
"""Build a WorkCandidate from a plain dict (tests / MCP inventory)."""
return WorkCandidate(
kind=str(data.get("kind") or "issue"),
number=int(data["number"]),
state=str(data.get("state") or "open"),
labels=tuple(data.get("labels") or ()),
title=str(data.get("title") or ""),
priority=int(data.get("priority") or 0),
head_sha=data.get("head_sha"),
request_changes_current_head=bool(data.get("request_changes_current_head")),
approval_on_current_head=bool(data.get("approval_on_current_head")),
approval_stale=bool(data.get("approval_stale")),
approval_contaminated=bool(data.get("approval_contaminated")),
mergeable=bool(data.get("mergeable")),
blocked=bool(data.get("blocked")),
dependency_unmet=bool(data.get("dependency_unmet")),
dependency_reason=data.get("dependency_reason"),
already_claimed_elsewhere=bool(data.get("already_claimed_elsewhere")),
)
+625 -25
View File
@@ -11,9 +11,9 @@ Implements the durable coordination store described by
shared Postgres or single allocator daemon can replace the connection shared Postgres or single allocator daemon can replace the connection
layer later without changing call sites. layer later without changing call sites.
This module is the **substrate** for #600 (allocator policy/tool) and #612 This module is the **substrate** for #600 (allocator policy/tool), #601
(incident bridge). It does **not** implement ``gitea_allocate_next_work`` (first-class lease lifecycle), and #612 (incident bridge). It does **not**
routing policy or provider adapters. implement ``gitea_allocate_next_work`` routing policy or provider adapters.
""" """
from __future__ import annotations from __future__ import annotations
@@ -29,7 +29,7 @@ from dataclasses import dataclass
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Any, Iterator, Sequence from typing import Any, Iterator, Sequence
SCHEMA_VERSION = 2 SCHEMA_VERSION = 3
# Assignable work kinds only — raw monitoring incidents are never work items. # Assignable work kinds only — raw monitoring incidents are never work items.
WORK_KINDS = frozenset({"issue", "pr"}) WORK_KINDS = frozenset({"issue", "pr"})
@@ -301,6 +301,7 @@ class ControlPlaneDB:
try: try:
conn.executescript(_SCHEMA_SQL) conn.executescript(_SCHEMA_SQL)
self._migrate_incident_links_null_scope(conn) self._migrate_incident_links_null_scope(conn)
self._migrate_lease_lifecycle_columns(conn)
conn.execute( conn.execute(
"INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)", "INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)",
("schema_version", str(SCHEMA_VERSION)), ("schema_version", str(SCHEMA_VERSION)),
@@ -604,6 +605,8 @@ class ControlPlaneDB:
lease_ttl_seconds: int = DEFAULT_LEASE_TTL_SECONDS, lease_ttl_seconds: int = DEFAULT_LEASE_TTL_SECONDS,
phase: str = "claimed", phase: str = "claimed",
now: datetime | None = None, now: datetime | None = None,
worktree_path: str | None = None,
owner_pid: int | None = None,
) -> AssignmentResult: ) -> AssignmentResult:
"""Atomically create assignment + lease for one Gitea work item. """Atomically create assignment + lease for one Gitea work item.
@@ -746,6 +749,32 @@ class ControlPlaneDB:
""", """,
(lease_id, work_item_id, session_id, role, phase, expires, now_s), (lease_id, work_item_id, session_id, role, phase, expires, now_s),
) )
# #601 optional lifecycle columns (present after schema v3 migration)
try:
lcols = {
r[1]
for r in conn.execute("PRAGMA table_info(leases)").fetchall()
}
if "worktree_path" in lcols and worktree_path:
conn.execute(
"UPDATE leases SET worktree_path = ? WHERE lease_id = ?",
(worktree_path, lease_id),
)
if "owner_pid" in lcols:
conn.execute(
"UPDATE leases SET owner_pid = ? WHERE lease_id = ?",
(
owner_pid if owner_pid is not None else os.getpid(),
lease_id,
),
)
if "expected_head_sha" in lcols and expected_head_sha:
conn.execute(
"UPDATE leases SET expected_head_sha = ? WHERE lease_id = ?",
(expected_head_sha, lease_id),
)
except sqlite3.Error:
pass
conn.execute( conn.execute(
""" """
INSERT INTO assignments( INSERT INTO assignments(
@@ -876,32 +905,15 @@ class ControlPlaneDB:
} }
def release_lease(self, lease_id: str, *, session_id: str) -> None: def release_lease(self, lease_id: str, *, session_id: str) -> None:
with self._tx() as conn: """Release a lease; unknown ids are no-ops for backward compatibility."""
with self._tx(immediate=False) as conn:
row = conn.execute( row = conn.execute(
"SELECT * FROM leases WHERE lease_id = ?", "SELECT lease_id FROM leases WHERE lease_id = ?",
(lease_id,), (lease_id,),
).fetchone() ).fetchone()
if not row: if not row:
return return
if row["session_id"] != session_id: self.release_lease_recorded(lease_id, session_id=session_id)
raise ForeignLeaseError(
f"cannot release lease {lease_id} owned by {row['session_id']}"
)
conn.execute(
"UPDATE leases SET status = 'released' WHERE lease_id = ?",
(lease_id,),
)
conn.execute(
"UPDATE assignments SET status = 'released' WHERE lease_id = ?",
(lease_id,),
)
conn.execute(
"""
INSERT INTO events(work_item_id, event_type, message, created_at)
VALUES (?, 'lease_released', ?, ?)
""",
(row["work_item_id"], f"session {session_id} released {lease_id}", _ts()),
)
def require_valid_assignment( def require_valid_assignment(
self, self,
@@ -1149,3 +1161,591 @@ class ControlPlaneDB:
(gitea_org, gitea_repo, int(gitea_issue_number)), (gitea_org, gitea_repo, int(gitea_issue_number)),
).fetchone() ).fetchone()
return dict(row) if row else None return dict(row) if row else None
def get_incident_link_by_provider(
self,
*,
provider: str,
provider_issue_id: str,
provider_base_url: str | None = None,
provider_org: str | None = None,
provider_project: str | None = None,
) -> dict[str, Any] | None:
"""Lookup canonical incident_links row by provider key (#612 / #613)."""
base_url = _norm_scope(provider_base_url)
p_org = _norm_scope(provider_org)
p_project = _norm_scope(provider_project)
with self._tx(immediate=False) as conn:
row = conn.execute(
"""
SELECT * FROM incident_links
WHERE provider = ?
AND provider_base_url = ?
AND provider_org = ?
AND provider_project = ?
AND provider_issue_id = ?
LIMIT 1
""",
(provider, base_url, p_org, p_project, str(provider_issue_id)),
).fetchone()
return dict(row) if row else None
# ── lease lifecycle (#601) ────────────────────────────────────────────
_LEASE_LIFECYCLE_COLUMNS: tuple[tuple[str, str], ...] = (
("worktree_path", "TEXT"),
("owner_pid", "INTEGER"),
("expected_head_sha", "TEXT"),
("adopted_from_session_id", "TEXT"),
("adopted_by_session_id", "TEXT"),
("provenance_json", "TEXT"),
("abandon_proof_json", "TEXT"),
)
def _migrate_lease_lifecycle_columns(self, conn: sqlite3.Connection) -> None:
"""Add provenance/worktree columns to leases for first-class lifecycle (#601)."""
cols = {
row[1]
for row in conn.execute("PRAGMA table_info(leases)").fetchall()
}
if not cols:
return
for name, decl in self._LEASE_LIFECYCLE_COLUMNS:
if name not in cols:
conn.execute(f"ALTER TABLE leases ADD COLUMN {name} {decl}")
def _lease_columns(self, conn: sqlite3.Connection) -> set[str]:
return {
row[1]
for row in conn.execute("PRAGMA table_info(leases)").fetchall()
}
def list_leases(
self,
*,
remote: str | None = None,
org: str | None = None,
repo: str | None = None,
role: str | None = None,
statuses: Sequence[str] | None = None,
limit: int = 100,
) -> list[dict[str, Any]]:
"""List leases joined with work_items as first-class workflow state."""
clauses: list[str] = []
params: list[Any] = []
if remote:
clauses.append("w.remote = ?")
params.append(remote)
if org:
clauses.append("w.org = ?")
params.append(org)
if repo:
clauses.append("w.repo = ?")
params.append(repo)
if role:
clauses.append("l.role = ?")
params.append(role)
if statuses:
placeholders = ", ".join("?" for _ in statuses)
clauses.append(f"l.status IN ({placeholders})")
params.extend(statuses)
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
sql = f"""
SELECT l.*, w.remote, w.org, w.repo, w.kind AS work_kind,
w.number AS work_number, w.state AS work_state,
w.current_head_sha AS work_head_sha,
s.pid AS session_pid, s.profile AS session_profile,
s.status AS session_status
FROM leases l
JOIN work_items w ON w.work_item_id = l.work_item_id
LEFT JOIN sessions s ON s.session_id = l.session_id
{where}
ORDER BY l.expires_at DESC
LIMIT ?
"""
params.append(max(1, int(limit)))
with self._tx(immediate=False) as conn:
rows = conn.execute(sql, params).fetchall()
return [dict(r) for r in rows]
def get_lease_workflow_state(self, lease_id: str) -> dict[str, Any] | None:
"""Return lease + assignment + work_item + session for one lease id."""
with self._tx(immediate=False) as conn:
lease = conn.execute(
"SELECT * FROM leases WHERE lease_id = ?",
(lease_id,),
).fetchone()
if not lease:
return None
work = conn.execute(
"SELECT * FROM work_items WHERE work_item_id = ?",
(lease["work_item_id"],),
).fetchone()
asn = conn.execute(
"""
SELECT * FROM assignments
WHERE lease_id = ?
ORDER BY created_at DESC LIMIT 1
""",
(lease_id,),
).fetchone()
sess = conn.execute(
"SELECT * FROM sessions WHERE session_id = ?",
(lease["session_id"],),
).fetchone()
provenance = None
if "provenance_json" in lease.keys() and lease["provenance_json"]:
try:
provenance = json.loads(lease["provenance_json"])
except (TypeError, json.JSONDecodeError):
provenance = {"raw": lease["provenance_json"]}
return {
"lease": dict(lease),
"work_item": dict(work) if work else None,
"assignment": dict(asn) if asn else None,
"session": dict(sess) if sess else None,
"provenance": provenance,
}
def attach_lease_provenance(
self, lease_id: str, provenance: dict[str, Any]
) -> None:
payload = json.dumps(provenance)
with self._tx() as conn:
cols = self._lease_columns(conn)
if "provenance_json" not in cols:
return
conn.execute(
"UPDATE leases SET provenance_json = ? WHERE lease_id = ?",
(payload, lease_id),
)
if provenance.get("adopted_from_session_id") and "adopted_from_session_id" in cols:
conn.execute(
"""
UPDATE leases
SET adopted_from_session_id = ?, adopted_by_session_id = ?
WHERE lease_id = ?
""",
(
provenance.get("adopted_from_session_id"),
provenance.get("adopted_by_session_id"),
lease_id,
),
)
if provenance.get("worktree_path") and "worktree_path" in cols:
conn.execute(
"UPDATE leases SET worktree_path = ? WHERE lease_id = ?",
(provenance.get("worktree_path"), lease_id),
)
if provenance.get("expected_head_sha") and "expected_head_sha" in cols:
conn.execute(
"UPDATE leases SET expected_head_sha = ? WHERE lease_id = ?",
(provenance.get("expected_head_sha"), lease_id),
)
def release_lease_recorded(
self, lease_id: str, *, session_id: str
) -> dict[str, Any]:
"""Explicit release with audit event + provenance proof (#601)."""
now_s = _ts()
with self._tx() as conn:
row = conn.execute(
"SELECT * FROM leases WHERE lease_id = ?",
(lease_id,),
).fetchone()
if not row:
raise ControlPlaneError(f"unknown lease_id {lease_id}")
if row["session_id"] != session_id:
raise ForeignLeaseError(
f"cannot release lease {lease_id} owned by {row['session_id']}"
)
if row["status"] not in ("active", "expired"):
# idempotent-ish for already released
if row["status"] == "released":
return {
"lease_id": lease_id,
"status": "released",
"session_id": session_id,
"released_at": now_s,
"idempotent": True,
}
conn.execute(
"UPDATE leases SET status = 'released' WHERE lease_id = ?",
(lease_id,),
)
conn.execute(
"UPDATE assignments SET status = 'released' WHERE lease_id = ?",
(lease_id,),
)
proof = {
"lease_id": lease_id,
"status": "released",
"session_id": session_id,
"released_at": now_s,
"prior_status": row["status"],
"work_item_id": row["work_item_id"],
}
cols = self._lease_columns(conn)
if "provenance_json" in cols:
prior = {}
if row["provenance_json"]:
try:
prior = json.loads(row["provenance_json"])
except (TypeError, json.JSONDecodeError):
prior = {}
prior["last_release"] = proof
conn.execute(
"UPDATE leases SET provenance_json = ? WHERE lease_id = ?",
(json.dumps(prior), lease_id),
)
conn.execute(
"""
INSERT INTO events(work_item_id, event_type, message, created_at)
VALUES (?, 'lease_released', ?, ?)
""",
(
row["work_item_id"],
f"session {session_id} released {lease_id}",
now_s,
),
)
return proof
def force_expire_lease(self, lease_id: str, *, reason: str = "") -> None:
now_s = _ts()
with self._tx() as conn:
row = conn.execute(
"SELECT * FROM leases WHERE lease_id = ?",
(lease_id,),
).fetchone()
if not row:
raise ControlPlaneError(f"unknown lease_id {lease_id}")
conn.execute(
"UPDATE leases SET status = 'expired' WHERE lease_id = ?",
(lease_id,),
)
conn.execute(
"UPDATE assignments SET status = 'expired' WHERE lease_id = ?",
(lease_id,),
)
conn.execute(
"""
INSERT INTO events(work_item_id, event_type, message, created_at)
VALUES (?, 'lease_expired', ?, ?)
""",
(
row["work_item_id"],
f"lease {lease_id} force-expired: {reason or 'unspecified'}",
now_s,
),
)
def abandon_lease(
self,
*,
lease_id: str,
requester_session_id: str,
proof: dict[str, Any],
) -> dict[str, Any]:
now_s = _ts()
with self._tx() as conn:
row = conn.execute(
"SELECT * FROM leases WHERE lease_id = ?",
(lease_id,),
).fetchone()
if not row:
raise ControlPlaneError(f"unknown lease_id {lease_id}")
conn.execute(
"UPDATE leases SET status = 'abandoned' WHERE lease_id = ?",
(lease_id,),
)
conn.execute(
"UPDATE assignments SET status = 'abandoned' WHERE lease_id = ?",
(lease_id,),
)
cols = self._lease_columns(conn)
if "abandon_proof_json" in cols:
conn.execute(
"UPDATE leases SET abandon_proof_json = ? WHERE lease_id = ?",
(json.dumps(proof), lease_id),
)
msg = (
f"session {requester_session_id} abandoned lease {lease_id} "
f"(prior owner {row['session_id']})"
)
conn.execute(
"""
INSERT INTO events(work_item_id, event_type, message, created_at)
VALUES (?, 'lease_abandoned', ?, ?)
""",
(row["work_item_id"], msg, now_s),
)
return {
"lease_id": lease_id,
"status": "abandoned",
"prior_owner_session_id": row["session_id"],
"requester_session_id": requester_session_id,
"abandoned_at": now_s,
"proof": proof,
}
def adopt_lease(
self,
*,
lease_id: str,
adopter_session_id: str,
role: str,
worktree_path: str | None = None,
expected_head_sha: str | None = None,
owner_pid: int | None = None,
provenance: dict[str, Any] | None = None,
lease_ttl_seconds: int = DEFAULT_LEASE_TTL_SECONDS,
) -> dict[str, Any]:
"""Transfer or refresh a lease with provenance (#601).
* Same owner + active → refresh (owner-resume).
* Expired/abandoned/released → create new assignment+lease with provenance.
* Active foreign → raise ForeignLeaseError (never silent steal).
"""
now = _utc_now()
now_s = _ts(now)
expires = _ts(now + timedelta(seconds=int(lease_ttl_seconds)))
provenance = provenance or {}
with self._tx(immediate=True) as conn:
lease = conn.execute(
"SELECT * FROM leases WHERE lease_id = ?",
(lease_id,),
).fetchone()
if not lease:
raise ControlPlaneError(f"unknown lease_id {lease_id}")
work = conn.execute(
"SELECT * FROM work_items WHERE work_item_id = ?",
(lease["work_item_id"],),
).fetchone()
if not work:
raise ControlPlaneError("lease has no work_item")
# Expire by time if needed
exp = _parse_ts(lease["expires_at"])
status = lease["status"]
if status == "active" and exp and exp <= now:
conn.execute(
"UPDATE leases SET status = 'expired' WHERE lease_id = ?",
(lease_id,),
)
conn.execute(
"UPDATE assignments SET status = 'expired' WHERE lease_id = ?",
(lease_id,),
)
status = "expired"
owner = lease["session_id"]
if status == "active" and owner != adopter_session_id:
raise ForeignLeaseError(
f"cannot adopt active foreign lease {lease_id} owned by {owner}"
)
# Ensure adopter session exists
sess = conn.execute(
"SELECT session_id FROM sessions WHERE session_id = ?",
(adopter_session_id,),
).fetchone()
if not sess:
conn.execute(
"""
INSERT INTO sessions(
session_id, role, profile, namespace, pid,
started_at, last_heartbeat_at, status
) VALUES (?, ?, NULL, NULL, ?, ?, ?, 'active')
""",
(adopter_session_id, role, owner_pid or os.getpid(), now_s, now_s),
)
cols = self._lease_columns(conn)
if status == "active" and owner == adopter_session_id:
# Owner-resume refresh
conn.execute(
"""
UPDATE leases
SET heartbeat_at = ?, expires_at = ?, phase = ?
WHERE lease_id = ?
""",
(now_s, expires, "adopted", lease_id),
)
if "worktree_path" in cols and worktree_path:
conn.execute(
"UPDATE leases SET worktree_path = ? WHERE lease_id = ?",
(worktree_path, lease_id),
)
if "owner_pid" in cols and owner_pid is not None:
conn.execute(
"UPDATE leases SET owner_pid = ? WHERE lease_id = ?",
(owner_pid, lease_id),
)
if "provenance_json" in cols:
prior = {}
if lease["provenance_json"]:
try:
prior = json.loads(lease["provenance_json"])
except (TypeError, json.JSONDecodeError):
prior = {}
prior["last_adopt"] = provenance
conn.execute(
"UPDATE leases SET provenance_json = ? WHERE lease_id = ?",
(json.dumps(prior), lease_id),
)
asn = conn.execute(
"""
SELECT * FROM assignments
WHERE lease_id = ? AND status = 'active'
ORDER BY created_at DESC LIMIT 1
""",
(lease_id,),
).fetchone()
lease2 = conn.execute(
"SELECT * FROM leases WHERE lease_id = ?", (lease_id,)
).fetchone()
conn.execute(
"""
INSERT INTO events(work_item_id, event_type, message, created_at)
VALUES (?, 'lease_adopted', ?, ?)
""",
(
lease["work_item_id"],
f"owner-resume adopt lease {lease_id} by {adopter_session_id}",
now_s,
),
)
return {
"outcome": "adopted_owner_resume",
"lease": dict(lease2) if lease2 else dict(lease),
"assignment": dict(asn) if asn else None,
"reasons": ["owner-resume: refreshed lease with provenance"],
}
# Non-active: create new lease + assignment (transfer)
new_lease_id = f"lease-{uuid.uuid4().hex[:16]}"
new_asn_id = f"asn-{uuid.uuid4().hex[:16]}"
# Mark prior non-active if still active somehow
if status == "active":
conn.execute(
"UPDATE leases SET status = 'released' WHERE lease_id = ?",
(lease_id,),
)
conn.execute(
"UPDATE assignments SET status = 'released' WHERE lease_id = ?",
(lease_id,),
)
# Prefer prior assignment allowed/forbidden
prior_asn = conn.execute(
"""
SELECT * FROM assignments WHERE lease_id = ?
ORDER BY created_at DESC LIMIT 1
""",
(lease_id,),
).fetchone()
allowed = (
prior_asn["allowed_actions"]
if prior_asn
else json.dumps(["implement", "comment", "push", "create_pr"])
)
forbidden = (
prior_asn["forbidden_actions"]
if prior_asn
else json.dumps(
["approve", "merge", "request_changes", "self_select_without_assignment"]
)
)
head = expected_head_sha or (
prior_asn["expected_head_sha"] if prior_asn else work["current_head_sha"]
)
# Dynamic insert for optional columns
base_cols = [
"lease_id",
"work_item_id",
"session_id",
"role",
"phase",
"expires_at",
"heartbeat_at",
"status",
]
base_vals: list[Any] = [
new_lease_id,
lease["work_item_id"],
adopter_session_id,
role,
"adopted",
expires,
now_s,
"active",
]
optional = {
"worktree_path": worktree_path,
"owner_pid": owner_pid,
"expected_head_sha": head,
"adopted_from_session_id": owner,
"adopted_by_session_id": adopter_session_id,
"provenance_json": json.dumps(provenance),
}
for col, val in optional.items():
if col in cols and val is not None:
base_cols.append(col)
base_vals.append(val)
placeholders = ", ".join("?" for _ in base_cols)
conn.execute(
f"INSERT INTO leases({', '.join(base_cols)}) VALUES ({placeholders})",
base_vals,
)
conn.execute(
"""
INSERT INTO assignments(
assignment_id, work_item_id, session_id, lease_id,
allowed_actions, forbidden_actions, expected_head_sha,
role, status, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'active', ?)
""",
(
new_asn_id,
lease["work_item_id"],
adopter_session_id,
new_lease_id,
allowed if isinstance(allowed, str) else json.dumps(list(allowed)),
forbidden if isinstance(forbidden, str) else json.dumps(list(forbidden)),
head,
role,
now_s,
),
)
conn.execute(
"""
INSERT INTO events(work_item_id, event_type, message, created_at)
VALUES (?, 'lease_adopted', ?, ?)
""",
(
lease["work_item_id"],
f"session {adopter_session_id} adopted from {owner} "
f"prior={lease_id} new={new_lease_id}",
now_s,
),
)
lease2 = conn.execute(
"SELECT * FROM leases WHERE lease_id = ?", (new_lease_id,)
).fetchone()
asn2 = conn.execute(
"SELECT * FROM assignments WHERE assignment_id = ?",
(new_asn_id,),
).fetchone()
return {
"outcome": "adopted_transfer",
"lease": dict(lease2) if lease2 else None,
"assignment": dict(asn2) if asn2 else None,
"reasons": [
f"transferred lease ownership from {owner} to {adopter_session_id}"
],
}
@@ -40,11 +40,63 @@
- **#600** must call this substrate (not file locks / comment-only leases alone) for completion. - **#600** must call this substrate (not file locks / comment-only leases alone) for completion.
- **#612** must write `incident_links` here and create **Gitea issues**; the allocator assigns those issues, not raw incidents. - **#612** must write `incident_links` here and create **Gitea issues**; the allocator assigns those issues, not raw incidents.
## Allocator API (#600)
Module: `allocator_service.py` · MCP tool: `gitea_allocate_next_work`
- Workers call `gitea_allocate_next_work(apply=false|true)` instead of self-selecting work.
- `apply=false` returns a dry-run selection (`outcome=preview`) with skip reasons.
- `apply=true` reserves via `ControlPlaneDB.assign_and_lease` (atomic assignment+lease).
- Coordination source is **always** the control-plane DB — not file locks or comment-only leases.
- Routing follows ADR §5.3 (REQUEST_CHANGES → author, terminal path first, foreign lease → wait).
- Raw monitoring incidents are never candidates.
## Lease lifecycle API (#601)
Module: `lease_lifecycle.py` · MCP tools: `gitea_*_workflow_lease(s)`
Active control-plane leases are **first-class workflow state**:
| Tool | Purpose |
|------|---------|
| `gitea_list_workflow_leases` | List active (or all) leases for a repo |
| `gitea_inspect_workflow_lease` | Freshness + `safe_next_action` for one lease id |
| `gitea_adopt_workflow_lease` | Owner-resume or sanctioned reclaim with provenance |
| `gitea_release_workflow_lease` | Explicit owner release (audited) |
| `gitea_expire_workflow_leases` | Deterministic expire of past-`expires_at` leases |
| `gitea_abandon_workflow_lease` | Abandon with required proof |
| `gitea_reclaim_expired_workflow_lease` | Expire + re-assign with provenance |
### Hard rules
1. Control-plane DB is the coordination authority — file locks / comment-only leases are not authoritative alone.
2. Active foreign leases cannot be stolen; inspect returns `wait_foreign_active`.
3. Abandon requires `(dead_process OR missing_worktree)` and `no_live_mutation_risk`; foreign abandon also needs `operator_authorized` or full dead+missing+no_open_pr proof.
4. Adopt provenance always records `adopted_from_session_id`, `adopted_by_session_id`, work identity, optional head SHA, and worktree path.
5. Reviewer→merger comment handoff (`gitea_adopt_merger_pr_lease`) is unchanged and remains the Gitea-thread durability path.
```bash
python3 -m pytest tests/test_lease_lifecycle.py tests/test_control_plane_db.py tests/test_allocator_service.py tests/test_merger_lease_adoption.py -q
```
## Incident bridge (#612)
Module: `incident_bridge.py` · MCP tools: `gitea_observability_*`
- Bridge converts Sentry/GlitchTip observations → **normal Gitea issues** + `incident_links`.
- Phase-1: `gitea_observability_reconcile_incident(apply=false|true)` with JSON observation.
- Dry-run performs **no** Gitea mutation and **no** DB write.
- Apply reuses existing links or creates one Gitea issue; never invents `work_items.kind=incident`.
- Config: `GITEA_OBSERVABILITY_PROJECTS_JSON` or `GITEA_OBSERVABILITY_PROJECTS_FILE`.
- Provider tokens never appear in issue bodies, links, or tool results.
- Allocator sees bridge work only after a Gitea issue exists.
## Non-goals (intentionally deferred) ## Non-goals (intentionally deferred)
- Full `gitea_allocate_next_work` routing policy (REQUEST_CHANGES → author, etc.) — **#600** - Full unsupervised watchdog auto-filing (prefer explicit reconcile first)
- Sentry/GlitchTip provider adapters and auto-watchdog — **#612** - Assuming GlitchTip writeback API equals Sentry without verification
- Gitea comment/label mirror writers (may be added by allocator tooling later) - Gitea comment/label mirror writers for every assignment (optional later)
## Tests ## Tests
+915 -40
View File
File diff suppressed because it is too large Load Diff
+788
View File
@@ -0,0 +1,788 @@
"""Observability incident bridge (#612).
Converts Sentry/GlitchTip **observations** into durable **Gitea issues** and
``incident_links`` rows on the #613 control-plane DB.
Hard rules (ADR):
* Gitea owns work; providers own incidents; the bridge only links them.
* Raw monitoring incidents are **never** assignable ``work_items``.
* The #600 allocator sees bridge work only as normal Gitea issues.
* Phase-1 prefers dry-run / explicit reconcile; apply creates/links issues.
* No tokens, DSNs, cookies, or other secrets in bodies, DB, or logs.
"""
from __future__ import annotations
import json
import os
import re
from dataclasses import dataclass, field
from typing import Any, Callable, Sequence
from control_plane_db import ControlPlaneDB, ControlPlaneError, WORK_KINDS, _norm_scope
PROVIDERS = frozenset({"sentry", "glitchtip"})
OUTCOME_PREVIEW = "preview"
OUTCOME_LINKED = "linked_existing"
OUTCOME_CREATED = "created_issue"
OUTCOME_UPDATED = "updated_link"
OUTCOME_BLOCKED = "blocked"
OUTCOME_NO_ACTION = "no_safe_action"
# Patterns scrubbed from any bridge-facing text (bodies, titles, tags, logs).
_SECRET_PATTERNS: tuple[re.Pattern[str], ...] = (
re.compile(
r"(?i)\b(api[_-]?token|token|secret|password|passwd)\s*[:=]\s*\S+"
),
re.compile(
r"(?i)\bAuthorization\s*[:=]\s*(?:Bearer\s+)?[A-Za-z0-9._\-+=/]{8,}"
),
re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._\-]{8,}"),
re.compile(r"(?i)\bBasic\s+[A-Za-z0-9+/=]{8,}"),
re.compile(r"(?i)\b(dsn|sentry_dsn|glitchtip_dsn)\s*[:=]\s*\S+"),
re.compile(r"https?://[^/\s]+:[^@/\s]+@"), # user:pass@host
re.compile(r"(?i)\b(keychain[_-]?id|private[_-]?key)\s*[:=]\s*\S+"),
re.compile(r"(?i)cookie\s*[:=]\s*[^\s;]+"),
re.compile(r"(?i)\bsession[_-]?id\s*[:=]\s*\S+"),
)
_SENSITIVE_TAG_KEYS = frozenset(
{
"authorization",
"cookie",
"set-cookie",
"password",
"passwd",
"secret",
"token",
"api_key",
"apikey",
"dsn",
"sentry_dsn",
"access_token",
"refresh_token",
}
)
DEFAULT_LABELS = (
"type:bug",
"observability",
"status:ready",
)
@dataclass(frozen=True)
class ProjectMapping:
"""Maps a monitoring project to a Gitea repository."""
name: str
provider: str
monitor_base_url: str
monitor_org: str
monitor_project: str
gitea_org: str
gitea_repo: str
default_labels: tuple[str, ...] = DEFAULT_LABELS
environment_filters: tuple[str, ...] = ()
severity_threshold: str | None = None
def __post_init__(self) -> None:
prov = (self.provider or "").strip().lower()
if prov not in PROVIDERS:
raise ControlPlaneError(
f"unknown provider '{self.provider}'; expected one of {sorted(PROVIDERS)}"
)
object.__setattr__(self, "provider", prov)
object.__setattr__(self, "monitor_base_url", (self.monitor_base_url or "").rstrip("/"))
object.__setattr__(self, "monitor_org", (self.monitor_org or "").strip())
object.__setattr__(self, "monitor_project", (self.monitor_project or "").strip())
object.__setattr__(self, "gitea_org", (self.gitea_org or "").strip())
object.__setattr__(self, "gitea_repo", (self.gitea_repo or "").strip())
object.__setattr__(
self,
"default_labels",
tuple(str(x).strip() for x in (self.default_labels or DEFAULT_LABELS) if str(x).strip()),
)
def as_dict(self) -> dict[str, Any]:
return {
"name": self.name,
"provider": self.provider,
"monitor_base_url": self.monitor_base_url,
"monitor_org": self.monitor_org,
"monitor_project": self.monitor_project,
"gitea_org": self.gitea_org,
"gitea_repo": self.gitea_repo,
"default_labels": list(self.default_labels),
"environment_filters": list(self.environment_filters),
"severity_threshold": self.severity_threshold,
}
def matches_observation(self, obs: dict[str, Any]) -> bool:
if (obs.get("provider") or "").strip().lower() != self.provider:
return False
base = (obs.get("provider_base_url") or obs.get("monitor_base_url") or "").rstrip("/")
if base and self.monitor_base_url and base != self.monitor_base_url:
return False
org = (obs.get("provider_org") or obs.get("monitor_org") or "").strip()
if org and self.monitor_org and org != self.monitor_org:
return False
project = (obs.get("provider_project") or obs.get("monitor_project") or "").strip()
if project and self.monitor_project and project != self.monitor_project:
return False
return True
@dataclass
class NormalizedIncident:
provider: str
provider_base_url: str
provider_org: str
provider_project: str
provider_issue_id: str
provider_short_id: str | None = None
provider_permalink: str | None = None
fingerprint: str | None = None
first_seen: str | None = None
last_seen: str | None = None
event_count: int | None = None
status: str = "open"
environment: str | None = None
severity: str | None = None
culprit: str | None = None
title: str = ""
summary: str = ""
tags: dict[str, str] = field(default_factory=dict)
gitea_org: str = ""
gitea_repo: str = ""
default_labels: tuple[str, ...] = DEFAULT_LABELS
def provider_key(self) -> tuple[str, str, str, str, str]:
return (
self.provider,
_norm_scope(self.provider_base_url),
_norm_scope(self.provider_org),
_norm_scope(self.provider_project),
str(self.provider_issue_id),
)
def as_dict(self) -> dict[str, Any]:
return {
"provider": self.provider,
"provider_base_url": self.provider_base_url,
"provider_org": self.provider_org,
"provider_project": self.provider_project,
"provider_issue_id": self.provider_issue_id,
"provider_short_id": self.provider_short_id,
"provider_permalink": self.provider_permalink,
"fingerprint": self.fingerprint,
"first_seen": self.first_seen,
"last_seen": self.last_seen,
"event_count": self.event_count,
"status": self.status,
"environment": self.environment,
"severity": self.severity,
"culprit": self.culprit,
"title": self.title,
"summary": self.summary,
"tags": dict(self.tags),
"gitea_org": self.gitea_org,
"gitea_repo": self.gitea_repo,
"default_labels": list(self.default_labels),
}
def redact_text(value: Any) -> str:
"""Redact secret-like material from a string (fail closed to empty)."""
if value is None:
return ""
text = str(value)
for pat in _SECRET_PATTERNS:
text = pat.sub("[REDACTED]", text)
return text
def sanitize_tags(tags: Any) -> dict[str, str]:
if not isinstance(tags, dict):
return {}
out: dict[str, str] = {}
for k, v in tags.items():
key = str(k).strip().lower()
if not key or key in _SENSITIVE_TAG_KEYS:
continue
if any(s in key for s in ("token", "secret", "password", "cookie", "auth", "dsn")):
continue
val = redact_text(v)
if "[REDACTED]" in val:
continue
if len(val) > 200:
val = val[:200] + ""
out[key] = val
return out
def project_mapping_from_dict(data: dict[str, Any]) -> ProjectMapping:
labels = data.get("default_labels") or DEFAULT_LABELS
envs = data.get("environment_filters") or ()
return ProjectMapping(
name=str(data.get("name") or data.get("monitor_project") or "unnamed"),
provider=str(data.get("provider") or ""),
monitor_base_url=str(data.get("monitor_base_url") or data.get("provider_base_url") or ""),
monitor_org=str(data.get("monitor_org") or data.get("provider_org") or ""),
monitor_project=str(data.get("monitor_project") or data.get("provider_project") or ""),
gitea_org=str(data.get("gitea_org") or ""),
gitea_repo=str(data.get("gitea_repo") or ""),
default_labels=tuple(labels),
environment_filters=tuple(envs),
severity_threshold=data.get("severity_threshold"),
)
def load_project_mappings(
*,
config_path: str | None = None,
mappings_json: str | None = None,
) -> list[ProjectMapping]:
"""Load project mappings from JSON env, file, or explicit JSON string.
Env: ``GITEA_OBSERVABILITY_PROJECTS_JSON`` or path
``GITEA_OBSERVABILITY_PROJECTS_FILE``.
"""
raw: Any = None
if mappings_json:
raw = json.loads(mappings_json)
else:
env_json = (os.environ.get("GITEA_OBSERVABILITY_PROJECTS_JSON") or "").strip()
path = (
(config_path or "").strip()
or (os.environ.get("GITEA_OBSERVABILITY_PROJECTS_FILE") or "").strip()
)
if env_json:
raw = json.loads(env_json)
elif path and os.path.isfile(path):
with open(path, encoding="utf-8") as fh:
raw = json.load(fh)
if raw is None:
return []
if isinstance(raw, dict) and "projects" in raw:
raw = raw["projects"]
if not isinstance(raw, list):
raise ControlPlaneError("observability project mappings must be a list")
return [project_mapping_from_dict(item) for item in raw if isinstance(item, dict)]
def resolve_mapping(
observation: dict[str, Any],
mappings: Sequence[ProjectMapping],
*,
explicit: ProjectMapping | None = None,
) -> ProjectMapping | None:
if explicit is not None:
return explicit
matches = [m for m in mappings if m.matches_observation(observation)]
if len(matches) == 1:
return matches[0]
if len(matches) > 1:
raise ControlPlaneError(
"ambiguous project mapping for observation: "
+ ", ".join(m.name for m in matches)
+ " (fail closed, #612)"
)
# Fallback: observation itself may carry gitea targets
g_org = (observation.get("gitea_org") or "").strip()
g_repo = (observation.get("gitea_repo") or "").strip()
provider = (observation.get("provider") or "").strip().lower()
if g_org and g_repo and provider in PROVIDERS:
return ProjectMapping(
name=f"inline-{provider}",
provider=provider,
monitor_base_url=str(
observation.get("provider_base_url")
or observation.get("monitor_base_url")
or ""
),
monitor_org=str(
observation.get("provider_org") or observation.get("monitor_org") or ""
),
monitor_project=str(
observation.get("provider_project")
or observation.get("monitor_project")
or ""
),
gitea_org=g_org,
gitea_repo=g_repo,
default_labels=tuple(observation.get("default_labels") or DEFAULT_LABELS),
)
return None
def normalize_incident(
observation: dict[str, Any],
mapping: ProjectMapping,
) -> NormalizedIncident:
"""Normalize + sanitize a raw provider observation (fail closed)."""
if not isinstance(observation, dict):
raise ControlPlaneError("observation must be a dict (fail closed, #612)")
provider = (observation.get("provider") or mapping.provider or "").strip().lower()
if provider not in PROVIDERS:
raise ControlPlaneError(
f"unknown provider '{provider}'; expected {sorted(PROVIDERS)} (fail closed)"
)
if provider != mapping.provider:
raise ControlPlaneError(
f"observation provider '{provider}' does not match mapping "
f"'{mapping.provider}' (fail closed, #612)"
)
issue_id = observation.get("provider_issue_id") or observation.get("id")
if issue_id is None or str(issue_id).strip() == "":
raise ControlPlaneError(
"observation missing provider_issue_id (fail closed, #612)"
)
base = (
observation.get("provider_base_url")
or observation.get("monitor_base_url")
or mapping.monitor_base_url
or ""
)
org = (
observation.get("provider_org")
or observation.get("monitor_org")
or mapping.monitor_org
or ""
)
project = (
observation.get("provider_project")
or observation.get("monitor_project")
or mapping.monitor_project
or ""
)
title = redact_text(
observation.get("title")
or observation.get("culprit")
or f"{provider} incident {issue_id}"
)
meta = observation.get("metadata")
meta_value = meta.get("value") if isinstance(meta, dict) else None
raw_sum = (
observation.get("summary")
or observation.get("message")
or meta_value
or title
)
summary = redact_text(raw_sum)
permalink = observation.get("provider_permalink") or observation.get("permalink")
if permalink:
permalink = redact_text(permalink)
if "[REDACTED]" in permalink:
permalink = None
tags = sanitize_tags(observation.get("tags") or {})
event_count = observation.get("event_count") or observation.get("count")
try:
event_count_i = int(event_count) if event_count is not None else None
except (TypeError, ValueError):
event_count_i = None
return NormalizedIncident(
provider=provider,
provider_base_url=str(base).rstrip("/"),
provider_org=str(org).strip(),
provider_project=str(project).strip(),
provider_issue_id=str(issue_id).strip(),
provider_short_id=redact_text(observation.get("provider_short_id") or observation.get("shortId") or "")
or None,
provider_permalink=permalink,
fingerprint=redact_text(observation.get("fingerprint") or "") or None,
first_seen=str(observation.get("first_seen") or observation.get("firstSeen") or "")
or None,
last_seen=str(observation.get("last_seen") or observation.get("lastSeen") or "")
or None,
event_count=event_count_i,
status=str(observation.get("status") or "open").strip().lower() or "open",
environment=redact_text(observation.get("environment") or "") or None,
severity=redact_text(observation.get("severity") or observation.get("level") or "")
or None,
culprit=redact_text(observation.get("culprit") or "") or None,
title=title[:200],
summary=summary[:2000],
tags=tags,
gitea_org=mapping.gitea_org,
gitea_repo=mapping.gitea_repo,
default_labels=mapping.default_labels,
)
def build_gitea_issue_title(inc: NormalizedIncident) -> str:
env = f" [{inc.environment}]" if inc.environment else ""
sev = f" ({inc.severity})" if inc.severity else ""
return redact_text(f"[obs:{inc.provider}]{env}{sev} {inc.title}")[:180]
def build_gitea_issue_body(inc: NormalizedIncident) -> str:
"""Sanitized durable issue body (no secrets)."""
lines = [
"## Observability incident (bridge #612)",
"",
"<!-- mcp-incident-bridge:v1 -->",
f"<!-- provider={inc.provider} issue_id={inc.provider_issue_id} -->",
"",
f"- **provider:** `{inc.provider}`",
f"- **provider_issue_id:** `{inc.provider_issue_id}`",
]
if inc.provider_short_id:
lines.append(f"- **provider_short_id:** `{inc.provider_short_id}`")
if inc.provider_permalink:
lines.append(f"- **provider_url:** {inc.provider_permalink}")
lines.extend(
[
f"- **provider_org/project:** `{inc.provider_org}` / `{inc.provider_project}`",
f"- **base_url:** `{inc.provider_base_url}`",
f"- **fingerprint:** `{inc.fingerprint or ''}`",
f"- **first_seen:** `{inc.first_seen or ''}`",
f"- **last_seen:** `{inc.last_seen or ''}`",
f"- **event_count:** `{inc.event_count if inc.event_count is not None else ''}`",
f"- **environment:** `{inc.environment or ''}`",
f"- **severity:** `{inc.severity or ''}`",
f"- **culprit:** `{inc.culprit or ''}`",
f"- **status:** `{inc.status}`",
"",
"### Summary",
"",
redact_text(inc.summary) or "(no summary)",
"",
"### Safe tags",
"",
]
)
if inc.tags:
for k, v in sorted(inc.tags.items()):
lines.append(f"- `{k}`: `{v}`")
else:
lines.append("- (none)")
lines.extend(
[
"",
"### Canonical next action",
"",
"Author: investigate and fix under normal Gitea workflow. "
"Allocator may select this issue as ordinary Gitea work "
"(never as a raw provider incident).",
"",
"### Bridge notes",
"",
"- Raw Sentry/GlitchTip incidents are **not** assignable work items.",
"- Gitea remains the durable work record; DB stores `incident_links` only.",
"- Provider tokens must never appear in this issue.",
]
)
return "\n".join(lines)
def _link_conflict(existing: dict[str, Any], inc: NormalizedIncident) -> str | None:
"""Fail closed if existing link targets a different Gitea issue/repo."""
eg_org = str(existing.get("gitea_org") or "")
eg_repo = str(existing.get("gitea_repo") or "")
eg_num = existing.get("gitea_issue_number")
if eg_org and eg_org != inc.gitea_org:
return (
f"existing link points to org '{eg_org}', mapping wants "
f"'{inc.gitea_org}' (fail closed, #612)"
)
if eg_repo and eg_repo != inc.gitea_repo:
return (
f"existing link points to repo '{eg_repo}', mapping wants "
f"'{inc.gitea_repo}' (fail closed, #612)"
)
# fingerprint conflict when both present and disagree
ef = (existing.get("fingerprint") or "").strip()
nf = (inc.fingerprint or "").strip()
if ef and nf and ef != nf:
# Same provider issue id with different fingerprints is ambiguous.
return (
f"fingerprint conflict for provider issue {inc.provider_issue_id}: "
f"link has '{ef}', observation has '{nf}' (fail closed, #612)"
)
return None
CreateIssueFn = Callable[[str, str, list[str], str, str], dict[str, Any]]
# create_issue_fn(title, body, labels, gitea_org, gitea_repo) -> {"number": int, ...}
def reconcile_incident(
db: ControlPlaneDB | None,
*,
observation: dict[str, Any],
mappings: Sequence[ProjectMapping] | None = None,
mapping: ProjectMapping | None = None,
apply: bool = False,
create_issue_fn: CreateIssueFn | None = None,
force_gitea_issue_number: int | None = None,
) -> dict[str, Any]:
"""Reconcile one observation into incident_links + optional Gitea issue.
*apply=False* (default): preview only — no DB mutation, no Gitea mutation.
*apply=True*: upsert link; create Gitea issue when none linked (requires
``create_issue_fn``) or use ``force_gitea_issue_number`` for explicit link.
Never creates control-plane ``work_items`` for raw incidents.
"""
base: dict[str, Any] = {
"success": False,
"apply": bool(apply),
"outcome": OUTCOME_NO_ACTION,
"performed": False,
"gitea_mutated": False,
"db_mutated": False,
"raw_incident_assignable": False,
"work_item_kind_would_be": None,
"reasons": [],
"skipped": None,
"incident": None,
"existing_link": None,
"gitea_issue": None,
"action": None,
"mapping": None,
"substrate": "control_plane_db.incident_links",
"durable_work_system": "gitea_issues",
}
if db is None:
base["reasons"] = [
"control-plane DB substrate unavailable (fail closed, #613/#612)"
]
base["outcome"] = OUTCOME_BLOCKED
return base
try:
maps = list(mappings or [])
resolved = resolve_mapping(observation, maps, explicit=mapping)
if resolved is None:
base["outcome"] = OUTCOME_BLOCKED
base["reasons"] = [
"no project mapping for observation (fail closed, #612); "
"configure GITEA_OBSERVABILITY_PROJECTS_JSON or pass mapping"
]
return base
base["mapping"] = resolved.as_dict()
inc = normalize_incident(observation, resolved)
base["incident"] = inc.as_dict()
except (ControlPlaneError, json.JSONDecodeError, TypeError, ValueError) as exc:
base["outcome"] = OUTCOME_BLOCKED
base["reasons"] = [str(exc)]
return base
# Environment filter (optional)
if resolved.environment_filters and inc.environment:
allowed = {e.lower() for e in resolved.environment_filters}
if inc.environment.lower() not in allowed:
base["outcome"] = OUTCOME_NO_ACTION
base["reasons"] = [
f"environment '{inc.environment}' not in filters "
f"{sorted(allowed)}; skip (policy)"
]
base["success"] = True
base["action"] = "skip_environment_filter"
return base
existing = db.get_incident_link_by_provider(
provider=inc.provider,
provider_issue_id=inc.provider_issue_id,
provider_base_url=inc.provider_base_url,
provider_org=inc.provider_org,
provider_project=inc.provider_project,
)
base["existing_link"] = existing
if existing:
conflict = _link_conflict(existing, inc)
if conflict:
base["outcome"] = OUTCOME_BLOCKED
base["reasons"] = [conflict]
return base
title = build_gitea_issue_title(inc)
body = build_gitea_issue_body(inc)
labels = list(inc.default_labels)
if inc.provider and f"{inc.provider}" not in labels:
labels.append(inc.provider)
if "observability" not in labels:
labels.append("observability")
preview_issue = {
"title": title,
"body_preview": body[:500] + ("" if len(body) > 500 else ""),
"labels": labels,
"gitea_org": inc.gitea_org,
"gitea_repo": inc.gitea_repo,
"would_create": existing is None and force_gitea_issue_number is None,
"would_reuse_issue": int(existing["gitea_issue_number"])
if existing
else force_gitea_issue_number,
}
base["gitea_issue_preview"] = preview_issue
if not apply:
base["success"] = True
base["outcome"] = OUTCOME_PREVIEW
base["action"] = (
"preview_reuse_link" if existing else "preview_create_issue"
)
base["reasons"] = [
"dry-run only (apply=false); no Gitea mutation and no DB write — "
"call again with apply=true to create/link"
]
if existing:
base["gitea_issue"] = {
"number": existing.get("gitea_issue_number"),
"org": existing.get("gitea_org"),
"repo": existing.get("gitea_repo"),
}
# Prove we never treat this as work_item kind incident
base["raw_incident_assignable"] = False
base["work_item_kind_would_be"] = "issue" # only after Gitea issue exists
return base
# --- apply path ---
issue_number: int | None = None
created = False
if existing:
issue_number = int(existing["gitea_issue_number"])
action = "updated_existing_link"
outcome = OUTCOME_UPDATED
elif force_gitea_issue_number is not None:
issue_number = int(force_gitea_issue_number)
action = "link_explicit_issue"
outcome = OUTCOME_LINKED
else:
if create_issue_fn is None:
base["outcome"] = OUTCOME_BLOCKED
base["reasons"] = [
"apply=true requires create_issue_fn when no existing link "
"(fail closed, #612)"
]
return base
try:
created_res = create_issue_fn(
title, body, labels, inc.gitea_org, inc.gitea_repo
)
except Exception as exc: # noqa: BLE001
base["outcome"] = OUTCOME_BLOCKED
base["reasons"] = [
f"Gitea issue creation failed: {redact_text(exc)} (fail closed)"
]
return base
number = created_res.get("number") if isinstance(created_res, dict) else None
if number is None:
base["outcome"] = OUTCOME_BLOCKED
base["reasons"] = [
"Gitea issue creation returned no issue number (fail closed, #612)"
]
base["create_result"] = created_res
return base
issue_number = int(number)
created = True
action = "created_gitea_issue"
outcome = OUTCOME_CREATED
base["gitea_mutated"] = True
base["create_result"] = {
k: created_res.get(k)
for k in ("number", "success", "performed", "reasons")
if isinstance(created_res, dict)
}
assert issue_number is not None
try:
link = db.upsert_incident_link(
provider=inc.provider,
provider_issue_id=inc.provider_issue_id,
gitea_org=inc.gitea_org,
gitea_repo=inc.gitea_repo,
gitea_issue_number=issue_number,
provider_base_url=inc.provider_base_url,
provider_org=inc.provider_org,
provider_project=inc.provider_project,
provider_short_id=inc.provider_short_id,
provider_permalink=inc.provider_permalink,
fingerprint=inc.fingerprint,
first_seen=inc.first_seen,
last_seen=inc.last_seen,
event_count=inc.event_count,
status=inc.status if not created else "open",
)
except Exception as exc: # noqa: BLE001
base["outcome"] = OUTCOME_BLOCKED
base["reasons"] = [
f"incident_links upsert failed: {redact_text(exc)} (fail closed, #613)"
]
base["gitea_issue"] = {
"number": issue_number,
"org": inc.gitea_org,
"repo": inc.gitea_repo,
"created": created,
}
return base
base["success"] = True
base["performed"] = True
base["db_mutated"] = True
base["outcome"] = outcome
base["action"] = action
base["gitea_issue"] = {
"number": issue_number,
"org": inc.gitea_org,
"repo": inc.gitea_repo,
"created": created,
}
base["link"] = {
k: link.get(k)
for k in (
"link_id",
"provider",
"provider_issue_id",
"gitea_org",
"gitea_repo",
"gitea_issue_number",
"fingerprint",
"event_count",
"status",
"last_sync_at",
)
}
base["reasons"] = [
(
f"reused Gitea issue #{issue_number} for provider "
f"{inc.provider}:{inc.provider_issue_id}"
if not created
else f"created Gitea issue #{issue_number} and stored incident_links row"
),
"raw provider incident is not an assignable work_item "
f"(WORK_KINDS={sorted(WORK_KINDS)})",
]
base["raw_incident_assignable"] = False
base["work_item_kind_would_be"] = "issue"
base["allocator_visible_as"] = {
"kind": "issue",
"number": issue_number,
"org": inc.gitea_org,
"repo": inc.gitea_repo,
"note": "allocator selects normal Gitea issues only (#600/#612)",
}
return base
def assert_not_raw_incident_work_item(kind: str) -> None:
"""Guard used by tests / callers: incidents must not enter WORK_KINDS."""
k = (kind or "").strip().lower()
if k not in WORK_KINDS:
if k in ("sentry_incident", "glitchtip_incident", "incident", "observation"):
raise ControlPlaneError(
f"kind '{k}' is not assignable; bridge must create Gitea issues first (#612)"
)
raise ControlPlaneError(f"kind '{k}' is not assignable")
+63
View File
@@ -375,6 +375,64 @@ def _same_realpath(left: str | None, right: str | None) -> bool:
return left == right return left == right
def assess_expired_lock_reclaim(
existing_lock: dict[str, Any] | None,
*,
now: datetime | None = None,
) -> dict[str, Any]:
"""Decide whether an expired/stale author issue lock may be reclaimed (#601).
Required proof (any reclaim of non-live lock):
* lease not live (expired or dead pid)
* owner process dead OR worktree missing
* no force-delete of live foreign ownership
"""
if not existing_lock:
return {
"reclaim_allowed": True,
"reasons": ["no existing lock"],
"freshness": assess_lock_freshness(None, now=now),
}
freshness = assess_lock_freshness(existing_lock, now=now)
if freshness.get("live"):
return {
"reclaim_allowed": False,
"reasons": ["lock is still live; cannot reclaim (fail closed)"],
"freshness": freshness,
}
pid = existing_lock.get("session_pid")
if pid is None:
pid = existing_lock.get("pid")
dead = not is_process_alive(pid) if pid is not None else True
wt = str(existing_lock.get("worktree_path") or "")
missing_wt = (not wt) or (not os.path.isdir(os.path.realpath(wt)))
if not (dead or missing_wt):
return {
"reclaim_allowed": False,
"reasons": [
"expired/stale lock still has live owner pid and present worktree; "
"recovery review required (fail closed)"
],
"freshness": freshness,
"owner_pid_dead": dead,
"worktree_missing": missing_wt,
}
return {
"reclaim_allowed": True,
"reasons": [
"non-live lock with dead process and/or missing worktree; "
"sanctioned reclaim allowed"
],
"freshness": freshness,
"owner_pid_dead": dead,
"worktree_missing": missing_wt,
"prior_branch": existing_lock.get("branch_name"),
"prior_worktree": existing_lock.get("worktree_path"),
"prior_pid": pid,
}
def assess_same_issue_lease_conflict( def assess_same_issue_lease_conflict(
existing_lock: dict[str, Any] | None, existing_lock: dict[str, Any] | None,
*, *,
@@ -405,6 +463,11 @@ def assess_same_issue_lease_conflict(
and _same_realpath(str(existing_worktree or ""), worktree_path) and _same_realpath(str(existing_worktree or ""), worktree_path)
) )
if is_lease_expired(existing_lock, now=now): if is_lease_expired(existing_lock, now=now):
reclaim = assess_expired_lock_reclaim(existing_lock, now=now)
if reclaim.get("reclaim_allowed"):
# #601: expired + dead pid / missing worktree may be reclaimed
# through the normal lock path (sanctioned overwrite).
return None
return ( return (
f"Issue #{issue_number} has an expired {operation_type} lease on " f"Issue #{issue_number} has an expired {operation_type} lease on "
f"branch '{existing_branch}' from worktree '{existing_worktree}'. " f"branch '{existing_branch}' from worktree '{existing_worktree}'. "
+723
View File
@@ -0,0 +1,723 @@
"""First-class control-plane lease lifecycle (#601).
Active leases are queryable workflow state. Canonical operations:
* list / inspect
* adopt (with provenance)
* release (explicit, recorded)
* expire / reclaim
* abandon (requires proof: dead process and/or missing worktree, etc.)
Authority model:
* Control-plane DB is the coordination source for assignment/lease.
* File locks and comment-only leases are **not** authoritative alone.
* Reviewer/merger comment leases (``reviewer_pr_lease`` / merger adoption)
remain for Gitea-thread durability; they must not bypass DB assignment when
the control-plane path is in use.
Fail closed on ambiguous ownership, active foreign leases, mismatched
worktree, stale head, missing capability, and unsafe cleanup.
"""
from __future__ import annotations
import json
import os
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Callable, Mapping
import control_plane_db as cpd
# Outcomes / safe next actions (stable vocabulary for tools + tests).
SAFE_OWNER_RESUME = "owner_resume"
SAFE_WAIT_FOREIGN = "wait_foreign_active"
SAFE_RECLAIM_EXPIRED = "reclaim_expired"
SAFE_ABANDON_ALLOWED = "abandon_allowed"
SAFE_RELEASE_OWNED = "release_owned"
SAFE_STALE_PROMPT = "stale_prompt_lease"
SAFE_UNKNOWN = "inspect_only"
SAFE_NO_AUTHORITY = "file_or_comment_not_authoritative"
LEASE_STATUS_ACTIVE = "active"
LEASE_STATUS_RELEASED = "released"
LEASE_STATUS_EXPIRED = "expired"
LEASE_STATUS_ABANDONED = "abandoned"
AUTHORITATIVE_SOURCE = "control_plane_db"
class LeaseLifecycleError(RuntimeError):
"""Fail-closed lifecycle policy error."""
@dataclass(frozen=True)
class AbandonProof:
"""Required evidence to abandon a non-owned or expired lease safely."""
dead_process: bool = False
missing_worktree: bool = False
same_owner: bool = False
no_open_pr: bool = False
no_live_mutation_risk: bool = False
operator_authorized: bool = False
worktree_path: str | None = None
owner_pid: int | None = None
notes: str = ""
def as_dict(self) -> dict[str, Any]:
return {
"dead_process": self.dead_process,
"missing_worktree": self.missing_worktree,
"same_owner": self.same_owner,
"no_open_pr": self.no_open_pr,
"no_live_mutation_risk": self.no_live_mutation_risk,
"operator_authorized": self.operator_authorized,
"worktree_path": self.worktree_path,
"owner_pid": self.owner_pid,
"notes": self.notes,
}
def is_sufficient(self) -> bool:
"""Abandon requires process death or missing worktree + no mutation risk.
Foreign active leases additionally need operator_authorized unless the
owner process is dead and the worktree is missing.
"""
if not self.no_live_mutation_risk:
return False
if not (self.dead_process or self.missing_worktree):
return False
if self.same_owner:
return True
# Foreign abandon: operator flag OR (dead + missing worktree + no risk)
if self.operator_authorized:
return True
return bool(self.dead_process and self.missing_worktree and self.no_open_pr)
def is_process_alive(pid: int | None) -> bool:
if pid is None:
return False
try:
pid_i = int(pid)
except (TypeError, ValueError):
return False
if pid_i <= 0:
return False
try:
os.kill(pid_i, 0)
return True
except ProcessLookupError:
return False
except PermissionError:
# Exists but not owned by us — treat as alive (fail closed).
return True
except OSError:
return False
def worktree_exists(path: str | None) -> bool:
if not path:
return False
try:
return os.path.isdir(os.path.realpath(path))
except OSError:
return False
def _utc_now() -> datetime:
return datetime.now(timezone.utc)
def _parse_ts(value: str | None) -> datetime | None:
return cpd._parse_ts(value)
def classify_lease_freshness(
lease: Mapping[str, Any],
*,
now: datetime | None = None,
pid_checker: Callable[[int | None], bool] = is_process_alive,
worktree_checker: Callable[[str | None], bool] = worktree_exists,
) -> dict[str, Any]:
"""Classify a control-plane lease row for workflow tooling."""
moment = now or _utc_now()
status = (lease.get("status") or "").strip().lower()
expires = _parse_ts(lease.get("expires_at"))
owner_pid = lease.get("owner_pid")
if owner_pid is None:
owner_pid = lease.get("session_pid")
wt = lease.get("worktree_path")
pid_alive = pid_checker(owner_pid) if owner_pid is not None else None
wt_present = worktree_checker(wt) if wt else None
expired_by_time = bool(expires and expires <= moment)
if status == LEASE_STATUS_ABANDONED:
freshness = "abandoned"
elif status == LEASE_STATUS_RELEASED:
freshness = "released"
elif status == LEASE_STATUS_EXPIRED or expired_by_time:
freshness = "expired"
elif status != LEASE_STATUS_ACTIVE:
freshness = status or "unknown"
elif pid_alive is False:
freshness = "stale_dead_process"
elif wt is not None and wt_present is False:
freshness = "stale_missing_worktree"
else:
freshness = "active"
return {
"freshness": freshness,
"status": status,
"expired_by_time": expired_by_time,
"owner_pid": owner_pid,
"owner_pid_alive": pid_alive,
"worktree_path": wt,
"worktree_present": wt_present,
"expires_at": lease.get("expires_at"),
"authoritative_source": AUTHORITATIVE_SOURCE,
"file_lock_only": False,
"comment_lease_only": False,
}
def decide_safe_next_action(
*,
lease: Mapping[str, Any] | None,
caller_session_id: str,
freshness: Mapping[str, Any] | None = None,
caller_worktree: str | None = None,
) -> dict[str, Any]:
"""Return safe_next_action + reasons for a caller inspecting a lease."""
if not lease:
return {
"safe_next_action": SAFE_STALE_PROMPT,
"reasons": ["lease not found in control-plane DB (stale prompt id?)"],
"block": True,
}
fr = freshness or classify_lease_freshness(lease)
owner = str(lease.get("session_id") or "")
same_owner = owner == str(caller_session_id)
status = fr.get("freshness")
reasons: list[str] = []
# Worktree mismatch for owner resume
lease_wt = lease.get("worktree_path")
if (
same_owner
and lease_wt
and caller_worktree
and os.path.realpath(str(lease_wt)) != os.path.realpath(str(caller_worktree))
):
return {
"safe_next_action": SAFE_UNKNOWN,
"reasons": [
f"worktree mismatch: lease has {lease_wt!r}, caller has "
f"{caller_worktree!r} (fail closed)"
],
"block": True,
"same_owner": True,
}
if status in ("abandoned", "released"):
return {
"safe_next_action": SAFE_STALE_PROMPT,
"reasons": [f"lease status is {status}; do not adopt blindly"],
"block": True,
"same_owner": same_owner,
}
if status == "expired" or fr.get("expired_by_time"):
return {
"safe_next_action": SAFE_RECLAIM_EXPIRED,
"reasons": ["lease expired; reclaim via expire+assign or adopt reclaim path"],
"block": False,
"same_owner": same_owner,
}
if status in ("stale_dead_process", "stale_missing_worktree"):
if same_owner:
return {
"safe_next_action": SAFE_OWNER_RESUME,
"reasons": [
f"caller owns lease with freshness={status}; rebind via "
"adopt/owner-resume or abandon with proof"
],
"block": False,
"same_owner": True,
"also_allowed": [SAFE_ABANDON_ALLOWED, SAFE_RELEASE_OWNED],
}
return {
"safe_next_action": SAFE_ABANDON_ALLOWED,
"reasons": [
f"lease freshness={status}; abandon with required proof then reassign"
],
"block": False,
"same_owner": False,
}
if same_owner and status == "active":
return {
"safe_next_action": SAFE_OWNER_RESUME,
"reasons": [
"caller owns active lease; resume via heartbeat/assign owner-resume "
"or release explicitly"
],
"block": False,
"same_owner": True,
"also_allowed": [SAFE_RELEASE_OWNED],
}
if not same_owner and status == "active":
return {
"safe_next_action": SAFE_WAIT_FOREIGN,
"reasons": [
f"foreign active lease held by session {owner}; "
"do not steal (fail closed)"
],
"block": True,
"same_owner": False,
"owner_session_id": owner,
}
return {
"safe_next_action": SAFE_UNKNOWN,
"reasons": [f"unclassified freshness={status}"],
"block": True,
"same_owner": same_owner,
}
def non_db_lease_authority_report(
*,
file_lock_present: bool = False,
comment_lease_present: bool = False,
db_lease_present: bool = False,
) -> dict[str, Any]:
"""File/comment leases alone are not coordination authority (#601 / #600)."""
authoritative = bool(db_lease_present)
reasons: list[str] = []
if file_lock_present and not db_lease_present:
reasons.append(
"file lock present but control-plane DB lease absent — "
"file lock is not authoritative alone"
)
if comment_lease_present and not db_lease_present:
reasons.append(
"comment-only lease present but control-plane DB lease absent — "
"comment lease is not authoritative alone"
)
if authoritative:
reasons.append("control-plane DB lease is the coordination authority")
return {
"authoritative_source": AUTHORITATIVE_SOURCE if authoritative else None,
"db_lease_present": db_lease_present,
"file_lock_present": file_lock_present,
"comment_lease_present": comment_lease_present,
"safe_next_action": (
SAFE_UNKNOWN if authoritative else SAFE_NO_AUTHORITY
),
"reasons": reasons,
"file_lock_only": bool(file_lock_present and not db_lease_present),
"comment_lease_only": bool(comment_lease_present and not db_lease_present),
}
def build_adopt_provenance(
*,
adopted_from_session_id: str,
adopted_by_session_id: str,
work_kind: str,
work_number: int,
remote: str,
org: str,
repo: str,
worktree_path: str | None,
expected_head_sha: str | None,
prior_lease_id: str | None,
reason: str,
) -> dict[str, Any]:
return {
"adopted_from_session_id": adopted_from_session_id,
"adopted_by_session_id": adopted_by_session_id,
"work_kind": work_kind,
"work_number": int(work_number),
"remote": remote,
"org": org,
"repo": repo,
"worktree_path": worktree_path,
"expected_head_sha": expected_head_sha,
"prior_lease_id": prior_lease_id,
"reason": reason,
"recorded_at": cpd._ts(),
"source": "lease_lifecycle.adopt",
}
def inspect_lease(
db: cpd.ControlPlaneDB,
lease_id: str,
*,
caller_session_id: str,
caller_worktree: str | None = None,
) -> dict[str, Any]:
"""Full first-class lease workflow state for one lease id."""
state = db.get_lease_workflow_state(lease_id)
if not state:
decision = decide_safe_next_action(
lease=None, caller_session_id=caller_session_id
)
return {
"success": True,
"found": False,
"lease_id": lease_id,
"lease": None,
"freshness": None,
**decision,
"authoritative_source": AUTHORITATIVE_SOURCE,
"file_lock_only": False,
"comment_lease_only": False,
}
lease = state["lease"]
freshness = classify_lease_freshness(lease)
decision = decide_safe_next_action(
lease=lease,
caller_session_id=caller_session_id,
freshness=freshness,
caller_worktree=caller_worktree,
)
return {
"success": True,
"found": True,
"lease_id": lease_id,
"lease": lease,
"assignment": state.get("assignment"),
"work_item": state.get("work_item"),
"session": state.get("session"),
"freshness": freshness,
"provenance": state.get("provenance"),
**decision,
"authoritative_source": AUTHORITATIVE_SOURCE,
"file_lock_only": False,
"comment_lease_only": False,
}
def list_active_leases(
db: cpd.ControlPlaneDB,
*,
remote: str | None = None,
org: str | None = None,
repo: str | None = None,
role: str | None = None,
include_non_active: bool = False,
limit: int = 100,
) -> dict[str, Any]:
statuses = None if include_non_active else (LEASE_STATUS_ACTIVE,)
rows = db.list_leases(
remote=remote,
org=org,
repo=repo,
role=role,
statuses=statuses,
limit=limit,
)
enriched = []
for row in rows:
fr = classify_lease_freshness(row)
enriched.append({**row, "freshness": fr})
return {
"success": True,
"count": len(enriched),
"leases": enriched,
"authoritative_source": AUTHORITATIVE_SOURCE,
"file_lock_only": False,
"comment_lease_only": False,
"include_non_active": include_non_active,
}
def adopt_lease(
db: cpd.ControlPlaneDB,
*,
lease_id: str,
adopter_session_id: str,
role: str,
worktree_path: str | None = None,
expected_head_sha: str | None = None,
owner_pid: int | None = None,
operator_authorized: bool = False,
) -> dict[str, Any]:
"""Sanctioned adopt path with provenance; never silent foreign steal."""
state = db.get_lease_workflow_state(lease_id)
if not state:
raise LeaseLifecycleError(
f"unknown lease_id {lease_id}; stale prompt id (fail closed)"
)
lease = state["lease"]
work = state["work_item"]
freshness = classify_lease_freshness(lease)
owner = str(lease.get("session_id") or "")
same_owner = owner == str(adopter_session_id)
if freshness["freshness"] == "active" and not same_owner:
raise LeaseLifecycleError(
f"refusing to steal active foreign lease {lease_id} owned by "
f"{owner} (fail closed)"
)
if freshness["freshness"] in ("abandoned", "released"):
raise LeaseLifecycleError(
f"lease {lease_id} is {freshness['freshness']}; cannot adopt "
"(fail closed)"
)
# Expired or stale: require abandon-style safety before ownership transfer
# when not same owner; same owner may reclaim.
if not same_owner and freshness["freshness"] in (
"expired",
"stale_dead_process",
"stale_missing_worktree",
):
if not operator_authorized and freshness["freshness"] == "expired":
# Deterministic reclaim of expired foreign lease is allowed
# without operator flag (sanctioned expire reclaim).
pass
elif freshness["freshness"] != "expired" and not operator_authorized:
# stale active-looking requires abandon proof path
raise LeaseLifecycleError(
f"lease {lease_id} freshness={freshness['freshness']}; "
"use abandon with proof before foreign adopt (fail closed)"
)
provenance = build_adopt_provenance(
adopted_from_session_id=owner,
adopted_by_session_id=adopter_session_id,
work_kind=str(work.get("kind")),
work_number=int(work.get("number")),
remote=str(work.get("remote")),
org=str(work.get("org")),
repo=str(work.get("repo")),
worktree_path=worktree_path,
expected_head_sha=expected_head_sha or lease.get("expected_head_sha"),
prior_lease_id=lease_id,
reason=(
"owner-resume-adopt" if same_owner else "sanctioned-reclaim-adopt"
),
)
result = db.adopt_lease(
lease_id=lease_id,
adopter_session_id=adopter_session_id,
role=role,
worktree_path=worktree_path,
expected_head_sha=expected_head_sha,
owner_pid=owner_pid if owner_pid is not None else os.getpid(),
provenance=provenance,
)
return {
"success": True,
"outcome": result.get("outcome"),
"same_owner": same_owner,
"prior_lease_id": lease_id,
"lease": result.get("lease"),
"assignment": result.get("assignment"),
"provenance": provenance,
"authoritative_source": AUTHORITATIVE_SOURCE,
"file_lock_only": False,
"comment_lease_only": False,
"reasons": result.get("reasons") or [],
}
def release_lease(
db: cpd.ControlPlaneDB,
*,
lease_id: str,
session_id: str,
) -> dict[str, Any]:
proof = db.release_lease_recorded(lease_id=lease_id, session_id=session_id)
return {
"success": True,
"outcome": "released",
"lease_id": lease_id,
"session_id": session_id,
"release_proof": proof,
"authoritative_source": AUTHORITATIVE_SOURCE,
"file_lock_only": False,
"comment_lease_only": False,
}
def expire_leases(db: cpd.ControlPlaneDB) -> dict[str, Any]:
n = db.expire_stale_leases()
return {
"success": True,
"outcome": "expired",
"expired_count": int(n),
"authoritative_source": AUTHORITATIVE_SOURCE,
"file_lock_only": False,
"comment_lease_only": False,
}
def reclaim_expired_lease(
db: cpd.ControlPlaneDB,
*,
lease_id: str,
session_id: str,
role: str,
worktree_path: str | None = None,
expected_head_sha: str | None = None,
) -> dict[str, Any]:
"""Expire-if-needed then assign to *session_id* for the same work item."""
state = db.get_lease_workflow_state(lease_id)
if not state:
raise LeaseLifecycleError(f"unknown lease_id {lease_id}")
lease = state["lease"]
work = state["work_item"]
freshness = classify_lease_freshness(lease)
if freshness["freshness"] == "active":
if lease.get("session_id") != session_id:
raise LeaseLifecycleError(
f"cannot reclaim active foreign lease {lease_id} (fail closed)"
)
# owner resume
return adopt_lease(
db,
lease_id=lease_id,
adopter_session_id=session_id,
role=role,
worktree_path=worktree_path,
expected_head_sha=expected_head_sha,
)
if freshness["freshness"] not in (
"expired",
"stale_dead_process",
"stale_missing_worktree",
"abandoned",
"released",
):
raise LeaseLifecycleError(
f"lease {lease_id} freshness={freshness['freshness']} not reclaimable"
)
# Ensure expired marker is applied
db.expire_stale_leases()
# If still active due to clock skew / non-time stale, force expire this lease
refreshed = db.get_lease_workflow_state(lease_id)
if refreshed and refreshed["lease"].get("status") == "active":
db.force_expire_lease(lease_id, reason="reclaim_expired_lease")
head = expected_head_sha or work.get("current_head_sha")
assigned = db.assign_and_lease(
session_id=session_id,
role=role,
remote=str(work["remote"]),
org=str(work["org"]),
repo=str(work["repo"]),
kind=str(work["kind"]),
number=int(work["number"]),
expected_head_sha=head,
phase="reclaimed",
worktree_path=worktree_path,
owner_pid=os.getpid(),
)
if assigned.outcome != "assigned":
raise LeaseLifecycleError(
f"reclaim assign failed: {assigned.outcome}{assigned.reason}"
)
provenance = build_adopt_provenance(
adopted_from_session_id=str(lease.get("session_id")),
adopted_by_session_id=session_id,
work_kind=str(work["kind"]),
work_number=int(work["number"]),
remote=str(work["remote"]),
org=str(work["org"]),
repo=str(work["repo"]),
worktree_path=worktree_path,
expected_head_sha=head,
prior_lease_id=lease_id,
reason="reclaim_expired",
)
db.attach_lease_provenance(assigned.lease_id, provenance)
return {
"success": True,
"outcome": "reclaimed",
"prior_lease_id": lease_id,
"assignment": assigned.as_dict(),
"provenance": provenance,
"authoritative_source": AUTHORITATIVE_SOURCE,
"file_lock_only": False,
"comment_lease_only": False,
}
def abandon_lease(
db: cpd.ControlPlaneDB,
*,
lease_id: str,
requester_session_id: str,
proof: AbandonProof,
) -> dict[str, Any]:
state = db.get_lease_workflow_state(lease_id)
if not state:
raise LeaseLifecycleError(f"unknown lease_id {lease_id}")
lease = state["lease"]
owner = str(lease.get("session_id") or "")
same_owner = owner == str(requester_session_id)
# Enrich proof with live checks when not provided
owner_pid = proof.owner_pid
if owner_pid is None:
owner_pid = lease.get("owner_pid")
wt = proof.worktree_path if proof.worktree_path is not None else lease.get(
"worktree_path"
)
dead = proof.dead_process or (
owner_pid is not None and not is_process_alive(owner_pid)
)
missing_wt = proof.missing_worktree or (
bool(wt) and not worktree_exists(str(wt))
)
enriched = AbandonProof(
dead_process=bool(dead),
missing_worktree=bool(missing_wt),
same_owner=same_owner or proof.same_owner,
no_open_pr=proof.no_open_pr,
no_live_mutation_risk=proof.no_live_mutation_risk,
operator_authorized=proof.operator_authorized,
worktree_path=str(wt) if wt else None,
owner_pid=int(owner_pid) if owner_pid is not None else None,
notes=proof.notes,
)
if not enriched.is_sufficient():
raise LeaseLifecycleError(
"abandon proof insufficient: need (dead_process or missing_worktree) "
"and no_live_mutation_risk; foreign abandon also needs "
"operator_authorized or (dead+missing_worktree+no_open_pr). "
f"proof={enriched.as_dict()}"
)
# Active foreign with live process + present worktree: never abandon without
# operator (already covered by is_sufficient).
result = db.abandon_lease(
lease_id=lease_id,
requester_session_id=requester_session_id,
proof=enriched.as_dict(),
)
return {
"success": True,
"outcome": "abandoned",
"lease_id": lease_id,
"requester_session_id": requester_session_id,
"prior_owner_session_id": owner,
"abandon_proof": enriched.as_dict(),
"audit": result,
"authoritative_source": AUTHORITATIVE_SOURCE,
"file_lock_only": False,
"comment_lease_only": False,
}
+194 -6
View File
@@ -1,4 +1,4 @@
"""Stale / moot #332 review-decision lock detection and cleanup policy (#594). """Stale / moot #332 review-decision lock detection and cleanup policy (#594/#620).
#332 correctly hard-stops a reviewer session after a terminal live review #332 correctly hard-stops a reviewer session after a terminal live review
mutation. After #559 those locks are durable on disk, so they can outlive the mutation. After #559 those locks are durable on disk, so they can outlive the
@@ -6,6 +6,12 @@ work they protect: when the referenced PR is already merged or closed, no
same-PR merge sequence remains, yet the durable ledger still blocks unrelated same-PR merge sequence remains, yet the durable ledger still blocks unrelated
new terminal reviews. new terminal reviews.
#620 scopes the terminal boundary by **reviewed head SHA**: a prior
REQUEST_CHANGES (or APPROVE) on head A must not block a fresh formal decision
on head B of the **same open PR**. Same-head duplicates remain fail-closed.
Open-PR lock *cleanup* remains forbidden unless the PR is truly merged/closed
(#594); new-head re-review is allowed without deleting the durable lock.
This module is pure policy (no Gitea I/O). The MCP tool This module is pure policy (no Gitea I/O). The MCP tool
``gitea_cleanup_stale_review_decision_lock`` fetches live PR state, calls these ``gitea_cleanup_stale_review_decision_lock`` fetches live PR state, calls these
helpers, and only then clears durable state when cleanup is allowed. helpers, and only then clears durable state when cleanup is allowed.
@@ -23,6 +29,45 @@ from typing import Any
TERMINAL_REVIEW_ACTIONS = frozenset({"approve", "request_changes"}) TERMINAL_REVIEW_ACTIONS = frozenset({"approve", "request_changes"})
def normalize_head_sha(value: Any) -> str | None:
"""Normalize a git SHA for equality comparison; None if empty/invalid."""
if value is None:
return None
text = str(value).strip().lower()
if not text:
return None
return text
def heads_equal(a: Any, b: Any) -> bool:
na = normalize_head_sha(a)
nb = normalize_head_sha(b)
if not na or not nb:
return False
return na == nb
def mutation_head_sha(mutation: dict | None, lock: dict | None = None) -> str | None:
"""Head SHA that bounds a live mutation (#620).
Prefers fields recorded on the mutation itself. For *legacy* mutations
that predate head-scoping, falls back to the lock's
``ready_expected_head_sha`` only when that ready binding is for the same
PR number (the frozen durable PR #619-style ledger).
"""
if not isinstance(mutation, dict):
return None
for key in ("head_sha", "expected_head_sha", "reviewed_head_sha"):
head = normalize_head_sha(mutation.get(key))
if head:
return head
if not isinstance(lock, dict):
return None
if lock.get("ready_pr_number") != mutation.get("pr_number"):
return None
return normalize_head_sha(lock.get("ready_expected_head_sha"))
def last_terminal_mutation(lock: dict | None) -> dict | None: def last_terminal_mutation(lock: dict | None) -> dict | None:
"""Return the last terminal live mutation on *lock*, or None.""" """Return the last terminal live mutation on *lock*, or None."""
if not lock: if not lock:
@@ -35,18 +80,125 @@ def last_terminal_mutation(lock: dict | None) -> dict | None:
return terminals[-1] if terminals else None return terminals[-1] if terminals else None
def prior_live_mutations_block_boundary(
lock: dict | None,
*,
pr_number: int,
expected_head_sha: str | None,
) -> bool:
"""True when prior live mutations block a new decision for PR+head (#620).
Historical terminals on a **different head of the same PR** do not block.
Any prior mutation on another PR, the same head, or without a comparable
head SHA fails closed (blocks).
Head resolution uses ``mutation_head_sha(m, lock)`` so pre-#620 ledgers
that only stored ``ready_expected_head_sha`` still compare correctly
(PR #619-style).
"""
if not lock:
return False
if lock.get("correction_authorized"):
return False
prior = list(lock.get("live_mutations") or [])
if not prior:
return False
target = normalize_head_sha(expected_head_sha)
for m in prior:
if not isinstance(m, dict):
return True
m_pr = m.get("pr_number")
m_head = mutation_head_sha(m, lock)
if (
m_pr == pr_number
and target
and m_head
and not heads_equal(m_head, target)
):
# Same open PR, different reviewed head — historical boundary only.
continue
return True
return False
def terminal_boundary_allows_fresh_decision(
lock: dict | None,
*,
pr_number: int,
expected_head_sha: str | None,
operation: str,
) -> bool:
"""Whether #332 hard-stop should yield for a new PR+head boundary (#620).
Only for ``mark_ready`` / ``review`` / ``resume`` on the **same PR** when
the requested head differs from the last terminal's head. Merge is never
reopened by head change (approved head merge path is separate).
"""
if operation not in ("mark_ready", "review", "resume"):
return False
if not lock:
return False
if lock.get("correction_authorized"):
return True
last = last_terminal_mutation(lock)
if last is None:
return True
if last.get("pr_number") != pr_number:
return False
locked_head = mutation_head_sha(last, lock)
target = normalize_head_sha(expected_head_sha)
if not locked_head or not target:
return False
return not heads_equal(locked_head, target)
def backfill_terminal_heads_from_ready(lock: dict) -> dict:
"""Stamp head_sha onto legacy terminal mutations before overwriting ready_*.
Called when marking a fresh decision on a new head so historical rows keep
their original boundary after ``ready_expected_head_sha`` advances (#620).
"""
if not isinstance(lock, dict):
return lock
ready_pr = lock.get("ready_pr_number")
ready_head = normalize_head_sha(lock.get("ready_expected_head_sha"))
if ready_pr is None or not ready_head:
return lock
mutations = list(lock.get("live_mutations") or [])
changed = False
for m in mutations:
if not isinstance(m, dict):
continue
if m.get("action") not in TERMINAL_REVIEW_ACTIONS:
continue
if m.get("pr_number") != ready_pr:
continue
if mutation_head_sha(m):
continue
m["head_sha"] = ready_head
changed = True
if changed:
lock["live_mutations"] = mutations
return lock
def classify_pr_live_state(pr_live: dict | None) -> dict[str, Any]: def classify_pr_live_state(pr_live: dict | None) -> dict[str, Any]:
"""Normalize a Gitea PR payload into merge/closed flags.""" """Normalize a Gitea PR payload into merge/closed flags."""
pr = pr_live or {} pr = pr_live or {}
merged = bool(pr.get("merged") or pr.get("merged_at")) merged = bool(pr.get("merged") or pr.get("merged_at"))
state = (pr.get("state") or "").strip().lower() or None state = (pr.get("state") or "").strip().lower() or None
closed = state == "closed" closed = state == "closed"
head = pr.get("head") if isinstance(pr.get("head"), dict) else {}
head_sha = normalize_head_sha(
pr.get("head_sha") or pr.get("head_commit_sha") or head.get("sha")
)
return { return {
"pr_state": state, "pr_state": state,
"pr_merged": merged, "pr_merged": merged,
"pr_merged_or_closed": merged or closed, "pr_merged_or_closed": merged or closed,
"merge_commit_sha": pr.get("merge_commit_sha") "merge_commit_sha": pr.get("merge_commit_sha")
or pr.get("merged_commit_sha"), or pr.get("merged_commit_sha"),
"current_pr_head_sha": head_sha,
} }
@@ -56,6 +208,7 @@ def lock_summary(lock: dict | None) -> dict[str, Any] | None:
return None return None
last = last_terminal_mutation(lock) last = last_terminal_mutation(lock)
mutations = list(lock.get("live_mutations") or []) mutations = list(lock.get("live_mutations") or [])
last_head = mutation_head_sha(last, lock) if last else None
return { return {
"task": lock.get("task"), "task": lock.get("task"),
"remote": lock.get("remote"), "remote": lock.get("remote"),
@@ -67,16 +220,21 @@ def lock_summary(lock: dict | None) -> dict[str, Any] | None:
"session_profile": lock.get("session_profile"), "session_profile": lock.get("session_profile"),
"ready_pr_number": lock.get("ready_pr_number"), "ready_pr_number": lock.get("ready_pr_number"),
"ready_action": lock.get("ready_action"), "ready_action": lock.get("ready_action"),
"ready_expected_head_sha": normalize_head_sha(
lock.get("ready_expected_head_sha")
),
"live_mutations_count": len(mutations), "live_mutations_count": len(mutations),
"last_terminal": ( "last_terminal": (
{ {
"pr_number": last.get("pr_number"), "pr_number": last.get("pr_number"),
"action": last.get("action"), "action": last.get("action"),
"review_id": last.get("review_id"), "review_id": last.get("review_id"),
"head_sha": last_head,
} }
if last if last
else None else None
), ),
"locked_head_sha": last_head,
"correction_authorized": bool(lock.get("correction_authorized")), "correction_authorized": bool(lock.get("correction_authorized")),
"updated_at": lock.get("updated_at") or lock.get("recorded_at"), "updated_at": lock.get("updated_at") or lock.get("recorded_at"),
} }
@@ -88,13 +246,16 @@ def assess_stale_review_decision_lock(
pr_live: dict | None = None, pr_live: dict | None = None,
pr_lookup_error: str | None = None, pr_lookup_error: str | None = None,
active_profile_identity: str | None = None, active_profile_identity: str | None = None,
current_pr_head_sha: str | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Assess whether a durable review-decision lock is stale/moot (#594). """Assess whether a durable review-decision lock is stale/moot (#594/#620).
*pr_live* must be the live Gitea payload for the **last terminal *pr_live* must be the live Gitea payload for the **last terminal
mutation's PR**. When no lock / no terminal mutation exists, cleanup is mutation's PR**. When no lock / no terminal mutation exists, cleanup is
not applicable. When the PR is still open (or lookup fails), cleanup is not applicable. When the PR is still open (or lookup fails), cleanup is
forbidden and #332 hard-stop remains in force. forbidden (#594). Open PR + different live head reports
``stale_by_head`` / ``fresh_review_on_current_head_allowed`` (#620)
without enabling cleanup.
""" """
summary = lock_summary(lock) summary = lock_summary(lock)
result: dict[str, Any] = { result: dict[str, Any] = {
@@ -102,6 +263,10 @@ def assess_stale_review_decision_lock(
"lock_summary": summary, "lock_summary": summary,
"last_terminal_pr": None, "last_terminal_pr": None,
"last_terminal_action": None, "last_terminal_action": None,
"locked_head_sha": None,
"current_pr_head_sha": normalize_head_sha(current_pr_head_sha),
"stale_by_head": False,
"fresh_review_on_current_head_allowed": False,
"is_moot": False, "is_moot": False,
"cleanup_allowed": False, "cleanup_allowed": False,
"profile_match": True, "profile_match": True,
@@ -141,8 +306,10 @@ def assess_stale_review_decision_lock(
pr_number = last.get("pr_number") pr_number = last.get("pr_number")
action = last.get("action") action = last.get("action")
locked_head = mutation_head_sha(last, lock)
result["last_terminal_pr"] = pr_number result["last_terminal_pr"] = pr_number
result["last_terminal_action"] = action result["last_terminal_action"] = action
result["locked_head_sha"] = locked_head
if pr_number is None: if pr_number is None:
result["reasons"].append( result["reasons"].append(
@@ -165,25 +332,46 @@ def assess_stale_review_decision_lock(
return result return result
live = classify_pr_live_state(pr_live) live = classify_pr_live_state(pr_live)
current_head = normalize_head_sha(
current_pr_head_sha or live.get("current_pr_head_sha")
)
result.update( result.update(
{ {
"pr_state": live["pr_state"], "pr_state": live["pr_state"],
"pr_merged": live["pr_merged"], "pr_merged": live["pr_merged"],
"pr_merged_or_closed": live["pr_merged_or_closed"], "pr_merged_or_closed": live["pr_merged_or_closed"],
"merge_commit_sha": live["merge_commit_sha"], "merge_commit_sha": live["merge_commit_sha"],
"current_pr_head_sha": current_head,
} }
) )
if not live["pr_merged_or_closed"]: if not live["pr_merged_or_closed"]:
result["reasons"].append( stale_by_head = bool(
f"terminal review mutation on open PR #{pr_number} is still active " locked_head and current_head and not heads_equal(locked_head, current_head)
f"({action}); #332 hard-stop remains — cleanup forbidden (#594)"
) )
result["stale_by_head"] = stale_by_head
# Fresh formal decision is allowed on the new head without cleanup (#620).
result["fresh_review_on_current_head_allowed"] = stale_by_head
if stale_by_head:
result["reasons"].append(
f"terminal {action} on PR #{pr_number} is bound to head "
f"{locked_head[:12]}…; live head is {current_head[:12]}… — "
"fresh formal review on current head is allowed without "
"cleanup (fail closed for cleanup, #620); #332 same-head "
"duplicate protection remains"
)
else:
result["reasons"].append(
f"terminal review mutation on open PR #{pr_number} is still active "
f"({action}); #332 hard-stop remains — cleanup forbidden (#594)"
)
return result return result
# Merged or closed: lock is moot. Same-PR merge continuation is impossible. # Merged or closed: lock is moot. Same-PR merge continuation is impossible.
result["is_moot"] = True result["is_moot"] = True
result["cleanup_allowed"] = True result["cleanup_allowed"] = True
result["stale_by_head"] = False
result["fresh_review_on_current_head_allowed"] = False
result["reasons"].append( result["reasons"].append(
f"terminal mutation ({action} on PR #{pr_number}) is moot: PR is " f"terminal mutation ({action} on PR #{pr_number}) is moot: PR is "
f"{'merged' if live['pr_merged'] else 'closed'}; canonical cleanup allowed (#594)" f"{'merged' if live['pr_merged'] else 'closed'}; canonical cleanup allowed (#594)"
+97
View File
@@ -139,6 +139,103 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = {
"permission": "gitea.pr.create", "permission": "gitea.pr.create",
"role": "author", "role": "author",
}, },
# #600: controller-owned allocator — any authenticated profile may call;
# routing enforces role match to selected work. Uses control-plane DB (#613).
"allocate_next_work": {
"permission": "gitea.read",
"role": "author",
},
"gitea_allocate_next_work": {
"permission": "gitea.read",
"role": "author",
},
# #601 first-class lease lifecycle — inspect/list need read; mutations gate on
# ownership in the control-plane DB (not a separate Gitea write permission).
"list_workflow_leases": {
"permission": "gitea.read",
"role": "author",
},
"gitea_list_workflow_leases": {
"permission": "gitea.read",
"role": "author",
},
"inspect_workflow_lease": {
"permission": "gitea.read",
"role": "author",
},
"gitea_inspect_workflow_lease": {
"permission": "gitea.read",
"role": "author",
},
"adopt_workflow_lease": {
"permission": "gitea.read",
"role": "author",
},
"gitea_adopt_workflow_lease": {
"permission": "gitea.read",
"role": "author",
},
"release_workflow_lease": {
"permission": "gitea.read",
"role": "author",
},
"gitea_release_workflow_lease": {
"permission": "gitea.read",
"role": "author",
},
"expire_workflow_leases": {
"permission": "gitea.read",
"role": "author",
},
"gitea_expire_workflow_leases": {
"permission": "gitea.read",
"role": "author",
},
"abandon_workflow_lease": {
"permission": "gitea.read",
"role": "author",
},
"gitea_abandon_workflow_lease": {
"permission": "gitea.read",
"role": "author",
},
"reclaim_expired_workflow_lease": {
"permission": "gitea.read",
"role": "author",
},
"gitea_reclaim_expired_workflow_lease": {
"permission": "gitea.read",
"role": "author",
},
# #612 incident bridge — reconcile uses create_issue for apply;
# dry-run needs read only. Tools gate apply paths themselves.
"observability_reconcile_incident": {
"permission": "gitea.read",
"role": "author",
},
"gitea_observability_reconcile_incident": {
"permission": "gitea.read",
"role": "author",
},
"observability_list_projects": {
"permission": "gitea.read",
"role": "author",
},
"gitea_observability_list_projects": {
"permission": "gitea.read",
"role": "author",
},
"observability_link_issue": {
"permission": "gitea.read",
"role": "author",
},
"gitea_observability_link_issue": {
"permission": "gitea.read",
"role": "author",
},
"reconcile_landed_pr": { "reconcile_landed_pr": {
"permission": "gitea.read", "permission": "gitea.read",
"role": "author", "role": "author",
+366
View File
@@ -0,0 +1,366 @@
"""Tests for controller-owned allocator (#600) on control-plane DB (#613)."""
from __future__ import annotations
import os
import tempfile
import threading
import unittest
from concurrent.futures import ThreadPoolExecutor, as_completed
from allocator_service import (
OUTCOME_ASSIGNED,
OUTCOME_BLOCKED_TERMINAL,
OUTCOME_NO_SAFE,
OUTCOME_PREVIEW,
OUTCOME_WAIT,
WorkCandidate,
allocate_next_work,
candidate_from_dict,
classify_skip,
expected_role_for_candidate,
)
from control_plane_db import ControlPlaneDB, InvalidWorkKindError
class AllocatorServiceTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self._tmp.name, "cp.sqlite3")
self.db = ControlPlaneDB(self.db_path)
def tearDown(self) -> None:
self._tmp.cleanup()
def _alloc(self, **kwargs):
defaults = dict(
db=self.db,
session_id="s-test",
role="author",
remote="prgs",
org="org",
repo="repo",
candidates=[],
apply=False,
profile_name="prgs-author",
username="jcwalker3",
)
defaults.update(kwargs)
return allocate_next_work(**defaults)
def test_selects_ready_issue_for_author(self) -> None:
cands = [
WorkCandidate(
kind="issue",
number=612,
labels=("status:ready",),
title="bridge",
dependency_unmet=True,
dependency_reason="downstream of #600",
),
WorkCandidate(
kind="issue",
number=600,
labels=("status:ready", "type:feature"),
title="allocator",
priority=50,
),
WorkCandidate(
kind="issue",
number=601,
labels=("status:blocked",),
title="blocked",
blocked=True,
),
]
res = self._alloc(candidates=cands, apply=False)
self.assertTrue(res["success"])
self.assertEqual(res["outcome"], OUTCOME_PREVIEW)
self.assertEqual(res["selected"]["number"], 600)
self.assertEqual(res["selected"]["kind"], "issue")
# When 600 is highest priority valid work, lower-priority blocked/dep
# candidates are not visited. Prove they are skipped when they sort first.
res2 = self._alloc(
candidates=[
WorkCandidate(
kind="issue",
number=612,
labels=("status:ready",),
priority=99,
dependency_unmet=True,
dependency_reason="downstream of #600",
),
WorkCandidate(
kind="issue",
number=601,
labels=("status:blocked",),
priority=98,
blocked=True,
),
WorkCandidate(
kind="issue",
number=600,
labels=("status:ready",),
priority=1,
),
],
apply=False,
)
skipped_nums = {s["number"] for s in res2["skipped"]}
self.assertIn(612, skipped_nums)
self.assertIn(601, skipped_nums)
self.assertEqual(res2["selected"]["number"], 600)
self.assertIsNone(res["assignment"])
self.assertFalse(res["file_lock_only"])
self.assertFalse(res["comment_lease_only"])
def test_atomic_assign_and_lease_on_apply(self) -> None:
cands = [
WorkCandidate(
kind="issue",
number=600,
labels=("status:ready",),
priority=10,
)
]
res = self._alloc(candidates=cands, apply=True, session_id="s-a")
self.assertEqual(res["outcome"], OUTCOME_ASSIGNED)
asn = res["assignment"]
self.assertEqual(asn["outcome"], "assigned")
self.assertIsNotNone(asn["assignment_id"])
self.assertIsNotNone(asn["lease_id"])
self.assertEqual(asn["work_number"], 600)
self.assertIn("implement", asn["allowed_actions"])
self.assertIn("merge", asn["forbidden_actions"])
proof = res["lease_proof"]
self.assertEqual(proof["source"], "control_plane_db.assign_and_lease")
def test_concurrent_allocators_no_double_assign(self) -> None:
cand = WorkCandidate(
kind="pr",
number=100,
head_sha="a" * 40,
priority=10,
)
# Seed work item so both race the same target
self.db.upsert_work_item(
remote="prgs",
org="org",
repo="repo",
kind="pr",
number=100,
current_head_sha="a" * 40,
)
results = []
lock = threading.Lock()
def worker(sid: str):
r = allocate_next_work(
self.db,
session_id=sid,
role="reviewer",
remote="prgs",
org="org",
repo="repo",
candidates=[cand],
apply=True,
profile_name="prgs-reviewer",
)
with lock:
results.append(r)
with ThreadPoolExecutor(max_workers=2) as pool:
futs = [pool.submit(worker, f"s-{i}") for i in range(2)]
for f in as_completed(futs):
f.result()
outcomes = [r["outcome"] for r in results]
self.assertEqual(outcomes.count(OUTCOME_ASSIGNED), 1, outcomes)
self.assertEqual(outcomes.count(OUTCOME_WAIT), 1, outcomes)
def test_blocked_and_dependency_skipped(self) -> None:
cands = [
WorkCandidate(kind="issue", number=1, blocked=True, labels=("status:blocked",)),
WorkCandidate(
kind="issue",
number=612,
labels=("status:ready",),
dependency_unmet=True,
dependency_reason="downstream of #600",
),
]
res = self._alloc(candidates=cands, apply=True, role="author")
self.assertEqual(res["outcome"], OUTCOME_NO_SAFE)
self.assertIsNone(res["selected"])
reasons = " ".join(s["reason"] for s in res["skipped"])
self.assertIn("blocked", reasons.lower())
self.assertIn("600", reasons)
def test_already_leased_returns_wait(self) -> None:
self.db.upsert_session(session_id="owner", role="author")
self.db.assign_and_lease(
session_id="owner",
role="author",
remote="prgs",
org="org",
repo="repo",
kind="issue",
number=50,
)
res = self._alloc(
session_id="other",
candidates=[
WorkCandidate(kind="issue", number=50, labels=("status:ready",))
],
apply=True,
)
self.assertEqual(res["outcome"], OUTCOME_WAIT)
self.assertEqual(res["owner_session_id"], "owner")
def test_role_ineligible_skipped(self) -> None:
# PR with current-head REQUEST_CHANGES expects author, not reviewer.
cand = WorkCandidate(
kind="pr",
number=9,
head_sha="b" * 40,
request_changes_current_head=True,
)
self.assertEqual(expected_role_for_candidate(cand), "author")
res = self._alloc(
role="reviewer",
profile_name="prgs-reviewer",
candidates=[cand],
apply=False,
)
self.assertEqual(res["outcome"], OUTCOME_NO_SAFE)
self.assertTrue(any("expects role" in s["reason"] for s in res["skipped"]))
def test_terminal_lock_blocks_downstream_reviewer_prs(self) -> None:
self.db.set_terminal_lock(
remote="prgs",
org="org",
repo="repo",
terminal_pr=10,
decision="request_changes",
status="active",
)
cands = [
WorkCandidate(kind="pr", number=11, head_sha="c" * 40, priority=5),
WorkCandidate(kind="pr", number=10, head_sha="d" * 40, priority=1),
]
res = self._alloc(
role="reviewer",
profile_name="prgs-reviewer",
candidates=cands,
apply=False,
)
# Terminal PR #10 is selectable; #11 skipped for terminal path.
self.assertEqual(res["selected"]["number"], 10)
self.assertTrue(
any(
s["number"] == 11 and "terminal-review lock" in s["reason"]
for s in res["skipped"]
)
)
def test_terminal_lock_blocks_all_when_only_downstream(self) -> None:
self.db.set_terminal_lock(
remote="prgs",
org="org",
repo="repo",
terminal_pr=10,
decision="request_changes",
status="active",
)
res = self._alloc(
role="reviewer",
profile_name="prgs-reviewer",
candidates=[
WorkCandidate(kind="pr", number=99, head_sha="e" * 40),
],
apply=False,
)
self.assertEqual(res["outcome"], OUTCOME_BLOCKED_TERMINAL)
def test_no_work_structured(self) -> None:
res = self._alloc(candidates=[], apply=True)
self.assertEqual(res["outcome"], OUTCOME_NO_SAFE)
self.assertIsNone(res["selected"])
self.assertTrue(res["reasons"])
def test_rejects_incident_kind(self) -> None:
with self.assertRaises(InvalidWorkKindError):
WorkCandidate(kind="sentry_incident", number=1)
def test_612_downstream_marker_in_result(self) -> None:
res = self._alloc(
candidates=[
WorkCandidate(kind="issue", number=600, labels=("status:ready",))
],
apply=True,
)
self.assertIn("612", res.get("downstream_note", ""))
def test_substrate_not_file_or_comment_lease(self) -> None:
res = self._alloc(
candidates=[
WorkCandidate(kind="issue", number=1, labels=("status:ready",))
],
apply=True,
)
self.assertEqual(res["substrate"], "control_plane_db")
self.assertFalse(res["file_lock_only"])
self.assertFalse(res["comment_lease_only"])
self.assertEqual(
res["lease_proof"]["source"], "control_plane_db.assign_and_lease"
)
def test_candidate_from_dict(self) -> None:
c = candidate_from_dict(
{
"kind": "pr",
"number": 7,
"head_sha": "f" * 40,
"approval_on_current_head": True,
"mergeable": True,
}
)
self.assertEqual(expected_role_for_candidate(c), "merger")
def test_merger_gets_clean_approval(self) -> None:
c = WorkCandidate(
kind="pr",
number=3,
head_sha="1" * 40,
approval_on_current_head=True,
mergeable=True,
priority=100,
)
res = self._alloc(
role="merger",
profile_name="prgs-merger",
candidates=[c],
apply=True,
session_id="merger-1",
)
self.assertEqual(res["outcome"], OUTCOME_ASSIGNED)
self.assertEqual(res["assignment"]["work_number"], 3)
self.assertIn("merge", res["assignment"]["allowed_actions"])
def test_db_unavailable_fails_closed(self) -> None:
res = allocate_next_work(
None, # type: ignore[arg-type]
session_id="x",
role="author",
remote="prgs",
org="o",
repo="r",
candidates=[],
apply=True,
)
self.assertFalse(res["success"])
self.assertIn("unavailable", res["reasons"][0].lower())
if __name__ == "__main__":
unittest.main()
+1 -1
View File
@@ -36,7 +36,7 @@ class ControlPlaneDBTest(unittest.TestCase):
rows = dict(conn.execute("SELECT key, value FROM schema_meta").fetchall()) rows = dict(conn.execute("SELECT key, value FROM schema_meta").fetchall())
finally: finally:
conn.close() conn.close()
self.assertEqual(rows["schema_version"], "2") self.assertEqual(rows["schema_version"], "3")
self.assertIn("DB coordinates", rows["architecture"]) self.assertIn("DB coordinates", rows["architecture"])
self.assertIn("bridge", rows["architecture"].lower()) self.assertIn("bridge", rows["architecture"].lower())
@@ -0,0 +1,397 @@
"""Head-scoped #332 review-decision locks (#620).
Proves:
* REQUEST_CHANGES on head A does not block mark_ready/submit on head B
* APPROVE on head B is allowed after RC on head A (same open PR)
* same-head duplicate terminal remains blocked
* open-PR cleanup remains forbidden; assessment reports stale_by_head
* historical mutations keep their head_sha (preserved ledger)
* PR #619-style: old RC at 036b78e…, current head 2429d9d…, approve allowed
"""
from __future__ import annotations
import os
import unittest
from unittest.mock import patch
import mcp_server
import stale_review_decision_lock as srdl
HEAD_A = "036b78e31ea5d036452b3a52e0e086b01e0c763f"
HEAD_B = "2429d9d2e826e519eb8eb4ade45987c00838aefb"
HEAD_C = "c" * 40
def _lock(mutations=None, correction=False, ready_pr=None, ready_head=None, ready_action=None):
return {
"task": "review_pr",
"remote": "prgs",
"org": "Scaled-Tech-Consulting",
"repo": "Gitea-Tools",
"session_pid": os.getpid(),
"session_profile": "prgs-reviewer",
"session_profile_lock": "prgs-reviewer",
"profile_identity": "prgs-reviewer",
"final_review_decision_ready": False,
"ready_pr_number": ready_pr,
"ready_action": ready_action,
"ready_expected_head_sha": ready_head,
"ready_remote": "prgs" if ready_pr else None,
"ready_org": "Scaled-Tech-Consulting" if ready_pr else None,
"ready_repo": "Gitea-Tools" if ready_pr else None,
"live_mutations": list(mutations or []),
"correction_authorized": correction,
"correction_reason": None,
}
RC_619_LEGACY = {
"pr_number": 619,
"action": "request_changes",
"review_id": 410,
"review_state": "request_changes",
# no head_sha — pre-#620 durable ledger shape
}
RC_619_HEAD_A = {
"pr_number": 619,
"action": "request_changes",
"review_id": 410,
"review_state": "request_changes",
"head_sha": HEAD_A,
}
APPROVE_619_HEAD_B = {
"pr_number": 619,
"action": "approve",
"review_id": 411,
"review_state": "approve",
"head_sha": HEAD_B,
}
def _seed(mutations=None, **kwargs):
import review_workflow_load
review_workflow_load.record_review_workflow_load(mcp_server.PROJECT_ROOT)
mcp_server._save_review_decision_lock(_lock(mutations, **kwargs))
mcp_server.gitea_load_review_workflow()
def _open_pr(pr_number=619, head=HEAD_B):
return {
"number": pr_number,
"state": "open",
"merged": False,
"merged_at": None,
"head": {"sha": head},
}
def _no_lease():
return {"block": False, "reasons": [], "mutation_allowed": True}
def _feedback(blocking=False, stale=False, success=True):
return {
"success": success,
"has_blocking_change_requests": blocking,
"review_feedback_stale": stale,
"current_head_sha": HEAD_B,
}
class TestPureHeadScopeHelpers(unittest.TestCase):
def test_mutation_head_prefers_recorded_sha(self):
m = {"pr_number": 1, "head_sha": HEAD_A}
self.assertEqual(srdl.mutation_head_sha(m), HEAD_A)
def test_legacy_mutation_uses_ready_expected_for_same_pr(self):
lock = _lock(
[RC_619_LEGACY],
ready_pr=619,
ready_head=HEAD_A,
ready_action="request_changes",
)
self.assertEqual(srdl.mutation_head_sha(RC_619_LEGACY, lock), HEAD_A)
def test_legacy_mutation_ignores_ready_for_other_pr(self):
lock = _lock([RC_619_LEGACY], ready_pr=1, ready_head=HEAD_A)
self.assertIsNone(srdl.mutation_head_sha(RC_619_LEGACY, lock))
def test_prior_blocks_same_head_not_other_head(self):
lock = _lock([RC_619_HEAD_A])
self.assertTrue(
srdl.prior_live_mutations_block_boundary(
lock, pr_number=619, expected_head_sha=HEAD_A
)
)
self.assertFalse(
srdl.prior_live_mutations_block_boundary(
lock, pr_number=619, expected_head_sha=HEAD_B
)
)
def test_backfill_stamps_legacy_terminals(self):
lock = _lock(
[dict(RC_619_LEGACY)],
ready_pr=619,
ready_head=HEAD_A,
ready_action="request_changes",
)
srdl.backfill_terminal_heads_from_ready(lock)
self.assertEqual(lock["live_mutations"][0]["head_sha"], HEAD_A)
class TestHardStopHeadScope(unittest.TestCase):
def tearDown(self):
mcp_server._save_review_decision_lock(None)
mcp_server.review_workflow_load.clear_review_workflow_load()
def test_rc_head_a_allows_mark_ready_head_b(self):
_seed(
[RC_619_HEAD_A],
ready_pr=619,
ready_head=HEAD_A,
ready_action="request_changes",
)
self.assertEqual(
mcp_server.terminal_review_hard_stop_reasons(
619, "mark_ready", expected_head_sha=HEAD_B
),
[],
)
def test_rc_head_a_blocks_mark_ready_same_head(self):
_seed(
[RC_619_HEAD_A],
ready_pr=619,
ready_head=HEAD_A,
ready_action="request_changes",
)
reasons = mcp_server.terminal_review_hard_stop_reasons(
619, "mark_ready", expected_head_sha=HEAD_A
)
self.assertTrue(reasons)
self.assertIn("#332", reasons[0])
def test_rc_head_a_blocks_other_pr(self):
_seed([RC_619_HEAD_A], ready_pr=619, ready_head=HEAD_A)
reasons = mcp_server.terminal_review_hard_stop_reasons(
700, "mark_ready", expected_head_sha=HEAD_B
)
self.assertTrue(reasons)
def test_legacy_619_ledger_allows_new_head(self):
"""PR #619-style durable lock without mutation head_sha."""
_seed(
[RC_619_LEGACY],
ready_pr=619,
ready_head=HEAD_A,
ready_action="request_changes",
)
self.assertEqual(
mcp_server.terminal_review_hard_stop_reasons(
619, "mark_ready", expected_head_sha=HEAD_B
),
[],
)
class TestMarkFinalAndRecord(unittest.TestCase):
def tearDown(self):
mcp_server._save_review_decision_lock(None)
mcp_server.review_workflow_load.clear_review_workflow_load()
def _mark(self, pr, action, head, feedback=None):
with patch("mcp_server._list_pr_lease_comments", return_value=[]), patch(
"mcp_server._pr_work_lease_reviewer_block", return_value=_no_lease()
), patch.object(
mcp_server,
"gitea_get_pr_review_feedback",
return_value=feedback or _feedback(blocking=False, stale=True),
), patch.object(
mcp_server,
"gitea_check_pr_eligibility",
return_value={"eligible": True, "head_sha": head},
):
return mcp_server.gitea_mark_final_review_decision(
pr_number=pr,
action=action,
expected_head_sha=head,
remote="prgs",
org="Scaled-Tech-Consulting",
repo="Gitea-Tools",
)
def test_rc_then_rc_on_new_head(self):
_seed(
[RC_619_HEAD_A],
ready_pr=619,
ready_head=HEAD_A,
ready_action="request_changes",
)
# Prior RC at head A is unresolved but head moved → feedback stale.
res = self._mark(
619,
"request_changes",
HEAD_B,
feedback=_feedback(blocking=True, stale=True),
)
self.assertTrue(res.get("marked_ready"), res)
lock = mcp_server._load_review_decision_lock()
# Historical head A preserved on mutation after backfill.
heads = {
m.get("head_sha")
for m in lock["live_mutations"]
if m.get("action") == "request_changes"
}
self.assertIn(HEAD_A, heads)
self.assertEqual(lock["ready_expected_head_sha"], HEAD_B)
def test_rc_then_approve_on_new_head(self):
_seed(
[RC_619_HEAD_A],
ready_pr=619,
ready_head=HEAD_A,
ready_action="request_changes",
)
res = self._mark(619, "approve", HEAD_B)
self.assertTrue(res.get("marked_ready"), res)
self.assertTrue(res.get("head_scoped"))
def test_same_head_rc_still_blocked_by_hard_stop(self):
_seed(
[RC_619_HEAD_A],
ready_pr=619,
ready_head=HEAD_A,
ready_action="request_changes",
)
res = self._mark(619, "approve", HEAD_A)
self.assertFalse(res.get("marked_ready"))
self.assertTrue(any("#332" in r for r in res.get("reasons") or []))
def test_pr619_style_legacy_lock_approve_on_current_head(self):
_seed(
[RC_619_LEGACY],
ready_pr=619,
ready_head=HEAD_A,
ready_action="request_changes",
)
res = self._mark(619, "approve", HEAD_B)
self.assertTrue(res.get("marked_ready"), res)
lock = mcp_server._load_review_decision_lock()
# Backfill should have stamped HEAD_A onto the legacy mutation.
self.assertEqual(lock["live_mutations"][0].get("head_sha"), HEAD_A)
self.assertEqual(lock["ready_expected_head_sha"], HEAD_B)
def test_record_live_mutation_stores_head(self):
_seed(ready_pr=619, ready_head=HEAD_B, ready_action="approve")
lock = mcp_server._load_review_decision_lock()
lock["final_review_decision_ready"] = True
lock["ready_pr_number"] = 619
lock["ready_expected_head_sha"] = HEAD_B
mcp_server._save_review_decision_lock(lock)
mcp_server.record_live_review_mutation(619, "approve", review_id=99)
stored = mcp_server._load_review_decision_lock()["live_mutations"][-1]
self.assertEqual(stored["head_sha"], HEAD_B)
self.assertEqual(stored["pr_number"], 619)
def test_submit_gate_allows_new_head_after_prior_terminal(self):
"""check_review_decision_gate must not block different-head submit."""
mutations = [RC_619_HEAD_A]
_seed(
mutations,
ready_pr=619,
ready_head=HEAD_B,
ready_action="approve",
)
lock = mcp_server._load_review_decision_lock()
lock["final_review_decision_ready"] = True
lock["ready_pr_number"] = 619
lock["ready_action"] = "approve"
lock["ready_expected_head_sha"] = HEAD_B
lock["ready_remote"] = "prgs"
lock["ready_org"] = "Scaled-Tech-Consulting"
lock["ready_repo"] = "Gitea-Tools"
mcp_server._save_review_decision_lock(lock)
reasons = mcp_server.check_review_decision_gate(
619,
"approve",
final_review_decision_ready=True,
remote="prgs",
org="Scaled-Tech-Consulting",
repo="Gitea-Tools",
)
self.assertEqual(reasons, [], reasons)
def test_submit_gate_blocks_same_head_duplicate(self):
mutations = [RC_619_HEAD_A]
_seed(
mutations,
ready_pr=619,
ready_head=HEAD_A,
ready_action="request_changes",
)
lock = mcp_server._load_review_decision_lock()
lock["final_review_decision_ready"] = True
lock["ready_pr_number"] = 619
lock["ready_action"] = "request_changes"
lock["ready_expected_head_sha"] = HEAD_A
lock["ready_remote"] = "prgs"
lock["ready_org"] = "Scaled-Tech-Consulting"
lock["ready_repo"] = "Gitea-Tools"
mcp_server._save_review_decision_lock(lock)
reasons = mcp_server.check_review_decision_gate(
619,
"request_changes",
final_review_decision_ready=True,
remote="prgs",
org="Scaled-Tech-Consulting",
repo="Gitea-Tools",
)
self.assertTrue(reasons)
class TestAssessmentOpenPrHead(unittest.TestCase):
def test_open_pr_stale_by_head_no_cleanup(self):
lock = _lock(
[RC_619_HEAD_A],
ready_pr=619,
ready_head=HEAD_A,
ready_action="request_changes",
)
a = srdl.assess_stale_review_decision_lock(
lock, pr_live=_open_pr(619, HEAD_B)
)
self.assertTrue(a["has_lock"])
self.assertFalse(a["is_moot"])
self.assertFalse(a["cleanup_allowed"])
self.assertTrue(a["stale_by_head"])
self.assertTrue(a["fresh_review_on_current_head_allowed"])
self.assertEqual(a["locked_head_sha"], HEAD_A)
self.assertEqual(a["current_pr_head_sha"], HEAD_B)
def test_open_pr_same_head_no_fresh(self):
lock = _lock([RC_619_HEAD_A], ready_pr=619, ready_head=HEAD_A)
a = srdl.assess_stale_review_decision_lock(
lock, pr_live=_open_pr(619, HEAD_A)
)
self.assertFalse(a["stale_by_head"])
self.assertFalse(a["fresh_review_on_current_head_allowed"])
self.assertFalse(a["cleanup_allowed"])
def test_historical_mutation_preserved_in_summary(self):
lock = _lock(
[RC_619_HEAD_A, APPROVE_619_HEAD_B],
ready_pr=619,
ready_head=HEAD_B,
ready_action="approve",
)
summary = srdl.lock_summary(lock)
self.assertEqual(summary["live_mutations_count"], 2)
self.assertEqual(summary["last_terminal"]["head_sha"], HEAD_B)
self.assertEqual(summary["locked_head_sha"], HEAD_B)
if __name__ == "__main__":
unittest.main()
+345
View File
@@ -0,0 +1,345 @@
"""Tests for observability incident bridge (#612) on #613 substrate."""
from __future__ import annotations
import json
import os
import tempfile
import unittest
from control_plane_db import ControlPlaneDB, InvalidWorkKindError, WORK_KINDS
from incident_bridge import (
OUTCOME_BLOCKED,
OUTCOME_CREATED,
OUTCOME_LINKED,
OUTCOME_PREVIEW,
OUTCOME_UPDATED,
ProjectMapping,
assert_not_raw_incident_work_item,
build_gitea_issue_body,
load_project_mappings,
normalize_incident,
project_mapping_from_dict,
reconcile_incident,
redact_text,
sanitize_tags,
)
def _mapping(**kwargs) -> ProjectMapping:
base = dict(
name="gitea-tools-mcp",
provider="sentry",
monitor_base_url="https://sentry.prgs.cc",
monitor_org="prgs",
monitor_project="gitea-tools-mcp",
gitea_org="Scaled-Tech-Consulting",
gitea_repo="Gitea-Tools",
default_labels=("type:bug", "observability", "sentry", "status:ready"),
)
base.update(kwargs)
return ProjectMapping(**base)
def _obs(**kwargs) -> dict:
base = dict(
provider="sentry",
provider_base_url="https://sentry.prgs.cc",
provider_org="prgs",
provider_project="gitea-tools-mcp",
provider_issue_id="ISSUE-100",
provider_short_id="GITEA-TOOLS-1A",
provider_permalink="https://sentry.prgs.cc/organizations/prgs/issues/100/",
fingerprint="fp-abc",
title="TypeError: boom",
summary="TypeError: boom in handle_request",
first_seen="2026-07-01T00:00:00Z",
last_seen="2026-07-10T00:00:00Z",
event_count=3,
environment="production",
severity="error",
culprit="handle_request",
tags={"runtime": "python", "release": "1.2.3"},
)
base.update(kwargs)
return base
class IncidentBridgeTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.db = ControlPlaneDB(os.path.join(self._tmp.name, "cp.sqlite3"))
self.mapping = _mapping()
self.created: list[dict] = []
def tearDown(self) -> None:
self._tmp.cleanup()
def _create_issue(self, title, body, labels, g_org, g_repo):
n = 9000 + len(self.created)
rec = {
"number": n,
"success": True,
"performed": True,
"title": title,
"body": body,
"labels": labels,
"org": g_org,
"repo": g_repo,
}
self.created.append(rec)
return rec
def test_redact_secrets(self) -> None:
dirty = "token=supersecret123 Authorization: Bearer abcdefghijklmnop"
clean = redact_text(dirty)
self.assertNotIn("supersecret", clean)
self.assertNotIn("abcdefghijklmnop", clean)
self.assertIn("[REDACTED]", clean)
tags = sanitize_tags(
{"token": "x", "runtime": "py", "password": "nope", "release": "1.0"}
)
self.assertEqual(tags, {"runtime": "py", "release": "1.0"})
def test_body_contains_no_secrets(self) -> None:
inc = normalize_incident(
_obs(
summary="fail password=hunter2 token: abc",
tags={"cookie": "sid=1", "runtime": "py"},
),
self.mapping,
)
body = build_gitea_issue_body(inc)
self.assertNotIn("hunter2", body)
self.assertNotIn("sid=1", body)
self.assertIn("provider_issue_id", body)
self.assertIn("not** assignable", body.lower())
def test_preview_no_mutation(self) -> None:
res = reconcile_incident(
self.db,
observation=_obs(),
mapping=self.mapping,
apply=False,
create_issue_fn=self._create_issue,
)
self.assertTrue(res["success"])
self.assertEqual(res["outcome"], OUTCOME_PREVIEW)
self.assertFalse(res["gitea_mutated"])
self.assertFalse(res["db_mutated"])
self.assertFalse(res["raw_incident_assignable"])
self.assertEqual(len(self.created), 0)
self.assertIsNone(
self.db.get_incident_link_by_provider(
provider="sentry",
provider_issue_id="ISSUE-100",
provider_base_url="https://sentry.prgs.cc",
provider_org="prgs",
provider_project="gitea-tools-mcp",
)
)
def test_apply_creates_issue_and_link(self) -> None:
res = reconcile_incident(
self.db,
observation=_obs(),
mapping=self.mapping,
apply=True,
create_issue_fn=self._create_issue,
)
self.assertTrue(res["success"])
self.assertEqual(res["outcome"], OUTCOME_CREATED)
self.assertTrue(res["gitea_mutated"])
self.assertTrue(res["db_mutated"])
self.assertEqual(len(self.created), 1)
self.assertEqual(res["gitea_issue"]["number"], self.created[0]["number"])
link = self.db.get_incident_link_by_provider(
provider="sentry",
provider_issue_id="ISSUE-100",
provider_base_url="https://sentry.prgs.cc",
provider_org="prgs",
provider_project="gitea-tools-mcp",
)
self.assertIsNotNone(link)
self.assertEqual(int(link["gitea_issue_number"]), self.created[0]["number"])
self.assertEqual(res["allocator_visible_as"]["kind"], "issue")
def test_duplicate_observation_reuses_issue(self) -> None:
first = reconcile_incident(
self.db,
observation=_obs(),
mapping=self.mapping,
apply=True,
create_issue_fn=self._create_issue,
)
second = reconcile_incident(
self.db,
observation=_obs(event_count=9, last_seen="2026-07-11T00:00:00Z"),
mapping=self.mapping,
apply=True,
create_issue_fn=self._create_issue,
)
self.assertEqual(first["outcome"], OUTCOME_CREATED)
self.assertEqual(second["outcome"], OUTCOME_UPDATED)
self.assertEqual(len(self.created), 1)
self.assertEqual(
second["gitea_issue"]["number"], first["gitea_issue"]["number"]
)
link = self.db.get_incident_link_by_provider(
provider="sentry",
provider_issue_id="ISSUE-100",
provider_base_url="https://sentry.prgs.cc",
provider_org="prgs",
provider_project="gitea-tools-mcp",
)
self.assertEqual(int(link["event_count"]), 9)
def test_explicit_link_existing_issue(self) -> None:
res = reconcile_incident(
self.db,
observation=_obs(provider_issue_id="ISSUE-200"),
mapping=self.mapping,
apply=True,
force_gitea_issue_number=555,
create_issue_fn=self._create_issue,
)
self.assertEqual(res["outcome"], OUTCOME_LINKED)
self.assertEqual(len(self.created), 0)
self.assertEqual(res["gitea_issue"]["number"], 555)
link = self.db.get_incident_link_for_gitea_issue(
gitea_org="Scaled-Tech-Consulting",
gitea_repo="Gitea-Tools",
gitea_issue_number=555,
)
self.assertIsNotNone(link)
def test_fingerprint_conflict_fails_closed(self) -> None:
reconcile_incident(
self.db,
observation=_obs(fingerprint="fp-a"),
mapping=self.mapping,
apply=True,
create_issue_fn=self._create_issue,
)
res = reconcile_incident(
self.db,
observation=_obs(fingerprint="fp-b"),
mapping=self.mapping,
apply=True,
create_issue_fn=self._create_issue,
)
self.assertEqual(res["outcome"], OUTCOME_BLOCKED)
self.assertIn("fingerprint conflict", res["reasons"][0].lower())
self.assertEqual(len(self.created), 1)
def test_ambiguous_mapping_fails_closed(self) -> None:
maps = [
_mapping(name="a", monitor_project="gitea-tools-mcp"),
_mapping(name="b", monitor_project="gitea-tools-mcp"),
]
res = reconcile_incident(
self.db,
observation=_obs(),
mappings=maps,
apply=False,
)
self.assertEqual(res["outcome"], OUTCOME_BLOCKED)
self.assertIn("ambiguous", res["reasons"][0].lower())
def test_missing_provider_issue_id_fails_closed(self) -> None:
res = reconcile_incident(
self.db,
observation=_obs(provider_issue_id=""),
mapping=self.mapping,
apply=False,
)
self.assertEqual(res["outcome"], OUTCOME_BLOCKED)
def test_raw_incident_not_work_kind(self) -> None:
self.assertNotIn("sentry_incident", WORK_KINDS)
with self.assertRaises(InvalidWorkKindError):
self.db.upsert_work_item(
remote="prgs",
org="o",
repo="r",
kind="sentry_incident",
number=1,
)
with self.assertRaises(Exception):
assert_not_raw_incident_work_item("glitchtip_incident")
def test_db_unavailable(self) -> None:
res = reconcile_incident(
None,
observation=_obs(),
mapping=self.mapping,
apply=True,
create_issue_fn=self._create_issue,
)
self.assertFalse(res["success"])
self.assertEqual(res["outcome"], OUTCOME_BLOCKED)
def test_structured_skip_reason_environment(self) -> None:
m = _mapping(environment_filters=("staging",))
res = reconcile_incident(
self.db,
observation=_obs(environment="production"),
mapping=m,
apply=True,
create_issue_fn=self._create_issue,
)
self.assertTrue(res["success"])
self.assertEqual(res["action"], "skip_environment_filter")
self.assertEqual(len(self.created), 0)
def test_load_mappings_json(self) -> None:
payload = json.dumps(
{
"projects": [
{
"name": "gt",
"provider": "glitchtip",
"monitor_base_url": "https://glitchtip.example",
"monitor_org": "org",
"monitor_project": "proj",
"gitea_org": "Scaled-Tech-Consulting",
"gitea_repo": "Gitea-Tools",
}
]
}
)
maps = load_project_mappings(mappings_json=payload)
self.assertEqual(len(maps), 1)
self.assertEqual(maps[0].provider, "glitchtip")
def test_glitchtip_provider_accepted(self) -> None:
m = _mapping(provider="glitchtip", monitor_base_url="https://glitchtip.example")
res = reconcile_incident(
self.db,
observation=_obs(
provider="glitchtip",
provider_base_url="https://glitchtip.example",
),
mapping=m,
apply=True,
create_issue_fn=self._create_issue,
)
self.assertEqual(res["outcome"], OUTCOME_CREATED)
self.assertEqual(res["incident"]["provider"], "glitchtip")
def test_allocator_only_sees_issue_kind(self) -> None:
res = reconcile_incident(
self.db,
observation=_obs(),
mapping=self.mapping,
apply=True,
create_issue_fn=self._create_issue,
)
vis = res["allocator_visible_as"]
self.assertEqual(vis["kind"], "issue")
self.assertIn(vis["kind"], WORK_KINDS)
self.assertFalse(res["raw_incident_assignable"])
if __name__ == "__main__":
unittest.main()
+27 -7
View File
@@ -103,20 +103,40 @@ class TestIssueLockStore(unittest.TestCase):
self.assertIn("live foreign issue lock", block or "") self.assertIn("live foreign issue lock", block or "")
def test_expired_lease_allows_takeover_with_conflict_check(self): def test_expired_lease_allows_takeover_with_conflict_check(self):
# #601: expired + dead pid / missing worktree → sanctioned reclaim (no block).
existing = _lock_record( existing = _lock_record(
branch_name="feat/issue-420-other", branch_name="feat/issue-420-other",
worktree_path="/tmp/other", worktree_path="/tmp/other-does-not-exist-420",
session_pid=1,
work_lease=_lease("2000-01-01T00:00:00Z"), work_lease=_lease("2000-01-01T00:00:00Z"),
) )
incoming = _lock_record(worktree_path="/tmp/mine") incoming = _lock_record(worktree_path="/tmp/mine")
self.assertIsNone(ils.assess_foreign_lock_overwrite(existing, incoming)) self.assertIsNone(ils.assess_foreign_lock_overwrite(existing, incoming))
block = ils.assess_same_issue_lease_conflict( with mock.patch.object(ils, "is_process_alive", return_value=False):
existing, block = ils.assess_same_issue_lease_conflict(
issue_number=420, existing,
branch_name="feat/issue-420-server-code-parity", issue_number=420,
worktree_path="/tmp/mine", branch_name="feat/issue-420-server-code-parity",
worktree_path="/tmp/mine",
)
self.assertIsNone(block)
# Expired but still-live owner pid AND present worktree → fail closed.
present_wt = self.lock_dir # exists
sticky = _lock_record(
branch_name="feat/issue-420-other",
worktree_path=present_wt,
session_pid=os.getpid(),
work_lease=_lease("2000-01-01T00:00:00Z"),
) )
self.assertIn("Recovery review is required", block or "") with mock.patch.object(ils, "is_process_alive", return_value=True):
blocked = ils.assess_same_issue_lease_conflict(
sticky,
issue_number=420,
branch_name="feat/issue-420-server-code-parity",
worktree_path="/tmp/mine",
)
self.assertIn("Recovery review is required", blocked or "")
def test_same_owner_lease_conflict_allows_refresh(self): def test_same_owner_lease_conflict_allows_refresh(self):
worktree = "/tmp/wt-420" worktree = "/tmp/wt-420"
+392
View File
@@ -0,0 +1,392 @@
"""Tests for first-class control-plane lease lifecycle (#601)."""
from __future__ import annotations
import os
import tempfile
import unittest
from datetime import timedelta
from unittest import mock
from allocator_service import allocate_next_work, WorkCandidate
from control_plane_db import ControlPlaneDB, ForeignLeaseError, _ts, _utc_now
import lease_lifecycle as ll
import merger_lease_adoption as mla
import reviewer_pr_lease as rpl
from datetime import datetime, timezone
class LeaseLifecycleTest(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self._tmp.name, "cp.sqlite3")
self.db = ControlPlaneDB(self.db_path)
self.db.upsert_session(session_id="owner", role="author", profile="prgs-author", pid=111)
self.db.upsert_session(session_id="other", role="author", profile="prgs-author", pid=222)
def tearDown(self) -> None:
self._tmp.cleanup()
def _assign(self, session_id="owner", number=601, **kwargs):
return self.db.assign_and_lease(
session_id=session_id,
role="author",
remote="prgs",
org="Scaled-Tech-Consulting",
repo="Gitea-Tools",
kind="issue",
number=number,
allowed_actions=("implement", "comment", "push", "create_pr"),
forbidden_actions=("approve", "merge", "self_select_without_assignment"),
worktree_path=kwargs.pop("worktree_path", self._tmp.name),
owner_pid=kwargs.pop("owner_pid", os.getpid()),
**kwargs,
)
def test_list_active_leases_as_workflow_state(self) -> None:
a = self._assign(number=601)
self._assign(session_id="other", number=602)
listed = ll.list_active_leases(
self.db, remote="prgs", org="Scaled-Tech-Consulting", repo="Gitea-Tools"
)
self.assertTrue(listed["success"])
self.assertEqual(listed["count"], 2)
self.assertEqual(listed["authoritative_source"], "control_plane_db")
self.assertFalse(listed["file_lock_only"])
self.assertFalse(listed["comment_lease_only"])
numbers = sorted(x["work_number"] for x in listed["leases"])
self.assertEqual(numbers, [601, 602])
self.assertIsNotNone(a.lease_id)
def test_adopt_lease_owner_resume_preserves_provenance(self) -> None:
a = self._assign()
res = ll.adopt_lease(
self.db,
lease_id=a.lease_id,
adopter_session_id="owner",
role="author",
worktree_path=self._tmp.name,
)
self.assertTrue(res["success"])
self.assertTrue(res["same_owner"])
prov = res["provenance"]
self.assertEqual(prov["adopted_from_session_id"], "owner")
self.assertEqual(prov["adopted_by_session_id"], "owner")
self.assertEqual(prov["work_number"], 601)
self.assertEqual(prov["worktree_path"], self._tmp.name)
state = self.db.get_lease_workflow_state(a.lease_id)
self.assertIsNotNone(state)
self.assertEqual(state["lease"]["status"], "active")
def test_release_lease_explicit_recorded(self) -> None:
a = self._assign()
res = ll.release_lease(self.db, lease_id=a.lease_id, session_id="owner")
self.assertEqual(res["outcome"], "released")
self.assertIn("release_proof", res)
self.assertEqual(res["release_proof"]["status"], "released")
state = self.db.get_lease_workflow_state(a.lease_id)
self.assertEqual(state["lease"]["status"], "released")
def test_expire_and_reclaim_expired_lease(self) -> None:
a = self._assign(lease_ttl_seconds=1)
past = _utc_now() - timedelta(hours=2)
import sqlite3
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"UPDATE leases SET expires_at = ? WHERE lease_id = ?",
(_ts(past), a.lease_id),
)
conn.commit()
finally:
conn.close()
exp = ll.expire_leases(self.db)
self.assertGreaterEqual(exp["expired_count"], 1)
reclaimed = ll.reclaim_expired_lease(
self.db,
lease_id=a.lease_id,
session_id="other",
role="author",
worktree_path=self._tmp.name,
)
self.assertEqual(reclaimed["outcome"], "reclaimed")
self.assertEqual(reclaimed["assignment"]["session_id"], "other")
self.assertEqual(
reclaimed["provenance"]["adopted_from_session_id"], "owner"
)
self.assertEqual(
reclaimed["provenance"]["adopted_by_session_id"], "other"
)
def test_abandon_stale_lease_with_required_proof(self) -> None:
a = self._assign(owner_pid=99999999, worktree_path="/nonexistent/path/for-601")
# Force active but dead pid + missing worktree
proof = ll.AbandonProof(
dead_process=True,
missing_worktree=True,
no_open_pr=True,
no_live_mutation_risk=True,
owner_pid=99999999,
worktree_path="/nonexistent/path/for-601",
)
res = ll.abandon_lease(
self.db,
lease_id=a.lease_id,
requester_session_id="other",
proof=proof,
)
self.assertEqual(res["outcome"], "abandoned")
self.assertEqual(res["prior_owner_session_id"], "owner")
self.assertTrue(res["abandon_proof"]["dead_process"])
state = self.db.get_lease_workflow_state(a.lease_id)
self.assertEqual(state["lease"]["status"], "abandoned")
def test_refuse_steal_active_foreign_lease(self) -> None:
a = self._assign()
with self.assertRaises(ll.LeaseLifecycleError) as ctx:
ll.adopt_lease(
self.db,
lease_id=a.lease_id,
adopter_session_id="other",
role="author",
)
self.assertIn("steal", str(ctx.exception).lower())
decision = ll.inspect_lease(
self.db, a.lease_id, caller_session_id="other"
)
self.assertEqual(decision["safe_next_action"], ll.SAFE_WAIT_FOREIGN)
self.assertTrue(decision["block"])
def test_refuse_ambiguous_lease_ownership_stale_id(self) -> None:
decision = ll.inspect_lease(
self.db, "lease-does-not-exist", caller_session_id="owner"
)
self.assertFalse(decision["found"])
self.assertEqual(decision["safe_next_action"], ll.SAFE_STALE_PROMPT)
with self.assertRaises(ll.LeaseLifecycleError):
ll.adopt_lease(
self.db,
lease_id="lease-does-not-exist",
adopter_session_id="owner",
role="author",
)
def test_provenance_on_adopt_release_abandon(self) -> None:
a = self._assign()
adopted = ll.adopt_lease(
self.db,
lease_id=a.lease_id,
adopter_session_id="owner",
role="author",
worktree_path=self._tmp.name,
expected_head_sha="abc",
)
self.assertIn("adopted_from_session_id", adopted["provenance"])
self.assertIn("adopted_by_session_id", adopted["provenance"])
released = ll.release_lease(
self.db, lease_id=a.lease_id, session_id="owner"
)
self.assertEqual(released["release_proof"]["session_id"], "owner")
b = self._assign(number=700, owner_pid=1, worktree_path="/no/such/wt")
abandoned = ll.abandon_lease(
self.db,
lease_id=b.lease_id,
requester_session_id="owner",
proof=ll.AbandonProof(
dead_process=True,
missing_worktree=True,
no_live_mutation_risk=True,
no_open_pr=True,
),
)
self.assertIn("abandon_proof", abandoned)
self.assertEqual(abandoned["abandon_proof"]["dead_process"], True)
def test_allocator_assignment_lease_compatible(self) -> None:
"""Allocator assign+lease still works and is listable/inspectable."""
cands = [
WorkCandidate(
kind="issue",
number=601,
labels=("status:ready",),
title="leases",
priority=20,
)
]
res = allocate_next_work(
self.db,
session_id="alloc-s",
role="author",
remote="prgs",
org="Scaled-Tech-Consulting",
repo="Gitea-Tools",
candidates=cands,
apply=True,
profile_name="prgs-author",
username="jcwalker3",
)
self.assertEqual(res["outcome"], "assigned_work")
lid = res["assignment"]["lease_id"]
listed = ll.list_active_leases(self.db, remote="prgs")
ids = [x["lease_id"] for x in listed["leases"]]
self.assertIn(lid, ids)
insp = ll.inspect_lease(self.db, lid, caller_session_id="alloc-s")
self.assertTrue(insp["found"])
self.assertIn(
insp["safe_next_action"],
(ll.SAFE_OWNER_RESUME, ll.SAFE_ABANDON_ALLOWED),
)
def test_reviewer_merger_lease_handoff_still_works(self) -> None:
"""#536 merger adoption path is independent and must not regress."""
rpl.clear_session_lease()
body = mla.format_adoption_body(
repo="Scaled-Tech-Consulting/Gitea-Tools",
pr_number=999,
issue_number=601,
adopter_identity="sysadmin",
adopter_profile="prgs-merger",
adopter_session_id="merger-1",
worktree="branches/merge-pr999",
candidate_head="a" * 40,
target_branch="master",
target_branch_sha="b" * 40,
adopted_from_session_id="reviewer-1",
adopted_from_profile="prgs-reviewer",
adopted_from_reviewer_identity="sysadmin",
adopted_from_comment_id=42,
)
self.assertIn(mla.ADOPTION_MARKER, body)
self.assertIn("adopted_from_session_id: reviewer-1", body)
self.assertIn("adopted_by_profile: prgs-merger", body)
prov = mla.build_lease_provenance(
source=mla.SOURCE_ADOPT,
comment_id=42,
adopted_from_session_id="reviewer-1",
adoption_reason=mla.DEFAULT_ADOPTION_REASON,
)
rpl.record_session_lease(
{
"pr_number": 999,
"session_id": "merger-1",
"comment_id": 42,
},
lease_provenance=prov,
)
proof = mla.describe_session_lease_proof(rpl.get_session_lease())
self.assertTrue(proof["lease_proof_sanctioned"])
self.assertEqual(proof["lease_proof_source"], mla.SOURCE_ADOPT)
rpl.clear_session_lease()
def test_missing_worktree_and_dead_pid_handled_safely(self) -> None:
a = self._assign(owner_pid=1, worktree_path="/definitely/missing/wt-601")
with mock.patch.object(ll, "is_process_alive", return_value=False):
fr = ll.classify_lease_freshness(
self.db.get_lease_workflow_state(a.lease_id)["lease"]
)
self.assertIn(fr["freshness"], ("stale_dead_process", "stale_missing_worktree"))
# Insufficient abandon proof fails closed
with self.assertRaises(ll.LeaseLifecycleError):
ll.abandon_lease(
self.db,
lease_id=a.lease_id,
requester_session_id="other",
proof=ll.AbandonProof(dead_process=True), # missing no_live_mutation_risk
)
def test_file_lock_or_comment_not_authoritative_alone(self) -> None:
report = ll.non_db_lease_authority_report(
file_lock_present=True,
comment_lease_present=False,
db_lease_present=False,
)
self.assertEqual(report["safe_next_action"], ll.SAFE_NO_AUTHORITY)
self.assertTrue(report["file_lock_only"])
self.assertIsNone(report["authoritative_source"])
report2 = ll.non_db_lease_authority_report(
file_lock_present=True,
comment_lease_present=True,
db_lease_present=True,
)
self.assertEqual(report2["authoritative_source"], "control_plane_db")
self.assertFalse(report2["file_lock_only"])
self.assertFalse(report2["comment_lease_only"])
def test_abandon_proof_is_sufficient_rules(self) -> None:
self.assertFalse(ll.AbandonProof(dead_process=True).is_sufficient())
self.assertTrue(
ll.AbandonProof(
dead_process=True,
missing_worktree=True,
no_live_mutation_risk=True,
no_open_pr=True,
).is_sufficient()
)
self.assertTrue(
ll.AbandonProof(
dead_process=True,
no_live_mutation_risk=True,
same_owner=True,
).is_sufficient()
)
self.assertTrue(
ll.AbandonProof(
missing_worktree=True,
no_live_mutation_risk=True,
operator_authorized=True,
).is_sufficient()
)
class IssueLockExpiredReclaimTest(unittest.TestCase):
def test_expired_lock_reclaim_allowed_when_dead_and_missing_wt(self) -> None:
import issue_lock_store as ils
lock = {
"issue_number": 601,
"branch_name": "feat/issue-601-x",
"worktree_path": "/no/such/worktree-601",
"session_pid": 1,
"work_lease": {
"operation_type": "author_issue_work",
"expires_at": "2000-01-01T00:00:00Z",
},
}
with mock.patch.object(ils, "is_process_alive", return_value=False):
res = ils.assess_expired_lock_reclaim(lock)
self.assertTrue(res["reclaim_allowed"])
# Conflict assessor allows takeover
block = ils.assess_same_issue_lease_conflict(
lock,
issue_number=601,
branch_name="feat/issue-601-first-class-leases",
worktree_path="/tmp/new",
)
self.assertIsNone(block)
def test_live_lock_reclaim_refused(self) -> None:
import issue_lock_store as ils
from datetime import datetime, timedelta, timezone
future = (datetime.now(timezone.utc) + timedelta(hours=2)).strftime(
"%Y-%m-%dT%H:%M:%SZ"
)
lock = {
"issue_number": 601,
"branch_name": "feat/issue-601-x",
"worktree_path": tempfile.gettempdir(),
"session_pid": os.getpid(),
"work_lease": {
"operation_type": "author_issue_work",
"expires_at": future,
"last_heartbeat_at": future,
},
}
res = ils.assess_expired_lock_reclaim(lock)
self.assertFalse(res["reclaim_allowed"])
if __name__ == "__main__":
unittest.main()
+19 -3
View File
@@ -5,6 +5,7 @@ the MCP protocol) with mocked API responses.
""" """
import json import json
import os import os
import shutil
import sys import sys
import tempfile import tempfile
import unittest import unittest
@@ -3811,7 +3812,15 @@ class TestIssueLocking(unittest.TestCase):
@patch("mcp_server.api_get_all", return_value=[]) @patch("mcp_server.api_get_all", return_value=[])
@patch("mcp_server.get_auth_header", return_value=FAKE_AUTH) @patch("mcp_server.get_auth_header", return_value=FAKE_AUTH)
def test_lock_issue_blocks_expired_same_operation_lease_for_recovery(self, _auth, _api, _git_state): def test_lock_issue_blocks_expired_same_operation_lease_for_recovery(self, _auth, _api, _git_state):
"""#601: expired lease still blocks takeover when owner pid is live AND worktree exists.
Dead-pid / missing-worktree reclaim is covered by issue_lock_store unit tests.
This MCP path must remain fail-closed for sticky expired foreign ownership.
"""
prgs_repo = mcp_server.REMOTES["prgs"]["repo"] prgs_repo = mcp_server.REMOTES["prgs"]["repo"]
# Present worktree + live pid => reclaim_allowed=False even though expires_at is past.
sticky_worktree = tempfile.mkdtemp(prefix="gitea-sticky-lease-")
self.addCleanup(lambda: shutil.rmtree(sticky_worktree, ignore_errors=True))
issue_lock_store.save_lock_file( issue_lock_store.save_lock_file(
issue_lock_store.lock_file_path( issue_lock_store.lock_file_path(
remote="prgs", remote="prgs",
@@ -3825,15 +3834,22 @@ class TestIssueLocking(unittest.TestCase):
"remote": "prgs", "remote": "prgs",
"org": "Scaled-Tech-Consulting", "org": "Scaled-Tech-Consulting",
"repo": prgs_repo, "repo": prgs_repo,
"worktree_path": "/tmp/other-worktree", "worktree_path": sticky_worktree,
"session_pid": os.getpid(),
"pid": os.getpid(),
"work_lease": { "work_lease": {
"operation_type": "author_issue_work", "operation_type": "author_issue_work",
"expires_at": "2000-01-01T00:00:00Z", "expires_at": "2000-01-01T00:00:00Z",
}, },
}, },
) )
with self.assertRaises(RuntimeError) as ctx: with patch.object(issue_lock_store, "is_process_alive", return_value=True):
gitea_lock_issue(issue_number=196, branch_name="feat/issue-196-mutations", remote="prgs") with self.assertRaises(RuntimeError) as ctx:
gitea_lock_issue(
issue_number=196,
branch_name="feat/issue-196-mutations",
remote="prgs",
)
self.assertIn("Recovery review is required before takeover", str(ctx.exception)) self.assertIn("Recovery review is required before takeover", str(ctx.exception))
@patch("mcp_server.api_get_all", return_value=[]) @patch("mcp_server.api_get_all", return_value=[])