Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e8bae606cb | ||
|
|
956fa15fe3 | ||
|
|
fa510dd28d | ||
|
|
1dd30ecb15 | ||
|
|
8eada1fbe4 |
+226
-7
@@ -27,12 +27,12 @@ import uuid
|
|||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any, Iterator, Sequence
|
from typing import Any, Iterator, Mapping, Sequence
|
||||||
|
|
||||||
import dependency_graph
|
import dependency_graph
|
||||||
import gitea_audit
|
import gitea_audit
|
||||||
|
|
||||||
SCHEMA_VERSION = 5
|
SCHEMA_VERSION = 6
|
||||||
|
|
||||||
# Assignable work kinds only — raw monitoring incidents are never work items.
|
# Assignable work kinds only — raw monitoring incidents are never work items.
|
||||||
WORK_KINDS = frozenset({"issue", "pr"})
|
WORK_KINDS = frozenset({"issue", "pr"})
|
||||||
@@ -419,6 +419,7 @@ class ControlPlaneDB:
|
|||||||
self._migrate_incident_links_null_scope(conn)
|
self._migrate_incident_links_null_scope(conn)
|
||||||
self._migrate_lease_lifecycle_columns(conn)
|
self._migrate_lease_lifecycle_columns(conn)
|
||||||
self._migrate_session_ownership_columns(conn)
|
self._migrate_session_ownership_columns(conn)
|
||||||
|
self._migrate_session_lifecycle_columns(conn)
|
||||||
self._migrate_usage_events_table(conn)
|
self._migrate_usage_events_table(conn)
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)",
|
"INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)",
|
||||||
@@ -812,6 +813,7 @@ class ControlPlaneDB:
|
|||||||
pid: int | None = None,
|
pid: int | None = None,
|
||||||
status: str = "active",
|
status: str = "active",
|
||||||
controller_instance_id: str | None = None,
|
controller_instance_id: str | None = None,
|
||||||
|
owner_process_started_at: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Register/refresh a session row.
|
"""Register/refresh a session row.
|
||||||
|
|
||||||
@@ -821,16 +823,22 @@ class ControlPlaneDB:
|
|||||||
controller instance can. It is never overwritten with ``None``, so a
|
controller instance can. It is never overwritten with ``None``, so a
|
||||||
heartbeat from a caller that does not supply one cannot erase
|
heartbeat from a caller that does not supply one cannot erase
|
||||||
ownership.
|
ownership.
|
||||||
|
|
||||||
|
*owner_process_started_at* (#969) is the OS start time of the owner
|
||||||
|
process at registration time. When present it is retained across
|
||||||
|
heartbeats (never cleared by ``None``) so later PID-reuse checks do
|
||||||
|
not depend on a live ``ps`` probe of a long-dead process.
|
||||||
"""
|
"""
|
||||||
now = _ts()
|
now = _ts()
|
||||||
instance = (controller_instance_id or "").strip() or None
|
instance = (controller_instance_id or "").strip() or None
|
||||||
|
proc_start = (owner_process_started_at or "").strip() or None
|
||||||
with self._tx() as conn:
|
with self._tx() as conn:
|
||||||
existing = conn.execute(
|
existing = conn.execute(
|
||||||
"SELECT session_id FROM sessions WHERE session_id = ?",
|
"SELECT session_id FROM sessions WHERE session_id = ?",
|
||||||
(session_id,),
|
(session_id,),
|
||||||
).fetchone()
|
).fetchone()
|
||||||
if existing:
|
if existing:
|
||||||
if instance is None:
|
if instance is None and proc_start is None:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""
|
"""
|
||||||
UPDATE sessions
|
UPDATE sessions
|
||||||
@@ -840,7 +848,21 @@ class ControlPlaneDB:
|
|||||||
""",
|
""",
|
||||||
(role, profile, namespace, pid, now, status, session_id),
|
(role, profile, namespace, pid, now, status, session_id),
|
||||||
)
|
)
|
||||||
else:
|
elif instance is None:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE sessions
|
||||||
|
SET role = ?, profile = ?, namespace = ?, pid = ?,
|
||||||
|
last_heartbeat_at = ?, status = ?,
|
||||||
|
owner_process_started_at = COALESCE(?, owner_process_started_at)
|
||||||
|
WHERE session_id = ?
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
role, profile, namespace, pid, now, status,
|
||||||
|
proc_start, session_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
elif proc_start is None:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""
|
"""
|
||||||
UPDATE sessions
|
UPDATE sessions
|
||||||
@@ -854,18 +876,33 @@ class ControlPlaneDB:
|
|||||||
instance, session_id,
|
instance, session_id,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE sessions
|
||||||
|
SET role = ?, profile = ?, namespace = ?, pid = ?,
|
||||||
|
last_heartbeat_at = ?, status = ?,
|
||||||
|
controller_instance_id = ?,
|
||||||
|
owner_process_started_at = COALESCE(?, owner_process_started_at)
|
||||||
|
WHERE session_id = ?
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
role, profile, namespace, pid, now, status,
|
||||||
|
instance, proc_start, session_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"""
|
"""
|
||||||
INSERT INTO sessions(
|
INSERT INTO sessions(
|
||||||
session_id, role, profile, namespace, pid,
|
session_id, role, profile, namespace, pid,
|
||||||
started_at, last_heartbeat_at, status,
|
started_at, last_heartbeat_at, status,
|
||||||
controller_instance_id
|
controller_instance_id, owner_process_started_at
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
session_id, role, profile, namespace, pid, now, now,
|
session_id, role, profile, namespace, pid, now, now,
|
||||||
status, instance,
|
status, instance, proc_start,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
@@ -881,6 +918,165 @@ class ControlPlaneDB:
|
|||||||
(_ts(), session_id),
|
(_ts(), session_id),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def retire_session(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
session_id: str,
|
||||||
|
reason: str,
|
||||||
|
actor_session_id: str | None = None,
|
||||||
|
details: Mapping[str, Any] | None = None,
|
||||||
|
now: datetime | None = None,
|
||||||
|
terminal_status: str = "retired",
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Terminalize one session row if it is still non-terminal (#969).
|
||||||
|
|
||||||
|
CAS on non-terminal status: concurrent retirements of the same row
|
||||||
|
yield exactly one ``retired`` outcome and subsequent
|
||||||
|
``already_terminal`` outcomes. Never deletes historical rows. Writes a
|
||||||
|
durable ``session_retired`` event for audit.
|
||||||
|
"""
|
||||||
|
moment = _ts(now)
|
||||||
|
reason_s = (reason or "").strip() or "unspecified"
|
||||||
|
term = (terminal_status or "retired").strip().lower() or "retired"
|
||||||
|
terminal_set = {
|
||||||
|
"retired",
|
||||||
|
"ended",
|
||||||
|
"terminal",
|
||||||
|
"dead",
|
||||||
|
"stale",
|
||||||
|
"orphaned",
|
||||||
|
}
|
||||||
|
with self._tx() as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT * FROM sessions WHERE session_id = ?",
|
||||||
|
(session_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
return {
|
||||||
|
"outcome": "missing",
|
||||||
|
"reason": reason_s,
|
||||||
|
"prior_status": None,
|
||||||
|
"new_status": None,
|
||||||
|
"details": {"session_id": session_id},
|
||||||
|
}
|
||||||
|
prior = dict(row)
|
||||||
|
prior_status = str(prior.get("status") or "").strip().lower()
|
||||||
|
if prior_status in terminal_set:
|
||||||
|
return {
|
||||||
|
"outcome": "already_terminal",
|
||||||
|
"reason": reason_s,
|
||||||
|
"prior_status": prior_status,
|
||||||
|
"new_status": prior_status,
|
||||||
|
"details": {"session_id": session_id, "idempotent": True},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Optional: refuse when an active lease still names this session.
|
||||||
|
active_lease = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT lease_id, status, expires_at FROM leases
|
||||||
|
WHERE session_id = ? AND status = 'active'
|
||||||
|
LIMIT 1
|
||||||
|
""",
|
||||||
|
(session_id,),
|
||||||
|
).fetchone()
|
||||||
|
if active_lease is not None:
|
||||||
|
return {
|
||||||
|
"outcome": "blocked",
|
||||||
|
"reason": "live_lease",
|
||||||
|
"prior_status": prior_status,
|
||||||
|
"new_status": prior_status,
|
||||||
|
"details": {
|
||||||
|
"session_id": session_id,
|
||||||
|
"lease_id": active_lease["lease_id"],
|
||||||
|
"blocker": "active_lease_row",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cols = {
|
||||||
|
r[1] for r in conn.execute("PRAGMA table_info(sessions)").fetchall()
|
||||||
|
}
|
||||||
|
if "retired_at" in cols and "retire_reason" in cols:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE sessions
|
||||||
|
SET status = ?, retired_at = ?, retire_reason = ?
|
||||||
|
WHERE session_id = ? AND status = ?
|
||||||
|
""",
|
||||||
|
(term, moment, reason_s, session_id, prior.get("status")),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE sessions
|
||||||
|
SET status = ?
|
||||||
|
WHERE session_id = ? AND status = ?
|
||||||
|
""",
|
||||||
|
(term, session_id, prior.get("status")),
|
||||||
|
)
|
||||||
|
changed = conn.execute(
|
||||||
|
"SELECT changes()"
|
||||||
|
).fetchone()[0]
|
||||||
|
if not changed:
|
||||||
|
# Lost CAS race — re-read.
|
||||||
|
refreshed = conn.execute(
|
||||||
|
"SELECT status FROM sessions WHERE session_id = ?",
|
||||||
|
(session_id,),
|
||||||
|
).fetchone()
|
||||||
|
cur = (
|
||||||
|
str(refreshed["status"]).strip().lower()
|
||||||
|
if refreshed is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"outcome": "already_terminal"
|
||||||
|
if cur in terminal_set
|
||||||
|
else "blocked",
|
||||||
|
"reason": reason_s,
|
||||||
|
"prior_status": prior_status,
|
||||||
|
"new_status": cur,
|
||||||
|
"details": {
|
||||||
|
"session_id": session_id,
|
||||||
|
"cas_lost": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
detail_payload: dict[str, Any] = {
|
||||||
|
"session_id": session_id,
|
||||||
|
"prior_status": prior_status,
|
||||||
|
"new_status": term,
|
||||||
|
"reason": reason_s,
|
||||||
|
"actor_session_id": actor_session_id,
|
||||||
|
"pid": prior.get("pid"),
|
||||||
|
"role": prior.get("role"),
|
||||||
|
"profile": prior.get("profile"),
|
||||||
|
}
|
||||||
|
if isinstance(details, Mapping):
|
||||||
|
for key, value in details.items():
|
||||||
|
if key not in detail_payload:
|
||||||
|
detail_payload[key] = value
|
||||||
|
message = (
|
||||||
|
f"session {session_id} retired reason={reason_s} "
|
||||||
|
f"prior_status={prior_status}"
|
||||||
|
)
|
||||||
|
# events.work_item_id is nullable; session retirement is not work-scoped.
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO events(work_item_id, event_type, message, created_at)
|
||||||
|
VALUES (NULL, 'session_retired', ?, ?)
|
||||||
|
""",
|
||||||
|
(message[:2000], moment),
|
||||||
|
)
|
||||||
|
# Also persist a structured JSON line in the message when short enough
|
||||||
|
# by appending a compact summary (full detail stays in return value /
|
||||||
|
# audit log; events.message is human-readable).
|
||||||
|
return {
|
||||||
|
"outcome": "retired",
|
||||||
|
"reason": reason_s,
|
||||||
|
"prior_status": prior_status,
|
||||||
|
"new_status": term,
|
||||||
|
"details": detail_payload,
|
||||||
|
}
|
||||||
|
|
||||||
def list_sessions(
|
def list_sessions(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -1639,6 +1835,29 @@ class ControlPlaneDB:
|
|||||||
if name not in cols:
|
if name not in cols:
|
||||||
conn.execute(f"ALTER TABLE sessions ADD COLUMN {name} {decl}")
|
conn.execute(f"ALTER TABLE sessions ADD COLUMN {name} {decl}")
|
||||||
|
|
||||||
|
_SESSION_LIFECYCLE_COLUMNS: tuple[tuple[str, str], ...] = (
|
||||||
|
("owner_process_started_at", "TEXT"),
|
||||||
|
("retired_at", "TEXT"),
|
||||||
|
("retire_reason", "TEXT"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _migrate_session_lifecycle_columns(self, conn: sqlite3.Connection) -> None:
|
||||||
|
"""Add session retirement / PID-reuse provenance columns (#969).
|
||||||
|
|
||||||
|
Additive and idempotent. Pre-existing rows migrate with NULL; retirement
|
||||||
|
fills ``retired_at`` / ``retire_reason``, and new upserts may record
|
||||||
|
``owner_process_started_at`` for stronger identity checks.
|
||||||
|
"""
|
||||||
|
cols = {
|
||||||
|
row[1]
|
||||||
|
for row in conn.execute("PRAGMA table_info(sessions)").fetchall()
|
||||||
|
}
|
||||||
|
if not cols:
|
||||||
|
return
|
||||||
|
for name, decl in self._SESSION_LIFECYCLE_COLUMNS:
|
||||||
|
if name not in cols:
|
||||||
|
conn.execute(f"ALTER TABLE sessions ADD COLUMN {name} {decl}")
|
||||||
|
|
||||||
def list_active_claims(
|
def list_active_claims(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
|
|||||||
@@ -19,7 +19,11 @@ The assessor classifies:
|
|||||||
|
|
||||||
- **service_health** — process healthy / parity mutation-safe
|
- **service_health** — process healthy / parity mutation-safe
|
||||||
- **clients** — connected client descriptors (optional inventory)
|
- **clients** — connected client descriptors (optional inventory)
|
||||||
- **sessions** — active session rows with dead owner pids are unresolved
|
- **sessions** — active session rows with dead or reused owner pids are
|
||||||
|
unresolved until retired via `#969` (`session_lifecycle` /
|
||||||
|
`apply_session_cleanup=true` on `gitea_reconcile_after_restart`, or
|
||||||
|
`gitea_retire_stale_workflow_sessions`). Live owners, live leases, and live
|
||||||
|
client-managed sessions are never retired.
|
||||||
- **checkpoints** — soft-depends on #660; skipped with reason when schema absent
|
- **checkpoints** — soft-depends on #660; skipped with reason when schema absent
|
||||||
- **leases** — live control-plane leases after restart
|
- **leases** — live control-plane leases after restart
|
||||||
- **capabilities** — master-parity / stale-runtime (#610)
|
- **capabilities** — master-parity / stale-runtime (#610)
|
||||||
|
|||||||
@@ -8,10 +8,10 @@
|
|||||||
"document. #930's inventory had no such guard and its gitea_mcp_server.py",
|
"document. #930's inventory had no such guard and its gitea_mcp_server.py",
|
||||||
"anchors drifted between 7bf4f125 and aad5c8b4."
|
"anchors drifted between 7bf4f125 and aad5c8b4."
|
||||||
],
|
],
|
||||||
"generated_against_commit": "ca5f078d8a575ea3e2991771f8b4ea85e3dcaaa0",
|
"generated_against_commit": "1dd30ecb1508b559868c2d5d94367bc055d5138e",
|
||||||
"anchors": [
|
"anchors": [
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:24864",
|
"anchor": "gitea_mcp_server.py:25087",
|
||||||
"expect": "mcp_daemon_guard.bind_native_mcp_transport()"
|
"expect": "mcp_daemon_guard.bind_native_mcp_transport()"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -27,11 +27,11 @@
|
|||||||
"expect": "def assess_transport_for_auth_mint"
|
"expect": "def assess_transport_for_auth_mint"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:9191",
|
"anchor": "gitea_mcp_server.py:9192",
|
||||||
"expect": "assess_transport_for_auth_mint()"
|
"expect": "assess_transport_for_auth_mint()"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:9440",
|
"anchor": "gitea_mcp_server.py:9441",
|
||||||
"expect": "assess_transport_for_auth_mint()"
|
"expect": "assess_transport_for_auth_mint()"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -39,31 +39,31 @@
|
|||||||
"expect": "The transport is selected by deployment configuration"
|
"expect": "The transport is selected by deployment configuration"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:15481",
|
"anchor": "gitea_mcp_server.py:15630",
|
||||||
"expect": "def _is_client_managed_process"
|
"expect": "def _is_client_managed_process"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:15511",
|
"anchor": "gitea_mcp_server.py:15644",
|
||||||
"expect": "def _provenance_mutation_block"
|
"expect": "def _provenance_mutation_block"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:15519",
|
"anchor": "gitea_mcp_server.py:15652",
|
||||||
"expect": "unsupported_manual_launch"
|
"expect": "unsupported_manual_launch"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:19070",
|
"anchor": "gitea_mcp_server.py:19217",
|
||||||
"expect": "server_provenance"
|
"expect": "server_provenance"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:21581",
|
"anchor": "gitea_mcp_server.py:21741",
|
||||||
"expect": "def _check_mcp_runtimes_diagnostics"
|
"expect": "def _check_mcp_runtimes_diagnostics"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:21601",
|
"anchor": "gitea_mcp_server.py:21761",
|
||||||
"expect": "\"ps\", \"-o\", \"pid,lstart,command\""
|
"expect": "\"ps\", \"-o\", \"pid,lstart,command\""
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:21645",
|
"anchor": "gitea_mcp_server.py:21805",
|
||||||
"expect": "\"ps\", \"eww\""
|
"expect": "\"ps\", \"eww\""
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -107,19 +107,19 @@
|
|||||||
"expect": "def assert_keychain_access_allowed"
|
"expect": "def assert_keychain_access_allowed"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:19327",
|
"anchor": "gitea_mcp_server.py:19487",
|
||||||
"expect": "def gitea_list_profiles"
|
"expect": "def gitea_list_profiles"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:19378",
|
"anchor": "gitea_mcp_server.py:19538",
|
||||||
"expect": "gitea_config.resolve_token(p)"
|
"expect": "gitea_config.resolve_token(p)"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:19691",
|
"anchor": "gitea_mcp_server.py:19851",
|
||||||
"expect": "def gitea_audit_config"
|
"expect": "def gitea_audit_config"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:19713",
|
"anchor": "gitea_mcp_server.py:19873",
|
||||||
"expect": "service_summaries(config)"
|
"expect": "service_summaries(config)"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -135,19 +135,19 @@
|
|||||||
"expect": "_keychain_token(auth.get(\"id\"))"
|
"expect": "_keychain_token(auth.get(\"id\"))"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:17776",
|
"anchor": "gitea_mcp_server.py:17909",
|
||||||
"expect": "\"jenkins-mcp\""
|
"expect": "\"jenkins-mcp\""
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:17782",
|
"anchor": "gitea_mcp_server.py:17915",
|
||||||
"expect": "external-mcp"
|
"expect": "external-mcp"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:17803",
|
"anchor": "gitea_mcp_server.py:17936",
|
||||||
"expect": "\"glitchtip-mcp\""
|
"expect": "\"glitchtip-mcp\""
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:17808",
|
"anchor": "gitea_mcp_server.py:17941",
|
||||||
"expect": "external-mcp"
|
"expect": "external-mcp"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -183,7 +183,7 @@
|
|||||||
"expect": "mutation_safe"
|
"expect": "mutation_safe"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:19171",
|
"anchor": "gitea_mcp_server.py:19331",
|
||||||
"expect": "def gitea_assess_master_parity"
|
"expect": "def gitea_assess_master_parity"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -195,11 +195,11 @@
|
|||||||
"expect": "AUTHOR_WORKTREE_ENV"
|
"expect": "AUTHOR_WORKTREE_ENV"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:2351",
|
"anchor": "gitea_mcp_server.py:2352",
|
||||||
"expect": "/tmp/gitea_issue_lock.json"
|
"expect": "/tmp/gitea_issue_lock.json"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:10956",
|
"anchor": "gitea_mcp_server.py:10957",
|
||||||
"expect": "def gitea_bootstrap_author_issue_worktree"
|
"expect": "def gitea_bootstrap_author_issue_worktree"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -239,7 +239,7 @@
|
|||||||
"expect": "os.getpid()"
|
"expect": "os.getpid()"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"anchor": "gitea_mcp_server.py:12870",
|
"anchor": "gitea_mcp_server.py:12871",
|
||||||
"expect": "owner_pid_alive"
|
"expect": "owner_pid_alive"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ What the adversary is, what each boundary protects, and which services may share
|
|||||||
- **Issue:** #956 (Remote-MCP threat model), child of epic #929, cross-linked to #955.
|
- **Issue:** #956 (Remote-MCP threat model), child of epic #929, cross-linked to #955.
|
||||||
- **Depends on:** #930 (closed) — `docs/remote-mcp/coupling-inventory.md`.
|
- **Depends on:** #930 (closed) — `docs/remote-mcp/coupling-inventory.md`.
|
||||||
- **Blocks:** #932, #933, #934, #938.
|
- **Blocks:** #932, #933, #934, #938.
|
||||||
- **Generated against commit:** `ca5f078d8a575ea3e2991771f8b4ea85e3dcaaa0` (#708's
|
- **Generated against commit:** `1dd30ecb1508b559868c2d5d94367bc055d5138e` (#708's
|
||||||
namespace-attachment gate). Originally generated against
|
namespace-attachment gate). Originally generated against
|
||||||
`aad5c8b42361d380a8eeb07b94b90815e594c2c5` (`master`), re-anchored at
|
`aad5c8b42361d380a8eeb07b94b90815e594c2c5` (`master`), re-anchored at
|
||||||
`a143cd065ba06e1a2bdc5143a19ec156e53650ef` when #931's transport bind seam shifted the
|
`a143cd065ba06e1a2bdc5143a19ec156e53650ef` when #931's transport bind seam shifted the
|
||||||
@@ -31,7 +31,7 @@ document cites an anchor the fixture does not cover.
|
|||||||
|
|
||||||
This guard exists because #930 did not have one. Its inventory was generated at
|
This guard exists because #930 did not have one. Its inventory was generated at
|
||||||
`7bf4f125`; by `aad5c8b4` its `gitea_mcp_server.py` anchors had drifted — the transport
|
`7bf4f125`; by `aad5c8b4` its `gitea_mcp_server.py` anchors had drifted — the transport
|
||||||
bind it cited at line 23750 now lives at `gitea_mcp_server.py:24864`, and its
|
bind it cited at line 23750 now lives at `gitea_mcp_server.py:25087`, and its
|
||||||
client-managed provenance anchor at 14588 now lands in an unrelated function. Nothing
|
client-managed provenance anchor at 14588 now lands in an unrelated function. Nothing
|
||||||
failed, because nothing checked. Anchors into a ~24,700-line module rot silently, and a
|
failed, because nothing checked. Anchors into a ~24,700-line module rot silently, and a
|
||||||
security document that cannot prove its own citations is worse than none, because it is
|
security document that cannot prove its own citations is worse than none, because it is
|
||||||
@@ -75,19 +75,19 @@ authenticate the *caller*, not the *intent*.
|
|||||||
## 3. Trust boundaries
|
## 3. Trust boundaries
|
||||||
|
|
||||||
"Crossing requires today" is what the code actually enforces at
|
"Crossing requires today" is what the code actually enforces at
|
||||||
`ca5f078d8a575ea3e2991771f8b4ea85e3dcaaa0`, not what the design intends.
|
`1dd30ecb1508b559868c2d5d94367bc055d5138e`, not what the design intends.
|
||||||
|
|
||||||
| ID | Boundary | Protects | Crossing requires today | Crossing must require remotely |
|
| ID | Boundary | Protects | Crossing requires today | Crossing must require remotely |
|
||||||
| -- | -------- | -------- | ----------------------- | ------------------------------ |
|
| -- | -------- | -------- | ----------------------- | ------------------------------ |
|
||||||
| B1 | LLM client ↔ MCP server session | A1, A3, A10 — that a mutating session was established through the sanctioned client path | A single configured bind (`gitea_mcp_server.py:24864`) validated against one closed allowlist (`mcp_daemon_guard.py:49`, `mcp_daemon_guard.py:195`) — since #931 the identifier comes from deployment configuration and defaults to the local transport, so the boundary no longer rests on a literal, but it still rests on the *bind* rather than on an authenticated caller; client-managed provenance (`gitea_mcp_server.py:15481`) or a refusal (`gitea_mcp_server.py:15519`); production transport before recovery-authorization mint (`irrecoverable_provenance.py:497`, consumed at `gitea_mcp_server.py:9191` and `gitea_mcp_server.py:9440`) | An authenticated handshake issuing a server-side session identity bound to a principal, with the transport recorded in provenance. The physical proof (a pipe) must become a cryptographic one. |
|
| B1 | LLM client ↔ MCP server session | A1, A3, A10 — that a mutating session was established through the sanctioned client path | A single configured bind (`gitea_mcp_server.py:25087`) validated against one closed allowlist (`mcp_daemon_guard.py:49`, `mcp_daemon_guard.py:195`) — since #931 the identifier comes from deployment configuration and defaults to the local transport, so the boundary no longer rests on a literal, but it still rests on the *bind* rather than on an authenticated caller; client-managed provenance (`gitea_mcp_server.py:15630`) or a refusal (`gitea_mcp_server.py:15652`); production transport before recovery-authorization mint (`irrecoverable_provenance.py:497`, consumed at `gitea_mcp_server.py:9192` and `gitea_mcp_server.py:9441`) | An authenticated handshake issuing a server-side session identity bound to a principal, with the transport recorded in provenance. The physical proof (a pipe) must become a cryptographic one. |
|
||||||
| B2 | Role ↔ role | A9 — that author, reviewer, merger, and reconciler are distinct authorities | **The process boundary only.** The role is a property of the process, read once from `GITEA_MCP_PROFILE` (`gitea_config.py:54`). A caller gets author permissions by connecting to the author process. Review and merge are the operations singled out for extra care (`gitea_config.py:97`) | A per-request principal, so the role follows from the credential presented and cannot be selected by reaching a different endpoint. |
|
| B2 | Role ↔ role | A9 — that author, reviewer, merger, and reconciler are distinct authorities | **The process boundary only.** The role is a property of the process, read once from `GITEA_MCP_PROFILE` (`gitea_config.py:54`). A caller gets author permissions by connecting to the author process. Review and merge are the operations singled out for extra care (`gitea_config.py:97`) | A per-request principal, so the role follows from the credential presented and cannot be selected by reaching a different endpoint. |
|
||||||
| B3 | MCP server ↔ credential store | A3, A8 — that only sanctioned code turns a profile into a token | `_keychain_token` shelling out to the login keychain (`gitea_config.py:956`), dispatched by `resolve_token` (`gitea_config.py:974`) with the reference type built at `gitea_config.py:1015`, gated by `assert_keychain_access_allowed` (`mcp_daemon_guard.py:583`). Inline secrets are rejected at config load (`gitea_config.py:294`) | A credential provider keyed by the *request* principal, returning only that principal's credential, with the source recorded and the value never returned. |
|
| B3 | MCP server ↔ credential store | A3, A8 — that only sanctioned code turns a profile into a token | `_keychain_token` shelling out to the login keychain (`gitea_config.py:956`), dispatched by `resolve_token` (`gitea_config.py:974`) with the reference type built at `gitea_config.py:1015`, gated by `assert_keychain_access_allowed` (`mcp_daemon_guard.py:583`). Inline secrets are rejected at config load (`gitea_config.py:294`) | A credential provider keyed by the *request* principal, returning only that principal's credential, with the source recorded and the value never returned. |
|
||||||
| B4 | MCP server ↔ Gitea | A1, A2 — that only authorized calls reach the forge | A bearer token over TLS. Server-side, nothing distinguishes one role's token from another beyond the account it belongs to | Unchanged at the forge; the endpoint in front of it must refuse unauthenticated and plaintext connections before tool dispatch. |
|
| B4 | MCP server ↔ Gitea | A1, A2 — that only authorized calls reach the forge | A bearer token over TLS. Server-side, nothing distinguishes one role's token from another beyond the account it belongs to | Unchanged at the forge; the endpoint in front of it must refuse unauthenticated and plaintext connections before tool dispatch. |
|
||||||
| B5 | MCP server ↔ caller's filesystem | A7 — that a tool acts on the *caller's* disk or refuses | Nothing. The server's disk *is* the caller's disk. Worktree bootstrap writes directly (`gitea_mcp_server.py:10956`); the active workspace is process-global (`gitea_mcp_server.py:193`, `gitea_mcp_server.py:194`) | An explicit per-tool classification, enforced at dispatch, refusing filesystem tools over a transport that cannot reach the caller's disk. A green verdict about the wrong disk is the failure to prevent. |
|
| B5 | MCP server ↔ caller's filesystem | A7 — that a tool acts on the *caller's* disk or refuses | Nothing. The server's disk *is* the caller's disk. Worktree bootstrap writes directly (`gitea_mcp_server.py:10957`); the active workspace is process-global (`gitea_mcp_server.py:193`, `gitea_mcp_server.py:194`) | An explicit per-tool classification, enforced at dispatch, refusing filesystem tools over a transport that cannot reach the caller's disk. A green verdict about the wrong disk is the failure to prevent. |
|
||||||
| B6 | MCP server ↔ coordination state | A6, A9 — mutual exclusion | Local files and a local SQLite database, with liveness judged from the local process table (`issue_lock_store.py:98`), keyed on paths under one user's home (`issue_lock_store.py:26`, `mcp_session_state.py:27`, `control_plane_db.py:47`) and on `os.getpid()` (`control_plane_db.py:1145`, `gitea_mcp_server.py:12870`). A legacy global slot still exists at `gitea_mcp_server.py:2351`, and the session-pointer file is named per PID (`issue_lock_store.py:83`) | One authority per ownership question, with liveness from session identity and expiry, and atomic acquire, renew, and release across hosts. |
|
| B6 | MCP server ↔ coordination state | A6, A9 — mutual exclusion | Local files and a local SQLite database, with liveness judged from the local process table (`issue_lock_store.py:98`), keyed on paths under one user's home (`issue_lock_store.py:26`, `mcp_session_state.py:27`, `control_plane_db.py:47`) and on `os.getpid()` (`control_plane_db.py:1145`, `gitea_mcp_server.py:12871`). A legacy global slot still exists at `gitea_mcp_server.py:2352`, and the session-pointer file is named per PID (`issue_lock_store.py:83`) | One authority per ownership question, with liveness from session identity and expiry, and atomic acquire, renew, and release across hosts. |
|
||||||
| B7 | Gitea integration ↔ unrelated integrations | A4, A5 — that a Gitea compromise is not a CI and observability compromise | **Nothing.** See §5. The Gitea server reads Jenkins and GlitchTip secrets (`gitea_config.py:851`, reached from `gitea_config.py:837`) and holds the Sentry token (`sentry_incident_bridge.py:190`) | A hard process boundary. This is the boundary #956 exists to create. |
|
| B7 | Gitea integration ↔ unrelated integrations | A4, A5 — that a Gitea compromise is not a CI and observability compromise | **Nothing.** See §5. The Gitea server reads Jenkins and GlitchTip secrets (`gitea_config.py:851`, reached from `gitea_config.py:837`) and holds the Sentry token (`sentry_incident_bridge.py:190`) | A hard process boundary. This is the boundary #956 exists to create. |
|
||||||
| B8 | Tenant ↔ tenant (`prgs` / `mdcps` / `local-lab`) | A2 — that one organization's compromise is not another's | Convention. One configuration declares all three contexts; `resolve_service` fails closed on a *disabled* context (`gitea_config.py:704`) but the credentials of enabled ones remain reachable in-process. A per-profile repository scope exists (`gitea_config.py:499`) | Separate deployments, or at minimum per-tenant credential scopes with no process able to resolve both. |
|
| B8 | Tenant ↔ tenant (`prgs` / `mdcps` / `local-lab`) | A2 — that one organization's compromise is not another's | Convention. One configuration declares all three contexts; `resolve_service` fails closed on a *disabled* context (`gitea_config.py:704`) but the credentials of enabled ones remain reachable in-process. A per-profile repository scope exists (`gitea_config.py:499`) | Separate deployments, or at minimum per-tenant credential scopes with no process able to resolve both. |
|
||||||
| B9 | Deployed code ↔ merged policy | A1, A10 — that the running server enforces the rules that were actually merged | Comparing this process's startup commit against this disk (`master_parity_gate.py:168`), conjoined into a single verdict (`master_parity_gate.py:255`) published by `gitea_mcp_server.py:19171` | Freshness defined against the deployed build identity, with an explicit fail-closed verdict when undeterminable. |
|
| B9 | Deployed code ↔ merged policy | A1, A10 — that the running server enforces the rules that were actually merged | Comparing this process's startup commit against this disk (`master_parity_gate.py:168`), conjoined into a single verdict (`master_parity_gate.py:255`) published by `gitea_mcp_server.py:19331` | Freshness defined against the deployed build identity, with an explicit fail-closed verdict when undeterminable. |
|
||||||
|
|
||||||
### What no boundary constrains
|
### What no boundary constrains
|
||||||
|
|
||||||
@@ -132,10 +132,10 @@ Two flows deserve attention because neither is obvious from the code:
|
|||||||
|
|
||||||
1. **The keychain flow fans out.** B3 is drawn once but resolves credentials for *every*
|
1. **The keychain flow fans out.** B3 is drawn once but resolves credentials for *every*
|
||||||
configured profile and service, not only the active one. `gitea_list_profiles`
|
configured profile and service, not only the active one. `gitea_list_profiles`
|
||||||
(`gitea_mcp_server.py:19327`) reports each profile's credential status by calling
|
(`gitea_mcp_server.py:19487`) reports each profile's credential status by calling
|
||||||
`resolve_token` on it (`gitea_mcp_server.py:19378`), and `gitea_audit_config`
|
`resolve_token` on it (`gitea_mcp_server.py:19538`), and `gitea_audit_config`
|
||||||
(`gitea_mcp_server.py:19691`) reports service credential status through
|
(`gitea_mcp_server.py:19851`) reports service credential status through
|
||||||
`service_summaries` (`gitea_mcp_server.py:19713`).
|
`service_summaries` (`gitea_mcp_server.py:19873`).
|
||||||
2. **The return path is a flow too.** Content read from Gitea travels back into the model
|
2. **The return path is a flow too.** Content read from Gitea travels back into the model
|
||||||
and is treated as instruction. This is the ADV2 edge, and it is the only edge in the
|
and is treated as instruction. This is the ADV2 edge, and it is the only edge in the
|
||||||
diagram with no authentication on it, because it is not a request.
|
diagram with no authentication on it, because it is not a request.
|
||||||
@@ -180,16 +180,16 @@ the credential, and an attacker holding the token does not call our tools.
|
|||||||
|
|
||||||
**Finding 3 — Any one role process can resolve every other role's credential.** This is not
|
**Finding 3 — Any one role process can resolve every other role's credential.** This is not
|
||||||
inferred; it is demonstrated by tool output. `gitea_list_profiles`
|
inferred; it is demonstrated by tool output. `gitea_list_profiles`
|
||||||
(`gitea_mcp_server.py:19327`) called from the **author** session reports
|
(`gitea_mcp_server.py:19487`) called from the **author** session reports
|
||||||
`identity_status: "credentials present"` for `prgs-merger`, `prgs-reviewer`,
|
`identity_status: "credentials present"` for `prgs-merger`, `prgs-reviewer`,
|
||||||
`prgs-reconciler`, and every `mdcps` profile, because it calls `resolve_token` on each one
|
`prgs-reconciler`, and every `mdcps` profile, because it calls `resolve_token` on each one
|
||||||
(`gitea_mcp_server.py:19378`). The author process does not merely *have access to* the
|
(`gitea_mcp_server.py:19538`). The author process does not merely *have access to* the
|
||||||
merger's credential — it reads it to answer a status query. B2 is not a credential boundary
|
merger's credential — it reads it to answer a status query. B2 is not a credential boundary
|
||||||
in either direction.
|
in either direction.
|
||||||
|
|
||||||
**Finding 4 — The Gitea server reads CI and observability secrets.** `gitea_audit_config`
|
**Finding 4 — The Gitea server reads CI and observability secrets.** `gitea_audit_config`
|
||||||
(`gitea_mcp_server.py:19691`) reports `MDCPS Jenkins: enabled, read-only, authenticated`.
|
(`gitea_mcp_server.py:19851`) reports `MDCPS Jenkins: enabled, read-only, authenticated`.
|
||||||
That word `authenticated` is produced by `service_summaries` (`gitea_mcp_server.py:19713`,
|
That word `authenticated` is produced by `service_summaries` (`gitea_mcp_server.py:19873`,
|
||||||
defined at `gitea_config.py:837`), whose default check calls `_keychain_token` on the
|
defined at `gitea_config.py:837`), whose default check calls `_keychain_token` on the
|
||||||
service's own keychain reference (`gitea_config.py:851`). Producing that one line requires
|
service's own keychain reference (`gitea_config.py:851`). Producing that one line requires
|
||||||
the Gitea MCP server to read the Jenkins secret and the GlitchTip secret out of the
|
the Gitea MCP server to read the Jenkins secret and the GlitchTip secret out of the
|
||||||
@@ -197,8 +197,8 @@ keychain. B7 does not exist.
|
|||||||
|
|
||||||
**Finding 5 — Jenkins and GlitchTip are already decomposed; the reach is residual.** Their
|
**Finding 5 — Jenkins and GlitchTip are already decomposed; the reach is residual.** Their
|
||||||
tools live in separately registered servers, marked `external-mcp`
|
tools live in separately registered servers, marked `external-mcp`
|
||||||
(`gitea_mcp_server.py:17776`, `gitea_mcp_server.py:17782`, `gitea_mcp_server.py:17803`,
|
(`gitea_mcp_server.py:17909`, `gitea_mcp_server.py:17915`, `gitea_mcp_server.py:17936`,
|
||||||
`gitea_mcp_server.py:17808`) with their own expected tool sets (`mcp_discoverability.py:9`,
|
`gitea_mcp_server.py:17941`) with their own expected tool sets (`mcp_discoverability.py:9`,
|
||||||
`mcp_discoverability.py:17`). The correct decomposition was already chosen. What remains is
|
`mcp_discoverability.py:17`). The correct decomposition was already chosen. What remains is
|
||||||
a leak across it: the credential *references* still live in the Gitea configuration and are
|
a leak across it: the credential *references* still live in the Gitea configuration and are
|
||||||
still resolved by the Gitea process. #75 bundled these services into one control-plane
|
still resolved by the Gitea process. #75 bundled these services into one control-plane
|
||||||
@@ -209,8 +209,8 @@ GlitchTip, the Sentry bridge runs *inside* the Gitea server, resolving its token
|
|||||||
process environment (`sentry_incident_bridge.py:190`) and sending it as a bearer header
|
process environment (`sentry_incident_bridge.py:190`) and sending it as a bearer header
|
||||||
(`sentry_incident_bridge.py:289`). Being an environment variable rather than a keychain item
|
(`sentry_incident_bridge.py:289`). Being an environment variable rather than a keychain item
|
||||||
makes it strictly worse: it needs no keychain prompt and is inherited by every subprocess the
|
makes it strictly worse: it needs no keychain prompt and is inherited by every subprocess the
|
||||||
server spawns — including the `ps` invocations at `gitea_mcp_server.py:21601` and
|
server spawns — including the `ps` invocations at `gitea_mcp_server.py:21761` and
|
||||||
`gitea_mcp_server.py:21645`, reached from `gitea_mcp_server.py:21581`.
|
`gitea_mcp_server.py:21805`, reached from `gitea_mcp_server.py:21741`.
|
||||||
|
|
||||||
**Finding 7 — The highest-value coordination asset has the weakest gate.** A6 is protected
|
**Finding 7 — The highest-value coordination asset has the weakest gate.** A6 is protected
|
||||||
by filesystem permissions alone (CR14). Corrupting a lease requires no Gitea credential,
|
by filesystem permissions alone (CR14). Corrupting a lease requires no Gitea credential,
|
||||||
@@ -219,8 +219,8 @@ assumes. Every other asset costs an attacker a credential; this one costs nothin
|
|||||||
local access, which is exactly ADV5's position.
|
local access, which is exactly ADV5's position.
|
||||||
|
|
||||||
**Finding 8 — Provenance authenticates the launch, not the caller.** `server_provenance` is
|
**Finding 8 — Provenance authenticates the launch, not the caller.** `server_provenance` is
|
||||||
reported as exactly `client_managed` or `manual_launch` (`gitea_mcp_server.py:19070`),
|
reported as exactly `client_managed` or `manual_launch` (`gitea_mcp_server.py:19217`),
|
||||||
derived from environment inspection (`gitea_mcp_server.py:15481`) with the recognized-key
|
derived from environment inspection (`gitea_mcp_server.py:15630`) with the recognized-key
|
||||||
allowlist at `gitea_config.py:1172` and the generator that emits the marker at
|
allowlist at `gitea_config.py:1172` and the generator that emits the marker at
|
||||||
`gitea_config.py:1233`. Every one of those facts is fixed at process start. A client that is
|
`gitea_config.py:1233`. Every one of those facts is fixed at process start. A client that is
|
||||||
trustworthy at launch and compromised a minute later remains `client_managed` for the life
|
trustworthy at launch and compromised a minute later remains `client_managed` for the life
|
||||||
@@ -259,7 +259,7 @@ holds the token and calls the API instead of the tool.
|
|||||||
|
|
||||||
**D3 — Credential resolution is scoped to the request principal.** A session must resolve its
|
**D3 — Credential resolution is scoped to the request principal.** A session must resolve its
|
||||||
own credential and must have no path to any other principal's. The resolve-every-profile
|
own credential and must have no path to any other principal's. The resolve-every-profile
|
||||||
behavior behind `gitea_mcp_server.py:19378` and `gitea_mcp_server.py:19713` must report
|
behavior behind `gitea_mcp_server.py:19538` and `gitea_mcp_server.py:19873` must report
|
||||||
configured-or-not from configuration alone, without resolving the secret.
|
configured-or-not from configuration alone, without resolving the secret.
|
||||||
|
|
||||||
*Rationale.* Finding 3. An audit surface that proves a credential exists by fetching it is a
|
*Rationale.* Finding 3. An audit surface that proves a credential exists by fetching it is a
|
||||||
@@ -336,11 +336,11 @@ The client is attached to the local fleet over stdio.
|
|||||||
|
|
||||||
| Boundary | What ADV1 reaches | Stopped by |
|
| Boundary | What ADV1 reaches | Stopped by |
|
||||||
| -------- | ----------------- | ---------- |
|
| -------- | ----------------- | ---------- |
|
||||||
| B1 | Everything the fleet serves. The client *is* the sanctioned launcher: it satisfies the client-managed check (`gitea_mcp_server.py:15481`) by construction, and provenance is never re-verified after launch (Finding 8). | Nothing. The guard authenticates the launch, not the caller. |
|
| B1 | Everything the fleet serves. The client *is* the sanctioned launcher: it satisfies the client-managed check (`gitea_mcp_server.py:15630`) by construction, and provenance is never re-verified after launch (Finding 8). | Nothing. The guard authenticates the launch, not the caller. |
|
||||||
| B2 | All five roles — it is attached to all five namespaces. It can author a PR, approve it from the reviewer namespace, and merge it from the merger namespace. | Only the in-process self-review check, which compares `jcwalker3` (author) against `sysadmin` (reviewer) and **passes**, because Finding 1 made them different accounts while leaving reviewer and merger identical. A9 falls in one sequence of legitimate calls. |
|
| B2 | All five roles — it is attached to all five namespaces. It can author a PR, approve it from the reviewer namespace, and merge it from the merger namespace. | Only the in-process self-review check, which compares `jcwalker3` (author) against `sysadmin` (reviewer) and **passes**, because Finding 1 made them different accounts while leaving reviewer and merger identical. A9 falls in one sequence of legitimate calls. |
|
||||||
| B3 | Every credential in CR1–CR10 via CR13, with no additional prompt — the daemon is already sanctioned, so `assert_keychain_access_allowed` (`mcp_daemon_guard.py:583`) returns immediately. | Nothing. |
|
| B3 | Every credential in CR1–CR10 via CR13, with no additional prompt — the daemon is already sanctioned, so `assert_keychain_access_allowed` (`mcp_daemon_guard.py:583`) returns immediately. | Nothing. |
|
||||||
| B4 | A1 and A2 in full. | Branch protection at the forge, to the extent configured. |
|
| B4 | A1 and A2 in full. | Branch protection at the forge, to the extent configured. |
|
||||||
| B5 | The operator's checkout and every worktree, through the author tools (`gitea_mcp_server.py:10956`), plus the shared stderr path at `mcp_server.py:13`. | Nothing; the server's disk is the target disk. |
|
| B5 | The operator's checkout and every worktree, through the author tools (`gitea_mcp_server.py:10957`), plus the shared stderr path at `mcp_server.py:13`. | Nothing; the server's disk is the target disk. |
|
||||||
| B6 | All coordination state — no credential required (CR14). It can forge lease ownership and clear decision locks. | Filesystem permissions, which it already satisfies. |
|
| B6 | All coordination state — no credential required (CR14). It can forge lease ownership and clear decision locks. | Filesystem permissions, which it already satisfies. |
|
||||||
| B7 | Jenkins (A4) and GlitchTip (A5) secrets via Finding 4, and CR11/CR12 from its own environment. | Nothing. |
|
| B7 | Jenkins (A4) and GlitchTip (A5) secrets via Finding 4, and CR11/CR12 from its own environment. | Nothing. |
|
||||||
| B8 | Both tenants. | Nothing in-process; only the disabled-context check (`gitea_config.py:704`), which does not apply to enabled contexts. |
|
| B8 | Both tenants. | Nothing in-process; only the disabled-context check (`gitea_config.py:704`), which does not apply to enabled contexts. |
|
||||||
|
|||||||
+400
-38
@@ -199,6 +199,7 @@ RECONCILER_WORKTREE_ENV = "GITEA_RECONCILER_WORKTREE"
|
|||||||
import namespace_workspace_binding as nwb # noqa: E402
|
import namespace_workspace_binding as nwb # noqa: E402
|
||||||
import canonical_repository_root as crr # noqa: E402 # #706 cross-repo canonical root
|
import canonical_repository_root as crr # noqa: E402 # #706 cross-repo canonical root
|
||||||
import mcp_namespace_health # noqa: E402
|
import mcp_namespace_health # noqa: E402
|
||||||
|
import mcp_worker_identity # noqa: E402 # #948 single client/session provenance authority
|
||||||
import stale_binding_recovery # noqa: E402
|
import stale_binding_recovery # noqa: E402
|
||||||
|
|
||||||
# Worktree env bindings inherited from the parent environment at daemon boot
|
# Worktree env bindings inherited from the parent environment at daemon boot
|
||||||
@@ -14916,7 +14917,7 @@ def _stale_runtime_reconnect_action() -> str:
|
|||||||
return (
|
return (
|
||||||
"blocker_kind=runtime_reconnect_required: call "
|
"blocker_kind=runtime_reconnect_required: call "
|
||||||
"gitea_request_mcp_reconnect(namespace=<active gitea-* namespace>, "
|
"gitea_request_mcp_reconnect(namespace=<active gitea-* namespace>, "
|
||||||
"reason='stale-runtime', client='codex') for a typed operator "
|
"reason='stale-runtime') for a typed operator "
|
||||||
"reconnect blocker with exact UI steps, then reconnect the IDE/client "
|
"reconnect blocker with exact UI steps, then reconnect the IDE/client "
|
||||||
"MCP session so the server reloads at the current master head. Do not "
|
"MCP session so the server reloads at the current master head. Do not "
|
||||||
"call gitea_activate_profile, pkill, touch configs, or switch MCP role "
|
"call gitea_activate_profile, pkill, touch configs, or switch MCP role "
|
||||||
@@ -15478,36 +15479,168 @@ def _session_context_mutation_block(
|
|||||||
return blocked
|
return blocked
|
||||||
|
|
||||||
|
|
||||||
def _is_client_managed_process() -> bool:
|
# --- #948 client/session runtime ownership -------------------------------
|
||||||
"""Check whether the current MCP server process has client-managed launch provenance (#686)."""
|
#
|
||||||
val = (
|
# One daemon launch is one *generation*. The session that owns it registers a
|
||||||
os.environ.get("GITEA_CLIENT_MANAGED")
|
# unique worker identity against it, so a second healthy client attaching its
|
||||||
or os.environ.get("GITEA_MCP_CLIENT_MANAGED")
|
# own generation is no longer indistinguishable from a duplicate process.
|
||||||
or os.environ.get("GITEA_SERVER_PROVENANCE")
|
# Everything here is process-local cache plus a local SQLite registry: no Gitea
|
||||||
or os.environ.get("GITEA_FORCE_CLIENT_MANAGED")
|
# call, no network, no config write, and every failure degrades to "unproven"
|
||||||
or ""
|
# rather than raising into a tool call.
|
||||||
).strip().lower()
|
|
||||||
|
|
||||||
if val in ("0", "false", "no", "manual", "manual_launch"):
|
_WORKER_REGISTRY = None
|
||||||
return False
|
_WORKER_IDENTITY: str | None = None
|
||||||
|
_WORKER_GENERATION: str | None = None
|
||||||
|
_WORKER_REGISTRATION_ATTEMPTED = False
|
||||||
|
|
||||||
if val in ("1", "true", "yes", "client_managed"):
|
#: Env a client launcher may set to name itself and its session. Absent values
|
||||||
return True
|
#: are reported as unknown; they are never guessed at, because guessing is what
|
||||||
|
#: produced Codex reconnect steps for a Gemini operator.
|
||||||
|
CLIENT_NAME_ENV = "GITEA_MCP_CLIENT"
|
||||||
|
CLIENT_INSTANCE_ENV = "GITEA_MCP_CLIENT_INSTANCE"
|
||||||
|
CLIENT_SESSION_ENV = "GITEA_MCP_CLIENT_SESSION"
|
||||||
|
|
||||||
# A terminal launch has an active TTY on stdin
|
|
||||||
|
def _worker_registry():
|
||||||
|
"""Local worker registry, or ``None`` when it cannot be opened.
|
||||||
|
|
||||||
|
Under pytest the registry is only opened when a test has pinned a path, so
|
||||||
|
a test run never writes into the operator's real registry.
|
||||||
|
"""
|
||||||
|
global _WORKER_REGISTRY
|
||||||
|
if _WORKER_REGISTRY is not None:
|
||||||
|
return _WORKER_REGISTRY
|
||||||
|
if mcp_daemon_guard.is_pytest_runtime() and not (
|
||||||
|
os.environ.get(mcp_worker_identity.REGISTRY_PATH_ENV) or ""
|
||||||
|
).strip():
|
||||||
|
return None
|
||||||
try:
|
try:
|
||||||
if sys.stdin and sys.stdin.isatty():
|
_WORKER_REGISTRY = mcp_worker_identity.WorkerRegistry()
|
||||||
return False
|
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
return None
|
||||||
|
return _WORKER_REGISTRY
|
||||||
|
|
||||||
# Standard client launch or test runner with stdio pipe and profile env
|
|
||||||
if "GITEA_MCP_CONFIG" in os.environ or "GITEA_MCP_PROFILE" in os.environ or "GITEA_PROFILE_NAME" in os.environ:
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
def _client_identity_hints() -> dict:
|
||||||
|
"""What the launcher told us about itself. Unset fields stay unset."""
|
||||||
|
return {
|
||||||
|
"client_name": (os.environ.get(CLIENT_NAME_ENV) or "").strip() or None,
|
||||||
|
"client_instance_id": (os.environ.get(CLIENT_INSTANCE_ENV) or "").strip()
|
||||||
|
or f"pid-{os.getpid()}",
|
||||||
|
"session_id": (os.environ.get(CLIENT_SESSION_ENV) or "").strip()
|
||||||
|
or f"proc-{os.getpid()}-{_process_boot_head_sha or 'nohead'}",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _active_worker_identity() -> str | None:
|
||||||
|
"""This runtime's worker identity, registering it once per process.
|
||||||
|
|
||||||
|
A collision does not adopt the existing registration: a fresh identity is
|
||||||
|
minted and registered instead, which is the #948 AC32 requirement and also
|
||||||
|
the only safe response — adopting would silently transfer another worker's
|
||||||
|
leases and fencing tokens.
|
||||||
|
"""
|
||||||
|
global _WORKER_IDENTITY, _WORKER_GENERATION, _WORKER_REGISTRATION_ATTEMPTED
|
||||||
|
if _WORKER_IDENTITY is not None:
|
||||||
|
return _WORKER_IDENTITY
|
||||||
|
if _WORKER_REGISTRATION_ATTEMPTED:
|
||||||
|
return None
|
||||||
|
_WORKER_REGISTRATION_ATTEMPTED = True
|
||||||
|
|
||||||
|
registry = _worker_registry()
|
||||||
|
if registry is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
hints = _client_identity_hints()
|
||||||
|
generation = _WORKER_GENERATION or mcp_worker_identity.new_generation_id(os.getpid())
|
||||||
|
native = mcp_daemon_guard.native_runtime_status()
|
||||||
|
|
||||||
|
try:
|
||||||
|
for _attempt in range(3):
|
||||||
|
identity = mcp_worker_identity.generate_worker_identity(
|
||||||
|
hints["client_name"], hints["session_id"]
|
||||||
|
)
|
||||||
|
outcome = registry.register(
|
||||||
|
worker_identity=identity,
|
||||||
|
client_name=hints["client_name"],
|
||||||
|
client_instance_id=hints["client_instance_id"],
|
||||||
|
session_id=hints["session_id"],
|
||||||
|
generation_id=generation,
|
||||||
|
role=_active_role_kind_safe(),
|
||||||
|
profile=(os.environ.get(gitea_config.ENV_PROFILE) or "").strip() or None,
|
||||||
|
remote=(os.environ.get("GITEA_MCP_REMOTE") or "").strip() or None,
|
||||||
|
repository_binding=PROJECT_ROOT,
|
||||||
|
pid=os.getpid(),
|
||||||
|
transport=native.get("bound_transport"),
|
||||||
|
token_fingerprint=native.get("token_fingerprint"),
|
||||||
|
pid_alive_probe=issue_lock_store.is_process_alive,
|
||||||
|
)
|
||||||
|
if outcome.get("registered"):
|
||||||
|
_WORKER_IDENTITY = identity
|
||||||
|
_WORKER_GENERATION = generation
|
||||||
|
return identity
|
||||||
|
if not outcome.get("collision"):
|
||||||
|
return None
|
||||||
|
# Collided: loop mints a different identity rather than reusing this
|
||||||
|
# one. The existing registration is left exactly as it was.
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _active_role_kind_safe() -> str | None:
|
||||||
|
"""Best-effort role for the registry record; never raises into a tool call.
|
||||||
|
|
||||||
|
The role is recorded for diagnostics only. It is deliberately not part of
|
||||||
|
the identity: role is a reusable capability definition that many live
|
||||||
|
workers may share (#948 AC37).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
profile = get_profile() or {}
|
||||||
|
return _role_kind(
|
||||||
|
profile.get("allowed_operations") or [],
|
||||||
|
profile.get("forbidden_operations") or [],
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _stdin_is_tty() -> bool:
|
||||||
|
"""Terminal evidence for launch provenance, tolerant of a detached stdin."""
|
||||||
|
try:
|
||||||
|
return bool(sys.stdin and sys.stdin.isatty())
|
||||||
|
except Exception:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _launch_provenance() -> dict:
|
||||||
|
"""This process's launch provenance, from the one shared authority (#948).
|
||||||
|
|
||||||
|
Previously each surface reimplemented this. ``gitea_get_runtime_context``
|
||||||
|
read the live environment while ``mcp_namespace_health`` read a filtered
|
||||||
|
summary that structurally could not see the provenance keys, so the two
|
||||||
|
reported different provenance for the same process. Both now call
|
||||||
|
``mcp_worker_identity``.
|
||||||
|
"""
|
||||||
|
return mcp_worker_identity.assess_launch_provenance(
|
||||||
|
dict(os.environ), stdin_is_tty=_stdin_is_tty()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_client_managed_process() -> bool:
|
||||||
|
"""Check whether the current MCP server process has client-managed launch provenance (#686).
|
||||||
|
|
||||||
|
#948: the decision logic moved to
|
||||||
|
``mcp_worker_identity.assess_launch_provenance`` unchanged — same env keys,
|
||||||
|
same precedence, same TTY fallback — so this keeps returning exactly what it
|
||||||
|
always returned while no longer being a second, divergent implementation.
|
||||||
|
Launch provenance answers "was this hand-launched from a terminal", which is
|
||||||
|
the #686 question. It does *not* answer which live session owns this
|
||||||
|
runtime; that is ``session_ownership`` and needs an attachment record.
|
||||||
|
"""
|
||||||
|
return bool(_launch_provenance()["client_managed"])
|
||||||
|
|
||||||
|
|
||||||
def _provenance_mutation_block(**extra_fields) -> dict | None:
|
def _provenance_mutation_block(**extra_fields) -> dict | None:
|
||||||
"""Refuse mutating tool calls on processes lacking client-managed launch provenance (#686)."""
|
"""Refuse mutating tool calls on processes lacking client-managed launch provenance (#686)."""
|
||||||
if _is_client_managed_process():
|
if _is_client_managed_process():
|
||||||
@@ -19048,7 +19181,21 @@ def gitea_get_runtime_context(
|
|||||||
source="gitea_get_runtime_context",
|
source="gitea_get_runtime_context",
|
||||||
)
|
)
|
||||||
|
|
||||||
is_client_managed = _is_client_managed_process()
|
# #948: one assessment, shared with mcp_namespace_health. Reporting both
|
||||||
|
# dimensions from a single call is what makes the two surfaces agree — the
|
||||||
|
# contradiction they used to produce came from two implementations, not from
|
||||||
|
# two genuinely different observations.
|
||||||
|
provenance_assessment = mcp_worker_identity.assess_provenance(
|
||||||
|
registry=_worker_registry(),
|
||||||
|
worker_identity=_active_worker_identity(),
|
||||||
|
env=dict(os.environ),
|
||||||
|
native_transport_bound=mcp_daemon_guard.bound_transport() is not None,
|
||||||
|
profile=profile.get("profile_name"),
|
||||||
|
role=_role_kind(allowed, forbidden),
|
||||||
|
stdin_is_tty=_stdin_is_tty(),
|
||||||
|
pid_alive_probe=issue_lock_store.is_process_alive,
|
||||||
|
)
|
||||||
|
is_client_managed = provenance_assessment["is_client_managed"]
|
||||||
unconsumed_env = gitea_config.get_unconsumed_gitea_env_overrides()
|
unconsumed_env = gitea_config.get_unconsumed_gitea_env_overrides()
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
@@ -19067,8 +19214,21 @@ def gitea_get_runtime_context(
|
|||||||
"review_merge_blocked_reasons": blocked_reasons,
|
"review_merge_blocked_reasons": blocked_reasons,
|
||||||
"suggested_fix": suggested_fix,
|
"suggested_fix": suggested_fix,
|
||||||
"safe_next_action": safe_next_action,
|
"safe_next_action": safe_next_action,
|
||||||
"server_provenance": "client_managed" if is_client_managed else "manual_launch",
|
"server_provenance": provenance_assessment["launch_provenance"],
|
||||||
"is_client_managed": is_client_managed,
|
"is_client_managed": is_client_managed,
|
||||||
|
# #948: the ownership dimension, which the environment cannot establish.
|
||||||
|
# A surface that needs "may this session mutate on behalf of its client"
|
||||||
|
# reads these, not server_provenance.
|
||||||
|
"session_ownership": provenance_assessment["session_ownership"],
|
||||||
|
"session_owned": provenance_assessment["session_owned"],
|
||||||
|
"worker_identity": provenance_assessment["worker_identity"],
|
||||||
|
"session_id": provenance_assessment["session_id"],
|
||||||
|
"client_instance_id": provenance_assessment["client_instance_id"],
|
||||||
|
"generation_id": provenance_assessment["generation_id"],
|
||||||
|
"client_name": provenance_assessment["client_name"],
|
||||||
|
"fencing_epoch": provenance_assessment["fencing_epoch"],
|
||||||
|
"conflicting_live_sessions": provenance_assessment["conflicting_live_sessions"],
|
||||||
|
"provenance_assessment": provenance_assessment,
|
||||||
"unconsumed_gitea_env": unconsumed_env,
|
"unconsumed_gitea_env": unconsumed_env,
|
||||||
"preflight_ready": preflight["preflight_ready"],
|
"preflight_ready": preflight["preflight_ready"],
|
||||||
"preflight_block_reasons": preflight["preflight_block_reasons"],
|
"preflight_block_reasons": preflight["preflight_block_reasons"],
|
||||||
@@ -21654,11 +21814,23 @@ def _check_mcp_runtimes_diagnostics(task: str, matching_profiles: list[str]) ->
|
|||||||
if match:
|
if match:
|
||||||
profile = match.group(1)
|
profile = match.group(1)
|
||||||
|
|
||||||
|
# #948: provenance for a scanned peer comes from the shared authority,
|
||||||
|
# fed by that peer's own environment, so the fleet scan cannot disagree
|
||||||
|
# with what that peer reports about itself.
|
||||||
|
peer_env = {
|
||||||
|
m.group(1): m.group(2)
|
||||||
|
for m in re.finditer(r'\b(GITEA_[A-Z0-9_]+)=([^\s]+)', env_out)
|
||||||
|
}
|
||||||
|
# declared_only: this is a peer process. Its stdin is not ours to
|
||||||
|
# inspect, and it inherits GITEA_MCP_PROFILE from any shell that
|
||||||
|
# exported it, so only an explicit declaration counts as evidence here.
|
||||||
is_client_managed = bool(
|
is_client_managed = bool(
|
||||||
re.search(r'\bGITEA_CLIENT_MANAGED=(1|true|yes|client_managed)\b', env_out, re.IGNORECASE)
|
mcp_worker_identity.assess_launch_provenance(
|
||||||
or re.search(r'\bGITEA_MCP_CLIENT_MANAGED=(1|true|yes|client_managed)\b', env_out, re.IGNORECASE)
|
peer_env, declared_only=True
|
||||||
or re.search(r'\bGITEA_SERVER_PROVENANCE=client_managed\b', env_out, re.IGNORECASE)
|
)["client_managed"]
|
||||||
)
|
)
|
||||||
|
peer_worker_identity = peer_env.get("GITEA_MCP_WORKER_IDENTITY")
|
||||||
|
peer_generation = peer_env.get("GITEA_MCP_GENERATION_ID")
|
||||||
|
|
||||||
for env_match in re.finditer(r'\b(GITEA_[A-Z0-9_]+)=([^\s]+)', env_out):
|
for env_match in re.finditer(r'\b(GITEA_[A-Z0-9_]+)=([^\s]+)', env_out):
|
||||||
k, v = env_match.group(1), env_match.group(2)
|
k, v = env_match.group(1), env_match.group(2)
|
||||||
@@ -21674,6 +21846,8 @@ def _check_mcp_runtimes_diagnostics(task: str, matching_profiles: list[str]) ->
|
|||||||
"start_time": start_time,
|
"start_time": start_time,
|
||||||
"is_stale": is_stale,
|
"is_stale": is_stale,
|
||||||
"is_client_managed": is_client_managed,
|
"is_client_managed": is_client_managed,
|
||||||
|
"worker_identity": peer_worker_identity,
|
||||||
|
"generation_id": peer_generation,
|
||||||
}
|
}
|
||||||
if profile not in all_profile_procs:
|
if profile not in all_profile_procs:
|
||||||
all_profile_procs[profile] = []
|
all_profile_procs[profile] = []
|
||||||
@@ -21681,12 +21855,44 @@ def _check_mcp_runtimes_diagnostics(task: str, matching_profiles: list[str]) ->
|
|||||||
|
|
||||||
running_profiles = {}
|
running_profiles = {}
|
||||||
for profile, procs in all_profile_procs.items():
|
for profile, procs in all_profile_procs.items():
|
||||||
if len(procs) > 1:
|
# #948 AC40/AC43: sharing a profile is legitimate — profile is a
|
||||||
|
# reusable capability definition, not a worker identity. What is *not*
|
||||||
|
# legitimate is reusing one worker identity, or two live sessions
|
||||||
|
# claiming one generation. Distinctness has to be proven, though:
|
||||||
|
# processes carrying no identity evidence are indistinguishable, so
|
||||||
|
# they stay classified as duplicates and keep the #686 wall intact.
|
||||||
|
identified = [p for p in procs if p.get("worker_identity")]
|
||||||
|
distinct_identities = {p["worker_identity"] for p in identified}
|
||||||
|
all_identified = len(identified) == len(procs)
|
||||||
|
duplicate_identity = len(identified) != len(distinct_identities)
|
||||||
|
contested_generation = any(
|
||||||
|
len({p["worker_identity"] for p in identified if p.get("generation_id") == gen}) > 1
|
||||||
|
for gen in {p.get("generation_id") for p in identified if p.get("generation_id")}
|
||||||
|
)
|
||||||
|
|
||||||
|
if len(procs) > 1 and (
|
||||||
|
not all_identified or duplicate_identity or contested_generation
|
||||||
|
):
|
||||||
pids_str = ", ".join(str(p["pid"]) for p in procs)
|
pids_str = ", ".join(str(p["pid"]) for p in procs)
|
||||||
|
if duplicate_identity or contested_generation:
|
||||||
|
detail = (
|
||||||
|
"The same worker identity or generation is claimed more than once, "
|
||||||
|
"so these are genuine duplicates rather than independent workers."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
detail = (
|
||||||
|
"Manual or duplicate launches defeat staleness detection and cannot "
|
||||||
|
"receive client stdio."
|
||||||
|
)
|
||||||
reasons.append(
|
reasons.append(
|
||||||
f"stale-runtime: Duplicate MCP server process(es) detected for profile '{profile}' (PIDs: {pids_str}). "
|
f"stale-runtime: Duplicate MCP server process(es) detected for profile '{profile}' (PIDs: {pids_str}). "
|
||||||
"Manual or duplicate launches defeat staleness detection and cannot receive client stdio."
|
+ detail
|
||||||
)
|
)
|
||||||
|
# Otherwise: several independently identified workers share one profile.
|
||||||
|
# No reason is appended, deliberately. Every reason this function
|
||||||
|
# returns is raised as a hard RuntimeError by its callers, so recording
|
||||||
|
# legitimate concurrency here as "informational" would block exactly the
|
||||||
|
# case #948 exists to permit.
|
||||||
client_procs = [p for p in procs if p["is_client_managed"]]
|
client_procs = [p for p in procs if p["is_client_managed"]]
|
||||||
if client_procs:
|
if client_procs:
|
||||||
client_procs.sort(key=lambda p: p["start_time"], reverse=True)
|
client_procs.sort(key=lambda p: p["start_time"], reverse=True)
|
||||||
@@ -22095,7 +22301,7 @@ def gitea_resolve_task_capability(
|
|||||||
next_safe_action = (
|
next_safe_action = (
|
||||||
"blocker_kind=runtime_reconnect_required: call "
|
"blocker_kind=runtime_reconnect_required: call "
|
||||||
"gitea_request_mcp_reconnect(namespace=<active gitea-* namespace>, "
|
"gitea_request_mcp_reconnect(namespace=<active gitea-* namespace>, "
|
||||||
"reason='stale-runtime', client='codex') for a typed operator "
|
"reason='stale-runtime') for a typed operator "
|
||||||
"blocker with exact UI steps, then reconnect/reload the IDE-managed "
|
"blocker with exact UI steps, then reconnect/reload the IDE-managed "
|
||||||
"Gitea MCP server for this profile so it reloads current master. "
|
"Gitea MCP server for this profile so it reloads current master. "
|
||||||
"Do not edit mcp_config.json by hand, pkill, or touch configs; the "
|
"Do not edit mcp_config.json by hand, pkill, or touch configs; the "
|
||||||
@@ -23663,7 +23869,7 @@ def gitea_workflow_dashboard(
|
|||||||
def gitea_request_mcp_reconnect(
|
def gitea_request_mcp_reconnect(
|
||||||
namespace: str | None = None,
|
namespace: str | None = None,
|
||||||
reason: str | None = None,
|
reason: str | None = None,
|
||||||
client: str = "codex",
|
client: str | None = None,
|
||||||
remote: str = "dadeschools",
|
remote: str = "dadeschools",
|
||||||
host: str | None = None,
|
host: str | None = None,
|
||||||
session_id: str | None = None,
|
session_id: str | None = None,
|
||||||
@@ -23693,8 +23899,12 @@ def gitea_request_mcp_reconnect(
|
|||||||
to the active profile's inferred namespace.
|
to the active profile's inferred namespace.
|
||||||
reason: Why reconnect is requested: ``stale-runtime``, ``transport_eof``,
|
reason: Why reconnect is requested: ``stale-runtime``, ``transport_eof``,
|
||||||
``missing_namespace``, ``not_required``, or free-form (normalized).
|
``missing_namespace``, ``not_required``, or free-form (normalized).
|
||||||
client: Operator UI surface — ``codex`` (default), ``claude_code``, or
|
client: Operator UI surface — ``codex``, ``claude_code``, or
|
||||||
``generic``.
|
``generic``. #948: omitting it resolves the client from this
|
||||||
|
runtime's live attachment record rather than assuming one vendor,
|
||||||
|
and falls back to ``generic`` when nothing identifies the client.
|
||||||
|
Emitting Codex panel steps to a Gemini/Antigravity operator left
|
||||||
|
them with no reachable recovery path.
|
||||||
remote: Known instance — ``dadeschools`` or ``prgs`` (parity context).
|
remote: Known instance — ``dadeschools`` or ``prgs`` (parity context).
|
||||||
host: Optional host override for parity context.
|
host: Optional host override for parity context.
|
||||||
session_id: Optional session id to echo in the report.
|
session_id: Optional session id to echo in the report.
|
||||||
@@ -23754,6 +23964,19 @@ def gitea_request_mcp_reconnect(
|
|||||||
else:
|
else:
|
||||||
effective_reason = mcp_client_reconnect.REASON_UNSPECIFIED
|
effective_reason = mcp_client_reconnect.REASON_UNSPECIFIED
|
||||||
|
|
||||||
|
# #948: an omitted client is resolved from the live attachment record, so
|
||||||
|
# the steps describe the UI the operator is actually in front of.
|
||||||
|
if not (client or "").strip():
|
||||||
|
client = mcp_worker_identity.reconnect_client_for(
|
||||||
|
mcp_worker_identity.assess_provenance(
|
||||||
|
registry=_worker_registry(),
|
||||||
|
worker_identity=_active_worker_identity(),
|
||||||
|
env=dict(os.environ),
|
||||||
|
stdin_is_tty=_stdin_is_tty(),
|
||||||
|
pid_alive_probe=issue_lock_store.is_process_alive,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
payload = mcp_client_reconnect.build_reconnect_request(
|
payload = mcp_client_reconnect.build_reconnect_request(
|
||||||
namespace=ns,
|
namespace=ns,
|
||||||
profile=profile_name,
|
profile=profile_name,
|
||||||
@@ -24209,10 +24432,18 @@ def _run_post_restart_reconcile(
|
|||||||
repo: str | None = None,
|
repo: str | None = None,
|
||||||
mode: str | None = None,
|
mode: str | None = None,
|
||||||
limit: int = 200,
|
limit: int = 200,
|
||||||
|
apply_session_cleanup: bool = False,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Gather + classify post-restart state; cache the latest proof (#662)."""
|
"""Gather + classify post-restart state; cache the latest proof (#662 / #969).
|
||||||
|
|
||||||
|
When *apply_session_cleanup* is true, confirmed-stale session rows (dead
|
||||||
|
owner / PID reuse, no live lease) are terminalized through
|
||||||
|
``session_lifecycle`` before re-classification so the sessions dimension
|
||||||
|
can resolve. Default remains dry (read-only) to preserve #662 rollout.
|
||||||
|
"""
|
||||||
global _POST_RESTART_LAST_PROOF, _POST_RESTART_BOOT_RAN
|
global _POST_RESTART_LAST_PROOF, _POST_RESTART_BOOT_RAN
|
||||||
import post_restart_reconcile as prr
|
import post_restart_reconcile as prr
|
||||||
|
import session_lifecycle as sl
|
||||||
|
|
||||||
try:
|
try:
|
||||||
_h, o, r = _resolve(remote, None, org, repo)
|
_h, o, r = _resolve(remote, None, org, repo)
|
||||||
@@ -24226,13 +24457,48 @@ def _run_post_restart_reconcile(
|
|||||||
inventory = _gather_post_restart_inventory(
|
inventory = _gather_post_restart_inventory(
|
||||||
remote=remote, org=o, repo=r, limit=limit
|
remote=remote, org=o, repo=r, limit=limit
|
||||||
)
|
)
|
||||||
|
|
||||||
|
session_cleanup: dict | None = None
|
||||||
|
if apply_session_cleanup:
|
||||||
|
db, db_errs = _control_plane_db_or_error()
|
||||||
|
if db is None:
|
||||||
|
session_cleanup = {
|
||||||
|
"success": False,
|
||||||
|
"reasons": db_errs
|
||||||
|
or ["control-plane DB unavailable; cannot retire sessions"],
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
profile = get_profile()
|
||||||
|
profile_name = (profile.get("profile_name") or "").strip() or "session"
|
||||||
|
actor = f"{profile_name}-{os.getpid()}"
|
||||||
|
session_cleanup = sl.retire_stale_sessions(
|
||||||
|
db,
|
||||||
|
sessions=inventory.get("sessions") or [],
|
||||||
|
leases=inventory.get("leases") or [],
|
||||||
|
dry_run=False,
|
||||||
|
actor_session_id=actor,
|
||||||
|
session_limit=max(1, int(limit)),
|
||||||
|
)
|
||||||
|
# Re-gather active sessions after mutation so classification sees
|
||||||
|
# the post-retirement fleet (retired rows drop out of active list).
|
||||||
|
try:
|
||||||
|
inventory["sessions"] = db.list_sessions(
|
||||||
|
statuses=("active",), limit=max(1, int(limit))
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
inventory["incomplete_reasons"] = list(
|
||||||
|
inventory.get("incomplete_reasons") or []
|
||||||
|
) + [f"post-retirement session re-list failed: {_redact(str(exc))}"]
|
||||||
|
fleet = (session_cleanup.get("fleet") or {}) if session_cleanup else {}
|
||||||
|
inventory["session_fleet"] = fleet
|
||||||
|
|
||||||
proof = prr.reconcile_after_restart(
|
proof = prr.reconcile_after_restart(
|
||||||
inventory,
|
inventory,
|
||||||
mode=mode or _post_restart_reconcile_mode(),
|
mode=mode or _post_restart_reconcile_mode(),
|
||||||
)
|
)
|
||||||
payload = proof.as_dict()
|
payload = proof.as_dict()
|
||||||
payload["success"] = True
|
payload["success"] = True
|
||||||
payload["read_only"] = True
|
payload["read_only"] = not apply_session_cleanup
|
||||||
payload["remote"] = remote
|
payload["remote"] = remote
|
||||||
payload["org"] = o
|
payload["org"] = o
|
||||||
payload["repo"] = r
|
payload["repo"] = r
|
||||||
@@ -24241,6 +24507,9 @@ def _run_post_restart_reconcile(
|
|||||||
"proposed_follow_ups lists durable issues the apply path may create; "
|
"proposed_follow_ups lists durable issues the apply path may create; "
|
||||||
"this tool never creates them (log-only by default, #662 rollout)"
|
"this tool never creates them (log-only by default, #662 rollout)"
|
||||||
)
|
)
|
||||||
|
if session_cleanup is not None:
|
||||||
|
payload["session_cleanup"] = session_cleanup
|
||||||
|
payload["apply_session_cleanup"] = bool(apply_session_cleanup)
|
||||||
_POST_RESTART_LAST_PROOF = payload
|
_POST_RESTART_LAST_PROOF = payload
|
||||||
_POST_RESTART_BOOT_RAN = True
|
_POST_RESTART_BOOT_RAN = True
|
||||||
return payload
|
return payload
|
||||||
@@ -24267,8 +24536,9 @@ def gitea_reconcile_after_restart(
|
|||||||
repo: str | None = None,
|
repo: str | None = None,
|
||||||
mode: str | None = None,
|
mode: str | None = None,
|
||||||
limit: int = 200,
|
limit: int = 200,
|
||||||
|
apply_session_cleanup: bool = False,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Run post-restart MCP reconciliation and return a completion proof (#662).
|
"""Run post-restart MCP reconciliation and return a completion proof (#662 / #969).
|
||||||
|
|
||||||
Gathers live control-plane sessions, leases, worktree bindings, and
|
Gathers live control-plane sessions, leases, worktree bindings, and
|
||||||
master-parity evidence, then classifies them with the pure
|
master-parity evidence, then classifies them with the pure
|
||||||
@@ -24276,12 +24546,17 @@ def gitea_reconcile_after_restart(
|
|||||||
machine-readable completion proof listing resolved / unresolved dimensions
|
machine-readable completion proof listing resolved / unresolved dimensions
|
||||||
and proposed durable follow-up issues.
|
and proposed durable follow-up issues.
|
||||||
|
|
||||||
Read-only by design: never restarts MCP, never auto-resumes write
|
Never restarts MCP, never auto-resumes write mutations, and never creates
|
||||||
mutations, and never creates Gitea issues (those are a separate apply
|
Gitea issues. Default mode is ``log_only``; set
|
||||||
path). Default mode is ``log_only``; set
|
|
||||||
``GITEA_POST_RESTART_RECONCILE_MODE=enforce`` (or pass ``mode='enforce'``)
|
``GITEA_POST_RESTART_RECONCILE_MODE=enforce`` (or pass ``mode='enforce'``)
|
||||||
to set ``mutation_hold`` when anything remains unresolved.
|
to set ``mutation_hold`` when anything remains unresolved.
|
||||||
|
|
||||||
|
*apply_session_cleanup* (#969): when true, confirmed-stale workflow session
|
||||||
|
rows (dead owner PID / PID reuse, no live lease, not a live client-managed
|
||||||
|
owner) are terminalized through the sanctioned ``session_lifecycle`` path
|
||||||
|
before re-classification. Default false preserves the historical read-only
|
||||||
|
gather+classify behaviour. Does not delete historical rows.
|
||||||
|
|
||||||
Soft-depends on #660 for session checkpoints: when the checkpoint schema
|
Soft-depends on #660 for session checkpoints: when the checkpoint schema
|
||||||
module is absent the checkpoints dimension is ``skipped`` with an explicit
|
module is absent the checkpoints dimension is ``skipped`` with an explicit
|
||||||
reason rather than inventing a schema.
|
reason rather than inventing a schema.
|
||||||
@@ -24301,9 +24576,96 @@ def gitea_reconcile_after_restart(
|
|||||||
repo=repo,
|
repo=repo,
|
||||||
mode=mode,
|
mode=mode,
|
||||||
limit=limit,
|
limit=limit,
|
||||||
|
apply_session_cleanup=bool(apply_session_cleanup),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@mcp.tool()
|
||||||
|
def gitea_retire_stale_workflow_sessions(
|
||||||
|
remote: str = "dadeschools",
|
||||||
|
host: str | None = None,
|
||||||
|
org: str | None = None,
|
||||||
|
repo: str | None = None,
|
||||||
|
apply: bool = False,
|
||||||
|
limit: int = 500,
|
||||||
|
) -> dict:
|
||||||
|
"""Retire workflow session rows whose owners are no longer live (#969).
|
||||||
|
|
||||||
|
Plans (and optionally applies) terminalization of control-plane session
|
||||||
|
rows that are confirmed stale:
|
||||||
|
|
||||||
|
* owner PID absent / dead
|
||||||
|
* PID reused by an unrelated process (process start after session start)
|
||||||
|
* no live workflow lease
|
||||||
|
* not a live client-managed owner
|
||||||
|
|
||||||
|
Default ``apply=false`` is dry-run only. ``apply=true`` performs CAS
|
||||||
|
status updates to ``retired`` and writes durable ``session_retired``
|
||||||
|
events. Idempotent under concurrent reconciliation. Never deletes rows
|
||||||
|
and never touches sessions protected by a live lease or live owner.
|
||||||
|
"""
|
||||||
|
read_block = _profile_operation_gate("gitea.read")
|
||||||
|
if read_block:
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"read_only": not apply,
|
||||||
|
"reasons": read_block,
|
||||||
|
"permission_report": _permission_block_report("gitea.read"),
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
_h, o, r = _resolve(remote, None, org, repo)
|
||||||
|
except ValueError as exc:
|
||||||
|
return {"success": False, "reasons": [str(exc)], "read_only": not apply}
|
||||||
|
|
||||||
|
db, db_errs = _control_plane_db_or_error()
|
||||||
|
if db is None:
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"read_only": not apply,
|
||||||
|
"reasons": db_errs or ["control-plane DB unavailable"],
|
||||||
|
}
|
||||||
|
|
||||||
|
import session_lifecycle as sl
|
||||||
|
|
||||||
|
profile = get_profile()
|
||||||
|
profile_name = (profile.get("profile_name") or "").strip() or "session"
|
||||||
|
actor = f"{profile_name}-{os.getpid()}"
|
||||||
|
|
||||||
|
# Prefer lease inventory scoped to the requested repo when available.
|
||||||
|
leases: list[dict] = []
|
||||||
|
try:
|
||||||
|
lease_result = lease_lifecycle.list_active_leases(
|
||||||
|
db,
|
||||||
|
remote=remote if remote in REMOTES else remote,
|
||||||
|
org=o,
|
||||||
|
repo=r,
|
||||||
|
role=None,
|
||||||
|
include_non_active=True,
|
||||||
|
limit=max(1, int(limit)),
|
||||||
|
)
|
||||||
|
leases = list(lease_result.get("leases") or [])
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
try:
|
||||||
|
leases = db.list_leases(statuses=("active",), limit=max(1, int(limit)))
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
leases = []
|
||||||
|
|
||||||
|
result = sl.retire_stale_sessions(
|
||||||
|
db,
|
||||||
|
leases=leases,
|
||||||
|
dry_run=not bool(apply),
|
||||||
|
actor_session_id=actor,
|
||||||
|
session_limit=max(1, int(limit)),
|
||||||
|
)
|
||||||
|
result["read_only"] = not bool(apply)
|
||||||
|
result["apply"] = bool(apply)
|
||||||
|
result["remote"] = remote
|
||||||
|
result["org"] = o
|
||||||
|
result["repo"] = r
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
@mcp.tool()
|
@mcp.tool()
|
||||||
def gitea_inspect_workflow_lease(
|
def gitea_inspect_workflow_lease(
|
||||||
lease_id: str,
|
lease_id: str,
|
||||||
|
|||||||
+21
-9
@@ -92,7 +92,15 @@ OPERATOR_UI_STEPS: dict[str, tuple[str, ...]] = {
|
|||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
DEFAULT_CLIENT = "codex"
|
#: What an *unidentified* client gets. #948: this is deliberately the
|
||||||
|
#: host-agnostic step set rather than a specific product. Defaulting to one
|
||||||
|
#: vendor emitted Codex UI steps to a Gemini/Antigravity operator, who then had
|
||||||
|
#: no reachable recovery path — the guidance named a panel they do not have.
|
||||||
|
DEFAULT_CLIENT = "generic"
|
||||||
|
|
||||||
|
#: The historical default, kept addressable by name so Codex callers still get
|
||||||
|
#: Codex steps, without it silently becoming the fallback for unknown clients.
|
||||||
|
LEGACY_DEFAULT_CLIENT = "codex"
|
||||||
|
|
||||||
|
|
||||||
def normalize_reason(reason: str | None) -> str:
|
def normalize_reason(reason: str | None) -> str:
|
||||||
@@ -128,14 +136,18 @@ def normalize_reason(reason: str | None) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def normalize_client(client: str | None) -> str:
|
def normalize_client(client: str | None) -> str:
|
||||||
"""Return a known client key for operator UI steps."""
|
"""Return the UI-step key for a client.
|
||||||
text = (client or "").strip().lower().replace(" ", "_").replace("-", "_")
|
|
||||||
if text in ("codex", "openai_codex", "openai"):
|
#948: alias resolution is shared with ``mcp_worker_identity`` so a client
|
||||||
return "codex"
|
name means the same thing wherever it is read. A name we recognise but have
|
||||||
if text in ("claude", "claude_code", "claude_desktop", "anthropic"):
|
no bespoke steps for — Gemini, Antigravity, Grok — resolves to the generic
|
||||||
return "claude_code"
|
host-agnostic steps rather than to another vendor's panel.
|
||||||
if text in OPERATOR_UI_STEPS:
|
"""
|
||||||
return text
|
import mcp_worker_identity
|
||||||
|
|
||||||
|
canonical = mcp_worker_identity.normalize_client_name(client)
|
||||||
|
if canonical in OPERATOR_UI_STEPS:
|
||||||
|
return canonical
|
||||||
return DEFAULT_CLIENT
|
return DEFAULT_CLIENT
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+58
-5
@@ -95,6 +95,15 @@ SAFE_ENV_KEYS = (
|
|||||||
"GITEA_MCP_CONFIG",
|
"GITEA_MCP_CONFIG",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# #948: provenance used to be derived from the summary this allowlist produces.
|
||||||
|
# The allowlist never carried a provenance key, so that derivation could only
|
||||||
|
# ever evaluate to ``manual_launch`` — whatever the process actually was — while
|
||||||
|
# ``gitea_get_runtime_context`` read the live environment and reported
|
||||||
|
# ``client_managed`` for the same process. Provenance is no longer derived here.
|
||||||
|
# It comes from ``mcp_worker_identity.assess_provenance``, the single authority
|
||||||
|
# every surface shares. This allowlist keeps its original and only job: deciding
|
||||||
|
# which env values are safe to echo back in diagnostics.
|
||||||
|
|
||||||
|
|
||||||
def assess_connected_namespace_attachment(
|
def assess_connected_namespace_attachment(
|
||||||
*,
|
*,
|
||||||
@@ -443,12 +452,23 @@ def classify_namespace_probe(
|
|||||||
profile: str | None = None,
|
profile: str | None = None,
|
||||||
configured: bool = True,
|
configured: bool = True,
|
||||||
probe_source: str | None = None,
|
probe_source: str | None = None,
|
||||||
|
worker_identity: str | None = None,
|
||||||
|
generation_id: str | None = None,
|
||||||
|
registry: Any | None = None,
|
||||||
|
pid_alive_probe: Any | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Classify whether a required tool is callable through a live namespace.
|
"""Classify whether a required tool is callable through a live namespace.
|
||||||
|
|
||||||
``registered_tools`` is static/server-side evidence. ``probe_result`` is
|
``registered_tools`` is static/server-side evidence. ``probe_result`` is
|
||||||
live invocation evidence. Only ``probe_source=client_namespace`` proves the
|
live invocation evidence. Only ``probe_source=client_namespace`` proves the
|
||||||
IDE-managed path; ``offline_spawn`` is an offline subprocess check only.
|
IDE-managed path; ``offline_spawn`` is an offline subprocess check only.
|
||||||
|
|
||||||
|
#948: ``worker_identity``/``generation_id``/``registry`` carry the
|
||||||
|
client/session ownership evidence. Provenance is resolved by
|
||||||
|
``mcp_worker_identity.assess_provenance`` — the same call
|
||||||
|
``gitea_get_runtime_context`` makes — so the two surfaces cannot report
|
||||||
|
different provenance for one process. Omitting them yields the fail-closed
|
||||||
|
``unproven`` verdict, never a fabricated ``client_managed``.
|
||||||
"""
|
"""
|
||||||
ns = (namespace or "").strip()
|
ns = (namespace or "").strip()
|
||||||
tool = required_tool or REQUIRED_NAMESPACE_TOOLS.get(ns) or "gitea_whoami"
|
tool = required_tool or REQUIRED_NAMESPACE_TOOLS.get(ns) or "gitea_whoami"
|
||||||
@@ -565,14 +585,31 @@ def classify_namespace_probe(
|
|||||||
blocks = namespace_health_blocks_task("merge_pr", healthy)
|
blocks = namespace_health_blocks_task("merge_pr", healthy)
|
||||||
|
|
||||||
import gitea_config
|
import gitea_config
|
||||||
|
import mcp_worker_identity
|
||||||
|
|
||||||
raw_env = process.get("env") if isinstance(process, dict) else None
|
raw_env = process.get("env") if isinstance(process, dict) else None
|
||||||
unconsumed_env = gitea_config.get_unconsumed_gitea_env_overrides(raw_env)
|
unconsumed_env = gitea_config.get_unconsumed_gitea_env_overrides(raw_env)
|
||||||
is_client_managed = bool(
|
|
||||||
env_summary.get("GITEA_CLIENT_MANAGED") in ("1", "true", "yes", "client_managed")
|
# #948: one authority, shared with gitea_get_runtime_context. The env is
|
||||||
or env_summary.get("GITEA_MCP_CLIENT_MANAGED") in ("1", "true", "yes", "client_managed")
|
# passed whole rather than through SAFE_ENV_KEYS — the allowlist exists to
|
||||||
or env_summary.get("GITEA_SERVER_PROVENANCE") == "client_managed"
|
# decide what may be *echoed*, and using it to decide what may be *believed*
|
||||||
|
# is what made this surface structurally unable to report client_managed.
|
||||||
|
# ``declared_only``: ``process`` describes an observed peer, not this
|
||||||
|
# interpreter. Its stdin is unavailable and its launcher-config env is
|
||||||
|
# inherited from whatever shell started it, so only an explicit declaration
|
||||||
|
# is evidence. Absence of one is ``unproven``, not an asserted manual launch.
|
||||||
|
provenance_verdict = mcp_worker_identity.assess_provenance(
|
||||||
|
registry=registry,
|
||||||
|
worker_identity=worker_identity,
|
||||||
|
generation_id=generation_id,
|
||||||
|
env=raw_env if isinstance(raw_env, dict) else {},
|
||||||
|
namespace=ns,
|
||||||
|
profile=profile_name,
|
||||||
|
pid_alive_probe=pid_alive_probe,
|
||||||
|
declared_only=True,
|
||||||
)
|
)
|
||||||
provenance = "client_managed" if is_client_managed else "manual_launch"
|
provenance = provenance_verdict["provenance"]
|
||||||
|
is_client_managed = provenance_verdict["is_client_managed"]
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"success": healthy,
|
"success": healthy,
|
||||||
@@ -591,6 +628,15 @@ def classify_namespace_probe(
|
|||||||
"remediation": remediation,
|
"remediation": remediation,
|
||||||
"provenance": provenance,
|
"provenance": provenance,
|
||||||
"is_client_managed": is_client_managed,
|
"is_client_managed": is_client_managed,
|
||||||
|
# Every non-client-session verdict fails closed. Consumers that only
|
||||||
|
# need "may this mutate?" read this and stay correct across the #948
|
||||||
|
# vocabulary split between ``manual_launch`` and ``unproven``.
|
||||||
|
"provenance_fail_closed": provenance_verdict["fail_closed"],
|
||||||
|
"provenance_assessment": provenance_verdict,
|
||||||
|
"worker_identity": provenance_verdict["worker_identity"],
|
||||||
|
"session_id": provenance_verdict["session_id"],
|
||||||
|
"generation_id": provenance_verdict["generation_id"],
|
||||||
|
"client_name": provenance_verdict["client_name"],
|
||||||
"unconsumed_gitea_env": unconsumed_env,
|
"unconsumed_gitea_env": unconsumed_env,
|
||||||
"diagnostics": {
|
"diagnostics": {
|
||||||
"namespace": ns,
|
"namespace": ns,
|
||||||
@@ -602,6 +648,13 @@ def classify_namespace_probe(
|
|||||||
"probe_source": source,
|
"probe_source": source,
|
||||||
"provenance": provenance,
|
"provenance": provenance,
|
||||||
"is_client_managed": is_client_managed,
|
"is_client_managed": is_client_managed,
|
||||||
|
"provenance_fail_closed": provenance_verdict["fail_closed"],
|
||||||
|
"provenance_blocker_kind": provenance_verdict["blocker_kind"],
|
||||||
|
"provenance_scope": provenance_verdict["scope"],
|
||||||
|
"worker_identity": provenance_verdict["worker_identity"],
|
||||||
|
"session_id": provenance_verdict["session_id"],
|
||||||
|
"generation_id": provenance_verdict["generation_id"],
|
||||||
|
"client_name": provenance_verdict["client_name"],
|
||||||
"unconsumed_gitea_env": unconsumed_env,
|
"unconsumed_gitea_env": unconsumed_env,
|
||||||
},
|
},
|
||||||
"blocks_merge_workflow": blocks,
|
"blocks_merge_workflow": blocks,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -475,8 +475,37 @@ def reconcile_after_restart(
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- sessions -------------------------------------------------------
|
# --- sessions (#969: dead-owner / PID-reuse lifecycle) ---------------
|
||||||
|
# Prefer a precomputed fleet report from the gather/apply path when present;
|
||||||
|
# otherwise classify pure from inventory (injectable checkers stay default).
|
||||||
sessions = [s for s in (inventory.get("sessions") or []) if isinstance(s, Mapping)]
|
sessions = [s for s in (inventory.get("sessions") or []) if isinstance(s, Mapping)]
|
||||||
|
leases_for_sessions = [
|
||||||
|
L for L in (inventory.get("leases") or []) if isinstance(L, Mapping)
|
||||||
|
]
|
||||||
|
fleet_report = inventory.get("session_fleet")
|
||||||
|
if isinstance(fleet_report, Mapping) and "retireable_session_ids" in fleet_report:
|
||||||
|
fleet_details = dict(fleet_report)
|
||||||
|
retireable_ids = list(fleet_details.get("retireable_session_ids") or [])
|
||||||
|
resolved = bool(fleet_details.get("sessions_dimension_resolved", not retireable_ids))
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
import session_lifecycle as _sl
|
||||||
|
|
||||||
|
client_managed = inventory.get("client_managed_session_ids")
|
||||||
|
cm_set = None
|
||||||
|
if isinstance(client_managed, (list, tuple, set, frozenset)):
|
||||||
|
cm_set = {str(x) for x in client_managed}
|
||||||
|
fleet = _sl.classify_sessions(
|
||||||
|
sessions,
|
||||||
|
leases=leases_for_sessions,
|
||||||
|
now=started,
|
||||||
|
client_managed_sessions=cm_set,
|
||||||
|
)
|
||||||
|
fleet_details = fleet.as_dict()
|
||||||
|
retireable_ids = list(fleet_details.get("retireable_session_ids") or [])
|
||||||
|
resolved = bool(fleet_details.get("sessions_dimension_resolved"))
|
||||||
|
except Exception as exc: # noqa: BLE001 — fail closed to legacy signal
|
||||||
|
# Legacy fallback: dead-pid active rows only (pre-#969 behaviour).
|
||||||
orphan_sessions = [
|
orphan_sessions = [
|
||||||
s
|
s
|
||||||
for s in sessions
|
for s in sessions
|
||||||
@@ -484,15 +513,27 @@ def reconcile_after_restart(
|
|||||||
and s.get("pid") is not None
|
and s.get("pid") is not None
|
||||||
and not lease_lifecycle.is_process_alive(s.get("pid"))
|
and not lease_lifecycle.is_process_alive(s.get("pid"))
|
||||||
]
|
]
|
||||||
if orphan_sessions:
|
retireable_ids = [s.get("session_id") for s in orphan_sessions]
|
||||||
|
resolved = not orphan_sessions
|
||||||
|
fleet_details = {
|
||||||
|
"total_sessions": len(sessions),
|
||||||
|
"retireable_session_ids": retireable_ids,
|
||||||
|
"legacy_fallback": True,
|
||||||
|
"fallback_error": str(exc),
|
||||||
|
}
|
||||||
|
|
||||||
|
if not resolved and retireable_ids:
|
||||||
items.append(
|
items.append(
|
||||||
_item(
|
_item(
|
||||||
DIM_SESSIONS,
|
DIM_SESSIONS,
|
||||||
ITEM_UNRESOLVED,
|
ITEM_UNRESOLVED,
|
||||||
f"{len(orphan_sessions)} active session row(s) with dead owner pid",
|
f"{len(retireable_ids)} session row(s) with dead/reused owner "
|
||||||
|
f"await retirement",
|
||||||
details={
|
details={
|
||||||
"orphan_session_ids": [s.get("session_id") for s in orphan_sessions],
|
"orphan_session_ids": retireable_ids,
|
||||||
|
"retireable_session_ids": retireable_ids,
|
||||||
"total_sessions": len(sessions),
|
"total_sessions": len(sessions),
|
||||||
|
"fleet": fleet_details,
|
||||||
},
|
},
|
||||||
follow_up=True,
|
follow_up=True,
|
||||||
)
|
)
|
||||||
@@ -502,8 +543,12 @@ def reconcile_after_restart(
|
|||||||
_item(
|
_item(
|
||||||
DIM_SESSIONS,
|
DIM_SESSIONS,
|
||||||
ITEM_RESOLVED,
|
ITEM_RESOLVED,
|
||||||
f"{len(sessions)} session row(s) reconciled (no dead-pid orphans)",
|
f"{len(sessions)} session row(s) reconciled "
|
||||||
details={"total_sessions": len(sessions)},
|
f"(no retireable dead/reused owners)",
|
||||||
|
details={
|
||||||
|
"total_sessions": len(sessions),
|
||||||
|
"fleet": fleet_details,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,703 @@
|
|||||||
|
"""Safe lifecycle for workflow session rows with dead or reused owners (#969).
|
||||||
|
|
||||||
|
Post-restart reconciliation previously left hundreds of ``active`` session rows
|
||||||
|
with dead owner PIDs permanently unresolved. A PID existence check alone is not
|
||||||
|
enough: operating systems reuse PIDs, so an unrelated live process can appear to
|
||||||
|
own a historical session.
|
||||||
|
|
||||||
|
This module is the pure classification + apply core for session retirement:
|
||||||
|
|
||||||
|
* distinguish live, disconnected, stale, protected, and terminal records
|
||||||
|
* refuse retirement when a live lease or live verified owner remains
|
||||||
|
* protect live client-managed sessions
|
||||||
|
* detect PID reuse via process start time vs session start / heartbeat
|
||||||
|
* terminalize confirmed-stale rows idempotently with durable audit events
|
||||||
|
* stay safe under concurrent reconciles (CAS on status)
|
||||||
|
|
||||||
|
Design mirrors ``lease_lifecycle`` / ``post_restart_reconcile``:
|
||||||
|
|
||||||
|
* Pure classification accepts injectable checkers so unit tests never touch
|
||||||
|
real processes.
|
||||||
|
* Apply mutations go only through :meth:`ControlPlaneDB.retire_session`.
|
||||||
|
* Historical rows are never deleted; status moves to a terminal value and an
|
||||||
|
events-row records the action + reason.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from typing import Any, Callable, Mapping, Sequence
|
||||||
|
|
||||||
|
import control_plane_db as cpd
|
||||||
|
import lease_lifecycle
|
||||||
|
|
||||||
|
# Session status vocabulary.
|
||||||
|
SESSION_STATUS_ACTIVE = "active"
|
||||||
|
SESSION_STATUS_RETIRED = "retired"
|
||||||
|
SESSION_STATUS_ENDED = "ended"
|
||||||
|
SESSION_STATUS_TERMINAL = "terminal"
|
||||||
|
|
||||||
|
TERMINAL_SESSION_STATUSES = frozenset(
|
||||||
|
{
|
||||||
|
SESSION_STATUS_RETIRED,
|
||||||
|
SESSION_STATUS_ENDED,
|
||||||
|
SESSION_STATUS_TERMINAL,
|
||||||
|
"dead",
|
||||||
|
"stale",
|
||||||
|
"orphaned",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Classification outcomes for one session row.
|
||||||
|
CLASS_LIVE = "live"
|
||||||
|
CLASS_DISCONNECTED = "disconnected"
|
||||||
|
CLASS_STALE = "stale"
|
||||||
|
CLASS_PROTECTED = "protected"
|
||||||
|
CLASS_TERMINAL = "terminal"
|
||||||
|
|
||||||
|
# Stable reason codes (audit + tests).
|
||||||
|
REASON_ALREADY_TERMINAL = "already_terminal"
|
||||||
|
REASON_LIVE_OWNER = "live_owner"
|
||||||
|
REASON_LIVE_LEASE = "live_lease"
|
||||||
|
REASON_CLIENT_MANAGED_LIVE = "client_managed_live"
|
||||||
|
REASON_DEAD_OWNER = "dead_owner"
|
||||||
|
REASON_PID_REUSE = "pid_reuse"
|
||||||
|
REASON_HEARTBEAT_STALE_DEAD = "heartbeat_stale_dead_owner"
|
||||||
|
REASON_MISSING_PID = "missing_pid_no_lease"
|
||||||
|
|
||||||
|
# Event type written to control-plane events table.
|
||||||
|
EVENT_SESSION_RETIRED = "session_retired"
|
||||||
|
|
||||||
|
# Default heartbeat window before a still-alive PID is treated as disconnected
|
||||||
|
# rather than live (does not alone authorize retirement).
|
||||||
|
DEFAULT_HEARTBEAT_STALE_SECONDS = 900
|
||||||
|
|
||||||
|
# PID reuse: process start must be strictly later than session started_at by
|
||||||
|
# more than this skew (ps lstart is second-resolution; clocks can lag).
|
||||||
|
PID_REUSE_SKEW = timedelta(seconds=2)
|
||||||
|
|
||||||
|
# Live lease freshness values that block retirement.
|
||||||
|
_LIVE_LEASE_FRESHNESS = frozenset({"active", "live"})
|
||||||
|
|
||||||
|
|
||||||
|
class SessionLifecycleError(RuntimeError):
|
||||||
|
"""Fail-closed session lifecycle policy error."""
|
||||||
|
|
||||||
|
|
||||||
|
def _utc_now() -> datetime:
|
||||||
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_ts(value: str | datetime | None) -> datetime | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
if value.tzinfo is None:
|
||||||
|
return value.replace(tzinfo=timezone.utc)
|
||||||
|
return value.astimezone(timezone.utc)
|
||||||
|
return cpd._parse_ts(str(value))
|
||||||
|
|
||||||
|
|
||||||
|
def _ts(dt: datetime | None = None) -> str:
|
||||||
|
return cpd._ts(dt)
|
||||||
|
|
||||||
|
|
||||||
|
def process_start_time(pid: int | None) -> datetime | None:
|
||||||
|
"""Return the OS start time of *pid*, or None when it cannot be resolved.
|
||||||
|
|
||||||
|
Uses ``ps -o lstart=`` (POSIX). Failures return None rather than inventing
|
||||||
|
evidence — missing start time never authorizes retirement of a live PID.
|
||||||
|
"""
|
||||||
|
if pid is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
pid_i = int(pid)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if pid_i <= 0:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(
|
||||||
|
["ps", "-o", "lstart=", "-p", str(pid_i)],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
timeout=2,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.SubprocessError):
|
||||||
|
return None
|
||||||
|
if proc.returncode != 0:
|
||||||
|
return None
|
||||||
|
text = (proc.stdout or "").strip()
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
# Example: "Wed Jul 29 09:14:36 2026"
|
||||||
|
naive = datetime.strptime(text, "%a %b %d %H:%M:%S %Y")
|
||||||
|
return naive.replace(tzinfo=timezone.utc)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _lease_freshness_label(lease: Mapping[str, Any]) -> str:
|
||||||
|
fr = lease.get("freshness")
|
||||||
|
if isinstance(fr, Mapping):
|
||||||
|
return str(fr.get("freshness") or "").strip().lower()
|
||||||
|
if fr:
|
||||||
|
return str(fr).strip().lower()
|
||||||
|
# Fall back to classifying a raw lease row.
|
||||||
|
try:
|
||||||
|
return str(
|
||||||
|
lease_lifecycle.classify_lease_freshness(lease).get("freshness") or ""
|
||||||
|
).strip().lower()
|
||||||
|
except Exception: # noqa: BLE001 — pure classifier must not raise on bad rows
|
||||||
|
status = str(lease.get("status") or "").strip().lower()
|
||||||
|
return status or "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def live_lease_session_ids(
|
||||||
|
leases: Sequence[Mapping[str, Any]] | None,
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||||
|
) -> set[str]:
|
||||||
|
"""Session ids that still hold a live (or ambiguous-active) workflow lease."""
|
||||||
|
live: set[str] = set()
|
||||||
|
moment = now or _utc_now()
|
||||||
|
for lease in leases or ():
|
||||||
|
if not isinstance(lease, Mapping):
|
||||||
|
continue
|
||||||
|
status = str(lease.get("status") or "").strip().lower()
|
||||||
|
if status and status not in {
|
||||||
|
lease_lifecycle.LEASE_STATUS_ACTIVE,
|
||||||
|
"",
|
||||||
|
}:
|
||||||
|
# Explicit terminal lease statuses never protect a session.
|
||||||
|
if status in {
|
||||||
|
lease_lifecycle.LEASE_STATUS_RELEASED,
|
||||||
|
lease_lifecycle.LEASE_STATUS_EXPIRED,
|
||||||
|
lease_lifecycle.LEASE_STATUS_ABANDONED,
|
||||||
|
}:
|
||||||
|
continue
|
||||||
|
freshness = _lease_freshness_label(lease)
|
||||||
|
if freshness in _LIVE_LEASE_FRESHNESS or freshness in {"", "unknown"}:
|
||||||
|
# Ambiguous active rows: re-check with authoritative classifier.
|
||||||
|
try:
|
||||||
|
fr = lease_lifecycle.classify_lease_freshness(
|
||||||
|
lease, now=moment, pid_checker=pid_checker
|
||||||
|
)
|
||||||
|
freshness = str(fr.get("freshness") or "").strip().lower()
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
freshness = "unknown"
|
||||||
|
if freshness in _LIVE_LEASE_FRESHNESS:
|
||||||
|
sid = str(lease.get("session_id") or "").strip()
|
||||||
|
if sid:
|
||||||
|
live.add(sid)
|
||||||
|
elif freshness == "unknown" and status in {
|
||||||
|
lease_lifecycle.LEASE_STATUS_ACTIVE,
|
||||||
|
"",
|
||||||
|
}:
|
||||||
|
# Fail closed: active lease with unknown freshness blocks retirement.
|
||||||
|
sid = str(lease.get("session_id") or "").strip()
|
||||||
|
if sid:
|
||||||
|
live.add(sid)
|
||||||
|
return live
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SessionClassification:
|
||||||
|
"""Classification of one workflow session row."""
|
||||||
|
|
||||||
|
session_id: str
|
||||||
|
classification: str
|
||||||
|
reason: str
|
||||||
|
retireable: bool
|
||||||
|
status: str | None
|
||||||
|
pid: int | None
|
||||||
|
pid_alive: bool | None
|
||||||
|
pid_reused: bool
|
||||||
|
heartbeat_stale: bool
|
||||||
|
has_live_lease: bool
|
||||||
|
client_managed: bool
|
||||||
|
details: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"session_id": self.session_id,
|
||||||
|
"classification": self.classification,
|
||||||
|
"reason": self.reason,
|
||||||
|
"retireable": self.retireable,
|
||||||
|
"status": self.status,
|
||||||
|
"pid": self.pid,
|
||||||
|
"pid_alive": self.pid_alive,
|
||||||
|
"pid_reused": self.pid_reused,
|
||||||
|
"heartbeat_stale": self.heartbeat_stale,
|
||||||
|
"has_live_lease": self.has_live_lease,
|
||||||
|
"client_managed": self.client_managed,
|
||||||
|
"details": dict(self.details),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def classify_session(
|
||||||
|
row: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||||
|
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
||||||
|
live_lease_sessions: set[str] | frozenset[str] | None = None,
|
||||||
|
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
||||||
|
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
||||||
|
) -> SessionClassification:
|
||||||
|
"""Classify one session row for retirement decisions (#969).
|
||||||
|
|
||||||
|
Rules (first match wins where noted):
|
||||||
|
|
||||||
|
1. Non-active / already-terminal status → ``terminal`` (not retireable).
|
||||||
|
2. Session holds a live lease → ``protected`` (never retire).
|
||||||
|
3. PID missing and no live lease → ``stale`` (retireable: missing owner).
|
||||||
|
4. PID alive + process start after session start → ``stale`` (PID reuse).
|
||||||
|
5. PID alive + client-managed → ``live`` protected (never retire).
|
||||||
|
6. PID alive + fresh heartbeat → ``live``.
|
||||||
|
7. PID alive + stale heartbeat → ``disconnected`` (not retireable alone).
|
||||||
|
8. PID dead → ``stale`` (retireable).
|
||||||
|
"""
|
||||||
|
moment = now or _utc_now()
|
||||||
|
session_id = str(row.get("session_id") or "").strip()
|
||||||
|
status = str(row.get("status") or "").strip().lower() or None
|
||||||
|
raw_pid = row.get("pid")
|
||||||
|
try:
|
||||||
|
pid = int(raw_pid) if raw_pid is not None else None
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pid = None
|
||||||
|
|
||||||
|
client_managed = bool(
|
||||||
|
row.get("client_managed")
|
||||||
|
or row.get("is_client_managed")
|
||||||
|
or (
|
||||||
|
client_managed_sessions is not None
|
||||||
|
and session_id in client_managed_sessions
|
||||||
|
)
|
||||||
|
)
|
||||||
|
has_live_lease = bool(
|
||||||
|
live_lease_sessions is not None and session_id in live_lease_sessions
|
||||||
|
)
|
||||||
|
|
||||||
|
hb = _parse_ts(row.get("last_heartbeat_at"))
|
||||||
|
heartbeat_stale = bool(
|
||||||
|
hb is not None
|
||||||
|
and (moment - hb).total_seconds() > max(0, int(heartbeat_stale_seconds))
|
||||||
|
)
|
||||||
|
if hb is None and status == SESSION_STATUS_ACTIVE:
|
||||||
|
# No heartbeat evidence: treat as stale for liveness bookkeeping only.
|
||||||
|
heartbeat_stale = True
|
||||||
|
|
||||||
|
started = _parse_ts(row.get("started_at"))
|
||||||
|
recorded_proc_start = _parse_ts(row.get("owner_process_started_at"))
|
||||||
|
|
||||||
|
if status in TERMINAL_SESSION_STATUSES:
|
||||||
|
return SessionClassification(
|
||||||
|
session_id=session_id,
|
||||||
|
classification=CLASS_TERMINAL,
|
||||||
|
reason=REASON_ALREADY_TERMINAL,
|
||||||
|
retireable=False,
|
||||||
|
status=status,
|
||||||
|
pid=pid,
|
||||||
|
pid_alive=None,
|
||||||
|
pid_reused=False,
|
||||||
|
heartbeat_stale=heartbeat_stale,
|
||||||
|
has_live_lease=has_live_lease,
|
||||||
|
client_managed=client_managed,
|
||||||
|
)
|
||||||
|
|
||||||
|
if has_live_lease:
|
||||||
|
return SessionClassification(
|
||||||
|
session_id=session_id,
|
||||||
|
classification=CLASS_PROTECTED,
|
||||||
|
reason=REASON_LIVE_LEASE,
|
||||||
|
retireable=False,
|
||||||
|
status=status,
|
||||||
|
pid=pid,
|
||||||
|
pid_alive=pid_checker(pid) if pid is not None else None,
|
||||||
|
pid_reused=False,
|
||||||
|
heartbeat_stale=heartbeat_stale,
|
||||||
|
has_live_lease=True,
|
||||||
|
client_managed=client_managed,
|
||||||
|
details={"blocker": "live_workflow_lease"},
|
||||||
|
)
|
||||||
|
|
||||||
|
if pid is None:
|
||||||
|
return SessionClassification(
|
||||||
|
session_id=session_id,
|
||||||
|
classification=CLASS_STALE,
|
||||||
|
reason=REASON_MISSING_PID,
|
||||||
|
retireable=True,
|
||||||
|
status=status,
|
||||||
|
pid=None,
|
||||||
|
pid_alive=False,
|
||||||
|
pid_reused=False,
|
||||||
|
heartbeat_stale=heartbeat_stale,
|
||||||
|
has_live_lease=False,
|
||||||
|
client_managed=client_managed,
|
||||||
|
)
|
||||||
|
|
||||||
|
pid_alive = bool(pid_checker(pid))
|
||||||
|
pid_reused = False
|
||||||
|
proc_start: datetime | None = None
|
||||||
|
|
||||||
|
if pid_alive:
|
||||||
|
proc_start = recorded_proc_start or process_start_probe(pid)
|
||||||
|
# PID reuse: live process started after the session row itself was
|
||||||
|
# created. Anchor on started_at only — last_heartbeat alone is not a
|
||||||
|
# safe bound (synthetic inventories and long-lived processes would
|
||||||
|
# false-positive against a live ``ps`` probe).
|
||||||
|
if proc_start is not None and started is not None:
|
||||||
|
if proc_start > (started + PID_REUSE_SKEW):
|
||||||
|
pid_reused = True
|
||||||
|
|
||||||
|
if pid_reused:
|
||||||
|
return SessionClassification(
|
||||||
|
session_id=session_id,
|
||||||
|
classification=CLASS_STALE,
|
||||||
|
reason=REASON_PID_REUSE,
|
||||||
|
retireable=True,
|
||||||
|
status=status,
|
||||||
|
pid=pid,
|
||||||
|
pid_alive=True,
|
||||||
|
pid_reused=True,
|
||||||
|
heartbeat_stale=heartbeat_stale,
|
||||||
|
has_live_lease=False,
|
||||||
|
client_managed=client_managed,
|
||||||
|
details={
|
||||||
|
"process_started_at": _ts(proc_start) if proc_start else None,
|
||||||
|
"session_started_at": row.get("started_at"),
|
||||||
|
"last_heartbeat_at": row.get("last_heartbeat_at"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
if pid_alive and client_managed:
|
||||||
|
return SessionClassification(
|
||||||
|
session_id=session_id,
|
||||||
|
classification=CLASS_LIVE,
|
||||||
|
reason=REASON_CLIENT_MANAGED_LIVE,
|
||||||
|
retireable=False,
|
||||||
|
status=status,
|
||||||
|
pid=pid,
|
||||||
|
pid_alive=True,
|
||||||
|
pid_reused=False,
|
||||||
|
heartbeat_stale=heartbeat_stale,
|
||||||
|
has_live_lease=False,
|
||||||
|
client_managed=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
if pid_alive and not heartbeat_stale:
|
||||||
|
return SessionClassification(
|
||||||
|
session_id=session_id,
|
||||||
|
classification=CLASS_LIVE,
|
||||||
|
reason=REASON_LIVE_OWNER,
|
||||||
|
retireable=False,
|
||||||
|
status=status,
|
||||||
|
pid=pid,
|
||||||
|
pid_alive=True,
|
||||||
|
pid_reused=False,
|
||||||
|
heartbeat_stale=False,
|
||||||
|
has_live_lease=False,
|
||||||
|
client_managed=client_managed,
|
||||||
|
)
|
||||||
|
|
||||||
|
if pid_alive and heartbeat_stale:
|
||||||
|
# Process still exists but has not heartbeated — disconnected, not
|
||||||
|
# confirmed stale. Do not retire; operator/reconnect owns next step.
|
||||||
|
return SessionClassification(
|
||||||
|
session_id=session_id,
|
||||||
|
classification=CLASS_DISCONNECTED,
|
||||||
|
reason=REASON_LIVE_OWNER,
|
||||||
|
retireable=False,
|
||||||
|
status=status,
|
||||||
|
pid=pid,
|
||||||
|
pid_alive=True,
|
||||||
|
pid_reused=False,
|
||||||
|
heartbeat_stale=True,
|
||||||
|
has_live_lease=False,
|
||||||
|
client_managed=client_managed,
|
||||||
|
details={"note": "alive_pid_stale_heartbeat_not_retired"},
|
||||||
|
)
|
||||||
|
|
||||||
|
# PID dead (or checker said not alive).
|
||||||
|
reason = REASON_DEAD_OWNER
|
||||||
|
if heartbeat_stale:
|
||||||
|
reason = REASON_HEARTBEAT_STALE_DEAD
|
||||||
|
return SessionClassification(
|
||||||
|
session_id=session_id,
|
||||||
|
classification=CLASS_STALE,
|
||||||
|
reason=reason,
|
||||||
|
retireable=True,
|
||||||
|
status=status,
|
||||||
|
pid=pid,
|
||||||
|
pid_alive=False,
|
||||||
|
pid_reused=False,
|
||||||
|
heartbeat_stale=heartbeat_stale,
|
||||||
|
has_live_lease=False,
|
||||||
|
client_managed=client_managed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SessionFleetReport:
|
||||||
|
"""Fleet-wide classification summary for reconcile + apply."""
|
||||||
|
|
||||||
|
classifications: tuple[SessionClassification, ...]
|
||||||
|
live_count: int
|
||||||
|
disconnected_count: int
|
||||||
|
stale_count: int
|
||||||
|
protected_count: int
|
||||||
|
terminal_count: int
|
||||||
|
retireable: tuple[SessionClassification, ...]
|
||||||
|
counts_by_reason: dict[str, int]
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"live_count": self.live_count,
|
||||||
|
"disconnected_count": self.disconnected_count,
|
||||||
|
"stale_count": self.stale_count,
|
||||||
|
"protected_count": self.protected_count,
|
||||||
|
"terminal_count": self.terminal_count,
|
||||||
|
"retireable_count": len(self.retireable),
|
||||||
|
"retireable_session_ids": [c.session_id for c in self.retireable],
|
||||||
|
"counts_by_reason": dict(self.counts_by_reason),
|
||||||
|
"classifications": [c.as_dict() for c in self.classifications],
|
||||||
|
"sessions_dimension_resolved": len(self.retireable) == 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def classify_sessions(
|
||||||
|
sessions: Sequence[Mapping[str, Any]] | None,
|
||||||
|
*,
|
||||||
|
leases: Sequence[Mapping[str, Any]] | None = None,
|
||||||
|
now: datetime | None = None,
|
||||||
|
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||||
|
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
||||||
|
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
||||||
|
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
||||||
|
) -> SessionFleetReport:
|
||||||
|
"""Classify a fleet of session rows against live leases (#969)."""
|
||||||
|
moment = now or _utc_now()
|
||||||
|
live_leases = live_lease_session_ids(
|
||||||
|
leases, now=moment, pid_checker=pid_checker
|
||||||
|
)
|
||||||
|
results: list[SessionClassification] = []
|
||||||
|
for row in sessions or ():
|
||||||
|
if not isinstance(row, Mapping):
|
||||||
|
continue
|
||||||
|
results.append(
|
||||||
|
classify_session(
|
||||||
|
row,
|
||||||
|
now=moment,
|
||||||
|
pid_checker=pid_checker,
|
||||||
|
process_start_probe=process_start_probe,
|
||||||
|
live_lease_sessions=live_leases,
|
||||||
|
client_managed_sessions=client_managed_sessions,
|
||||||
|
heartbeat_stale_seconds=heartbeat_stale_seconds,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
live_count = sum(1 for c in results if c.classification == CLASS_LIVE)
|
||||||
|
disconnected_count = sum(
|
||||||
|
1 for c in results if c.classification == CLASS_DISCONNECTED
|
||||||
|
)
|
||||||
|
stale_count = sum(1 for c in results if c.classification == CLASS_STALE)
|
||||||
|
protected_count = sum(1 for c in results if c.classification == CLASS_PROTECTED)
|
||||||
|
terminal_count = sum(1 for c in results if c.classification == CLASS_TERMINAL)
|
||||||
|
retireable = tuple(c for c in results if c.retireable)
|
||||||
|
by_reason: dict[str, int] = {}
|
||||||
|
for c in results:
|
||||||
|
by_reason[c.reason] = by_reason.get(c.reason, 0) + 1
|
||||||
|
|
||||||
|
return SessionFleetReport(
|
||||||
|
classifications=tuple(results),
|
||||||
|
live_count=live_count,
|
||||||
|
disconnected_count=disconnected_count,
|
||||||
|
stale_count=stale_count,
|
||||||
|
protected_count=protected_count,
|
||||||
|
terminal_count=terminal_count,
|
||||||
|
retireable=retireable,
|
||||||
|
counts_by_reason=by_reason,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RetirementResult:
|
||||||
|
"""Outcome of one session retirement attempt."""
|
||||||
|
|
||||||
|
session_id: str
|
||||||
|
outcome: str # retired | already_terminal | skipped | blocked | missing
|
||||||
|
reason: str
|
||||||
|
prior_status: str | None = None
|
||||||
|
new_status: str | None = None
|
||||||
|
details: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"session_id": self.session_id,
|
||||||
|
"outcome": self.outcome,
|
||||||
|
"reason": self.reason,
|
||||||
|
"prior_status": self.prior_status,
|
||||||
|
"new_status": self.new_status,
|
||||||
|
"details": dict(self.details),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def apply_session_retirements(
|
||||||
|
db: cpd.ControlPlaneDB,
|
||||||
|
report: SessionFleetReport,
|
||||||
|
*,
|
||||||
|
dry_run: bool = False,
|
||||||
|
actor_session_id: str | None = None,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Terminalize every retireable session in *report* (idempotent).
|
||||||
|
|
||||||
|
Concurrent reconciles are safe: each retirement is a CAS on
|
||||||
|
``status='active'`` (or other non-terminal). A second pass that sees the
|
||||||
|
same session already retired records ``already_terminal`` rather than
|
||||||
|
duplicating audit noise beyond a single no-op outcome.
|
||||||
|
"""
|
||||||
|
moment = now or _utc_now()
|
||||||
|
results: list[RetirementResult] = []
|
||||||
|
retired = 0
|
||||||
|
already = 0
|
||||||
|
blocked = 0
|
||||||
|
missing = 0
|
||||||
|
|
||||||
|
for classification in report.retireable:
|
||||||
|
if not classification.retireable:
|
||||||
|
continue
|
||||||
|
if dry_run:
|
||||||
|
results.append(
|
||||||
|
RetirementResult(
|
||||||
|
session_id=classification.session_id,
|
||||||
|
outcome="skipped",
|
||||||
|
reason=classification.reason,
|
||||||
|
prior_status=classification.status,
|
||||||
|
new_status=SESSION_STATUS_RETIRED,
|
||||||
|
details={"dry_run": True, **classification.details},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
applied = db.retire_session(
|
||||||
|
session_id=classification.session_id,
|
||||||
|
reason=classification.reason,
|
||||||
|
actor_session_id=actor_session_id,
|
||||||
|
details={
|
||||||
|
"classification": classification.classification,
|
||||||
|
"pid": classification.pid,
|
||||||
|
"pid_alive": classification.pid_alive,
|
||||||
|
"pid_reused": classification.pid_reused,
|
||||||
|
"client_managed": classification.client_managed,
|
||||||
|
**classification.details,
|
||||||
|
},
|
||||||
|
now=moment,
|
||||||
|
)
|
||||||
|
outcome = str(applied.get("outcome") or "missing")
|
||||||
|
results.append(
|
||||||
|
RetirementResult(
|
||||||
|
session_id=classification.session_id,
|
||||||
|
outcome=outcome,
|
||||||
|
reason=str(applied.get("reason") or classification.reason),
|
||||||
|
prior_status=applied.get("prior_status"),
|
||||||
|
new_status=applied.get("new_status"),
|
||||||
|
details=dict(applied.get("details") or {}),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if outcome == "retired":
|
||||||
|
retired += 1
|
||||||
|
elif outcome == "already_terminal":
|
||||||
|
already += 1
|
||||||
|
elif outcome == "blocked":
|
||||||
|
blocked += 1
|
||||||
|
else:
|
||||||
|
missing += 1
|
||||||
|
|
||||||
|
# Non-retireable classifications are recorded for audit completeness when
|
||||||
|
# dry-run lists the fleet, but apply only mutates retireable rows.
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"dry_run": dry_run,
|
||||||
|
"retired_count": retired,
|
||||||
|
"already_terminal_count": already,
|
||||||
|
"blocked_count": blocked,
|
||||||
|
"missing_count": missing,
|
||||||
|
"planned_count": len(report.retireable),
|
||||||
|
"results": [r.as_dict() for r in results],
|
||||||
|
"audit_action": EVENT_SESSION_RETIRED,
|
||||||
|
"actor_session_id": actor_session_id,
|
||||||
|
"recorded_at": _ts(moment),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def retire_stale_sessions(
|
||||||
|
db: cpd.ControlPlaneDB,
|
||||||
|
*,
|
||||||
|
sessions: Sequence[Mapping[str, Any]] | None = None,
|
||||||
|
leases: Sequence[Mapping[str, Any]] | None = None,
|
||||||
|
dry_run: bool = False,
|
||||||
|
actor_session_id: str | None = None,
|
||||||
|
now: datetime | None = None,
|
||||||
|
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||||
|
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
||||||
|
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
||||||
|
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
||||||
|
session_limit: int = 500,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""End-to-end plan + apply for stale session retirement (#969).
|
||||||
|
|
||||||
|
When *sessions* is omitted the control-plane DB is inventoried (active
|
||||||
|
rows only). Callers that already gathered inventory should pass it.
|
||||||
|
"""
|
||||||
|
moment = now or _utc_now()
|
||||||
|
if sessions is None:
|
||||||
|
sessions = db.list_sessions(
|
||||||
|
statuses=(SESSION_STATUS_ACTIVE,),
|
||||||
|
limit=max(1, int(session_limit)),
|
||||||
|
)
|
||||||
|
if leases is None:
|
||||||
|
try:
|
||||||
|
leases = db.list_leases(statuses=("active",), limit=max(1, int(session_limit)))
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
leases = []
|
||||||
|
|
||||||
|
report = classify_sessions(
|
||||||
|
sessions,
|
||||||
|
leases=leases,
|
||||||
|
now=moment,
|
||||||
|
pid_checker=pid_checker,
|
||||||
|
process_start_probe=process_start_probe,
|
||||||
|
client_managed_sessions=client_managed_sessions,
|
||||||
|
heartbeat_stale_seconds=heartbeat_stale_seconds,
|
||||||
|
)
|
||||||
|
apply_result = apply_session_retirements(
|
||||||
|
db,
|
||||||
|
report,
|
||||||
|
dry_run=dry_run,
|
||||||
|
actor_session_id=actor_session_id,
|
||||||
|
now=moment,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"fleet": report.as_dict(),
|
||||||
|
"apply": apply_result,
|
||||||
|
"sessions_dimension_resolved": (
|
||||||
|
report.as_dict()["sessions_dimension_resolved"]
|
||||||
|
if dry_run
|
||||||
|
else apply_result["planned_count"]
|
||||||
|
== (
|
||||||
|
apply_result["retired_count"]
|
||||||
|
+ apply_result["already_terminal_count"]
|
||||||
|
)
|
||||||
|
and apply_result["blocked_count"] == 0
|
||||||
|
),
|
||||||
|
}
|
||||||
+12
-2
@@ -153,8 +153,9 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = {
|
|||||||
"permission": "gitea.branch.push",
|
"permission": "gitea.branch.push",
|
||||||
"role": "author",
|
"role": "author",
|
||||||
},
|
},
|
||||||
# #662: post-restart reconcile is read-only inventory + pure classification.
|
# #662: post-restart reconcile is inventory + pure classification.
|
||||||
# Durable follow-up issue creation is a separate apply path (not this task).
|
# #969: optional apply_session_cleanup retires confirmed-stale session rows
|
||||||
|
# through the same tool; durable follow-up Gitea issues remain separate.
|
||||||
"reconcile_after_restart": {
|
"reconcile_after_restart": {
|
||||||
"permission": "gitea.read",
|
"permission": "gitea.read",
|
||||||
"role": "author",
|
"role": "author",
|
||||||
@@ -163,6 +164,15 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = {
|
|||||||
"permission": "gitea.read",
|
"permission": "gitea.read",
|
||||||
"role": "author",
|
"role": "author",
|
||||||
},
|
},
|
||||||
|
# #969: explicit plan/apply path for dead-owner session retirement.
|
||||||
|
"retire_stale_workflow_sessions": {
|
||||||
|
"permission": "gitea.read",
|
||||||
|
"role": "author",
|
||||||
|
},
|
||||||
|
"gitea_retire_stale_workflow_sessions": {
|
||||||
|
"permission": "gitea.read",
|
||||||
|
"role": "author",
|
||||||
|
},
|
||||||
# #644: Phase 2 Web Console recovery tasks.
|
# #644: Phase 2 Web Console recovery tasks.
|
||||||
"clear_stale_binding": {
|
"clear_stale_binding": {
|
||||||
"permission": "gitea.read",
|
"permission": "gitea.read",
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ class ControlPlaneDBTest(unittest.TestCase):
|
|||||||
rows = dict(conn.execute("SELECT key, value FROM schema_meta").fetchall())
|
rows = dict(conn.execute("SELECT key, value FROM schema_meta").fetchall())
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
self.assertEqual(rows["schema_version"], "5")
|
self.assertEqual(rows["schema_version"], "6")
|
||||||
self.assertIn("DB coordinates", rows["architecture"])
|
self.assertIn("DB coordinates", rows["architecture"])
|
||||||
self.assertIn("bridge", rows["architecture"].lower())
|
self.assertIn("bridge", rows["architecture"].lower())
|
||||||
|
|
||||||
@@ -868,7 +868,7 @@ class SessionCheckpointTest(unittest.TestCase):
|
|||||||
conn.close()
|
conn.close()
|
||||||
self.assertIn("session_checkpoints", names)
|
self.assertIn("session_checkpoints", names)
|
||||||
record = self._write()
|
record = self._write()
|
||||||
self.assertEqual(record["checkpoint_schema_version"], 5)
|
self.assertEqual(record["checkpoint_schema_version"], 6)
|
||||||
|
|
||||||
# AC2 — checkpoints written for multi-role session fixtures.
|
# AC2 — checkpoints written for multi-role session fixtures.
|
||||||
def test_multi_role_fixtures_each_get_a_row(self) -> None:
|
def test_multi_role_fixtures_each_get_a_row(self) -> None:
|
||||||
|
|||||||
@@ -112,7 +112,19 @@ class TestIssue686ManualMcpProvenance(unittest.TestCase):
|
|||||||
self.assertTrue(any("All matching profiles for task 'create_issue' (['prgs-author']) are running but stale" in r for r in reasons))
|
self.assertTrue(any("All matching profiles for task 'create_issue' (['prgs-author']) are running but stale" in r for r in reasons))
|
||||||
|
|
||||||
def test_namespace_health_classification_includes_provenance(self):
|
def test_namespace_health_classification_includes_provenance(self):
|
||||||
"""AC 1 & 4: mcp_namespace_health diagnostics include provenance and unconsumed_gitea_env."""
|
"""AC 1 & 4: mcp_namespace_health diagnostics include provenance and unconsumed_gitea_env.
|
||||||
|
|
||||||
|
#948 narrowed the vocabulary here. This process carries no client-managed
|
||||||
|
declaration, so the old code labelled it ``manual_launch`` — asserting a
|
||||||
|
hand-launched terminal process it had no evidence for, and contradicting
|
||||||
|
``gitea_get_runtime_context``, which read the same process and reported
|
||||||
|
``client_managed``. Absence of proof is now reported as ``unproven``.
|
||||||
|
|
||||||
|
The #686 wall itself is unchanged and still asserted below:
|
||||||
|
``is_client_managed`` stays False, so nothing previously refused is now
|
||||||
|
permitted. Only the label on the *reason* changed, so remediation names
|
||||||
|
the proof that is actually missing.
|
||||||
|
"""
|
||||||
process = {
|
process = {
|
||||||
"pid": 5555,
|
"pid": 5555,
|
||||||
"profile": "prgs-author",
|
"profile": "prgs-author",
|
||||||
@@ -129,10 +141,12 @@ class TestIssue686ManualMcpProvenance(unittest.TestCase):
|
|||||||
process=process,
|
process=process,
|
||||||
probe_source="client_namespace",
|
probe_source="client_namespace",
|
||||||
)
|
)
|
||||||
self.assertEqual(res["provenance"], "manual_launch")
|
self.assertEqual(res["provenance"], "unproven")
|
||||||
self.assertFalse(res["is_client_managed"])
|
self.assertFalse(res["is_client_managed"])
|
||||||
|
# The wall is intact: no client-managed proof still fails closed.
|
||||||
|
self.assertTrue(res["provenance_fail_closed"])
|
||||||
self.assertEqual(res["unconsumed_gitea_env"], {"GITEA_DUMMY": "99"})
|
self.assertEqual(res["unconsumed_gitea_env"], {"GITEA_DUMMY": "99"})
|
||||||
self.assertEqual(res["diagnostics"]["provenance"], "manual_launch")
|
self.assertEqual(res["diagnostics"]["provenance"], "unproven")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -0,0 +1,662 @@
|
|||||||
|
"""Client/session-aware runtime ownership and provenance (#948).
|
||||||
|
|
||||||
|
Covers the reproduced contradiction that motivated the issue: one surface
|
||||||
|
reporting ``client_managed`` while another reported ``manual_launch`` for the
|
||||||
|
same process, remediation hardcoded to one vendor, and a profile-wide duplicate
|
||||||
|
wall that could not tell two healthy clients apart.
|
||||||
|
|
||||||
|
All client and session identifiers here are synthetic.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import mcp_client_reconnect
|
||||||
|
import mcp_namespace_health
|
||||||
|
import mcp_worker_identity as mwi
|
||||||
|
|
||||||
|
|
||||||
|
NOW = datetime(2026, 7, 29, 6, 0, 0, tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _registry() -> mwi.WorkerRegistry:
|
||||||
|
"""A registry on a throwaway path; never the operator's real one."""
|
||||||
|
handle, path = tempfile.mkstemp(suffix=".sqlite3")
|
||||||
|
os.close(handle)
|
||||||
|
os.unlink(path)
|
||||||
|
return mwi.WorkerRegistry(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _attach(
|
||||||
|
registry: mwi.WorkerRegistry,
|
||||||
|
*,
|
||||||
|
client: str,
|
||||||
|
session: str,
|
||||||
|
generation: str,
|
||||||
|
profile: str = "prgs-reviewer",
|
||||||
|
role: str = "reviewer",
|
||||||
|
pid: int = 4242,
|
||||||
|
now: datetime = NOW,
|
||||||
|
ttl: float = 900.0,
|
||||||
|
) -> dict:
|
||||||
|
"""Register one synthetic worker and return the outcome."""
|
||||||
|
identity = mwi.generate_worker_identity(client, session, now=now)
|
||||||
|
outcome = registry.register(
|
||||||
|
worker_identity=identity,
|
||||||
|
client_name=client,
|
||||||
|
client_instance_id=f"inst-{session}",
|
||||||
|
session_id=session,
|
||||||
|
generation_id=generation,
|
||||||
|
role=role,
|
||||||
|
profile=profile,
|
||||||
|
pid=pid,
|
||||||
|
heartbeat_ttl_seconds=ttl,
|
||||||
|
now=now,
|
||||||
|
)
|
||||||
|
outcome["identity"] = identity
|
||||||
|
return outcome
|
||||||
|
|
||||||
|
|
||||||
|
class IdentityFormatTests(unittest.TestCase):
|
||||||
|
"""AC27-29: collision-resistant `<llm-name>-<UTC-timestamp>-<short-sha>`."""
|
||||||
|
|
||||||
|
def test_identity_matches_required_format(self):
|
||||||
|
identity = mwi.generate_worker_identity("Gemini", "sess-0001", now=NOW)
|
||||||
|
parsed = mwi.parse_worker_identity(identity)
|
||||||
|
self.assertTrue(parsed["valid"], parsed["reasons"])
|
||||||
|
self.assertEqual(parsed["client_name"], "gemini")
|
||||||
|
self.assertEqual(parsed["minted_at"], "20260729T060000Z")
|
||||||
|
self.assertEqual(len(parsed["digest"]), 12)
|
||||||
|
|
||||||
|
def test_digest_varies_with_session_and_nonce(self):
|
||||||
|
base = dict(timestamp_ns=1, now=NOW)
|
||||||
|
a = mwi.generate_worker_identity("codex", "sess-A", nonce="n", **base)
|
||||||
|
b = mwi.generate_worker_identity("codex", "sess-B", nonce="n", **base)
|
||||||
|
c = mwi.generate_worker_identity("codex", "sess-A", nonce="m", **base)
|
||||||
|
self.assertNotEqual(a, b, "session must feed the digest")
|
||||||
|
self.assertNotEqual(a, c, "nonce must feed the digest")
|
||||||
|
|
||||||
|
def test_identity_is_not_role_or_profile(self):
|
||||||
|
"""AC26: identity is independent of role and profile."""
|
||||||
|
args = dict(timestamp_ns=7, nonce="fixed", now=NOW)
|
||||||
|
same = mwi.generate_worker_identity("claude", "sess-1", **args)
|
||||||
|
self.assertEqual(same, mwi.generate_worker_identity("claude", "sess-1", **args))
|
||||||
|
# Nothing role- or profile-derived appears in the identity.
|
||||||
|
self.assertNotIn("reviewer", same)
|
||||||
|
self.assertNotIn("prgs", same)
|
||||||
|
|
||||||
|
def test_malformed_identity_rejected(self):
|
||||||
|
self.assertFalse(mwi.parse_worker_identity("prgs-reviewer")["valid"])
|
||||||
|
self.assertFalse(mwi.parse_worker_identity("")["valid"])
|
||||||
|
self.assertFalse(mwi.parse_worker_identity(None)["valid"])
|
||||||
|
|
||||||
|
|
||||||
|
class PerClientAttachmentTests(unittest.TestCase):
|
||||||
|
"""Every supported client attaches and is reported as itself."""
|
||||||
|
|
||||||
|
def _assert_attached_as(self, client: str, expected_name: str):
|
||||||
|
registry = _registry()
|
||||||
|
outcome = _attach(
|
||||||
|
registry, client=client, session=f"sess-{client}", generation="gen-1"
|
||||||
|
)
|
||||||
|
self.assertTrue(outcome["registered"], outcome["reasons"])
|
||||||
|
|
||||||
|
verdict = mwi.assess_provenance(
|
||||||
|
registry=registry, worker_identity=outcome["identity"], env={}, now=NOW
|
||||||
|
)
|
||||||
|
self.assertEqual(verdict["session_ownership"], mwi.OWNERSHIP_OWNED)
|
||||||
|
self.assertEqual(verdict["provenance"], mwi.PROVENANCE_CLIENT_SESSION)
|
||||||
|
self.assertEqual(verdict["client_name"], expected_name)
|
||||||
|
self.assertTrue(verdict["session_owned"])
|
||||||
|
self.assertFalse(verdict["fail_closed"])
|
||||||
|
return verdict
|
||||||
|
|
||||||
|
def test_codex_attachment(self):
|
||||||
|
self._assert_attached_as("codex", "codex")
|
||||||
|
|
||||||
|
def test_gemini_attachment(self):
|
||||||
|
self._assert_attached_as("gemini", "gemini")
|
||||||
|
|
||||||
|
def test_antigravity_attachment(self):
|
||||||
|
self._assert_attached_as("antigravity", "antigravity")
|
||||||
|
|
||||||
|
def test_claude_attachment(self):
|
||||||
|
self._assert_attached_as("claude", "claude_code")
|
||||||
|
|
||||||
|
def test_unknown_client_is_named_not_guessed(self):
|
||||||
|
verdict = self._assert_attached_as("some_new_llm", "some_new_llm")
|
||||||
|
self.assertNotEqual(verdict["client_name"], "codex")
|
||||||
|
|
||||||
|
|
||||||
|
class SessionLifecycleTests(unittest.TestCase):
|
||||||
|
def test_same_client_new_session_gets_distinct_identity(self):
|
||||||
|
registry = _registry()
|
||||||
|
first = _attach(registry, client="codex", session="sess-1", generation="gen-1")
|
||||||
|
second = _attach(registry, client="codex", session="sess-2", generation="gen-2")
|
||||||
|
self.assertTrue(first["registered"])
|
||||||
|
self.assertTrue(second["registered"])
|
||||||
|
self.assertNotEqual(first["identity"], second["identity"])
|
||||||
|
|
||||||
|
# Both are live and neither blocks the other.
|
||||||
|
cohort = mwi.classify_cohort(registry.list_workers(), now=NOW)
|
||||||
|
self.assertEqual(cohort["live_worker_count"], 2)
|
||||||
|
self.assertFalse(cohort["blocked"], cohort["reasons"])
|
||||||
|
|
||||||
|
def test_different_client_attaches_after_previous_session_ends(self):
|
||||||
|
"""AC14: expiry then takeover with a higher fencing epoch."""
|
||||||
|
registry = _registry()
|
||||||
|
gone = _attach(
|
||||||
|
registry, client="codex", session="sess-old", generation="gen-shared", ttl=60
|
||||||
|
)
|
||||||
|
later = NOW + timedelta(hours=1)
|
||||||
|
self.assertFalse(
|
||||||
|
registry.is_live(registry.get(gone["identity"]), now=later)["live"]
|
||||||
|
)
|
||||||
|
|
||||||
|
arriving = _attach(
|
||||||
|
registry,
|
||||||
|
client="gemini",
|
||||||
|
session="sess-new",
|
||||||
|
generation="gen-other",
|
||||||
|
now=later,
|
||||||
|
)
|
||||||
|
claim = registry.claim_generation(
|
||||||
|
worker_identity=arriving["identity"],
|
||||||
|
generation_id="gen-shared",
|
||||||
|
now=later,
|
||||||
|
)
|
||||||
|
self.assertTrue(claim["claimed"], claim["reasons"])
|
||||||
|
self.assertIn(gone["identity"], claim["superseded_workers"])
|
||||||
|
self.assertGreater(claim["fencing_epoch"], gone["fencing_epoch"])
|
||||||
|
|
||||||
|
def test_superseded_session_is_fenced_on_resume(self):
|
||||||
|
"""AC15/AC16: the prior session cannot heartbeat its way back."""
|
||||||
|
registry = _registry()
|
||||||
|
old = _attach(
|
||||||
|
registry, client="codex", session="sess-old", generation="gen-shared", ttl=60
|
||||||
|
)
|
||||||
|
later = NOW + timedelta(hours=1)
|
||||||
|
new = _attach(
|
||||||
|
registry, client="gemini", session="sess-new", generation="gen-x", now=later
|
||||||
|
)
|
||||||
|
registry.claim_generation(
|
||||||
|
worker_identity=new["identity"], generation_id="gen-shared", now=later
|
||||||
|
)
|
||||||
|
|
||||||
|
resumed = registry.heartbeat(
|
||||||
|
worker_identity=old["identity"],
|
||||||
|
fencing_epoch=old["fencing_epoch"],
|
||||||
|
now=later,
|
||||||
|
)
|
||||||
|
self.assertFalse(resumed["renewed"])
|
||||||
|
self.assertFalse(resumed["mutation_performed"])
|
||||||
|
self.assertEqual(resumed["blocker_kind"], mwi.BLOCKER_FENCED)
|
||||||
|
|
||||||
|
def test_heartbeat_renews_only_the_owning_lease(self):
|
||||||
|
"""AC11: a wrong epoch never renews, and never mutates."""
|
||||||
|
registry = _registry()
|
||||||
|
worker = _attach(registry, client="codex", session="s", generation="g")
|
||||||
|
good = registry.heartbeat(
|
||||||
|
worker_identity=worker["identity"],
|
||||||
|
fencing_epoch=worker["fencing_epoch"],
|
||||||
|
now=NOW + timedelta(minutes=5),
|
||||||
|
)
|
||||||
|
self.assertTrue(good["renewed"])
|
||||||
|
|
||||||
|
bad = registry.heartbeat(
|
||||||
|
worker_identity=worker["identity"],
|
||||||
|
fencing_epoch=worker["fencing_epoch"] + 99,
|
||||||
|
now=NOW + timedelta(minutes=6),
|
||||||
|
)
|
||||||
|
self.assertFalse(bad["renewed"])
|
||||||
|
self.assertFalse(bad["mutation_performed"])
|
||||||
|
self.assertEqual(
|
||||||
|
registry.get(worker["identity"])["last_heartbeat_at"],
|
||||||
|
good["last_heartbeat_at"],
|
||||||
|
"a refused heartbeat must not advance the record",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ConflictAndCollisionTests(unittest.TestCase):
|
||||||
|
def test_two_live_sessions_cannot_claim_one_generation(self):
|
||||||
|
registry = _registry()
|
||||||
|
first = _attach(registry, client="codex", session="s1", generation="gen-shared")
|
||||||
|
second = _attach(registry, client="gemini", session="s2", generation="gen-other")
|
||||||
|
|
||||||
|
claim = registry.claim_generation(
|
||||||
|
worker_identity=second["identity"],
|
||||||
|
generation_id="gen-shared",
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
self.assertFalse(claim["claimed"])
|
||||||
|
self.assertFalse(claim["mutation_performed"])
|
||||||
|
self.assertEqual(claim["blocker_kind"], mwi.BLOCKER_CONFLICTING_SESSIONS)
|
||||||
|
self.assertEqual(
|
||||||
|
claim["conflicting_owners"][0]["worker_identity"], first["identity"]
|
||||||
|
)
|
||||||
|
# The sanctioned recovery must never be "kill the other process".
|
||||||
|
self.assertIn("Do not kill", claim["exact_next_action"])
|
||||||
|
|
||||||
|
def test_contested_generation_fails_closed_in_assessment(self):
|
||||||
|
registry = _registry()
|
||||||
|
first = _attach(registry, client="codex", session="s1", generation="gen-shared")
|
||||||
|
_attach(registry, client="gemini", session="s2", generation="gen-shared")
|
||||||
|
|
||||||
|
verdict = mwi.assess_provenance(
|
||||||
|
registry=registry, worker_identity=first["identity"], env={}, now=NOW
|
||||||
|
)
|
||||||
|
self.assertEqual(verdict["session_ownership"], mwi.OWNERSHIP_CONTESTED)
|
||||||
|
self.assertTrue(verdict["fail_closed"])
|
||||||
|
self.assertEqual(verdict["blocker_kind"], mwi.BLOCKER_CONTRADICTORY)
|
||||||
|
self.assertTrue(verdict["conflicting_live_sessions"])
|
||||||
|
|
||||||
|
def test_identity_collision_is_refused_without_corrupting_existing(self):
|
||||||
|
"""AC31: never replace, adopt, merge with, or corrupt the incumbent."""
|
||||||
|
registry = _registry()
|
||||||
|
incumbent = _attach(registry, client="codex", session="s1", generation="gen-1")
|
||||||
|
before = registry.get(incumbent["identity"])
|
||||||
|
|
||||||
|
collided = registry.register(
|
||||||
|
worker_identity=incumbent["identity"],
|
||||||
|
client_name="gemini",
|
||||||
|
client_instance_id="inst-other",
|
||||||
|
session_id="s2",
|
||||||
|
generation_id="gen-2",
|
||||||
|
pid=9999,
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
self.assertFalse(collided["registered"])
|
||||||
|
self.assertTrue(collided["collision"])
|
||||||
|
self.assertFalse(collided["mutation_performed"])
|
||||||
|
self.assertEqual(collided["blocker_kind"], mwi.BLOCKER_IDENTITY_COLLISION)
|
||||||
|
self.assertEqual(collided["collision_kind"], "active_worker")
|
||||||
|
self.assertEqual(
|
||||||
|
registry.get(incumbent["identity"]), before, "incumbent must be untouched"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_after_collision_a_regenerated_identity_registers(self):
|
||||||
|
"""AC32/AC35: forced collision, safe regeneration, successful replacement."""
|
||||||
|
registry = _registry()
|
||||||
|
fixed = dict(timestamp_ns=99, nonce="deterministic", now=NOW)
|
||||||
|
forced = mwi.generate_worker_identity("codex", "sess-collide", **fixed)
|
||||||
|
first = registry.register(
|
||||||
|
worker_identity=forced,
|
||||||
|
client_name="codex",
|
||||||
|
client_instance_id="inst-1",
|
||||||
|
session_id="sess-collide",
|
||||||
|
generation_id="gen-1",
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
self.assertTrue(first["registered"])
|
||||||
|
|
||||||
|
# A second worker deriving the same inputs collides deterministically.
|
||||||
|
again = mwi.generate_worker_identity("codex", "sess-collide", **fixed)
|
||||||
|
self.assertEqual(again, forced)
|
||||||
|
self.assertTrue(
|
||||||
|
registry.register(
|
||||||
|
worker_identity=again,
|
||||||
|
client_name="codex",
|
||||||
|
client_instance_id="inst-2",
|
||||||
|
session_id="sess-collide",
|
||||||
|
generation_id="gen-2",
|
||||||
|
now=NOW,
|
||||||
|
)["collision"]
|
||||||
|
)
|
||||||
|
|
||||||
|
replacement = mwi.generate_worker_identity(
|
||||||
|
"codex", "sess-collide", timestamp_ns=100, nonce="different", now=NOW
|
||||||
|
)
|
||||||
|
self.assertNotEqual(replacement, forced)
|
||||||
|
self.assertTrue(
|
||||||
|
registry.register(
|
||||||
|
worker_identity=replacement,
|
||||||
|
client_name="codex",
|
||||||
|
client_instance_id="inst-2",
|
||||||
|
session_id="sess-collide",
|
||||||
|
generation_id="gen-2",
|
||||||
|
now=NOW,
|
||||||
|
)["registered"]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_restarted_worker_inherits_nothing(self):
|
||||||
|
"""AC33/AC34: a restart mints a new identity and no prior epoch."""
|
||||||
|
registry = _registry()
|
||||||
|
before = _attach(
|
||||||
|
registry, client="codex", session="sess-before", generation="gen-1", ttl=60
|
||||||
|
)
|
||||||
|
later = NOW + timedelta(hours=2)
|
||||||
|
after = _attach(
|
||||||
|
registry, client="codex", session="sess-after", generation="gen-2", now=later
|
||||||
|
)
|
||||||
|
self.assertNotEqual(before["identity"], after["identity"])
|
||||||
|
self.assertNotEqual(
|
||||||
|
registry.get(after["identity"])["generation_id"],
|
||||||
|
registry.get(before["identity"])["generation_id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class LivenessTests(unittest.TestCase):
|
||||||
|
def test_stale_session_record_is_not_live(self):
|
||||||
|
registry = _registry()
|
||||||
|
worker = _attach(registry, client="codex", session="s", generation="g", ttl=300)
|
||||||
|
stale = registry.is_live(
|
||||||
|
registry.get(worker["identity"]), now=NOW + timedelta(hours=1)
|
||||||
|
)
|
||||||
|
self.assertFalse(stale["live"])
|
||||||
|
self.assertFalse(stale["heartbeat_fresh"])
|
||||||
|
|
||||||
|
def test_liveness_is_not_pid_comparison_alone(self):
|
||||||
|
"""AC7: a live PID does not resurrect an expired registration."""
|
||||||
|
registry = _registry()
|
||||||
|
worker = _attach(registry, client="codex", session="s", generation="g", ttl=60)
|
||||||
|
verdict = registry.is_live(
|
||||||
|
registry.get(worker["identity"]),
|
||||||
|
now=NOW + timedelta(hours=1),
|
||||||
|
pid_alive=True,
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
verdict["live"], "a live PID must not override a dead heartbeat"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_dead_pid_withdraws_liveness_from_a_fresh_heartbeat(self):
|
||||||
|
registry = _registry()
|
||||||
|
worker = _attach(registry, client="codex", session="s", generation="g")
|
||||||
|
verdict = registry.is_live(
|
||||||
|
registry.get(worker["identity"]), now=NOW, pid_alive=False
|
||||||
|
)
|
||||||
|
self.assertFalse(verdict["live"])
|
||||||
|
|
||||||
|
def test_stale_ownership_does_not_permanently_strand_a_daemon(self):
|
||||||
|
registry = _registry()
|
||||||
|
stranded = _attach(
|
||||||
|
registry, client="codex", session="s-old", generation="gen-daemon", ttl=60
|
||||||
|
)
|
||||||
|
later = NOW + timedelta(hours=3)
|
||||||
|
rescuer = _attach(
|
||||||
|
registry, client="claude", session="s-new", generation="gen-tmp", now=later
|
||||||
|
)
|
||||||
|
claim = registry.claim_generation(
|
||||||
|
worker_identity=rescuer["identity"],
|
||||||
|
generation_id="gen-daemon",
|
||||||
|
now=later,
|
||||||
|
)
|
||||||
|
self.assertTrue(claim["claimed"], claim["reasons"])
|
||||||
|
self.assertIn(stranded["identity"], claim["superseded_workers"])
|
||||||
|
|
||||||
|
|
||||||
|
class EvidenceTests(unittest.TestCase):
|
||||||
|
def test_env_flag_alone_does_not_prove_session_ownership(self):
|
||||||
|
verdict = mwi.assess_provenance(
|
||||||
|
registry=None,
|
||||||
|
worker_identity=None,
|
||||||
|
env={"GITEA_CLIENT_MANAGED": "1", "GITEA_MCP_SANCTIONED_DAEMON": "1"},
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
self.assertFalse(verdict["session_owned"])
|
||||||
|
self.assertEqual(verdict["session_ownership"], mwi.OWNERSHIP_UNOWNED)
|
||||||
|
self.assertTrue(verdict["env_flag_only"])
|
||||||
|
self.assertTrue(verdict["fail_closed"])
|
||||||
|
self.assertNotIn(mwi.EVIDENCE_ATTACHMENT_RECORD, verdict["evidence"])
|
||||||
|
self.assertFalse(verdict["env_signal"]["proves_session_ownership"])
|
||||||
|
|
||||||
|
def test_env_flag_still_answers_the_launch_question(self):
|
||||||
|
"""The #686 wall is preserved: env decides launch, not ownership."""
|
||||||
|
self.assertTrue(
|
||||||
|
mwi.assess_launch_provenance({"GITEA_CLIENT_MANAGED": "1"})["client_managed"]
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
mwi.assess_launch_provenance({"GITEA_CLIENT_MANAGED": "0"})["client_managed"]
|
||||||
|
)
|
||||||
|
self.assertFalse(
|
||||||
|
mwi.assess_launch_provenance({}, stdin_is_tty=True)["client_managed"]
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
mwi.assess_launch_provenance({"GITEA_MCP_PROFILE": "prgs-author"})[
|
||||||
|
"client_managed"
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_missing_evidence_is_unproven_not_manual(self):
|
||||||
|
"""A missing proof must not be reported as a hand-launched process."""
|
||||||
|
verdict = mwi.assess_provenance(
|
||||||
|
registry=None, worker_identity=None, env={}, now=NOW
|
||||||
|
)
|
||||||
|
self.assertEqual(verdict["provenance"], mwi.PROVENANCE_UNPROVEN)
|
||||||
|
self.assertNotEqual(verdict["provenance"], mwi.PROVENANCE_MANUAL)
|
||||||
|
self.assertTrue(verdict["fail_closed"])
|
||||||
|
|
||||||
|
def test_declared_manual_launch_is_reported_as_manual(self):
|
||||||
|
verdict = mwi.assess_provenance(
|
||||||
|
registry=None,
|
||||||
|
worker_identity=None,
|
||||||
|
env={"GITEA_CLIENT_MANAGED": "0"},
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
self.assertEqual(verdict["provenance"], mwi.PROVENANCE_MANUAL)
|
||||||
|
|
||||||
|
def test_fail_closed_refusal_names_its_scope_not_the_profile(self):
|
||||||
|
"""AC17/AC41: no refusal is profile-wide."""
|
||||||
|
verdict = mwi.assess_provenance(
|
||||||
|
registry=None,
|
||||||
|
worker_identity=None,
|
||||||
|
env={},
|
||||||
|
profile="prgs-reviewer",
|
||||||
|
role="reviewer",
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
self.assertFalse(verdict["scope"]["profile_wide"])
|
||||||
|
self.assertEqual(verdict["blocker_kind"], mwi.BLOCKER_NO_ATTACHMENT)
|
||||||
|
|
||||||
|
|
||||||
|
class CohortScopingTests(unittest.TestCase):
|
||||||
|
def test_shared_profile_with_distinct_identities_does_not_block(self):
|
||||||
|
"""AC40: profile is not a singleton identity."""
|
||||||
|
registry = _registry()
|
||||||
|
_attach(
|
||||||
|
registry,
|
||||||
|
client="codex",
|
||||||
|
session="s1",
|
||||||
|
generation="g1",
|
||||||
|
profile="prgs-reviewer",
|
||||||
|
)
|
||||||
|
_attach(
|
||||||
|
registry,
|
||||||
|
client="gemini",
|
||||||
|
session="s2",
|
||||||
|
generation="g2",
|
||||||
|
profile="prgs-reviewer",
|
||||||
|
)
|
||||||
|
|
||||||
|
cohort = mwi.classify_cohort(registry.list_workers(), now=NOW)
|
||||||
|
self.assertFalse(cohort["blocked"], cohort["reasons"])
|
||||||
|
self.assertEqual(cohort["blocker_kind"], mwi.BLOCKER_NONE)
|
||||||
|
self.assertIn("prgs-reviewer", cohort["shared_profiles"])
|
||||||
|
self.assertTrue(cohort["profile_sharing_permitted"])
|
||||||
|
self.assertEqual(cohort["blocked_worker_identities"], [])
|
||||||
|
|
||||||
|
def test_duplicate_cohort_records_block_only_the_offenders(self):
|
||||||
|
registry = _registry()
|
||||||
|
_attach(registry, client="codex", session="s1", generation="gen-contested")
|
||||||
|
_attach(registry, client="gemini", session="s2", generation="gen-contested")
|
||||||
|
_attach(registry, client="claude", session="s3", generation="gen-fine")
|
||||||
|
|
||||||
|
cohort = mwi.classify_cohort(registry.list_workers(), now=NOW)
|
||||||
|
self.assertTrue(cohort["blocked"])
|
||||||
|
self.assertEqual(cohort["contested_generations"], ["gen-contested"])
|
||||||
|
self.assertEqual(len(cohort["blocked_worker_identities"]), 2)
|
||||||
|
|
||||||
|
def test_mixed_runtime_generations_are_scoped_independently(self):
|
||||||
|
"""AC17: one stale generation does not wall unrelated healthy ones."""
|
||||||
|
registry = _registry()
|
||||||
|
stale = _attach(
|
||||||
|
registry, client="codex", session="s1", generation="gen-stale", ttl=60
|
||||||
|
)
|
||||||
|
healthy_a = _attach(registry, client="gemini", session="s2", generation="gen-a")
|
||||||
|
healthy_b = _attach(registry, client="claude", session="s3", generation="gen-b")
|
||||||
|
|
||||||
|
scoped = mwi.scope_runtime_failure(
|
||||||
|
failure_kind="stale-runtime",
|
||||||
|
worker_identity=stale["identity"],
|
||||||
|
profile="prgs-reviewer",
|
||||||
|
all_live_workers=registry.list_workers(),
|
||||||
|
)
|
||||||
|
self.assertFalse(scoped["profile_wide"])
|
||||||
|
self.assertFalse(scoped["fleet_wide"])
|
||||||
|
self.assertEqual(len(scoped["affected_workers"]), 1)
|
||||||
|
self.assertEqual(scoped["unaffected_worker_count"], 2)
|
||||||
|
unaffected = {w["worker_identity"] for w in scoped["unaffected_workers"]}
|
||||||
|
self.assertEqual(unaffected, {healthy_a["identity"], healthy_b["identity"]})
|
||||||
|
|
||||||
|
|
||||||
|
class HardcodedClientRegressionTests(unittest.TestCase):
|
||||||
|
def test_unknown_client_does_not_resolve_to_codex(self):
|
||||||
|
for name in ("gemini", "antigravity", "grok", "some_new_llm", "", None):
|
||||||
|
with self.subTest(client=name):
|
||||||
|
self.assertNotEqual(
|
||||||
|
mcp_client_reconnect.normalize_client(name),
|
||||||
|
"codex",
|
||||||
|
"an unidentified client must never be handed Codex UI steps",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_known_clients_still_get_their_own_steps(self):
|
||||||
|
self.assertEqual(mcp_client_reconnect.normalize_client("codex"), "codex")
|
||||||
|
self.assertEqual(
|
||||||
|
mcp_client_reconnect.normalize_client("claude_code"), "claude_code"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_generic_steps_do_not_name_a_specific_vendor(self):
|
||||||
|
steps = " ".join(mcp_client_reconnect.operator_ui_steps("gemini"))
|
||||||
|
self.assertNotIn("Codex", steps)
|
||||||
|
|
||||||
|
def test_reconnect_client_is_derived_from_the_attachment_record(self):
|
||||||
|
registry = _registry()
|
||||||
|
worker = _attach(registry, client="antigravity", session="s", generation="g")
|
||||||
|
verdict = mwi.assess_provenance(
|
||||||
|
registry=registry, worker_identity=worker["identity"], env={}, now=NOW
|
||||||
|
)
|
||||||
|
self.assertEqual(mwi.reconnect_client_for(verdict), "antigravity")
|
||||||
|
|
||||||
|
def test_reconnect_client_is_unknown_rather_than_guessed(self):
|
||||||
|
self.assertEqual(mwi.reconnect_client_for({}), mwi.UNKNOWN_CLIENT)
|
||||||
|
|
||||||
|
|
||||||
|
class RemoteBindingTests(unittest.TestCase):
|
||||||
|
def test_explicit_prgs_selection_is_honoured(self):
|
||||||
|
resolved = mwi.resolve_bound_remote(
|
||||||
|
requested_remote="prgs", bound_remote="prgs", default_remote="dadeschools"
|
||||||
|
)
|
||||||
|
self.assertEqual(resolved["remote"], "prgs")
|
||||||
|
self.assertFalse(resolved["drifted"])
|
||||||
|
|
||||||
|
def test_omitted_remote_uses_the_binding_not_the_library_default(self):
|
||||||
|
"""The reported dadeschools host drift."""
|
||||||
|
resolved = mwi.resolve_bound_remote(
|
||||||
|
requested_remote=None, bound_remote="prgs", default_remote="dadeschools"
|
||||||
|
)
|
||||||
|
self.assertEqual(resolved["remote"], "prgs")
|
||||||
|
self.assertNotEqual(resolved["remote"], "dadeschools")
|
||||||
|
self.assertEqual(resolved["resolved_from"], "session_binding")
|
||||||
|
|
||||||
|
def test_contradicting_the_binding_is_refused(self):
|
||||||
|
resolved = mwi.resolve_bound_remote(
|
||||||
|
requested_remote="dadeschools",
|
||||||
|
bound_remote="prgs",
|
||||||
|
default_remote="dadeschools",
|
||||||
|
)
|
||||||
|
self.assertEqual(resolved["remote"], "prgs")
|
||||||
|
self.assertTrue(resolved["drifted"])
|
||||||
|
self.assertFalse(resolved["honoured_request"])
|
||||||
|
|
||||||
|
def test_unbound_session_falls_back_and_says_so(self):
|
||||||
|
resolved = mwi.resolve_bound_remote(
|
||||||
|
requested_remote=None, bound_remote=None, default_remote="dadeschools"
|
||||||
|
)
|
||||||
|
self.assertEqual(resolved["remote"], "dadeschools")
|
||||||
|
self.assertEqual(resolved["resolved_from"], "library_default")
|
||||||
|
self.assertTrue(resolved["reasons"])
|
||||||
|
|
||||||
|
|
||||||
|
class SurfaceAgreementTests(unittest.TestCase):
|
||||||
|
"""The reproduced contradiction: two surfaces, one process, two answers."""
|
||||||
|
|
||||||
|
def test_namespace_health_and_direct_assessment_agree(self):
|
||||||
|
registry = _registry()
|
||||||
|
worker = _attach(
|
||||||
|
registry,
|
||||||
|
client="gemini",
|
||||||
|
session="sess-agree",
|
||||||
|
generation="gen-agree",
|
||||||
|
profile="prgs-reviewer",
|
||||||
|
)
|
||||||
|
env = {"GITEA_MCP_PROFILE": "prgs-reviewer", "GITEA_CLIENT_MANAGED": "1"}
|
||||||
|
|
||||||
|
direct = mwi.assess_provenance(
|
||||||
|
registry=registry,
|
||||||
|
worker_identity=worker["identity"],
|
||||||
|
env=env,
|
||||||
|
profile="prgs-reviewer",
|
||||||
|
)
|
||||||
|
health = mcp_namespace_health.classify_namespace_probe(
|
||||||
|
"gitea-reviewer",
|
||||||
|
configured=True,
|
||||||
|
registered_tools=["gitea_whoami"],
|
||||||
|
probe_result={"success": True},
|
||||||
|
probe_source="client_namespace",
|
||||||
|
process={"pid": 4242, "profile": "prgs-reviewer", "env": env},
|
||||||
|
registry=registry,
|
||||||
|
worker_identity=worker["identity"],
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(health["provenance"], direct["provenance"])
|
||||||
|
self.assertEqual(health["is_client_managed"], direct["is_client_managed"])
|
||||||
|
self.assertEqual(health["worker_identity"], direct["worker_identity"])
|
||||||
|
self.assertEqual(health["session_id"], "sess-agree")
|
||||||
|
self.assertEqual(health["client_name"], "gemini")
|
||||||
|
|
||||||
|
def test_namespace_health_can_report_client_managed_at_all(self):
|
||||||
|
"""The old derivation was structurally incapable of this."""
|
||||||
|
env = {"GITEA_CLIENT_MANAGED": "1", "GITEA_MCP_PROFILE": "prgs-author"}
|
||||||
|
health = mcp_namespace_health.classify_namespace_probe(
|
||||||
|
"gitea-author",
|
||||||
|
configured=True,
|
||||||
|
registered_tools=["gitea_whoami"],
|
||||||
|
probe_result={"success": True},
|
||||||
|
probe_source="client_namespace",
|
||||||
|
process={"pid": 1234, "profile": "prgs-author", "env": env},
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
health["is_client_managed"],
|
||||||
|
"a client-managed launch must be reportable as client-managed",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_namespace_health_without_attachment_fails_closed(self):
|
||||||
|
health = mcp_namespace_health.classify_namespace_probe(
|
||||||
|
"gitea-author",
|
||||||
|
configured=True,
|
||||||
|
registered_tools=["gitea_whoami"],
|
||||||
|
probe_result={"success": True},
|
||||||
|
probe_source="client_namespace",
|
||||||
|
process={"pid": 1234, "profile": "prgs-author", "env": {}},
|
||||||
|
)
|
||||||
|
self.assertTrue(health["provenance_fail_closed"])
|
||||||
|
self.assertEqual(health["provenance"], mwi.PROVENANCE_UNPROVEN)
|
||||||
|
self.assertIsNone(health["session_id"])
|
||||||
|
|
||||||
|
def test_no_false_reconnect_loop_for_an_owned_session(self):
|
||||||
|
"""A proven owner must not be told to reconnect."""
|
||||||
|
registry = _registry()
|
||||||
|
worker = _attach(registry, client="claude", session="s", generation="g")
|
||||||
|
verdict = mwi.assess_provenance(
|
||||||
|
registry=registry, worker_identity=worker["identity"], env={}, now=NOW
|
||||||
|
)
|
||||||
|
self.assertFalse(verdict["fail_closed"])
|
||||||
|
self.assertEqual(verdict["blocker_kind"], mwi.BLOCKER_NONE)
|
||||||
|
self.assertEqual(verdict["reasons"], [])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,480 @@
|
|||||||
|
"""Tests for dead-owner / PID-reuse session retirement (#969)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import unittest
|
||||||
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import control_plane_db as cpd
|
||||||
|
import post_restart_reconcile as prr
|
||||||
|
import session_lifecycle as sl
|
||||||
|
|
||||||
|
NOW = datetime(2026, 7, 29, 12, 0, 0, tzinfo=timezone.utc)
|
||||||
|
EARLIER = NOW - timedelta(hours=2)
|
||||||
|
LATER = NOW + timedelta(minutes=5)
|
||||||
|
|
||||||
|
|
||||||
|
def _session(
|
||||||
|
session_id: str,
|
||||||
|
*,
|
||||||
|
pid: int | None = 4242,
|
||||||
|
status: str = "active",
|
||||||
|
started_at: datetime = EARLIER,
|
||||||
|
last_heartbeat_at: datetime | None = None,
|
||||||
|
client_managed: bool = False,
|
||||||
|
owner_process_started_at: datetime | None = None,
|
||||||
|
role: str = "author",
|
||||||
|
) -> dict:
|
||||||
|
hb = last_heartbeat_at if last_heartbeat_at is not None else started_at
|
||||||
|
row = {
|
||||||
|
"session_id": session_id,
|
||||||
|
"role": role,
|
||||||
|
"profile": "prgs-author",
|
||||||
|
"pid": pid,
|
||||||
|
"status": status,
|
||||||
|
"started_at": cpd._ts(started_at),
|
||||||
|
"last_heartbeat_at": cpd._ts(hb),
|
||||||
|
"client_managed": client_managed,
|
||||||
|
}
|
||||||
|
if owner_process_started_at is not None:
|
||||||
|
row["owner_process_started_at"] = cpd._ts(owner_process_started_at)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def _alive(pids: set[int]):
|
||||||
|
def _check(pid):
|
||||||
|
try:
|
||||||
|
return int(pid) in pids
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return _check
|
||||||
|
|
||||||
|
|
||||||
|
def _starts(mapping: dict[int, datetime]):
|
||||||
|
def _probe(pid):
|
||||||
|
try:
|
||||||
|
return mapping.get(int(pid))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
return _probe
|
||||||
|
|
||||||
|
|
||||||
|
class ClassifyDeadOwnerTests(unittest.TestCase):
|
||||||
|
def test_dead_owner_is_stale_and_retireable(self) -> None:
|
||||||
|
c = sl.classify_session(
|
||||||
|
_session("ghost", pid=2_000_000_000),
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive(set()),
|
||||||
|
process_start_probe=_starts({}),
|
||||||
|
)
|
||||||
|
self.assertEqual(c.classification, sl.CLASS_STALE)
|
||||||
|
self.assertTrue(c.retireable)
|
||||||
|
self.assertIn(c.reason, {sl.REASON_DEAD_OWNER, sl.REASON_HEARTBEAT_STALE_DEAD})
|
||||||
|
|
||||||
|
def test_missing_pid_is_stale(self) -> None:
|
||||||
|
c = sl.classify_session(
|
||||||
|
_session("no-pid", pid=None),
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive(set()),
|
||||||
|
)
|
||||||
|
self.assertEqual(c.classification, sl.CLASS_STALE)
|
||||||
|
self.assertEqual(c.reason, sl.REASON_MISSING_PID)
|
||||||
|
self.assertTrue(c.retireable)
|
||||||
|
|
||||||
|
|
||||||
|
class PidReuseTests(unittest.TestCase):
|
||||||
|
def test_pid_reuse_marks_stale_not_live(self) -> None:
|
||||||
|
# Process with same PID started AFTER the session was recorded.
|
||||||
|
c = sl.classify_session(
|
||||||
|
_session("reused", pid=77, started_at=EARLIER, last_heartbeat_at=EARLIER),
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive({77}),
|
||||||
|
process_start_probe=_starts({77: LATER}),
|
||||||
|
)
|
||||||
|
self.assertEqual(c.classification, sl.CLASS_STALE)
|
||||||
|
self.assertEqual(c.reason, sl.REASON_PID_REUSE)
|
||||||
|
self.assertTrue(c.pid_reused)
|
||||||
|
self.assertTrue(c.retireable)
|
||||||
|
|
||||||
|
def test_matching_process_start_is_live(self) -> None:
|
||||||
|
c = sl.classify_session(
|
||||||
|
_session(
|
||||||
|
"same-proc",
|
||||||
|
pid=88,
|
||||||
|
started_at=EARLIER,
|
||||||
|
last_heartbeat_at=NOW - timedelta(seconds=30),
|
||||||
|
owner_process_started_at=EARLIER - timedelta(seconds=5),
|
||||||
|
),
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive({88}),
|
||||||
|
process_start_probe=_starts({88: EARLIER - timedelta(seconds=5)}),
|
||||||
|
)
|
||||||
|
self.assertEqual(c.classification, sl.CLASS_LIVE)
|
||||||
|
self.assertFalse(c.retireable)
|
||||||
|
|
||||||
|
|
||||||
|
class LiveOwnerAndLeaseTests(unittest.TestCase):
|
||||||
|
def test_live_owner_not_retired(self) -> None:
|
||||||
|
c = sl.classify_session(
|
||||||
|
_session(
|
||||||
|
"live",
|
||||||
|
pid=os.getpid(),
|
||||||
|
last_heartbeat_at=NOW - timedelta(seconds=10),
|
||||||
|
),
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive({os.getpid()}),
|
||||||
|
process_start_probe=_starts({os.getpid(): EARLIER}),
|
||||||
|
)
|
||||||
|
self.assertEqual(c.classification, sl.CLASS_LIVE)
|
||||||
|
self.assertFalse(c.retireable)
|
||||||
|
|
||||||
|
def test_live_lease_blocks_retirement_even_if_pid_dead(self) -> None:
|
||||||
|
c = sl.classify_session(
|
||||||
|
_session("leased", pid=99999),
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive(set()),
|
||||||
|
live_lease_sessions={"leased"},
|
||||||
|
)
|
||||||
|
self.assertEqual(c.classification, sl.CLASS_PROTECTED)
|
||||||
|
self.assertEqual(c.reason, sl.REASON_LIVE_LEASE)
|
||||||
|
self.assertFalse(c.retireable)
|
||||||
|
|
||||||
|
def test_client_managed_live_never_retired(self) -> None:
|
||||||
|
c = sl.classify_session(
|
||||||
|
_session(
|
||||||
|
"client",
|
||||||
|
pid=55,
|
||||||
|
client_managed=True,
|
||||||
|
last_heartbeat_at=NOW - timedelta(seconds=5),
|
||||||
|
),
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive({55}),
|
||||||
|
process_start_probe=_starts({55: EARLIER}),
|
||||||
|
)
|
||||||
|
self.assertEqual(c.classification, sl.CLASS_LIVE)
|
||||||
|
self.assertEqual(c.reason, sl.REASON_CLIENT_MANAGED_LIVE)
|
||||||
|
self.assertFalse(c.retireable)
|
||||||
|
|
||||||
|
def test_client_managed_dead_pid_is_retireable(self) -> None:
|
||||||
|
# Dead client process is not a live client-managed session.
|
||||||
|
c = sl.classify_session(
|
||||||
|
_session("client-dead", pid=56, client_managed=True),
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive(set()),
|
||||||
|
)
|
||||||
|
self.assertEqual(c.classification, sl.CLASS_STALE)
|
||||||
|
self.assertTrue(c.retireable)
|
||||||
|
|
||||||
|
|
||||||
|
class TerminalAndDisconnectedTests(unittest.TestCase):
|
||||||
|
def test_already_terminal_not_retireable(self) -> None:
|
||||||
|
c = sl.classify_session(
|
||||||
|
_session("done", status="retired", pid=1),
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive(set()),
|
||||||
|
)
|
||||||
|
self.assertEqual(c.classification, sl.CLASS_TERMINAL)
|
||||||
|
self.assertFalse(c.retireable)
|
||||||
|
|
||||||
|
def test_alive_stale_heartbeat_is_disconnected_not_retired(self) -> None:
|
||||||
|
c = sl.classify_session(
|
||||||
|
_session(
|
||||||
|
"quiet",
|
||||||
|
pid=66,
|
||||||
|
last_heartbeat_at=NOW - timedelta(hours=5),
|
||||||
|
),
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive({66}),
|
||||||
|
process_start_probe=_starts({66: EARLIER}),
|
||||||
|
)
|
||||||
|
self.assertEqual(c.classification, sl.CLASS_DISCONNECTED)
|
||||||
|
self.assertFalse(c.retireable)
|
||||||
|
|
||||||
|
|
||||||
|
class FleetAndApplyTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self._tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db_path = os.path.join(self._tmp.name, "cp.sqlite3")
|
||||||
|
self.db = cpd.ControlPlaneDB(self.db_path)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self._tmp.cleanup()
|
||||||
|
|
||||||
|
def test_mixed_fleet_and_apply_retires_only_stale(self) -> None:
|
||||||
|
live_pid = os.getpid()
|
||||||
|
# Use wall-clock "now" so upsert timestamps align with classification.
|
||||||
|
moment = datetime.now(timezone.utc)
|
||||||
|
proc_start = moment - timedelta(hours=1)
|
||||||
|
self.db.upsert_session(
|
||||||
|
session_id="s-live",
|
||||||
|
role="author",
|
||||||
|
pid=live_pid,
|
||||||
|
status="active",
|
||||||
|
owner_process_started_at=cpd._ts(proc_start),
|
||||||
|
)
|
||||||
|
self.db.upsert_session(
|
||||||
|
session_id="s-dead", role="reviewer", pid=2_000_000_001, status="active"
|
||||||
|
)
|
||||||
|
self.db.upsert_session(
|
||||||
|
session_id="s-ended", role="merger", pid=3, status="ended"
|
||||||
|
)
|
||||||
|
|
||||||
|
sessions = self.db.list_sessions(limit=50)
|
||||||
|
report = sl.classify_sessions(
|
||||||
|
sessions,
|
||||||
|
leases=[],
|
||||||
|
now=moment,
|
||||||
|
pid_checker=_alive({live_pid}),
|
||||||
|
process_start_probe=_starts({live_pid: proc_start}),
|
||||||
|
)
|
||||||
|
self.assertGreaterEqual(report.stale_count, 1)
|
||||||
|
self.assertTrue(
|
||||||
|
any(c.session_id == "s-dead" and c.retireable for c in report.classifications)
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
any(
|
||||||
|
c.session_id == "s-live" and not c.retireable
|
||||||
|
for c in report.classifications
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
first = sl.apply_session_retirements(
|
||||||
|
self.db, report, dry_run=False, actor_session_id="actor-1", now=moment
|
||||||
|
)
|
||||||
|
self.assertGreaterEqual(first["retired_count"], 1)
|
||||||
|
# After retirement, active list should exclude s-dead.
|
||||||
|
active = {
|
||||||
|
s["session_id"]
|
||||||
|
for s in self.db.list_sessions(statuses=("active",), limit=50)
|
||||||
|
}
|
||||||
|
self.assertNotIn("s-dead", active)
|
||||||
|
self.assertIn("s-live", active)
|
||||||
|
|
||||||
|
# Repeated cleanup is idempotent.
|
||||||
|
report2 = sl.classify_sessions(
|
||||||
|
self.db.list_sessions(limit=50),
|
||||||
|
leases=[],
|
||||||
|
now=moment,
|
||||||
|
pid_checker=_alive({live_pid}),
|
||||||
|
process_start_probe=_starts({live_pid: proc_start}),
|
||||||
|
)
|
||||||
|
second = sl.apply_session_retirements(
|
||||||
|
self.db, report2, dry_run=False, actor_session_id="actor-1", now=moment
|
||||||
|
)
|
||||||
|
# No double-retirement of the same row as a new mutation.
|
||||||
|
self.assertEqual(second["retired_count"], 0)
|
||||||
|
|
||||||
|
# Durable audit event present.
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
conn = sqlite3.connect(self.db_path)
|
||||||
|
try:
|
||||||
|
events = conn.execute(
|
||||||
|
"SELECT event_type, message FROM events WHERE event_type = ?",
|
||||||
|
("session_retired",),
|
||||||
|
).fetchall()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
self.assertTrue(events)
|
||||||
|
self.assertTrue(any("s-dead" in (m or "") for _, m in events))
|
||||||
|
|
||||||
|
def test_live_lease_blocks_db_retirement(self) -> None:
|
||||||
|
self.db.upsert_session(
|
||||||
|
session_id="s-leased", role="author", pid=2_000_000_002, status="active"
|
||||||
|
)
|
||||||
|
self.db.upsert_work_item(
|
||||||
|
remote="prgs",
|
||||||
|
org="org",
|
||||||
|
repo="repo",
|
||||||
|
kind="issue",
|
||||||
|
number=969,
|
||||||
|
)
|
||||||
|
result = self.db.assign_and_lease(
|
||||||
|
session_id="s-leased",
|
||||||
|
role="author",
|
||||||
|
remote="prgs",
|
||||||
|
org="org",
|
||||||
|
repo="repo",
|
||||||
|
kind="issue",
|
||||||
|
number=969,
|
||||||
|
)
|
||||||
|
self.assertEqual(result.outcome, "assigned")
|
||||||
|
|
||||||
|
# Inventory-style lease with explicit live freshness (authoritative for
|
||||||
|
# the pure classifier). DB apply also blocks on the active lease row.
|
||||||
|
leases = [
|
||||||
|
{
|
||||||
|
"lease_id": result.lease_id,
|
||||||
|
"session_id": "s-leased",
|
||||||
|
"status": "active",
|
||||||
|
"freshness": {"freshness": "active"},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
report = sl.classify_sessions(
|
||||||
|
self.db.list_sessions(statuses=("active",), limit=20),
|
||||||
|
leases=leases,
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive(set()),
|
||||||
|
)
|
||||||
|
# Classifier protects via live lease set.
|
||||||
|
self.assertTrue(
|
||||||
|
any(
|
||||||
|
c.session_id == "s-leased" and c.classification == sl.CLASS_PROTECTED
|
||||||
|
for c in report.classifications
|
||||||
|
)
|
||||||
|
)
|
||||||
|
apply = sl.apply_session_retirements(
|
||||||
|
self.db, report, dry_run=False, actor_session_id="actor", now=NOW
|
||||||
|
)
|
||||||
|
self.assertEqual(apply["retired_count"], 0)
|
||||||
|
active = {
|
||||||
|
s["session_id"]
|
||||||
|
for s in self.db.list_sessions(statuses=("active",), limit=20)
|
||||||
|
}
|
||||||
|
self.assertIn("s-leased", active)
|
||||||
|
|
||||||
|
# Direct DB CAS also refuses while an active lease row remains.
|
||||||
|
blocked = self.db.retire_session(
|
||||||
|
session_id="s-leased",
|
||||||
|
reason=sl.REASON_DEAD_OWNER,
|
||||||
|
actor_session_id="actor",
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
self.assertEqual(blocked["outcome"], "blocked")
|
||||||
|
self.assertEqual(blocked["reason"], "live_lease")
|
||||||
|
|
||||||
|
def test_concurrent_retirement_is_idempotent(self) -> None:
|
||||||
|
for i in range(20):
|
||||||
|
self.db.upsert_session(
|
||||||
|
session_id=f"ghost-{i}",
|
||||||
|
role="author",
|
||||||
|
pid=3_000_000 + i,
|
||||||
|
status="active",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _worker() -> dict:
|
||||||
|
return sl.retire_stale_sessions(
|
||||||
|
self.db,
|
||||||
|
dry_run=False,
|
||||||
|
actor_session_id=f"actor-{threading.get_ident()}",
|
||||||
|
now=NOW,
|
||||||
|
pid_checker=_alive(set()),
|
||||||
|
process_start_probe=_starts({}),
|
||||||
|
session_limit=100,
|
||||||
|
)
|
||||||
|
|
||||||
|
outcomes = []
|
||||||
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||||
|
futs = [pool.submit(_worker) for _ in range(4)]
|
||||||
|
for fut in as_completed(futs):
|
||||||
|
outcomes.append(fut.result())
|
||||||
|
|
||||||
|
total_retired = sum(o["apply"]["retired_count"] for o in outcomes)
|
||||||
|
# Exactly one successful retirement per ghost row across all workers.
|
||||||
|
self.assertEqual(total_retired, 20)
|
||||||
|
active = {
|
||||||
|
s["session_id"]
|
||||||
|
for s in self.db.list_sessions(statuses=("active",), limit=100)
|
||||||
|
}
|
||||||
|
for i in range(20):
|
||||||
|
self.assertNotIn(f"ghost-{i}", active)
|
||||||
|
|
||||||
|
|
||||||
|
class ReconcileIntegrationTests(unittest.TestCase):
|
||||||
|
def test_unresolved_until_retired_then_resolved(self) -> None:
|
||||||
|
inv = {
|
||||||
|
"inventory_complete": True,
|
||||||
|
"incomplete_reasons": [],
|
||||||
|
"service_health": {"healthy": True},
|
||||||
|
"clients": [{"session_id": "c1", "connected": True}],
|
||||||
|
"sessions": [
|
||||||
|
_session("ghost", pid=2_000_000_099, last_heartbeat_at=EARLIER),
|
||||||
|
],
|
||||||
|
"leases": [],
|
||||||
|
"checkpoints_available": False,
|
||||||
|
"worktree_bindings": [],
|
||||||
|
"pending_mutations": [],
|
||||||
|
"capabilities": {"stale": False},
|
||||||
|
"boot_head_sha": "a" * 40,
|
||||||
|
"current_head_sha": "a" * 40,
|
||||||
|
"queue_state": {"safe_to_resume": True},
|
||||||
|
}
|
||||||
|
proof = prr.reconcile_after_restart(inv, now=NOW, mode=prr.MODE_LOG_ONLY)
|
||||||
|
sess = next(i for i in proof.items if i.dimension == prr.DIM_SESSIONS)
|
||||||
|
self.assertEqual(sess.status, prr.ITEM_UNRESOLVED)
|
||||||
|
self.assertIn("ghost", sess.details.get("orphan_session_ids") or [])
|
||||||
|
|
||||||
|
# After retirement inventory (no active orphans) resolves.
|
||||||
|
inv2 = dict(inv)
|
||||||
|
inv2["sessions"] = []
|
||||||
|
inv2["session_fleet"] = {
|
||||||
|
"retireable_session_ids": [],
|
||||||
|
"sessions_dimension_resolved": True,
|
||||||
|
"live_count": 0,
|
||||||
|
"stale_count": 0,
|
||||||
|
}
|
||||||
|
proof2 = prr.reconcile_after_restart(inv2, now=NOW, mode=prr.MODE_LOG_ONLY)
|
||||||
|
sess2 = next(i for i in proof2.items if i.dimension == prr.DIM_SESSIONS)
|
||||||
|
self.assertEqual(sess2.status, prr.ITEM_RESOLVED)
|
||||||
|
|
||||||
|
def test_legacy_orphan_key_still_populated(self) -> None:
|
||||||
|
inv = {
|
||||||
|
"inventory_complete": True,
|
||||||
|
"service_health": {"healthy": True},
|
||||||
|
"clients": [],
|
||||||
|
"sessions": [_session("ghost", pid=2_000_000_100)],
|
||||||
|
"leases": [],
|
||||||
|
"checkpoints_available": False,
|
||||||
|
"worktree_bindings": [],
|
||||||
|
"pending_mutations": [],
|
||||||
|
"capabilities": {"stale": False},
|
||||||
|
"boot_head_sha": "a" * 40,
|
||||||
|
"current_head_sha": "a" * 40,
|
||||||
|
"queue_state": {"safe_to_resume": True},
|
||||||
|
}
|
||||||
|
proof = prr.reconcile_after_restart(inv, now=NOW)
|
||||||
|
sess = next(i for i in proof.items if i.dimension == prr.DIM_SESSIONS)
|
||||||
|
self.assertIn("orphan_session_ids", sess.details)
|
||||||
|
|
||||||
|
|
||||||
|
class SchemaMigrationTests(unittest.TestCase):
|
||||||
|
def test_lifecycle_columns_present(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = os.path.join(tmp, "cp.sqlite3")
|
||||||
|
db = cpd.ControlPlaneDB(path)
|
||||||
|
db.upsert_session(session_id="s1", role="author", pid=1)
|
||||||
|
out = db.retire_session(
|
||||||
|
session_id="s1",
|
||||||
|
reason=sl.REASON_DEAD_OWNER,
|
||||||
|
actor_session_id="tester",
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
self.assertEqual(out["outcome"], "retired")
|
||||||
|
rows = db.list_sessions(limit=5)
|
||||||
|
# May not appear under active filter
|
||||||
|
all_rows = db.list_sessions(limit=5)
|
||||||
|
# Re-open raw to check columns
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
conn = sqlite3.connect(path)
|
||||||
|
try:
|
||||||
|
cols = {r[1] for r in conn.execute("PRAGMA table_info(sessions)")}
|
||||||
|
version = conn.execute(
|
||||||
|
"SELECT value FROM schema_meta WHERE key='schema_version'"
|
||||||
|
).fetchone()[0]
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
self.assertIn("retired_at", cols)
|
||||||
|
self.assertIn("retire_reason", cols)
|
||||||
|
self.assertIn("owner_process_started_at", cols)
|
||||||
|
self.assertEqual(version, "6")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user