Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e8bae606cb | ||
|
|
956fa15fe3 | ||
|
|
fa510dd28d | ||
|
|
1dd30ecb15 | ||
|
|
8eada1fbe4 |
+226
-7
@@ -27,12 +27,12 @@ import uuid
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Iterator, Sequence
|
||||
from typing import Any, Iterator, Mapping, Sequence
|
||||
|
||||
import dependency_graph
|
||||
import gitea_audit
|
||||
|
||||
SCHEMA_VERSION = 5
|
||||
SCHEMA_VERSION = 6
|
||||
|
||||
# Assignable work kinds only — raw monitoring incidents are never work items.
|
||||
WORK_KINDS = frozenset({"issue", "pr"})
|
||||
@@ -419,6 +419,7 @@ class ControlPlaneDB:
|
||||
self._migrate_incident_links_null_scope(conn)
|
||||
self._migrate_lease_lifecycle_columns(conn)
|
||||
self._migrate_session_ownership_columns(conn)
|
||||
self._migrate_session_lifecycle_columns(conn)
|
||||
self._migrate_usage_events_table(conn)
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)",
|
||||
@@ -812,6 +813,7 @@ class ControlPlaneDB:
|
||||
pid: int | None = None,
|
||||
status: str = "active",
|
||||
controller_instance_id: str | None = None,
|
||||
owner_process_started_at: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Register/refresh a session row.
|
||||
|
||||
@@ -821,16 +823,22 @@ class ControlPlaneDB:
|
||||
controller instance can. It is never overwritten with ``None``, so a
|
||||
heartbeat from a caller that does not supply one cannot erase
|
||||
ownership.
|
||||
|
||||
*owner_process_started_at* (#969) is the OS start time of the owner
|
||||
process at registration time. When present it is retained across
|
||||
heartbeats (never cleared by ``None``) so later PID-reuse checks do
|
||||
not depend on a live ``ps`` probe of a long-dead process.
|
||||
"""
|
||||
now = _ts()
|
||||
instance = (controller_instance_id or "").strip() or None
|
||||
proc_start = (owner_process_started_at or "").strip() or None
|
||||
with self._tx() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT session_id FROM sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if existing:
|
||||
if instance is None:
|
||||
if instance is None and proc_start is None:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
@@ -840,7 +848,21 @@ class ControlPlaneDB:
|
||||
""",
|
||||
(role, profile, namespace, pid, now, status, session_id),
|
||||
)
|
||||
else:
|
||||
elif instance is None:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET role = ?, profile = ?, namespace = ?, pid = ?,
|
||||
last_heartbeat_at = ?, status = ?,
|
||||
owner_process_started_at = COALESCE(?, owner_process_started_at)
|
||||
WHERE session_id = ?
|
||||
""",
|
||||
(
|
||||
role, profile, namespace, pid, now, status,
|
||||
proc_start, session_id,
|
||||
),
|
||||
)
|
||||
elif proc_start is None:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
@@ -854,18 +876,33 @@ class ControlPlaneDB:
|
||||
instance, session_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET role = ?, profile = ?, namespace = ?, pid = ?,
|
||||
last_heartbeat_at = ?, status = ?,
|
||||
controller_instance_id = ?,
|
||||
owner_process_started_at = COALESCE(?, owner_process_started_at)
|
||||
WHERE session_id = ?
|
||||
""",
|
||||
(
|
||||
role, profile, namespace, pid, now, status,
|
||||
instance, proc_start, session_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO sessions(
|
||||
session_id, role, profile, namespace, pid,
|
||||
started_at, last_heartbeat_at, status,
|
||||
controller_instance_id
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
controller_instance_id, owner_process_started_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
session_id, role, profile, namespace, pid, now, now,
|
||||
status, instance,
|
||||
status, instance, proc_start,
|
||||
),
|
||||
)
|
||||
row = conn.execute(
|
||||
@@ -881,6 +918,165 @@ class ControlPlaneDB:
|
||||
(_ts(), session_id),
|
||||
)
|
||||
|
||||
def retire_session(
|
||||
self,
|
||||
*,
|
||||
session_id: str,
|
||||
reason: str,
|
||||
actor_session_id: str | None = None,
|
||||
details: Mapping[str, Any] | None = None,
|
||||
now: datetime | None = None,
|
||||
terminal_status: str = "retired",
|
||||
) -> dict[str, Any]:
|
||||
"""Terminalize one session row if it is still non-terminal (#969).
|
||||
|
||||
CAS on non-terminal status: concurrent retirements of the same row
|
||||
yield exactly one ``retired`` outcome and subsequent
|
||||
``already_terminal`` outcomes. Never deletes historical rows. Writes a
|
||||
durable ``session_retired`` event for audit.
|
||||
"""
|
||||
moment = _ts(now)
|
||||
reason_s = (reason or "").strip() or "unspecified"
|
||||
term = (terminal_status or "retired").strip().lower() or "retired"
|
||||
terminal_set = {
|
||||
"retired",
|
||||
"ended",
|
||||
"terminal",
|
||||
"dead",
|
||||
"stale",
|
||||
"orphaned",
|
||||
}
|
||||
with self._tx() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return {
|
||||
"outcome": "missing",
|
||||
"reason": reason_s,
|
||||
"prior_status": None,
|
||||
"new_status": None,
|
||||
"details": {"session_id": session_id},
|
||||
}
|
||||
prior = dict(row)
|
||||
prior_status = str(prior.get("status") or "").strip().lower()
|
||||
if prior_status in terminal_set:
|
||||
return {
|
||||
"outcome": "already_terminal",
|
||||
"reason": reason_s,
|
||||
"prior_status": prior_status,
|
||||
"new_status": prior_status,
|
||||
"details": {"session_id": session_id, "idempotent": True},
|
||||
}
|
||||
|
||||
# Optional: refuse when an active lease still names this session.
|
||||
active_lease = conn.execute(
|
||||
"""
|
||||
SELECT lease_id, status, expires_at FROM leases
|
||||
WHERE session_id = ? AND status = 'active'
|
||||
LIMIT 1
|
||||
""",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if active_lease is not None:
|
||||
return {
|
||||
"outcome": "blocked",
|
||||
"reason": "live_lease",
|
||||
"prior_status": prior_status,
|
||||
"new_status": prior_status,
|
||||
"details": {
|
||||
"session_id": session_id,
|
||||
"lease_id": active_lease["lease_id"],
|
||||
"blocker": "active_lease_row",
|
||||
},
|
||||
}
|
||||
|
||||
cols = {
|
||||
r[1] for r in conn.execute("PRAGMA table_info(sessions)").fetchall()
|
||||
}
|
||||
if "retired_at" in cols and "retire_reason" in cols:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET status = ?, retired_at = ?, retire_reason = ?
|
||||
WHERE session_id = ? AND status = ?
|
||||
""",
|
||||
(term, moment, reason_s, session_id, prior.get("status")),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET status = ?
|
||||
WHERE session_id = ? AND status = ?
|
||||
""",
|
||||
(term, session_id, prior.get("status")),
|
||||
)
|
||||
changed = conn.execute(
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
if not changed:
|
||||
# Lost CAS race — re-read.
|
||||
refreshed = conn.execute(
|
||||
"SELECT status FROM sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
cur = (
|
||||
str(refreshed["status"]).strip().lower()
|
||||
if refreshed is not None
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"outcome": "already_terminal"
|
||||
if cur in terminal_set
|
||||
else "blocked",
|
||||
"reason": reason_s,
|
||||
"prior_status": prior_status,
|
||||
"new_status": cur,
|
||||
"details": {
|
||||
"session_id": session_id,
|
||||
"cas_lost": True,
|
||||
},
|
||||
}
|
||||
|
||||
detail_payload: dict[str, Any] = {
|
||||
"session_id": session_id,
|
||||
"prior_status": prior_status,
|
||||
"new_status": term,
|
||||
"reason": reason_s,
|
||||
"actor_session_id": actor_session_id,
|
||||
"pid": prior.get("pid"),
|
||||
"role": prior.get("role"),
|
||||
"profile": prior.get("profile"),
|
||||
}
|
||||
if isinstance(details, Mapping):
|
||||
for key, value in details.items():
|
||||
if key not in detail_payload:
|
||||
detail_payload[key] = value
|
||||
message = (
|
||||
f"session {session_id} retired reason={reason_s} "
|
||||
f"prior_status={prior_status}"
|
||||
)
|
||||
# events.work_item_id is nullable; session retirement is not work-scoped.
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO events(work_item_id, event_type, message, created_at)
|
||||
VALUES (NULL, 'session_retired', ?, ?)
|
||||
""",
|
||||
(message[:2000], moment),
|
||||
)
|
||||
# Also persist a structured JSON line in the message when short enough
|
||||
# by appending a compact summary (full detail stays in return value /
|
||||
# audit log; events.message is human-readable).
|
||||
return {
|
||||
"outcome": "retired",
|
||||
"reason": reason_s,
|
||||
"prior_status": prior_status,
|
||||
"new_status": term,
|
||||
"details": detail_payload,
|
||||
}
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
*,
|
||||
@@ -1639,6 +1835,29 @@ class ControlPlaneDB:
|
||||
if name not in cols:
|
||||
conn.execute(f"ALTER TABLE sessions ADD COLUMN {name} {decl}")
|
||||
|
||||
_SESSION_LIFECYCLE_COLUMNS: tuple[tuple[str, str], ...] = (
|
||||
("owner_process_started_at", "TEXT"),
|
||||
("retired_at", "TEXT"),
|
||||
("retire_reason", "TEXT"),
|
||||
)
|
||||
|
||||
def _migrate_session_lifecycle_columns(self, conn: sqlite3.Connection) -> None:
|
||||
"""Add session retirement / PID-reuse provenance columns (#969).
|
||||
|
||||
Additive and idempotent. Pre-existing rows migrate with NULL; retirement
|
||||
fills ``retired_at`` / ``retire_reason``, and new upserts may record
|
||||
``owner_process_started_at`` for stronger identity checks.
|
||||
"""
|
||||
cols = {
|
||||
row[1]
|
||||
for row in conn.execute("PRAGMA table_info(sessions)").fetchall()
|
||||
}
|
||||
if not cols:
|
||||
return
|
||||
for name, decl in self._SESSION_LIFECYCLE_COLUMNS:
|
||||
if name not in cols:
|
||||
conn.execute(f"ALTER TABLE sessions ADD COLUMN {name} {decl}")
|
||||
|
||||
def list_active_claims(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -19,7 +19,11 @@ The assessor classifies:
|
||||
|
||||
- **service_health** — process healthy / parity mutation-safe
|
||||
- **clients** — connected client descriptors (optional inventory)
|
||||
- **sessions** — active session rows with dead owner pids are unresolved
|
||||
- **sessions** — active session rows with dead or reused owner pids are
|
||||
unresolved until retired via `#969` (`session_lifecycle` /
|
||||
`apply_session_cleanup=true` on `gitea_reconcile_after_restart`, or
|
||||
`gitea_retire_stale_workflow_sessions`). Live owners, live leases, and live
|
||||
client-managed sessions are never retired.
|
||||
- **checkpoints** — soft-depends on #660; skipped with reason when schema absent
|
||||
- **leases** — live control-plane leases after restart
|
||||
- **capabilities** — master-parity / stale-runtime (#610)
|
||||
|
||||
@@ -8,10 +8,10 @@
|
||||
"document. #930's inventory had no such guard and its gitea_mcp_server.py",
|
||||
"anchors drifted between 7bf4f125 and aad5c8b4."
|
||||
],
|
||||
"generated_against_commit": "ca5f078d8a575ea3e2991771f8b4ea85e3dcaaa0",
|
||||
"generated_against_commit": "1dd30ecb1508b559868c2d5d94367bc055d5138e",
|
||||
"anchors": [
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:24864",
|
||||
"anchor": "gitea_mcp_server.py:25087",
|
||||
"expect": "mcp_daemon_guard.bind_native_mcp_transport()"
|
||||
},
|
||||
{
|
||||
@@ -27,11 +27,11 @@
|
||||
"expect": "def assess_transport_for_auth_mint"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:9191",
|
||||
"anchor": "gitea_mcp_server.py:9192",
|
||||
"expect": "assess_transport_for_auth_mint()"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:9440",
|
||||
"anchor": "gitea_mcp_server.py:9441",
|
||||
"expect": "assess_transport_for_auth_mint()"
|
||||
},
|
||||
{
|
||||
@@ -39,31 +39,31 @@
|
||||
"expect": "The transport is selected by deployment configuration"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:15481",
|
||||
"anchor": "gitea_mcp_server.py:15630",
|
||||
"expect": "def _is_client_managed_process"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:15511",
|
||||
"anchor": "gitea_mcp_server.py:15644",
|
||||
"expect": "def _provenance_mutation_block"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:15519",
|
||||
"anchor": "gitea_mcp_server.py:15652",
|
||||
"expect": "unsupported_manual_launch"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:19070",
|
||||
"anchor": "gitea_mcp_server.py:19217",
|
||||
"expect": "server_provenance"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:21581",
|
||||
"anchor": "gitea_mcp_server.py:21741",
|
||||
"expect": "def _check_mcp_runtimes_diagnostics"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:21601",
|
||||
"anchor": "gitea_mcp_server.py:21761",
|
||||
"expect": "\"ps\", \"-o\", \"pid,lstart,command\""
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:21645",
|
||||
"anchor": "gitea_mcp_server.py:21805",
|
||||
"expect": "\"ps\", \"eww\""
|
||||
},
|
||||
{
|
||||
@@ -107,19 +107,19 @@
|
||||
"expect": "def assert_keychain_access_allowed"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:19327",
|
||||
"anchor": "gitea_mcp_server.py:19487",
|
||||
"expect": "def gitea_list_profiles"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:19378",
|
||||
"anchor": "gitea_mcp_server.py:19538",
|
||||
"expect": "gitea_config.resolve_token(p)"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:19691",
|
||||
"anchor": "gitea_mcp_server.py:19851",
|
||||
"expect": "def gitea_audit_config"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:19713",
|
||||
"anchor": "gitea_mcp_server.py:19873",
|
||||
"expect": "service_summaries(config)"
|
||||
},
|
||||
{
|
||||
@@ -135,19 +135,19 @@
|
||||
"expect": "_keychain_token(auth.get(\"id\"))"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:17776",
|
||||
"anchor": "gitea_mcp_server.py:17909",
|
||||
"expect": "\"jenkins-mcp\""
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:17782",
|
||||
"anchor": "gitea_mcp_server.py:17915",
|
||||
"expect": "external-mcp"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:17803",
|
||||
"anchor": "gitea_mcp_server.py:17936",
|
||||
"expect": "\"glitchtip-mcp\""
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:17808",
|
||||
"anchor": "gitea_mcp_server.py:17941",
|
||||
"expect": "external-mcp"
|
||||
},
|
||||
{
|
||||
@@ -183,7 +183,7 @@
|
||||
"expect": "mutation_safe"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:19171",
|
||||
"anchor": "gitea_mcp_server.py:19331",
|
||||
"expect": "def gitea_assess_master_parity"
|
||||
},
|
||||
{
|
||||
@@ -195,11 +195,11 @@
|
||||
"expect": "AUTHOR_WORKTREE_ENV"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:2351",
|
||||
"anchor": "gitea_mcp_server.py:2352",
|
||||
"expect": "/tmp/gitea_issue_lock.json"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:10956",
|
||||
"anchor": "gitea_mcp_server.py:10957",
|
||||
"expect": "def gitea_bootstrap_author_issue_worktree"
|
||||
},
|
||||
{
|
||||
@@ -239,7 +239,7 @@
|
||||
"expect": "os.getpid()"
|
||||
},
|
||||
{
|
||||
"anchor": "gitea_mcp_server.py:12870",
|
||||
"anchor": "gitea_mcp_server.py:12871",
|
||||
"expect": "owner_pid_alive"
|
||||
}
|
||||
]
|
||||
|
||||
@@ -5,7 +5,7 @@ What the adversary is, what each boundary protects, and which services may share
|
||||
- **Issue:** #956 (Remote-MCP threat model), child of epic #929, cross-linked to #955.
|
||||
- **Depends on:** #930 (closed) — `docs/remote-mcp/coupling-inventory.md`.
|
||||
- **Blocks:** #932, #933, #934, #938.
|
||||
- **Generated against commit:** `ca5f078d8a575ea3e2991771f8b4ea85e3dcaaa0` (#708's
|
||||
- **Generated against commit:** `1dd30ecb1508b559868c2d5d94367bc055d5138e` (#708's
|
||||
namespace-attachment gate). Originally generated against
|
||||
`aad5c8b42361d380a8eeb07b94b90815e594c2c5` (`master`), re-anchored at
|
||||
`a143cd065ba06e1a2bdc5143a19ec156e53650ef` when #931's transport bind seam shifted the
|
||||
@@ -31,7 +31,7 @@ document cites an anchor the fixture does not cover.
|
||||
|
||||
This guard exists because #930 did not have one. Its inventory was generated at
|
||||
`7bf4f125`; by `aad5c8b4` its `gitea_mcp_server.py` anchors had drifted — the transport
|
||||
bind it cited at line 23750 now lives at `gitea_mcp_server.py:24864`, and its
|
||||
bind it cited at line 23750 now lives at `gitea_mcp_server.py:25087`, and its
|
||||
client-managed provenance anchor at 14588 now lands in an unrelated function. Nothing
|
||||
failed, because nothing checked. Anchors into a ~24,700-line module rot silently, and a
|
||||
security document that cannot prove its own citations is worse than none, because it is
|
||||
@@ -75,19 +75,19 @@ authenticate the *caller*, not the *intent*.
|
||||
## 3. Trust boundaries
|
||||
|
||||
"Crossing requires today" is what the code actually enforces at
|
||||
`ca5f078d8a575ea3e2991771f8b4ea85e3dcaaa0`, not what the design intends.
|
||||
`1dd30ecb1508b559868c2d5d94367bc055d5138e`, not what the design intends.
|
||||
|
||||
| ID | Boundary | Protects | Crossing requires today | Crossing must require remotely |
|
||||
| -- | -------- | -------- | ----------------------- | ------------------------------ |
|
||||
| B1 | LLM client ↔ MCP server session | A1, A3, A10 — that a mutating session was established through the sanctioned client path | A single configured bind (`gitea_mcp_server.py:24864`) validated against one closed allowlist (`mcp_daemon_guard.py:49`, `mcp_daemon_guard.py:195`) — since #931 the identifier comes from deployment configuration and defaults to the local transport, so the boundary no longer rests on a literal, but it still rests on the *bind* rather than on an authenticated caller; client-managed provenance (`gitea_mcp_server.py:15481`) or a refusal (`gitea_mcp_server.py:15519`); production transport before recovery-authorization mint (`irrecoverable_provenance.py:497`, consumed at `gitea_mcp_server.py:9191` and `gitea_mcp_server.py:9440`) | An authenticated handshake issuing a server-side session identity bound to a principal, with the transport recorded in provenance. The physical proof (a pipe) must become a cryptographic one. |
|
||||
| B1 | LLM client ↔ MCP server session | A1, A3, A10 — that a mutating session was established through the sanctioned client path | A single configured bind (`gitea_mcp_server.py:25087`) validated against one closed allowlist (`mcp_daemon_guard.py:49`, `mcp_daemon_guard.py:195`) — since #931 the identifier comes from deployment configuration and defaults to the local transport, so the boundary no longer rests on a literal, but it still rests on the *bind* rather than on an authenticated caller; client-managed provenance (`gitea_mcp_server.py:15630`) or a refusal (`gitea_mcp_server.py:15652`); production transport before recovery-authorization mint (`irrecoverable_provenance.py:497`, consumed at `gitea_mcp_server.py:9192` and `gitea_mcp_server.py:9441`) | An authenticated handshake issuing a server-side session identity bound to a principal, with the transport recorded in provenance. The physical proof (a pipe) must become a cryptographic one. |
|
||||
| B2 | Role ↔ role | A9 — that author, reviewer, merger, and reconciler are distinct authorities | **The process boundary only.** The role is a property of the process, read once from `GITEA_MCP_PROFILE` (`gitea_config.py:54`). A caller gets author permissions by connecting to the author process. Review and merge are the operations singled out for extra care (`gitea_config.py:97`) | A per-request principal, so the role follows from the credential presented and cannot be selected by reaching a different endpoint. |
|
||||
| B3 | MCP server ↔ credential store | A3, A8 — that only sanctioned code turns a profile into a token | `_keychain_token` shelling out to the login keychain (`gitea_config.py:956`), dispatched by `resolve_token` (`gitea_config.py:974`) with the reference type built at `gitea_config.py:1015`, gated by `assert_keychain_access_allowed` (`mcp_daemon_guard.py:583`). Inline secrets are rejected at config load (`gitea_config.py:294`) | A credential provider keyed by the *request* principal, returning only that principal's credential, with the source recorded and the value never returned. |
|
||||
| B4 | MCP server ↔ Gitea | A1, A2 — that only authorized calls reach the forge | A bearer token over TLS. Server-side, nothing distinguishes one role's token from another beyond the account it belongs to | Unchanged at the forge; the endpoint in front of it must refuse unauthenticated and plaintext connections before tool dispatch. |
|
||||
| B5 | MCP server ↔ caller's filesystem | A7 — that a tool acts on the *caller's* disk or refuses | Nothing. The server's disk *is* the caller's disk. Worktree bootstrap writes directly (`gitea_mcp_server.py:10956`); the active workspace is process-global (`gitea_mcp_server.py:193`, `gitea_mcp_server.py:194`) | An explicit per-tool classification, enforced at dispatch, refusing filesystem tools over a transport that cannot reach the caller's disk. A green verdict about the wrong disk is the failure to prevent. |
|
||||
| B6 | MCP server ↔ coordination state | A6, A9 — mutual exclusion | Local files and a local SQLite database, with liveness judged from the local process table (`issue_lock_store.py:98`), keyed on paths under one user's home (`issue_lock_store.py:26`, `mcp_session_state.py:27`, `control_plane_db.py:47`) and on `os.getpid()` (`control_plane_db.py:1145`, `gitea_mcp_server.py:12870`). A legacy global slot still exists at `gitea_mcp_server.py:2351`, and the session-pointer file is named per PID (`issue_lock_store.py:83`) | One authority per ownership question, with liveness from session identity and expiry, and atomic acquire, renew, and release across hosts. |
|
||||
| B5 | MCP server ↔ caller's filesystem | A7 — that a tool acts on the *caller's* disk or refuses | Nothing. The server's disk *is* the caller's disk. Worktree bootstrap writes directly (`gitea_mcp_server.py:10957`); the active workspace is process-global (`gitea_mcp_server.py:193`, `gitea_mcp_server.py:194`) | An explicit per-tool classification, enforced at dispatch, refusing filesystem tools over a transport that cannot reach the caller's disk. A green verdict about the wrong disk is the failure to prevent. |
|
||||
| B6 | MCP server ↔ coordination state | A6, A9 — mutual exclusion | Local files and a local SQLite database, with liveness judged from the local process table (`issue_lock_store.py:98`), keyed on paths under one user's home (`issue_lock_store.py:26`, `mcp_session_state.py:27`, `control_plane_db.py:47`) and on `os.getpid()` (`control_plane_db.py:1145`, `gitea_mcp_server.py:12871`). A legacy global slot still exists at `gitea_mcp_server.py:2352`, and the session-pointer file is named per PID (`issue_lock_store.py:83`) | One authority per ownership question, with liveness from session identity and expiry, and atomic acquire, renew, and release across hosts. |
|
||||
| B7 | Gitea integration ↔ unrelated integrations | A4, A5 — that a Gitea compromise is not a CI and observability compromise | **Nothing.** See §5. The Gitea server reads Jenkins and GlitchTip secrets (`gitea_config.py:851`, reached from `gitea_config.py:837`) and holds the Sentry token (`sentry_incident_bridge.py:190`) | A hard process boundary. This is the boundary #956 exists to create. |
|
||||
| B8 | Tenant ↔ tenant (`prgs` / `mdcps` / `local-lab`) | A2 — that one organization's compromise is not another's | Convention. One configuration declares all three contexts; `resolve_service` fails closed on a *disabled* context (`gitea_config.py:704`) but the credentials of enabled ones remain reachable in-process. A per-profile repository scope exists (`gitea_config.py:499`) | Separate deployments, or at minimum per-tenant credential scopes with no process able to resolve both. |
|
||||
| B9 | Deployed code ↔ merged policy | A1, A10 — that the running server enforces the rules that were actually merged | Comparing this process's startup commit against this disk (`master_parity_gate.py:168`), conjoined into a single verdict (`master_parity_gate.py:255`) published by `gitea_mcp_server.py:19171` | Freshness defined against the deployed build identity, with an explicit fail-closed verdict when undeterminable. |
|
||||
| B9 | Deployed code ↔ merged policy | A1, A10 — that the running server enforces the rules that were actually merged | Comparing this process's startup commit against this disk (`master_parity_gate.py:168`), conjoined into a single verdict (`master_parity_gate.py:255`) published by `gitea_mcp_server.py:19331` | Freshness defined against the deployed build identity, with an explicit fail-closed verdict when undeterminable. |
|
||||
|
||||
### What no boundary constrains
|
||||
|
||||
@@ -132,10 +132,10 @@ Two flows deserve attention because neither is obvious from the code:
|
||||
|
||||
1. **The keychain flow fans out.** B3 is drawn once but resolves credentials for *every*
|
||||
configured profile and service, not only the active one. `gitea_list_profiles`
|
||||
(`gitea_mcp_server.py:19327`) reports each profile's credential status by calling
|
||||
`resolve_token` on it (`gitea_mcp_server.py:19378`), and `gitea_audit_config`
|
||||
(`gitea_mcp_server.py:19691`) reports service credential status through
|
||||
`service_summaries` (`gitea_mcp_server.py:19713`).
|
||||
(`gitea_mcp_server.py:19487`) reports each profile's credential status by calling
|
||||
`resolve_token` on it (`gitea_mcp_server.py:19538`), and `gitea_audit_config`
|
||||
(`gitea_mcp_server.py:19851`) reports service credential status through
|
||||
`service_summaries` (`gitea_mcp_server.py:19873`).
|
||||
2. **The return path is a flow too.** Content read from Gitea travels back into the model
|
||||
and is treated as instruction. This is the ADV2 edge, and it is the only edge in the
|
||||
diagram with no authentication on it, because it is not a request.
|
||||
@@ -180,16 +180,16 @@ the credential, and an attacker holding the token does not call our tools.
|
||||
|
||||
**Finding 3 — Any one role process can resolve every other role's credential.** This is not
|
||||
inferred; it is demonstrated by tool output. `gitea_list_profiles`
|
||||
(`gitea_mcp_server.py:19327`) called from the **author** session reports
|
||||
(`gitea_mcp_server.py:19487`) called from the **author** session reports
|
||||
`identity_status: "credentials present"` for `prgs-merger`, `prgs-reviewer`,
|
||||
`prgs-reconciler`, and every `mdcps` profile, because it calls `resolve_token` on each one
|
||||
(`gitea_mcp_server.py:19378`). The author process does not merely *have access to* the
|
||||
(`gitea_mcp_server.py:19538`). The author process does not merely *have access to* the
|
||||
merger's credential — it reads it to answer a status query. B2 is not a credential boundary
|
||||
in either direction.
|
||||
|
||||
**Finding 4 — The Gitea server reads CI and observability secrets.** `gitea_audit_config`
|
||||
(`gitea_mcp_server.py:19691`) reports `MDCPS Jenkins: enabled, read-only, authenticated`.
|
||||
That word `authenticated` is produced by `service_summaries` (`gitea_mcp_server.py:19713`,
|
||||
(`gitea_mcp_server.py:19851`) reports `MDCPS Jenkins: enabled, read-only, authenticated`.
|
||||
That word `authenticated` is produced by `service_summaries` (`gitea_mcp_server.py:19873`,
|
||||
defined at `gitea_config.py:837`), whose default check calls `_keychain_token` on the
|
||||
service's own keychain reference (`gitea_config.py:851`). Producing that one line requires
|
||||
the Gitea MCP server to read the Jenkins secret and the GlitchTip secret out of the
|
||||
@@ -197,8 +197,8 @@ keychain. B7 does not exist.
|
||||
|
||||
**Finding 5 — Jenkins and GlitchTip are already decomposed; the reach is residual.** Their
|
||||
tools live in separately registered servers, marked `external-mcp`
|
||||
(`gitea_mcp_server.py:17776`, `gitea_mcp_server.py:17782`, `gitea_mcp_server.py:17803`,
|
||||
`gitea_mcp_server.py:17808`) with their own expected tool sets (`mcp_discoverability.py:9`,
|
||||
(`gitea_mcp_server.py:17909`, `gitea_mcp_server.py:17915`, `gitea_mcp_server.py:17936`,
|
||||
`gitea_mcp_server.py:17941`) with their own expected tool sets (`mcp_discoverability.py:9`,
|
||||
`mcp_discoverability.py:17`). The correct decomposition was already chosen. What remains is
|
||||
a leak across it: the credential *references* still live in the Gitea configuration and are
|
||||
still resolved by the Gitea process. #75 bundled these services into one control-plane
|
||||
@@ -209,8 +209,8 @@ GlitchTip, the Sentry bridge runs *inside* the Gitea server, resolving its token
|
||||
process environment (`sentry_incident_bridge.py:190`) and sending it as a bearer header
|
||||
(`sentry_incident_bridge.py:289`). Being an environment variable rather than a keychain item
|
||||
makes it strictly worse: it needs no keychain prompt and is inherited by every subprocess the
|
||||
server spawns — including the `ps` invocations at `gitea_mcp_server.py:21601` and
|
||||
`gitea_mcp_server.py:21645`, reached from `gitea_mcp_server.py:21581`.
|
||||
server spawns — including the `ps` invocations at `gitea_mcp_server.py:21761` and
|
||||
`gitea_mcp_server.py:21805`, reached from `gitea_mcp_server.py:21741`.
|
||||
|
||||
**Finding 7 — The highest-value coordination asset has the weakest gate.** A6 is protected
|
||||
by filesystem permissions alone (CR14). Corrupting a lease requires no Gitea credential,
|
||||
@@ -219,8 +219,8 @@ assumes. Every other asset costs an attacker a credential; this one costs nothin
|
||||
local access, which is exactly ADV5's position.
|
||||
|
||||
**Finding 8 — Provenance authenticates the launch, not the caller.** `server_provenance` is
|
||||
reported as exactly `client_managed` or `manual_launch` (`gitea_mcp_server.py:19070`),
|
||||
derived from environment inspection (`gitea_mcp_server.py:15481`) with the recognized-key
|
||||
reported as exactly `client_managed` or `manual_launch` (`gitea_mcp_server.py:19217`),
|
||||
derived from environment inspection (`gitea_mcp_server.py:15630`) with the recognized-key
|
||||
allowlist at `gitea_config.py:1172` and the generator that emits the marker at
|
||||
`gitea_config.py:1233`. Every one of those facts is fixed at process start. A client that is
|
||||
trustworthy at launch and compromised a minute later remains `client_managed` for the life
|
||||
@@ -259,7 +259,7 @@ holds the token and calls the API instead of the tool.
|
||||
|
||||
**D3 — Credential resolution is scoped to the request principal.** A session must resolve its
|
||||
own credential and must have no path to any other principal's. The resolve-every-profile
|
||||
behavior behind `gitea_mcp_server.py:19378` and `gitea_mcp_server.py:19713` must report
|
||||
behavior behind `gitea_mcp_server.py:19538` and `gitea_mcp_server.py:19873` must report
|
||||
configured-or-not from configuration alone, without resolving the secret.
|
||||
|
||||
*Rationale.* Finding 3. An audit surface that proves a credential exists by fetching it is a
|
||||
@@ -336,11 +336,11 @@ The client is attached to the local fleet over stdio.
|
||||
|
||||
| Boundary | What ADV1 reaches | Stopped by |
|
||||
| -------- | ----------------- | ---------- |
|
||||
| B1 | Everything the fleet serves. The client *is* the sanctioned launcher: it satisfies the client-managed check (`gitea_mcp_server.py:15481`) by construction, and provenance is never re-verified after launch (Finding 8). | Nothing. The guard authenticates the launch, not the caller. |
|
||||
| B1 | Everything the fleet serves. The client *is* the sanctioned launcher: it satisfies the client-managed check (`gitea_mcp_server.py:15630`) by construction, and provenance is never re-verified after launch (Finding 8). | Nothing. The guard authenticates the launch, not the caller. |
|
||||
| B2 | All five roles — it is attached to all five namespaces. It can author a PR, approve it from the reviewer namespace, and merge it from the merger namespace. | Only the in-process self-review check, which compares `jcwalker3` (author) against `sysadmin` (reviewer) and **passes**, because Finding 1 made them different accounts while leaving reviewer and merger identical. A9 falls in one sequence of legitimate calls. |
|
||||
| B3 | Every credential in CR1–CR10 via CR13, with no additional prompt — the daemon is already sanctioned, so `assert_keychain_access_allowed` (`mcp_daemon_guard.py:583`) returns immediately. | Nothing. |
|
||||
| B4 | A1 and A2 in full. | Branch protection at the forge, to the extent configured. |
|
||||
| B5 | The operator's checkout and every worktree, through the author tools (`gitea_mcp_server.py:10956`), plus the shared stderr path at `mcp_server.py:13`. | Nothing; the server's disk is the target disk. |
|
||||
| B5 | The operator's checkout and every worktree, through the author tools (`gitea_mcp_server.py:10957`), plus the shared stderr path at `mcp_server.py:13`. | Nothing; the server's disk is the target disk. |
|
||||
| B6 | All coordination state — no credential required (CR14). It can forge lease ownership and clear decision locks. | Filesystem permissions, which it already satisfies. |
|
||||
| B7 | Jenkins (A4) and GlitchTip (A5) secrets via Finding 4, and CR11/CR12 from its own environment. | Nothing. |
|
||||
| B8 | Both tenants. | Nothing in-process; only the disabled-context check (`gitea_config.py:704`), which does not apply to enabled contexts. |
|
||||
|
||||
+402
-40
@@ -199,6 +199,7 @@ RECONCILER_WORKTREE_ENV = "GITEA_RECONCILER_WORKTREE"
|
||||
import namespace_workspace_binding as nwb # noqa: E402
|
||||
import canonical_repository_root as crr # noqa: E402 # #706 cross-repo canonical root
|
||||
import mcp_namespace_health # noqa: E402
|
||||
import mcp_worker_identity # noqa: E402 # #948 single client/session provenance authority
|
||||
import stale_binding_recovery # noqa: E402
|
||||
|
||||
# Worktree env bindings inherited from the parent environment at daemon boot
|
||||
@@ -14916,7 +14917,7 @@ def _stale_runtime_reconnect_action() -> str:
|
||||
return (
|
||||
"blocker_kind=runtime_reconnect_required: call "
|
||||
"gitea_request_mcp_reconnect(namespace=<active gitea-* namespace>, "
|
||||
"reason='stale-runtime', client='codex') for a typed operator "
|
||||
"reason='stale-runtime') for a typed operator "
|
||||
"reconnect blocker with exact UI steps, then reconnect the IDE/client "
|
||||
"MCP session so the server reloads at the current master head. Do not "
|
||||
"call gitea_activate_profile, pkill, touch configs, or switch MCP role "
|
||||
@@ -15478,34 +15479,166 @@ def _session_context_mutation_block(
|
||||
return blocked
|
||||
|
||||
|
||||
def _is_client_managed_process() -> bool:
|
||||
"""Check whether the current MCP server process has client-managed launch provenance (#686)."""
|
||||
val = (
|
||||
os.environ.get("GITEA_CLIENT_MANAGED")
|
||||
or os.environ.get("GITEA_MCP_CLIENT_MANAGED")
|
||||
or os.environ.get("GITEA_SERVER_PROVENANCE")
|
||||
or os.environ.get("GITEA_FORCE_CLIENT_MANAGED")
|
||||
or ""
|
||||
).strip().lower()
|
||||
# --- #948 client/session runtime ownership -------------------------------
|
||||
#
|
||||
# One daemon launch is one *generation*. The session that owns it registers a
|
||||
# unique worker identity against it, so a second healthy client attaching its
|
||||
# own generation is no longer indistinguishable from a duplicate process.
|
||||
# Everything here is process-local cache plus a local SQLite registry: no Gitea
|
||||
# call, no network, no config write, and every failure degrades to "unproven"
|
||||
# rather than raising into a tool call.
|
||||
|
||||
if val in ("0", "false", "no", "manual", "manual_launch"):
|
||||
_WORKER_REGISTRY = None
|
||||
_WORKER_IDENTITY: str | None = None
|
||||
_WORKER_GENERATION: str | None = None
|
||||
_WORKER_REGISTRATION_ATTEMPTED = False
|
||||
|
||||
#: Env a client launcher may set to name itself and its session. Absent values
|
||||
#: are reported as unknown; they are never guessed at, because guessing is what
|
||||
#: produced Codex reconnect steps for a Gemini operator.
|
||||
CLIENT_NAME_ENV = "GITEA_MCP_CLIENT"
|
||||
CLIENT_INSTANCE_ENV = "GITEA_MCP_CLIENT_INSTANCE"
|
||||
CLIENT_SESSION_ENV = "GITEA_MCP_CLIENT_SESSION"
|
||||
|
||||
|
||||
def _worker_registry():
|
||||
"""Local worker registry, or ``None`` when it cannot be opened.
|
||||
|
||||
Under pytest the registry is only opened when a test has pinned a path, so
|
||||
a test run never writes into the operator's real registry.
|
||||
"""
|
||||
global _WORKER_REGISTRY
|
||||
if _WORKER_REGISTRY is not None:
|
||||
return _WORKER_REGISTRY
|
||||
if mcp_daemon_guard.is_pytest_runtime() and not (
|
||||
os.environ.get(mcp_worker_identity.REGISTRY_PATH_ENV) or ""
|
||||
).strip():
|
||||
return None
|
||||
try:
|
||||
_WORKER_REGISTRY = mcp_worker_identity.WorkerRegistry()
|
||||
except Exception:
|
||||
return None
|
||||
return _WORKER_REGISTRY
|
||||
|
||||
|
||||
def _client_identity_hints() -> dict:
|
||||
"""What the launcher told us about itself. Unset fields stay unset."""
|
||||
return {
|
||||
"client_name": (os.environ.get(CLIENT_NAME_ENV) or "").strip() or None,
|
||||
"client_instance_id": (os.environ.get(CLIENT_INSTANCE_ENV) or "").strip()
|
||||
or f"pid-{os.getpid()}",
|
||||
"session_id": (os.environ.get(CLIENT_SESSION_ENV) or "").strip()
|
||||
or f"proc-{os.getpid()}-{_process_boot_head_sha or 'nohead'}",
|
||||
}
|
||||
|
||||
|
||||
def _active_worker_identity() -> str | None:
|
||||
"""This runtime's worker identity, registering it once per process.
|
||||
|
||||
A collision does not adopt the existing registration: a fresh identity is
|
||||
minted and registered instead, which is the #948 AC32 requirement and also
|
||||
the only safe response — adopting would silently transfer another worker's
|
||||
leases and fencing tokens.
|
||||
"""
|
||||
global _WORKER_IDENTITY, _WORKER_GENERATION, _WORKER_REGISTRATION_ATTEMPTED
|
||||
if _WORKER_IDENTITY is not None:
|
||||
return _WORKER_IDENTITY
|
||||
if _WORKER_REGISTRATION_ATTEMPTED:
|
||||
return None
|
||||
_WORKER_REGISTRATION_ATTEMPTED = True
|
||||
|
||||
registry = _worker_registry()
|
||||
if registry is None:
|
||||
return None
|
||||
|
||||
hints = _client_identity_hints()
|
||||
generation = _WORKER_GENERATION or mcp_worker_identity.new_generation_id(os.getpid())
|
||||
native = mcp_daemon_guard.native_runtime_status()
|
||||
|
||||
try:
|
||||
for _attempt in range(3):
|
||||
identity = mcp_worker_identity.generate_worker_identity(
|
||||
hints["client_name"], hints["session_id"]
|
||||
)
|
||||
outcome = registry.register(
|
||||
worker_identity=identity,
|
||||
client_name=hints["client_name"],
|
||||
client_instance_id=hints["client_instance_id"],
|
||||
session_id=hints["session_id"],
|
||||
generation_id=generation,
|
||||
role=_active_role_kind_safe(),
|
||||
profile=(os.environ.get(gitea_config.ENV_PROFILE) or "").strip() or None,
|
||||
remote=(os.environ.get("GITEA_MCP_REMOTE") or "").strip() or None,
|
||||
repository_binding=PROJECT_ROOT,
|
||||
pid=os.getpid(),
|
||||
transport=native.get("bound_transport"),
|
||||
token_fingerprint=native.get("token_fingerprint"),
|
||||
pid_alive_probe=issue_lock_store.is_process_alive,
|
||||
)
|
||||
if outcome.get("registered"):
|
||||
_WORKER_IDENTITY = identity
|
||||
_WORKER_GENERATION = generation
|
||||
return identity
|
||||
if not outcome.get("collision"):
|
||||
return None
|
||||
# Collided: loop mints a different identity rather than reusing this
|
||||
# one. The existing registration is left exactly as it was.
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _active_role_kind_safe() -> str | None:
|
||||
"""Best-effort role for the registry record; never raises into a tool call.
|
||||
|
||||
The role is recorded for diagnostics only. It is deliberately not part of
|
||||
the identity: role is a reusable capability definition that many live
|
||||
workers may share (#948 AC37).
|
||||
"""
|
||||
try:
|
||||
profile = get_profile() or {}
|
||||
return _role_kind(
|
||||
profile.get("allowed_operations") or [],
|
||||
profile.get("forbidden_operations") or [],
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _stdin_is_tty() -> bool:
|
||||
"""Terminal evidence for launch provenance, tolerant of a detached stdin."""
|
||||
try:
|
||||
return bool(sys.stdin and sys.stdin.isatty())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
if val in ("1", "true", "yes", "client_managed"):
|
||||
return True
|
||||
|
||||
# A terminal launch has an active TTY on stdin
|
||||
try:
|
||||
if sys.stdin and sys.stdin.isatty():
|
||||
return False
|
||||
except Exception:
|
||||
pass
|
||||
def _launch_provenance() -> dict:
|
||||
"""This process's launch provenance, from the one shared authority (#948).
|
||||
|
||||
# Standard client launch or test runner with stdio pipe and profile env
|
||||
if "GITEA_MCP_CONFIG" in os.environ or "GITEA_MCP_PROFILE" in os.environ or "GITEA_PROFILE_NAME" in os.environ:
|
||||
return True
|
||||
Previously each surface reimplemented this. ``gitea_get_runtime_context``
|
||||
read the live environment while ``mcp_namespace_health`` read a filtered
|
||||
summary that structurally could not see the provenance keys, so the two
|
||||
reported different provenance for the same process. Both now call
|
||||
``mcp_worker_identity``.
|
||||
"""
|
||||
return mcp_worker_identity.assess_launch_provenance(
|
||||
dict(os.environ), stdin_is_tty=_stdin_is_tty()
|
||||
)
|
||||
|
||||
return False
|
||||
|
||||
def _is_client_managed_process() -> bool:
|
||||
"""Check whether the current MCP server process has client-managed launch provenance (#686).
|
||||
|
||||
#948: the decision logic moved to
|
||||
``mcp_worker_identity.assess_launch_provenance`` unchanged — same env keys,
|
||||
same precedence, same TTY fallback — so this keeps returning exactly what it
|
||||
always returned while no longer being a second, divergent implementation.
|
||||
Launch provenance answers "was this hand-launched from a terminal", which is
|
||||
the #686 question. It does *not* answer which live session owns this
|
||||
runtime; that is ``session_ownership`` and needs an attachment record.
|
||||
"""
|
||||
return bool(_launch_provenance()["client_managed"])
|
||||
|
||||
|
||||
def _provenance_mutation_block(**extra_fields) -> dict | None:
|
||||
@@ -19048,7 +19181,21 @@ def gitea_get_runtime_context(
|
||||
source="gitea_get_runtime_context",
|
||||
)
|
||||
|
||||
is_client_managed = _is_client_managed_process()
|
||||
# #948: one assessment, shared with mcp_namespace_health. Reporting both
|
||||
# dimensions from a single call is what makes the two surfaces agree — the
|
||||
# contradiction they used to produce came from two implementations, not from
|
||||
# two genuinely different observations.
|
||||
provenance_assessment = mcp_worker_identity.assess_provenance(
|
||||
registry=_worker_registry(),
|
||||
worker_identity=_active_worker_identity(),
|
||||
env=dict(os.environ),
|
||||
native_transport_bound=mcp_daemon_guard.bound_transport() is not None,
|
||||
profile=profile.get("profile_name"),
|
||||
role=_role_kind(allowed, forbidden),
|
||||
stdin_is_tty=_stdin_is_tty(),
|
||||
pid_alive_probe=issue_lock_store.is_process_alive,
|
||||
)
|
||||
is_client_managed = provenance_assessment["is_client_managed"]
|
||||
unconsumed_env = gitea_config.get_unconsumed_gitea_env_overrides()
|
||||
|
||||
result = {
|
||||
@@ -19067,8 +19214,21 @@ def gitea_get_runtime_context(
|
||||
"review_merge_blocked_reasons": blocked_reasons,
|
||||
"suggested_fix": suggested_fix,
|
||||
"safe_next_action": safe_next_action,
|
||||
"server_provenance": "client_managed" if is_client_managed else "manual_launch",
|
||||
"server_provenance": provenance_assessment["launch_provenance"],
|
||||
"is_client_managed": is_client_managed,
|
||||
# #948: the ownership dimension, which the environment cannot establish.
|
||||
# A surface that needs "may this session mutate on behalf of its client"
|
||||
# reads these, not server_provenance.
|
||||
"session_ownership": provenance_assessment["session_ownership"],
|
||||
"session_owned": provenance_assessment["session_owned"],
|
||||
"worker_identity": provenance_assessment["worker_identity"],
|
||||
"session_id": provenance_assessment["session_id"],
|
||||
"client_instance_id": provenance_assessment["client_instance_id"],
|
||||
"generation_id": provenance_assessment["generation_id"],
|
||||
"client_name": provenance_assessment["client_name"],
|
||||
"fencing_epoch": provenance_assessment["fencing_epoch"],
|
||||
"conflicting_live_sessions": provenance_assessment["conflicting_live_sessions"],
|
||||
"provenance_assessment": provenance_assessment,
|
||||
"unconsumed_gitea_env": unconsumed_env,
|
||||
"preflight_ready": preflight["preflight_ready"],
|
||||
"preflight_block_reasons": preflight["preflight_block_reasons"],
|
||||
@@ -21654,11 +21814,23 @@ def _check_mcp_runtimes_diagnostics(task: str, matching_profiles: list[str]) ->
|
||||
if match:
|
||||
profile = match.group(1)
|
||||
|
||||
# #948: provenance for a scanned peer comes from the shared authority,
|
||||
# fed by that peer's own environment, so the fleet scan cannot disagree
|
||||
# with what that peer reports about itself.
|
||||
peer_env = {
|
||||
m.group(1): m.group(2)
|
||||
for m in re.finditer(r'\b(GITEA_[A-Z0-9_]+)=([^\s]+)', env_out)
|
||||
}
|
||||
# declared_only: this is a peer process. Its stdin is not ours to
|
||||
# inspect, and it inherits GITEA_MCP_PROFILE from any shell that
|
||||
# exported it, so only an explicit declaration counts as evidence here.
|
||||
is_client_managed = bool(
|
||||
re.search(r'\bGITEA_CLIENT_MANAGED=(1|true|yes|client_managed)\b', env_out, re.IGNORECASE)
|
||||
or re.search(r'\bGITEA_MCP_CLIENT_MANAGED=(1|true|yes|client_managed)\b', env_out, re.IGNORECASE)
|
||||
or re.search(r'\bGITEA_SERVER_PROVENANCE=client_managed\b', env_out, re.IGNORECASE)
|
||||
mcp_worker_identity.assess_launch_provenance(
|
||||
peer_env, declared_only=True
|
||||
)["client_managed"]
|
||||
)
|
||||
peer_worker_identity = peer_env.get("GITEA_MCP_WORKER_IDENTITY")
|
||||
peer_generation = peer_env.get("GITEA_MCP_GENERATION_ID")
|
||||
|
||||
for env_match in re.finditer(r'\b(GITEA_[A-Z0-9_]+)=([^\s]+)', env_out):
|
||||
k, v = env_match.group(1), env_match.group(2)
|
||||
@@ -21674,6 +21846,8 @@ def _check_mcp_runtimes_diagnostics(task: str, matching_profiles: list[str]) ->
|
||||
"start_time": start_time,
|
||||
"is_stale": is_stale,
|
||||
"is_client_managed": is_client_managed,
|
||||
"worker_identity": peer_worker_identity,
|
||||
"generation_id": peer_generation,
|
||||
}
|
||||
if profile not in all_profile_procs:
|
||||
all_profile_procs[profile] = []
|
||||
@@ -21681,12 +21855,44 @@ def _check_mcp_runtimes_diagnostics(task: str, matching_profiles: list[str]) ->
|
||||
|
||||
running_profiles = {}
|
||||
for profile, procs in all_profile_procs.items():
|
||||
if len(procs) > 1:
|
||||
# #948 AC40/AC43: sharing a profile is legitimate — profile is a
|
||||
# reusable capability definition, not a worker identity. What is *not*
|
||||
# legitimate is reusing one worker identity, or two live sessions
|
||||
# claiming one generation. Distinctness has to be proven, though:
|
||||
# processes carrying no identity evidence are indistinguishable, so
|
||||
# they stay classified as duplicates and keep the #686 wall intact.
|
||||
identified = [p for p in procs if p.get("worker_identity")]
|
||||
distinct_identities = {p["worker_identity"] for p in identified}
|
||||
all_identified = len(identified) == len(procs)
|
||||
duplicate_identity = len(identified) != len(distinct_identities)
|
||||
contested_generation = any(
|
||||
len({p["worker_identity"] for p in identified if p.get("generation_id") == gen}) > 1
|
||||
for gen in {p.get("generation_id") for p in identified if p.get("generation_id")}
|
||||
)
|
||||
|
||||
if len(procs) > 1 and (
|
||||
not all_identified or duplicate_identity or contested_generation
|
||||
):
|
||||
pids_str = ", ".join(str(p["pid"]) for p in procs)
|
||||
if duplicate_identity or contested_generation:
|
||||
detail = (
|
||||
"The same worker identity or generation is claimed more than once, "
|
||||
"so these are genuine duplicates rather than independent workers."
|
||||
)
|
||||
else:
|
||||
detail = (
|
||||
"Manual or duplicate launches defeat staleness detection and cannot "
|
||||
"receive client stdio."
|
||||
)
|
||||
reasons.append(
|
||||
f"stale-runtime: Duplicate MCP server process(es) detected for profile '{profile}' (PIDs: {pids_str}). "
|
||||
"Manual or duplicate launches defeat staleness detection and cannot receive client stdio."
|
||||
+ detail
|
||||
)
|
||||
# Otherwise: several independently identified workers share one profile.
|
||||
# No reason is appended, deliberately. Every reason this function
|
||||
# returns is raised as a hard RuntimeError by its callers, so recording
|
||||
# legitimate concurrency here as "informational" would block exactly the
|
||||
# case #948 exists to permit.
|
||||
client_procs = [p for p in procs if p["is_client_managed"]]
|
||||
if client_procs:
|
||||
client_procs.sort(key=lambda p: p["start_time"], reverse=True)
|
||||
@@ -22095,7 +22301,7 @@ def gitea_resolve_task_capability(
|
||||
next_safe_action = (
|
||||
"blocker_kind=runtime_reconnect_required: call "
|
||||
"gitea_request_mcp_reconnect(namespace=<active gitea-* namespace>, "
|
||||
"reason='stale-runtime', client='codex') for a typed operator "
|
||||
"reason='stale-runtime') for a typed operator "
|
||||
"blocker with exact UI steps, then reconnect/reload the IDE-managed "
|
||||
"Gitea MCP server for this profile so it reloads current master. "
|
||||
"Do not edit mcp_config.json by hand, pkill, or touch configs; the "
|
||||
@@ -23663,7 +23869,7 @@ def gitea_workflow_dashboard(
|
||||
def gitea_request_mcp_reconnect(
|
||||
namespace: str | None = None,
|
||||
reason: str | None = None,
|
||||
client: str = "codex",
|
||||
client: str | None = None,
|
||||
remote: str = "dadeschools",
|
||||
host: str | None = None,
|
||||
session_id: str | None = None,
|
||||
@@ -23693,8 +23899,12 @@ def gitea_request_mcp_reconnect(
|
||||
to the active profile's inferred namespace.
|
||||
reason: Why reconnect is requested: ``stale-runtime``, ``transport_eof``,
|
||||
``missing_namespace``, ``not_required``, or free-form (normalized).
|
||||
client: Operator UI surface — ``codex`` (default), ``claude_code``, or
|
||||
``generic``.
|
||||
client: Operator UI surface — ``codex``, ``claude_code``, or
|
||||
``generic``. #948: omitting it resolves the client from this
|
||||
runtime's live attachment record rather than assuming one vendor,
|
||||
and falls back to ``generic`` when nothing identifies the client.
|
||||
Emitting Codex panel steps to a Gemini/Antigravity operator left
|
||||
them with no reachable recovery path.
|
||||
remote: Known instance — ``dadeschools`` or ``prgs`` (parity context).
|
||||
host: Optional host override for parity context.
|
||||
session_id: Optional session id to echo in the report.
|
||||
@@ -23754,6 +23964,19 @@ def gitea_request_mcp_reconnect(
|
||||
else:
|
||||
effective_reason = mcp_client_reconnect.REASON_UNSPECIFIED
|
||||
|
||||
# #948: an omitted client is resolved from the live attachment record, so
|
||||
# the steps describe the UI the operator is actually in front of.
|
||||
if not (client or "").strip():
|
||||
client = mcp_worker_identity.reconnect_client_for(
|
||||
mcp_worker_identity.assess_provenance(
|
||||
registry=_worker_registry(),
|
||||
worker_identity=_active_worker_identity(),
|
||||
env=dict(os.environ),
|
||||
stdin_is_tty=_stdin_is_tty(),
|
||||
pid_alive_probe=issue_lock_store.is_process_alive,
|
||||
)
|
||||
)
|
||||
|
||||
payload = mcp_client_reconnect.build_reconnect_request(
|
||||
namespace=ns,
|
||||
profile=profile_name,
|
||||
@@ -24209,10 +24432,18 @@ def _run_post_restart_reconcile(
|
||||
repo: str | None = None,
|
||||
mode: str | None = None,
|
||||
limit: int = 200,
|
||||
apply_session_cleanup: bool = False,
|
||||
) -> dict:
|
||||
"""Gather + classify post-restart state; cache the latest proof (#662)."""
|
||||
"""Gather + classify post-restart state; cache the latest proof (#662 / #969).
|
||||
|
||||
When *apply_session_cleanup* is true, confirmed-stale session rows (dead
|
||||
owner / PID reuse, no live lease) are terminalized through
|
||||
``session_lifecycle`` before re-classification so the sessions dimension
|
||||
can resolve. Default remains dry (read-only) to preserve #662 rollout.
|
||||
"""
|
||||
global _POST_RESTART_LAST_PROOF, _POST_RESTART_BOOT_RAN
|
||||
import post_restart_reconcile as prr
|
||||
import session_lifecycle as sl
|
||||
|
||||
try:
|
||||
_h, o, r = _resolve(remote, None, org, repo)
|
||||
@@ -24226,13 +24457,48 @@ def _run_post_restart_reconcile(
|
||||
inventory = _gather_post_restart_inventory(
|
||||
remote=remote, org=o, repo=r, limit=limit
|
||||
)
|
||||
|
||||
session_cleanup: dict | None = None
|
||||
if apply_session_cleanup:
|
||||
db, db_errs = _control_plane_db_or_error()
|
||||
if db is None:
|
||||
session_cleanup = {
|
||||
"success": False,
|
||||
"reasons": db_errs
|
||||
or ["control-plane DB unavailable; cannot retire sessions"],
|
||||
}
|
||||
else:
|
||||
profile = get_profile()
|
||||
profile_name = (profile.get("profile_name") or "").strip() or "session"
|
||||
actor = f"{profile_name}-{os.getpid()}"
|
||||
session_cleanup = sl.retire_stale_sessions(
|
||||
db,
|
||||
sessions=inventory.get("sessions") or [],
|
||||
leases=inventory.get("leases") or [],
|
||||
dry_run=False,
|
||||
actor_session_id=actor,
|
||||
session_limit=max(1, int(limit)),
|
||||
)
|
||||
# Re-gather active sessions after mutation so classification sees
|
||||
# the post-retirement fleet (retired rows drop out of active list).
|
||||
try:
|
||||
inventory["sessions"] = db.list_sessions(
|
||||
statuses=("active",), limit=max(1, int(limit))
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
inventory["incomplete_reasons"] = list(
|
||||
inventory.get("incomplete_reasons") or []
|
||||
) + [f"post-retirement session re-list failed: {_redact(str(exc))}"]
|
||||
fleet = (session_cleanup.get("fleet") or {}) if session_cleanup else {}
|
||||
inventory["session_fleet"] = fleet
|
||||
|
||||
proof = prr.reconcile_after_restart(
|
||||
inventory,
|
||||
mode=mode or _post_restart_reconcile_mode(),
|
||||
)
|
||||
payload = proof.as_dict()
|
||||
payload["success"] = True
|
||||
payload["read_only"] = True
|
||||
payload["read_only"] = not apply_session_cleanup
|
||||
payload["remote"] = remote
|
||||
payload["org"] = o
|
||||
payload["repo"] = r
|
||||
@@ -24241,6 +24507,9 @@ def _run_post_restart_reconcile(
|
||||
"proposed_follow_ups lists durable issues the apply path may create; "
|
||||
"this tool never creates them (log-only by default, #662 rollout)"
|
||||
)
|
||||
if session_cleanup is not None:
|
||||
payload["session_cleanup"] = session_cleanup
|
||||
payload["apply_session_cleanup"] = bool(apply_session_cleanup)
|
||||
_POST_RESTART_LAST_PROOF = payload
|
||||
_POST_RESTART_BOOT_RAN = True
|
||||
return payload
|
||||
@@ -24267,8 +24536,9 @@ def gitea_reconcile_after_restart(
|
||||
repo: str | None = None,
|
||||
mode: str | None = None,
|
||||
limit: int = 200,
|
||||
apply_session_cleanup: bool = False,
|
||||
) -> dict:
|
||||
"""Run post-restart MCP reconciliation and return a completion proof (#662).
|
||||
"""Run post-restart MCP reconciliation and return a completion proof (#662 / #969).
|
||||
|
||||
Gathers live control-plane sessions, leases, worktree bindings, and
|
||||
master-parity evidence, then classifies them with the pure
|
||||
@@ -24276,12 +24546,17 @@ def gitea_reconcile_after_restart(
|
||||
machine-readable completion proof listing resolved / unresolved dimensions
|
||||
and proposed durable follow-up issues.
|
||||
|
||||
Read-only by design: never restarts MCP, never auto-resumes write
|
||||
mutations, and never creates Gitea issues (those are a separate apply
|
||||
path). Default mode is ``log_only``; set
|
||||
Never restarts MCP, never auto-resumes write mutations, and never creates
|
||||
Gitea issues. Default mode is ``log_only``; set
|
||||
``GITEA_POST_RESTART_RECONCILE_MODE=enforce`` (or pass ``mode='enforce'``)
|
||||
to set ``mutation_hold`` when anything remains unresolved.
|
||||
|
||||
*apply_session_cleanup* (#969): when true, confirmed-stale workflow session
|
||||
rows (dead owner PID / PID reuse, no live lease, not a live client-managed
|
||||
owner) are terminalized through the sanctioned ``session_lifecycle`` path
|
||||
before re-classification. Default false preserves the historical read-only
|
||||
gather+classify behaviour. Does not delete historical rows.
|
||||
|
||||
Soft-depends on #660 for session checkpoints: when the checkpoint schema
|
||||
module is absent the checkpoints dimension is ``skipped`` with an explicit
|
||||
reason rather than inventing a schema.
|
||||
@@ -24301,9 +24576,96 @@ def gitea_reconcile_after_restart(
|
||||
repo=repo,
|
||||
mode=mode,
|
||||
limit=limit,
|
||||
apply_session_cleanup=bool(apply_session_cleanup),
|
||||
)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def gitea_retire_stale_workflow_sessions(
|
||||
remote: str = "dadeschools",
|
||||
host: str | None = None,
|
||||
org: str | None = None,
|
||||
repo: str | None = None,
|
||||
apply: bool = False,
|
||||
limit: int = 500,
|
||||
) -> dict:
|
||||
"""Retire workflow session rows whose owners are no longer live (#969).
|
||||
|
||||
Plans (and optionally applies) terminalization of control-plane session
|
||||
rows that are confirmed stale:
|
||||
|
||||
* owner PID absent / dead
|
||||
* PID reused by an unrelated process (process start after session start)
|
||||
* no live workflow lease
|
||||
* not a live client-managed owner
|
||||
|
||||
Default ``apply=false`` is dry-run only. ``apply=true`` performs CAS
|
||||
status updates to ``retired`` and writes durable ``session_retired``
|
||||
events. Idempotent under concurrent reconciliation. Never deletes rows
|
||||
and never touches sessions protected by a live lease or live owner.
|
||||
"""
|
||||
read_block = _profile_operation_gate("gitea.read")
|
||||
if read_block:
|
||||
return {
|
||||
"success": False,
|
||||
"read_only": not apply,
|
||||
"reasons": read_block,
|
||||
"permission_report": _permission_block_report("gitea.read"),
|
||||
}
|
||||
|
||||
try:
|
||||
_h, o, r = _resolve(remote, None, org, repo)
|
||||
except ValueError as exc:
|
||||
return {"success": False, "reasons": [str(exc)], "read_only": not apply}
|
||||
|
||||
db, db_errs = _control_plane_db_or_error()
|
||||
if db is None:
|
||||
return {
|
||||
"success": False,
|
||||
"read_only": not apply,
|
||||
"reasons": db_errs or ["control-plane DB unavailable"],
|
||||
}
|
||||
|
||||
import session_lifecycle as sl
|
||||
|
||||
profile = get_profile()
|
||||
profile_name = (profile.get("profile_name") or "").strip() or "session"
|
||||
actor = f"{profile_name}-{os.getpid()}"
|
||||
|
||||
# Prefer lease inventory scoped to the requested repo when available.
|
||||
leases: list[dict] = []
|
||||
try:
|
||||
lease_result = lease_lifecycle.list_active_leases(
|
||||
db,
|
||||
remote=remote if remote in REMOTES else remote,
|
||||
org=o,
|
||||
repo=r,
|
||||
role=None,
|
||||
include_non_active=True,
|
||||
limit=max(1, int(limit)),
|
||||
)
|
||||
leases = list(lease_result.get("leases") or [])
|
||||
except Exception: # noqa: BLE001
|
||||
try:
|
||||
leases = db.list_leases(statuses=("active",), limit=max(1, int(limit)))
|
||||
except Exception: # noqa: BLE001
|
||||
leases = []
|
||||
|
||||
result = sl.retire_stale_sessions(
|
||||
db,
|
||||
leases=leases,
|
||||
dry_run=not bool(apply),
|
||||
actor_session_id=actor,
|
||||
session_limit=max(1, int(limit)),
|
||||
)
|
||||
result["read_only"] = not bool(apply)
|
||||
result["apply"] = bool(apply)
|
||||
result["remote"] = remote
|
||||
result["org"] = o
|
||||
result["repo"] = r
|
||||
return result
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def gitea_inspect_workflow_lease(
|
||||
lease_id: str,
|
||||
|
||||
+21
-9
@@ -92,7 +92,15 @@ OPERATOR_UI_STEPS: dict[str, tuple[str, ...]] = {
|
||||
),
|
||||
}
|
||||
|
||||
DEFAULT_CLIENT = "codex"
|
||||
#: What an *unidentified* client gets. #948: this is deliberately the
|
||||
#: host-agnostic step set rather than a specific product. Defaulting to one
|
||||
#: vendor emitted Codex UI steps to a Gemini/Antigravity operator, who then had
|
||||
#: no reachable recovery path — the guidance named a panel they do not have.
|
||||
DEFAULT_CLIENT = "generic"
|
||||
|
||||
#: The historical default, kept addressable by name so Codex callers still get
|
||||
#: Codex steps, without it silently becoming the fallback for unknown clients.
|
||||
LEGACY_DEFAULT_CLIENT = "codex"
|
||||
|
||||
|
||||
def normalize_reason(reason: str | None) -> str:
|
||||
@@ -128,14 +136,18 @@ def normalize_reason(reason: str | None) -> str:
|
||||
|
||||
|
||||
def normalize_client(client: str | None) -> str:
|
||||
"""Return a known client key for operator UI steps."""
|
||||
text = (client or "").strip().lower().replace(" ", "_").replace("-", "_")
|
||||
if text in ("codex", "openai_codex", "openai"):
|
||||
return "codex"
|
||||
if text in ("claude", "claude_code", "claude_desktop", "anthropic"):
|
||||
return "claude_code"
|
||||
if text in OPERATOR_UI_STEPS:
|
||||
return text
|
||||
"""Return the UI-step key for a client.
|
||||
|
||||
#948: alias resolution is shared with ``mcp_worker_identity`` so a client
|
||||
name means the same thing wherever it is read. A name we recognise but have
|
||||
no bespoke steps for — Gemini, Antigravity, Grok — resolves to the generic
|
||||
host-agnostic steps rather than to another vendor's panel.
|
||||
"""
|
||||
import mcp_worker_identity
|
||||
|
||||
canonical = mcp_worker_identity.normalize_client_name(client)
|
||||
if canonical in OPERATOR_UI_STEPS:
|
||||
return canonical
|
||||
return DEFAULT_CLIENT
|
||||
|
||||
|
||||
|
||||
+58
-5
@@ -95,6 +95,15 @@ SAFE_ENV_KEYS = (
|
||||
"GITEA_MCP_CONFIG",
|
||||
)
|
||||
|
||||
# #948: provenance used to be derived from the summary this allowlist produces.
|
||||
# The allowlist never carried a provenance key, so that derivation could only
|
||||
# ever evaluate to ``manual_launch`` — whatever the process actually was — while
|
||||
# ``gitea_get_runtime_context`` read the live environment and reported
|
||||
# ``client_managed`` for the same process. Provenance is no longer derived here.
|
||||
# It comes from ``mcp_worker_identity.assess_provenance``, the single authority
|
||||
# every surface shares. This allowlist keeps its original and only job: deciding
|
||||
# which env values are safe to echo back in diagnostics.
|
||||
|
||||
|
||||
def assess_connected_namespace_attachment(
|
||||
*,
|
||||
@@ -443,12 +452,23 @@ def classify_namespace_probe(
|
||||
profile: str | None = None,
|
||||
configured: bool = True,
|
||||
probe_source: str | None = None,
|
||||
worker_identity: str | None = None,
|
||||
generation_id: str | None = None,
|
||||
registry: Any | None = None,
|
||||
pid_alive_probe: Any | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Classify whether a required tool is callable through a live namespace.
|
||||
|
||||
``registered_tools`` is static/server-side evidence. ``probe_result`` is
|
||||
live invocation evidence. Only ``probe_source=client_namespace`` proves the
|
||||
IDE-managed path; ``offline_spawn`` is an offline subprocess check only.
|
||||
|
||||
#948: ``worker_identity``/``generation_id``/``registry`` carry the
|
||||
client/session ownership evidence. Provenance is resolved by
|
||||
``mcp_worker_identity.assess_provenance`` — the same call
|
||||
``gitea_get_runtime_context`` makes — so the two surfaces cannot report
|
||||
different provenance for one process. Omitting them yields the fail-closed
|
||||
``unproven`` verdict, never a fabricated ``client_managed``.
|
||||
"""
|
||||
ns = (namespace or "").strip()
|
||||
tool = required_tool or REQUIRED_NAMESPACE_TOOLS.get(ns) or "gitea_whoami"
|
||||
@@ -565,14 +585,31 @@ def classify_namespace_probe(
|
||||
blocks = namespace_health_blocks_task("merge_pr", healthy)
|
||||
|
||||
import gitea_config
|
||||
import mcp_worker_identity
|
||||
|
||||
raw_env = process.get("env") if isinstance(process, dict) else None
|
||||
unconsumed_env = gitea_config.get_unconsumed_gitea_env_overrides(raw_env)
|
||||
is_client_managed = bool(
|
||||
env_summary.get("GITEA_CLIENT_MANAGED") in ("1", "true", "yes", "client_managed")
|
||||
or env_summary.get("GITEA_MCP_CLIENT_MANAGED") in ("1", "true", "yes", "client_managed")
|
||||
or env_summary.get("GITEA_SERVER_PROVENANCE") == "client_managed"
|
||||
|
||||
# #948: one authority, shared with gitea_get_runtime_context. The env is
|
||||
# passed whole rather than through SAFE_ENV_KEYS — the allowlist exists to
|
||||
# decide what may be *echoed*, and using it to decide what may be *believed*
|
||||
# is what made this surface structurally unable to report client_managed.
|
||||
# ``declared_only``: ``process`` describes an observed peer, not this
|
||||
# interpreter. Its stdin is unavailable and its launcher-config env is
|
||||
# inherited from whatever shell started it, so only an explicit declaration
|
||||
# is evidence. Absence of one is ``unproven``, not an asserted manual launch.
|
||||
provenance_verdict = mcp_worker_identity.assess_provenance(
|
||||
registry=registry,
|
||||
worker_identity=worker_identity,
|
||||
generation_id=generation_id,
|
||||
env=raw_env if isinstance(raw_env, dict) else {},
|
||||
namespace=ns,
|
||||
profile=profile_name,
|
||||
pid_alive_probe=pid_alive_probe,
|
||||
declared_only=True,
|
||||
)
|
||||
provenance = "client_managed" if is_client_managed else "manual_launch"
|
||||
provenance = provenance_verdict["provenance"]
|
||||
is_client_managed = provenance_verdict["is_client_managed"]
|
||||
|
||||
return {
|
||||
"success": healthy,
|
||||
@@ -591,6 +628,15 @@ def classify_namespace_probe(
|
||||
"remediation": remediation,
|
||||
"provenance": provenance,
|
||||
"is_client_managed": is_client_managed,
|
||||
# Every non-client-session verdict fails closed. Consumers that only
|
||||
# need "may this mutate?" read this and stay correct across the #948
|
||||
# vocabulary split between ``manual_launch`` and ``unproven``.
|
||||
"provenance_fail_closed": provenance_verdict["fail_closed"],
|
||||
"provenance_assessment": provenance_verdict,
|
||||
"worker_identity": provenance_verdict["worker_identity"],
|
||||
"session_id": provenance_verdict["session_id"],
|
||||
"generation_id": provenance_verdict["generation_id"],
|
||||
"client_name": provenance_verdict["client_name"],
|
||||
"unconsumed_gitea_env": unconsumed_env,
|
||||
"diagnostics": {
|
||||
"namespace": ns,
|
||||
@@ -602,6 +648,13 @@ def classify_namespace_probe(
|
||||
"probe_source": source,
|
||||
"provenance": provenance,
|
||||
"is_client_managed": is_client_managed,
|
||||
"provenance_fail_closed": provenance_verdict["fail_closed"],
|
||||
"provenance_blocker_kind": provenance_verdict["blocker_kind"],
|
||||
"provenance_scope": provenance_verdict["scope"],
|
||||
"worker_identity": provenance_verdict["worker_identity"],
|
||||
"session_id": provenance_verdict["session_id"],
|
||||
"generation_id": provenance_verdict["generation_id"],
|
||||
"client_name": provenance_verdict["client_name"],
|
||||
"unconsumed_gitea_env": unconsumed_env,
|
||||
},
|
||||
"blocks_merge_workflow": blocks,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+57
-12
@@ -475,24 +475,65 @@ def reconcile_after_restart(
|
||||
)
|
||||
)
|
||||
|
||||
# --- sessions -------------------------------------------------------
|
||||
# --- sessions (#969: dead-owner / PID-reuse lifecycle) ---------------
|
||||
# Prefer a precomputed fleet report from the gather/apply path when present;
|
||||
# otherwise classify pure from inventory (injectable checkers stay default).
|
||||
sessions = [s for s in (inventory.get("sessions") or []) if isinstance(s, Mapping)]
|
||||
orphan_sessions = [
|
||||
s
|
||||
for s in sessions
|
||||
if str(s.get("status") or "").lower() == "active"
|
||||
and s.get("pid") is not None
|
||||
and not lease_lifecycle.is_process_alive(s.get("pid"))
|
||||
leases_for_sessions = [
|
||||
L for L in (inventory.get("leases") or []) if isinstance(L, Mapping)
|
||||
]
|
||||
if orphan_sessions:
|
||||
fleet_report = inventory.get("session_fleet")
|
||||
if isinstance(fleet_report, Mapping) and "retireable_session_ids" in fleet_report:
|
||||
fleet_details = dict(fleet_report)
|
||||
retireable_ids = list(fleet_details.get("retireable_session_ids") or [])
|
||||
resolved = bool(fleet_details.get("sessions_dimension_resolved", not retireable_ids))
|
||||
else:
|
||||
try:
|
||||
import session_lifecycle as _sl
|
||||
|
||||
client_managed = inventory.get("client_managed_session_ids")
|
||||
cm_set = None
|
||||
if isinstance(client_managed, (list, tuple, set, frozenset)):
|
||||
cm_set = {str(x) for x in client_managed}
|
||||
fleet = _sl.classify_sessions(
|
||||
sessions,
|
||||
leases=leases_for_sessions,
|
||||
now=started,
|
||||
client_managed_sessions=cm_set,
|
||||
)
|
||||
fleet_details = fleet.as_dict()
|
||||
retireable_ids = list(fleet_details.get("retireable_session_ids") or [])
|
||||
resolved = bool(fleet_details.get("sessions_dimension_resolved"))
|
||||
except Exception as exc: # noqa: BLE001 — fail closed to legacy signal
|
||||
# Legacy fallback: dead-pid active rows only (pre-#969 behaviour).
|
||||
orphan_sessions = [
|
||||
s
|
||||
for s in sessions
|
||||
if str(s.get("status") or "").lower() == "active"
|
||||
and s.get("pid") is not None
|
||||
and not lease_lifecycle.is_process_alive(s.get("pid"))
|
||||
]
|
||||
retireable_ids = [s.get("session_id") for s in orphan_sessions]
|
||||
resolved = not orphan_sessions
|
||||
fleet_details = {
|
||||
"total_sessions": len(sessions),
|
||||
"retireable_session_ids": retireable_ids,
|
||||
"legacy_fallback": True,
|
||||
"fallback_error": str(exc),
|
||||
}
|
||||
|
||||
if not resolved and retireable_ids:
|
||||
items.append(
|
||||
_item(
|
||||
DIM_SESSIONS,
|
||||
ITEM_UNRESOLVED,
|
||||
f"{len(orphan_sessions)} active session row(s) with dead owner pid",
|
||||
f"{len(retireable_ids)} session row(s) with dead/reused owner "
|
||||
f"await retirement",
|
||||
details={
|
||||
"orphan_session_ids": [s.get("session_id") for s in orphan_sessions],
|
||||
"orphan_session_ids": retireable_ids,
|
||||
"retireable_session_ids": retireable_ids,
|
||||
"total_sessions": len(sessions),
|
||||
"fleet": fleet_details,
|
||||
},
|
||||
follow_up=True,
|
||||
)
|
||||
@@ -502,8 +543,12 @@ def reconcile_after_restart(
|
||||
_item(
|
||||
DIM_SESSIONS,
|
||||
ITEM_RESOLVED,
|
||||
f"{len(sessions)} session row(s) reconciled (no dead-pid orphans)",
|
||||
details={"total_sessions": len(sessions)},
|
||||
f"{len(sessions)} session row(s) reconciled "
|
||||
f"(no retireable dead/reused owners)",
|
||||
details={
|
||||
"total_sessions": len(sessions),
|
||||
"fleet": fleet_details,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,703 @@
|
||||
"""Safe lifecycle for workflow session rows with dead or reused owners (#969).
|
||||
|
||||
Post-restart reconciliation previously left hundreds of ``active`` session rows
|
||||
with dead owner PIDs permanently unresolved. A PID existence check alone is not
|
||||
enough: operating systems reuse PIDs, so an unrelated live process can appear to
|
||||
own a historical session.
|
||||
|
||||
This module is the pure classification + apply core for session retirement:
|
||||
|
||||
* distinguish live, disconnected, stale, protected, and terminal records
|
||||
* refuse retirement when a live lease or live verified owner remains
|
||||
* protect live client-managed sessions
|
||||
* detect PID reuse via process start time vs session start / heartbeat
|
||||
* terminalize confirmed-stale rows idempotently with durable audit events
|
||||
* stay safe under concurrent reconciles (CAS on status)
|
||||
|
||||
Design mirrors ``lease_lifecycle`` / ``post_restart_reconcile``:
|
||||
|
||||
* Pure classification accepts injectable checkers so unit tests never touch
|
||||
real processes.
|
||||
* Apply mutations go only through :meth:`ControlPlaneDB.retire_session`.
|
||||
* Historical rows are never deleted; status moves to a terminal value and an
|
||||
events-row records the action + reason.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Callable, Mapping, Sequence
|
||||
|
||||
import control_plane_db as cpd
|
||||
import lease_lifecycle
|
||||
|
||||
# Session status vocabulary.
|
||||
SESSION_STATUS_ACTIVE = "active"
|
||||
SESSION_STATUS_RETIRED = "retired"
|
||||
SESSION_STATUS_ENDED = "ended"
|
||||
SESSION_STATUS_TERMINAL = "terminal"
|
||||
|
||||
TERMINAL_SESSION_STATUSES = frozenset(
|
||||
{
|
||||
SESSION_STATUS_RETIRED,
|
||||
SESSION_STATUS_ENDED,
|
||||
SESSION_STATUS_TERMINAL,
|
||||
"dead",
|
||||
"stale",
|
||||
"orphaned",
|
||||
}
|
||||
)
|
||||
|
||||
# Classification outcomes for one session row.
|
||||
CLASS_LIVE = "live"
|
||||
CLASS_DISCONNECTED = "disconnected"
|
||||
CLASS_STALE = "stale"
|
||||
CLASS_PROTECTED = "protected"
|
||||
CLASS_TERMINAL = "terminal"
|
||||
|
||||
# Stable reason codes (audit + tests).
|
||||
REASON_ALREADY_TERMINAL = "already_terminal"
|
||||
REASON_LIVE_OWNER = "live_owner"
|
||||
REASON_LIVE_LEASE = "live_lease"
|
||||
REASON_CLIENT_MANAGED_LIVE = "client_managed_live"
|
||||
REASON_DEAD_OWNER = "dead_owner"
|
||||
REASON_PID_REUSE = "pid_reuse"
|
||||
REASON_HEARTBEAT_STALE_DEAD = "heartbeat_stale_dead_owner"
|
||||
REASON_MISSING_PID = "missing_pid_no_lease"
|
||||
|
||||
# Event type written to control-plane events table.
|
||||
EVENT_SESSION_RETIRED = "session_retired"
|
||||
|
||||
# Default heartbeat window before a still-alive PID is treated as disconnected
|
||||
# rather than live (does not alone authorize retirement).
|
||||
DEFAULT_HEARTBEAT_STALE_SECONDS = 900
|
||||
|
||||
# PID reuse: process start must be strictly later than session started_at by
|
||||
# more than this skew (ps lstart is second-resolution; clocks can lag).
|
||||
PID_REUSE_SKEW = timedelta(seconds=2)
|
||||
|
||||
# Live lease freshness values that block retirement.
|
||||
_LIVE_LEASE_FRESHNESS = frozenset({"active", "live"})
|
||||
|
||||
|
||||
class SessionLifecycleError(RuntimeError):
|
||||
"""Fail-closed session lifecycle policy error."""
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _parse_ts(value: str | datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
return cpd._parse_ts(str(value))
|
||||
|
||||
|
||||
def _ts(dt: datetime | None = None) -> str:
|
||||
return cpd._ts(dt)
|
||||
|
||||
|
||||
def process_start_time(pid: int | None) -> datetime | None:
|
||||
"""Return the OS start time of *pid*, or None when it cannot be resolved.
|
||||
|
||||
Uses ``ps -o lstart=`` (POSIX). Failures return None rather than inventing
|
||||
evidence — missing start time never authorizes retirement of a live PID.
|
||||
"""
|
||||
if pid is None:
|
||||
return None
|
||||
try:
|
||||
pid_i = int(pid)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if pid_i <= 0:
|
||||
return None
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["ps", "-o", "lstart=", "-p", str(pid_i)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=2,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
text = (proc.stdout or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
# Example: "Wed Jul 29 09:14:36 2026"
|
||||
naive = datetime.strptime(text, "%a %b %d %H:%M:%S %Y")
|
||||
return naive.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _lease_freshness_label(lease: Mapping[str, Any]) -> str:
|
||||
fr = lease.get("freshness")
|
||||
if isinstance(fr, Mapping):
|
||||
return str(fr.get("freshness") or "").strip().lower()
|
||||
if fr:
|
||||
return str(fr).strip().lower()
|
||||
# Fall back to classifying a raw lease row.
|
||||
try:
|
||||
return str(
|
||||
lease_lifecycle.classify_lease_freshness(lease).get("freshness") or ""
|
||||
).strip().lower()
|
||||
except Exception: # noqa: BLE001 — pure classifier must not raise on bad rows
|
||||
status = str(lease.get("status") or "").strip().lower()
|
||||
return status or "unknown"
|
||||
|
||||
|
||||
def live_lease_session_ids(
|
||||
leases: Sequence[Mapping[str, Any]] | None,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||
) -> set[str]:
|
||||
"""Session ids that still hold a live (or ambiguous-active) workflow lease."""
|
||||
live: set[str] = set()
|
||||
moment = now or _utc_now()
|
||||
for lease in leases or ():
|
||||
if not isinstance(lease, Mapping):
|
||||
continue
|
||||
status = str(lease.get("status") or "").strip().lower()
|
||||
if status and status not in {
|
||||
lease_lifecycle.LEASE_STATUS_ACTIVE,
|
||||
"",
|
||||
}:
|
||||
# Explicit terminal lease statuses never protect a session.
|
||||
if status in {
|
||||
lease_lifecycle.LEASE_STATUS_RELEASED,
|
||||
lease_lifecycle.LEASE_STATUS_EXPIRED,
|
||||
lease_lifecycle.LEASE_STATUS_ABANDONED,
|
||||
}:
|
||||
continue
|
||||
freshness = _lease_freshness_label(lease)
|
||||
if freshness in _LIVE_LEASE_FRESHNESS or freshness in {"", "unknown"}:
|
||||
# Ambiguous active rows: re-check with authoritative classifier.
|
||||
try:
|
||||
fr = lease_lifecycle.classify_lease_freshness(
|
||||
lease, now=moment, pid_checker=pid_checker
|
||||
)
|
||||
freshness = str(fr.get("freshness") or "").strip().lower()
|
||||
except Exception: # noqa: BLE001
|
||||
freshness = "unknown"
|
||||
if freshness in _LIVE_LEASE_FRESHNESS:
|
||||
sid = str(lease.get("session_id") or "").strip()
|
||||
if sid:
|
||||
live.add(sid)
|
||||
elif freshness == "unknown" and status in {
|
||||
lease_lifecycle.LEASE_STATUS_ACTIVE,
|
||||
"",
|
||||
}:
|
||||
# Fail closed: active lease with unknown freshness blocks retirement.
|
||||
sid = str(lease.get("session_id") or "").strip()
|
||||
if sid:
|
||||
live.add(sid)
|
||||
return live
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SessionClassification:
|
||||
"""Classification of one workflow session row."""
|
||||
|
||||
session_id: str
|
||||
classification: str
|
||||
reason: str
|
||||
retireable: bool
|
||||
status: str | None
|
||||
pid: int | None
|
||||
pid_alive: bool | None
|
||||
pid_reused: bool
|
||||
heartbeat_stale: bool
|
||||
has_live_lease: bool
|
||||
client_managed: bool
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"session_id": self.session_id,
|
||||
"classification": self.classification,
|
||||
"reason": self.reason,
|
||||
"retireable": self.retireable,
|
||||
"status": self.status,
|
||||
"pid": self.pid,
|
||||
"pid_alive": self.pid_alive,
|
||||
"pid_reused": self.pid_reused,
|
||||
"heartbeat_stale": self.heartbeat_stale,
|
||||
"has_live_lease": self.has_live_lease,
|
||||
"client_managed": self.client_managed,
|
||||
"details": dict(self.details),
|
||||
}
|
||||
|
||||
|
||||
def classify_session(
|
||||
row: Mapping[str, Any],
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
||||
live_lease_sessions: set[str] | frozenset[str] | None = None,
|
||||
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
||||
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
||||
) -> SessionClassification:
|
||||
"""Classify one session row for retirement decisions (#969).
|
||||
|
||||
Rules (first match wins where noted):
|
||||
|
||||
1. Non-active / already-terminal status → ``terminal`` (not retireable).
|
||||
2. Session holds a live lease → ``protected`` (never retire).
|
||||
3. PID missing and no live lease → ``stale`` (retireable: missing owner).
|
||||
4. PID alive + process start after session start → ``stale`` (PID reuse).
|
||||
5. PID alive + client-managed → ``live`` protected (never retire).
|
||||
6. PID alive + fresh heartbeat → ``live``.
|
||||
7. PID alive + stale heartbeat → ``disconnected`` (not retireable alone).
|
||||
8. PID dead → ``stale`` (retireable).
|
||||
"""
|
||||
moment = now or _utc_now()
|
||||
session_id = str(row.get("session_id") or "").strip()
|
||||
status = str(row.get("status") or "").strip().lower() or None
|
||||
raw_pid = row.get("pid")
|
||||
try:
|
||||
pid = int(raw_pid) if raw_pid is not None else None
|
||||
except (TypeError, ValueError):
|
||||
pid = None
|
||||
|
||||
client_managed = bool(
|
||||
row.get("client_managed")
|
||||
or row.get("is_client_managed")
|
||||
or (
|
||||
client_managed_sessions is not None
|
||||
and session_id in client_managed_sessions
|
||||
)
|
||||
)
|
||||
has_live_lease = bool(
|
||||
live_lease_sessions is not None and session_id in live_lease_sessions
|
||||
)
|
||||
|
||||
hb = _parse_ts(row.get("last_heartbeat_at"))
|
||||
heartbeat_stale = bool(
|
||||
hb is not None
|
||||
and (moment - hb).total_seconds() > max(0, int(heartbeat_stale_seconds))
|
||||
)
|
||||
if hb is None and status == SESSION_STATUS_ACTIVE:
|
||||
# No heartbeat evidence: treat as stale for liveness bookkeeping only.
|
||||
heartbeat_stale = True
|
||||
|
||||
started = _parse_ts(row.get("started_at"))
|
||||
recorded_proc_start = _parse_ts(row.get("owner_process_started_at"))
|
||||
|
||||
if status in TERMINAL_SESSION_STATUSES:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_TERMINAL,
|
||||
reason=REASON_ALREADY_TERMINAL,
|
||||
retireable=False,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=None,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=has_live_lease,
|
||||
client_managed=client_managed,
|
||||
)
|
||||
|
||||
if has_live_lease:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_PROTECTED,
|
||||
reason=REASON_LIVE_LEASE,
|
||||
retireable=False,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=pid_checker(pid) if pid is not None else None,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=True,
|
||||
client_managed=client_managed,
|
||||
details={"blocker": "live_workflow_lease"},
|
||||
)
|
||||
|
||||
if pid is None:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_STALE,
|
||||
reason=REASON_MISSING_PID,
|
||||
retireable=True,
|
||||
status=status,
|
||||
pid=None,
|
||||
pid_alive=False,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=False,
|
||||
client_managed=client_managed,
|
||||
)
|
||||
|
||||
pid_alive = bool(pid_checker(pid))
|
||||
pid_reused = False
|
||||
proc_start: datetime | None = None
|
||||
|
||||
if pid_alive:
|
||||
proc_start = recorded_proc_start or process_start_probe(pid)
|
||||
# PID reuse: live process started after the session row itself was
|
||||
# created. Anchor on started_at only — last_heartbeat alone is not a
|
||||
# safe bound (synthetic inventories and long-lived processes would
|
||||
# false-positive against a live ``ps`` probe).
|
||||
if proc_start is not None and started is not None:
|
||||
if proc_start > (started + PID_REUSE_SKEW):
|
||||
pid_reused = True
|
||||
|
||||
if pid_reused:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_STALE,
|
||||
reason=REASON_PID_REUSE,
|
||||
retireable=True,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=True,
|
||||
pid_reused=True,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=False,
|
||||
client_managed=client_managed,
|
||||
details={
|
||||
"process_started_at": _ts(proc_start) if proc_start else None,
|
||||
"session_started_at": row.get("started_at"),
|
||||
"last_heartbeat_at": row.get("last_heartbeat_at"),
|
||||
},
|
||||
)
|
||||
|
||||
if pid_alive and client_managed:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_LIVE,
|
||||
reason=REASON_CLIENT_MANAGED_LIVE,
|
||||
retireable=False,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=True,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=False,
|
||||
client_managed=True,
|
||||
)
|
||||
|
||||
if pid_alive and not heartbeat_stale:
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_LIVE,
|
||||
reason=REASON_LIVE_OWNER,
|
||||
retireable=False,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=True,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=False,
|
||||
has_live_lease=False,
|
||||
client_managed=client_managed,
|
||||
)
|
||||
|
||||
if pid_alive and heartbeat_stale:
|
||||
# Process still exists but has not heartbeated — disconnected, not
|
||||
# confirmed stale. Do not retire; operator/reconnect owns next step.
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_DISCONNECTED,
|
||||
reason=REASON_LIVE_OWNER,
|
||||
retireable=False,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=True,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=True,
|
||||
has_live_lease=False,
|
||||
client_managed=client_managed,
|
||||
details={"note": "alive_pid_stale_heartbeat_not_retired"},
|
||||
)
|
||||
|
||||
# PID dead (or checker said not alive).
|
||||
reason = REASON_DEAD_OWNER
|
||||
if heartbeat_stale:
|
||||
reason = REASON_HEARTBEAT_STALE_DEAD
|
||||
return SessionClassification(
|
||||
session_id=session_id,
|
||||
classification=CLASS_STALE,
|
||||
reason=reason,
|
||||
retireable=True,
|
||||
status=status,
|
||||
pid=pid,
|
||||
pid_alive=False,
|
||||
pid_reused=False,
|
||||
heartbeat_stale=heartbeat_stale,
|
||||
has_live_lease=False,
|
||||
client_managed=client_managed,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SessionFleetReport:
|
||||
"""Fleet-wide classification summary for reconcile + apply."""
|
||||
|
||||
classifications: tuple[SessionClassification, ...]
|
||||
live_count: int
|
||||
disconnected_count: int
|
||||
stale_count: int
|
||||
protected_count: int
|
||||
terminal_count: int
|
||||
retireable: tuple[SessionClassification, ...]
|
||||
counts_by_reason: dict[str, int]
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"live_count": self.live_count,
|
||||
"disconnected_count": self.disconnected_count,
|
||||
"stale_count": self.stale_count,
|
||||
"protected_count": self.protected_count,
|
||||
"terminal_count": self.terminal_count,
|
||||
"retireable_count": len(self.retireable),
|
||||
"retireable_session_ids": [c.session_id for c in self.retireable],
|
||||
"counts_by_reason": dict(self.counts_by_reason),
|
||||
"classifications": [c.as_dict() for c in self.classifications],
|
||||
"sessions_dimension_resolved": len(self.retireable) == 0,
|
||||
}
|
||||
|
||||
|
||||
def classify_sessions(
|
||||
sessions: Sequence[Mapping[str, Any]] | None,
|
||||
*,
|
||||
leases: Sequence[Mapping[str, Any]] | None = None,
|
||||
now: datetime | None = None,
|
||||
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
||||
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
||||
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
||||
) -> SessionFleetReport:
|
||||
"""Classify a fleet of session rows against live leases (#969)."""
|
||||
moment = now or _utc_now()
|
||||
live_leases = live_lease_session_ids(
|
||||
leases, now=moment, pid_checker=pid_checker
|
||||
)
|
||||
results: list[SessionClassification] = []
|
||||
for row in sessions or ():
|
||||
if not isinstance(row, Mapping):
|
||||
continue
|
||||
results.append(
|
||||
classify_session(
|
||||
row,
|
||||
now=moment,
|
||||
pid_checker=pid_checker,
|
||||
process_start_probe=process_start_probe,
|
||||
live_lease_sessions=live_leases,
|
||||
client_managed_sessions=client_managed_sessions,
|
||||
heartbeat_stale_seconds=heartbeat_stale_seconds,
|
||||
)
|
||||
)
|
||||
|
||||
live_count = sum(1 for c in results if c.classification == CLASS_LIVE)
|
||||
disconnected_count = sum(
|
||||
1 for c in results if c.classification == CLASS_DISCONNECTED
|
||||
)
|
||||
stale_count = sum(1 for c in results if c.classification == CLASS_STALE)
|
||||
protected_count = sum(1 for c in results if c.classification == CLASS_PROTECTED)
|
||||
terminal_count = sum(1 for c in results if c.classification == CLASS_TERMINAL)
|
||||
retireable = tuple(c for c in results if c.retireable)
|
||||
by_reason: dict[str, int] = {}
|
||||
for c in results:
|
||||
by_reason[c.reason] = by_reason.get(c.reason, 0) + 1
|
||||
|
||||
return SessionFleetReport(
|
||||
classifications=tuple(results),
|
||||
live_count=live_count,
|
||||
disconnected_count=disconnected_count,
|
||||
stale_count=stale_count,
|
||||
protected_count=protected_count,
|
||||
terminal_count=terminal_count,
|
||||
retireable=retireable,
|
||||
counts_by_reason=by_reason,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RetirementResult:
|
||||
"""Outcome of one session retirement attempt."""
|
||||
|
||||
session_id: str
|
||||
outcome: str # retired | already_terminal | skipped | blocked | missing
|
||||
reason: str
|
||||
prior_status: str | None = None
|
||||
new_status: str | None = None
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"session_id": self.session_id,
|
||||
"outcome": self.outcome,
|
||||
"reason": self.reason,
|
||||
"prior_status": self.prior_status,
|
||||
"new_status": self.new_status,
|
||||
"details": dict(self.details),
|
||||
}
|
||||
|
||||
|
||||
def apply_session_retirements(
|
||||
db: cpd.ControlPlaneDB,
|
||||
report: SessionFleetReport,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
actor_session_id: str | None = None,
|
||||
now: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Terminalize every retireable session in *report* (idempotent).
|
||||
|
||||
Concurrent reconciles are safe: each retirement is a CAS on
|
||||
``status='active'`` (or other non-terminal). A second pass that sees the
|
||||
same session already retired records ``already_terminal`` rather than
|
||||
duplicating audit noise beyond a single no-op outcome.
|
||||
"""
|
||||
moment = now or _utc_now()
|
||||
results: list[RetirementResult] = []
|
||||
retired = 0
|
||||
already = 0
|
||||
blocked = 0
|
||||
missing = 0
|
||||
|
||||
for classification in report.retireable:
|
||||
if not classification.retireable:
|
||||
continue
|
||||
if dry_run:
|
||||
results.append(
|
||||
RetirementResult(
|
||||
session_id=classification.session_id,
|
||||
outcome="skipped",
|
||||
reason=classification.reason,
|
||||
prior_status=classification.status,
|
||||
new_status=SESSION_STATUS_RETIRED,
|
||||
details={"dry_run": True, **classification.details},
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
applied = db.retire_session(
|
||||
session_id=classification.session_id,
|
||||
reason=classification.reason,
|
||||
actor_session_id=actor_session_id,
|
||||
details={
|
||||
"classification": classification.classification,
|
||||
"pid": classification.pid,
|
||||
"pid_alive": classification.pid_alive,
|
||||
"pid_reused": classification.pid_reused,
|
||||
"client_managed": classification.client_managed,
|
||||
**classification.details,
|
||||
},
|
||||
now=moment,
|
||||
)
|
||||
outcome = str(applied.get("outcome") or "missing")
|
||||
results.append(
|
||||
RetirementResult(
|
||||
session_id=classification.session_id,
|
||||
outcome=outcome,
|
||||
reason=str(applied.get("reason") or classification.reason),
|
||||
prior_status=applied.get("prior_status"),
|
||||
new_status=applied.get("new_status"),
|
||||
details=dict(applied.get("details") or {}),
|
||||
)
|
||||
)
|
||||
if outcome == "retired":
|
||||
retired += 1
|
||||
elif outcome == "already_terminal":
|
||||
already += 1
|
||||
elif outcome == "blocked":
|
||||
blocked += 1
|
||||
else:
|
||||
missing += 1
|
||||
|
||||
# Non-retireable classifications are recorded for audit completeness when
|
||||
# dry-run lists the fleet, but apply only mutates retireable rows.
|
||||
return {
|
||||
"success": True,
|
||||
"dry_run": dry_run,
|
||||
"retired_count": retired,
|
||||
"already_terminal_count": already,
|
||||
"blocked_count": blocked,
|
||||
"missing_count": missing,
|
||||
"planned_count": len(report.retireable),
|
||||
"results": [r.as_dict() for r in results],
|
||||
"audit_action": EVENT_SESSION_RETIRED,
|
||||
"actor_session_id": actor_session_id,
|
||||
"recorded_at": _ts(moment),
|
||||
}
|
||||
|
||||
|
||||
def retire_stale_sessions(
|
||||
db: cpd.ControlPlaneDB,
|
||||
*,
|
||||
sessions: Sequence[Mapping[str, Any]] | None = None,
|
||||
leases: Sequence[Mapping[str, Any]] | None = None,
|
||||
dry_run: bool = False,
|
||||
actor_session_id: str | None = None,
|
||||
now: datetime | None = None,
|
||||
pid_checker: Callable[[int | None], bool] = lease_lifecycle.is_process_alive,
|
||||
process_start_probe: Callable[[int | None], datetime | None] = process_start_time,
|
||||
client_managed_sessions: set[str] | frozenset[str] | None = None,
|
||||
heartbeat_stale_seconds: int = DEFAULT_HEARTBEAT_STALE_SECONDS,
|
||||
session_limit: int = 500,
|
||||
) -> dict[str, Any]:
|
||||
"""End-to-end plan + apply for stale session retirement (#969).
|
||||
|
||||
When *sessions* is omitted the control-plane DB is inventoried (active
|
||||
rows only). Callers that already gathered inventory should pass it.
|
||||
"""
|
||||
moment = now or _utc_now()
|
||||
if sessions is None:
|
||||
sessions = db.list_sessions(
|
||||
statuses=(SESSION_STATUS_ACTIVE,),
|
||||
limit=max(1, int(session_limit)),
|
||||
)
|
||||
if leases is None:
|
||||
try:
|
||||
leases = db.list_leases(statuses=("active",), limit=max(1, int(session_limit)))
|
||||
except Exception: # noqa: BLE001
|
||||
leases = []
|
||||
|
||||
report = classify_sessions(
|
||||
sessions,
|
||||
leases=leases,
|
||||
now=moment,
|
||||
pid_checker=pid_checker,
|
||||
process_start_probe=process_start_probe,
|
||||
client_managed_sessions=client_managed_sessions,
|
||||
heartbeat_stale_seconds=heartbeat_stale_seconds,
|
||||
)
|
||||
apply_result = apply_session_retirements(
|
||||
db,
|
||||
report,
|
||||
dry_run=dry_run,
|
||||
actor_session_id=actor_session_id,
|
||||
now=moment,
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"fleet": report.as_dict(),
|
||||
"apply": apply_result,
|
||||
"sessions_dimension_resolved": (
|
||||
report.as_dict()["sessions_dimension_resolved"]
|
||||
if dry_run
|
||||
else apply_result["planned_count"]
|
||||
== (
|
||||
apply_result["retired_count"]
|
||||
+ apply_result["already_terminal_count"]
|
||||
)
|
||||
and apply_result["blocked_count"] == 0
|
||||
),
|
||||
}
|
||||
+12
-2
@@ -153,8 +153,9 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = {
|
||||
"permission": "gitea.branch.push",
|
||||
"role": "author",
|
||||
},
|
||||
# #662: post-restart reconcile is read-only inventory + pure classification.
|
||||
# Durable follow-up issue creation is a separate apply path (not this task).
|
||||
# #662: post-restart reconcile is inventory + pure classification.
|
||||
# #969: optional apply_session_cleanup retires confirmed-stale session rows
|
||||
# through the same tool; durable follow-up Gitea issues remain separate.
|
||||
"reconcile_after_restart": {
|
||||
"permission": "gitea.read",
|
||||
"role": "author",
|
||||
@@ -163,6 +164,15 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = {
|
||||
"permission": "gitea.read",
|
||||
"role": "author",
|
||||
},
|
||||
# #969: explicit plan/apply path for dead-owner session retirement.
|
||||
"retire_stale_workflow_sessions": {
|
||||
"permission": "gitea.read",
|
||||
"role": "author",
|
||||
},
|
||||
"gitea_retire_stale_workflow_sessions": {
|
||||
"permission": "gitea.read",
|
||||
"role": "author",
|
||||
},
|
||||
# #644: Phase 2 Web Console recovery tasks.
|
||||
"clear_stale_binding": {
|
||||
"permission": "gitea.read",
|
||||
|
||||
@@ -37,7 +37,7 @@ class ControlPlaneDBTest(unittest.TestCase):
|
||||
rows = dict(conn.execute("SELECT key, value FROM schema_meta").fetchall())
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(rows["schema_version"], "5")
|
||||
self.assertEqual(rows["schema_version"], "6")
|
||||
self.assertIn("DB coordinates", rows["architecture"])
|
||||
self.assertIn("bridge", rows["architecture"].lower())
|
||||
|
||||
@@ -868,7 +868,7 @@ class SessionCheckpointTest(unittest.TestCase):
|
||||
conn.close()
|
||||
self.assertIn("session_checkpoints", names)
|
||||
record = self._write()
|
||||
self.assertEqual(record["checkpoint_schema_version"], 5)
|
||||
self.assertEqual(record["checkpoint_schema_version"], 6)
|
||||
|
||||
# AC2 — checkpoints written for multi-role session fixtures.
|
||||
def test_multi_role_fixtures_each_get_a_row(self) -> None:
|
||||
|
||||
@@ -112,7 +112,19 @@ class TestIssue686ManualMcpProvenance(unittest.TestCase):
|
||||
self.assertTrue(any("All matching profiles for task 'create_issue' (['prgs-author']) are running but stale" in r for r in reasons))
|
||||
|
||||
def test_namespace_health_classification_includes_provenance(self):
|
||||
"""AC 1 & 4: mcp_namespace_health diagnostics include provenance and unconsumed_gitea_env."""
|
||||
"""AC 1 & 4: mcp_namespace_health diagnostics include provenance and unconsumed_gitea_env.
|
||||
|
||||
#948 narrowed the vocabulary here. This process carries no client-managed
|
||||
declaration, so the old code labelled it ``manual_launch`` — asserting a
|
||||
hand-launched terminal process it had no evidence for, and contradicting
|
||||
``gitea_get_runtime_context``, which read the same process and reported
|
||||
``client_managed``. Absence of proof is now reported as ``unproven``.
|
||||
|
||||
The #686 wall itself is unchanged and still asserted below:
|
||||
``is_client_managed`` stays False, so nothing previously refused is now
|
||||
permitted. Only the label on the *reason* changed, so remediation names
|
||||
the proof that is actually missing.
|
||||
"""
|
||||
process = {
|
||||
"pid": 5555,
|
||||
"profile": "prgs-author",
|
||||
@@ -129,10 +141,12 @@ class TestIssue686ManualMcpProvenance(unittest.TestCase):
|
||||
process=process,
|
||||
probe_source="client_namespace",
|
||||
)
|
||||
self.assertEqual(res["provenance"], "manual_launch")
|
||||
self.assertEqual(res["provenance"], "unproven")
|
||||
self.assertFalse(res["is_client_managed"])
|
||||
# The wall is intact: no client-managed proof still fails closed.
|
||||
self.assertTrue(res["provenance_fail_closed"])
|
||||
self.assertEqual(res["unconsumed_gitea_env"], {"GITEA_DUMMY": "99"})
|
||||
self.assertEqual(res["diagnostics"]["provenance"], "manual_launch")
|
||||
self.assertEqual(res["diagnostics"]["provenance"], "unproven")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,662 @@
|
||||
"""Client/session-aware runtime ownership and provenance (#948).
|
||||
|
||||
Covers the reproduced contradiction that motivated the issue: one surface
|
||||
reporting ``client_managed`` while another reported ``manual_launch`` for the
|
||||
same process, remediation hardcoded to one vendor, and a profile-wide duplicate
|
||||
wall that could not tell two healthy clients apart.
|
||||
|
||||
All client and session identifiers here are synthetic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import mcp_client_reconnect
|
||||
import mcp_namespace_health
|
||||
import mcp_worker_identity as mwi
|
||||
|
||||
|
||||
NOW = datetime(2026, 7, 29, 6, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _registry() -> mwi.WorkerRegistry:
|
||||
"""A registry on a throwaway path; never the operator's real one."""
|
||||
handle, path = tempfile.mkstemp(suffix=".sqlite3")
|
||||
os.close(handle)
|
||||
os.unlink(path)
|
||||
return mwi.WorkerRegistry(path)
|
||||
|
||||
|
||||
def _attach(
|
||||
registry: mwi.WorkerRegistry,
|
||||
*,
|
||||
client: str,
|
||||
session: str,
|
||||
generation: str,
|
||||
profile: str = "prgs-reviewer",
|
||||
role: str = "reviewer",
|
||||
pid: int = 4242,
|
||||
now: datetime = NOW,
|
||||
ttl: float = 900.0,
|
||||
) -> dict:
|
||||
"""Register one synthetic worker and return the outcome."""
|
||||
identity = mwi.generate_worker_identity(client, session, now=now)
|
||||
outcome = registry.register(
|
||||
worker_identity=identity,
|
||||
client_name=client,
|
||||
client_instance_id=f"inst-{session}",
|
||||
session_id=session,
|
||||
generation_id=generation,
|
||||
role=role,
|
||||
profile=profile,
|
||||
pid=pid,
|
||||
heartbeat_ttl_seconds=ttl,
|
||||
now=now,
|
||||
)
|
||||
outcome["identity"] = identity
|
||||
return outcome
|
||||
|
||||
|
||||
class IdentityFormatTests(unittest.TestCase):
|
||||
"""AC27-29: collision-resistant `<llm-name>-<UTC-timestamp>-<short-sha>`."""
|
||||
|
||||
def test_identity_matches_required_format(self):
|
||||
identity = mwi.generate_worker_identity("Gemini", "sess-0001", now=NOW)
|
||||
parsed = mwi.parse_worker_identity(identity)
|
||||
self.assertTrue(parsed["valid"], parsed["reasons"])
|
||||
self.assertEqual(parsed["client_name"], "gemini")
|
||||
self.assertEqual(parsed["minted_at"], "20260729T060000Z")
|
||||
self.assertEqual(len(parsed["digest"]), 12)
|
||||
|
||||
def test_digest_varies_with_session_and_nonce(self):
|
||||
base = dict(timestamp_ns=1, now=NOW)
|
||||
a = mwi.generate_worker_identity("codex", "sess-A", nonce="n", **base)
|
||||
b = mwi.generate_worker_identity("codex", "sess-B", nonce="n", **base)
|
||||
c = mwi.generate_worker_identity("codex", "sess-A", nonce="m", **base)
|
||||
self.assertNotEqual(a, b, "session must feed the digest")
|
||||
self.assertNotEqual(a, c, "nonce must feed the digest")
|
||||
|
||||
def test_identity_is_not_role_or_profile(self):
|
||||
"""AC26: identity is independent of role and profile."""
|
||||
args = dict(timestamp_ns=7, nonce="fixed", now=NOW)
|
||||
same = mwi.generate_worker_identity("claude", "sess-1", **args)
|
||||
self.assertEqual(same, mwi.generate_worker_identity("claude", "sess-1", **args))
|
||||
# Nothing role- or profile-derived appears in the identity.
|
||||
self.assertNotIn("reviewer", same)
|
||||
self.assertNotIn("prgs", same)
|
||||
|
||||
def test_malformed_identity_rejected(self):
|
||||
self.assertFalse(mwi.parse_worker_identity("prgs-reviewer")["valid"])
|
||||
self.assertFalse(mwi.parse_worker_identity("")["valid"])
|
||||
self.assertFalse(mwi.parse_worker_identity(None)["valid"])
|
||||
|
||||
|
||||
class PerClientAttachmentTests(unittest.TestCase):
|
||||
"""Every supported client attaches and is reported as itself."""
|
||||
|
||||
def _assert_attached_as(self, client: str, expected_name: str):
|
||||
registry = _registry()
|
||||
outcome = _attach(
|
||||
registry, client=client, session=f"sess-{client}", generation="gen-1"
|
||||
)
|
||||
self.assertTrue(outcome["registered"], outcome["reasons"])
|
||||
|
||||
verdict = mwi.assess_provenance(
|
||||
registry=registry, worker_identity=outcome["identity"], env={}, now=NOW
|
||||
)
|
||||
self.assertEqual(verdict["session_ownership"], mwi.OWNERSHIP_OWNED)
|
||||
self.assertEqual(verdict["provenance"], mwi.PROVENANCE_CLIENT_SESSION)
|
||||
self.assertEqual(verdict["client_name"], expected_name)
|
||||
self.assertTrue(verdict["session_owned"])
|
||||
self.assertFalse(verdict["fail_closed"])
|
||||
return verdict
|
||||
|
||||
def test_codex_attachment(self):
|
||||
self._assert_attached_as("codex", "codex")
|
||||
|
||||
def test_gemini_attachment(self):
|
||||
self._assert_attached_as("gemini", "gemini")
|
||||
|
||||
def test_antigravity_attachment(self):
|
||||
self._assert_attached_as("antigravity", "antigravity")
|
||||
|
||||
def test_claude_attachment(self):
|
||||
self._assert_attached_as("claude", "claude_code")
|
||||
|
||||
def test_unknown_client_is_named_not_guessed(self):
|
||||
verdict = self._assert_attached_as("some_new_llm", "some_new_llm")
|
||||
self.assertNotEqual(verdict["client_name"], "codex")
|
||||
|
||||
|
||||
class SessionLifecycleTests(unittest.TestCase):
|
||||
def test_same_client_new_session_gets_distinct_identity(self):
|
||||
registry = _registry()
|
||||
first = _attach(registry, client="codex", session="sess-1", generation="gen-1")
|
||||
second = _attach(registry, client="codex", session="sess-2", generation="gen-2")
|
||||
self.assertTrue(first["registered"])
|
||||
self.assertTrue(second["registered"])
|
||||
self.assertNotEqual(first["identity"], second["identity"])
|
||||
|
||||
# Both are live and neither blocks the other.
|
||||
cohort = mwi.classify_cohort(registry.list_workers(), now=NOW)
|
||||
self.assertEqual(cohort["live_worker_count"], 2)
|
||||
self.assertFalse(cohort["blocked"], cohort["reasons"])
|
||||
|
||||
def test_different_client_attaches_after_previous_session_ends(self):
|
||||
"""AC14: expiry then takeover with a higher fencing epoch."""
|
||||
registry = _registry()
|
||||
gone = _attach(
|
||||
registry, client="codex", session="sess-old", generation="gen-shared", ttl=60
|
||||
)
|
||||
later = NOW + timedelta(hours=1)
|
||||
self.assertFalse(
|
||||
registry.is_live(registry.get(gone["identity"]), now=later)["live"]
|
||||
)
|
||||
|
||||
arriving = _attach(
|
||||
registry,
|
||||
client="gemini",
|
||||
session="sess-new",
|
||||
generation="gen-other",
|
||||
now=later,
|
||||
)
|
||||
claim = registry.claim_generation(
|
||||
worker_identity=arriving["identity"],
|
||||
generation_id="gen-shared",
|
||||
now=later,
|
||||
)
|
||||
self.assertTrue(claim["claimed"], claim["reasons"])
|
||||
self.assertIn(gone["identity"], claim["superseded_workers"])
|
||||
self.assertGreater(claim["fencing_epoch"], gone["fencing_epoch"])
|
||||
|
||||
def test_superseded_session_is_fenced_on_resume(self):
|
||||
"""AC15/AC16: the prior session cannot heartbeat its way back."""
|
||||
registry = _registry()
|
||||
old = _attach(
|
||||
registry, client="codex", session="sess-old", generation="gen-shared", ttl=60
|
||||
)
|
||||
later = NOW + timedelta(hours=1)
|
||||
new = _attach(
|
||||
registry, client="gemini", session="sess-new", generation="gen-x", now=later
|
||||
)
|
||||
registry.claim_generation(
|
||||
worker_identity=new["identity"], generation_id="gen-shared", now=later
|
||||
)
|
||||
|
||||
resumed = registry.heartbeat(
|
||||
worker_identity=old["identity"],
|
||||
fencing_epoch=old["fencing_epoch"],
|
||||
now=later,
|
||||
)
|
||||
self.assertFalse(resumed["renewed"])
|
||||
self.assertFalse(resumed["mutation_performed"])
|
||||
self.assertEqual(resumed["blocker_kind"], mwi.BLOCKER_FENCED)
|
||||
|
||||
def test_heartbeat_renews_only_the_owning_lease(self):
|
||||
"""AC11: a wrong epoch never renews, and never mutates."""
|
||||
registry = _registry()
|
||||
worker = _attach(registry, client="codex", session="s", generation="g")
|
||||
good = registry.heartbeat(
|
||||
worker_identity=worker["identity"],
|
||||
fencing_epoch=worker["fencing_epoch"],
|
||||
now=NOW + timedelta(minutes=5),
|
||||
)
|
||||
self.assertTrue(good["renewed"])
|
||||
|
||||
bad = registry.heartbeat(
|
||||
worker_identity=worker["identity"],
|
||||
fencing_epoch=worker["fencing_epoch"] + 99,
|
||||
now=NOW + timedelta(minutes=6),
|
||||
)
|
||||
self.assertFalse(bad["renewed"])
|
||||
self.assertFalse(bad["mutation_performed"])
|
||||
self.assertEqual(
|
||||
registry.get(worker["identity"])["last_heartbeat_at"],
|
||||
good["last_heartbeat_at"],
|
||||
"a refused heartbeat must not advance the record",
|
||||
)
|
||||
|
||||
|
||||
class ConflictAndCollisionTests(unittest.TestCase):
|
||||
def test_two_live_sessions_cannot_claim_one_generation(self):
|
||||
registry = _registry()
|
||||
first = _attach(registry, client="codex", session="s1", generation="gen-shared")
|
||||
second = _attach(registry, client="gemini", session="s2", generation="gen-other")
|
||||
|
||||
claim = registry.claim_generation(
|
||||
worker_identity=second["identity"],
|
||||
generation_id="gen-shared",
|
||||
now=NOW,
|
||||
)
|
||||
self.assertFalse(claim["claimed"])
|
||||
self.assertFalse(claim["mutation_performed"])
|
||||
self.assertEqual(claim["blocker_kind"], mwi.BLOCKER_CONFLICTING_SESSIONS)
|
||||
self.assertEqual(
|
||||
claim["conflicting_owners"][0]["worker_identity"], first["identity"]
|
||||
)
|
||||
# The sanctioned recovery must never be "kill the other process".
|
||||
self.assertIn("Do not kill", claim["exact_next_action"])
|
||||
|
||||
def test_contested_generation_fails_closed_in_assessment(self):
|
||||
registry = _registry()
|
||||
first = _attach(registry, client="codex", session="s1", generation="gen-shared")
|
||||
_attach(registry, client="gemini", session="s2", generation="gen-shared")
|
||||
|
||||
verdict = mwi.assess_provenance(
|
||||
registry=registry, worker_identity=first["identity"], env={}, now=NOW
|
||||
)
|
||||
self.assertEqual(verdict["session_ownership"], mwi.OWNERSHIP_CONTESTED)
|
||||
self.assertTrue(verdict["fail_closed"])
|
||||
self.assertEqual(verdict["blocker_kind"], mwi.BLOCKER_CONTRADICTORY)
|
||||
self.assertTrue(verdict["conflicting_live_sessions"])
|
||||
|
||||
def test_identity_collision_is_refused_without_corrupting_existing(self):
|
||||
"""AC31: never replace, adopt, merge with, or corrupt the incumbent."""
|
||||
registry = _registry()
|
||||
incumbent = _attach(registry, client="codex", session="s1", generation="gen-1")
|
||||
before = registry.get(incumbent["identity"])
|
||||
|
||||
collided = registry.register(
|
||||
worker_identity=incumbent["identity"],
|
||||
client_name="gemini",
|
||||
client_instance_id="inst-other",
|
||||
session_id="s2",
|
||||
generation_id="gen-2",
|
||||
pid=9999,
|
||||
now=NOW,
|
||||
)
|
||||
self.assertFalse(collided["registered"])
|
||||
self.assertTrue(collided["collision"])
|
||||
self.assertFalse(collided["mutation_performed"])
|
||||
self.assertEqual(collided["blocker_kind"], mwi.BLOCKER_IDENTITY_COLLISION)
|
||||
self.assertEqual(collided["collision_kind"], "active_worker")
|
||||
self.assertEqual(
|
||||
registry.get(incumbent["identity"]), before, "incumbent must be untouched"
|
||||
)
|
||||
|
||||
def test_after_collision_a_regenerated_identity_registers(self):
|
||||
"""AC32/AC35: forced collision, safe regeneration, successful replacement."""
|
||||
registry = _registry()
|
||||
fixed = dict(timestamp_ns=99, nonce="deterministic", now=NOW)
|
||||
forced = mwi.generate_worker_identity("codex", "sess-collide", **fixed)
|
||||
first = registry.register(
|
||||
worker_identity=forced,
|
||||
client_name="codex",
|
||||
client_instance_id="inst-1",
|
||||
session_id="sess-collide",
|
||||
generation_id="gen-1",
|
||||
now=NOW,
|
||||
)
|
||||
self.assertTrue(first["registered"])
|
||||
|
||||
# A second worker deriving the same inputs collides deterministically.
|
||||
again = mwi.generate_worker_identity("codex", "sess-collide", **fixed)
|
||||
self.assertEqual(again, forced)
|
||||
self.assertTrue(
|
||||
registry.register(
|
||||
worker_identity=again,
|
||||
client_name="codex",
|
||||
client_instance_id="inst-2",
|
||||
session_id="sess-collide",
|
||||
generation_id="gen-2",
|
||||
now=NOW,
|
||||
)["collision"]
|
||||
)
|
||||
|
||||
replacement = mwi.generate_worker_identity(
|
||||
"codex", "sess-collide", timestamp_ns=100, nonce="different", now=NOW
|
||||
)
|
||||
self.assertNotEqual(replacement, forced)
|
||||
self.assertTrue(
|
||||
registry.register(
|
||||
worker_identity=replacement,
|
||||
client_name="codex",
|
||||
client_instance_id="inst-2",
|
||||
session_id="sess-collide",
|
||||
generation_id="gen-2",
|
||||
now=NOW,
|
||||
)["registered"]
|
||||
)
|
||||
|
||||
def test_restarted_worker_inherits_nothing(self):
|
||||
"""AC33/AC34: a restart mints a new identity and no prior epoch."""
|
||||
registry = _registry()
|
||||
before = _attach(
|
||||
registry, client="codex", session="sess-before", generation="gen-1", ttl=60
|
||||
)
|
||||
later = NOW + timedelta(hours=2)
|
||||
after = _attach(
|
||||
registry, client="codex", session="sess-after", generation="gen-2", now=later
|
||||
)
|
||||
self.assertNotEqual(before["identity"], after["identity"])
|
||||
self.assertNotEqual(
|
||||
registry.get(after["identity"])["generation_id"],
|
||||
registry.get(before["identity"])["generation_id"],
|
||||
)
|
||||
|
||||
|
||||
class LivenessTests(unittest.TestCase):
|
||||
def test_stale_session_record_is_not_live(self):
|
||||
registry = _registry()
|
||||
worker = _attach(registry, client="codex", session="s", generation="g", ttl=300)
|
||||
stale = registry.is_live(
|
||||
registry.get(worker["identity"]), now=NOW + timedelta(hours=1)
|
||||
)
|
||||
self.assertFalse(stale["live"])
|
||||
self.assertFalse(stale["heartbeat_fresh"])
|
||||
|
||||
def test_liveness_is_not_pid_comparison_alone(self):
|
||||
"""AC7: a live PID does not resurrect an expired registration."""
|
||||
registry = _registry()
|
||||
worker = _attach(registry, client="codex", session="s", generation="g", ttl=60)
|
||||
verdict = registry.is_live(
|
||||
registry.get(worker["identity"]),
|
||||
now=NOW + timedelta(hours=1),
|
||||
pid_alive=True,
|
||||
)
|
||||
self.assertFalse(
|
||||
verdict["live"], "a live PID must not override a dead heartbeat"
|
||||
)
|
||||
|
||||
def test_dead_pid_withdraws_liveness_from_a_fresh_heartbeat(self):
|
||||
registry = _registry()
|
||||
worker = _attach(registry, client="codex", session="s", generation="g")
|
||||
verdict = registry.is_live(
|
||||
registry.get(worker["identity"]), now=NOW, pid_alive=False
|
||||
)
|
||||
self.assertFalse(verdict["live"])
|
||||
|
||||
def test_stale_ownership_does_not_permanently_strand_a_daemon(self):
|
||||
registry = _registry()
|
||||
stranded = _attach(
|
||||
registry, client="codex", session="s-old", generation="gen-daemon", ttl=60
|
||||
)
|
||||
later = NOW + timedelta(hours=3)
|
||||
rescuer = _attach(
|
||||
registry, client="claude", session="s-new", generation="gen-tmp", now=later
|
||||
)
|
||||
claim = registry.claim_generation(
|
||||
worker_identity=rescuer["identity"],
|
||||
generation_id="gen-daemon",
|
||||
now=later,
|
||||
)
|
||||
self.assertTrue(claim["claimed"], claim["reasons"])
|
||||
self.assertIn(stranded["identity"], claim["superseded_workers"])
|
||||
|
||||
|
||||
class EvidenceTests(unittest.TestCase):
|
||||
def test_env_flag_alone_does_not_prove_session_ownership(self):
|
||||
verdict = mwi.assess_provenance(
|
||||
registry=None,
|
||||
worker_identity=None,
|
||||
env={"GITEA_CLIENT_MANAGED": "1", "GITEA_MCP_SANCTIONED_DAEMON": "1"},
|
||||
now=NOW,
|
||||
)
|
||||
self.assertFalse(verdict["session_owned"])
|
||||
self.assertEqual(verdict["session_ownership"], mwi.OWNERSHIP_UNOWNED)
|
||||
self.assertTrue(verdict["env_flag_only"])
|
||||
self.assertTrue(verdict["fail_closed"])
|
||||
self.assertNotIn(mwi.EVIDENCE_ATTACHMENT_RECORD, verdict["evidence"])
|
||||
self.assertFalse(verdict["env_signal"]["proves_session_ownership"])
|
||||
|
||||
def test_env_flag_still_answers_the_launch_question(self):
|
||||
"""The #686 wall is preserved: env decides launch, not ownership."""
|
||||
self.assertTrue(
|
||||
mwi.assess_launch_provenance({"GITEA_CLIENT_MANAGED": "1"})["client_managed"]
|
||||
)
|
||||
self.assertFalse(
|
||||
mwi.assess_launch_provenance({"GITEA_CLIENT_MANAGED": "0"})["client_managed"]
|
||||
)
|
||||
self.assertFalse(
|
||||
mwi.assess_launch_provenance({}, stdin_is_tty=True)["client_managed"]
|
||||
)
|
||||
self.assertTrue(
|
||||
mwi.assess_launch_provenance({"GITEA_MCP_PROFILE": "prgs-author"})[
|
||||
"client_managed"
|
||||
]
|
||||
)
|
||||
|
||||
def test_missing_evidence_is_unproven_not_manual(self):
|
||||
"""A missing proof must not be reported as a hand-launched process."""
|
||||
verdict = mwi.assess_provenance(
|
||||
registry=None, worker_identity=None, env={}, now=NOW
|
||||
)
|
||||
self.assertEqual(verdict["provenance"], mwi.PROVENANCE_UNPROVEN)
|
||||
self.assertNotEqual(verdict["provenance"], mwi.PROVENANCE_MANUAL)
|
||||
self.assertTrue(verdict["fail_closed"])
|
||||
|
||||
def test_declared_manual_launch_is_reported_as_manual(self):
|
||||
verdict = mwi.assess_provenance(
|
||||
registry=None,
|
||||
worker_identity=None,
|
||||
env={"GITEA_CLIENT_MANAGED": "0"},
|
||||
now=NOW,
|
||||
)
|
||||
self.assertEqual(verdict["provenance"], mwi.PROVENANCE_MANUAL)
|
||||
|
||||
def test_fail_closed_refusal_names_its_scope_not_the_profile(self):
|
||||
"""AC17/AC41: no refusal is profile-wide."""
|
||||
verdict = mwi.assess_provenance(
|
||||
registry=None,
|
||||
worker_identity=None,
|
||||
env={},
|
||||
profile="prgs-reviewer",
|
||||
role="reviewer",
|
||||
now=NOW,
|
||||
)
|
||||
self.assertFalse(verdict["scope"]["profile_wide"])
|
||||
self.assertEqual(verdict["blocker_kind"], mwi.BLOCKER_NO_ATTACHMENT)
|
||||
|
||||
|
||||
class CohortScopingTests(unittest.TestCase):
|
||||
def test_shared_profile_with_distinct_identities_does_not_block(self):
|
||||
"""AC40: profile is not a singleton identity."""
|
||||
registry = _registry()
|
||||
_attach(
|
||||
registry,
|
||||
client="codex",
|
||||
session="s1",
|
||||
generation="g1",
|
||||
profile="prgs-reviewer",
|
||||
)
|
||||
_attach(
|
||||
registry,
|
||||
client="gemini",
|
||||
session="s2",
|
||||
generation="g2",
|
||||
profile="prgs-reviewer",
|
||||
)
|
||||
|
||||
cohort = mwi.classify_cohort(registry.list_workers(), now=NOW)
|
||||
self.assertFalse(cohort["blocked"], cohort["reasons"])
|
||||
self.assertEqual(cohort["blocker_kind"], mwi.BLOCKER_NONE)
|
||||
self.assertIn("prgs-reviewer", cohort["shared_profiles"])
|
||||
self.assertTrue(cohort["profile_sharing_permitted"])
|
||||
self.assertEqual(cohort["blocked_worker_identities"], [])
|
||||
|
||||
def test_duplicate_cohort_records_block_only_the_offenders(self):
|
||||
registry = _registry()
|
||||
_attach(registry, client="codex", session="s1", generation="gen-contested")
|
||||
_attach(registry, client="gemini", session="s2", generation="gen-contested")
|
||||
_attach(registry, client="claude", session="s3", generation="gen-fine")
|
||||
|
||||
cohort = mwi.classify_cohort(registry.list_workers(), now=NOW)
|
||||
self.assertTrue(cohort["blocked"])
|
||||
self.assertEqual(cohort["contested_generations"], ["gen-contested"])
|
||||
self.assertEqual(len(cohort["blocked_worker_identities"]), 2)
|
||||
|
||||
def test_mixed_runtime_generations_are_scoped_independently(self):
|
||||
"""AC17: one stale generation does not wall unrelated healthy ones."""
|
||||
registry = _registry()
|
||||
stale = _attach(
|
||||
registry, client="codex", session="s1", generation="gen-stale", ttl=60
|
||||
)
|
||||
healthy_a = _attach(registry, client="gemini", session="s2", generation="gen-a")
|
||||
healthy_b = _attach(registry, client="claude", session="s3", generation="gen-b")
|
||||
|
||||
scoped = mwi.scope_runtime_failure(
|
||||
failure_kind="stale-runtime",
|
||||
worker_identity=stale["identity"],
|
||||
profile="prgs-reviewer",
|
||||
all_live_workers=registry.list_workers(),
|
||||
)
|
||||
self.assertFalse(scoped["profile_wide"])
|
||||
self.assertFalse(scoped["fleet_wide"])
|
||||
self.assertEqual(len(scoped["affected_workers"]), 1)
|
||||
self.assertEqual(scoped["unaffected_worker_count"], 2)
|
||||
unaffected = {w["worker_identity"] for w in scoped["unaffected_workers"]}
|
||||
self.assertEqual(unaffected, {healthy_a["identity"], healthy_b["identity"]})
|
||||
|
||||
|
||||
class HardcodedClientRegressionTests(unittest.TestCase):
|
||||
def test_unknown_client_does_not_resolve_to_codex(self):
|
||||
for name in ("gemini", "antigravity", "grok", "some_new_llm", "", None):
|
||||
with self.subTest(client=name):
|
||||
self.assertNotEqual(
|
||||
mcp_client_reconnect.normalize_client(name),
|
||||
"codex",
|
||||
"an unidentified client must never be handed Codex UI steps",
|
||||
)
|
||||
|
||||
def test_known_clients_still_get_their_own_steps(self):
|
||||
self.assertEqual(mcp_client_reconnect.normalize_client("codex"), "codex")
|
||||
self.assertEqual(
|
||||
mcp_client_reconnect.normalize_client("claude_code"), "claude_code"
|
||||
)
|
||||
|
||||
def test_generic_steps_do_not_name_a_specific_vendor(self):
|
||||
steps = " ".join(mcp_client_reconnect.operator_ui_steps("gemini"))
|
||||
self.assertNotIn("Codex", steps)
|
||||
|
||||
def test_reconnect_client_is_derived_from_the_attachment_record(self):
|
||||
registry = _registry()
|
||||
worker = _attach(registry, client="antigravity", session="s", generation="g")
|
||||
verdict = mwi.assess_provenance(
|
||||
registry=registry, worker_identity=worker["identity"], env={}, now=NOW
|
||||
)
|
||||
self.assertEqual(mwi.reconnect_client_for(verdict), "antigravity")
|
||||
|
||||
def test_reconnect_client_is_unknown_rather_than_guessed(self):
|
||||
self.assertEqual(mwi.reconnect_client_for({}), mwi.UNKNOWN_CLIENT)
|
||||
|
||||
|
||||
class RemoteBindingTests(unittest.TestCase):
|
||||
def test_explicit_prgs_selection_is_honoured(self):
|
||||
resolved = mwi.resolve_bound_remote(
|
||||
requested_remote="prgs", bound_remote="prgs", default_remote="dadeschools"
|
||||
)
|
||||
self.assertEqual(resolved["remote"], "prgs")
|
||||
self.assertFalse(resolved["drifted"])
|
||||
|
||||
def test_omitted_remote_uses_the_binding_not_the_library_default(self):
|
||||
"""The reported dadeschools host drift."""
|
||||
resolved = mwi.resolve_bound_remote(
|
||||
requested_remote=None, bound_remote="prgs", default_remote="dadeschools"
|
||||
)
|
||||
self.assertEqual(resolved["remote"], "prgs")
|
||||
self.assertNotEqual(resolved["remote"], "dadeschools")
|
||||
self.assertEqual(resolved["resolved_from"], "session_binding")
|
||||
|
||||
def test_contradicting_the_binding_is_refused(self):
|
||||
resolved = mwi.resolve_bound_remote(
|
||||
requested_remote="dadeschools",
|
||||
bound_remote="prgs",
|
||||
default_remote="dadeschools",
|
||||
)
|
||||
self.assertEqual(resolved["remote"], "prgs")
|
||||
self.assertTrue(resolved["drifted"])
|
||||
self.assertFalse(resolved["honoured_request"])
|
||||
|
||||
def test_unbound_session_falls_back_and_says_so(self):
|
||||
resolved = mwi.resolve_bound_remote(
|
||||
requested_remote=None, bound_remote=None, default_remote="dadeschools"
|
||||
)
|
||||
self.assertEqual(resolved["remote"], "dadeschools")
|
||||
self.assertEqual(resolved["resolved_from"], "library_default")
|
||||
self.assertTrue(resolved["reasons"])
|
||||
|
||||
|
||||
class SurfaceAgreementTests(unittest.TestCase):
|
||||
"""The reproduced contradiction: two surfaces, one process, two answers."""
|
||||
|
||||
def test_namespace_health_and_direct_assessment_agree(self):
|
||||
registry = _registry()
|
||||
worker = _attach(
|
||||
registry,
|
||||
client="gemini",
|
||||
session="sess-agree",
|
||||
generation="gen-agree",
|
||||
profile="prgs-reviewer",
|
||||
)
|
||||
env = {"GITEA_MCP_PROFILE": "prgs-reviewer", "GITEA_CLIENT_MANAGED": "1"}
|
||||
|
||||
direct = mwi.assess_provenance(
|
||||
registry=registry,
|
||||
worker_identity=worker["identity"],
|
||||
env=env,
|
||||
profile="prgs-reviewer",
|
||||
)
|
||||
health = mcp_namespace_health.classify_namespace_probe(
|
||||
"gitea-reviewer",
|
||||
configured=True,
|
||||
registered_tools=["gitea_whoami"],
|
||||
probe_result={"success": True},
|
||||
probe_source="client_namespace",
|
||||
process={"pid": 4242, "profile": "prgs-reviewer", "env": env},
|
||||
registry=registry,
|
||||
worker_identity=worker["identity"],
|
||||
)
|
||||
|
||||
self.assertEqual(health["provenance"], direct["provenance"])
|
||||
self.assertEqual(health["is_client_managed"], direct["is_client_managed"])
|
||||
self.assertEqual(health["worker_identity"], direct["worker_identity"])
|
||||
self.assertEqual(health["session_id"], "sess-agree")
|
||||
self.assertEqual(health["client_name"], "gemini")
|
||||
|
||||
def test_namespace_health_can_report_client_managed_at_all(self):
|
||||
"""The old derivation was structurally incapable of this."""
|
||||
env = {"GITEA_CLIENT_MANAGED": "1", "GITEA_MCP_PROFILE": "prgs-author"}
|
||||
health = mcp_namespace_health.classify_namespace_probe(
|
||||
"gitea-author",
|
||||
configured=True,
|
||||
registered_tools=["gitea_whoami"],
|
||||
probe_result={"success": True},
|
||||
probe_source="client_namespace",
|
||||
process={"pid": 1234, "profile": "prgs-author", "env": env},
|
||||
)
|
||||
self.assertTrue(
|
||||
health["is_client_managed"],
|
||||
"a client-managed launch must be reportable as client-managed",
|
||||
)
|
||||
|
||||
def test_namespace_health_without_attachment_fails_closed(self):
|
||||
health = mcp_namespace_health.classify_namespace_probe(
|
||||
"gitea-author",
|
||||
configured=True,
|
||||
registered_tools=["gitea_whoami"],
|
||||
probe_result={"success": True},
|
||||
probe_source="client_namespace",
|
||||
process={"pid": 1234, "profile": "prgs-author", "env": {}},
|
||||
)
|
||||
self.assertTrue(health["provenance_fail_closed"])
|
||||
self.assertEqual(health["provenance"], mwi.PROVENANCE_UNPROVEN)
|
||||
self.assertIsNone(health["session_id"])
|
||||
|
||||
def test_no_false_reconnect_loop_for_an_owned_session(self):
|
||||
"""A proven owner must not be told to reconnect."""
|
||||
registry = _registry()
|
||||
worker = _attach(registry, client="claude", session="s", generation="g")
|
||||
verdict = mwi.assess_provenance(
|
||||
registry=registry, worker_identity=worker["identity"], env={}, now=NOW
|
||||
)
|
||||
self.assertFalse(verdict["fail_closed"])
|
||||
self.assertEqual(verdict["blocker_kind"], mwi.BLOCKER_NONE)
|
||||
self.assertEqual(verdict["reasons"], [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,480 @@
|
||||
"""Tests for dead-owner / PID-reuse session retirement (#969)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import control_plane_db as cpd
|
||||
import post_restart_reconcile as prr
|
||||
import session_lifecycle as sl
|
||||
|
||||
NOW = datetime(2026, 7, 29, 12, 0, 0, tzinfo=timezone.utc)
|
||||
EARLIER = NOW - timedelta(hours=2)
|
||||
LATER = NOW + timedelta(minutes=5)
|
||||
|
||||
|
||||
def _session(
|
||||
session_id: str,
|
||||
*,
|
||||
pid: int | None = 4242,
|
||||
status: str = "active",
|
||||
started_at: datetime = EARLIER,
|
||||
last_heartbeat_at: datetime | None = None,
|
||||
client_managed: bool = False,
|
||||
owner_process_started_at: datetime | None = None,
|
||||
role: str = "author",
|
||||
) -> dict:
|
||||
hb = last_heartbeat_at if last_heartbeat_at is not None else started_at
|
||||
row = {
|
||||
"session_id": session_id,
|
||||
"role": role,
|
||||
"profile": "prgs-author",
|
||||
"pid": pid,
|
||||
"status": status,
|
||||
"started_at": cpd._ts(started_at),
|
||||
"last_heartbeat_at": cpd._ts(hb),
|
||||
"client_managed": client_managed,
|
||||
}
|
||||
if owner_process_started_at is not None:
|
||||
row["owner_process_started_at"] = cpd._ts(owner_process_started_at)
|
||||
return row
|
||||
|
||||
|
||||
def _alive(pids: set[int]):
|
||||
def _check(pid):
|
||||
try:
|
||||
return int(pid) in pids
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
return _check
|
||||
|
||||
|
||||
def _starts(mapping: dict[int, datetime]):
|
||||
def _probe(pid):
|
||||
try:
|
||||
return mapping.get(int(pid))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
return _probe
|
||||
|
||||
|
||||
class ClassifyDeadOwnerTests(unittest.TestCase):
|
||||
def test_dead_owner_is_stale_and_retireable(self) -> None:
|
||||
c = sl.classify_session(
|
||||
_session("ghost", pid=2_000_000_000),
|
||||
now=NOW,
|
||||
pid_checker=_alive(set()),
|
||||
process_start_probe=_starts({}),
|
||||
)
|
||||
self.assertEqual(c.classification, sl.CLASS_STALE)
|
||||
self.assertTrue(c.retireable)
|
||||
self.assertIn(c.reason, {sl.REASON_DEAD_OWNER, sl.REASON_HEARTBEAT_STALE_DEAD})
|
||||
|
||||
def test_missing_pid_is_stale(self) -> None:
|
||||
c = sl.classify_session(
|
||||
_session("no-pid", pid=None),
|
||||
now=NOW,
|
||||
pid_checker=_alive(set()),
|
||||
)
|
||||
self.assertEqual(c.classification, sl.CLASS_STALE)
|
||||
self.assertEqual(c.reason, sl.REASON_MISSING_PID)
|
||||
self.assertTrue(c.retireable)
|
||||
|
||||
|
||||
class PidReuseTests(unittest.TestCase):
|
||||
def test_pid_reuse_marks_stale_not_live(self) -> None:
|
||||
# Process with same PID started AFTER the session was recorded.
|
||||
c = sl.classify_session(
|
||||
_session("reused", pid=77, started_at=EARLIER, last_heartbeat_at=EARLIER),
|
||||
now=NOW,
|
||||
pid_checker=_alive({77}),
|
||||
process_start_probe=_starts({77: LATER}),
|
||||
)
|
||||
self.assertEqual(c.classification, sl.CLASS_STALE)
|
||||
self.assertEqual(c.reason, sl.REASON_PID_REUSE)
|
||||
self.assertTrue(c.pid_reused)
|
||||
self.assertTrue(c.retireable)
|
||||
|
||||
def test_matching_process_start_is_live(self) -> None:
|
||||
c = sl.classify_session(
|
||||
_session(
|
||||
"same-proc",
|
||||
pid=88,
|
||||
started_at=EARLIER,
|
||||
last_heartbeat_at=NOW - timedelta(seconds=30),
|
||||
owner_process_started_at=EARLIER - timedelta(seconds=5),
|
||||
),
|
||||
now=NOW,
|
||||
pid_checker=_alive({88}),
|
||||
process_start_probe=_starts({88: EARLIER - timedelta(seconds=5)}),
|
||||
)
|
||||
self.assertEqual(c.classification, sl.CLASS_LIVE)
|
||||
self.assertFalse(c.retireable)
|
||||
|
||||
|
||||
class LiveOwnerAndLeaseTests(unittest.TestCase):
|
||||
def test_live_owner_not_retired(self) -> None:
|
||||
c = sl.classify_session(
|
||||
_session(
|
||||
"live",
|
||||
pid=os.getpid(),
|
||||
last_heartbeat_at=NOW - timedelta(seconds=10),
|
||||
),
|
||||
now=NOW,
|
||||
pid_checker=_alive({os.getpid()}),
|
||||
process_start_probe=_starts({os.getpid(): EARLIER}),
|
||||
)
|
||||
self.assertEqual(c.classification, sl.CLASS_LIVE)
|
||||
self.assertFalse(c.retireable)
|
||||
|
||||
def test_live_lease_blocks_retirement_even_if_pid_dead(self) -> None:
|
||||
c = sl.classify_session(
|
||||
_session("leased", pid=99999),
|
||||
now=NOW,
|
||||
pid_checker=_alive(set()),
|
||||
live_lease_sessions={"leased"},
|
||||
)
|
||||
self.assertEqual(c.classification, sl.CLASS_PROTECTED)
|
||||
self.assertEqual(c.reason, sl.REASON_LIVE_LEASE)
|
||||
self.assertFalse(c.retireable)
|
||||
|
||||
def test_client_managed_live_never_retired(self) -> None:
|
||||
c = sl.classify_session(
|
||||
_session(
|
||||
"client",
|
||||
pid=55,
|
||||
client_managed=True,
|
||||
last_heartbeat_at=NOW - timedelta(seconds=5),
|
||||
),
|
||||
now=NOW,
|
||||
pid_checker=_alive({55}),
|
||||
process_start_probe=_starts({55: EARLIER}),
|
||||
)
|
||||
self.assertEqual(c.classification, sl.CLASS_LIVE)
|
||||
self.assertEqual(c.reason, sl.REASON_CLIENT_MANAGED_LIVE)
|
||||
self.assertFalse(c.retireable)
|
||||
|
||||
def test_client_managed_dead_pid_is_retireable(self) -> None:
|
||||
# Dead client process is not a live client-managed session.
|
||||
c = sl.classify_session(
|
||||
_session("client-dead", pid=56, client_managed=True),
|
||||
now=NOW,
|
||||
pid_checker=_alive(set()),
|
||||
)
|
||||
self.assertEqual(c.classification, sl.CLASS_STALE)
|
||||
self.assertTrue(c.retireable)
|
||||
|
||||
|
||||
class TerminalAndDisconnectedTests(unittest.TestCase):
|
||||
def test_already_terminal_not_retireable(self) -> None:
|
||||
c = sl.classify_session(
|
||||
_session("done", status="retired", pid=1),
|
||||
now=NOW,
|
||||
pid_checker=_alive(set()),
|
||||
)
|
||||
self.assertEqual(c.classification, sl.CLASS_TERMINAL)
|
||||
self.assertFalse(c.retireable)
|
||||
|
||||
def test_alive_stale_heartbeat_is_disconnected_not_retired(self) -> None:
|
||||
c = sl.classify_session(
|
||||
_session(
|
||||
"quiet",
|
||||
pid=66,
|
||||
last_heartbeat_at=NOW - timedelta(hours=5),
|
||||
),
|
||||
now=NOW,
|
||||
pid_checker=_alive({66}),
|
||||
process_start_probe=_starts({66: EARLIER}),
|
||||
)
|
||||
self.assertEqual(c.classification, sl.CLASS_DISCONNECTED)
|
||||
self.assertFalse(c.retireable)
|
||||
|
||||
|
||||
class FleetAndApplyTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self._tmp.name, "cp.sqlite3")
|
||||
self.db = cpd.ControlPlaneDB(self.db_path)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def test_mixed_fleet_and_apply_retires_only_stale(self) -> None:
|
||||
live_pid = os.getpid()
|
||||
# Use wall-clock "now" so upsert timestamps align with classification.
|
||||
moment = datetime.now(timezone.utc)
|
||||
proc_start = moment - timedelta(hours=1)
|
||||
self.db.upsert_session(
|
||||
session_id="s-live",
|
||||
role="author",
|
||||
pid=live_pid,
|
||||
status="active",
|
||||
owner_process_started_at=cpd._ts(proc_start),
|
||||
)
|
||||
self.db.upsert_session(
|
||||
session_id="s-dead", role="reviewer", pid=2_000_000_001, status="active"
|
||||
)
|
||||
self.db.upsert_session(
|
||||
session_id="s-ended", role="merger", pid=3, status="ended"
|
||||
)
|
||||
|
||||
sessions = self.db.list_sessions(limit=50)
|
||||
report = sl.classify_sessions(
|
||||
sessions,
|
||||
leases=[],
|
||||
now=moment,
|
||||
pid_checker=_alive({live_pid}),
|
||||
process_start_probe=_starts({live_pid: proc_start}),
|
||||
)
|
||||
self.assertGreaterEqual(report.stale_count, 1)
|
||||
self.assertTrue(
|
||||
any(c.session_id == "s-dead" and c.retireable for c in report.classifications)
|
||||
)
|
||||
self.assertTrue(
|
||||
any(
|
||||
c.session_id == "s-live" and not c.retireable
|
||||
for c in report.classifications
|
||||
)
|
||||
)
|
||||
|
||||
first = sl.apply_session_retirements(
|
||||
self.db, report, dry_run=False, actor_session_id="actor-1", now=moment
|
||||
)
|
||||
self.assertGreaterEqual(first["retired_count"], 1)
|
||||
# After retirement, active list should exclude s-dead.
|
||||
active = {
|
||||
s["session_id"]
|
||||
for s in self.db.list_sessions(statuses=("active",), limit=50)
|
||||
}
|
||||
self.assertNotIn("s-dead", active)
|
||||
self.assertIn("s-live", active)
|
||||
|
||||
# Repeated cleanup is idempotent.
|
||||
report2 = sl.classify_sessions(
|
||||
self.db.list_sessions(limit=50),
|
||||
leases=[],
|
||||
now=moment,
|
||||
pid_checker=_alive({live_pid}),
|
||||
process_start_probe=_starts({live_pid: proc_start}),
|
||||
)
|
||||
second = sl.apply_session_retirements(
|
||||
self.db, report2, dry_run=False, actor_session_id="actor-1", now=moment
|
||||
)
|
||||
# No double-retirement of the same row as a new mutation.
|
||||
self.assertEqual(second["retired_count"], 0)
|
||||
|
||||
# Durable audit event present.
|
||||
import sqlite3
|
||||
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
events = conn.execute(
|
||||
"SELECT event_type, message FROM events WHERE event_type = ?",
|
||||
("session_retired",),
|
||||
).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertTrue(events)
|
||||
self.assertTrue(any("s-dead" in (m or "") for _, m in events))
|
||||
|
||||
def test_live_lease_blocks_db_retirement(self) -> None:
|
||||
self.db.upsert_session(
|
||||
session_id="s-leased", role="author", pid=2_000_000_002, status="active"
|
||||
)
|
||||
self.db.upsert_work_item(
|
||||
remote="prgs",
|
||||
org="org",
|
||||
repo="repo",
|
||||
kind="issue",
|
||||
number=969,
|
||||
)
|
||||
result = self.db.assign_and_lease(
|
||||
session_id="s-leased",
|
||||
role="author",
|
||||
remote="prgs",
|
||||
org="org",
|
||||
repo="repo",
|
||||
kind="issue",
|
||||
number=969,
|
||||
)
|
||||
self.assertEqual(result.outcome, "assigned")
|
||||
|
||||
# Inventory-style lease with explicit live freshness (authoritative for
|
||||
# the pure classifier). DB apply also blocks on the active lease row.
|
||||
leases = [
|
||||
{
|
||||
"lease_id": result.lease_id,
|
||||
"session_id": "s-leased",
|
||||
"status": "active",
|
||||
"freshness": {"freshness": "active"},
|
||||
}
|
||||
]
|
||||
report = sl.classify_sessions(
|
||||
self.db.list_sessions(statuses=("active",), limit=20),
|
||||
leases=leases,
|
||||
now=NOW,
|
||||
pid_checker=_alive(set()),
|
||||
)
|
||||
# Classifier protects via live lease set.
|
||||
self.assertTrue(
|
||||
any(
|
||||
c.session_id == "s-leased" and c.classification == sl.CLASS_PROTECTED
|
||||
for c in report.classifications
|
||||
)
|
||||
)
|
||||
apply = sl.apply_session_retirements(
|
||||
self.db, report, dry_run=False, actor_session_id="actor", now=NOW
|
||||
)
|
||||
self.assertEqual(apply["retired_count"], 0)
|
||||
active = {
|
||||
s["session_id"]
|
||||
for s in self.db.list_sessions(statuses=("active",), limit=20)
|
||||
}
|
||||
self.assertIn("s-leased", active)
|
||||
|
||||
# Direct DB CAS also refuses while an active lease row remains.
|
||||
blocked = self.db.retire_session(
|
||||
session_id="s-leased",
|
||||
reason=sl.REASON_DEAD_OWNER,
|
||||
actor_session_id="actor",
|
||||
now=NOW,
|
||||
)
|
||||
self.assertEqual(blocked["outcome"], "blocked")
|
||||
self.assertEqual(blocked["reason"], "live_lease")
|
||||
|
||||
def test_concurrent_retirement_is_idempotent(self) -> None:
|
||||
for i in range(20):
|
||||
self.db.upsert_session(
|
||||
session_id=f"ghost-{i}",
|
||||
role="author",
|
||||
pid=3_000_000 + i,
|
||||
status="active",
|
||||
)
|
||||
|
||||
def _worker() -> dict:
|
||||
return sl.retire_stale_sessions(
|
||||
self.db,
|
||||
dry_run=False,
|
||||
actor_session_id=f"actor-{threading.get_ident()}",
|
||||
now=NOW,
|
||||
pid_checker=_alive(set()),
|
||||
process_start_probe=_starts({}),
|
||||
session_limit=100,
|
||||
)
|
||||
|
||||
outcomes = []
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
futs = [pool.submit(_worker) for _ in range(4)]
|
||||
for fut in as_completed(futs):
|
||||
outcomes.append(fut.result())
|
||||
|
||||
total_retired = sum(o["apply"]["retired_count"] for o in outcomes)
|
||||
# Exactly one successful retirement per ghost row across all workers.
|
||||
self.assertEqual(total_retired, 20)
|
||||
active = {
|
||||
s["session_id"]
|
||||
for s in self.db.list_sessions(statuses=("active",), limit=100)
|
||||
}
|
||||
for i in range(20):
|
||||
self.assertNotIn(f"ghost-{i}", active)
|
||||
|
||||
|
||||
class ReconcileIntegrationTests(unittest.TestCase):
|
||||
def test_unresolved_until_retired_then_resolved(self) -> None:
|
||||
inv = {
|
||||
"inventory_complete": True,
|
||||
"incomplete_reasons": [],
|
||||
"service_health": {"healthy": True},
|
||||
"clients": [{"session_id": "c1", "connected": True}],
|
||||
"sessions": [
|
||||
_session("ghost", pid=2_000_000_099, last_heartbeat_at=EARLIER),
|
||||
],
|
||||
"leases": [],
|
||||
"checkpoints_available": False,
|
||||
"worktree_bindings": [],
|
||||
"pending_mutations": [],
|
||||
"capabilities": {"stale": False},
|
||||
"boot_head_sha": "a" * 40,
|
||||
"current_head_sha": "a" * 40,
|
||||
"queue_state": {"safe_to_resume": True},
|
||||
}
|
||||
proof = prr.reconcile_after_restart(inv, now=NOW, mode=prr.MODE_LOG_ONLY)
|
||||
sess = next(i for i in proof.items if i.dimension == prr.DIM_SESSIONS)
|
||||
self.assertEqual(sess.status, prr.ITEM_UNRESOLVED)
|
||||
self.assertIn("ghost", sess.details.get("orphan_session_ids") or [])
|
||||
|
||||
# After retirement inventory (no active orphans) resolves.
|
||||
inv2 = dict(inv)
|
||||
inv2["sessions"] = []
|
||||
inv2["session_fleet"] = {
|
||||
"retireable_session_ids": [],
|
||||
"sessions_dimension_resolved": True,
|
||||
"live_count": 0,
|
||||
"stale_count": 0,
|
||||
}
|
||||
proof2 = prr.reconcile_after_restart(inv2, now=NOW, mode=prr.MODE_LOG_ONLY)
|
||||
sess2 = next(i for i in proof2.items if i.dimension == prr.DIM_SESSIONS)
|
||||
self.assertEqual(sess2.status, prr.ITEM_RESOLVED)
|
||||
|
||||
def test_legacy_orphan_key_still_populated(self) -> None:
|
||||
inv = {
|
||||
"inventory_complete": True,
|
||||
"service_health": {"healthy": True},
|
||||
"clients": [],
|
||||
"sessions": [_session("ghost", pid=2_000_000_100)],
|
||||
"leases": [],
|
||||
"checkpoints_available": False,
|
||||
"worktree_bindings": [],
|
||||
"pending_mutations": [],
|
||||
"capabilities": {"stale": False},
|
||||
"boot_head_sha": "a" * 40,
|
||||
"current_head_sha": "a" * 40,
|
||||
"queue_state": {"safe_to_resume": True},
|
||||
}
|
||||
proof = prr.reconcile_after_restart(inv, now=NOW)
|
||||
sess = next(i for i in proof.items if i.dimension == prr.DIM_SESSIONS)
|
||||
self.assertIn("orphan_session_ids", sess.details)
|
||||
|
||||
|
||||
class SchemaMigrationTests(unittest.TestCase):
|
||||
def test_lifecycle_columns_present(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "cp.sqlite3")
|
||||
db = cpd.ControlPlaneDB(path)
|
||||
db.upsert_session(session_id="s1", role="author", pid=1)
|
||||
out = db.retire_session(
|
||||
session_id="s1",
|
||||
reason=sl.REASON_DEAD_OWNER,
|
||||
actor_session_id="tester",
|
||||
now=NOW,
|
||||
)
|
||||
self.assertEqual(out["outcome"], "retired")
|
||||
rows = db.list_sessions(limit=5)
|
||||
# May not appear under active filter
|
||||
all_rows = db.list_sessions(limit=5)
|
||||
# Re-open raw to check columns
|
||||
import sqlite3
|
||||
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
cols = {r[1] for r in conn.execute("PRAGMA table_info(sessions)")}
|
||||
version = conn.execute(
|
||||
"SELECT value FROM schema_meta WHERE key='schema_version'"
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertIn("retired_at", cols)
|
||||
self.assertIn("retire_reason", cols)
|
||||
self.assertIn("owner_process_started_at", cols)
|
||||
self.assertEqual(version, "6")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user