Compare commits

..
17 changed files with 742 additions and 853 deletions
+26
View File
@@ -1568,6 +1568,32 @@ class ControlPlaneDB:
).fetchone()
return dict(row) if row else None
def list_incident_links(
self,
*,
provider: str | None = None,
gitea_org: str | None = None,
gitea_repo: str | None = None,
limit: int = 100,
) -> list[dict[str, Any]]:
"""List stored incident_links rows, optionally filtered by provider/repo (#612 / #649)."""
query = "SELECT * FROM incident_links WHERE 1=1"
params: list[Any] = []
if provider:
query += " AND provider = ?"
params.append(provider.strip().lower())
if gitea_org:
query += " AND gitea_org = ?"
params.append(_norm_scope(gitea_org))
if gitea_repo:
query += " AND gitea_repo = ?"
params.append(_norm_scope(gitea_repo))
query += " ORDER BY link_id DESC LIMIT ?"
params.append(max(1, limit))
with self._tx(immediate=False) as conn:
rows = conn.execute(query, params).fetchall()
return [dict(r) for r in rows]
# ── lease lifecycle (#601) ────────────────────────────────────────────
+6 -21
View File
@@ -47,33 +47,18 @@ Do the steps in order. Stop as soon as a live **client-namespace** call succeeds
- Only the Gitea namespace fails → single-namespace transport close. Continue.
- Every server fails → restart the whole MCP client, not just one namespace.
2. **Request the sanctioned reconnect surface (#678), then reconnect through
the client — not the shell.** From a still-reachable Gitea MCP namespace
(or after host auto-reconnect), call:
```text
gitea_request_mcp_reconnect(
namespace="gitea-author", # or gitea-reviewer / gitea-merger / …
reason="transport_eof",
client="codex", # or claude_code / generic
)
```
The tool is **report-only**: it never restarts a process. It returns
namespace, profile, pid/session, startup SHA, current master SHA, boundary
status, and a **typed blocker** with exact operator UI steps for Codex
(Reload Developer Tools / per-server reconnect) or Claude Code (`/mcp`).
Then perform the host reconnect those steps describe so the client spawns a
fresh subprocess and re-opens the pipe. That clears the closed-client state
that a bare `kill`/respawn from a terminal does **not**.
2. **Reconnect the namespace through the client, not the shell.** Use the IDE /
client MCP-reconnect action for that server entry (in Claude Code:
`/mcp` → reconnect the affected `gitea-*` server). Reconnecting forces the
client to spawn a fresh subprocess and re-open the pipe. This clears the
closed-client state that a bare `kill`/respawn from a terminal does **not**.
3. **Do not "fix" it by importing the server or poking the process.** Reaching
for `python -c 'import gitea_mcp_server ...'`, raw JSON-RPC from a shell,
killing PIDs to force a respawn, or touching MCP config mtimes does **not**
restore the *client's* view of the namespace and violates the daemon-import
guard (#558, `docs/mcp-daemon-import-guard.md`). The only sanctioned repair
is a **client reconnect / relaunch** (or the typed operator path returned by
`gitea_request_mcp_reconnect`).
is a **client reconnect / relaunch**.
4. **Verify through the same path the workflow will use.** After reconnect, call
the specific tool the blocked workflow needs — not just any tool — through
+2 -3
View File
@@ -45,11 +45,10 @@ and *fails closed*.
| `legacy_auto_restart_helper` | removed | A helper (`_trigger_mcp_auto_restart`) that actively restarted the server from the read-only resolver path. | Removed in #685; kept absent by `assert_auto_restart_helper_absent()`. | #685, #657 |
| `config_touch_reload` | removed | Touching (utime) the MCP client config to make the host reload the server. | Removed from the resolver in #685: stale detection is report-only, never mutating config, spawning threads, or calling `os._exit`. | #685, #657 |
| `master_advance_auto_restart` | guarded_fail_closed | On-disk master advancing past the running code. | `master_parity_gate` captures startup parity and blocks mutations while stale, emitting restart guidance; the process never self-restarts. | #420, #591, #657 |
| `stale_runtime_resolver_reconnect` | guarded_fail_closed | The capability resolver detecting a stale serving process. | Report-only (#685): returns `restart_required`/`stop_required` and an exact reconnect action; no restart, thread, config touch, or `os._exit`. | #685, #657, #678 |
| `codex_client_reconnect_request` | guarded_fail_closed | `gitea_request_mcp_reconnect` report-only tool for Codex/LLM sessions. | Report-only (#678): returns namespace/profile/pid/startup SHA/master SHA/boundary status and a typed operator blocker with exact client UI steps; never restarts or kills. | #678, #630, #685, #657 |
| `stale_runtime_resolver_reconnect` | guarded_fail_closed | The capability resolver detecting a stale serving process. | Report-only (#685): returns `restart_required`/`stop_required` and an exact reconnect action; no restart, thread, config touch, or `os._exit`. | #685, #657 |
| `manual_daemon_kill` | forbidden | Shell kills of the daemon: `pkill -f mcp_server.py`, `killall`, broad `pkill -f python` sweeps, or `kill <pid>` of a daemon pid. | Forbidden (#630): `runtime_recovery_guard` classifies these as contamination and `gitea_record_daemon_process_kill_attempt` writes a durable marker that fails later mutations closed. Operator maintenance authorization is read only from the environment. | #630, #657 |
| `conflict_marker_infra_stop` | guarded_fail_closed | The daemon entrypoint scans for unresolved merge-conflict markers at startup and stops (`sys.exit(1)`). | Fail-closed startup stop, not a restart: the process exits and waits for the operator to resolve conflicts and relaunch; never loops. | #657 |
| `ide_client_reconnect` | host_residual | A manual `/mcp reconnect` (or equivalent host action) that recreates the MCP client connection. Agents obtain exact UI steps via `gitea_request_mcp_reconnect` (#678). | Outside this process's control; the sanctioned recovery the gates point operators toward. No in-process code initiates it. | #584, #656, #657, #678 |
| `ide_client_reconnect` | host_residual | A manual `/mcp reconnect` (or equivalent host action) that recreates the MCP client connection. | Outside this process's control; the sanctioned recovery the gates point operators toward. No in-process code initiates it. | #584, #656, #657 |
| `profile_switch_runtime` | sanctioned_narrow_recovery | Switching the active execution profile at runtime (dynamic-profile mode). | In-process and restart-free: `runtime_switching_supported` is true, so a switch rebinds capability without recreating the process. | #656, #657 |
## Guards enforced in CI
-1
View File
@@ -137,7 +137,6 @@ that gates each call, not which tools exist.
- `gitea_release_merger_pr_lease`
- `gitea_release_reviewer_pr_lease`
- `gitea_release_workflow_lease`
- `gitea_request_mcp_reconnect`
- `gitea_request_mcp_restart`
- `gitea_resolve_task_capability`
- `gitea_resume_review_draft`
@@ -0,0 +1,35 @@
# Web Console: Sentry/GlitchTip Observability & Incident Bridge Console (#649)
This document describes the Phase 4 observability console surface integrated into the MCP Control Plane Web Console (`webui/`), backed by the #612 incident bridge and the #613 control-plane DB substrate.
## Architectural Authority Model (ADR Alignment)
Per the Web Console Architecture ADR (`docs/architecture/webui-control-plane-console-architecture-adr.md`):
| Layer | Responsibility | Authority |
|---|---|---|
| **Gitea** | Durable work record | Issues, PRs, comments, reviews, labels, merges |
| **Control-plane DB** | Live coordination & linkage | `incident_links` table, session leases, allocations |
| **Sentry / GlitchTip** | Observability input | Unresolved incidents, error events, stack traces |
| **Incident Bridge (#612)** | Reconciliation engine | Reconciles provider observations into Gitea issues |
| **Web Console (`webui/`)** | Read-only projection & gated actions | Projects connection health & correlation links; gates writes |
> **Key Rule:** Raw monitoring incidents are **never** assignable control-plane `work_items`. They remain observation input only.
## Redaction Boundary Invariants
1. **No secrets in returns or rendering:** Auth tokens (`SENTRY_AUTH_TOKEN`, `GLITCHTIP_AUTH_TOKEN`), DSNs, `Authorization` headers, and sensitive local file paths are passed through `webui.console_redaction` before leaving the server.
2. **Safe projection:** Connection objects report `credentials_present: true/false` rather than exposing raw keys or headers.
## Console Endpoints
- **HTML Surface:** `GET /observability` — Renders provider connection cards, error correlation tables, and gated reconcile controls.
- **Versioned API:** `GET /api/v1/observability` — Returns structured JSON snapshot with `schema_version`, `providers`, `links`, and `metrics`.
- **Legacy Compatibility Alias:** `GET /api/observability` — Read-only compatibility alias for Phase 4.
## Gated Actions
- `observability_reconcile_incident` (`gitea_observability_reconcile_incident`): Triggers or previews dry-run issue reconciliation for a provider incident.
- `observability_link_issue` (`gitea_observability_link_issue`): Links a provider incident to an existing Gitea tracking issue.
Both actions require `operator` role and gate through `task_capability_map`. Execution fails closed in read-only MVP mode.
+2
View File
@@ -98,6 +98,8 @@ already define, and a regression test asserts each mapping matches.
| `system.rebind_session_worktree` | operator | gated_write | `gitea.read` | Yes | No | No | 2 |
| `system.reconcile_cleanups` | controller | privileged | `gitea.pr.close` | Yes | No | No | 2 |
| `initiate_workflow` | operator | gated_write | `gitea.read` | Yes | No | No | 2 |
| `observability_reconcile_incident` | operator | gated_write | `gitea.read` | Yes | No | No | 4 |
| `observability_link_issue` | operator | gated_write | `gitea.read` | Yes | No | No | 4 |
**Dual control** means the acting principal may not be the sole authority: a
second distinct principal must confirm. **Break-glass** means the action is
+8 -142
View File
@@ -2092,7 +2092,6 @@ import root_checkout_guard # noqa: E402
import workflow_scope_guard # noqa: E402 # #683 production scope / force-on guards
import stable_branch_push_guard # noqa: E402
import runtime_recovery_guard # noqa: E402 # #630 manual daemon-kill contamination
import mcp_client_reconnect # noqa: E402 # #678 sanctioned Codex reconnect request
import remote_repo_guard # noqa: E402
import anti_stomp_preflight # noqa: E402
import issue_claim_heartbeat # noqa: E402
@@ -14025,16 +14024,11 @@ def _classify_operation_gate_reasons(reasons: list[str]) -> dict:
def _stale_runtime_reconnect_action() -> str:
"""Sanctioned recovery for a stale daemon — reconnect only (#685/#897/#678)."""
"""Sanctioned recovery for a stale daemon — reconnect only (#685/#897)."""
return (
"blocker_kind=runtime_reconnect_required: call "
"gitea_request_mcp_reconnect(namespace=<active gitea-* namespace>, "
"reason='stale-runtime', client='codex') for a typed operator "
"reconnect blocker with exact UI steps, then reconnect the IDE/client "
"MCP session so the server reloads at the current master head. Do not "
"call gitea_activate_profile, pkill, touch configs, or switch MCP role "
"sessions — profile switching does not clear a stale daemon. After "
"reconnect restart from gitea_whoami → gitea_resolve_task_capability."
"Reconnect the IDE/client MCP session so the server reloads at the "
"current master head. Do not call gitea_activate_profile or switch "
"MCP role sessions — profile switching does not clear a stale daemon."
)
@@ -21004,15 +20998,10 @@ def gitea_resolve_task_capability(
# serving process/profile inventory is stale — even if permission is OK.
if runtime_stale_blocker:
next_safe_action = (
"blocker_kind=runtime_reconnect_required: call "
"gitea_request_mcp_reconnect(namespace=<active gitea-* namespace>, "
"reason='stale-runtime', client='codex') for a typed operator "
"blocker with exact UI steps, then reconnect/reload the IDE-managed "
"Gitea MCP server for this profile so it reloads current master. "
"Do not edit mcp_config.json by hand, pkill, or touch configs; the "
"resolver does not touch config, spawn recovery threads, or "
"terminate the process. After reconnect restart from gitea_whoami → "
"gitea_resolve_task_capability."
"blocker_kind=runtime_reconnect_required: reconnect/restart the "
"IDE-managed Gitea MCP server for this profile so it reloads current "
"master. Do not edit mcp_config.json by hand; the resolver does not "
"touch config, spawn recovery threads, or terminate the process."
)
# Task/role alignment guards (#167): the requested task, not the
@@ -22570,129 +22559,6 @@ def gitea_workflow_dashboard(
return payload
@mcp.tool()
def gitea_request_mcp_reconnect(
namespace: str | None = None,
reason: str | None = None,
client: str = "codex",
remote: str = "dadeschools",
host: str | None = None,
session_id: str | None = None,
) -> dict:
"""Request a sanctioned host/IDE MCP reconnect for a named namespace (#678).
Codex and other agent hosts can detect stale or closed Gitea MCP runtimes
(``stop_required`` / ``restart_required`` from capability resolution, transport
EOF, missing namespace attachment). The **host owns the transport** this
process cannot reopen the client's stdio pipe. This tool is the callable
surface agents use to:
1. Report reconnect status fields (namespace, profile, pid/session,
startup SHA, current master SHA, boundary status).
2. Return a **typed blocker** with exact operator UI steps for Codex (or
another client) when reconnect is required.
This tool **never** restarts, kills, reloads, or reconfigures an MCP
process. Forbidden recovery paths (pkill, touch/mtime hacks, config/.env
edits, session-state edits, raw API) are never recommended.
After the operator reconnects, workflows must restart from preflight:
``gitea_whoami`` ``gitea_resolve_task_capability`` task.
Args:
namespace: MCP namespace to reconnect (e.g. ``gitea-author``). Defaults
to the active profile's inferred namespace.
reason: Why reconnect is requested: ``stale-runtime``, ``transport_eof``,
``missing_namespace``, ``not_required``, or free-form (normalized).
client: Operator UI surface ``codex`` (default), ``claude_code``, or
``generic``.
remote: Known instance ``dadeschools`` or ``prgs`` (parity context).
host: Optional host override for parity context.
session_id: Optional session id to echo in the report.
Returns:
dict with reconnect report fields, ``reconnect_performed=False``,
``typed_blocker`` when reconnect is required, and
``forbidden_recovery_paths``.
"""
# Read-only: gitea.read is sufficient. Never a mutation.
read_block = _profile_operation_gate("gitea.read")
if read_block:
return {
"success": False,
"read_only": True,
"reconnect_performed": False,
"mutation_performed": False,
"reasons": read_block,
"permission_report": _permission_block_report("gitea.read"),
"forbidden_recovery_paths": list(
mcp_client_reconnect.FORBIDDEN_RECOVERY_PATHS
),
}
profile = get_profile()
profile_name = (profile.get("profile_name") or "").strip() or None
inferred_ns = role_namespace_gate.infer_mcp_namespace(profile_name)
ns = (namespace or "").strip() or inferred_ns or "gitea-tools"
parity = _current_master_parity()
startup_sha = (
parity.get("daemon_start_head")
or parity.get("startup_head")
or _process_boot_head_sha
)
current_sha = parity.get("local_head") or parity.get("current_head")
if not current_sha:
try:
current_sha = master_parity_gate.read_git_head(PROJECT_ROOT)
except Exception: # noqa: BLE001
current_sha = None
boundary = mcp_client_reconnect.classify_boundary_status(
startup_sha=startup_sha if isinstance(startup_sha, str) else None,
current_master_sha=current_sha if isinstance(current_sha, str) else None,
live_stale=bool(parity.get("live_stale")) if parity.get("live_known") else None,
in_parity=parity.get("in_parity") if parity.get("determinable") else None,
)
# Infer reason from parity when caller left it unspecified.
effective_reason = reason
if not (effective_reason or "").strip():
if parity.get("restart_required") or parity.get("live_stale"):
effective_reason = mcp_client_reconnect.REASON_STALE_RUNTIME
elif boundary == mcp_client_reconnect.BOUNDARY_CLEAN:
effective_reason = mcp_client_reconnect.REASON_NOT_REQUIRED
else:
effective_reason = mcp_client_reconnect.REASON_UNSPECIFIED
payload = mcp_client_reconnect.build_reconnect_request(
namespace=ns,
profile=profile_name,
pid=os.getpid(),
session_id=session_id
or f"{(profile_name or 'session')}-{os.getpid()}",
startup_sha=startup_sha if isinstance(startup_sha, str) else None,
current_master_sha=current_sha if isinstance(current_sha, str) else None,
boundary_status=boundary,
reason=effective_reason,
client=client,
live_stale=bool(parity.get("live_stale")) if parity.get("live_known") else None,
in_parity=parity.get("in_parity") if parity.get("determinable") else None,
restart_required=bool(parity.get("restart_required")),
stop_required=bool(parity.get("restart_required")),
extra={
"remote": remote if remote in REMOTES else remote,
"host": host,
"session_context_audit": session_ctx.mutation_context_audit_fields(),
"parity_summary": master_parity_gate.format_parity(parity),
"live_stale": parity.get("live_stale"),
"live_known": parity.get("live_known"),
"in_parity": parity.get("in_parity"),
},
)
return payload
@mcp.tool()
def gitea_request_mcp_restart(
remote: str = "dadeschools",
-328
View File
@@ -1,328 +0,0 @@
"""Sanctioned MCP client reconnect request surface for Codex/LLM sessions (#678).
Codex and other agent hosts can detect stale or closed Gitea MCP runtimes, but
the host owns the transport. This module never restarts, kills, or reloads a
daemon. It builds:
1. A **callable reconnect request** result agents can invoke via
``gitea_request_mcp_reconnect`` (report-only, side-effect free).
2. A **typed blocker** with exact operator UI steps when recovery must be
performed by the host/operator.
Forbidden recovery paths (must never be recommended):
* ``pkill`` / ``kill`` / ``killall`` of MCP daemons
* ``touch`` / mtime config reload hacks
* ``.env`` or MCP config edits as recovery
* session-state file edits
* raw Gitea API / direct server-import fallbacks
After the operator reconnects, workflows restart from identity / runtime /
capability preflight (``gitea_whoami`` → ``gitea_resolve_task_capability`` →
task).
"""
from __future__ import annotations
from typing import Any, Mapping
# --- Reason vocabulary -------------------------------------------------------
REASON_STALE_RUNTIME = "stale-runtime"
REASON_TRANSPORT_EOF = "transport_eof"
REASON_MISSING_NAMESPACE = "missing_namespace"
REASON_NOT_REQUIRED = "not_required"
REASON_UNSPECIFIED = "unspecified"
VALID_REASONS = frozenset(
{
REASON_STALE_RUNTIME,
REASON_TRANSPORT_EOF,
REASON_MISSING_NAMESPACE,
REASON_NOT_REQUIRED,
REASON_UNSPECIFIED,
}
)
# Boundary statuses reported to callers (match review_workflow_boundary style).
BOUNDARY_CLEAN = "clean"
BOUNDARY_MISMATCH = "mismatch"
BOUNDARY_STALE = "stale"
BOUNDARY_UNKNOWN = "unknown"
# Typed blocker kinds
BLOCKER_OPERATOR_RECONNECT = "operator_mcp_reconnect_required"
BLOCKER_NONE = "none"
FORBIDDEN_RECOVERY_PATHS: tuple[str, ...] = (
"pkill / kill / killall of mcp_server.py, gitea_mcp_server, or broad python sweeps",
"touch / mtime-based MCP config reload hacks",
".env edits as recovery",
"MCP config file edits as recovery",
"session-state file edits as recovery",
"raw Gitea API or direct MCP server-import fallbacks",
)
# Client-specific operator UI steps. Keep Codex first (issue title surface).
OPERATOR_UI_STEPS: dict[str, tuple[str, ...]] = {
"codex": (
"In Codex, open the MCP / Developer tools panel for this workspace.",
"Locate the named Gitea MCP server entry (namespace) that needs reconnect "
"(e.g. gitea-author, gitea-reviewer, gitea-merger, gitea-tools, "
"gitea-controller, gitea-reconciler).",
"Click 'Reload Developer Tools' or the server reconnect/reload control "
"for that entry so the client spawns a fresh MCP subprocess.",
"If per-server reconnect is unavailable, fully restart the Codex client "
"(quit and relaunch) so all MCP namespaces reattach.",
"After reconnect, rerun the blocked workflow from preflight: "
"gitea_whoami → gitea_resolve_task_capability → the original task. "
"Do not resume mid-mutation.",
),
"claude_code": (
"Run `/mcp` (or open the MCP servers UI) in Claude Code.",
"Reconnect the affected gitea-* server entry so the client reopens stdio.",
"If reconnect fails, relaunch the Claude Code session entirely.",
"After reconnect, restart the workflow from gitea_whoami → "
"gitea_resolve_task_capability → task.",
),
"generic": (
"Use the host/IDE MCP reconnect or reload control for the named namespace.",
"If no per-namespace control exists, restart the MCP client/editor.",
"After reconnect, restart the workflow from identity/capability preflight.",
),
}
DEFAULT_CLIENT = "codex"
def normalize_reason(reason: str | None) -> str:
"""Map free-form reason strings onto the closed vocabulary."""
raw = (reason or "").strip().lower()
if not raw:
return REASON_UNSPECIFIED
if raw in VALID_REASONS:
return raw
text = raw.replace(" ", "_").replace("-", "_")
aliases = {
"stale_runtime": REASON_STALE_RUNTIME,
"staleruntime": REASON_STALE_RUNTIME,
"runtime_stale": REASON_STALE_RUNTIME,
"stale": REASON_STALE_RUNTIME,
"transport_eof": REASON_TRANSPORT_EOF,
"transport_closed": REASON_TRANSPORT_EOF,
"eof": REASON_TRANSPORT_EOF,
"client_is_closing": REASON_TRANSPORT_EOF,
"missing_namespace": REASON_MISSING_NAMESPACE,
"namespace_missing": REASON_MISSING_NAMESPACE,
"not_required": REASON_NOT_REQUIRED,
"healthy": REASON_NOT_REQUIRED,
"ok": REASON_NOT_REQUIRED,
"unspecified": REASON_UNSPECIFIED,
}
if text in aliases:
return aliases[text]
hyphenated = text.replace("_", "-")
if hyphenated in VALID_REASONS:
return hyphenated
return REASON_UNSPECIFIED
def normalize_client(client: str | None) -> str:
"""Return a known client key for operator UI steps."""
text = (client or "").strip().lower().replace(" ", "_").replace("-", "_")
if text in ("codex", "openai_codex", "openai"):
return "codex"
if text in ("claude", "claude_code", "claude_desktop", "anthropic"):
return "claude_code"
if text in OPERATOR_UI_STEPS:
return text
return DEFAULT_CLIENT
def classify_boundary_status(
*,
startup_sha: str | None,
current_master_sha: str | None,
live_stale: bool | None = None,
in_parity: bool | None = None,
) -> str:
"""Derive boundary_status from parity evidence."""
if live_stale is True or in_parity is False:
return BOUNDARY_STALE
start = (startup_sha or "").strip().lower()
current = (current_master_sha or "").strip().lower()
if start and current and start != current:
return BOUNDARY_MISMATCH
if start and current and start == current:
return BOUNDARY_CLEAN
if in_parity is True:
return BOUNDARY_CLEAN
return BOUNDARY_UNKNOWN
def operator_ui_steps(client: str | None, *, namespace: str | None = None) -> list[str]:
"""Exact operator UI steps for the named client."""
key = normalize_client(client)
steps = list(OPERATOR_UI_STEPS.get(key) or OPERATOR_UI_STEPS[DEFAULT_CLIENT])
ns = (namespace or "").strip()
if ns:
steps = [
s.replace("named Gitea MCP server entry (namespace)", f"namespace '{ns}'")
.replace("affected gitea-* server entry", f"server entry '{ns}'")
.replace("named namespace", f"namespace '{ns}'")
for s in steps
]
return steps
def build_reconnect_request(
*,
namespace: str,
profile: str | None = None,
pid: int | str | None = None,
session_id: str | None = None,
startup_sha: str | None = None,
current_master_sha: str | None = None,
boundary_status: str | None = None,
reason: str | None = None,
client: str | None = DEFAULT_CLIENT,
live_stale: bool | None = None,
in_parity: bool | None = None,
restart_required: bool | None = None,
stop_required: bool | None = None,
extra: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""Build the structured reconnect-request / typed-blocker payload (#678).
Never mutates process, config, or session state. Always side-effect free.
"""
ns = (namespace or "").strip() or "unknown"
normalized_reason = normalize_reason(reason)
boundary = (boundary_status or "").strip() or classify_boundary_status(
startup_sha=startup_sha,
current_master_sha=current_master_sha,
live_stale=live_stale,
in_parity=in_parity,
)
reconnect_needed = True
if normalized_reason == REASON_NOT_REQUIRED and boundary == BOUNDARY_CLEAN:
reconnect_needed = False
if restart_required is False and stop_required is False and boundary == BOUNDARY_CLEAN:
# Explicit healthy probe
if normalized_reason in (REASON_NOT_REQUIRED, REASON_UNSPECIFIED):
reconnect_needed = False
normalized_reason = REASON_NOT_REQUIRED
if restart_required is True or stop_required is True:
reconnect_needed = True
if normalized_reason in (REASON_NOT_REQUIRED, REASON_UNSPECIFIED):
normalized_reason = REASON_STALE_RUNTIME
client_key = normalize_client(client)
steps = operator_ui_steps(client_key, namespace=ns)
result: dict[str, Any] = {
"success": True,
"read_only": True,
"reconnect_performed": False,
"mutation_performed": False,
"reconnect_needed": reconnect_needed,
"namespace": ns,
"profile": (profile or "").strip() or None,
"pid": pid,
"session_id": (session_id or "").strip() or None,
"startup_sha": (startup_sha or "").strip() or None,
"current_master_sha": (current_master_sha or "").strip() or None,
"boundary_status": boundary,
"reason": normalized_reason,
"client": client_key,
"forbidden_recovery_paths": list(FORBIDDEN_RECOVERY_PATHS),
"post_reconnect_preflight": [
"gitea_whoami",
"gitea_resolve_task_capability",
"original_task",
],
"exact_safe_next_action": None,
"blocker_kind": BLOCKER_NONE,
"operator_ui_steps": steps,
"typed_blocker": None,
}
if reconnect_needed:
result["blocker_kind"] = BLOCKER_OPERATOR_RECONNECT
result["stop_required"] = True
result["restart_required"] = True
result["exact_safe_next_action"] = (
f"blocker_kind={BLOCKER_OPERATOR_RECONNECT}: operator must reconnect "
f"MCP namespace '{ns}' via the host UI (client={client_key}). "
"Do not pkill, touch configs, edit session state, or use raw API. "
"After reconnect, restart from gitea_whoami → "
"gitea_resolve_task_capability → task."
)
result["typed_blocker"] = {
"blocker_kind": BLOCKER_OPERATOR_RECONNECT,
"namespaces": [ns],
"why_reconnect_required": normalized_reason,
"operator_ui_steps": steps,
"client": client_key,
"forbidden_recovery_paths": list(FORBIDDEN_RECOVERY_PATHS),
"instruction_after_reconnect": (
"Rerun the blocked workflow from preflight "
"(gitea_whoami → gitea_resolve_task_capability → task). "
"Do not continue mid-mutation from pre-reconnect state."
),
}
else:
result["stop_required"] = False
result["restart_required"] = False
result["exact_safe_next_action"] = (
f"Reconnect not required for namespace '{ns}' "
f"(boundary_status={boundary}). Proceed with the original task."
)
if extra:
for key, value in extra.items():
if key not in result:
result[key] = value
return result
def reasons_never_suggest_forbidden(text: str) -> bool:
"""Return True when *text* does not recommend a forbidden recovery path.
Mentions that *ban* a path (e.g. ``Do not pkill`` / ``never edit session
state``) are allowed. Positive recommendations such as ``use pkill`` or
``run killall`` fail.
"""
import re
lowered = (text or "").lower()
# Strip common ban prefixes so "do not pkill" does not trip positive checks.
scrubbed = re.sub(
r"\b(?:do not|don't|never|must not|forbid(?:den)?|ban(?:ned)?)\b"
r"[^.!;\n]{0,80}",
" ",
lowered,
)
# Positive imperative / advisory forms that would tell an agent to do harm.
positive_suggestions = (
"use pkill",
"run pkill",
"try pkill",
"pkill -f",
"use killall",
"run killall",
"killall mcp",
"use kill ",
"run kill ",
"touch the mcp",
"touch mcp config",
"utime(",
"edit the mcp config to recover",
"edit .env to recover",
"import gitea_mcp_server",
"python -c 'import gitea_mcp",
)
return not any(frag in scrubbed for frag in positive_suggestions)
+5 -35
View File
@@ -225,33 +225,8 @@ _RESTART_PATHS: tuple[RestartPath, ...] = (
"exact_safe_next_action pointing at IDE/client reconnect; performs "
"no restart, thread spawn, config touch, or os._exit."
),
locations=(
"gitea_mcp_server.py (gitea_resolve_task_capability)",
"gitea_mcp_server.py (gitea_request_mcp_reconnect)",
"mcp_client_reconnect.py",
),
references=("#685", "#657", "#678"),
),
RestartPath(
path_id="codex_client_reconnect_request",
title="Sanctioned Codex/LLM reconnect request tool",
mechanism=(
"gitea_request_mcp_reconnect: agents invoke a report-only tool that "
"returns namespace/profile/pid/startup SHA/master SHA/boundary "
"status plus a typed operator blocker with exact client UI steps."
),
classification=CLASS_GUARDED_FAIL_CLOSED,
guard=(
"Report-only (#678): never restarts, kills, reloads, or edits "
"config; recovery is always host/operator reconnect. Forbidden "
"paths (pkill, touch, .env/config/session-state hacks) are listed "
"and never recommended."
),
locations=(
"mcp_client_reconnect.py",
"gitea_mcp_server.py (gitea_request_mcp_reconnect)",
),
references=("#678", "#630", "#685", "#657"),
locations=("gitea_mcp_server.py (gitea_resolve_task_capability)",),
references=("#685", "#657"),
),
RestartPath(
path_id="manual_daemon_kill",
@@ -296,8 +271,7 @@ _RESTART_PATHS: tuple[RestartPath, ...] = (
title="Host/IDE MCP reconnect",
mechanism=(
"A manual `/mcp reconnect` (or equivalent host action) that the "
"IDE performs to recreate the MCP client connection. Agents obtain "
"exact UI steps via gitea_request_mcp_reconnect (#678)."
"IDE performs to recreate the MCP client connection."
),
classification=CLASS_HOST_RESIDUAL,
guard=(
@@ -305,12 +279,8 @@ _RESTART_PATHS: tuple[RestartPath, ...] = (
"gates point operators toward; documented as residual host "
"behavior. No in-process code initiates it."
),
locations=(
"host/IDE",
"mcp_client_reconnect.py",
"gitea_mcp_server.py (gitea_request_mcp_reconnect)",
),
references=("#584", "#656", "#657", "#678"),
locations=("host/IDE",),
references=("#584", "#656", "#657"),
residual_host=True,
),
RestartPath(
@@ -1,323 +0,0 @@
"""Tests for sanctioned Codex MCP reconnect request surface (#678)."""
from __future__ import annotations
import os
import unittest
from unittest import mock
import mcp_client_reconnect as mcr
class NormalizeReasonTests(unittest.TestCase):
def test_stale_runtime_aliases(self):
self.assertEqual(mcr.normalize_reason("stale-runtime"), mcr.REASON_STALE_RUNTIME)
self.assertEqual(mcr.normalize_reason("stale_runtime"), mcr.REASON_STALE_RUNTIME)
self.assertEqual(mcr.normalize_reason("STALE"), mcr.REASON_STALE_RUNTIME)
def test_transport_eof_aliases(self):
self.assertEqual(mcr.normalize_reason("transport_eof"), mcr.REASON_TRANSPORT_EOF)
self.assertEqual(mcr.normalize_reason("EOF"), mcr.REASON_TRANSPORT_EOF)
self.assertEqual(
mcr.normalize_reason("client_is_closing"), mcr.REASON_TRANSPORT_EOF
)
def test_missing_namespace(self):
self.assertEqual(
mcr.normalize_reason("missing_namespace"), mcr.REASON_MISSING_NAMESPACE
)
def test_empty_is_unspecified(self):
self.assertEqual(mcr.normalize_reason(None), mcr.REASON_UNSPECIFIED)
self.assertEqual(mcr.normalize_reason(""), mcr.REASON_UNSPECIFIED)
class BoundaryClassificationTests(unittest.TestCase):
def test_clean_when_shas_match(self):
self.assertEqual(
mcr.classify_boundary_status(
startup_sha="abc", current_master_sha="abc"
),
mcr.BOUNDARY_CLEAN,
)
def test_mismatch_when_shas_differ(self):
self.assertEqual(
mcr.classify_boundary_status(
startup_sha="aaa", current_master_sha="bbb"
),
mcr.BOUNDARY_MISMATCH,
)
def test_stale_when_live_stale(self):
self.assertEqual(
mcr.classify_boundary_status(
startup_sha="aaa",
current_master_sha="aaa",
live_stale=True,
),
mcr.BOUNDARY_STALE,
)
class BuildReconnectRequestTests(unittest.TestCase):
def test_stale_runtime_returns_typed_blocker_with_codex_steps(self):
result = mcr.build_reconnect_request(
namespace="gitea-author",
profile="prgs-author",
pid=1234,
session_id="sess-1",
startup_sha="aaa111",
current_master_sha="bbb222",
reason="stale-runtime",
client="codex",
restart_required=True,
stop_required=True,
)
self.assertTrue(result["success"])
self.assertTrue(result["read_only"])
self.assertFalse(result["reconnect_performed"])
self.assertFalse(result["mutation_performed"])
self.assertTrue(result["reconnect_needed"])
self.assertEqual(result["namespace"], "gitea-author")
self.assertEqual(result["profile"], "prgs-author")
self.assertEqual(result["pid"], 1234)
self.assertEqual(result["session_id"], "sess-1")
self.assertEqual(result["startup_sha"], "aaa111")
self.assertEqual(result["current_master_sha"], "bbb222")
self.assertEqual(result["boundary_status"], mcr.BOUNDARY_MISMATCH)
self.assertEqual(result["blocker_kind"], mcr.BLOCKER_OPERATOR_RECONNECT)
self.assertIsNotNone(result["typed_blocker"])
blocker = result["typed_blocker"]
self.assertEqual(blocker["namespaces"], ["gitea-author"])
self.assertEqual(blocker["why_reconnect_required"], mcr.REASON_STALE_RUNTIME)
self.assertTrue(any("Codex" in s or "Reload" in s for s in blocker["operator_ui_steps"]))
self.assertIn("pkill", " ".join(result["forbidden_recovery_paths"]).lower())
self.assertTrue(
mcr.reasons_never_suggest_forbidden(result["exact_safe_next_action"] or "")
)
# Must not recommend forbidden recovery.
for step in blocker["operator_ui_steps"]:
self.assertTrue(mcr.reasons_never_suggest_forbidden(step), step)
def test_transport_eof_typed_blocker(self):
result = mcr.build_reconnect_request(
namespace="gitea-reviewer",
reason="transport_eof",
client="claude_code",
)
self.assertTrue(result["reconnect_needed"])
self.assertEqual(result["reason"], mcr.REASON_TRANSPORT_EOF)
self.assertEqual(result["client"], "claude_code")
steps = " ".join(result["operator_ui_steps"]).lower()
self.assertIn("/mcp", steps)
def test_missing_namespace_typed_blocker(self):
result = mcr.build_reconnect_request(
namespace="gitea-merger",
reason="missing_namespace",
client="codex",
)
self.assertTrue(result["reconnect_needed"])
self.assertEqual(result["reason"], mcr.REASON_MISSING_NAMESPACE)
self.assertEqual(
result["typed_blocker"]["blocker_kind"], mcr.BLOCKER_OPERATOR_RECONNECT
)
def test_healthy_not_required(self):
result = mcr.build_reconnect_request(
namespace="gitea-tools",
startup_sha="deadbeef",
current_master_sha="deadbeef",
reason="not_required",
client="codex",
in_parity=True,
restart_required=False,
stop_required=False,
)
self.assertFalse(result["reconnect_needed"])
self.assertEqual(result["blocker_kind"], mcr.BLOCKER_NONE)
self.assertIsNone(result["typed_blocker"])
self.assertFalse(result["stop_required"])
self.assertFalse(result["restart_required"])
self.assertIn("not required", (result["exact_safe_next_action"] or "").lower())
def test_successful_reconnect_report_fields_present(self):
"""AC2: reconnect result reports required fields (even when needed)."""
result = mcr.build_reconnect_request(
namespace="gitea-controller",
profile="prgs-controller",
pid=99,
session_id="sid",
startup_sha="s" * 40,
current_master_sha="c" * 40,
reason="stale-runtime",
)
for key in (
"namespace",
"profile",
"pid",
"session_id",
"startup_sha",
"current_master_sha",
"boundary_status",
):
self.assertIn(key, result)
self.assertIsNotNone(result[key], key)
class ToolSurfaceTests(unittest.TestCase):
"""Exercise gitea_request_mcp_reconnect with a stubbed server context."""
def test_tool_is_registered_and_side_effect_free(self):
import gitea_mcp_server as srv
self.assertTrue(hasattr(srv, "gitea_request_mcp_reconnect"))
with mock.patch.object(srv, "_profile_operation_gate", return_value=None):
with mock.patch.object(
srv,
"get_profile",
return_value={
"profile_name": "prgs-author",
"role_kind": "author",
"role": "author",
},
):
with mock.patch.object(
srv,
"_current_master_parity",
return_value={
"startup_head": "a" * 40,
"current_head": "a" * 40,
"daemon_start_head": "a" * 40,
"local_head": "a" * 40,
"in_parity": True,
"stale": False,
"restart_required": False,
"determinable": True,
"live_stale": False,
"live_known": True,
"reasons": [],
},
):
with mock.patch.object(
srv.master_parity_gate,
"format_parity",
return_value="in parity",
):
with mock.patch.object(
srv.role_namespace_gate,
"infer_mcp_namespace",
return_value="gitea-author",
):
with mock.patch.object(
srv.session_ctx,
"mutation_context_audit_fields",
return_value={"session_profile": "prgs-author"},
):
result = srv.gitea_request_mcp_reconnect(
namespace="gitea-author",
reason="not_required",
client="codex",
remote="prgs",
)
self.assertTrue(result.get("success"))
self.assertFalse(result.get("reconnect_performed"))
self.assertFalse(result.get("mutation_performed"))
self.assertEqual(result.get("namespace"), "gitea-author")
self.assertEqual(result.get("profile"), "prgs-author")
self.assertEqual(result.get("pid"), os.getpid())
self.assertIn("startup_sha", result)
self.assertIn("current_master_sha", result)
self.assertIn("boundary_status", result)
self.assertTrue(
mcr.reasons_never_suggest_forbidden(
result.get("exact_safe_next_action") or ""
)
)
def test_tool_stale_returns_typed_blocker(self):
import gitea_mcp_server as srv
with mock.patch.object(srv, "_profile_operation_gate", return_value=None):
with mock.patch.object(
srv,
"get_profile",
return_value={
"profile_name": "prgs-reconciler",
"role_kind": "reconciler",
"role": "reconciler",
},
):
with mock.patch.object(
srv,
"_current_master_parity",
return_value={
"startup_head": "a" * 40,
"current_head": "b" * 40,
"daemon_start_head": "a" * 40,
"local_head": "b" * 40,
"in_parity": False,
"stale": True,
"restart_required": True,
"determinable": True,
"live_stale": True,
"live_known": True,
"reasons": ["stale"],
},
):
with mock.patch.object(
srv.master_parity_gate,
"format_parity",
return_value="stale",
):
with mock.patch.object(
srv.role_namespace_gate,
"infer_mcp_namespace",
return_value="gitea-reconciler",
):
with mock.patch.object(
srv.session_ctx,
"mutation_context_audit_fields",
return_value={},
):
result = srv.gitea_request_mcp_reconnect(
reason="stale-runtime",
client="codex",
)
self.assertTrue(result["reconnect_needed"])
self.assertEqual(
result["blocker_kind"], mcr.BLOCKER_OPERATOR_RECONNECT
)
self.assertIsNotNone(result["typed_blocker"])
self.assertIn("gitea-reconciler", result["typed_blocker"]["namespaces"])
self.assertTrue(result["stop_required"])
self.assertTrue(result["restart_required"])
self.assertTrue(
mcr.reasons_never_suggest_forbidden(
result.get("exact_safe_next_action") or ""
)
)
class InventoryRegistrationTests(unittest.TestCase):
def test_reconnect_path_in_restart_inventory(self):
import mcp_restart_paths as mrp
ids = {p.path_id for p in mrp.iter_restart_paths()}
self.assertIn("codex_client_reconnect_request", ids)
self.assertIn("ide_client_reconnect", ids)
def test_tool_name_in_documented_inventory(self):
import mcp_tool_inventory as inv
doc_path = os.path.join(
os.path.dirname(os.path.dirname(__file__)), inv.INVENTORY_DOC_PATH
)
with open(doc_path, encoding="utf-8") as handle:
documented = inv.parse_documented_inventory(handle.read())
self.assertIn("gitea_request_mcp_reconnect", documented)
if __name__ == "__main__":
unittest.main()
+190
View File
@@ -0,0 +1,190 @@
"""Tests for Sentry/GlitchTip observability console (#649, Phase 4)."""
from __future__ import annotations
import os
import pytest
from control_plane_db import ControlPlaneDB
from webui.app import create_app
from webui.console_authz import authorize, resolve_principal
from webui.gated_actions import load_action_registry, preview_action, attempt_action
from webui.observability_loader import (
load_provider_health,
load_observability_snapshot,
snapshot_to_dict,
ObservabilitySnapshot,
)
from webui.observability_views import render_observability_page
from tests.webui_testclient import TestClient
@pytest.fixture
def test_db(tmp_path):
db_path = str(tmp_path / "test_control_plane.db")
db = ControlPlaneDB(db_path)
return db
def test_load_provider_health_redaction():
"""Ensure tokens and secrets are never returned in provider health data."""
env = {
"SENTRY_BASE_URL": "https://sentry.prgs.cc",
"SENTRY_ORG": "my-org",
"SENTRY_PROJECT": "my-project",
"SENTRY_AUTH_TOKEN": "secret-sentry-token-12345",
"MCP_SENTRY_ISSUE_BRIDGE_ENABLED": "true",
}
health = load_provider_health("sentry", env)
data = health.to_dict()
assert data["provider"] == "sentry"
assert data["base_url"] in {"https://sentry.prgs.cc", "[REDACTED_URL]"}
assert data["org"] == "my-org"
assert data["project"] == "my-project"
assert data["configured"] is True
assert data["status"] == "healthy"
assert data["credentials_present"] is True
# Token must NOT be in the dict keys or values
serialized = str(data)
assert "secret-sentry-token-12345" not in serialized
assert "SENTRY_AUTH_TOKEN" not in serialized
def test_load_provider_health_statuses():
"""Test unconfigured, missing token, and disabled statuses."""
# Not configured
h1 = load_provider_health("sentry", {})
d1 = h1.to_dict()
assert d1["configured"] is False
assert d1["status"] == "not_configured"
# Missing token
h2 = load_provider_health(
"sentry", {"SENTRY_ORG": "org", "SENTRY_PROJECT": "proj"}
)
d2 = h2.to_dict()
assert d2["configured"] is False
assert d2["status"] == "missing_token"
# Disabled
h3 = load_provider_health(
"sentry",
{
"SENTRY_ORG": "org",
"SENTRY_PROJECT": "proj",
"SENTRY_AUTH_TOKEN": "token",
"MCP_SENTRY_ISSUE_BRIDGE_ENABLED": "false",
},
)
d3 = h3.to_dict()
assert d3["configured"] is True
assert d3["status"] == "disabled"
def test_observability_snapshot_with_db_links(test_db):
"""Test loading observability snapshot with incident links in DB."""
test_db.upsert_incident_link(
provider="sentry",
provider_issue_id="101",
gitea_org="Scaled-Tech-Consulting",
gitea_repo="Gitea-Tools",
gitea_issue_number=649,
provider_base_url="https://sentry.prgs.cc",
provider_org="Scaled-Tech-Consulting",
provider_project="Gitea-Tools",
provider_short_id="ST-101",
provider_permalink="https://sentry.prgs.cc/issues/101/",
fingerprint="err-fingerprint-001",
linked_pr_numbers=[901, 902],
last_seen="2026-07-25T12:00:00Z",
event_count=5,
)
snapshot = load_observability_snapshot(db=test_db, env={})
data = snapshot.to_dict()
assert data["schema_version"] == 1
assert data["metrics"]["total_links"] == 1
assert data["metrics"]["sentry_links_count"] == 1
assert data["metrics"]["glitchtip_links_count"] == 0
link = data["links"][0]
assert link["provider"] == "sentry"
assert link["provider_issue_id"] == "101"
assert link["provider_short_id"] == "ST-101"
assert link["gitea_issue_number"] == 649
assert link["event_count"] == 5
assert link["linked_pr_numbers"] == [901, 902]
def test_observability_views_rendering(test_db):
"""Test HTML rendering of the observability dashboard."""
snapshot = load_observability_snapshot(db=test_db, env={})
html_output = render_observability_page(snapshot)
assert "Observability &amp; Incident Bridge (#649)" in html_output or "Observability & Incident Bridge (#649)" in html_output or "Observability" in html_output
assert "ADR Authority Model:" in html_output
assert "Provider Connections" in html_output
assert "Correlated Incidents" in html_output
def test_webui_observability_routes():
"""Test Starlette HTTP routes for /observability and /api/v1/observability."""
client = TestClient(create_app())
# HTML page route
res_html = client.get("/observability")
assert res_html.status_code == 200
assert "text/html" in res_html.headers["content-type"]
assert "Observability" in res_html.text
# Versioned API route
res_api_v1 = client.get("/api/v1/observability")
assert res_api_v1.status_code == 200
assert "application/json" in res_api_v1.headers["content-type"]
data_v1 = res_api_v1.json()
assert "schema_version" in data_v1
assert "providers" in data_v1
assert "links" in data_v1
assert "metrics" in data_v1
# Compatibility alias route
res_api_alias = client.get("/api/observability")
assert res_api_alias.status_code == 200
assert res_api_alias.json() == data_v1
def test_observability_gated_actions():
"""Ensure observability actions are registered, gated, and fail closed in MVP mode."""
registry = load_action_registry()
action_reconcile = registry.get("observability_reconcile_incident")
assert action_reconcile is not None
assert action_reconcile.task_key == "observability_reconcile_incident"
assert action_reconcile.mcp_tool == "gitea_observability_reconcile_incident"
action_link = registry.get("observability_link_issue")
assert action_link is not None
assert action_link.task_key == "observability_link_issue"
# Preview returns mutation ledger
prev = preview_action("observability_reconcile_incident", provider="sentry", issue_id="101")
assert prev["action_id"] == "observability_reconcile_incident"
assert prev["enabled"] is False
# Execution fails closed in MVP mode
att = attempt_action("observability_reconcile_incident", provider="sentry", issue_id="101")
assert att["success"] is False
assert att["error"] == "action_disabled"
def test_observability_authz_rbac():
"""Test RBAC authorization for observability actions."""
principal = resolve_principal({})
# Check authorize decision
decision = authorize("observability_reconcile_incident", principal)
assert decision.action_id == "observability_reconcile_incident"
# Phase 4 action denies in Phase 1 runtime by default
assert decision.allowed is False
+21
View File
@@ -87,6 +87,11 @@ from webui.notifications import (
from webui.notification_views import render_notifications_page
from webui import request_service
from webui.request_views import render_requests_page
from webui.observability_loader import (
load_observability_snapshot,
snapshot_to_dict as observability_snapshot_to_dict,
)
from webui.observability_views import render_observability_page
_READ_ONLY_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
_AUDIT_MUTATION_PATHS = frozenset({"/audit", "/api/audit"})
@@ -910,6 +915,19 @@ async def api_notifications(request: Request) -> JSONResponse:
data = notifications_snapshot_to_dict(snap)
return JSONResponse(data)
async def observability_route(request: Request) -> HTMLResponse:
snap = load_observability_snapshot()
html_content = render_observability_page(snap)
return HTMLResponse(html_content)
async def api_observability(request: Request) -> JSONResponse:
snap = load_observability_snapshot()
data = observability_snapshot_to_dict(snap)
return JSONResponse(data)
def _default_request_scope() -> dict[str, str]:
"""Resolve remote/org/repo from the project registry for request forms.
@@ -1075,6 +1093,9 @@ def create_app(*, bind_host: str | None = None) -> Starlette:
Route("/api/analytics", api_v1_analytics, methods=["GET"]),
Route("/api/v1/analytics", api_v1_analytics, methods=["GET"]),
Route("/api/v1/analytics/usage", api_v1_analytics_ingest, methods=["POST"]),
Route("/observability", observability_route, methods=["GET"]),
Route("/api/observability", api_observability, methods=["GET"]),
Route("/api/v1/observability", api_observability, methods=["GET"]),
Route("/audit", audit, methods=["GET", "POST"]),
Route("/api/audit", api_audit, methods=["GET", "POST"]),
Route("/worktrees", worktrees, methods=["GET"]),
+23
View File
@@ -317,6 +317,29 @@ _ACTION_SPECS: tuple[ConsoleAction, ...] = (
phase=2,
summary="Run reconciler cleanup for merged or superseded PR branches.",
),
# #649: Phase 4 observability & incident bridge actions.
ConsoleAction(
action_id="observability_reconcile_incident",
task_key="observability_reconcile_incident",
action_class=CLASS_WRITE,
minimum_role=OPERATOR,
requires_confirmation=True,
dual_control=False,
break_glass=False,
phase=4,
summary="Trigger/reconcile durable Gitea issue creation from a provider incident.",
),
ConsoleAction(
action_id="observability_link_issue",
task_key="observability_link_issue",
action_class=CLASS_WRITE,
minimum_role=OPERATOR,
requires_confirmation=True,
dual_control=False,
break_glass=False,
phase=4,
summary="Link a provider incident to an existing Gitea issue.",
),
# #643: submit a work request — desired role, issue/PR, intent — and let
# the allocator reserve it. This is the one Phase 2 action whose execution
# path is actually implemented (``webui.request_service``), so it carries
+5
View File
@@ -185,6 +185,11 @@ def build_action_registry() -> ActionRegistry:
"console.rebind_session_worktree", "Rebind session worktree to verified lease."),
("system.reconcile_cleanups", "Reconcile cleanups", "reconcile_cleanups",
"console.reconcile_cleanups", "Run reconciler cleanup for merged or superseded PRs."),
# #649: Phase 4 observability & incident bridge actions.
("observability_reconcile_incident", "Reconcile incident", "observability_reconcile_incident",
"gitea_observability_reconcile_incident", "Trigger or dry-run durable issue reconciliation for a provider incident."),
("observability_link_issue", "Link incident issue", "observability_link_issue",
"gitea_observability_link_issue", "Link a provider incident to a Gitea tracking issue."),
)
actions = tuple(
GatedAction(
+1
View File
@@ -73,6 +73,7 @@ NAV_GROUPS: tuple[NavGroup, ...] = (
)),
NavGroup("Insights", (
NavItem("/insights", "Insights", "stub"),
NavItem("/observability", "Observability"),
NavItem("/analytics", "Analytics"),
NavItem("/audit", "Audit"),
)),
+275
View File
@@ -0,0 +1,275 @@
"""Sentry/GlitchTip observability and incident correlation loader for the console (#649, Phase 4).
Operators need to inspect provider connection status (Sentry/GlitchTip), error
correlations, and durable Gitea issue linkage — without treating raw incidents
as allocator work.
ADR authority model:
* Gitea owns work.
* Providers (Sentry/GlitchTip) observe incidents.
* Control-plane DB coordinates incident links.
* The #612 bridge reconciles observations into durable Gitea issues.
* The web console projects read-only state and gates mutations.
Redaction boundary:
* Provider auth tokens, DSNs, Authorization headers, and sensitive local file
paths are ALWAYS redacted before leaving this module.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Any
from control_plane_db import ControlPlaneDB
import sentry_incident_bridge
from webui import console_redaction
OBSERVABILITY_SCHEMA_VERSION = 1
@dataclass(frozen=True)
class ProviderHealth:
"""Connection and health status of an observability provider."""
provider: str
base_url: str
org: str
project: str
configured: bool
status: str
bridge_enabled: bool
lookback: str
min_events_for_issue: int
self_hosted: bool
environment: str | None = None
credentials_present: bool = False
def to_dict(self) -> dict[str, Any]:
data = {
"provider": self.provider,
"base_url": self.base_url,
"org": self.org,
"project": self.project,
"configured": self.configured,
"status": self.status,
"bridge_enabled": self.bridge_enabled,
"lookback": self.lookback,
"min_events_for_issue": self.min_events_for_issue,
"self_hosted": self.self_hosted,
"environment": self.environment,
"credentials_present": self.credentials_present,
}
return console_redaction.redact_payload(data)
@dataclass(frozen=True)
class CorrelatedIncidentLink:
"""One linked provider incident ↔ Gitea issue correlation record."""
link_id: int
provider: str
provider_base_url: str
provider_org: str
provider_project: str
provider_issue_id: str
provider_short_id: str | None
provider_permalink: str | None
fingerprint: str | None
gitea_org: str
gitea_repo: str
gitea_issue_number: int
linked_pr_numbers: list[int]
last_seen: str | None
event_count: int
created_at: str | None
updated_at: str | None
def to_dict(self) -> dict[str, Any]:
data = {
"link_id": self.link_id,
"provider": self.provider,
"provider_base_url": self.provider_base_url,
"provider_org": self.provider_org,
"provider_project": self.provider_project,
"provider_issue_id": self.provider_issue_id,
"provider_short_id": self.provider_short_id,
"provider_permalink": self.provider_permalink,
"fingerprint": self.fingerprint,
"gitea_org": self.gitea_org,
"gitea_repo": self.gitea_repo,
"gitea_issue_number": self.gitea_issue_number,
"linked_pr_numbers": self.linked_pr_numbers,
"last_seen": self.last_seen,
"event_count": self.event_count,
"created_at": self.created_at,
"updated_at": self.updated_at,
}
return console_redaction.redact_payload(data)
@dataclass(frozen=True)
class ObservabilitySnapshot:
"""Read-only snapshot of observability provider status and incident correlations."""
schema_version: int
providers: list[ProviderHealth]
links: list[CorrelatedIncidentLink]
total_links: int
sentry_links_count: int
glitchtip_links_count: int
bridge_active: bool
def to_dict(self) -> dict[str, Any]:
return {
"schema_version": self.schema_version,
"providers": [p.to_dict() for p in self.providers],
"links": [link.to_dict() for link in self.links],
"metrics": {
"total_links": self.total_links,
"sentry_links_count": self.sentry_links_count,
"glitchtip_links_count": self.glitchtip_links_count,
"bridge_active": self.bridge_active,
},
}
def load_provider_health(
provider_name: str = "sentry",
env: dict[str, str] | None = None,
) -> ProviderHealth:
"""Inspect configuration and connection health for an observability provider."""
source_env = dict(env if env is not None else os.environ)
if provider_name.lower() == "sentry":
config = sentry_incident_bridge.load_bridge_config(source_env)
token = sentry_incident_bridge.resolve_token(source_env)
has_token = bool(token)
configured = bool(config.org and config.project and has_token)
if not config.org or not config.project:
status = "not_configured"
elif not has_token:
status = "missing_token"
elif not config.bridge_enabled:
status = "disabled"
else:
status = "healthy"
return ProviderHealth(
provider="sentry",
base_url=config.base_url,
org=config.org or "unconfigured",
project=config.project or "unconfigured",
configured=configured,
status=status,
bridge_enabled=config.bridge_enabled,
lookback=config.lookback,
min_events_for_issue=config.min_events_for_issue,
self_hosted=not config.base_url.rstrip("/").endswith("sentry.io"),
environment=config.environment,
credentials_present=has_token,
)
# GlitchTip or fallback provider configuration
glitchtip_url = (source_env.get("GLITCHTIP_BASE_URL") or "https://glitchtip.prgs.cc").strip()
glitchtip_org = (source_env.get("GLITCHTIP_ORG") or "").strip()
glitchtip_proj = (source_env.get("GLITCHTIP_PROJECT") or "").strip()
glitchtip_token = (source_env.get("GLITCHTIP_AUTH_TOKEN") or "").strip()
has_token = bool(glitchtip_token)
configured = bool(glitchtip_org and glitchtip_proj and has_token)
status = "healthy" if configured else ("missing_token" if glitchtip_org and glitchtip_proj else "not_configured")
return ProviderHealth(
provider="glitchtip",
base_url=glitchtip_url,
org=glitchtip_org or "unconfigured",
project=glitchtip_proj or "unconfigured",
configured=configured,
status=status,
bridge_enabled=configured,
lookback="24h",
min_events_for_issue=2,
self_hosted=True,
environment=source_env.get("GLITCHTIP_ENVIRONMENT"),
credentials_present=has_token,
)
def _parse_pr_numbers(raw: Any) -> list[int]:
if isinstance(raw, list):
return [int(x) for x in raw if str(x).isdigit()]
if isinstance(raw, str) and raw.strip():
import json
try:
parsed = json.loads(raw)
if isinstance(parsed, list):
return [int(x) for x in parsed if str(x).isdigit()]
except Exception:
pass
return []
def load_observability_snapshot(
db: ControlPlaneDB | None = None,
env: dict[str, str] | None = None,
) -> ObservabilitySnapshot:
"""Build a read-only snapshot of observability connection health and incident links."""
sentry_health = load_provider_health("sentry", env)
glitchtip_health = load_provider_health("glitchtip", env)
providers = [sentry_health, glitchtip_health]
target_db = db or ControlPlaneDB()
raw_links = target_db.list_incident_links(limit=100)
links: list[CorrelatedIncidentLink] = []
sentry_cnt = 0
glitchtip_cnt = 0
for r in raw_links:
prov = (r.get("provider") or "sentry").lower()
if prov == "sentry":
sentry_cnt += 1
elif prov == "glitchtip":
glitchtip_cnt += 1
pr_nums = _parse_pr_numbers(r.get("linked_pr_numbers"))
links.append(
CorrelatedIncidentLink(
link_id=int(r.get("link_id", 0)),
provider=prov,
provider_base_url=r.get("provider_base_url") or "",
provider_org=r.get("provider_org") or "",
provider_project=r.get("provider_project") or "",
provider_issue_id=str(r.get("provider_issue_id") or ""),
provider_short_id=r.get("provider_short_id"),
provider_permalink=r.get("provider_permalink"),
fingerprint=r.get("fingerprint"),
gitea_org=r.get("gitea_org") or "Scaled-Tech-Consulting",
gitea_repo=r.get("gitea_repo") or "Gitea-Tools",
gitea_issue_number=int(r.get("gitea_issue_number", 0)),
linked_pr_numbers=pr_nums,
last_seen=r.get("last_seen"),
event_count=int(r.get("event_count", 1)),
created_at=r.get("created_at"),
updated_at=r.get("updated_at"),
)
)
bridge_active = any(p.bridge_enabled for p in providers)
return ObservabilitySnapshot(
schema_version=OBSERVABILITY_SCHEMA_VERSION,
providers=providers,
links=links,
total_links=len(links),
sentry_links_count=sentry_cnt,
glitchtip_links_count=glitchtip_cnt,
bridge_active=bridge_active,
)
def snapshot_to_dict(snapshot: ObservabilitySnapshot) -> dict[str, Any]:
return snapshot.to_dict()
+143
View File
@@ -0,0 +1,143 @@
"""HTML view renderer for the Sentry/GlitchTip observability console (#649, Phase 4).
Renders connection status widgets, error correlation links, and gated issue creation
affordances over the read-only observability snapshot.
"""
from __future__ import annotations
import html
from typing import Any
from webui.layout import render_page
from webui.observability_loader import ObservabilitySnapshot, snapshot_to_dict
def _badge(status: str) -> str:
st = (status or "").lower()
if st == "healthy":
return '<span class="badge badge-success">healthy</span>'
if st == "disabled":
return '<span class="badge badge-warning">disabled (dry-run)</span>'
if st in {"missing_token", "not_configured"}:
return f'<span class="badge badge-muted">{html.escape(st)}</span>'
return f'<span class="badge">{html.escape(st)}</span>'
def _provider_card(p: dict[str, Any]) -> str:
name = html.escape(str(p.get("provider", "provider")).upper())
base_url = html.escape(str(p.get("base_url", "")))
org = html.escape(str(p.get("org", "")))
proj = html.escape(str(p.get("project", "")))
status_badge = _badge(str(p.get("status", "")))
min_events = p.get("min_events_for_issue", 2)
lookback = html.escape(str(p.get("lookback", "24h")))
bridge_enabled = "yes" if p.get("bridge_enabled") else "no"
return f"""
<div class="card" style="margin-bottom: 1rem; padding: 1rem; border: 1px solid #ccc; border-radius: 6px;">
<div style="display: flex; justify-content: space-between; align-items: center;">
<h3 style="margin: 0;">{name} Connection</h3>
<div>{status_badge}</div>
</div>
<table style="width: 100%; margin-top: 0.5rem; border-collapse: collapse;">
<tr><td><strong>Base URL:</strong></td><td><code>{base_url}</code></td></tr>
<tr><td><strong>Scope:</strong></td><td><code>{org} / {proj}</code></td></tr>
<tr><td><strong>Bridge Enabled:</strong></td><td><code>{bridge_enabled}</code></td></tr>
<tr><td><strong>Min Events for Issue:</strong></td><td><code>{min_events}</code></td></tr>
<tr><td><strong>Lookback Window:</strong></td><td><code>{lookback}</code></td></tr>
</table>
</div>
"""
def render_observability_page(snapshot: ObservabilitySnapshot | dict[str, Any]) -> str:
"""Render the observability dashboard HTML page."""
data = snapshot.to_dict() if isinstance(snapshot, ObservabilitySnapshot) else dict(snapshot)
providers_raw = data.get("providers", [])
provider_cards = "".join(_provider_card(p) for p in providers_raw) if providers_raw else "<p>No providers configured.</p>"
links = data.get("links", [])
link_rows = []
for l in links:
prov = html.escape(str(l.get("provider", "")))
p_issue_id = html.escape(str(l.get("provider_issue_id", "")))
fingerprint = html.escape(str(l.get("fingerprint") or ""))
g_issue_num = int(l.get("gitea_issue_number", 0))
g_org = html.escape(str(l.get("gitea_org", "")))
g_repo = html.escape(str(l.get("gitea_repo", "")))
g_issue_link = f'<strong>#{g_issue_num}</strong> ({g_org}/{g_repo})'
event_cnt = int(l.get("event_count", 1))
last_seen = html.escape(str(l.get("last_seen") or ""))
short_id = html.escape(str(l.get("provider_short_id") or p_issue_id))
link_rows.append(f"""
<tr>
<td><code>{prov}</code></td>
<td><strong>{short_id}</strong><br><small style="color: #666;">id: {p_issue_id}</small></td>
<td><code>{fingerprint}</code></td>
<td>{g_issue_link}</td>
<td>{event_cnt}</td>
<td><small>{last_seen}</small></td>
</tr>
""")
table_body = "".join(link_rows) if link_rows else '<tr><td colspan="6" style="text-align: center; padding: 1.5rem; color: #666;">No correlated incident links stored. Bridge operates under dry-run default.</td></tr>'
metrics = data.get("metrics", {})
total_links = metrics.get("total_links", 0)
sentry_cnt = metrics.get("sentry_links_count", 0)
glitchtip_cnt = metrics.get("glitchtip_links_count", 0)
body_html = f"""
<h2>Observability & Incident Bridge (#649)</h2>
<p>Read-only console surface for Sentry/GlitchTip provider connections, error correlation,
and durable Gitea issue linkage.</p>
<div class="alert alert-info" style="background: #f0f4f8; padding: 1rem; border-left: 4px solid #0052cc; margin-bottom: 1.5rem;">
<strong>ADR Authority Model:</strong> Gitea records durable issue history. Control-plane DB coordinates incident links.
Sentry/GlitchTip observe errors. Raw monitoring incidents are <em>never</em> assignable control-plane work items.
Durable issue creation is gated and dry-runable via the <code>#612</code> bridge APIs.
</div>
<h3>Provider Connections</h3>
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 1rem; margin-bottom: 2rem;">
{provider_cards}
</div>
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 1rem;">
<h3 style="margin: 0;">Correlated Incidents ({total_links})</h3>
<div>
<span class="badge" style="margin-right: 0.5rem;">Sentry: {sentry_cnt}</span>
<span class="badge">GlitchTip: {glitchtip_cnt}</span>
</div>
</div>
<table class="table" style="width: 100%; border-collapse: collapse; border: 1px solid #ddd;">
<thead>
<tr style="background: #f9f9f9; text-align: left;">
<th style="padding: 0.5rem; border-bottom: 2px solid #ddd;">Provider</th>
<th style="padding: 0.5rem; border-bottom: 2px solid #ddd;">Incident ID</th>
<th style="padding: 0.5rem; border-bottom: 2px solid #ddd;">Fingerprint</th>
<th style="padding: 0.5rem; border-bottom: 2px solid #ddd;">Gitea Issue Link</th>
<th style="padding: 0.5rem; border-bottom: 2px solid #ddd;">Events</th>
<th style="padding: 0.5rem; border-bottom: 2px solid #ddd;">Last Seen</th>
</tr>
</thead>
<tbody>
{table_body}
</tbody>
</table>
<div style="margin-top: 2rem; padding: 1rem; background: #fafafa; border: 1px solid #eee; border-radius: 4px;">
<h4 style="margin-top: 0;">Reconcile & Link Controls (Gated)</h4>
<p style="margin-bottom: 0.5rem; color: #555;">
Create or reconcile durable Gitea issues from provider observations using the <code>#612</code> incident bridge:
</p>
<code>mcp call gitea_observability_reconcile_incident --provider sentry --apply false</code>
</div>
"""
return render_page(title="Observability", body_html=body_html)