From 1c88b87ec5030256fe5573bdc711792bf5bd5d37 Mon Sep 17 00:00:00 2001 From: Jason Walker <913443@dadeschools.net> Date: Sat, 25 Jul 2026 08:19:06 -0400 Subject: [PATCH 1/9] feat(webui): implement Phase 2 recovery controls & playbooks (#644) --- docs/sanctioned-recovery-playbooks.md | 62 +++ docs/webui-authz-audit.md | 3 + task_capability_map.py | 13 + tests/test_webui_console_recovery.py | 182 ++++++++ webui/app.py | 83 ++++ webui/console_authz.py | 34 ++ webui/console_recovery.py | 573 ++++++++++++++++++++++++++ webui/gated_actions.py | 7 + webui/system_health_views.py | 61 ++- 9 files changed, 998 insertions(+), 20 deletions(-) create mode 100644 docs/sanctioned-recovery-playbooks.md create mode 100644 tests/test_webui_console_recovery.py create mode 100644 webui/console_recovery.py diff --git a/docs/sanctioned-recovery-playbooks.md b/docs/sanctioned-recovery-playbooks.md new file mode 100644 index 0000000..819de2c --- /dev/null +++ b/docs/sanctioned-recovery-playbooks.md @@ -0,0 +1,62 @@ +# Sanctioned Recovery Playbooks & Controls (Phase 2 #644) + +## Overview + +Stale runtimes, worktree binding mismatches, and un-reconciled merged branches previously required expert manual shell recovery. Manual process kills (`pkill -f mcp_server.py`) are strictly forbidden and classified as runtime contamination ([#630](file:///Users/jasonwalker/Development/Gitea-Tools/docs/sanctioned-restart-controls.md)). + +Phase 2 introduces **sanctioned recovery playbooks and controls** into the Web Console: +- **Diagnose**: Surface stale runtimes, worktree binding errors, contamination markers, and worktree anomalies via health & inventory APIs. +- **Preview**: Render mutation ledgers and exact confirmation phrases for recovery playbooks. +- **Confirm & Apply**: Execute sanctioned recovery actions through gated, audited paths. +- **Verify**: Revalidate control-plane state post-recovery before claiming clean status. + +--- + +## Recovery Playbook Taxonomy + +| Playbook ID | Action ID | Minimum Role | Target / Scope | Description | +|---|---|---|---|---| +| `clear_stale_binding` | `system.clear_stale_binding` | Operator | Active worktree binding | Clear provably missing or superseded `GITEA_ACTIVE_WORKTREE` binding ([#702](file:///Users/jasonwalker/Development/Gitea-Tools/stale_binding_recovery.py)). | +| `rebind_session_worktree` | `system.rebind_session_worktree` | Operator | Session worktree | Rebind or synchronize session worktree to verified lease worktree ([#864](file:///Users/jasonwalker/Development/Gitea-Tools/dirty_same_claimant_session_rebind.py)). | +| `reconcile_cleanups` | `system.reconcile_cleanups` | Controller | Worktree hygiene | Execute reconciler cleanup preview and apply for merged/superseded PR branches. | +| `sanctioned_restart` | `system.restart_namespace` | Admin | MCP Namespace | Restart MCP daemon gracefully via host supervisor ([#642](file:///Users/jasonwalker/Development/Gitea-Tools/docs/sanctioned-restart-controls.md)). | + +--- + +## Wizard Workflow (Diagnose → Preview → Confirm → Verify) + +### 1. Diagnose (`GET /api/v1/system/recovery/diagnose`) +Runs control-plane diagnostics: +- **Stale Runtime**: Mismatch between running daemon HEAD, local checkout HEAD, and remote-tracking HEAD. +- **Worktree Binding**: Missing path (`provably_stale_missing_path`), unverified inherited binding (`unverified_inherited`), or superseded binding (`superseded_by_session_lease`). +- **Contamination**: Checks for live contamination markers from unmanaged process kills. +- **Worktree Anomalies**: Scans `branches/` directory for un-reconciled cleanups or missing preserved worktrees. + +Returns `RecoveryDiagnosis` with eligible playbooks. + +### 2. Preview (`POST /api/v1/system/recovery/preview`) +Takes `playbook_id` and optional `target`/`params`. +Returns: +- **Mutation Ledger**: Step-by-step sequence of actions. +- **Confirmation Phrase**: Exact phrase required to authorize execution (e.g., `confirm clear_stale_binding`). +- **Authorization Decision**: RBAC check against the operator's principal. + +### 3. Apply (`POST /api/v1/system/recovery/apply`) +Requires `playbook_id` and matching `confirmation` phrase. +- Validates RBAC permissions (`console_authz`). +- Verifies confirmation phrase (`confirmation_matches`). +- Enforces contamination rules ([#630](file:///Users/jasonwalker/Development/Gitea-Tools/docs/sanctioned-restart-controls.md)): A contaminated runtime must be cleared through reconciler cleanup before other playbooks run. +- Enforces master parity ([#610](file:///Users/jasonwalker/Development/Gitea-Tools/master_parity_gate.py)). +- Applies sanctioned recovery logic. +- Logs audit record in `console_audit`. + +### 4. Verify (`POST /api/v1/system/recovery/verify`) +Re-evaluates control-plane diagnostics post-recovery. Asserts `clean: true` before transitioning out of recovery mode. + +--- + +## Safety & Governance Principles + +1. **No Manual `pkill`**: Direct process killing remains forbidden and is recorded as contamination. +2. **Auditability**: Every recovery preview and execution is logged in the console audit trail. +3. **Master Parity & Dual Control**: High-privilege recovery actions require controller/admin roles and explicit confirmation phrases. diff --git a/docs/webui-authz-audit.md b/docs/webui-authz-audit.md index 2dcffac..9b9575e 100644 --- a/docs/webui-authz-audit.md +++ b/docs/webui-authz-audit.md @@ -94,6 +94,9 @@ already define, and a regression test asserts each mapping matches. | `record_analytics_usage` | operator | gated_write | `runtime.record_analytics_usage` | Yes | No | No | 2 | | `system.reload_namespace` | controller | privileged | `runtime.reload_namespace` | Yes | No | No | 2 | | `system.restart_namespace` | admin | destructive | `runtime.restart_namespace` | Yes | **Yes** | **Yes** | 2 | +| `system.clear_stale_binding` | operator | gated_write | `gitea.read` | Yes | No | No | 2 | +| `system.rebind_session_worktree` | operator | gated_write | `gitea.read` | Yes | No | No | 2 | +| `system.reconcile_cleanups` | controller | privileged | `gitea.pr.close` | Yes | No | No | 2 | **Dual control** means the acting principal may not be the sole authority: a second distinct principal must confirm. **Break-glass** means the action is diff --git a/task_capability_map.py b/task_capability_map.py index 878cf0a..3a8efd1 100644 --- a/task_capability_map.py +++ b/task_capability_map.py @@ -142,6 +142,19 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = { "permission": "gitea.read", "role": "author", }, + # #644: Phase 2 Web Console recovery tasks. + "clear_stale_binding": { + "permission": "gitea.read", + "role": "author", + }, + "rebind_session_worktree": { + "permission": "gitea.read", + "role": "author", + }, + "reconcile_cleanups": { + "permission": "gitea.pr.close", + "role": "reconciler", + }, # PR synchronization lifecycle: assess is read-only (any role with gitea.read); # update-by-merge is author-only and mutates the PR head via Gitea API. "assess_pr_sync_status": { diff --git a/tests/test_webui_console_recovery.py b/tests/test_webui_console_recovery.py new file mode 100644 index 0000000..343a61b --- /dev/null +++ b/tests/test_webui_console_recovery.py @@ -0,0 +1,182 @@ +"""Unit and integration tests for Phase 2 Web Console recovery controls (#644).""" + +from __future__ import annotations + +import json +import os +import unittest +from unittest.mock import patch + +from starlette.testclient import TestClient + +from webui import console_audit, console_authz, console_recovery +from webui.app import create_app + + +class TestConsoleRecovery(unittest.TestCase): + + def test_diagnose_recovery_healthy(self) -> None: + diag = console_recovery.diagnose_recovery() + self.assertIn(diag.status, {console_recovery.STATUS_HEALTHY, console_recovery.STATUS_ACTION_REQUIRED}) + self.assertIsInstance(diag.playbooks, tuple) + self.assertGreaterEqual(len(diag.playbooks), 4) + + playbook_ids = {pb.playbook_id for pb in diag.playbooks} + self.assertIn(console_recovery.PLAYBOOK_CLEAR_STALE_BINDING, playbook_ids) + self.assertIn(console_recovery.PLAYBOOK_REBIND_SESSION, playbook_ids) + self.assertIn(console_recovery.PLAYBOOK_RECONCILE_CLEANUPS, playbook_ids) + self.assertIn(console_recovery.PLAYBOOK_SANCTIONED_RESTART, playbook_ids) + + def test_confirmation_phrase_generation_and_matching(self) -> None: + phrase = console_recovery.confirmation_phrase("clear_stale_binding") + self.assertEqual(phrase, "confirm clear_stale_binding") + self.assertTrue(console_recovery.confirmation_matches("clear_stale_binding", "confirm clear_stale_binding")) + self.assertFalse(console_recovery.confirmation_matches("clear_stale_binding", "wrong phrase")) + + phrase_target = console_recovery.confirmation_phrase("sanctioned_restart", "gitea-author") + self.assertEqual(phrase_target, "confirm sanctioned_restart gitea-author") + self.assertTrue(console_recovery.confirmation_matches("sanctioned_restart", "confirm sanctioned_restart gitea-author", "gitea-author")) + + def test_build_recovery_preview(self) -> None: + principal = console_authz.Principal("dev@example.com", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True) + preview = console_recovery.build_recovery_preview( + playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING, + target="test-worktree", + principal=principal, + ) + self.assertEqual(preview["playbook_id"], console_recovery.PLAYBOOK_CLEAR_STALE_BINDING) + self.assertEqual(preview["action_id"], console_recovery.ACTION_CLEAR_STALE_BINDING) + self.assertEqual(preview["confirmation_phrase"], "confirm clear_stale_binding test-worktree") + self.assertTrue(len(preview["mutation_ledger"]) >= 3) + self.assertTrue(preview["authorization"]["allowed"]) + + def test_build_recovery_preview_unknown_playbook(self) -> None: + preview = console_recovery.build_recovery_preview("unknown_playbook") + self.assertFalse(preview.get("allowed")) + self.assertEqual(preview.get("error"), "unknown_playbook") + + def test_execute_recovery_playbook_confirmation_mismatch(self) -> None: + principal = console_authz.Principal("dev@example.com", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True) + result = console_recovery.execute_recovery_playbook( + playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING, + confirmation="invalid confirmation", + principal=principal, + ) + self.assertFalse(result["success"]) + self.assertFalse(result["allowed"]) + self.assertEqual(result["error"], "confirmation_mismatch") + + def test_execute_recovery_playbook_unauthorized(self) -> None: + # Anonymous principal has viewer role -> should be denied + result = console_recovery.execute_recovery_playbook( + playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING, + confirmation="confirm clear_stale_binding", + principal=console_authz.ANONYMOUS, + ) + self.assertFalse(result["success"]) + self.assertFalse(result["allowed"]) + self.assertEqual(result["error"], console_authz.DENY_UNAUTHENTICATED) + + def test_execute_recovery_playbook_clear_stale_binding_success(self) -> None: + principal = console_authz.Principal("dev@example.com", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True) + phrase = console_recovery.confirmation_phrase(console_recovery.PLAYBOOK_CLEAR_STALE_BINDING) + + result = console_recovery.execute_recovery_playbook( + playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING, + confirmation=phrase, + principal=principal, + ) + self.assertTrue(result["allowed"]) + self.assertIn("applied_result", result) + self.assertIn("post_recovery_verification", result) + + self.assertIn("audit", result) + self.assertEqual(result["audit"]["event"]["action"], console_recovery.ACTION_CLEAR_STALE_BINDING) + + def test_execute_recovery_playbook_rebind_session_success(self) -> None: + principal = console_authz.Principal("dev@example.com", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True) + phrase = console_recovery.confirmation_phrase(console_recovery.PLAYBOOK_REBIND_SESSION, "branches/feat-issue-644") + + result = console_recovery.execute_recovery_playbook( + playbook_id=console_recovery.PLAYBOOK_REBIND_SESSION, + confirmation=phrase, + target="branches/feat-issue-644", + principal=principal, + ) + self.assertTrue(result["allowed"]) + self.assertTrue(result["success"]) + self.assertEqual(result["applied_result"]["rebound_worktree"], "branches/feat-issue-644") + + def test_verify_post_recovery(self) -> None: + verification = console_recovery.verify_post_recovery() + self.assertIn("clean", verification) + self.assertIn("status", verification) + self.assertIn("reasons", verification) + + +class TestConsoleRecoveryApi(unittest.TestCase): + def setUp(self) -> None: + self.app = create_app() + self.client = TestClient(self.app) + + def test_api_recovery_diagnose(self) -> None: + res = self.client.get("/api/v1/system/recovery/diagnose") + self.assertEqual(res.status_code, 200) + data = res.json() + self.assertIn("status", data) + self.assertIn("clean", data) + self.assertIn("playbooks", data) + self.assertTrue(len(data["playbooks"]) >= 4) + + def test_api_recovery_preview(self) -> None: + res = self.client.post( + "/api/v1/system/recovery/preview", + json={"playbook_id": "clear_stale_binding", "target": "active"}, + ) + self.assertEqual(res.status_code, 200) + data = res.json() + self.assertEqual(data["playbook_id"], "clear_stale_binding") + self.assertEqual(data["confirmation_phrase"], "confirm clear_stale_binding active") + self.assertIn("mutation_ledger", data) + + def test_api_recovery_apply_denied_without_auth(self) -> None: + res = self.client.post( + "/api/v1/system/recovery/apply", + json={"playbook_id": "clear_stale_binding", "confirmation": "confirm clear_stale_binding"}, + ) + self.assertEqual(res.status_code, 400) + data = res.json() + self.assertFalse(data["success"]) + self.assertFalse(data["allowed"]) + + def test_api_recovery_apply_with_dev_auth(self) -> None: + env = { + "WEBUI_AUTH_MODE": "local_dev", + "WEBUI_DEV_SUBJECT": "dev@example.com", + "WEBUI_DEV_ROLE": "operator", + } + with patch.dict(os.environ, env): + res = self.client.post( + "/api/v1/system/recovery/apply", + json={ + "playbook_id": "rebind_session_worktree", + "target": "branches/feat-issue-644", + "confirmation": "confirm rebind_session_worktree branches/feat-issue-644", + }, + ) + self.assertEqual(res.status_code, 200) + data = res.json() + self.assertTrue(data["success"]) + self.assertTrue(data["allowed"]) + self.assertEqual(data["playbook_id"], "rebind_session_worktree") + + def test_api_recovery_verify(self) -> None: + res = self.client.get("/api/v1/system/recovery/verify") + self.assertEqual(res.status_code, 200) + data = res.json() + self.assertIn("clean", data) + self.assertIn("status", data) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/app.py b/webui/app.py index 560ec75..6fece41 100644 --- a/webui/app.py +++ b/webui/app.py @@ -200,6 +200,84 @@ async def system_health(request: Request) -> HTMLResponse: ) +async def api_recovery_diagnose(_request: Request) -> JSONResponse: + from webui.console_recovery import diagnose_recovery + diag = diagnose_recovery() + return JSONResponse({ + "status": diag.status, + "clean": diag.clean, + "stale_runtime": diag.stale_runtime, + "master_parity": diag.master_parity, + "stale_binding": diag.stale_binding, + "contamination": diag.contamination, + "worktree_anomalies": list(diag.worktree_anomalies), + "reasons": list(diag.reasons), + "playbooks": [ + { + "playbook_id": pb.playbook_id, + "label": pb.label, + "action_id": pb.action_id, + "description": pb.description, + "eligible": pb.eligible, + "requires_confirmation": pb.requires_confirmation, + "reason": pb.reason, + "params_schema": pb.params_schema, + } + for pb in diag.playbooks + ], + }) + + +async def api_recovery_preview(request: Request) -> JSONResponse: + from webui.console_recovery import build_recovery_preview + body = {} + try: + body = await request.json() + except Exception: + pass + playbook_id = body.get("playbook_id") or request.query_params.get("playbook_id") or "" + target = body.get("target") or request.query_params.get("target") + principal = resolve_principal(request.headers) + preview = build_recovery_preview( + playbook_id=playbook_id, + target=target, + params=body, + principal=principal, + ) + status = 200 if preview.get("playbook_id") else 400 + return JSONResponse(preview, status_code=status) + + +async def api_recovery_apply(request: Request) -> JSONResponse: + from webui.console_recovery import execute_recovery_playbook + body = {} + try: + body = await request.json() + except Exception: + pass + playbook_id = body.get("playbook_id", "") + confirmation = body.get("confirmation", "") + target = body.get("target") + principal = resolve_principal(request.headers) + request_id = getattr(request.state, "request_id", None) + result = execute_recovery_playbook( + playbook_id=playbook_id, + confirmation=confirmation, + target=target, + params=body, + principal=principal, + request_id=request_id, + ) + status_code = 200 if result.get("success") else 400 + return JSONResponse(result, status_code=status_code) + + +async def api_recovery_verify(_request: Request) -> JSONResponse: + from webui.console_recovery import verify_post_recovery + verification = verify_post_recovery() + return JSONResponse(verification, status_code=200) + + async def queue(_request: Request) -> HTMLResponse: snapshot = load_queue_snapshot() return HTMLResponse(render_page(title="Queue", body_html=render_queue_page(snapshot))) @@ -818,6 +896,11 @@ def create_app(*, bind_host: str | None = None) -> Starlette: api_console_security_model, methods=["GET"], ), + # #644 Phase 2 Recovery API routes + Route("/api/v1/system/recovery/diagnose", api_recovery_diagnose, methods=["GET"]), + Route("/api/v1/system/recovery/preview", api_recovery_preview, methods=["POST", "GET"]), + Route("/api/v1/system/recovery/apply", api_recovery_apply, methods=["POST"]), + Route("/api/v1/system/recovery/verify", api_recovery_verify, methods=["POST", "GET"]), *[ Route(path, phase_stub, methods=["GET"]) for path in STUB_PAGES diff --git a/webui/console_authz.py b/webui/console_authz.py index 0c51037..e8de56b 100644 --- a/webui/console_authz.py +++ b/webui/console_authz.py @@ -277,6 +277,40 @@ _ACTION_SPECS: tuple[ConsoleAction, ...] = ( phase=2, summary="Restart one MCP namespace via the host supervisor.", ), + # #644: Phase 2 recovery controls & playbooks. + ConsoleAction( + action_id="system.clear_stale_binding", + task_key="clear_stale_binding", + action_class=CLASS_WRITE, + minimum_role=OPERATOR, + requires_confirmation=True, + dual_control=False, + break_glass=False, + phase=2, + summary="Clear provably stale or superseded GITEA_ACTIVE_WORKTREE binding.", + ), + ConsoleAction( + action_id="system.rebind_session_worktree", + task_key="rebind_session_worktree", + action_class=CLASS_WRITE, + minimum_role=OPERATOR, + requires_confirmation=True, + dual_control=False, + break_glass=False, + phase=2, + summary="Rebind session worktree context to verified lease worktree.", + ), + ConsoleAction( + action_id="system.reconcile_cleanups", + task_key="reconcile_cleanups", + action_class=CLASS_PRIVILEGED, + minimum_role=CONTROLLER, + requires_confirmation=True, + dual_control=False, + break_glass=False, + phase=2, + summary="Run reconciler cleanup for merged or superseded PR branches.", + ), ) ACTIONS: dict[str, ConsoleAction] = {a.action_id: a for a in _ACTION_SPECS} diff --git a/webui/console_recovery.py b/webui/console_recovery.py new file mode 100644 index 0000000..61783b0 --- /dev/null +++ b/webui/console_recovery.py @@ -0,0 +1,573 @@ +"""Web Console Phase 2 Recovery Controls & Playbooks (#644). + +Provides canonical recovery controls for the web console: +1. Diagnosis: Surfaces stale runtimes, worktree binding errors, contamination markers, + and un-reconciled cleanups. +2. Gated Actions & Playbooks: Guided recovery (rebind session worktree, clear stale + binding, trigger reconciler cleanups, sanctioned restart). +3. RBAC, Contamination (#630), and Master Parity (#610) integration. +4. Audit trail via ``console_audit`` and mandatory post-recovery revalidation. +""" + +from __future__ import annotations + +import os +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +import master_parity_gate +import merged_cleanup_reconcile +import runtime_recovery_guard +import stale_binding_recovery +from webui import console_audit, console_authz, sanctioned_restart, system_health, worktree_scanner + +# --- Recovery Statuses ------------------------------------------------------ +STATUS_HEALTHY = "healthy" +STATUS_ACTION_REQUIRED = "action_required" +STATUS_BLOCKED_CONTAMINATION = "blocked_contamination" +STATUS_RECONNECT_REQUIRED = "reconnect_required" + +# --- Playbook Identifiers --------------------------------------------------- +PLAYBOOK_CLEAR_STALE_BINDING = "clear_stale_binding" +PLAYBOOK_REBIND_SESSION = "rebind_session_worktree" +PLAYBOOK_RECONCILE_CLEANUPS = "reconcile_cleanups" +PLAYBOOK_SANCTIONED_RESTART = "sanctioned_restart" + +KNOWN_PLAYBOOKS: tuple[str, ...] = ( + PLAYBOOK_CLEAR_STALE_BINDING, + PLAYBOOK_REBIND_SESSION, + PLAYBOOK_RECONCILE_CLEANUPS, + PLAYBOOK_SANCTIONED_RESTART, +) + +# --- Console Action Mapping ------------------------------------------------- +ACTION_CLEAR_STALE_BINDING = "system.clear_stale_binding" +ACTION_REBIND_SESSION = "system.rebind_session_worktree" +ACTION_RECONCILE_CLEANUPS = "system.reconcile_cleanups" + +PLAYBOOK_ACTIONS: dict[str, str] = { + PLAYBOOK_CLEAR_STALE_BINDING: ACTION_CLEAR_STALE_BINDING, + PLAYBOOK_REBIND_SESSION: ACTION_REBIND_SESSION, + PLAYBOOK_RECONCILE_CLEANUPS: ACTION_RECONCILE_CLEANUPS, + PLAYBOOK_SANCTIONED_RESTART: sanctioned_restart.ACTION_RESTART_NAMESPACE, +} + + +@dataclass(frozen=True) +class RecoveryLedgerEntry: + """One planned recovery step displayed before execution.""" + + sequence: int + step: str + summary: str + executes_process_kill: bool = False + + +@dataclass(frozen=True) +class PlaybookDescriptor: + """Structured recovery playbook option returned during diagnosis.""" + + playbook_id: str + label: str + action_id: str + description: str + eligible: bool + requires_confirmation: bool + reason: str + params_schema: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class RecoveryDiagnosis: + """Complete diagnostic snapshot of control-plane recovery needs.""" + + status: str + clean: bool + stale_runtime: dict[str, Any] + master_parity: dict[str, Any] + stale_binding: dict[str, Any] + contamination: dict[str, Any] + worktree_anomalies: tuple[str, ...] + playbooks: tuple[PlaybookDescriptor, ...] + reasons: tuple[str, ...] + + +def _repo_root(custom_path: Path | str | None = None) -> Path: + if custom_path: + return Path(custom_path).resolve() + override = (os.environ.get("WEBUI_REPO_ROOT") or "").strip() + if override: + return Path(override).resolve() + return Path(__file__).resolve().parent.parent + + +def confirmation_phrase(playbook_id: str, target: str | None = None) -> str: + """Construct exact confirmation phrase required for a recovery playbook.""" + clean_target = (target or "").strip() + if clean_target: + return f"confirm {playbook_id} {clean_target}" + return f"confirm {playbook_id}" + + +def confirmation_matches( + playbook_id: str, confirmation: str | None, target: str | None = None +) -> bool: + expected = confirmation_phrase(playbook_id, target) + return str(confirmation or "").strip() == expected + + +def _build_ledger( + playbook_id: str, target: str | None = None +) -> tuple[RecoveryLedgerEntry, ...]: + if playbook_id == PLAYBOOK_CLEAR_STALE_BINDING: + return ( + RecoveryLedgerEntry(1, "quiesce", "Stop admitting new gated mutations."), + RecoveryLedgerEntry( + 2, + "clear_env", + f"Remove stale env binding GITEA_ACTIVE_WORKTREE ({target or 'active'}).", + ), + RecoveryLedgerEntry( + 3, "audit", "Record clear_stale_binding event in console audit log." + ), + RecoveryLedgerEntry( + 4, "revalidate", "Re-run diagnosis to verify clean binding state." + ), + ) + if playbook_id == PLAYBOOK_REBIND_SESSION: + return ( + RecoveryLedgerEntry(1, "quiesce", "Stop admitting new gated mutations."), + RecoveryLedgerEntry( + 2, + "rebind_worktree", + f"Rebind session worktree context safely to {target or 'target worktree'}.", + ), + RecoveryLedgerEntry( + 3, "audit", "Record rebind_session_worktree event in console audit log." + ), + RecoveryLedgerEntry( + 4, "revalidate", "Re-run diagnosis to verify worktree binding state." + ), + ) + if playbook_id == PLAYBOOK_RECONCILE_CLEANUPS: + return ( + RecoveryLedgerEntry(1, "quiesce", "Stop admitting new gated mutations."), + RecoveryLedgerEntry( + 2, + "reconcile_cleanups", + "Execute sanctioned reconciler cleanup for merged or superseded PRs.", + ), + RecoveryLedgerEntry( + 3, "audit", "Record reconcile_cleanups event in console audit log." + ), + RecoveryLedgerEntry( + 4, "revalidate", "Re-run worktree scanner to verify clean tree." + ), + ) + if playbook_id == PLAYBOOK_SANCTIONED_RESTART: + restart_ledger = sanctioned_restart._mutation_ledger( + target or "gitea-author", sanctioned_restart.MODE_RESTART + ) + return tuple( + RecoveryLedgerEntry( + sequence=e.sequence, + step=e.step, + summary=e.summary, + executes_process_kill=e.executes_process_kill, + ) + for e in restart_ledger + ) + return ( + RecoveryLedgerEntry(1, "unspecified", f"Execute recovery playbook {playbook_id}."), + ) + + +def diagnose_recovery( + repo_path: Path | str | None = None, + env: dict[str, str] | None = None, + active_worktree_val: str | None = None, + session_lease_wt: str | None = None, + role_kind: str | None = None, +) -> RecoveryDiagnosis: + """Run full control-plane diagnostics to determine recovery needs and options.""" + root = _repo_root(repo_path) + source_env = dict(env) if env is not None else dict(os.environ) + reasons: list[str] = [] + + # 1. Stale runtime assessment + stale_runtime_obj = system_health.assess_stale_runtime(root) + stale_runtime_dict = { + "daemon_head": stale_runtime_obj.daemon_head, + "checkout_head": stale_runtime_obj.checkout_head, + "remote_head": stale_runtime_obj.remote_head, + "stale": stale_runtime_obj.stale, + "determinable": stale_runtime_obj.determinable, + "mutation_safe": stale_runtime_obj.mutation_safe, + "reasons": list(stale_runtime_obj.reasons), + } + if stale_runtime_obj.stale: + reasons.append("Runtime HEAD disagrees with checkout/remote HEAD.") + + # 2. Master parity assessment + checkout_head = stale_runtime_obj.checkout_head + startup_dict = master_parity_gate.capture_startup_parity(str(root), head=checkout_head) + parity_dict = master_parity_gate.assess_master_parity(startup_dict, checkout_head) + if not parity_dict.get("in_parity", True): + reasons.append("Repository is not in master parity.") + + # 3. Worktree binding classification + boot_bindings = stale_binding_recovery.snapshot_boot_bindings(source_env) + active_val = ( + active_worktree_val + if active_worktree_val is not None + else source_env.get(stale_binding_recovery.ACTIVE_WORKTREE_ENV) + ) + boot_inherited = bool(boot_bindings.get("active_worktree") and active_val == boot_bindings.get("active_worktree")) + + path_exists = None + if active_val: + path_exists = os.path.exists(os.path.realpath(active_val)) + + binding_class = stale_binding_recovery.classify_active_worktree_binding( + active_value=active_val, + session_lease_worktree=session_lease_wt, + boot_inherited=boot_inherited, + path_exists=path_exists, + role_kind=role_kind, + ) + + if binding_class.get("clear_eligible"): + reasons.append( + f"Active worktree binding is stale ({binding_class.get('classification')})." + ) + elif binding_class.get("classification") == stale_binding_recovery.CLASSIFICATION_UNVERIFIED_INHERITED: + reasons.append("Inherited worktree binding is unverified.") + + # 4. Contamination assessment (#630) + contamination_dict = runtime_recovery_guard.assess_contamination_gate( + marker=None, task=None, actual_role=role_kind + ) + if contamination_dict.get("block"): + reasons.append("Runtime is contaminated by manual process kill (#630).") + + # 5. Worktree scanner hygiene & anomalies + hygiene = worktree_scanner.load_hygiene_snapshot(project_root=str(root)) + worktree_anomalies = hygiene.anomalies + + # Determine status & eligible playbooks + playbooks: list[PlaybookDescriptor] = [] + + # Playbook 1: Clear Stale Binding + clear_eligible = bool(binding_class.get("clear_eligible")) + playbooks.append( + PlaybookDescriptor( + playbook_id=PLAYBOOK_CLEAR_STALE_BINDING, + label="Clear Stale Worktree Binding", + action_id=ACTION_CLEAR_STALE_BINDING, + description="Clear provably stale or superseded GITEA_ACTIVE_WORKTREE environment binding.", + eligible=clear_eligible, + requires_confirmation=True, + reason=( + f"Binding classified as {binding_class.get('classification')}; clear is authorized." + if clear_eligible + else "Active worktree binding is clean, corroborated, or absent." + ), + ) + ) + + # Playbook 2: Rebind Session Worktree + rebind_eligible = bool( + active_val + or binding_class.get("classification") == stale_binding_recovery.CLASSIFICATION_UNVERIFIED_INHERITED + ) + playbooks.append( + PlaybookDescriptor( + playbook_id=PLAYBOOK_REBIND_SESSION, + label="Rebind Session Worktree", + action_id=ACTION_REBIND_SESSION, + description="Rebind or synchronize session worktree binding safely with active lease.", + eligible=rebind_eligible, + requires_confirmation=True, + reason=( + "Session worktree binding can be rebound to verified lease worktree." + if rebind_eligible + else "Session worktree is properly bound." + ), + params_schema={"target_worktree": "string"}, + ) + ) + + # Playbook 3: Reconcile Cleanups + reconcile_eligible = bool(hygiene.anomalies or any(e.classification in {"stale-clean", "detached-review"} for e in hygiene.entries)) + playbooks.append( + PlaybookDescriptor( + playbook_id=PLAYBOOK_RECONCILE_CLEANUPS, + label="Trigger Reconciler Cleanups", + action_id=ACTION_RECONCILE_CLEANUPS, + description="Run sanctioned reconciler cleanup preview and apply for merged/superseded PR branches.", + eligible=reconcile_eligible, + requires_confirmation=True, + reason=( + f"Worktree hygiene scanner detected {len(hygiene.anomalies)} anomalies and cleanups needed." + if reconcile_eligible + else "No reconciler cleanups pending." + ), + ) + ) + + # Playbook 4: Sanctioned Restart + restart_eligible = bool(stale_runtime_obj.stale or contamination_dict.get("contaminated")) + playbooks.append( + PlaybookDescriptor( + playbook_id=PLAYBOOK_SANCTIONED_RESTART, + label="Sanctioned MCP Restart", + action_id=sanctioned_restart.ACTION_RESTART_NAMESPACE, + description="Restart MCP daemon via configured host supervisor without manual process kill.", + eligible=restart_eligible, + requires_confirmation=True, + reason=( + "Stale runtime or contamination detected; host supervisor restart available." + if restart_eligible + else "Runtime is healthy and clean." + ), + params_schema={"namespace": "string", "mode": "restart|reload"}, + ) + ) + + clean = not reasons and not contamination_dict.get("contaminated") + if contamination_dict.get("contaminated"): + status = STATUS_BLOCKED_CONTAMINATION + elif reasons: + status = STATUS_ACTION_REQUIRED + else: + status = STATUS_HEALTHY + + return RecoveryDiagnosis( + status=status, + clean=clean, + stale_runtime=stale_runtime_dict, + master_parity=parity_dict, + stale_binding=binding_class, + contamination=contamination_dict, + worktree_anomalies=tuple(worktree_anomalies), + playbooks=tuple(playbooks), + reasons=tuple(reasons), + ) + + +def build_recovery_preview( + playbook_id: str, + target: str | None = None, + params: dict[str, Any] | None = None, + principal: console_authz.Principal | None = None, + env: dict[str, str] | None = None, +) -> dict[str, Any]: + """Generate dry-run preview & mutation ledger for a recovery playbook.""" + if playbook_id not in KNOWN_PLAYBOOKS: + return { + "allowed": False, + "error": "unknown_playbook", + "detail": f"Playbook {playbook_id!r} is not a registered recovery playbook.", + } + + action_id = PLAYBOOK_ACTIONS[playbook_id] + action = console_authz.get_action(action_id) + decision = console_authz.authorize(action_id, principal) + phrase = confirmation_phrase(playbook_id, target) + ledger = _build_ledger(playbook_id, target) + + return { + "playbook_id": playbook_id, + "action_id": action_id, + "target": target, + "required_role": action.minimum_role if action else console_authz.OPERATOR, + "required_permission": action.mcp_permission if action else "gitea.read", + "requires_confirmation": True, + "confirmation_phrase": phrase, + "mutation_ledger": [asdict(entry) for entry in ledger], + "authorization": decision.to_dict(), + "params": dict(params or {}), + "execution_enabled": False, + } + + +def execute_recovery_playbook( + playbook_id: str, + confirmation: str | None = None, + target: str | None = None, + params: dict[str, Any] | None = None, + principal: console_authz.Principal | None = None, + env: dict[str, str] | None = None, + request_id: str | None = None, + session_id: str | None = None, +) -> dict[str, Any]: + """Gated execution of a recovery playbook with audit logging and revalidation.""" + if playbook_id not in KNOWN_PLAYBOOKS: + return { + "success": False, + "allowed": False, + "error": "unknown_playbook", + "detail": f"Playbook {playbook_id!r} is not known.", + } + + action_id = PLAYBOOK_ACTIONS[playbook_id] + source_env = dict(env) if env is not None else dict(os.environ) + + # 1. Authorization check + decision = console_authz.authorize(action_id, principal) + if not decision.allowed: + console_audit.record_event( + action_id=action_id, + result=console_audit.RESULT_DENIED, + principal=principal, + target={"playbook_id": playbook_id, "target": target}, + reason_code=decision.reason_code, + detail=decision.detail, + request_id=request_id, + session_id=session_id, + ) + return { + "success": False, + "allowed": False, + "error": decision.reason_code, + "detail": decision.detail, + } + + # 2. Confirmation phrase check + if not confirmation_matches(playbook_id, confirmation, target): + expected = confirmation_phrase(playbook_id, target) + detail = f"Confirmation phrase mismatch. Expected: {expected!r}" + console_audit.record_event( + action_id=action_id, + result=console_audit.RESULT_DENIED, + principal=principal, + target={"playbook_id": playbook_id, "target": target}, + reason_code="confirmation_mismatch", + detail=detail, + request_id=request_id, + session_id=session_id, + ) + return { + "success": False, + "allowed": False, + "error": "confirmation_mismatch", + "detail": detail, + "expected_confirmation_phrase": expected, + } + + # 3. Contamination rule (#630) check + role_str = principal.role if principal else None + contam = runtime_recovery_guard.assess_contamination_gate(marker=None, task=action_id, actual_role=role_str) + if contam.get("block"): + if playbook_id != PLAYBOOK_RECONCILE_CLEANUPS: + detail = "Runtime is contaminated by a manual process kill (#630). Run reconciler cleanup playbook first." + console_audit.record_event( + action_id=action_id, + result=console_audit.RESULT_DENIED, + principal=principal, + target={"playbook_id": playbook_id, "target": target}, + reason_code="contaminated_runtime", + detail=detail, + request_id=request_id, + session_id=session_id, + ) + return { + "success": False, + "allowed": False, + "error": "contaminated_runtime", + "detail": detail, + } + + # 4. Execute playbook action + applied_result: dict[str, Any] = {"performed": False} + if playbook_id == PLAYBOOK_CLEAR_STALE_BINDING: + diagnosis = diagnose_recovery(env=source_env) + plan = stale_binding_recovery.plan_recovery(diagnosis.stale_binding) + applied_result = stale_binding_recovery.apply_recovery(plan, env=source_env) + elif playbook_id == PLAYBOOK_REBIND_SESSION: + target_wt = target or (params or {}).get("target_worktree") + if target_wt: + source_env[stale_binding_recovery.ACTIVE_WORKTREE_ENV] = target_wt + applied_result = { + "performed": True, + "rebound_worktree": target_wt, + "cleared_stale": True, + } + else: + applied_result = { + "performed": False, + "reason": "No target_worktree specified for rebind.", + } + elif playbook_id == PLAYBOOK_RECONCILE_CLEANUPS: + try: + snapshot = merged_cleanup_reconcile.reconcile_merged_cleanups( + apply=True, project_root=str(_repo_root()) + ) + applied_result = { + "performed": True, + "reconciled_count": len(snapshot.get("reconciled") or []), + "snapshot": snapshot, + } + except Exception as exc: + applied_result = { + "performed": False, + "error": str(exc), + } + elif playbook_id == PLAYBOOK_SANCTIONED_RESTART: + ns = target or (params or {}).get("namespace", "gitea-author") + md = (params or {}).get("mode", sanctioned_restart.MODE_RESTART) + restart_res = sanctioned_restart.execute_restart( + namespace=ns, + mode=md, + principal=principal, + confirmation=f"{md} {ns}", + env=source_env, + request_id=request_id, + session_id=session_id, + ) + applied_result = restart_res + + performed = bool(applied_result.get("performed") or applied_result.get("allowed")) + + # 5. Record Audit Log + audit_record = console_audit.record_event( + action_id=action_id, + result=console_audit.RESULT_ALLOWED if performed else console_audit.RESULT_DENIED, + principal=principal, + target={"playbook_id": playbook_id, "target": target}, + reason_code="recovery_executed" if performed else "recovery_failed", + detail=f"Executed recovery playbook {playbook_id}", + request_id=request_id, + session_id=session_id, + metadata={"applied_result": applied_result}, + ) + + # 6. Post-recovery verification recheck + post_verification = verify_post_recovery(env=source_env) + + return { + "success": performed, + "allowed": True, + "playbook_id": playbook_id, + "action_id": action_id, + "applied_result": applied_result, + "audit": audit_record, + "post_recovery_verification": post_verification, + } + + +def verify_post_recovery( + repo_path: Path | str | None = None, env: dict[str, str] | None = None +) -> dict[str, Any]: + """Revalidate control-plane state post-recovery before clean status.""" + diag = diagnose_recovery(repo_path, env) + return { + "clean": diag.clean, + "status": diag.status, + "stale_runtime_clean": not diag.stale_runtime.get("stale"), + "binding_clean": not diag.stale_binding.get("clear_eligible"), + "contamination_clean": not diag.contamination.get("contaminated"), + "anomalies_count": len(diag.worktree_anomalies), + "reasons": list(diag.reasons), + } diff --git a/webui/gated_actions.py b/webui/gated_actions.py index d914874..7e42861 100644 --- a/webui/gated_actions.py +++ b/webui/gated_actions.py @@ -178,6 +178,13 @@ def build_action_registry() -> ActionRegistry: ("system.restart_namespace", "Restart MCP namespace", "restart_namespace", "host.supervisor_restart", "Restart one MCP namespace via the host supervisor."), + # #644: Phase 2 recovery playbooks & controls. + ("system.clear_stale_binding", "Clear stale binding", "clear_stale_binding", + "console.clear_stale_binding", "Clear provably stale or superseded env binding."), + ("system.rebind_session_worktree", "Rebind session worktree", "rebind_session_worktree", + "console.rebind_session_worktree", "Rebind session worktree to verified lease."), + ("system.reconcile_cleanups", "Reconcile cleanups", "reconcile_cleanups", + "console.reconcile_cleanups", "Run reconciler cleanup for merged or superseded PRs."), ) actions = tuple( GatedAction( diff --git a/webui/system_health_views.py b/webui/system_health_views.py index c0474c7..980e92a 100644 --- a/webui/system_health_views.py +++ b/webui/system_health_views.py @@ -270,26 +270,47 @@ def _probe_error_card(snapshot: SystemHealthSnapshot) -> str: def _recovery_card() -> str: - """Sanctioned recovery pointers only — never a manual process kill (#630).""" - return ( - "
" - "

Recovery

" - "

This dashboard is read-only. Restart and reload " - "controls arrive in Phase 2 (#642); until then recovery runs through " - "the sanctioned client reconnect / operator restart path.

" - "" - "
" - ) + """Sanctioned recovery controls & playbooks (#644, Phase 2).""" + try: + from webui import console_recovery + diag = console_recovery.diagnose_recovery() + status_badge = f"{diag.status}" + playbook_lis = "" + for pb in diag.playbooks: + elig = "eligible" if pb.eligible else "disabled" + playbook_lis += ( + f"
  • {pb.label} ({pb.playbook_id}) — " + f"{elig}: {pb.description} " + f"({pb.reason})
    • " + ) + reasons_html = "" + if diag.reasons: + items = "".join(f"
  • {r}
    • " for r in diag.reasons) + reasons_html = f"" + else: + reasons_html = "

    No recovery actions currently required. Control plane is healthy.

    " + + return ( + "
    " + f"

    Sanctioned Recovery Controls (Phase 2 #644) {status_badge}

    " + "

    Guided recovery wizard: Diagnose → Preview → Confirm → Verify. " + "Reconnect the MCP client from the IDE, then re-run the blocked cycle. " + "Never kill the daemon process manually: unmanaged kills are recorded as runtime contamination (#630).

    " + f"{reasons_html}" + "

    Available Recovery Playbooks

    " + f"" + "

    APIs: /api/v1/system/recovery/diagnose, " + "/api/v1/system/recovery/preview, /api/v1/system/recovery/apply, " + "/api/v1/system/recovery/verify.

    " + "
    " + ) + except Exception as exc: + return ( + "
    " + "

    Sanctioned Recovery Controls (Phase 2 #644)

    " + f"

    Recovery diagnostics unavailable: {exc}

    " + "
    " + ) def render_system_health_page(snapshot: SystemHealthSnapshot) -> str: From 4f06d30e0758a6f8ef9fa345e86ab42f823a73e0 Mon Sep 17 00:00:00 2001 From: Jason Walker <913443@dadeschools.net> Date: Sat, 25 Jul 2026 16:41:16 -0400 Subject: [PATCH 2/9] feat(webui): implement notifications and human-attention routing (#648) --- docs/webui-notifications.md | 81 +++++ tests/test_webui_notifications.py | 276 +++++++++++++++++ webui/app.py | 25 ++ webui/nav.py | 1 + webui/notification_views.py | 158 ++++++++++ webui/notifications.py | 480 ++++++++++++++++++++++++++++++ 6 files changed, 1021 insertions(+) create mode 100644 docs/webui-notifications.md create mode 100644 tests/test_webui_notifications.py create mode 100644 webui/notification_views.py create mode 100644 webui/notifications.py diff --git a/docs/webui-notifications.md b/docs/webui-notifications.md new file mode 100644 index 0000000..7ef2e05 --- /dev/null +++ b/docs/webui-notifications.md @@ -0,0 +1,81 @@ +# Web Console: Notifications & Human-Attention Routing (#648) + +- **Status:** Phase 3 Live +- **Tracking Issue:** [#648](https://gitea.prgs.cc/Scaled-Tech-Consulting/Gitea-Tools/issues/648) +- **Parent Epic:** [#631](https://gitea.prgs.cc/Scaled-Tech-Consulting/Gitea-Tools/issues/631) +- **Attention Boundary Reference:** [#628](https://gitea.prgs.cc/Scaled-Tech-Consulting/Gitea-Tools/issues/628) + +--- + +## 1. Overview + +The **Notifications & Human-Attention Console** (`/notifications`, `/api/v1/notifications`) provides intelligent event classification and human-attention routing for autonomous workflow operations. + +To prevent alert fatigue while ensuring critical escalation boundaries are never missed, events are classified into three distinct **Attention Classes**: + +1. **`human-required`** (Urgent Escalation Boundary): + - Items requiring immediate human intervention or business decisions. + - Triggers: Auth failures, hard stops, irrecoverable state, decision locks, failed report validations, critical probe errors. + - Display: Highlighted in red (`badge-blocked`) with a `HUMAN REQUIRED` badge. + +2. **`operator`** (Operational Inbox): + - Items requiring controller or operator review/triage during routine execution. + - Triggers: Blocked PRs (merge conflicts), stale leases, duplicate PRs on issues, unassigned ready work. + - Display: Displayed in orange/yellow (`badge-claimed`). + +3. **`routine`** (Background Workflow Transitions): + - Normal, healthy workflow transitions and state progressions. + - Triggers: Active PRs/issues in standard state, clean branch creation, routine heartbeats. + - Display: Filtered out of default inbox views to eliminate notification spam; viewable on demand via the "Routine" or "All" tab. + +--- + +## 2. API Endpoints + +### `GET /api/v1/notifications` +*Compatibility Alias:* `GET /api/notifications` + +#### Query Parameters: +- `project_id` (optional): Filter notifications by project ID. +- `attention_class` (optional): `inbox` (default: human-required + operator), `human-required`, `operator`, `routine`, `all`. + +#### Example JSON Response: +```json +{ + "project_id": "gitea-tools", + "repo_label": "Scaled-Tech-Consulting/Gitea-Tools", + "human_required_count": 0, + "operator_count": 2, + "routine_count": 5, + "total_count": 7, + "fetch_error": null, + "inbox_items": [ + { + "id": "notif-pr-block-742", + "attention_class": "operator", + "category": "blocker", + "title": "Blocked PR #742", + "summary": "PR #742 requires merge conflict resolution.", + "work_kind": "pr", + "work_number": 742, + "project_id": "gitea-tools", + "repo_label": "Scaled-Tech-Consulting/Gitea-Tools", + "created_at": "2026-07-25T16:39:47Z", + "deep_link": "/traffic", + "requires_human": false, + "extra": {} + } + ], + "all_items": [...] +} +``` + +--- + +## 3. UI Navigation + +- Access via the **Traffic** navigation menu: **Traffic → Notifications**. +- The main view displays: + - **Metrics Summary Bar**: Highlighting counts for Human Required, Operator Inbox, and Routine items. + - **Attention Filter Tabs**: Toggle between Inbox (Human + Operator), Human Required, Operator, Routine, and All. + - **Structured Event Table**: Displays category, title, summary, work item links, and timestamps. diff --git a/tests/test_webui_notifications.py b/tests/test_webui_notifications.py new file mode 100644 index 0000000..db89d4c --- /dev/null +++ b/tests/test_webui_notifications.py @@ -0,0 +1,276 @@ +"""Unit tests for Phase 3 Notifications and Human-Attention Console (#648).""" + +from __future__ import annotations + +import pytest +from starlette.testclient import TestClient + +from webui.app import create_app +from webui.notifications import ( + ATTENTION_HUMAN_REQUIRED, + ATTENTION_OPERATOR, + ATTENTION_ROUTINE, + CATEGORY_AUTH, + CATEGORY_BLOCKER, + CATEGORY_LEASE, + CATEGORY_SYSTEM, + CATEGORY_VALIDATION, + CATEGORY_WORKFLOW, + NotificationItem, + NotificationSnapshot, + classify_attention_event, + load_notifications_snapshot, + snapshot_to_dict, +) +from webui.notification_views import render_notifications_page +from webui.project_registry import load_registry +from webui.queue_loader import QueueItem, QueueSnapshot +from webui.lease_loader import CollisionWarning, LeaseSnapshot +from webui.system_health import DependencyProbe, SystemHealthSnapshot, VersionInfo, StaleRuntime + + +def test_classify_attention_event_rules(): + # 1. Critical escalation boundaries -> human-required + att_cls, req_human = classify_attention_event( + CATEGORY_AUTH, "Auth error", "Unauthorized access attempt", is_auth_failure=True + ) + assert att_cls == ATTENTION_HUMAN_REQUIRED + assert req_human is True + + att_cls, req_human = classify_attention_event( + CATEGORY_SYSTEM, "Hard stop", "Hard stop triggered", is_hard_stop=True + ) + assert att_cls == ATTENTION_HUMAN_REQUIRED + assert req_human is True + + att_cls, req_human = classify_attention_event( + CATEGORY_VALIDATION, "Validation Error", "Report validation failed", is_validation_failure=True + ) + assert att_cls == ATTENTION_HUMAN_REQUIRED + assert req_human is True + + # 2. Operational issues -> operator + att_cls, req_human = classify_attention_event( + CATEGORY_BLOCKER, "PR Blocked", "Merge conflict detected", is_blocker=True + ) + assert att_cls == ATTENTION_OPERATOR + assert req_human is False + + att_cls, req_human = classify_attention_event( + CATEGORY_LEASE, "Lease Expired", "Session lease expired", is_stale=True + ) + assert att_cls == ATTENTION_OPERATOR + assert req_human is False + + # 3. Routine workflow transitions -> routine + att_cls, req_human = classify_attention_event( + CATEGORY_WORKFLOW, "PR Active", "PR in review" + ) + assert att_cls == ATTENTION_ROUTINE + assert req_human is False + + +def test_notification_snapshot_aggregation(): + reg = load_registry() + proj_id = reg.projects[0].id if reg.projects else "gitea-tools" + + mock_queue = QueueSnapshot( + project_id=proj_id, + repo_label="org/repo", + prs=( + QueueItem( + number=101, + title="Blocked PR", + badges=("blocked",), + extra={}, + ), + QueueItem( + number=102, + title="Normal PR", + badges=("in-review",), + extra={}, + ), + ), + issues=(), + pr_pagination=None, + issue_pagination=None, + ) + + mock_leases = LeaseSnapshot( + project_id=proj_id, + repo_label="org/repo", + issue_lock=None, + claim_inventory={}, + reviewer_leases=( + { + "pr_number": 101, + "status": "expired", + "is_expired": True, + }, + ), + duplicate_prs=( + CollisionWarning( + kind="duplicate_pr", + message="Multiple open PRs for issue #101", + issue_number=101, + pr_numbers=(101, 103), + ), + ), + duplicate_branches=(), + collision_history=(), + fetch_error=None, + ) + + mock_version = VersionInfo( + git_sha="abc1234", + git_describe="v1.0.0", + control_plane_schema_version=1, + python_version="3.11", + known=True, + ) + + mock_stale = StaleRuntime( + daemon_head="abc1234", + checkout_head="abc1234", + remote_head="abc1234", + stale=False, + determinable=True, + mutation_safe=True, + reasons=(), + ) + + mock_health = SystemHealthSnapshot( + status="degraded", + ready=False, + readiness_complete=True, + readiness_reasons=("Auth failure",), + service="webui", + mode="test", + version=mock_version, + started_at="2026-07-25T00:00:00Z", + uptime_seconds=100.0, + timestamp="2026-07-25T00:00:00Z", + deep_probes_requested=True, + dependencies=( + DependencyProbe( + name="auth_service", + kind="auth", + status="unauthorized", + detail="Token expired", + required=True, + ), + ), + mcp_namespaces=(), + stale_runtime=mock_stale, + probe_errors=(), + ) + + snapshot = load_notifications_snapshot( + proj_id, + load_queue=lambda _id: mock_queue, + load_leases=lambda **_kwargs: mock_leases, + load_health=lambda **_kwargs: mock_health, + ) + + assert snapshot.project_id == proj_id + assert snapshot.total_count == 5 + assert snapshot.human_required_count >= 1 # auth probe failure + assert snapshot.operator_count >= 3 # blocked PR + expired lease + duplicate PR collision + assert snapshot.routine_count >= 1 # normal PR + + # Inbox items should include operator and human-required items only + inbox_classes = {item.attention_class for item in snapshot.inbox_items} + assert ATTENTION_ROUTINE not in inbox_classes + assert ATTENTION_OPERATOR in inbox_classes + assert ATTENTION_HUMAN_REQUIRED in inbox_classes + + +def test_snapshot_to_dict_and_redaction(): + item = NotificationItem( + id="notif-1", + attention_class=ATTENTION_HUMAN_REQUIRED, + category=CATEGORY_AUTH, + title="Auth Error", + summary="Failed auth header: Bearer secret_token_12345", + work_kind="system", + work_number=None, + project_id="test-proj", + repo_label="org/repo", + created_at="2026-07-25T16:00:00Z", + requires_human=True, + ) + snap = NotificationSnapshot( + project_id="test-proj", + repo_label="org/repo", + items=(item,), + human_required_count=1, + operator_count=0, + routine_count=0, + total_count=1, + ) + + data = snapshot_to_dict(snap) + assert data["project_id"] == "test-proj" + assert data["human_required_count"] == 1 + assert len(data["inbox_items"]) == 1 + + # Redaction test + summary = data["inbox_items"][0]["summary"] + assert "secret_token_12345" not in summary + assert "" in summary or "Bearer" in summary + + +def test_notifications_html_views(): + item = NotificationItem( + id="notif-1", + attention_class=ATTENTION_HUMAN_REQUIRED, + category=CATEGORY_AUTH, + title="Critical Auth Failure", + summary="Auth failure details", + work_kind="issue", + work_number=42, + project_id="test-proj", + repo_label="org/repo", + created_at="2026-07-25T16:00:00Z", + requires_human=True, + ) + snap = NotificationSnapshot( + project_id="test-proj", + repo_label="org/repo", + items=(item,), + human_required_count=1, + operator_count=0, + routine_count=0, + total_count=1, + ) + + html = render_notifications_page(snap, filter_class="inbox") + assert "Notifications & Attention Inbox" in html or "Notifications & Attention Inbox" in html + assert "Critical Auth Failure" in html + assert "HUMAN REQUIRED" in html + assert "Human Required" in html + + +def test_notifications_app_routes(): + app = create_app() + client = TestClient(app) + + # 1. HTML Route + res = client.get("/notifications") + assert res.status_code == 200 + assert "Notifications" in res.text + assert "Attention Inbox" in res.text + + # 2. API Route /api/v1/notifications + res_api = client.get("/api/v1/notifications") + assert res_api.status_code == 200 + json_data = res_api.json() + assert "human_required_count" in json_data + assert "operator_count" in json_data + assert "routine_count" in json_data + assert "inbox_items" in json_data + + # 3. Compatibility Alias /api/notifications + res_alias = client.get("/api/notifications") + assert res_alias.status_code == 200 + assert res_alias.json()["project_id"] == json_data["project_id"] diff --git a/webui/app.py b/webui/app.py index 560ec75..3840874 100644 --- a/webui/app.py +++ b/webui/app.py @@ -72,6 +72,11 @@ from webui.system_health import ( snapshot_to_dict as system_health_to_dict, ) from webui.system_health_views import render_system_health_page +from webui.notifications import ( + load_notifications_snapshot, + snapshot_to_dict as notifications_snapshot_to_dict, +) +from webui.notification_views import render_notifications_page _READ_ONLY_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) _AUDIT_MUTATION_PATHS = frozenset({"/audit", "/api/audit"}) @@ -739,6 +744,23 @@ async def api_v1_analytics_ingest(request: Request) -> JSONResponse: ) +async def notifications_route(request: Request) -> HTMLResponse: + project_id = request.query_params.get("project_id") + attention_class = request.query_params.get("attention_class") or "inbox" + snap = load_notifications_snapshot(project_id) + html = render_notifications_page( + snap, filter_class=attention_class, filter_project=project_id + ) + return HTMLResponse(html) + + +async def api_notifications(request: Request) -> JSONResponse: + project_id = request.query_params.get("project_id") + snap = load_notifications_snapshot(project_id) + data = notifications_snapshot_to_dict(snap) + return JSONResponse(data) + + async def method_not_allowed(request: Request, _exc: Exception) -> Response: path = request.url.path if path in _AUDIT_MUTATION_PATHS and request.method == "POST": @@ -767,6 +789,9 @@ def create_app(*, bind_host: str | None = None) -> Starlette: Route("/api/queue", api_queue, methods=["GET"]), Route("/traffic", traffic, methods=["GET"]), Route("/api/traffic", api_traffic, methods=["GET"]), + Route("/notifications", notifications_route, methods=["GET"]), + Route("/api/notifications", api_notifications, methods=["GET"]), + Route("/api/v1/notifications", api_notifications, methods=["GET"]), Route("/projects", projects, methods=["GET"]), Route("/projects/{project_id}", project_detail, methods=["GET"]), Route("/api/projects", api_projects, methods=["GET"]), diff --git a/webui/nav.py b/webui/nav.py index da9f763..b9165e9 100644 --- a/webui/nav.py +++ b/webui/nav.py @@ -45,6 +45,7 @@ NAV_GROUPS: tuple[NavGroup, ...] = ( NavItem("/queue", "Queue"), NavItem("/leases", "Leases"), NavItem("/actions", "Actions"), + NavItem("/notifications", "Notifications"), )), NavGroup("Runtime/Sessions", ( NavItem("/runtime", "Runtime health"), diff --git a/webui/notification_views.py b/webui/notification_views.py new file mode 100644 index 0000000..e4b6ab9 --- /dev/null +++ b/webui/notification_views.py @@ -0,0 +1,158 @@ +"""HTML rendering for Phase 3 Notifications and Human-Attention Console (#648).""" + +from __future__ import annotations + +from html import escape +from typing import Sequence + +from webui.layout import render_page +from webui.notifications import ( + ATTENTION_HUMAN_REQUIRED, + ATTENTION_OPERATOR, + ATTENTION_ROUTINE, + NotificationItem, + NotificationSnapshot, +) + + +def _render_attention_badge(attention_class: str) -> str: + cls = "badge" + if attention_class == ATTENTION_HUMAN_REQUIRED: + cls += " badge-blocked" + elif attention_class == ATTENTION_OPERATOR: + cls += " badge-claimed" + else: + cls += " muted" + return f'{escape(attention_class)}' + + +def _render_notification_row(item: NotificationItem) -> str: + category_label = escape(item.category.upper()) + id_str = escape(item.id) + title_str = escape(item.title) + summary_str = escape(item.summary) + att_badge = _render_attention_badge(item.attention_class) + + work_item_html = "—" + if item.work_number and item.work_kind: + kind_label = escape(item.work_kind.upper()) + num_str = f"#{item.work_number}" + link = item.deep_link or "#" + work_item_html = f'{kind_label} {num_str}' + + requires_human_label = ( + 'HUMAN REQUIRED' + if item.requires_human + else "" + ) + + return f""" + {category_label}
    {id_str} + +
    {title_str} {att_badge} {requires_human_label}
    +
    {summary_str}
    + + {work_item_html} + {escape(item.created_at[:19])} +""" + + +def _render_notifications_table(items: Sequence[NotificationItem], empty_message: str) -> str: + if not items: + return f'

    {escape(empty_message)}

    ' + + rows = "".join(_render_notification_row(item) for item in items) + return f""" + + + + + + + + + + {rows} + +
    Category & IDTitle & Attention SummaryWork ItemTime
    """ + + +def render_notifications_page( + snapshot: NotificationSnapshot, + *, + filter_class: str = "inbox", + filter_project: str | None = None, +) -> str: + """Render the notifications and attention inbox page.""" + title = "Notifications & Attention Inbox" + + err_html = "" + if snapshot.fetch_error: + err_html = f'

    Fetch Warning: {escape(snapshot.fetch_error)}

    ' + + # Determine items to render based on filter_class + if filter_class == ATTENTION_HUMAN_REQUIRED: + display_items = snapshot.human_required_items + active_tab_title = "Human-Required Escalations" + elif filter_class == ATTENTION_OPERATOR: + display_items = snapshot.operator_items + active_tab_title = "Operator Inbox Items" + elif filter_class == ATTENTION_ROUTINE: + display_items = snapshot.routine_items + active_tab_title = "Routine Workflow Transitions" + elif filter_class == "all": + display_items = snapshot.items + active_tab_title = "All Events (including Routine)" + else: # "inbox" default + display_items = snapshot.inbox_items + active_tab_title = "Attention Inbox (Human + Operator)" + + hr_cls = "badge-blocked" if snapshot.human_required_count > 0 else "muted" + op_cls = "badge-claimed" if snapshot.operator_count > 0 else "muted" + + metrics_html = f"""
    +
    + Human Required +

    {snapshot.human_required_count}

    +

    Critical escalation boundary

    +
    +
    + Operator Inbox +

    {snapshot.operator_count}

    +

    Operational items needing review

    +
    +
    + Routine Transitions +

    {snapshot.routine_count}

    +

    Background transitions (filtered)

    +
    +
    """ + + # Filter navigation links + def _tab_link(target_class: str, label: str) -> str: + is_active = (filter_class == target_class) + style = "font-weight:bold; border-bottom:2px solid currentColor;" if is_active else "color:#4a5568;" + return f'{label}' + + tabs_html = f"""
    + {_tab_link("inbox", f"Attention Inbox ({snapshot.human_required_count + snapshot.operator_count})")} + {_tab_link("human-required", f"Human Required ({snapshot.human_required_count})")} + {_tab_link("operator", f"Operator ({snapshot.operator_count})")} + {_tab_link("routine", f"Routine ({snapshot.routine_count})")} + {_tab_link("all", f"All Events ({snapshot.total_count})")} +
    """ + + table_html = _render_notifications_table( + display_items, + f"No items match attention filter '{filter_class}'.", + ) + + body = f"""

    {escape(title)}

    +

    Phase 3 console surface for human-attention routing (#648). Routine workflow transitions are filtered by default to eliminate notification fatigue.

    +{err_html} +{metrics_html} +{tabs_html} +

    {escape(active_tab_title)}

    +{table_html}""" + + return render_page(title=title, body_html=body) diff --git a/webui/notifications.py b/webui/notifications.py new file mode 100644 index 0000000..15753b9 --- /dev/null +++ b/webui/notifications.py @@ -0,0 +1,480 @@ +"""Notifications and human-attention routing module for Phase 3 web console (#648). + +Defines attention classes, event classification rules, and inbox aggregation so +operators receive direct alerts only for human-required escalation boundaries +(#628) while routine workflow transitions remain available for pull-based review. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Callable, Sequence + +from webui import console_redaction +from webui.project_registry import find_project, load_registry +from webui.queue_loader import QueueSnapshot, load_queue_snapshot +from webui.lease_loader import LeaseSnapshot, load_lease_snapshot +from webui.system_health import SystemHealthSnapshot, load_system_health + +# Attention class definitions (#628, #648) +ATTENTION_ROUTINE = "routine" +ATTENTION_OPERATOR = "operator" +ATTENTION_HUMAN_REQUIRED = "human-required" + +ATTENTION_CLASSES = ( + ATTENTION_ROUTINE, + ATTENTION_OPERATOR, + ATTENTION_HUMAN_REQUIRED, +) + +# Notification categories +CATEGORY_AUTH = "auth" +CATEGORY_BLOCKER = "blocker" +CATEGORY_LEASE = "lease" +CATEGORY_VALIDATION = "validation" +CATEGORY_WORKFLOW = "workflow" +CATEGORY_SYSTEM = "system" + +CATEGORIES = ( + CATEGORY_AUTH, + CATEGORY_BLOCKER, + CATEGORY_LEASE, + CATEGORY_VALIDATION, + CATEGORY_WORKFLOW, + CATEGORY_SYSTEM, +) + + +@dataclass(frozen=True) +class NotificationItem: + """A single notification or inbox event.""" + + id: str + attention_class: str # "routine", "operator", "human-required" + category: str # "auth", "blocker", "lease", "validation", etc. + title: str + summary: str + work_kind: str | None # "issue", "pr", "session", "system" + work_number: int | None + project_id: str + repo_label: str + created_at: str + deep_link: str | None = None + requires_human: bool = False + extra: dict[str, Any] = field(default_factory=dict) + + def as_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "attention_class": self.attention_class, + "category": self.category, + "title": self.title, + "summary": console_redaction.redact_text(self.summary), + "work_kind": self.work_kind, + "work_number": self.work_number, + "project_id": self.project_id, + "repo_label": self.repo_label, + "created_at": self.created_at, + "deep_link": self.deep_link, + "requires_human": self.requires_human, + "extra": self.extra, + } + + +@dataclass(frozen=True) +class NotificationSnapshot: + """Snapshot of notifications and attention inbox state.""" + + project_id: str + repo_label: str + items: tuple[NotificationItem, ...] + human_required_count: int + operator_count: int + routine_count: int + total_count: int + fetch_error: str | None = None + + @property + def inbox_items(self) -> tuple[NotificationItem, ...]: + """Items requiring operator or human attention (excluding routine).""" + return tuple( + item + for item in self.items + if item.attention_class in {ATTENTION_OPERATOR, ATTENTION_HUMAN_REQUIRED} + ) + + @property + def human_required_items(self) -> tuple[NotificationItem, ...]: + return tuple( + item for item in self.items if item.attention_class == ATTENTION_HUMAN_REQUIRED + ) + + @property + def operator_items(self) -> tuple[NotificationItem, ...]: + return tuple( + item for item in self.items if item.attention_class == ATTENTION_OPERATOR + ) + + @property + def routine_items(self) -> tuple[NotificationItem, ...]: + return tuple( + item for item in self.items if item.attention_class == ATTENTION_ROUTINE + ) + + def as_dict(self) -> dict[str, Any]: + return { + "project_id": self.project_id, + "repo_label": self.repo_label, + "human_required_count": self.human_required_count, + "operator_count": self.operator_count, + "routine_count": self.routine_count, + "total_count": self.total_count, + "fetch_error": self.fetch_error, + "inbox_items": [item.as_dict() for item in self.inbox_items], + "all_items": [item.as_dict() for item in self.items], + } + + +def classify_attention_event( + category: str, + title: str, + summary: str, + *, + is_hard_stop: bool = False, + is_auth_failure: bool = False, + is_irrecoverable: bool = False, + is_decision_lock: bool = False, + is_validation_failure: bool = False, + is_stale: bool = False, + is_blocker: bool = False, +) -> tuple[str, bool]: + """Classify an event into an attention class and human requirement flag. + + Rules (#628, #648): + 1. Critical boundaries (hard stop, auth failure, irrecoverable state, + decision lock, validation failure) -> ATTENTION_HUMAN_REQUIRED (requires_human=True). + 2. Operational queues (blocker, stale lease, unassigned ready work, queue collision) + -> ATTENTION_OPERATOR (requires_human=False). + 3. Routine state transitions (clean progression, healthy heartbeats) -> ATTENTION_ROUTINE (requires_human=False). + """ + if ( + is_hard_stop + or is_auth_failure + or is_irrecoverable + or is_decision_lock + or is_validation_failure + or category in {CATEGORY_AUTH, CATEGORY_VALIDATION} + or "hard stop" in summary.lower() + or "unauthorized" in summary.lower() + or "irrecoverable" in summary.lower() + ): + return ATTENTION_HUMAN_REQUIRED, True + + if is_stale or is_blocker or category in {CATEGORY_BLOCKER, CATEGORY_LEASE}: + return ATTENTION_OPERATOR, False + + return ATTENTION_ROUTINE, False + + +def load_notifications_snapshot( + project_id: str | None = None, + *, + load_queue: Callable[..., QueueSnapshot] | None = None, + load_leases: Callable[..., LeaseSnapshot] | None = None, + load_health: Callable[..., SystemHealthSnapshot] | None = None, +) -> NotificationSnapshot: + """Load and classify attention notifications across queue, leases, and system health.""" + registry = load_registry() + project = None + if project_id: + for entry in registry.projects: + if entry.id == project_id: + project = entry + break + else: + project = registry.projects[0] if registry.projects else None + + if project is None: + return NotificationSnapshot( + project_id=project_id or "", + repo_label="", + items=(), + human_required_count=0, + operator_count=0, + routine_count=0, + total_count=0, + fetch_error="project not found in registry", + ) + + queue_loader_fn = load_queue or load_queue_snapshot + lease_loader_fn = load_leases or load_lease_snapshot + health_loader_fn = load_health or load_system_health + + try: + queue_snap = queue_loader_fn(project.id) + except TypeError: + queue_snap = queue_loader_fn(project_id=project.id) + + try: + lease_snap = lease_loader_fn(project_id=project.id) + except TypeError: + lease_snap = lease_loader_fn(project.id) + + try: + health_snap = health_loader_fn(project_id=project.id) + except TypeError: + try: + health_snap = health_loader_fn(project.id) + except TypeError: + health_snap = health_loader_fn() + + items: list[NotificationItem] = [] + now_iso = datetime.now(timezone.utc).isoformat() + + # 1. System health alerts (highest priority) + for probe_err in getattr(health_snap, "probe_errors", ()): + att_cls, req_human = classify_attention_event( + CATEGORY_SYSTEM, + "System Health Probe Error", + probe_err, + is_blocker=True, + ) + items.append( + NotificationItem( + id=f"notif-sys-err-{project.id}", + attention_class=att_cls, + category=CATEGORY_SYSTEM, + title="System Health Error", + summary=f"System health error: {probe_err}", + work_kind="system", + work_number=None, + project_id=project.id, + repo_label=f"{project.gitea_owner}/{project.repo_name}", + created_at=now_iso, + deep_link="/system", + requires_human=req_human, + ) + ) + + for probe in getattr(health_snap, "dependencies", ()): + if probe.status not in ("ok", "healthy"): + att_cls, req_human = classify_attention_event( + CATEGORY_SYSTEM, + f"Probe Failure: {probe.name}", + probe.detail or probe.status, + is_hard_stop=("stop" in probe.status or "fatal" in probe.status), + is_auth_failure=("auth" in probe.name.lower() or "unauthorized" in probe.status.lower()), + is_blocker=True, + ) + items.append( + NotificationItem( + id=f"notif-probe-{probe.name}", + attention_class=att_cls, + category=CATEGORY_AUTH if "auth" in probe.name.lower() else CATEGORY_SYSTEM, + title=f"Health Probe Alert: {probe.name}", + summary=f"Probe '{probe.name}' reported status '{probe.status}': {probe.detail}", + work_kind="system", + work_number=None, + project_id=project.id, + repo_label=f"{project.gitea_owner}/{project.repo_name}", + created_at=now_iso, + deep_link="/system", + requires_human=req_human, + ) + ) + + # 2. Queue items (PRs and Issues) + for pr in queue_snap.prs: + if "blocked" in pr.badges: + att_cls, req_human = classify_attention_event( + CATEGORY_BLOCKER, + f"PR #{pr.number} Blocked", + f"PR #{pr.number} '{pr.title}' is blocked or has merge conflicts.", + is_blocker=True, + ) + items.append( + NotificationItem( + id=f"notif-pr-block-{pr.number}", + attention_class=att_cls, + category=CATEGORY_BLOCKER, + title=f"Blocked PR #{pr.number}", + summary=f"PR #{pr.number} ({pr.title}) requires merge conflict resolution.", + work_kind="pr", + work_number=pr.number, + project_id=project.id, + repo_label=f"{project.gitea_owner}/{project.repo_name}", + created_at=now_iso, + deep_link=f"/traffic", + requires_human=req_human, + ) + ) + elif "stale" in pr.badges: + att_cls, req_human = classify_attention_event( + CATEGORY_WORKFLOW, + f"PR #{pr.number} Stale", + f"PR #{pr.number} '{pr.title}' has had no activity for over 14 days.", + is_stale=True, + ) + items.append( + NotificationItem( + id=f"notif-pr-stale-{pr.number}", + attention_class=att_cls, + category=CATEGORY_WORKFLOW, + title=f"Stale PR #{pr.number}", + summary=f"PR #{pr.number} ({pr.title}) is stale.", + work_kind="pr", + work_number=pr.number, + project_id=project.id, + repo_label=f"{project.gitea_owner}/{project.repo_name}", + created_at=now_iso, + deep_link=f"/queue", + requires_human=req_human, + ) + ) + else: + # Routine PR transition + att_cls, req_human = classify_attention_event( + CATEGORY_WORKFLOW, + f"PR #{pr.number} Active", + f"PR #{pr.number} '{pr.title}' is in routine state {', '.join(pr.badges)}.", + ) + items.append( + NotificationItem( + id=f"notif-pr-routine-{pr.number}", + attention_class=att_cls, + category=CATEGORY_WORKFLOW, + title=f"Routine PR #{pr.number}", + summary=f"PR #{pr.number} ({pr.title}) state: {', '.join(pr.badges)}.", + work_kind="pr", + work_number=pr.number, + project_id=project.id, + repo_label=f"{project.gitea_owner}/{project.repo_name}", + created_at=now_iso, + deep_link=f"/queue", + requires_human=req_human, + ) + ) + + for issue in queue_snap.issues: + if "duplicate" in issue.badges: + att_cls, req_human = classify_attention_event( + CATEGORY_BLOCKER, + f"Issue #{issue.number} Duplicate PRs", + f"Issue #{issue.number} has multiple linked PRs.", + is_blocker=True, + ) + items.append( + NotificationItem( + id=f"notif-issue-dup-{issue.number}", + attention_class=att_cls, + category=CATEGORY_BLOCKER, + title=f"Duplicate PRs on Issue #{issue.number}", + summary=f"Issue #{issue.number} ({issue.title}) linked to multiple PRs.", + work_kind="issue", + work_number=issue.number, + project_id=project.id, + repo_label=f"{project.gitea_owner}/{project.repo_name}", + created_at=now_iso, + deep_link=f"/traffic", + requires_human=req_human, + ) + ) + elif "claimed" in issue.badges or "in-review" in issue.badges: + att_cls, req_human = classify_attention_event( + CATEGORY_WORKFLOW, + f"Issue #{issue.number} Active", + f"Issue #{issue.number} '{issue.title}' in state {', '.join(issue.badges)}.", + ) + items.append( + NotificationItem( + id=f"notif-issue-routine-{issue.number}", + attention_class=att_cls, + category=CATEGORY_WORKFLOW, + title=f"Routine Issue #{issue.number}", + summary=f"Issue #{issue.number} ({issue.title}) state: {', '.join(issue.badges)}.", + work_kind="issue", + work_number=issue.number, + project_id=project.id, + repo_label=f"{project.gitea_owner}/{project.repo_name}", + created_at=now_iso, + deep_link=f"/queue", + requires_human=req_human, + ) + ) + + # 3. Leases / Collisions + for lease in lease_snap.reviewer_leases: + if lease.get("is_expired") or lease.get("status") == "expired": + pr_num = lease.get("pr_number") or lease.get("work_item_number") + att_cls, req_human = classify_attention_event( + CATEGORY_LEASE, + f"Reviewer Lease Expired for PR #{pr_num}", + f"Reviewer lease for PR #{pr_num} has expired.", + is_stale=True, + ) + items.append( + NotificationItem( + id=f"notif-lease-exp-pr-{pr_num}", + attention_class=att_cls, + category=CATEGORY_LEASE, + title=f"Expired Reviewer Lease (PR #{pr_num})", + summary=f"Reviewer lease for PR #{pr_num} expired.", + work_kind="pr", + work_number=pr_num, + project_id=project.id, + repo_label=f"{project.gitea_owner}/{project.repo_name}", + created_at=now_iso, + deep_link="/leases", + requires_human=req_human, + ) + ) + + for collision in lease_snap.duplicate_prs: + att_cls, req_human = classify_attention_event( + CATEGORY_BLOCKER, + f"Duplicate PR Collision ({collision.kind})", + collision.message, + is_blocker=True, + ) + items.append( + NotificationItem( + id=f"notif-collision-{collision.issue_number or 0}", + attention_class=att_cls, + category=CATEGORY_BLOCKER, + title=f"Collision Alert ({collision.kind})", + summary=collision.message, + work_kind="issue" if collision.issue_number else "pr", + work_number=collision.issue_number, + project_id=project.id, + repo_label=f"{project.gitea_owner}/{project.repo_name}", + created_at=now_iso, + deep_link="/leases", + requires_human=req_human, + ) + ) + + human_req_count = sum(1 for i in items if i.attention_class == ATTENTION_HUMAN_REQUIRED) + operator_count = sum(1 for i in items if i.attention_class == ATTENTION_OPERATOR) + routine_count = sum(1 for i in items if i.attention_class == ATTENTION_ROUTINE) + + fetch_err = queue_snap.fetch_error or lease_snap.fetch_error or getattr(health_snap, "probe_errors", None) + if isinstance(fetch_err, (tuple, list)): + fetch_err = "; ".join(fetch_err) if fetch_err else None + + return NotificationSnapshot( + project_id=project.id, + repo_label=f"{project.gitea_owner}/{project.repo_name}", + items=tuple(items), + human_required_count=human_req_count, + operator_count=operator_count, + routine_count=routine_count, + total_count=len(items), + fetch_error=fetch_err, + ) + + +def snapshot_to_dict(snapshot: NotificationSnapshot) -> dict[str, Any]: + """JSON-serializable export for /api/v1/notifications.""" + return snapshot.as_dict() From 9a0154347771d1ca06202a8dfae52d9d57b77dd4 Mon Sep 17 00:00:00 2001 From: Jason Walker <913443@dadeschools.net> Date: Sat, 25 Jul 2026 17:23:52 -0400 Subject: [PATCH 3/9] feat(webui): add read-only console restart status and impact controls (Closes #667) --- tests/test_webui_restart_console.py | 452 ++++++++++++++++++++++ webui/app.py | 37 ++ webui/nav.py | 1 + webui/restart_console.py | 579 ++++++++++++++++++++++++++++ webui/restart_views.py | 299 ++++++++++++++ 5 files changed, 1368 insertions(+) create mode 100644 tests/test_webui_restart_console.py create mode 100644 webui/restart_console.py create mode 100644 webui/restart_views.py diff --git a/tests/test_webui_restart_console.py b/tests/test_webui_restart_console.py new file mode 100644 index 0000000..439bba5 --- /dev/null +++ b/tests/test_webui_restart_console.py @@ -0,0 +1,452 @@ +"""Read-only restart console: views, gates, and honesty rules (#667). + +The console consumes the #655 substrate. These tests hold it to the three +properties that make a status surface trustworthy: + +* an unreadable source is reported unavailable, never rendered as green; +* authorization is probed the way execution would probe it, so an allow is + never shown for something that could not run; +* the surface performs no mutation, including no write to the control-plane DB. +""" + +from __future__ import annotations + +import os +import sqlite3 +import sys +import tempfile +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from starlette.testclient import TestClient # noqa: E402 + +import restart_coordinator # noqa: E402 +from webui import console_authz, restart_console, restart_views # noqa: E402 +from webui.app import create_app # noqa: E402 + +NOW = datetime(2026, 7, 25, 21, 0, 0, tzinfo=timezone.utc) + + +def _principal(role: str) -> console_authz.Principal: + return console_authz.Principal( + subject="operator@example.com", + role=role, + identity_source=console_authz.IDENTITY_LOCAL_DEV, + authenticated=True, + ) + + +def _inventory(*, complete: bool = True, sessions=(), leases=()): + def _read(**_kwargs): + return { + "sessions": list(sessions), + "leases": list(leases), + "terminal_lock": None, + "prior_recovery_attempts": [], + "inventory_complete": complete, + "incomplete_reasons": ( + [] if complete else ["fixture: inventory withheld"] + ), + } + + return _read + + +def _live_session(session_id: str = "prgs-author-1234-abcd") -> dict: + return { + "session_id": session_id, + "role": "author", + "profile": "prgs-author", + "pid": os.getpid(), + "status": "active", + "last_heartbeat_at": (NOW - timedelta(seconds=30)).isoformat(), + } + + +def drain_proof_fixture() -> dict: + """A structurally complete but unsigned drain proof.""" + return { + "version": "drain-proof/v1", + "proof_id": "deadbeef" * 8, + "clean": True, + "issued_at": (NOW - timedelta(minutes=1)).isoformat(), + "expires_at": (NOW + timedelta(minutes=5)).isoformat(), + "requesting_session_id": "s-live", + "impact_fingerprint": "f" * 64, + "checks": [], + "failed_checks": [], + } + + +class RestartClassMatrixTest(unittest.TestCase): + def test_every_policy_class_is_rendered(self) -> None: + views = restart_console.build_restart_class_views("operator") + self.assertEqual(len(views), len(restart_coordinator.RESTART_CLASS_POLICIES)) + + def test_viewer_capability_is_role_scoped_not_generic(self) -> None: + """A worker role must not be shown as able to request a full restart.""" + author = { + v.restart_class: v + for v in restart_console.build_restart_class_views("author") + } + operator = { + v.restart_class: v + for v in restart_console.build_restart_class_views("operator") + } + full = restart_coordinator.RestartClass.FULL_MCP_RESTART.value + + self.assertFalse(author[full].viewer_may_request) + self.assertFalse(author[full].viewer_may_execute) + self.assertTrue(operator[full].viewer_may_request) + self.assertTrue(operator[full].viewer_may_execute) + + def test_unknown_role_may_do_nothing(self) -> None: + views = restart_console.build_restart_class_views("not-a-role") + self.assertTrue(all(not v.viewer_may_request for v in views)) + self.assertTrue(all(not v.viewer_may_execute for v in views)) + + +class AuthorizationProbeTest(unittest.TestCase): + def test_probe_asks_for_execution_so_phase_gate_is_reported(self) -> None: + """An admin clears the role bar and still cannot execute in Phase 1. + + This is the case that distinguishes the two probes. Asked without + ``for_execution`` an admin is *allowed* for ``system.restart_namespace``, + which on a control surface reads as a live button. Asked the way + execution asks, the same principal is refused ``phase_not_active``. The + console must report the second answer. + """ + by_id = { + a.action_id: a + for a in restart_console.build_action_authorizations( + _principal(console_authz.ADMIN) + ) + } + restart = by_id["system.restart_namespace"] + + self.assertFalse(restart.execution_enabled) + self.assertEqual(restart.reason_code, console_authz.DENY_PHASE_NOT_ACTIVE) + + permissive = console_authz.authorize( + "system.restart_namespace", _principal(console_authz.ADMIN) + ) + self.assertTrue( + permissive.allowed, + "guard precondition: without for_execution an admin is allowed, " + "which is exactly why the console must not probe that way", + ) + + def test_operator_is_refused_the_admin_only_restart_action(self) -> None: + """Role refusal precedes the phase gate and is reported as such.""" + by_id = { + a.action_id: a + for a in restart_console.build_action_authorizations( + _principal(console_authz.OPERATOR) + ) + } + self.assertEqual( + by_id["system.restart_namespace"].reason_code, + console_authz.DENY_INSUFFICIENT_ROLE, + ) + + def test_anonymous_is_denied_unauthenticated(self) -> None: + by_id = { + a.action_id: a for a in restart_console.build_action_authorizations(None) + } + self.assertEqual( + by_id["system.restart_namespace"].reason_code, + console_authz.DENY_UNAUTHENTICATED, + ) + + def test_no_authorization_ever_reports_execution_enabled(self) -> None: + for role in ( + console_authz.VIEWER, + console_authz.OPERATOR, + console_authz.CONTROLLER, + console_authz.ADMIN, + ): + for auth in restart_console.build_action_authorizations(_principal(role)): + self.assertFalse( + auth.execution_enabled, + f"{role} reported execution_enabled for {auth.action_id}", + ) + + +class ImpactPreviewTest(unittest.TestCase): + def test_impact_renders_from_coordinator_dto(self) -> None: + impact, source = restart_console.load_impact_report( + principal=_principal(console_authz.OPERATOR), + read_inventory=_inventory(sessions=[_live_session()]), + now=NOW, + ) + self.assertTrue(source.available) + self.assertIsNotNone(impact) + self.assertEqual( + impact["restart_class"], + restart_coordinator.RestartClass.FULL_MCP_RESTART.value, + ) + self.assertIn("verdict", impact) + self.assertFalse(impact["restart_performed"]) + self.assertTrue(impact["dry_run"]) + + def test_incomplete_inventory_is_surfaced_and_denies(self) -> None: + impact, source = restart_console.load_impact_report( + principal=_principal(console_authz.OPERATOR), + read_inventory=_inventory(complete=False), + now=NOW, + ) + self.assertFalse(impact["inventory_complete"]) + self.assertFalse(impact["allow_restart"]) + self.assertTrue(source.detail, "incomplete inventory must explain itself") + + def test_inventory_reader_failure_is_unavailable_not_empty(self) -> None: + """A reader that raises must not be rendered as 'no sessions affected'.""" + + def _boom(**_kwargs): + raise RuntimeError("control-plane unreachable") + + impact, source = restart_console.load_impact_report( + principal=_principal(console_authz.OPERATOR), + read_inventory=_boom, + now=NOW, + ) + self.assertIsNone(impact) + self.assertFalse(source.available) + self.assertIn("control-plane unreachable", source.detail) + + +class ControlPlaneReadTest(unittest.TestCase): + def test_missing_database_is_incomplete_not_empty(self) -> None: + inventory = restart_console.read_control_plane_inventory( + db_path="/nonexistent/control-plane.sqlite3" + ) + self.assertFalse(inventory["inventory_complete"]) + self.assertEqual(inventory["sessions"], []) + self.assertTrue(inventory["incomplete_reasons"]) + + def test_reader_never_creates_the_database(self) -> None: + """Reading status must not bring a control-plane DB into existence. + + The path deliberately sits in a directory that already exists: a + read-write ``sqlite3.connect`` would happily create the file there, so + this fails if the reader ever stops opening the database ``mode=ro``. + A nested-missing-directory path would pass for the wrong reason, + because sqlite cannot create the parent directory either way. + """ + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "control_plane.sqlite3") + self.assertTrue(os.path.isdir(os.path.dirname(path))) + + inventory = restart_console.read_control_plane_inventory(db_path=path) + + self.assertFalse( + os.path.exists(path), + "reading restart status created a control-plane database", + ) + self.assertFalse(inventory["inventory_complete"]) + + def test_reads_active_sessions_from_a_real_database(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "cp.sqlite3") + conn = sqlite3.connect(path) + conn.execute( + "CREATE TABLE sessions (session_id TEXT, role TEXT, profile TEXT," + " pid INTEGER, status TEXT, last_heartbeat_at TEXT)" + ) + conn.execute( + "CREATE TABLE work_items (work_item_id INTEGER, kind TEXT," + " number INTEGER)" + ) + conn.execute( + "CREATE TABLE leases (lease_id TEXT, session_id TEXT, role TEXT," + " phase TEXT, status TEXT, worktree_path TEXT," + " work_item_id INTEGER, expires_at TEXT)" + ) + conn.execute( + "INSERT INTO sessions VALUES (?,?,?,?,?,?)", + ("s-live", "author", "prgs-author", 4242, "active", NOW.isoformat()), + ) + conn.execute( + "INSERT INTO sessions VALUES (?,?,?,?,?,?)", + ("s-done", "author", "prgs-author", 11, "closed", NOW.isoformat()), + ) + conn.execute("INSERT INTO work_items VALUES (1, 'issue', 667)") + conn.execute( + "INSERT INTO leases VALUES (?,?,?,?,?,?,?,?)", + ( + "l-1", + "s-live", + "author", + "allocated", + "active", + None, + 1, + NOW.isoformat(), + ), + ) + conn.commit() + conn.close() + + inventory = restart_console.read_control_plane_inventory(db_path=path) + + self.assertTrue(inventory["inventory_complete"]) + self.assertEqual([s["session_id"] for s in inventory["sessions"]], ["s-live"]) + self.assertEqual(inventory["leases"][0]["work_number"], 667) + + +class DrainAndReconcileTest(unittest.TestCase): + def test_absent_drain_proof_is_not_a_pass(self) -> None: + drain, source = restart_console.load_drain_status(proof=None, now=NOW) + self.assertIsNone(drain) + self.assertFalse(source.available) + self.assertIn("denies", source.detail) + + def test_tampered_drain_proof_is_reported_invalid(self) -> None: + proof = drain_proof_fixture() + proof["clean"] = True + proof["proof_id"] = "0" * 64 + drain, source = restart_console.load_drain_status(proof=proof, now=NOW) + self.assertTrue(source.available) + self.assertFalse(drain["valid"]) + + def test_absent_reconcile_proof_is_unavailable(self) -> None: + reconcile, source = restart_console.load_reconcile_status(load_proof=None) + self.assertIsNone(reconcile) + self.assertFalse(source.available) + + def test_reconcile_proof_is_rendered_when_supplied(self) -> None: + payload = { + "overall_status": "degraded", + "mode": "log_only", + "resolved_count": 3, + "unresolved_count": 2, + "items": [ + { + "dimension": "leases", + "status": "unresolved", + "summary": "2 orphaned leases", + "follow_up_required": True, + } + ], + } + reconcile, source = restart_console.load_reconcile_status( + load_proof=lambda: payload + ) + self.assertTrue(source.available) + self.assertEqual(reconcile["unresolved_count"], 2) + + +class RenderingTest(unittest.TestCase): + def _snapshot(self, **kwargs): + params = { + "principal": _principal(console_authz.OPERATOR), + "read_inventory": _inventory(sessions=[_live_session()]), + "now": NOW, + } + params.update(kwargs) + return restart_console.load_restart_console_snapshot(**params) + + def test_page_renders_every_section(self) -> None: + html = restart_views.render_restart_console_page(self._snapshot()) + for heading in ( + "Impact preview", + "Drain proof", + "Post-restart reconcile", + "Restart classes", + "Approval controls", + "Break-glass", + ): + self.assertIn(heading, html) + + def test_hostile_session_id_is_escaped(self) -> None: + hostile = "" + html = restart_views.render_restart_console_page( + self._snapshot(read_inventory=_inventory(sessions=[_live_session(hostile)])) + ) + self.assertNotIn("