From 25bc2a32918d7b068559665022ec2c28a600aff4 Mon Sep 17 00:00:00 2001 From: Jason Walker <913443@dadeschools.net> Date: Thu, 23 Jul 2026 14:48:46 -0400 Subject: [PATCH] feat(webui): workflow-event and conversation timeline model (Closes #637) Phase 1 child of the Web Console epic #631. Adds a durable, versioned WorkflowEvent schema with per-source adapters and a read-only query API so operators can browse a unified timeline of workflow events, decisions, tool calls, and handoffs instead of scattered evidence. - webui/timeline.py (new): versioned WorkflowEvent schema; control-plane event adapter and Gitea CTH handoff-comment adapter; read-only mode=ro control-plane reader; conjunctive filter by issue/PR/session; stable (timestamp, source_rank, event_key) ordering; bounded pagination; fail-soft per-source status; redaction at the boundary, fail closed. - webui/app.py: GET /api/v1/timeline read-only route with thread-scoped, fail-soft handoff comment source. - tests/test_webui_timeline.py (new): schema, adapters, redaction of secret-like payloads, filter/sort/pagination, scoped CP reader, fail-soft composition, and API integration. - docs/webui-local-dev.md: timeline route and field-authority notes. Read-only Phase 1; no mutation of historical events; no full chat replay; no unredacted tool-argument storage. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/webui-local-dev.md | 70 +++++ tests/test_webui_timeline.py | 364 +++++++++++++++++++++++ webui/app.py | 97 +++++++ webui/timeline.py | 539 +++++++++++++++++++++++++++++++++++ 4 files changed, 1070 insertions(+) create mode 100644 tests/test_webui_timeline.py create mode 100644 webui/timeline.py diff --git a/docs/webui-local-dev.md b/docs/webui-local-dev.md index f5799b2..7f24212 100644 --- a/docs/webui-local-dev.md +++ b/docs/webui-local-dev.md @@ -212,6 +212,76 @@ health, workflow/schema SHA-256 hashes, and stale-runtime warnings when the checkout is behind merged safety-gate changes. Restart guidance links to #420; no tokens or MCP restart actions are exposed. +## Workflow-event timeline (#637) + +`GET /api/v1/timeline` is a read-only, versioned aggregation of workflow +events from every available source into one normalised, filterable stream. It +is the model layer for the Phase 1 timeline console view (a later child issue +of #631); this issue ships the schema, adapters, and read API only. + +### Schema (versioned) + +`webui/timeline.py` declares `TIMELINE_SCHEMA_VERSION` (currently `1`) and the +frozen `WorkflowEvent` record. Every response carries `schema_version` so a +consumer can branch on shape. One event: + +```json +{ + "source": "control_plane", + "event_type": "lease.renew", + "event_key": "cp:1421", + "timestamp": "2026-07-23T02:00:00Z", + "actor": null, + "role": null, + "issue_number": 637, + "pr_number": null, + "session_id": null, + "tool_name": null, + "decision": null, + "message": "lease renewed", + "correlation_id": "issue#637", + "evidence_refs": [], + "sensitive": true +} +``` + +`event_key` is stable and unique per source (`cp:`, +`cth:::`), so pagination and dedup are deterministic. + +### Sources and field authority + +| Source | Adapter | Authority | +|---|---|---| +| Control-plane `events` ⋈ `work_items` | `adapt_cp_events` | `event_type`, `message`, `timestamp`, issue/PR scope come from the CP database, read through a `mode=ro` URI (never creates the DB or runs migrations) | +| Gitea Canonical Thread Handoff comments | `adapt_cth_comments` | `actor`, `role` (next owner), `decision`, `evidence_refs`, `timestamp` come from the parsed CTH comment body (`canonical_thread_handoff`) | + +Handoff comments are thread-scoped: they are only read when the request filters +by a single `issue` or `pr`. Otherwise the handoff source reports `not run` +with a reason — it is never rendered as empty-and-healthy. Each source degrades +independently: an unavailable control-plane DB or a failed comment fetch is a +`sources[]` entry with `ok:false` and a `reason`, never a dropped timeline. + +### Query parameters + +`issue`, `pr`, `session` (conjunctive filters); `limit` (default 50, max 500) +and `offset` for pagination; `remote`, `org`, `repo` to override the default +registry-project scope. Events sort ascending by +`(timestamp, source_rank, event_key)`; missing timestamps sort last. + +### Redaction + +Every free-text field (event messages, decision/proof text, roles) is passed +through the console redaction policy (`webui.console_redaction`, backed by +`gitea_audit.redact`) before it leaves the module, failing closed to the +placeholder. No unredacted tool arguments or secrets are ever emitted, and a +generation error never drops raw data to a caller or a log. + +### Tests + +```bash +pytest tests/test_webui_timeline.py -q +``` + ## Tests ```bash diff --git a/tests/test_webui_timeline.py b/tests/test_webui_timeline.py new file mode 100644 index 0000000..35384c0 --- /dev/null +++ b/tests/test_webui_timeline.py @@ -0,0 +1,364 @@ +"""Tests for the workflow-event timeline model and read API (#637). + +Covers the acceptance criteria: versioned schema, adaptation of control-plane +events and Gitea handoff comments, filter by issue/PR/session, redaction of +secret-like payloads, and stable pagination. +""" +import os +import sqlite3 +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from starlette.testclient import TestClient + +import control_plane_db +from canonical_thread_handoff import format_cth_body +from webui import timeline +from webui.app import create_app + + +def _seed_db(path: str) -> None: + """Create a control-plane DB and seed scoped work_items + events.""" + # Constructing ControlPlaneDB runs the schema migration once. + control_plane_db.ControlPlaneDB(db_path=path) + conn = sqlite3.connect(path) + try: + conn.execute( + "INSERT INTO work_items(remote, org, repo, kind, number, state, updated_at) " + "VALUES (?,?,?,?,?,?,?)", + ("prgs", "Scaled-Tech-Consulting", "Gitea-Tools", "issue", 637, "open", "2026-07-23T00:00:00Z"), + ) + issue_wid = conn.execute("SELECT last_insert_rowid()").fetchone()[0] + conn.execute( + "INSERT INTO work_items(remote, org, repo, kind, number, state, updated_at) " + "VALUES (?,?,?,?,?,?,?)", + ("prgs", "Scaled-Tech-Consulting", "Gitea-Tools", "pr", 813, "open", "2026-07-23T00:00:00Z"), + ) + pr_wid = conn.execute("SELECT last_insert_rowid()").fetchone()[0] + # A work item for a different repo — must never appear in prgs/Gitea-Tools scope. + conn.execute( + "INSERT INTO work_items(remote, org, repo, kind, number, state, updated_at) " + "VALUES (?,?,?,?,?,?,?)", + ("dadeschools", "Other", "Elsewhere", "issue", 1, "open", "2026-07-23T00:00:00Z"), + ) + other_wid = conn.execute("SELECT last_insert_rowid()").fetchone()[0] + + rows = [ + (issue_wid, "allocation", "assigned author work", "2026-07-23T01:00:00Z"), + (issue_wid, "lease.renew", "token=ghs_ABCDEF1234567890abcdef lease renewed", "2026-07-23T02:00:00Z"), + (pr_wid, "pr.opened", "PR opened for review", "2026-07-23T03:00:00Z"), + (other_wid, "allocation", "off-scope event", "2026-07-23T04:00:00Z"), + ] + conn.executemany( + "INSERT INTO events(work_item_id, event_type, message, created_at) VALUES (?,?,?,?)", + rows, + ) + conn.commit() + finally: + conn.close() + + +class TestSchema(unittest.TestCase): + def test_schema_is_versioned(self): + self.assertIsInstance(timeline.TIMELINE_SCHEMA_VERSION, int) + self.assertGreaterEqual(timeline.TIMELINE_SCHEMA_VERSION, 1) + + def test_event_to_dict_shape(self): + ev = timeline.WorkflowEvent( + source=timeline.SOURCE_CONTROL_PLANE, + event_type="allocation", + event_key="cp:1", + timestamp="2026-07-23T01:00:00Z", + issue_number=637, + ) + d = ev.to_dict() + for key in ( + "source", "event_type", "event_key", "timestamp", "actor", "role", + "issue_number", "pr_number", "session_id", "tool_name", "decision", + "message", "correlation_id", "evidence_refs", "sensitive", + ): + self.assertIn(key, d) + self.assertEqual(d["evidence_refs"], []) + + +class TestCpAdapter(unittest.TestCase): + def test_issue_and_pr_mapping(self): + rows = [ + {"event_id": 1, "event_type": "allocation", "message": "x", "created_at": "2026-07-23T01:00:00Z", "kind": "issue", "number": 637}, + {"event_id": 2, "event_type": "pr.opened", "message": "y", "created_at": "2026-07-23T02:00:00Z", "kind": "pr", "number": 813}, + ] + events = timeline.adapt_cp_events(rows) + self.assertEqual(len(events), 2) + self.assertEqual(events[0].issue_number, 637) + self.assertIsNone(events[0].pr_number) + self.assertEqual(events[0].correlation_id, "issue#637") + self.assertIsNone(events[1].issue_number) + self.assertEqual(events[1].pr_number, 813) + + def test_malformed_rows_skipped(self): + rows = [ + {"event_id": None, "event_type": "x", "kind": "issue", "number": 1}, + {"event_id": 5, "event_type": "", "kind": "issue", "number": 1}, + {"event_id": 6, "event_type": "ok", "message": "m", "created_at": None, "kind": "issue", "number": 1}, + ] + events = timeline.adapt_cp_events(rows) + self.assertEqual(len(events), 1) + self.assertIsNone(events[0].timestamp) + + def test_sensitive_event_flagged(self): + rows = [{"event_id": 1, "event_type": "lease.renew", "message": "m", "created_at": "2026-07-23T01:00:00Z", "kind": "issue", "number": 1}] + events = timeline.adapt_cp_events(rows) + self.assertTrue(events[0].sensitive) + + +class TestCthAdapter(unittest.TestCase): + def test_cth_comment_becomes_event(self): + body = format_cth_body( + cth_type="Author Handoff", + status="ready", + next_owner="reviewer", + decision="implement timeline", + proof="commit abc1234 closes #637", + next_action="review PR", + ready_to_paste_prompt="Review PR #900 as reviewer", + ) + comments = [{"id": 42, "body": body, "created_at": "2026-07-23T05:00:00Z", "user": {"login": "jcwalker3"}}] + events = timeline.adapt_cth_comments(comments, kind="issue", number=637) + self.assertEqual(len(events), 1) + ev = events[0] + self.assertEqual(ev.source, timeline.SOURCE_GITEA_HANDOFF) + self.assertEqual(ev.event_type, "handoff:Author Handoff") + self.assertEqual(ev.actor, "jcwalker3") + self.assertEqual(ev.issue_number, 637) + self.assertEqual(ev.event_key, "cth:issue:637:42") + self.assertIn("#637", ev.evidence_refs) + self.assertIn("abc1234", ev.evidence_refs) + + def test_non_cth_comment_ignored(self): + comments = [{"id": 1, "body": "just a normal comment", "created_at": "2026-07-23T05:00:00Z", "user": {"login": "x"}}] + self.assertEqual(timeline.adapt_cth_comments(comments, kind="issue", number=1), []) + + +class TestRedaction(unittest.TestCase): + def test_cp_message_redacted(self): + rows = [{"event_id": 1, "event_type": "lease", "message": "token=ghs_ABCDEF1234567890abcdef here", "created_at": "2026-07-23T01:00:00Z", "kind": "issue", "number": 1}] + events = timeline.adapt_cp_events(rows) + self.assertNotIn("ghs_ABCDEF1234567890abcdef", events[0].message or "") + + def test_handoff_decision_redacted(self): + body = format_cth_body( + cth_type="Blocker", + status="blocked", + next_owner="author", + decision="password=SuperSecret123! must rotate", + proof="none", + next_action="rotate", + ready_to_paste_prompt="Rotate the credential and retry", + ) + comments = [{"id": 7, "body": body, "created_at": "2026-07-23T05:00:00Z", "user": {"login": "x"}}] + events = timeline.adapt_cth_comments(comments, kind="issue", number=1) + self.assertNotIn("SuperSecret123!", events[0].decision or "") + + +class TestFilterSortPaginate(unittest.TestCase): + def _events(self): + return [ + timeline.WorkflowEvent(source="control_plane", event_type="a", event_key="cp:3", timestamp="2026-07-23T03:00:00Z", pr_number=813), + timeline.WorkflowEvent(source="control_plane", event_type="b", event_key="cp:1", timestamp="2026-07-23T01:00:00Z", issue_number=637), + timeline.WorkflowEvent(source="control_plane", event_type="c", event_key="cp:2", timestamp="2026-07-23T02:00:00Z", issue_number=637, session_id="sess-1"), + ] + + def test_filter_by_issue(self): + out = timeline.filter_events(self._events(), issue=637) + self.assertEqual({e.event_key for e in out}, {"cp:1", "cp:2"}) + + def test_filter_by_pr(self): + out = timeline.filter_events(self._events(), pr=813) + self.assertEqual([e.event_key for e in out], ["cp:3"]) + + def test_filter_by_session(self): + out = timeline.filter_events(self._events(), session="sess-1") + self.assertEqual([e.event_key for e in out], ["cp:2"]) + + def test_stable_sort_ascending(self): + out = timeline.sort_events(self._events()) + self.assertEqual([e.event_key for e in out], ["cp:1", "cp:2", "cp:3"]) + + def test_missing_timestamp_sorts_last(self): + evs = self._events() + [ + timeline.WorkflowEvent(source="control_plane", event_type="z", event_key="cp:9", timestamp=None) + ] + out = timeline.sort_events(evs) + self.assertEqual(out[-1].event_key, "cp:9") + + def test_pagination_windows_and_next_offset(self): + evs = timeline.sort_events(self._events()) + page1 = timeline.paginate(evs, limit=2, offset=0) + self.assertEqual(len(page1.events), 2) + self.assertEqual(page1.total, 3) + self.assertEqual(page1.next_offset, 2) + page2 = timeline.paginate(evs, limit=2, offset=2) + self.assertEqual(len(page2.events), 1) + self.assertIsNone(page2.next_offset) + + def test_pagination_bounds_coerced(self): + evs = self._events() + page = timeline.paginate(evs, limit=-5, offset=-3) + self.assertGreaterEqual(page.limit, 1) + self.assertEqual(page.offset, 0) + + +class TestCpReader(unittest.TestCase): + def test_reads_scoped_events_only(self): + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + db = os.path.join(tmp, "cp.sqlite3") + _seed_db(db) + events, status = timeline.read_cp_events( + remote="prgs", org="Scaled-Tech-Consulting", repo="Gitea-Tools", db_path=db + ) + self.assertTrue(status.ok) + # 3 scoped events; the dadeschools/Other event is excluded. + self.assertEqual(len(events), 3) + self.assertTrue(all(e.source == "control_plane" for e in events)) + # Redaction applied to the token-bearing message. + joined = " ".join(e.message or "" for e in events) + self.assertNotIn("ghs_ABCDEF1234567890abcdef", joined) + + def test_missing_db_degrades(self): + events, status = timeline.read_cp_events( + remote="prgs", org="Scaled-Tech-Consulting", repo="Gitea-Tools", + db_path="/nonexistent/path/to/cp.sqlite3", + ) + self.assertEqual(events, []) + self.assertFalse(status.ok) + self.assertIsNotNone(status.reason) + + +class TestLoadTimeline(unittest.TestCase): + def test_handoff_not_run_without_thread_filter(self): + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + db = os.path.join(tmp, "cp.sqlite3") + _seed_db(db) + snap = timeline.load_timeline( + remote="prgs", org="Scaled-Tech-Consulting", repo="Gitea-Tools", db_path=db + ) + d = snap.to_dict() + handoff = [s for s in d["sources"] if s["name"] == "gitea_handoff"][0] + self.assertFalse(handoff["ok"]) + self.assertIn("thread-scoped", handoff["reason"]) + self.assertEqual(d["schema_version"], timeline.TIMELINE_SCHEMA_VERSION) + + def test_handoff_included_via_injected_source(self): + import tempfile + + body = format_cth_body( + cth_type="Author Handoff", status="ready", next_owner="reviewer", + decision="d", proof="#637", next_action="review", ready_to_paste_prompt="Review PR #1 now", + ) + + def source(kind, number): + return [{"id": 1, "body": body, "created_at": "2026-07-23T09:00:00Z", "user": {"login": "jcwalker3"}}] + + with tempfile.TemporaryDirectory() as tmp: + db = os.path.join(tmp, "cp.sqlite3") + _seed_db(db) + snap = timeline.load_timeline( + remote="prgs", org="Scaled-Tech-Consulting", repo="Gitea-Tools", + issue=637, db_path=db, comment_source=source, + ) + d = snap.to_dict() + handoff = [s for s in d["sources"] if s["name"] == "gitea_handoff"][0] + self.assertTrue(handoff["ok"]) + self.assertEqual(handoff["count"], 1) + # Both a CP event and the handoff event for issue 637 appear, sorted. + kinds = {e["source"] for e in d["events"]} + self.assertEqual(kinds, {"control_plane", "gitea_handoff"}) + + def test_failing_comment_source_degrades_only_handoff(self): + import tempfile + + def boom(kind, number): + raise RuntimeError("network down") + + with tempfile.TemporaryDirectory() as tmp: + db = os.path.join(tmp, "cp.sqlite3") + _seed_db(db) + snap = timeline.load_timeline( + remote="prgs", org="Scaled-Tech-Consulting", repo="Gitea-Tools", + issue=637, db_path=db, comment_source=boom, + ) + d = snap.to_dict() + cp = [s for s in d["sources"] if s["name"] == "control_plane"][0] + handoff = [s for s in d["sources"] if s["name"] == "gitea_handoff"][0] + self.assertTrue(cp["ok"]) + self.assertFalse(handoff["ok"]) + self.assertIn("network down", handoff["reason"]) + + +class TestTimelineApi(unittest.TestCase): + def setUp(self): + self._prev_db = os.environ.get(control_plane_db.DB_PATH_ENV) + self._prev_offline = os.environ.get("WEBUI_TEST_OFFLINE") + import tempfile + + self._tmpdir = tempfile.TemporaryDirectory() + self._db = os.path.join(self._tmpdir.name, "cp.sqlite3") + _seed_db(self._db) + os.environ[control_plane_db.DB_PATH_ENV] = self._db + os.environ["WEBUI_TEST_OFFLINE"] = "1" + self.client = TestClient(create_app()) + + def tearDown(self): + if self._prev_db is None: + os.environ.pop(control_plane_db.DB_PATH_ENV, None) + else: + os.environ[control_plane_db.DB_PATH_ENV] = self._prev_db + if self._prev_offline is None: + os.environ.pop("WEBUI_TEST_OFFLINE", None) + else: + os.environ["WEBUI_TEST_OFFLINE"] = self._prev_offline + self._tmpdir.cleanup() + + def test_api_returns_timeline(self): + resp = self.client.get("/api/v1/timeline") + self.assertEqual(resp.status_code, 200) + body = resp.json() + self.assertEqual(body["schema_version"], timeline.TIMELINE_SCHEMA_VERSION) + self.assertIn("events", body) + self.assertIn("pagination", body) + self.assertGreaterEqual(body["pagination"]["total"], 1) + + def test_api_filter_by_issue(self): + resp = self.client.get("/api/v1/timeline?issue=637") + self.assertEqual(resp.status_code, 200) + events = resp.json()["events"] + self.assertTrue(events) + self.assertTrue(all(e["issue_number"] == 637 for e in events)) + + def test_api_pagination(self): + resp = self.client.get("/api/v1/timeline?limit=1&offset=0") + self.assertEqual(resp.status_code, 200) + pg = resp.json()["pagination"] + self.assertEqual(pg["limit"], 1) + self.assertEqual(len(resp.json()["events"]), 1) + if pg["total"] > 1: + self.assertTrue(pg["has_more"]) + + def test_api_is_read_only(self): + resp = self.client.post("/api/v1/timeline") + self.assertIn(resp.status_code, (404, 405)) + + def test_api_no_secret_leak(self): + resp = self.client.get("/api/v1/timeline?issue=637") + self.assertNotIn("ghs_ABCDEF1234567890abcdef", resp.text) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/app.py b/webui/app.py index cd8ab8b..26675bd 100644 --- a/webui/app.py +++ b/webui/app.py @@ -45,6 +45,7 @@ from webui.worktree_scanner import load_hygiene_snapshot, snapshot_to_dict as wo from webui.worktree_views import render_worktrees_page from webui.runtime_health import load_runtime_snapshot, snapshot_to_dict as runtime_snapshot_to_dict from webui.runtime_views import render_runtime_page +from webui.timeline import load_timeline, snapshot_to_dict as timeline_snapshot_to_dict _READ_ONLY_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) _AUDIT_MUTATION_PATHS = frozenset({"/audit", "/api/audit"}) @@ -377,6 +378,101 @@ async def api_console_security_model(_request: Request) -> JSONResponse: }) +def _query_int(request: Request, key: str) -> int | None: + """Parse an optional integer query parameter; None when absent/invalid.""" + raw = request.query_params.get(key) + if raw is None or not str(raw).strip(): + return None + try: + return int(str(raw).strip()) + except (TypeError, ValueError): + return None + + +def _derive_remote(host: str) -> str: + """Map a Gitea host to its known short remote name (control-plane scope key).""" + text = (host or "").lower() + if "prgs" in text: + return "prgs" + if "dadeschools" in text: + return "dadeschools" + return text.split(".")[0] if text else "" + + +def _timeline_comment_source(host: str, org: str, repo: str): + """Build a fail-soft CTH-comment fetcher for one repo, or None when offline. + + Returns a callable ``(kind, number) -> list[comment]``. Credentials or + network failures raise inside the callable so ``load_timeline`` degrades the + handoff source rather than the whole timeline. Offline test mode yields no + live source so the handoff section reports ``not run``. + """ + import os + + from gitea_auth import api_fetch_page, get_auth_header, repo_api_url + + offline = (os.environ.get("WEBUI_TEST_OFFLINE") or "").strip().lower() in {"1", "true", "yes"} + if offline: + return None + auth = get_auth_header(host) + if not auth: + return None + + def _fetch(kind: str, number: int) -> list: + segment = "pulls" if kind == "pr" else "issues" + url = f"{repo_api_url(host, org, repo)}/{segment}/{int(number)}/comments" + comments: list = [] + page = 1 + while page <= 20: + raw, meta = api_fetch_page(url, auth, page=page, limit=50) + comments.extend(raw) + if bool(meta["is_final_page"]): + break + page += 1 + return comments + + return _fetch + + +async def api_v1_timeline(request: Request) -> JSONResponse: + """Read-only workflow-event timeline (#637). Filter by issue/PR/session.""" + from webui.queue_loader import _host_from_url # host normalisation helper + + registry, error = _load_project_registry() + if error is not None: + return JSONResponse(error.to_dict(), status_code=500) + project = registry.projects[0] if registry.projects else None + + org = request.query_params.get("org") or (project.gitea_owner if project else "") + repo = request.query_params.get("repo") or (project.repo_name if project else "") + host = _host_from_url(project.remote_host) if project else "" + remote = request.query_params.get("remote") or _derive_remote(host) + + if not (remote and org and repo): + return JSONResponse( + { + "error": "timeline_scope_unresolved", + "detail": "no project in registry and no remote/org/repo query params provided", + }, + status_code=400, + ) + + comment_source = _timeline_comment_source(host, org, repo) if (host and org and repo) else None + + snapshot = load_timeline( + remote=remote, + org=org, + repo=repo, + issue=_query_int(request, "issue"), + pr=_query_int(request, "pr"), + session=(request.query_params.get("session") or None), + limit=_query_int(request, "limit"), + offset=_query_int(request, "offset"), + comment_source=comment_source, + ) + return JSONResponse(timeline_snapshot_to_dict(snapshot)) + + async def method_not_allowed(request: Request, _exc: Exception) -> Response: path = request.url.path if path in _AUDIT_MUTATION_PATHS and request.method == "POST": @@ -415,6 +511,7 @@ def create_app(*, bind_host: str | None = None) -> Starlette: Route("/api/prompts", api_prompts, methods=["GET"]), Route("/runtime", runtime, methods=["GET"]), Route("/api/runtime", api_runtime, methods=["GET"]), + Route("/api/v1/timeline", api_v1_timeline, methods=["GET"]), Route("/audit", audit, methods=["GET", "POST"]), Route("/api/audit", api_audit, methods=["GET", "POST"]), Route("/worktrees", worktrees, methods=["GET"]), diff --git a/webui/timeline.py b/webui/timeline.py new file mode 100644 index 0000000..e4597af --- /dev/null +++ b/webui/timeline.py @@ -0,0 +1,539 @@ +"""Workflow-event and conversation timeline model (#637, Phase 1). + +Operators cannot browse a unified timeline of workflow events, decisions, +tool calls, and handoffs: the evidence is scattered across control-plane +events, Gitea canonical handoff comments, and local logs. This module defines +one durable, versioned event schema and per-source adapters that normalise +those scattered records into a single ``WorkflowEvent`` stream, plus a +read-only query layer (filter by issue / PR / session, stable ordering, +pagination) that the ``/api/v1/timeline`` route serves. + +Design rules honoured here: + +- **Read-only.** Sources are read; nothing is mutated. The control-plane + database is opened through a ``mode=ro`` URI so a missing or unwritable DB + degrades to a reason instead of creating directories or running migrations. +- **Fail-soft per source.** An unavailable source degrades to a status with a + reason rather than raising, and a source that could not run is never + rendered as an empty-and-healthy timeline. +- **Redaction at the boundary, fail closed.** Every free-text field (event + messages, redacted tool arguments, decision/proof text) is run through the + console redaction policy before it leaves this module. An unredactable value + becomes the placeholder — an unredacted payload is never emitted, and a + generation error never drops raw data to a caller or a log. +- **Stable ordering.** Events sort by ``(timestamp, source_rank, event_key)`` + with a deterministic tiebreak, so pagination is stable across calls and + events with equal or missing timestamps keep a fixed order. + +Non-goals (from the issue): no full chat replay, no mutation of historical +events, no unredacted tool-argument storage. +""" + +from __future__ import annotations + +import re +import sqlite3 +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Callable, Iterable + +import control_plane_db +from webui import console_redaction + +# The schema is versioned so consumers can branch on shape. Bump on any +# breaking change to WorkflowEvent's serialized form. +TIMELINE_SCHEMA_VERSION = 1 + +# Known event sources and their deterministic ordering rank. When two events +# carry the same timestamp, the source rank breaks the tie before the +# per-source event key, so a control-plane event and a handoff comment minted +# in the same second always sort in a fixed order. +SOURCE_CONTROL_PLANE = "control_plane" +SOURCE_GITEA_HANDOFF = "gitea_handoff" +_SOURCE_RANK = { + SOURCE_CONTROL_PLANE: 0, + SOURCE_GITEA_HANDOFF: 1, +} + +# A timestamp far in the future so events with no parseable timestamp sort +# last (after everything real) instead of first, without raising. +_MISSING_TS_SORT = "9999-12-31T23:59:59Z" + + +def _parse_ts(value: str | None) -> str | None: + """Normalise a timestamp to ``...Z`` UTC, or None when unparseable.""" + if not value: + return None + text = str(value).strip() + if not text: + return None + candidate = text[:-1] + "+00:00" if text.endswith("Z") else text + try: + parsed = datetime.fromisoformat(candidate) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _redact(value: Any) -> Any: + """Redact a single free-text field, failing closed to the placeholder.""" + if value is None: + return None + return console_redaction.redact_text(str(value)) + + +@dataclass(frozen=True) +class WorkflowEvent: + """One normalised timeline event. + + Every field is optional except ``source``/``event_type``/``event_key`` + because sources carry different subsets. The class is frozen so an adapted + event is an immutable record; a consumer that needs a variant builds a new + one rather than mutating history. + """ + + source: str + event_type: str + event_key: str + timestamp: str | None = None + actor: str | None = None + role: str | None = None + issue_number: int | None = None + pr_number: int | None = None + session_id: str | None = None + tool_name: str | None = None + decision: str | None = None + message: str | None = None + correlation_id: str | None = None + evidence_refs: tuple[str, ...] = () + sensitive: bool = False + + def sort_key(self) -> tuple[str, int, str]: + return ( + self.timestamp or _MISSING_TS_SORT, + _SOURCE_RANK.get(self.source, 99), + self.event_key, + ) + + def to_dict(self) -> dict[str, Any]: + return { + "source": self.source, + "event_type": self.event_type, + "event_key": self.event_key, + "timestamp": self.timestamp, + "actor": self.actor, + "role": self.role, + "issue_number": self.issue_number, + "pr_number": self.pr_number, + "session_id": self.session_id, + "tool_name": self.tool_name, + "decision": self.decision, + "message": self.message, + "correlation_id": self.correlation_id, + "evidence_refs": list(self.evidence_refs), + "sensitive": self.sensitive, + } + + +# --------------------------------------------------------------------------- # +# Adapters — pure functions from a source's raw records to WorkflowEvents. # +# Each is total: a malformed record is skipped, never raised on. # +# --------------------------------------------------------------------------- # + +# Event types whose payload is treated as sensitive and always redaction-hard +# (they can carry lease/session provenance or tool arguments). +_SENSITIVE_EVENT_HINTS = ("lease", "capability", "token", "auth", "secret") + +# Reference tokens (issue/PR/comment ids) and SHAs parsed out of proof text. +_EVIDENCE_REF_RE = re.compile(r"(?:#|PR\s*#?|issue\s*#?|comment\s*#?)(\d+)", re.IGNORECASE) +_SHA_RE = re.compile(r"\b[0-9a-f]{7,40}\b") + + +def _kind_to_numbers(kind: str | None, number: int | None) -> tuple[int | None, int | None]: + """Map a control-plane work-item (kind, number) to (issue_no, pr_no).""" + if number is None: + return (None, None) + if kind == "pr": + return (None, int(number)) + if kind == "issue": + return (int(number), None) + return (None, None) + + +def _correlation_for(kind: str | None, number: int | None) -> str | None: + if number is None or kind not in ("issue", "pr"): + return None + return f"{kind}#{number}" + + +def _extract_evidence_refs(*texts: str | None) -> tuple[str, ...]: + refs: list[str] = [] + for text in texts: + if not text: + continue + for match in _EVIDENCE_REF_RE.finditer(text): + token = f"#{match.group(1)}" + if token not in refs: + refs.append(token) + for match in _SHA_RE.finditer(text): + token = match.group(0) + if token not in refs: + refs.append(token) + return tuple(refs) + + +def adapt_cp_events(rows: Iterable[dict[str, Any]]) -> list[WorkflowEvent]: + """Adapt control-plane ``events`` rows (joined to work_items) into events. + + Each row is expected to carry ``event_id``, ``event_type``, ``message``, + ``created_at`` and the joined work-item ``kind``/``number``. Rows missing + an id or type are skipped so a partially written table never raises. + """ + events: list[WorkflowEvent] = [] + for row in rows or []: + try: + event_id = row.get("event_id") + event_type = (row.get("event_type") or "").strip() + if event_id is None or not event_type: + continue + kind = row.get("kind") + number = row.get("number") + issue_no, pr_no = _kind_to_numbers(kind, number) + sensitive = any(hint in event_type.lower() for hint in _SENSITIVE_EVENT_HINTS) + events.append( + WorkflowEvent( + source=SOURCE_CONTROL_PLANE, + event_type=event_type, + event_key=f"cp:{event_id}", + timestamp=_parse_ts(row.get("created_at")), + issue_number=issue_no, + pr_number=pr_no, + session_id=(row.get("session_id") or None), + message=_redact(row.get("message")), + correlation_id=_correlation_for(kind, number), + sensitive=sensitive, + ) + ) + except Exception: + # A single malformed row must not sink the whole adaptation. + continue + return events + + +def adapt_cth_comments( + comments: Iterable[dict[str, Any]], + *, + kind: str, + number: int, +) -> list[WorkflowEvent]: + """Adapt Gitea Canonical Thread Handoff (CTH) comments into events. + + Only comments that parse as a CTH (``canonical_thread_handoff.parse_cth_comment``) + become events; ordinary comments are ignored. ``kind``/``number`` scope the + events to the issue or PR the comments belong to. + """ + # Imported lazily so this module has no import-time dependency on the + # handoff parser when only the control-plane adapter is used. + from canonical_thread_handoff import parse_cth_comment + + issue_no, pr_no = _kind_to_numbers(kind, number) + correlation = _correlation_for(kind, number) + events: list[WorkflowEvent] = [] + for comment in comments or []: + try: + body = comment.get("body") or "" + parsed = parse_cth_comment(body) + if not parsed: + continue + fields = parsed.get("fields") or {} + cth_type = parsed.get("cth_type") or "handoff" + comment_id = comment.get("id") + actor = (comment.get("user") or {}).get("login") + decision = fields.get("decision") + proof = fields.get("proof") + next_action = fields.get("next action") + events.append( + WorkflowEvent( + source=SOURCE_GITEA_HANDOFF, + event_type=f"handoff:{cth_type}", + event_key=f"cth:{kind}:{number}:{comment_id}", + timestamp=_parse_ts(comment.get("created_at")), + actor=actor, + role=_redact(fields.get("next owner")), + issue_number=issue_no, + pr_number=pr_no, + decision=_redact(decision), + message=_redact(next_action or fields.get("status")), + correlation_id=correlation, + evidence_refs=_extract_evidence_refs(proof, decision), + sensitive=False, + ) + ) + except Exception: + continue + return events + + +# --------------------------------------------------------------------------- # +# Read-only control-plane event source. # +# --------------------------------------------------------------------------- # + +_CP_EVENTS_QUERY = """ +SELECT e.event_id AS event_id, + e.event_type AS event_type, + e.message AS message, + e.created_at AS created_at, + w.kind AS kind, + w.number AS number +FROM events e +JOIN work_items w ON e.work_item_id = w.work_item_id +WHERE w.remote = ? AND w.org = ? AND w.repo = ? +""" + + +@dataclass(frozen=True) +class SourceStatus: + """Fail-soft status for one timeline source.""" + + name: str + ok: bool + reason: str | None = None + count: int = 0 + + def to_dict(self) -> dict[str, Any]: + return {"name": self.name, "ok": self.ok, "reason": self.reason, "count": self.count} + + +def read_cp_events( + *, + remote: str, + org: str, + repo: str, + db_path: str | None = None, +) -> tuple[list[WorkflowEvent], SourceStatus]: + """Read scoped control-plane events read-only. Never creates the DB. + + Opens the SQLite file through a ``mode=ro`` URI: a health/timeline read + must never create directories or run the schema migration that + ``ControlPlaneDB()`` performs on construction. A missing or unreadable DB + degrades to a status with a reason. + """ + path = (db_path or control_plane_db.default_db_path()).strip() + conn: sqlite3.Connection | None = None + try: + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + cursor = conn.execute(_CP_EVENTS_QUERY, (remote, org, repo)) + rows = [dict(r) for r in cursor.fetchall()] + except sqlite3.OperationalError as exc: + return ([], SourceStatus(SOURCE_CONTROL_PLANE, ok=False, reason=f"control-plane DB unavailable: {exc}")) + except sqlite3.Error as exc: + return ([], SourceStatus(SOURCE_CONTROL_PLANE, ok=False, reason=f"control-plane read failed: {exc}")) + finally: + if conn is not None: + conn.close() + events = adapt_cp_events(rows) + return (events, SourceStatus(SOURCE_CONTROL_PLANE, ok=True, count=len(events))) + + +# --------------------------------------------------------------------------- # +# Filter, sort, paginate. # +# --------------------------------------------------------------------------- # + + +def filter_events( + events: Iterable[WorkflowEvent], + *, + issue: int | None = None, + pr: int | None = None, + session: str | None = None, +) -> list[WorkflowEvent]: + """Filter events by issue number, PR number, and/or session id. + + Filters are conjunctive. A filter that names a dimension an event does not + carry excludes that event (an issue filter excludes PR-only events). + """ + out: list[WorkflowEvent] = [] + for ev in events: + if issue is not None and ev.issue_number != issue: + continue + if pr is not None and ev.pr_number != pr: + continue + if session is not None and ev.session_id != session: + continue + out.append(ev) + return out + + +def sort_events(events: Iterable[WorkflowEvent]) -> list[WorkflowEvent]: + """Return events in stable timeline order (ascending).""" + return sorted(events, key=lambda ev: ev.sort_key()) + + +@dataclass(frozen=True) +class TimelinePage: + """One page of the sorted, filtered timeline.""" + + events: tuple[WorkflowEvent, ...] + total: int + limit: int + offset: int + + @property + def next_offset(self) -> int | None: + nxt = self.offset + len(self.events) + return nxt if nxt < self.total else None + + def to_dict(self) -> dict[str, Any]: + return { + "events": [ev.to_dict() for ev in self.events], + "pagination": { + "total": self.total, + "limit": self.limit, + "offset": self.offset, + "returned": len(self.events), + "next_offset": self.next_offset, + "has_more": self.next_offset is not None, + }, + } + + +_MAX_LIMIT = 500 +_DEFAULT_LIMIT = 50 + + +def _coerce_bounds(limit: int | None, offset: int | None) -> tuple[int, int]: + try: + lim = int(limit) if limit is not None else _DEFAULT_LIMIT + except (TypeError, ValueError): + lim = _DEFAULT_LIMIT + try: + off = int(offset) if offset is not None else 0 + except (TypeError, ValueError): + off = 0 + lim = max(1, min(lim, _MAX_LIMIT)) + off = max(0, off) + return (lim, off) + + +def paginate(events: list[WorkflowEvent], *, limit: int | None, offset: int | None) -> TimelinePage: + lim, off = _coerce_bounds(limit, offset) + window = events[off : off + lim] + return TimelinePage(events=tuple(window), total=len(events), limit=lim, offset=off) + + +# --------------------------------------------------------------------------- # +# Composition — load_timeline aggregates all sources, fail-soft. # +# --------------------------------------------------------------------------- # + +# A comment source is a callable that, given (kind, number), returns the raw +# Gitea comment list for that issue/PR. The route supplies a live fail-soft +# fetcher; tests supply a fixture. When None, the handoff source is reported as +# not-run (never silently empty-and-healthy). +CommentSource = Callable[[str, int], list[dict[str, Any]]] + + +@dataclass(frozen=True) +class TimelineSnapshot: + schema_version: int + remote: str + org: str + repo: str + filters: dict[str, Any] + page: TimelinePage + sources: tuple[SourceStatus, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "scope": {"remote": self.remote, "org": self.org, "repo": self.repo}, + "filters": self.filters, + "sources": [s.to_dict() for s in self.sources], + **self.page.to_dict(), + } + + +def load_timeline( + *, + remote: str, + org: str, + repo: str, + issue: int | None = None, + pr: int | None = None, + session: str | None = None, + limit: int | None = None, + offset: int | None = None, + db_path: str | None = None, + comment_source: CommentSource | None = None, +) -> TimelineSnapshot: + """Aggregate every timeline source into one filtered, paginated snapshot. + + Sources are read independently and fail soft: an unavailable source + contributes a ``SourceStatus`` with ``ok=False`` and a reason, and never + collapses the whole timeline. The handoff source only runs when a specific + issue or PR is requested (a handoff comment belongs to one thread) and a + ``comment_source`` is available; otherwise it is reported as ``not run`` + rather than as an empty-and-healthy source. + """ + all_events: list[WorkflowEvent] = [] + statuses: list[SourceStatus] = [] + + cp_events, cp_status = read_cp_events(remote=remote, org=org, repo=repo, db_path=db_path) + all_events.extend(cp_events) + statuses.append(cp_status) + + # Gitea handoff comments are thread-scoped: only fetch when the caller + # narrowed to one issue or PR, and only when a source was provided. + handoff_target: tuple[str, int] | None = None + if pr is not None: + handoff_target = ("pr", pr) + elif issue is not None: + handoff_target = ("issue", issue) + + if handoff_target is None: + statuses.append( + SourceStatus( + SOURCE_GITEA_HANDOFF, + ok=False, + reason="not run: handoff comments are thread-scoped; filter by issue or pr to include them", + ) + ) + elif comment_source is None: + statuses.append( + SourceStatus( + SOURCE_GITEA_HANDOFF, + ok=False, + reason="not run: no comment source configured for this timeline read", + ) + ) + else: + kind, number = handoff_target + try: + comments = comment_source(kind, number) or [] + handoff_events = adapt_cth_comments(comments, kind=kind, number=number) + all_events.extend(handoff_events) + statuses.append(SourceStatus(SOURCE_GITEA_HANDOFF, ok=True, count=len(handoff_events))) + except Exception as exc: # fail soft: a fetch/parse error degrades this source only + statuses.append( + SourceStatus(SOURCE_GITEA_HANDOFF, ok=False, reason=f"handoff source failed: {exc}") + ) + + filtered = filter_events(all_events, issue=issue, pr=pr, session=session) + ordered = sort_events(filtered) + page = paginate(ordered, limit=limit, offset=offset) + + return TimelineSnapshot( + schema_version=TIMELINE_SCHEMA_VERSION, + remote=remote, + org=org, + repo=repo, + filters={"issue": issue, "pr": pr, "session": session}, + page=page, + sources=tuple(statuses), + ) + + +def snapshot_to_dict(snapshot: TimelineSnapshot) -> dict[str, Any]: + return snapshot.to_dict()