feat(webui): workflow-event and conversation timeline model (Closes #637) #849
@@ -46,6 +46,17 @@ _FIELD_RE = re.compile(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_known_cth_type(value: str | None) -> bool:
|
||||||
|
"""True when *value* is a declared member of the :data:`CTH_TYPES` contract.
|
||||||
|
|
||||||
|
``CTH_TYPES`` is the single authority for what a CTH type may be. The
|
||||||
|
heading a comment carries is free text, so a *read* path that turns a parsed
|
||||||
|
type into something durable — a serialized field, a routing decision — must
|
||||||
|
check membership here rather than trust the parse or keep a list of its own.
|
||||||
|
"""
|
||||||
|
return (value or "").strip() in CTH_TYPES
|
||||||
|
|
||||||
|
|
||||||
def format_cth_body(
|
def format_cth_body(
|
||||||
*,
|
*,
|
||||||
cth_type: str,
|
cth_type: str,
|
||||||
@@ -60,7 +71,7 @@ def format_cth_body(
|
|||||||
) -> str:
|
) -> str:
|
||||||
"""Render a canonical CTH comment body."""
|
"""Render a canonical CTH comment body."""
|
||||||
normalized_type = (cth_type or "").strip()
|
normalized_type = (cth_type or "").strip()
|
||||||
if normalized_type not in CTH_TYPES:
|
if not is_known_cth_type(normalized_type):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"unknown CTH type '{cth_type}'; expected one of {sorted(CTH_TYPES)}"
|
f"unknown CTH type '{cth_type}'; expected one of {sorted(CTH_TYPES)}"
|
||||||
)
|
)
|
||||||
@@ -101,6 +112,12 @@ def parse_cth_comment(body: str) -> dict[str, Any] | None:
|
|||||||
fields[key] = match.group(2).strip()
|
fields[key] = match.group(2).strip()
|
||||||
return {
|
return {
|
||||||
"cth_type": cth_type,
|
"cth_type": cth_type,
|
||||||
|
# The heading capture is unconstrained free text, so the parse states
|
||||||
|
# whether it satisfies the CTH_TYPES contract instead of leaving every
|
||||||
|
# reader to decide (or forget). Parsing stays total — an unknown type is
|
||||||
|
# still parsed and reported, never raised on — but a reader that turns
|
||||||
|
# the type into a durable value can now tell the two apart.
|
||||||
|
"cth_type_known": is_known_cth_type(cth_type),
|
||||||
"fields": fields,
|
"fields": fields,
|
||||||
"raw_body": text,
|
"raw_body": text,
|
||||||
}
|
}
|
||||||
@@ -119,7 +136,7 @@ def assess_cth_comment(body: str) -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
cth_type = parsed.get("cth_type") or ""
|
cth_type = parsed.get("cth_type") or ""
|
||||||
if cth_type not in CTH_TYPES:
|
if not is_known_cth_type(cth_type):
|
||||||
reasons.append(
|
reasons.append(
|
||||||
f"unknown CTH type '{cth_type}'; expected one of {sorted(CTH_TYPES)}"
|
f"unknown CTH type '{cth_type}'; expected one of {sorted(CTH_TYPES)}"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -16,7 +16,13 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|||||||
from starlette.testclient import TestClient
|
from starlette.testclient import TestClient
|
||||||
|
|
||||||
import control_plane_db
|
import control_plane_db
|
||||||
from canonical_thread_handoff import format_cth_body
|
from canonical_thread_handoff import (
|
||||||
|
CTH_TYPES,
|
||||||
|
MARKER,
|
||||||
|
format_cth_body,
|
||||||
|
is_known_cth_type,
|
||||||
|
parse_cth_comment,
|
||||||
|
)
|
||||||
from webui import timeline
|
from webui import timeline
|
||||||
from webui.app import create_app
|
from webui.app import create_app
|
||||||
|
|
||||||
@@ -614,5 +620,402 @@ class TestTimelineApi(unittest.TestCase):
|
|||||||
self.assertIsNone(body["error"])
|
self.assertIsNone(body["error"])
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_event(path: str, *, event_type: str, message: str, created_at: str) -> None:
|
||||||
|
"""Append one control-plane event with a caller-chosen ``event_type``.
|
||||||
|
|
||||||
|
The ``events`` table stores whatever a producer writes, so this seeds the
|
||||||
|
adapter the way a hostile or buggy producer would.
|
||||||
|
"""
|
||||||
|
conn = sqlite3.connect(path)
|
||||||
|
try:
|
||||||
|
wid = conn.execute(
|
||||||
|
"SELECT work_item_id FROM work_items WHERE kind='issue' AND number=637"
|
||||||
|
).fetchone()[0]
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO events(work_item_id, event_type, message, created_at) VALUES (?,?,?,?)",
|
||||||
|
(wid, event_type, message, created_at),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _raw_cth_body(heading: str, **fields) -> str:
|
||||||
|
"""Build a CTH comment with an arbitrary heading.
|
||||||
|
|
||||||
|
``format_cth_body`` refuses an undeclared type, which is exactly the write
|
||||||
|
path already covered. The read path must cope with a body that never went
|
||||||
|
through it, so this writes the marker and heading directly.
|
||||||
|
"""
|
||||||
|
lines = [MARKER, f"## CTH: {heading}", ""]
|
||||||
|
base = {
|
||||||
|
"Status": "ready",
|
||||||
|
"Next owner": "reviewer",
|
||||||
|
"Current blocker": "none",
|
||||||
|
"Decision": "d",
|
||||||
|
"Proof": "none",
|
||||||
|
"Next action": "review",
|
||||||
|
"Ready-to-paste prompt": "Review PR #1 now",
|
||||||
|
}
|
||||||
|
base.update(fields)
|
||||||
|
lines.extend(f"{key}: {value}" for key, value in base.items())
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
class TestEventTypeBoundary(unittest.TestCase):
|
||||||
|
"""F2 residual: event_type must cross the same boundary as every other field.
|
||||||
|
|
||||||
|
The canary is seeded through ``event_type`` *only*, with a benign message,
|
||||||
|
so message redaction cannot be what makes these pass. Each case inspects
|
||||||
|
``event_type`` explicitly and then the complete serialized payload.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# ---- control-plane stored event_type ---------------------------------- #
|
||||||
|
|
||||||
|
def test_cp_event_type_canary_never_serialized(self):
|
||||||
|
rows = [{
|
||||||
|
"event_id": 1,
|
||||||
|
"event_type": SYNTHETIC_SECRET_40_HEX,
|
||||||
|
"message": "deploy completed", # benign: no redaction happens here
|
||||||
|
"created_at": "2026-07-23T01:00:00Z",
|
||||||
|
"kind": "issue",
|
||||||
|
"number": 637,
|
||||||
|
}]
|
||||||
|
events = timeline.adapt_cp_events(rows)
|
||||||
|
self.assertEqual(len(events), 1)
|
||||||
|
ev = events[0]
|
||||||
|
# The field itself, inspected directly.
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, ev.event_type)
|
||||||
|
self.assertEqual(ev.event_type, timeline.UNSAFE_EVENT_TYPE)
|
||||||
|
# Message redaction is provably not what saved us: it is untouched.
|
||||||
|
self.assertEqual(ev.message, "deploy completed")
|
||||||
|
# And nowhere in the serialized record.
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, json.dumps(ev.to_dict()))
|
||||||
|
# An unsafe value is a visible fact, not a silent substitution.
|
||||||
|
self.assertTrue(ev.sensitive)
|
||||||
|
|
||||||
|
def test_cp_same_canary_in_message_and_event_type(self):
|
||||||
|
"""The decisive case: one record, one value, two fields, one verdict."""
|
||||||
|
rows = [{
|
||||||
|
"event_id": 2,
|
||||||
|
"event_type": SYNTHETIC_SECRET_40_HEX,
|
||||||
|
"message": f"deploy token={SYNTHETIC_SECRET_40_HEX}",
|
||||||
|
"created_at": "2026-07-23T01:00:00Z",
|
||||||
|
"kind": "issue",
|
||||||
|
"number": 637,
|
||||||
|
}]
|
||||||
|
ev = timeline.adapt_cp_events(rows)[0]
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, ev.message or "")
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, ev.event_type)
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, json.dumps(ev.to_dict()))
|
||||||
|
|
||||||
|
def test_cp_unsafe_event_type_is_not_rewritten_as_a_valid_one(self):
|
||||||
|
"""A refused value must not be disguised as some other real event type."""
|
||||||
|
rows = [{
|
||||||
|
"event_id": 3,
|
||||||
|
"event_type": SYNTHETIC_SECRET_40_HEX,
|
||||||
|
"message": "m",
|
||||||
|
"created_at": "2026-07-23T01:00:00Z",
|
||||||
|
"kind": "issue",
|
||||||
|
"number": 637,
|
||||||
|
}]
|
||||||
|
ev = timeline.adapt_cp_events(rows)[0]
|
||||||
|
for legitimate in ("allocation", "pr.opened", "lease.renew", "assigned"):
|
||||||
|
self.assertNotEqual(ev.event_type, legitimate)
|
||||||
|
|
||||||
|
def test_cp_assigned_secret_in_event_type_refused(self):
|
||||||
|
rows = [{
|
||||||
|
"event_id": 4,
|
||||||
|
"event_type": f"token={SYNTHETIC_SECRET_40_HEX}",
|
||||||
|
"message": "m",
|
||||||
|
"created_at": "2026-07-23T01:00:00Z",
|
||||||
|
"kind": "issue",
|
||||||
|
"number": 637,
|
||||||
|
}]
|
||||||
|
ev = timeline.adapt_cp_events(rows)[0]
|
||||||
|
self.assertEqual(ev.event_type, timeline.UNSAFE_EVENT_TYPE)
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, json.dumps(ev.to_dict()))
|
||||||
|
|
||||||
|
def test_cp_legitimate_event_types_preserved(self):
|
||||||
|
"""Every real producer type in this repo must survive untouched."""
|
||||||
|
legitimate = [
|
||||||
|
"allocation", "pr.opened", "lease.renew", "assigned",
|
||||||
|
"lease_released", "lease_expired", "lease_abandoned",
|
||||||
|
"lease_adopted", "dependency_edge_state_change",
|
||||||
|
]
|
||||||
|
rows = [
|
||||||
|
{
|
||||||
|
"event_id": i,
|
||||||
|
"event_type": name,
|
||||||
|
"message": "m",
|
||||||
|
"created_at": "2026-07-23T01:00:00Z",
|
||||||
|
"kind": "issue",
|
||||||
|
"number": 637,
|
||||||
|
}
|
||||||
|
for i, name in enumerate(legitimate, start=1)
|
||||||
|
]
|
||||||
|
events = timeline.adapt_cp_events(rows)
|
||||||
|
self.assertEqual([e.event_type for e in events], legitimate)
|
||||||
|
|
||||||
|
def test_cp_malformed_event_type_shapes_refused(self):
|
||||||
|
for bad in ("has space", "1leading-digit", "x" * 200, "semi;colon", "new\nline"):
|
||||||
|
with self.subTest(event_type=bad):
|
||||||
|
rows = [{
|
||||||
|
"event_id": 1,
|
||||||
|
"event_type": bad,
|
||||||
|
"message": "m",
|
||||||
|
"created_at": "2026-07-23T01:00:00Z",
|
||||||
|
"kind": "issue",
|
||||||
|
"number": 637,
|
||||||
|
}]
|
||||||
|
events = timeline.adapt_cp_events(rows)
|
||||||
|
self.assertEqual(events[0].event_type, timeline.UNSAFE_EVENT_TYPE)
|
||||||
|
self.assertNotIn(bad, json.dumps(events[0].to_dict()))
|
||||||
|
|
||||||
|
# ---- CTH heading event_type ------------------------------------------- #
|
||||||
|
|
||||||
|
def test_cth_heading_canary_never_serialized(self):
|
||||||
|
comments = [{
|
||||||
|
"id": 11,
|
||||||
|
"body": _raw_cth_body(SYNTHETIC_SECRET_40_HEX),
|
||||||
|
"created_at": "2026-07-23T05:00:00Z",
|
||||||
|
"user": {"login": "jcwalker3"},
|
||||||
|
}]
|
||||||
|
events = timeline.adapt_cth_comments(comments, kind="issue", number=637)
|
||||||
|
self.assertEqual(len(events), 1)
|
||||||
|
ev = events[0]
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, ev.event_type)
|
||||||
|
self.assertEqual(ev.event_type, timeline.UNKNOWN_HANDOFF_EVENT_TYPE)
|
||||||
|
# No message redaction is doing the work here — the message is benign.
|
||||||
|
self.assertEqual(ev.message, "review")
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, json.dumps(ev.to_dict()))
|
||||||
|
self.assertTrue(ev.sensitive)
|
||||||
|
|
||||||
|
def test_cth_assigned_secret_heading_refused(self):
|
||||||
|
comments = [{
|
||||||
|
"id": 12,
|
||||||
|
"body": _raw_cth_body(f"token={SYNTHETIC_SECRET_40_HEX}"),
|
||||||
|
"created_at": "2026-07-23T05:00:00Z",
|
||||||
|
"user": {"login": "jcwalker3"},
|
||||||
|
}]
|
||||||
|
ev = timeline.adapt_cth_comments(comments, kind="issue", number=637)[0]
|
||||||
|
self.assertEqual(ev.event_type, timeline.UNKNOWN_HANDOFF_EVENT_TYPE)
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, json.dumps(ev.to_dict()))
|
||||||
|
|
||||||
|
def test_cth_unknown_and_whitespace_headings_normalized(self):
|
||||||
|
for heading in ("Totally Made Up", "Author Handoff", "author handoff"):
|
||||||
|
with self.subTest(heading=heading):
|
||||||
|
comments = [{
|
||||||
|
"id": 13,
|
||||||
|
"body": _raw_cth_body(heading),
|
||||||
|
"created_at": "2026-07-23T05:00:00Z",
|
||||||
|
"user": {"login": "x"},
|
||||||
|
}]
|
||||||
|
events = timeline.adapt_cth_comments(comments, kind="issue", number=637)
|
||||||
|
self.assertEqual(len(events), 1)
|
||||||
|
self.assertEqual(events[0].event_type, timeline.UNKNOWN_HANDOFF_EVENT_TYPE)
|
||||||
|
|
||||||
|
def test_cth_declared_types_all_preserved(self):
|
||||||
|
"""Point 5: no legitimate declared type is lost to the new check."""
|
||||||
|
for cth_type in sorted(CTH_TYPES):
|
||||||
|
with self.subTest(cth_type=cth_type):
|
||||||
|
comments = [{
|
||||||
|
"id": 14,
|
||||||
|
"body": _raw_cth_body(cth_type),
|
||||||
|
"created_at": "2026-07-23T05:00:00Z",
|
||||||
|
"user": {"login": "x"},
|
||||||
|
}]
|
||||||
|
ev = timeline.adapt_cth_comments(comments, kind="issue", number=637)[0]
|
||||||
|
self.assertEqual(ev.event_type, f"handoff:{cth_type}")
|
||||||
|
self.assertFalse(ev.sensitive)
|
||||||
|
|
||||||
|
def test_cth_surrounding_whitespace_still_matches_contract(self):
|
||||||
|
comments = [{
|
||||||
|
"id": 15,
|
||||||
|
"body": _raw_cth_body(" Author Handoff "),
|
||||||
|
"created_at": "2026-07-23T05:00:00Z",
|
||||||
|
"user": {"login": "x"},
|
||||||
|
}]
|
||||||
|
ev = timeline.adapt_cth_comments(comments, kind="issue", number=637)[0]
|
||||||
|
self.assertEqual(ev.event_type, "handoff:Author Handoff")
|
||||||
|
|
||||||
|
# ---- complete serialized payload, both paths -------------------------- #
|
||||||
|
|
||||||
|
def _payload_clean(self, snapshot):
|
||||||
|
payload = json.dumps(snapshot.to_dict())
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, payload)
|
||||||
|
from webui import console_redaction
|
||||||
|
|
||||||
|
self.assertEqual(console_redaction.scan_for_secrets(snapshot.to_dict()), [])
|
||||||
|
return snapshot.to_dict()
|
||||||
|
|
||||||
|
def test_canary_absent_from_full_payload_via_cp_event_type(self):
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
db = os.path.join(tmp, "cp.sqlite3")
|
||||||
|
_seed_db(db)
|
||||||
|
_seed_event(
|
||||||
|
db,
|
||||||
|
event_type=SYNTHETIC_SECRET_40_HEX,
|
||||||
|
message="routine deploy",
|
||||||
|
created_at="2026-07-23T06:00:00Z",
|
||||||
|
)
|
||||||
|
snap = timeline.load_timeline(
|
||||||
|
remote="prgs", org="Scaled-Tech-Consulting", repo="Gitea-Tools",
|
||||||
|
issue=637, db_path=db, comment_source=lambda k, n: [],
|
||||||
|
)
|
||||||
|
d = self._payload_clean(snap)
|
||||||
|
types = [e["event_type"] for e in d["events"]]
|
||||||
|
self.assertIn(timeline.UNSAFE_EVENT_TYPE, types)
|
||||||
|
# The legitimate seeded types are still present and unchanged.
|
||||||
|
self.assertIn("allocation", types)
|
||||||
|
|
||||||
|
def test_canary_absent_from_full_payload_via_cth_heading(self):
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
comments = [{
|
||||||
|
"id": 21,
|
||||||
|
"body": _raw_cth_body(SYNTHETIC_SECRET_40_HEX),
|
||||||
|
"created_at": "2026-07-23T09:00:00Z",
|
||||||
|
"user": {"login": "jcwalker3"},
|
||||||
|
}]
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
db = os.path.join(tmp, "cp.sqlite3")
|
||||||
|
_seed_db(db)
|
||||||
|
snap = timeline.load_timeline(
|
||||||
|
remote="prgs", org="Scaled-Tech-Consulting", repo="Gitea-Tools",
|
||||||
|
issue=637, db_path=db, comment_source=lambda k, n: comments,
|
||||||
|
)
|
||||||
|
d = self._payload_clean(snap)
|
||||||
|
self.assertIn(
|
||||||
|
timeline.UNKNOWN_HANDOFF_EVENT_TYPE,
|
||||||
|
[e["event_type"] for e in d["events"]],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_canary_absent_when_seeded_through_both_paths_at_once(self):
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
comments = [{
|
||||||
|
"id": 22,
|
||||||
|
"body": _raw_cth_body(SYNTHETIC_SECRET_40_HEX),
|
||||||
|
"created_at": "2026-07-23T09:00:00Z",
|
||||||
|
"user": {"login": "jcwalker3"},
|
||||||
|
}]
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
db = os.path.join(tmp, "cp.sqlite3")
|
||||||
|
_seed_db(db)
|
||||||
|
_seed_event(
|
||||||
|
db,
|
||||||
|
event_type=SYNTHETIC_SECRET_40_HEX,
|
||||||
|
message=f"deploy token={SYNTHETIC_SECRET_40_HEX}",
|
||||||
|
created_at="2026-07-23T06:00:00Z",
|
||||||
|
)
|
||||||
|
snap = timeline.load_timeline(
|
||||||
|
remote="prgs", org="Scaled-Tech-Consulting", repo="Gitea-Tools",
|
||||||
|
issue=637, db_path=db, comment_source=lambda k, n: comments,
|
||||||
|
)
|
||||||
|
self._payload_clean(snap)
|
||||||
|
|
||||||
|
|
||||||
|
class TestSerializedFieldAudit(unittest.TestCase):
|
||||||
|
"""The remaining externally influenced strings that reach the payload."""
|
||||||
|
|
||||||
|
def test_event_key_ids_must_be_plain_identifiers(self):
|
||||||
|
rows = [
|
||||||
|
{"event_id": SYNTHETIC_SECRET_40_HEX, "event_type": "allocation",
|
||||||
|
"message": "m", "created_at": "2026-07-23T01:00:00Z",
|
||||||
|
"kind": "issue", "number": 637},
|
||||||
|
{"event_id": 8, "event_type": "allocation", "message": "m",
|
||||||
|
"created_at": "2026-07-23T01:00:00Z", "kind": "issue", "number": 637},
|
||||||
|
]
|
||||||
|
events = timeline.adapt_cp_events(rows)
|
||||||
|
self.assertEqual([e.event_key for e in events], ["cp:8"])
|
||||||
|
|
||||||
|
def test_comment_id_must_be_a_plain_identifier(self):
|
||||||
|
comments = [{
|
||||||
|
"id": SYNTHETIC_SECRET_40_HEX,
|
||||||
|
"body": _raw_cth_body("Author Handoff"),
|
||||||
|
"created_at": "2026-07-23T05:00:00Z",
|
||||||
|
"user": {"login": "x"},
|
||||||
|
}]
|
||||||
|
self.assertEqual(timeline.adapt_cth_comments(comments, kind="issue", number=637), [])
|
||||||
|
|
||||||
|
def test_adapter_refuses_a_scope_it_cannot_express(self):
|
||||||
|
comments = [{
|
||||||
|
"id": 1,
|
||||||
|
"body": _raw_cth_body("Author Handoff"),
|
||||||
|
"created_at": "2026-07-23T05:00:00Z",
|
||||||
|
"user": {"login": "x"},
|
||||||
|
}]
|
||||||
|
self.assertEqual(
|
||||||
|
timeline.adapt_cth_comments(comments, kind="issue", number=SYNTHETIC_SECRET_40_HEX),
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
timeline.adapt_cth_comments(comments, kind=SYNTHETIC_SECRET_40_HEX, number=1),
|
||||||
|
[],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_source_failure_reason_is_redacted(self):
|
||||||
|
def boom(kind, number):
|
||||||
|
raise RuntimeError(f"auth failed with token={SYNTHETIC_SECRET_40_HEX}")
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
db = os.path.join(tmp, "cp.sqlite3")
|
||||||
|
_seed_db(db)
|
||||||
|
snap = timeline.load_timeline(
|
||||||
|
remote="prgs", org="Scaled-Tech-Consulting", repo="Gitea-Tools",
|
||||||
|
issue=637, db_path=db, comment_source=boom,
|
||||||
|
)
|
||||||
|
d = snap.to_dict()
|
||||||
|
handoff = [s for s in d["sources"] if s["name"] == "gitea_handoff"][0]
|
||||||
|
self.assertFalse(handoff["ok"])
|
||||||
|
self.assertIn("handoff source failed", handoff["reason"])
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, json.dumps(d))
|
||||||
|
|
||||||
|
def test_echoed_scope_and_filters_are_guarded(self):
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
db = os.path.join(tmp, "cp.sqlite3")
|
||||||
|
_seed_db(db)
|
||||||
|
d = timeline.load_timeline(
|
||||||
|
remote="prgs", org="Scaled-Tech-Consulting", repo="Gitea-Tools",
|
||||||
|
issue=637, session=SYNTHETIC_SECRET_40_HEX, db_path=db,
|
||||||
|
comment_source=lambda k, n: [],
|
||||||
|
).to_dict()
|
||||||
|
self.assertNotIn(SYNTHETIC_SECRET_40_HEX, json.dumps(d))
|
||||||
|
# Ordinary scope values are untouched, so the echo stays useful.
|
||||||
|
self.assertEqual(
|
||||||
|
d["scope"],
|
||||||
|
{"remote": "prgs", "org": "Scaled-Tech-Consulting", "repo": "Gitea-Tools"},
|
||||||
|
)
|
||||||
|
self.assertEqual(d["filters"]["issue"], 637)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCthTypeContract(unittest.TestCase):
|
||||||
|
"""The contract has one authority; the read path consults it."""
|
||||||
|
|
||||||
|
def test_declared_types_are_known(self):
|
||||||
|
for cth_type in CTH_TYPES:
|
||||||
|
self.assertTrue(is_known_cth_type(cth_type))
|
||||||
|
|
||||||
|
def test_undeclared_types_are_not_known(self):
|
||||||
|
for value in ("", None, "Made Up", SYNTHETIC_SECRET_40_HEX, "author handoff"):
|
||||||
|
self.assertFalse(is_known_cth_type(value))
|
||||||
|
|
||||||
|
def test_parse_reports_contract_membership(self):
|
||||||
|
known = parse_cth_comment(_raw_cth_body("Author Handoff"))
|
||||||
|
self.assertTrue(known["cth_type_known"])
|
||||||
|
self.assertEqual(known["cth_type"], "Author Handoff")
|
||||||
|
unknown = parse_cth_comment(_raw_cth_body("Not A Real Type"))
|
||||||
|
# Parsing stays total: the type is still reported, just not endorsed.
|
||||||
|
self.assertEqual(unknown["cth_type"], "Not A Real Type")
|
||||||
|
self.assertFalse(unknown["cth_type_known"])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+135
-13
@@ -203,6 +203,28 @@ _REF_SHA_SHAPE = re.compile(r"^[0-9a-f]{7,40}$")
|
|||||||
# material wherever it appears, never as an identifier.
|
# material wherever it appears, never as an identifier.
|
||||||
_BARE_SECRET_SHAPE = re.compile(r"^[0-9a-f]{32,}$")
|
_BARE_SECRET_SHAPE = re.compile(r"^[0-9a-f]{32,}$")
|
||||||
|
|
||||||
|
# An event type reads like an identifier, but a stored one is externally
|
||||||
|
# influenced: any producer that writes the control-plane ``events`` table
|
||||||
|
# chooses the string. It reaches ``to_dict`` verbatim, so it is validated here
|
||||||
|
# rather than trusted because of where it came from.
|
||||||
|
_CP_EVENT_TYPE_SHAPE = re.compile(r"^[A-Za-z][A-Za-z0-9._:+-]{0,63}$")
|
||||||
|
|
||||||
|
# Emitted in place of a value that cannot be proven safe. Deliberately not a
|
||||||
|
# plausible workflow type: an unsafe value is refused, never quietly rewritten
|
||||||
|
# into a different valid-looking one that would misdescribe the record.
|
||||||
|
UNSAFE_EVENT_TYPE = "unsafe:redacted"
|
||||||
|
|
||||||
|
# Emitted for a CTH heading that is not a declared member of ``CTH_TYPES``. The
|
||||||
|
# contract is enforced on write (``format_cth_body``) and on assess; the read
|
||||||
|
# path the timeline uses enforces it too rather than assuming it was.
|
||||||
|
UNKNOWN_HANDOFF_EVENT_TYPE = "handoff:unrecognized"
|
||||||
|
|
||||||
|
# A source record id is a plain integer in both sources it comes from: the
|
||||||
|
# control-plane ``events`` primary key and a Gitea comment id. ``event_key`` is
|
||||||
|
# serialized verbatim and is the pagination tiebreak, so anything else is
|
||||||
|
# refused rather than interpolated into it.
|
||||||
|
_RECORD_ID_SHAPE = re.compile(r"^[0-9]{1,19}$")
|
||||||
|
|
||||||
|
|
||||||
def _kind_to_numbers(kind: str | None, number: int | None) -> tuple[int | None, int | None]:
|
def _kind_to_numbers(kind: str | None, number: int | None) -> tuple[int | None, int | None]:
|
||||||
"""Map a control-plane work-item (kind, number) to (issue_no, pr_no)."""
|
"""Map a control-plane work-item (kind, number) to (issue_no, pr_no)."""
|
||||||
@@ -297,6 +319,64 @@ def _safe_session_id(value: Any) -> str | None:
|
|||||||
return text if _redact(text) == text else None
|
return text if _redact(text) == text else None
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_record_id(value: Any) -> str | None:
|
||||||
|
"""Return a source record id only when it is a plain numeric identifier.
|
||||||
|
|
||||||
|
``event_key`` is serialized verbatim and is the deterministic pagination
|
||||||
|
tiebreak, so an id is interpolated into it only when it has the shape both
|
||||||
|
real sources actually produce. A record whose identity cannot be trusted is
|
||||||
|
refused by the caller rather than keyed on.
|
||||||
|
"""
|
||||||
|
if value is None or isinstance(value, bool):
|
||||||
|
return None
|
||||||
|
if isinstance(value, int):
|
||||||
|
return str(value)
|
||||||
|
text = str(value).strip()
|
||||||
|
return text if _RECORD_ID_SHAPE.match(text) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_cp_event_type(value: Any) -> tuple[str, bool]:
|
||||||
|
"""Validate a stored control-plane event type. Returns ``(type, unsafe)``.
|
||||||
|
|
||||||
|
The stored value is externally influenced — whichever producer wrote the
|
||||||
|
``events`` row chose the string — and ``to_dict`` serializes it verbatim, so
|
||||||
|
it passes a boundary of its own instead of relying on the one ``message``
|
||||||
|
passes. A value survives only when it is an ordinary identifier, is not a
|
||||||
|
bare secret-shaped hex run, and is unchanged by a redaction pass. Anything
|
||||||
|
else fails closed to :data:`UNSAFE_EVENT_TYPE`: the record stays visible as
|
||||||
|
an audit entry, but the value itself is never republished — not verbatim,
|
||||||
|
not partially sanitized, and not rewritten into some other valid-looking
|
||||||
|
type that would misdescribe what happened.
|
||||||
|
"""
|
||||||
|
text = ("" if value is None else str(value)).strip()
|
||||||
|
if not text:
|
||||||
|
return ("", False)
|
||||||
|
if _BARE_SECRET_SHAPE.match(text):
|
||||||
|
return (UNSAFE_EVENT_TYPE, True)
|
||||||
|
if not _CP_EVENT_TYPE_SHAPE.match(text):
|
||||||
|
return (UNSAFE_EVENT_TYPE, True)
|
||||||
|
if _redact(text) != text:
|
||||||
|
return (UNSAFE_EVENT_TYPE, True)
|
||||||
|
return (text, False)
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_echo(value: Any) -> Any:
|
||||||
|
"""Guard a scalar that is echoed back rather than derived from a record.
|
||||||
|
|
||||||
|
Query scope and filter values are caller-supplied and are reflected in the
|
||||||
|
response so an operator can see what was asked. Reflection is still
|
||||||
|
emission: a value redaction would alter, or a bare secret-shaped hex run, is
|
||||||
|
replaced by the placeholder instead of being echoed verbatim. Ordinary
|
||||||
|
scope and filter values pass through untouched.
|
||||||
|
"""
|
||||||
|
if value is None or isinstance(value, (int, bool)):
|
||||||
|
return value
|
||||||
|
text = str(value)
|
||||||
|
if _BARE_SECRET_SHAPE.match(text.strip()):
|
||||||
|
return console_redaction.REDACTED
|
||||||
|
return _redact(text)
|
||||||
|
|
||||||
|
|
||||||
def adapt_cp_events(rows: Iterable[dict[str, Any]]) -> list[WorkflowEvent]:
|
def adapt_cp_events(rows: Iterable[dict[str, Any]]) -> list[WorkflowEvent]:
|
||||||
"""Adapt control-plane ``events`` rows (joined to work_items) into events.
|
"""Adapt control-plane ``events`` rows (joined to work_items) into events.
|
||||||
|
|
||||||
@@ -307,14 +387,20 @@ def adapt_cp_events(rows: Iterable[dict[str, Any]]) -> list[WorkflowEvent]:
|
|||||||
events: list[WorkflowEvent] = []
|
events: list[WorkflowEvent] = []
|
||||||
for row in rows or []:
|
for row in rows or []:
|
||||||
try:
|
try:
|
||||||
event_id = row.get("event_id")
|
event_id = _safe_record_id(row.get("event_id"))
|
||||||
event_type = (row.get("event_type") or "").strip()
|
raw_event_type = (row.get("event_type") or "").strip()
|
||||||
if event_id is None or not event_type:
|
if event_id is None or not raw_event_type:
|
||||||
continue
|
continue
|
||||||
|
# The stored type is source data, not a trusted constant: validate
|
||||||
|
# it before it is serialized, exactly as `message` below is redacted
|
||||||
|
# before it is serialized.
|
||||||
|
event_type, event_type_unsafe = _safe_cp_event_type(raw_event_type)
|
||||||
kind = row.get("kind")
|
kind = row.get("kind")
|
||||||
number = row.get("number")
|
number = row.get("number")
|
||||||
issue_no, pr_no = _kind_to_numbers(kind, number)
|
issue_no, pr_no = _kind_to_numbers(kind, number)
|
||||||
sensitive = any(hint in event_type.lower() for hint in _SENSITIVE_EVENT_HINTS)
|
sensitive = event_type_unsafe or any(
|
||||||
|
hint in raw_event_type.lower() for hint in _SENSITIVE_EVENT_HINTS
|
||||||
|
)
|
||||||
events.append(
|
events.append(
|
||||||
WorkflowEvent(
|
WorkflowEvent(
|
||||||
source=SOURCE_CONTROL_PLANE,
|
source=SOURCE_CONTROL_PLANE,
|
||||||
@@ -355,7 +441,18 @@ def adapt_cth_comments(
|
|||||||
"""
|
"""
|
||||||
# Imported lazily so this module has no import-time dependency on the
|
# Imported lazily so this module has no import-time dependency on the
|
||||||
# handoff parser when only the control-plane adapter is used.
|
# handoff parser when only the control-plane adapter is used.
|
||||||
from canonical_thread_handoff import parse_cth_comment
|
from canonical_thread_handoff import is_known_cth_type, parse_cth_comment
|
||||||
|
|
||||||
|
# ``kind``/``number`` are interpolated into event_key and correlation_id, so
|
||||||
|
# they are normalised once here. A scope this adapter cannot express is
|
||||||
|
# refused outright rather than serialized into an identifier.
|
||||||
|
kind = (kind or "").strip().lower()
|
||||||
|
if kind not in ("issue", "pr"):
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
number = int(number)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return []
|
||||||
|
|
||||||
issue_no, pr_no = _kind_to_numbers(kind, number)
|
issue_no, pr_no = _kind_to_numbers(kind, number)
|
||||||
correlation = _correlation_for(kind, number)
|
correlation = _correlation_for(kind, number)
|
||||||
@@ -367,8 +464,17 @@ def adapt_cth_comments(
|
|||||||
if not parsed:
|
if not parsed:
|
||||||
continue
|
continue
|
||||||
fields = parsed.get("fields") or {}
|
fields = parsed.get("fields") or {}
|
||||||
cth_type = parsed.get("cth_type") or "handoff"
|
cth_type = parsed.get("cth_type") or ""
|
||||||
comment_id = comment.get("id")
|
comment_id = _safe_record_id(comment.get("id"))
|
||||||
|
if comment_id is None:
|
||||||
|
continue
|
||||||
|
# The CTH heading is free text: the parser accepts whatever follows
|
||||||
|
# "## CTH:", and only the write and assess paths check it against
|
||||||
|
# the contract. Check it here too — an unrecognised heading is
|
||||||
|
# reported as such rather than serialized into event_type, so
|
||||||
|
# arbitrary, malformed, or secret-shaped heading content has no way
|
||||||
|
# through. Declared types are preserved exactly.
|
||||||
|
cth_type_known = is_known_cth_type(cth_type)
|
||||||
# Redaction runs first, and every derived value is taken from the
|
# Redaction runs first, and every derived value is taken from the
|
||||||
# redacted text — deriving evidence refs from the raw proof would
|
# redacted text — deriving evidence refs from the raw proof would
|
||||||
# re-emit exactly what redaction was about to remove.
|
# re-emit exactly what redaction was about to remove.
|
||||||
@@ -381,7 +487,11 @@ def adapt_cth_comments(
|
|||||||
events.append(
|
events.append(
|
||||||
WorkflowEvent(
|
WorkflowEvent(
|
||||||
source=SOURCE_GITEA_HANDOFF,
|
source=SOURCE_GITEA_HANDOFF,
|
||||||
event_type=f"handoff:{cth_type}",
|
event_type=(
|
||||||
|
f"handoff:{cth_type.strip()}"
|
||||||
|
if cth_type_known
|
||||||
|
else UNKNOWN_HANDOFF_EVENT_TYPE
|
||||||
|
),
|
||||||
event_key=f"cth:{kind}:{number}:{comment_id}",
|
event_key=f"cth:{kind}:{number}:{comment_id}",
|
||||||
timestamp=_parse_ts(comment.get("created_at")),
|
timestamp=_parse_ts(comment.get("created_at")),
|
||||||
actor=_redact((comment.get("user") or {}).get("login")),
|
actor=_redact((comment.get("user") or {}).get("login")),
|
||||||
@@ -396,7 +506,7 @@ def adapt_cth_comments(
|
|||||||
message=next_action or _redact(fields.get("status")),
|
message=next_action or _redact(fields.get("status")),
|
||||||
correlation_id=correlation,
|
correlation_id=correlation,
|
||||||
evidence_refs=refs,
|
evidence_refs=refs,
|
||||||
sensitive=refs_dropped,
|
sensitive=refs_dropped or not cth_type_known,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -453,7 +563,10 @@ def _cp_status(*, ok: bool, reason: str | None = None, count: int = 0) -> Source
|
|||||||
return SourceStatus(
|
return SourceStatus(
|
||||||
SOURCE_CONTROL_PLANE,
|
SOURCE_CONTROL_PLANE,
|
||||||
ok=ok,
|
ok=ok,
|
||||||
reason=reason,
|
# A failure reason is serialized like any other field and is often an
|
||||||
|
# exception string carrying a path or a transport error, so it crosses
|
||||||
|
# the redaction boundary too. Static reasons pass through unchanged.
|
||||||
|
reason=_redact(reason),
|
||||||
count=count,
|
count=count,
|
||||||
supported_filters=_SOURCE_FILTER_SUPPORT[SOURCE_CONTROL_PLANE],
|
supported_filters=_SOURCE_FILTER_SUPPORT[SOURCE_CONTROL_PLANE],
|
||||||
)
|
)
|
||||||
@@ -463,7 +576,9 @@ def _handoff_status(*, ok: bool, reason: str | None = None, count: int = 0) -> S
|
|||||||
return SourceStatus(
|
return SourceStatus(
|
||||||
SOURCE_GITEA_HANDOFF,
|
SOURCE_GITEA_HANDOFF,
|
||||||
ok=ok,
|
ok=ok,
|
||||||
reason=reason,
|
# Same boundary as the control-plane status: this reason can quote an
|
||||||
|
# error raised by a live authenticated fetch.
|
||||||
|
reason=_redact(reason),
|
||||||
count=count,
|
count=count,
|
||||||
supported_filters=_SOURCE_FILTER_SUPPORT[SOURCE_GITEA_HANDOFF],
|
supported_filters=_SOURCE_FILTER_SUPPORT[SOURCE_GITEA_HANDOFF],
|
||||||
)
|
)
|
||||||
@@ -623,8 +738,15 @@ class TimelineSnapshot:
|
|||||||
"ok": self.ok,
|
"ok": self.ok,
|
||||||
"error": self.error,
|
"error": self.error,
|
||||||
"schema_version": self.schema_version,
|
"schema_version": self.schema_version,
|
||||||
"scope": {"remote": self.remote, "org": self.org, "repo": self.repo},
|
# Scope and filters are echoed caller input, not derived record
|
||||||
"filters": self.filters,
|
# data. Reflecting a value is still emitting it, so both cross the
|
||||||
|
# same boundary; ordinary scope and filter values are unchanged.
|
||||||
|
"scope": {
|
||||||
|
"remote": _safe_echo(self.remote),
|
||||||
|
"org": _safe_echo(self.org),
|
||||||
|
"repo": _safe_echo(self.repo),
|
||||||
|
},
|
||||||
|
"filters": {key: _safe_echo(value) for key, value in self.filters.items()},
|
||||||
"sources": [s.to_dict() for s in self.sources],
|
"sources": [s.to_dict() for s in self.sources],
|
||||||
**self.page.to_dict(),
|
**self.page.to_dict(),
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user