feat(webui): model usage, token cost, latency, and performance analytics (Closes #651)
This commit is contained in:
+198
-1
@@ -31,7 +31,7 @@ from typing import Any, Iterator, Sequence
|
||||
|
||||
import dependency_graph
|
||||
|
||||
SCHEMA_VERSION = 4
|
||||
SCHEMA_VERSION = 5
|
||||
|
||||
# Assignable work kinds only — raw monitoring incidents are never work items.
|
||||
WORK_KINDS = frozenset({"issue", "pr"})
|
||||
@@ -186,6 +186,34 @@ CREATE INDEX IF NOT EXISTS idx_dependency_edges_target
|
||||
ON dependency_edges(remote, org, repo, target_kind, target_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_assignments_session ON assignments(session_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_incident_gitea ON incident_links(gitea_org, gitea_repo, gitea_issue_number);
|
||||
|
||||
-- Model usage, token cost, latency, and performance events (#651)
|
||||
CREATE TABLE IF NOT EXISTS usage_events (
|
||||
usage_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT,
|
||||
remote TEXT NOT NULL DEFAULT 'dadeschools',
|
||||
org TEXT NOT NULL DEFAULT '',
|
||||
repo TEXT NOT NULL DEFAULT '',
|
||||
project_id TEXT,
|
||||
role TEXT NOT NULL DEFAULT 'unknown',
|
||||
model TEXT NOT NULL DEFAULT 'unknown',
|
||||
issue_number INTEGER,
|
||||
pr_number INTEGER,
|
||||
stage TEXT NOT NULL DEFAULT 'unknown',
|
||||
input_tokens INTEGER,
|
||||
output_tokens INTEGER,
|
||||
total_tokens INTEGER,
|
||||
estimated_cost_usd REAL,
|
||||
latency_ms INTEGER,
|
||||
duration_ms INTEGER,
|
||||
status TEXT NOT NULL DEFAULT 'success',
|
||||
metadata TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_usage_events_scope ON usage_events(remote, org, repo);
|
||||
CREATE INDEX IF NOT EXISTS idx_usage_events_role_model ON usage_events(role, model);
|
||||
CREATE INDEX IF NOT EXISTS idx_usage_events_stage ON usage_events(stage);
|
||||
"""
|
||||
|
||||
|
||||
@@ -339,6 +367,7 @@ class ControlPlaneDB:
|
||||
self._migrate_incident_links_null_scope(conn)
|
||||
self._migrate_lease_lifecycle_columns(conn)
|
||||
self._migrate_session_ownership_columns(conn)
|
||||
self._migrate_usage_events_table(conn)
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)",
|
||||
("schema_version", str(SCHEMA_VERSION)),
|
||||
@@ -518,6 +547,174 @@ class ControlPlaneDB:
|
||||
f"UPDATE incident_links SET {col} = '' WHERE {col} IS NULL"
|
||||
)
|
||||
|
||||
def _migrate_usage_events_table(self, conn: sqlite3.Connection) -> None:
|
||||
"""Create usage_events table and indexes if they do not exist (#651)."""
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS usage_events (
|
||||
usage_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT,
|
||||
remote TEXT NOT NULL DEFAULT 'dadeschools',
|
||||
org TEXT NOT NULL DEFAULT '',
|
||||
repo TEXT NOT NULL DEFAULT '',
|
||||
project_id TEXT,
|
||||
role TEXT NOT NULL DEFAULT 'unknown',
|
||||
model TEXT NOT NULL DEFAULT 'unknown',
|
||||
issue_number INTEGER,
|
||||
pr_number INTEGER,
|
||||
stage TEXT NOT NULL DEFAULT 'unknown',
|
||||
input_tokens INTEGER,
|
||||
output_tokens INTEGER,
|
||||
total_tokens INTEGER,
|
||||
estimated_cost_usd REAL,
|
||||
latency_ms INTEGER,
|
||||
duration_ms INTEGER,
|
||||
status TEXT NOT NULL DEFAULT 'success',
|
||||
metadata TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_events_scope ON usage_events(remote, org, repo);")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_events_role_model ON usage_events(role, model);")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_events_stage ON usage_events(stage);")
|
||||
|
||||
def record_usage_event(
|
||||
self,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
remote: str = "dadeschools",
|
||||
org: str = "",
|
||||
repo: str = "",
|
||||
project_id: str | None = None,
|
||||
role: str = "unknown",
|
||||
model: str = "unknown",
|
||||
issue_number: int | None = None,
|
||||
pr_number: int | None = None,
|
||||
stage: str = "unknown",
|
||||
input_tokens: int | None = None,
|
||||
output_tokens: int | None = None,
|
||||
total_tokens: int | None = None,
|
||||
estimated_cost_usd: float | None = None,
|
||||
latency_ms: int | None = None,
|
||||
duration_ms: int | None = None,
|
||||
status: str = "success",
|
||||
metadata: str | dict[str, Any] | None = None,
|
||||
created_at: str | None = None,
|
||||
) -> int:
|
||||
"""Record a model usage, token cost, latency, or stage performance event (#651)."""
|
||||
ts = created_at or _ts()
|
||||
meta_str: str | None = None
|
||||
if metadata is not None:
|
||||
from webui import console_redaction
|
||||
redacted_meta = console_redaction.redact_payload(metadata)
|
||||
if isinstance(redacted_meta, str):
|
||||
meta_str = redacted_meta
|
||||
else:
|
||||
try:
|
||||
meta_str = json.dumps(redacted_meta, default=str)
|
||||
except Exception:
|
||||
meta_str = str(redacted_meta)
|
||||
|
||||
if total_tokens is None and (input_tokens is not None or output_tokens is not None):
|
||||
total_tokens = (input_tokens or 0) + (output_tokens or 0)
|
||||
|
||||
with self._tx(immediate=True) as conn:
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT INTO usage_events (
|
||||
session_id, remote, org, repo, project_id, role, model,
|
||||
issue_number, pr_number, stage, input_tokens, output_tokens,
|
||||
total_tokens, estimated_cost_usd, latency_ms, duration_ms,
|
||||
status, metadata, created_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
session_id,
|
||||
remote,
|
||||
org,
|
||||
repo,
|
||||
project_id,
|
||||
role,
|
||||
model,
|
||||
issue_number,
|
||||
pr_number,
|
||||
stage,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
total_tokens,
|
||||
estimated_cost_usd,
|
||||
latency_ms,
|
||||
duration_ms,
|
||||
status,
|
||||
meta_str,
|
||||
ts,
|
||||
),
|
||||
)
|
||||
return cursor.lastrowid
|
||||
|
||||
def query_usage_events(
|
||||
self,
|
||||
*,
|
||||
remote: str | None = None,
|
||||
org: str | None = None,
|
||||
repo: str | None = None,
|
||||
project_id: str | None = None,
|
||||
role: str | None = None,
|
||||
model: str | None = None,
|
||||
issue_number: int | None = None,
|
||||
pr_number: int | None = None,
|
||||
stage: str | None = None,
|
||||
session_id: str | None = None,
|
||||
limit: int = 500,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Query stored usage events matching filters (#651)."""
|
||||
conditions = []
|
||||
params = []
|
||||
if remote:
|
||||
conditions.append("remote = ?")
|
||||
params.append(remote)
|
||||
if org:
|
||||
conditions.append("org = ?")
|
||||
params.append(org)
|
||||
if repo:
|
||||
conditions.append("repo = ?")
|
||||
params.append(repo)
|
||||
if project_id:
|
||||
conditions.append("project_id = ?")
|
||||
params.append(project_id)
|
||||
if role:
|
||||
conditions.append("role = ?")
|
||||
params.append(role)
|
||||
if model:
|
||||
conditions.append("model = ?")
|
||||
params.append(model)
|
||||
if issue_number is not None:
|
||||
conditions.append("issue_number = ?")
|
||||
params.append(issue_number)
|
||||
if pr_number is not None:
|
||||
conditions.append("pr_number = ?")
|
||||
params.append(pr_number)
|
||||
if stage:
|
||||
conditions.append("stage = ?")
|
||||
params.append(stage)
|
||||
if session_id:
|
||||
conditions.append("session_id = ?")
|
||||
params.append(session_id)
|
||||
|
||||
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
|
||||
sql = f"""
|
||||
SELECT * FROM usage_events
|
||||
{where_clause}
|
||||
ORDER BY usage_id ASC
|
||||
LIMIT ? OFFSET ?
|
||||
"""
|
||||
params.extend([limit, offset])
|
||||
|
||||
with self._tx(immediate=False) as conn:
|
||||
cursor = conn.execute(sql, params)
|
||||
rows = cursor.fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
# ── sessions ──────────────────────────────────────────────────────────
|
||||
|
||||
def upsert_session(
|
||||
|
||||
Reference in New Issue
Block a user