feat(webui): model usage, token cost, latency, and performance analytics (Closes #651)
This commit is contained in:
@@ -0,0 +1,428 @@
|
||||
"""Model usage, token cost, latency, and workflow-performance analytics (#651, Phase 4).
|
||||
|
||||
Ingests session instrumentation metrics, aggregates usage/cost/latency percentiles
|
||||
by project, role, model, issue/PR, and stage, enforcing secret redaction and
|
||||
explicitly rendering missing metrics as "Unknown" without zero-fabrication.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any, Sequence
|
||||
|
||||
import control_plane_db
|
||||
from webui import console_redaction
|
||||
|
||||
ANALYTICS_SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UsageEvent:
|
||||
usage_id: int
|
||||
session_id: str | None
|
||||
remote: str
|
||||
org: str
|
||||
repo: str
|
||||
project_id: str | None
|
||||
role: str
|
||||
model: str
|
||||
issue_number: int | None
|
||||
pr_number: int | None
|
||||
stage: str
|
||||
input_tokens: int | None
|
||||
output_tokens: int | None
|
||||
total_tokens: int | None
|
||||
estimated_cost_usd: float | None
|
||||
latency_ms: int | None
|
||||
duration_ms: int | None
|
||||
status: str
|
||||
metadata: str | None
|
||||
created_at: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
d = asdict(self)
|
||||
if d["metadata"]:
|
||||
d["metadata"] = console_redaction.redact_text(d["metadata"])
|
||||
return d
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GroupMetrics:
|
||||
name: str
|
||||
total_events: int
|
||||
events_with_tokens: int
|
||||
input_tokens: int | None
|
||||
output_tokens: int | None
|
||||
total_tokens: int | None
|
||||
events_with_cost: int
|
||||
estimated_cost_usd: float | None
|
||||
events_with_latency: int
|
||||
latency_p50_ms: float | None
|
||||
latency_p90_ms: float | None
|
||||
latency_p95_ms: float | None
|
||||
latency_p99_ms: float | None
|
||||
latency_avg_ms: float | None
|
||||
events_with_duration: int
|
||||
duration_avg_ms: float | None
|
||||
display_tokens: str
|
||||
display_cost: str
|
||||
display_latency_p50: str
|
||||
display_latency_p90: str
|
||||
display_duration_avg: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AnalyticsSnapshot:
|
||||
ok: bool
|
||||
reason: str
|
||||
schema_version: int
|
||||
remote: str
|
||||
org: str
|
||||
repo: str
|
||||
total_events: int
|
||||
overall_summary: GroupMetrics
|
||||
by_project: dict[str, GroupMetrics]
|
||||
by_role: dict[str, GroupMetrics]
|
||||
by_model: dict[str, GroupMetrics]
|
||||
by_work_item: dict[str, GroupMetrics]
|
||||
by_stage: dict[str, GroupMetrics]
|
||||
events: tuple[UsageEvent, ...]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"ok": self.ok,
|
||||
"reason": self.reason,
|
||||
"schema_version": self.schema_version,
|
||||
"remote": self.remote,
|
||||
"org": self.org,
|
||||
"repo": self.repo,
|
||||
"total_events": self.total_events,
|
||||
"overall_summary": self.overall_summary.to_dict(),
|
||||
"by_project": {k: v.to_dict() for k, v in self.by_project.items()},
|
||||
"by_role": {k: v.to_dict() for k, v in self.by_role.items()},
|
||||
"by_model": {k: v.to_dict() for k, v in self.by_model.items()},
|
||||
"by_work_item": {k: v.to_dict() for k, v in self.by_work_item.items()},
|
||||
"by_stage": {k: v.to_dict() for k, v in self.by_stage.items()},
|
||||
"events": [e.to_dict() for e in self.events],
|
||||
}
|
||||
|
||||
|
||||
def compute_percentile(values: Sequence[float | int], percentile: float) -> float | None:
|
||||
if not values:
|
||||
return None
|
||||
sorted_vals = sorted(values)
|
||||
n = len(sorted_vals)
|
||||
if n == 1:
|
||||
return float(sorted_vals[0])
|
||||
k = (n - 1) * (percentile / 100.0)
|
||||
f = math.floor(k)
|
||||
c = math.ceil(k)
|
||||
if f == c:
|
||||
return float(sorted_vals[int(f)])
|
||||
d0 = sorted_vals[int(f)] * (c - k)
|
||||
d1 = sorted_vals[int(c)] * (k - f)
|
||||
return float(d0 + d1)
|
||||
|
||||
|
||||
def aggregate_events(group_name: str, events: Sequence[UsageEvent]) -> GroupMetrics:
|
||||
total_events = len(events)
|
||||
if total_events == 0:
|
||||
return GroupMetrics(
|
||||
name=group_name,
|
||||
total_events=0,
|
||||
events_with_tokens=0,
|
||||
input_tokens=None,
|
||||
output_tokens=None,
|
||||
total_tokens=None,
|
||||
events_with_cost=0,
|
||||
estimated_cost_usd=None,
|
||||
events_with_latency=0,
|
||||
latency_p50_ms=None,
|
||||
latency_p90_ms=None,
|
||||
latency_p95_ms=None,
|
||||
latency_p99_ms=None,
|
||||
latency_avg_ms=None,
|
||||
events_with_duration=0,
|
||||
duration_avg_ms=None,
|
||||
display_tokens="Unknown",
|
||||
display_cost="Unknown",
|
||||
display_latency_p50="Unknown",
|
||||
display_latency_p90="Unknown",
|
||||
display_duration_avg="Unknown",
|
||||
)
|
||||
|
||||
token_events = [
|
||||
e for e in events
|
||||
if e.total_tokens is not None or e.input_tokens is not None or e.output_tokens is not None
|
||||
]
|
||||
events_with_tokens = len(token_events)
|
||||
if events_with_tokens > 0:
|
||||
input_tokens = sum(e.input_tokens or 0 for e in token_events)
|
||||
output_tokens = sum(e.output_tokens or 0 for e in token_events)
|
||||
total_tokens = sum(
|
||||
e.total_tokens if e.total_tokens is not None else ((e.input_tokens or 0) + (e.output_tokens or 0))
|
||||
for e in token_events
|
||||
)
|
||||
display_tokens = f"{total_tokens:,}"
|
||||
else:
|
||||
input_tokens = None
|
||||
output_tokens = None
|
||||
total_tokens = None
|
||||
display_tokens = "Unknown"
|
||||
|
||||
cost_events = [e for e in events if e.estimated_cost_usd is not None]
|
||||
events_with_cost = len(cost_events)
|
||||
if events_with_cost > 0:
|
||||
estimated_cost_usd = round(sum(e.estimated_cost_usd for e in cost_events), 6)
|
||||
display_cost = f"${estimated_cost_usd:.4f}"
|
||||
else:
|
||||
estimated_cost_usd = None
|
||||
display_cost = "Unknown"
|
||||
|
||||
latency_vals = [e.latency_ms for e in events if e.latency_ms is not None]
|
||||
events_with_latency = len(latency_vals)
|
||||
if events_with_latency > 0:
|
||||
latency_p50_ms = compute_percentile(latency_vals, 50.0)
|
||||
latency_p90_ms = compute_percentile(latency_vals, 90.0)
|
||||
latency_p95_ms = compute_percentile(latency_vals, 95.0)
|
||||
latency_p99_ms = compute_percentile(latency_vals, 99.0)
|
||||
latency_avg_ms = round(sum(latency_vals) / events_with_latency, 2)
|
||||
display_latency_p50 = f"{round(latency_p50_ms, 1)} ms" if latency_p50_ms is not None else "Unknown"
|
||||
display_latency_p90 = f"{round(latency_p90_ms, 1)} ms" if latency_p90_ms is not None else "Unknown"
|
||||
else:
|
||||
latency_p50_ms = None
|
||||
latency_p90_ms = None
|
||||
latency_p95_ms = None
|
||||
latency_p99_ms = None
|
||||
latency_avg_ms = None
|
||||
display_latency_p50 = "Unknown"
|
||||
display_latency_p90 = "Unknown"
|
||||
|
||||
duration_vals = [e.duration_ms for e in events if e.duration_ms is not None]
|
||||
events_with_duration = len(duration_vals)
|
||||
if events_with_duration > 0:
|
||||
duration_avg_ms = round(sum(duration_vals) / events_with_duration, 2)
|
||||
display_duration_avg = f"{round(duration_avg_ms / 1000.0, 2)} s" if duration_avg_ms >= 1000 else f"{round(duration_avg_ms, 1)} ms"
|
||||
else:
|
||||
duration_avg_ms = None
|
||||
display_duration_avg = "Unknown"
|
||||
|
||||
return GroupMetrics(
|
||||
name=group_name,
|
||||
total_events=total_events,
|
||||
events_with_tokens=events_with_tokens,
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=total_tokens,
|
||||
events_with_cost=events_with_cost,
|
||||
estimated_cost_usd=estimated_cost_usd,
|
||||
events_with_latency=events_with_latency,
|
||||
latency_p50_ms=latency_p50_ms,
|
||||
latency_p90_ms=latency_p90_ms,
|
||||
latency_p95_ms=latency_p95_ms,
|
||||
latency_p99_ms=latency_p99_ms,
|
||||
latency_avg_ms=latency_avg_ms,
|
||||
events_with_duration=events_with_duration,
|
||||
duration_avg_ms=duration_avg_ms,
|
||||
display_tokens=display_tokens,
|
||||
display_cost=display_cost,
|
||||
display_latency_p50=display_latency_p50,
|
||||
display_latency_p90=display_latency_p90,
|
||||
display_duration_avg=display_duration_avg,
|
||||
)
|
||||
|
||||
|
||||
def record_usage(
|
||||
*,
|
||||
db_path: str | None = None,
|
||||
session_id: str | None = None,
|
||||
remote: str = "dadeschools",
|
||||
org: str = "",
|
||||
repo: str = "",
|
||||
project_id: str | None = None,
|
||||
role: str = "unknown",
|
||||
model: str = "unknown",
|
||||
issue_number: int | None = None,
|
||||
pr_number: int | None = None,
|
||||
stage: str = "unknown",
|
||||
input_tokens: int | None = None,
|
||||
output_tokens: int | None = None,
|
||||
total_tokens: int | None = None,
|
||||
estimated_cost_usd: float | None = None,
|
||||
latency_ms: int | None = None,
|
||||
duration_ms: int | None = None,
|
||||
status: str = "success",
|
||||
metadata: str | dict[str, Any] | None = None,
|
||||
created_at: str | None = None,
|
||||
) -> int:
|
||||
"""Ingest/record a single usage event with optional metrics."""
|
||||
db = control_plane_db.ControlPlaneDB(db_path=db_path)
|
||||
return db.record_usage_event(
|
||||
session_id=session_id,
|
||||
remote=remote,
|
||||
org=org,
|
||||
repo=repo,
|
||||
project_id=project_id,
|
||||
role=role,
|
||||
model=model,
|
||||
issue_number=issue_number,
|
||||
pr_number=pr_number,
|
||||
stage=stage,
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=total_tokens,
|
||||
estimated_cost_usd=estimated_cost_usd,
|
||||
latency_ms=latency_ms,
|
||||
duration_ms=duration_ms,
|
||||
status=status,
|
||||
metadata=metadata,
|
||||
created_at=created_at,
|
||||
)
|
||||
|
||||
|
||||
def load_analytics(
|
||||
*,
|
||||
db_path: str | None = None,
|
||||
remote: str | None = None,
|
||||
org: str | None = None,
|
||||
repo: str | None = None,
|
||||
project_id: str | None = None,
|
||||
role: str | None = None,
|
||||
model: str | None = None,
|
||||
stage: str | None = None,
|
||||
issue_number: int | None = None,
|
||||
pr_number: int | None = None,
|
||||
limit: int = 500,
|
||||
) -> AnalyticsSnapshot:
|
||||
"""Load analytics snapshot aggregated by project, role, model, issue/PR, and stage."""
|
||||
remote_filter = (remote or "").strip() or None
|
||||
org_filter = (org or "").strip() or None
|
||||
repo_filter = (repo or "").strip() or None
|
||||
role_filter = (role or "").strip() or None
|
||||
model_filter = (model or "").strip() or None
|
||||
stage_filter = (stage or "").strip() or None
|
||||
|
||||
try:
|
||||
db = control_plane_db.ControlPlaneDB(db_path=db_path)
|
||||
rows = db.query_usage_events(
|
||||
remote=remote_filter,
|
||||
org=org_filter,
|
||||
repo=repo_filter,
|
||||
project_id=project_id,
|
||||
role=role_filter,
|
||||
model=model_filter,
|
||||
stage=stage_filter,
|
||||
issue_number=issue_number,
|
||||
pr_number=pr_number,
|
||||
limit=limit,
|
||||
)
|
||||
except Exception as exc:
|
||||
empty_summary = aggregate_events("Overall", [])
|
||||
return AnalyticsSnapshot(
|
||||
ok=False,
|
||||
reason=f"control_plane_db_unavailable: {exc}",
|
||||
schema_version=ANALYTICS_SCHEMA_VERSION,
|
||||
remote=remote,
|
||||
org=org,
|
||||
repo=repo,
|
||||
total_events=0,
|
||||
overall_summary=empty_summary,
|
||||
by_project={},
|
||||
by_role={},
|
||||
by_model={},
|
||||
by_work_item={},
|
||||
by_stage={},
|
||||
events=(),
|
||||
)
|
||||
|
||||
parsed_events: list[UsageEvent] = []
|
||||
for r in rows:
|
||||
meta = console_redaction.redact_text(r.get("metadata")) if r.get("metadata") else None
|
||||
parsed_events.append(
|
||||
UsageEvent(
|
||||
usage_id=r["usage_id"],
|
||||
session_id=r.get("session_id"),
|
||||
remote=r.get("remote", remote),
|
||||
org=r.get("org", org),
|
||||
repo=r.get("repo", repo),
|
||||
project_id=r.get("project_id"),
|
||||
role=r.get("role", "unknown"),
|
||||
model=r.get("model", "unknown"),
|
||||
issue_number=r.get("issue_number"),
|
||||
pr_number=r.get("pr_number"),
|
||||
stage=r.get("stage", "unknown"),
|
||||
input_tokens=r.get("input_tokens"),
|
||||
output_tokens=r.get("output_tokens"),
|
||||
total_tokens=r.get("total_tokens"),
|
||||
estimated_cost_usd=r.get("estimated_cost_usd"),
|
||||
latency_ms=r.get("latency_ms"),
|
||||
duration_ms=r.get("duration_ms"),
|
||||
status=r.get("status", "success"),
|
||||
metadata=meta,
|
||||
created_at=r.get("created_at", ""),
|
||||
)
|
||||
)
|
||||
|
||||
overall_summary = aggregate_events("Overall", parsed_events)
|
||||
|
||||
# Group by project
|
||||
groups_by_project: dict[str, list[UsageEvent]] = {}
|
||||
for e in parsed_events:
|
||||
key = e.project_id or (f"{e.org}/{e.repo}" if e.org and e.repo else "default")
|
||||
groups_by_project.setdefault(key, []).append(e)
|
||||
by_project = {k: aggregate_events(k, v) for k, v in groups_by_project.items()}
|
||||
|
||||
# Group by role
|
||||
groups_by_role: dict[str, list[UsageEvent]] = {}
|
||||
for e in parsed_events:
|
||||
groups_by_role.setdefault(e.role, []).append(e)
|
||||
by_role = {k: aggregate_events(k, v) for k, v in groups_by_role.items()}
|
||||
|
||||
# Group by model
|
||||
groups_by_model: dict[str, list[UsageEvent]] = {}
|
||||
for e in parsed_events:
|
||||
groups_by_model.setdefault(e.model, []).append(e)
|
||||
by_model = {k: aggregate_events(k, v) for k, v in groups_by_model.items()}
|
||||
|
||||
# Group by work item
|
||||
groups_by_work_item: dict[str, list[UsageEvent]] = {}
|
||||
for e in parsed_events:
|
||||
if e.issue_number:
|
||||
key = f"issue #{e.issue_number}"
|
||||
elif e.pr_number:
|
||||
key = f"pr #{e.pr_number}"
|
||||
else:
|
||||
key = "unlinked"
|
||||
groups_by_work_item.setdefault(key, []).append(e)
|
||||
by_work_item = {k: aggregate_events(k, v) for k, v in groups_by_work_item.items()}
|
||||
|
||||
# Group by stage
|
||||
groups_by_stage: dict[str, list[UsageEvent]] = {}
|
||||
for e in parsed_events:
|
||||
groups_by_stage.setdefault(e.stage, []).append(e)
|
||||
by_stage = {k: aggregate_events(k, v) for k, v in groups_by_stage.items()}
|
||||
|
||||
return AnalyticsSnapshot(
|
||||
ok=True,
|
||||
reason="ok",
|
||||
schema_version=ANALYTICS_SCHEMA_VERSION,
|
||||
remote=remote,
|
||||
org=org,
|
||||
repo=repo,
|
||||
total_events=len(parsed_events),
|
||||
overall_summary=overall_summary,
|
||||
by_project=by_project,
|
||||
by_role=by_role,
|
||||
by_model=by_model,
|
||||
by_work_item=by_work_item,
|
||||
by_stage=by_stage,
|
||||
events=tuple(parsed_events),
|
||||
)
|
||||
|
||||
|
||||
def snapshot_to_dict(snapshot: AnalyticsSnapshot) -> dict[str, Any]:
|
||||
return snapshot.to_dict()
|
||||
Reference in New Issue
Block a user