202 lines
7.4 KiB
Python
202 lines
7.4 KiB
Python
"""HTML views for the Model Usage & Performance Analytics console (#651)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from webui.analytics_loader import AnalyticsSnapshot, GroupMetrics, UsageEvent
|
|
from webui.layout import render_page
|
|
|
|
|
|
def _render_badge(text: str, badge_type: str = "muted") -> str:
|
|
return f'<span class="badge badge-{badge_type}">{text}</span>'
|
|
|
|
|
|
def _render_group_table(title: str, groups: dict[str, GroupMetrics], key_header: str = "Group") -> str:
|
|
if not groups:
|
|
return (
|
|
f"<h3>{title}</h3>"
|
|
'<div class="card"><p class="muted">No telemetry events recorded for this dimension.</p></div>'
|
|
)
|
|
|
|
rows = []
|
|
for key, g in sorted(groups.items(), key=lambda x: x[1].total_events, reverse=True):
|
|
cost_cell = (
|
|
f'<span class="accent">{g.display_cost}</span>'
|
|
if g.events_with_cost > 0
|
|
else _render_badge("Unknown")
|
|
)
|
|
tokens_cell = (
|
|
g.display_tokens
|
|
if g.events_with_tokens > 0
|
|
else _render_badge("Unknown")
|
|
)
|
|
lat_p50 = (
|
|
g.display_latency_p50
|
|
if g.events_with_latency > 0
|
|
else _render_badge("Unknown")
|
|
)
|
|
lat_p90 = (
|
|
g.display_latency_p90
|
|
if g.events_with_latency > 0
|
|
else _render_badge("Unknown")
|
|
)
|
|
dur_avg = (
|
|
g.display_duration_avg
|
|
if g.events_with_duration > 0
|
|
else _render_badge("Unknown")
|
|
)
|
|
|
|
rows.append(
|
|
"<tr>"
|
|
f"<td><strong>{key}</strong></td>"
|
|
f"<td>{g.total_events}</td>"
|
|
f"<td>{tokens_cell}</td>"
|
|
f"<td>{cost_cell}</td>"
|
|
f"<td>{lat_p50}</td>"
|
|
f"<td>{lat_p90}</td>"
|
|
f"<td>{dur_avg}</td>"
|
|
"</tr>"
|
|
)
|
|
|
|
rows_html = "".join(rows)
|
|
return f"""
|
|
<h3>{title}</h3>
|
|
<div class="card" style="overflow-x: auto;">
|
|
<table class="data-table">
|
|
<thead>
|
|
<tr>
|
|
<th>{key_header}</th>
|
|
<th>Events</th>
|
|
<th>Total Tokens</th>
|
|
<th>Est. Cost</th>
|
|
<th>Latency (p50)</th>
|
|
<th>Latency (p90)</th>
|
|
<th>Avg Stage Duration</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{rows_html}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
"""
|
|
|
|
|
|
def _render_events_table(events: tuple[UsageEvent, ...]) -> str:
|
|
if not events:
|
|
return (
|
|
"<h3>Recent Usage & Instrumentation Events</h3>"
|
|
'<div class="card"><p class="muted">No individual telemetry events recorded yet. Opt-in instrumentation via session logging or POST /api/v1/analytics/usage.</p></div>'
|
|
)
|
|
|
|
rows = []
|
|
for e in list(events)[-50:]: # Display latest 50
|
|
work_item = f"issue #{e.issue_number}" if e.issue_number else (f"pr #{e.pr_number}" if e.pr_number else "unlinked")
|
|
tokens = f"{e.total_tokens:,}" if e.total_tokens is not None else _render_badge("Unknown")
|
|
cost = f"${e.estimated_cost_usd:.4f}" if e.estimated_cost_usd is not None else _render_badge("Unknown")
|
|
latency = f"{e.latency_ms} ms" if e.latency_ms is not None else _render_badge("Unknown")
|
|
duration = f"{e.duration_ms} ms" if e.duration_ms is not None else _render_badge("Unknown")
|
|
status_badge = _render_badge(e.status, "success" if e.status == "success" else "danger")
|
|
|
|
rows.append(
|
|
"<tr>"
|
|
f"<td>#{e.usage_id}</td>"
|
|
f"<td><small>{e.created_at}</small></td>"
|
|
f"<td><span class=\"badge\">{e.role}</span></td>"
|
|
f"<td><strong>{e.model}</strong></td>"
|
|
f"<td>{e.stage}</td>"
|
|
f"<td>{work_item}</td>"
|
|
f"<td>{tokens}</td>"
|
|
f"<td>{cost}</td>"
|
|
f"<td>{latency}</td>"
|
|
f"<td>{duration}</td>"
|
|
f"<td>{status_badge}</td>"
|
|
"</tr>"
|
|
)
|
|
|
|
rows_html = "".join(rows)
|
|
return f"""
|
|
<h3>Recent Telemetry Events</h3>
|
|
<div class="card" style="overflow-x: auto;">
|
|
<table class="data-table">
|
|
<thead>
|
|
<tr>
|
|
<th>ID</th>
|
|
<th>Timestamp</th>
|
|
<th>Role</th>
|
|
<th>Model</th>
|
|
<th>Stage</th>
|
|
<th>Work Item</th>
|
|
<th>Tokens</th>
|
|
<th>Cost</th>
|
|
<th>Latency</th>
|
|
<th>Duration</th>
|
|
<th>Status</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{rows_html}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
"""
|
|
|
|
|
|
def render_analytics_page(snapshot: AnalyticsSnapshot) -> str:
|
|
"""Render the main Model Usage & Performance Analytics console page."""
|
|
summary = snapshot.overall_summary
|
|
|
|
kpi_tokens = summary.display_tokens if summary.events_with_tokens > 0 else _render_badge("Unknown")
|
|
kpi_cost = summary.display_cost if summary.events_with_cost > 0 else _render_badge("Unknown")
|
|
kpi_lat_p50 = summary.display_latency_p50 if summary.events_with_latency > 0 else _render_badge("Unknown")
|
|
kpi_dur_avg = summary.display_duration_avg if summary.events_with_duration > 0 else _render_badge("Unknown")
|
|
|
|
status_notice = ""
|
|
if not snapshot.ok:
|
|
status_notice = (
|
|
f'<div class="card warning-card"><strong>Degraded Data Source:</strong> {snapshot.reason}</div>'
|
|
)
|
|
|
|
body_html = f"""
|
|
<h2>Model Usage & Performance Analytics (Phase 4)</h2>
|
|
<p class="muted">
|
|
Durable console analytics for model usage, token cost, latency percentiles, and workflow-stage performance correlated to issues, PRs, and worker roles.
|
|
</p>
|
|
|
|
{status_notice}
|
|
|
|
<div class="notice-card" style="background: rgba(91, 159, 212, 0.1); border: 1px solid var(--border); padding: 0.75rem 1rem; border-radius: 6px; margin-bottom: 1.5rem;">
|
|
<small><strong>Note on telemetry fidelity:</strong> Missing data or untracked metrics are explicitly labeled as <em>Unknown</em>. No token costs or latency metrics are zero-fabricated.</small>
|
|
</div>
|
|
|
|
<div class="card-grid" style="display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 1rem; margin-bottom: 1.5rem;">
|
|
<div class="card">
|
|
<span class="muted" style="font-size: 0.85rem;">Total Events</span>
|
|
<h3 style="margin: 0.25rem 0 0 0;">{summary.total_events}</h3>
|
|
</div>
|
|
<div class="card">
|
|
<span class="muted" style="font-size: 0.85rem;">Total Tokens</span>
|
|
<h3 style="margin: 0.25rem 0 0 0;">{kpi_tokens}</h3>
|
|
</div>
|
|
<div class="card">
|
|
<span class="muted" style="font-size: 0.85rem;">Est. Token Cost</span>
|
|
<h3 style="margin: 0.25rem 0 0 0;">{kpi_cost}</h3>
|
|
</div>
|
|
<div class="card">
|
|
<span class="muted" style="font-size: 0.85rem;">Latency (p50)</span>
|
|
<h3 style="margin: 0.25rem 0 0 0;">{kpi_lat_p50}</h3>
|
|
</div>
|
|
<div class="card">
|
|
<span class="muted" style="font-size: 0.85rem;">Avg Stage Duration</span>
|
|
<h3 style="margin: 0.25rem 0 0 0;">{kpi_dur_avg}</h3>
|
|
</div>
|
|
</div>
|
|
|
|
{_render_group_table("Usage & Cost by Model", snapshot.by_model, "Model")}
|
|
{_render_group_table("Performance by Workflow Stage", snapshot.by_stage, "Stage")}
|
|
{_render_group_table("Usage & Cost by Role", snapshot.by_role, "Role")}
|
|
{_render_group_table("Work Item Analytics", snapshot.by_work_item, "Work Item")}
|
|
{_render_events_table(snapshot.events)}
|
|
"""
|
|
|
|
return render_page(title="Model Usage & Performance Analytics", body_html=body_html)
|