Files
Gitea-Tools/webui/insights_loader.py
T
sysadminandClaude Opus 4.8 983e8ac2c7 feat(webui): AI-provider connections and evidence-backed insights (Closes #650)
Add Phase 4 advisory surfaces for declared AI-provider connection status
(no secrets, no live probe claims) and operational insights derived only
from durable traffic, health, provider, and analytics evidence.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
2026-07-25 16:53:36 -04:00

714 lines
24 KiB
Python

"""AI-provider connections and evidence-backed operational insights (#650, Phase 4).
Operators need two related, **advisory** surfaces:
1. **Provider connection status** — which AI runtimes are *declared* in the
worker registry (#798), without ever exposing API keys or inventing a live
probe that this process cannot perform.
2. **Evidence-backed insights** — short cards derived only from durable
console evidence (traffic, system health, analytics, the same registry).
Every insight carries explicit evidence refs (issue/PR/provider/event ids).
Insights never claim that a workflow action completed without proof, and
they never mutate anything.
Design rules matching the rest of the console:
- **Read-only.** No endpoint registered here mutates Gitea, the control plane,
or the registry.
- **Advisory only.** Insights carry ``advisory_only=True`` and never emit an
"action completed" claim. The allocator, review, and merge paths remain the
only authorities for work selection and terminal state.
- **Qualified absence.** When a source could not run, the insight list says so
rather than inventing an empty-and-healthy fleet or zero blocked items.
- **Redaction.** Free-text titles, reasons, and notes pass through
``webui.console_redaction`` before they leave this module.
- **No secrets.** Provider records are taken from the credential-free worker
registry. Keys never appear in this surface.
Non-goals (from the issue): free-form chatbot that overrides gates, secret
provider keys in the UI, auto-merge or auto-close from insights.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Any, Callable, Sequence
from webui import console_redaction
from webui.worker_registry import (
ProviderRecord,
WorkerRegistry,
WorkerRecord,
load_registry as load_worker_registry,
workers_for_provider,
)
INSIGHTS_SCHEMA_VERSION = 1
# Provider connection vocabulary. Declared availability is not a live probe —
# the worker registry owns the declaration, and adapters (#800) own live checks.
CONNECTION_DECLARED_AVAILABLE = "declared_available"
CONNECTION_DECLARED_UNAVAILABLE = "declared_unavailable"
CONNECTION_REGISTRY_UNAVAILABLE = "registry_unavailable"
# Insight kinds. Each generator is a pure function over one evidence source.
INSIGHT_BLOCKED_QUEUE = "blocked_queue_pressure"
INSIGHT_CONTROLLER_ATTENTION = "controller_attention"
INSIGHT_STALE_RUNTIME = "stale_runtime_risk"
INSIGHT_PROVIDER_WITHOUT_WORKERS = "provider_without_workers"
INSIGHT_ANALYTICS_FAILURE_RATE = "analytics_failure_pressure"
SEVERITY_INFO = "info"
SEVERITY_WARN = "warn"
SEVERITY_CRITICAL = "critical"
SEVERITY_UNPROVEN = "unproven"
CONFIDENCE_HIGH = "high"
CONFIDENCE_MEDIUM = "medium"
CONFIDENCE_LOW = "low"
CONFIDENCE_UNPROVEN = "unproven"
def _redact(value: Any) -> Any:
if value is None:
return None
return console_redaction.redact_text(str(value))
def _offline_test_mode() -> bool:
return (os.environ.get("WEBUI_TEST_OFFLINE") or "").strip().lower() in {
"1",
"true",
"yes",
"on",
}
# --- Provider connection status ------------------------------------------------
@dataclass(frozen=True)
class ProviderConnection:
"""One AI provider's declared connection status (no secrets, no live probe)."""
provider_id: str
display_name: str
vendor: str
executable: str
connection_status: str
available_declared: bool
models: tuple[str, ...]
worker_count: int
enabled_worker_count: int
notes: str
#: Explicit statement of what was *not* proven (live process health, etc.).
probe_limit: str
def to_dict(self) -> dict[str, Any]:
return {
"provider_id": self.provider_id,
"display_name": self.display_name,
"vendor": self.vendor,
"executable": self.executable,
"connection_status": self.connection_status,
"available_declared": self.available_declared,
"models": list(self.models),
"worker_count": self.worker_count,
"enabled_worker_count": self.enabled_worker_count,
"notes": self.notes,
"probe_limit": self.probe_limit,
# Always true for this surface: keys are never loaded.
"secrets_exposed": False,
}
@dataclass(frozen=True)
class ProviderSnapshot:
ok: bool
providers: tuple[ProviderConnection, ...] = ()
registry_revision: int | None = None
registry_path: str | None = None
fetch_error: str | None = None
schema_version: int = INSIGHTS_SCHEMA_VERSION
def to_dict(self) -> dict[str, Any]:
return {
"ok": self.ok,
"schema_version": self.schema_version,
"registry_revision": self.registry_revision,
"registry_path": self.registry_path,
"fetch_error": self.fetch_error,
"providers": [p.to_dict() for p in self.providers],
"interpretation_limits": [
"connection_status reflects the worker registry declaration only",
"no API keys or credential material are loaded or rendered",
"live executable health is not probed on this surface (#800 owns that)",
],
}
_PROBE_LIMIT = (
"Declared status only. This console does not probe the provider executable "
"or call vendor APIs; live health belongs to the provider adapter framework."
)
def connection_status_for(provider: ProviderRecord) -> str:
return (
CONNECTION_DECLARED_AVAILABLE
if provider.available
else CONNECTION_DECLARED_UNAVAILABLE
)
def build_provider_connection(
provider: ProviderRecord,
workers: Sequence[WorkerRecord],
) -> ProviderConnection:
enabled = sum(1 for worker in workers if worker.enabled)
return ProviderConnection(
provider_id=provider.id,
display_name=str(_redact(provider.display_name) or provider.id),
vendor=str(_redact(provider.vendor) or ""),
executable=str(_redact(provider.executable) or ""),
connection_status=connection_status_for(provider),
available_declared=bool(provider.available),
models=tuple(str(_redact(m) or m) for m in provider.models),
worker_count=len(workers),
enabled_worker_count=enabled,
notes=str(_redact(provider.notes) or ""),
probe_limit=_PROBE_LIMIT,
)
def load_provider_snapshot(
*,
registry: WorkerRegistry | None = None,
registry_loader: Callable[[], WorkerRegistry] | None = None,
) -> ProviderSnapshot:
"""Load declared provider connections. Never raises for missing registry."""
if registry is None:
loader = registry_loader or load_worker_registry
try:
if _offline_test_mode() and registry_loader is None:
return ProviderSnapshot(
ok=False,
fetch_error=(
"provider registry not loaded in offline test mode "
"(inject a registry for unit tests)"
),
)
registry = loader()
except Exception as exc: # fail soft — operator-visible reason
return ProviderSnapshot(
ok=False,
fetch_error=str(_redact(f"worker registry unavailable: {exc}")),
)
connections = tuple(
build_provider_connection(provider, workers_for_provider(registry, provider.id))
for provider in registry.providers
)
return ProviderSnapshot(
ok=True,
providers=connections,
registry_revision=registry.revision,
registry_path=str(registry.source_path),
)
# --- Evidence-backed insights --------------------------------------------------
@dataclass(frozen=True)
class EvidenceRef:
"""One durable reference an insight is allowed to cite."""
kind: str # issue | pr | provider | health | analytics | traffic
ref: str
detail: str
def to_dict(self) -> dict[str, Any]:
return {
"kind": self.kind,
"ref": self.ref,
"detail": str(_redact(self.detail) or ""),
}
@dataclass(frozen=True)
class Insight:
"""One advisory finding. Never a claim that an action completed."""
insight_id: str
kind: str
severity: str
confidence: str
title: str
summary: str
evidence: tuple[EvidenceRef, ...]
advisory_only: bool = True
claims_action_completed: bool = False
def to_dict(self) -> dict[str, Any]:
return {
"insight_id": self.insight_id,
"kind": self.kind,
"severity": self.severity,
"confidence": self.confidence,
"title": str(_redact(self.title) or ""),
"summary": str(_redact(self.summary) or ""),
"evidence": [item.to_dict() for item in self.evidence],
"advisory_only": self.advisory_only,
"claims_action_completed": self.claims_action_completed,
}
@dataclass(frozen=True)
class InsightsSnapshot:
ok: bool
insights: tuple[Insight, ...] = ()
sources_used: tuple[str, ...] = ()
sources_unavailable: tuple[dict[str, str], ...] = ()
fetch_error: str | None = None
schema_version: int = INSIGHTS_SCHEMA_VERSION
def to_dict(self) -> dict[str, Any]:
return {
"ok": self.ok,
"schema_version": self.schema_version,
"insights": [insight.to_dict() for insight in self.insights],
"sources_used": list(self.sources_used),
"sources_unavailable": list(self.sources_unavailable),
"fetch_error": self.fetch_error,
"interpretation_limits": [
"insights are advisory only and never authorize merge, review, or close",
"an insight without evidence refs is refused rather than emitted",
"a missing source is listed under sources_unavailable, not as an empty success",
"insights never claim a workflow action completed",
],
}
def _require_evidence(evidence: Sequence[EvidenceRef]) -> tuple[EvidenceRef, ...]:
"""Fail closed: an insight with no evidence must not be emitted."""
items = tuple(evidence)
if not items:
raise ValueError("insight requires at least one evidence ref")
return items
def insight_blocked_queue(traffic: Any) -> Insight | None:
"""Traffic blocked bucket pressure with per-item evidence."""
blocked = tuple(getattr(traffic, "blocked", ()) or ())
if not blocked:
return None
evidence = []
for item in blocked[:20]:
kind = str(getattr(item, "kind", "issue") or "issue")
number = int(getattr(item, "number", 0) or 0)
if number <= 0:
continue
reason = getattr(item, "block_reason", None) or "blocked"
evidence.append(
EvidenceRef(
kind=kind,
ref=f"#{number}",
detail=f"traffic_state=blocked; reason={reason}",
)
)
if not evidence:
return None
count = len(blocked)
severity = SEVERITY_CRITICAL if count >= 10 else SEVERITY_WARN
return Insight(
insight_id=f"{INSIGHT_BLOCKED_QUEUE}:{count}",
kind=INSIGHT_BLOCKED_QUEUE,
severity=severity,
confidence=(
CONFIDENCE_HIGH
if getattr(traffic, "inventory_complete", False)
else CONFIDENCE_MEDIUM
),
title=f"{count} blocked work item(s) in traffic control",
summary=(
f"Traffic control reports {count} blocked item(s). "
"This is an observation of the loaded window, not a claim that "
"any remediation ran."
),
evidence=_require_evidence(evidence),
)
def insight_controller_attention(traffic: Any) -> Insight | None:
needs = tuple(getattr(traffic, "needs_controller", ()) or ())
if not needs:
return None
evidence = []
for item in needs[:20]:
kind = str(getattr(item, "kind", "issue") or "issue")
number = int(getattr(item, "number", 0) or 0)
if number <= 0:
continue
evidence.append(
EvidenceRef(
kind=kind,
ref=f"#{number}",
detail="traffic_state=needs_controller",
)
)
if not evidence:
return None
count = len(needs)
return Insight(
insight_id=f"{INSIGHT_CONTROLLER_ATTENTION}:{count}",
kind=INSIGHT_CONTROLLER_ATTENTION,
severity=SEVERITY_WARN if count else SEVERITY_INFO,
confidence=(
CONFIDENCE_HIGH
if getattr(traffic, "inventory_complete", False)
else CONFIDENCE_MEDIUM
),
title=f"{count} item(s) need controller attention",
summary=(
f"Traffic control marks {count} item(s) as needs_controller. "
"Advisory only — the controller allocator remains the authority "
"for routing."
),
evidence=_require_evidence(evidence),
)
def insight_stale_runtime(health: Any) -> Insight | None:
stale = getattr(health, "stale_runtime", None)
if stale is None:
return None
mutation_safe = bool(getattr(stale, "mutation_safe", False))
is_stale = bool(getattr(stale, "stale", False))
determinable = bool(getattr(stale, "determinable", False))
if mutation_safe and not is_stale:
return None
daemon = getattr(stale, "daemon_head", None) or "unknown"
checkout = getattr(stale, "checkout_head", None) or "unknown"
remote = getattr(stale, "remote_head", None) or "unknown"
if not determinable:
severity = SEVERITY_UNPROVEN
confidence = CONFIDENCE_UNPROVEN
title = "Runtime parity is not determinable"
summary = (
"System health could not prove mutation_safe. This is not proof "
"that the runtime is stale — only that parity was unproven."
)
else:
severity = SEVERITY_CRITICAL if is_stale else SEVERITY_WARN
confidence = CONFIDENCE_HIGH
title = "Stale or mutation-unsafe runtime"
summary = (
"System health reports a runtime that is not mutation_safe. "
"No restart or recovery is claimed by this insight."
)
return Insight(
insight_id=f"{INSIGHT_STALE_RUNTIME}:{daemon}:{checkout}",
kind=INSIGHT_STALE_RUNTIME,
severity=severity,
confidence=confidence,
title=title,
summary=summary,
evidence=_require_evidence(
(
EvidenceRef(
kind="health",
ref="stale_runtime",
detail=(
f"stale={is_stale}; mutation_safe={mutation_safe}; "
f"determinable={determinable}; daemon={daemon}; "
f"checkout={checkout}; remote={remote}"
),
),
)
),
)
def insight_providers_without_workers(
providers: Sequence[ProviderConnection],
) -> Insight | None:
lonely = [
provider
for provider in providers
if provider.available_declared and provider.worker_count == 0
]
if not lonely:
return None
evidence = tuple(
EvidenceRef(
kind="provider",
ref=provider.provider_id,
detail=(
f"available_declared=true; worker_count=0; "
f"vendor={provider.vendor}"
),
)
for provider in lonely
)
return Insight(
insight_id=f"{INSIGHT_PROVIDER_WITHOUT_WORKERS}:{len(lonely)}",
kind=INSIGHT_PROVIDER_WITHOUT_WORKERS,
severity=SEVERITY_INFO,
confidence=CONFIDENCE_HIGH,
title=f"{len(lonely)} declared-available provider(s) have no workers",
summary=(
"The worker registry declares these providers available but no "
"worker instance names them. This is a configuration observation, "
"not a claim that a provider process is running or idle."
),
evidence=_require_evidence(evidence),
)
def insight_analytics_failures(analytics: Any) -> Insight | None:
"""Flag elevated non-ok stage status in analytics when events exist."""
if analytics is None or not getattr(analytics, "ok", False):
return None
events = tuple(getattr(analytics, "events", ()) or ())
if not events:
return None
failed = [
event
for event in events
if str(getattr(event, "status", "") or "").lower()
in {"error", "failed", "failure"}
]
if not failed:
return None
# Cap evidence so a large window stays readable.
evidence = []
for event in failed[:20]:
usage_id = getattr(event, "usage_id", None)
issue = getattr(event, "issue_number", None)
pr = getattr(event, "pr_number", None)
if pr is not None:
ref_kind, ref = "pr", f"#{int(pr)}"
elif issue is not None:
ref_kind, ref = "issue", f"#{int(issue)}"
else:
ref_kind, ref = "analytics", f"usage:{usage_id}"
evidence.append(
EvidenceRef(
kind=ref_kind,
ref=ref,
detail=(
f"status={getattr(event, 'status', '')}; "
f"stage={getattr(event, 'stage', '')}; "
f"model={getattr(event, 'model', '')}"
),
)
)
if not evidence:
return None
rate = len(failed) / max(len(events), 1)
return Insight(
insight_id=f"{INSIGHT_ANALYTICS_FAILURE_RATE}:{len(failed)}:{len(events)}",
kind=INSIGHT_ANALYTICS_FAILURE_RATE,
severity=SEVERITY_WARN if rate >= 0.1 else SEVERITY_INFO,
confidence=CONFIDENCE_MEDIUM,
title=f"{len(failed)} analytics event(s) reported failure status",
summary=(
f"{len(failed)} of {len(events)} loaded analytics events carry a "
"failure status. Advisory only — this is not a gate decision."
),
evidence=_require_evidence(evidence),
)
def generate_insights(
*,
traffic: Any | None = None,
health: Any | None = None,
provider_snapshot: ProviderSnapshot | None = None,
analytics: Any | None = None,
) -> tuple[tuple[Insight, ...], tuple[str, ...], tuple[dict[str, str], ...]]:
"""Pure multi-source insight generation. Never mutates inputs."""
insights: list[Insight] = []
used: list[str] = []
unavailable: list[dict[str, str]] = []
if traffic is None:
unavailable.append(
{"source": "traffic", "reason": "traffic snapshot not supplied"}
)
elif getattr(traffic, "fetch_error", None):
unavailable.append(
{
"source": "traffic",
"reason": str(_redact(traffic.fetch_error) or "traffic fetch failed"),
}
)
else:
used.append("traffic")
for builder in (insight_blocked_queue, insight_controller_attention):
try:
item = builder(traffic)
except ValueError:
continue
if item is not None:
insights.append(item)
if health is None:
unavailable.append(
{"source": "system_health", "reason": "system health snapshot not supplied"}
)
else:
used.append("system_health")
try:
item = insight_stale_runtime(health)
except ValueError:
item = None
if item is not None:
insights.append(item)
if provider_snapshot is None:
unavailable.append(
{"source": "providers", "reason": "provider snapshot not supplied"}
)
elif not provider_snapshot.ok:
unavailable.append(
{
"source": "providers",
"reason": str(
_redact(provider_snapshot.fetch_error)
or "provider registry unavailable"
),
}
)
else:
used.append("providers")
try:
item = insight_providers_without_workers(provider_snapshot.providers)
except ValueError:
item = None
if item is not None:
insights.append(item)
if analytics is None:
unavailable.append(
{"source": "analytics", "reason": "analytics snapshot not supplied"}
)
elif not getattr(analytics, "ok", False):
unavailable.append(
{
"source": "analytics",
"reason": str(
_redact(getattr(analytics, "fetch_error", None))
or "analytics snapshot not ok"
),
}
)
else:
used.append("analytics")
try:
item = insight_analytics_failures(analytics)
except ValueError:
item = None
if item is not None:
insights.append(item)
# Stable ordering: severity then kind.
_sev_rank = {
SEVERITY_CRITICAL: 0,
SEVERITY_WARN: 1,
SEVERITY_INFO: 2,
SEVERITY_UNPROVEN: 3,
}
insights.sort(key=lambda i: (_sev_rank.get(i.severity, 9), i.kind, i.insight_id))
return tuple(insights), tuple(used), tuple(unavailable)
def load_insights_snapshot(
*,
traffic: Any | None = None,
health: Any | None = None,
provider_snapshot: ProviderSnapshot | None = None,
analytics: Any | None = None,
load_live: bool = True,
) -> InsightsSnapshot:
"""Compose insights from injected or live console evidence sources."""
sources_unavailable: list[dict[str, str]] = []
if load_live and traffic is None and not _offline_test_mode():
try:
from webui.traffic_loader import load_traffic_snapshot
traffic = load_traffic_snapshot()
except Exception as exc: # fail soft
sources_unavailable.append(
{
"source": "traffic",
"reason": str(_redact(f"traffic load failed: {exc}")),
}
)
traffic = None
if load_live and health is None and not _offline_test_mode():
try:
from webui.system_health import load_system_health
health = load_system_health()
except Exception as exc:
sources_unavailable.append(
{
"source": "system_health",
"reason": str(_redact(f"system health load failed: {exc}")),
}
)
health = None
if provider_snapshot is None:
provider_snapshot = load_provider_snapshot()
if load_live and analytics is None and not _offline_test_mode():
try:
from webui.analytics_loader import load_analytics
analytics = load_analytics()
except Exception as exc:
sources_unavailable.append(
{
"source": "analytics",
"reason": str(_redact(f"analytics load failed: {exc}")),
}
)
analytics = None
insights, used, unavailable = generate_insights(
traffic=traffic,
health=health,
provider_snapshot=provider_snapshot,
analytics=analytics,
)
merged_unavailable = tuple(sources_unavailable) + unavailable
# ok when at least one source contributed or we can honestly report absence.
ok = bool(used) or bool(merged_unavailable)
return InsightsSnapshot(
ok=ok,
insights=insights,
sources_used=used,
sources_unavailable=merged_unavailable,
fetch_error=None
if used
else (
"no evidence sources produced a usable snapshot"
if merged_unavailable
else "no insight sources ran"
),
)
def snapshot_providers_to_dict(snapshot: ProviderSnapshot) -> dict[str, Any]:
return snapshot.to_dict()
def snapshot_insights_to_dict(snapshot: InsightsSnapshot) -> dict[str, Any]:
return snapshot.to_dict()