"""AI-provider connections and evidence-backed operational insights (#650, Phase 4). Operators need two related, **advisory** surfaces: 1. **Provider connection status** — which AI runtimes are *declared* in the worker registry (#798), without ever exposing API keys or inventing a live probe that this process cannot perform. 2. **Evidence-backed insights** — short cards derived only from durable console evidence (traffic, system health, analytics, the same registry). Every insight carries explicit evidence refs (issue/PR/provider/event ids). Insights never claim that a workflow action completed without proof, and they never mutate anything. Design rules matching the rest of the console: - **Read-only.** No endpoint registered here mutates Gitea, the control plane, or the registry. - **Advisory only.** Insights carry ``advisory_only=True`` and never emit an "action completed" claim. The allocator, review, and merge paths remain the only authorities for work selection and terminal state. - **Qualified absence.** When a source could not run, the insight list says so rather than inventing an empty-and-healthy fleet or zero blocked items. - **Redaction.** Free-text titles, reasons, and notes pass through ``webui.console_redaction`` before they leave this module. - **No secrets.** Provider records are taken from the credential-free worker registry. Keys never appear in this surface. Non-goals (from the issue): free-form chatbot that overrides gates, secret provider keys in the UI, auto-merge or auto-close from insights. """ from __future__ import annotations import os from dataclasses import dataclass from typing import Any, Callable, Sequence from webui import console_redaction from webui.worker_registry import ( ProviderRecord, WorkerRegistry, WorkerRecord, load_registry as load_worker_registry, workers_for_provider, ) INSIGHTS_SCHEMA_VERSION = 1 # Provider connection vocabulary. Declared availability is not a live probe — # the worker registry owns the declaration, and adapters (#800) own live checks. CONNECTION_DECLARED_AVAILABLE = "declared_available" CONNECTION_DECLARED_UNAVAILABLE = "declared_unavailable" CONNECTION_REGISTRY_UNAVAILABLE = "registry_unavailable" # Insight kinds. Each generator is a pure function over one evidence source. INSIGHT_BLOCKED_QUEUE = "blocked_queue_pressure" INSIGHT_CONTROLLER_ATTENTION = "controller_attention" INSIGHT_STALE_RUNTIME = "stale_runtime_risk" INSIGHT_PROVIDER_WITHOUT_WORKERS = "provider_without_workers" INSIGHT_ANALYTICS_FAILURE_RATE = "analytics_failure_pressure" SEVERITY_INFO = "info" SEVERITY_WARN = "warn" SEVERITY_CRITICAL = "critical" SEVERITY_UNPROVEN = "unproven" CONFIDENCE_HIGH = "high" CONFIDENCE_MEDIUM = "medium" CONFIDENCE_LOW = "low" CONFIDENCE_UNPROVEN = "unproven" def _redact(value: Any) -> Any: if value is None: return None return console_redaction.redact_text(str(value)) def _offline_test_mode() -> bool: return (os.environ.get("WEBUI_TEST_OFFLINE") or "").strip().lower() in { "1", "true", "yes", "on", } # --- Provider connection status ------------------------------------------------ @dataclass(frozen=True) class ProviderConnection: """One AI provider's declared connection status (no secrets, no live probe).""" provider_id: str display_name: str vendor: str executable: str connection_status: str available_declared: bool models: tuple[str, ...] worker_count: int enabled_worker_count: int notes: str #: Explicit statement of what was *not* proven (live process health, etc.). probe_limit: str def to_dict(self) -> dict[str, Any]: return { "provider_id": self.provider_id, "display_name": self.display_name, "vendor": self.vendor, "executable": self.executable, "connection_status": self.connection_status, "available_declared": self.available_declared, "models": list(self.models), "worker_count": self.worker_count, "enabled_worker_count": self.enabled_worker_count, "notes": self.notes, "probe_limit": self.probe_limit, # Always true for this surface: keys are never loaded. "secrets_exposed": False, } @dataclass(frozen=True) class ProviderSnapshot: ok: bool providers: tuple[ProviderConnection, ...] = () registry_revision: int | None = None registry_path: str | None = None fetch_error: str | None = None schema_version: int = INSIGHTS_SCHEMA_VERSION def to_dict(self) -> dict[str, Any]: return { "ok": self.ok, "schema_version": self.schema_version, "registry_revision": self.registry_revision, "registry_path": self.registry_path, "fetch_error": self.fetch_error, "providers": [p.to_dict() for p in self.providers], "interpretation_limits": [ "connection_status reflects the worker registry declaration only", "no API keys or credential material are loaded or rendered", "live executable health is not probed on this surface (#800 owns that)", ], } _PROBE_LIMIT = ( "Declared status only. This console does not probe the provider executable " "or call vendor APIs; live health belongs to the provider adapter framework." ) def connection_status_for(provider: ProviderRecord) -> str: return ( CONNECTION_DECLARED_AVAILABLE if provider.available else CONNECTION_DECLARED_UNAVAILABLE ) def build_provider_connection( provider: ProviderRecord, workers: Sequence[WorkerRecord], ) -> ProviderConnection: enabled = sum(1 for worker in workers if worker.enabled) return ProviderConnection( provider_id=provider.id, display_name=str(_redact(provider.display_name) or provider.id), vendor=str(_redact(provider.vendor) or ""), executable=str(_redact(provider.executable) or ""), connection_status=connection_status_for(provider), available_declared=bool(provider.available), models=tuple(str(_redact(m) or m) for m in provider.models), worker_count=len(workers), enabled_worker_count=enabled, notes=str(_redact(provider.notes) or ""), probe_limit=_PROBE_LIMIT, ) def load_provider_snapshot( *, registry: WorkerRegistry | None = None, registry_loader: Callable[[], WorkerRegistry] | None = None, ) -> ProviderSnapshot: """Load declared provider connections. Never raises for missing registry.""" if registry is None: loader = registry_loader or load_worker_registry try: if _offline_test_mode() and registry_loader is None: return ProviderSnapshot( ok=False, fetch_error=( "provider registry not loaded in offline test mode " "(inject a registry for unit tests)" ), ) registry = loader() except Exception as exc: # fail soft — operator-visible reason return ProviderSnapshot( ok=False, fetch_error=str(_redact(f"worker registry unavailable: {exc}")), ) connections = tuple( build_provider_connection(provider, workers_for_provider(registry, provider.id)) for provider in registry.providers ) return ProviderSnapshot( ok=True, providers=connections, registry_revision=registry.revision, registry_path=str(registry.source_path), ) # --- Evidence-backed insights -------------------------------------------------- @dataclass(frozen=True) class EvidenceRef: """One durable reference an insight is allowed to cite.""" kind: str # issue | pr | provider | health | analytics | traffic ref: str detail: str def to_dict(self) -> dict[str, Any]: return { "kind": self.kind, "ref": self.ref, "detail": str(_redact(self.detail) or ""), } @dataclass(frozen=True) class Insight: """One advisory finding. Never a claim that an action completed.""" insight_id: str kind: str severity: str confidence: str title: str summary: str evidence: tuple[EvidenceRef, ...] advisory_only: bool = True claims_action_completed: bool = False def to_dict(self) -> dict[str, Any]: return { "insight_id": self.insight_id, "kind": self.kind, "severity": self.severity, "confidence": self.confidence, "title": str(_redact(self.title) or ""), "summary": str(_redact(self.summary) or ""), "evidence": [item.to_dict() for item in self.evidence], "advisory_only": self.advisory_only, "claims_action_completed": self.claims_action_completed, } @dataclass(frozen=True) class InsightsSnapshot: ok: bool insights: tuple[Insight, ...] = () sources_used: tuple[str, ...] = () sources_unavailable: tuple[dict[str, str], ...] = () fetch_error: str | None = None schema_version: int = INSIGHTS_SCHEMA_VERSION def to_dict(self) -> dict[str, Any]: return { "ok": self.ok, "schema_version": self.schema_version, "insights": [insight.to_dict() for insight in self.insights], "sources_used": list(self.sources_used), "sources_unavailable": list(self.sources_unavailable), "fetch_error": self.fetch_error, "interpretation_limits": [ "insights are advisory only and never authorize merge, review, or close", "an insight without evidence refs is refused rather than emitted", "a missing source is listed under sources_unavailable, not as an empty success", "insights never claim a workflow action completed", ], } def _require_evidence(evidence: Sequence[EvidenceRef]) -> tuple[EvidenceRef, ...]: """Fail closed: an insight with no evidence must not be emitted.""" items = tuple(evidence) if not items: raise ValueError("insight requires at least one evidence ref") return items def insight_blocked_queue(traffic: Any) -> Insight | None: """Traffic blocked bucket pressure with per-item evidence.""" blocked = tuple(getattr(traffic, "blocked", ()) or ()) if not blocked: return None evidence = [] for item in blocked[:20]: kind = str(getattr(item, "kind", "issue") or "issue") number = int(getattr(item, "number", 0) or 0) if number <= 0: continue reason = getattr(item, "block_reason", None) or "blocked" evidence.append( EvidenceRef( kind=kind, ref=f"#{number}", detail=f"traffic_state=blocked; reason={reason}", ) ) if not evidence: return None count = len(blocked) severity = SEVERITY_CRITICAL if count >= 10 else SEVERITY_WARN return Insight( insight_id=f"{INSIGHT_BLOCKED_QUEUE}:{count}", kind=INSIGHT_BLOCKED_QUEUE, severity=severity, confidence=( CONFIDENCE_HIGH if getattr(traffic, "inventory_complete", False) else CONFIDENCE_MEDIUM ), title=f"{count} blocked work item(s) in traffic control", summary=( f"Traffic control reports {count} blocked item(s). " "This is an observation of the loaded window, not a claim that " "any remediation ran." ), evidence=_require_evidence(evidence), ) def insight_controller_attention(traffic: Any) -> Insight | None: needs = tuple(getattr(traffic, "needs_controller", ()) or ()) if not needs: return None evidence = [] for item in needs[:20]: kind = str(getattr(item, "kind", "issue") or "issue") number = int(getattr(item, "number", 0) or 0) if number <= 0: continue evidence.append( EvidenceRef( kind=kind, ref=f"#{number}", detail="traffic_state=needs_controller", ) ) if not evidence: return None count = len(needs) return Insight( insight_id=f"{INSIGHT_CONTROLLER_ATTENTION}:{count}", kind=INSIGHT_CONTROLLER_ATTENTION, severity=SEVERITY_WARN if count else SEVERITY_INFO, confidence=( CONFIDENCE_HIGH if getattr(traffic, "inventory_complete", False) else CONFIDENCE_MEDIUM ), title=f"{count} item(s) need controller attention", summary=( f"Traffic control marks {count} item(s) as needs_controller. " "Advisory only — the controller allocator remains the authority " "for routing." ), evidence=_require_evidence(evidence), ) def insight_stale_runtime(health: Any) -> Insight | None: stale = getattr(health, "stale_runtime", None) if stale is None: return None mutation_safe = bool(getattr(stale, "mutation_safe", False)) is_stale = bool(getattr(stale, "stale", False)) determinable = bool(getattr(stale, "determinable", False)) if mutation_safe and not is_stale: return None daemon = getattr(stale, "daemon_head", None) or "unknown" checkout = getattr(stale, "checkout_head", None) or "unknown" remote = getattr(stale, "remote_head", None) or "unknown" if not determinable: severity = SEVERITY_UNPROVEN confidence = CONFIDENCE_UNPROVEN title = "Runtime parity is not determinable" summary = ( "System health could not prove mutation_safe. This is not proof " "that the runtime is stale — only that parity was unproven." ) else: severity = SEVERITY_CRITICAL if is_stale else SEVERITY_WARN confidence = CONFIDENCE_HIGH title = "Stale or mutation-unsafe runtime" summary = ( "System health reports a runtime that is not mutation_safe. " "No restart or recovery is claimed by this insight." ) return Insight( insight_id=f"{INSIGHT_STALE_RUNTIME}:{daemon}:{checkout}", kind=INSIGHT_STALE_RUNTIME, severity=severity, confidence=confidence, title=title, summary=summary, evidence=_require_evidence( ( EvidenceRef( kind="health", ref="stale_runtime", detail=( f"stale={is_stale}; mutation_safe={mutation_safe}; " f"determinable={determinable}; daemon={daemon}; " f"checkout={checkout}; remote={remote}" ), ), ) ), ) def insight_providers_without_workers( providers: Sequence[ProviderConnection], ) -> Insight | None: lonely = [ provider for provider in providers if provider.available_declared and provider.worker_count == 0 ] if not lonely: return None evidence = tuple( EvidenceRef( kind="provider", ref=provider.provider_id, detail=( f"available_declared=true; worker_count=0; " f"vendor={provider.vendor}" ), ) for provider in lonely ) return Insight( insight_id=f"{INSIGHT_PROVIDER_WITHOUT_WORKERS}:{len(lonely)}", kind=INSIGHT_PROVIDER_WITHOUT_WORKERS, severity=SEVERITY_INFO, confidence=CONFIDENCE_HIGH, title=f"{len(lonely)} declared-available provider(s) have no workers", summary=( "The worker registry declares these providers available but no " "worker instance names them. This is a configuration observation, " "not a claim that a provider process is running or idle." ), evidence=_require_evidence(evidence), ) def insight_analytics_failures(analytics: Any) -> Insight | None: """Flag elevated non-ok stage status in analytics when events exist.""" if analytics is None or not getattr(analytics, "ok", False): return None events = tuple(getattr(analytics, "events", ()) or ()) if not events: return None failed = [ event for event in events if str(getattr(event, "status", "") or "").lower() in {"error", "failed", "failure"} ] if not failed: return None # Cap evidence so a large window stays readable. evidence = [] for event in failed[:20]: usage_id = getattr(event, "usage_id", None) issue = getattr(event, "issue_number", None) pr = getattr(event, "pr_number", None) if pr is not None: ref_kind, ref = "pr", f"#{int(pr)}" elif issue is not None: ref_kind, ref = "issue", f"#{int(issue)}" else: ref_kind, ref = "analytics", f"usage:{usage_id}" evidence.append( EvidenceRef( kind=ref_kind, ref=ref, detail=( f"status={getattr(event, 'status', '')}; " f"stage={getattr(event, 'stage', '')}; " f"model={getattr(event, 'model', '')}" ), ) ) if not evidence: return None rate = len(failed) / max(len(events), 1) return Insight( insight_id=f"{INSIGHT_ANALYTICS_FAILURE_RATE}:{len(failed)}:{len(events)}", kind=INSIGHT_ANALYTICS_FAILURE_RATE, severity=SEVERITY_WARN if rate >= 0.1 else SEVERITY_INFO, confidence=CONFIDENCE_MEDIUM, title=f"{len(failed)} analytics event(s) reported failure status", summary=( f"{len(failed)} of {len(events)} loaded analytics events carry a " "failure status. Advisory only — this is not a gate decision." ), evidence=_require_evidence(evidence), ) def generate_insights( *, traffic: Any | None = None, health: Any | None = None, provider_snapshot: ProviderSnapshot | None = None, analytics: Any | None = None, ) -> tuple[tuple[Insight, ...], tuple[str, ...], tuple[dict[str, str], ...]]: """Pure multi-source insight generation. Never mutates inputs.""" insights: list[Insight] = [] used: list[str] = [] unavailable: list[dict[str, str]] = [] if traffic is None: unavailable.append( {"source": "traffic", "reason": "traffic snapshot not supplied"} ) elif getattr(traffic, "fetch_error", None): unavailable.append( { "source": "traffic", "reason": str(_redact(traffic.fetch_error) or "traffic fetch failed"), } ) else: used.append("traffic") for builder in (insight_blocked_queue, insight_controller_attention): try: item = builder(traffic) except ValueError: continue if item is not None: insights.append(item) if health is None: unavailable.append( {"source": "system_health", "reason": "system health snapshot not supplied"} ) else: used.append("system_health") try: item = insight_stale_runtime(health) except ValueError: item = None if item is not None: insights.append(item) if provider_snapshot is None: unavailable.append( {"source": "providers", "reason": "provider snapshot not supplied"} ) elif not provider_snapshot.ok: unavailable.append( { "source": "providers", "reason": str( _redact(provider_snapshot.fetch_error) or "provider registry unavailable" ), } ) else: used.append("providers") try: item = insight_providers_without_workers(provider_snapshot.providers) except ValueError: item = None if item is not None: insights.append(item) if analytics is None: unavailable.append( {"source": "analytics", "reason": "analytics snapshot not supplied"} ) elif not getattr(analytics, "ok", False): unavailable.append( { "source": "analytics", "reason": str( _redact(getattr(analytics, "fetch_error", None)) or "analytics snapshot not ok" ), } ) else: used.append("analytics") try: item = insight_analytics_failures(analytics) except ValueError: item = None if item is not None: insights.append(item) # Stable ordering: severity then kind. _sev_rank = { SEVERITY_CRITICAL: 0, SEVERITY_WARN: 1, SEVERITY_INFO: 2, SEVERITY_UNPROVEN: 3, } insights.sort(key=lambda i: (_sev_rank.get(i.severity, 9), i.kind, i.insight_id)) return tuple(insights), tuple(used), tuple(unavailable) def load_insights_snapshot( *, traffic: Any | None = None, health: Any | None = None, provider_snapshot: ProviderSnapshot | None = None, analytics: Any | None = None, load_live: bool = True, ) -> InsightsSnapshot: """Compose insights from injected or live console evidence sources.""" sources_unavailable: list[dict[str, str]] = [] if load_live and traffic is None and not _offline_test_mode(): try: from webui.traffic_loader import load_traffic_snapshot traffic = load_traffic_snapshot() except Exception as exc: # fail soft sources_unavailable.append( { "source": "traffic", "reason": str(_redact(f"traffic load failed: {exc}")), } ) traffic = None if load_live and health is None and not _offline_test_mode(): try: from webui.system_health import load_system_health health = load_system_health() except Exception as exc: sources_unavailable.append( { "source": "system_health", "reason": str(_redact(f"system health load failed: {exc}")), } ) health = None if provider_snapshot is None: provider_snapshot = load_provider_snapshot() if load_live and analytics is None and not _offline_test_mode(): try: from webui.analytics_loader import load_analytics analytics = load_analytics() except Exception as exc: sources_unavailable.append( { "source": "analytics", "reason": str(_redact(f"analytics load failed: {exc}")), } ) analytics = None insights, used, unavailable = generate_insights( traffic=traffic, health=health, provider_snapshot=provider_snapshot, analytics=analytics, ) merged_unavailable = tuple(sources_unavailable) + unavailable # ok when at least one source contributed or we can honestly report absence. ok = bool(used) or bool(merged_unavailable) return InsightsSnapshot( ok=ok, insights=insights, sources_used=used, sources_unavailable=merged_unavailable, fetch_error=None if used else ( "no evidence sources produced a usable snapshot" if merged_unavailable else "no insight sources ran" ), ) def snapshot_providers_to_dict(snapshot: ProviderSnapshot) -> dict[str, Any]: return snapshot.to_dict() def snapshot_insights_to_dict(snapshot: InsightsSnapshot) -> dict[str, Any]: return snapshot.to_dict()