Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
22e0a41bd5 |
+24
-98
@@ -23,7 +23,6 @@ import json
|
||||
import os
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
from control_plane_db import (
|
||||
@@ -739,46 +738,6 @@ def normalize_exclude_issue_numbers(
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def _claim_expires_at(claim: Any) -> datetime | None:
|
||||
"""Parse a claim's ``expires_at``, or ``None`` when it is absent/malformed."""
|
||||
if not isinstance(claim, Mapping):
|
||||
return None
|
||||
text = str(claim.get("expires_at") or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
if text.endswith("Z"):
|
||||
text = text[:-1] + "+00:00"
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _drop_expired_claims(
|
||||
claims: Mapping[tuple[str, int], dict[str, Any]],
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> dict[tuple[str, int], dict[str, Any]]:
|
||||
"""Claims minus those whose lease has already expired (#643).
|
||||
|
||||
The read-only mirror of ``expire_stale_leases``: the sweep marks such rows
|
||||
``expired`` so they stop being returned as claims, and this reaches the same
|
||||
view without writing. A claim with no parseable ``expires_at`` is **kept** —
|
||||
an unreadable expiry is not evidence that work is free.
|
||||
"""
|
||||
moment = now or datetime.now(timezone.utc)
|
||||
kept: dict[tuple[str, int], dict[str, Any]] = {}
|
||||
for key, claim in (claims or {}).items():
|
||||
expires_at = _claim_expires_at(claim)
|
||||
if expires_at is not None and expires_at <= moment:
|
||||
continue
|
||||
kept[key] = claim
|
||||
return kept
|
||||
|
||||
|
||||
def candidate_set_fingerprint(
|
||||
candidates: Sequence[WorkCandidate],
|
||||
*,
|
||||
@@ -867,22 +826,12 @@ def allocate_next_work(
|
||||
exclude_issue_numbers: Sequence[int] | None = None,
|
||||
expected_candidate_set_fingerprint: str | None = None,
|
||||
allocation_mode: str | None = None,
|
||||
side_effect_free: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Select and optionally reserve the next work unit via control-plane DB.
|
||||
|
||||
*apply=False* (default): dry-run selection only — no lease/assignment.
|
||||
*apply=True*: atomic ``assign_and_lease`` for the selected candidate.
|
||||
|
||||
*side_effect_free* (#643): a dry run that writes **nothing** to the
|
||||
control-plane DB. A plain ``apply=False`` still registered a session row and
|
||||
swept stale leases globally, so a caller advertising a read-only preview was
|
||||
mutating on every call. Under this flag both writes are suppressed and stale
|
||||
leases are instead filtered out of the claim map in memory, which yields the
|
||||
same selection the sweep would have produced without persisting anything.
|
||||
Incompatible with *apply* — the combination fails closed rather than
|
||||
silently reserving.
|
||||
|
||||
*allocation_mode* (#840): ``cross_role`` (default for controller) inspects
|
||||
the complete queue and returns one authoritative selection naming the
|
||||
required downstream role/profile/action. ``role_scoped`` keeps prior
|
||||
@@ -936,57 +885,40 @@ def allocate_next_work(
|
||||
"allocation_mode": (allocation_mode or "").strip() or None,
|
||||
}
|
||||
|
||||
# A side-effect-free run may never reserve: reserving is a write, and the
|
||||
# flag is the caller's assertion that this call writes nothing (#643).
|
||||
if side_effect_free and apply:
|
||||
session_id = (session_id or "").strip() or f"alloc-{uuid.uuid4().hex[:12]}"
|
||||
try:
|
||||
db.upsert_session(
|
||||
session_id=session_id,
|
||||
role=role_norm,
|
||||
profile=profile_name,
|
||||
pid=os.getpid(),
|
||||
controller_instance_id=controller_instance_id,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface structured
|
||||
return {
|
||||
"success": False,
|
||||
"outcome": OUTCOME_NO_SAFE,
|
||||
"apply": True,
|
||||
"reasons": [
|
||||
"side_effect_free is incompatible with apply=True; an "
|
||||
"assignment is a write (fail closed, #643)"
|
||||
f"failed to register session in control-plane DB: {exc} "
|
||||
"(fail closed, #613)"
|
||||
],
|
||||
"skipped": [],
|
||||
"assignment": None,
|
||||
"substrate": "control_plane_db",
|
||||
}
|
||||
|
||||
session_id = (session_id or "").strip() or f"alloc-{uuid.uuid4().hex[:12]}"
|
||||
if not side_effect_free:
|
||||
try:
|
||||
db.upsert_session(
|
||||
session_id=session_id,
|
||||
role=role_norm,
|
||||
profile=profile_name,
|
||||
pid=os.getpid(),
|
||||
controller_instance_id=controller_instance_id,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface structured
|
||||
return {
|
||||
"success": False,
|
||||
"outcome": OUTCOME_NO_SAFE,
|
||||
"reasons": [
|
||||
f"failed to register session in control-plane DB: {exc} "
|
||||
"(fail closed, #613)"
|
||||
],
|
||||
"skipped": [],
|
||||
"assignment": None,
|
||||
"substrate": "control_plane_db",
|
||||
}
|
||||
|
||||
# Expire stale leases globally before selection.
|
||||
try:
|
||||
db.expire_stale_leases()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {
|
||||
"success": False,
|
||||
"outcome": OUTCOME_NO_SAFE,
|
||||
"reasons": [f"lease expiry failed: {exc} (fail closed)"],
|
||||
"skipped": [],
|
||||
"assignment": None,
|
||||
"substrate": "control_plane_db",
|
||||
}
|
||||
# Expire stale leases globally before selection.
|
||||
try:
|
||||
db.expire_stale_leases()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {
|
||||
"success": False,
|
||||
"outcome": OUTCOME_NO_SAFE,
|
||||
"reasons": [f"lease expiry failed: {exc} (fail closed)"],
|
||||
"skipped": [],
|
||||
"assignment": None,
|
||||
"substrate": "control_plane_db",
|
||||
}
|
||||
|
||||
terminal = None
|
||||
try:
|
||||
@@ -1021,12 +953,6 @@ def allocate_next_work(
|
||||
"assignment": None,
|
||||
"substrate": "control_plane_db",
|
||||
}
|
||||
if side_effect_free:
|
||||
# ``list_active_claims`` filters on status alone, so without the
|
||||
# global sweep an already-expired lease would still read as a live
|
||||
# claim and the preview would report work as taken that is free.
|
||||
# Drop those in memory: same view the sweep produces, no write.
|
||||
claims = _drop_expired_claims(claims)
|
||||
|
||||
try:
|
||||
exclude_nums = normalize_exclude_issue_numbers(exclude_issue_numbers)
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# MCP restart audit events and incidents (#665)
|
||||
|
||||
Restarts and recovery attempts leave a forensic trail. Failed drains and
|
||||
break-glass paths also raise durable Gitea incident issues so unsafe restarts
|
||||
cannot be silently repeated.
|
||||
|
||||
Parent umbrella: **#655**. Related: impact coordinator **#658**, drain proof
|
||||
**#661**, restart classes **#663**, break-glass **#664**, post-restart reconcile
|
||||
**#662**, vision **#652**, roadmap **#653**, console recovery **#642**.
|
||||
|
||||
## Components
|
||||
|
||||
| Piece | Where | Responsibility |
|
||||
|-------|-------|----------------|
|
||||
| Event schema + emission | `restart_audit.py` | `mcp.restart.*` vocabulary, redacted payload builder, append-only sink via `gitea_audit` |
|
||||
| Fail-closed privileged gate | `restart_audit.require_audit_or_deny` | When `GITEA_AUDIT_LOG` is set and the write fails, privileged apply is denied |
|
||||
| Incident descriptors | `restart_audit.build_incident_descriptor` | Durable follow-up issues (failed drain, break-glass, reconcile unresolved, unguarded) |
|
||||
| Materializer | `restart_audit.materialize_incident` | Injected `create_issue_fn` (network kept out of pure tests) |
|
||||
| Wiring | `gitea_request_mcp_restart` | Correlation id, impact-preview audit, apply-gate / break-glass audit + incident creation |
|
||||
|
||||
## Event vocabulary
|
||||
|
||||
| Event type | When |
|
||||
|------------|------|
|
||||
| `mcp.restart.impact_preview` | Every `gitea_request_mcp_restart` evaluation |
|
||||
| `mcp.restart.drain_enter` | Drain window starts (schema reserved; emit from drain path) |
|
||||
| `mcp.restart.drain_exit` | Drain window ends |
|
||||
| `mcp.restart.drain_proof` | Drain-proof verification result |
|
||||
| `mcp.restart.apply_gate` | Apply hard gate (`dry_run=False`) |
|
||||
| `mcp.restart.break_glass` | Authorized break-glass bypass |
|
||||
| `mcp.restart.post_restart_reconcile` | Post-restart reconcile outcome |
|
||||
| `mcp.restart.narrower_recovery` | Narrower recovery attempt recorded |
|
||||
| `mcp.restart.unguarded_detected` | Unguarded restart path detected |
|
||||
|
||||
All free text is redacted before sink write or issue body assembly. Emission
|
||||
never raises; callers decide fail-closed policy.
|
||||
|
||||
## Correlation
|
||||
|
||||
Each restart lifecycle mints a short `correlation_id` (`rst-` + 16 hex) shared
|
||||
across impact preview → apply gate → incident descriptors so operators can join
|
||||
the trail.
|
||||
|
||||
## Privileged deny-on-audit-fail
|
||||
|
||||
Rollout policy (issue #665):
|
||||
|
||||
1. Configure `GITEA_AUDIT_LOG` so writes land.
|
||||
2. Only then enforce deny when a privileged restart path cannot audit.
|
||||
|
||||
When audit is **not** configured, privileged apply still proceeds (no false
|
||||
denials during rollout). When audit **is** configured and the write fails,
|
||||
`apply_authorized` is cleared.
|
||||
|
||||
## Incidents
|
||||
|
||||
| Kind | Trigger |
|
||||
|------|---------|
|
||||
| `restart_failed_drain` | Apply denied by drain hard gate / failed proof |
|
||||
| `restart_break_glass` | Any authorized break-glass apply |
|
||||
| `restart_reconcile_unresolved` | Post-restart reconcile left work unresolved |
|
||||
| `restart_unguarded_detected` | Unguarded restart attempt detected |
|
||||
|
||||
Break-glass **always** creates an incident descriptor (and a Gitea issue when
|
||||
the create path is available). Failed drain does the same. Incident bodies
|
||||
include correlation id, session, class, scope, proof id, and redacted reasons.
|
||||
|
||||
Default labels: `mcp-health`, `safety`, `observability`, `status:ready`,
|
||||
`type:bug`, `workflow-hardening`.
|
||||
|
||||
## Tool payload surface
|
||||
|
||||
`gitea_request_mcp_restart` returns:
|
||||
|
||||
* `correlation_id` — lifecycle join key
|
||||
* `restart_audit.impact_preview_written` — sink success for the preview event
|
||||
* `restart_audit.apply_gate_written` — sink success for apply/break-glass (apply only)
|
||||
* `restart_audit.incident_result` — materialization outcome when an incident was required
|
||||
* `incident` — durable descriptor (when gate requires follow-up)
|
||||
|
||||
## Security
|
||||
|
||||
* No secrets in audit payloads or issue bodies.
|
||||
* This module never restarts a process.
|
||||
* Drain proof verification remains #661; audit only records the decision.
|
||||
* Incident creation failures are recorded in `incident_result.reasons` and never
|
||||
crash the restart evaluation path (audit write failure still fails closed for
|
||||
privileged apply when the sink is enabled).
|
||||
|
||||
## Tests
|
||||
|
||||
See `tests/test_restart_audit.py`: schema, redaction, emission, deny policy,
|
||||
incident materialization mocks, break-glass / failed-drain selection.
|
||||
@@ -33,6 +33,7 @@ recovery behavior for all nine classes.
|
||||
| `ControlPlaneDB.list_sessions` | `control_plane_db.py` | Read-only session inventory (the process-level unit a restart kills). |
|
||||
| `gitea_request_mcp_restart` | `gitea_mcp_server.py` | MCP tool: gathers inventory from the #613 DB, calls the coordinator, returns the report, and on `dry_run=False` runs the #661 drain-proof hard gate. Never restarts a process. |
|
||||
| `drain_proof.gate_apply_restart` | `drain_proof.py` | The #661 hard gate: verifies a drain proof against the current impact fingerprint, or records an authorized break-glass bypass. |
|
||||
| `restart_audit` | `restart_audit.py` | #665 forensic trail: `mcp.restart.*` events via `gitea_audit`, correlation ids, durable incidents for failed drain / break-glass. See [`mcp-restart-audit.md`](./mcp-restart-audit.md). |
|
||||
|
||||
## Dimensions evaluated
|
||||
|
||||
|
||||
@@ -94,7 +94,6 @@ already define, and a regression test asserts each mapping matches.
|
||||
| `record_analytics_usage` | operator | gated_write | `runtime.record_analytics_usage` | Yes | No | No | 2 |
|
||||
| `system.reload_namespace` | controller | privileged | `runtime.reload_namespace` | Yes | No | No | 2 |
|
||||
| `system.restart_namespace` | admin | destructive | `runtime.restart_namespace` | Yes | **Yes** | **Yes** | 2 |
|
||||
| `initiate_workflow` | operator | gated_write | `gitea.read` | Yes | No | No | 2 |
|
||||
|
||||
**Dual control** means the acting principal may not be the sole authority: a
|
||||
second distinct principal must confirm. **Break-glass** means the action is
|
||||
@@ -113,12 +112,6 @@ by the console — both hand off to a host supervisor, and neither exposes a raw
|
||||
process kill. See
|
||||
[`sanctioned-restart-controls.md`](sanctioned-restart-controls.md) (#642).
|
||||
|
||||
`initiate_workflow` (#643) is operator-class because its outcome is a *claim*,
|
||||
not a Gitea verdict. Requesting reviewer or merger work reserves that work
|
||||
through the allocator; it does not grant the right to approve or merge, which
|
||||
stays with the MCP role profile and its own capability gates. See
|
||||
[`webui-requests.md`](webui-requests.md).
|
||||
|
||||
### Authorization decision
|
||||
|
||||
`authorize(action_id, principal, for_execution=False)` returns a decision
|
||||
@@ -133,24 +126,9 @@ record and **denies by default**. The deny reasons are closed and enumerated:
|
||||
| `phase_not_active` | Execution requested for an action whose phase is not open. |
|
||||
| `allowed_preview_only` | Authorized — preview only, execution still disabled. |
|
||||
|
||||
There is no implicit allow branch.
|
||||
|
||||
`execution_enabled` on the decision reports whether the action has a live
|
||||
execution path at all, and is computed by `execution_wired(action)`. There are
|
||||
exactly two ways to be wired:
|
||||
|
||||
1. the action's `phase` is at or below `ACTIVE_PHASE`; or
|
||||
2. the action declares an `execution_env_flag` **and** that variable is set.
|
||||
|
||||
Every action that declares no flag therefore reports `execution_enabled: false`
|
||||
while the console is in Phase 1, so no caller can read an allow as permission
|
||||
to mutate. The per-action flag exists because raising `ACTIVE_PHASE` would
|
||||
enable execution for every action of that phase at once, including ones whose
|
||||
execution path is not implemented. One implemented action goes live on its own
|
||||
flag instead of dragging its unimplemented phase-mates with it.
|
||||
|
||||
`initiate_workflow` is the only action that currently declares a flag
|
||||
(`WEBUI_REQUESTS_EXECUTION`), and it stays denied until an operator sets it.
|
||||
There is no implicit allow branch. Even the allow result reports
|
||||
`execution_enabled: false` while the console is in Phase 1, so no caller can
|
||||
read an allow as permission to mutate.
|
||||
|
||||
## Secret redaction
|
||||
|
||||
@@ -257,22 +235,13 @@ second one. The integration points are already wired and observable:
|
||||
instead of adding a parallel check.
|
||||
- **`GET /api/console/security-model`** publishes the RBAC matrix, redaction
|
||||
policy, and audit policy as JSON for operators and tests.
|
||||
- **`POST /api/v1/requests/preview` and `.../apply`** (#643) are the first
|
||||
actions to use this model for a real execution path. Preview always returns a
|
||||
decision and an audited `previewed` record; apply requires `confirm=true`,
|
||||
emits `succeeded` or `denied`, and reserves work only through the allocator.
|
||||
See [`webui-requests.md`](webui-requests.md).
|
||||
|
||||
A Phase 2 action must: use `execution_wired` rather than a private enable flag,
|
||||
implement the confirmation and dual-control flow the matrix already declares,
|
||||
emit a `succeeded` or `failed` record alongside the `gitea_audit` mutation
|
||||
record, and keep `viewer` unable to reach any of it. Turning on execution
|
||||
without the confirmation flow contradicts a declared requirement and is a
|
||||
review failure, not a shortcut.
|
||||
|
||||
Raising `ACTIVE_PHASE` remains the way to open a whole phase at once, and is
|
||||
deliberately *not* what #643 did: an action-scoped opt-in cannot enable an
|
||||
action whose execution path nobody wrote.
|
||||
To open Phase 2, a child issue must: raise `ACTIVE_PHASE`, implement the
|
||||
confirmation and dual-control flow the matrix already declares, emit a
|
||||
`succeeded` or `failed` record alongside the `gitea_audit` mutation record, and
|
||||
keep `viewer` unable to reach any of it. Turning on execution without the
|
||||
confirmation flow contradicts a declared requirement and is a review failure,
|
||||
not a shortcut.
|
||||
|
||||
## Local-dev mode
|
||||
|
||||
@@ -325,7 +294,6 @@ Until Phase 2 wires it, probe protection rests on network placement alone, as
|
||||
| `WEBUI_ROLE_MAP` | unset | JSON subject → role map |
|
||||
| `WEBUI_REQUIRE_PROBE_AUTH` | unset | Require auth for non-public probes |
|
||||
| `WEBUI_CONSOLE_AUDIT_LOG` | unset | Append-only audit sink path |
|
||||
| `WEBUI_REQUESTS_EXECUTION` | unset | Opt in to `initiate_workflow` execution (#643) |
|
||||
|
||||
All are read server-side only. None is ever rendered into a page or returned by
|
||||
an API.
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
# Web console requests: intent preview and workflow initiation (#643)
|
||||
|
||||
**Phase 2. Preview is always live and always read-only. Initiation is wired but
|
||||
denied until an operator opts in.**
|
||||
|
||||
Before this surface, starting role work meant pasting a prompt into a terminal
|
||||
and trusting the operator to have checked the allocator first. Nothing enforced
|
||||
that check, so two sessions could reach for the same issue and each believe it
|
||||
was theirs. This page replaces the paste with a *request*: a desired role, an
|
||||
issue or PR, and a stated intent, answered by an authorization decision and —
|
||||
on confirmation — an exclusive assignment from the allocator.
|
||||
|
||||
| Concern | Module |
|
||||
|---------|--------|
|
||||
| Request model, preview, initiation | `webui/request_service.py` |
|
||||
| Form and preview rendering | `webui/request_views.py` |
|
||||
| Authorization | `webui/console_authz.py` (`initiate_workflow`) |
|
||||
| Audit | `webui/console_audit.py` |
|
||||
| Ownership substrate | `allocator_service.py` + `control_plane_db.py` |
|
||||
|
||||
## Surfaces
|
||||
|
||||
| Path | Method | Purpose |
|
||||
|------|--------|---------|
|
||||
| `/requests` | GET | Request form |
|
||||
| `/requests` | POST | Render an intent preview. **Never assigns.** |
|
||||
| `/api/v1/requests/preview` | POST | Intent preview as JSON |
|
||||
| `/api/v1/requests/apply` | POST | Initiate — confirmed, audited, allocator-owned |
|
||||
|
||||
The HTML form has no initiate button on purpose. Initiating requires a
|
||||
confirmed POST to `/api/v1/requests/apply`, so a stray form submission cannot
|
||||
reserve work as a side effect.
|
||||
|
||||
## The request
|
||||
|
||||
```json
|
||||
{
|
||||
"desired_role": "author",
|
||||
"work_kind": "issue",
|
||||
"work_number": 643,
|
||||
"intent_summary": "implement request preview and initiation",
|
||||
"remote": "prgs",
|
||||
"org": "Scaled-Tech-Consulting",
|
||||
"repo": "Gitea-Tools",
|
||||
"expected_head_sha": null
|
||||
}
|
||||
```
|
||||
|
||||
`desired_role` is one of `author`, `reviewer`, `merger`, `reconciler`,
|
||||
`controller`. `work_kind` is `issue` or `pr`. `remote`/`org`/`repo` default to
|
||||
the first project in the registry when omitted; when neither the request nor
|
||||
the registry resolves them, the request is rejected rather than pointed at some
|
||||
other repository. `intent_summary` is required — it is what the audit record
|
||||
states as the reason — and is truncated to 500 characters.
|
||||
|
||||
Parsing rejects rather than corrects. An unknown role, an unknown work kind, a
|
||||
non-positive number, or a missing intent each return `400` with a `reason_code`
|
||||
and the offending `field`.
|
||||
|
||||
## Preview
|
||||
|
||||
Five checks, each with its own verdict, reason code, and detail:
|
||||
|
||||
| Check | Passes when |
|
||||
|-------|-------------|
|
||||
| `authorization` | The console principal holds `operator` or above |
|
||||
| `capability` | The desired role maps to a declared profile and MCP namespace |
|
||||
| `lease_availability` | No active claim holds the work unit |
|
||||
| `next_safe_action` | The allocator would independently select this exact work unit |
|
||||
| `head_pin` | PR work resolves to a head SHA, and a supplied SHA still matches |
|
||||
|
||||
A preview also returns the role's `allowed_actions` and `prohibited_actions`
|
||||
(from `allocator_service.ROLE_ACTIONS`), the `required_profile` and
|
||||
`required_namespace` the work must run under, and a `correlation_id` that ties
|
||||
the preview to its audit record and to any assignment that follows.
|
||||
|
||||
Preview is read-only in the strict sense: it calls the allocator with
|
||||
`apply=false` and writes nothing but an audit line. An unauthorized principal
|
||||
never reaches the allocator or the control-plane DB at all, so a denial cannot
|
||||
be used to enumerate the queue.
|
||||
|
||||
## Initiation
|
||||
|
||||
`POST /api/v1/requests/apply` refuses in this order, and every refusal returns
|
||||
before any assignment is attempted:
|
||||
|
||||
| Condition | Outcome | Status |
|
||||
|-----------|---------|--------|
|
||||
| Unparseable request | `invalid_request` | 400 |
|
||||
| Not authorized, or execution not wired | `denied` | 403 |
|
||||
| `confirm` not set | `denied` / `confirmation_required` | 409 |
|
||||
| Work unit already claimed | `blocked` / `duplicate_assignment` | 409 |
|
||||
| Allocator would select other work | `wait` / `not_next_safe_work` | 409 |
|
||||
| Allocator declines on apply | `blocked` or `wait` | 409 |
|
||||
| Evidence unavailable | `wait` / `evidence_unavailable` | 503 |
|
||||
| Assigned | `assigned_work` | 201 |
|
||||
|
||||
A success returns the assignment plus a `handoff` block naming the profile, the
|
||||
namespace, and the actions that stay forbidden — enough for the operator to
|
||||
continue in the right MCP namespace without guessing.
|
||||
|
||||
### Why apply runs the allocator twice
|
||||
|
||||
The allocator is the only source of exclusive ownership (#600 / #613), and it
|
||||
selects work; it does not take orders. So `apply` runs a dry-run first and
|
||||
proceeds only when the allocator would independently pick the requested work
|
||||
unit. If it would not, the request reports `wait` and mutates nothing.
|
||||
|
||||
A request is therefore a *confirmation* of the allocator's decision, never an
|
||||
override of it. The apply call carries the dry-run's
|
||||
`candidate_set_fingerprint` as a CAS pin (#776), so a queue that changed
|
||||
between the two calls fails closed rather than assigning against a stale view.
|
||||
The result is checked again on the way out: an assignment naming a different
|
||||
work unit is not read as success.
|
||||
|
||||
### Fail-closed defaults
|
||||
|
||||
- An unreadable control-plane DB denies. It is never treated as "nothing holds
|
||||
this work unit".
|
||||
- An incomplete queue inventory denies (#758). Ranking a partial candidate set
|
||||
can select the wrong work.
|
||||
- An allocator that raises denies.
|
||||
- PR work with no resolvable head SHA denies; a supplied SHA that no longer
|
||||
matches denies with `head_moved`.
|
||||
|
||||
## Enabling initiation
|
||||
|
||||
Execution is wired off. Set `WEBUI_REQUESTS_EXECUTION=1` to enable it for the
|
||||
`initiate_workflow` action only — see
|
||||
[`webui-authz-audit.md`](webui-authz-audit.md) for why this is an
|
||||
action-scoped flag rather than a phase bump. With the variable unset, `apply`
|
||||
returns `403` with `reason_code: unauthorized` no matter who asks.
|
||||
|
||||
Enabling execution does **not** enable approvals or merges. Those are phase 3
|
||||
console actions and remain forbidden in every path here; the console reserves
|
||||
work and hands off, and the MCP role profile enforces what that role may then
|
||||
do.
|
||||
|
||||
## Audit
|
||||
|
||||
Every preview and every apply emits a console audit record (schema in
|
||||
[`webui-authz-audit.md`](webui-authz-audit.md)):
|
||||
|
||||
| Event | `result` |
|
||||
|-------|----------|
|
||||
| Preview | `previewed` |
|
||||
| Refusal at any stage | `denied` |
|
||||
| Assignment created | `succeeded` |
|
||||
|
||||
`correlation.request_id` carries the request's `correlation_id`, and a
|
||||
successful record's `metadata` carries `assignment_id` and `lease_id`, so an
|
||||
assignment can be traced back to the intent that produced it. The operator's
|
||||
`intent_summary` travels in `metadata` and passes through the standard
|
||||
redaction pass before persistence like every other field.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- No browser-initiated approve or merge, in this phase or any other.
|
||||
- No bypass of allocator exclusive ownership; no self-selection of work.
|
||||
- No auto-start from raw monitoring incidents (#612 stays downstream).
|
||||
@@ -2069,6 +2069,7 @@ import lease_policy # noqa: E402
|
||||
import workflow_dashboard # noqa: E402 # #605 live queue/lease dashboard
|
||||
import restart_coordinator # noqa: E402 # #658 MCP restart coordinator/impact
|
||||
import drain_proof # noqa: E402 # #661 pre-restart drain proof and hard gate
|
||||
import restart_audit # noqa: E402 # #665 restart audit events + incidents
|
||||
import incident_bridge # noqa: E402
|
||||
import sentry_observability # noqa: E402 (#606 optional Sentry observability)
|
||||
import sentry_incident_bridge # noqa: E402 (#607 Sentry→Gitea incident bridge)
|
||||
@@ -22750,6 +22751,38 @@ def gitea_request_mcp_restart(
|
||||
# and a durable incident is raised. Break-glass is the only bypass and its
|
||||
# authorization is read from the environment, never self-asserted.
|
||||
payload["apply_supported"] = False
|
||||
# #665: correlation id threads impact preview → apply gate → incidents.
|
||||
correlation_id = restart_audit.new_correlation_id()
|
||||
payload["correlation_id"] = correlation_id
|
||||
auth_user = None
|
||||
try:
|
||||
# remote-only: host override is for operator diagnostics, not required here
|
||||
auth_user = (gitea_whoami(remote=remote) or {}).get("username")
|
||||
except Exception: # noqa: BLE001 — identity is best-effort for audit
|
||||
auth_user = None
|
||||
preview_audit = restart_audit.record_restart_lifecycle(
|
||||
event_type=restart_audit.EVENT_IMPACT_PREVIEW,
|
||||
outcome=str(report.verdict or "unknown"),
|
||||
correlation_id=correlation_id,
|
||||
remote=remote,
|
||||
org=o,
|
||||
repo=r,
|
||||
requesting_session_id=sid,
|
||||
restart_class=restart_class,
|
||||
profile_name=profile_name,
|
||||
authenticated_username=auth_user,
|
||||
reasons=list(report.reasons or []),
|
||||
details={
|
||||
"dry_run": True,
|
||||
"allow_restart": bool(report.allow_restart),
|
||||
"inventory_complete": inventory_complete,
|
||||
},
|
||||
privileged=False,
|
||||
)
|
||||
payload["restart_audit"] = {
|
||||
"correlation_id": correlation_id,
|
||||
"impact_preview_written": preview_audit["audit_written"],
|
||||
}
|
||||
if not dry_run:
|
||||
proof_obj: dict | None = None
|
||||
proof_parse_error: str | None = None
|
||||
@@ -22809,6 +22842,86 @@ def gitea_request_mcp_restart(
|
||||
)
|
||||
if not gate.allow and gate.incident is not None:
|
||||
payload["incident"] = gate.incident
|
||||
|
||||
# #665: audit apply-gate + materialize durable incidents for failed
|
||||
# drain and break-glass. Privileged apply denies if audit is enabled
|
||||
# and the sink write fails.
|
||||
incident_desc = restart_audit.incident_from_apply_gate(
|
||||
gate_payload={
|
||||
**gate_payload,
|
||||
"incident": gate.incident,
|
||||
"allow": gate.allow,
|
||||
},
|
||||
break_glass=break_glass,
|
||||
correlation_id=correlation_id,
|
||||
requesting_session_id=sid,
|
||||
restart_class=restart_class,
|
||||
remote=remote,
|
||||
org=o,
|
||||
repo=r,
|
||||
)
|
||||
if incident_desc is not None:
|
||||
payload["incident"] = incident_desc
|
||||
|
||||
def _create_restart_incident_issue(
|
||||
*, title, body, labels, org=None, repo=None, **_kw
|
||||
):
|
||||
return gitea_create_issue(
|
||||
title=title,
|
||||
body=body,
|
||||
labels=labels,
|
||||
remote=remote,
|
||||
host=h,
|
||||
org=org or o,
|
||||
repo=repo or r,
|
||||
)
|
||||
|
||||
apply_audit = restart_audit.record_restart_lifecycle(
|
||||
event_type=(
|
||||
restart_audit.EVENT_BREAK_GLASS
|
||||
if break_glass
|
||||
else restart_audit.EVENT_APPLY_GATE
|
||||
),
|
||||
outcome=(
|
||||
"break_glass"
|
||||
if break_glass
|
||||
else ("allow" if payload["apply_authorized"] else "deny")
|
||||
),
|
||||
correlation_id=correlation_id,
|
||||
remote=remote,
|
||||
org=o,
|
||||
repo=r,
|
||||
requesting_session_id=sid,
|
||||
restart_class=restart_class,
|
||||
profile_name=profile_name,
|
||||
authenticated_username=auth_user,
|
||||
reasons=list(gate_payload.get("reasons") or []),
|
||||
details={
|
||||
"apply_authorized": payload["apply_authorized"],
|
||||
"drain_gate_allow": gate_payload.get("drain_gate_allow"),
|
||||
"restart_class_authorized": restart_class_authorized,
|
||||
"break_glass": break_glass,
|
||||
"proof_id": gate_payload.get("proof_id"),
|
||||
},
|
||||
privileged=True,
|
||||
create_incident=incident_desc,
|
||||
create_issue_fn=_create_restart_incident_issue
|
||||
if incident_desc is not None
|
||||
else None,
|
||||
dry_run_incident=False,
|
||||
)
|
||||
payload["restart_audit"] = {
|
||||
"correlation_id": correlation_id,
|
||||
"impact_preview_written": preview_audit["audit_written"],
|
||||
"apply_gate_written": apply_audit["audit_written"],
|
||||
"incident_result": apply_audit.get("incident_result"),
|
||||
}
|
||||
if apply_audit["deny_reasons"]:
|
||||
payload["apply_authorized"] = False
|
||||
payload["reasons"] = list(payload.get("reasons") or []) + list(
|
||||
apply_audit["deny_reasons"]
|
||||
)
|
||||
payload["success"] = True
|
||||
return payload
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
"""MCP restart lifecycle audit events and incident materialization (#665).
|
||||
|
||||
Restarts and recovery attempts must leave a forensic trail: impact previews,
|
||||
drain enter/exit, drain-proof results, apply gate verdicts, break-glass, and
|
||||
post-restart reconcile outcomes. Failed drains and break-glass must also raise
|
||||
durable Gitea incident issues so they cannot be silently repeated.
|
||||
|
||||
This module is the pure + sink layer for that trail:
|
||||
|
||||
* **Schema** — ``mcp.restart.*`` event names and a redacted payload builder.
|
||||
* **Emission** — append-only via :mod:`gitea_audit` (off when ``GITEA_AUDIT_LOG``
|
||||
is unset; privileged apply can still *require* a successful write).
|
||||
* **Incidents** — descriptors for failed drain / break-glass / unguarded restart,
|
||||
plus an optional materializer that creates a Gitea issue through an injected
|
||||
``create_issue_fn`` (keeps this module free of network I/O in tests).
|
||||
|
||||
Design rules:
|
||||
|
||||
* **No secrets.** All free text is redacted before write or issue body assembly.
|
||||
* **Never raises from emission.** ``emit_restart_event`` returns False on sink
|
||||
failure so callers can decide fail-closed policy for privileged restarts.
|
||||
* **Does not restart.** Audit never executes a process restart.
|
||||
* **Drain proof stays #661.** This module records what the gate decided; it
|
||||
does not re-verify proofs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Callable, Mapping, Sequence
|
||||
|
||||
import gitea_audit
|
||||
|
||||
# ── Event vocabulary (stable identifiers for operators + tests) ───────────────
|
||||
|
||||
EVENT_IMPACT_PREVIEW = "mcp.restart.impact_preview"
|
||||
EVENT_DRAIN_ENTER = "mcp.restart.drain_enter"
|
||||
EVENT_DRAIN_EXIT = "mcp.restart.drain_exit"
|
||||
EVENT_DRAIN_PROOF = "mcp.restart.drain_proof"
|
||||
EVENT_APPLY_GATE = "mcp.restart.apply_gate"
|
||||
EVENT_BREAK_GLASS = "mcp.restart.break_glass"
|
||||
EVENT_POST_RESTART_RECONCILE = "mcp.restart.post_restart_reconcile"
|
||||
EVENT_NARROWER_RECOVERY = "mcp.restart.narrower_recovery"
|
||||
EVENT_UNGUARDED_DETECTED = "mcp.restart.unguarded_detected"
|
||||
|
||||
RESTART_EVENT_TYPES: frozenset[str] = frozenset(
|
||||
{
|
||||
EVENT_IMPACT_PREVIEW,
|
||||
EVENT_DRAIN_ENTER,
|
||||
EVENT_DRAIN_EXIT,
|
||||
EVENT_DRAIN_PROOF,
|
||||
EVENT_APPLY_GATE,
|
||||
EVENT_BREAK_GLASS,
|
||||
EVENT_POST_RESTART_RECONCILE,
|
||||
EVENT_NARROWER_RECOVERY,
|
||||
EVENT_UNGUARDED_DETECTED,
|
||||
}
|
||||
)
|
||||
|
||||
# Incident kinds (durable Gitea issues).
|
||||
INCIDENT_FAILED_DRAIN = "restart_failed_drain"
|
||||
INCIDENT_BREAK_GLASS = "restart_break_glass"
|
||||
INCIDENT_RECONCILE_UNRESOLVED = "restart_reconcile_unresolved"
|
||||
INCIDENT_UNGUARDED = "restart_unguarded_detected"
|
||||
|
||||
DEFAULT_INCIDENT_LABELS: tuple[str, ...] = (
|
||||
"mcp-health",
|
||||
"safety",
|
||||
"observability",
|
||||
"status:ready",
|
||||
"type:bug",
|
||||
"workflow-hardening",
|
||||
)
|
||||
|
||||
CreateIssueFn = Callable[..., dict[str, Any]]
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def new_correlation_id() -> str:
|
||||
"""Mint a short correlation id shared across a restart lifecycle."""
|
||||
return f"rst-{uuid.uuid4().hex[:16]}"
|
||||
|
||||
|
||||
def build_restart_event(
|
||||
*,
|
||||
event_type: str,
|
||||
outcome: str,
|
||||
correlation_id: str | None = None,
|
||||
remote: str | None = None,
|
||||
org: str | None = None,
|
||||
repo: str | None = None,
|
||||
requesting_session_id: str | None = None,
|
||||
restart_class: str | None = None,
|
||||
profile_name: str | None = None,
|
||||
authenticated_username: str | None = None,
|
||||
reasons: Sequence[str] | None = None,
|
||||
details: Mapping[str, Any] | None = None,
|
||||
now: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a redacted ``mcp.restart.*`` audit event.
|
||||
|
||||
Raises ``ValueError`` on unknown event types so a typo cannot silently land
|
||||
under a free-form action name.
|
||||
"""
|
||||
name = str(event_type or "").strip()
|
||||
if name not in RESTART_EVENT_TYPES:
|
||||
raise ValueError(
|
||||
f"unknown restart audit event_type {name!r}; expected one of "
|
||||
f"{sorted(RESTART_EVENT_TYPES)}"
|
||||
)
|
||||
redacted_reasons = [
|
||||
gitea_audit.redact(str(r)) for r in (reasons or []) if str(r).strip()
|
||||
]
|
||||
redacted_details = gitea_audit.redact(dict(details or {}))
|
||||
if not isinstance(redacted_details, dict):
|
||||
redacted_details = {"value": redacted_details}
|
||||
|
||||
event = gitea_audit.build_event(
|
||||
action=name,
|
||||
result=str(outcome or "unknown"),
|
||||
remote=remote,
|
||||
repository=f"{org}/{repo}" if org and repo else None,
|
||||
profile_name=profile_name,
|
||||
authenticated_username=authenticated_username,
|
||||
reason="; ".join(redacted_reasons) if redacted_reasons else None,
|
||||
request_metadata={
|
||||
"event_family": "mcp.restart",
|
||||
"correlation_id": correlation_id or new_correlation_id(),
|
||||
"restart_class": restart_class,
|
||||
"requesting_session_id": requesting_session_id,
|
||||
"org": org,
|
||||
"repo": repo,
|
||||
"details": redacted_details,
|
||||
"reasons": redacted_reasons,
|
||||
},
|
||||
now=now or _utc_now_iso(),
|
||||
operation=name,
|
||||
)
|
||||
event["action_type"] = "restart_lifecycle"
|
||||
event["event_type"] = name
|
||||
event["correlation_id"] = (event.get("request_metadata") or {}).get(
|
||||
"correlation_id"
|
||||
)
|
||||
return event
|
||||
|
||||
|
||||
def emit_restart_event(event: Mapping[str, Any], *, path: str | None = None) -> bool:
|
||||
"""Append *event* to the audit sink. Never raises. Returns write success."""
|
||||
try:
|
||||
return bool(gitea_audit.write_event(dict(event), path=path))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def require_audit_or_deny(
|
||||
*,
|
||||
privileged: bool,
|
||||
written: bool,
|
||||
audit_enabled: bool | None = None,
|
||||
) -> list[str]:
|
||||
"""Return deny reasons when a privileged restart path fails to audit.
|
||||
|
||||
When audit is not configured (``GITEA_AUDIT_LOG`` unset), privileged apply
|
||||
still proceeds under the rollout policy "enable audit before enforcing
|
||||
deny-on-audit-fail" — but *if* audit is enabled and the write fails,
|
||||
privileged apply is denied (fail closed).
|
||||
"""
|
||||
enabled = (
|
||||
gitea_audit.audit_enabled() if audit_enabled is None else bool(audit_enabled)
|
||||
)
|
||||
if not privileged:
|
||||
return []
|
||||
if not enabled:
|
||||
return []
|
||||
if written:
|
||||
return []
|
||||
return [
|
||||
"privileged restart path requires a successful audit write; "
|
||||
"audit sink failed (fail closed, #665)"
|
||||
]
|
||||
|
||||
|
||||
# ── Incident descriptors ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def build_incident_descriptor(
|
||||
*,
|
||||
kind: str,
|
||||
reasons: Sequence[str],
|
||||
correlation_id: str | None = None,
|
||||
requesting_session_id: str | None = None,
|
||||
restart_class: str | None = None,
|
||||
remote: str | None = None,
|
||||
org: str | None = None,
|
||||
repo: str | None = None,
|
||||
proof_id: str | None = None,
|
||||
at: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a durable incident descriptor (no network)."""
|
||||
titles = {
|
||||
INCIDENT_FAILED_DRAIN: "Restart denied: drain proof failed the hard gate",
|
||||
INCIDENT_BREAK_GLASS: "Break-glass MCP restart authorized",
|
||||
INCIDENT_RECONCILE_UNRESOLVED: "Post-restart reconcile left unresolved work",
|
||||
INCIDENT_UNGUARDED: "Unguarded MCP restart attempt detected",
|
||||
}
|
||||
title = titles.get(kind, f"MCP restart incident ({kind})")
|
||||
redacted_reasons = [
|
||||
gitea_audit.redact(str(r)) for r in reasons if str(r).strip()
|
||||
]
|
||||
return {
|
||||
"kind": kind,
|
||||
"title": title,
|
||||
"labels": list(DEFAULT_INCIDENT_LABELS),
|
||||
"reasons": redacted_reasons,
|
||||
"correlation_id": correlation_id,
|
||||
"requesting_session_id": requesting_session_id,
|
||||
"restart_class": restart_class,
|
||||
"remote": remote,
|
||||
"org": org,
|
||||
"repo": repo,
|
||||
"proof_id": proof_id,
|
||||
"at": at or _utc_now_iso(),
|
||||
"source": "restart_audit#665",
|
||||
}
|
||||
|
||||
|
||||
def incident_body(descriptor: Mapping[str, Any]) -> str:
|
||||
"""Render a redacted markdown body for a Gitea incident issue."""
|
||||
reasons = descriptor.get("reasons") or []
|
||||
reason_lines = "\n".join(f"- {gitea_audit.redact(str(r))}" for r in reasons) or (
|
||||
"- (no reasons recorded)"
|
||||
)
|
||||
return "\n".join(
|
||||
[
|
||||
"<!-- mcp-restart-incident:v1 -->",
|
||||
f"## MCP restart incident (`{descriptor.get('kind')}`)",
|
||||
"",
|
||||
f"**Correlation:** `{descriptor.get('correlation_id') or 'none'}`",
|
||||
f"**Session:** `{descriptor.get('requesting_session_id') or 'none'}`",
|
||||
f"**Class:** `{descriptor.get('restart_class') or 'none'}`",
|
||||
f"**Scope:** `{descriptor.get('remote')}/{descriptor.get('org')}/"
|
||||
f"{descriptor.get('repo')}`",
|
||||
f"**At:** `{descriptor.get('at')}`",
|
||||
f"**Proof id:** `{descriptor.get('proof_id') or 'none'}`",
|
||||
"",
|
||||
"### Reasons",
|
||||
reason_lines,
|
||||
"",
|
||||
"### Operator next steps",
|
||||
"- Treat this as durable follow-up work under the restart-governance umbrella (#655).",
|
||||
"- Do not invent a second restart path; use sanctioned coordinator tools only.",
|
||||
"- Raw secrets must never appear in this issue (already redacted).",
|
||||
"",
|
||||
f"_Source: {descriptor.get('source')}_",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def materialize_incident(
|
||||
descriptor: Mapping[str, Any],
|
||||
*,
|
||||
create_issue_fn: CreateIssueFn | None,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a Gitea issue from *descriptor* when *create_issue_fn* is provided.
|
||||
|
||||
Returns a result dict with ``created`` / ``issue_number`` / ``dry_run`` /
|
||||
``reasons``. Never raises.
|
||||
"""
|
||||
base: dict[str, Any] = {
|
||||
"created": False,
|
||||
"dry_run": bool(dry_run),
|
||||
"issue_number": None,
|
||||
"kind": descriptor.get("kind"),
|
||||
"reasons": [],
|
||||
"descriptor": dict(descriptor),
|
||||
}
|
||||
if dry_run:
|
||||
base["reasons"] = ["dry-run only; no Gitea issue created"]
|
||||
return base
|
||||
if create_issue_fn is None:
|
||||
base["reasons"] = [
|
||||
"create_issue_fn not provided; incident descriptor retained only"
|
||||
]
|
||||
return base
|
||||
try:
|
||||
result = create_issue_fn(
|
||||
title=str(descriptor.get("title") or "MCP restart incident"),
|
||||
body=incident_body(descriptor),
|
||||
labels=list(descriptor.get("labels") or DEFAULT_INCIDENT_LABELS),
|
||||
org=descriptor.get("org"),
|
||||
repo=descriptor.get("repo"),
|
||||
)
|
||||
number = None
|
||||
if isinstance(result, dict):
|
||||
number = result.get("number") or result.get("issue_number")
|
||||
if number is not None:
|
||||
base["created"] = True
|
||||
base["issue_number"] = int(number)
|
||||
base["reasons"] = [f"created incident issue #{int(number)}"]
|
||||
else:
|
||||
base["reasons"] = ["create_issue_fn returned no issue number"]
|
||||
except Exception as exc: # noqa: BLE001 — never break restart path here
|
||||
base["reasons"] = [
|
||||
f"incident issue creation failed: {gitea_audit.redact(str(exc))}"
|
||||
]
|
||||
return base
|
||||
|
||||
|
||||
def record_restart_lifecycle(
|
||||
*,
|
||||
event_type: str,
|
||||
outcome: str,
|
||||
correlation_id: str,
|
||||
remote: str | None = None,
|
||||
org: str | None = None,
|
||||
repo: str | None = None,
|
||||
requesting_session_id: str | None = None,
|
||||
restart_class: str | None = None,
|
||||
profile_name: str | None = None,
|
||||
authenticated_username: str | None = None,
|
||||
reasons: Sequence[str] | None = None,
|
||||
details: Mapping[str, Any] | None = None,
|
||||
privileged: bool = False,
|
||||
create_incident: Mapping[str, Any] | None = None,
|
||||
create_issue_fn: CreateIssueFn | None = None,
|
||||
dry_run_incident: bool = False,
|
||||
audit_path: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Emit one restart audit event and optionally materialize an incident.
|
||||
|
||||
Returns ``{event, audit_written, deny_reasons, incident_result}``.
|
||||
"""
|
||||
event = build_restart_event(
|
||||
event_type=event_type,
|
||||
outcome=outcome,
|
||||
correlation_id=correlation_id,
|
||||
remote=remote,
|
||||
org=org,
|
||||
repo=repo,
|
||||
requesting_session_id=requesting_session_id,
|
||||
restart_class=restart_class,
|
||||
profile_name=profile_name,
|
||||
authenticated_username=authenticated_username,
|
||||
reasons=reasons,
|
||||
details=details,
|
||||
)
|
||||
written = emit_restart_event(event, path=audit_path)
|
||||
deny = require_audit_or_deny(privileged=privileged, written=written)
|
||||
incident_result = None
|
||||
if create_incident is not None:
|
||||
incident_result = materialize_incident(
|
||||
create_incident,
|
||||
create_issue_fn=create_issue_fn,
|
||||
dry_run=dry_run_incident,
|
||||
)
|
||||
return {
|
||||
"event": event,
|
||||
"audit_written": written,
|
||||
"deny_reasons": deny,
|
||||
"incident_result": incident_result,
|
||||
"correlation_id": correlation_id,
|
||||
}
|
||||
|
||||
|
||||
def incident_from_apply_gate(
|
||||
*,
|
||||
gate_payload: Mapping[str, Any],
|
||||
break_glass: bool,
|
||||
correlation_id: str,
|
||||
requesting_session_id: str | None,
|
||||
restart_class: str | None,
|
||||
remote: str | None,
|
||||
org: str | None,
|
||||
repo: str | None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Choose an incident descriptor from an apply-gate payload, if required."""
|
||||
reasons = list(gate_payload.get("reasons") or [])
|
||||
proof_id = gate_payload.get("proof_id")
|
||||
if break_glass:
|
||||
return build_incident_descriptor(
|
||||
kind=INCIDENT_BREAK_GLASS,
|
||||
reasons=reasons
|
||||
or ["break-glass restart path used; durable incident required (#665)"],
|
||||
correlation_id=correlation_id,
|
||||
requesting_session_id=requesting_session_id,
|
||||
restart_class=restart_class,
|
||||
remote=remote,
|
||||
org=org,
|
||||
repo=repo,
|
||||
proof_id=proof_id if isinstance(proof_id, str) else None,
|
||||
)
|
||||
# Failed drain / deny path.
|
||||
incident = gate_payload.get("incident")
|
||||
if isinstance(incident, Mapping) and incident:
|
||||
# Normalize gate-provided descriptor into our schema.
|
||||
return build_incident_descriptor(
|
||||
kind=INCIDENT_FAILED_DRAIN,
|
||||
reasons=list(incident.get("reasons") or reasons),
|
||||
correlation_id=correlation_id,
|
||||
requesting_session_id=requesting_session_id
|
||||
or incident.get("requesting_session_id"),
|
||||
restart_class=restart_class,
|
||||
remote=remote,
|
||||
org=org,
|
||||
repo=repo,
|
||||
proof_id=incident.get("proof_id") or proof_id,
|
||||
at=incident.get("at"),
|
||||
)
|
||||
if not gate_payload.get("allow") and not gate_payload.get("drain_gate_allow", True):
|
||||
return build_incident_descriptor(
|
||||
kind=INCIDENT_FAILED_DRAIN,
|
||||
reasons=reasons or ["restart apply denied"],
|
||||
correlation_id=correlation_id,
|
||||
requesting_session_id=requesting_session_id,
|
||||
restart_class=restart_class,
|
||||
remote=remote,
|
||||
org=org,
|
||||
repo=repo,
|
||||
proof_id=proof_id if isinstance(proof_id, str) else None,
|
||||
)
|
||||
return None
|
||||
@@ -7,7 +7,6 @@ import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from allocator_service import (
|
||||
OUTCOME_ASSIGNED,
|
||||
@@ -16,7 +15,6 @@ from allocator_service import (
|
||||
OUTCOME_PREVIEW,
|
||||
OUTCOME_WAIT,
|
||||
WorkCandidate,
|
||||
_drop_expired_claims,
|
||||
allocate_next_work,
|
||||
candidate_from_dict,
|
||||
classify_skip,
|
||||
@@ -364,161 +362,5 @@ class AllocatorServiceTest(unittest.TestCase):
|
||||
self.assertIn("unavailable", res["reasons"][0].lower())
|
||||
|
||||
|
||||
class SideEffectFreeAllocationTest(unittest.TestCase):
|
||||
"""``side_effect_free`` dry runs write nothing to the control plane (#643).
|
||||
|
||||
A plain ``apply=False`` still called ``upsert_session`` and
|
||||
``expire_stale_leases`` before the apply branch was consulted, so a caller
|
||||
advertising a read-only preview mutated on every call — one unreferenced
|
||||
session row per preview, plus a global lease sweep.
|
||||
"""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.db = ControlPlaneDB(os.path.join(self._tmp.name, "cp.sqlite3"))
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _alloc(self, **kwargs):
|
||||
defaults = dict(
|
||||
db=self.db,
|
||||
session_id="s-preview",
|
||||
role="author",
|
||||
remote="prgs",
|
||||
org="org",
|
||||
repo="repo",
|
||||
candidates=[
|
||||
WorkCandidate(kind="issue", number=643, labels=("status:ready",))
|
||||
],
|
||||
apply=False,
|
||||
profile_name="prgs-author",
|
||||
username="jcwalker3",
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return allocate_next_work(**defaults)
|
||||
|
||||
def _session_ids(self) -> set[str]:
|
||||
return {str(r.get("session_id")) for r in self.db.list_sessions()}
|
||||
|
||||
def test_side_effect_free_preview_writes_no_session_row(self):
|
||||
before = self._session_ids()
|
||||
result = self._alloc(side_effect_free=True)
|
||||
self.assertEqual(result["outcome"], OUTCOME_PREVIEW)
|
||||
self.assertEqual(self._session_ids(), before)
|
||||
self.assertNotIn("s-preview", self._session_ids())
|
||||
|
||||
def test_plain_dry_run_still_registers_a_session(self):
|
||||
# The default is unchanged for every existing caller.
|
||||
self._alloc()
|
||||
self.assertIn("s-preview", self._session_ids())
|
||||
|
||||
def test_repeated_previews_do_not_accumulate_rows(self):
|
||||
for index in range(5):
|
||||
self._alloc(side_effect_free=True, session_id=f"s-{index}")
|
||||
self.assertEqual(self._session_ids(), set())
|
||||
|
||||
def test_side_effect_free_does_not_sweep_stale_leases(self):
|
||||
self.db.upsert_session(session_id="owner", role="author", pid=1)
|
||||
assigned = self.db.assign_and_lease(
|
||||
session_id="owner",
|
||||
role="author",
|
||||
remote="prgs",
|
||||
org="org",
|
||||
repo="repo",
|
||||
kind="issue",
|
||||
number=999,
|
||||
lease_ttl_seconds=-60, # already expired
|
||||
)
|
||||
self.assertEqual(assigned.outcome, "assigned")
|
||||
|
||||
self._alloc(side_effect_free=True)
|
||||
|
||||
# The expired row is still 'active' in the DB: nothing swept it.
|
||||
statuses = {
|
||||
r["lease_id"]: r["status"]
|
||||
for r in self.db.list_leases(
|
||||
remote="prgs", org="org", repo="repo",
|
||||
statuses=("active", "expired"),
|
||||
)
|
||||
}
|
||||
self.assertEqual(statuses.get(assigned.lease_id), "active")
|
||||
|
||||
def test_expired_claims_are_filtered_in_memory_so_work_stays_selectable(self):
|
||||
"""The read-only mirror of the sweep: expired claims must not block."""
|
||||
self.db.upsert_session(session_id="owner", role="author", pid=1)
|
||||
self.db.assign_and_lease(
|
||||
session_id="owner",
|
||||
role="author",
|
||||
remote="prgs",
|
||||
org="org",
|
||||
repo="repo",
|
||||
kind="issue",
|
||||
number=643,
|
||||
lease_ttl_seconds=-60, # expired: must not withhold #643
|
||||
)
|
||||
result = self._alloc(side_effect_free=True)
|
||||
self.assertEqual(result["outcome"], OUTCOME_PREVIEW)
|
||||
self.assertEqual(result["selected"]["number"], 643)
|
||||
|
||||
def test_a_live_claim_still_withholds_the_work(self):
|
||||
self.db.upsert_session(session_id="owner", role="author", pid=1)
|
||||
self.db.assign_and_lease(
|
||||
session_id="owner",
|
||||
role="author",
|
||||
remote="prgs",
|
||||
org="org",
|
||||
repo="repo",
|
||||
kind="issue",
|
||||
number=643,
|
||||
lease_ttl_seconds=3600,
|
||||
)
|
||||
result = self._alloc(side_effect_free=True)
|
||||
self.assertNotEqual(result["outcome"], OUTCOME_ASSIGNED)
|
||||
self.assertNotEqual((result.get("selected") or {}).get("number"), 643)
|
||||
|
||||
def test_side_effect_free_with_apply_fails_closed(self):
|
||||
result = self._alloc(side_effect_free=True, apply=True)
|
||||
self.assertFalse(result["success"])
|
||||
self.assertEqual(result["outcome"], OUTCOME_NO_SAFE)
|
||||
self.assertIsNone(result["assignment"])
|
||||
self.assertIn("incompatible with apply", result["reasons"][0])
|
||||
# And it reserved nothing.
|
||||
self.assertEqual(
|
||||
self.db.list_leases(remote="prgs", org="org", repo="repo"), []
|
||||
)
|
||||
|
||||
|
||||
class DropExpiredClaimsTest(unittest.TestCase):
|
||||
"""The in-memory expiry filter behind side-effect-free previews (#643)."""
|
||||
|
||||
def test_unparseable_expiry_is_kept_rather_than_assumed_free(self):
|
||||
claims = {
|
||||
("issue", 1): {"lease_id": "l1", "expires_at": "not-a-date"},
|
||||
("issue", 2): {"lease_id": "l2"},
|
||||
("issue", 3): {"lease_id": "l3", "expires_at": None},
|
||||
}
|
||||
self.assertEqual(_drop_expired_claims(claims), claims)
|
||||
|
||||
def test_expired_dropped_and_future_kept(self):
|
||||
now = datetime(2026, 7, 25, 12, 0, tzinfo=timezone.utc)
|
||||
claims = {
|
||||
("issue", 1): {"expires_at": "2026-07-25T11:59:59+00:00"},
|
||||
("issue", 2): {"expires_at": "2026-07-25T12:00:01+00:00"},
|
||||
("issue", 3): {"expires_at": "2026-07-25T12:00:00+00:00"}, # boundary
|
||||
}
|
||||
kept = _drop_expired_claims(claims, now=now)
|
||||
self.assertEqual(set(kept), {("issue", 2)})
|
||||
|
||||
def test_naive_and_zulu_timestamps_are_treated_as_utc(self):
|
||||
now = datetime(2026, 7, 25, 12, 0, tzinfo=timezone.utc)
|
||||
claims = {
|
||||
("issue", 1): {"expires_at": "2026-07-25T11:00:00"}, # naive, past
|
||||
("issue", 2): {"expires_at": "2026-07-25T13:00:00Z"}, # zulu, future
|
||||
}
|
||||
kept = _drop_expired_claims(claims, now=now)
|
||||
self.assertEqual(set(kept), {("issue", 2)})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
"""Tests for MCP restart lifecycle audit events and incidents (#665)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import gitea_audit
|
||||
import restart_audit as ra
|
||||
|
||||
|
||||
class TestEventSchema(unittest.TestCase):
|
||||
def test_all_lifecycle_event_types_are_named(self):
|
||||
expected = {
|
||||
"mcp.restart.impact_preview",
|
||||
"mcp.restart.drain_enter",
|
||||
"mcp.restart.drain_exit",
|
||||
"mcp.restart.drain_proof",
|
||||
"mcp.restart.apply_gate",
|
||||
"mcp.restart.break_glass",
|
||||
"mcp.restart.post_restart_reconcile",
|
||||
"mcp.restart.narrower_recovery",
|
||||
"mcp.restart.unguarded_detected",
|
||||
}
|
||||
self.assertEqual(set(ra.RESTART_EVENT_TYPES), expected)
|
||||
|
||||
def test_build_restart_event_core_fields(self):
|
||||
event = ra.build_restart_event(
|
||||
event_type=ra.EVENT_IMPACT_PREVIEW,
|
||||
outcome="safe",
|
||||
correlation_id="rst-abc123",
|
||||
remote="prgs",
|
||||
org="Scaled-Tech-Consulting",
|
||||
repo="Gitea-Tools",
|
||||
requesting_session_id="sess-1",
|
||||
restart_class="full_mcp_restart",
|
||||
profile_name="prgs-author",
|
||||
authenticated_username="bot",
|
||||
reasons=["inventory complete"],
|
||||
details={"allow_restart": True},
|
||||
now="2026-07-25T12:00:00+00:00",
|
||||
)
|
||||
self.assertEqual(event["event_type"], ra.EVENT_IMPACT_PREVIEW)
|
||||
self.assertEqual(event["action"], ra.EVENT_IMPACT_PREVIEW)
|
||||
self.assertEqual(event["action_type"], "restart_lifecycle")
|
||||
self.assertEqual(event["result"], "safe")
|
||||
self.assertEqual(event["correlation_id"], "rst-abc123")
|
||||
self.assertEqual(event["profile_name"], "prgs-author")
|
||||
self.assertEqual(event["authenticated_username"], "bot")
|
||||
meta = event["request_metadata"]
|
||||
self.assertEqual(meta["event_family"], "mcp.restart")
|
||||
self.assertEqual(meta["correlation_id"], "rst-abc123")
|
||||
self.assertEqual(meta["restart_class"], "full_mcp_restart")
|
||||
self.assertEqual(meta["details"]["allow_restart"], True)
|
||||
|
||||
def test_unknown_event_type_raises(self):
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
ra.build_restart_event(
|
||||
event_type="mcp.restart.not_a_real_event",
|
||||
outcome="x",
|
||||
correlation_id="rst-1",
|
||||
)
|
||||
self.assertIn("unknown restart audit event_type", str(ctx.exception))
|
||||
|
||||
def test_reasons_and_details_are_redacted(self):
|
||||
event = ra.build_restart_event(
|
||||
event_type=ra.EVENT_APPLY_GATE,
|
||||
outcome="deny",
|
||||
correlation_id="rst-sec",
|
||||
reasons=["token secret-xyz rejected", "ok"],
|
||||
details={"token": "leak-token", "status": "denied"},
|
||||
)
|
||||
self.assertNotIn("secret-xyz", event.get("reason") or "")
|
||||
meta = event["request_metadata"]
|
||||
self.assertEqual(meta["details"]["token"], gitea_audit.REDACTED)
|
||||
self.assertEqual(meta["details"]["status"], "denied")
|
||||
for reason in meta["reasons"]:
|
||||
self.assertNotIn("secret-xyz", reason)
|
||||
|
||||
def test_new_correlation_id_shape(self):
|
||||
cid = ra.new_correlation_id()
|
||||
self.assertTrue(cid.startswith("rst-"))
|
||||
self.assertEqual(len(cid), len("rst-") + 16)
|
||||
|
||||
|
||||
class TestEmitAndRequire(unittest.TestCase):
|
||||
def test_emit_appends_json_line(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "audit.log")
|
||||
event = ra.build_restart_event(
|
||||
event_type=ra.EVENT_DRAIN_PROOF,
|
||||
outcome="pass",
|
||||
correlation_id="rst-write",
|
||||
)
|
||||
self.assertTrue(ra.emit_restart_event(event, path=path))
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
lines = fh.read().splitlines()
|
||||
self.assertEqual(len(lines), 1)
|
||||
loaded = json.loads(lines[0])
|
||||
self.assertEqual(loaded["event_type"], ra.EVENT_DRAIN_PROOF)
|
||||
self.assertEqual(loaded["correlation_id"], "rst-write")
|
||||
|
||||
def test_emit_never_raises(self):
|
||||
self.assertFalse(
|
||||
ra.emit_restart_event({"action": "x"}, path="/no/such/dir/audit.log")
|
||||
)
|
||||
|
||||
def test_require_audit_denies_privileged_when_write_fails_and_enabled(self):
|
||||
deny = ra.require_audit_or_deny(
|
||||
privileged=True, written=False, audit_enabled=True
|
||||
)
|
||||
self.assertEqual(len(deny), 1)
|
||||
self.assertIn("fail closed", deny[0])
|
||||
|
||||
def test_require_audit_allows_when_audit_disabled(self):
|
||||
# Rollout policy: enable audit before enforcing deny-on-audit-fail.
|
||||
deny = ra.require_audit_or_deny(
|
||||
privileged=True, written=False, audit_enabled=False
|
||||
)
|
||||
self.assertEqual(deny, [])
|
||||
|
||||
def test_require_audit_noop_for_non_privileged(self):
|
||||
deny = ra.require_audit_or_deny(
|
||||
privileged=False, written=False, audit_enabled=True
|
||||
)
|
||||
self.assertEqual(deny, [])
|
||||
|
||||
def test_require_audit_allows_when_written(self):
|
||||
deny = ra.require_audit_or_deny(
|
||||
privileged=True, written=True, audit_enabled=True
|
||||
)
|
||||
self.assertEqual(deny, [])
|
||||
|
||||
|
||||
class TestIncidents(unittest.TestCase):
|
||||
def test_break_glass_descriptor(self):
|
||||
desc = ra.build_incident_descriptor(
|
||||
kind=ra.INCIDENT_BREAK_GLASS,
|
||||
reasons=["break-glass authorized"],
|
||||
correlation_id="rst-bg",
|
||||
requesting_session_id="s1",
|
||||
restart_class="full_mcp_restart",
|
||||
remote="prgs",
|
||||
org="O",
|
||||
repo="R",
|
||||
)
|
||||
self.assertEqual(desc["kind"], ra.INCIDENT_BREAK_GLASS)
|
||||
self.assertIn("Break-glass", desc["title"])
|
||||
self.assertIn("mcp-health", desc["labels"])
|
||||
self.assertEqual(desc["source"], "restart_audit#665")
|
||||
|
||||
def test_incident_body_redacts_and_includes_correlation(self):
|
||||
desc = ra.build_incident_descriptor(
|
||||
kind=ra.INCIDENT_FAILED_DRAIN,
|
||||
reasons=["token secret-xyz failed proof"],
|
||||
correlation_id="rst-body",
|
||||
remote="prgs",
|
||||
org="O",
|
||||
repo="R",
|
||||
proof_id="proof-1",
|
||||
)
|
||||
body = ra.incident_body(desc)
|
||||
self.assertIn("rst-body", body)
|
||||
self.assertIn("proof-1", body)
|
||||
self.assertIn("mcp-restart-incident:v1", body)
|
||||
self.assertNotIn("secret-xyz", body)
|
||||
|
||||
def test_materialize_dry_run(self):
|
||||
desc = ra.build_incident_descriptor(
|
||||
kind=ra.INCIDENT_FAILED_DRAIN,
|
||||
reasons=["denied"],
|
||||
correlation_id="rst-dr",
|
||||
)
|
||||
result = ra.materialize_incident(desc, create_issue_fn=lambda **k: {}, dry_run=True)
|
||||
self.assertFalse(result["created"])
|
||||
self.assertTrue(result["dry_run"])
|
||||
self.assertIn("dry-run", result["reasons"][0])
|
||||
|
||||
def test_materialize_without_create_fn(self):
|
||||
desc = ra.build_incident_descriptor(
|
||||
kind=ra.INCIDENT_FAILED_DRAIN,
|
||||
reasons=["denied"],
|
||||
correlation_id="rst-nfn",
|
||||
)
|
||||
result = ra.materialize_incident(desc, create_issue_fn=None)
|
||||
self.assertFalse(result["created"])
|
||||
self.assertIn("create_issue_fn not provided", result["reasons"][0])
|
||||
|
||||
def test_materialize_creates_issue(self):
|
||||
created = {}
|
||||
|
||||
def _create(*, title, body, labels, org=None, repo=None, **_kw):
|
||||
created["title"] = title
|
||||
created["body"] = body
|
||||
created["labels"] = labels
|
||||
created["org"] = org
|
||||
created["repo"] = repo
|
||||
return {"number": 999}
|
||||
|
||||
desc = ra.build_incident_descriptor(
|
||||
kind=ra.INCIDENT_BREAK_GLASS,
|
||||
reasons=["break-glass"],
|
||||
correlation_id="rst-create",
|
||||
org="O",
|
||||
repo="R",
|
||||
)
|
||||
result = ra.materialize_incident(desc, create_issue_fn=_create)
|
||||
self.assertTrue(result["created"])
|
||||
self.assertEqual(result["issue_number"], 999)
|
||||
self.assertIn("Break-glass", created["title"])
|
||||
self.assertIn("rst-create", created["body"])
|
||||
self.assertEqual(created["org"], "O")
|
||||
|
||||
def test_materialize_never_raises_on_create_failure(self):
|
||||
def _boom(**_kw):
|
||||
raise RuntimeError("token secret-xyz network")
|
||||
|
||||
desc = ra.build_incident_descriptor(
|
||||
kind=ra.INCIDENT_FAILED_DRAIN,
|
||||
reasons=["x"],
|
||||
correlation_id="rst-boom",
|
||||
)
|
||||
result = ra.materialize_incident(desc, create_issue_fn=_boom)
|
||||
self.assertFalse(result["created"])
|
||||
self.assertIn("failed", result["reasons"][0])
|
||||
self.assertNotIn("secret-xyz", result["reasons"][0])
|
||||
|
||||
|
||||
class TestIncidentFromApplyGate(unittest.TestCase):
|
||||
def test_break_glass_always_incident(self):
|
||||
desc = ra.incident_from_apply_gate(
|
||||
gate_payload={"allow": True, "reasons": [], "proof_id": None},
|
||||
break_glass=True,
|
||||
correlation_id="rst-bg2",
|
||||
requesting_session_id="s",
|
||||
restart_class="full_mcp_restart",
|
||||
remote="prgs",
|
||||
org="O",
|
||||
repo="R",
|
||||
)
|
||||
self.assertIsNotNone(desc)
|
||||
self.assertEqual(desc["kind"], ra.INCIDENT_BREAK_GLASS)
|
||||
|
||||
def test_failed_drain_from_gate_incident(self):
|
||||
desc = ra.incident_from_apply_gate(
|
||||
gate_payload={
|
||||
"allow": False,
|
||||
"drain_gate_allow": False,
|
||||
"reasons": ["proof expired"],
|
||||
"incident": {
|
||||
"reasons": ["proof expired"],
|
||||
"proof_id": "p1",
|
||||
},
|
||||
},
|
||||
break_glass=False,
|
||||
correlation_id="rst-fd",
|
||||
requesting_session_id="s",
|
||||
restart_class="full_mcp_restart",
|
||||
remote="prgs",
|
||||
org="O",
|
||||
repo="R",
|
||||
)
|
||||
self.assertIsNotNone(desc)
|
||||
self.assertEqual(desc["kind"], ra.INCIDENT_FAILED_DRAIN)
|
||||
self.assertEqual(desc["proof_id"], "p1")
|
||||
|
||||
def test_allow_without_break_glass_no_incident(self):
|
||||
desc = ra.incident_from_apply_gate(
|
||||
gate_payload={
|
||||
"allow": True,
|
||||
"drain_gate_allow": True,
|
||||
"reasons": [],
|
||||
},
|
||||
break_glass=False,
|
||||
correlation_id="rst-ok",
|
||||
requesting_session_id="s",
|
||||
restart_class="full_mcp_restart",
|
||||
remote="prgs",
|
||||
org="O",
|
||||
repo="R",
|
||||
)
|
||||
self.assertIsNone(desc)
|
||||
|
||||
|
||||
class TestRecordLifecycle(unittest.TestCase):
|
||||
def test_record_emits_and_materializes(self):
|
||||
created = []
|
||||
|
||||
def _create(**kwargs):
|
||||
created.append(kwargs)
|
||||
return {"number": 42}
|
||||
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "audit.log")
|
||||
with patch.dict(os.environ, {"GITEA_AUDIT_LOG": path}, clear=False):
|
||||
incident = ra.build_incident_descriptor(
|
||||
kind=ra.INCIDENT_BREAK_GLASS,
|
||||
reasons=["bg"],
|
||||
correlation_id="rst-lc",
|
||||
org="O",
|
||||
repo="R",
|
||||
)
|
||||
out = ra.record_restart_lifecycle(
|
||||
event_type=ra.EVENT_BREAK_GLASS,
|
||||
outcome="break_glass",
|
||||
correlation_id="rst-lc",
|
||||
remote="prgs",
|
||||
org="O",
|
||||
repo="R",
|
||||
privileged=True,
|
||||
create_incident=incident,
|
||||
create_issue_fn=_create,
|
||||
audit_path=path,
|
||||
)
|
||||
self.assertTrue(out["audit_written"])
|
||||
self.assertEqual(out["deny_reasons"], [])
|
||||
self.assertTrue(out["incident_result"]["created"])
|
||||
self.assertEqual(out["incident_result"]["issue_number"], 42)
|
||||
self.assertEqual(len(created), 1)
|
||||
|
||||
def test_privileged_deny_when_audit_write_fails(self):
|
||||
with patch.dict(
|
||||
os.environ, {"GITEA_AUDIT_LOG": "/no/such/dir/a.log"}, clear=False
|
||||
):
|
||||
with patch("restart_audit.emit_restart_event", return_value=False):
|
||||
with patch("gitea_audit.audit_enabled", return_value=True):
|
||||
out = ra.record_restart_lifecycle(
|
||||
event_type=ra.EVENT_APPLY_GATE,
|
||||
outcome="deny",
|
||||
correlation_id="rst-deny",
|
||||
privileged=True,
|
||||
audit_path="/no/such/dir/a.log",
|
||||
)
|
||||
self.assertFalse(out["audit_written"])
|
||||
self.assertEqual(len(out["deny_reasons"]), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
File diff suppressed because it is too large
Load Diff
-116
@@ -72,8 +72,6 @@ from webui.system_health import (
|
||||
snapshot_to_dict as system_health_to_dict,
|
||||
)
|
||||
from webui.system_health_views import render_system_health_page
|
||||
from webui import request_service
|
||||
from webui.request_views import render_requests_page
|
||||
|
||||
_READ_ONLY_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
|
||||
_AUDIT_MUTATION_PATHS = frozenset({"/audit", "/api/audit"})
|
||||
@@ -741,109 +739,6 @@ async def api_v1_analytics_ingest(request: Request) -> JSONResponse:
|
||||
)
|
||||
|
||||
|
||||
def _default_request_scope() -> dict[str, str]:
|
||||
"""Resolve remote/org/repo from the project registry for request forms.
|
||||
|
||||
Returns an empty mapping when the registry cannot be read, which makes
|
||||
``parse_request`` reject a request that did not name its own scope rather
|
||||
than letting it default to some other repository.
|
||||
"""
|
||||
from webui.queue_loader import _host_from_url # host normalisation helper
|
||||
|
||||
registry, error = _load_project_registry()
|
||||
if error is not None or not registry.projects:
|
||||
return {}
|
||||
project = registry.projects[0]
|
||||
host = _host_from_url(project.remote_host)
|
||||
return {
|
||||
"remote": _derive_remote(host),
|
||||
"org": project.gitea_owner or "",
|
||||
"repo": project.repo_name or "",
|
||||
}
|
||||
|
||||
|
||||
async def _request_payload(request: Request) -> dict[str, object]:
|
||||
"""Read a request body as JSON or form-encoded. Never raises."""
|
||||
content_type = (request.headers.get("content-type") or "").lower()
|
||||
if "application/json" in content_type:
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
return {}
|
||||
return dict(body) if isinstance(body, dict) else {}
|
||||
try:
|
||||
form = await request.form()
|
||||
except Exception:
|
||||
return {}
|
||||
return {key: form[key] for key in form}
|
||||
|
||||
|
||||
async def requests_page(request: Request) -> HTMLResponse:
|
||||
"""Operator request form and intent preview (#643).
|
||||
|
||||
POST here only ever *previews*. Initiation is a separate confirmed call to
|
||||
``/api/v1/requests/apply`` so that submitting this form cannot reserve
|
||||
work as a side effect.
|
||||
"""
|
||||
submitted: dict[str, object] = {}
|
||||
preview = None
|
||||
error = None
|
||||
if request.method == "POST":
|
||||
submitted = await _request_payload(request)
|
||||
work_request, error = request_service.parse_request(
|
||||
submitted, default_scope=_default_request_scope()
|
||||
)
|
||||
if work_request is not None:
|
||||
preview = request_service.preview_request(
|
||||
work_request,
|
||||
principal=resolve_principal(headers=dict(request.headers)),
|
||||
)
|
||||
return HTMLResponse(
|
||||
render_requests_page(
|
||||
preview=preview, error=error, submitted=submitted
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def api_v1_request_preview(request: Request) -> JSONResponse:
|
||||
"""Dry-run authorization and intent preview for a work request (#643)."""
|
||||
payload = await _request_payload(request)
|
||||
work_request, error = request_service.parse_request(
|
||||
payload, default_scope=_default_request_scope()
|
||||
)
|
||||
if work_request is None:
|
||||
return JSONResponse(error.to_dict(), status_code=400)
|
||||
preview = request_service.preview_request(
|
||||
work_request,
|
||||
principal=resolve_principal(headers=dict(request.headers)),
|
||||
)
|
||||
return JSONResponse(
|
||||
preview.to_dict(), status_code=200 if preview.authorized else 403
|
||||
)
|
||||
|
||||
|
||||
async def api_v1_request_apply(request: Request) -> JSONResponse:
|
||||
"""Initiate a previewed work request through the allocator (#643).
|
||||
|
||||
Fail-closed at every step: unauthorized, unconfirmed, not-next-safe, and
|
||||
already-claimed all return without attempting an assignment.
|
||||
"""
|
||||
payload = await _request_payload(request)
|
||||
work_request, error = request_service.parse_request(
|
||||
payload, default_scope=_default_request_scope()
|
||||
)
|
||||
if work_request is None:
|
||||
return JSONResponse(error.to_dict(), status_code=400)
|
||||
confirm = _truthy_flag(str(payload.get("confirm") or ""))
|
||||
result = request_service.apply_request(
|
||||
work_request,
|
||||
principal=resolve_principal(headers=dict(request.headers)),
|
||||
confirm=confirm,
|
||||
)
|
||||
status = int(result.pop("status_code", 403))
|
||||
return JSONResponse(result, status_code=status)
|
||||
|
||||
|
||||
async def method_not_allowed(request: Request, _exc: Exception) -> Response:
|
||||
path = request.url.path
|
||||
if path in _AUDIT_MUTATION_PATHS and request.method == "POST":
|
||||
@@ -911,17 +806,6 @@ def create_app(*, bind_host: str | None = None) -> Starlette:
|
||||
api_action_attempt,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route("/requests", requests_page, methods=["GET", "POST"]),
|
||||
Route(
|
||||
"/api/v1/requests/preview",
|
||||
api_v1_request_preview,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/api/v1/requests/apply",
|
||||
api_v1_request_apply,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route("/api/leases", api_leases, methods=["GET"]),
|
||||
Route("/api/v1/inventory", api_inventory, methods=["GET"]),
|
||||
Route(
|
||||
|
||||
+8
-71
@@ -115,12 +115,6 @@ class ConsoleAction:
|
||||
break_glass: bool
|
||||
phase: int
|
||||
summary: str
|
||||
# Opt-in switch for an action whose execution path is genuinely wired
|
||||
# ahead of its phase becoming globally active (#643). Naming a variable
|
||||
# here enables nothing on its own: the variable must also be set in the
|
||||
# environment. An action that leaves this ``None`` can only execute once
|
||||
# ACTIVE_PHASE reaches its phase, exactly as before.
|
||||
execution_env_flag: str | None = None
|
||||
|
||||
@property
|
||||
def mcp_permission(self) -> str:
|
||||
@@ -283,27 +277,6 @@ _ACTION_SPECS: tuple[ConsoleAction, ...] = (
|
||||
phase=2,
|
||||
summary="Restart one MCP namespace via the host supervisor.",
|
||||
),
|
||||
# #643: submit a work request — desired role, issue/PR, intent — and let
|
||||
# the allocator reserve it. This is the one Phase 2 action whose execution
|
||||
# path is actually implemented (``webui.request_service``), so it carries
|
||||
# the opt-in flag; it stays denied until an operator sets that variable.
|
||||
# Authority is operator-class because the outcome is a claim, not a Gitea
|
||||
# verdict: initiating reviewer or merger *work* does not grant the right
|
||||
# to approve or merge, which stays with the MCP role profile.
|
||||
ConsoleAction(
|
||||
action_id="initiate_workflow",
|
||||
task_key="allocate_next_work",
|
||||
action_class=CLASS_WRITE,
|
||||
minimum_role=OPERATOR,
|
||||
requires_confirmation=True,
|
||||
dual_control=False,
|
||||
break_glass=False,
|
||||
phase=2,
|
||||
summary=(
|
||||
"Preview and initiate allocator-owned workflow work for a role."
|
||||
),
|
||||
execution_env_flag="WEBUI_REQUESTS_EXECUTION",
|
||||
),
|
||||
)
|
||||
|
||||
ACTIONS: dict[str, ConsoleAction] = {a.action_id: a for a in _ACTION_SPECS}
|
||||
@@ -457,33 +430,6 @@ ALLOW_PREVIEW = "allowed_preview_only"
|
||||
# gated on this model landing; nothing here enables it.
|
||||
ACTIVE_PHASE = 1
|
||||
|
||||
_TRUTHY = frozenset({"1", "true", "yes", "on"})
|
||||
|
||||
|
||||
def execution_wired(
|
||||
action: ConsoleAction | None, env: dict[str, str] | None = None
|
||||
) -> bool:
|
||||
"""Whether *action* has a live execution path right now.
|
||||
|
||||
Two ways to be wired, and only two. The action's phase is active, or the
|
||||
action declares an opt-in environment variable *and* that variable is set.
|
||||
Everything else — including every action that never declares a flag — is
|
||||
unwired, so the default across the registry stays deny.
|
||||
|
||||
Bumping ``ACTIVE_PHASE`` would enable execution for every action of that
|
||||
phase at once. The per-action flag exists so a single implemented action
|
||||
can go live without dragging its unimplemented phase-mates with it.
|
||||
"""
|
||||
if action is None:
|
||||
return False
|
||||
if action.phase <= ACTIVE_PHASE:
|
||||
return True
|
||||
flag = (action.execution_env_flag or "").strip()
|
||||
if not flag:
|
||||
return False
|
||||
source = env if env is not None else os.environ
|
||||
return (source.get(flag) or "").strip().lower() in _TRUTHY
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AuthorizationDecision:
|
||||
@@ -523,19 +469,16 @@ def authorize(
|
||||
principal: Principal | None = None,
|
||||
*,
|
||||
for_execution: bool = False,
|
||||
env: dict[str, str] | None = None,
|
||||
) -> AuthorizationDecision:
|
||||
"""Decide whether *principal* may invoke *action_id*. Deny by default.
|
||||
|
||||
``for_execution`` distinguishes a read-only preview from a real invocation.
|
||||
``execution_enabled`` reports whether the action has a live execution path
|
||||
at all (:func:`execution_wired`) — for every action without an explicit
|
||||
opt-in flag that stays ``False`` while the console is in Phase 1, so no
|
||||
caller can read an allow as permission to mutate.
|
||||
Even an allowed decision reports ``execution_enabled=False`` while the
|
||||
console is in Phase 1, so no caller can read an allow as permission to
|
||||
mutate.
|
||||
"""
|
||||
who = principal if principal is not None else ANONYMOUS
|
||||
action = get_action(action_id)
|
||||
wired = execution_wired(action, env)
|
||||
|
||||
if action is None:
|
||||
return AuthorizationDecision(
|
||||
@@ -554,7 +497,7 @@ def authorize(
|
||||
"requires_confirmation": action.requires_confirmation,
|
||||
"dual_control": action.dual_control,
|
||||
"break_glass": action.break_glass,
|
||||
"execution_enabled": wired,
|
||||
"execution_enabled": False,
|
||||
}
|
||||
|
||||
if not who.authenticated:
|
||||
@@ -587,19 +530,13 @@ def authorize(
|
||||
**base,
|
||||
)
|
||||
|
||||
if for_execution and not wired:
|
||||
if for_execution and action.phase > ACTIVE_PHASE:
|
||||
return AuthorizationDecision(
|
||||
allowed=False,
|
||||
reason_code=DENY_PHASE_NOT_ACTIVE,
|
||||
detail=(
|
||||
f"Action {action_id!r} belongs to phase {action.phase}; the "
|
||||
f"console is in phase {ACTIVE_PHASE}"
|
||||
+ (
|
||||
f" and {action.execution_env_flag} is not set"
|
||||
if action.execution_env_flag
|
||||
else ""
|
||||
)
|
||||
+ ". Execution is not wired."
|
||||
f"console is in phase {ACTIVE_PHASE}. Execution is not wired."
|
||||
),
|
||||
**base,
|
||||
)
|
||||
@@ -608,8 +545,8 @@ def authorize(
|
||||
allowed=True,
|
||||
reason_code=ALLOW_PREVIEW,
|
||||
detail=(
|
||||
"Principal holds the required role. Execution proceeds only for an "
|
||||
"action with a wired execution path; everything else is preview."
|
||||
"Principal holds the required role. Preview only — execution "
|
||||
"remains disabled until the Phase 2 action framework ships."
|
||||
),
|
||||
**base,
|
||||
)
|
||||
|
||||
@@ -45,7 +45,6 @@ NAV_GROUPS: tuple[NavGroup, ...] = (
|
||||
NavItem("/queue", "Queue"),
|
||||
NavItem("/leases", "Leases"),
|
||||
NavItem("/actions", "Actions"),
|
||||
NavItem("/requests", "Requests"),
|
||||
)),
|
||||
NavGroup("Runtime/Sessions", (
|
||||
NavItem("/runtime", "Runtime health"),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,164 +0,0 @@
|
||||
"""HTML views for the operator request surface (#643).
|
||||
|
||||
The form is deliberately a *preview* form. It has no initiate button, because
|
||||
initiating requires a confirmed POST to ``/api/v1/requests/apply`` and a stray
|
||||
form submission must not be able to produce one by accident.
|
||||
|
||||
Nothing rendered here is trusted input: every interpolated value is escaped,
|
||||
and the page renders only values the service already produced rather than
|
||||
echoing a raw request body back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from webui.layout import render_page
|
||||
from webui.request_service import (
|
||||
REQUESTABLE_ROLES,
|
||||
WORK_KINDS,
|
||||
RequestError,
|
||||
RequestPreview,
|
||||
)
|
||||
|
||||
REQUESTS_PATH = "/requests"
|
||||
PREVIEW_API_PATH = "/api/v1/requests/preview"
|
||||
APPLY_API_PATH = "/api/v1/requests/apply"
|
||||
|
||||
|
||||
def _escape(text: Any) -> str:
|
||||
return html.escape(str(text if text is not None else ""), quote=True)
|
||||
|
||||
|
||||
REQUEST_PAGE_STYLES = """
|
||||
<style>
|
||||
.request-form { display: grid; gap: 0.75rem; max-width: 44rem; }
|
||||
.request-form label { display: grid; gap: 0.25rem; font-size: 0.9rem; }
|
||||
.request-check { margin: 0.35rem 0; }
|
||||
.request-check .verdict-ok { color: var(--accent); }
|
||||
.request-check .verdict-fail { color: #d14; }
|
||||
.request-prohibited code { margin-right: 0.4rem; }
|
||||
</style>
|
||||
"""
|
||||
|
||||
|
||||
def _options(values: tuple[str, ...], selected: Any) -> str:
|
||||
return "".join(
|
||||
f"<option value='{_escape(value)}'"
|
||||
+ (" selected" if selected == value else "")
|
||||
+ f">{_escape(value)}</option>"
|
||||
for value in values
|
||||
)
|
||||
|
||||
|
||||
def _form(values: dict[str, Any] | None = None) -> str:
|
||||
current = dict(values or {})
|
||||
number = current.get("work_number")
|
||||
return (
|
||||
f"<form class='request-form' method='post' action='{REQUESTS_PATH}'>"
|
||||
"<label>Desired role<select name='desired_role'>"
|
||||
f"{_options(REQUESTABLE_ROLES, current.get('desired_role'))}"
|
||||
"</select></label>"
|
||||
"<label>Work kind<select name='work_kind'>"
|
||||
f"{_options(WORK_KINDS, current.get('work_kind'))}"
|
||||
"</select></label>"
|
||||
"<label>Issue or PR number"
|
||||
"<input type='number' name='work_number' min='1' required "
|
||||
f"value='{_escape(number) if number else ''}'></label>"
|
||||
"<label>Intent summary"
|
||||
"<input type='text' name='intent_summary' maxlength='500' required "
|
||||
f"value='{_escape(current.get('intent_summary'))}'></label>"
|
||||
"<label>Expected head SHA <span class='muted'>(PR work only)</span>"
|
||||
"<input type='text' name='expected_head_sha' "
|
||||
f"value='{_escape(current.get('expected_head_sha'))}'></label>"
|
||||
"<button type='submit' class='copy-btn'>Preview request</button>"
|
||||
"<p class='muted meta'>Preview is read-only and creates no assignment. "
|
||||
f"Initiating requires a confirmed POST to <code>{APPLY_API_PATH}</code>."
|
||||
"</p>"
|
||||
"</form>"
|
||||
)
|
||||
|
||||
|
||||
def _checks_block(preview: RequestPreview) -> str:
|
||||
rows = []
|
||||
for check in preview.checks:
|
||||
verdict = "PASS" if check.ok else "FAIL"
|
||||
css = "verdict-ok" if check.ok else "verdict-fail"
|
||||
rows.append(
|
||||
"<li class='request-check'>"
|
||||
f"<span class='{css}'><strong>{verdict}</strong></span> "
|
||||
f"<code>{_escape(check.name)}</code> — {_escape(check.detail)} "
|
||||
f"<span class='muted meta'>({_escape(check.reason_code)})</span>"
|
||||
"</li>"
|
||||
)
|
||||
return "<ul>" + "".join(rows) + "</ul>"
|
||||
|
||||
|
||||
def _preview_block(preview: RequestPreview) -> str:
|
||||
verdict = "AUTHORIZED" if preview.authorized else "DENIED"
|
||||
prohibited = "".join(
|
||||
f"<code>{_escape(action)}</code>" for action in preview.prohibited_actions
|
||||
)
|
||||
request = preview.request
|
||||
evidence = json.dumps(preview.allocator_evidence, indent=2, default=str)
|
||||
return (
|
||||
"<h3>Intent preview</h3>"
|
||||
f"<p><strong>{verdict}</strong> — {_escape(preview.detail)}</p>"
|
||||
"<p class='meta'>"
|
||||
f"Role <code>{_escape(request.desired_role)}</code> · "
|
||||
f"{_escape(request.work_kind)} <code>{_escape(request.display_ref)}</code>"
|
||||
f" · profile <code>{_escape(preview.required_profile)}</code> · "
|
||||
f"namespace <code>{_escape(preview.required_namespace)}</code> · "
|
||||
f"permission <code>{_escape(preview.required_permission)}</code>"
|
||||
"</p>"
|
||||
f"<p>Intent: {_escape(request.intent_summary)}</p>"
|
||||
f"{_checks_block(preview)}"
|
||||
f"<p><strong>Next safe action:</strong> "
|
||||
f"{_escape(preview.next_safe_action)}</p>"
|
||||
"<p class='request-prohibited'><strong>Prohibited for this role:</strong> "
|
||||
+ (prohibited or "<span class='muted'>none declared</span>")
|
||||
+ "</p>"
|
||||
"<p class='muted meta'>Correlation id "
|
||||
f"<code>{_escape(preview.correlation_id)}</code></p>"
|
||||
"<details><summary>Allocator evidence</summary>"
|
||||
f"<pre class='prompt-text'>{_escape(evidence)}</pre>"
|
||||
"</details>"
|
||||
)
|
||||
|
||||
|
||||
def _error_block(error: RequestError) -> str:
|
||||
field = (
|
||||
f"<p class='meta'>Field: <code>{_escape(error.field_name)}</code></p>"
|
||||
if error.field_name
|
||||
else ""
|
||||
)
|
||||
return (
|
||||
"<h3>Request rejected</h3>"
|
||||
f"<p><strong>{_escape(error.reason_code)}</strong> — "
|
||||
f"{_escape(error.detail)}</p>{field}"
|
||||
)
|
||||
|
||||
|
||||
def render_requests_page(
|
||||
*,
|
||||
preview: RequestPreview | None = None,
|
||||
error: RequestError | None = None,
|
||||
submitted: dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
"""Render the request form, plus a preview or rejection when one exists."""
|
||||
body = (
|
||||
"<h2>Requests</h2>"
|
||||
"<p>Submit a work request — desired role, issue or PR, and intent — "
|
||||
"and see whether it would be authorized before anything is reserved. "
|
||||
"Initiation goes through the allocator (#600/#613); this console never "
|
||||
"self-selects work, never approves, and never merges.</p>"
|
||||
+ _form(submitted)
|
||||
+ (_error_block(error) if error is not None else "")
|
||||
+ (_preview_block(preview) if preview is not None else "")
|
||||
+ f"<p class='meta'><a href='{PREVIEW_API_PATH}'>Preview API</a> · "
|
||||
"<a href='/api/console/security-model'>RBAC model</a></p>"
|
||||
+ REQUEST_PAGE_STYLES
|
||||
)
|
||||
return render_page(title="Requests", body_html=body)
|
||||
@@ -201,16 +201,6 @@ def _candidates_from_queue_snapshot(q_snap: QueueSnapshot) -> list[WorkCandidate
|
||||
return candidates
|
||||
|
||||
|
||||
def candidates_from_queue_snapshot(q_snap: QueueSnapshot) -> list[WorkCandidate]:
|
||||
"""Public alias for :func:`_candidates_from_queue_snapshot` (#643).
|
||||
|
||||
The request-initiation service ranks the same candidate set this view
|
||||
renders, so both must agree on how a queue row becomes a candidate. One
|
||||
construction, two callers — not two that can drift apart.
|
||||
"""
|
||||
return _candidates_from_queue_snapshot(q_snap)
|
||||
|
||||
|
||||
def _claim_lease_records(inventory: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
"""Normalize ``build_claim_inventory`` entries into lease records.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user