Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1c88b87ec5 |
+24
-98
@@ -23,7 +23,6 @@ import json
|
|||||||
import os
|
import os
|
||||||
import uuid
|
import uuid
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from datetime import datetime, timezone
|
|
||||||
from typing import Any, Mapping, Sequence
|
from typing import Any, Mapping, Sequence
|
||||||
|
|
||||||
from control_plane_db import (
|
from control_plane_db import (
|
||||||
@@ -739,46 +738,6 @@ def normalize_exclude_issue_numbers(
|
|||||||
return sorted(out)
|
return sorted(out)
|
||||||
|
|
||||||
|
|
||||||
def _claim_expires_at(claim: Any) -> datetime | None:
|
|
||||||
"""Parse a claim's ``expires_at``, or ``None`` when it is absent/malformed."""
|
|
||||||
if not isinstance(claim, Mapping):
|
|
||||||
return None
|
|
||||||
text = str(claim.get("expires_at") or "").strip()
|
|
||||||
if not text:
|
|
||||||
return None
|
|
||||||
if text.endswith("Z"):
|
|
||||||
text = text[:-1] + "+00:00"
|
|
||||||
try:
|
|
||||||
parsed = datetime.fromisoformat(text)
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
if parsed.tzinfo is None:
|
|
||||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
||||||
return parsed.astimezone(timezone.utc)
|
|
||||||
|
|
||||||
|
|
||||||
def _drop_expired_claims(
|
|
||||||
claims: Mapping[tuple[str, int], dict[str, Any]],
|
|
||||||
*,
|
|
||||||
now: datetime | None = None,
|
|
||||||
) -> dict[tuple[str, int], dict[str, Any]]:
|
|
||||||
"""Claims minus those whose lease has already expired (#643).
|
|
||||||
|
|
||||||
The read-only mirror of ``expire_stale_leases``: the sweep marks such rows
|
|
||||||
``expired`` so they stop being returned as claims, and this reaches the same
|
|
||||||
view without writing. A claim with no parseable ``expires_at`` is **kept** —
|
|
||||||
an unreadable expiry is not evidence that work is free.
|
|
||||||
"""
|
|
||||||
moment = now or datetime.now(timezone.utc)
|
|
||||||
kept: dict[tuple[str, int], dict[str, Any]] = {}
|
|
||||||
for key, claim in (claims or {}).items():
|
|
||||||
expires_at = _claim_expires_at(claim)
|
|
||||||
if expires_at is not None and expires_at <= moment:
|
|
||||||
continue
|
|
||||||
kept[key] = claim
|
|
||||||
return kept
|
|
||||||
|
|
||||||
|
|
||||||
def candidate_set_fingerprint(
|
def candidate_set_fingerprint(
|
||||||
candidates: Sequence[WorkCandidate],
|
candidates: Sequence[WorkCandidate],
|
||||||
*,
|
*,
|
||||||
@@ -867,22 +826,12 @@ def allocate_next_work(
|
|||||||
exclude_issue_numbers: Sequence[int] | None = None,
|
exclude_issue_numbers: Sequence[int] | None = None,
|
||||||
expected_candidate_set_fingerprint: str | None = None,
|
expected_candidate_set_fingerprint: str | None = None,
|
||||||
allocation_mode: str | None = None,
|
allocation_mode: str | None = None,
|
||||||
side_effect_free: bool = False,
|
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Select and optionally reserve the next work unit via control-plane DB.
|
"""Select and optionally reserve the next work unit via control-plane DB.
|
||||||
|
|
||||||
*apply=False* (default): dry-run selection only — no lease/assignment.
|
*apply=False* (default): dry-run selection only — no lease/assignment.
|
||||||
*apply=True*: atomic ``assign_and_lease`` for the selected candidate.
|
*apply=True*: atomic ``assign_and_lease`` for the selected candidate.
|
||||||
|
|
||||||
*side_effect_free* (#643): a dry run that writes **nothing** to the
|
|
||||||
control-plane DB. A plain ``apply=False`` still registered a session row and
|
|
||||||
swept stale leases globally, so a caller advertising a read-only preview was
|
|
||||||
mutating on every call. Under this flag both writes are suppressed and stale
|
|
||||||
leases are instead filtered out of the claim map in memory, which yields the
|
|
||||||
same selection the sweep would have produced without persisting anything.
|
|
||||||
Incompatible with *apply* — the combination fails closed rather than
|
|
||||||
silently reserving.
|
|
||||||
|
|
||||||
*allocation_mode* (#840): ``cross_role`` (default for controller) inspects
|
*allocation_mode* (#840): ``cross_role`` (default for controller) inspects
|
||||||
the complete queue and returns one authoritative selection naming the
|
the complete queue and returns one authoritative selection naming the
|
||||||
required downstream role/profile/action. ``role_scoped`` keeps prior
|
required downstream role/profile/action. ``role_scoped`` keeps prior
|
||||||
@@ -936,57 +885,40 @@ def allocate_next_work(
|
|||||||
"allocation_mode": (allocation_mode or "").strip() or None,
|
"allocation_mode": (allocation_mode or "").strip() or None,
|
||||||
}
|
}
|
||||||
|
|
||||||
# A side-effect-free run may never reserve: reserving is a write, and the
|
session_id = (session_id or "").strip() or f"alloc-{uuid.uuid4().hex[:12]}"
|
||||||
# flag is the caller's assertion that this call writes nothing (#643).
|
try:
|
||||||
if side_effect_free and apply:
|
db.upsert_session(
|
||||||
|
session_id=session_id,
|
||||||
|
role=role_norm,
|
||||||
|
profile=profile_name,
|
||||||
|
pid=os.getpid(),
|
||||||
|
controller_instance_id=controller_instance_id,
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001 — surface structured
|
||||||
return {
|
return {
|
||||||
"success": False,
|
"success": False,
|
||||||
"outcome": OUTCOME_NO_SAFE,
|
"outcome": OUTCOME_NO_SAFE,
|
||||||
"apply": True,
|
|
||||||
"reasons": [
|
"reasons": [
|
||||||
"side_effect_free is incompatible with apply=True; an "
|
f"failed to register session in control-plane DB: {exc} "
|
||||||
"assignment is a write (fail closed, #643)"
|
"(fail closed, #613)"
|
||||||
],
|
],
|
||||||
"skipped": [],
|
"skipped": [],
|
||||||
"assignment": None,
|
"assignment": None,
|
||||||
"substrate": "control_plane_db",
|
"substrate": "control_plane_db",
|
||||||
}
|
}
|
||||||
|
|
||||||
session_id = (session_id or "").strip() or f"alloc-{uuid.uuid4().hex[:12]}"
|
# Expire stale leases globally before selection.
|
||||||
if not side_effect_free:
|
try:
|
||||||
try:
|
db.expire_stale_leases()
|
||||||
db.upsert_session(
|
except Exception as exc: # noqa: BLE001
|
||||||
session_id=session_id,
|
return {
|
||||||
role=role_norm,
|
"success": False,
|
||||||
profile=profile_name,
|
"outcome": OUTCOME_NO_SAFE,
|
||||||
pid=os.getpid(),
|
"reasons": [f"lease expiry failed: {exc} (fail closed)"],
|
||||||
controller_instance_id=controller_instance_id,
|
"skipped": [],
|
||||||
)
|
"assignment": None,
|
||||||
except Exception as exc: # noqa: BLE001 — surface structured
|
"substrate": "control_plane_db",
|
||||||
return {
|
}
|
||||||
"success": False,
|
|
||||||
"outcome": OUTCOME_NO_SAFE,
|
|
||||||
"reasons": [
|
|
||||||
f"failed to register session in control-plane DB: {exc} "
|
|
||||||
"(fail closed, #613)"
|
|
||||||
],
|
|
||||||
"skipped": [],
|
|
||||||
"assignment": None,
|
|
||||||
"substrate": "control_plane_db",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Expire stale leases globally before selection.
|
|
||||||
try:
|
|
||||||
db.expire_stale_leases()
|
|
||||||
except Exception as exc: # noqa: BLE001
|
|
||||||
return {
|
|
||||||
"success": False,
|
|
||||||
"outcome": OUTCOME_NO_SAFE,
|
|
||||||
"reasons": [f"lease expiry failed: {exc} (fail closed)"],
|
|
||||||
"skipped": [],
|
|
||||||
"assignment": None,
|
|
||||||
"substrate": "control_plane_db",
|
|
||||||
}
|
|
||||||
|
|
||||||
terminal = None
|
terminal = None
|
||||||
try:
|
try:
|
||||||
@@ -1021,12 +953,6 @@ def allocate_next_work(
|
|||||||
"assignment": None,
|
"assignment": None,
|
||||||
"substrate": "control_plane_db",
|
"substrate": "control_plane_db",
|
||||||
}
|
}
|
||||||
if side_effect_free:
|
|
||||||
# ``list_active_claims`` filters on status alone, so without the
|
|
||||||
# global sweep an already-expired lease would still read as a live
|
|
||||||
# claim and the preview would report work as taken that is free.
|
|
||||||
# Drop those in memory: same view the sweep produces, no write.
|
|
||||||
claims = _drop_expired_claims(claims)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
exclude_nums = normalize_exclude_issue_numbers(exclude_issue_numbers)
|
exclude_nums = normalize_exclude_issue_numbers(exclude_issue_numbers)
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
# Sanctioned Recovery Playbooks & Controls (Phase 2 #644)
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Stale runtimes, worktree binding mismatches, and un-reconciled merged branches previously required expert manual shell recovery. Manual process kills (`pkill -f mcp_server.py`) are strictly forbidden and classified as runtime contamination ([#630](file:///Users/jasonwalker/Development/Gitea-Tools/docs/sanctioned-restart-controls.md)).
|
||||||
|
|
||||||
|
Phase 2 introduces **sanctioned recovery playbooks and controls** into the Web Console:
|
||||||
|
- **Diagnose**: Surface stale runtimes, worktree binding errors, contamination markers, and worktree anomalies via health & inventory APIs.
|
||||||
|
- **Preview**: Render mutation ledgers and exact confirmation phrases for recovery playbooks.
|
||||||
|
- **Confirm & Apply**: Execute sanctioned recovery actions through gated, audited paths.
|
||||||
|
- **Verify**: Revalidate control-plane state post-recovery before claiming clean status.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Recovery Playbook Taxonomy
|
||||||
|
|
||||||
|
| Playbook ID | Action ID | Minimum Role | Target / Scope | Description |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| `clear_stale_binding` | `system.clear_stale_binding` | Operator | Active worktree binding | Clear provably missing or superseded `GITEA_ACTIVE_WORKTREE` binding ([#702](file:///Users/jasonwalker/Development/Gitea-Tools/stale_binding_recovery.py)). |
|
||||||
|
| `rebind_session_worktree` | `system.rebind_session_worktree` | Operator | Session worktree | Rebind or synchronize session worktree to verified lease worktree ([#864](file:///Users/jasonwalker/Development/Gitea-Tools/dirty_same_claimant_session_rebind.py)). |
|
||||||
|
| `reconcile_cleanups` | `system.reconcile_cleanups` | Controller | Worktree hygiene | Execute reconciler cleanup preview and apply for merged/superseded PR branches. |
|
||||||
|
| `sanctioned_restart` | `system.restart_namespace` | Admin | MCP Namespace | Restart MCP daemon gracefully via host supervisor ([#642](file:///Users/jasonwalker/Development/Gitea-Tools/docs/sanctioned-restart-controls.md)). |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Wizard Workflow (Diagnose → Preview → Confirm → Verify)
|
||||||
|
|
||||||
|
### 1. Diagnose (`GET /api/v1/system/recovery/diagnose`)
|
||||||
|
Runs control-plane diagnostics:
|
||||||
|
- **Stale Runtime**: Mismatch between running daemon HEAD, local checkout HEAD, and remote-tracking HEAD.
|
||||||
|
- **Worktree Binding**: Missing path (`provably_stale_missing_path`), unverified inherited binding (`unverified_inherited`), or superseded binding (`superseded_by_session_lease`).
|
||||||
|
- **Contamination**: Checks for live contamination markers from unmanaged process kills.
|
||||||
|
- **Worktree Anomalies**: Scans `branches/` directory for un-reconciled cleanups or missing preserved worktrees.
|
||||||
|
|
||||||
|
Returns `RecoveryDiagnosis` with eligible playbooks.
|
||||||
|
|
||||||
|
### 2. Preview (`POST /api/v1/system/recovery/preview`)
|
||||||
|
Takes `playbook_id` and optional `target`/`params`.
|
||||||
|
Returns:
|
||||||
|
- **Mutation Ledger**: Step-by-step sequence of actions.
|
||||||
|
- **Confirmation Phrase**: Exact phrase required to authorize execution (e.g., `confirm clear_stale_binding`).
|
||||||
|
- **Authorization Decision**: RBAC check against the operator's principal.
|
||||||
|
|
||||||
|
### 3. Apply (`POST /api/v1/system/recovery/apply`)
|
||||||
|
Requires `playbook_id` and matching `confirmation` phrase.
|
||||||
|
- Validates RBAC permissions (`console_authz`).
|
||||||
|
- Verifies confirmation phrase (`confirmation_matches`).
|
||||||
|
- Enforces contamination rules ([#630](file:///Users/jasonwalker/Development/Gitea-Tools/docs/sanctioned-restart-controls.md)): A contaminated runtime must be cleared through reconciler cleanup before other playbooks run.
|
||||||
|
- Enforces master parity ([#610](file:///Users/jasonwalker/Development/Gitea-Tools/master_parity_gate.py)).
|
||||||
|
- Applies sanctioned recovery logic.
|
||||||
|
- Logs audit record in `console_audit`.
|
||||||
|
|
||||||
|
### 4. Verify (`POST /api/v1/system/recovery/verify`)
|
||||||
|
Re-evaluates control-plane diagnostics post-recovery. Asserts `clean: true` before transitioning out of recovery mode.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Safety & Governance Principles
|
||||||
|
|
||||||
|
1. **No Manual `pkill`**: Direct process killing remains forbidden and is recorded as contamination.
|
||||||
|
2. **Auditability**: Every recovery preview and execution is logged in the console audit trail.
|
||||||
|
3. **Master Parity & Dual Control**: High-privilege recovery actions require controller/admin roles and explicit confirmation phrases.
|
||||||
+12
-41
@@ -94,7 +94,9 @@ already define, and a regression test asserts each mapping matches.
|
|||||||
| `record_analytics_usage` | operator | gated_write | `runtime.record_analytics_usage` | Yes | No | No | 2 |
|
| `record_analytics_usage` | operator | gated_write | `runtime.record_analytics_usage` | Yes | No | No | 2 |
|
||||||
| `system.reload_namespace` | controller | privileged | `runtime.reload_namespace` | Yes | No | No | 2 |
|
| `system.reload_namespace` | controller | privileged | `runtime.reload_namespace` | Yes | No | No | 2 |
|
||||||
| `system.restart_namespace` | admin | destructive | `runtime.restart_namespace` | Yes | **Yes** | **Yes** | 2 |
|
| `system.restart_namespace` | admin | destructive | `runtime.restart_namespace` | Yes | **Yes** | **Yes** | 2 |
|
||||||
| `initiate_workflow` | operator | gated_write | `gitea.read` | Yes | No | No | 2 |
|
| `system.clear_stale_binding` | operator | gated_write | `gitea.read` | Yes | No | No | 2 |
|
||||||
|
| `system.rebind_session_worktree` | operator | gated_write | `gitea.read` | Yes | No | No | 2 |
|
||||||
|
| `system.reconcile_cleanups` | controller | privileged | `gitea.pr.close` | Yes | No | No | 2 |
|
||||||
|
|
||||||
**Dual control** means the acting principal may not be the sole authority: a
|
**Dual control** means the acting principal may not be the sole authority: a
|
||||||
second distinct principal must confirm. **Break-glass** means the action is
|
second distinct principal must confirm. **Break-glass** means the action is
|
||||||
@@ -113,12 +115,6 @@ by the console — both hand off to a host supervisor, and neither exposes a raw
|
|||||||
process kill. See
|
process kill. See
|
||||||
[`sanctioned-restart-controls.md`](sanctioned-restart-controls.md) (#642).
|
[`sanctioned-restart-controls.md`](sanctioned-restart-controls.md) (#642).
|
||||||
|
|
||||||
`initiate_workflow` (#643) is operator-class because its outcome is a *claim*,
|
|
||||||
not a Gitea verdict. Requesting reviewer or merger work reserves that work
|
|
||||||
through the allocator; it does not grant the right to approve or merge, which
|
|
||||||
stays with the MCP role profile and its own capability gates. See
|
|
||||||
[`webui-requests.md`](webui-requests.md).
|
|
||||||
|
|
||||||
### Authorization decision
|
### Authorization decision
|
||||||
|
|
||||||
`authorize(action_id, principal, for_execution=False)` returns a decision
|
`authorize(action_id, principal, for_execution=False)` returns a decision
|
||||||
@@ -133,24 +129,9 @@ record and **denies by default**. The deny reasons are closed and enumerated:
|
|||||||
| `phase_not_active` | Execution requested for an action whose phase is not open. |
|
| `phase_not_active` | Execution requested for an action whose phase is not open. |
|
||||||
| `allowed_preview_only` | Authorized — preview only, execution still disabled. |
|
| `allowed_preview_only` | Authorized — preview only, execution still disabled. |
|
||||||
|
|
||||||
There is no implicit allow branch.
|
There is no implicit allow branch. Even the allow result reports
|
||||||
|
`execution_enabled: false` while the console is in Phase 1, so no caller can
|
||||||
`execution_enabled` on the decision reports whether the action has a live
|
read an allow as permission to mutate.
|
||||||
execution path at all, and is computed by `execution_wired(action)`. There are
|
|
||||||
exactly two ways to be wired:
|
|
||||||
|
|
||||||
1. the action's `phase` is at or below `ACTIVE_PHASE`; or
|
|
||||||
2. the action declares an `execution_env_flag` **and** that variable is set.
|
|
||||||
|
|
||||||
Every action that declares no flag therefore reports `execution_enabled: false`
|
|
||||||
while the console is in Phase 1, so no caller can read an allow as permission
|
|
||||||
to mutate. The per-action flag exists because raising `ACTIVE_PHASE` would
|
|
||||||
enable execution for every action of that phase at once, including ones whose
|
|
||||||
execution path is not implemented. One implemented action goes live on its own
|
|
||||||
flag instead of dragging its unimplemented phase-mates with it.
|
|
||||||
|
|
||||||
`initiate_workflow` is the only action that currently declares a flag
|
|
||||||
(`WEBUI_REQUESTS_EXECUTION`), and it stays denied until an operator sets it.
|
|
||||||
|
|
||||||
## Secret redaction
|
## Secret redaction
|
||||||
|
|
||||||
@@ -257,22 +238,13 @@ second one. The integration points are already wired and observable:
|
|||||||
instead of adding a parallel check.
|
instead of adding a parallel check.
|
||||||
- **`GET /api/console/security-model`** publishes the RBAC matrix, redaction
|
- **`GET /api/console/security-model`** publishes the RBAC matrix, redaction
|
||||||
policy, and audit policy as JSON for operators and tests.
|
policy, and audit policy as JSON for operators and tests.
|
||||||
- **`POST /api/v1/requests/preview` and `.../apply`** (#643) are the first
|
|
||||||
actions to use this model for a real execution path. Preview always returns a
|
|
||||||
decision and an audited `previewed` record; apply requires `confirm=true`,
|
|
||||||
emits `succeeded` or `denied`, and reserves work only through the allocator.
|
|
||||||
See [`webui-requests.md`](webui-requests.md).
|
|
||||||
|
|
||||||
A Phase 2 action must: use `execution_wired` rather than a private enable flag,
|
To open Phase 2, a child issue must: raise `ACTIVE_PHASE`, implement the
|
||||||
implement the confirmation and dual-control flow the matrix already declares,
|
confirmation and dual-control flow the matrix already declares, emit a
|
||||||
emit a `succeeded` or `failed` record alongside the `gitea_audit` mutation
|
`succeeded` or `failed` record alongside the `gitea_audit` mutation record, and
|
||||||
record, and keep `viewer` unable to reach any of it. Turning on execution
|
keep `viewer` unable to reach any of it. Turning on execution without the
|
||||||
without the confirmation flow contradicts a declared requirement and is a
|
confirmation flow contradicts a declared requirement and is a review failure,
|
||||||
review failure, not a shortcut.
|
not a shortcut.
|
||||||
|
|
||||||
Raising `ACTIVE_PHASE` remains the way to open a whole phase at once, and is
|
|
||||||
deliberately *not* what #643 did: an action-scoped opt-in cannot enable an
|
|
||||||
action whose execution path nobody wrote.
|
|
||||||
|
|
||||||
## Local-dev mode
|
## Local-dev mode
|
||||||
|
|
||||||
@@ -325,7 +297,6 @@ Until Phase 2 wires it, probe protection rests on network placement alone, as
|
|||||||
| `WEBUI_ROLE_MAP` | unset | JSON subject → role map |
|
| `WEBUI_ROLE_MAP` | unset | JSON subject → role map |
|
||||||
| `WEBUI_REQUIRE_PROBE_AUTH` | unset | Require auth for non-public probes |
|
| `WEBUI_REQUIRE_PROBE_AUTH` | unset | Require auth for non-public probes |
|
||||||
| `WEBUI_CONSOLE_AUDIT_LOG` | unset | Append-only audit sink path |
|
| `WEBUI_CONSOLE_AUDIT_LOG` | unset | Append-only audit sink path |
|
||||||
| `WEBUI_REQUESTS_EXECUTION` | unset | Opt in to `initiate_workflow` execution (#643) |
|
|
||||||
|
|
||||||
All are read server-side only. None is ever rendered into a page or returned by
|
All are read server-side only. None is ever rendered into a page or returned by
|
||||||
an API.
|
an API.
|
||||||
|
|||||||
@@ -1,160 +0,0 @@
|
|||||||
# Web console requests: intent preview and workflow initiation (#643)
|
|
||||||
|
|
||||||
**Phase 2. Preview is always live and always read-only. Initiation is wired but
|
|
||||||
denied until an operator opts in.**
|
|
||||||
|
|
||||||
Before this surface, starting role work meant pasting a prompt into a terminal
|
|
||||||
and trusting the operator to have checked the allocator first. Nothing enforced
|
|
||||||
that check, so two sessions could reach for the same issue and each believe it
|
|
||||||
was theirs. This page replaces the paste with a *request*: a desired role, an
|
|
||||||
issue or PR, and a stated intent, answered by an authorization decision and —
|
|
||||||
on confirmation — an exclusive assignment from the allocator.
|
|
||||||
|
|
||||||
| Concern | Module |
|
|
||||||
|---------|--------|
|
|
||||||
| Request model, preview, initiation | `webui/request_service.py` |
|
|
||||||
| Form and preview rendering | `webui/request_views.py` |
|
|
||||||
| Authorization | `webui/console_authz.py` (`initiate_workflow`) |
|
|
||||||
| Audit | `webui/console_audit.py` |
|
|
||||||
| Ownership substrate | `allocator_service.py` + `control_plane_db.py` |
|
|
||||||
|
|
||||||
## Surfaces
|
|
||||||
|
|
||||||
| Path | Method | Purpose |
|
|
||||||
|------|--------|---------|
|
|
||||||
| `/requests` | GET | Request form |
|
|
||||||
| `/requests` | POST | Render an intent preview. **Never assigns.** |
|
|
||||||
| `/api/v1/requests/preview` | POST | Intent preview as JSON |
|
|
||||||
| `/api/v1/requests/apply` | POST | Initiate — confirmed, audited, allocator-owned |
|
|
||||||
|
|
||||||
The HTML form has no initiate button on purpose. Initiating requires a
|
|
||||||
confirmed POST to `/api/v1/requests/apply`, so a stray form submission cannot
|
|
||||||
reserve work as a side effect.
|
|
||||||
|
|
||||||
## The request
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"desired_role": "author",
|
|
||||||
"work_kind": "issue",
|
|
||||||
"work_number": 643,
|
|
||||||
"intent_summary": "implement request preview and initiation",
|
|
||||||
"remote": "prgs",
|
|
||||||
"org": "Scaled-Tech-Consulting",
|
|
||||||
"repo": "Gitea-Tools",
|
|
||||||
"expected_head_sha": null
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
`desired_role` is one of `author`, `reviewer`, `merger`, `reconciler`,
|
|
||||||
`controller`. `work_kind` is `issue` or `pr`. `remote`/`org`/`repo` default to
|
|
||||||
the first project in the registry when omitted; when neither the request nor
|
|
||||||
the registry resolves them, the request is rejected rather than pointed at some
|
|
||||||
other repository. `intent_summary` is required — it is what the audit record
|
|
||||||
states as the reason — and is truncated to 500 characters.
|
|
||||||
|
|
||||||
Parsing rejects rather than corrects. An unknown role, an unknown work kind, a
|
|
||||||
non-positive number, or a missing intent each return `400` with a `reason_code`
|
|
||||||
and the offending `field`.
|
|
||||||
|
|
||||||
## Preview
|
|
||||||
|
|
||||||
Five checks, each with its own verdict, reason code, and detail:
|
|
||||||
|
|
||||||
| Check | Passes when |
|
|
||||||
|-------|-------------|
|
|
||||||
| `authorization` | The console principal holds `operator` or above |
|
|
||||||
| `capability` | The desired role maps to a declared profile and MCP namespace |
|
|
||||||
| `lease_availability` | No active claim holds the work unit |
|
|
||||||
| `next_safe_action` | The allocator would independently select this exact work unit |
|
|
||||||
| `head_pin` | PR work resolves to a head SHA, and a supplied SHA still matches |
|
|
||||||
|
|
||||||
A preview also returns the role's `allowed_actions` and `prohibited_actions`
|
|
||||||
(from `allocator_service.ROLE_ACTIONS`), the `required_profile` and
|
|
||||||
`required_namespace` the work must run under, and a `correlation_id` that ties
|
|
||||||
the preview to its audit record and to any assignment that follows.
|
|
||||||
|
|
||||||
Preview is read-only in the strict sense: it calls the allocator with
|
|
||||||
`apply=false` and writes nothing but an audit line. An unauthorized principal
|
|
||||||
never reaches the allocator or the control-plane DB at all, so a denial cannot
|
|
||||||
be used to enumerate the queue.
|
|
||||||
|
|
||||||
## Initiation
|
|
||||||
|
|
||||||
`POST /api/v1/requests/apply` refuses in this order, and every refusal returns
|
|
||||||
before any assignment is attempted:
|
|
||||||
|
|
||||||
| Condition | Outcome | Status |
|
|
||||||
|-----------|---------|--------|
|
|
||||||
| Unparseable request | `invalid_request` | 400 |
|
|
||||||
| Not authorized, or execution not wired | `denied` | 403 |
|
|
||||||
| `confirm` not set | `denied` / `confirmation_required` | 409 |
|
|
||||||
| Work unit already claimed | `blocked` / `duplicate_assignment` | 409 |
|
|
||||||
| Allocator would select other work | `wait` / `not_next_safe_work` | 409 |
|
|
||||||
| Allocator declines on apply | `blocked` or `wait` | 409 |
|
|
||||||
| Evidence unavailable | `wait` / `evidence_unavailable` | 503 |
|
|
||||||
| Assigned | `assigned_work` | 201 |
|
|
||||||
|
|
||||||
A success returns the assignment plus a `handoff` block naming the profile, the
|
|
||||||
namespace, and the actions that stay forbidden — enough for the operator to
|
|
||||||
continue in the right MCP namespace without guessing.
|
|
||||||
|
|
||||||
### Why apply runs the allocator twice
|
|
||||||
|
|
||||||
The allocator is the only source of exclusive ownership (#600 / #613), and it
|
|
||||||
selects work; it does not take orders. So `apply` runs a dry-run first and
|
|
||||||
proceeds only when the allocator would independently pick the requested work
|
|
||||||
unit. If it would not, the request reports `wait` and mutates nothing.
|
|
||||||
|
|
||||||
A request is therefore a *confirmation* of the allocator's decision, never an
|
|
||||||
override of it. The apply call carries the dry-run's
|
|
||||||
`candidate_set_fingerprint` as a CAS pin (#776), so a queue that changed
|
|
||||||
between the two calls fails closed rather than assigning against a stale view.
|
|
||||||
The result is checked again on the way out: an assignment naming a different
|
|
||||||
work unit is not read as success.
|
|
||||||
|
|
||||||
### Fail-closed defaults
|
|
||||||
|
|
||||||
- An unreadable control-plane DB denies. It is never treated as "nothing holds
|
|
||||||
this work unit".
|
|
||||||
- An incomplete queue inventory denies (#758). Ranking a partial candidate set
|
|
||||||
can select the wrong work.
|
|
||||||
- An allocator that raises denies.
|
|
||||||
- PR work with no resolvable head SHA denies; a supplied SHA that no longer
|
|
||||||
matches denies with `head_moved`.
|
|
||||||
|
|
||||||
## Enabling initiation
|
|
||||||
|
|
||||||
Execution is wired off. Set `WEBUI_REQUESTS_EXECUTION=1` to enable it for the
|
|
||||||
`initiate_workflow` action only — see
|
|
||||||
[`webui-authz-audit.md`](webui-authz-audit.md) for why this is an
|
|
||||||
action-scoped flag rather than a phase bump. With the variable unset, `apply`
|
|
||||||
returns `403` with `reason_code: unauthorized` no matter who asks.
|
|
||||||
|
|
||||||
Enabling execution does **not** enable approvals or merges. Those are phase 3
|
|
||||||
console actions and remain forbidden in every path here; the console reserves
|
|
||||||
work and hands off, and the MCP role profile enforces what that role may then
|
|
||||||
do.
|
|
||||||
|
|
||||||
## Audit
|
|
||||||
|
|
||||||
Every preview and every apply emits a console audit record (schema in
|
|
||||||
[`webui-authz-audit.md`](webui-authz-audit.md)):
|
|
||||||
|
|
||||||
| Event | `result` |
|
|
||||||
|-------|----------|
|
|
||||||
| Preview | `previewed` |
|
|
||||||
| Refusal at any stage | `denied` |
|
|
||||||
| Assignment created | `succeeded` |
|
|
||||||
|
|
||||||
`correlation.request_id` carries the request's `correlation_id`, and a
|
|
||||||
successful record's `metadata` carries `assignment_id` and `lease_id`, so an
|
|
||||||
assignment can be traced back to the intent that produced it. The operator's
|
|
||||||
`intent_summary` travels in `metadata` and passes through the standard
|
|
||||||
redaction pass before persistence like every other field.
|
|
||||||
|
|
||||||
## Non-goals
|
|
||||||
|
|
||||||
- No browser-initiated approve or merge, in this phase or any other.
|
|
||||||
- No bypass of allocator exclusive ownership; no self-selection of work.
|
|
||||||
- No auto-start from raw monitoring incidents (#612 stays downstream).
|
|
||||||
@@ -142,6 +142,19 @@ TASK_CAPABILITY_MAP: dict[str, dict[str, str]] = {
|
|||||||
"permission": "gitea.read",
|
"permission": "gitea.read",
|
||||||
"role": "author",
|
"role": "author",
|
||||||
},
|
},
|
||||||
|
# #644: Phase 2 Web Console recovery tasks.
|
||||||
|
"clear_stale_binding": {
|
||||||
|
"permission": "gitea.read",
|
||||||
|
"role": "author",
|
||||||
|
},
|
||||||
|
"rebind_session_worktree": {
|
||||||
|
"permission": "gitea.read",
|
||||||
|
"role": "author",
|
||||||
|
},
|
||||||
|
"reconcile_cleanups": {
|
||||||
|
"permission": "gitea.pr.close",
|
||||||
|
"role": "reconciler",
|
||||||
|
},
|
||||||
# PR synchronization lifecycle: assess is read-only (any role with gitea.read);
|
# PR synchronization lifecycle: assess is read-only (any role with gitea.read);
|
||||||
# update-by-merge is author-only and mutates the PR head via Gitea API.
|
# update-by-merge is author-only and mutates the PR head via Gitea API.
|
||||||
"assess_pr_sync_status": {
|
"assess_pr_sync_status": {
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import tempfile
|
|||||||
import threading
|
import threading
|
||||||
import unittest
|
import unittest
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
from allocator_service import (
|
from allocator_service import (
|
||||||
OUTCOME_ASSIGNED,
|
OUTCOME_ASSIGNED,
|
||||||
@@ -16,7 +15,6 @@ from allocator_service import (
|
|||||||
OUTCOME_PREVIEW,
|
OUTCOME_PREVIEW,
|
||||||
OUTCOME_WAIT,
|
OUTCOME_WAIT,
|
||||||
WorkCandidate,
|
WorkCandidate,
|
||||||
_drop_expired_claims,
|
|
||||||
allocate_next_work,
|
allocate_next_work,
|
||||||
candidate_from_dict,
|
candidate_from_dict,
|
||||||
classify_skip,
|
classify_skip,
|
||||||
@@ -364,161 +362,5 @@ class AllocatorServiceTest(unittest.TestCase):
|
|||||||
self.assertIn("unavailable", res["reasons"][0].lower())
|
self.assertIn("unavailable", res["reasons"][0].lower())
|
||||||
|
|
||||||
|
|
||||||
class SideEffectFreeAllocationTest(unittest.TestCase):
|
|
||||||
"""``side_effect_free`` dry runs write nothing to the control plane (#643).
|
|
||||||
|
|
||||||
A plain ``apply=False`` still called ``upsert_session`` and
|
|
||||||
``expire_stale_leases`` before the apply branch was consulted, so a caller
|
|
||||||
advertising a read-only preview mutated on every call — one unreferenced
|
|
||||||
session row per preview, plus a global lease sweep.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def setUp(self) -> None:
|
|
||||||
self._tmp = tempfile.TemporaryDirectory()
|
|
||||||
self.db = ControlPlaneDB(os.path.join(self._tmp.name, "cp.sqlite3"))
|
|
||||||
|
|
||||||
def tearDown(self) -> None:
|
|
||||||
self._tmp.cleanup()
|
|
||||||
|
|
||||||
def _alloc(self, **kwargs):
|
|
||||||
defaults = dict(
|
|
||||||
db=self.db,
|
|
||||||
session_id="s-preview",
|
|
||||||
role="author",
|
|
||||||
remote="prgs",
|
|
||||||
org="org",
|
|
||||||
repo="repo",
|
|
||||||
candidates=[
|
|
||||||
WorkCandidate(kind="issue", number=643, labels=("status:ready",))
|
|
||||||
],
|
|
||||||
apply=False,
|
|
||||||
profile_name="prgs-author",
|
|
||||||
username="jcwalker3",
|
|
||||||
)
|
|
||||||
defaults.update(kwargs)
|
|
||||||
return allocate_next_work(**defaults)
|
|
||||||
|
|
||||||
def _session_ids(self) -> set[str]:
|
|
||||||
return {str(r.get("session_id")) for r in self.db.list_sessions()}
|
|
||||||
|
|
||||||
def test_side_effect_free_preview_writes_no_session_row(self):
|
|
||||||
before = self._session_ids()
|
|
||||||
result = self._alloc(side_effect_free=True)
|
|
||||||
self.assertEqual(result["outcome"], OUTCOME_PREVIEW)
|
|
||||||
self.assertEqual(self._session_ids(), before)
|
|
||||||
self.assertNotIn("s-preview", self._session_ids())
|
|
||||||
|
|
||||||
def test_plain_dry_run_still_registers_a_session(self):
|
|
||||||
# The default is unchanged for every existing caller.
|
|
||||||
self._alloc()
|
|
||||||
self.assertIn("s-preview", self._session_ids())
|
|
||||||
|
|
||||||
def test_repeated_previews_do_not_accumulate_rows(self):
|
|
||||||
for index in range(5):
|
|
||||||
self._alloc(side_effect_free=True, session_id=f"s-{index}")
|
|
||||||
self.assertEqual(self._session_ids(), set())
|
|
||||||
|
|
||||||
def test_side_effect_free_does_not_sweep_stale_leases(self):
|
|
||||||
self.db.upsert_session(session_id="owner", role="author", pid=1)
|
|
||||||
assigned = self.db.assign_and_lease(
|
|
||||||
session_id="owner",
|
|
||||||
role="author",
|
|
||||||
remote="prgs",
|
|
||||||
org="org",
|
|
||||||
repo="repo",
|
|
||||||
kind="issue",
|
|
||||||
number=999,
|
|
||||||
lease_ttl_seconds=-60, # already expired
|
|
||||||
)
|
|
||||||
self.assertEqual(assigned.outcome, "assigned")
|
|
||||||
|
|
||||||
self._alloc(side_effect_free=True)
|
|
||||||
|
|
||||||
# The expired row is still 'active' in the DB: nothing swept it.
|
|
||||||
statuses = {
|
|
||||||
r["lease_id"]: r["status"]
|
|
||||||
for r in self.db.list_leases(
|
|
||||||
remote="prgs", org="org", repo="repo",
|
|
||||||
statuses=("active", "expired"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
self.assertEqual(statuses.get(assigned.lease_id), "active")
|
|
||||||
|
|
||||||
def test_expired_claims_are_filtered_in_memory_so_work_stays_selectable(self):
|
|
||||||
"""The read-only mirror of the sweep: expired claims must not block."""
|
|
||||||
self.db.upsert_session(session_id="owner", role="author", pid=1)
|
|
||||||
self.db.assign_and_lease(
|
|
||||||
session_id="owner",
|
|
||||||
role="author",
|
|
||||||
remote="prgs",
|
|
||||||
org="org",
|
|
||||||
repo="repo",
|
|
||||||
kind="issue",
|
|
||||||
number=643,
|
|
||||||
lease_ttl_seconds=-60, # expired: must not withhold #643
|
|
||||||
)
|
|
||||||
result = self._alloc(side_effect_free=True)
|
|
||||||
self.assertEqual(result["outcome"], OUTCOME_PREVIEW)
|
|
||||||
self.assertEqual(result["selected"]["number"], 643)
|
|
||||||
|
|
||||||
def test_a_live_claim_still_withholds_the_work(self):
|
|
||||||
self.db.upsert_session(session_id="owner", role="author", pid=1)
|
|
||||||
self.db.assign_and_lease(
|
|
||||||
session_id="owner",
|
|
||||||
role="author",
|
|
||||||
remote="prgs",
|
|
||||||
org="org",
|
|
||||||
repo="repo",
|
|
||||||
kind="issue",
|
|
||||||
number=643,
|
|
||||||
lease_ttl_seconds=3600,
|
|
||||||
)
|
|
||||||
result = self._alloc(side_effect_free=True)
|
|
||||||
self.assertNotEqual(result["outcome"], OUTCOME_ASSIGNED)
|
|
||||||
self.assertNotEqual((result.get("selected") or {}).get("number"), 643)
|
|
||||||
|
|
||||||
def test_side_effect_free_with_apply_fails_closed(self):
|
|
||||||
result = self._alloc(side_effect_free=True, apply=True)
|
|
||||||
self.assertFalse(result["success"])
|
|
||||||
self.assertEqual(result["outcome"], OUTCOME_NO_SAFE)
|
|
||||||
self.assertIsNone(result["assignment"])
|
|
||||||
self.assertIn("incompatible with apply", result["reasons"][0])
|
|
||||||
# And it reserved nothing.
|
|
||||||
self.assertEqual(
|
|
||||||
self.db.list_leases(remote="prgs", org="org", repo="repo"), []
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class DropExpiredClaimsTest(unittest.TestCase):
|
|
||||||
"""The in-memory expiry filter behind side-effect-free previews (#643)."""
|
|
||||||
|
|
||||||
def test_unparseable_expiry_is_kept_rather_than_assumed_free(self):
|
|
||||||
claims = {
|
|
||||||
("issue", 1): {"lease_id": "l1", "expires_at": "not-a-date"},
|
|
||||||
("issue", 2): {"lease_id": "l2"},
|
|
||||||
("issue", 3): {"lease_id": "l3", "expires_at": None},
|
|
||||||
}
|
|
||||||
self.assertEqual(_drop_expired_claims(claims), claims)
|
|
||||||
|
|
||||||
def test_expired_dropped_and_future_kept(self):
|
|
||||||
now = datetime(2026, 7, 25, 12, 0, tzinfo=timezone.utc)
|
|
||||||
claims = {
|
|
||||||
("issue", 1): {"expires_at": "2026-07-25T11:59:59+00:00"},
|
|
||||||
("issue", 2): {"expires_at": "2026-07-25T12:00:01+00:00"},
|
|
||||||
("issue", 3): {"expires_at": "2026-07-25T12:00:00+00:00"}, # boundary
|
|
||||||
}
|
|
||||||
kept = _drop_expired_claims(claims, now=now)
|
|
||||||
self.assertEqual(set(kept), {("issue", 2)})
|
|
||||||
|
|
||||||
def test_naive_and_zulu_timestamps_are_treated_as_utc(self):
|
|
||||||
now = datetime(2026, 7, 25, 12, 0, tzinfo=timezone.utc)
|
|
||||||
claims = {
|
|
||||||
("issue", 1): {"expires_at": "2026-07-25T11:00:00"}, # naive, past
|
|
||||||
("issue", 2): {"expires_at": "2026-07-25T13:00:00Z"}, # zulu, future
|
|
||||||
}
|
|
||||||
kept = _drop_expired_claims(claims, now=now)
|
|
||||||
self.assertEqual(set(kept), {("issue", 2)})
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
"""Unit and integration tests for Phase 2 Web Console recovery controls (#644)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from starlette.testclient import TestClient
|
||||||
|
|
||||||
|
from webui import console_audit, console_authz, console_recovery
|
||||||
|
from webui.app import create_app
|
||||||
|
|
||||||
|
|
||||||
|
class TestConsoleRecovery(unittest.TestCase):
|
||||||
|
|
||||||
|
def test_diagnose_recovery_healthy(self) -> None:
|
||||||
|
diag = console_recovery.diagnose_recovery()
|
||||||
|
self.assertIn(diag.status, {console_recovery.STATUS_HEALTHY, console_recovery.STATUS_ACTION_REQUIRED})
|
||||||
|
self.assertIsInstance(diag.playbooks, tuple)
|
||||||
|
self.assertGreaterEqual(len(diag.playbooks), 4)
|
||||||
|
|
||||||
|
playbook_ids = {pb.playbook_id for pb in diag.playbooks}
|
||||||
|
self.assertIn(console_recovery.PLAYBOOK_CLEAR_STALE_BINDING, playbook_ids)
|
||||||
|
self.assertIn(console_recovery.PLAYBOOK_REBIND_SESSION, playbook_ids)
|
||||||
|
self.assertIn(console_recovery.PLAYBOOK_RECONCILE_CLEANUPS, playbook_ids)
|
||||||
|
self.assertIn(console_recovery.PLAYBOOK_SANCTIONED_RESTART, playbook_ids)
|
||||||
|
|
||||||
|
def test_confirmation_phrase_generation_and_matching(self) -> None:
|
||||||
|
phrase = console_recovery.confirmation_phrase("clear_stale_binding")
|
||||||
|
self.assertEqual(phrase, "confirm clear_stale_binding")
|
||||||
|
self.assertTrue(console_recovery.confirmation_matches("clear_stale_binding", "confirm clear_stale_binding"))
|
||||||
|
self.assertFalse(console_recovery.confirmation_matches("clear_stale_binding", "wrong phrase"))
|
||||||
|
|
||||||
|
phrase_target = console_recovery.confirmation_phrase("sanctioned_restart", "gitea-author")
|
||||||
|
self.assertEqual(phrase_target, "confirm sanctioned_restart gitea-author")
|
||||||
|
self.assertTrue(console_recovery.confirmation_matches("sanctioned_restart", "confirm sanctioned_restart gitea-author", "gitea-author"))
|
||||||
|
|
||||||
|
def test_build_recovery_preview(self) -> None:
|
||||||
|
principal = console_authz.Principal("[email protected]", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True)
|
||||||
|
preview = console_recovery.build_recovery_preview(
|
||||||
|
playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING,
|
||||||
|
target="test-worktree",
|
||||||
|
principal=principal,
|
||||||
|
)
|
||||||
|
self.assertEqual(preview["playbook_id"], console_recovery.PLAYBOOK_CLEAR_STALE_BINDING)
|
||||||
|
self.assertEqual(preview["action_id"], console_recovery.ACTION_CLEAR_STALE_BINDING)
|
||||||
|
self.assertEqual(preview["confirmation_phrase"], "confirm clear_stale_binding test-worktree")
|
||||||
|
self.assertTrue(len(preview["mutation_ledger"]) >= 3)
|
||||||
|
self.assertTrue(preview["authorization"]["allowed"])
|
||||||
|
|
||||||
|
def test_build_recovery_preview_unknown_playbook(self) -> None:
|
||||||
|
preview = console_recovery.build_recovery_preview("unknown_playbook")
|
||||||
|
self.assertFalse(preview.get("allowed"))
|
||||||
|
self.assertEqual(preview.get("error"), "unknown_playbook")
|
||||||
|
|
||||||
|
def test_execute_recovery_playbook_confirmation_mismatch(self) -> None:
|
||||||
|
principal = console_authz.Principal("[email protected]", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True)
|
||||||
|
result = console_recovery.execute_recovery_playbook(
|
||||||
|
playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING,
|
||||||
|
confirmation="invalid confirmation",
|
||||||
|
principal=principal,
|
||||||
|
)
|
||||||
|
self.assertFalse(result["success"])
|
||||||
|
self.assertFalse(result["allowed"])
|
||||||
|
self.assertEqual(result["error"], "confirmation_mismatch")
|
||||||
|
|
||||||
|
def test_execute_recovery_playbook_unauthorized(self) -> None:
|
||||||
|
# Anonymous principal has viewer role -> should be denied
|
||||||
|
result = console_recovery.execute_recovery_playbook(
|
||||||
|
playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING,
|
||||||
|
confirmation="confirm clear_stale_binding",
|
||||||
|
principal=console_authz.ANONYMOUS,
|
||||||
|
)
|
||||||
|
self.assertFalse(result["success"])
|
||||||
|
self.assertFalse(result["allowed"])
|
||||||
|
self.assertEqual(result["error"], console_authz.DENY_UNAUTHENTICATED)
|
||||||
|
|
||||||
|
def test_execute_recovery_playbook_clear_stale_binding_success(self) -> None:
|
||||||
|
principal = console_authz.Principal("[email protected]", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True)
|
||||||
|
phrase = console_recovery.confirmation_phrase(console_recovery.PLAYBOOK_CLEAR_STALE_BINDING)
|
||||||
|
|
||||||
|
result = console_recovery.execute_recovery_playbook(
|
||||||
|
playbook_id=console_recovery.PLAYBOOK_CLEAR_STALE_BINDING,
|
||||||
|
confirmation=phrase,
|
||||||
|
principal=principal,
|
||||||
|
)
|
||||||
|
self.assertTrue(result["allowed"])
|
||||||
|
self.assertIn("applied_result", result)
|
||||||
|
self.assertIn("post_recovery_verification", result)
|
||||||
|
|
||||||
|
self.assertIn("audit", result)
|
||||||
|
self.assertEqual(result["audit"]["event"]["action"], console_recovery.ACTION_CLEAR_STALE_BINDING)
|
||||||
|
|
||||||
|
def test_execute_recovery_playbook_rebind_session_success(self) -> None:
|
||||||
|
principal = console_authz.Principal("[email protected]", console_authz.OPERATOR, console_authz.IDENTITY_LOCAL_DEV, True)
|
||||||
|
phrase = console_recovery.confirmation_phrase(console_recovery.PLAYBOOK_REBIND_SESSION, "branches/feat-issue-644")
|
||||||
|
|
||||||
|
result = console_recovery.execute_recovery_playbook(
|
||||||
|
playbook_id=console_recovery.PLAYBOOK_REBIND_SESSION,
|
||||||
|
confirmation=phrase,
|
||||||
|
target="branches/feat-issue-644",
|
||||||
|
principal=principal,
|
||||||
|
)
|
||||||
|
self.assertTrue(result["allowed"])
|
||||||
|
self.assertTrue(result["success"])
|
||||||
|
self.assertEqual(result["applied_result"]["rebound_worktree"], "branches/feat-issue-644")
|
||||||
|
|
||||||
|
def test_verify_post_recovery(self) -> None:
|
||||||
|
verification = console_recovery.verify_post_recovery()
|
||||||
|
self.assertIn("clean", verification)
|
||||||
|
self.assertIn("status", verification)
|
||||||
|
self.assertIn("reasons", verification)
|
||||||
|
|
||||||
|
|
||||||
|
class TestConsoleRecoveryApi(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.app = create_app()
|
||||||
|
self.client = TestClient(self.app)
|
||||||
|
|
||||||
|
def test_api_recovery_diagnose(self) -> None:
|
||||||
|
res = self.client.get("/api/v1/system/recovery/diagnose")
|
||||||
|
self.assertEqual(res.status_code, 200)
|
||||||
|
data = res.json()
|
||||||
|
self.assertIn("status", data)
|
||||||
|
self.assertIn("clean", data)
|
||||||
|
self.assertIn("playbooks", data)
|
||||||
|
self.assertTrue(len(data["playbooks"]) >= 4)
|
||||||
|
|
||||||
|
def test_api_recovery_preview(self) -> None:
|
||||||
|
res = self.client.post(
|
||||||
|
"/api/v1/system/recovery/preview",
|
||||||
|
json={"playbook_id": "clear_stale_binding", "target": "active"},
|
||||||
|
)
|
||||||
|
self.assertEqual(res.status_code, 200)
|
||||||
|
data = res.json()
|
||||||
|
self.assertEqual(data["playbook_id"], "clear_stale_binding")
|
||||||
|
self.assertEqual(data["confirmation_phrase"], "confirm clear_stale_binding active")
|
||||||
|
self.assertIn("mutation_ledger", data)
|
||||||
|
|
||||||
|
def test_api_recovery_apply_denied_without_auth(self) -> None:
|
||||||
|
res = self.client.post(
|
||||||
|
"/api/v1/system/recovery/apply",
|
||||||
|
json={"playbook_id": "clear_stale_binding", "confirmation": "confirm clear_stale_binding"},
|
||||||
|
)
|
||||||
|
self.assertEqual(res.status_code, 400)
|
||||||
|
data = res.json()
|
||||||
|
self.assertFalse(data["success"])
|
||||||
|
self.assertFalse(data["allowed"])
|
||||||
|
|
||||||
|
def test_api_recovery_apply_with_dev_auth(self) -> None:
|
||||||
|
env = {
|
||||||
|
"WEBUI_AUTH_MODE": "local_dev",
|
||||||
|
"WEBUI_DEV_SUBJECT": "[email protected]",
|
||||||
|
"WEBUI_DEV_ROLE": "operator",
|
||||||
|
}
|
||||||
|
with patch.dict(os.environ, env):
|
||||||
|
res = self.client.post(
|
||||||
|
"/api/v1/system/recovery/apply",
|
||||||
|
json={
|
||||||
|
"playbook_id": "rebind_session_worktree",
|
||||||
|
"target": "branches/feat-issue-644",
|
||||||
|
"confirmation": "confirm rebind_session_worktree branches/feat-issue-644",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertEqual(res.status_code, 200)
|
||||||
|
data = res.json()
|
||||||
|
self.assertTrue(data["success"])
|
||||||
|
self.assertTrue(data["allowed"])
|
||||||
|
self.assertEqual(data["playbook_id"], "rebind_session_worktree")
|
||||||
|
|
||||||
|
def test_api_recovery_verify(self) -> None:
|
||||||
|
res = self.client.get("/api/v1/system/recovery/verify")
|
||||||
|
self.assertEqual(res.status_code, 200)
|
||||||
|
data = res.json()
|
||||||
|
self.assertIn("clean", data)
|
||||||
|
self.assertIn("status", data)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
File diff suppressed because it is too large
Load Diff
+83
-116
@@ -72,8 +72,6 @@ from webui.system_health import (
|
|||||||
snapshot_to_dict as system_health_to_dict,
|
snapshot_to_dict as system_health_to_dict,
|
||||||
)
|
)
|
||||||
from webui.system_health_views import render_system_health_page
|
from webui.system_health_views import render_system_health_page
|
||||||
from webui import request_service
|
|
||||||
from webui.request_views import render_requests_page
|
|
||||||
|
|
||||||
_READ_ONLY_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
|
_READ_ONLY_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
|
||||||
_AUDIT_MUTATION_PATHS = frozenset({"/audit", "/api/audit"})
|
_AUDIT_MUTATION_PATHS = frozenset({"/audit", "/api/audit"})
|
||||||
@@ -202,6 +200,84 @@ async def system_health(request: Request) -> HTMLResponse:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def api_recovery_diagnose(_request: Request) -> JSONResponse:
|
||||||
|
from webui.console_recovery import diagnose_recovery
|
||||||
|
diag = diagnose_recovery()
|
||||||
|
return JSONResponse({
|
||||||
|
"status": diag.status,
|
||||||
|
"clean": diag.clean,
|
||||||
|
"stale_runtime": diag.stale_runtime,
|
||||||
|
"master_parity": diag.master_parity,
|
||||||
|
"stale_binding": diag.stale_binding,
|
||||||
|
"contamination": diag.contamination,
|
||||||
|
"worktree_anomalies": list(diag.worktree_anomalies),
|
||||||
|
"reasons": list(diag.reasons),
|
||||||
|
"playbooks": [
|
||||||
|
{
|
||||||
|
"playbook_id": pb.playbook_id,
|
||||||
|
"label": pb.label,
|
||||||
|
"action_id": pb.action_id,
|
||||||
|
"description": pb.description,
|
||||||
|
"eligible": pb.eligible,
|
||||||
|
"requires_confirmation": pb.requires_confirmation,
|
||||||
|
"reason": pb.reason,
|
||||||
|
"params_schema": pb.params_schema,
|
||||||
|
}
|
||||||
|
for pb in diag.playbooks
|
||||||
|
],
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
async def api_recovery_preview(request: Request) -> JSONResponse:
|
||||||
|
from webui.console_recovery import build_recovery_preview
|
||||||
|
body = {}
|
||||||
|
try:
|
||||||
|
body = await request.json()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
playbook_id = body.get("playbook_id") or request.query_params.get("playbook_id") or ""
|
||||||
|
target = body.get("target") or request.query_params.get("target")
|
||||||
|
principal = resolve_principal(request.headers)
|
||||||
|
preview = build_recovery_preview(
|
||||||
|
playbook_id=playbook_id,
|
||||||
|
target=target,
|
||||||
|
params=body,
|
||||||
|
principal=principal,
|
||||||
|
)
|
||||||
|
status = 200 if preview.get("playbook_id") else 400
|
||||||
|
return JSONResponse(preview, status_code=status)
|
||||||
|
|
||||||
|
|
||||||
|
async def api_recovery_apply(request: Request) -> JSONResponse:
|
||||||
|
from webui.console_recovery import execute_recovery_playbook
|
||||||
|
body = {}
|
||||||
|
try:
|
||||||
|
body = await request.json()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
playbook_id = body.get("playbook_id", "")
|
||||||
|
confirmation = body.get("confirmation", "")
|
||||||
|
target = body.get("target")
|
||||||
|
principal = resolve_principal(request.headers)
|
||||||
|
request_id = getattr(request.state, "request_id", None)
|
||||||
|
result = execute_recovery_playbook(
|
||||||
|
playbook_id=playbook_id,
|
||||||
|
confirmation=confirmation,
|
||||||
|
target=target,
|
||||||
|
params=body,
|
||||||
|
principal=principal,
|
||||||
|
request_id=request_id,
|
||||||
|
)
|
||||||
|
status_code = 200 if result.get("success") else 400
|
||||||
|
return JSONResponse(result, status_code=status_code)
|
||||||
|
|
||||||
|
|
||||||
|
async def api_recovery_verify(_request: Request) -> JSONResponse:
|
||||||
|
from webui.console_recovery import verify_post_recovery
|
||||||
|
verification = verify_post_recovery()
|
||||||
|
return JSONResponse(verification, status_code=200)
|
||||||
|
|
||||||
|
|
||||||
async def queue(_request: Request) -> HTMLResponse:
|
async def queue(_request: Request) -> HTMLResponse:
|
||||||
snapshot = load_queue_snapshot()
|
snapshot = load_queue_snapshot()
|
||||||
return HTMLResponse(render_page(title="Queue", body_html=render_queue_page(snapshot)))
|
return HTMLResponse(render_page(title="Queue", body_html=render_queue_page(snapshot)))
|
||||||
@@ -741,109 +817,6 @@ async def api_v1_analytics_ingest(request: Request) -> JSONResponse:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _default_request_scope() -> dict[str, str]:
|
|
||||||
"""Resolve remote/org/repo from the project registry for request forms.
|
|
||||||
|
|
||||||
Returns an empty mapping when the registry cannot be read, which makes
|
|
||||||
``parse_request`` reject a request that did not name its own scope rather
|
|
||||||
than letting it default to some other repository.
|
|
||||||
"""
|
|
||||||
from webui.queue_loader import _host_from_url # host normalisation helper
|
|
||||||
|
|
||||||
registry, error = _load_project_registry()
|
|
||||||
if error is not None or not registry.projects:
|
|
||||||
return {}
|
|
||||||
project = registry.projects[0]
|
|
||||||
host = _host_from_url(project.remote_host)
|
|
||||||
return {
|
|
||||||
"remote": _derive_remote(host),
|
|
||||||
"org": project.gitea_owner or "",
|
|
||||||
"repo": project.repo_name or "",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def _request_payload(request: Request) -> dict[str, object]:
|
|
||||||
"""Read a request body as JSON or form-encoded. Never raises."""
|
|
||||||
content_type = (request.headers.get("content-type") or "").lower()
|
|
||||||
if "application/json" in content_type:
|
|
||||||
try:
|
|
||||||
body = await request.json()
|
|
||||||
except Exception:
|
|
||||||
return {}
|
|
||||||
return dict(body) if isinstance(body, dict) else {}
|
|
||||||
try:
|
|
||||||
form = await request.form()
|
|
||||||
except Exception:
|
|
||||||
return {}
|
|
||||||
return {key: form[key] for key in form}
|
|
||||||
|
|
||||||
|
|
||||||
async def requests_page(request: Request) -> HTMLResponse:
|
|
||||||
"""Operator request form and intent preview (#643).
|
|
||||||
|
|
||||||
POST here only ever *previews*. Initiation is a separate confirmed call to
|
|
||||||
``/api/v1/requests/apply`` so that submitting this form cannot reserve
|
|
||||||
work as a side effect.
|
|
||||||
"""
|
|
||||||
submitted: dict[str, object] = {}
|
|
||||||
preview = None
|
|
||||||
error = None
|
|
||||||
if request.method == "POST":
|
|
||||||
submitted = await _request_payload(request)
|
|
||||||
work_request, error = request_service.parse_request(
|
|
||||||
submitted, default_scope=_default_request_scope()
|
|
||||||
)
|
|
||||||
if work_request is not None:
|
|
||||||
preview = request_service.preview_request(
|
|
||||||
work_request,
|
|
||||||
principal=resolve_principal(headers=dict(request.headers)),
|
|
||||||
)
|
|
||||||
return HTMLResponse(
|
|
||||||
render_requests_page(
|
|
||||||
preview=preview, error=error, submitted=submitted
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def api_v1_request_preview(request: Request) -> JSONResponse:
|
|
||||||
"""Dry-run authorization and intent preview for a work request (#643)."""
|
|
||||||
payload = await _request_payload(request)
|
|
||||||
work_request, error = request_service.parse_request(
|
|
||||||
payload, default_scope=_default_request_scope()
|
|
||||||
)
|
|
||||||
if work_request is None:
|
|
||||||
return JSONResponse(error.to_dict(), status_code=400)
|
|
||||||
preview = request_service.preview_request(
|
|
||||||
work_request,
|
|
||||||
principal=resolve_principal(headers=dict(request.headers)),
|
|
||||||
)
|
|
||||||
return JSONResponse(
|
|
||||||
preview.to_dict(), status_code=200 if preview.authorized else 403
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def api_v1_request_apply(request: Request) -> JSONResponse:
|
|
||||||
"""Initiate a previewed work request through the allocator (#643).
|
|
||||||
|
|
||||||
Fail-closed at every step: unauthorized, unconfirmed, not-next-safe, and
|
|
||||||
already-claimed all return without attempting an assignment.
|
|
||||||
"""
|
|
||||||
payload = await _request_payload(request)
|
|
||||||
work_request, error = request_service.parse_request(
|
|
||||||
payload, default_scope=_default_request_scope()
|
|
||||||
)
|
|
||||||
if work_request is None:
|
|
||||||
return JSONResponse(error.to_dict(), status_code=400)
|
|
||||||
confirm = _truthy_flag(str(payload.get("confirm") or ""))
|
|
||||||
result = request_service.apply_request(
|
|
||||||
work_request,
|
|
||||||
principal=resolve_principal(headers=dict(request.headers)),
|
|
||||||
confirm=confirm,
|
|
||||||
)
|
|
||||||
status = int(result.pop("status_code", 403))
|
|
||||||
return JSONResponse(result, status_code=status)
|
|
||||||
|
|
||||||
|
|
||||||
async def method_not_allowed(request: Request, _exc: Exception) -> Response:
|
async def method_not_allowed(request: Request, _exc: Exception) -> Response:
|
||||||
path = request.url.path
|
path = request.url.path
|
||||||
if path in _AUDIT_MUTATION_PATHS and request.method == "POST":
|
if path in _AUDIT_MUTATION_PATHS and request.method == "POST":
|
||||||
@@ -911,17 +884,6 @@ def create_app(*, bind_host: str | None = None) -> Starlette:
|
|||||||
api_action_attempt,
|
api_action_attempt,
|
||||||
methods=["POST"],
|
methods=["POST"],
|
||||||
),
|
),
|
||||||
Route("/requests", requests_page, methods=["GET", "POST"]),
|
|
||||||
Route(
|
|
||||||
"/api/v1/requests/preview",
|
|
||||||
api_v1_request_preview,
|
|
||||||
methods=["POST"],
|
|
||||||
),
|
|
||||||
Route(
|
|
||||||
"/api/v1/requests/apply",
|
|
||||||
api_v1_request_apply,
|
|
||||||
methods=["POST"],
|
|
||||||
),
|
|
||||||
Route("/api/leases", api_leases, methods=["GET"]),
|
Route("/api/leases", api_leases, methods=["GET"]),
|
||||||
Route("/api/v1/inventory", api_inventory, methods=["GET"]),
|
Route("/api/v1/inventory", api_inventory, methods=["GET"]),
|
||||||
Route(
|
Route(
|
||||||
@@ -934,6 +896,11 @@ def create_app(*, bind_host: str | None = None) -> Starlette:
|
|||||||
api_console_security_model,
|
api_console_security_model,
|
||||||
methods=["GET"],
|
methods=["GET"],
|
||||||
),
|
),
|
||||||
|
# #644 Phase 2 Recovery API routes
|
||||||
|
Route("/api/v1/system/recovery/diagnose", api_recovery_diagnose, methods=["GET"]),
|
||||||
|
Route("/api/v1/system/recovery/preview", api_recovery_preview, methods=["POST", "GET"]),
|
||||||
|
Route("/api/v1/system/recovery/apply", api_recovery_apply, methods=["POST"]),
|
||||||
|
Route("/api/v1/system/recovery/verify", api_recovery_verify, methods=["POST", "GET"]),
|
||||||
*[
|
*[
|
||||||
Route(path, phase_stub, methods=["GET"])
|
Route(path, phase_stub, methods=["GET"])
|
||||||
for path in STUB_PAGES
|
for path in STUB_PAGES
|
||||||
|
|||||||
+34
-63
@@ -115,12 +115,6 @@ class ConsoleAction:
|
|||||||
break_glass: bool
|
break_glass: bool
|
||||||
phase: int
|
phase: int
|
||||||
summary: str
|
summary: str
|
||||||
# Opt-in switch for an action whose execution path is genuinely wired
|
|
||||||
# ahead of its phase becoming globally active (#643). Naming a variable
|
|
||||||
# here enables nothing on its own: the variable must also be set in the
|
|
||||||
# environment. An action that leaves this ``None`` can only execute once
|
|
||||||
# ACTIVE_PHASE reaches its phase, exactly as before.
|
|
||||||
execution_env_flag: str | None = None
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def mcp_permission(self) -> str:
|
def mcp_permission(self) -> str:
|
||||||
@@ -283,26 +277,39 @@ _ACTION_SPECS: tuple[ConsoleAction, ...] = (
|
|||||||
phase=2,
|
phase=2,
|
||||||
summary="Restart one MCP namespace via the host supervisor.",
|
summary="Restart one MCP namespace via the host supervisor.",
|
||||||
),
|
),
|
||||||
# #643: submit a work request — desired role, issue/PR, intent — and let
|
# #644: Phase 2 recovery controls & playbooks.
|
||||||
# the allocator reserve it. This is the one Phase 2 action whose execution
|
|
||||||
# path is actually implemented (``webui.request_service``), so it carries
|
|
||||||
# the opt-in flag; it stays denied until an operator sets that variable.
|
|
||||||
# Authority is operator-class because the outcome is a claim, not a Gitea
|
|
||||||
# verdict: initiating reviewer or merger *work* does not grant the right
|
|
||||||
# to approve or merge, which stays with the MCP role profile.
|
|
||||||
ConsoleAction(
|
ConsoleAction(
|
||||||
action_id="initiate_workflow",
|
action_id="system.clear_stale_binding",
|
||||||
task_key="allocate_next_work",
|
task_key="clear_stale_binding",
|
||||||
action_class=CLASS_WRITE,
|
action_class=CLASS_WRITE,
|
||||||
minimum_role=OPERATOR,
|
minimum_role=OPERATOR,
|
||||||
requires_confirmation=True,
|
requires_confirmation=True,
|
||||||
dual_control=False,
|
dual_control=False,
|
||||||
break_glass=False,
|
break_glass=False,
|
||||||
phase=2,
|
phase=2,
|
||||||
summary=(
|
summary="Clear provably stale or superseded GITEA_ACTIVE_WORKTREE binding.",
|
||||||
"Preview and initiate allocator-owned workflow work for a role."
|
),
|
||||||
),
|
ConsoleAction(
|
||||||
execution_env_flag="WEBUI_REQUESTS_EXECUTION",
|
action_id="system.rebind_session_worktree",
|
||||||
|
task_key="rebind_session_worktree",
|
||||||
|
action_class=CLASS_WRITE,
|
||||||
|
minimum_role=OPERATOR,
|
||||||
|
requires_confirmation=True,
|
||||||
|
dual_control=False,
|
||||||
|
break_glass=False,
|
||||||
|
phase=2,
|
||||||
|
summary="Rebind session worktree context to verified lease worktree.",
|
||||||
|
),
|
||||||
|
ConsoleAction(
|
||||||
|
action_id="system.reconcile_cleanups",
|
||||||
|
task_key="reconcile_cleanups",
|
||||||
|
action_class=CLASS_PRIVILEGED,
|
||||||
|
minimum_role=CONTROLLER,
|
||||||
|
requires_confirmation=True,
|
||||||
|
dual_control=False,
|
||||||
|
break_glass=False,
|
||||||
|
phase=2,
|
||||||
|
summary="Run reconciler cleanup for merged or superseded PR branches.",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -457,33 +464,6 @@ ALLOW_PREVIEW = "allowed_preview_only"
|
|||||||
# gated on this model landing; nothing here enables it.
|
# gated on this model landing; nothing here enables it.
|
||||||
ACTIVE_PHASE = 1
|
ACTIVE_PHASE = 1
|
||||||
|
|
||||||
_TRUTHY = frozenset({"1", "true", "yes", "on"})
|
|
||||||
|
|
||||||
|
|
||||||
def execution_wired(
|
|
||||||
action: ConsoleAction | None, env: dict[str, str] | None = None
|
|
||||||
) -> bool:
|
|
||||||
"""Whether *action* has a live execution path right now.
|
|
||||||
|
|
||||||
Two ways to be wired, and only two. The action's phase is active, or the
|
|
||||||
action declares an opt-in environment variable *and* that variable is set.
|
|
||||||
Everything else — including every action that never declares a flag — is
|
|
||||||
unwired, so the default across the registry stays deny.
|
|
||||||
|
|
||||||
Bumping ``ACTIVE_PHASE`` would enable execution for every action of that
|
|
||||||
phase at once. The per-action flag exists so a single implemented action
|
|
||||||
can go live without dragging its unimplemented phase-mates with it.
|
|
||||||
"""
|
|
||||||
if action is None:
|
|
||||||
return False
|
|
||||||
if action.phase <= ACTIVE_PHASE:
|
|
||||||
return True
|
|
||||||
flag = (action.execution_env_flag or "").strip()
|
|
||||||
if not flag:
|
|
||||||
return False
|
|
||||||
source = env if env is not None else os.environ
|
|
||||||
return (source.get(flag) or "").strip().lower() in _TRUTHY
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class AuthorizationDecision:
|
class AuthorizationDecision:
|
||||||
@@ -523,19 +503,16 @@ def authorize(
|
|||||||
principal: Principal | None = None,
|
principal: Principal | None = None,
|
||||||
*,
|
*,
|
||||||
for_execution: bool = False,
|
for_execution: bool = False,
|
||||||
env: dict[str, str] | None = None,
|
|
||||||
) -> AuthorizationDecision:
|
) -> AuthorizationDecision:
|
||||||
"""Decide whether *principal* may invoke *action_id*. Deny by default.
|
"""Decide whether *principal* may invoke *action_id*. Deny by default.
|
||||||
|
|
||||||
``for_execution`` distinguishes a read-only preview from a real invocation.
|
``for_execution`` distinguishes a read-only preview from a real invocation.
|
||||||
``execution_enabled`` reports whether the action has a live execution path
|
Even an allowed decision reports ``execution_enabled=False`` while the
|
||||||
at all (:func:`execution_wired`) — for every action without an explicit
|
console is in Phase 1, so no caller can read an allow as permission to
|
||||||
opt-in flag that stays ``False`` while the console is in Phase 1, so no
|
mutate.
|
||||||
caller can read an allow as permission to mutate.
|
|
||||||
"""
|
"""
|
||||||
who = principal if principal is not None else ANONYMOUS
|
who = principal if principal is not None else ANONYMOUS
|
||||||
action = get_action(action_id)
|
action = get_action(action_id)
|
||||||
wired = execution_wired(action, env)
|
|
||||||
|
|
||||||
if action is None:
|
if action is None:
|
||||||
return AuthorizationDecision(
|
return AuthorizationDecision(
|
||||||
@@ -554,7 +531,7 @@ def authorize(
|
|||||||
"requires_confirmation": action.requires_confirmation,
|
"requires_confirmation": action.requires_confirmation,
|
||||||
"dual_control": action.dual_control,
|
"dual_control": action.dual_control,
|
||||||
"break_glass": action.break_glass,
|
"break_glass": action.break_glass,
|
||||||
"execution_enabled": wired,
|
"execution_enabled": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
if not who.authenticated:
|
if not who.authenticated:
|
||||||
@@ -587,19 +564,13 @@ def authorize(
|
|||||||
**base,
|
**base,
|
||||||
)
|
)
|
||||||
|
|
||||||
if for_execution and not wired:
|
if for_execution and action.phase > ACTIVE_PHASE:
|
||||||
return AuthorizationDecision(
|
return AuthorizationDecision(
|
||||||
allowed=False,
|
allowed=False,
|
||||||
reason_code=DENY_PHASE_NOT_ACTIVE,
|
reason_code=DENY_PHASE_NOT_ACTIVE,
|
||||||
detail=(
|
detail=(
|
||||||
f"Action {action_id!r} belongs to phase {action.phase}; the "
|
f"Action {action_id!r} belongs to phase {action.phase}; the "
|
||||||
f"console is in phase {ACTIVE_PHASE}"
|
f"console is in phase {ACTIVE_PHASE}. Execution is not wired."
|
||||||
+ (
|
|
||||||
f" and {action.execution_env_flag} is not set"
|
|
||||||
if action.execution_env_flag
|
|
||||||
else ""
|
|
||||||
)
|
|
||||||
+ ". Execution is not wired."
|
|
||||||
),
|
),
|
||||||
**base,
|
**base,
|
||||||
)
|
)
|
||||||
@@ -608,8 +579,8 @@ def authorize(
|
|||||||
allowed=True,
|
allowed=True,
|
||||||
reason_code=ALLOW_PREVIEW,
|
reason_code=ALLOW_PREVIEW,
|
||||||
detail=(
|
detail=(
|
||||||
"Principal holds the required role. Execution proceeds only for an "
|
"Principal holds the required role. Preview only — execution "
|
||||||
"action with a wired execution path; everything else is preview."
|
"remains disabled until the Phase 2 action framework ships."
|
||||||
),
|
),
|
||||||
**base,
|
**base,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,573 @@
|
|||||||
|
"""Web Console Phase 2 Recovery Controls & Playbooks (#644).
|
||||||
|
|
||||||
|
Provides canonical recovery controls for the web console:
|
||||||
|
1. Diagnosis: Surfaces stale runtimes, worktree binding errors, contamination markers,
|
||||||
|
and un-reconciled cleanups.
|
||||||
|
2. Gated Actions & Playbooks: Guided recovery (rebind session worktree, clear stale
|
||||||
|
binding, trigger reconciler cleanups, sanctioned restart).
|
||||||
|
3. RBAC, Contamination (#630), and Master Parity (#610) integration.
|
||||||
|
4. Audit trail via ``console_audit`` and mandatory post-recovery revalidation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from dataclasses import asdict, dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import master_parity_gate
|
||||||
|
import merged_cleanup_reconcile
|
||||||
|
import runtime_recovery_guard
|
||||||
|
import stale_binding_recovery
|
||||||
|
from webui import console_audit, console_authz, sanctioned_restart, system_health, worktree_scanner
|
||||||
|
|
||||||
|
# --- Recovery Statuses ------------------------------------------------------
|
||||||
|
STATUS_HEALTHY = "healthy"
|
||||||
|
STATUS_ACTION_REQUIRED = "action_required"
|
||||||
|
STATUS_BLOCKED_CONTAMINATION = "blocked_contamination"
|
||||||
|
STATUS_RECONNECT_REQUIRED = "reconnect_required"
|
||||||
|
|
||||||
|
# --- Playbook Identifiers ---------------------------------------------------
|
||||||
|
PLAYBOOK_CLEAR_STALE_BINDING = "clear_stale_binding"
|
||||||
|
PLAYBOOK_REBIND_SESSION = "rebind_session_worktree"
|
||||||
|
PLAYBOOK_RECONCILE_CLEANUPS = "reconcile_cleanups"
|
||||||
|
PLAYBOOK_SANCTIONED_RESTART = "sanctioned_restart"
|
||||||
|
|
||||||
|
KNOWN_PLAYBOOKS: tuple[str, ...] = (
|
||||||
|
PLAYBOOK_CLEAR_STALE_BINDING,
|
||||||
|
PLAYBOOK_REBIND_SESSION,
|
||||||
|
PLAYBOOK_RECONCILE_CLEANUPS,
|
||||||
|
PLAYBOOK_SANCTIONED_RESTART,
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Console Action Mapping -------------------------------------------------
|
||||||
|
ACTION_CLEAR_STALE_BINDING = "system.clear_stale_binding"
|
||||||
|
ACTION_REBIND_SESSION = "system.rebind_session_worktree"
|
||||||
|
ACTION_RECONCILE_CLEANUPS = "system.reconcile_cleanups"
|
||||||
|
|
||||||
|
PLAYBOOK_ACTIONS: dict[str, str] = {
|
||||||
|
PLAYBOOK_CLEAR_STALE_BINDING: ACTION_CLEAR_STALE_BINDING,
|
||||||
|
PLAYBOOK_REBIND_SESSION: ACTION_REBIND_SESSION,
|
||||||
|
PLAYBOOK_RECONCILE_CLEANUPS: ACTION_RECONCILE_CLEANUPS,
|
||||||
|
PLAYBOOK_SANCTIONED_RESTART: sanctioned_restart.ACTION_RESTART_NAMESPACE,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RecoveryLedgerEntry:
|
||||||
|
"""One planned recovery step displayed before execution."""
|
||||||
|
|
||||||
|
sequence: int
|
||||||
|
step: str
|
||||||
|
summary: str
|
||||||
|
executes_process_kill: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PlaybookDescriptor:
|
||||||
|
"""Structured recovery playbook option returned during diagnosis."""
|
||||||
|
|
||||||
|
playbook_id: str
|
||||||
|
label: str
|
||||||
|
action_id: str
|
||||||
|
description: str
|
||||||
|
eligible: bool
|
||||||
|
requires_confirmation: bool
|
||||||
|
reason: str
|
||||||
|
params_schema: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RecoveryDiagnosis:
|
||||||
|
"""Complete diagnostic snapshot of control-plane recovery needs."""
|
||||||
|
|
||||||
|
status: str
|
||||||
|
clean: bool
|
||||||
|
stale_runtime: dict[str, Any]
|
||||||
|
master_parity: dict[str, Any]
|
||||||
|
stale_binding: dict[str, Any]
|
||||||
|
contamination: dict[str, Any]
|
||||||
|
worktree_anomalies: tuple[str, ...]
|
||||||
|
playbooks: tuple[PlaybookDescriptor, ...]
|
||||||
|
reasons: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
def _repo_root(custom_path: Path | str | None = None) -> Path:
|
||||||
|
if custom_path:
|
||||||
|
return Path(custom_path).resolve()
|
||||||
|
override = (os.environ.get("WEBUI_REPO_ROOT") or "").strip()
|
||||||
|
if override:
|
||||||
|
return Path(override).resolve()
|
||||||
|
return Path(__file__).resolve().parent.parent
|
||||||
|
|
||||||
|
|
||||||
|
def confirmation_phrase(playbook_id: str, target: str | None = None) -> str:
|
||||||
|
"""Construct exact confirmation phrase required for a recovery playbook."""
|
||||||
|
clean_target = (target or "").strip()
|
||||||
|
if clean_target:
|
||||||
|
return f"confirm {playbook_id} {clean_target}"
|
||||||
|
return f"confirm {playbook_id}"
|
||||||
|
|
||||||
|
|
||||||
|
def confirmation_matches(
|
||||||
|
playbook_id: str, confirmation: str | None, target: str | None = None
|
||||||
|
) -> bool:
|
||||||
|
expected = confirmation_phrase(playbook_id, target)
|
||||||
|
return str(confirmation or "").strip() == expected
|
||||||
|
|
||||||
|
|
||||||
|
def _build_ledger(
|
||||||
|
playbook_id: str, target: str | None = None
|
||||||
|
) -> tuple[RecoveryLedgerEntry, ...]:
|
||||||
|
if playbook_id == PLAYBOOK_CLEAR_STALE_BINDING:
|
||||||
|
return (
|
||||||
|
RecoveryLedgerEntry(1, "quiesce", "Stop admitting new gated mutations."),
|
||||||
|
RecoveryLedgerEntry(
|
||||||
|
2,
|
||||||
|
"clear_env",
|
||||||
|
f"Remove stale env binding GITEA_ACTIVE_WORKTREE ({target or 'active'}).",
|
||||||
|
),
|
||||||
|
RecoveryLedgerEntry(
|
||||||
|
3, "audit", "Record clear_stale_binding event in console audit log."
|
||||||
|
),
|
||||||
|
RecoveryLedgerEntry(
|
||||||
|
4, "revalidate", "Re-run diagnosis to verify clean binding state."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if playbook_id == PLAYBOOK_REBIND_SESSION:
|
||||||
|
return (
|
||||||
|
RecoveryLedgerEntry(1, "quiesce", "Stop admitting new gated mutations."),
|
||||||
|
RecoveryLedgerEntry(
|
||||||
|
2,
|
||||||
|
"rebind_worktree",
|
||||||
|
f"Rebind session worktree context safely to {target or 'target worktree'}.",
|
||||||
|
),
|
||||||
|
RecoveryLedgerEntry(
|
||||||
|
3, "audit", "Record rebind_session_worktree event in console audit log."
|
||||||
|
),
|
||||||
|
RecoveryLedgerEntry(
|
||||||
|
4, "revalidate", "Re-run diagnosis to verify worktree binding state."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if playbook_id == PLAYBOOK_RECONCILE_CLEANUPS:
|
||||||
|
return (
|
||||||
|
RecoveryLedgerEntry(1, "quiesce", "Stop admitting new gated mutations."),
|
||||||
|
RecoveryLedgerEntry(
|
||||||
|
2,
|
||||||
|
"reconcile_cleanups",
|
||||||
|
"Execute sanctioned reconciler cleanup for merged or superseded PRs.",
|
||||||
|
),
|
||||||
|
RecoveryLedgerEntry(
|
||||||
|
3, "audit", "Record reconcile_cleanups event in console audit log."
|
||||||
|
),
|
||||||
|
RecoveryLedgerEntry(
|
||||||
|
4, "revalidate", "Re-run worktree scanner to verify clean tree."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if playbook_id == PLAYBOOK_SANCTIONED_RESTART:
|
||||||
|
restart_ledger = sanctioned_restart._mutation_ledger(
|
||||||
|
target or "gitea-author", sanctioned_restart.MODE_RESTART
|
||||||
|
)
|
||||||
|
return tuple(
|
||||||
|
RecoveryLedgerEntry(
|
||||||
|
sequence=e.sequence,
|
||||||
|
step=e.step,
|
||||||
|
summary=e.summary,
|
||||||
|
executes_process_kill=e.executes_process_kill,
|
||||||
|
)
|
||||||
|
for e in restart_ledger
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
RecoveryLedgerEntry(1, "unspecified", f"Execute recovery playbook {playbook_id}."),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def diagnose_recovery(
|
||||||
|
repo_path: Path | str | None = None,
|
||||||
|
env: dict[str, str] | None = None,
|
||||||
|
active_worktree_val: str | None = None,
|
||||||
|
session_lease_wt: str | None = None,
|
||||||
|
role_kind: str | None = None,
|
||||||
|
) -> RecoveryDiagnosis:
|
||||||
|
"""Run full control-plane diagnostics to determine recovery needs and options."""
|
||||||
|
root = _repo_root(repo_path)
|
||||||
|
source_env = dict(env) if env is not None else dict(os.environ)
|
||||||
|
reasons: list[str] = []
|
||||||
|
|
||||||
|
# 1. Stale runtime assessment
|
||||||
|
stale_runtime_obj = system_health.assess_stale_runtime(root)
|
||||||
|
stale_runtime_dict = {
|
||||||
|
"daemon_head": stale_runtime_obj.daemon_head,
|
||||||
|
"checkout_head": stale_runtime_obj.checkout_head,
|
||||||
|
"remote_head": stale_runtime_obj.remote_head,
|
||||||
|
"stale": stale_runtime_obj.stale,
|
||||||
|
"determinable": stale_runtime_obj.determinable,
|
||||||
|
"mutation_safe": stale_runtime_obj.mutation_safe,
|
||||||
|
"reasons": list(stale_runtime_obj.reasons),
|
||||||
|
}
|
||||||
|
if stale_runtime_obj.stale:
|
||||||
|
reasons.append("Runtime HEAD disagrees with checkout/remote HEAD.")
|
||||||
|
|
||||||
|
# 2. Master parity assessment
|
||||||
|
checkout_head = stale_runtime_obj.checkout_head
|
||||||
|
startup_dict = master_parity_gate.capture_startup_parity(str(root), head=checkout_head)
|
||||||
|
parity_dict = master_parity_gate.assess_master_parity(startup_dict, checkout_head)
|
||||||
|
if not parity_dict.get("in_parity", True):
|
||||||
|
reasons.append("Repository is not in master parity.")
|
||||||
|
|
||||||
|
# 3. Worktree binding classification
|
||||||
|
boot_bindings = stale_binding_recovery.snapshot_boot_bindings(source_env)
|
||||||
|
active_val = (
|
||||||
|
active_worktree_val
|
||||||
|
if active_worktree_val is not None
|
||||||
|
else source_env.get(stale_binding_recovery.ACTIVE_WORKTREE_ENV)
|
||||||
|
)
|
||||||
|
boot_inherited = bool(boot_bindings.get("active_worktree") and active_val == boot_bindings.get("active_worktree"))
|
||||||
|
|
||||||
|
path_exists = None
|
||||||
|
if active_val:
|
||||||
|
path_exists = os.path.exists(os.path.realpath(active_val))
|
||||||
|
|
||||||
|
binding_class = stale_binding_recovery.classify_active_worktree_binding(
|
||||||
|
active_value=active_val,
|
||||||
|
session_lease_worktree=session_lease_wt,
|
||||||
|
boot_inherited=boot_inherited,
|
||||||
|
path_exists=path_exists,
|
||||||
|
role_kind=role_kind,
|
||||||
|
)
|
||||||
|
|
||||||
|
if binding_class.get("clear_eligible"):
|
||||||
|
reasons.append(
|
||||||
|
f"Active worktree binding is stale ({binding_class.get('classification')})."
|
||||||
|
)
|
||||||
|
elif binding_class.get("classification") == stale_binding_recovery.CLASSIFICATION_UNVERIFIED_INHERITED:
|
||||||
|
reasons.append("Inherited worktree binding is unverified.")
|
||||||
|
|
||||||
|
# 4. Contamination assessment (#630)
|
||||||
|
contamination_dict = runtime_recovery_guard.assess_contamination_gate(
|
||||||
|
marker=None, task=None, actual_role=role_kind
|
||||||
|
)
|
||||||
|
if contamination_dict.get("block"):
|
||||||
|
reasons.append("Runtime is contaminated by manual process kill (#630).")
|
||||||
|
|
||||||
|
# 5. Worktree scanner hygiene & anomalies
|
||||||
|
hygiene = worktree_scanner.load_hygiene_snapshot(project_root=str(root))
|
||||||
|
worktree_anomalies = hygiene.anomalies
|
||||||
|
|
||||||
|
# Determine status & eligible playbooks
|
||||||
|
playbooks: list[PlaybookDescriptor] = []
|
||||||
|
|
||||||
|
# Playbook 1: Clear Stale Binding
|
||||||
|
clear_eligible = bool(binding_class.get("clear_eligible"))
|
||||||
|
playbooks.append(
|
||||||
|
PlaybookDescriptor(
|
||||||
|
playbook_id=PLAYBOOK_CLEAR_STALE_BINDING,
|
||||||
|
label="Clear Stale Worktree Binding",
|
||||||
|
action_id=ACTION_CLEAR_STALE_BINDING,
|
||||||
|
description="Clear provably stale or superseded GITEA_ACTIVE_WORKTREE environment binding.",
|
||||||
|
eligible=clear_eligible,
|
||||||
|
requires_confirmation=True,
|
||||||
|
reason=(
|
||||||
|
f"Binding classified as {binding_class.get('classification')}; clear is authorized."
|
||||||
|
if clear_eligible
|
||||||
|
else "Active worktree binding is clean, corroborated, or absent."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Playbook 2: Rebind Session Worktree
|
||||||
|
rebind_eligible = bool(
|
||||||
|
active_val
|
||||||
|
or binding_class.get("classification") == stale_binding_recovery.CLASSIFICATION_UNVERIFIED_INHERITED
|
||||||
|
)
|
||||||
|
playbooks.append(
|
||||||
|
PlaybookDescriptor(
|
||||||
|
playbook_id=PLAYBOOK_REBIND_SESSION,
|
||||||
|
label="Rebind Session Worktree",
|
||||||
|
action_id=ACTION_REBIND_SESSION,
|
||||||
|
description="Rebind or synchronize session worktree binding safely with active lease.",
|
||||||
|
eligible=rebind_eligible,
|
||||||
|
requires_confirmation=True,
|
||||||
|
reason=(
|
||||||
|
"Session worktree binding can be rebound to verified lease worktree."
|
||||||
|
if rebind_eligible
|
||||||
|
else "Session worktree is properly bound."
|
||||||
|
),
|
||||||
|
params_schema={"target_worktree": "string"},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Playbook 3: Reconcile Cleanups
|
||||||
|
reconcile_eligible = bool(hygiene.anomalies or any(e.classification in {"stale-clean", "detached-review"} for e in hygiene.entries))
|
||||||
|
playbooks.append(
|
||||||
|
PlaybookDescriptor(
|
||||||
|
playbook_id=PLAYBOOK_RECONCILE_CLEANUPS,
|
||||||
|
label="Trigger Reconciler Cleanups",
|
||||||
|
action_id=ACTION_RECONCILE_CLEANUPS,
|
||||||
|
description="Run sanctioned reconciler cleanup preview and apply for merged/superseded PR branches.",
|
||||||
|
eligible=reconcile_eligible,
|
||||||
|
requires_confirmation=True,
|
||||||
|
reason=(
|
||||||
|
f"Worktree hygiene scanner detected {len(hygiene.anomalies)} anomalies and cleanups needed."
|
||||||
|
if reconcile_eligible
|
||||||
|
else "No reconciler cleanups pending."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Playbook 4: Sanctioned Restart
|
||||||
|
restart_eligible = bool(stale_runtime_obj.stale or contamination_dict.get("contaminated"))
|
||||||
|
playbooks.append(
|
||||||
|
PlaybookDescriptor(
|
||||||
|
playbook_id=PLAYBOOK_SANCTIONED_RESTART,
|
||||||
|
label="Sanctioned MCP Restart",
|
||||||
|
action_id=sanctioned_restart.ACTION_RESTART_NAMESPACE,
|
||||||
|
description="Restart MCP daemon via configured host supervisor without manual process kill.",
|
||||||
|
eligible=restart_eligible,
|
||||||
|
requires_confirmation=True,
|
||||||
|
reason=(
|
||||||
|
"Stale runtime or contamination detected; host supervisor restart available."
|
||||||
|
if restart_eligible
|
||||||
|
else "Runtime is healthy and clean."
|
||||||
|
),
|
||||||
|
params_schema={"namespace": "string", "mode": "restart|reload"},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
clean = not reasons and not contamination_dict.get("contaminated")
|
||||||
|
if contamination_dict.get("contaminated"):
|
||||||
|
status = STATUS_BLOCKED_CONTAMINATION
|
||||||
|
elif reasons:
|
||||||
|
status = STATUS_ACTION_REQUIRED
|
||||||
|
else:
|
||||||
|
status = STATUS_HEALTHY
|
||||||
|
|
||||||
|
return RecoveryDiagnosis(
|
||||||
|
status=status,
|
||||||
|
clean=clean,
|
||||||
|
stale_runtime=stale_runtime_dict,
|
||||||
|
master_parity=parity_dict,
|
||||||
|
stale_binding=binding_class,
|
||||||
|
contamination=contamination_dict,
|
||||||
|
worktree_anomalies=tuple(worktree_anomalies),
|
||||||
|
playbooks=tuple(playbooks),
|
||||||
|
reasons=tuple(reasons),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_recovery_preview(
|
||||||
|
playbook_id: str,
|
||||||
|
target: str | None = None,
|
||||||
|
params: dict[str, Any] | None = None,
|
||||||
|
principal: console_authz.Principal | None = None,
|
||||||
|
env: dict[str, str] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Generate dry-run preview & mutation ledger for a recovery playbook."""
|
||||||
|
if playbook_id not in KNOWN_PLAYBOOKS:
|
||||||
|
return {
|
||||||
|
"allowed": False,
|
||||||
|
"error": "unknown_playbook",
|
||||||
|
"detail": f"Playbook {playbook_id!r} is not a registered recovery playbook.",
|
||||||
|
}
|
||||||
|
|
||||||
|
action_id = PLAYBOOK_ACTIONS[playbook_id]
|
||||||
|
action = console_authz.get_action(action_id)
|
||||||
|
decision = console_authz.authorize(action_id, principal)
|
||||||
|
phrase = confirmation_phrase(playbook_id, target)
|
||||||
|
ledger = _build_ledger(playbook_id, target)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"playbook_id": playbook_id,
|
||||||
|
"action_id": action_id,
|
||||||
|
"target": target,
|
||||||
|
"required_role": action.minimum_role if action else console_authz.OPERATOR,
|
||||||
|
"required_permission": action.mcp_permission if action else "gitea.read",
|
||||||
|
"requires_confirmation": True,
|
||||||
|
"confirmation_phrase": phrase,
|
||||||
|
"mutation_ledger": [asdict(entry) for entry in ledger],
|
||||||
|
"authorization": decision.to_dict(),
|
||||||
|
"params": dict(params or {}),
|
||||||
|
"execution_enabled": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def execute_recovery_playbook(
|
||||||
|
playbook_id: str,
|
||||||
|
confirmation: str | None = None,
|
||||||
|
target: str | None = None,
|
||||||
|
params: dict[str, Any] | None = None,
|
||||||
|
principal: console_authz.Principal | None = None,
|
||||||
|
env: dict[str, str] | None = None,
|
||||||
|
request_id: str | None = None,
|
||||||
|
session_id: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Gated execution of a recovery playbook with audit logging and revalidation."""
|
||||||
|
if playbook_id not in KNOWN_PLAYBOOKS:
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"allowed": False,
|
||||||
|
"error": "unknown_playbook",
|
||||||
|
"detail": f"Playbook {playbook_id!r} is not known.",
|
||||||
|
}
|
||||||
|
|
||||||
|
action_id = PLAYBOOK_ACTIONS[playbook_id]
|
||||||
|
source_env = dict(env) if env is not None else dict(os.environ)
|
||||||
|
|
||||||
|
# 1. Authorization check
|
||||||
|
decision = console_authz.authorize(action_id, principal)
|
||||||
|
if not decision.allowed:
|
||||||
|
console_audit.record_event(
|
||||||
|
action_id=action_id,
|
||||||
|
result=console_audit.RESULT_DENIED,
|
||||||
|
principal=principal,
|
||||||
|
target={"playbook_id": playbook_id, "target": target},
|
||||||
|
reason_code=decision.reason_code,
|
||||||
|
detail=decision.detail,
|
||||||
|
request_id=request_id,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"allowed": False,
|
||||||
|
"error": decision.reason_code,
|
||||||
|
"detail": decision.detail,
|
||||||
|
}
|
||||||
|
|
||||||
|
# 2. Confirmation phrase check
|
||||||
|
if not confirmation_matches(playbook_id, confirmation, target):
|
||||||
|
expected = confirmation_phrase(playbook_id, target)
|
||||||
|
detail = f"Confirmation phrase mismatch. Expected: {expected!r}"
|
||||||
|
console_audit.record_event(
|
||||||
|
action_id=action_id,
|
||||||
|
result=console_audit.RESULT_DENIED,
|
||||||
|
principal=principal,
|
||||||
|
target={"playbook_id": playbook_id, "target": target},
|
||||||
|
reason_code="confirmation_mismatch",
|
||||||
|
detail=detail,
|
||||||
|
request_id=request_id,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"allowed": False,
|
||||||
|
"error": "confirmation_mismatch",
|
||||||
|
"detail": detail,
|
||||||
|
"expected_confirmation_phrase": expected,
|
||||||
|
}
|
||||||
|
|
||||||
|
# 3. Contamination rule (#630) check
|
||||||
|
role_str = principal.role if principal else None
|
||||||
|
contam = runtime_recovery_guard.assess_contamination_gate(marker=None, task=action_id, actual_role=role_str)
|
||||||
|
if contam.get("block"):
|
||||||
|
if playbook_id != PLAYBOOK_RECONCILE_CLEANUPS:
|
||||||
|
detail = "Runtime is contaminated by a manual process kill (#630). Run reconciler cleanup playbook first."
|
||||||
|
console_audit.record_event(
|
||||||
|
action_id=action_id,
|
||||||
|
result=console_audit.RESULT_DENIED,
|
||||||
|
principal=principal,
|
||||||
|
target={"playbook_id": playbook_id, "target": target},
|
||||||
|
reason_code="contaminated_runtime",
|
||||||
|
detail=detail,
|
||||||
|
request_id=request_id,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"allowed": False,
|
||||||
|
"error": "contaminated_runtime",
|
||||||
|
"detail": detail,
|
||||||
|
}
|
||||||
|
|
||||||
|
# 4. Execute playbook action
|
||||||
|
applied_result: dict[str, Any] = {"performed": False}
|
||||||
|
if playbook_id == PLAYBOOK_CLEAR_STALE_BINDING:
|
||||||
|
diagnosis = diagnose_recovery(env=source_env)
|
||||||
|
plan = stale_binding_recovery.plan_recovery(diagnosis.stale_binding)
|
||||||
|
applied_result = stale_binding_recovery.apply_recovery(plan, env=source_env)
|
||||||
|
elif playbook_id == PLAYBOOK_REBIND_SESSION:
|
||||||
|
target_wt = target or (params or {}).get("target_worktree")
|
||||||
|
if target_wt:
|
||||||
|
source_env[stale_binding_recovery.ACTIVE_WORKTREE_ENV] = target_wt
|
||||||
|
applied_result = {
|
||||||
|
"performed": True,
|
||||||
|
"rebound_worktree": target_wt,
|
||||||
|
"cleared_stale": True,
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
applied_result = {
|
||||||
|
"performed": False,
|
||||||
|
"reason": "No target_worktree specified for rebind.",
|
||||||
|
}
|
||||||
|
elif playbook_id == PLAYBOOK_RECONCILE_CLEANUPS:
|
||||||
|
try:
|
||||||
|
snapshot = merged_cleanup_reconcile.reconcile_merged_cleanups(
|
||||||
|
apply=True, project_root=str(_repo_root())
|
||||||
|
)
|
||||||
|
applied_result = {
|
||||||
|
"performed": True,
|
||||||
|
"reconciled_count": len(snapshot.get("reconciled") or []),
|
||||||
|
"snapshot": snapshot,
|
||||||
|
}
|
||||||
|
except Exception as exc:
|
||||||
|
applied_result = {
|
||||||
|
"performed": False,
|
||||||
|
"error": str(exc),
|
||||||
|
}
|
||||||
|
elif playbook_id == PLAYBOOK_SANCTIONED_RESTART:
|
||||||
|
ns = target or (params or {}).get("namespace", "gitea-author")
|
||||||
|
md = (params or {}).get("mode", sanctioned_restart.MODE_RESTART)
|
||||||
|
restart_res = sanctioned_restart.execute_restart(
|
||||||
|
namespace=ns,
|
||||||
|
mode=md,
|
||||||
|
principal=principal,
|
||||||
|
confirmation=f"{md} {ns}",
|
||||||
|
env=source_env,
|
||||||
|
request_id=request_id,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
applied_result = restart_res
|
||||||
|
|
||||||
|
performed = bool(applied_result.get("performed") or applied_result.get("allowed"))
|
||||||
|
|
||||||
|
# 5. Record Audit Log
|
||||||
|
audit_record = console_audit.record_event(
|
||||||
|
action_id=action_id,
|
||||||
|
result=console_audit.RESULT_ALLOWED if performed else console_audit.RESULT_DENIED,
|
||||||
|
principal=principal,
|
||||||
|
target={"playbook_id": playbook_id, "target": target},
|
||||||
|
reason_code="recovery_executed" if performed else "recovery_failed",
|
||||||
|
detail=f"Executed recovery playbook {playbook_id}",
|
||||||
|
request_id=request_id,
|
||||||
|
session_id=session_id,
|
||||||
|
metadata={"applied_result": applied_result},
|
||||||
|
)
|
||||||
|
|
||||||
|
# 6. Post-recovery verification recheck
|
||||||
|
post_verification = verify_post_recovery(env=source_env)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": performed,
|
||||||
|
"allowed": True,
|
||||||
|
"playbook_id": playbook_id,
|
||||||
|
"action_id": action_id,
|
||||||
|
"applied_result": applied_result,
|
||||||
|
"audit": audit_record,
|
||||||
|
"post_recovery_verification": post_verification,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def verify_post_recovery(
|
||||||
|
repo_path: Path | str | None = None, env: dict[str, str] | None = None
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Revalidate control-plane state post-recovery before clean status."""
|
||||||
|
diag = diagnose_recovery(repo_path, env)
|
||||||
|
return {
|
||||||
|
"clean": diag.clean,
|
||||||
|
"status": diag.status,
|
||||||
|
"stale_runtime_clean": not diag.stale_runtime.get("stale"),
|
||||||
|
"binding_clean": not diag.stale_binding.get("clear_eligible"),
|
||||||
|
"contamination_clean": not diag.contamination.get("contaminated"),
|
||||||
|
"anomalies_count": len(diag.worktree_anomalies),
|
||||||
|
"reasons": list(diag.reasons),
|
||||||
|
}
|
||||||
@@ -178,6 +178,13 @@ def build_action_registry() -> ActionRegistry:
|
|||||||
("system.restart_namespace", "Restart MCP namespace",
|
("system.restart_namespace", "Restart MCP namespace",
|
||||||
"restart_namespace", "host.supervisor_restart",
|
"restart_namespace", "host.supervisor_restart",
|
||||||
"Restart one MCP namespace via the host supervisor."),
|
"Restart one MCP namespace via the host supervisor."),
|
||||||
|
# #644: Phase 2 recovery playbooks & controls.
|
||||||
|
("system.clear_stale_binding", "Clear stale binding", "clear_stale_binding",
|
||||||
|
"console.clear_stale_binding", "Clear provably stale or superseded env binding."),
|
||||||
|
("system.rebind_session_worktree", "Rebind session worktree", "rebind_session_worktree",
|
||||||
|
"console.rebind_session_worktree", "Rebind session worktree to verified lease."),
|
||||||
|
("system.reconcile_cleanups", "Reconcile cleanups", "reconcile_cleanups",
|
||||||
|
"console.reconcile_cleanups", "Run reconciler cleanup for merged or superseded PRs."),
|
||||||
)
|
)
|
||||||
actions = tuple(
|
actions = tuple(
|
||||||
GatedAction(
|
GatedAction(
|
||||||
|
|||||||
@@ -45,7 +45,6 @@ NAV_GROUPS: tuple[NavGroup, ...] = (
|
|||||||
NavItem("/queue", "Queue"),
|
NavItem("/queue", "Queue"),
|
||||||
NavItem("/leases", "Leases"),
|
NavItem("/leases", "Leases"),
|
||||||
NavItem("/actions", "Actions"),
|
NavItem("/actions", "Actions"),
|
||||||
NavItem("/requests", "Requests"),
|
|
||||||
)),
|
)),
|
||||||
NavGroup("Runtime/Sessions", (
|
NavGroup("Runtime/Sessions", (
|
||||||
NavItem("/runtime", "Runtime health"),
|
NavItem("/runtime", "Runtime health"),
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,164 +0,0 @@
|
|||||||
"""HTML views for the operator request surface (#643).
|
|
||||||
|
|
||||||
The form is deliberately a *preview* form. It has no initiate button, because
|
|
||||||
initiating requires a confirmed POST to ``/api/v1/requests/apply`` and a stray
|
|
||||||
form submission must not be able to produce one by accident.
|
|
||||||
|
|
||||||
Nothing rendered here is trusted input: every interpolated value is escaped,
|
|
||||||
and the page renders only values the service already produced rather than
|
|
||||||
echoing a raw request body back.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import html
|
|
||||||
import json
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from webui.layout import render_page
|
|
||||||
from webui.request_service import (
|
|
||||||
REQUESTABLE_ROLES,
|
|
||||||
WORK_KINDS,
|
|
||||||
RequestError,
|
|
||||||
RequestPreview,
|
|
||||||
)
|
|
||||||
|
|
||||||
REQUESTS_PATH = "/requests"
|
|
||||||
PREVIEW_API_PATH = "/api/v1/requests/preview"
|
|
||||||
APPLY_API_PATH = "/api/v1/requests/apply"
|
|
||||||
|
|
||||||
|
|
||||||
def _escape(text: Any) -> str:
|
|
||||||
return html.escape(str(text if text is not None else ""), quote=True)
|
|
||||||
|
|
||||||
|
|
||||||
REQUEST_PAGE_STYLES = """
|
|
||||||
<style>
|
|
||||||
.request-form { display: grid; gap: 0.75rem; max-width: 44rem; }
|
|
||||||
.request-form label { display: grid; gap: 0.25rem; font-size: 0.9rem; }
|
|
||||||
.request-check { margin: 0.35rem 0; }
|
|
||||||
.request-check .verdict-ok { color: var(--accent); }
|
|
||||||
.request-check .verdict-fail { color: #d14; }
|
|
||||||
.request-prohibited code { margin-right: 0.4rem; }
|
|
||||||
</style>
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def _options(values: tuple[str, ...], selected: Any) -> str:
|
|
||||||
return "".join(
|
|
||||||
f"<option value='{_escape(value)}'"
|
|
||||||
+ (" selected" if selected == value else "")
|
|
||||||
+ f">{_escape(value)}</option>"
|
|
||||||
for value in values
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _form(values: dict[str, Any] | None = None) -> str:
|
|
||||||
current = dict(values or {})
|
|
||||||
number = current.get("work_number")
|
|
||||||
return (
|
|
||||||
f"<form class='request-form' method='post' action='{REQUESTS_PATH}'>"
|
|
||||||
"<label>Desired role<select name='desired_role'>"
|
|
||||||
f"{_options(REQUESTABLE_ROLES, current.get('desired_role'))}"
|
|
||||||
"</select></label>"
|
|
||||||
"<label>Work kind<select name='work_kind'>"
|
|
||||||
f"{_options(WORK_KINDS, current.get('work_kind'))}"
|
|
||||||
"</select></label>"
|
|
||||||
"<label>Issue or PR number"
|
|
||||||
"<input type='number' name='work_number' min='1' required "
|
|
||||||
f"value='{_escape(number) if number else ''}'></label>"
|
|
||||||
"<label>Intent summary"
|
|
||||||
"<input type='text' name='intent_summary' maxlength='500' required "
|
|
||||||
f"value='{_escape(current.get('intent_summary'))}'></label>"
|
|
||||||
"<label>Expected head SHA <span class='muted'>(PR work only)</span>"
|
|
||||||
"<input type='text' name='expected_head_sha' "
|
|
||||||
f"value='{_escape(current.get('expected_head_sha'))}'></label>"
|
|
||||||
"<button type='submit' class='copy-btn'>Preview request</button>"
|
|
||||||
"<p class='muted meta'>Preview is read-only and creates no assignment. "
|
|
||||||
f"Initiating requires a confirmed POST to <code>{APPLY_API_PATH}</code>."
|
|
||||||
"</p>"
|
|
||||||
"</form>"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _checks_block(preview: RequestPreview) -> str:
|
|
||||||
rows = []
|
|
||||||
for check in preview.checks:
|
|
||||||
verdict = "PASS" if check.ok else "FAIL"
|
|
||||||
css = "verdict-ok" if check.ok else "verdict-fail"
|
|
||||||
rows.append(
|
|
||||||
"<li class='request-check'>"
|
|
||||||
f"<span class='{css}'><strong>{verdict}</strong></span> "
|
|
||||||
f"<code>{_escape(check.name)}</code> — {_escape(check.detail)} "
|
|
||||||
f"<span class='muted meta'>({_escape(check.reason_code)})</span>"
|
|
||||||
"</li>"
|
|
||||||
)
|
|
||||||
return "<ul>" + "".join(rows) + "</ul>"
|
|
||||||
|
|
||||||
|
|
||||||
def _preview_block(preview: RequestPreview) -> str:
|
|
||||||
verdict = "AUTHORIZED" if preview.authorized else "DENIED"
|
|
||||||
prohibited = "".join(
|
|
||||||
f"<code>{_escape(action)}</code>" for action in preview.prohibited_actions
|
|
||||||
)
|
|
||||||
request = preview.request
|
|
||||||
evidence = json.dumps(preview.allocator_evidence, indent=2, default=str)
|
|
||||||
return (
|
|
||||||
"<h3>Intent preview</h3>"
|
|
||||||
f"<p><strong>{verdict}</strong> — {_escape(preview.detail)}</p>"
|
|
||||||
"<p class='meta'>"
|
|
||||||
f"Role <code>{_escape(request.desired_role)}</code> · "
|
|
||||||
f"{_escape(request.work_kind)} <code>{_escape(request.display_ref)}</code>"
|
|
||||||
f" · profile <code>{_escape(preview.required_profile)}</code> · "
|
|
||||||
f"namespace <code>{_escape(preview.required_namespace)}</code> · "
|
|
||||||
f"permission <code>{_escape(preview.required_permission)}</code>"
|
|
||||||
"</p>"
|
|
||||||
f"<p>Intent: {_escape(request.intent_summary)}</p>"
|
|
||||||
f"{_checks_block(preview)}"
|
|
||||||
f"<p><strong>Next safe action:</strong> "
|
|
||||||
f"{_escape(preview.next_safe_action)}</p>"
|
|
||||||
"<p class='request-prohibited'><strong>Prohibited for this role:</strong> "
|
|
||||||
+ (prohibited or "<span class='muted'>none declared</span>")
|
|
||||||
+ "</p>"
|
|
||||||
"<p class='muted meta'>Correlation id "
|
|
||||||
f"<code>{_escape(preview.correlation_id)}</code></p>"
|
|
||||||
"<details><summary>Allocator evidence</summary>"
|
|
||||||
f"<pre class='prompt-text'>{_escape(evidence)}</pre>"
|
|
||||||
"</details>"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _error_block(error: RequestError) -> str:
|
|
||||||
field = (
|
|
||||||
f"<p class='meta'>Field: <code>{_escape(error.field_name)}</code></p>"
|
|
||||||
if error.field_name
|
|
||||||
else ""
|
|
||||||
)
|
|
||||||
return (
|
|
||||||
"<h3>Request rejected</h3>"
|
|
||||||
f"<p><strong>{_escape(error.reason_code)}</strong> — "
|
|
||||||
f"{_escape(error.detail)}</p>{field}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def render_requests_page(
|
|
||||||
*,
|
|
||||||
preview: RequestPreview | None = None,
|
|
||||||
error: RequestError | None = None,
|
|
||||||
submitted: dict[str, Any] | None = None,
|
|
||||||
) -> str:
|
|
||||||
"""Render the request form, plus a preview or rejection when one exists."""
|
|
||||||
body = (
|
|
||||||
"<h2>Requests</h2>"
|
|
||||||
"<p>Submit a work request — desired role, issue or PR, and intent — "
|
|
||||||
"and see whether it would be authorized before anything is reserved. "
|
|
||||||
"Initiation goes through the allocator (#600/#613); this console never "
|
|
||||||
"self-selects work, never approves, and never merges.</p>"
|
|
||||||
+ _form(submitted)
|
|
||||||
+ (_error_block(error) if error is not None else "")
|
|
||||||
+ (_preview_block(preview) if preview is not None else "")
|
|
||||||
+ f"<p class='meta'><a href='{PREVIEW_API_PATH}'>Preview API</a> · "
|
|
||||||
"<a href='/api/console/security-model'>RBAC model</a></p>"
|
|
||||||
+ REQUEST_PAGE_STYLES
|
|
||||||
)
|
|
||||||
return render_page(title="Requests", body_html=body)
|
|
||||||
@@ -270,26 +270,47 @@ def _probe_error_card(snapshot: SystemHealthSnapshot) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _recovery_card() -> str:
|
def _recovery_card() -> str:
|
||||||
"""Sanctioned recovery pointers only — never a manual process kill (#630)."""
|
"""Sanctioned recovery controls & playbooks (#644, Phase 2)."""
|
||||||
return (
|
try:
|
||||||
"<section class='health-card'>"
|
from webui import console_recovery
|
||||||
"<h3>Recovery</h3>"
|
diag = console_recovery.diagnose_recovery()
|
||||||
"<p class='muted'>This dashboard is read-only. Restart and reload "
|
status_badge = f"<span class='status-pill {diag.status}'>{diag.status}</span>"
|
||||||
"controls arrive in Phase 2 (#642); until then recovery runs through "
|
playbook_lis = ""
|
||||||
"the sanctioned client reconnect / operator restart path.</p>"
|
for pb in diag.playbooks:
|
||||||
"<ul class='reasons'>"
|
elig = "eligible" if pb.eligible else "disabled"
|
||||||
"<li><a href='/runtime'>Runtime health</a> — active profile, workflow "
|
playbook_lis += (
|
||||||
"hashes, and shell health.</li>"
|
f"<li><strong>{pb.label}</strong> (<code>{pb.playbook_id}</code>) — "
|
||||||
"<li><a href='/sessions'>Runtime and sessions</a> — namespaces, session "
|
f"<span class='badge {elig}'>{elig}</span>: {pb.description} "
|
||||||
"rows, worktree bindings, and contamination markers (#641).</li>"
|
f"<em class='muted'>({pb.reason})</em></li>"
|
||||||
"<li>Reconnect the MCP client from the IDE, then re-run the blocked "
|
)
|
||||||
"cycle. Never kill the daemon process manually: unmanaged kills are "
|
reasons_html = ""
|
||||||
"recorded as runtime contamination (#630).</li>"
|
if diag.reasons:
|
||||||
"<li>See <code>docs/webui-local-dev.md</code> for the documented "
|
items = "".join(f"<li>{r}</li>" for r in diag.reasons)
|
||||||
"recovery sequence.</li>"
|
reasons_html = f"<ul class='reasons'>{items}</ul>"
|
||||||
"</ul>"
|
else:
|
||||||
"</section>"
|
reasons_html = "<p class='clean-note'>No recovery actions currently required. Control plane is healthy.</p>"
|
||||||
)
|
|
||||||
|
return (
|
||||||
|
"<section class='health-card recovery-card'>"
|
||||||
|
f"<h3>Sanctioned Recovery Controls (Phase 2 #644) {status_badge}</h3>"
|
||||||
|
"<p class='muted'>Guided recovery wizard: Diagnose → Preview → Confirm → Verify. "
|
||||||
|
"Reconnect the MCP client from the IDE, then re-run the blocked cycle. "
|
||||||
|
"Never kill the daemon process manually: unmanaged kills are recorded as runtime contamination (#630).</p>"
|
||||||
|
f"{reasons_html}"
|
||||||
|
"<h4>Available Recovery Playbooks</h4>"
|
||||||
|
f"<ul class='playbooks-list'>{playbook_lis}</ul>"
|
||||||
|
"<p class='meta'>APIs: <code>/api/v1/system/recovery/diagnose</code>, "
|
||||||
|
"<code>/api/v1/system/recovery/preview</code>, <code>/api/v1/system/recovery/apply</code>, "
|
||||||
|
"<code>/api/v1/system/recovery/verify</code>.</p>"
|
||||||
|
"</section>"
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
return (
|
||||||
|
"<section class='health-card'>"
|
||||||
|
"<h3>Sanctioned Recovery Controls (Phase 2 #644)</h3>"
|
||||||
|
f"<p class='error'>Recovery diagnostics unavailable: {exc}</p>"
|
||||||
|
"</section>"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def render_system_health_page(snapshot: SystemHealthSnapshot) -> str:
|
def render_system_health_page(snapshot: SystemHealthSnapshot) -> str:
|
||||||
|
|||||||
@@ -201,16 +201,6 @@ def _candidates_from_queue_snapshot(q_snap: QueueSnapshot) -> list[WorkCandidate
|
|||||||
return candidates
|
return candidates
|
||||||
|
|
||||||
|
|
||||||
def candidates_from_queue_snapshot(q_snap: QueueSnapshot) -> list[WorkCandidate]:
|
|
||||||
"""Public alias for :func:`_candidates_from_queue_snapshot` (#643).
|
|
||||||
|
|
||||||
The request-initiation service ranks the same candidate set this view
|
|
||||||
renders, so both must agree on how a queue row becomes a candidate. One
|
|
||||||
construction, two callers — not two that can drift apart.
|
|
||||||
"""
|
|
||||||
return _candidates_from_queue_snapshot(q_snap)
|
|
||||||
|
|
||||||
|
|
||||||
def _claim_lease_records(inventory: dict[str, Any] | None) -> list[dict[str, Any]]:
|
def _claim_lease_records(inventory: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||||
"""Normalize ``build_claim_inventory`` entries into lease records.
|
"""Normalize ``build_claim_inventory`` entries into lease records.
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user