sysadmin
a6a2243aad
Address review 443 on PR #616: mandatory operator-guide/runbook cross-links, LLM vs operator restart/relaunch split, and sanctioned post-merge parity staleness response (stop, report, operator reload, re-verify). Add docs tests enforcing F1–F3. Refs: #615
140 lines
5.3 KiB
Python
140 lines
5.3 KiB
Python
"""Documentation acceptance for the stable control runtime ADR (#615 / PR #616).
|
|
|
|
Enforces review 443 remediation:
|
|
|
|
* F1 — operator guide / runbooks cross-link the ADR (issue #615 AC2).
|
|
* F2 — LLM sessions are not instructed to kill/restart/relaunch MCP; process
|
|
restart is operator-owned; ADR is the authoritative split.
|
|
* F3 — routine post-merge master-parity staleness has a sanctioned response
|
|
(stop → report → operator reload → re-verify parity) and is not a full
|
|
promotion ledger requirement.
|
|
"""
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
ADR = (
|
|
REPO_ROOT
|
|
/ "docs"
|
|
/ "architecture"
|
|
/ "mcp-stable-control-runtime-policy-adr.md"
|
|
)
|
|
ADR_REL = "architecture/mcp-stable-control-runtime-policy-adr.md"
|
|
ADR_BASENAME = "mcp-stable-control-runtime-policy-adr.md"
|
|
|
|
CROSS_LINK_DOCS = (
|
|
REPO_ROOT / "docs" / "wiki" / "Operator-Guide.md",
|
|
REPO_ROOT / "docs" / "wiki" / "Runbooks.md",
|
|
REPO_ROOT / "docs" / "llm-workflow-runbooks.md",
|
|
)
|
|
|
|
|
|
def _read(path: Path) -> str:
|
|
assert path.is_file(), f"missing {path.relative_to(REPO_ROOT)}"
|
|
return path.read_text(encoding="utf-8")
|
|
|
|
|
|
def test_adr_exists_with_policy_core():
|
|
text = _read(ADR)
|
|
assert text.lstrip().startswith("#"), "ADR lacks a title"
|
|
assert "#615" in text
|
|
assert "stable control runtime" in text.lower()
|
|
assert "2.3" in text and "2.4" in text and "2.5" in text and "2.6" in text
|
|
|
|
|
|
def test_f1_operator_docs_cross_link_adr():
|
|
for path in CROSS_LINK_DOCS:
|
|
text = _read(path)
|
|
assert ADR_BASENAME in text, (
|
|
f"{path.relative_to(REPO_ROOT)} must cross-link {ADR_BASENAME} "
|
|
f"(issue #615 acceptance criterion 2)"
|
|
)
|
|
|
|
|
|
def test_f1_adr_lists_cross_links_as_acceptance_not_optional_tooling():
|
|
text = _read(ADR)
|
|
# Acceptance section must require guide/runbook cross-links.
|
|
assert "Operator guide / runbooks cross-link" in text or (
|
|
"operator guide" in text.lower() and "cross-link" in text.lower()
|
|
and "Acceptance" in text
|
|
)
|
|
# Cross-link must not remain only as optional tooling item #4.
|
|
optional = text.split("## 5. Implementation follow-ups", 1)[-1].split(
|
|
"## 6. Acceptance", 1
|
|
)[0]
|
|
assert "Operator Guide wiki cross-link to this ADR" not in optional, (
|
|
"ADR §5 must not list operator-guide cross-link as optional tooling"
|
|
)
|
|
assert "Not optional" in text or "must** cross-link" in text.lower() or (
|
|
"must cross-link" in text.lower()
|
|
)
|
|
|
|
|
|
def test_f2_runbook_fallback_is_operator_owned_not_llm_restart():
|
|
runbooks = _read(REPO_ROOT / "docs" / "llm-workflow-runbooks.md")
|
|
# Forbidden historical self-service instruction.
|
|
forbidden = (
|
|
"the LLM must relaunch or restart the client/MCP with the correct "
|
|
"profile environment variable before claiming or working on any tasks"
|
|
)
|
|
assert forbidden not in runbooks, (
|
|
"llm-workflow-runbooks must not instruct the LLM to relaunch/restart MCP"
|
|
)
|
|
assert "operator-owned" in runbooks.lower() or "Operator-owned" in runbooks
|
|
assert ADR_BASENAME in runbooks
|
|
|
|
|
|
def test_f2_workspace_rebind_does_not_require_llm_process_relaunch():
|
|
runbooks = _read(REPO_ROOT / "docs" / "llm-workflow-runbooks.md")
|
|
section = runbooks.split("### Safe reconnect / rebind procedure", 1)[-1]
|
|
section = section.split("## Safety notes", 1)[0]
|
|
# Must not tell the LLM alone to relaunch the MCP process as step 2.
|
|
assert "Reconnect or relaunch the correct namespace MCP server from the intended" not in section
|
|
assert "Operator-owned" in section or "operator" in section.lower()
|
|
assert "worktree_path" in section
|
|
collapsed = " ".join(section.lower().split())
|
|
assert "client reconnect" in collapsed
|
|
|
|
|
|
def test_f2_adr_forbids_llm_restart_and_supersedes_self_service_relaunch():
|
|
text = _read(ADR)
|
|
lower = text.lower()
|
|
assert "must not" in lower and "restart" in lower
|
|
assert "operator" in lower and "reload" in lower
|
|
assert "supersedes" in lower
|
|
assert "llm" in lower
|
|
|
|
|
|
def test_f3_adr_defines_post_merge_parity_staleness_response():
|
|
text = _read(ADR)
|
|
lower = text.lower()
|
|
assert "2.6" in text
|
|
assert "post-merge" in lower or "post merge" in lower
|
|
assert "parity" in lower and "stale" in lower
|
|
# Sanctioned steps: stop, report, operator reload, re-verify.
|
|
assert "stop" in lower
|
|
assert "report" in lower
|
|
assert "operator" in lower
|
|
assert "parity is verified" in lower or "re-verif" in lower or (
|
|
"startup/current-head" in lower
|
|
)
|
|
# Not a full promotion ledger for routine reload.
|
|
assert "not a §2.4 promotion" in lower or "not a section 2.4 promotion" in lower or (
|
|
"not a §2.4" in text or "Not a §2.4 promotion" in text
|
|
)
|
|
# Stale parity listed among unhealthy triggers.
|
|
assert "master parity is stale" in lower or "stale master parity" in lower
|
|
|
|
|
|
def test_f3_adr_forbids_llm_bypass_of_parity_gate():
|
|
text = _read(ADR)
|
|
lower = text.lower()
|
|
assert "bypass" in lower or "self-reset" in lower or "self-service" in lower
|
|
assert "parity" in lower
|
|
|
|
|
|
def test_cross_links_do_not_embed_secrets_or_raw_hosts_in_wiki_snippets():
|
|
for path in CROSS_LINK_DOCS:
|
|
text = _read(path)
|
|
for marker in ("ghp_", "BEGIN PRIVATE KEY", "Authorization: Bearer"):
|
|
assert marker not in text, f"{path} contains {marker!r}"
|