sysadminandClaude Opus 4.8
f20e0fc338
Reviewer runs that observe a validation failure and later rerun to green must report the failure and its investigation; a later pass never erases the earlier failure: - New reviewer_validation_failure_history.py verifier: reports must carry a Validation failure history section naming every observed failure with command, failing test or error, suspected cause, reproduced status, baseline master status, and PR-caused evidence; rerun-pass claims must explain what changed between runs, whether environmental state (such as stale /tmp files) was cleaned, and why the pass is trustworthy; an unexplained transient failure cannot be reported as a bare pass and must use the status 'passed after transient failure investigation' (exported as TRANSIENT_INVESTIGATION_STATUS). - Re-export from review_proofs and wire into build_final_report as a downgrade check. - Review-merge final-report schema gains the Validation failure history template section; contract test locks the markers and an export test locks the re-export. - 12 tests covering the issue's scenarios: failing-then-passing rerun, baseline-equivalent failure, /tmp state contamination, unexplained transient failure, and a report omitting an earlier failure. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
293 lines
9.6 KiB
Python
293 lines
9.6 KiB
Python
"""Doc-contract checks for task-specific LLM workflow split (#333, #334)."""
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
SKILL_DIR = REPO_ROOT / "skills" / "llm-project-workflow"
|
|
SKILL = (SKILL_DIR / "SKILL.md").read_text(encoding="utf-8")
|
|
|
|
|
|
def test_skill_md_exists():
|
|
assert SKILL_DIR.joinpath("SKILL.md").is_file()
|
|
|
|
|
|
def test_all_workflow_files_exist():
|
|
for name in (
|
|
"review-merge-pr.md",
|
|
"reconcile-landed-pr.md",
|
|
"create-issue.md",
|
|
"work-issue.md",
|
|
"pr-queue-cleanup.md",
|
|
):
|
|
assert (SKILL_DIR / "workflows" / name).is_file(), name
|
|
|
|
|
|
def test_skill_references_all_workflow_files():
|
|
for name in (
|
|
"workflows/review-merge-pr.md",
|
|
"workflows/reconcile-landed-pr.md",
|
|
"workflows/create-issue.md",
|
|
"workflows/work-issue.md",
|
|
"workflows/pr-queue-cleanup.md",
|
|
):
|
|
assert name in SKILL
|
|
|
|
|
|
def test_skill_contains_mode_isolation_language():
|
|
assert "## Mode isolation" in SKILL
|
|
assert "review-merge-pr" in SKILL
|
|
assert "reconcile-landed-pr" in SKILL
|
|
assert "create-issue" in SKILL
|
|
assert "work-issue" in SKILL
|
|
assert "pr-queue-cleanup" in SKILL
|
|
assert "Do not mix modes" in SKILL or "do not mix modes" in SKILL.lower()
|
|
|
|
|
|
def test_skill_is_router_not_monolithic_review_body():
|
|
assert "This skill is a **router**" in SKILL or "router" in SKILL.lower()
|
|
assert "## F. Review workflow" not in SKILL
|
|
assert "gitea_mark_final_review_decision" not in SKILL
|
|
assert "gitea_submit_pr_review" not in SKILL
|
|
|
|
|
|
def test_skill_still_declares_controller_handoff_contract():
|
|
assert "## Controller Handoff" in SKILL
|
|
assert "assess_controller_handoff" in SKILL
|
|
assert "## Global LLM Worktree Rule" in SKILL
|
|
|
|
|
|
def test_review_merge_workflow_contract():
|
|
text = (SKILL_DIR / "workflows" / "review-merge-pr.md").read_text(encoding="utf-8")
|
|
assert "canonical: true" in text
|
|
assert "## 0. Load the canonical workflow first" in text
|
|
assert "## 26A. Terminal review mutation hard-stop" in text
|
|
assert "## 11A. Skipped PRs are read-only" in text
|
|
assert "## 35. Duplicate request-changes prevention" in text
|
|
assert "## 37. Controller handoff schema" in text
|
|
assert "INVENTORY_PAGINATION_UNPROVEN" in text
|
|
assert "ALREADY_LANDED_RECONCILE_REQUIRED" in text
|
|
|
|
|
|
def test_review_merge_final_report_schema_exists():
|
|
path = SKILL_DIR / "schemas" / "review-merge-final-report.md"
|
|
text = path.read_text(encoding="utf-8")
|
|
assert "Controller Handoff" in text
|
|
assert "Candidate head SHA:" in text
|
|
assert "Terminal review mutation:" in text
|
|
assert "Workspace mutations" in text
|
|
assert "### Validation failure history (#396)" in text
|
|
assert "passed after transient failure investigation" in text
|
|
|
|
|
|
def test_validation_failure_history_verifier_exported():
|
|
from review_proofs import assess_validation_failure_history_report
|
|
|
|
assert callable(assess_validation_failure_history_report)
|
|
|
|
|
|
def test_reconcile_landed_workflow_contract():
|
|
text = (SKILL_DIR / "workflows" / "reconcile-landed-pr.md").read_text(encoding="utf-8")
|
|
assert "canonical: true" in text
|
|
assert "Do not review or merge normal PRs" in text
|
|
assert "Already-landed proof" in text
|
|
# Issue #302: partial reconciliation policy must stay documented.
|
|
assert "## 12A. Partial reconciliation policy (#302)" in text
|
|
assert "comment-then-stop" in text
|
|
assert "PARTIAL_RECONCILE_COMMENT_THEN_STOP" in text
|
|
assert "RECOVERY_HANDOFF_ONLY" in text
|
|
assert "resolve_partial_reconciliation_plan" in text
|
|
|
|
|
|
def test_create_issue_workflow_contract():
|
|
text = (SKILL_DIR / "workflows" / "create-issue.md").read_text(encoding="utf-8")
|
|
assert "canonical: true" in text
|
|
assert "## 9. Duplicate search before mutation" in text
|
|
|
|
|
|
def test_work_issue_workflow_contract():
|
|
text = (SKILL_DIR / "workflows" / "work-issue.md").read_text(encoding="utf-8")
|
|
assert "canonical: true" in text
|
|
assert "Do not merge your own PR" in text
|
|
assert "## 21. PR creation rules" in text
|
|
|
|
|
|
def test_pr_queue_cleanup_workflow_contract():
|
|
text = (SKILL_DIR / "workflows" / "pr-queue-cleanup.md").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
assert "canonical: true" in text
|
|
assert "exactly one" in text
|
|
assert "check_cleanup_task_allowed" in text
|
|
assert "resolve_cleanup_run_state" in text
|
|
assert "Next suggested PR" in text
|
|
|
|
|
|
def test_final_report_schemas_exist():
|
|
for name in (
|
|
"review-merge-final-report.md",
|
|
"reconcile-landed-final-report.md",
|
|
"create-issue-final-report.md",
|
|
"work-issue-final-report.md",
|
|
"pr-queue-cleanup-final-report.md",
|
|
):
|
|
assert (SKILL_DIR / "schemas" / name).is_file(), name
|
|
|
|
|
|
def test_review_pr_template_references_workflow():
|
|
text = (SKILL_DIR / "templates" / "review-pr.md").read_text(encoding="utf-8")
|
|
assert "workflows/review-merge-pr.md" in text
|
|
|
|
|
|
def test_merge_pr_template_references_workflow():
|
|
text = (SKILL_DIR / "templates" / "merge-pr.md").read_text(encoding="utf-8")
|
|
assert "workflows/review-merge-pr.md" in text
|
|
|
|
|
|
def test_start_issue_template_references_work_issue_workflow():
|
|
text = (SKILL_DIR / "templates" / "start-issue.md").read_text(encoding="utf-8")
|
|
assert "workflows/work-issue.md" in text
|
|
|
|
|
|
def test_reconcile_skill_registered_in_mcp_server():
|
|
from mcp_server import _PROJECT_SKILLS
|
|
|
|
assert "gitea-reconcile-landed-pr" in _PROJECT_SKILLS
|
|
steps = _PROJECT_SKILLS["gitea-reconcile-landed-pr"]["steps"]
|
|
joined = " ".join(steps).lower()
|
|
assert "gitea_scan_already_landed_open_prs" in joined
|
|
assert "gitea_assess_already_landed_reconciliation" in joined
|
|
|
|
|
|
def test_pr_queue_cleanup_template_references_workflow():
|
|
text = (SKILL_DIR / "templates" / "pr-queue-cleanup.md").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
assert "workflows/pr-queue-cleanup.md" in text
|
|
assert "pr_queue_cleanup" in text
|
|
|
|
|
|
def test_pr_queue_cleanup_verifier_exported():
|
|
from review_proofs import assess_pr_queue_cleanup_report
|
|
|
|
assert callable(assess_pr_queue_cleanup_report)
|
|
|
|
|
|
def test_reviewer_fallback_verifier_exported():
|
|
from review_proofs import assess_reviewer_fallback_report
|
|
|
|
assert callable(assess_reviewer_fallback_report)
|
|
|
|
|
|
def test_final_report_validator_exported():
|
|
from review_proofs import assess_final_report_validator
|
|
|
|
assert callable(assess_final_report_validator)
|
|
|
|
|
|
def test_review_final_report_schema_verifier_exported():
|
|
from review_proofs import assess_review_final_report_schema
|
|
|
|
assert callable(assess_review_final_report_schema)
|
|
|
|
|
|
def test_create_issue_final_report_verifier_exported():
|
|
from review_proofs import assess_create_issue_final_report
|
|
|
|
assert callable(assess_create_issue_final_report)
|
|
|
|
|
|
def test_merge_simulation_verifier_exported():
|
|
from review_proofs import assess_merge_simulation_report
|
|
|
|
assert callable(assess_merge_simulation_report)
|
|
|
|
|
|
def test_validation_integrity_verifier_exported():
|
|
from review_proofs import assess_validation_integrity_report
|
|
|
|
assert callable(assess_validation_integrity_report)
|
|
|
|
|
|
def test_prior_blocker_skip_verifier_exported():
|
|
from review_proofs import assess_prior_blocker_skip_proof
|
|
|
|
assert callable(assess_prior_blocker_skip_proof)
|
|
|
|
|
|
def test_non_mergeable_skip_verifier_exported():
|
|
from review_proofs import assess_non_mergeable_skip_proof
|
|
|
|
assert callable(assess_non_mergeable_skip_proof)
|
|
|
|
|
|
def test_validation_worktree_edit_verifier_exported():
|
|
from review_proofs import assess_validation_worktree_edit_report
|
|
|
|
assert callable(assess_validation_worktree_edit_report)
|
|
|
|
|
|
def test_reconcile_inventory_verifier_exported():
|
|
from review_proofs import assess_reconcile_inventory_report
|
|
|
|
assert callable(assess_reconcile_inventory_report)
|
|
|
|
|
|
def test_reconcile_linked_issue_verifier_exported():
|
|
from review_proofs import assess_reconcile_linked_issue_report
|
|
|
|
assert callable(assess_reconcile_linked_issue_report)
|
|
|
|
|
|
def test_already_landed_handoff_verifier_exported():
|
|
from review_proofs import assess_already_landed_handoff_report
|
|
|
|
assert callable(assess_already_landed_handoff_report)
|
|
|
|
|
|
def test_inventory_worktree_verifier_exported():
|
|
from review_proofs import assess_inventory_worktree_report
|
|
|
|
assert callable(assess_inventory_worktree_report)
|
|
|
|
|
|
def test_infra_stop_handoff_verifier_exported():
|
|
from review_proofs import assess_infra_stop_handoff_report
|
|
|
|
assert callable(assess_infra_stop_handoff_report)
|
|
|
|
|
|
def test_mutation_categories_verifier_exported():
|
|
from review_proofs import assess_mutation_categories_report
|
|
|
|
assert callable(assess_mutation_categories_report)
|
|
|
|
|
|
def test_worktree_ownership_verifier_exported():
|
|
from review_proofs import assess_worktree_ownership_report
|
|
|
|
assert callable(assess_worktree_ownership_report)
|
|
|
|
|
|
def test_already_landed_classification_verifier_exported():
|
|
from review_proofs import assess_already_landed_classification_report
|
|
|
|
assert callable(assess_already_landed_classification_report)
|
|
|
|
|
|
def test_pr_queue_cleanup_verifier_exported():
|
|
from review_proofs import assess_pr_queue_cleanup_report
|
|
|
|
assert callable(assess_pr_queue_cleanup_report)
|
|
|
|
|
|
def test_pr_queue_cleanup_workflow_contract():
|
|
text = (SKILL_DIR / "workflows" / "pr-queue-cleanup.md").read_text(encoding="utf-8")
|
|
assert "canonical: true" in text
|
|
assert "task_mode: pr-queue-cleanup" in text
|
|
assert "## 1. Mode isolation" in text
|
|
assert "## 4. Terminal mutation chain" in text
|
|
assert "## 5. Fresh run per PR" in text
|
|
assert "exactly one" in text.lower()
|
|
assert "review-merge-pr.md" in text
|
|
assert "Next suggested PR" in text
|
|
assert "schemas/pr-queue-cleanup-final-report.md" in text
|