diff --git a/review_proofs.py b/review_proofs.py index be71582..c820d08 100644 --- a/review_proofs.py +++ b/review_proofs.py @@ -1571,8 +1571,55 @@ HANDOFF_ROLE_FIELDS = { )), ("Related issues", ("related issues",)), ), + "create_issue": ( + ("Active profile", ("active profile",)), + ("Runtime context", ("runtime context",)), + ("Requested issue task", ("requested issue task",)), + ("Workflow source", ("workflow source",)), + ("Capability proof", ("capability proof",)), + ("Duplicate search terms", ("duplicate search terms",)), + ("Duplicate search pagination proof", ( + "duplicate search pagination proof", + )), + ("Duplicates found", ("duplicates found",)), + ("Issues created", ("issues created",)), + ("Issues commented", ("issues commented",)), + ("Issues edited", ("issues edited",)), + ("Issues skipped as duplicates", ( + "issues skipped as duplicates", + )), + ("File edits by issue creator", ("file edits by issue creator",)), + ("Safety statement", ("safety statement",)), + ), } +CREATE_ISSUE_WORKFLOW_MARKERS = ( + "workflows/create-issue.md", + "create-issue.md", +) + +CREATE_ISSUE_TASK_MARKERS = ( + "create-issue", + "create issue", +) + +CREATE_ISSUE_FORBIDDEN_CROSS_MODE_FIELDS = ( + ("Selected issue", ("selected issue",)), + ("Branch name", ("branch name",)), + ("PR number", ("pr number",)), + ("PR URL", ("pr url",)), + ("Commit SHA", ("commit sha",)), + ("Push result", ("push result",)), + ("Baseline comparison", ("baseline comparison",)), + ("Selected PR", ("selected pr",)), + ("Review decision", ("review decision",)), + ("Merge result", ("merge result",)), + ("Merge preflight", ("merge preflight",)), + ("Already-landed gate", ("already-landed gate",)), + ("Pinned reviewed head", ("pinned reviewed head",)), + ("Terminal review mutation", ("terminal review mutation",)), +) + def _handoff_section_lines(report_text): """Return the lines of the Controller Handoff section, or None.""" @@ -1623,6 +1670,11 @@ def assess_controller_handoff(report_text, role=None, local_edits=False): fields_dict[label] = v.strip() required = list(HANDOFF_BASE_FIELDS) + if role == "create_issue": + required = [ + field for field in required + if field[0] not in ("Workspace mutations", "Mutations") + ] required.extend(HANDOFF_ROLE_FIELDS.get(role or "", ())) missing = [] @@ -2647,6 +2699,141 @@ def assess_issue_filing_duplicate_summary( } +def assess_create_issue_workflow_source(report_text: str) -> dict: + """#337: create-issue reports must cite the canonical workflow file.""" + lower = (report_text or "").lower() + reasons = [] + if not any(marker in lower for marker in CREATE_ISSUE_WORKFLOW_MARKERS): + reasons.append( + "create-issue report missing workflow source " + "(workflows/create-issue.md)" + ) + if not any(marker in lower for marker in CREATE_ISSUE_TASK_MARKERS): + reasons.append( + "create-issue report missing task mode declaration " + "(create-issue)" + ) + return { + "complete": not reasons, + "downgraded": bool(reasons), + "reasons": reasons, + } + + +def assess_create_issue_mode_isolation(report_text: str) -> dict: + """#337: reject work-issue/review-merge handoff fields in create-issue runs.""" + section = _handoff_section_lines(report_text) + reasons = [] + if section is None: + return { + "complete": True, + "downgraded": False, + "reasons": [], + } + + labels = [] + for line in section: + stripped = line.strip().lstrip("-*").strip() + if ":" in stripped: + labels.append(stripped.split(":", 1)[0].strip().lower()) + + for name, aliases in CREATE_ISSUE_FORBIDDEN_CROSS_MODE_FIELDS: + if any( + label.startswith(alias) + for label in labels + for alias in aliases + ): + reasons.append( + f"create-issue handoff must not include cross-mode field " + f"'{name}'" + ) + + legacy_workspace = any( + label.startswith("workspace mutations") for label in labels + ) + precise_categories = any( + label.startswith("file edits by issue creator") + for label in labels + ) + if legacy_workspace and not precise_categories: + reasons.append( + "create-issue handoff must use precise mutation categories " + "instead of legacy 'Workspace mutations'" + ) + + return { + "complete": not reasons, + "downgraded": bool(reasons), + "reasons": reasons, + } + + +def assess_create_issue_final_report( + report_text: str, + *, + issue_number: int | None = None, + issue_title: str | None = None, + action: str = "created", + mutations: list[str] | None = None, + resolved_capabilities: dict | None = None, + issues_searched: int | None = None, + closest_matches: list[dict] | None = None, + duplicate_result: dict | None = None, + performed_mutations: list[str] | None = None, +) -> dict: + """#337: composite verifier for create-issue final reports.""" + checks = { + "workflow_source": assess_create_issue_workflow_source(report_text), + "mode_isolation": assess_create_issue_mode_isolation(report_text), + "controller_handoff": assess_controller_handoff( + report_text, role="create_issue" + ), + "issue_reference": assess_issue_filing_issue_reference( + report_text, + issue_number=issue_number, + issue_title=issue_title, + action=action, + ), + "mutation_capability": assess_issue_filing_mutation_capability( + report_text, + mutations or performed_mutations, + resolved_capabilities, + ), + "mutation_scope": assess_issue_filing_mutation_scope( + report_text, + performed_mutations=performed_mutations or mutations, + ), + "duplicate_summary": assess_issue_filing_duplicate_summary( + report_text, + issues_searched=issues_searched, + closest_matches=closest_matches, + duplicate_result=duplicate_result, + ), + } + + reasons = [] + downgraded = False + for name, result in checks.items(): + verdict = result.get("verdict") + if verdict in ("missing", "incomplete"): + downgraded = True + reasons.extend(result.get("reasons") or []) + elif result.get("downgraded") or not result.get("complete", True): + downgraded = True + reasons.extend( + f"{name}: {r}" for r in (result.get("reasons") or []) + ) + + grade = "A" if not downgraded else "downgraded" + return { + "grade": grade, + "downgraded": downgraded, + "checks": checks, + "reasons": reasons, + "complete": not downgraded, + } + + def assess_issue_filing_final_report( report_text: str, *, diff --git a/tests/test_llm_workflow_split.py b/tests/test_llm_workflow_split.py index 327b17d..3e23692 100644 --- a/tests/test_llm_workflow_split.py +++ b/tests/test_llm_workflow_split.py @@ -97,4 +97,10 @@ def test_merge_pr_template_references_workflow(): def test_start_issue_template_references_work_issue_workflow(): text = (SKILL_DIR / "templates" / "start-issue.md").read_text(encoding="utf-8") - assert "workflows/work-issue.md" in text \ No newline at end of file + assert "workflows/work-issue.md" in text + + +def test_create_issue_final_report_verifier_exported(): + from review_proofs import assess_create_issue_final_report + + assert callable(assess_create_issue_final_report) \ No newline at end of file diff --git a/tests/test_review_proofs.py b/tests/test_review_proofs.py index 7166cca..b4683dc 100644 --- a/tests/test_review_proofs.py +++ b/tests/test_review_proofs.py @@ -29,6 +29,9 @@ from review_proofs import ( # noqa: E402 assess_contradictory_no_pr_claim, assess_continuation_mode_report, assess_controller_handoff, + assess_create_issue_final_report, + assess_create_issue_mode_isolation, + assess_create_issue_workflow_source, assess_edited_pr_inventory_coverage, assess_empty_queue_report, assess_fresh_issue_selection, @@ -2042,5 +2045,120 @@ class TestIssueFilingFinalReport(unittest.TestCase): self.assertTrue(result["downgraded"]) +class TestCreateIssueFinalReport(unittest.TestCase): + """#337: create-issue final-report verifier coverage.""" + + ISSUE_TITLE = "Extract create-issue workflow from monolithic LLM project skill" + CAPABILITIES = { + "create_issue": { + "requested_task": "create_issue", + "required_operation_permission": "gitea.issue.create", + "allowed_in_current_session": True, + }, + } + CLOSEST = [{"number": 333, "title": "Split monolithic workflow", "state": "closed"}] + + def _good_report(self): + return "\n".join([ + "Task mode: create-issue", + "Workflow source: skills/llm-project-workflow/workflows/create-issue.md", + f"Updated issue #337 — `{self.ISSUE_TITLE}` (remaining verifier AC).", + "Duplicate check: searched 54 open issues.", + "Closest existing: #333 — Split monolithic workflow (partial overlap).", + "Why new issue justified: dedicated create-issue extraction.", + "Only mutation: issue creation (gitea.issue.create).", + "Confirm no labels, comments, PRs, reviews, merges, or closes.", + "## Controller Handoff", + "- Task: create-issue", + "- Repo: Scaled-Tech-Consulting/Gitea-Tools", + "- Role: author", + "- Identity: jcwalker3 / prgs-author", + "- Active profile: prgs-author", + "- Runtime context: prgs-author on prgs", + "- Requested issue task: extract create-issue workflow verifier", + "- Workflow source: workflows/create-issue.md @ fd396df", + "- Capability proof: create_issue allowed (gitea.issue.create)", + "- Duplicate search terms: create-issue, workflow extract", + "- Duplicate search pagination proof: 54 open issues < limit 500", + "- Duplicates found: #333 partial only", + "- Issues created: none (implementation PR for remaining AC)", + "- Issues commented: none", + "- Issues edited: none", + "- Issues skipped as duplicates: none", + "- File edits by issue creator: review_proofs.py, tests", + "- Worktree/index mutations: feat/issue-337 worktree created", + "- Git ref mutations: fetch prgs/master", + "- MCP/Gitea mutations: gitea_mark_issue start on #337", + "- Issue mutations: claimed #337", + "- Label/assignment/milestone mutations: none", + "- External-state mutations: none", + "- Read-only diagnostics: whoami, list issues", + "- Blockers: none", + "- Current status: implementing verifier", + "- Safe next action: open PR", + "- Safety statement: no review/merge/repo main-checkout edits", + "- Issue/PR: #337", + "- Branch/SHA: feat/issue-337-create-issue-verifier", + "- Files changed: review_proofs.py, tests", + "- Validation: pytest tests/test_review_proofs.py", + "- Mutations: create_issue capability resolved; mark_issue only", + "- Next: open PR", + "- Safety: no review/merge", + ]) + + def test_complete_create_issue_report_earns_a(self): + result = assess_create_issue_final_report( + self._good_report(), + issue_number=337, + issue_title=self.ISSUE_TITLE, + action="updated", + mutations=["create_issue"], + resolved_capabilities=self.CAPABILITIES, + issues_searched=54, + closest_matches=self.CLOSEST, + performed_mutations=["create_issue"], + ) + self.assertEqual(result["grade"], "A") + self.assertFalse(result["downgraded"]) + + def test_missing_workflow_source_downgrades(self): + report = self._good_report().replace("workflows/create-issue.md", "SKILL.md only") + result = assess_create_issue_workflow_source(report) + self.assertTrue(result["downgraded"]) + + def test_work_issue_field_in_handoff_downgrades(self): + report = self._good_report().replace( + "- Safety statement:", + "- Selected issue: #123\n- Safety statement:", + ) + result = assess_create_issue_mode_isolation(report) + self.assertTrue(result["downgraded"]) + self.assertTrue( + any("Selected issue" in r for r in result["reasons"]) + ) + + def test_review_merge_field_in_handoff_downgrades(self): + report = self._good_report().replace( + "- Safety statement:", + "- Review decision: APPROVE\n- Safety statement:", + ) + result = assess_create_issue_mode_isolation(report) + self.assertTrue(result["downgraded"]) + + def test_legacy_workspace_mutations_without_precise_categories_downgrades(self): + report = self._good_report().replace( + "- File edits by issue creator:", + "- Workspace mutations: none\n- File edits by issue creator:", + ) + lines = report.splitlines() + cleaned = [ + line for line in lines + if not line.strip().startswith("- File edits by issue creator:") + ] + report = "\n".join(cleaned) + result = assess_create_issue_mode_isolation(report) + self.assertTrue(result["downgraded"]) + + if __name__ == "__main__": unittest.main()