feat: add create-issue final-report verifier (#337)
Extend review_proofs with assess_create_issue_final_report to enforce workflow-source proof, create-issue handoff schema fields, and rejection of work-issue/review-merge cross-mode handoff fields. Closes #337
This commit is contained in:
@@ -97,4 +97,10 @@ def test_merge_pr_template_references_workflow():
|
||||
|
||||
def test_start_issue_template_references_work_issue_workflow():
|
||||
text = (SKILL_DIR / "templates" / "start-issue.md").read_text(encoding="utf-8")
|
||||
assert "workflows/work-issue.md" in text
|
||||
assert "workflows/work-issue.md" in text
|
||||
|
||||
|
||||
def test_create_issue_final_report_verifier_exported():
|
||||
from review_proofs import assess_create_issue_final_report
|
||||
|
||||
assert callable(assess_create_issue_final_report)
|
||||
@@ -29,6 +29,9 @@ from review_proofs import ( # noqa: E402
|
||||
assess_contradictory_no_pr_claim,
|
||||
assess_continuation_mode_report,
|
||||
assess_controller_handoff,
|
||||
assess_create_issue_final_report,
|
||||
assess_create_issue_mode_isolation,
|
||||
assess_create_issue_workflow_source,
|
||||
assess_edited_pr_inventory_coverage,
|
||||
assess_empty_queue_report,
|
||||
assess_fresh_issue_selection,
|
||||
@@ -2042,5 +2045,120 @@ class TestIssueFilingFinalReport(unittest.TestCase):
|
||||
self.assertTrue(result["downgraded"])
|
||||
|
||||
|
||||
class TestCreateIssueFinalReport(unittest.TestCase):
|
||||
"""#337: create-issue final-report verifier coverage."""
|
||||
|
||||
ISSUE_TITLE = "Extract create-issue workflow from monolithic LLM project skill"
|
||||
CAPABILITIES = {
|
||||
"create_issue": {
|
||||
"requested_task": "create_issue",
|
||||
"required_operation_permission": "gitea.issue.create",
|
||||
"allowed_in_current_session": True,
|
||||
},
|
||||
}
|
||||
CLOSEST = [{"number": 333, "title": "Split monolithic workflow", "state": "closed"}]
|
||||
|
||||
def _good_report(self):
|
||||
return "\n".join([
|
||||
"Task mode: create-issue",
|
||||
"Workflow source: skills/llm-project-workflow/workflows/create-issue.md",
|
||||
f"Updated issue #337 — `{self.ISSUE_TITLE}` (remaining verifier AC).",
|
||||
"Duplicate check: searched 54 open issues.",
|
||||
"Closest existing: #333 — Split monolithic workflow (partial overlap).",
|
||||
"Why new issue justified: dedicated create-issue extraction.",
|
||||
"Only mutation: issue creation (gitea.issue.create).",
|
||||
"Confirm no labels, comments, PRs, reviews, merges, or closes.",
|
||||
"## Controller Handoff",
|
||||
"- Task: create-issue",
|
||||
"- Repo: Scaled-Tech-Consulting/Gitea-Tools",
|
||||
"- Role: author",
|
||||
"- Identity: jcwalker3 / prgs-author",
|
||||
"- Active profile: prgs-author",
|
||||
"- Runtime context: prgs-author on prgs",
|
||||
"- Requested issue task: extract create-issue workflow verifier",
|
||||
"- Workflow source: workflows/create-issue.md @ fd396df",
|
||||
"- Capability proof: create_issue allowed (gitea.issue.create)",
|
||||
"- Duplicate search terms: create-issue, workflow extract",
|
||||
"- Duplicate search pagination proof: 54 open issues < limit 500",
|
||||
"- Duplicates found: #333 partial only",
|
||||
"- Issues created: none (implementation PR for remaining AC)",
|
||||
"- Issues commented: none",
|
||||
"- Issues edited: none",
|
||||
"- Issues skipped as duplicates: none",
|
||||
"- File edits by issue creator: review_proofs.py, tests",
|
||||
"- Worktree/index mutations: feat/issue-337 worktree created",
|
||||
"- Git ref mutations: fetch prgs/master",
|
||||
"- MCP/Gitea mutations: gitea_mark_issue start on #337",
|
||||
"- Issue mutations: claimed #337",
|
||||
"- Label/assignment/milestone mutations: none",
|
||||
"- External-state mutations: none",
|
||||
"- Read-only diagnostics: whoami, list issues",
|
||||
"- Blockers: none",
|
||||
"- Current status: implementing verifier",
|
||||
"- Safe next action: open PR",
|
||||
"- Safety statement: no review/merge/repo main-checkout edits",
|
||||
"- Issue/PR: #337",
|
||||
"- Branch/SHA: feat/issue-337-create-issue-verifier",
|
||||
"- Files changed: review_proofs.py, tests",
|
||||
"- Validation: pytest tests/test_review_proofs.py",
|
||||
"- Mutations: create_issue capability resolved; mark_issue only",
|
||||
"- Next: open PR",
|
||||
"- Safety: no review/merge",
|
||||
])
|
||||
|
||||
def test_complete_create_issue_report_earns_a(self):
|
||||
result = assess_create_issue_final_report(
|
||||
self._good_report(),
|
||||
issue_number=337,
|
||||
issue_title=self.ISSUE_TITLE,
|
||||
action="updated",
|
||||
mutations=["create_issue"],
|
||||
resolved_capabilities=self.CAPABILITIES,
|
||||
issues_searched=54,
|
||||
closest_matches=self.CLOSEST,
|
||||
performed_mutations=["create_issue"],
|
||||
)
|
||||
self.assertEqual(result["grade"], "A")
|
||||
self.assertFalse(result["downgraded"])
|
||||
|
||||
def test_missing_workflow_source_downgrades(self):
|
||||
report = self._good_report().replace("workflows/create-issue.md", "SKILL.md only")
|
||||
result = assess_create_issue_workflow_source(report)
|
||||
self.assertTrue(result["downgraded"])
|
||||
|
||||
def test_work_issue_field_in_handoff_downgrades(self):
|
||||
report = self._good_report().replace(
|
||||
"- Safety statement:",
|
||||
"- Selected issue: #123\n- Safety statement:",
|
||||
)
|
||||
result = assess_create_issue_mode_isolation(report)
|
||||
self.assertTrue(result["downgraded"])
|
||||
self.assertTrue(
|
||||
any("Selected issue" in r for r in result["reasons"])
|
||||
)
|
||||
|
||||
def test_review_merge_field_in_handoff_downgrades(self):
|
||||
report = self._good_report().replace(
|
||||
"- Safety statement:",
|
||||
"- Review decision: APPROVE\n- Safety statement:",
|
||||
)
|
||||
result = assess_create_issue_mode_isolation(report)
|
||||
self.assertTrue(result["downgraded"])
|
||||
|
||||
def test_legacy_workspace_mutations_without_precise_categories_downgrades(self):
|
||||
report = self._good_report().replace(
|
||||
"- File edits by issue creator:",
|
||||
"- Workspace mutations: none\n- File edits by issue creator:",
|
||||
)
|
||||
lines = report.splitlines()
|
||||
cleaned = [
|
||||
line for line in lines
|
||||
if not line.strip().startswith("- File edits by issue creator:")
|
||||
]
|
||||
report = "\n".join(cleaned)
|
||||
result = assess_create_issue_mode_isolation(report)
|
||||
self.assertTrue(result["downgraded"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user