Scaled-Tech-Consulting/Gitea-Tools
2
0
Fork 0
Code Issues 73 Pull Requests 9 Actions Packages Projects Releases Wiki Activity

feat: add create-issue final-report verifier (#337)

Browse Source 
Extend review_proofs with assess_create_issue_final_report to enforce
workflow-source proof, create-issue handoff schema fields, and rejection
of work-issue/review-merge cross-mode handoff fields.

Closes #337
This commit is contained in:
sysadmin
2026-07-07 04:23:45 -04:00
parent fd396df334
commit 14f5c865b3
3 changed files with 312 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
review_proofs.py
+187
View File
@@ -1571,8 +1571,55 @@ HANDOFF_ROLE_FIELDS = {
)),
("Related issues", ("related issues",)),
),
"create_issue": (
("Active profile", ("active profile",)),
("Runtime context", ("runtime context",)),
("Requested issue task", ("requested issue task",)),
("Workflow source", ("workflow source",)),
("Capability proof", ("capability proof",)),
("Duplicate search terms", ("duplicate search terms",)),
("Duplicate search pagination proof", (
"duplicate search pagination proof",
)),
("Duplicates found", ("duplicates found",)),
("Issues created", ("issues created",)),
("Issues commented", ("issues commented",)),
("Issues edited", ("issues edited",)),
("Issues skipped as duplicates", (
"issues skipped as duplicates",
)),
("File edits by issue creator", ("file edits by issue creator",)),
("Safety statement", ("safety statement",)),
),
}
CREATE_ISSUE_WORKFLOW_MARKERS = (
"workflows/create-issue.md",
"create-issue.md",
)
CREATE_ISSUE_TASK_MARKERS = (
"create-issue",
"create issue",
)
CREATE_ISSUE_FORBIDDEN_CROSS_MODE_FIELDS = (
("Selected issue", ("selected issue",)),
("Branch name", ("branch name",)),
("PR number", ("pr number",)),
("PR URL", ("pr url",)),
("Commit SHA", ("commit sha",)),
("Push result", ("push result",)),
("Baseline comparison", ("baseline comparison",)),
("Selected PR", ("selected pr",)),
("Review decision", ("review decision",)),
("Merge result", ("merge result",)),
("Merge preflight", ("merge preflight",)),
("Already-landed gate", ("already-landed gate",)),
("Pinned reviewed head", ("pinned reviewed head",)),
("Terminal review mutation", ("terminal review mutation",)),
)
def _handoff_section_lines(report_text):
"""Return the lines of the Controller Handoff section, or None."""
@@ -1623,6 +1670,11 @@ def assess_controller_handoff(report_text, role=None, local_edits=False):
fields_dict[label] = v.strip()
required = list(HANDOFF_BASE_FIELDS)
if role == "create_issue":
required = [
field for field in required
if field[0] not in ("Workspace mutations", "Mutations")
]
required.extend(HANDOFF_ROLE_FIELDS.get(role or "", ()))
missing = []
@@ -2647,6 +2699,141 @@ def assess_issue_filing_duplicate_summary(
}
def assess_create_issue_workflow_source(report_text: str) -> dict:
"""#337: create-issue reports must cite the canonical workflow file."""
lower = (report_text or "").lower()
reasons = []
if not any(marker in lower for marker in CREATE_ISSUE_WORKFLOW_MARKERS):
reasons.append(
"create-issue report missing workflow source "
"(workflows/create-issue.md)"
)
if not any(marker in lower for marker in CREATE_ISSUE_TASK_MARKERS):
reasons.append(
"create-issue report missing task mode declaration "
"(create-issue)"
)
return {
"complete": not reasons,
"downgraded": bool(reasons),
"reasons": reasons,
}
def assess_create_issue_mode_isolation(report_text: str) -> dict:
"""#337: reject work-issue/review-merge handoff fields in create-issue runs."""
section = _handoff_section_lines(report_text)
reasons = []
if section is None:
return {
"complete": True,
"downgraded": False,
"reasons": [],
}
labels = []
for line in section:
stripped = line.strip().lstrip("-*").strip()
if ":" in stripped:
labels.append(stripped.split(":", 1)[0].strip().lower())
for name, aliases in CREATE_ISSUE_FORBIDDEN_CROSS_MODE_FIELDS:
if any(
label.startswith(alias)
for label in labels
for alias in aliases
):
reasons.append(
f"create-issue handoff must not include cross-mode field "
f"'{name}'"
)
legacy_workspace = any(
label.startswith("workspace mutations") for label in labels
)
precise_categories = any(
label.startswith("file edits by issue creator")
for label in labels
)
if legacy_workspace and not precise_categories:
reasons.append(
"create-issue handoff must use precise mutation categories "
"instead of legacy 'Workspace mutations'"
)
return {
"complete": not reasons,
"downgraded": bool(reasons),
"reasons": reasons,
}
def assess_create_issue_final_report(
report_text: str,
*,
issue_number: int | None = None,
issue_title: str | None = None,
action: str = "created",
mutations: list[str] | None = None,
resolved_capabilities: dict | None = None,
issues_searched: int | None = None,
closest_matches: list[dict] | None = None,
duplicate_result: dict | None = None,
performed_mutations: list[str] | None = None,
) -> dict:
"""#337: composite verifier for create-issue final reports."""
checks = {
"workflow_source": assess_create_issue_workflow_source(report_text),
"mode_isolation": assess_create_issue_mode_isolation(report_text),
"controller_handoff": assess_controller_handoff(
report_text, role="create_issue"
),
"issue_reference": assess_issue_filing_issue_reference(
report_text,
issue_number=issue_number,
issue_title=issue_title,
action=action,
),
"mutation_capability": assess_issue_filing_mutation_capability(
report_text,
mutations or performed_mutations,
resolved_capabilities,
),
"mutation_scope": assess_issue_filing_mutation_scope(
report_text,
performed_mutations=performed_mutations or mutations,
),
"duplicate_summary": assess_issue_filing_duplicate_summary(
report_text,
issues_searched=issues_searched,
closest_matches=closest_matches,
duplicate_result=duplicate_result,
),
}
reasons = []
downgraded = False
for name, result in checks.items():
verdict = result.get("verdict")
if verdict in ("missing", "incomplete"):
downgraded = True
reasons.extend(result.get("reasons") or [])
elif result.get("downgraded") or not result.get("complete", True):
downgraded = True
reasons.extend(
f"{name}: {r}" for r in (result.get("reasons") or [])
)
grade = "A" if not downgraded else "downgraded"
return {
"grade": grade,
"downgraded": downgraded,
"checks": checks,
"reasons": reasons,
"complete": not downgraded,
}
def assess_issue_filing_final_report(
report_text: str,
*,
tests/test_llm_workflow_split.py
+7 -1
View File
@@ -97,4 +97,10 @@ def test_merge_pr_template_references_workflow():
def test_start_issue_template_references_work_issue_workflow():
text = (SKILL_DIR / "templates" / "start-issue.md").read_text(encoding="utf-8")
assert "workflows/work-issue.md" in text
assert "workflows/work-issue.md" in text
def test_create_issue_final_report_verifier_exported():
from review_proofs import assess_create_issue_final_report
assert callable(assess_create_issue_final_report)
tests/test_review_proofs.py
+118
View File
@@ -29,6 +29,9 @@ from review_proofs import ( # noqa: E402
assess_contradictory_no_pr_claim,
assess_continuation_mode_report,
assess_controller_handoff,
assess_create_issue_final_report,
assess_create_issue_mode_isolation,
assess_create_issue_workflow_source,
assess_edited_pr_inventory_coverage,
assess_empty_queue_report,
assess_fresh_issue_selection,
@@ -2042,5 +2045,120 @@ class TestIssueFilingFinalReport(unittest.TestCase):
self.assertTrue(result["downgraded"])
class TestCreateIssueFinalReport(unittest.TestCase):
"""#337: create-issue final-report verifier coverage."""
ISSUE_TITLE = "Extract create-issue workflow from monolithic LLM project skill"
CAPABILITIES = {
"create_issue": {
"requested_task": "create_issue",
"required_operation_permission": "gitea.issue.create",
"allowed_in_current_session": True,
},
}
CLOSEST = [{"number": 333, "title": "Split monolithic workflow", "state": "closed"}]
def _good_report(self):
return "\n".join([
"Task mode: create-issue",
"Workflow source: skills/llm-project-workflow/workflows/create-issue.md",
f"Updated issue #337 — `{self.ISSUE_TITLE}` (remaining verifier AC).",
"Duplicate check: searched 54 open issues.",
"Closest existing: #333 — Split monolithic workflow (partial overlap).",
"Why new issue justified: dedicated create-issue extraction.",
"Only mutation: issue creation (gitea.issue.create).",
"Confirm no labels, comments, PRs, reviews, merges, or closes.",
"## Controller Handoff",
"- Task: create-issue",
"- Repo: Scaled-Tech-Consulting/Gitea-Tools",
"- Role: author",
"- Identity: jcwalker3 / prgs-author",
"- Active profile: prgs-author",
"- Runtime context: prgs-author on prgs",
"- Requested issue task: extract create-issue workflow verifier",
"- Workflow source: workflows/create-issue.md @ fd396df",
"- Capability proof: create_issue allowed (gitea.issue.create)",
"- Duplicate search terms: create-issue, workflow extract",
"- Duplicate search pagination proof: 54 open issues < limit 500",
"- Duplicates found: #333 partial only",
"- Issues created: none (implementation PR for remaining AC)",
"- Issues commented: none",
"- Issues edited: none",
"- Issues skipped as duplicates: none",
"- File edits by issue creator: review_proofs.py, tests",
"- Worktree/index mutations: feat/issue-337 worktree created",
"- Git ref mutations: fetch prgs/master",
"- MCP/Gitea mutations: gitea_mark_issue start on #337",
"- Issue mutations: claimed #337",
"- Label/assignment/milestone mutations: none",
"- External-state mutations: none",
"- Read-only diagnostics: whoami, list issues",
"- Blockers: none",
"- Current status: implementing verifier",
"- Safe next action: open PR",
"- Safety statement: no review/merge/repo main-checkout edits",
"- Issue/PR: #337",
"- Branch/SHA: feat/issue-337-create-issue-verifier",
"- Files changed: review_proofs.py, tests",
"- Validation: pytest tests/test_review_proofs.py",
"- Mutations: create_issue capability resolved; mark_issue only",
"- Next: open PR",
"- Safety: no review/merge",
])
def test_complete_create_issue_report_earns_a(self):
result = assess_create_issue_final_report(
self._good_report(),
issue_number=337,
issue_title=self.ISSUE_TITLE,
action="updated",
mutations=["create_issue"],
resolved_capabilities=self.CAPABILITIES,
issues_searched=54,
closest_matches=self.CLOSEST,
performed_mutations=["create_issue"],
)
self.assertEqual(result["grade"], "A")
self.assertFalse(result["downgraded"])
def test_missing_workflow_source_downgrades(self):
report = self._good_report().replace("workflows/create-issue.md", "SKILL.md only")
result = assess_create_issue_workflow_source(report)
self.assertTrue(result["downgraded"])
def test_work_issue_field_in_handoff_downgrades(self):
report = self._good_report().replace(
"- Safety statement:",
"- Selected issue: #123\n- Safety statement:",
)
result = assess_create_issue_mode_isolation(report)
self.assertTrue(result["downgraded"])
self.assertTrue(
any("Selected issue" in r for r in result["reasons"])
)
def test_review_merge_field_in_handoff_downgrades(self):
report = self._good_report().replace(
"- Safety statement:",
"- Review decision: APPROVE\n- Safety statement:",
)
result = assess_create_issue_mode_isolation(report)
self.assertTrue(result["downgraded"])
def test_legacy_workspace_mutations_without_precise_categories_downgrades(self):
report = self._good_report().replace(
"- File edits by issue creator:",
"- Workspace mutations: none\n- File edits by issue creator:",
)
lines = report.splitlines()
cleaned = [
line for line in lines
if not line.strip().startswith("- File edits by issue creator:")
]
report = "\n".join(cleaned)
result = assess_create_issue_mode_isolation(report)
self.assertTrue(result["downgraded"])
if __name__ == "__main__":
unittest.main()