From 68a34f9f9c6f19c33f080412b2dbb21571d266fb Mon Sep 17 00:00:00 2001 From: yashoza19 Date: Thu, 3 Sep 2026 19:19:57 -0400 Subject: [PATCH] Add pluggable LLM backends and Granite-friendly drafter postprocessing. Enable skills/messages/openai drafter backends with draft-compare tooling, compact prompts and epic-grouped payloads for small models, and deterministic link/gap enrichment so OpenAI-compatible endpoints produce reviewable drafts. Co-authored-by: Cursor --- .env.example | 8 + src/status/cli.py | 123 +++++++++++- src/status/config.py | 8 + src/status/skills/drafter.py | 177 +++++++++++++---- src/status/skills/evidence.py | 87 ++++++++ src/status/skills/llm_backends.py | 287 +++++++++++++++++++++++++++ src/status/skills/payload_summary.py | 125 ++++++++++++ src/status/skills/schemas.py | 33 ++- tests/test_drafter.py | 20 +- tests/test_evidence.py | 85 ++++++++ tests/test_llm_backends.py | 42 ++++ tests/test_payload_summary.py | 48 +++++ 12 files changed, 990 insertions(+), 53 deletions(-) create mode 100644 src/status/skills/llm_backends.py create mode 100644 src/status/skills/payload_summary.py create mode 100644 tests/test_llm_backends.py create mode 100644 tests/test_payload_summary.py diff --git a/.env.example b/.env.example index 0b43503..0c342c3 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,14 @@ GITHUB_LOGIN= # Claude / Slack # ANTHROPIC_API_KEY= +# CLAUDE_MODEL=claude-sonnet-5 +# DRAFTER_LLM_BACKEND=skills +# DRAFTER_MAX_TOKENS=4096 +# DRAFTER_MESSAGES_MAX_TOKENS=16384 +# DRAFTER_SKILL_MAX_TOKENS=16384 +# OPENAI_API_KEY= +# OPENAI_BASE_URL=https://your-inference-host/agentic-workflow/your-model-serving-route +# OPENAI_SSL_VERIFY=false # set for internal clusters with self-signed certs # SLACK_BOT_TOKEN= # SLACK_APP_TOKEN= # REPORT_CHANNEL_ID= diff --git a/src/status/cli.py b/src/status/cli.py index cf81f6d..ddbe062 100644 --- a/src/status/cli.py +++ b/src/status/cli.py @@ -3,7 +3,7 @@ import json from datetime import date, datetime from pathlib import Path -from typing import Annotated, Optional +from typing import Annotated, Any, Optional import typer from rich.console import Console @@ -14,6 +14,7 @@ from status.db import get_session from status.skills.client import SkillClient from status.skills.drafter import DraftPersistError, draft_and_persist, load_fixture, run_drafter +from status.skills.llm_backends import DrafterBackend, backend_config_error, prompt_version_for from status.skills.synthesizer import run_synthesizer app = typer.Typer(no_args_is_help=True, help="Weekly status pipeline CLI") @@ -112,6 +113,126 @@ def draft( console.print_json(json.dumps(output, indent=2)) +def _parse_drafter_backend(value: str) -> DrafterBackend: + try: + return DrafterBackend.parse(value) + except ValueError as exc: + console.print(f"[red]{exc}[/]") + raise typer.Exit(1) from exc + + +@app.command("draft-compare") +def draft_compare( + fixture: Annotated[ + Optional[Path], typer.Option("--fixture", "-f", help="Collector payload JSON") + ] = None, + person: Annotated[ + Optional[str], typer.Option("--person", "-p", help="Person ID (collects live data)") + ] = None, + week: Annotated[ + Optional[str], typer.Option("--week", "-w", help="Week ending Friday (YYYY-MM-DD)") + ] = None, + backend: Annotated[ + list[str], + typer.Option( + "--backend", + "-b", + help="Backends to compare: skills (hosted Claude skill), messages, openai", + ), + ] = ["skills", "openai"], + output_dir: Annotated[ + Path, + typer.Option("--output-dir", "-o", help="Directory for per-backend JSON output"), + ] = Path("draft-compare"), + no_persist: Annotated[ + bool, + typer.Option( + "--no-persist", + help="Accepted for parity with status draft (compare never writes to Postgres)", + ), + ] = False, +) -> None: + """Run the drafter across multiple LLM backends and write JSON for comparison.""" + _ = no_persist + + if fixture is not None: + try: + payload = load_fixture(fixture) + except FileNotFoundError as exc: + console.print(f"[red]{exc}[/]") + raise typer.Exit(1) from exc + elif person and week: + payload = run_collect(person, _parse_week(week)) + else: + console.print("[red]Provide --fixture or both --person and --week[/]") + raise typer.Exit(1) + + if not backend: + console.print("[red]Provide at least one --backend[/]") + raise typer.Exit(1) + + output_dir.mkdir(parents=True, exist_ok=True) + summary: dict[str, Any] = { + "person": payload.get("person"), + "week_ending": payload.get("week_end") or payload.get("week_ending"), + "backends": {}, + } + + table = Table("Backend", "Status", "Entries", "Flags", "Output file") + for raw_backend in backend: + resolved = _parse_drafter_backend(raw_backend) + config_error = backend_config_error(resolved) + output_path = output_dir / f"draft-{resolved.value}.json" + + if config_error: + result_payload = { + "backend": resolved.value, + "error": config_error, + "draft": None, + } + output_path.write_text(json.dumps(result_payload, indent=2), encoding="utf-8") + summary["backends"][resolved.value] = result_payload + table.add_row(resolved.value, "skipped", "-", "-", str(output_path)) + continue + + try: + draft = run_drafter(payload, backend=resolved) + except Exception as exc: # noqa: BLE001 - surface any unexpected compare failure + result_payload = { + "backend": resolved.value, + "error": str(exc), + "draft": None, + } + output_path.write_text(json.dumps(result_payload, indent=2), encoding="utf-8") + summary["backends"][resolved.value] = result_payload + table.add_row(resolved.value, "error", "-", "-", str(output_path)) + continue + + result_payload = { + "backend": resolved.value, + "prompt_version": prompt_version_for(resolved), + "draft": draft.model_dump(), + } + output_path.write_text(json.dumps(result_payload, indent=2), encoding="utf-8") + summary["backends"][resolved.value] = { + "prompt_version": result_payload["prompt_version"], + "entry_count": len(draft.entries), + "flag_count": len(draft.flags), + "output_file": str(output_path), + } + table.add_row( + resolved.value, + "ok", + str(len(draft.entries)), + str(len(draft.flags)), + str(output_path), + ) + + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + console.print(table) + console.print(f"[dim]Wrote summary to {output_dir / 'summary.json'}[/]") + + @app.command() def send( person: Annotated[str, typer.Option("--person", "-p")], diff --git a/src/status/config.py b/src/status/config.py index 2aaac17..8bcca6b 100644 --- a/src/status/config.py +++ b/src/status/config.py @@ -56,6 +56,14 @@ class Settings(BaseSettings): drafter_skill_id: str | None = Field(default=None, alias="DRAFTER_SKILL_ID") drafter_skill_version: str = Field(default="latest", alias="DRAFTER_SKILL_VERSION") + drafter_llm_backend: str = Field(default="skills", alias="DRAFTER_LLM_BACKEND") + drafter_max_tokens: int = Field(default=4096, alias="DRAFTER_MAX_TOKENS") + drafter_messages_max_tokens: int = Field(default=16384, alias="DRAFTER_MESSAGES_MAX_TOKENS") + drafter_skill_max_tokens: int = Field(default=16384, alias="DRAFTER_SKILL_MAX_TOKENS") + openai_api_key: str | None = Field(default=None, alias="OPENAI_API_KEY") + openai_base_url: str | None = Field(default=None, alias="OPENAI_BASE_URL") + openai_model: str | None = Field(default=None, alias="OPENAI_MODEL") + openai_ssl_verify: bool = Field(default=True, alias="OPENAI_SSL_VERIFY") synthesizer_skill_id: str | None = Field(default=None, alias="SYNTHESIZER_SKILL_ID") synthesizer_skill_version: str = Field(default="latest", alias="SYNTHESIZER_SKILL_VERSION") diff --git a/src/status/skills/drafter.py b/src/status/skills/drafter.py index d6cf04d..6d7b10e 100644 --- a/src/status/skills/drafter.py +++ b/src/status/skills/drafter.py @@ -16,15 +16,21 @@ from status.config import get_settings from status.db import get_session from status.db.draft import persist_draft_output -from status.skills.client import SkillClient, SkillError, SkillRef +from status.skills.llm_backends import ( + DrafterBackend, + LlmBackendError, + backend_config_error, + invoke_drafter_backend, + prompt_version_for, +) from status.skills.evidence import ( - MILESTONE_ONLY_RE, build_evidence_labels, + enrich_outcome_links, filter_evidence_to_payload, - inject_markdown_links, issue_summary_index, - outcome_from_linked_evidence, + jira_keys_from_evidence, payload_jira_keys, + pr_url_index, ) from status.skills.schemas import DraftEntry, DraftOutput @@ -115,11 +121,71 @@ def attach_evidence_labels(draft: DraftOutput, payload: dict[str, Any]) -> Draft return draft.model_copy(update={"entries": enriched}) +def _detect_gap_flags( + draft: DraftOutput, + payload: dict[str, Any], +) -> tuple[list[str], str]: + """Deterministic gap flags small models often omit.""" + flags = list(draft.flags) + unticketed_prompt = draft.unticketed_prompt.strip() + + previous_epics: dict[str, str] = {} + for entry in payload.get("previous_entries") or []: + epic_key = entry.get("epic_key") + if epic_key: + previous_epics[str(epic_key)] = str(entry.get("epic_name") or epic_key) + + active_epics: set[str] = set() + cited_pr_urls: set[str] = set() + for entry in draft.entries: + if entry.epic_key: + active_epics.add(entry.epic_key) + for key in jira_keys_from_evidence(entry.evidence): + active_epics.add(key) + for item in entry.evidence: + if item.startswith("http"): + cited_pr_urls.add(item) + + for epic_key, epic_name in previous_epics.items(): + if epic_key not in active_epics: + message = f"{epic_key} ({epic_name}) had no activity this week." + if message not in flags: + flags.append(message) + + unticketed_prs: list[dict[str, Any]] = [] + for pr in payload.get("pull_requests") or []: + if pr.get("linked_issue_keys"): + continue + url = pr.get("url") + title = str(pr.get("title") or "untitled PR") + message = f'PR "{title}" has no linked Jira ticket.' + if message not in flags: + flags.append(message) + if url and url not in cited_pr_urls: + unticketed_prs.append(pr) + + if not unticketed_prompt and unticketed_prs: + titles = [str(pr.get("title") or "a pull request") for pr in unticketed_prs[:2]] + if len(titles) == 1: + unticketed_prompt = ( + f'Should "{titles[0]}" be tied to a Jira ticket for this week\'s status?' + ) + else: + unticketed_prompt = ( + f'Nothing here links "{titles[0]}" or "{titles[1]}" to a ticket — ' + "which epic should they roll up to?" + ) + + flags = list(dict.fromkeys(flags)) + return flags, unticketed_prompt + + def postprocess_draft(draft: DraftOutput, payload: dict[str, Any]) -> DraftOutput: """Filter evidence to this person's payload and enrich outcomes with Jira links.""" settings = get_settings() jira_base_url = settings.jira_base_url or "https://redhat.atlassian.net" allowed_keys = payload_jira_keys(payload) + pr_titles = pr_url_index(list(payload.get("pull_requests") or [])) processed: list[DraftEntry] = [] for entry in draft.entries: @@ -134,23 +200,26 @@ def postprocess_draft(draft: DraftOutput, payload: dict[str, Any]) -> DraftOutpu for key, label in entry.evidence_labels.items() if key in allowed_keys } - outcome = entry.outcome - if MILESTONE_ONLY_RE.search(outcome) and "[" not in outcome: - rewritten = outcome_from_linked_evidence( - entry.state, - evidence, - labels, - jira_base_url=jira_base_url, - ) - if rewritten: - outcome = rewritten - outcome = inject_markdown_links(outcome, labels, jira_base_url=jira_base_url) + outcome = enrich_outcome_links( + entry.outcome, + entry.state, + evidence, + labels, + pr_titles, + jira_base_url=jira_base_url, + ) updates: dict[str, Any] = { "evidence": evidence, "evidence_labels": labels, "outcome": outcome, } + if entry.epic_key is None and not entry.needs_human: + updates["needs_human"] = True + if not entry.why_flagged: + updates["why_flagged"] = ( + "No epic on this work — which initiative should it roll up to?" + ) if len(evidence) < len(entry.evidence): updates["needs_human"] = True if not entry.why_flagged: @@ -160,39 +229,64 @@ def postprocess_draft(draft: DraftOutput, payload: dict[str, Any]) -> DraftOutpu ) processed.append(entry.model_copy(update=updates)) - return draft.model_copy(update={"entries": processed}) + flags, unticketed_prompt = _detect_gap_flags( + draft.model_copy(update={"entries": processed}), + payload, + ) + return draft.model_copy( + update={ + "entries": processed, + "flags": flags, + "unticketed_prompt": unticketed_prompt, + } + ) -def run_drafter(payload: dict[str, Any], *, dry_run: bool = False) -> DraftOutput: +def run_drafter( + payload: dict[str, Any], + *, + dry_run: bool = False, + backend: DrafterBackend | None = None, +) -> DraftOutput: settings = get_settings() - if dry_run or not settings.drafter_skill_id: - return _empty_draft( - payload, - flags=["dry-run: no skill invocation"] if dry_run else ["no DRAFTER_SKILL_ID configured"], + if dry_run: + return _empty_draft(payload, flags=["dry-run: no skill invocation"]) + + resolved = backend or DrafterBackend.parse(settings.drafter_llm_backend) + config_error = backend_config_error(resolved) + if config_error: + return _empty_draft(payload, flags=[config_error]) + + instruction = DRAFTER_INSTRUCTION + regeneration_notes = str(payload.get("regeneration_notes") or "").strip() + if regeneration_notes: + instruction = ( + f"{instruction}\n\nThe user asked to regenerate this draft with this guidance: " + f"{regeneration_notes}" ) - if not settings.anthropic_api_key: - return _empty_draft(payload, flags=["ANTHROPIC_API_KEY not configured"]) - - client = SkillClient(api_key=settings.anthropic_api_key, model=settings.claude_model) - skill = SkillRef( - skill_id=settings.drafter_skill_id, - version=settings.drafter_skill_version, - ) - - last_error: SkillError | None = None + last_error: Exception | None = None for attempt in range(2): try: - result = client.invoke_json(skill, payload, DRAFTER_INSTRUCTION, DraftOutput) + result = invoke_drafter_backend( + resolved, + payload, + DraftOutput, + instruction=instruction, + ) assert isinstance(result, DraftOutput) normalized = _normalize_draft(result, payload) labeled = attach_evidence_labels(normalized, payload) return postprocess_draft(labeled, payload) - except SkillError as exc: + except LlmBackendError as exc: last_error = exc - log.warning("drafter attempt %s failed: %s", attempt + 1, exc) + log.warning("drafter attempt %s failed (%s): %s", attempt + 1, resolved.value, exc) - flag = f"drafter failed after retry: {last_error}" if last_error else "drafter failed after retry" + flag = ( + f"drafter failed after retry ({resolved.value}): {last_error}" + if last_error + else f"drafter failed after retry ({resolved.value})" + ) return _empty_draft(payload, flags=[flag]) @@ -253,13 +347,14 @@ def draft_and_persist( settings = get_settings() draft = run_drafter(payload, dry_run=dry_run) - if settings.drafter_skill_id and not dry_run: - prompt_version = SkillRef( - skill_id=settings.drafter_skill_id, - version=settings.drafter_skill_version, - ).prompt_version - else: + if dry_run: prompt_version = "dry-run" + else: + try: + backend = DrafterBackend.parse(settings.drafter_llm_backend) + prompt_version = prompt_version_for(backend) + except ValueError: + prompt_version = settings.drafter_llm_backend if dry_run or not persist: return DraftRunResult( diff --git a/src/status/skills/evidence.py b/src/status/skills/evidence.py index 8dbe85d..8e39dd1 100644 --- a/src/status/skills/evidence.py +++ b/src/status/skills/evidence.py @@ -131,6 +131,93 @@ def filter_evidence_to_payload( return kept +def outcome_has_markdown_links(text: str) -> bool: + return bool(text and MD_LINK_RE.search(text)) + + +def pr_url_index(pull_requests: list[dict] | None) -> dict[str, str]: + """Map PR URLs to titles from the collector payload.""" + titles: dict[str, str] = {} + for pr in pull_requests or []: + url = pr.get("url") + if url: + titles[str(url)] = str(pr.get("title") or "pull request").strip() + return titles + + +def inject_pr_markdown_links( + text: str, + evidence: list[str], + pr_titles: dict[str, str], +) -> str: + """Turn bare PR URLs in evidence into markdown links in the outcome.""" + if not evidence or not pr_titles: + return text + + updated = text or "" + appended: list[str] = [] + for item in evidence: + if not item.startswith("http"): + continue + title = pr_titles.get(item) + if not title: + continue + link = f"[{title}]({item})" + if f"]({item})" in updated: + continue + if item in updated: + updated = updated.replace(item, link, 1) + else: + appended.append(link) + + if appended: + if updated and not updated.endswith("."): + updated += "." + updated = f"{updated} {'; '.join(appended)}.".strip() + return updated + + +def enrich_outcome_links( + outcome: str, + entry_state: str, + evidence: list[str], + evidence_labels: dict[str, str], + pr_titles: dict[str, str], + *, + jira_base_url: str = "https://redhat.atlassian.net", +) -> str: + """Add Jira and PR markdown links when the model omitted them.""" + text = outcome + jira_labels = { + key: label + for key, label in evidence_labels.items() + if key in set(jira_keys_from_evidence(evidence)) + } + + if not outcome_has_markdown_links(text): + if MILESTONE_ONLY_RE.search(text) and jira_labels: + rewritten = outcome_from_linked_evidence( + entry_state, + evidence, + jira_labels, + jira_base_url=jira_base_url, + ) + if rewritten: + text = rewritten + elif jira_labels: + rewritten = outcome_from_linked_evidence( + entry_state, + evidence, + jira_labels, + jira_base_url=jira_base_url, + ) + if rewritten: + text = rewritten + + text = inject_markdown_links(text, evidence_labels, jira_base_url=jira_base_url) + return inject_pr_markdown_links(text, evidence, pr_titles) + + def inject_markdown_links( text: str, evidence_labels: dict[str, str], diff --git a/src/status/skills/llm_backends.py b/src/status/skills/llm_backends.py new file mode 100644 index 0000000..db94a21 --- /dev/null +++ b/src/status/skills/llm_backends.py @@ -0,0 +1,287 @@ +"""Pluggable LLM backends for the weekly-status drafter.""" + +from __future__ import annotations + +import json +import logging +import ssl +import urllib.error +import urllib.request +from enum import Enum +from typing import Any + +from pydantic import BaseModel + +from status.config import SKILLS_DIR, get_settings +from status.skills.client import SkillClient, SkillError, SkillRef +from status.skills.json_output import JsonOutputError, parse_json_model + +from status.skills.payload_summary import summarize_payload_for_openai + +log = logging.getLogger(__name__) + +DRAFTER_SKILL_DIR = SKILLS_DIR / "weekly-status-drafter" + +PROMPT_INSTRUCTION = ( + "Apply the weekly-status-drafter instructions to the collector payload below. " + "Return only the JSON object defined in the skill as plain text in your reply. " + "Use markdown links [text](url) in outcome fields for Jira and GitHub evidence." +) + +OPENAI_DRAFTER_PROMPT = """\ +You draft one engineer's weekly status from grouped Jira and GitHub activity. + +Rules: +- One entry per epic in `epic_groups`. Tickets with no epic become one entry with `epic_key: null`. +- Group PRs and commits with their epic; cite every PR URL you mention in `evidence`. +- Write `outcome` in past tense, one or two sentences, with markdown links: + `[summary phrase](https://redhat.atlassian.net/browse/KEY)` and `[PR title](pr-url)`. +- Set `needs_human: true` when `epic_key` is null or confidence is low. +- Add `flags` for epics in `previous_epics` with no activity this week. +- Add `flags` for PRs with no linked Jira ticket. +- End with a specific `unticketed_prompt` when PRs lack tickets. + +Return only JSON (no markdown fences): +{ + "person": "string", + "week_ending": "YYYY-MM-DD", + "entries": [{ + "project": "string", + "epic_key": "string | null", + "epic_name": "string | null", + "state": "shipped | progressing | slipped | blocked | quiet", + "outcome": "one or two linked sentences", + "evidence": ["JIRA-KEY", "https://github.com/org/repo/pull/N"], + "blocker": "string | null", + "ask": "string | null", + "confidence": "high | medium | low", + "needs_human": false, + "why_flagged": "string | null" + }], + "flags": ["string"], + "unticketed_prompt": "string" +} + +Example entry outcome: +Working on [agentic weekly status pipeline](https://redhat.atlassian.net/browse/EET-5519); +merged [Improve drafter linked outcomes](https://github.com/opdev/agentic-status-report/pull/18). +""" + + +class DrafterBackend(str, Enum): + SKILLS = "skills" + MESSAGES = "messages" + OPENAI = "openai" + + @classmethod + def parse(cls, value: str) -> DrafterBackend: + normalized = value.strip().lower() + try: + return cls(normalized) + except ValueError as exc: + supported = ", ".join(member.value for member in cls) + raise ValueError(f"Unknown drafter backend {value!r}; use one of: {supported}") from exc + + +class LlmBackendError(RuntimeError): + """Raised when an LLM backend cannot complete a drafter call.""" + + +def load_drafter_skill_text() -> str: + skill_path = DRAFTER_SKILL_DIR / "SKILL.md" + if not skill_path.is_file(): + raise LlmBackendError(f"drafter skill file not found: {skill_path}") + return skill_path.read_text(encoding="utf-8") + + +def backend_config_error(backend: DrafterBackend) -> str | None: + settings = get_settings() + if backend is DrafterBackend.SKILLS: + if not settings.anthropic_api_key: + return "ANTHROPIC_API_KEY not set" + if not settings.drafter_skill_id: + return "DRAFTER_SKILL_ID not set" + return None + if backend is DrafterBackend.MESSAGES: + if not settings.anthropic_api_key: + return "ANTHROPIC_API_KEY not set" + return None + if backend is DrafterBackend.OPENAI: + if not settings.openai_api_key: + return "OPENAI_API_KEY not set" + if not settings.openai_base_url: + return "OPENAI_BASE_URL not set" + if not settings.openai_model: + return "OPENAI_MODEL not set" + return None + return f"unsupported backend: {backend}" + + +def prompt_version_for(backend: DrafterBackend) -> str: + settings = get_settings() + if backend is DrafterBackend.SKILLS and settings.drafter_skill_id: + return SkillRef( + skill_id=settings.drafter_skill_id, + version=settings.drafter_skill_version, + ).prompt_version + if backend is DrafterBackend.MESSAGES: + return f"messages:{settings.claude_model}" + if backend is DrafterBackend.OPENAI: + return f"openai:{settings.openai_model}" + return backend.value + + +def invoke_drafter_backend( + backend: DrafterBackend, + payload: dict[str, Any], + schema: type[BaseModel], + *, + instruction: str | None = None, +) -> BaseModel: + config_error = backend_config_error(backend) + if config_error: + raise LlmBackendError(config_error) + + if backend is DrafterBackend.SKILLS: + return _invoke_skills_backend(payload, schema, instruction=instruction) + if backend is DrafterBackend.MESSAGES: + return _invoke_messages_backend(payload, schema, instruction=instruction) + return _invoke_openai_backend(payload, schema, instruction=instruction) + + +def _invoke_skills_backend( + payload: dict[str, Any], + schema: type[BaseModel], + *, + instruction: str | None, +) -> BaseModel: + settings = get_settings() + assert settings.anthropic_api_key and settings.drafter_skill_id + client = SkillClient(api_key=settings.anthropic_api_key, model=settings.claude_model) + skill = SkillRef( + skill_id=settings.drafter_skill_id, + version=settings.drafter_skill_version, + ) + try: + result = client.invoke_json( + skill, + payload, + instruction or PROMPT_INSTRUCTION, + schema, + ) + except SkillError as exc: + raise LlmBackendError(str(exc)) from exc + if not isinstance(result, schema): + raise LlmBackendError(f"skills backend returned unexpected type: {type(result)}") + return result + + +def _invoke_messages_backend( + payload: dict[str, Any], + schema: type[BaseModel], + *, + instruction: str | None, +) -> BaseModel: + settings = get_settings() + assert settings.anthropic_api_key + skill_text = load_drafter_skill_text() + user_text = json.dumps(payload, indent=2) + system_text = f"{skill_text}\n\n{instruction or PROMPT_INSTRUCTION}" + + try: + import anthropic + except ImportError as exc: + raise LlmBackendError("anthropic package is required for messages backend") from exc + + client = anthropic.Anthropic(api_key=settings.anthropic_api_key) + response = client.messages.create( + model=settings.claude_model, + max_tokens=settings.drafter_messages_max_tokens, + system=system_text, + messages=[{"role": "user", "content": user_text}], + ) + text = _anthropic_text(response) + try: + return parse_json_model(text, schema, source="messages") + except JsonOutputError as exc: + raise LlmBackendError(str(exc)) from exc + + +def _openai_ssl_context() -> ssl.SSLContext | None: + settings = get_settings() + if settings.openai_ssl_verify: + return None + return ssl._create_unverified_context() + + +def openai_chat_completions_url(base_url: str) -> str: + """Resolve an OpenAI-compatible chat completions URL from a flexible base URL.""" + base = base_url.rstrip("/") + if base.endswith("/chat/completions"): + return base + if base.endswith("/v1"): + return f"{base}/chat/completions" + return f"{base}/v1/chat/completions" + + +def _invoke_openai_backend( + payload: dict[str, Any], + schema: type[BaseModel], + *, + instruction: str | None, +) -> BaseModel: + settings = get_settings() + assert settings.openai_api_key and settings.openai_base_url and settings.openai_model + summarized = summarize_payload_for_openai(payload) + user_text = json.dumps(summarized, indent=2) + system_text = f"{OPENAI_DRAFTER_PROMPT}\n\n{instruction or PROMPT_INSTRUCTION}" + url = openai_chat_completions_url(settings.openai_base_url) + body = { + "model": settings.openai_model, + "max_tokens": settings.drafter_max_tokens, + "messages": [ + {"role": "system", "content": system_text}, + {"role": "user", "content": user_text}, + ], + "temperature": 0, + } + request = urllib.request.Request( + url, + data=json.dumps(body).encode("utf-8"), + headers={ + "Authorization": f"Bearer {settings.openai_api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with urllib.request.urlopen( + request, + timeout=600, + context=_openai_ssl_context(), + ) as response: + raw = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise LlmBackendError(f"openai backend HTTP {exc.code}: {detail[:500]}") from exc + except urllib.error.URLError as exc: + raise LlmBackendError(f"openai backend request failed: {exc}") from exc + + try: + text = raw["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError) as exc: + raise LlmBackendError(f"openai backend returned unexpected response: {raw!r}") from exc + + try: + return parse_json_model(str(text), schema, source="openai") + except JsonOutputError as exc: + raise LlmBackendError(str(exc)) from exc + + +def _anthropic_text(response: Any) -> str: + parts: list[str] = [] + for block in getattr(response, "content", []) or []: + text = getattr(block, "text", None) + if text: + parts.append(str(text)) + return "\n".join(parts).strip() diff --git a/src/status/skills/payload_summary.py b/src/status/skills/payload_summary.py new file mode 100644 index 0000000..0d37c3c --- /dev/null +++ b/src/status/skills/payload_summary.py @@ -0,0 +1,125 @@ +"""Pre-group collector payloads for small-model drafter prompts.""" + +from __future__ import annotations + +from typing import Any + + +def _epic_bucket(epic_key: str | None, project: str) -> str: + if epic_key: + return str(epic_key) + return f"__no_epic__:{project}" + + +def summarize_payload_for_openai(payload: dict[str, Any]) -> dict[str, Any]: + """Collapse raw Jira/PR/commit lists into epic-oriented groups.""" + groups: dict[str, dict[str, Any]] = {} + + def ensure_group(bucket: str, *, project: str, epic_key: str | None, epic_name: str | None) -> dict[str, Any]: + if bucket not in groups: + groups[bucket] = { + "project": project, + "epic_key": epic_key, + "epic_name": epic_name, + "jira_issues": [], + "pull_requests": [], + "commits": [], + } + return groups[bucket] + + issue_by_key: dict[str, dict[str, Any]] = {} + for issue in payload.get("jira_issues") or []: + key = issue.get("key") + if not key: + continue + issue_by_key[str(key)] = issue + project = str(issue.get("project") or "unknown") + epic_key = issue.get("epic_key") + epic_name = issue.get("epic_name") + bucket = _epic_bucket(str(epic_key) if epic_key else None, project) + group = ensure_group( + bucket, + project=project, + epic_key=str(epic_key) if epic_key else None, + epic_name=str(epic_name) if epic_name else None, + ) + group["jira_issues"].append( + { + "key": key, + "summary": issue.get("summary"), + "status": issue.get("status"), + "is_assignee": issue.get("is_assignee"), + } + ) + + def assign_pr_or_commit( + item: dict[str, Any], + *, + field: str, + title_key: str, + extra: dict[str, Any] | None = None, + ) -> None: + linked = [str(key) for key in (item.get("linked_issue_keys") or []) if key] + target_buckets: set[str] = set() + for key in linked: + issue = issue_by_key.get(key) + if issue: + project = str(issue.get("project") or "unknown") + epic_key = issue.get("epic_key") + target_buckets.add(_epic_bucket(str(epic_key) if epic_key else None, project)) + if not target_buckets: + if groups: + if len(groups) == 1: + target_buckets = set(groups.keys()) + else: + with_issues = [bucket for bucket, group in groups.items() if group["jira_issues"]] + if len(with_issues) == 1: + target_buckets = {with_issues[0]} + if not target_buckets: + repo = str(item.get("repo") or "") + project = repo.split("/")[-1] if "/" in repo else "unknown" + target_buckets.add(_epic_bucket(None, project)) + + row = {title_key: item.get(title_key), "url": item.get("url"), **(extra or {})} + for bucket in target_buckets: + group = groups.get(bucket) + if group is None: + project = bucket.split(":", 1)[-1] if bucket.startswith("__no_epic__:") else "unknown" + epic_key = None if bucket.startswith("__no_epic__:") else bucket + group = ensure_group(bucket, project=project, epic_key=epic_key, epic_name=None) + group[field].append(row) + + for pr in payload.get("pull_requests") or []: + assign_pr_or_commit( + pr, + field="pull_requests", + title_key="title", + extra={"state": pr.get("state"), "merged_at": pr.get("merged_at")}, + ) + + for commit in payload.get("commits") or []: + assign_pr_or_commit( + commit, + field="commits", + title_key="summary", + extra={"committed_at": commit.get("committed_at")}, + ) + + previous_epics = [ + { + "epic_key": entry.get("epic_key"), + "epic_name": entry.get("epic_name"), + "week_ending": entry.get("week_ending"), + "state": entry.get("state"), + } + for entry in payload.get("previous_entries") or [] + if entry.get("epic_key") + ] + + return { + "person": payload.get("person"), + "week_start": payload.get("week_start"), + "week_end": payload.get("week_end"), + "epic_groups": list(groups.values()), + "previous_epics": previous_epics, + } diff --git a/src/status/skills/schemas.py b/src/status/skills/schemas.py index 833d005..c0087ee 100644 --- a/src/status/skills/schemas.py +++ b/src/status/skills/schemas.py @@ -2,7 +2,7 @@ from typing import Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator class DraftEntry(BaseModel): @@ -19,6 +19,25 @@ class DraftEntry(BaseModel): needs_human: bool = False why_flagged: str | None = None + @field_validator("state", mode="before") + @classmethod + def _normalize_state(cls, value: str) -> str: + if not isinstance(value, str): + return value + normalized = value.strip().lower().replace("-", " ").replace("_", " ") + aliases = { + "in progress": "progressing", + "done": "shipped", + "complete": "shipped", + "completed": "shipped", + "shipped": "shipped", + "progressing": "progressing", + "slipped": "slipped", + "blocked": "blocked", + "quiet": "quiet", + } + return aliases.get(normalized, value) + class DraftOutput(BaseModel): person: str @@ -27,6 +46,18 @@ class DraftOutput(BaseModel): flags: list[str] = Field(default_factory=list) unticketed_prompt: str = "" + @field_validator("entries", "flags", mode="before") + @classmethod + def _coerce_lists(cls, value: list[DraftEntry] | list[str] | None) -> list[DraftEntry] | list[str]: + if value is None: + return [] + return value + + @field_validator("unticketed_prompt", mode="before") + @classmethod + def _coerce_unticketed_prompt(cls, value: str | None) -> str: + return "" if value is None else value + class SynthesisParticipation(BaseModel): person_id: str diff --git a/tests/test_drafter.py b/tests/test_drafter.py index a2d41b1..056d989 100644 --- a/tests/test_drafter.py +++ b/tests/test_drafter.py @@ -12,6 +12,7 @@ _empty_draft, _normalize_draft, ) +from status.skills.llm_backends import LlmBackendError from status.skills.schemas import DraftEntry, DraftOutput @@ -49,42 +50,41 @@ def test_run_drafter_retries_once_on_skill_error() -> None: with patch("status.skills.drafter.get_settings") as settings_mock: settings = settings_mock.return_value + settings.drafter_llm_backend = "skills" settings.drafter_skill_id = "skill_test" settings.drafter_skill_version = "latest" settings.anthropic_api_key = "key" settings.claude_model = "claude-sonnet-5" - with patch("status.skills.drafter.SkillClient") as client_cls: - client = client_cls.return_value - client.invoke_json.side_effect = [ - __import__("status.skills.client", fromlist=["SkillError"]).SkillError("bad json"), + with patch("status.skills.drafter.invoke_drafter_backend") as invoke_mock: + invoke_mock.side_effect = [ + LlmBackendError("bad json"), draft, ] result = run_drafter(payload) assert result.entries[0].epic_key == "EET-5493" - assert client.invoke_json.call_count == 2 + assert invoke_mock.call_count == 2 def test_run_drafter_returns_flagged_empty_after_two_failures() -> None: payload = {"person": "pilot", "week_end": "2026-08-14", "jira_issues": [], "pull_requests": []} - from status.skills.client import SkillError with patch("status.skills.drafter.get_settings") as settings_mock: settings = settings_mock.return_value + settings.drafter_llm_backend = "skills" settings.drafter_skill_id = "skill_test" settings.drafter_skill_version = "latest" settings.anthropic_api_key = "key" settings.claude_model = "claude-sonnet-5" - with patch("status.skills.drafter.SkillClient") as client_cls: - client = client_cls.return_value - client.invoke_json.side_effect = SkillError("still bad") + with patch("status.skills.drafter.invoke_drafter_backend") as invoke_mock: + invoke_mock.side_effect = LlmBackendError("still bad") result = run_drafter(payload) assert result.entries == [] assert any("failed after retry" in flag for flag in result.flags) - assert client.invoke_json.call_count == 2 + assert invoke_mock.call_count == 2 def test_normalize_draft_uses_payload_week_end() -> None: diff --git a/tests/test_evidence.py b/tests/test_evidence.py index 4fd6321..4d12a8d 100644 --- a/tests/test_evidence.py +++ b/tests/test_evidence.py @@ -2,14 +2,17 @@ from status.skills.evidence import ( build_evidence_labels, + enrich_outcome_links, filter_evidence_to_payload, inject_markdown_links, + inject_pr_markdown_links, issue_summary_index, jira_keys_from_evidence, link_phrase_from_summary, markdown_links_to_slack, merge_evidence_labels, outcome_from_linked_evidence, + pr_url_index, ) from status.skills.drafter import attach_evidence_labels, postprocess_draft from status.skills.schemas import DraftEntry, DraftOutput @@ -95,6 +98,88 @@ def test_filter_evidence_to_payload_drops_unowned_jira_keys() -> None: assert evidence == ["EET-5528", "https://github.com/org/repo/pull/1"] +def test_enrich_outcome_links_adds_jira_and_pr_links() -> None: + pr_url = "https://github.com/org/repo/pull/18" + outcome = enrich_outcome_links( + "Completed drafter improvements this week.", + "progressing", + ["EET-5519", pr_url], + {"EET-5519": "agentic weekly status pipeline"}, + {pr_url: "Improve drafter linked outcomes"}, + ) + assert "[agentic weekly status pipeline]" in outcome + assert "[Improve drafter linked outcomes]" in outcome + + +def test_postprocess_draft_sets_needs_human_for_null_epic() -> None: + draft = DraftOutput( + person="yoza", + week_ending="2026-08-28", + entries=[ + DraftEntry( + project="EET", + epic_key=None, + epic_name=None, + state="progressing", + outcome="Worked on pipeline improvements.", + evidence=["EET-5519"], + evidence_labels={"EET-5519": "agentic weekly status pipeline"}, + confidence="high", + ) + ], + ) + payload = { + "jira_issues": [{"key": "EET-5519", "summary": "Agentic Weekly Status Pipeline"}], + "pull_requests": [], + "commits": [], + "previous_entries": [], + } + processed = postprocess_draft(draft, payload) + assert processed.entries[0].needs_human is True + assert processed.entries[0].why_flagged + + +def test_postprocess_draft_adds_stale_epic_and_unticketed_flags() -> None: + draft = DraftOutput( + person="yoza", + week_ending="2026-08-28", + entries=[ + DraftEntry( + project="EET", + epic_key=None, + epic_name=None, + state="progressing", + outcome="Worked on pipeline.", + evidence=["EET-5519"], + evidence_labels={"EET-5519": "agentic weekly status pipeline"}, + confidence="high", + ) + ], + ) + payload = { + "jira_issues": [{"key": "EET-5519", "summary": "Agentic Weekly Status Pipeline"}], + "pull_requests": [ + { + "url": "https://github.com/opdev/agentic-status-report/pull/17", + "title": "M5 synthesizer pipeline", + "linked_issue_keys": [], + } + ], + "commits": [], + "previous_entries": [ + { + "epic_key": "EET-5493", + "epic_name": "OpenShift Cluster Management Bot", + "week_ending": "2026-08-14", + } + ], + } + processed = postprocess_draft(draft, payload) + assert any("EET-5493" in flag for flag in processed.flags) + assert any("no linked Jira ticket" in flag for flag in processed.flags) + assert processed.unticketed_prompt + + def test_markdown_links_to_slack() -> None: text = "See [edit flows](https://redhat.atlassian.net/browse/EET-5527)." assert text.replace( diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py new file mode 100644 index 0000000..925272b --- /dev/null +++ b/tests/test_llm_backends.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from status.skills.llm_backends import ( + DrafterBackend, + backend_config_error, + load_drafter_skill_text, + openai_chat_completions_url, +) + + +def test_backend_config_error_openai_requires_endpoint_fields() -> None: + with patch("status.skills.llm_backends.get_settings") as mock_settings: + settings = MagicMock() + settings.openai_api_key = "key" + settings.openai_base_url = None + settings.openai_model = "model" + mock_settings.return_value = settings + assert backend_config_error(DrafterBackend.OPENAI) == "OPENAI_BASE_URL not set" + + +def test_load_drafter_skill_text_reads_local_skill() -> None: + text = load_drafter_skill_text() + assert "weekly-status-drafter" in text.lower() or "entries" in text + + +def test_drafter_backend_parse_rejects_unknown() -> None: + with pytest.raises(ValueError, match="Unknown drafter backend"): + DrafterBackend.parse("anthropic") + + +def test_openai_chat_completions_url_appends_v1_path() -> None: + base = "http://example.com/agentic-workflow/model-route" + assert openai_chat_completions_url(base) == ( + "http://example.com/agentic-workflow/model-route/v1/chat/completions" + ) + assert openai_chat_completions_url(base + "/v1") == ( + "http://example.com/agentic-workflow/model-route/v1/chat/completions" + ) diff --git a/tests/test_payload_summary.py b/tests/test_payload_summary.py new file mode 100644 index 0000000..0eed4ae --- /dev/null +++ b/tests/test_payload_summary.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from status.skills.payload_summary import summarize_payload_for_openai + + +def test_summarize_payload_for_openai_groups_by_epic() -> None: + payload = { + "person": "yoza", + "week_start": "2026-08-22", + "week_end": "2026-08-28", + "jira_issues": [ + { + "key": "EET-5519", + "summary": "Agentic Weekly Status Pipeline", + "status": "In Progress", + "project": "EET", + "epic_key": None, + "epic_name": None, + "is_assignee": True, + } + ], + "pull_requests": [ + { + "url": "https://github.com/opdev/agentic-status-report/pull/18", + "title": "Improve drafter linked outcomes", + "repo": "opdev/agentic-status-report", + "state": "merged", + "linked_issue_keys": [], + } + ], + "commits": [], + "previous_entries": [ + { + "epic_key": "EET-5493", + "epic_name": "OpenShift Cluster Management Bot", + "week_ending": "2026-08-14", + } + ], + } + + summary = summarize_payload_for_openai(payload) + assert summary["person"] == "yoza" + assert len(summary["epic_groups"]) == 1 + group = summary["epic_groups"][0] + assert group["epic_key"] is None + assert group["jira_issues"][0]["key"] == "EET-5519" + assert group["pull_requests"][0]["title"] == "Improve drafter linked outcomes" + assert summary["previous_epics"][0]["epic_key"] == "EET-5493"