diff --git a/.github/scripts/validate_community_preset.py b/.github/scripts/validate_community_preset.py new file mode 100644 index 0000000000..d53134f1a0 --- /dev/null +++ b/.github/scripts/validate_community_preset.py @@ -0,0 +1,505 @@ +"""Verify a community preset submission and its generated catalog changes. + +Exit 1: confirmed submission mismatch; 2: check blocked; 3: generated files +need repair. The workflow decides labels and PR creation from these outcomes. +""" + +import argparse +import hashlib +import json +import re +import sys +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlsplit + +try: + import yaml +except ImportError: + print("BLOCKED: PyYAML is unavailable; cannot inspect published preset.yml") + sys.exit(2) + +MAX_MANIFEST_COUNT = 100 +MAX_MANIFEST_SIZE = 1024 * 1024 +MAX_TOTAL_MANIFEST_SIZE = 10 * 1024 * 1024 + + +class SubmissionMismatch(Exception): + pass + + +class Blocked(Exception): + pass + + +class GeneratedError(Exception): + pass + + +def read_text( + path: Path, context: str, error_type: type[Exception] = Blocked +) -> str: + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise error_type(f"cannot read {context} ({path}): {exc}") from exc + + +def read_json(path: Path, context: str, error_type: type[Exception]) -> dict: + try: + value = json.loads(read_text(path, context, error_type)) + except json.JSONDecodeError as exc: + raise error_type(f"{context} is invalid JSON: {exc}") from exc + if not isinstance(value, dict): + raise error_type(f"{context} must be a JSON object") + return value + + +def field(issue: dict, key: str) -> str: + value = issue.get(key) + if not isinstance(value, str) or not value.strip(): + raise SubmissionMismatch(f"missing or invalid issue field: {key}") + return value.strip() + + +def csv_values(value: str) -> list[str]: + return [part.strip() for part in value.split(",") if part.strip()] + + +def repository_parts(url: str) -> tuple[str, str]: + parsed = urlsplit(url) + match = re.fullmatch(r"/([^/]+)/([^/]+)/?", parsed.path) + if parsed.scheme != "https" or parsed.netloc.lower() != "github.com" or not match: + raise SubmissionMismatch(f"invalid GitHub repository URL: {url}") + return match[1].lower(), match[2].lower() + + +def release_tag(issue: dict) -> str: + owner, repo = repository_parts(field(issue, "repository")) + url = field(issue, "download_url") + parsed = urlsplit(url) + prefix = f"/{owner}/{repo}/" + path = parsed.path + if parsed.scheme != "https" or parsed.netloc.lower() != "github.com": + raise SubmissionMismatch(f"download URL is not a GitHub URL: {url}") + if not path.lower().startswith(prefix): + raise SubmissionMismatch("download URL does not belong to the submitted repository") + suffix = path[len(prefix):] + archive = re.fullmatch(r"archive/refs/tags/([^/]+)\.zip", suffix) + release = re.fullmatch(r"releases/download/([^/]+)/[^/]+\.zip", suffix) + if not (archive or release) or parsed.query or parsed.fragment: + raise SubmissionMismatch(f"download URL is not a pinned ZIP release: {url}") + tag = (archive or release)[1] + version = field(issue, "version") + if not re.fullmatch(r"\d+\.\d+\.\d+", version) or not re.fullmatch( + rf"(?:v?{re.escape(version)}|.+-v?{re.escape(version)})", tag + ): + raise SubmissionMismatch( + f"release tag {tag!r} does not match submitted version {version!r}" + ) + return tag + + +def published_manifest(archive_path: Path, preset_id: str) -> tuple[dict, bool]: + try: + with zipfile.ZipFile(archive_path) as archive: + matching = [] + invalid = [] + manifests = [ + member for member in archive.infolist() + if not member.is_dir() + and member.filename.rsplit("/", 1)[-1] == "preset.yml" + ] + if len(manifests) > MAX_MANIFEST_COUNT: + raise SubmissionMismatch( + f"archive contains more than {MAX_MANIFEST_COUNT} preset.yml files" + ) + total_size = sum(member.file_size for member in manifests) + if total_size > MAX_TOTAL_MANIFEST_SIZE: + raise SubmissionMismatch( + "archive preset.yml files exceed the " + f"{MAX_TOTAL_MANIFEST_SIZE // (1024 * 1024)} MiB total limit" + ) + for member in manifests: + try: + if member.file_size > MAX_MANIFEST_SIZE: + raise ValueError("preset.yml exceeds 1 MiB") + with archive.open(member) as stream: + data = yaml.safe_load(stream.read().decode("utf-8")) + except (yaml.YAMLError, UnicodeError, ValueError) as exc: + invalid.append(f"{member.filename}: {exc}") + continue + if isinstance(data, dict) and isinstance(data.get("preset"), dict): + if data["preset"].get("id") == preset_id: + matching.append((member.filename, data)) + except FileNotFoundError as exc: + raise Blocked(f"downloaded archive is unavailable at {archive_path}: {exc}") from exc + except PermissionError as exc: + raise Blocked(f"cannot read downloaded archive at {archive_path}: {exc}") from exc + except OSError as exc: + raise Blocked(f"cannot read downloaded archive at {archive_path}: {exc}") from exc + except (zipfile.BadZipFile, EOFError) as exc: + raise SubmissionMismatch(f"downloaded archive is not a readable ZIP: {exc}") from exc + if len(matching) != 1: + if not matching and invalid: + raise SubmissionMismatch( + f"no published preset.yml for {preset_id!r}; invalid manifests: " + + "; ".join(invalid) + ) + raise SubmissionMismatch( + f"expected one published preset.yml for {preset_id!r}, found {len(matching)}" + ) + return matching[0][1], len(manifests) == 1 + + +def required_extensions(manifest: dict) -> list[str]: + requires = manifest.get("requires") + if not isinstance(requires, dict): + raise SubmissionMismatch("published preset.yml has no requires mapping") + extensions = requires.get("extensions", []) + if not isinstance(extensions, list): + raise SubmissionMismatch("published requires.extensions must be a list") + result = [] + for item in extensions: + if isinstance(item, str): + result.append(item) + elif isinstance(item, dict) and isinstance(item.get("id"), str): + if item.get("required", True) is not False: + result.append(item["id"]) + else: + raise SubmissionMismatch("invalid published requires.extensions entry") + return result + + +def check_readme(text: str, issue: dict, *, single_preset_archive: bool) -> None: + expected = field(issue, "download_url") + preset_id = field(issue, "preset_id") + owner, repo = repository_parts(field(issue, "repository")) + submitted_scope = re.fullmatch(r"(.+)-v?\d+\.\d+\.\d+", release_tag(issue)) + accepted_scopes = {preset_id} + if submitted_scope: + accepted_scopes.add(submitted_scope[1]) + expected_path = urlsplit(expected).path.split("/") + accepted = False + for match in re.finditer( + r"(?--from|--dev|[a-z][a-z0-9-]*)" + r"(?:\s+(?P[^\s`]+))?", + text, + ): + option = match["option"] + raw_value = match["value"] or "" + if option == "--from": + value = raw_value.strip("'\"<>(),.;") + if value == expected: + accepted = True + continue + parsed = urlsplit(value) + parts = parsed.path.split("/") + tag = ( + parts[5] if len(parts) > 5 and parts[3] == "releases" + else parts[6].removesuffix(".zip") + if len(parts) > 6 and parts[3] == "archive" + else "" + ) + scoped = re.fullmatch(r"(.+)-v?\d+\.\d+\.\d+", tag) + same_repository = ( + parsed.netloc.lower() == "github.com" + and parsed.path.lower().startswith(f"/{owner}/{repo}/") + ) + # An unscoped tag alone does not identify a preset in a monorepo. + same_release_asset = ( + len(expected_path) > 6 + and expected_path[3:5] == ["releases", "download"] + and len(parts) > 6 + and parts[3:5] == ["releases", "download"] + and expected_path[6] == parts[6] + ) + unscoped_archive = ( + single_preset_archive + and submitted_scope is None + and expected_path[3:6] == ["archive", "refs", "tags"] + and parts[3:6] == ["archive", "refs", "tags"] + ) + if (scoped and scoped[1] in accepted_scopes) or ( + same_repository and not scoped and (same_release_asset or unscoped_archive) + ): + raise SubmissionMismatch( + f"README --from URL for {preset_id} differs from Download URL: {value}" + ) + elif option == "--dev": + value = raw_value.strip("'\"") + if value and not value.startswith("-"): + accepted = True + elif option == preset_id: + accepted = True + if not accepted: + raise SubmissionMismatch( + "README has no valid specify preset add command for the submitted preset" + ) + + +def count_items(value: str) -> int: + return sum(line.lstrip().startswith("- ") for line in value.splitlines()) + + +def expected_values(issue: dict, manifest: dict, digest: str) -> dict: + pack = manifest.get("preset") + requires = manifest.get("requires") + if not isinstance(pack, dict) or not isinstance(requires, dict): + raise SubmissionMismatch("published preset.yml lacks preset or requires metadata") + extension_ids = csv_values(issue.get("required_extensions", "")) + actual_extensions = required_extensions(manifest) + comparisons = { + "preset.id": (pack.get("id"), field(issue, "preset_id")), + "preset.version": (pack.get("version"), field(issue, "version")), + "requires.speckit_version": ( + requires.get("speckit_version"), field(issue, "speckit_version") + ), + "requires.extensions": (sorted(actual_extensions), sorted(extension_ids)), + } + for name, (actual, expected) in comparisons.items(): + if actual != expected: + raise SubmissionMismatch( + f"published {name} {actual!r} differs from submitted {expected!r}" + ) + dependencies = {"speckit_version": field(issue, "speckit_version")} + if extension_ids: + dependencies["extensions"] = extension_ids + provided = { + "templates": count_items(field(issue, "templates_provided")), + "commands": count_items(field(issue, "commands_provided")), + } + scripts = issue.get("scripts_count", "0") + if scripts: + try: + number = int(scripts) + except (ValueError, TypeError) as exc: + raise SubmissionMismatch("scripts_count must be a non-negative integer") from exc + if number < 0: + raise SubmissionMismatch("scripts_count must be a non-negative integer") + if number: + provided["scripts"] = number + return { + "id": field(issue, "preset_id"), + "name": field(issue, "preset_name"), + "version": field(issue, "version"), + "description": field(issue, "description"), + "author": field(issue, "author"), + "repository": field(issue, "repository"), + "homepage": field(issue, "repository"), + "download_url": field(issue, "download_url"), + "sha256": digest, + "documentation": field(issue, "documentation"), + "license": field(issue, "license"), + "requires": dependencies, + "provides": provided, + "tags": csv_values(field(issue, "tags")), + } + + +def submission(args: argparse.Namespace) -> None: + issue = read_json(args.issue, "issue input", Blocked) + release_tag(issue) + manifest, single_preset_archive = published_manifest( + args.archive, field(issue, "preset_id") + ) + check_readme( + read_text(args.readme, "fetched README"), + issue, + single_preset_archive=single_preset_archive, + ) + try: + with args.archive.open("rb") as archive: + digest = hashlib.file_digest(archive, "sha256").hexdigest() + except OSError as exc: + raise Blocked(f"cannot hash downloaded archive: {exc}") from exc + expected = expected_values(issue, manifest, digest) + catalog = read_json(args.catalog, "original catalog", Blocked) + entries = catalog.get("presets") + if not isinstance(entries, dict): + raise Blocked("original catalog has no presets object") + previous = entries.get(expected["id"]) + if previous is not None and not isinstance(previous, dict): + raise Blocked("original catalog entry is not an object") + if previous is not None and not isinstance(previous.get("created_at"), str): + raise Blocked("original catalog entry has no created_at to preserve") + original_rows = documentation_rows( + read_text(args.docs, "original documentation", Blocked), Blocked + ) + expected_row = documentation_row(expected) + previous_row = None + if previous is not None: + try: + previous_row = documentation_row(previous) + except (AttributeError, KeyError, TypeError) as exc: + raise Blocked( + f"original catalog entry cannot produce a documentation row: {exc}" + ) from exc + snapshot = { + "expected": expected, + "created_at": previous.get("created_at") if previous else None, + "expected_timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z"), + "documentation": { + "expected_row_count": original_rows.count(expected_row), + "previous_row": previous_row, + "previous_row_count": original_rows.count(previous_row) + if previous_row is not None else 0, + }, + } + try: + args.snapshot.write_text(json.dumps(snapshot), encoding="utf-8") + except OSError as exc: + raise Blocked(f"cannot write verifier snapshot: {exc}") from exc + print("PASSED: published preset.yml, release tag, README, and issue fields agree") + + +def documentation_row(entry: dict) -> str: + def cell(value: str) -> str: + return value.replace("|", r"\|").replace("\n", " ") + + provided = entry["provides"] + quantities = [ + f"{count} {name[:-1] if count == 1 else name}" + for name, count in provided.items() if count + ] + dependencies = entry["requires"].get("extensions", []) + requires = ", ".join(f"{name} extension" for name in dependencies) or "—" + repo = entry["repository"].rstrip("/").rsplit("/", 1)[-1] + return ( + f"| {cell(entry['name'])} | {cell(entry['description'])} | " + f"{', '.join(quantities)} | {requires} | " + f"[{repo}]({entry['repository']}) |" + ) + + +def markdown_table_cells(row: str) -> list[str]: + cells = [] + cell = [] + escaped = False + for character in row: + if escaped: + if character == "|": + cell.append(character) + else: + cell.extend(("\\", character)) + escaped = False + elif character == "\\": + escaped = True + elif character == "|": + cells.append("".join(cell).strip()) + cell = [] + else: + cell.append(character) + if escaped: + cell.append("\\") + cells.append("".join(cell).strip()) + return cells + + +def documentation_rows(text: str, error_type: type[Exception]) -> list[str]: + lines = text.splitlines() + try: + start = next(i for i, line in enumerate(lines) if line.startswith("| Preset |")) + except StopIteration as exc: + raise error_type("documentation has no Community Presets table") from exc + rows = [] + for line in lines[start + 2:]: + if not line.startswith("|"): + break + rows.append(line.strip()) + return rows + + +def generated(args: argparse.Namespace) -> None: + snapshot = read_json(args.snapshot, "verifier snapshot", Blocked) + expected = snapshot.get("expected") + if not isinstance(expected, dict) or not isinstance(expected.get("id"), str): + raise Blocked("verifier snapshot lacks validated expected values") + expected_timestamp = snapshot.get("expected_timestamp") + if not isinstance(expected_timestamp, str): + raise Blocked("verifier snapshot lacks the expected UTC timestamp") + catalog = read_json(args.catalog, "generated catalog", GeneratedError) + entries = catalog.get("presets") + if not isinstance(entries, dict): + raise GeneratedError("generated catalog has no presets object") + keys = list(entries) + if keys != sorted(keys): + raise GeneratedError("catalog preset IDs are not in alphabetical order") + entry = entries.get(expected["id"]) + if not isinstance(entry, dict): + raise GeneratedError(f"catalog is missing entry {expected['id']}") + for key, value in expected.items(): + if entry.get(key) != value: + raise GeneratedError(f"catalog {key} does not match validated value: {value!r}") + if snapshot.get("created_at") is not None: + if entry.get("created_at") != snapshot["created_at"]: + raise GeneratedError("catalog created_at was not preserved on update") + elif entry.get("created_at") != expected_timestamp: + raise GeneratedError("new catalog created_at does not match the expected UTC date") + if entry.get("updated_at") != expected_timestamp: + raise GeneratedError("catalog entry updated_at does not match the expected UTC date") + if catalog.get("updated_at") != expected_timestamp: + raise GeneratedError("catalog top-level updated_at does not match the expected UTC date") + documentation = snapshot.get("documentation") + if not isinstance(documentation, dict): + raise Blocked("verifier snapshot lacks original documentation state") + expected_row_count = documentation.get("expected_row_count") + previous_row = documentation.get("previous_row") + previous_row_count = documentation.get("previous_row_count") + if ( + not isinstance(expected_row_count, int) + or isinstance(expected_row_count, bool) + or expected_row_count < 0 + or previous_row is not None and not isinstance(previous_row, str) + or not isinstance(previous_row_count, int) + or isinstance(previous_row_count, bool) + or previous_row_count < 0 + ): + raise Blocked("verifier snapshot has invalid original documentation state") + rows = documentation_rows( + read_text(args.docs, "generated documentation", GeneratedError), + GeneratedError, + ) + names = [markdown_table_cells(row)[1] for row in rows] + if names != sorted(names, key=str.casefold): + raise GeneratedError("documentation preset names are not in alphabetical order") + expected_row = documentation_row(expected) + expected_generated_count = expected_row_count + ( + 0 if previous_row == expected_row else 1 + ) + if rows.count(expected_row) != expected_generated_count: + raise GeneratedError(f"documentation row does not match validated values: {expected_row}") + if previous_row is not None and previous_row != expected_row: + expected_previous_count = max(0, previous_row_count - 1) + if rows.count(previous_row) != expected_previous_count: + raise GeneratedError("previous documentation row was not replaced") + print("PASSED: generated catalog and documentation match validated submission") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("submission", "generated")) + for name in ("issue", "archive", "readme", "catalog", "docs", "snapshot"): + parser.add_argument(f"--{name}", required=True, type=Path) + args = parser.parse_args() + try: + if args.phase == "submission": + submission(args) + else: + generated(args) + except SubmissionMismatch as exc: + print(f"FAILED: {exc}") + return 1 + except Blocked as exc: + print(f"BLOCKED: {exc}") + return 2 + except GeneratedError as exc: + print(f"REPAIR: {exc}") + return 3 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/add-community-preset.lock.yml b/.github/workflows/add-community-preset.lock.yml index 5b6cb10812..5fd10b3cf1 100644 --- a/.github/workflows/add-community-preset.lock.yml +++ b/.github/workflows/add-community-preset.lock.yml @@ -1,5 +1,5 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"e692354fdb52cf49c3f46096529618b23eb09336fe58fe2392b77210b374b568","body_hash":"cd1920f05bf8608b46191a3802a1c2f6c595ab3ce28034116e3346d4988bed4a","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_pull_request","missing_data","missing_tool","noop","remove_labels"]}]} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"499cb209b6fc93a3b306cb0113aefb2367be2a84d952c4bb4516f982f535c25c","body_hash":"67f2a3313f59103002b9e362e4d16ad1c3ef4035afb6c96f3446a345e4da4852","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/setup-python","sha":"5fda3b95a4ea91299a34e894583c3862153e4b97","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_pull_request","missing_data","missing_tool","noop","remove_labels"]}]} # This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # # ___ _ _ @@ -41,6 +41,7 @@ # - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 # - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) # - actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 +# - actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 # - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 # - github/gh-aw-actions/setup@5e508589e03a7757a7e05b26e834292f5445bfb6 # v0.88.7 # @@ -491,6 +492,15 @@ jobs: with: name: activation path: /tmp/gh-aw + - continue-on-error: true + name: Set up Python for preset verification + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + - continue-on-error: true + name: Install preset verifier dependency + run: python3 -m pip install 'PyYAML==6.0.3' + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/add-community-preset.md b/.github/workflows/add-community-preset.md index f6ccf01c8f..80608481e7 100644 --- a/.github/workflows/add-community-preset.md +++ b/.github/workflows/add-community-preset.md @@ -39,6 +39,16 @@ permissions: checkout: fetch-depth: 0 +steps: + - name: Set up Python for preset verification + continue-on-error: true + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + - name: Install preset verifier dependency + continue-on-error: true + run: python3 -m pip install 'PyYAML==6.0.3' + safe-outputs: noop: report-as-issue: false @@ -130,7 +140,8 @@ deciding pass/fail: ### 2c. Repository validation - Fetch the repository URL — confirm it exists and is publicly accessible -- Confirm the repository contains a `preset.yml` file +- Confirm the repository contains a `preset.yml` file (in the preset's + subdirectory for a monorepo) - Confirm the repository contains a `LICENSE` file > The README requirement is enforced once, in **Step 2d**, against the specific file the @@ -170,10 +181,15 @@ preset** — not just any file named `README.md`, and not a product/framework pi - `specify preset add --dev ` A `specify preset add --from ` command only counts when its `` **matches the - submitted Download URL exactly**. A `--from` command pointing at a *different* URL does - **not** satisfy the install-command requirement (treat it as if absent) — but the README - may still pass on one of the other accepted forms (`specify preset add ` or - `specify preset add --dev `). + submitted Download URL exactly**. If the README also contains a `--from` release URL + identifiable as this preset by its tag scope or matching release asset that differs + from the Download URL, **fail** even if another accepted command (`specify preset add + ` or `specify preset add --dev `) is present. Do not flag a different + preset's unscoped release URL in a monorepo as stale. Bare archive tags are only + compared when the downloaded archive contains one `preset.yml`: a bare tag alone + cannot identify which preset it belongs to in a multi-preset archive. A README + with only a valid `--dev` command remains acceptable. The verifier in Step 2g + enforces this comparison. If **no** accepted `specify preset add ...` command is present, the README is treated as a generic description/pitch rather than preset-usage documentation — **fail this check** and @@ -258,6 +274,35 @@ substitute for fetching the archive. Never execute downloaded content. - Confirm that all required checkboxes in the Testing Checklist and Submission Requirements sections are checked (`[x]`) +### 2g. Reproducible published-artifact and README comparison + +After the pinned download has returned HTTP 200 and the optional checksum comparison +has succeeded, use the edit tool to save the fetched **exact documentation README** +as `/tmp/gh-aw/preset-readme.md`. Use the edit tool to save +`/tmp/gh-aw/preset-submission.json` as a JSON object with these keys copied from the +issue form (not inferred from the README or archive): +`preset_id`, `preset_name`, `version`, `description`, `author`, `repository`, +`download_url`, `documentation`, `license`, `speckit_version`, +`required_extensions` (empty string when absent), `templates_provided`, +`commands_provided`, `scripts_count` (string `"0"` when absent), and `tags`. +Retain multiline list values as strings. Do not interpolate issue or README text +into shell commands. Run this fixed command unchanged: + +```bash +python3 .github/scripts/validate_community_preset.py submission --issue /tmp/gh-aw/preset-submission.json --archive /tmp/gh-aw/community-archive.zip --readme /tmp/gh-aw/preset-readme.md --catalog presets/catalog.community.json --docs docs/community/presets.md --snapshot /tmp/gh-aw/preset-validation.json +``` + +The verifier reads `preset.yml` from the downloaded ZIP, including preset-scoped +paths in monorepos; it parses YAML without executing archive content. It checks +the published ID, version, Spec Kit requirement, required extension IDs, release +tag, and README install references against the issue, then records the validated +values, current UTC date at submission validation, and existing `created_at` +for the generated-file check. Exit 1 (`FAILED`) +is a confirmed submission mismatch: report it on the issue under Failed, with +no PR. Exit 2 (`BLOCKED`) means the verifier could not read the archive or other +required inputs (including a missing Python dependency): report the exact error +and run link under Blocked. Do not treat either exit as a pass. + ### Validation outcome Choose exactly one outcome below, in order. A check that could not run is @@ -291,9 +336,10 @@ If there are no environment blockers and a completed check found a submission de #### Passed If there are no environment blockers and every required check completed and passed: -1. Remove `validation-failed` -2. Add the `validation-passed` label -3. Continue to Step 3 +1. Remove any stale `validation-passed` and `validation-failed` labels from + earlier runs. +2. Continue to Step 3. **Do not add `validation-passed` yet**: the generated + catalog and documentation must pass Step 5's verifier before success is recorded. ## Step 3 — Determine Add vs Update @@ -329,7 +375,7 @@ Insert the entry in **alphabetical order by preset ID** within the "repository": "", "download_url": "", "sha256": "", - "homepage": "", + "homepage": "", "documentation": "", "license": "", "requires": { @@ -340,8 +386,8 @@ Insert the entry in **alphabetical order by preset ID** within the "commands": }, "tags": ["", ""], - "created_at": "T00:00:00Z", - "updated_at": "T00:00:00Z" + "created_at": "T00:00:00Z", + "updated_at": "T00:00:00Z" } } ``` @@ -361,8 +407,11 @@ If the preset provides scripts, add `"scripts": ` inside `"provides"`. ### For an update Replace only the changed fields (typically `version`, `download_url`, -`description`, `provides`, `requires`, `tags`, `updated_at`). **Preserve** -`created_at` from the existing entry. +`description`, `homepage`, `provides`, `requires`, `tags`, `updated_at`). Set +`homepage` to the submitted repository URL; the form has no separate homepage +field. **Preserve** +`created_at` from the existing entry. Use the verifier snapshot's validated UTC +date for `updated_at`, even if the UTC date changes during the run. ### Counting templates and commands @@ -372,16 +421,11 @@ Parse the "Templates Provided" and "Commands Provided" issue fields: ### After editing -Update the **top-level `"updated_at"` timestamp** in the catalog to today's date -in ISO 8601 format. +Update the **top-level `"updated_at"` timestamp** in the catalog to the +verifier snapshot's UTC date in ISO 8601 format. -Validate the JSON by running: - -```bash -python3 -c "import json; json.load(open('presets/catalog.community.json')); print('Valid JSON')" -``` - -If validation fails, fix the JSON and re-validate before continuing. +The generated-file verifier in Step 5 parses and checks the JSON. Fix any +catalog error it reports and rerun it before continuing. ## Step 5 — Update `docs/community/presets.md` @@ -398,7 +442,8 @@ Insert a new row in **alphabetical order by preset name**: For the Requires column: - Use `—` if no extensions are required -- List required extension names if any (e.g., `AIDE extension`) +- List required extension IDs if any (e.g., `aide extension`), comma-separated +- Omit zero-count kinds from Provides and use singular nouns for counts of one If the preset provides scripts, include them: ` templates, commands, scripts` @@ -406,9 +451,34 @@ If the preset provides scripts, include them: ` templates, commands, Find the existing row and update any changed fields in-place. +### Verify the generated files + +Before labeling success or requesting a PR, run this fixed command unchanged: + +```bash +python3 .github/scripts/validate_community_preset.py generated --issue /tmp/gh-aw/preset-submission.json --archive /tmp/gh-aw/community-archive.zip --readme /tmp/gh-aw/preset-readme.md --catalog presets/catalog.community.json --docs docs/community/presets.md --snapshot /tmp/gh-aw/preset-validation.json +``` + +The verifier checks JSON parsing, the validated catalog metadata (including +`homepage` set to the submitted repository URL) and digest, +the top-level and entry `updated_at` timestamps against the recorded UTC date +(and `created_at` for new entries), +alphabetical ID order, the documentation row's name, purpose, counts, extension +requirements and repository link, alphabetical name order, and preservation of +`created_at` on updates. Exit 3 (`REPAIR`) is an agent-generated catalog or +documentation error, **not** a submitter defect: fix the files and re-run this +command until it exits 0. Do not label `validation-failed` for an error in +generated files. Exit 2 (`BLOCKED`) means an input or tool is unavailable: report +the exact error and workflow run link to the maintainer, remove +`validation-passed`, and stop without a PR. If a generated-file error cannot be +corrected, remove `validation-passed`, stop without a PR, and report the problem for +maintainer investigation. + ## Step 6 — Create Pull Request -Create a pull request with the changes. Use this branch naming convention: +Only after the generated-file verifier exits 0, add the `validation-passed` +label and request the configured draft pull request with the changes. Use this +branch naming convention: This repository-owned gh-aw maintenance workflow does not perform the contributor open-PR count check or request confirmation. After successful validation and diff --git a/tests/test_community_preset_validation.py b/tests/test_community_preset_validation.py new file mode 100644 index 0000000000..847734791c --- /dev/null +++ b/tests/test_community_preset_validation.py @@ -0,0 +1,682 @@ +"""Regression coverage for the repository-owned preset submission verifier.""" + +import hashlib +import json +import subprocess +import sys +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parent.parent +VERIFIER = ROOT / ".github" / "scripts" / "validate_community_preset.py" +WORKFLOW = ROOT / ".github" / "workflows" / "add-community-preset.md" + + +@pytest.fixture +def submission(tmp_path): + issue = { + "preset_id": "sample", + "preset_name": "Sample Preset", + "version": "1.2.3", + "description": "Sample usage", + "author": "Contributor", + "repository": "https://github.com/example/presets", + "download_url": "https://github.com/example/presets/releases/download/sample-v1.2.3/sample.zip", + "documentation": "https://github.com/example/presets/blob/main/sample/README.md", + "license": "MIT", + "speckit_version": ">=1.0.0", + "required_extensions": "aide, canon", + "templates_provided": "- spec-template.md", + "commands_provided": "- speckit.plan.md", + "scripts_count": "0", + "tags": "sample, example", + } + paths = {name: tmp_path / name for name in ( + "issue.json", "archive.zip", "README.md", "catalog.json", "presets.md", + "snapshot.json", + )} + paths["README.md"].write_text( + f"specify preset add --from {issue['download_url']}\n", encoding="utf-8" + ) + manifest = { + "preset": {"id": "sample", "version": "1.2.3"}, + "requires": {"speckit_version": ">=1.0.0", "extensions": ["aide", "canon"]}, + } + with zipfile.ZipFile(paths["archive.zip"], "w") as archive: + archive.writestr("presets-release/sample/preset.yml", yaml.safe_dump(manifest)) + archive.writestr("presets-release/other/preset.yml", yaml.safe_dump({ + "preset": {"id": "other", "version": "9.9.9"}, + "requires": {"speckit_version": ">=0.1.0"}, + })) + issue["actual_sha256"] = hashlib.sha256(paths["archive.zip"].read_bytes()).hexdigest() + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["catalog.json"].write_text( + json.dumps({"presets": {}}), encoding="utf-8" + ) + paths["presets.md"].write_text( + "| Preset | Purpose | Provides | Requires | URL |\n" + "|--------|---------|----------|----------|-----|\n", + encoding="utf-8", + ) + return issue, manifest, paths + + +def run_verifier(paths, phase="submission"): + return subprocess.run( + [ + sys.executable, str(VERIFIER), phase, + "--issue", str(paths["issue.json"]), + "--archive", str(paths["archive.zip"]), + "--readme", str(paths["README.md"]), + "--catalog", str(paths["catalog.json"]), + "--docs", str(paths["presets.md"]), + "--snapshot", str(paths["snapshot.json"]), + ], + capture_output=True, text=True, check=False, + ) + + +def write_archive(paths, manifest): + with zipfile.ZipFile(paths["archive.zip"], "w") as archive: + archive.writestr("release/sample/preset.yml", yaml.safe_dump(manifest)) + + +def write_generated(issue, paths, *, created_at=None): + timestamp = ( + json.loads(paths["snapshot.json"].read_text(encoding="utf-8"))["expected_timestamp"] + if paths["snapshot.json"].exists() + else "2026-01-01T00:00:00Z" + ) + entry = { + "id": issue["preset_id"], "name": issue["preset_name"], + "version": issue["version"], "description": issue["description"], + "author": issue["author"], "repository": issue["repository"], + "download_url": issue["download_url"], "sha256": issue["actual_sha256"], + "homepage": issue["repository"], "documentation": issue["documentation"], + "license": issue["license"], + "requires": {"speckit_version": issue["speckit_version"], + "extensions": ["aide", "canon"]}, + "provides": {"templates": 1, "commands": 1}, + "tags": ["sample", "example"], + "created_at": created_at if created_at is not None else timestamp, + "updated_at": timestamp, + } + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], "presets": {"sample": entry}, + }), encoding="utf-8") + preset_name = issue["preset_name"].replace("|", r"\|") + description = issue["description"].replace("|", r"\|") + paths["presets.md"].write_text( + "| Preset | Purpose | Provides | Requires | URL |\n" + "|--------|---------|----------|----------|-----|\n" + f"| {preset_name} | {description} | 1 template, 1 command | " + "aide extension, canon extension | " + "[presets](https://github.com/example/presets) |\n", + encoding="utf-8", + ) + return entry + + +def test_matching_monorepo_submission_and_generated_files_pass(submission): + issue, _, paths = submission + before = datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z") + first = run_verifier(paths) + assert first.returncode == 0, first.stdout + first.stderr + snapshot = json.loads(paths["snapshot.json"].read_text(encoding="utf-8")) + after = datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z") + assert snapshot["expected_timestamp"] in (before, after) + write_generated(issue, paths) + second = run_verifier(paths, "generated") + assert second.returncode == 0, second.stdout + second.stderr + + +def test_update_preserving_created_at_passes(submission): + issue, _, paths = submission + original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + paths["catalog.json"].write_text(json.dumps({ + "updated_at": original["updated_at"], "presets": {"sample": original}, + }), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + assert run_verifier(paths, "generated").returncode == 0 + + +@pytest.mark.parametrize(("change", "message"), [ + ({"preset": {"id": "sample", "version": "1.2.4"}}, "version"), + ({"requires": {"speckit_version": ">=2.0.0", + "extensions": ["aide", "canon"]}}, "speckit_version"), + ({"requires": {"speckit_version": ">=1.0.0", + "extensions": ["aide"]}}, "extensions"), +]) +def test_published_manifest_mismatch_is_submission_failure(submission, change, message): + _, manifest, paths = submission + manifest.update(change) + write_archive(paths, manifest) + result = run_verifier(paths) + assert result.returncode == 1 + assert message in result.stdout + assert not paths["snapshot.json"].exists() + + +def test_release_tag_mismatch_is_submission_failure(submission): + issue, _, paths = submission + issue["download_url"] = issue["download_url"].replace("v1.2.3", "v1.2.4") + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + assert run_verifier(paths).returncode == 1 + + +def test_stale_from_url_fails_even_with_valid_dev_command(submission): + issue, _, paths = submission + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n", + encoding="utf-8", + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "README" in result.stdout + + +@pytest.mark.parametrize("archive_url", [False, True]) +def test_stale_from_url_with_submitted_scoped_tag_fails(submission, archive_url): + issue, _, paths = submission + if archive_url: + issue["download_url"] = ( + "https://github.com/example/presets/archive/refs/tags/" + "spec-kit-sample-v1.2.3.zip" + ) + else: + issue["download_url"] = issue["download_url"].replace( + "sample-v1.2.3", "spec-kit-sample-v1.2.3" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n", + encoding="utf-8", + ) + result = run_verifier(paths) + assert result.returncode == 1, result.stdout + result.stderr + assert "README --from URL" in result.stdout + + +def test_dev_only_readme_is_accepted(submission): + _, _, paths = submission + paths["README.md"].write_text( + "specify preset add --dev ./sample\n", encoding="utf-8" + ) + assert run_verifier(paths).returncode == 0 + + +def test_dev_current_directory_path_is_accepted(submission): + _, _, paths = submission + paths["README.md"].write_text( + "specify preset add --dev .\n", encoding="utf-8" + ) + assert run_verifier(paths).returncode == 0 + + +def test_dev_option_without_path_is_rejected(submission): + _, _, paths = submission + paths["README.md"].write_text( + "specify preset add --dev --priority 20\n", encoding="utf-8" + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "README" in result.stdout + + +def test_quoted_from_url_with_sentence_punctuation_is_accepted(submission): + issue, _, paths = submission + paths["README.md"].write_text( + f'Spec Kit install: `specify preset add --from "{issue["download_url"]}".`\n', + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + +def test_id_install_and_unrelated_monorepo_release_are_accepted(submission): + _, _, paths = submission + paths["README.md"].write_text( + "specify preset add sample\n" + "specify preset add --from " + "https://github.com/example/presets/releases/download/other-v2.0.0/other.zip\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + +def test_unrelated_scoped_release_stays_accepted_with_submitted_scope(submission): + issue, _, paths = submission + issue["download_url"] = issue["download_url"].replace( + "sample-v1.2.3", "spec-kit-sample-v1.2.3" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add sample\n" + "specify preset add --from " + "https://github.com/example/presets/releases/download/other-v2.0.0/other.zip\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + +@pytest.mark.parametrize("archive_url", [False, True]) +def test_unrelated_unscoped_monorepo_release_stays_accepted(submission, archive_url): + issue, _, paths = submission + if archive_url: + issue["download_url"] = ( + "https://github.com/example/presets/archive/refs/tags/" + "spec-kit-sample-v1.2.3.zip" + ) + unrelated = ( + "https://github.com/example/presets/archive/refs/tags/v2.0.0.zip" + ) + else: + issue["download_url"] = issue["download_url"].replace( + "sample-v1.2.3", "spec-kit-sample-v1.2.3" + ) + unrelated = ( + "https://github.com/example/presets/releases/download/v2.0.0/other.zip" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + f"specify preset add --from {unrelated}\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + +def test_unrelated_unscoped_archive_stays_accepted_in_monorepo(submission): + issue, _, paths = submission + issue["download_url"] = ( + "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + "specify preset add --from " + "https://github.com/example/presets/archive/refs/tags/v2.0.0.zip\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + +def test_stale_unscoped_archive_fails_for_single_preset(submission): + issue, manifest, paths = submission + write_archive(paths, manifest) + issue["download_url"] = ( + "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + "specify preset add --from " + "https://github.com/example/presets/archive/refs/tags/v1.2.2.zip\n", + encoding="utf-8", + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "README --from URL" in result.stdout + + +def test_matching_unscoped_archive_passes_for_single_preset(submission): + issue, manifest, paths = submission + write_archive(paths, manifest) + issue["download_url"] = ( + "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + f"specify preset add --from {issue['download_url']}\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + +def test_same_asset_on_unscoped_tag_is_reported_as_stale(submission): + issue, _, paths = submission + issue["download_url"] = issue["download_url"].replace( + "sample-v1.2.3", "v1.2.3" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n", + encoding="utf-8", + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "README --from URL" in result.stdout + + +def test_same_asset_on_bare_tag_stays_stale_for_scoped_submission(submission): + issue, _, paths = submission + issue["download_url"] = issue["download_url"].replace( + "sample-v1.2.3", "spec-kit-sample-v1.2.3" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + "specify preset add --from " + "https://github.com/example/presets/releases/download/v1.2.2/sample.zip\n", + encoding="utf-8", + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "README --from URL" in result.stdout + + +def test_stale_scoped_release_in_another_repository_fails(submission): + _, _, paths = submission + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + "specify preset add --from " + "https://github.com/elsewhere/presets/releases/download/sample-v1.2.2/sample.zip\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 1 + + +def test_optional_manifest_extension_is_not_required(submission): + issue, manifest, paths = submission + manifest["requires"]["extensions"].append({ + "id": "optional", "version": ">=1.0.0", "required": False, + }) + write_archive(paths, manifest) + issue["actual_sha256"] = hashlib.sha256(paths["archive.zip"].read_bytes()).hexdigest() + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + + +def test_invalid_unrelated_monorepo_manifest_does_not_mask_match(submission): + _, manifest, paths = submission + with zipfile.ZipFile(paths["archive.zip"], "w") as archive: + archive.writestr("release/sample/preset.yml", yaml.safe_dump(manifest)) + archive.writestr("release/other/preset.yml", "preset: [invalid\n") + assert run_verifier(paths).returncode == 0 + + +def test_archive_with_too_many_manifests_is_rejected_before_parsing(submission): + _, _, paths = submission + with zipfile.ZipFile(paths["archive.zip"], "w") as archive: + for index in range(101): + archive.writestr(f"release/{index}/preset.yml", "preset: [invalid\n") + result = run_verifier(paths) + assert result.returncode == 1 + assert "more than 100 preset.yml files" in result.stdout + assert "invalid manifests" not in result.stdout + + +def test_archive_with_excessive_total_manifest_size_is_rejected(submission): + _, manifest, paths = submission + padding = "#" * (1024 * 1024 - len(yaml.safe_dump(manifest)) - 2) + with zipfile.ZipFile( + paths["archive.zip"], "w", compression=zipfile.ZIP_DEFLATED + ) as archive: + for index in range(11): + archive.writestr( + f"release/{index}/preset.yml", + f"{yaml.safe_dump(manifest)}\n{padding}", + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "10 MiB total limit" in result.stdout + + +def test_missing_archive_is_blocked_not_failed(submission): + _, _, paths = submission + paths["archive.zip"].unlink() + result = run_verifier(paths) + assert result.returncode == 2 + assert "BLOCKED" in result.stdout + + +def test_missing_generated_documentation_is_repairable(submission): + issue, _, paths = submission + assert run_verifier(paths).returncode == 0 + write_generated(issue, paths) + paths["presets.md"].unlink() + result = run_verifier(paths, "generated") + assert result.returncode == 3 + assert "REPAIR" in result.stdout + + +def test_existing_entry_without_creation_date_blocks_update(submission): + issue, _, paths = submission + entry = write_generated(issue, paths) + del entry["created_at"] + paths["catalog.json"].write_text( + json.dumps({"presets": {"sample": entry}}), encoding="utf-8" + ) + result = run_verifier(paths) + assert result.returncode == 2 + assert "created_at" in result.stdout + + +@pytest.mark.parametrize("timestamp_field", ["created_at", "updated_at"]) +def test_generated_new_entry_rejects_stale_dates(submission, timestamp_field): + issue, _, paths = submission + assert run_verifier(paths).returncode == 0 + entry = write_generated(issue, paths) + entry[timestamp_field] = "2000-01-01T00:00:00Z" + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], "presets": {"sample": entry}, + }), encoding="utf-8") + result = run_verifier(paths, "generated") + assert result.returncode == 3, result.stdout + result.stderr + assert timestamp_field in result.stdout + + +def test_generated_update_rejects_matching_stale_updated_dates(submission): + issue, _, paths = submission + entry = write_generated(issue, paths) + assert run_verifier(paths).returncode == 0 + entry["updated_at"] = "2000-01-01T00:00:00Z" + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], "presets": {"sample": entry}, + }), encoding="utf-8") + result = run_verifier(paths, "generated") + assert result.returncode == 3, result.stdout + result.stderr + assert "updated_at" in result.stdout + + +def test_generated_update_rejects_stale_homepage(submission): + issue, _, paths = submission + entry = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + entry["homepage"] = "https://example.com/old" + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], "presets": {"sample": entry}, + }), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + entry = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + entry["homepage"] = "https://example.com/old" + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], "presets": {"sample": entry}, + }), encoding="utf-8") + result = run_verifier(paths, "generated") + assert result.returncode == 3 + assert "homepage" in result.stdout + + +def test_generated_documentation_accepts_escaped_pipe_in_preset_name(submission): + issue, _, paths = submission + issue["preset_name"] = "Data | Governance" + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + write_generated(issue, paths) + result = run_verifier(paths, "generated") + assert result.returncode == 0, result.stdout + result.stderr + + +def test_generated_documentation_allows_duplicate_display_names(submission): + issue, _, paths = submission + other_row = ( + "| Sample Preset | Other usage | 1 command | — | " + "[other](https://github.com/example/other) |" + ) + paths["catalog.json"].write_text(json.dumps({ + "presets": {"other": {"name": issue["preset_name"]}}, + }), encoding="utf-8") + paths["presets.md"].write_text( + "| Preset | Purpose | Provides | Requires | URL |\n" + "|--------|---------|----------|----------|-----|\n" + f"{other_row}\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + entry = write_generated(issue, paths) + catalog = json.loads(paths["catalog.json"].read_text(encoding="utf-8")) + catalog["presets"] = { + "other": {"name": issue["preset_name"]}, + "sample": entry, + } + paths["catalog.json"].write_text(json.dumps(catalog), encoding="utf-8") + paths["presets.md"].write_text( + "| Preset | Purpose | Provides | Requires | URL |\n" + "|--------|---------|----------|----------|-----|\n" + f"{other_row}\n" + "| Sample Preset | Sample usage | 1 template, 1 command | " + "aide extension, canon extension | " + "[presets](https://github.com/example/presets) |\n", + encoding="utf-8", + ) + result = run_verifier(paths, "generated") + assert result.returncode == 0, result.stdout + result.stderr + + +def test_generated_documentation_accepts_replaced_renamed_row(submission): + issue, _, paths = submission + original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + paths["catalog.json"].write_text(json.dumps({ + "updated_at": original["updated_at"], "presets": {"sample": original}, + }), encoding="utf-8") + issue["preset_name"] = "Renamed Preset" + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + result = run_verifier(paths, "generated") + assert result.returncode == 0, result.stdout + result.stderr + + +def test_generated_documentation_rejects_stale_row_after_rename(submission): + issue, _, paths = submission + original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + previous_row = paths["presets.md"].read_text(encoding="utf-8").splitlines()[2] + paths["catalog.json"].write_text(json.dumps({ + "updated_at": original["updated_at"], "presets": {"sample": original}, + }), encoding="utf-8") + issue["preset_name"] = "Renamed Preset" + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + paths["presets.md"].write_text( + paths["presets.md"].read_text(encoding="utf-8") + previous_row + "\n", + encoding="utf-8", + ) + result = run_verifier(paths, "generated") + assert result.returncode == 3 + assert "previous documentation row" in result.stdout + + +@pytest.mark.parametrize(("damage", "message"), [ + ("catalog-json", "catalog"), + ("catalog-order", "alphabetical"), + ("catalog-metadata", "version"), + ("homepage-missing", "homepage"), + ("homepage-stale", "homepage"), + ("catalog-timestamp", "top-level updated_at"), + ("docs-order", "alphabetical"), + ("docs-row", "documentation row"), + ("created-at", "created_at"), +]) +def test_generated_defects_are_fixable_not_submission_failures(submission, damage, message): + issue, _, paths = submission + if damage == "created-at": + original = write_generated(issue, paths) + paths["catalog.json"].write_text( + json.dumps({"updated_at": original["updated_at"], "presets": { + "sample": original, + }}), encoding="utf-8" + ) + assert run_verifier(paths).returncode == 0 + entry = write_generated(issue, paths) + if damage == "catalog-json": + paths["catalog.json"].write_text("{", encoding="utf-8") + elif damage == "catalog-order": + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], + "presets": {"sample": entry, "aaa": {"name": "AAA"}}, + }), encoding="utf-8") + elif damage == "catalog-metadata": + entry["version"] = "1.2.4" + paths["catalog.json"].write_text( + json.dumps({"updated_at": entry["updated_at"], "presets": { + "sample": entry, + }}), encoding="utf-8" + ) + elif damage in ("homepage-missing", "homepage-stale"): + if damage == "homepage-missing": + del entry["homepage"] + else: + entry["homepage"] = "https://github.com/example/other" + paths["catalog.json"].write_text( + json.dumps({"updated_at": entry["updated_at"], "presets": { + "sample": entry, + }}), encoding="utf-8" + ) + elif damage == "catalog-timestamp": + paths["catalog.json"].write_text( + json.dumps({"updated_at": "2020-01-01T00:00:00Z", "presets": { + "sample": entry, + }}), encoding="utf-8" + ) + elif damage == "docs-order": + paths["presets.md"].write_text( + paths["presets.md"].read_text(encoding="utf-8") + + "| AAA | x | 1 command | — | [aaa](https://github.com/aaa/aaa) |\n", + encoding="utf-8", + ) + elif damage == "docs-row": + paths["presets.md"].write_text( + paths["presets.md"].read_text(encoding="utf-8").replace( + "Sample usage", "Wrong purpose" + ), encoding="utf-8", + ) + elif damage == "created-at": + entry["created_at"] = "2000-01-01T00:00:00Z" + paths["catalog.json"].write_text( + json.dumps({"updated_at": entry["updated_at"], "presets": { + "sample": entry, + }}), encoding="utf-8" + ) + result = run_verifier(paths, "generated") + assert result.returncode == 3, result.stdout + result.stderr + assert message in result.stdout + + +def test_workflow_gates_success_on_both_verifier_phases(): + source = WORKFLOW.read_text(encoding="utf-8") + assert "python3 .github/scripts/validate_community_preset.py submission" in source + assert "python3 .github/scripts/validate_community_preset.py generated" in source + assert (source.index("validate_community_preset.py generated") + < source.index("## Step 6") + < source.index("add the `validation-passed`", source.index("## Step 6"))) + frontmatter = yaml.safe_load(source.split("---", 2)[1]) + assert [step["name"] for step in frontmatter["steps"]] == [ + "Set up Python for preset verification", + "Install preset verifier dependency", + ] + compiled = yaml.safe_load( + (ROOT / ".github/workflows/add-community-preset.lock.yml").read_text( + encoding="utf-8" + ) + ) + steps = compiled["jobs"]["agent"]["steps"] + for setup in frontmatter["steps"]: + assert setup in steps diff --git a/tests/test_github_workflows.py b/tests/test_github_workflows.py index 5268ac8c42..5b6d1d8873 100644 --- a/tests/test_github_workflows.py +++ b/tests/test_github_workflows.py @@ -817,10 +817,21 @@ def test_community_archive_permission_failures_are_not_submission_failures(kind) "If there are no environment blockers and every required check completed " "and passed:" ) - assert re.search( - r"remove `validation-failed`.*add (?:the )?`validation-passed`", - passed, re.IGNORECASE | re.DOTALL, - ) + if kind == "preset": + assert "Remove any stale `validation-passed` and `validation-failed`" in passed + assert "Do not add `validation-passed` yet" in passed + generated = source_text.split("### Verify the generated files", 1)[1].split( + "\n## Step 6", 1 + )[0] + assert "validate_community_preset.py generated" in generated + assert "add the `validation-passed`" in source_text.split( + "\n## Step 6", 1 + )[1] + else: + assert re.search( + r"remove `validation-failed`.*add (?:the )?`validation-passed`", + passed, re.IGNORECASE | re.DOTALL, + ) assert "validation-failed" in source["safe-outputs"]["remove-labels"]["allowed"] assert "validation-failed" in _safe_output_config(compiled)["remove_labels"]["allowed"] assert "validation-passed" in source["safe-outputs"]["remove-labels"]["allowed"]