From 2305285072f0af2aae0857443d118d12b703201d Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 29 Sep 2026 07:36:10 -0500 Subject: [PATCH 1/6] fix: verify preset submissions before catalog PR Compare published manifests and README release references with issue fields, then verify generated catalog and documentation before success labeling. Regenerate the workflow lock file and cover submission, blocked, and repair outcomes. Refs github/spec-kit#4746 Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/validate_community_preset.py | 384 ++++++++++++++++++ .../workflows/add-community-preset.lock.yml | 14 +- .github/workflows/add-community-preset.md | 94 ++++- tests/test_community_preset_validation.py | 342 ++++++++++++++++ tests/test_github_workflows.py | 19 +- 5 files changed, 830 insertions(+), 23 deletions(-) create mode 100644 .github/scripts/validate_community_preset.py create mode 100644 tests/test_community_preset_validation.py diff --git a/.github/scripts/validate_community_preset.py b/.github/scripts/validate_community_preset.py new file mode 100644 index 0000000000..fb6f9803cf --- /dev/null +++ b/.github/scripts/validate_community_preset.py @@ -0,0 +1,384 @@ +"""Verify a community preset submission and its generated catalog changes. + +Exit 1: confirmed submission mismatch; 2: check blocked; 3: generated files +need repair. The workflow decides labels and PR creation from these outcomes. +""" + +import argparse +import hashlib +import json +import re +import sys +import zipfile +from pathlib import Path +from urllib.parse import urlsplit + +try: + import yaml +except ImportError: + print("BLOCKED: PyYAML is unavailable; cannot inspect published preset.yml") + sys.exit(2) + + +class SubmissionMismatch(Exception): + pass + + +class Blocked(Exception): + pass + + +class GeneratedError(Exception): + pass + + +def read_text( + path: Path, context: str, error_type: type[Exception] = Blocked +) -> str: + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as exc: + raise error_type(f"cannot read {context} ({path}): {exc}") from exc + + +def read_json(path: Path, context: str, error_type: type[Exception]) -> dict: + try: + value = json.loads(read_text(path, context, error_type)) + except json.JSONDecodeError as exc: + raise error_type(f"{context} is invalid JSON: {exc}") from exc + if not isinstance(value, dict): + raise error_type(f"{context} must be a JSON object") + return value + + +def field(issue: dict, key: str) -> str: + value = issue.get(key) + if not isinstance(value, str) or not value.strip(): + raise SubmissionMismatch(f"missing or invalid issue field: {key}") + return value.strip() + + +def csv_values(value: str) -> list[str]: + return [part.strip() for part in value.split(",") if part.strip()] + + +def repository_parts(url: str) -> tuple[str, str]: + parsed = urlsplit(url) + match = re.fullmatch(r"/([^/]+)/([^/]+)/?", parsed.path) + if parsed.scheme != "https" or parsed.netloc.lower() != "github.com" or not match: + raise SubmissionMismatch(f"invalid GitHub repository URL: {url}") + return match[1].lower(), match[2].lower() + + +def release_tag(issue: dict) -> str: + owner, repo = repository_parts(field(issue, "repository")) + url = field(issue, "download_url") + parsed = urlsplit(url) + prefix = f"/{owner}/{repo}/" + path = parsed.path + if parsed.scheme != "https" or parsed.netloc.lower() != "github.com": + raise SubmissionMismatch(f"download URL is not a GitHub URL: {url}") + if not path.lower().startswith(prefix): + raise SubmissionMismatch("download URL does not belong to the submitted repository") + suffix = path[len(prefix):] + archive = re.fullmatch(r"archive/refs/tags/([^/]+)\.zip", suffix) + release = re.fullmatch(r"releases/download/([^/]+)/[^/]+\.zip", suffix) + if not (archive or release) or parsed.query or parsed.fragment: + raise SubmissionMismatch(f"download URL is not a pinned ZIP release: {url}") + tag = (archive or release)[1] + version = field(issue, "version") + if not re.fullmatch(r"\d+\.\d+\.\d+", version) or not re.fullmatch( + rf"(?:v?{re.escape(version)}|.+-v?{re.escape(version)})", tag + ): + raise SubmissionMismatch( + f"release tag {tag!r} does not match submitted version {version!r}" + ) + return tag + + +def published_manifest(archive_path: Path, preset_id: str) -> dict: + try: + with zipfile.ZipFile(archive_path) as archive: + matching = [] + invalid = [] + for member in archive.infolist(): + if member.is_dir() or member.filename.rsplit("/", 1)[-1] != "preset.yml": + continue + try: + if member.file_size > 1024 * 1024: + raise ValueError("preset.yml exceeds 1 MiB") + with archive.open(member) as stream: + data = yaml.safe_load(stream.read().decode("utf-8")) + except (yaml.YAMLError, UnicodeError, ValueError) as exc: + invalid.append(f"{member.filename}: {exc}") + continue + if isinstance(data, dict) and isinstance(data.get("preset"), dict): + if data["preset"].get("id") == preset_id: + matching.append((member.filename, data)) + except FileNotFoundError as exc: + raise Blocked(f"downloaded archive is unavailable at {archive_path}: {exc}") from exc + except PermissionError as exc: + raise Blocked(f"cannot read downloaded archive at {archive_path}: {exc}") from exc + except OSError as exc: + raise Blocked(f"cannot read downloaded archive at {archive_path}: {exc}") from exc + except (zipfile.BadZipFile, EOFError) as exc: + raise SubmissionMismatch(f"downloaded archive is not a readable ZIP: {exc}") from exc + if len(matching) != 1: + if not matching and invalid: + raise SubmissionMismatch( + f"no published preset.yml for {preset_id!r}; invalid manifests: " + + "; ".join(invalid) + ) + raise SubmissionMismatch( + f"expected one published preset.yml for {preset_id!r}, found {len(matching)}" + ) + return matching[0][1] + + +def required_extensions(manifest: dict) -> list[str]: + requires = manifest.get("requires") + if not isinstance(requires, dict): + raise SubmissionMismatch("published preset.yml has no requires mapping") + extensions = requires.get("extensions", []) + if not isinstance(extensions, list): + raise SubmissionMismatch("published requires.extensions must be a list") + result = [] + for item in extensions: + if isinstance(item, str): + result.append(item) + elif isinstance(item, dict) and isinstance(item.get("id"), str): + if item.get("required", True) is not False: + result.append(item["id"]) + else: + raise SubmissionMismatch("invalid published requires.extensions entry") + return result + + +def check_readme(text: str, issue: dict) -> None: + expected = field(issue, "download_url") + preset_id = field(issue, "preset_id") + owner, repo = repository_parts(field(issue, "repository")) + accepted = False + for match in re.finditer( + r"(?--from|--dev|[a-z][a-z0-9-]*)" + r"(?:\s+(?P[^\s`]+))?", + text, + ): + option = match["option"] + value = (match["value"] or "").strip("'\"<>(),.;") + if option == "--from": + if value == expected: + accepted = True + continue + parsed = urlsplit(value) + parts = parsed.path.split("/") + tag = ( + parts[5] if len(parts) > 5 and parts[3] == "releases" + else parts[6].removesuffix(".zip") + if len(parts) > 6 and parts[3] == "archive" + else "" + ) + scoped = re.fullmatch(r"(.+)-v?\d+\.\d+\.\d+", tag) + same_repository = ( + parsed.netloc.lower() == "github.com" + and parsed.path.lower().startswith(f"/{owner}/{repo}/") + ) + if (scoped and scoped[1] == preset_id) or ( + same_repository and (not scoped or scoped[1] == preset_id) + ): + raise SubmissionMismatch( + f"README --from URL for {preset_id} differs from Download URL: {value}" + ) + elif option == "--dev" and value: + accepted = True + elif option == preset_id: + accepted = True + if not accepted: + raise SubmissionMismatch( + "README has no valid specify preset add command for the submitted preset" + ) + + +def count_items(value: str) -> int: + return sum(line.lstrip().startswith("- ") for line in value.splitlines()) + + +def expected_values(issue: dict, manifest: dict, digest: str) -> dict: + pack = manifest.get("preset") + requires = manifest.get("requires") + if not isinstance(pack, dict) or not isinstance(requires, dict): + raise SubmissionMismatch("published preset.yml lacks preset or requires metadata") + extension_ids = csv_values(issue.get("required_extensions", "")) + actual_extensions = required_extensions(manifest) + comparisons = { + "preset.id": (pack.get("id"), field(issue, "preset_id")), + "preset.version": (pack.get("version"), field(issue, "version")), + "requires.speckit_version": ( + requires.get("speckit_version"), field(issue, "speckit_version") + ), + "requires.extensions": (sorted(actual_extensions), sorted(extension_ids)), + } + for name, (actual, expected) in comparisons.items(): + if actual != expected: + raise SubmissionMismatch( + f"published {name} {actual!r} differs from submitted {expected!r}" + ) + dependencies = {"speckit_version": field(issue, "speckit_version")} + if extension_ids: + dependencies["extensions"] = extension_ids + provided = { + "templates": count_items(field(issue, "templates_provided")), + "commands": count_items(field(issue, "commands_provided")), + } + scripts = issue.get("scripts_count", "0") + if scripts: + try: + number = int(scripts) + except (ValueError, TypeError) as exc: + raise SubmissionMismatch("scripts_count must be a non-negative integer") from exc + if number < 0: + raise SubmissionMismatch("scripts_count must be a non-negative integer") + if number: + provided["scripts"] = number + return { + "id": field(issue, "preset_id"), + "name": field(issue, "preset_name"), + "version": field(issue, "version"), + "description": field(issue, "description"), + "author": field(issue, "author"), + "repository": field(issue, "repository"), + "download_url": field(issue, "download_url"), + "sha256": digest, + "documentation": field(issue, "documentation"), + "license": field(issue, "license"), + "requires": dependencies, + "provides": provided, + "tags": csv_values(field(issue, "tags")), + } + + +def submission(args: argparse.Namespace) -> None: + issue = read_json(args.issue, "issue input", Blocked) + release_tag(issue) + manifest = published_manifest(args.archive, field(issue, "preset_id")) + check_readme(read_text(args.readme, "fetched README"), issue) + try: + with args.archive.open("rb") as archive: + digest = hashlib.file_digest(archive, "sha256").hexdigest() + except OSError as exc: + raise Blocked(f"cannot hash downloaded archive: {exc}") from exc + expected = expected_values(issue, manifest, digest) + catalog = read_json(args.catalog, "original catalog", Blocked) + entries = catalog.get("presets") + if not isinstance(entries, dict): + raise Blocked("original catalog has no presets object") + previous = entries.get(expected["id"]) + if previous is not None and not isinstance(previous, dict): + raise Blocked("original catalog entry is not an object") + if previous is not None and not isinstance(previous.get("created_at"), str): + raise Blocked("original catalog entry has no created_at to preserve") + snapshot = { + "expected": expected, + "created_at": previous.get("created_at") if previous else None, + } + try: + args.snapshot.write_text(json.dumps(snapshot), encoding="utf-8") + except OSError as exc: + raise Blocked(f"cannot write verifier snapshot: {exc}") from exc + print("PASSED: published preset.yml, release tag, README, and issue fields agree") + + +def documentation_row(entry: dict) -> str: + def cell(value: str) -> str: + return value.replace("|", r"\|").replace("\n", " ") + + provided = entry["provides"] + quantities = [ + f"{count} {name[:-1] if count == 1 else name}" + for name, count in provided.items() if count + ] + dependencies = entry["requires"].get("extensions", []) + requires = ", ".join(f"{name} extension" for name in dependencies) or "—" + repo = entry["repository"].rstrip("/").rsplit("/", 1)[-1] + return ( + f"| {cell(entry['name'])} | {cell(entry['description'])} | " + f"{', '.join(quantities)} | {requires} | " + f"[{repo}]({entry['repository']}) |" + ) + + +def generated(args: argparse.Namespace) -> None: + snapshot = read_json(args.snapshot, "verifier snapshot", Blocked) + expected = snapshot.get("expected") + if not isinstance(expected, dict) or not isinstance(expected.get("id"), str): + raise Blocked("verifier snapshot lacks validated expected values") + catalog = read_json(args.catalog, "generated catalog", GeneratedError) + entries = catalog.get("presets") + if not isinstance(entries, dict): + raise GeneratedError("generated catalog has no presets object") + keys = list(entries) + if keys != sorted(keys): + raise GeneratedError("catalog preset IDs are not in alphabetical order") + entry = entries.get(expected["id"]) + if not isinstance(entry, dict): + raise GeneratedError(f"catalog is missing entry {expected['id']}") + for key, value in expected.items(): + if entry.get(key) != value: + raise GeneratedError(f"catalog {key} does not match validated value: {value!r}") + if snapshot.get("created_at") is not None: + if entry.get("created_at") != snapshot["created_at"]: + raise GeneratedError("catalog created_at was not preserved on update") + elif not isinstance(entry.get("created_at"), str): + raise GeneratedError("new catalog entry has no created_at") + if not isinstance(entry.get("updated_at"), str): + raise GeneratedError("catalog entry has no updated_at") + if catalog.get("updated_at") != entry["updated_at"]: + raise GeneratedError("catalog top-level updated_at does not match entry updated_at") + docs = read_text(args.docs, "generated documentation", GeneratedError) + lines = docs.splitlines() + try: + start = next(i for i, line in enumerate(lines) if line.startswith("| Preset |")) + except StopIteration as exc: + raise GeneratedError("documentation has no Community Presets table") from exc + rows = [] + for line in lines[start + 2:]: + if not line.startswith("|"): + break + rows.append(line.strip()) + names = [row.split("|", 2)[1].strip() for row in rows] + if names != sorted(names, key=str.casefold): + raise GeneratedError("documentation preset names are not in alphabetical order") + expected_row = documentation_row(expected) + if rows.count(expected_row) != 1: + raise GeneratedError(f"documentation row does not match validated values: {expected_row}") + if names.count(expected["name"]) != 1: + raise GeneratedError("documentation contains duplicate preset names") + print("PASSED: generated catalog and documentation match validated submission") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("submission", "generated")) + for name in ("issue", "archive", "readme", "catalog", "docs", "snapshot"): + parser.add_argument(f"--{name}", required=True, type=Path) + args = parser.parse_args() + try: + if args.phase == "submission": + submission(args) + else: + generated(args) + except SubmissionMismatch as exc: + print(f"FAILED: {exc}") + return 1 + except Blocked as exc: + print(f"BLOCKED: {exc}") + return 2 + except GeneratedError as exc: + print(f"REPAIR: {exc}") + return 3 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/add-community-preset.lock.yml b/.github/workflows/add-community-preset.lock.yml index 5b6cb10812..9387f0b94e 100644 --- a/.github/workflows/add-community-preset.lock.yml +++ b/.github/workflows/add-community-preset.lock.yml @@ -1,5 +1,5 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"e692354fdb52cf49c3f46096529618b23eb09336fe58fe2392b77210b374b568","body_hash":"cd1920f05bf8608b46191a3802a1c2f6c595ab3ce28034116e3346d4988bed4a","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_pull_request","missing_data","missing_tool","noop","remove_labels"]}]} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"499cb209b6fc93a3b306cb0113aefb2367be2a84d952c4bb4516f982f535c25c","body_hash":"f17cb4834d73fa5067f13ac54372f90eb0553ab153bc7f6df7c90494a97994f0","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/setup-python","sha":"5fda3b95a4ea91299a34e894583c3862153e4b97","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_pull_request","missing_data","missing_tool","noop","remove_labels"]}]} # This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # # ___ _ _ @@ -41,6 +41,7 @@ # - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 # - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) # - actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 +# - actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 # - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 # - github/gh-aw-actions/setup@5e508589e03a7757a7e05b26e834292f5445bfb6 # v0.88.7 # @@ -491,6 +492,15 @@ jobs: with: name: activation path: /tmp/gh-aw + - continue-on-error: true + name: Set up Python for preset verification + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + - continue-on-error: true + name: Install preset verifier dependency + run: python3 -m pip install 'PyYAML==6.0.3' + - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/add-community-preset.md b/.github/workflows/add-community-preset.md index f6ccf01c8f..8b171e67bb 100644 --- a/.github/workflows/add-community-preset.md +++ b/.github/workflows/add-community-preset.md @@ -39,6 +39,16 @@ permissions: checkout: fetch-depth: 0 +steps: + - name: Set up Python for preset verification + continue-on-error: true + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + - name: Install preset verifier dependency + continue-on-error: true + run: python3 -m pip install 'PyYAML==6.0.3' + safe-outputs: noop: report-as-issue: false @@ -130,7 +140,8 @@ deciding pass/fail: ### 2c. Repository validation - Fetch the repository URL — confirm it exists and is publicly accessible -- Confirm the repository contains a `preset.yml` file +- Confirm the repository contains a `preset.yml` file (in the preset's + subdirectory for a monorepo) - Confirm the repository contains a `LICENSE` file > The README requirement is enforced once, in **Step 2d**, against the specific file the @@ -170,10 +181,11 @@ preset** — not just any file named `README.md`, and not a product/framework pi - `specify preset add --dev ` A `specify preset add --from ` command only counts when its `` **matches the - submitted Download URL exactly**. A `--from` command pointing at a *different* URL does - **not** satisfy the install-command requirement (treat it as if absent) — but the README - may still pass on one of the other accepted forms (`specify preset add ` or - `specify preset add --dev `). + submitted Download URL exactly**. If the README also contains a `--from` release URL + for this preset in the submitted repository that differs from the Download URL, **fail** + even if another accepted command (`specify preset add ` or + `specify preset add --dev `) is present. A README with only a valid `--dev` + command remains acceptable. The verifier in Step 2g enforces this comparison. If **no** accepted `specify preset add ...` command is present, the README is treated as a generic description/pitch rather than preset-usage documentation — **fail this check** and @@ -258,6 +270,34 @@ substitute for fetching the archive. Never execute downloaded content. - Confirm that all required checkboxes in the Testing Checklist and Submission Requirements sections are checked (`[x]`) +### 2g. Reproducible published-artifact and README comparison + +After the pinned download has returned HTTP 200 and the optional checksum comparison +has succeeded, use the edit tool to save the fetched **exact documentation README** +as `/tmp/gh-aw/preset-readme.md`. Use the edit tool to save +`/tmp/gh-aw/preset-submission.json` as a JSON object with these keys copied from the +issue form (not inferred from the README or archive): +`preset_id`, `preset_name`, `version`, `description`, `author`, `repository`, +`download_url`, `documentation`, `license`, `speckit_version`, +`required_extensions` (empty string when absent), `templates_provided`, +`commands_provided`, `scripts_count` (string `"0"` when absent), and `tags`. +Retain multiline list values as strings. Do not interpolate issue or README text +into shell commands. Run this fixed command unchanged: + +```bash +python3 .github/scripts/validate_community_preset.py submission --issue /tmp/gh-aw/preset-submission.json --archive /tmp/gh-aw/community-archive.zip --readme /tmp/gh-aw/preset-readme.md --catalog presets/catalog.community.json --docs docs/community/presets.md --snapshot /tmp/gh-aw/preset-validation.json +``` + +The verifier reads `preset.yml` from the downloaded ZIP, including preset-scoped +paths in monorepos; it parses YAML without executing archive content. It checks +the published ID, version, Spec Kit requirement, required extension IDs, release +tag, and README install references against the issue, then records the validated +values and existing `created_at` for the generated-file check. Exit 1 (`FAILED`) +is a confirmed submission mismatch: report it on the issue under Failed, with +no PR. Exit 2 (`BLOCKED`) means the verifier could not read the archive or other +required inputs (including a missing Python dependency): report the exact error +and run link under Blocked. Do not treat either exit as a pass. + ### Validation outcome Choose exactly one outcome below, in order. A check that could not run is @@ -291,9 +331,10 @@ If there are no environment blockers and a completed check found a submission de #### Passed If there are no environment blockers and every required check completed and passed: -1. Remove `validation-failed` -2. Add the `validation-passed` label -3. Continue to Step 3 +1. Remove any stale `validation-passed` and `validation-failed` labels from + earlier runs. +2. Continue to Step 3. **Do not add `validation-passed` yet**: the generated + catalog and documentation must pass Step 5's verifier before success is recorded. ## Step 3 — Determine Add vs Update @@ -375,13 +416,8 @@ Parse the "Templates Provided" and "Commands Provided" issue fields: Update the **top-level `"updated_at"` timestamp** in the catalog to today's date in ISO 8601 format. -Validate the JSON by running: - -```bash -python3 -c "import json; json.load(open('presets/catalog.community.json')); print('Valid JSON')" -``` - -If validation fails, fix the JSON and re-validate before continuing. +The generated-file verifier in Step 5 parses and checks the JSON. Fix any +catalog error it reports and rerun it before continuing. ## Step 5 — Update `docs/community/presets.md` @@ -398,7 +434,8 @@ Insert a new row in **alphabetical order by preset name**: For the Requires column: - Use `—` if no extensions are required -- List required extension names if any (e.g., `AIDE extension`) +- List required extension IDs if any (e.g., `aide extension`), comma-separated +- Omit zero-count kinds from Provides and use singular nouns for counts of one If the preset provides scripts, include them: ` templates, commands, scripts` @@ -406,9 +443,32 @@ If the preset provides scripts, include them: ` templates, commands, Find the existing row and update any changed fields in-place. +### Verify the generated files + +Before labeling success or requesting a PR, run this fixed command unchanged: + +```bash +python3 .github/scripts/validate_community_preset.py generated --issue /tmp/gh-aw/preset-submission.json --archive /tmp/gh-aw/community-archive.zip --readme /tmp/gh-aw/preset-readme.md --catalog presets/catalog.community.json --docs docs/community/presets.md --snapshot /tmp/gh-aw/preset-validation.json +``` + +The verifier checks JSON parsing, the validated catalog metadata and digest, +the top-level and entry `updated_at` timestamps, +alphabetical ID order, the documentation row's name, purpose, counts, extension +requirements and repository link, alphabetical name order, and preservation of +`created_at` on updates. Exit 3 (`REPAIR`) is an agent-generated catalog or +documentation error, **not** a submitter defect: fix the files and re-run this +command until it exits 0. Do not label `validation-failed` for an error in +generated files. Exit 2 (`BLOCKED`) means an input or tool is unavailable: report +the exact error and workflow run link to the maintainer, remove +`validation-passed`, and stop without a PR. If a generated-file error cannot be +corrected, remove `validation-passed`, stop without a PR, and report the problem for +maintainer investigation. + ## Step 6 — Create Pull Request -Create a pull request with the changes. Use this branch naming convention: +Only after the generated-file verifier exits 0, add the `validation-passed` +label and request the configured draft pull request with the changes. Use this +branch naming convention: This repository-owned gh-aw maintenance workflow does not perform the contributor open-PR count check or request confirmation. After successful validation and diff --git a/tests/test_community_preset_validation.py b/tests/test_community_preset_validation.py new file mode 100644 index 0000000000..a0ed7906f3 --- /dev/null +++ b/tests/test_community_preset_validation.py @@ -0,0 +1,342 @@ +"""Regression coverage for the repository-owned preset submission verifier.""" + +import hashlib +import json +import subprocess +import sys +import zipfile +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parent.parent +VERIFIER = ROOT / ".github" / "scripts" / "validate_community_preset.py" +WORKFLOW = ROOT / ".github" / "workflows" / "add-community-preset.md" + + +@pytest.fixture +def submission(tmp_path): + issue = { + "preset_id": "sample", + "preset_name": "Sample Preset", + "version": "1.2.3", + "description": "Sample usage", + "author": "Contributor", + "repository": "https://github.com/example/presets", + "download_url": "https://github.com/example/presets/releases/download/sample-v1.2.3/sample.zip", + "documentation": "https://github.com/example/presets/blob/main/sample/README.md", + "license": "MIT", + "speckit_version": ">=1.0.0", + "required_extensions": "aide, canon", + "templates_provided": "- spec-template.md", + "commands_provided": "- speckit.plan.md", + "scripts_count": "0", + "tags": "sample, example", + } + paths = {name: tmp_path / name for name in ( + "issue.json", "archive.zip", "README.md", "catalog.json", "presets.md", + "snapshot.json", + )} + paths["README.md"].write_text( + f"specify preset add --from {issue['download_url']}\n", encoding="utf-8" + ) + manifest = { + "preset": {"id": "sample", "version": "1.2.3"}, + "requires": {"speckit_version": ">=1.0.0", "extensions": ["aide", "canon"]}, + } + with zipfile.ZipFile(paths["archive.zip"], "w") as archive: + archive.writestr("presets-release/sample/preset.yml", yaml.safe_dump(manifest)) + archive.writestr("presets-release/other/preset.yml", yaml.safe_dump({ + "preset": {"id": "other", "version": "9.9.9"}, + "requires": {"speckit_version": ">=0.1.0"}, + })) + issue["actual_sha256"] = hashlib.sha256(paths["archive.zip"].read_bytes()).hexdigest() + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["catalog.json"].write_text( + json.dumps({"presets": {}}), encoding="utf-8" + ) + paths["presets.md"].write_text( + "| Preset | Purpose | Provides | Requires | URL |\n" + "|--------|---------|----------|----------|-----|\n", + encoding="utf-8", + ) + return issue, manifest, paths + + +def run_verifier(paths, phase="submission"): + return subprocess.run( + [ + sys.executable, str(VERIFIER), phase, + "--issue", str(paths["issue.json"]), + "--archive", str(paths["archive.zip"]), + "--readme", str(paths["README.md"]), + "--catalog", str(paths["catalog.json"]), + "--docs", str(paths["presets.md"]), + "--snapshot", str(paths["snapshot.json"]), + ], + capture_output=True, text=True, check=False, + ) + + +def write_archive(paths, manifest): + with zipfile.ZipFile(paths["archive.zip"], "w") as archive: + archive.writestr("release/sample/preset.yml", yaml.safe_dump(manifest)) + + +def write_generated(issue, paths, *, created_at="2025-01-01T00:00:00Z"): + entry = { + "id": issue["preset_id"], "name": issue["preset_name"], + "version": issue["version"], "description": issue["description"], + "author": issue["author"], "repository": issue["repository"], + "download_url": issue["download_url"], "sha256": issue["actual_sha256"], + "homepage": issue["repository"], "documentation": issue["documentation"], + "license": issue["license"], + "requires": {"speckit_version": issue["speckit_version"], + "extensions": ["aide", "canon"]}, + "provides": {"templates": 1, "commands": 1}, + "tags": ["sample", "example"], + "created_at": created_at, "updated_at": "2026-01-01T00:00:00Z", + } + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], "presets": {"sample": entry}, + }), encoding="utf-8") + paths["presets.md"].write_text( + "| Preset | Purpose | Provides | Requires | URL |\n" + "|--------|---------|----------|----------|-----|\n" + "| Sample Preset | Sample usage | 1 template, 1 command | " + "aide extension, canon extension | " + "[presets](https://github.com/example/presets) |\n", + encoding="utf-8", + ) + return entry + + +def test_matching_monorepo_submission_and_generated_files_pass(submission): + issue, _, paths = submission + first = run_verifier(paths) + assert first.returncode == 0, first.stdout + first.stderr + write_generated(issue, paths) + second = run_verifier(paths, "generated") + assert second.returncode == 0, second.stdout + second.stderr + + +def test_update_preserving_created_at_passes(submission): + issue, _, paths = submission + original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + paths["catalog.json"].write_text(json.dumps({ + "updated_at": original["updated_at"], "presets": {"sample": original}, + }), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + assert run_verifier(paths, "generated").returncode == 0 + + +@pytest.mark.parametrize(("change", "message"), [ + ({"preset": {"id": "sample", "version": "1.2.4"}}, "version"), + ({"requires": {"speckit_version": ">=2.0.0", + "extensions": ["aide", "canon"]}}, "speckit_version"), + ({"requires": {"speckit_version": ">=1.0.0", + "extensions": ["aide"]}}, "extensions"), +]) +def test_published_manifest_mismatch_is_submission_failure(submission, change, message): + _, manifest, paths = submission + manifest.update(change) + write_archive(paths, manifest) + result = run_verifier(paths) + assert result.returncode == 1 + assert message in result.stdout + assert not paths["snapshot.json"].exists() + + +def test_release_tag_mismatch_is_submission_failure(submission): + issue, _, paths = submission + issue["download_url"] = issue["download_url"].replace("v1.2.3", "v1.2.4") + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + assert run_verifier(paths).returncode == 1 + + +def test_stale_from_url_fails_even_with_valid_dev_command(submission): + issue, _, paths = submission + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n", + encoding="utf-8", + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "README" in result.stdout + + +def test_dev_only_readme_is_accepted(submission): + _, _, paths = submission + paths["README.md"].write_text( + "specify preset add --dev ./sample\n", encoding="utf-8" + ) + assert run_verifier(paths).returncode == 0 + + +def test_quoted_from_url_with_sentence_punctuation_is_accepted(submission): + issue, _, paths = submission + paths["README.md"].write_text( + f'Spec Kit install: `specify preset add --from "{issue["download_url"]}".`\n', + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + +def test_id_install_and_unrelated_monorepo_release_are_accepted(submission): + _, _, paths = submission + paths["README.md"].write_text( + "specify preset add sample\n" + "specify preset add --from " + "https://github.com/example/presets/releases/download/other-v2.0.0/other.zip\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + +def test_stale_scoped_release_in_another_repository_fails(submission): + _, _, paths = submission + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + "specify preset add --from " + "https://github.com/elsewhere/presets/releases/download/sample-v1.2.2/sample.zip\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 1 + + +def test_optional_manifest_extension_is_not_required(submission): + issue, manifest, paths = submission + manifest["requires"]["extensions"].append({ + "id": "optional", "version": ">=1.0.0", "required": False, + }) + write_archive(paths, manifest) + issue["actual_sha256"] = hashlib.sha256(paths["archive.zip"].read_bytes()).hexdigest() + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + + +def test_invalid_unrelated_monorepo_manifest_does_not_mask_match(submission): + _, manifest, paths = submission + with zipfile.ZipFile(paths["archive.zip"], "w") as archive: + archive.writestr("release/sample/preset.yml", yaml.safe_dump(manifest)) + archive.writestr("release/other/preset.yml", "preset: [invalid\n") + assert run_verifier(paths).returncode == 0 + + +def test_missing_archive_is_blocked_not_failed(submission): + _, _, paths = submission + paths["archive.zip"].unlink() + result = run_verifier(paths) + assert result.returncode == 2 + assert "BLOCKED" in result.stdout + + +def test_missing_generated_documentation_is_repairable(submission): + issue, _, paths = submission + assert run_verifier(paths).returncode == 0 + write_generated(issue, paths) + paths["presets.md"].unlink() + result = run_verifier(paths, "generated") + assert result.returncode == 3 + assert "REPAIR" in result.stdout + + +def test_existing_entry_without_creation_date_blocks_update(submission): + issue, _, paths = submission + entry = write_generated(issue, paths) + del entry["created_at"] + paths["catalog.json"].write_text( + json.dumps({"presets": {"sample": entry}}), encoding="utf-8" + ) + result = run_verifier(paths) + assert result.returncode == 2 + assert "created_at" in result.stdout + + +@pytest.mark.parametrize(("damage", "message"), [ + ("catalog-json", "catalog"), + ("catalog-order", "alphabetical"), + ("catalog-metadata", "version"), + ("catalog-timestamp", "top-level updated_at"), + ("docs-order", "alphabetical"), + ("docs-row", "documentation row"), + ("created-at", "created_at"), +]) +def test_generated_defects_are_fixable_not_submission_failures(submission, damage, message): + issue, _, paths = submission + if damage == "created-at": + original = write_generated(issue, paths) + paths["catalog.json"].write_text( + json.dumps({"updated_at": original["updated_at"], "presets": { + "sample": original, + }}), encoding="utf-8" + ) + assert run_verifier(paths).returncode == 0 + entry = write_generated(issue, paths) + if damage == "catalog-json": + paths["catalog.json"].write_text("{", encoding="utf-8") + elif damage == "catalog-order": + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], + "presets": {"sample": entry, "aaa": {"name": "AAA"}}, + }), encoding="utf-8") + elif damage == "catalog-metadata": + entry["version"] = "1.2.4" + paths["catalog.json"].write_text( + json.dumps({"updated_at": entry["updated_at"], "presets": { + "sample": entry, + }}), encoding="utf-8" + ) + elif damage == "catalog-timestamp": + paths["catalog.json"].write_text( + json.dumps({"updated_at": "2020-01-01T00:00:00Z", "presets": { + "sample": entry, + }}), encoding="utf-8" + ) + elif damage == "docs-order": + paths["presets.md"].write_text( + paths["presets.md"].read_text(encoding="utf-8") + + "| AAA | x | 1 command | — | [aaa](https://github.com/aaa/aaa) |\n", + encoding="utf-8", + ) + elif damage == "docs-row": + paths["presets.md"].write_text( + paths["presets.md"].read_text(encoding="utf-8").replace( + "Sample usage", "Wrong purpose" + ), encoding="utf-8", + ) + elif damage == "created-at": + entry["created_at"] = "2026-01-01T00:00:00Z" + paths["catalog.json"].write_text( + json.dumps({"updated_at": entry["updated_at"], "presets": { + "sample": entry, + }}), encoding="utf-8" + ) + result = run_verifier(paths, "generated") + assert result.returncode == 3, result.stdout + result.stderr + assert message in result.stdout + + +def test_workflow_gates_success_on_both_verifier_phases(): + source = WORKFLOW.read_text(encoding="utf-8") + assert "python3 .github/scripts/validate_community_preset.py submission" in source + assert "python3 .github/scripts/validate_community_preset.py generated" in source + assert (source.index("validate_community_preset.py generated") + < source.index("## Step 6") + < source.index("add the `validation-passed`", source.index("## Step 6"))) + frontmatter = yaml.safe_load(source.split("---", 2)[1]) + assert [step["name"] for step in frontmatter["steps"]] == [ + "Set up Python for preset verification", + "Install preset verifier dependency", + ] + compiled = yaml.safe_load( + (ROOT / ".github/workflows/add-community-preset.lock.yml").read_text( + encoding="utf-8" + ) + ) + steps = compiled["jobs"]["agent"]["steps"] + for setup in frontmatter["steps"]: + assert setup in steps diff --git a/tests/test_github_workflows.py b/tests/test_github_workflows.py index 5268ac8c42..5b6d1d8873 100644 --- a/tests/test_github_workflows.py +++ b/tests/test_github_workflows.py @@ -817,10 +817,21 @@ def test_community_archive_permission_failures_are_not_submission_failures(kind) "If there are no environment blockers and every required check completed " "and passed:" ) - assert re.search( - r"remove `validation-failed`.*add (?:the )?`validation-passed`", - passed, re.IGNORECASE | re.DOTALL, - ) + if kind == "preset": + assert "Remove any stale `validation-passed` and `validation-failed`" in passed + assert "Do not add `validation-passed` yet" in passed + generated = source_text.split("### Verify the generated files", 1)[1].split( + "\n## Step 6", 1 + )[0] + assert "validate_community_preset.py generated" in generated + assert "add the `validation-passed`" in source_text.split( + "\n## Step 6", 1 + )[1] + else: + assert re.search( + r"remove `validation-failed`.*add (?:the )?`validation-passed`", + passed, re.IGNORECASE | re.DOTALL, + ) assert "validation-failed" in source["safe-outputs"]["remove-labels"]["allowed"] assert "validation-failed" in _safe_output_config(compiled)["remove_labels"]["allowed"] assert "validation-passed" in source["safe-outputs"]["remove-labels"]["allowed"] From 6acc33db6132d35eb25a987bc479aa631fe07fd2 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:05:08 -0500 Subject: [PATCH 2/6] fix: validate UTC dates and scoped preset release URLs Record the submission UTC date and require generated catalog timestamps to match it; keep update creation dates intact. Detect stale README release links whose scoped tag prefix matches the submitted release, including ZIP archive URLs. Refs github/spec-kit#4746 Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/validate_community_preset.py | 25 ++++-- .../workflows/add-community-preset.lock.yml | 2 +- .github/workflows/add-community-preset.md | 17 ++-- tests/test_community_preset_validation.py | 82 ++++++++++++++++++- 4 files changed, 107 insertions(+), 19 deletions(-) diff --git a/.github/scripts/validate_community_preset.py b/.github/scripts/validate_community_preset.py index fb6f9803cf..2d1e2c2618 100644 --- a/.github/scripts/validate_community_preset.py +++ b/.github/scripts/validate_community_preset.py @@ -10,6 +10,7 @@ import re import sys import zipfile +from datetime import datetime, timezone from pathlib import Path from urllib.parse import urlsplit @@ -158,6 +159,10 @@ def check_readme(text: str, issue: dict) -> None: expected = field(issue, "download_url") preset_id = field(issue, "preset_id") owner, repo = repository_parts(field(issue, "repository")) + submitted_scope = re.fullmatch(r"(.+)-v?\d+\.\d+\.\d+", release_tag(issue)) + accepted_scopes = {preset_id} + if submitted_scope: + accepted_scopes.add(submitted_scope[1]) accepted = False for match in re.finditer( r"(? None: parsed.netloc.lower() == "github.com" and parsed.path.lower().startswith(f"/{owner}/{repo}/") ) - if (scoped and scoped[1] == preset_id) or ( - same_repository and (not scoped or scoped[1] == preset_id) + if (scoped and scoped[1] in accepted_scopes) or ( + same_repository and not scoped ): raise SubmissionMismatch( f"README --from URL for {preset_id} differs from Download URL: {value}" @@ -281,6 +286,7 @@ def submission(args: argparse.Namespace) -> None: snapshot = { "expected": expected, "created_at": previous.get("created_at") if previous else None, + "expected_timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z"), } try: args.snapshot.write_text(json.dumps(snapshot), encoding="utf-8") @@ -313,6 +319,9 @@ def generated(args: argparse.Namespace) -> None: expected = snapshot.get("expected") if not isinstance(expected, dict) or not isinstance(expected.get("id"), str): raise Blocked("verifier snapshot lacks validated expected values") + expected_timestamp = snapshot.get("expected_timestamp") + if not isinstance(expected_timestamp, str): + raise Blocked("verifier snapshot lacks the expected UTC timestamp") catalog = read_json(args.catalog, "generated catalog", GeneratedError) entries = catalog.get("presets") if not isinstance(entries, dict): @@ -329,12 +338,12 @@ def generated(args: argparse.Namespace) -> None: if snapshot.get("created_at") is not None: if entry.get("created_at") != snapshot["created_at"]: raise GeneratedError("catalog created_at was not preserved on update") - elif not isinstance(entry.get("created_at"), str): - raise GeneratedError("new catalog entry has no created_at") - if not isinstance(entry.get("updated_at"), str): - raise GeneratedError("catalog entry has no updated_at") - if catalog.get("updated_at") != entry["updated_at"]: - raise GeneratedError("catalog top-level updated_at does not match entry updated_at") + elif entry.get("created_at") != expected_timestamp: + raise GeneratedError("new catalog created_at does not match the expected UTC date") + if entry.get("updated_at") != expected_timestamp: + raise GeneratedError("catalog entry updated_at does not match the expected UTC date") + if catalog.get("updated_at") != expected_timestamp: + raise GeneratedError("catalog top-level updated_at does not match the expected UTC date") docs = read_text(args.docs, "generated documentation", GeneratedError) lines = docs.splitlines() try: diff --git a/.github/workflows/add-community-preset.lock.yml b/.github/workflows/add-community-preset.lock.yml index 9387f0b94e..23cc790ba0 100644 --- a/.github/workflows/add-community-preset.lock.yml +++ b/.github/workflows/add-community-preset.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"499cb209b6fc93a3b306cb0113aefb2367be2a84d952c4bb4516f982f535c25c","body_hash":"f17cb4834d73fa5067f13ac54372f90eb0553ab153bc7f6df7c90494a97994f0","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"499cb209b6fc93a3b306cb0113aefb2367be2a84d952c4bb4516f982f535c25c","body_hash":"3b3c6aba8ccc13b6354b67fbde26368a41b687e089010a1e77617f363b3fa7c3","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/setup-python","sha":"5fda3b95a4ea91299a34e894583c3862153e4b97","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_pull_request","missing_data","missing_tool","noop","remove_labels"]}]} # This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/add-community-preset.md b/.github/workflows/add-community-preset.md index 8b171e67bb..f6fbedcb49 100644 --- a/.github/workflows/add-community-preset.md +++ b/.github/workflows/add-community-preset.md @@ -292,7 +292,8 @@ The verifier reads `preset.yml` from the downloaded ZIP, including preset-scoped paths in monorepos; it parses YAML without executing archive content. It checks the published ID, version, Spec Kit requirement, required extension IDs, release tag, and README install references against the issue, then records the validated -values and existing `created_at` for the generated-file check. Exit 1 (`FAILED`) +values, current UTC date at submission validation, and existing `created_at` +for the generated-file check. Exit 1 (`FAILED`) is a confirmed submission mismatch: report it on the issue under Failed, with no PR. Exit 2 (`BLOCKED`) means the verifier could not read the archive or other required inputs (including a missing Python dependency): report the exact error @@ -381,8 +382,8 @@ Insert the entry in **alphabetical order by preset ID** within the "commands": }, "tags": ["", ""], - "created_at": "T00:00:00Z", - "updated_at": "T00:00:00Z" + "created_at": "T00:00:00Z", + "updated_at": "T00:00:00Z" } } ``` @@ -403,7 +404,8 @@ If the preset provides scripts, add `"scripts": ` inside `"provides"`. Replace only the changed fields (typically `version`, `download_url`, `description`, `provides`, `requires`, `tags`, `updated_at`). **Preserve** -`created_at` from the existing entry. +`created_at` from the existing entry. Use the verifier snapshot's validated UTC +date for `updated_at`, even if the UTC date changes during the run. ### Counting templates and commands @@ -413,8 +415,8 @@ Parse the "Templates Provided" and "Commands Provided" issue fields: ### After editing -Update the **top-level `"updated_at"` timestamp** in the catalog to today's date -in ISO 8601 format. +Update the **top-level `"updated_at"` timestamp** in the catalog to the +verifier snapshot's UTC date in ISO 8601 format. The generated-file verifier in Step 5 parses and checks the JSON. Fix any catalog error it reports and rerun it before continuing. @@ -452,7 +454,8 @@ python3 .github/scripts/validate_community_preset.py generated --issue /tmp/gh-a ``` The verifier checks JSON parsing, the validated catalog metadata and digest, -the top-level and entry `updated_at` timestamps, +the top-level and entry `updated_at` timestamps against the recorded UTC date +(and `created_at` for new entries), alphabetical ID order, the documentation row's name, purpose, counts, extension requirements and repository link, alphabetical name order, and preservation of `created_at` on updates. Exit 3 (`REPAIR`) is an agent-generated catalog or diff --git a/tests/test_community_preset_validation.py b/tests/test_community_preset_validation.py index a0ed7906f3..7b55599af5 100644 --- a/tests/test_community_preset_validation.py +++ b/tests/test_community_preset_validation.py @@ -5,6 +5,7 @@ import subprocess import sys import zipfile +from datetime import datetime, timezone from pathlib import Path import pytest @@ -84,7 +85,12 @@ def write_archive(paths, manifest): archive.writestr("release/sample/preset.yml", yaml.safe_dump(manifest)) -def write_generated(issue, paths, *, created_at="2025-01-01T00:00:00Z"): +def write_generated(issue, paths, *, created_at=None): + timestamp = ( + json.loads(paths["snapshot.json"].read_text(encoding="utf-8"))["expected_timestamp"] + if paths["snapshot.json"].exists() + else "2026-01-01T00:00:00Z" + ) entry = { "id": issue["preset_id"], "name": issue["preset_name"], "version": issue["version"], "description": issue["description"], @@ -96,7 +102,8 @@ def write_generated(issue, paths, *, created_at="2025-01-01T00:00:00Z"): "extensions": ["aide", "canon"]}, "provides": {"templates": 1, "commands": 1}, "tags": ["sample", "example"], - "created_at": created_at, "updated_at": "2026-01-01T00:00:00Z", + "created_at": created_at if created_at is not None else timestamp, + "updated_at": timestamp, } paths["catalog.json"].write_text(json.dumps({ "updated_at": entry["updated_at"], "presets": {"sample": entry}, @@ -114,8 +121,12 @@ def write_generated(issue, paths, *, created_at="2025-01-01T00:00:00Z"): def test_matching_monorepo_submission_and_generated_files_pass(submission): issue, _, paths = submission + before = datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z") first = run_verifier(paths) assert first.returncode == 0, first.stdout + first.stderr + snapshot = json.loads(paths["snapshot.json"].read_text(encoding="utf-8")) + after = datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z") + assert snapshot["expected_timestamp"] in (before, after) write_generated(issue, paths) second = run_verifier(paths, "generated") assert second.returncode == 0, second.stdout + second.stderr @@ -168,6 +179,29 @@ def test_stale_from_url_fails_even_with_valid_dev_command(submission): assert "README" in result.stdout +@pytest.mark.parametrize("archive_url", [False, True]) +def test_stale_from_url_with_submitted_scoped_tag_fails(submission, archive_url): + issue, _, paths = submission + if archive_url: + issue["download_url"] = ( + "https://github.com/example/presets/archive/refs/tags/" + "spec-kit-sample-v1.2.3.zip" + ) + else: + issue["download_url"] = issue["download_url"].replace( + "sample-v1.2.3", "spec-kit-sample-v1.2.3" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n", + encoding="utf-8", + ) + result = run_verifier(paths) + assert result.returncode == 1, result.stdout + result.stderr + assert "README --from URL" in result.stdout + + def test_dev_only_readme_is_accepted(submission): _, _, paths = submission paths["README.md"].write_text( @@ -196,6 +230,21 @@ def test_id_install_and_unrelated_monorepo_release_are_accepted(submission): assert run_verifier(paths).returncode == 0 +def test_unrelated_scoped_release_stays_accepted_with_submitted_scope(submission): + issue, _, paths = submission + issue["download_url"] = issue["download_url"].replace( + "sample-v1.2.3", "spec-kit-sample-v1.2.3" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add sample\n" + "specify preset add --from " + "https://github.com/example/presets/releases/download/other-v2.0.0/other.zip\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + def test_stale_scoped_release_in_another_repository_fails(submission): _, _, paths = submission paths["README.md"].write_text( @@ -256,6 +305,33 @@ def test_existing_entry_without_creation_date_blocks_update(submission): assert "created_at" in result.stdout +@pytest.mark.parametrize("timestamp_field", ["created_at", "updated_at"]) +def test_generated_new_entry_rejects_stale_dates(submission, timestamp_field): + issue, _, paths = submission + assert run_verifier(paths).returncode == 0 + entry = write_generated(issue, paths) + entry[timestamp_field] = "2000-01-01T00:00:00Z" + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], "presets": {"sample": entry}, + }), encoding="utf-8") + result = run_verifier(paths, "generated") + assert result.returncode == 3, result.stdout + result.stderr + assert timestamp_field in result.stdout + + +def test_generated_update_rejects_matching_stale_updated_dates(submission): + issue, _, paths = submission + entry = write_generated(issue, paths) + assert run_verifier(paths).returncode == 0 + entry["updated_at"] = "2000-01-01T00:00:00Z" + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], "presets": {"sample": entry}, + }), encoding="utf-8") + result = run_verifier(paths, "generated") + assert result.returncode == 3, result.stdout + result.stderr + assert "updated_at" in result.stdout + + @pytest.mark.parametrize(("damage", "message"), [ ("catalog-json", "catalog"), ("catalog-order", "alphabetical"), @@ -309,7 +385,7 @@ def test_generated_defects_are_fixable_not_submission_failures(submission, damag ), encoding="utf-8", ) elif damage == "created-at": - entry["created_at"] = "2026-01-01T00:00:00Z" + entry["created_at"] = "2000-01-01T00:00:00Z" paths["catalog.json"].write_text( json.dumps({"updated_at": entry["updated_at"], "presets": { "sample": entry, From 0007401e7d3e7e1409dd216f7f8d5f2b37f16155 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:26:41 -0500 Subject: [PATCH 3/6] fix: scope monorepo README links and validate homepage Avoid treating unrelated unscoped monorepo release URLs as stale while retaining checks for matching preset scopes and release assets. Require generated catalog homepage to equal the submitted repository URL for new and updated presets. Refs github/spec-kit#4746 Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/validate_community_preset.py | 17 +++- .../workflows/add-community-preset.lock.yml | 2 +- .github/workflows/add-community-preset.md | 18 ++-- tests/test_community_preset_validation.py | 90 +++++++++++++++++++ 4 files changed, 118 insertions(+), 9 deletions(-) diff --git a/.github/scripts/validate_community_preset.py b/.github/scripts/validate_community_preset.py index 2d1e2c2618..18b41704e2 100644 --- a/.github/scripts/validate_community_preset.py +++ b/.github/scripts/validate_community_preset.py @@ -163,6 +163,7 @@ def check_readme(text: str, issue: dict) -> None: accepted_scopes = {preset_id} if submitted_scope: accepted_scopes.add(submitted_scope[1]) + expected_path = urlsplit(expected).path.split("/") accepted = False for match in re.finditer( r"(? None: parsed.netloc.lower() == "github.com" and parsed.path.lower().startswith(f"/{owner}/{repo}/") ) + # An unscoped tag alone does not identify a preset in a monorepo. + same_release_asset = ( + len(expected_path) > 6 + and expected_path[3:5] == ["releases", "download"] + and len(parts) > 6 + and parts[3:5] == ["releases", "download"] + and expected_path[6] == parts[6] + ) + unscoped_archive = ( + submitted_scope is None + and expected_path[3:6] == ["archive", "refs", "tags"] + and parts[3:6] == ["archive", "refs", "tags"] + ) if (scoped and scoped[1] in accepted_scopes) or ( - same_repository and not scoped + same_repository and not scoped and (same_release_asset or unscoped_archive) ): raise SubmissionMismatch( f"README --from URL for {preset_id} differs from Download URL: {value}" @@ -253,6 +267,7 @@ def expected_values(issue: dict, manifest: dict, digest: str) -> dict: "description": field(issue, "description"), "author": field(issue, "author"), "repository": field(issue, "repository"), + "homepage": field(issue, "repository"), "download_url": field(issue, "download_url"), "sha256": digest, "documentation": field(issue, "documentation"), diff --git a/.github/workflows/add-community-preset.lock.yml b/.github/workflows/add-community-preset.lock.yml index 23cc790ba0..9e250ea110 100644 --- a/.github/workflows/add-community-preset.lock.yml +++ b/.github/workflows/add-community-preset.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"499cb209b6fc93a3b306cb0113aefb2367be2a84d952c4bb4516f982f535c25c","body_hash":"3b3c6aba8ccc13b6354b67fbde26368a41b687e089010a1e77617f363b3fa7c3","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"499cb209b6fc93a3b306cb0113aefb2367be2a84d952c4bb4516f982f535c25c","body_hash":"c518de62479c532553dd1d5f995a672776fcf13ef8ca04c0c109998e8598bdd4","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/setup-python","sha":"5fda3b95a4ea91299a34e894583c3862153e4b97","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_pull_request","missing_data","missing_tool","noop","remove_labels"]}]} # This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/add-community-preset.md b/.github/workflows/add-community-preset.md index f6fbedcb49..f4d36eeeea 100644 --- a/.github/workflows/add-community-preset.md +++ b/.github/workflows/add-community-preset.md @@ -182,10 +182,11 @@ preset** — not just any file named `README.md`, and not a product/framework pi A `specify preset add --from ` command only counts when its `` **matches the submitted Download URL exactly**. If the README also contains a `--from` release URL - for this preset in the submitted repository that differs from the Download URL, **fail** - even if another accepted command (`specify preset add ` or - `specify preset add --dev `) is present. A README with only a valid `--dev` - command remains acceptable. The verifier in Step 2g enforces this comparison. + identifiable as this preset by its tag scope or matching release asset that differs + from the Download URL, **fail** even if another accepted command (`specify preset add + ` or `specify preset add --dev `) is present. Do not flag a different + preset's unscoped release URL in a monorepo as stale. A README with only a valid + `--dev` command remains acceptable. The verifier in Step 2g enforces this comparison. If **no** accepted `specify preset add ...` command is present, the README is treated as a generic description/pitch rather than preset-usage documentation — **fail this check** and @@ -371,7 +372,7 @@ Insert the entry in **alphabetical order by preset ID** within the "repository": "", "download_url": "", "sha256": "", - "homepage": "", + "homepage": "", "documentation": "", "license": "", "requires": { @@ -403,7 +404,9 @@ If the preset provides scripts, add `"scripts": ` inside `"provides"`. ### For an update Replace only the changed fields (typically `version`, `download_url`, -`description`, `provides`, `requires`, `tags`, `updated_at`). **Preserve** +`description`, `homepage`, `provides`, `requires`, `tags`, `updated_at`). Set +`homepage` to the submitted repository URL; the form has no separate homepage +field. **Preserve** `created_at` from the existing entry. Use the verifier snapshot's validated UTC date for `updated_at`, even if the UTC date changes during the run. @@ -453,7 +456,8 @@ Before labeling success or requesting a PR, run this fixed command unchanged: python3 .github/scripts/validate_community_preset.py generated --issue /tmp/gh-aw/preset-submission.json --archive /tmp/gh-aw/community-archive.zip --readme /tmp/gh-aw/preset-readme.md --catalog presets/catalog.community.json --docs docs/community/presets.md --snapshot /tmp/gh-aw/preset-validation.json ``` -The verifier checks JSON parsing, the validated catalog metadata and digest, +The verifier checks JSON parsing, the validated catalog metadata (including +`homepage` set to the submitted repository URL) and digest, the top-level and entry `updated_at` timestamps against the recorded UTC date (and `created_at` for new entries), alphabetical ID order, the documentation row's name, purpose, counts, extension diff --git a/tests/test_community_preset_validation.py b/tests/test_community_preset_validation.py index 7b55599af5..31b87d32cb 100644 --- a/tests/test_community_preset_validation.py +++ b/tests/test_community_preset_validation.py @@ -245,6 +245,66 @@ def test_unrelated_scoped_release_stays_accepted_with_submitted_scope(submission assert run_verifier(paths).returncode == 0 +@pytest.mark.parametrize("archive_url", [False, True]) +def test_unrelated_unscoped_monorepo_release_stays_accepted(submission, archive_url): + issue, _, paths = submission + if archive_url: + issue["download_url"] = ( + "https://github.com/example/presets/archive/refs/tags/" + "spec-kit-sample-v1.2.3.zip" + ) + unrelated = ( + "https://github.com/example/presets/archive/refs/tags/v2.0.0.zip" + ) + else: + issue["download_url"] = issue["download_url"].replace( + "sample-v1.2.3", "spec-kit-sample-v1.2.3" + ) + unrelated = ( + "https://github.com/example/presets/releases/download/v2.0.0/other.zip" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + f"specify preset add --from {unrelated}\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + +def test_same_asset_on_unscoped_tag_is_reported_as_stale(submission): + issue, _, paths = submission + issue["download_url"] = issue["download_url"].replace( + "sample-v1.2.3", "v1.2.3" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + f"specify preset add --from {issue['download_url'].replace('v1.2.3', 'v1.2.2')}\n", + encoding="utf-8", + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "README --from URL" in result.stdout + + +def test_same_asset_on_bare_tag_stays_stale_for_scoped_submission(submission): + issue, _, paths = submission + issue["download_url"] = issue["download_url"].replace( + "sample-v1.2.3", "spec-kit-sample-v1.2.3" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + "specify preset add --from " + "https://github.com/example/presets/releases/download/v1.2.2/sample.zip\n", + encoding="utf-8", + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "README --from URL" in result.stdout + + def test_stale_scoped_release_in_another_repository_fails(submission): _, _, paths = submission paths["README.md"].write_text( @@ -332,10 +392,30 @@ def test_generated_update_rejects_matching_stale_updated_dates(submission): assert "updated_at" in result.stdout +def test_generated_update_rejects_stale_homepage(submission): + issue, _, paths = submission + entry = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + entry["homepage"] = "https://example.com/old" + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], "presets": {"sample": entry}, + }), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + entry = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + entry["homepage"] = "https://example.com/old" + paths["catalog.json"].write_text(json.dumps({ + "updated_at": entry["updated_at"], "presets": {"sample": entry}, + }), encoding="utf-8") + result = run_verifier(paths, "generated") + assert result.returncode == 3 + assert "homepage" in result.stdout + + @pytest.mark.parametrize(("damage", "message"), [ ("catalog-json", "catalog"), ("catalog-order", "alphabetical"), ("catalog-metadata", "version"), + ("homepage-missing", "homepage"), + ("homepage-stale", "homepage"), ("catalog-timestamp", "top-level updated_at"), ("docs-order", "alphabetical"), ("docs-row", "documentation row"), @@ -366,6 +446,16 @@ def test_generated_defects_are_fixable_not_submission_failures(submission, damag "sample": entry, }}), encoding="utf-8" ) + elif damage in ("homepage-missing", "homepage-stale"): + if damage == "homepage-missing": + del entry["homepage"] + else: + entry["homepage"] = "https://github.com/example/other" + paths["catalog.json"].write_text( + json.dumps({"updated_at": entry["updated_at"], "presets": { + "sample": entry, + }}), encoding="utf-8" + ) elif damage == "catalog-timestamp": paths["catalog.json"].write_text( json.dumps({"updated_at": "2020-01-01T00:00:00Z", "presets": { From 33815e53ac16fca7945b1b71bf44e3b857b38722 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:24:53 -0500 Subject: [PATCH 4/6] fix: distinguish unscoped archive URLs in monorepos Use the downloaded archive manifest count to apply bare-tag stale URL checks only when the archive contains one preset. Keep scoped tags and matching release assets checked, and document the monorepo exception with regression coverage. Refs github/spec-kit#4746 Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/validate_community_preset.py | 21 ++++++--- .../workflows/add-community-preset.lock.yml | 2 +- .github/workflows/add-community-preset.md | 7 ++- tests/test_community_preset_validation.py | 47 +++++++++++++++++++ 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/.github/scripts/validate_community_preset.py b/.github/scripts/validate_community_preset.py index 18b41704e2..f6ba3af278 100644 --- a/.github/scripts/validate_community_preset.py +++ b/.github/scripts/validate_community_preset.py @@ -97,14 +97,16 @@ def release_tag(issue: dict) -> str: return tag -def published_manifest(archive_path: Path, preset_id: str) -> dict: +def published_manifest(archive_path: Path, preset_id: str) -> tuple[dict, bool]: try: with zipfile.ZipFile(archive_path) as archive: matching = [] invalid = [] + manifest_count = 0 for member in archive.infolist(): if member.is_dir() or member.filename.rsplit("/", 1)[-1] != "preset.yml": continue + manifest_count += 1 try: if member.file_size > 1024 * 1024: raise ValueError("preset.yml exceeds 1 MiB") @@ -133,7 +135,7 @@ def published_manifest(archive_path: Path, preset_id: str) -> dict: raise SubmissionMismatch( f"expected one published preset.yml for {preset_id!r}, found {len(matching)}" ) - return matching[0][1] + return matching[0][1], manifest_count == 1 def required_extensions(manifest: dict) -> list[str]: @@ -155,7 +157,7 @@ def required_extensions(manifest: dict) -> list[str]: return result -def check_readme(text: str, issue: dict) -> None: +def check_readme(text: str, issue: dict, *, single_preset_archive: bool) -> None: expected = field(issue, "download_url") preset_id = field(issue, "preset_id") owner, repo = repository_parts(field(issue, "repository")) @@ -199,7 +201,8 @@ def check_readme(text: str, issue: dict) -> None: and expected_path[6] == parts[6] ) unscoped_archive = ( - submitted_scope is None + single_preset_archive + and submitted_scope is None and expected_path[3:6] == ["archive", "refs", "tags"] and parts[3:6] == ["archive", "refs", "tags"] ) @@ -281,8 +284,14 @@ def expected_values(issue: dict, manifest: dict, digest: str) -> dict: def submission(args: argparse.Namespace) -> None: issue = read_json(args.issue, "issue input", Blocked) release_tag(issue) - manifest = published_manifest(args.archive, field(issue, "preset_id")) - check_readme(read_text(args.readme, "fetched README"), issue) + manifest, single_preset_archive = published_manifest( + args.archive, field(issue, "preset_id") + ) + check_readme( + read_text(args.readme, "fetched README"), + issue, + single_preset_archive=single_preset_archive, + ) try: with args.archive.open("rb") as archive: digest = hashlib.file_digest(archive, "sha256").hexdigest() diff --git a/.github/workflows/add-community-preset.lock.yml b/.github/workflows/add-community-preset.lock.yml index 9e250ea110..5fd10b3cf1 100644 --- a/.github/workflows/add-community-preset.lock.yml +++ b/.github/workflows/add-community-preset.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"499cb209b6fc93a3b306cb0113aefb2367be2a84d952c4bb4516f982f535c25c","body_hash":"c518de62479c532553dd1d5f995a672776fcf13ef8ca04c0c109998e8598bdd4","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"499cb209b6fc93a3b306cb0113aefb2367be2a84d952c4bb4516f982f535c25c","body_hash":"67f2a3313f59103002b9e362e4d16ad1c3ef4035afb6c96f3446a345e4da4852","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/setup-python","sha":"5fda3b95a4ea91299a34e894583c3862153e4b97","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0daa8971fa4732b647150cb6524a6b0804b68d5d24f6f58b5dd1af23bd63fb23"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","create_pull_request","missing_data","missing_tool","noop","remove_labels"]}]} # This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/add-community-preset.md b/.github/workflows/add-community-preset.md index f4d36eeeea..80608481e7 100644 --- a/.github/workflows/add-community-preset.md +++ b/.github/workflows/add-community-preset.md @@ -185,8 +185,11 @@ preset** — not just any file named `README.md`, and not a product/framework pi identifiable as this preset by its tag scope or matching release asset that differs from the Download URL, **fail** even if another accepted command (`specify preset add ` or `specify preset add --dev `) is present. Do not flag a different - preset's unscoped release URL in a monorepo as stale. A README with only a valid - `--dev` command remains acceptable. The verifier in Step 2g enforces this comparison. + preset's unscoped release URL in a monorepo as stale. Bare archive tags are only + compared when the downloaded archive contains one `preset.yml`: a bare tag alone + cannot identify which preset it belongs to in a multi-preset archive. A README + with only a valid `--dev` command remains acceptable. The verifier in Step 2g + enforces this comparison. If **no** accepted `specify preset add ...` command is present, the README is treated as a generic description/pitch rather than preset-usage documentation — **fail this check** and diff --git a/tests/test_community_preset_validation.py b/tests/test_community_preset_validation.py index 31b87d32cb..e27458f518 100644 --- a/tests/test_community_preset_validation.py +++ b/tests/test_community_preset_validation.py @@ -272,6 +272,53 @@ def test_unrelated_unscoped_monorepo_release_stays_accepted(submission, archive_ assert run_verifier(paths).returncode == 0 +def test_unrelated_unscoped_archive_stays_accepted_in_monorepo(submission): + issue, _, paths = submission + issue["download_url"] = ( + "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + "specify preset add --from " + "https://github.com/example/presets/archive/refs/tags/v2.0.0.zip\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + +def test_stale_unscoped_archive_fails_for_single_preset(submission): + issue, manifest, paths = submission + write_archive(paths, manifest) + issue["download_url"] = ( + "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + "specify preset add --dev ./sample\n" + "specify preset add --from " + "https://github.com/example/presets/archive/refs/tags/v1.2.2.zip\n", + encoding="utf-8", + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "README --from URL" in result.stdout + + +def test_matching_unscoped_archive_passes_for_single_preset(submission): + issue, manifest, paths = submission + write_archive(paths, manifest) + issue["download_url"] = ( + "https://github.com/example/presets/archive/refs/tags/v1.2.3.zip" + ) + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + paths["README.md"].write_text( + f"specify preset add --from {issue['download_url']}\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + + def test_same_asset_on_unscoped_tag_is_reported_as_stale(submission): issue, _, paths = submission issue["download_url"] = issue["download_url"].replace( From 6cbc4851a6cbcc67ac818ae957617816f600be12 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:39:57 -0500 Subject: [PATCH 5/6] fix(presets): bound submitted archive validation Assisted-by: GitHub Copilot (model: GPT-5.6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/validate_community_preset.py | 55 +++++++++++++++++--- tests/test_community_preset_validation.py | 41 ++++++++++++++- 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/.github/scripts/validate_community_preset.py b/.github/scripts/validate_community_preset.py index f6ba3af278..5ec76b557e 100644 --- a/.github/scripts/validate_community_preset.py +++ b/.github/scripts/validate_community_preset.py @@ -20,6 +20,10 @@ print("BLOCKED: PyYAML is unavailable; cannot inspect published preset.yml") sys.exit(2) +MAX_MANIFEST_COUNT = 100 +MAX_MANIFEST_SIZE = 1024 * 1024 +MAX_TOTAL_MANIFEST_SIZE = 10 * 1024 * 1024 + class SubmissionMismatch(Exception): pass @@ -102,13 +106,24 @@ def published_manifest(archive_path: Path, preset_id: str) -> tuple[dict, bool]: with zipfile.ZipFile(archive_path) as archive: matching = [] invalid = [] - manifest_count = 0 - for member in archive.infolist(): - if member.is_dir() or member.filename.rsplit("/", 1)[-1] != "preset.yml": - continue - manifest_count += 1 + manifests = [ + member for member in archive.infolist() + if not member.is_dir() + and member.filename.rsplit("/", 1)[-1] == "preset.yml" + ] + if len(manifests) > MAX_MANIFEST_COUNT: + raise SubmissionMismatch( + f"archive contains more than {MAX_MANIFEST_COUNT} preset.yml files" + ) + total_size = sum(member.file_size for member in manifests) + if total_size > MAX_TOTAL_MANIFEST_SIZE: + raise SubmissionMismatch( + "archive preset.yml files exceed the " + f"{MAX_TOTAL_MANIFEST_SIZE // (1024 * 1024)} MiB total limit" + ) + for member in manifests: try: - if member.file_size > 1024 * 1024: + if member.file_size > MAX_MANIFEST_SIZE: raise ValueError("preset.yml exceeds 1 MiB") with archive.open(member) as stream: data = yaml.safe_load(stream.read().decode("utf-8")) @@ -135,7 +150,7 @@ def published_manifest(archive_path: Path, preset_id: str) -> tuple[dict, bool]: raise SubmissionMismatch( f"expected one published preset.yml for {preset_id!r}, found {len(matching)}" ) - return matching[0][1], manifest_count == 1 + return matching[0][1], len(manifests) == 1 def required_extensions(manifest: dict) -> list[str]: @@ -338,6 +353,30 @@ def cell(value: str) -> str: ) +def markdown_table_cells(row: str) -> list[str]: + cells = [] + cell = [] + escaped = False + for character in row: + if escaped: + if character == "|": + cell.append(character) + else: + cell.extend(("\\", character)) + escaped = False + elif character == "\\": + escaped = True + elif character == "|": + cells.append("".join(cell).strip()) + cell = [] + else: + cell.append(character) + if escaped: + cell.append("\\") + cells.append("".join(cell).strip()) + return cells + + def generated(args: argparse.Namespace) -> None: snapshot = read_json(args.snapshot, "verifier snapshot", Blocked) expected = snapshot.get("expected") @@ -379,7 +418,7 @@ def generated(args: argparse.Namespace) -> None: if not line.startswith("|"): break rows.append(line.strip()) - names = [row.split("|", 2)[1].strip() for row in rows] + names = [markdown_table_cells(row)[1] for row in rows] if names != sorted(names, key=str.casefold): raise GeneratedError("documentation preset names are not in alphabetical order") expected_row = documentation_row(expected) diff --git a/tests/test_community_preset_validation.py b/tests/test_community_preset_validation.py index e27458f518..29b42cacaa 100644 --- a/tests/test_community_preset_validation.py +++ b/tests/test_community_preset_validation.py @@ -108,10 +108,12 @@ def write_generated(issue, paths, *, created_at=None): paths["catalog.json"].write_text(json.dumps({ "updated_at": entry["updated_at"], "presets": {"sample": entry}, }), encoding="utf-8") + preset_name = issue["preset_name"].replace("|", r"\|") + description = issue["description"].replace("|", r"\|") paths["presets.md"].write_text( "| Preset | Purpose | Provides | Requires | URL |\n" "|--------|---------|----------|----------|-----|\n" - "| Sample Preset | Sample usage | 1 template, 1 command | " + f"| {preset_name} | {description} | 1 template, 1 command | " "aide extension, canon extension | " "[presets](https://github.com/example/presets) |\n", encoding="utf-8", @@ -382,6 +384,33 @@ def test_invalid_unrelated_monorepo_manifest_does_not_mask_match(submission): assert run_verifier(paths).returncode == 0 +def test_archive_with_too_many_manifests_is_rejected_before_parsing(submission): + _, _, paths = submission + with zipfile.ZipFile(paths["archive.zip"], "w") as archive: + for index in range(101): + archive.writestr(f"release/{index}/preset.yml", "preset: [invalid\n") + result = run_verifier(paths) + assert result.returncode == 1 + assert "more than 100 preset.yml files" in result.stdout + assert "invalid manifests" not in result.stdout + + +def test_archive_with_excessive_total_manifest_size_is_rejected(submission): + _, manifest, paths = submission + padding = "#" * (1024 * 1024 - len(yaml.safe_dump(manifest)) - 2) + with zipfile.ZipFile( + paths["archive.zip"], "w", compression=zipfile.ZIP_DEFLATED + ) as archive: + for index in range(11): + archive.writestr( + f"release/{index}/preset.yml", + f"{yaml.safe_dump(manifest)}\n{padding}", + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "10 MiB total limit" in result.stdout + + def test_missing_archive_is_blocked_not_failed(submission): _, _, paths = submission paths["archive.zip"].unlink() @@ -457,6 +486,16 @@ def test_generated_update_rejects_stale_homepage(submission): assert "homepage" in result.stdout +def test_generated_documentation_accepts_escaped_pipe_in_preset_name(submission): + issue, _, paths = submission + issue["preset_name"] = "Data | Governance" + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + write_generated(issue, paths) + result = run_verifier(paths, "generated") + assert result.returncode == 0, result.stdout + result.stderr + + @pytest.mark.parametrize(("damage", "message"), [ ("catalog-json", "catalog"), ("catalog-order", "alphabetical"), From db8baccb1f45fa07ec80e89d61f99c920ef9eba3 Mon Sep 17 00:00:00 2001 From: Manfred Riem <15701806+mnriem@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:59:15 -0500 Subject: [PATCH 6/6] fix(presets): validate README and documentation identity Assisted-by: GitHub Copilot (model: GPT-5.6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/validate_community_preset.py | 83 ++++++++++++++---- tests/test_community_preset_validation.py | 88 ++++++++++++++++++++ 2 files changed, 154 insertions(+), 17 deletions(-) diff --git a/.github/scripts/validate_community_preset.py b/.github/scripts/validate_community_preset.py index 5ec76b557e..d53134f1a0 100644 --- a/.github/scripts/validate_community_preset.py +++ b/.github/scripts/validate_community_preset.py @@ -189,8 +189,9 @@ def check_readme(text: str, issue: dict, *, single_preset_archive: bool) -> None text, ): option = match["option"] - value = (match["value"] or "").strip("'\"<>(),.;") + raw_value = match["value"] or "" if option == "--from": + value = raw_value.strip("'\"<>(),.;") if value == expected: accepted = True continue @@ -227,8 +228,10 @@ def check_readme(text: str, issue: dict, *, single_preset_archive: bool) -> None raise SubmissionMismatch( f"README --from URL for {preset_id} differs from Download URL: {value}" ) - elif option == "--dev" and value: - accepted = True + elif option == "--dev": + value = raw_value.strip("'\"") + if value and not value.startswith("-"): + accepted = True elif option == preset_id: accepted = True if not accepted: @@ -322,10 +325,28 @@ def submission(args: argparse.Namespace) -> None: raise Blocked("original catalog entry is not an object") if previous is not None and not isinstance(previous.get("created_at"), str): raise Blocked("original catalog entry has no created_at to preserve") + original_rows = documentation_rows( + read_text(args.docs, "original documentation", Blocked), Blocked + ) + expected_row = documentation_row(expected) + previous_row = None + if previous is not None: + try: + previous_row = documentation_row(previous) + except (AttributeError, KeyError, TypeError) as exc: + raise Blocked( + f"original catalog entry cannot produce a documentation row: {exc}" + ) from exc snapshot = { "expected": expected, "created_at": previous.get("created_at") if previous else None, "expected_timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT00:00:00Z"), + "documentation": { + "expected_row_count": original_rows.count(expected_row), + "previous_row": previous_row, + "previous_row_count": original_rows.count(previous_row) + if previous_row is not None else 0, + }, } try: args.snapshot.write_text(json.dumps(snapshot), encoding="utf-8") @@ -377,6 +398,20 @@ def markdown_table_cells(row: str) -> list[str]: return cells +def documentation_rows(text: str, error_type: type[Exception]) -> list[str]: + lines = text.splitlines() + try: + start = next(i for i, line in enumerate(lines) if line.startswith("| Preset |")) + except StopIteration as exc: + raise error_type("documentation has no Community Presets table") from exc + rows = [] + for line in lines[start + 2:]: + if not line.startswith("|"): + break + rows.append(line.strip()) + return rows + + def generated(args: argparse.Namespace) -> None: snapshot = read_json(args.snapshot, "verifier snapshot", Blocked) expected = snapshot.get("expected") @@ -407,25 +442,39 @@ def generated(args: argparse.Namespace) -> None: raise GeneratedError("catalog entry updated_at does not match the expected UTC date") if catalog.get("updated_at") != expected_timestamp: raise GeneratedError("catalog top-level updated_at does not match the expected UTC date") - docs = read_text(args.docs, "generated documentation", GeneratedError) - lines = docs.splitlines() - try: - start = next(i for i, line in enumerate(lines) if line.startswith("| Preset |")) - except StopIteration as exc: - raise GeneratedError("documentation has no Community Presets table") from exc - rows = [] - for line in lines[start + 2:]: - if not line.startswith("|"): - break - rows.append(line.strip()) + documentation = snapshot.get("documentation") + if not isinstance(documentation, dict): + raise Blocked("verifier snapshot lacks original documentation state") + expected_row_count = documentation.get("expected_row_count") + previous_row = documentation.get("previous_row") + previous_row_count = documentation.get("previous_row_count") + if ( + not isinstance(expected_row_count, int) + or isinstance(expected_row_count, bool) + or expected_row_count < 0 + or previous_row is not None and not isinstance(previous_row, str) + or not isinstance(previous_row_count, int) + or isinstance(previous_row_count, bool) + or previous_row_count < 0 + ): + raise Blocked("verifier snapshot has invalid original documentation state") + rows = documentation_rows( + read_text(args.docs, "generated documentation", GeneratedError), + GeneratedError, + ) names = [markdown_table_cells(row)[1] for row in rows] if names != sorted(names, key=str.casefold): raise GeneratedError("documentation preset names are not in alphabetical order") expected_row = documentation_row(expected) - if rows.count(expected_row) != 1: + expected_generated_count = expected_row_count + ( + 0 if previous_row == expected_row else 1 + ) + if rows.count(expected_row) != expected_generated_count: raise GeneratedError(f"documentation row does not match validated values: {expected_row}") - if names.count(expected["name"]) != 1: - raise GeneratedError("documentation contains duplicate preset names") + if previous_row is not None and previous_row != expected_row: + expected_previous_count = max(0, previous_row_count - 1) + if rows.count(previous_row) != expected_previous_count: + raise GeneratedError("previous documentation row was not replaced") print("PASSED: generated catalog and documentation match validated submission") diff --git a/tests/test_community_preset_validation.py b/tests/test_community_preset_validation.py index 29b42cacaa..847734791c 100644 --- a/tests/test_community_preset_validation.py +++ b/tests/test_community_preset_validation.py @@ -212,6 +212,24 @@ def test_dev_only_readme_is_accepted(submission): assert run_verifier(paths).returncode == 0 +def test_dev_current_directory_path_is_accepted(submission): + _, _, paths = submission + paths["README.md"].write_text( + "specify preset add --dev .\n", encoding="utf-8" + ) + assert run_verifier(paths).returncode == 0 + + +def test_dev_option_without_path_is_rejected(submission): + _, _, paths = submission + paths["README.md"].write_text( + "specify preset add --dev --priority 20\n", encoding="utf-8" + ) + result = run_verifier(paths) + assert result.returncode == 1 + assert "README" in result.stdout + + def test_quoted_from_url_with_sentence_punctuation_is_accepted(submission): issue, _, paths = submission paths["README.md"].write_text( @@ -496,6 +514,76 @@ def test_generated_documentation_accepts_escaped_pipe_in_preset_name(submission) assert result.returncode == 0, result.stdout + result.stderr +def test_generated_documentation_allows_duplicate_display_names(submission): + issue, _, paths = submission + other_row = ( + "| Sample Preset | Other usage | 1 command | — | " + "[other](https://github.com/example/other) |" + ) + paths["catalog.json"].write_text(json.dumps({ + "presets": {"other": {"name": issue["preset_name"]}}, + }), encoding="utf-8") + paths["presets.md"].write_text( + "| Preset | Purpose | Provides | Requires | URL |\n" + "|--------|---------|----------|----------|-----|\n" + f"{other_row}\n", + encoding="utf-8", + ) + assert run_verifier(paths).returncode == 0 + entry = write_generated(issue, paths) + catalog = json.loads(paths["catalog.json"].read_text(encoding="utf-8")) + catalog["presets"] = { + "other": {"name": issue["preset_name"]}, + "sample": entry, + } + paths["catalog.json"].write_text(json.dumps(catalog), encoding="utf-8") + paths["presets.md"].write_text( + "| Preset | Purpose | Provides | Requires | URL |\n" + "|--------|---------|----------|----------|-----|\n" + f"{other_row}\n" + "| Sample Preset | Sample usage | 1 template, 1 command | " + "aide extension, canon extension | " + "[presets](https://github.com/example/presets) |\n", + encoding="utf-8", + ) + result = run_verifier(paths, "generated") + assert result.returncode == 0, result.stdout + result.stderr + + +def test_generated_documentation_accepts_replaced_renamed_row(submission): + issue, _, paths = submission + original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + paths["catalog.json"].write_text(json.dumps({ + "updated_at": original["updated_at"], "presets": {"sample": original}, + }), encoding="utf-8") + issue["preset_name"] = "Renamed Preset" + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + result = run_verifier(paths, "generated") + assert result.returncode == 0, result.stdout + result.stderr + + +def test_generated_documentation_rejects_stale_row_after_rename(submission): + issue, _, paths = submission + original = write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + previous_row = paths["presets.md"].read_text(encoding="utf-8").splitlines()[2] + paths["catalog.json"].write_text(json.dumps({ + "updated_at": original["updated_at"], "presets": {"sample": original}, + }), encoding="utf-8") + issue["preset_name"] = "Renamed Preset" + paths["issue.json"].write_text(json.dumps(issue), encoding="utf-8") + assert run_verifier(paths).returncode == 0 + write_generated(issue, paths, created_at="2024-12-01T00:00:00Z") + paths["presets.md"].write_text( + paths["presets.md"].read_text(encoding="utf-8") + previous_row + "\n", + encoding="utf-8", + ) + result = run_verifier(paths, "generated") + assert result.returncode == 3 + assert "previous documentation row" in result.stdout + + @pytest.mark.parametrize(("damage", "message"), [ ("catalog-json", "catalog"), ("catalog-order", "alphabetical"),