From f2b94e1619fed05fd88d7b473ce6b3926556544e Mon Sep 17 00:00:00 2001 From: zengchuang Date: Mon, 2 Nov 2026 15:21:22 +0800 Subject: [PATCH 01/18] ci: add optional GPT summaries before release publication Keep GPT in a read-only notes job, preserve the GitHub changelog, and fall back to ordinary notes if summary generation or artifact delivery fails. Agent-assisted-by: Codex (implementation) Signed-off-by: zengchuang --- .github/workflows/release.yml | 74 +++++++++++++++++++++++++++++++-- docs/releases.md | 51 ++++++++++++++++++++--- tests/test_release_publisher.py | 61 ++++++++++++++++++++++++++- 3 files changed, 176 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 753ff0ccb..afbc047de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -122,9 +122,57 @@ jobs: if-no-files-found: error retention-days: 7 - publish: + notes: needs: [plan, build] + if: needs.plan.outputs.release == 'true' && needs.build.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + permissions: + contents: read + pull-requests: none + outputs: + available: ${{ steps.notes.outputs.available }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.plan.outputs.commit }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Summarize the validated source with the GPT API + id: notes + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_MODEL: ${{ vars.OPENAI_RELEASE_NOTES_MODEL || 'gpt-5.4-mini' }} + GH_REPO: ${{ github.repository }} + RELEASE_TAG: ${{ needs.plan.outputs.tag }} + RELEASE_COMMIT: ${{ needs.plan.outputs.commit }} + PREVIOUS_TAG: ${{ needs.plan.outputs.previous_tag }} + shell: bash + run: | + python tools/generate_release_notes.py --tag "$RELEASE_TAG" \ + --commit "$RELEASE_COMMIT" --previous-tag "$PREVIOUS_TAG" \ + --model "$OPENAI_MODEL" --output "$RUNNER_TEMP/release-notes/ai-notes.md" + if [[ -s "$RUNNER_TEMP/release-notes/ai-notes.md" ]]; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + fi + - uses: actions/upload-artifact@v4 + if: steps.notes.outputs.available == 'true' + with: + name: release-notes + path: ${{ runner.temp }}/release-notes/ai-notes.md + if-no-files-found: error + retention-days: 7 + + publish: + needs: [plan, build, notes] if: >- + always() && !cancelled() && needs.plan.outputs.release == 'true' && needs.build.result == 'success' && (github.event_name == 'schedule' || !inputs.dry_run) runs-on: ubuntu-latest @@ -156,6 +204,13 @@ jobs: with: name: release-distributions path: dist + - uses: actions/download-artifact@v4 + id: download_notes + if: needs.notes.outputs.available == 'true' + continue-on-error: true + with: + name: release-notes + path: notes - name: Verify downloaded checksums working-directory: dist run: sha256sum --check SHA256SUMS @@ -177,16 +232,27 @@ jobs: [[ "$latest" == "$PREVIOUS_TAG" ]] - name: Create the remote tag, upload packages, then publish shell: bash + env: + AI_NOTES_DOWNLOADED: ${{ steps.download_notes.outcome }} run: | gh api --method POST "repos/$GH_REPO/git/refs" \ -f ref="refs/tags/$RELEASE_TAG" -f sha="$RELEASE_COMMIT" > /dev/null - notes="Source: https://github.com/$GH_REPO/tree/$RELEASE_COMMIT" - notes+=$'\n\nThese notes describe software changes. Release builds and scaffold checks do not establish campaign PASS, model quality or service performance. Any such claim requires its acceptance issue and complete frozen evidence.' + notes_file="$RUNNER_TEMP/release-body.md" + { + printf 'Source: https://github.com/%s/tree/%s\n' "$GH_REPO" "$RELEASE_COMMIT" + printf '\nThese notes describe software changes. Release builds and scaffold checks do not establish campaign PASS, model quality or service performance. Any such claim requires its acceptance issue and complete frozen evidence.\n' + if [[ "${AI_NOTES_DOWNLOADED:-}" == success && -f notes/ai-notes.md && ! -L notes/ai-notes.md ]] && + [[ "$(wc -c < notes/ai-notes.md)" -gt 0 && "$(wc -c < notes/ai-notes.md)" -le 20000 ]]; then + printf '\n## AI-generated summary\n\n' + cat notes/ai-notes.md + printf '\n' + fi + } > "$notes_file" options=() if [[ -n "$PREVIOUS_TAG" ]]; then options+=(--notes-start-tag "$PREVIOUS_TAG"); fi gh release create "$RELEASE_TAG" dist/*.whl dist/*.tar.gz dist/*.zip dist/SHA256SUMS \ --verify-tag --draft --title "InferRSI $RELEASE_TAG" \ - --generate-notes --notes "$notes" "${options[@]}" + --generate-notes --notes-file "$notes_file" "${options[@]}" assets=$(gh release view "$RELEASE_TAG" --json assets --jq '.assets[].name') for file in dist/*.whl dist/*.tar.gz dist/*.zip dist/SHA256SUMS; do grep -Fxq "$(basename "$file")" <<< "$assets" diff --git a/docs/releases.md b/docs/releases.md index d40b9f407..e9097ba9d 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -26,7 +26,7 @@ Release. It must not create code commits, edit repository files, create/update/ delete code branches, create/approve/merge PRs or enable auto-merge. Configuration development and review are contributor work; they are not bot capabilities. -Plan/build jobs have Contents read access. Only the publisher receives Contents +Plan/build/notes jobs have Contents read access. Only the publisher receives Contents write; Pull requests permission is explicitly none. The publisher validates its tag/SHA inputs and never checks out or executes repository code. Regression tests execute its actual shell commands against a recording GitHub mock that @@ -64,7 +64,12 @@ configuration does not silently change their access or existing branch rules. Its generated plugin metadata, bundled runtime and tag must have the same version. Verify every bundled runtime hash and exercise the extracted launcher/knowledge reads outside the checkout with isolated Python. -4. A separate write job verifies the artifacts, source ancestry and release +4. After a successful build, an optional read-only notes job calls the GPT API + with bounded change metadata for exactly the planned revision range. Its + Markdown summary is transferred as a separate artifact. Missing credentials, + API/validation errors and unavailable notes artifacts fall back to the + existing GitHub-generated notes. +5. A separate write job verifies the artifacts, source ancestry and release state again. It creates the remote tag only after validation, creates an internal draft, uploads wheel/sdist/plugin ZIP/checksums and automatically publishes after confirming the attachments and tag identity. It does not execute @@ -79,6 +84,39 @@ software changes; CPU/package checks do not establish campaign `PASS`, model quality or service performance. Such claims retain the acceptance and evidence requirements in [contributing](contributing.md#evidence-and-claims). +## GPT release summaries + +To enable GPT summaries, add the Actions repository secret `OPENAI_API_KEY` under +**Settings → Secrets and variables → Actions**. The key is injected only into +the notes step. Do not put it in workflow source, command arguments, PR text or +logs. OpenAI API access and usage are billed to the key's API project; GitHub +Copilot access is not required. + +The default model is `gpt-5.4-mini`. Set the optional Actions repository variable +`OPENAI_RELEASE_NOTES_MODEL` to another model available to your API project and +compatible with the Responses API. No custom API endpoint is used. Without the +secret, the workflow skips GPT and continues ordinary publication. + +The [notes helper](../tools/generate_release_notes.py) sends bounded commit +subjects, source links and file-change statistics from the previous successful +tag through the planned commit. It omits authors, email addresses, commit bodies +and raw patches. Inputs are treated as source data, not executable instructions. +The prompt asks for a concise English overview of software features, fixes and +compatibility changes; it must not invent performance/quality or campaign PASS +claims. The response is labeled **AI-generated summary** and prepended to the +original GitHub changelog, preserving its PR links and source/evidence notice. +Model output is read as a file and never executed as shell code. + +The helper makes one request to the +[OpenAI Responses API](https://developers.openai.com/api/docs/guides/text), with +bounded input/output, a timeout, no tools and `store: false`. That setting does +not establish zero data retention. Missing credentials, failed/incomplete/refused +or unusable responses leave no summary. The optional notes job or artifact may +fail without skipping an otherwise valid publication; failed package checks and +workflow cancellation still prevent publication. The AI job has no repository +write permission, and the publisher receives no OpenAI key. There is no AI code +commit, merge, account installation or update to an existing public Release. + Each Release contains a Python wheel, source distribution, the versioned Codex plugin ZIP and `SHA256SUMS` covering all three packages. Extract the ZIP and follow the [Codex plugin installation guide](codex-plugin.md#install-from-the-release-zip). @@ -92,13 +130,16 @@ wrapper is packaged; the downloaded ZIP is the only Codex plugin release asset. Merge the configuration into the default branch. GitHub Actions and the listed actions must be allowed by organization policy, and the publisher needs -`contents: write`. No extra secret, PAT, GitHub App or repository-wide write -default is needed. The workflow guards the canonical repository and default +`contents: write`. Ordinary releases need no extra secret, PAT, GitHub App or +repository-wide write default. GPT summaries additionally use the optional +`OPENAI_API_KEY` secret described above. The workflow guards the canonical repository and default branch; copies must deliberately update that repository guard. Under **Actions → Release bot → Run workflow**, select the default branch and leave **dry_run** checked. It plans and builds the next release without creating -a remote tag or Release; unchanged trees skip here too. Clearing **dry_run** +a remote tag or Release; unchanged trees skip here too. With `OPENAI_API_KEY` +configured, a changed-source dry run also makes the GPT request and uploads the +summary artifact for inspection, consuming API usage. Clearing **dry_run** runs the same automatic publication immediately. It takes no version/tag input. ```bash diff --git a/tests/test_release_publisher.py b/tests/test_release_publisher.py index fff623ef5..e6be38588 100644 --- a/tests/test_release_publisher.py +++ b/tests/test_release_publisher.py @@ -59,6 +59,19 @@ def save(): installer = "dist/inferrsi-codex-plugin-v1.2.3.zip" if installer not in args: raise SystemExit("Release upload must include the versioned Codex installer") + if "--generate-notes" not in args or "--notes-file" not in args: + raise SystemExit("Keep the GitHub changelog and read summary as a file") + notes = Path(args[args.index("--notes-file") + 1]).read_text(encoding="utf-8") + if "Source: https://github.com/example/release/tree/" + commit not in notes: + raise SystemExit("Release notes must retain exact source provenance") + state["release_notes"] = notes + save() + if (state.get("ai_summary") and not state.get("failed_notes_download") + and len(state["ai_summary"].encode("utf-8")) <= 20000): + if "## AI-generated summary" not in notes or state["ai_summary"] not in notes: + raise SystemExit("Missing AI summary") + elif "## AI-generated summary" in notes: + raise SystemExit("Missing AI output must fall back to ordinary release notes") record("upload_plugin", name=Path(installer).name) state["assets"] = ["inferrsi-1.2.3.whl", "inferrsi-1.2.3.tar.gz", "SHA256SUMS", Path(installer).name] @@ -84,12 +97,29 @@ def test_write_token_is_confined_to_a_no_checkout_publisher(): config = workflow() assert config["permissions"] == {"contents": "read", "pull-requests": "none"} for name, job in config["jobs"].items(): + if name == "notes": + assert job["permissions"] == {"contents": "read", "pull-requests": "none"} + assert job["continue-on-error"] is True + continue if name != "publish": assert not job.get("permissions") continue assert job["permissions"] == {"contents": "write", "pull-requests": "none"} actions = [step["uses"] for step in job["steps"] if "uses" in step] - assert actions == ["actions/download-artifact@v4"] + assert actions == ["actions/download-artifact@v4", "actions/download-artifact@v4"] + assert "OPENAI_API_KEY" not in json.dumps(job) + + +def test_optional_notes_failure_cannot_skip_an_otherwise_valid_publication(): + job = workflow()["jobs"]["publish"] + assert job["needs"] == ["plan", "build", "notes"] + assert "always() && !cancelled()" in job["if"] + assert "needs.build.result == 'success'" in job["if"] + assert "needs.notes.result" not in job["if"] + optional = [step for step in job["steps"] + if step.get("with", {}).get("name") == "release-notes"] + assert len(optional) == 1 + assert optional[0]["continue-on-error"] is True @pytest.fixture @@ -124,6 +154,9 @@ def publisher(tmp_path): encoding="utf-8", newline="\n") def run(state=None, **inputs): + if state and state.get("ai_summary"): + (tmp_path / "notes").mkdir() + (tmp_path / "notes/ai-notes.md").write_text(state["ai_summary"], encoding="utf-8") if state and state.get("corrupt_zip"): (dist / "inferrsi-codex-plugin-v1.2.3.zip").write_bytes(b"changed installer") if state and state.get("absent_zip"): @@ -133,10 +166,15 @@ def run(state=None, **inputs): env = dict(os.environ) env.update(PATH=str(mock_bin) + os.pathsep + env["PATH"], FAKE_GH_STATE=str(state_path), FAKE_GH_LOG=str(log_path), + RUNNER_TEMP=str(tmp_path), GH_TOKEN="fake-no-access", GH_REPO="example/release", DEFAULT_BRANCH="main", RELEASE_TAG="v1.2.3", RELEASE_COMMIT=COMMIT, PREVIOUS_TAG="v1.2.2", PRERELEASE="true") env.update(inputs) + env["AI_NOTES_DOWNLOADED"] = ( + "failure" if state and state.get("failed_notes_download") else + "success" if state and state.get("ai_summary") else "skipped" + ) result = subprocess.run([bash, str(script)], cwd=tmp_path, env=env, capture_output=True, text=True, timeout=30) entries = [json.loads(line) for line in log_path.read_text().splitlines()] if ( @@ -158,6 +196,27 @@ def test_publisher_only_creates_a_tag_and_publishes_its_release(publisher): ] +def test_ai_notes_are_text_and_keep_the_publication_mutation_boundary(publisher, tmp_path): + # Treat model output as data even if it contains shell syntax. + summary = "Features: a new command. `touch should-not-exist` $(touch should-not-exist)" + result, mutations = publisher({"ai_summary": summary}) + assert result.returncode == 0, result.stderr + assert [entry["operation"] for entry in mutations] == [ + "create_tag", "create_release", "upload_plugin", "publish_release", + ] + assert not (tmp_path / "should-not-exist").exists() + + +@pytest.mark.parametrize("state", [ + {"ai_summary": "A partial download", "failed_notes_download": True}, + {"ai_summary": "x" * 20001}, +]) +def test_unusable_notes_artifact_does_not_block_ordinary_publication(publisher, state): + result, mutations = publisher(state) + assert result.returncode == 0, result.stderr + assert mutations[-1] == {"operation": "publish_release"} + + @pytest.mark.parametrize("inputs", [ {"RELEASE_TAG": "main"}, {"RELEASE_TAG": "refs/heads/main"}, {"RELEASE_TAG": "--help"}, {"RELEASE_TAG": "v1.2.3\nrefs/heads/main"}, From dca654668bdb6a509d71fd46e14980cc0aab6fab Mon Sep 17 00:00:00 2001 From: zengchuang Date: Mon, 2 Nov 2026 15:26:40 +0800 Subject: [PATCH 02/18] feat: generate bounded release overviews through the GPT API Generate an optional Markdown overview from a bounded immutable commit range and file statistics. Isolate API credentials from subprocesses, reject redirects and invalid replies, and fall back without stale output when generation is unavailable. Agent-assisted-by: Codex (implementation) Signed-off-by: zengchuang --- tests/test_release_notes.py | 299 ++++++++++++++++++++++++++++++++ tools/generate_release_notes.py | 242 ++++++++++++++++++++++++++ 2 files changed, 541 insertions(+) create mode 100644 tests/test_release_notes.py create mode 100644 tools/generate_release_notes.py diff --git a/tests/test_release_notes.py b/tests/test_release_notes.py new file mode 100644 index 000000000..e76794edc --- /dev/null +++ b/tests/test_release_notes.py @@ -0,0 +1,299 @@ +"""Release overviews use mocked HTTP and synthetic local source histories only.""" + +import io +import json +import subprocess +import sys +import urllib.error +from pathlib import Path + +import pytest + +from tools import generate_release_notes as notes + +KEY = "sk-test-no-access-credential" +SHA = "1" * 40 + + +def completed(text="## Features\n- Preserve source identity."): + return {"status": "completed", "output": [ + {"type": "reasoning", "summary": []}, + {"type": "message", "role": "assistant", "status": "completed", + "content": [{"type": "output_text", "text": text}]}, + ]} + + +@pytest.fixture +def source_repo(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + def git(*args): + return subprocess.check_output(["git", *args], text=True, stderr=subprocess.PIPE).strip() + + git("init", "-b", "main") + git("config", "user.name", "Release Test") + git("config", "user.email", "release@example.invalid") + (tmp_path / "initial.txt").write_text("initial\n", encoding="utf-8") + git("add", ".") + git("commit", "-m", "initial subject") + git("tag", "v0.1.0") + (tmp_path / "feature.txt").write_text("feature\n", encoding="utf-8") + git("add", ".") + git("commit", "-m", "feat: retain identity (#42)\n\n" + "PRIVATE_BODY_MUST_NOT_BE_SENT\nSigned-off-by: Release Test ") + return git, git("rev-parse", "HEAD") + + +def test_exact_source_range_contains_subjects_links_and_statistics_only(source_repo, monkeypatch): + _, sha = source_repo + monkeypatch.setenv("OPENAI_API_KEY", KEY) + monkeypatch.setenv("GH_TOKEN", "fake-github-token") + monkeypatch.setenv("GITHUB_TOKEN", "fake-actions-token") + original = subprocess.Popen + calls = [] + + def checked(*args, **kwargs): + env = kwargs["env"] + assert "OPENAI_API_KEY" not in env and "GH_TOKEN" not in env and "GITHUB_TOKEN" not in env + calls.append(args[0]) + return original(*args, **kwargs) + + monkeypatch.setattr(subprocess, "Popen", checked) + result = notes.source_snapshot("v0.1.1", sha, "v0.1.0", "example/release") + value = json.loads(result) + assert len(value["commits"]) == 1 + assert value["commits"][0] == { + "sha": sha, "subject": "feat: retain identity (#42)", + "url": f"https://github.com/example/release/commit/{sha}", + } + assert "feature.txt" in value["file_change_statistics"] + assert "initial subject" not in result + assert "PRIVATE_BODY" not in result and "release@example.invalid" not in result + assert "Signed-off-by" not in result and KEY not in result + assert calls and not any("--patch" in call or "-p" in call for call in calls) + + +def test_first_release_uses_complete_history_and_empty_tree_statistics(source_repo): + _, sha = source_repo + value = json.loads(notes.source_snapshot("v0.1.1", sha, "", "example/release")) + assert len(value["commits"]) == 2 + assert "initial.txt" in value["file_change_statistics"] + assert "feature.txt" in value["file_change_statistics"] + + +@pytest.mark.parametrize("tag,commit,previous,repository", [ + ("main", SHA, "", "example/release"), ("v01.1.0", SHA, "", "example/release"), + ("v0.1.1", "main", "", "example/release"), + ("v0.1.1", SHA, "refs/heads/main", "example/release"), + ("v0.1.1", SHA, "", "https://example.invalid/repo"), +]) +def test_invalid_source_identity_never_runs_git(monkeypatch, tag, commit, previous, repository): + monkeypatch.setattr(notes, "git", lambda *args: pytest.fail("Git must not run")) + with pytest.raises(ValueError): + notes.source_snapshot(tag, commit, previous, repository) + + +def test_source_checkout_must_match_planned_commit(source_repo): + with pytest.raises(ValueError, match="does not match"): + notes.source_snapshot("v0.1.1", SHA, "", "example/release") + + +def test_previous_release_must_be_an_ancestor(source_repo): + git, sha = source_repo + git("checkout", "-b", "unmerged", "v0.1.0") + git("commit", "--allow-empty", "-m", "unmerged branch") + git("tag", "v0.9.0") + git("checkout", "main") + with pytest.raises(ValueError, match="Source inspection failed"): + notes.source_snapshot("v0.1.1", sha, "v0.9.0", "example/release") + + +def test_source_budget_limits_subjects_commits_and_total_utf8_bytes(monkeypatch): + def git(args, limit=256): + if args[0] == "rev-parse": + return SHA, False + if args[0] == "hash-object": + return "2" * 40, False + if args[0] == "log": + return "".join(f"{number:040x}\t" + "知" * 500 + "\n" for number in range(101)), True + assert args[0] == "diff" and "--numstat" in args + return "1\t0\t" + "file" * 2_500, True + + monkeypatch.setattr(notes, "git", git) + result = notes.source_snapshot("v0.1.1", SHA, "", "example/" + "r" * 150) + value = json.loads(result) + assert len(result.encode("utf-8")) <= notes.MAX_SOURCE_BYTES + assert len(value["commits"]) <= 100 + assert all(len(item["subject"].encode("utf-8")) <= 240 for item in value["commits"]) + assert value["commits_truncated"] and value["file_statistics_truncated"] + + +def test_git_output_is_bounded_before_capture(monkeypatch): + original = subprocess.Popen + + def output_process(command, **kwargs): + assert "OPENAI_API_KEY" not in kwargs["env"] + return original([sys.executable, "-c", "import sys; sys.stdout.write('x' * 20000)"], + **kwargs) + + monkeypatch.setattr(subprocess, "Popen", output_process) + value, truncated = notes.git(["ignored"], 1024) + assert len(value) == 1024 and truncated + + +@pytest.fixture +def invocation(tmp_path, monkeypatch): + output = tmp_path / "overview.md" + monkeypatch.setenv("OPENAI_API_KEY", KEY) + monkeypatch.setenv("GH_REPO", "example/release") + monkeypatch.setattr(notes, "source_snapshot", lambda *args: '{"commits": []}') + arguments = ["--tag", "v0.1.1", "--commit", SHA, "--previous-tag", "", "--output", str(output)] + + def run(body=None, *, error=None, status=200): + calls = [] + + class Response(io.BytesIO): + pass + + def open_request(request, timeout): + calls.append((request, timeout)) + if error: + raise error + response = Response(body if isinstance(body, bytes) else json.dumps(body).encode()) + response.status = status + return response + + class Opener: + open = staticmethod(open_request) + + def build_opener(handler): + assert isinstance(handler, notes.NoRedirect) + return Opener() + + monkeypatch.setattr(notes.urllib.request, "build_opener", build_opener) + return notes.main(arguments), output, calls + + return run, output, arguments + + +def test_completed_mixed_output_items_make_one_bounded_markdown_file(invocation, capsys): + run, _, _ = invocation + response = completed() + response["output"].append({ + "type": "message", "role": "assistant", "status": "completed", + "content": [{"type": "output_text", "text": "## Compatibility\n"}, + {"type": "output_text", "text": "- Existing interfaces remain available."}], + }) + code, output, calls = run(response) + assert code == 0 and output.is_file() + text = output.read_text(encoding="utf-8") + assert "## Features" in text and "## Compatibility" in text + request, timeout = calls[0] + assert len(calls) == 1 and timeout == 45 + assert request.full_url == notes.ENDPOINT and request.method == "POST" + assert request.get_header("Authorization") == f"Bearer {KEY}" + payload = json.loads(request.data) + assert payload["model"] == "gpt-5.4-mini" and payload["store"] is False + assert payload["max_output_tokens"] == 1600 + assert "temperature" not in payload and "tools" not in payload + assert KEY not in request.data.decode() + assert "untrusted" in payload["instructions"] + captured = capsys.readouterr() + assert captured.out == "GPT release overview generated.\n" and captured.err == "" + + +@pytest.mark.parametrize("body", [ + b"not JSON", [], {"status": "incomplete", "output": []}, + {"status": "completed", "output": [], "error": {"message": KEY}}, + completed(""), completed("## Features\n" + "知" * 8_000), + completed("## Features\n\x1b[31munsafe"), completed("## Features\n\u202eunsafe"), + completed("```markdown\n## Features\n```"), completed("## Features\n