diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index a1c61d1d..d91c1bd6 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -119,6 +119,8 @@ Releases are managed by Release Please. After feature and fix PRs land on `main` Release notes require either `ANTHROPIC_API_KEY` or `OPENAI_API_KEY`; when using OpenAI-compatible credentials, set `COMMUNIQUE_MODEL` so Communique knows which model to call. The pinned Communique Linux binary currently requires a glibc 2.39-compatible environment (for example Ubuntu 24.04 or the GitHub-hosted `ubuntu-latest` runner); on older Linux hosts, run the dry-run in a newer container or runner. +`communique.toml` sets `max_tokens = 16384`. Communique 1.1.3 defaults to 4096 output tokens, and its one `submit_release_notes` call (changelog, title, and body) does not fit in that for a release with about 140 PRs. The model's call is then cut off, Communique makes three attempts in total, and the job passes only when the last attempt still holds a `changelog` or a `release_body`. `npm --prefix scripts run test:release-please-runner` checks that the pinned Communique and this config fit a submission larger than the default. + To dogfood release automation without mutating GitHub, install the pinned tools from `mise.toml`, install the locked Release Please dependency graph under `scripts/`, and run the runner in dry-run mode against a branch that already contains the release config: ```bash diff --git a/README.md b/README.md index a2f604c8..f23fc116 100644 --- a/README.md +++ b/README.md @@ -1014,7 +1014,8 @@ The plugin runs the CLI through a [terminal provider](./lua/claudecode/terminal/ - Run: `mise run all` (format, lint, test) - Run: `mise run test` (Busted suite with coverage, in a random order whose seed it prints; `BUSTED_SEED= mise run test` repeats that order, and a test registered inside a `pairs()` loop fails the run) or one file with `busted -v tests/unit/_spec.lua` - Run: `mise run check` (Lua syntax and `luacheck` on `lua/` and `tests/`) -- Run: `npm --prefix scripts run test:release-please-runner` (release runner, after `npm --prefix scripts ci`) +- Run: `npm --prefix scripts run test:release-please-runner` (release runner and the `communique.toml` output budget, + after `npm --prefix scripts ci`, with `communique` on `PATH`) ## Contributing diff --git a/communique.toml b/communique.toml index 0d1c41e4..d691d094 100644 --- a/communique.toml +++ b/communique.toml @@ -19,6 +19,9 @@ Do not invent changes that are not supported by the commits, pull requests, or d [defaults] model = "claude-opus-4-8" +# Communique 1.1.3 defaults to 4096 output tokens, which cuts off the one +# submit_release_notes call that carries changelog, title, and body. +max_tokens = 16384 repo = "coder/claudecode.nvim" verify_links = true match_style = true diff --git a/scripts/README.md b/scripts/README.md index 053f6442..8f1f10e6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,7 +8,7 @@ 2. Interactive and probe clients built on those libraries (`claude_interactive.sh`, `mcp_test.sh`, `research_messages.sh`, `websocat.sh`, `test_neovim_websocket.sh`). 3. Test harnesses (`run_integration_tests_individually.sh`, `test_opendiff.lua`, `test_opendiff_simple.sh`, `manual_test_helper.lua`). 4. Standalone reproductions of GitHub issues (`repro_issue_*.lua`, `repro_issue_*.sh`). -5. Release tooling: a release-please wrapper that writes changelog entries with Communique (`release-please-runner.js`, `run-release-please.sh`, `package.json`). `test-release-please-runner.js` holds the tests for `release-please-runner.js`. +5. Release tooling: a release-please wrapper that writes changelog entries with Communique (`release-please-runner.js`, `run-release-please.sh`, `package.json`). `test-release-please-runner.js` holds the tests for `release-please-runner.js`. It also runs the pinned `communique` binary with the repo's `communique.toml` against a fake Anthropic API, to check that one release-notes submission fits the configured output budget. CI runs four of these files: `.github/workflows/release-please.yaml` runs `./scripts/run-release-please.sh`, and `.github/workflows/test.yml` runs `./scripts/run_integration_tests_individually.sh` and `nvim --headless -u NONE -c "luafile scripts/repro_issue_238.lua" -c "cquit 2"` on Neovim stable and v0.8.0, and `npm --prefix scripts run test:release-please-runner` (which runs `test-release-please-runner.js`). @@ -119,9 +119,10 @@ The Lua repros run from the repository root with `nvim --headless -u NONE -l scr - `repro_issue_283.lua` binds TCP sockets on `127.0.0.1`. `repro_issue_283.sh` starts a background Neovim and kills it on exit [repro_issue_283.lua:listener:bind] [repro_issue_283.sh:cleanup]. - `release-please-runner.js` without `--dry-run` creates GitHub tags, releases, and pull requests through the GitHub API [release-please-runner.js:main]. - `release-please-runner.js` spawns `communique generate` with the full environment (API keys included) in a temp directory `communique-notes-*` that it removes afterward. It runs `prettier` on each changelog update and appends outputs to `GITHUB_OUTPUT`, or prints them when that variable is unset [release-please-runner.js:createCommuniqueChangelogNotes, formatChangelogMarkdown, writeGithubOutputs]. +- `test-release-please-runner.js` starts an HTTP server on a random `127.0.0.1` port and runs `git` and `communique` in a temp directory `communique-config-*` that it removes afterward. It passes `communique` only a placeholder API key [test-release-please-runner.js:testCommuniqueConfigFitsReleaseNotes]. ## Verification -- Run: `npm --prefix scripts run test:release-please-runner` (needs `npm --prefix scripts ci` first). +- Run: `npm --prefix scripts run test:release-please-runner` (needs `npm --prefix scripts ci` first, and `communique` on `PATH`, which `mise install` provides). - Run: `mise run format-check`. treefmt formats and lints `scripts/` with `shfmt` and `shellcheck` (`*.sh`), `stylua` (`*.lua`), and `prettier` (`*.js`, `*.json`, `*.md`). - `mise run check` runs `luacheck` on `lua/` and `tests/` only, so it does not cover `scripts/`. diff --git a/scripts/test-release-please-runner.js b/scripts/test-release-please-runner.js index d5a14944..1a5fdbac 100755 --- a/scripts/test-release-please-runner.js +++ b/scripts/test-release-please-runner.js @@ -2,10 +2,23 @@ "use strict"; const assert = require("node:assert/strict"); -const { writeFileSync } = require("node:fs"); +const { execFile, execFileSync } = require("node:child_process"); +const { + copyFileSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} = require("node:fs"); +const http = require("node:http"); +const { tmpdir } = require("node:os"); +const { join } = require("node:path"); +const { promisify } = require("node:util"); const runner = require("./release-please-runner.js"); +const execFileAsync = promisify(execFile); + function commit(type, bareMessage) { return { bareMessage, @@ -144,6 +157,156 @@ async function testReleasableCommitRunsCommunique() { assert.ok(body.includes("### [Features]\n\n- Fixed x"), body); } +const CHARS_PER_TOKEN = 4; + +/** + * Serves the Anthropic Messages API with one fixed submit_release_notes call. + * Like the real API, it cuts the call off at the request's max_tokens and keeps + * only the fields that finished before the cut. + */ +function startFakeAnthropic(submission) { + const requests = []; + const server = http.createServer((req, res) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + const request = JSON.parse(Buffer.concat(chunks).toString("utf8")); + const submitTool = request.tools.find( + (tool) => tool.name === "submit_release_notes", + ); + assert.ok(submitTool, "request must offer submit_release_notes"); + let usedTokens = 0; + let truncated = false; + const input = {}; + for (const field of submitTool.input_schema.required) { + const text = submission[field]; + assert.equal(typeof text, "string", `no text for ${field}`); + const tokens = Math.ceil(text.length / CHARS_PER_TOKEN); + if (usedTokens + tokens > request.max_tokens) { + usedTokens = request.max_tokens; + truncated = true; + break; + } + usedTokens += tokens; + input[field] = text; + } + requests.push({ maxTokens: request.max_tokens, truncated }); + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + id: "msg_test", + type: "message", + role: "assistant", + model: request.model, + content: [ + { + type: "tool_use", + id: "toolu_test", + name: "submit_release_notes", + input, + }, + ], + stop_reason: truncated ? "max_tokens" : "tool_use", + usage: { input_tokens: 100, output_tokens: usedTokens }, + }), + ); + }); + }); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + resolve({ + requests, + url: `http://127.0.0.1:${server.address().port}`, + close: () => { + server.closeAllConnections(); + server.close(); + }, + }); + }); + }); +} + +// Communique 1.1.3 asks the model for changelog, title, and body in one call. +// For the ~140 PRs since v0.3.0 that call did not fit in the 4096-token +// default: all three attempts stopped at exactly 4096 output tokens. This +// submission is bigger than the default and smaller than communique.toml allows. +async function testCommuniqueConfigFitsReleaseNotes() { + const changelog = `## Added\n${"- A user-visible change with a PR link.\n".repeat(230)}`; + const submission = { + changelog, + release_title: "A release title", + release_body: `${"- A longer release body line with detail.\n".repeat(350)}`, + }; + const submittedTokens = Math.ceil( + Object.values(submission).join("").length / CHARS_PER_TOKEN, + ); + assert.ok( + submittedTokens > 4096 && submittedTokens < 8192, + `submission must exceed communique's default budget, got ${submittedTokens} tokens`, + ); + + const root = mkdtempSync(join(tmpdir(), "communique-config-")); + const fake = await startFakeAnthropic(submission); + try { + const gitEnv = { + PATH: process.env.PATH, + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_NOSYSTEM: "1", + GIT_AUTHOR_NAME: "test", + GIT_AUTHOR_EMAIL: "test@example.invalid", + GIT_COMMITTER_NAME: "test", + GIT_COMMITTER_EMAIL: "test@example.invalid", + }; + const git = (...args) => + execFileSync("git", args, { cwd: root, env: gitEnv, stdio: "pipe" }); + git("init", "-q"); + git("commit", "-q", "--allow-empty", "-m", "chore: first"); + git("tag", "v0.0.1"); + git("commit", "-q", "--allow-empty", "-m", "fix: second (#1)"); + copyFileSync( + join(__dirname, "..", "communique.toml"), + join(root, "communique.toml"), + ); + + const outputFile = join(root, "notes.md"); + // The environment lists every variable, so no real API key reaches communique. + await execFileAsync( + "communique", + [ + "generate", + "HEAD", + "v0.0.1", + "--concise", + "--repo", + "coder/claudecode.nvim", + "--base-url", + fake.url, + "--output", + outputFile, + ], + { + cwd: root, + env: { + ...gitEnv, + ANTHROPIC_API_KEY: "test-key-not-real", + HOME: root, + }, + timeout: 60_000, + }, + ); + + assert.deepEqual( + fake.requests.map(({ truncated }) => truncated), + [false], + `communique must get its whole submission through on the first try, requests: ${JSON.stringify(fake.requests)}`, + ); + assert.equal(readFileSync(outputFile, "utf8").trim(), changelog.trim()); + } finally { + fake.close(); + rmSync(root, { recursive: true, force: true }); + } +} + async function main() { testPrettierResolution(); await testCommuniqueArgs(); @@ -152,6 +315,7 @@ async function main() { testPullRequestOutputs(); await testInternalCommitSkipsCommunique(); await testReleasableCommitRunsCommunique(); + await testCommuniqueConfigFitsReleaseNotes(); } main().catch((error) => {