From 5c251fca5c7e2f72d89fb18c9d26d507a4189f58 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 31 Oct 2026 00:33:51 +0000 Subject: [PATCH 1/2] feat(action): run eval suites with pass-rate gating --- CHANGELOG.md | 4 + README.md | 29 ++++++++ action.yml | 40 +++++++++- action/run.sh | 67 ++++++++++++++++- action_eval_test.go | 165 +++++++++++++++++++++++++++++++++++++++++ docs/eval-framework.md | 51 +++++++++++-- 6 files changed, 342 insertions(+), 14 deletions(-) create mode 100644 action_eval_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 10326ab..9c2f1c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Features + +- Experimental GitHub Action eval mode (#260): `eval`, `eval-out`, `eval-min-pass-rate`, `cost-limit` and `time-limit` inputs run suites with the existing verified installer, expose report paths, gate CI via `fail-on-error`, and optionally append `report.md` to the job summary. Artifact uploads remain workflow-managed. + ## [v0.10.3] - 2026-10-30 ### Features diff --git a/README.md b/README.md index 0264a19..fbe6c2b 100644 --- a/README.md +++ b/README.md @@ -163,3 +163,32 @@ Run `tinyagent --help` (or plain `tinyagent` at a terminal) for the overview: us | Built-in tool notes | [docs/tools.md](docs/tools.md) | | Malformed tool calls, output truncation, troubleshooting | [docs/troubleshooting.md](docs/troubleshooting.md) | | Evaluation framework | [docs/eval-framework.md](docs/eval-framework.md) | + +### Evaluate changes in GitHub Actions + +The experimental Action eval mode runs suites with the verified installer and +fails below a pass-rate threshold: + +```yaml +- uses: actions/checkout@v4 + with: { fetch-depth: 0 } +- uses: vicgupta/tinyagent@main + with: + version: latest + eval: evals/suite.yaml + eval-out: evals/out + eval-min-pass-rate: '0.8' + cost-limit: '5' + job-summary: 'true' + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} +- uses: actions/upload-artifact@v4 + if: always() + with: { name: eval-results, path: evals/out } +``` + +Use full history (`fetch-depth: 0`) and set a cost limit. Run only trusted suites; +do not expose secrets to untrusted fork PRs. Pin the Action ref and binary +version for reproducible CI once released. See the [complete CI example and +outputs](docs/eval-framework.md#9-ci-usage-and-pass-rate-gating), including fork +PR filtering and dry runs. diff --git a/action.yml b/action.yml index 3e86330..7d55e8e 100644 --- a/action.yml +++ b/action.yml @@ -1,5 +1,5 @@ name: tinyagent -description: Install a checksum-verified tinyagent release and run one task or review, with the result as step outputs. +description: Install a checksum-verified tinyagent release and run a task, review or eval suite, with the result as step outputs. author: vicgupta branding: icon: terminal @@ -8,8 +8,28 @@ branding: # Every input reaches the scripts through env:, never through ${{ }} inside a # run: body, so untrusted text (issue bodies, PR titles) is data, not shell. inputs: + eval: + description: Experimental eval suite YAML path. Mutually exclusive with task, task-file, reviews, attempts, model and profile (the suite defines its matrix). + required: false + default: "" + eval-out: + description: Eval report directory. Defaults to a fresh directory under RUNNER_TEMP. + required: false + default: "" + eval-min-pass-rate: + description: With eval, fail if any configuration is below this pass rate (0-1). + required: false + default: "" + cost-limit: + description: Maximum USD (0 is unlimited); with eval, shared across the suite. Recommended for evals. + required: false + default: "" + time-limit: + description: Maximum wall-clock seconds (0 is unlimited); with eval, shared across the suite. + required: false + default: "" task: - description: The task. Fed to `tinyagent -t -` on stdin. Required unless task-file, review-base or review-commit is set; with a review it adds focus instructions. + description: The task. Fed to `tinyagent -t -` on stdin. Required unless eval, task-file, review-base or review-commit is set; with a review it adds focus instructions. required: false default: "" task-file: @@ -89,7 +109,7 @@ inputs: required: false default: "" job-summary: - description: Append the `tinyagent --transcript` Markdown to the job summary (model-authored content, fenced and redacted). true or false. + description: Append eval report.md or the `tinyagent --transcript` Markdown to the job summary (model-authored content, fenced and redacted). true or false. required: false default: "false" fail-on-error: @@ -98,6 +118,15 @@ inputs: default: "true" outputs: + eval-out: + description: Eval output directory (also on failure; dry runs do not write reports). + value: ${{ steps.run.outputs.eval-out }} + eval-results: + description: Compact JSON with per-configuration pass rates and costs, and below_threshold; empty when no report was written. + value: ${{ steps.run.outputs.eval-results }} + eval-report: + description: Path of eval report.json; inspect configs for pass rates, costs and below_threshold for failures. Empty if no report was written. + value: ${{ steps.run.outputs.eval-report }} status: description: Final run status (Submitted, Completed, Error, ...), from the result event. Empty if the run never started. value: ${{ steps.run.outputs.status }} @@ -146,6 +175,11 @@ runs: shell: bash env: TINYAGENT_BIN: ${{ steps.install.outputs.path }} + INPUT_EVAL: ${{ inputs.eval }} + INPUT_EVAL_OUT: ${{ inputs.eval-out }} + INPUT_EVAL_MIN_PASS_RATE: ${{ inputs.eval-min-pass-rate }} + INPUT_COST_LIMIT: ${{ inputs.cost-limit }} + INPUT_TIME_LIMIT: ${{ inputs.time-limit }} INPUT_TASK: ${{ inputs.task }} INPUT_TASK_FILE: ${{ inputs.task-file }} INPUT_MODEL: ${{ inputs.model }} diff --git a/action/run.sh b/action/run.sh index a62851e..4ee0b88 100755 --- a/action/run.sh +++ b/action/run.sh @@ -26,6 +26,15 @@ bool job-summary "$INPUT_JOB_SUMMARY" bool fail-on-error "$INPUT_FAIL_ON_ERROR" command -v jq >/dev/null 2>&1 || usage "jq is required to read tinyagent's result; install it before this step" +if [ -n "$INPUT_EVAL" ]; then + for name in task:TASK task-file:TASK_FILE review-base:REVIEW_BASE review-commit:REVIEW_COMMIT attempts:ATTEMPTS model:MODEL profile:PROFILE; do + var=INPUT_${name#*:} + [ -z "${!var}" ] || usage "eval can't be combined with ${name%%:*}" + done +elif [ -n "$INPUT_EVAL_OUT$INPUT_EVAL_MIN_PASS_RATE" ]; then + usage "eval-out and eval-min-pass-rate need eval" +fi + if [ -n "$INPUT_TASK" ] && [ -n "$INPUT_TASK_FILE" ]; then usage "set task or task-file, not both" fi @@ -46,11 +55,27 @@ fi out=$(mktemp -d "${RUNNER_TEMP:-/tmp}/tinyagent.XXXXXX") traj=$out/trajectory.json events=$out/events.jsonl -args=(--json -o "$traj") +args=() +eval_out= +if [ -n "$INPUT_EVAL" ]; then + [ -z "$INPUT_WORKING_DIRECTORY" ] || cd -- "$INPUT_WORKING_DIRECTORY" + eval_out=${INPUT_EVAL_OUT:-$out/eval} + # Make outputs unambiguous for later workflow steps. Do not create the + # directory here: tinyagent validates the suite before writing reports. + [[ $eval_out = /* ]] || eval_out=$PWD/$eval_out + args+=(--eval "$INPUT_EVAL" --eval-out "$eval_out") + [ -z "$INPUT_EVAL_MIN_PASS_RATE" ] || args+=(--eval-min-pass-rate "$INPUT_EVAL_MIN_PASS_RATE") +else + args+=(--json -o "$traj") +fi +[ -z "$INPUT_COST_LIMIT" ] || args+=(--cost-limit "$INPUT_COST_LIMIT") +[ -z "$INPUT_TIME_LIMIT" ] || args+=(--time-limit "$INPUT_TIME_LIMIT") [ -z "$INPUT_MODEL" ] || args+=(-m "$INPUT_MODEL") [ -z "$INPUT_PROFILE" ] || args+=(--profile "$INPUT_PROFILE") [ -z "$INPUT_CONFIG" ] || args+=(-c "$INPUT_CONFIG") -[ -z "$INPUT_WORKING_DIRECTORY" ] || args+=(--cwd "$INPUT_WORKING_DIRECTORY") +if [ -z "$INPUT_EVAL" ]; then + [ -z "$INPUT_WORKING_DIRECTORY" ] || args+=(--cwd "$INPUT_WORKING_DIRECTORY") +fi [ -z "$INPUT_REVIEW_BASE" ] || args+=(--review-base "$INPUT_REVIEW_BASE") [ -z "$INPUT_REVIEW_COMMIT" ] || args+=(--review-commit "$INPUT_REVIEW_COMMIT") [ "$INPUT_READ_ONLY" = false ] || args+=(--read-only) @@ -75,7 +100,9 @@ fi # The task goes to tinyagent on stdin (`-t -`), never through the shell. # --attempts can't read stdin (every attempt needs the task), so there it # is one argv element instead, still never shell-evaluated. -if [ -n "$attempts" ]; then +if [ -n "$INPUT_EVAL" ]; then + exec 3>"$GITHUB_OUTPUT" +[ -n "$INPUT_EVAL" ] || echo "events=$events" >>"$GITHUB_OUTPUT" # Stop the runner from acting on workflow commands (::add-mask::, ::error:: # ...) in tinyagent's model-influenced log output. @@ -125,6 +152,38 @@ set_output() { # set_output } >>"$GITHUB_OUTPUT" } +if [ -n "$INPUT_EVAL" ]; then + # Eval stdout is a human-readable table, not JSON events. Never parse it + # as a result event, even if a trial prints something resembling one. + cat "$events" + value=$out/value + printf '%s' "$eval_out" >"$value" + set_output eval-out "$value" + : >"$value" + [ ! -f "$eval_out/report.json" ] || printf '%s' "$eval_out/report.json" >"$value" + set_output eval-report "$value" + : >"$value" + if [ -f "$eval_out/report.json" ]; then + jq -c '{configs: [.configs[]? | {model, profile, passed, attempted, pass_rate, cost_usd}], below_threshold: (.below_threshold // [])}' "$eval_out/report.json" >"$value" + fi + set_output eval-results "$value" + echo "exit-code=$code" >>"$GITHUB_OUTPUT" + if [ "$INPUT_JOB_SUMMARY" = true ] && [ -f "$eval_out/report.md" ] && [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + size=$(wc -c <"$eval_out/report.md" | tr -d ' ') + if [ "$size" -le 1048576 ]; then + cat "$eval_out/report.md" >>"$GITHUB_STEP_SUMMARY" + else + echo "tinyagent: the eval report ($size bytes) exceeds the 1 MiB job summary limit and was not added." >>"$GITHUB_STEP_SUMMARY" + fi + fi + echo "::$token::" + if [ "$code" -eq 2 ] || { [ "$code" -ne 0 ] && [ "$INPUT_FAIL_ON_ERROR" = true ]; }; then + echo "::error::tinyagent exited with status $code" + exit "$code" + fi + exit 0 +fi + result=$out/result.json if [ -n "$attempts" ]; then # The outputs describe the winner: its result event (tagged with its diff --git a/action_eval_test.go b/action_eval_test.go new file mode 100644 index 0000000..049471c --- /dev/null +++ b/action_eval_test.go @@ -0,0 +1,165 @@ +package tinyagent_test + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func actionEvalSuite(t *testing.T, ws, check string) string { + t.Helper() + suite := filepath.Join(ws, "suite $(touch PWNED).yaml") + data := fmt.Sprintf("version: 1\ntrials: 1\nmodels: [openai/gpt-4o]\ntasks:\n - {id: one, repo: %q, base: HEAD, task: x, checks: [{argv: [git, %s]}]}\n", ws, check) + if err := os.WriteFile(suite, []byte(data), 0600); err != nil { + t.Fatal(err) + } + return suite +} + +func TestActionEval(t *testing.T) { + bin := buildTinyagent(t) + for _, tc := range []struct{ name, check, fail, code string }{ + {"pass", "--version", "true", "0"}, + {"threshold fails", "no-such-command", "true", "1"}, + {"threshold tolerated", "no-such-command", "false", "1"}, + } { + t.Run(tc.name, func(t *testing.T) { + ws := gitWorkspace(t) + suite := actionEvalSuite(t, ws, tc.check) + stub := newStubModel(t, submitReply("done")) + out := "reports $(touch PWNED)" + r := runAction(t, actionEnv{workspace: ws, env: stub.env()}, map[string]string{ + "tinyagent-path": bin, "eval": suite, "eval-out": out, "eval-min-pass-rate": "0.8", + "cost-limit": "5", "time-limit": "60", "args": "--no-stream", "job-summary": "true", "fail-on-error": tc.fail, + }) + if (r.failed != "") != (tc.code != "0" && tc.fail == "true") || r.outputs["exit-code"] != tc.code { + t.Fatalf("failed %q outputs %v\n%s", r.failed, r.outputs, r.log) + } + if len(stub.requests()) != 1 { + t.Fatalf("requests %d\n%s", len(stub.requests()), r.log) + } + dir := filepath.Join(ws, out) + if r.outputs["eval-out"] != dir || r.outputs["eval-report"] != filepath.Join(dir, "report.json") { + t.Fatal(r.outputs) + } + var results struct { + Configs []struct { + PassRate float64 `json:"pass_rate"` + } `json:"configs"` + BelowThreshold []string `json:"below_threshold"` + } + if err := json.Unmarshal([]byte(r.outputs["eval-results"]), &results); err != nil || len(results.Configs) != 1 { + t.Fatalf("eval-results %q: %v", r.outputs["eval-results"], err) + } + if (len(results.BelowThreshold) > 0) != (tc.code == "1") { + t.Fatal(r.outputs["eval-results"]) + } + + for _, f := range []string{"results.jsonl", "report.json", "report.md"} { + if st, err := os.Stat(filepath.Join(dir, f)); err != nil || st.Size() == 0 { + t.Fatalf("%s: %v", f, err) + } + } + for _, key := range []string{"status", "submission", "structured-output", "steps", "cost-usd", "trajectory", "events"} { + if r.outputs[key] != "" { + t.Errorf("%s = %q", key, r.outputs[key]) + } + } + if !strings.Contains(r.summary, "# tinyagent eval:") { + t.Fatal(r.summary) + } + if _, err := os.Stat(filepath.Join(ws, "PWNED")); !os.IsNotExist(err) { + t.Fatal("shell metacharacters executed") + } + }) + } +} + +func TestActionEvalValidation(t *testing.T) { + bin := buildTinyagent(t) + for _, key := range []string{"task", "task-file", "review-base", "review-commit", "attempts", "model", "profile"} { + t.Run(key, func(t *testing.T) { + stub := newStubModel(t) + r := runAction(t, actionEnv{env: stub.env()}, map[string]string{"tinyagent-path": bin, "eval": "suite.yaml", key: "1"}) + if r.failed != "run" || r.outputs["exit-code"] != "2" || !strings.Contains(r.log, key) || len(stub.requests()) != 0 { + t.Fatalf("%v\n%s", r.outputs, r.log) + } + }) + } + for _, key := range []string{"eval-out", "eval-min-pass-rate"} { + t.Run(key, func(t *testing.T) { + r := runAction(t, actionEnv{}, map[string]string{"tinyagent-path": bin, "task": "x", key: "1"}) + if r.failed != "run" || r.outputs["exit-code"] != "2" || !strings.Contains(r.log, "need eval") { + t.Fatalf("%v\n%s", r.outputs, r.log) + } + }) + } + for _, invalid := range []bool{false, true} { + t.Run(fmt.Sprint("suite invalid ", invalid), func(t *testing.T) { + ws := gitWorkspace(t) + suite := filepath.Join(ws, "missing.yaml") + if invalid { + os.WriteFile(suite, []byte("invalid: suite\n"), 0600) + } + stub := newStubModel(t) + r := runAction(t, actionEnv{workspace: ws, env: stub.env()}, map[string]string{"tinyagent-path": bin, "eval": suite, "fail-on-error": "false"}) + if r.failed != "run" || r.outputs["exit-code"] != "2" || len(stub.requests()) != 0 { + t.Fatalf("%v\n%s", r.outputs, r.log) + } + }) + } +} + +func TestActionEvalDryRun(t *testing.T) { + bin := buildTinyagent(t) + ws := gitWorkspace(t) + suite := actionEvalSuite(t, ws, "--version") + stub := newStubModel(t) + r := runAction(t, actionEnv{workspace: ws, env: stub.env()}, map[string]string{"tinyagent-path": bin, "eval": suite, "args": "--dry-run", "job-summary": "true"}) + if r.failed != "" || r.outputs["exit-code"] != "0" || !strings.Contains(r.log, "1 trial(s)") || len(stub.requests()) != 0 || r.outputs["eval-report"] != "" || r.summary != "" { + t.Fatalf("%v\n%s", r.outputs, r.log) + } + if r.outputs["eval-out"] == "" { + t.Fatal("missing default output directory") + } +} + +func TestActionEvalSummaryLimitAndArgv(t *testing.T) { + ws := t.TempDir() + bin := filepath.Join(ws, "tinyagent") + script := `#!/usr/bin/env bash +set -eu +printf '%s\n' "$@" > "$ARGV_FILE" +while [ "$1" != --eval-out ]; do shift; done +shift +mkdir -p "$1" +printf '{}\n' > "$1/report.json" +head -c 1048577 /dev/zero | tr '\0' x > "$1/report.md" +` + if err := os.WriteFile(bin, []byte(script), 0700); err != nil { + t.Fatal(err) + } + argv := filepath.Join(ws, "argv") + malicious := "5; touch PWNED" + r := runAction(t, actionEnv{workspace: ws, env: map[string]string{"ARGV_FILE": argv}}, map[string]string{ + "tinyagent-path": bin, "eval": "suite $(touch PWNED).yaml", "cost-limit": malicious, "time-limit": "60", "config": "config with spaces.yaml", "args": "--dry-run", "job-summary": "true", + }) + if r.failed != "" || !strings.Contains(r.summary, "exceeds the 1 MiB") || len(r.summary) > 1000 { + t.Fatalf("failed %s summary %.200s\n%s", r.failed, r.summary, r.log) + } + b, err := os.ReadFile(argv) + if err != nil { + t.Fatal(err) + } + for _, flag := range []string{"--eval\nsuite $(touch PWNED).yaml\n", "--cost-limit\n" + malicious + "\n", "--time-limit\n60\n", "-c\nconfig with spaces.yaml\n", "--dry-run\n"} { + if !strings.Contains(string(b), flag) { + t.Errorf("argv %q lacks %q", b, flag) + } + } + if _, err := os.Stat(filepath.Join(ws, "PWNED")); !os.IsNotExist(err) { + t.Fatal("flag executed shell commands") + } +} diff --git a/docs/eval-framework.md b/docs/eval-framework.md index cb284ff..c5f79dc 100644 --- a/docs/eval-framework.md +++ b/docs/eval-framework.md @@ -3,8 +3,9 @@ > **Status: experimental; v1 implemented.** `tinyagent --eval suite.yaml --eval-out dir` runs the suite sequentially (code in `internal/eval/` and `cmd/tinyagent/eval.go`). What v1 ships and how it differs from the target design below: > > - **Shipped:** strict `version: 1` suite files (`version`, `models`, `profiles`, `trials`, and per task `id`, `repo`, `base`, `task`/`task_file`, `model`, `profile`, `checks`, `checks_on_limit`, `limits`, `tags`); a detached worktree per trial at `base`, removed afterwards (also on Ctrl-C) unless `--eval-keep`; one child tinyagent run per trial with `-o` written to `/trajectories/.json` and its output to `logs/.log`; `results.jsonl`, `report.json`, `report.md` (pass rate, pass@k and pass^k, cost, tokens, steps, wall time, outcome counts, per-task passes); suite-level `--cost-limit` / `--time-limit`; `--dry-run`; `--eval-min-pass-rate` (exit 1); per-task `limits.token_limit`, forwarded to each trial as `--token-limit`. Other flags on the command line (e.g. `-c`, `--no-stream`, `--ask-user`, `--step-limit`) are forwarded to every trial. +> - **GitHub Action:** experimental `eval`, `eval-out`, `eval-min-pass-rate`, `cost-limit` and `time-limit` inputs use the verified installer and expose report paths; see §9. > - **Differences:** `trials` is required. `--time-limit` takes seconds (as in single runs), not `30m`. Check output kept in `results.jsonl` is capped to the last 4096 bytes. Checks run on the host in the worktree. A trial's `--profile` is passed only for names other than `default` (unless the config defines a profile called `default`). -> - **Not yet implemented:** `--eval-jobs` (trials are sequential), `--json` eval events, `output_schema` / `expected_json` (rejected as unknown keys), `profile_digest`, the GitHub Action `eval` input. +> - **Not yet implemented:** `--eval-jobs` (trials are sequential), `--json` eval events, `output_schema` / `expected_json` (rejected as unknown keys), `profile_digest`. ## Contents @@ -328,22 +329,33 @@ Records contain no message content by default. ## 9. CI usage and pass-rate gating -The planned rollout includes a GitHub Action input for eval mode (PR 4 in the split below); the existing `action.yml` has no such input today. Until then, the target CI pattern is to run the binary directly with a threshold, which makes the job fail when quality regresses: +The experimental Action `eval` input runs the suite through the checksum-verified +installer and gates CI with `eval-min-pass-rate`. Use an Action ref and binary +release containing eval support (`v0.10.3` supports the CLI); `version: latest` +selects the latest binary when running the Action from `main`. Pin both for +reproducible CI after this Action change is released. ```yaml -# Target design — the action has no eval input yet; call the binary directly on: pull_request permissions: contents: read jobs: eval: + if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: { fetch-depth: 0 } # base commits must be reachable - - run: | - tinyagent --eval evals/suite.yaml --eval-out evals/out \ - --cost-limit 5 --eval-min-pass-rate 0.8 + - uses: vicgupta/tinyagent@main + id: eval + with: + version: latest + eval: evals/suite.yaml + eval-out: evals/out + eval-min-pass-rate: '0.8' + cost-limit: '5' + time-limit: '600' + job-summary: 'true' env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - uses: actions/upload-artifact@v4 @@ -351,7 +363,32 @@ jobs: with: { name: eval-results, path: evals/out } ``` -Guidance: use `fetch-depth: 0` so every `base` exists; always set a `--cost-limit`; upload `evals/out` as an artifact; don't run eval suites on untrusted fork PRs with secrets (see the README's "Untrusted input" section). +Use `fetch-depth: 0` so every `base` exists; always set `cost-limit` (omitting it +allows unlimited spend). Do not give secrets to untrusted fork PRs or run their +suites with `pull_request_target`. Review suite changes before running them: +checks execute commands on the runner. + +`eval` cannot be combined with `task`, `task-file`, `review-base`, +`review-commit`, `attempts`, `model` or `profile`; define the model/profile matrix +in the suite. `config` and trial flags in `args` are forwarded by the CLI. +`working-directory` selects the directory where the eval command runs, so +relative suite/output paths resolve there. `cost-limit` and `time-limit` are +suite budgets; both inputs are also available to ordinary task/review runs. + +`eval-out` defaults to a fresh directory under `RUNNER_TEMP`. The `eval-out` +output gives its absolute path; `eval-report` gives `report.json` when present. +The `eval-results` output provides compact JSON with per-configuration +pass rates and costs (`configs`) and failing configurations (`below_threshold`). The directory also contains `results.jsonl`, `report.md`, logs +and trajectories. Artifact upload is the workflow's responsibility. Ordinary +result-event outputs (`status`, `submission`, `steps`, `cost-usd`, etc.) are empty +for eval runs. `exit-code` preserves the CLI exit status, including threshold +failure (1); `fail-on-error: false` tolerates failures except usage errors (2). + +`job-summary: true` appends `report.md`, including on threshold failure, up to +1 MiB. Report text can contain suite-authored content; treat it as untrusted. +Use `args: --dry-run` to validate and print the trial matrix without model +requests; dry runs do not create report files. A missing or invalid suite fails +with exit 2 before any model request. ## 10. Cost control, security and privacy From 3152febbc3ca7d74faeaecd36ec4d4f1201b1b44 Mon Sep 17 00:00:00 2001 From: Vic Gupta Date: Fri, 30 Oct 2026 20:39:59 -0400 Subject: [PATCH 2/2] test(action): only flag eval in command position, not --eval/eval-out text --- action_test.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/action_test.go b/action_test.go index 9123b08..bff9360 100644 --- a/action_test.go +++ b/action_test.go @@ -261,6 +261,23 @@ func buildTinyagent(t *testing.T) string { return bin } +// setXOrEval matches `set -x` and the eval builtin in command position, not the +// word "eval" in flags, output names or messages (--eval, eval-out, "need eval"). +var setXOrEval = regexp.MustCompile(`set -[a-z]*x|(?m)(^|[;&|(]|\b(then|do|else)\b)[ \t]*eval([ \t]|$)`) + +func TestSetXOrEvalPattern(t *testing.T) { + for _, bad := range []string{"set -x", "set -ex", "eval \"$x\"", "\teval $cmd", "true; eval x", "if a; then eval x; fi", "( eval x )"} { + if !setXOrEval.MatchString(bad) { + t.Errorf("not caught: %q", bad) + } + } + for _, ok := range []string{"args+=(--eval \"$f\" --eval-out \"$d\")", "set_output eval-out \"$v\"", "usage \"eval can't be combined\"", "eval_out=$out/eval", "evaluate"} { + if setXOrEval.MatchString(ok) { + t.Errorf("false positive: %q", ok) + } + } +} + func TestActionNoUntrustedContextInScripts(t *testing.T) { a := loadAction(t) if a.Runs.Using != "composite" { @@ -279,7 +296,7 @@ func TestActionNoUntrustedContextInScripts(t *testing.T) { if err != nil { t.Fatal(err) } - if regexp.MustCompile(`set -[a-z]*x|\beval\b`).Match(data) { + if setXOrEval.Match(data) { t.Errorf("%s uses set -x or eval", f) } }