Fix #260: GitHub Action eval input with pass-rate gating

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Features
+
+- Experimental GitHub Action eval mode (#260): `eval`, `eval-out`, `eval-min-pass-rate`, `cost-limit` and `time-limit` inputs run suites with the existing verified installer, expose report paths, gate CI via `fail-on-error`, and optionally append `report.md` to the job summary. Artifact uploads remain workflow-managed.
+
 ## [v0.10.3] - 2026-10-30
 
 ### Features

README.md

@@ -163,3 +163,32 @@ Run `tinyagent --help` (or plain `tinyagent` at a terminal) for the overview: us
 | Built-in tool notes | [docs/tools.md](docs/tools.md) |
 | Malformed tool calls, output truncation, troubleshooting | [docs/troubleshooting.md](docs/troubleshooting.md) |
 | Evaluation framework | [docs/eval-framework.md](docs/eval-framework.md) |
+
+### Evaluate changes in GitHub Actions
+
+The experimental Action eval mode runs suites with the verified installer and
+fails below a pass-rate threshold:
+
+```yaml
+- uses: actions/checkout@v4
+  with: { fetch-depth: 0 }
+- uses: vicgupta/tinyagent@main
+  with:
+    version: latest
+    eval: evals/suite.yaml
+    eval-out: evals/out
+    eval-min-pass-rate: '0.8'
+    cost-limit: '5'
+    job-summary: 'true'
+  env:
+    ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+- uses: actions/upload-artifact@v4
+  if: always()
+  with: { name: eval-results, path: evals/out }
+```
+
+Use full history (`fetch-depth: 0`) and set a cost limit. Run only trusted suites;
+do not expose secrets to untrusted fork PRs. Pin the Action ref and binary
+version for reproducible CI once released. See the [complete CI example and
+outputs](docs/eval-framework.md#9-ci-usage-and-pass-rate-gating), including fork
+PR filtering and dry runs.

action.yml

@@ -1,5 +1,5 @@
 name: tinyagent
-description: Install a checksum-verified tinyagent release and run one task or review, with the result as step outputs.
+description: Install a checksum-verified tinyagent release and run a task, review or eval suite, with the result as step outputs.
 author: vicgupta
 branding:
   icon: terminal
@@ -8,8 +8,28 @@ branding:
 # Every input reaches the scripts through env:, never through ${{ }} inside a
 # run: body, so untrusted text (issue bodies, PR titles) is data, not shell.
 inputs:
+  eval:
+    description: Experimental eval suite YAML path. Mutually exclusive with task, task-file, reviews, attempts, model and profile (the suite defines its matrix).
+    required: false
+    default: ""
+  eval-out:
+    description: Eval report directory. Defaults to a fresh directory under RUNNER_TEMP.
+    required: false
+    default: ""
+  eval-min-pass-rate:
+    description: With eval, fail if any configuration is below this pass rate (0-1).
+    required: false
+    default: ""
+  cost-limit:
+    description: Maximum USD (0 is unlimited); with eval, shared across the suite. Recommended for evals.
+    required: false
+    default: ""
+  time-limit:
+    description: Maximum wall-clock seconds (0 is unlimited); with eval, shared across the suite.
+    required: false
+    default: ""
   task:
-    description: The task. Fed to `tinyagent -t -` on stdin. Required unless task-file, review-base or review-commit is set; with a review it adds focus instructions.
+    description: The task. Fed to `tinyagent -t -` on stdin. Required unless eval, task-file, review-base or review-commit is set; with a review it adds focus instructions.
     required: false
     default: ""
   task-file:
@@ -89,7 +109,7 @@ inputs:
     required: false
     default: ""
   job-summary:
-    description: Append the `tinyagent --transcript` Markdown to the job summary (model-authored content, fenced and redacted). true or false.
+    description: Append eval report.md or the `tinyagent --transcript` Markdown to the job summary (model-authored content, fenced and redacted). true or false.
     required: false
     default: "false"
   fail-on-error:
@@ -98,6 +118,15 @@ inputs:
     default: "true"
 
 outputs:
+  eval-out:
+    description: Eval output directory (also on failure; dry runs do not write reports).
+    value: ${{ steps.run.outputs.eval-out }}
+  eval-results:
+    description: Compact JSON with per-configuration pass rates and costs, and below_threshold; empty when no report was written.
+    value: ${{ steps.run.outputs.eval-results }}
+  eval-report:
+    description: Path of eval report.json; inspect configs for pass rates, costs and below_threshold for failures. Empty if no report was written.
+    value: ${{ steps.run.outputs.eval-report }}
   status:
     description: Final run status (Submitted, Completed, Error, ...), from the result event. Empty if the run never started.
     value: ${{ steps.run.outputs.status }}
@@ -146,6 +175,11 @@ runs:
       shell: bash
       env:
         TINYAGENT_BIN: ${{ steps.install.outputs.path }}
+        INPUT_EVAL: ${{ inputs.eval }}
+        INPUT_EVAL_OUT: ${{ inputs.eval-out }}
+        INPUT_EVAL_MIN_PASS_RATE: ${{ inputs.eval-min-pass-rate }}
+        INPUT_COST_LIMIT: ${{ inputs.cost-limit }}
+        INPUT_TIME_LIMIT: ${{ inputs.time-limit }}
         INPUT_TASK: ${{ inputs.task }}
         INPUT_TASK_FILE: ${{ inputs.task-file }}
         INPUT_MODEL: ${{ inputs.model }}

action/run.sh

@@ -26,6 +26,15 @@ bool job-summary "$INPUT_JOB_SUMMARY"
 bool fail-on-error "$INPUT_FAIL_ON_ERROR"
 command -v jq >/dev/null 2>&1 || usage "jq is required to read tinyagent's result; install it before this step"
 
+if [ -n "$INPUT_EVAL" ]; then
+	for name in task:TASK task-file:TASK_FILE review-base:REVIEW_BASE review-commit:REVIEW_COMMIT attempts:ATTEMPTS model:MODEL profile:PROFILE; do
+		var=INPUT_${name#*:}
+		[ -z "${!var}" ] || usage "eval can't be combined with ${name%%:*}"
+	done
+elif [ -n "$INPUT_EVAL_OUT$INPUT_EVAL_MIN_PASS_RATE" ]; then
+	usage "eval-out and eval-min-pass-rate need eval"
+fi
+
 if [ -n "$INPUT_TASK" ] && [ -n "$INPUT_TASK_FILE" ]; then
 	usage "set task or task-file, not both"
 fi
@@ -46,11 +55,27 @@ fi
 out=$(mktemp -d "${RUNNER_TEMP:-/tmp}/tinyagent.XXXXXX")
 traj=$out/trajectory.json
 events=$out/events.jsonl
-args=(--json -o "$traj")
+args=()
+eval_out=
+if [ -n "$INPUT_EVAL" ]; then
+	[ -z "$INPUT_WORKING_DIRECTORY" ] || cd -- "$INPUT_WORKING_DIRECTORY"
+	eval_out=${INPUT_EVAL_OUT:-$out/eval}
+	# Make outputs unambiguous for later workflow steps. Do not create the
+	# directory here: tinyagent validates the suite before writing reports.
+	[[ $eval_out = /* ]] || eval_out=$PWD/$eval_out
+	args+=(--eval "$INPUT_EVAL" --eval-out "$eval_out")
+	[ -z "$INPUT_EVAL_MIN_PASS_RATE" ] || args+=(--eval-min-pass-rate "$INPUT_EVAL_MIN_PASS_RATE")
+else
+	args+=(--json -o "$traj")
+fi
+[ -z "$INPUT_COST_LIMIT" ] || args+=(--cost-limit "$INPUT_COST_LIMIT")
+[ -z "$INPUT_TIME_LIMIT" ] || args+=(--time-limit "$INPUT_TIME_LIMIT")
 [ -z "$INPUT_MODEL" ] || args+=(-m "$INPUT_MODEL")
 [ -z "$INPUT_PROFILE" ] || args+=(--profile "$INPUT_PROFILE")
 [ -z "$INPUT_CONFIG" ] || args+=(-c "$INPUT_CONFIG")
-[ -z "$INPUT_WORKING_DIRECTORY" ] || args+=(--cwd "$INPUT_WORKING_DIRECTORY")
+if [ -z "$INPUT_EVAL" ]; then
+	[ -z "$INPUT_WORKING_DIRECTORY" ] || args+=(--cwd "$INPUT_WORKING_DIRECTORY")
+fi
 [ -z "$INPUT_REVIEW_BASE" ] || args+=(--review-base "$INPUT_REVIEW_BASE")
 [ -z "$INPUT_REVIEW_COMMIT" ] || args+=(--review-commit "$INPUT_REVIEW_COMMIT")
 [ "$INPUT_READ_ONLY" = false ] || args+=(--read-only)
@@ -75,7 +100,9 @@ fi
 # The task goes to tinyagent on stdin (`-t -`), never through the shell.
 # --attempts can't read stdin (every attempt needs the task), so there it
 # is one argv element instead, still never shell-evaluated.
-if [ -n "$attempts" ]; then
+if [ -n "$INPUT_EVAL" ]; then
+	exec 3</dev/null
+elif [ -n "$attempts" ]; then
 	if [ -n "$INPUT_TASK_FILE" ]; then
 		[ -f "$INPUT_TASK_FILE" ] || usage "task-file is not a file"
 		task=$(cat "$INPUT_TASK_FILE")
@@ -99,7 +126,7 @@ else
 	exec 3</dev/null
 fi
 
-echo "events=$events" >>"$GITHUB_OUTPUT"
+[ -n "$INPUT_EVAL" ] || echo "events=$events" >>"$GITHUB_OUTPUT"
 
 # Stop the runner from acting on workflow commands (::add-mask::, ::error::
 # ...) in tinyagent's model-influenced log output.
@@ -125,6 +152,38 @@ set_output() { # set_output <name> <file>
 	} >>"$GITHUB_OUTPUT"
 }
 
+if [ -n "$INPUT_EVAL" ]; then
+	# Eval stdout is a human-readable table, not JSON events. Never parse it
+	# as a result event, even if a trial prints something resembling one.
+	cat "$events"
+	value=$out/value
+	printf '%s' "$eval_out" >"$value"
+	set_output eval-out "$value"
+	: >"$value"
+	[ ! -f "$eval_out/report.json" ] || printf '%s' "$eval_out/report.json" >"$value"
+	set_output eval-report "$value"
+	: >"$value"
+	if [ -f "$eval_out/report.json" ]; then
+		jq -c '{configs: [.configs[]? | {model, profile, passed, attempted, pass_rate, cost_usd}], below_threshold: (.below_threshold // [])}' "$eval_out/report.json" >"$value"
+	fi
+	set_output eval-results "$value"
+	echo "exit-code=$code" >>"$GITHUB_OUTPUT"
+	if [ "$INPUT_JOB_SUMMARY" = true ] && [ -f "$eval_out/report.md" ] && [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+		size=$(wc -c <"$eval_out/report.md" | tr -d ' ')
+		if [ "$size" -le 1048576 ]; then
+			cat "$eval_out/report.md" >>"$GITHUB_STEP_SUMMARY"
+		else
+			echo "tinyagent: the eval report ($size bytes) exceeds the 1 MiB job summary limit and was not added." >>"$GITHUB_STEP_SUMMARY"
+		fi
+	fi
+	echo "::$token::"
+	if [ "$code" -eq 2 ] || { [ "$code" -ne 0 ] && [ "$INPUT_FAIL_ON_ERROR" = true ]; }; then
+		echo "::error::tinyagent exited with status $code"
+		exit "$code"
+	fi
+	exit 0
+fi
+
 result=$out/result.json
 if [ -n "$attempts" ]; then
 	# The outputs describe the winner: its result event (tagged with its

action_eval_test.go

@@ -0,0 +1,165 @@
+package tinyagent_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func actionEvalSuite(t *testing.T, ws, check string) string {
+	t.Helper()
+	suite := filepath.Join(ws, "suite $(touch PWNED).yaml")
+	data := fmt.Sprintf("version: 1\ntrials: 1\nmodels: [openai/gpt-4o]\ntasks:\n  - {id: one, repo: %q, base: HEAD, task: x, checks: [{argv: [git, %s]}]}\n", ws, check)
+	if err := os.WriteFile(suite, []byte(data), 0600); err != nil {
+		t.Fatal(err)
+	}
+	return suite
+}
+
+func TestActionEval(t *testing.T) {
+	bin := buildTinyagent(t)
+	for _, tc := range []struct{ name, check, fail, code string }{
+		{"pass", "--version", "true", "0"},
+		{"threshold fails", "no-such-command", "true", "1"},
+		{"threshold tolerated", "no-such-command", "false", "1"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			ws := gitWorkspace(t)
+			suite := actionEvalSuite(t, ws, tc.check)
+			stub := newStubModel(t, submitReply("done"))
+			out := "reports $(touch PWNED)"
+			r := runAction(t, actionEnv{workspace: ws, env: stub.env()}, map[string]string{
+				"tinyagent-path": bin, "eval": suite, "eval-out": out, "eval-min-pass-rate": "0.8",
+				"cost-limit": "5", "time-limit": "60", "args": "--no-stream", "job-summary": "true", "fail-on-error": tc.fail,
+			})
+			if (r.failed != "") != (tc.code != "0" && tc.fail == "true") || r.outputs["exit-code"] != tc.code {
+				t.Fatalf("failed %q outputs %v\n%s", r.failed, r.outputs, r.log)
+			}
+			if len(stub.requests()) != 1 {
+				t.Fatalf("requests %d\n%s", len(stub.requests()), r.log)
+			}
+			dir := filepath.Join(ws, out)
+			if r.outputs["eval-out"] != dir || r.outputs["eval-report"] != filepath.Join(dir, "report.json") {
+				t.Fatal(r.outputs)
+			}
+			var results struct {
+				Configs []struct {
+					PassRate float64 `json:"pass_rate"`
+				} `json:"configs"`
+				BelowThreshold []string `json:"below_threshold"`
+			}
+			if err := json.Unmarshal([]byte(r.outputs["eval-results"]), &results); err != nil || len(results.Configs) != 1 {
+				t.Fatalf("eval-results %q: %v", r.outputs["eval-results"], err)
+			}
+			if (len(results.BelowThreshold) > 0) != (tc.code == "1") {
+				t.Fatal(r.outputs["eval-results"])
+			}
+
+			for _, f := range []string{"results.jsonl", "report.json", "report.md"} {
+				if st, err := os.Stat(filepath.Join(dir, f)); err != nil || st.Size() == 0 {
+					t.Fatalf("%s: %v", f, err)
+				}
+			}
+			for _, key := range []string{"status", "submission", "structured-output", "steps", "cost-usd", "trajectory", "events"} {
+				if r.outputs[key] != "" {
+					t.Errorf("%s = %q", key, r.outputs[key])
+				}
+			}
+			if !strings.Contains(r.summary, "# tinyagent eval:") {
+				t.Fatal(r.summary)
+			}
+			if _, err := os.Stat(filepath.Join(ws, "PWNED")); !os.IsNotExist(err) {
+				t.Fatal("shell metacharacters executed")
+			}
+		})
+	}
+}
+
+func TestActionEvalValidation(t *testing.T) {
+	bin := buildTinyagent(t)
+	for _, key := range []string{"task", "task-file", "review-base", "review-commit", "attempts", "model", "profile"} {
+		t.Run(key, func(t *testing.T) {
+			stub := newStubModel(t)
+			r := runAction(t, actionEnv{env: stub.env()}, map[string]string{"tinyagent-path": bin, "eval": "suite.yaml", key: "1"})
+			if r.failed != "run" || r.outputs["exit-code"] != "2" || !strings.Contains(r.log, key) || len(stub.requests()) != 0 {
+				t.Fatalf("%v\n%s", r.outputs, r.log)
+			}
+		})
+	}
+	for _, key := range []string{"eval-out", "eval-min-pass-rate"} {
+		t.Run(key, func(t *testing.T) {
+			r := runAction(t, actionEnv{}, map[string]string{"tinyagent-path": bin, "task": "x", key: "1"})
+			if r.failed != "run" || r.outputs["exit-code"] != "2" || !strings.Contains(r.log, "need eval") {
+				t.Fatalf("%v\n%s", r.outputs, r.log)
+			}
+		})
+	}
+	for _, invalid := range []bool{false, true} {
+		t.Run(fmt.Sprint("suite invalid ", invalid), func(t *testing.T) {
+			ws := gitWorkspace(t)
+			suite := filepath.Join(ws, "missing.yaml")
+			if invalid {
+				os.WriteFile(suite, []byte("invalid: suite\n"), 0600)
+			}
+			stub := newStubModel(t)
+			r := runAction(t, actionEnv{workspace: ws, env: stub.env()}, map[string]string{"tinyagent-path": bin, "eval": suite, "fail-on-error": "false"})
+			if r.failed != "run" || r.outputs["exit-code"] != "2" || len(stub.requests()) != 0 {
+				t.Fatalf("%v\n%s", r.outputs, r.log)
+			}
+		})
+	}
+}
+
+func TestActionEvalDryRun(t *testing.T) {
+	bin := buildTinyagent(t)
+	ws := gitWorkspace(t)
+	suite := actionEvalSuite(t, ws, "--version")
+	stub := newStubModel(t)
+	r := runAction(t, actionEnv{workspace: ws, env: stub.env()}, map[string]string{"tinyagent-path": bin, "eval": suite, "args": "--dry-run", "job-summary": "true"})
+	if r.failed != "" || r.outputs["exit-code"] != "0" || !strings.Contains(r.log, "1 trial(s)") || len(stub.requests()) != 0 || r.outputs["eval-report"] != "" || r.summary != "" {
+		t.Fatalf("%v\n%s", r.outputs, r.log)
+	}
+	if r.outputs["eval-out"] == "" {
+		t.Fatal("missing default output directory")
+	}
+}
+
+func TestActionEvalSummaryLimitAndArgv(t *testing.T) {
+	ws := t.TempDir()
+	bin := filepath.Join(ws, "tinyagent")
+	script := `#!/usr/bin/env bash
+set -eu
+printf '%s\n' "$@" > "$ARGV_FILE"
+while [ "$1" != --eval-out ]; do shift; done
+shift
+mkdir -p "$1"
+printf '{}\n' > "$1/report.json"
+head -c 1048577 /dev/zero | tr '\0' x > "$1/report.md"
+`
+	if err := os.WriteFile(bin, []byte(script), 0700); err != nil {
+		t.Fatal(err)
+	}
+	argv := filepath.Join(ws, "argv")
+	malicious := "5; touch PWNED"
+	r := runAction(t, actionEnv{workspace: ws, env: map[string]string{"ARGV_FILE": argv}}, map[string]string{
+		"tinyagent-path": bin, "eval": "suite $(touch PWNED).yaml", "cost-limit": malicious, "time-limit": "60", "config": "config with spaces.yaml", "args": "--dry-run", "job-summary": "true",
+	})
+	if r.failed != "" || !strings.Contains(r.summary, "exceeds the 1 MiB") || len(r.summary) > 1000 {
+		t.Fatalf("failed %s summary %.200s\n%s", r.failed, r.summary, r.log)
+	}
+	b, err := os.ReadFile(argv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, flag := range []string{"--eval\nsuite $(touch PWNED).yaml\n", "--cost-limit\n" + malicious + "\n", "--time-limit\n60\n", "-c\nconfig with spaces.yaml\n", "--dry-run\n"} {
+		if !strings.Contains(string(b), flag) {
+			t.Errorf("argv %q lacks %q", b, flag)
+		}
+	}
+	if _, err := os.Stat(filepath.Join(ws, "PWNED")); !os.IsNotExist(err) {
+		t.Fatal("flag executed shell commands")
+	}
+}