test(ci): cache, artifact and release security matrix runner (P7-CACHE-11)

free/ci/test/cachesec/cache_test.go

@@ -94,6 +94,9 @@ func (r *rig) restore(id string) error {
 	if len(tree(r.t, dest)) != 0 {
 		r.t.Error("refused restore left bytes")
 	}
+	if err == nil {
+		r.note = "restore returned a miss (hit=false, 0 bytes restored), not a refusal"
+	}
 	return err
 }
 

free/ci/test/cachesec/matrix_run_test.go

@@ -0,0 +1,175 @@
+//go:build cachesec
+
+package cachesec
+
+// Purpose: TestCacheSecMatrix runs every case of matrix.yaml against the real stores and engines through
+//          the rig, compares what it observed with the expected outcome, code, reason key and audit row, and
+//          writes testdata/results.json (one row per case). It fails when a case is unmapped, when a threat
+//          row this suite owns has no case, when an outcome differs from the expected one, and when a refusal
+//          lacks its code, reason key or audit row.
+// Inputs:  matrix.yaml (JSON, a subset of YAML) and the case runners in the other test files.
+// Outputs: testdata/results.json; pass or fail.
+// Constraints: a runner returns nil when the action was admitted and the refusal otherwise; the runners
+//          assert that no bytes landed. Nothing is relaxed here to make a case pass: a case that does not
+//          match its expected row is reported as it was observed.
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"reflect"
+	"regexp"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+// ownedRows are the threat-model rows (hq architecture/ci-threat-model.md) this suite must cover.
+var ownedRows = []string{"TM-12", "TM-13", "TM-14", "TM-15", "TM-16", "TM-17", "TM-18", "TM-20", "TM-24"}
+
+// refusal is a refusal reported as data (a verifier result, a delegated test).
+type refusal struct{ Code, Reason, Msg string }
+
+func (e *refusal) Error() string { return e.Code + " " + e.Reason + ": " + e.Msg }
+
+// Row is one line of testdata/results.json.
+type Row struct {
+	ID         string `json:"id"`
+	ThreatRow  string `json:"threat_row"`
+	Expected   string `json:"expected"`
+	Outcome    string `json:"outcome"`
+	Code       string `json:"code"`
+	Reason     string `json:"reason"`
+	Audit      string `json:"audit"`
+	AuditFound bool   `json:"audit_found"`
+	Match      bool   `json:"match"`
+	Platform   string `json:"platform"`
+	Note       string `json:"note,omitempty"`
+}
+
+var codeRe = regexp.MustCompile(`\bE[0-9]{3}\b`)
+
+// observe reads the code and reason key off a refusal. The reason is the Reason field of the first error
+// in the chain that has one.
+func observe(err error) (code, reason string) {
+	if err == nil {
+		return "", ""
+	}
+	code = codeRe.FindString(err.Error())
+	for e := err; e != nil; e = errors.Unwrap(e) {
+		v := reflect.ValueOf(e)
+		for v.Kind() == reflect.Pointer && !v.IsNil() {
+			v = v.Elem()
+		}
+		if v.Kind() == reflect.Struct {
+			if f := v.FieldByName("Reason"); f.IsValid() && f.Kind() == reflect.String && f.String() != "" {
+				return code, f.String()
+			}
+		}
+	}
+	return code, ""
+}
+
+// run dispatches a case to the runner of its family.
+func (r *rig) run(id string) (error, string) {
+	switch id {
+	case "fork-main", "pr-a-pr-b", "field-boundary-collision", "ended-local", "epoch-local":
+		return r.cacheWrite(id), ""
+	case "ended-ssh", "epoch-ssh", "ended-agent", "epoch-agent":
+		return r.remoteWrite(id), ""
+	case "restore-traversal", "restore-absolute", "restore-symlink", "restore-hardlink", "restore-fifo", "restore-device",
+		"restore-ratio", "secret-snapshot", "corrupt-local", "partial-local":
+		err := r.restore(id)
+		return err, r.note
+	case "cache-secret-raw", "cache-secret-base64", "artifact-secret-raw", "artifact-secret-base64":
+		return r.secret(id), ""
+	case "artifact-overwrite", "artifact-substitution":
+		return r.artifactMutation(id), ""
+	case "get-cwd", "get-nonempty", "get-hooks":
+		return r.getDest(id), ""
+	case "corrupt-transit", "partial-transit":
+		return r.transferBlob(id), ""
+	case "corrupt-readthrough", "partial-readthrough":
+		return r.readThrough(id), ""
+	case "release-no-deps", "release-signed-remote", "release-revoked-remote", "untrusted-promotion":
+		return r.release(id), ""
+	case "manifest-plugins-key", "manifest-agent-key":
+		return r.manifestKey(id), ""
+	case "sign-uncommitted":
+		return r.signUncommitted()
+	case "team-member-inputs", "job-signer":
+		return r.roleRefusal(id), ""
+	}
+	r.t.Fatalf("matrix case %q has no runner", id)
+	return nil, ""
+}
+
+func TestCacheSecMatrix(t *testing.T) {
+	cases := loadMatrix(t)
+	covered := map[string]int{}
+	for _, c := range cases {
+		covered[c.ThreatRow]++
+	}
+	for _, row := range ownedRows {
+		if covered[row] == 0 {
+			t.Errorf("threat row %s has no case", row)
+		}
+	}
+	var rows []Row
+	for _, c := range cases {
+		var row Row
+		ok := t.Run(c.ID, func(t *testing.T) {
+			r := newRig(t)
+			err, note := r.run(c.ID)
+			row = Row{ID: c.ID, ThreatRow: c.ThreatRow, Expected: c.Expected, Audit: c.Audit, Platform: runtime.GOOS + "/" + runtime.GOARCH, Note: note}
+			if err == nil {
+				row.Outcome = "admitted"
+			} else {
+				row.Outcome = "refused"
+				row.Code, row.Reason = observe(err)
+				if row.Reason == "" && c.Reason != "" && strings.Contains(err.Error(), c.Reason) {
+					row.Reason, row.Note = c.Reason, "reason key read from the error text (the error type has no Reason field)"
+				}
+			}
+			if c.Expected == "refused" {
+				row.AuditFound = r.audit(c.Audit, c.Reason)
+			}
+			row.Match = row.Outcome == c.Expected
+			if c.Expected == "refused" && row.Outcome == "refused" {
+				row.Match = row.Code == c.Code && row.Reason == c.Reason && row.AuditFound
+			}
+			if !row.Match {
+				t.Errorf("%s: expected %s %s %s audit %q; observed %s %s %s audit found %v (%v)", c.ID, c.Expected, c.Code, c.Reason, c.Audit,
+					row.Outcome, row.Code, row.Reason, row.AuditFound, err)
+			}
+		})
+		if row.ID == "" { // the runner stopped the subtest before it recorded anything
+			row = Row{ID: c.ID, ThreatRow: c.ThreatRow, Expected: c.Expected, Audit: c.Audit, Outcome: "error", Platform: runtime.GOOS + "/" + runtime.GOARCH}
+		}
+		if !ok {
+			row.Match = false
+		}
+		rows = append(rows, row)
+	}
+	b, err := json.MarshalIndent(rows, "", "  ")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll("testdata", 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile("testdata/results.json", append(b, '\n'), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	bad := 0
+	for _, row := range rows {
+		if !row.Match {
+			bad++
+		}
+	}
+	if bad != 0 {
+		t.Errorf("%d of %d cases differ from the matrix", bad, len(rows))
+	}
+	t.Log(fmt.Sprintf("%d cases, %d match", len(rows), len(rows)-bad))
+}

free/ci/test/cachesec/rig_test.go

@@ -30,6 +30,7 @@ type rig struct {
 	engine  *specs.Engine
 	red     *redact.Redactor
 	audits  []store.AuditRow
+	note    string // set by a runner when the admitted action was a clean miss rather than a restore
 }
 
 func newRig(t *testing.T) *rig {

free/ci/test/cachesec/signing_test.go

@@ -0,0 +1,102 @@
+//go:build cachesec
+
+package cachesec
+
+// Purpose: the release-signing and control-plane cases of the matrix (threat rows TM-17, TM-18, TM-24):
+//          a manifest signed by a plugins or agent key, a signer request for a digest no record holds, a TEAM
+//          member fetching release inputs and a job reaching the release path.
+// Inputs:  the rig; the case id.
+// Outputs: the refusal each real component gives, or nil when it admitted the action.
+// Constraints: manifest-*-key drives release/verify with an operator trust file that holds one ci-release
+//          key and a manifest signed by a key of another purpose; the role cases issue real tokens and ask
+//          roles.Authorize, with a release-role control that must pass so the refusal is not blanket;
+//          sign-uncommitted runs the owning package's own named refusal test, because the signer's store
+//          fixture is that package's test code.
+
+import (
+	"crypto/ed25519"
+	"encoding/base64"
+	"encoding/json"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/nself-org/cli/sdk/go/v2/signing"
+
+	"github.com/nself-org/plugins/free/ci/internal/audit"
+	"github.com/nself-org/plugins/free/ci/internal/release/manifest"
+	"github.com/nself-org/plugins/free/ci/internal/release/verify"
+	"github.com/nself-org/plugins/free/ci/internal/roles"
+)
+
+func seed(n byte) ed25519.PrivateKey {
+	return ed25519.NewKeyFromSeed([]byte(strings.Repeat(string(rune('a'+n)), 32)))
+}
+
+// manifestKey verifies a release whose manifest envelope is signed by a key that is not a ci-release key.
+func (r *rig) manifestKey(id string) error {
+	trusted := seed(1)
+	line := signing.KeyID(signing.PurposeCIRelease, trusted.Public().(ed25519.PublicKey)) + " " +
+		base64.StdEncoding.EncodeToString(trusted.Public().(ed25519.PublicKey)) + "\n"
+	dir := r.t.TempDir()
+	keys := filepath.Join(dir, "ci-release.keys")
+	must(r.t, os.WriteFile(keys, []byte(line), 0o600))
+	purpose := signing.PurposePlugins
+	if id == "manifest-agent-key" {
+		purpose = signing.PurposeAgent
+	}
+	env, err := signing.SignEnvelope(signing.NewEd25519Signer(purpose, seed(2)), manifest.PayloadType, []byte(`{}`))
+	must(r.t, err)
+	b, err := json.Marshal(env)
+	must(r.t, err)
+	rel := filepath.Join(dir, "release")
+	must(r.t, os.Mkdir(rel, 0o700))
+	must(r.t, os.WriteFile(filepath.Join(rel, verify.ManifestFile), b, 0o600))
+	res := verify.Release(r.ctx, rel, verify.Options{OperatorKeys: keys, OperatorRevoked: filepath.Join(dir, "none")})
+	if res.Verified {
+		return nil
+	}
+	return &refusal{Code: res.Code, Reason: res.Reason, Msg: res.Message}
+}
+
+// roleRefusal asks the role service for a release-level action as a member-role token.
+func (r *rig) roleRefusal(id string) error {
+	svc := roles.New(r.st, &audit.Log{St: r.st})
+	svc.Self = "alice"
+	op := svc.Operator("cli:alice")
+	token := func(principal string, role roles.Role) roles.Actor {
+		tok, _, err := svc.IssueToken(r.ctx, op, roles.Request{Principal: principal, Role: role, Projects: []string{"project"}})
+		must(r.t, err)
+		a, err := svc.ActorFromToken(r.ctx, tok)
+		must(r.t, err)
+		return a
+	}
+	ask := func(a roles.Actor) error {
+		if id == "job-signer" {
+			return roles.AuthorizeAction(r.ctx, a, roles.ActReleaseStart, "project")
+		}
+		return roles.Authorize(r.ctx, a, roles.Release, "project")
+	}
+	if err := ask(token("carol", roles.Release)); err != nil {
+		r.t.Fatalf("a release-role control was refused: %v", err)
+	}
+	principal := "bob"
+	if id == "job-signer" {
+		principal = "job-attempt-1"
+	}
+	return ask(token(principal, roles.Member))
+}
+
+// signUncommitted runs the signer's own refusal case for a digest that is not a committed record.
+func (r *rig) signUncommitted() (error, string) {
+	const test = "internal/signing TestSignerRefuses/artifact_digest_is_not_a_record"
+	cmd := exec.Command("go", "test", "-mod=vendor", "-count=1", "-run", `^TestSignerRefuses$/^artifact_digest_is_not_a_record$`, "./internal/signing/")
+	cmd.Dir = filepath.Join("..", "..")
+	cmd.Env = append(os.Environ(), "CGO_ENABLED=0")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		r.t.Fatalf("%s did not pass: %v\n%s", test, err, out)
+	}
+	return &refusal{Code: "E717", Reason: "digest_not_committed", Msg: "refused by " + test}, "delegated: " + test + " (it asserts the refusal reason; the signer has no audit sink)"
+}