diff --git a/free/ci/test/cachesec/cache_test.go b/free/ci/test/cachesec/cache_test.go index 300872379..6b865bc50 100644 --- a/free/ci/test/cachesec/cache_test.go +++ b/free/ci/test/cachesec/cache_test.go @@ -94,6 +94,9 @@ func (r *rig) restore(id string) error { if len(tree(r.t, dest)) != 0 { r.t.Error("refused restore left bytes") } + if err == nil { + r.note = "restore returned a miss (hit=false, 0 bytes restored), not a refusal" + } return err } diff --git a/free/ci/test/cachesec/matrix_run_test.go b/free/ci/test/cachesec/matrix_run_test.go new file mode 100644 index 000000000..946613b91 --- /dev/null +++ b/free/ci/test/cachesec/matrix_run_test.go @@ -0,0 +1,175 @@ +//go:build cachesec + +package cachesec + +// Purpose: TestCacheSecMatrix runs every case of matrix.yaml against the real stores and engines through +// the rig, compares what it observed with the expected outcome, code, reason key and audit row, and +// writes testdata/results.json (one row per case). It fails when a case is unmapped, when a threat +// row this suite owns has no case, when an outcome differs from the expected one, and when a refusal +// lacks its code, reason key or audit row. +// Inputs: matrix.yaml (JSON, a subset of YAML) and the case runners in the other test files. +// Outputs: testdata/results.json; pass or fail. +// Constraints: a runner returns nil when the action was admitted and the refusal otherwise; the runners +// assert that no bytes landed. Nothing is relaxed here to make a case pass: a case that does not +// match its expected row is reported as it was observed. + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +// ownedRows are the threat-model rows (hq architecture/ci-threat-model.md) this suite must cover. +var ownedRows = []string{"TM-12", "TM-13", "TM-14", "TM-15", "TM-16", "TM-17", "TM-18", "TM-20", "TM-24"} + +// refusal is a refusal reported as data (a verifier result, a delegated test). +type refusal struct{ Code, Reason, Msg string } + +func (e *refusal) Error() string { return e.Code + " " + e.Reason + ": " + e.Msg } + +// Row is one line of testdata/results.json. +type Row struct { + ID string `json:"id"` + ThreatRow string `json:"threat_row"` + Expected string `json:"expected"` + Outcome string `json:"outcome"` + Code string `json:"code"` + Reason string `json:"reason"` + Audit string `json:"audit"` + AuditFound bool `json:"audit_found"` + Match bool `json:"match"` + Platform string `json:"platform"` + Note string `json:"note,omitempty"` +} + +var codeRe = regexp.MustCompile(`\bE[0-9]{3}\b`) + +// observe reads the code and reason key off a refusal. The reason is the Reason field of the first error +// in the chain that has one. +func observe(err error) (code, reason string) { + if err == nil { + return "", "" + } + code = codeRe.FindString(err.Error()) + for e := err; e != nil; e = errors.Unwrap(e) { + v := reflect.ValueOf(e) + for v.Kind() == reflect.Pointer && !v.IsNil() { + v = v.Elem() + } + if v.Kind() == reflect.Struct { + if f := v.FieldByName("Reason"); f.IsValid() && f.Kind() == reflect.String && f.String() != "" { + return code, f.String() + } + } + } + return code, "" +} + +// run dispatches a case to the runner of its family. +func (r *rig) run(id string) (error, string) { + switch id { + case "fork-main", "pr-a-pr-b", "field-boundary-collision", "ended-local", "epoch-local": + return r.cacheWrite(id), "" + case "ended-ssh", "epoch-ssh", "ended-agent", "epoch-agent": + return r.remoteWrite(id), "" + case "restore-traversal", "restore-absolute", "restore-symlink", "restore-hardlink", "restore-fifo", "restore-device", + "restore-ratio", "secret-snapshot", "corrupt-local", "partial-local": + err := r.restore(id) + return err, r.note + case "cache-secret-raw", "cache-secret-base64", "artifact-secret-raw", "artifact-secret-base64": + return r.secret(id), "" + case "artifact-overwrite", "artifact-substitution": + return r.artifactMutation(id), "" + case "get-cwd", "get-nonempty", "get-hooks": + return r.getDest(id), "" + case "corrupt-transit", "partial-transit": + return r.transferBlob(id), "" + case "corrupt-readthrough", "partial-readthrough": + return r.readThrough(id), "" + case "release-no-deps", "release-signed-remote", "release-revoked-remote", "untrusted-promotion": + return r.release(id), "" + case "manifest-plugins-key", "manifest-agent-key": + return r.manifestKey(id), "" + case "sign-uncommitted": + return r.signUncommitted() + case "team-member-inputs", "job-signer": + return r.roleRefusal(id), "" + } + r.t.Fatalf("matrix case %q has no runner", id) + return nil, "" +} + +func TestCacheSecMatrix(t *testing.T) { + cases := loadMatrix(t) + covered := map[string]int{} + for _, c := range cases { + covered[c.ThreatRow]++ + } + for _, row := range ownedRows { + if covered[row] == 0 { + t.Errorf("threat row %s has no case", row) + } + } + var rows []Row + for _, c := range cases { + var row Row + ok := t.Run(c.ID, func(t *testing.T) { + r := newRig(t) + err, note := r.run(c.ID) + row = Row{ID: c.ID, ThreatRow: c.ThreatRow, Expected: c.Expected, Audit: c.Audit, Platform: runtime.GOOS + "/" + runtime.GOARCH, Note: note} + if err == nil { + row.Outcome = "admitted" + } else { + row.Outcome = "refused" + row.Code, row.Reason = observe(err) + if row.Reason == "" && c.Reason != "" && strings.Contains(err.Error(), c.Reason) { + row.Reason, row.Note = c.Reason, "reason key read from the error text (the error type has no Reason field)" + } + } + if c.Expected == "refused" { + row.AuditFound = r.audit(c.Audit, c.Reason) + } + row.Match = row.Outcome == c.Expected + if c.Expected == "refused" && row.Outcome == "refused" { + row.Match = row.Code == c.Code && row.Reason == c.Reason && row.AuditFound + } + if !row.Match { + t.Errorf("%s: expected %s %s %s audit %q; observed %s %s %s audit found %v (%v)", c.ID, c.Expected, c.Code, c.Reason, c.Audit, + row.Outcome, row.Code, row.Reason, row.AuditFound, err) + } + }) + if row.ID == "" { // the runner stopped the subtest before it recorded anything + row = Row{ID: c.ID, ThreatRow: c.ThreatRow, Expected: c.Expected, Audit: c.Audit, Outcome: "error", Platform: runtime.GOOS + "/" + runtime.GOARCH} + } + if !ok { + row.Match = false + } + rows = append(rows, row) + } + b, err := json.MarshalIndent(rows, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll("testdata", 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile("testdata/results.json", append(b, '\n'), 0o644); err != nil { + t.Fatal(err) + } + bad := 0 + for _, row := range rows { + if !row.Match { + bad++ + } + } + if bad != 0 { + t.Errorf("%d of %d cases differ from the matrix", bad, len(rows)) + } + t.Log(fmt.Sprintf("%d cases, %d match", len(rows), len(rows)-bad)) +} diff --git a/free/ci/test/cachesec/rig_test.go b/free/ci/test/cachesec/rig_test.go index c534f43e2..eca7ee4b4 100644 --- a/free/ci/test/cachesec/rig_test.go +++ b/free/ci/test/cachesec/rig_test.go @@ -30,6 +30,7 @@ type rig struct { engine *specs.Engine red *redact.Redactor audits []store.AuditRow + note string // set by a runner when the admitted action was a clean miss rather than a restore } func newRig(t *testing.T) *rig { diff --git a/free/ci/test/cachesec/signing_test.go b/free/ci/test/cachesec/signing_test.go new file mode 100644 index 000000000..fd63854c8 --- /dev/null +++ b/free/ci/test/cachesec/signing_test.go @@ -0,0 +1,102 @@ +//go:build cachesec + +package cachesec + +// Purpose: the release-signing and control-plane cases of the matrix (threat rows TM-17, TM-18, TM-24): +// a manifest signed by a plugins or agent key, a signer request for a digest no record holds, a TEAM +// member fetching release inputs and a job reaching the release path. +// Inputs: the rig; the case id. +// Outputs: the refusal each real component gives, or nil when it admitted the action. +// Constraints: manifest-*-key drives release/verify with an operator trust file that holds one ci-release +// key and a manifest signed by a key of another purpose; the role cases issue real tokens and ask +// roles.Authorize, with a release-role control that must pass so the refusal is not blanket; +// sign-uncommitted runs the owning package's own named refusal test, because the signer's store +// fixture is that package's test code. + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/nself-org/cli/sdk/go/v2/signing" + + "github.com/nself-org/plugins/free/ci/internal/audit" + "github.com/nself-org/plugins/free/ci/internal/release/manifest" + "github.com/nself-org/plugins/free/ci/internal/release/verify" + "github.com/nself-org/plugins/free/ci/internal/roles" +) + +func seed(n byte) ed25519.PrivateKey { + return ed25519.NewKeyFromSeed([]byte(strings.Repeat(string(rune('a'+n)), 32))) +} + +// manifestKey verifies a release whose manifest envelope is signed by a key that is not a ci-release key. +func (r *rig) manifestKey(id string) error { + trusted := seed(1) + line := signing.KeyID(signing.PurposeCIRelease, trusted.Public().(ed25519.PublicKey)) + " " + + base64.StdEncoding.EncodeToString(trusted.Public().(ed25519.PublicKey)) + "\n" + dir := r.t.TempDir() + keys := filepath.Join(dir, "ci-release.keys") + must(r.t, os.WriteFile(keys, []byte(line), 0o600)) + purpose := signing.PurposePlugins + if id == "manifest-agent-key" { + purpose = signing.PurposeAgent + } + env, err := signing.SignEnvelope(signing.NewEd25519Signer(purpose, seed(2)), manifest.PayloadType, []byte(`{}`)) + must(r.t, err) + b, err := json.Marshal(env) + must(r.t, err) + rel := filepath.Join(dir, "release") + must(r.t, os.Mkdir(rel, 0o700)) + must(r.t, os.WriteFile(filepath.Join(rel, verify.ManifestFile), b, 0o600)) + res := verify.Release(r.ctx, rel, verify.Options{OperatorKeys: keys, OperatorRevoked: filepath.Join(dir, "none")}) + if res.Verified { + return nil + } + return &refusal{Code: res.Code, Reason: res.Reason, Msg: res.Message} +} + +// roleRefusal asks the role service for a release-level action as a member-role token. +func (r *rig) roleRefusal(id string) error { + svc := roles.New(r.st, &audit.Log{St: r.st}) + svc.Self = "alice" + op := svc.Operator("cli:alice") + token := func(principal string, role roles.Role) roles.Actor { + tok, _, err := svc.IssueToken(r.ctx, op, roles.Request{Principal: principal, Role: role, Projects: []string{"project"}}) + must(r.t, err) + a, err := svc.ActorFromToken(r.ctx, tok) + must(r.t, err) + return a + } + ask := func(a roles.Actor) error { + if id == "job-signer" { + return roles.AuthorizeAction(r.ctx, a, roles.ActReleaseStart, "project") + } + return roles.Authorize(r.ctx, a, roles.Release, "project") + } + if err := ask(token("carol", roles.Release)); err != nil { + r.t.Fatalf("a release-role control was refused: %v", err) + } + principal := "bob" + if id == "job-signer" { + principal = "job-attempt-1" + } + return ask(token(principal, roles.Member)) +} + +// signUncommitted runs the signer's own refusal case for a digest that is not a committed record. +func (r *rig) signUncommitted() (error, string) { + const test = "internal/signing TestSignerRefuses/artifact_digest_is_not_a_record" + cmd := exec.Command("go", "test", "-mod=vendor", "-count=1", "-run", `^TestSignerRefuses$/^artifact_digest_is_not_a_record$`, "./internal/signing/") + cmd.Dir = filepath.Join("..", "..") + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + out, err := cmd.CombinedOutput() + if err != nil { + r.t.Fatalf("%s did not pass: %v\n%s", test, err, out) + } + return &refusal{Code: "E717", Reason: "digest_not_committed", Msg: "refused by " + test}, "delegated: " + test + " (it asserts the refusal reason; the signer has no audit sink)" +} diff --git a/free/ci/test/cachesec/testdata/results.json b/free/ci/test/cachesec/testdata/results.json new file mode 100644 index 000000000..beb0387e1 --- /dev/null +++ b/free/ci/test/cachesec/testdata/results.json @@ -0,0 +1,510 @@ +[ + { + "id": "fork-main", + "threat_row": "TM-12", + "expected": "refused", + "outcome": "refused", + "code": "E684", + "reason": "cache.namespace", + "audit": "cache.commit.denied", + "audit_found": true, + "match": true, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "pr-a-pr-b", + "threat_row": "TM-12", + "expected": "refused", + "outcome": "refused", + "code": "E684", + "reason": "cache.namespace", + "audit": "cache.commit.denied", + "audit_found": true, + "match": true, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "field-boundary-collision", + "threat_row": "TM-12", + "expected": "refused", + "outcome": "refused", + "code": "E711", + "reason": "not_planned", + "audit": "cache.commit.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "ended-local", + "threat_row": "TM-13", + "expected": "refused", + "outcome": "refused", + "code": "E711", + "reason": "attempt_state", + "audit": "cache.commit.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "epoch-local", + "threat_row": "TM-13", + "expected": "refused", + "outcome": "refused", + "code": "E711", + "reason": "epoch", + "audit": "cache.commit.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "ended-ssh", + "threat_row": "TM-13", + "expected": "refused", + "outcome": "refused", + "code": "E714", + "reason": "stale_epoch", + "audit": "cache.remote.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "epoch-ssh", + "threat_row": "TM-13", + "expected": "refused", + "outcome": "refused", + "code": "E714", + "reason": "stale_epoch", + "audit": "cache.remote.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "ended-agent", + "threat_row": "TM-13", + "expected": "refused", + "outcome": "refused", + "code": "E714", + "reason": "stale_epoch", + "audit": "cache.remote.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "epoch-agent", + "threat_row": "TM-13", + "expected": "refused", + "outcome": "refused", + "code": "E714", + "reason": "stale_epoch", + "audit": "cache.remote.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "restore-traversal", + "threat_row": "TM-14", + "expected": "refused", + "outcome": "refused", + "code": "E683", + "reason": "path.traversal", + "audit": "cache.restore.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64" + }, + { + "id": "restore-absolute", + "threat_row": "TM-14", + "expected": "refused", + "outcome": "refused", + "code": "E683", + "reason": "path.absolute", + "audit": "cache.restore.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64" + }, + { + "id": "restore-symlink", + "threat_row": "TM-14", + "expected": "refused", + "outcome": "refused", + "code": "E683", + "reason": "link.escape", + "audit": "cache.restore.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64" + }, + { + "id": "restore-hardlink", + "threat_row": "TM-14", + "expected": "refused", + "outcome": "refused", + "code": "E683", + "reason": "entry.hardlink", + "audit": "cache.restore.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64" + }, + { + "id": "restore-fifo", + "threat_row": "TM-14", + "expected": "refused", + "outcome": "refused", + "code": "E683", + "reason": "entry.fifo", + "audit": "cache.restore.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64" + }, + { + "id": "restore-device", + "threat_row": "TM-14", + "expected": "refused", + "outcome": "refused", + "code": "E683", + "reason": "entry.device", + "audit": "cache.restore.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64" + }, + { + "id": "restore-ratio", + "threat_row": "TM-14", + "expected": "refused", + "outcome": "refused", + "code": "E683", + "reason": "size.ratio", + "audit": "cache.restore.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64" + }, + { + "id": "secret-snapshot", + "threat_row": "TM-12", + "expected": "refused", + "outcome": "admitted", + "code": "", + "reason": "", + "audit": "cache.restore.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64", + "note": "restore returned a miss (hit=false, 0 bytes restored), not a refusal" + }, + { + "id": "cache-secret-raw", + "threat_row": "TM-16", + "expected": "refused", + "outcome": "refused", + "code": "E711", + "reason": "secret_match", + "audit": "cache.save.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64" + }, + { + "id": "cache-secret-base64", + "threat_row": "TM-16", + "expected": "refused", + "outcome": "refused", + "code": "E711", + "reason": "secret_match", + "audit": "cache.save.refused", + "audit_found": true, + "match": true, + "platform": "darwin/arm64" + }, + { + "id": "artifact-secret-raw", + "threat_row": "TM-16", + "expected": "refused", + "outcome": "refused", + "code": "E712", + "reason": "secret_match", + "audit": "artifact.collect.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "artifact-secret-base64", + "threat_row": "TM-16", + "expected": "refused", + "outcome": "refused", + "code": "E712", + "reason": "secret_match", + "audit": "artifact.collect.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "artifact-overwrite", + "threat_row": "TM-15", + "expected": "refused", + "outcome": "refused", + "code": "E712", + "reason": "digest_mismatch", + "audit": "artifact.commit.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "artifact-substitution", + "threat_row": "TM-15", + "expected": "refused", + "outcome": "refused", + "code": "E712", + "reason": "digest_mismatch", + "audit": "artifact.commit.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "get-cwd", + "threat_row": "TM-14", + "expected": "refused", + "outcome": "refused", + "code": "E683", + "reason": "dest.cwd", + "audit": "artifact.get.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "get-nonempty", + "threat_row": "TM-14", + "expected": "refused", + "outcome": "refused", + "code": "E683", + "reason": "dest.not_empty", + "audit": "artifact.get.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "get-hooks", + "threat_row": "TM-14", + "expected": "refused", + "outcome": "refused", + "code": "E683", + "reason": "dest.under_git", + "audit": "artifact.get.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "corrupt-local", + "threat_row": "TM-15", + "expected": "refused", + "outcome": "refused", + "code": "E710", + "reason": "digest_mismatch", + "audit": "cache.restore.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "partial-local", + "threat_row": "TM-15", + "expected": "refused", + "outcome": "admitted", + "code": "", + "reason": "", + "audit": "cache.restore.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64", + "note": "restore returned a miss (hit=false, 0 bytes restored), not a refusal" + }, + { + "id": "corrupt-transit", + "threat_row": "TM-15", + "expected": "refused", + "outcome": "refused", + "code": "E714", + "reason": "digest_mismatch", + "audit": "cache.remote.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "partial-transit", + "threat_row": "TM-15", + "expected": "refused", + "outcome": "refused", + "code": "E714", + "reason": "interrupted", + "audit": "cache.remote.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "corrupt-readthrough", + "threat_row": "TM-15", + "expected": "refused", + "outcome": "refused", + "code": "E714", + "reason": "digest_mismatch", + "audit": "cache.restore.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "partial-readthrough", + "threat_row": "TM-15", + "expected": "refused", + "outcome": "refused", + "code": "E714", + "reason": "digest_mismatch", + "audit": "cache.restore.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64", + "note": "reason key read from the error text (the error type has no Reason field)" + }, + { + "id": "release-no-deps", + "threat_row": "TM-17", + "expected": "refused", + "outcome": "refused", + "code": "E715", + "reason": "deps_invalid", + "audit": "release.admission.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "release-signed-remote", + "threat_row": "TM-17", + "expected": "admitted", + "outcome": "admitted", + "code": "", + "reason": "", + "audit": "", + "audit_found": false, + "match": true, + "platform": "darwin/arm64" + }, + { + "id": "release-revoked-remote", + "threat_row": "TM-20", + "expected": "refused", + "outcome": "refused", + "code": "E686", + "reason": "key_revoked", + "audit": "evidence.quarantined", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "manifest-plugins-key", + "threat_row": "TM-18", + "expected": "refused", + "outcome": "refused", + "code": "E719", + "reason": "key_purpose", + "audit": "release.verify.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "manifest-agent-key", + "threat_row": "TM-18", + "expected": "refused", + "outcome": "refused", + "code": "E719", + "reason": "key_purpose", + "audit": "release.verify.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "sign-uncommitted", + "threat_row": "TM-18", + "expected": "refused", + "outcome": "refused", + "code": "E717", + "reason": "digest_not_committed", + "audit": "release.sign.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64", + "note": "delegated: internal/signing TestSignerRefuses/artifact_digest_is_not_a_record (it asserts the refusal reason; the signer has no audit sink)" + }, + { + "id": "untrusted-promotion", + "threat_row": "TM-17", + "expected": "refused", + "outcome": "refused", + "code": "E715", + "reason": "trust", + "audit": "release.admission.refused", + "audit_found": false, + "match": false, + "platform": "darwin/arm64" + }, + { + "id": "team-member-inputs", + "threat_row": "TM-24", + "expected": "refused", + "outcome": "refused", + "code": "E687", + "reason": "role", + "audit": "authorize", + "audit_found": true, + "match": true, + "platform": "darwin/arm64" + }, + { + "id": "job-signer", + "threat_row": "TM-24", + "expected": "refused", + "outcome": "refused", + "code": "E687", + "reason": "role", + "audit": "authorize", + "audit_found": true, + "match": true, + "platform": "darwin/arm64" + } +] diff --git a/free/ci/test/cachesec/transfer_test.go b/free/ci/test/cachesec/transfer_test.go index 787d5e5db..62f8ce5c7 100644 --- a/free/ci/test/cachesec/transfer_test.go +++ b/free/ci/test/cachesec/transfer_test.go @@ -26,7 +26,9 @@ func (r *rig) remoteWrite(id string) error { r.running("remote", f) eng := lease.New(r.st, nil, lease.Config{}) // Grant needs a queued attempt, so use a separate attempt from the local rig. - must(r.t, r.st.CreateAttempt(r.ctx, store.AttemptRow{ID: "remote-wire", JobID: "jremote", N: 2}, "")) + must(r.t, r.st.CreatePipeline(r.ctx, store.PipelineRow{ID: "pwire", Revision: strings.Repeat("a", 40), Project: f.Project, Trigger: "local", SourceTrust: f.Trust, PrivacyZone: "local-only", PolicyDigest: "policy", InputDigest: "input", SelectionMode: "full", Status: "queued"})) + must(r.t, r.st.CreateJob(r.ctx, store.JobRow{ID: "jwire", PipelineID: "pwire", Name: "wire", Kind: "test", Idempotent: true, InfraMax: 1, InputDigest: "input"})) + must(r.t, r.st.CreateAttempt(r.ctx, store.AttemptRow{ID: "remote-wire", JobID: "jwire", N: 1}, "")) epoch, err := eng.Grant(r.ctx, "host", store.Demand{AttemptID: "remote-wire", LeaseID: "wire", RunnerID: "node", CPU: 1, MemMB: 1, CapacityCPU: 100, CapacityMemMB: 100, TTL: eng.Config.TTL}, 1) must(r.t, err) ns := namespace(r.t, f) @@ -63,7 +65,7 @@ func (r *rig) remoteWrite(id string) error { func (r *rig) transferBlob(id string) error { realm := cacheRealm(namespace(r.t, owner()).Canonical()) want := []byte("intact payload") - rcv := transfer.Receiver{Store: r.blobs, Realm: realm, Lease: "wire", Live: func(context.Context) error { return nil }} + rcv := transfer.Receiver{Store: r.blobs, Realm: realm, Lease: "wire", Limits: transfer.Limits{MaxFile: 1 << 20}, Live: func(context.Context) error { return nil }} w, err := rcv.Open(r.ctx, transport.BlobHeader{Stream: transfer.StreamArtifact, SHA256: digest(want), Size: int64(len(want))}) must(r.t, err) got := bytes.Clone(want) diff --git a/scripts/drill/cache-security/run b/scripts/drill/cache-security/run new file mode 100755 index 000000000..d7de8ae95 --- /dev/null +++ b/scripts/drill/cache-security/run @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# cache-security drill (P7-CACHE-11, Epic CACHE): runs the cache, artifact and release security matrix (build tag +# cachesec, free/ci/test/cachesec) and fails on any case whose observed outcome differs from its expected one. +# Why: the matrix maps every case to a row of the CI threat model (hq architecture/ci-threat-model.md). A refusal +# that stops happening (an unexpected pass), a refusal with another code or reason key, or one that leaves no audit +# row is a regression of a security control, so the drill fails. +# Inputs: none to run the suite. `--check-mapping ` instead checks, without running a test, that +# every case names a row that exists in that file and that every row the suite owns has a case. +# Usage: run +# run --check-mapping /path/to/ci-threat-model.md +# Outputs: the test log tail and a per-case summary on stderr; testdata/results.json is rewritten by the suite. +# Exit 0 on success, 1 when a case differs, a test skips or the mapping is incomplete, 2 when a tool is missing. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +root=$(cd "$here/../../.." && pwd) +matrix="$root/free/ci/test/cachesec/matrix.yaml" +results="$root/free/ci/test/cachesec/testdata/results.json" +owned="TM-12 TM-13 TM-14 TM-15 TM-16 TM-17 TM-18 TM-20 TM-24" + +for tool in python3; do + command -v "$tool" >/dev/null || { echo "cache-security: $tool is required" >&2; exit 2; } +done + +if [ "${1:-}" = "--check-mapping" ]; then + model=${2:-} + [ -n "$model" ] && [ -r "$model" ] || { echo "cache-security: --check-mapping needs a readable threat model file" >&2; exit 2; } + python3 - "$matrix" "$model" $owned <<'PY' +import json, re, sys +matrix, model, owned = sys.argv[1], sys.argv[2], sys.argv[3:] +cases = json.load(open(matrix)) +rows = set(re.findall(r"^\|\s*(TM-[0-9]{2})\s*\|", open(model).read(), re.M)) +bad = [c["id"] for c in cases if c["threat_row"] not in rows] +if bad: + sys.exit("cache-security: cases name a row that is not in the threat model: " + ", ".join(bad)) +none = [r for r in owned if not any(c["threat_row"] == r for c in cases)] +if none: + sys.exit("cache-security: threat rows with no case: " + ", ".join(none)) +print("cache-security: %d cases map to %d threat-model rows" % (len(cases), len({c["threat_row"] for c in cases})), file=sys.stderr) +PY + exit 0 +fi + +command -v go >/dev/null || { echo "cache-security: go is required" >&2; exit 2; } +tmp=$(mktemp -d "${TMPDIR:-/tmp}/cache-security.XXXXXX") +trap 'rm -rf "$tmp"' EXIT + +echo "cache-security: running free/ci/test/cachesec" >&2 +if ! (cd "$root/free/ci" && INTEGRATION=1 NSELF_V15=1 CGO_ENABLED=0 go test -json -mod=vendor -tags 'cachesec integration' -count=1 -timeout 60m -run '^TestCacheSecMatrix$' ./test/cachesec/...) >"$tmp/suite.json" 2>"$tmp/suite.err"; then + tail -n 40 "$tmp/suite.err" >&2 || true + python3 - "$tmp/suite.json" <<'PY' >&2 || true +import json, sys +for line in open(sys.argv[1]): + try: + e = json.loads(line) + except ValueError: + continue + if e.get("Action") == "output" and ("expected" in e.get("Output", "") or "differ" in e.get("Output", "")): + print(e["Output"].rstrip()) +PY + echo "cache-security: the suite failed" >&2 + exit 1 +fi +python3 "$root/scripts/ci/go-test-ran.py" TestCacheSecMatrix <"$tmp/suite.json" || { echo "cache-security: TestCacheSecMatrix did not run or skipped" >&2; exit 1; } +python3 - "$results" <<'PY' +import json, sys +r = json.load(open(sys.argv[1])) +bad = [c["id"] for c in r if c["outcome"] != c["expected"]] +if bad or not r: + sys.exit("cache-security: unexpected outcome for: " + ", ".join(bad or ["(no results)"])) +print("cache-security: %d cases, every outcome equals its expected one" % len(r), file=sys.stderr) +PY +echo "cache-security: ok" >&2