diff --git a/free/ci/cmd/artifacts.go b/free/ci/cmd/artifacts.go index bf6edf0cf..0af37d75d 100644 --- a/free/ci/cmd/artifacts.go +++ b/free/ci/cmd/artifacts.go @@ -131,7 +131,7 @@ func openArtifacts(fs *flag.FlagSet) (*artifactsEnv, error) { } } role := authz.Role(firstOf(os.Getenv("NSELF_CI_ROLE"), string(authz.RoleAdmin))) - return &artifactsEnv{st: st, blobs: blobs, reader: records.Reader{Ledger: st, Blobs: blobs}, viewer: authz.Viewer{Project: project, Role: role}, + return &artifactsEnv{st: st, blobs: blobs, reader: records.Reader{Ledger: st, Blobs: blobs, Audit: st.AppendAudit}, viewer: authz.Viewer{Project: project, Role: role}, json: fs.Lookup("json").Value.String() == "true"}, nil } diff --git a/free/ci/cmd/artifacts_release.go b/free/ci/cmd/artifacts_release.go index b237c27e1..ec1c19f53 100644 --- a/free/ci/cmd/artifacts_release.go +++ b/free/ci/cmd/artifacts_release.go @@ -49,7 +49,7 @@ func publishDeps(st *store.Store, home, root string) (publish.Deps, error) { } env := []string{"NSELF_CI_REPO=" + root, "NSELF_CI_HOME=" + home} return publish.Deps{Src: st, Blobs: blobs, Ledger: st, Home: home, Root: root, Capture: report.Capture, Team: teamCoordinator(), - Signer: &signing.Client{Runner: &signing.FunnelRunner{Home: home}, Src: st, Env: env, Team: teamCoordinator()}}, nil + Signer: &signing.Client{Runner: &signing.FunnelRunner{Home: home}, Src: st, Env: env, Team: teamCoordinator(), Audit: st.AppendAudit}}, nil } // releasePublisher is the run package's publication seam. diff --git a/free/ci/cmd/artifacts_verify.go b/free/ci/cmd/artifacts_verify.go index 5dca5fafe..1d78f4849 100644 --- a/free/ci/cmd/artifacts_verify.go +++ b/free/ci/cmd/artifacts_verify.go @@ -12,9 +12,10 @@ package main // Outputs: a summary or one v1 envelope (--json) whose data always carries // `slsa_build_level: "L1"`; exit 0 verified, 11 invalid (the reason key names // the check), 1 evidence failing or quarantined. -// Constraints: V15 only; side-effect class read: it writes nothing (resolving -// the pin fetches the default branch into the repository's remote-tracking ref, -// as the policy resolver does for every run). The trust +// Constraints: V15 only; side-effect class read: it writes nothing but, on a +// machine with a coordinator store, one release.verify.refused audit row per +// refusal (resolving the pin fetches the default branch into the repository's +// remote-tracking ref, as the policy resolver does for every run). The trust // roots are the operator files and the protected default branch at the pinned // tip; a pin that cannot be resolved fails the verification (signature), it // never silently drops the branch's keys and revocations; the working tree and @@ -46,6 +47,9 @@ var ( releaseDirOf func(ctx context.Context, run string) (dir string, ok bool, err error) // releaseQuarantined reports evidence quarantine (P7-TRUST-20 registers it). releaseQuarantined func(ctx context.Context, ev model.Evidence) (bool, error) + // releaseVerifyAudit records a refused verification in the coordinator store (registered with + // releaseQuarantined); nil records nothing. + releaseVerifyAudit func(ctx context.Context, actor, action, target, detail string) error ) func init() { registerArtifactsSub("verify", artifactsVerify) } @@ -66,7 +70,7 @@ func artifactsVerify(args []string) int { asJSON := fs.Lookup("json").Value.String() == "true" ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - opt := verify.Options{Capture: report.Capture, Quarantined: releaseQuarantined} + opt := verify.Options{Capture: report.Capture, Quarantined: releaseQuarantined, Audit: releaseVerifyAudit} root, err := evidenceRoot(*repo) if err != nil { return artifactsExit(err, command, asJSON) diff --git a/free/ci/cmd/serve_admin_checkpoint.go b/free/ci/cmd/serve_admin_checkpoint.go index a6a9578a1..bb820d068 100644 --- a/free/ci/cmd/serve_admin_checkpoint.go +++ b/free/ci/cmd/serve_admin_checkpoint.go @@ -3,7 +3,7 @@ package main // Purpose: `nself ci serve admin audit checkpoint export ` and `... verify --pubkey // ` (P7-TRUST-20, Epic D13/D20): the TEAM coordinator signs the head of the audit chain with its own // ci-audit key; verify checks a checkpoint file against a fingerprint pinned off-host and against the live -// chain. It also registers the release quarantine seam of `ci artifacts verify`. +// chain. It also registers the release quarantine and refusal-audit seams of `ci artifacts verify`. // Inputs: a path; --pubkey (the fingerprint `export` printed); --json; $NSELF_CI_HOME (state.db and the // 0600 key audit-checkpoint.key, created at the first export); $NSELF_CI_COORDINATOR_MODE=team. // Outputs: export writes the checkpoint file and prints the public-key fingerprint; verify exits 0 when @@ -58,6 +58,25 @@ func init() { Handler: func(args []string) int { return runAdminCheckpoint(args, productionAdminEnv()) }, }) releaseQuarantined = storeQuarantined + releaseVerifyAudit = storeVerifyAudit +} + +// storeVerifyAudit appends a refused release verification to the coordinator store's audit log; a machine +// without a store (a release checked away from its coordinator) records nothing. +func storeVerifyAudit(ctx context.Context, actor, action, target, detail string) error { + path := filepath.Join(lifecycle.CIHome(), "state.db") + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + st, err := store.Open(path, store.Options{}) + if err != nil { + return err + } + defer func() { _ = st.Close() }() + return st.AppendAudit(ctx, actor, action, target, detail) } // storeQuarantined reads the quarantine index of the coordinator store; a machine without one has nothing diff --git a/free/ci/internal/artifacts/records/collect.go b/free/ci/internal/artifacts/records/collect.go index c8c8c4caa..84f8ab283 100644 --- a/free/ci/internal/artifacts/records/collect.go +++ b/free/ci/internal/artifacts/records/collect.go @@ -9,10 +9,13 @@ package records // entry passes authz.ArtifactPath through an Extractor; a declaration that // selects nothing, a file over the cap, a registered secret, a low disk and a // duplicate record name each fail the whole collection (E712 or E683) before -// the attempt can pass; blobs are written only after the scan of their bytes. +// the attempt can pass; blobs are written only after the scan of their bytes; +// with Audit set, each refusal leaves an artifact.collect.refused row. import ( "context" + "errors" + "fmt" "io" "math" "os" @@ -55,6 +58,25 @@ type Collector struct { Reserve func(total int64) error // MaxRecords bounds the files one attempt may collect (0 = DefaultMaxRecords). MaxRecords int + // Audit, when set, records every refused collection (artifact.collect.refused + // with its reason key; TM-14, TM-16). Target names the attempt or lease. + Audit AuditSink + Target string +} + +// AuditSink writes one audit row (store.AppendAudit). +type AuditSink func(ctx context.Context, actor, action, target, detail string) error + +// audited records a refusal of the collection and returns it; a lost row is joined to it. +func (c Collector) audited(ctx context.Context, err error) error { + var e *Error + if err == nil || c.Audit == nil || !errors.As(err, &e) { + return err + } + if aerr := c.Audit(ctx, "artifacts", "artifact.collect.refused", c.Target, "reason="+e.Reason); aerr != nil { + return errors.Join(err, fmt.Errorf("artifact collect audit: %w", aerr)) + } + return err } // DefaultMaxRecords is the most files one attempt may collect across all its declarations. @@ -71,11 +93,11 @@ type planned struct { // (opened here; callers that hold a pinned root use CollectRoot). func (c Collector) Collect(ctx context.Context, project, base string, decls []Decl) ([]Staged, error) { if info, err := os.Lstat(base); err != nil || !info.IsDir() { - return nil, failf(ReasonUnreadable, "job workdir %q is missing, not a directory or a symlink", base) + return nil, c.audited(ctx, failf(ReasonUnreadable, "job workdir %q is missing, not a directory or a symlink", base)) } root, err := os.OpenRoot(base) if err != nil { - return nil, failf(ReasonUnreadable, "job workdir: %v", err) + return nil, c.audited(ctx, failf(ReasonUnreadable, "job workdir: %v", err)) } defer func() { _ = root.Close() }() return c.CollectRoot(ctx, project, root, decls) @@ -87,6 +109,11 @@ func (c Collector) Collect(ctx context.Context, project, base string, decls []De // refusal leaves nothing behind; the bytes actually streamed are held to the // same budget. func (c Collector) CollectRoot(ctx context.Context, project string, root *os.Root, decls []Decl) ([]Staged, error) { + out, err := c.collect(ctx, project, root, decls) + return out, c.audited(ctx, err) +} + +func (c Collector) collect(ctx context.Context, project string, root *os.Root, decls []Decl) ([]Staged, error) { plan, err := c.plan(ctx, root, decls) if err != nil { return nil, err diff --git a/free/ci/internal/artifacts/records/read.go b/free/ci/internal/artifacts/records/read.go index 191f9c8da..e3c35551e 100644 --- a/free/ci/internal/artifacts/records/read.go +++ b/free/ci/internal/artifacts/records/read.go @@ -8,13 +8,15 @@ package records // record whose blob is gone is reported missing (and remembered) and Get // refuses it (E712 missing_blob); Get writes only inside a directory that // authz.OpenArtifactGetDest pinned, creates every file exclusively and -// re-verifies size and digest of what it wrote. +// re-verifies size and digest of what it wrote; with Audit set, every refused +// Get but "not found" leaves an artifact.get.refused row. import ( "context" "crypto/sha256" "encoding/hex" "errors" + "fmt" "io" "os" "path/filepath" @@ -28,6 +30,9 @@ import ( type Reader struct { Ledger Ledger Blobs Blobs + // Audit, when set, records every refused Get (artifact.get.refused with the + // reason key: a destination, path or digest refusal; TM-14, TM-15). + Audit AuditSink } // List returns the committed records the viewer may see, newest first. @@ -81,6 +86,26 @@ func (r Reader) checkBlob(ctx context.Context, rec Record) Record { // Get writes the record's bytes under dest/ and returns the path. func (r Reader) Get(ctx context.Context, v authz.Viewer, id, dest, cwd string) (string, Record, error) { + p, rec, err := r.get(ctx, v, id, dest, cwd) + if err == nil || r.Audit == nil || errors.Is(err, ErrNotFound) { + return p, rec, err + } + reason := ReasonUnreadable + var pe *authz.PathError + var re *Error + switch { + case errors.As(err, &pe): + reason = string(pe.Reason) + case errors.As(err, &re): + reason = re.Reason + } + if aerr := r.Audit(ctx, "artifacts", "artifact.get.refused", id, "reason="+reason); aerr != nil { + err = errors.Join(err, fmt.Errorf("artifact get audit: %w", aerr)) + } + return p, rec, err +} + +func (r Reader) get(ctx context.Context, v authz.Viewer, id, dest, cwd string) (string, Record, error) { rec, err := r.visible(ctx, v, id) if err != nil { return "", Record{}, err diff --git a/free/ci/internal/attest/revoke.go b/free/ci/internal/attest/revoke.go index 9d8572805..96fa13a84 100644 --- a/free/ci/internal/attest/revoke.go +++ b/free/ci/internal/attest/revoke.go @@ -2,8 +2,9 @@ package attest // Purpose: Revoke(node) (Epic D13, X15): the node's evidence keys become untrusted, the evidence they // signed is quarantined, and the caller gets the re-evaluation set (gates, posted statuses, promoted -// releases). The statuses are re-posted as errors with reason node.revoked (P7-HOST-02's Repost), and -// each promoted release gets one audit event, so a revoked node's success does not stand. +// releases). The statuses are re-posted as errors with reason node.revoked (P7-HOST-02's Repost), each +// quarantined run gets an evidence.quarantined event (reason key_revoked) and each promoted release one +// release.quarantine event, so a revoked node's success does not stand. // Inputs: the node id, the actor, a RevokeLedger, an AuditSink and a Reposter (all injected: this package // cannot import store, audit or report). // Outputs: Affected; errors from the audit and repost steps are joined and returned with it. @@ -33,7 +34,9 @@ type ReleaseRef struct{ RunID, Project, Revision, ManifestSHA256 string } // Affected is what a revocation re-evaluates. type Affected struct { // Keys are the evidence key ids revoked (ci-node-<16 hex>): those the node pins and every one it used. - Keys []string + Keys []string + // Runs are the runs whose evidence the revocation quarantined. + Runs []string Gates []GateRef Statuses []StatusKey Releases []ReleaseRef @@ -74,6 +77,9 @@ func (r Revoker) Revoke(ctx context.Context, nodeID, actor string) (Affected, er if r.Audit != nil { errs = append(errs, r.Audit(ctx, actor, "evidence.revoke", nodeID, fmt.Sprintf("keys=%d gates=%d statuses=%d releases=%d", len(out.Keys), len(out.Gates), len(out.Statuses), len(out.Releases)))) + for _, run := range out.Runs { // TM-20: each quarantined run's evidence is refused from now on + errs = append(errs, r.Audit(ctx, actor, "evidence.quarantined", run, "reason="+string(RemoteKeyRevoked)+" node="+nodeID)) + } for _, rel := range out.Releases { errs = append(errs, r.Audit(ctx, actor, "release.quarantine", rel.RunID, fmt.Sprintf("node=%s project=%s revision=%s manifest=%s", nodeID, rel.Project, rel.Revision, rel.ManifestSHA256))) diff --git a/free/ci/internal/blob/transfer/receiver.go b/free/ci/internal/blob/transfer/receiver.go index a7d95dfec..c89162fa1 100644 --- a/free/ci/internal/blob/transfer/receiver.go +++ b/free/ci/internal/blob/transfer/receiver.go @@ -55,8 +55,9 @@ type Receiver struct { // Live returns nil while the lease epoch is the current one. An error wrapping ErrStale is a // stale epoch (refused, audited); any other error (a store fault) is a transient failure. Live func(ctx context.Context) error - // Refused is told once per refused blob (by digest) when the epoch is stale, so the caller can - // write its audit row. It must not block. + // Refused is told once per refused blob and reason (by digest): a stale epoch, and a blob whose + // bytes end short or fail their digest at publish (TM-13, TM-15), so the caller can write its + // audit row. It must not block. Refused func(ctx context.Context, sha string, err *Error) mu sync.Mutex @@ -83,13 +84,18 @@ func (r *Receiver) epoch(ctx context.Context, sha string) error { if !errors.Is(err, ErrStale) { return Failf(ReasonInterrupted, "the lease check failed: %v", err) } - e := Failf(ReasonStaleEpoch, "the lease epoch is no longer current; blob %s is refused", short(sha)) + return r.tell(ctx, sha, Failf(ReasonStaleEpoch, "the lease epoch is no longer current; blob %s is refused", short(sha))) +} + +// tell reports e to Refused once per blob and reason, and returns it. +func (r *Receiver) tell(ctx context.Context, sha string, e *Error) *Error { r.mu.Lock() - first := !r.told[sha] + k := sha + "/" + e.Reason + first := !r.told[k] if r.told == nil { r.told = map[string]bool{} } - r.told[sha] = true + r.told[k] = true r.mu.Unlock() if first && r.Refused != nil { r.Refused(ctx, sha, e) diff --git a/free/ci/internal/blob/transfer/writer.go b/free/ci/internal/blob/transfer/writer.go index df87013a3..9bd84aef7 100644 --- a/free/ci/internal/blob/transfer/writer.go +++ b/free/ci/internal/blob/transfer/writer.go @@ -69,7 +69,7 @@ func (w *Writer) publish() error { return Failf(ReasonProtocol, "blob %s was aborted", short(w.h.SHA256)) } if w.offset != w.h.Size { - return Failf(ReasonInterrupted, "blob %s holds %d of %d B", short(w.h.SHA256), w.offset, w.h.Size) + return w.r.tell(context.Background(), w.h.SHA256, Failf(ReasonInterrupted, "blob %s holds %d of %d B", short(w.h.SHA256), w.offset, w.h.Size)) } if err := w.r.epoch(context.Background(), w.h.SHA256); err != nil { return err @@ -77,9 +77,9 @@ func (w *Writer) publish() error { if err := w.p.Finalize(); err != nil { if errors.Is(err, cas.ErrPartialConflict) { w.offset = 0 // the partial is gone: a retry starts clean - return Failf(ReasonDigest, "blob %s does not match its digest; nothing was published", short(w.h.SHA256)) + return w.r.tell(context.Background(), w.h.SHA256, Failf(ReasonDigest, "blob %s does not match its digest; nothing was published", short(w.h.SHA256))) } - return Failf(ReasonInterrupted, "blob %s could not be published: %v", short(w.h.SHA256), err) + return w.r.tell(context.Background(), w.h.SHA256, Failf(ReasonInterrupted, "blob %s could not be published: %v", short(w.h.SHA256), err)) } w.done, w.published = true, true return nil diff --git a/free/ci/internal/cache/entries/codes_entries.go b/free/ci/internal/cache/entries/codes_entries.go index c9eaa8d60..b779e1ab7 100644 --- a/free/ci/internal/cache/entries/codes_entries.go +++ b/free/ci/internal/cache/entries/codes_entries.go @@ -25,8 +25,12 @@ const ( ReasonSizeCap = "size_cap" ReasonDiskFloor = "disk_floor" ReasonPathOutside = "path_outside" + // ReasonSecretMatch: a save the value scan refused, or a read-only snapshot + // whose writer received secrets (refused to every other reader, TM-12). + ReasonSecretMatch = "secret_match" // ReasonBlobInvalid extends the Epic D20 list: a blob is missing, has the wrong - // size or was not freshly Put when its entry commits. + // size or was not freshly Put when its entry commits, or a restore found it + // missing, of the wrong size or failing its digest (TM-15). ReasonBlobInvalid = "blob_invalid" ) @@ -34,13 +38,19 @@ const ( type Refused struct { Reason string Err error // the cause, when there is one (ErrBlobStale, os.ErrNotExist) + // Op names the refused operation in the message; empty means "write". + Op string } func (e *Refused) Error() string { + op := e.Op + if op == "" { + op = "write" + } if e.Err != nil { - return CodeCacheRefused + " cache write refused: " + e.Reason + ": " + e.Err.Error() + return CodeCacheRefused + " cache " + op + " refused: " + e.Reason + ": " + e.Err.Error() } - return CodeCacheRefused + " cache write refused: " + e.Reason + return CodeCacheRefused + " cache " + op + " refused: " + e.Reason } // Unwrap exposes the cause. diff --git a/free/ci/internal/cache/entries/lookup.go b/free/ci/internal/cache/entries/lookup.go index ec773f582..da2df0406 100644 --- a/free/ci/internal/cache/entries/lookup.go +++ b/free/ci/internal/cache/entries/lookup.go @@ -4,7 +4,8 @@ package entries // every blob through cas, and Verify over all entries. // Inputs: the namespace being read, its key and the authz access mode. // Outputs: the Manifest and a hit flag; every refusal is a miss, never a -// fallback. +// fallback. ResolveForRestore also names why an entry found under the key was +// not served (secret_match, blob_invalid), so a restore can refuse it. // Constraints: no restore-key or prefix fallback, no other key, no other // namespace. The stored namespace is compared byte for byte (a mismatch is a // miss plus an audit row). ro-snapshot serves only authz.SnapshotEligible @@ -67,7 +68,8 @@ func (s *Store) intactBy(m Manifest, rehash bool) (reason string, err error) { // every blob is intact. The caller must derive access from authz.CacheAccess for // the attempt's own namespace and ns; Lookup does not know the attempt. func (s *Store) Lookup(ctx context.Context, ns authz.Namespace, key string, access authz.Mode) (Manifest, bool, error) { - return s.lookup(ctx, ns, key, access, true) + m, ok, _, err := s.lookup(ctx, ns, key, access, true) + return m, ok, err } // LookupForRestore is Lookup for a caller that reads every blob through @@ -76,34 +78,47 @@ func (s *Store) Lookup(ctx context.Context, ns authz.Namespace, key string, acce // hashes a 100 MB build cache once, not three times. A blob that fails the // digest is quarantined by that read, and the caller drops the entry. func (s *Store) LookupForRestore(ctx context.Context, ns authz.Namespace, key string, access authz.Mode) (Manifest, bool, error) { + m, ok, _, err := s.lookup(ctx, ns, key, access, false) + return m, ok, err +} + +// ResolveForRestore is LookupForRestore for a restore that refuses, rather than +// misses, an entry it found under the key and must not serve (TM-12, TM-15): +// refused is ReasonSecretMatch for a snapshot whose writer received secrets and +// ReasonBlobInvalid for a blob that is missing or has the wrong size (the entry +// is dropped, as Lookup drops it). Any other unusable entry is a plain miss. +func (s *Store) ResolveForRestore(ctx context.Context, ns authz.Namespace, key string, access authz.Mode) (m Manifest, hit bool, refused string, err error) { return s.lookup(ctx, ns, key, access, false) } -func (s *Store) lookup(ctx context.Context, ns authz.Namespace, key string, access authz.Mode, rehash bool) (Manifest, bool, error) { +func (s *Store) lookup(ctx context.Context, ns authz.Namespace, key string, access authz.Mode, rehash bool) (Manifest, bool, string, error) { if (access != authz.ModeRWOwn && access != authz.ModeROSnapshot) || !ns.Valid() || !hexKey.MatchString(key) { - return Manifest{}, false, nil + return Manifest{}, false, "", nil } list, err := s.be.CacheEntries(ctx, key) if err != nil || len(list) == 0 { - return Manifest{}, false, err + return Manifest{}, false, "", err } i := slices.IndexFunc(list, func(m Manifest) bool { return bytes.Equal(m.Namespace, ns.Canonical()) }) if i < 0 { // the key exists, but only under other namespaces: a miss, never a fallback - return Manifest{}, false, s.be.CacheAuditOnce(ctx, "cache.lookup.namespace_mismatch", key, "") + return Manifest{}, false, "", s.be.CacheAuditOnce(ctx, "cache.lookup.namespace_mismatch", key, "") } m := list[i] if access == authz.ModeROSnapshot && !s.snapshotOK(m) { - return Manifest{}, false, nil + if m.Writer.SecretsReceived { // a snapshot of a secret-bearing job is refused to others, never served + return Manifest{}, false, ReasonSecretMatch, nil + } + return Manifest{}, false, "", nil } reason, err := s.intactBy(m, rehash) if err != nil || reason == "unstable" { - return Manifest{}, false, err + return Manifest{}, false, "", err } if reason != "" { _, err = s.be.CacheDrop(ctx, m, reason) - return Manifest{}, false, err + return Manifest{}, false, ReasonBlobInvalid, err } - return m, true, s.be.CacheTouch(ctx, m.Namespace, key) + return m, true, "", s.be.CacheTouch(ctx, m.Namespace, key) } // snapshotOK applies authz.SnapshotEligible to the facts stored with m. diff --git a/free/ci/internal/cache/remote/restore_coord.go b/free/ci/internal/cache/remote/restore_coord.go index 3f410707e..2b725c9e8 100644 --- a/free/ci/internal/cache/remote/restore_coord.go +++ b/free/ci/internal/cache/remote/restore_coord.go @@ -11,7 +11,8 @@ package remote // is done when the node confirms it holds every byte (a held copy answers at the // announce, a fetched one after its re-hash). A node that reports a failed // re-hash after the final chunk gets the blob again, as many times as the -// transfer allows. Locality is advisory and never authorises a read. +// transfer allows, and leaves a cache.restore.refused row (digest_mismatch). +// Locality is advisory and never authorises a read. import ( "context" @@ -141,7 +142,10 @@ func (l *Lease) sendRestore(ctx context.Context, snd *transfer.Sender, up *final if got == b.Size { return nil } - // The node re-hashed the bytes, found them wrong, discarded them and reset to 0: send again. + // The node re-hashed the bytes, found them wrong, discarded them and reset to 0: audit, send again. + if err := l.h.Ledger.CacheAuditOnce(ctx, AuditRestoreRefused, l.b.AttemptID, "reason="+transfer.ReasonDigest+" node="+l.b.Node); err != nil && l.h.Warn != nil { + l.h.Warn(fmt.Errorf("cache restore audit: %w", err)) + } } return transfer.Failf(transfer.ReasonDigest, "blob %s failed verification on the node %d times", b.SHA256[:12], tries) } diff --git a/free/ci/internal/cache/specs/engine.go b/free/ci/internal/cache/specs/engine.go index 3fcf9daad..f45b8afe2 100644 --- a/free/ci/internal/cache/specs/engine.go +++ b/free/ci/internal/cache/specs/engine.go @@ -11,7 +11,10 @@ package specs // only after every blob was Put; nothing is committed for a cache with no // files in it (a hit on an empty entry would keep the next run from filling // it); restore refuses (and drops) an entry whose blob set is not exactly the -// spec's dirs or whose archive the path sanitiser rejects. +// spec's dirs or whose archive the path sanitiser rejects, refuses (E711 +// blob_invalid) an entry whose stored blob is missing, short or fails its +// digest, and refuses (E711 secret_match) a read-only snapshot of a +// secret-bearing writer; each refusal writes a cache.restore.refused row. import ( "context" @@ -68,10 +71,18 @@ func (d Dirs) names() []string { // is (false, nil). A refused entry is dropped, the dirs are emptied and the // error carries the sanitiser's E683 or the reason. func (e *Engine) Restore(ctx context.Context, base *os.Root, ns authz.Namespace, key string, access authz.Mode, spec Spec, dirs Dirs, opt UnpackOptions) (hit bool, bytes int64, err error) { - m, ok, err := e.Entries.LookupForRestore(ctx, ns, key, access) - if err != nil || !ok { + m, ok, refused, err := e.Entries.ResolveForRestore(ctx, ns, key, access) + if err != nil { return false, 0, err } + if refused != "" { // found under the key but never served: refused, not missed (TM-12, TM-15) + r := &entries.Refused{Op: "restore", Reason: refused} + _ = e.Backend.AppendAudit(ctx, "cache", "cache.restore.refused", key, "reason="+refused) + return false, 0, r + } + if !ok { + return false, 0, nil + } if !e.writerPassed(ctx, m) { // a cancel won after the commit and nobody retracted it (or a crash): never serve it return false, 0, nil } @@ -90,6 +101,9 @@ func (e *Engine) Restore(ctx context.Context, base *os.Root, ns authz.Namespace, if err == nil { return true, m.Bytes(), nil } + if blobFault(err) { // the stored bytes fail their digest or size: the entry's blob is invalid + err = &entries.Refused{Op: "restore", Reason: entries.ReasonBlobInvalid, Err: err} + } for _, d := range dirs { _ = removeIn(base, d) } @@ -100,9 +114,20 @@ func (e *Engine) Restore(ctx context.Context, base *os.Root, ns authz.Namespace, _, _ = e.Backend.CacheDrop(ctx, m, "unrestorable") } _ = e.Backend.AppendAudit(ctx, "cache", "cache.restore.refused", key, "reason="+refusalKey(err)) + var refusal *entries.Refused + if errors.As(err, &refusal) { + return false, 0, err + } return false, 0, fmt.Errorf("cache restore refused: %w", err) } +// blobFault reports whether err is the blob store refusing the stored bytes +// (E710 digest or size mismatch, or a file that changed under the read). +func blobFault(err error) bool { + var ce *cas.Error + return errors.As(err, &ce) || errors.Is(err, cas.ErrInvalid) +} + // writerPassed reports whether the attempt that committed m settled passed. An // attempt still finalizing is not served yet; one that ended any other way, or // that the store does not know, has its entry dropped: a missing attempt is no @@ -175,5 +200,9 @@ func refusalKey(err error) string { if errors.As(err, &pe) { return string(pe.Reason) } + var re *entries.Refused + if errors.As(err, &re) { + return re.Reason + } return "unreadable" } diff --git a/free/ci/internal/release/job/stash.go b/free/ci/internal/release/job/stash.go index b2f0b270d..d11ff5ee5 100644 --- a/free/ci/internal/release/job/stash.go +++ b/free/ci/internal/release/job/stash.go @@ -7,13 +7,16 @@ package job // digester must never run under the writer lock). // Inputs: a Provider per release attempt (Guard) or static Facts (Put, tests). // Outputs: Source.ReleaseFacts: the latest Facts, or an E715 when the last -// refresh failed or is older than maxAge (the attempt then stays queued). +// refresh failed or is older than maxAge (the attempt then stays queued); one +// release.admission.refused audit row per refusal Guard returns (Stash.Audit). // Constraints: safe for concurrent use; the refresher stops when the submission // returns; an entry is forgotten with it. maxAge is several refresh intervals, so // a change made while the attempt waits is seen within about refreshEvery. import ( "context" + "errors" + "fmt" "sync" "time" ) @@ -41,6 +44,29 @@ type Stash struct { // between Guard and Admit is seen atomically; nil revokes nothing. An error // (the ledger cannot answer) refuses the attempt: the signer is unverified. Revoked func(keyID string) (bool, error) + // Audit writes one audit row (store.AppendAudit); Guard records every + // refusal it returns as release.admission.refused (TM-17). Nil records none. + Audit AuditSink +} + +// AuditSink writes one audit row. +type AuditSink func(ctx context.Context, actor, action, target, detail string) error + +// AuditAction is the audit action of a refused release admission. +const AuditAction = "release.admission.refused" + +// refused records e (when it is a refusal) and returns it. A lost row is joined +// to the refusal, never silent, and never turns it into an admission. +func (s *Stash) refused(ctx context.Context, attemptID string, e *Error) error { + if e == nil { + return nil + } + if s.Audit != nil { + if err := s.Audit(ctx, "release", AuditAction, attemptID, "reason="+e.Reason+" code="+e.Code); err != nil { + return errors.Join(e, fmt.Errorf("release admission audit: %w", err)) + } + } + return e } // NewStash returns an empty stash. @@ -108,21 +134,21 @@ func (s *Stash) Guard(ctx context.Context, attemptID string, p Provider, next fu _, busy := s.m[attemptID] s.mu.Unlock() if busy { // two submissions of one attempt would share, and one would delete, the other's facts - return refuse(CodeNotAdmitted, Duplicate, "attempt %s is already being submitted", attemptID) + return s.refused(ctx, attemptID, refuse(CodeNotAdmitted, Duplicate, "attempt %s is already being submitted", attemptID)) } asked := now() f, perr := p(ctx) if perr != nil { - return perr + return s.refused(ctx, attemptID, perr) } if e := Evaluate(f); e != nil { - return e + return s.refused(ctx, attemptID, e) } own := &entry{f: f, at: asked, live: true} s.mu.Lock() if _, busy = s.m[attemptID]; busy { s.mu.Unlock() - return refuse(CodeNotAdmitted, Duplicate, "attempt %s is already being submitted", attemptID) + return s.refused(ctx, attemptID, refuse(CodeNotAdmitted, Duplicate, "attempt %s is already being submitted", attemptID)) } s.m[attemptID] = own s.mu.Unlock() diff --git a/free/ci/internal/release/verify/verify.go b/free/ci/internal/release/verify/verify.go index ed0605dfe..600b4c53d 100644 --- a/free/ci/internal/release/verify/verify.go +++ b/free/ci/internal/release/verify/verify.go @@ -12,7 +12,7 @@ package verify // SHA256SUMS and every file digest, evidence digest and revision, provenance // signature and subjects, SBOM links, then the evidence verdict at the released // revision. Nothing runs a process but git through the exec funnel; nothing -// writes. The claimed level is exactly Build L1 and Result carries no other. +// writes but the optional audit row of a refusal. The claimed level is exactly Build L1 and Result carries no other. import ( "context" @@ -59,6 +59,9 @@ type Options struct { // Quarantined tells whether the evidence was quarantined; nil quarantines // nothing. An error fails closed as quarantined. Quarantined func(ctx context.Context, ev model.Evidence) (bool, error) + // Audit, when set, records a refused verification (release.verify.refused + // with the reason key; TM-18). Its error is a Result warning, never a pass. + Audit func(ctx context.Context, actor, action, target, detail string) error } // Result is the outcome of Release; it is what `artifacts verify --json` prints. @@ -109,11 +112,11 @@ func Release(ctx context.Context, dir string, opt Options) Result { {"provenance", r.provenance}, {"sbom_link", r.sbomLinks}, {"evidence_verdict", r.evidenceVerdict}, } if st, err := os.Stat(dir); err != nil || !st.IsDir() { - return res.fail(fail(ReasonReleaseMissing, "%s is not a release directory", dir)) + return r.refused(res.fail(fail(ReasonReleaseMissing, "%s is not a release directory", dir))) } for _, s := range steps { if f := s.fn(); f != nil { - return res.fail(f) + return r.refused(res.fail(f)) } res.Passed = append(res.Passed, s.name) } @@ -121,6 +124,17 @@ func Release(ctx context.Context, dir string, opt Options) Result { return res } +// refused writes the refusal's audit row when Options.Audit is set. +func (r *run) refused(res Result) Result { + if r.opt.Audit == nil { + return res + } + if err := r.opt.Audit(r.ctx, "release", "release.verify.refused", r.dir, "reason="+res.Reason+" code="+res.Code); err != nil { + res.Warnings = append(res.Warnings, "the refusal could not be audited: "+err.Error()) + } + return res +} + func (res Result) fail(f *Failure) Result { res.Verified, res.Exit, res.Code, res.Reason, res.Message = false, f.ExitCode(), Code, f.Reason, f.Message return res diff --git a/free/ci/internal/run/artifacts_hook.go b/free/ci/internal/run/artifacts_hook.go index 3b85904ef..94b209ab0 100644 --- a/free/ci/internal/run/artifacts_hook.go +++ b/free/ci/internal/run/artifacts_hook.go @@ -190,7 +190,8 @@ func (h *artifactsHook) afterJob(ctx context.Context, a AttemptInfo, workspace * } facts.Producer = "nself-ci/" + a.JobName reserve := func(total int64) error { return a.Store.ArtifactReserve(ctx, a.AttemptID, epoch, total, pol) } - staged, err := records.Collector{Blobs: blobs, Scanner: h.scanner(), Policy: pol, Usage: usage, Reserve: reserve}.CollectRoot(ctx, facts.Project, root, decls) + staged, err := records.Collector{Blobs: blobs, Scanner: h.scanner(), Policy: pol, Usage: usage, Reserve: reserve, + Audit: a.Store.AppendAudit, Target: a.AttemptID}.CollectRoot(ctx, facts.Project, root, decls) if err != nil { return err } diff --git a/free/ci/internal/run/release_hook.go b/free/ci/internal/run/release_hook.go index 94888f0a6..ea79f964d 100644 --- a/free/ci/internal/run/release_hook.go +++ b/free/ci/internal/run/release_hook.go @@ -10,7 +10,8 @@ package run // internal/release/job. // Inputs: AttemptSpec (Dispatch wrapper), JobSpec (isolator wrapper), // AttemptInfo (attempt-start hook). -// Outputs: the previous hook's answer, or an E715 refusal. +// Outputs: the previous hook's answer, or an E715 refusal (Guard's refusals +// also leave a release.admission.refused row in the run's store). // Constraints: a release job runs only under the V15 engine (the v1.4 gate has // no release class): anywhere else it is refused. The facts come from the run's // own store rows and git worktree, never from a file the job could write. @@ -48,6 +49,7 @@ func init() { panic("run: release_hook.go must initialise after dispatch_hook.go and isolation_hook.go") } releaseStash.Revoked = func(keyID string) (bool, error) { return releaseRevoked(attestLedger(), keyID) } + releaseStash.Audit = releaseAudit if err := job.Register(releaseStash); err != nil { panic(err) } @@ -66,6 +68,15 @@ func releaseRevoked(l attest.Ledger, keyID string) (bool, error) { return l.Revoked(context.Background(), keyID) } +// releaseAudit writes a release admission refusal into the run's own store; +// releaseDispatch refuses an attempt without a session before Guard runs. +func releaseAudit(ctx context.Context, actor, action, target, detail string) error { + if sess := SessionFrom(ctx); sess != nil && sess.Store != nil { + return sess.Store.AppendAudit(ctx, actor, action, target, detail) + } + return errors.New("the run has no store for the release admission audit row") +} + // releaseFailure carries a release refusal as a run error with its code. func releaseFailure(e *job.Error) error { return fail(e.Code, "%s: %s", e.Reason, e.Message) } @@ -98,8 +109,12 @@ func releaseDispatch(ctx context.Context, s AttemptSpec, prev DispatchHook) (Adm } var adm Admitted err = releaseStash.Guard(ctx, id, provider, func() (e error) { adm, e = prev(ctx, s); return }) - if je, ok := err.(*job.Error); ok { - return Admitted{}, releaseFailure(je) + var je *job.Error + if errors.As(err, &je) { + if err == error(je) { + return Admitted{}, releaseFailure(je) + } + return Admitted{}, errors.Join(releaseFailure(je), err) // the refusal and its lost audit row } return adm, err } diff --git a/free/ci/internal/signing/client.go b/free/ci/internal/signing/client.go index 66e179c88..50a81248c 100644 --- a/free/ci/internal/signing/client.go +++ b/free/ci/internal/signing/client.go @@ -5,7 +5,8 @@ package signing // one request and read the envelopes back (cap:ci.release-signing: Sign, // PublicKey; P7-CACHE-17 and P7-CACHE-18 call it). // Inputs: a Runner, the run's Source and a run id with payloads. -// Outputs: DSSE envelopes in request order, or an E717 refusal. +// Outputs: DSSE envelopes in request order, or an E717 refusal (with Audit set, +// one release.sign.refused row per refused Sign). // Constraints: the signer's answer is checked for shape (one ci-release key id and one // 64-byte signature per item); its cryptographic check belongs to the verifier // that holds the trust file (P7-CACHE-09/18); the signer starts only when every attempt of the run is @@ -21,6 +22,8 @@ import ( "crypto/ed25519" "encoding/base64" "encoding/json" + "errors" + "fmt" "os" "time" @@ -43,6 +46,10 @@ type Client struct { // Team is set on a TEAM coordinator, which holds secret references only // (Epic X8): it never starts the signer and never resolves the release key. Team bool + // Audit, when set, records every refused Sign as release.sign.refused with + // its reason key (TM-18). The signer process never writes the store, so the + // coordinator records its refusals here. + Audit func(ctx context.Context, actor, action, target, detail string) error } func (c *Client) exe() (string, error) { @@ -110,8 +117,20 @@ func signerRefusal(out []byte, code int) error { return Failf(ReasonSignerFailed, "the signer failed (exit %d)", code) } -// Sign signs the payloads of a finished run, or refuses. +// Sign signs the payloads of a finished run, or refuses (and audits the refusal). func (c *Client) Sign(ctx context.Context, runID string, items []Item) ([]signing.Envelope, error) { + envs, err := c.sign(ctx, runID, items) + var e *Error + if err == nil || c.Audit == nil || !errors.As(err, &e) { + return envs, err + } + if aerr := c.Audit(ctx, "release", "release.sign.refused", runID, "reason="+e.Reason); aerr != nil { + return envs, errors.Join(err, fmt.Errorf("signer refusal audit: %w", aerr)) + } + return envs, err +} + +func (c *Client) sign(ctx context.Context, runID string, items []Item) ([]signing.Envelope, error) { if c.Src == nil || c.Runner == nil { return nil, Failf(ReasonSignerFailed, "the signer client has no run store or runner") } diff --git a/free/ci/internal/store/cache_artifacts_commit.go b/free/ci/internal/store/cache_artifacts_commit.go index fb985ed1d..27217a422 100644 --- a/free/ci/internal/store/cache_artifacts_commit.go +++ b/free/ci/internal/store/cache_artifacts_commit.go @@ -5,7 +5,7 @@ package store // the current epoch, after the caller re-hashed every blob. // Inputs: attempt id, epoch, the verified staged records, a records.Policy. // Outputs: the 80 % warning text; *records.Error (E711 stale capability, -// E712 size_cap) after one audit row. +// E712 size_cap, digest_mismatch, uncommitted) after one audit row. // Constraints: one writer transaction re-reads the attempt, the staged rows // and the committed byte sums; the rows the caller verified must be exactly // the rows that are staged now; released-kind bytes count toward the cap. @@ -56,14 +56,16 @@ func (s *Store) ArtifactCommit(ctx context.Context, attempt string, epoch int64, var add int64 for _, v := range verified { r, ok := staged[v.ID] - if !ok || r.SHA256 != v.SHA256 || r.Size != v.Size { - return records.Failf(records.ReasonDigest, "record %q is not the staged record that was verified", v.Name) + if !ok || r.SHA256 != v.SHA256 || r.Size != v.Size { // overwrite or substitution (TM-15): refused and audited + refusal = records.Failf(records.ReasonDigest, "record %q is not the staged record that was verified", v.Name) + return s.appendAudit(ctx, tx, "artifacts", "artifact.commit.refused", attempt, "reason="+records.ReasonDigest) } add += r.Size delete(staged, v.ID) } if len(staged) > 0 { - return records.Failf(records.ReasonUncommitted, "%d staged record(s) of %s were not verified", len(staged), attempt) + refusal = records.Failf(records.ReasonUncommitted, "%d staged record(s) of %s were not verified", len(staged), attempt) + return s.appendAudit(ctx, tx, "artifacts", "artifact.commit.refused", attempt, "reason="+records.ReasonUncommitted) } u, err := artUsage(ctx, tx, project) if err != nil { diff --git a/free/ci/internal/store/trust_attest.go b/free/ci/internal/store/trust_attest.go index 3b0b79457..7e5383878 100644 --- a/free/ci/internal/store/trust_attest.go +++ b/free/ci/internal/store/trust_attest.go @@ -200,6 +200,10 @@ func (l AttestLedger) RevokeNode(ctx context.Context, nodeID, reason string) (at func affected(ctx context.Context, tx *sql.Tx, keyIDs []string, out *attest.Affected) (err error) { ids, _ := json.Marshal(keyIDs) + runs, err := tx.QueryContext(ctx, `SELECT DISTINCT run_id FROM attest_quarantine WHERE key_id IN (SELECT value FROM json_each(?)) ORDER BY run_id`, string(ids)) + if out.Runs, err = pkiCollect(runs, err, func(r *sql.Rows) (id string, e error) { return id, r.Scan(&id) }); err != nil { + return err + } gates, err := tx.QueryContext(ctx, `SELECT DISTINCT repo_id,revision,job FROM attest_use WHERE key_id IN (SELECT value FROM json_each(?)) ORDER BY repo_id,revision,job`, string(ids)) if out.Gates, err = pkiCollect(gates, err, func(r *sql.Rows) (g attest.GateRef, e error) { return g, r.Scan(&g.RepoID, &g.Revision, &g.Job) }); err != nil { return err diff --git a/free/ci/test/cachesec/artifact_test.go b/free/ci/test/cachesec/artifact_test.go index ea7a8efb7..8e4546894 100644 --- a/free/ci/test/cachesec/artifact_test.go +++ b/free/ci/test/cachesec/artifact_test.go @@ -38,7 +38,7 @@ func (r *rig) secret(id string) error { defer base.Close() _, err = r.engine.Save(r.ctx, specs.SaveRequest{Base: base, Attempt: "writer", Epoch: 3, Namespace: namespace(r.t, owner()), Key: key(r.t, owner()), Spec: specs.Spec{ID: "go"}, Dirs: specs.Dirs{"data": "data"}}) } else { - c := records.Collector{Blobs: r.blobs, Scanner: r.red, Policy: artifactPolicy()} + c := records.Collector{Blobs: r.blobs, Scanner: r.red, Policy: artifactPolicy(), Audit: r.st.AppendAudit, Target: "writer"} _, err = c.Collect(r.ctx, "project", ws, []records.Decl{{Path: "data/payload", Kind: records.KindBinary, Retention: "30d", Visibility: authz.VisibilityProject}}) } r.noEntries() @@ -110,7 +110,7 @@ func (r *rig) getDest(id string) error { _, err := r.st.ArtifactCommit(r.ctx, "download", 3, []records.Record{rec}, artifactPolicy()) must(r.t, err) // A visible committed record ensures the real download reaches the destination check. - reader := records.Reader{Ledger: r.st, Blobs: r.blobs} + reader := records.Reader{Ledger: r.st, Blobs: r.blobs, Audit: r.st.AppendAudit} _, _, err = reader.Get(r.ctx, authz.Viewer{Project: "project", Role: authz.RoleMember}, rec.ID, dest, cwd) if !reflect.DeepEqual(before, tree(r.t, cwd)) { r.t.Error("refused download changed the destination") diff --git a/free/ci/test/cachesec/readthrough_test.go b/free/ci/test/cachesec/readthrough_test.go new file mode 100644 index 000000000..08d6a4505 --- /dev/null +++ b/free/ci/test/cachesec/readthrough_test.go @@ -0,0 +1,92 @@ +//go:build cachesec + +package cachesec + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "time" + + cacheRemote "github.com/nself-org/plugins/free/ci/internal/cache/remote" + "github.com/nself-org/plugins/free/ci/internal/protocol" + "github.com/nself-org/plugins/free/ci/internal/transport" +) + +// wire joins the coordinator's Restore to a node's NodeRestorer in memory; mangle changes every chunk +// on its way to the node (the bytes the node's read-through copy receives). +type wire struct { + frames chan transport.BlobFrame + offs chan protocol.Offset + mangle func(*protocol.Chunk) +} + +func (w *wire) send(f transport.BlobFrame) error { + select { + case w.frames <- f: + return nil + case <-time.After(5 * time.Second): + return errors.New("wire: node not reading") + } +} + +func (w *wire) Digest(d protocol.Digest) error { return w.send(transport.BlobFrame{Digest: &d}) } +func (w *wire) Chunk(c protocol.Chunk) error { + w.mangle(&c) + return w.send(transport.BlobFrame{Chunk: &c}) +} +func (w *wire) Offsets() <-chan protocol.Offset { return w.offs } + +type wireFeed struct{ w *wire } + +func (f wireFeed) Frames() <-chan transport.BlobFrame { return f.w.frames } +func (f wireFeed) Offset(o protocol.Offset) error { f.w.offs <- o; return nil } +func (f wireFeed) Close() {} + +// readThrough restores one planned blob from the coordinator into a node's read-through copies while +// every chunk is corrupted or cut short in transit; the node's re-hash must refuse each copy. +func (r *rig) readThrough(id string) error { + want := []byte("intact payload") + ns := namespace(r.t, owner()) + sha, size, err := r.blobs.Put(r.ctx, cacheRealm(ns.Canonical()), bytes.NewReader(want)) + must(r.t, err) + b := protocol.CacheBlob{Path: "data", SHA256: sha, Size: size} + l, realm := r.bindRemote(false, b) + dir := r.t.TempDir() + copies, err := cacheRemote.NewReadThrough(dir, 4<<20) + must(r.t, err) + w := &wire{frames: make(chan transport.BlobFrame, 8), offs: make(chan protocol.Offset, 64), mangle: func(c *protocol.Chunk) { + c.Data = mangled(r, c.Data, id == "partial-readthrough") + }} + ctx, cancel := context.WithTimeout(r.ctx, 60*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Restore(ctx, w) }() + n := &cacheRemote.NodeRestorer{Copies: copies, Feed: wireFeed{w}, Realm: realm, Wait: 5 * time.Second} + held := n.Fetch(ctx, []protocol.CacheBlob{b}) + err = <-done + if held[sha] { + r.t.Error("invalid read-through copy was held") + } + if f, e := copies.Open(ctx, realm, sha, size, nil); e == nil { + f.Close() + r.t.Error("invalid read-through copy was served") + } + if len(tree(r.t, dir)) != 0 { + r.t.Error("read-through published invalid bytes") + } + return err +} + +// mangled flips the first byte of a chunk's data, or keeps only its first half. +func mangled(r *rig, data string, partial bool) string { + b, err := base64.StdEncoding.DecodeString(data) + must(r.t, err) + if partial { + b = b[:len(b)/2] + } else if len(b) > 0 { + b[0] ^= 1 + } + return base64.StdEncoding.EncodeToString(b) +} diff --git a/free/ci/test/cachesec/release_test.go b/free/ci/test/cachesec/release_test.go index b4998e7c9..7e068b6cd 100644 --- a/free/ci/test/cachesec/release_test.go +++ b/free/ci/test/cachesec/release_test.go @@ -16,11 +16,13 @@ import ( "github.com/nself-org/plugins/free/ci/internal/store" ) -func releaseFacts() job.Facts { +// releaseFacts builds the facts of a release whose one dep ended with status +// (the coordinator's own evidence, consistent by construction). +func releaseFacts(status string) job.Facts { d := model.InputDigests{Config: strings.Repeat("c", 64), Lockfiles: []model.DigestRef{{Path: "go.sum", SHA256: strings.Repeat("1", 64)}}} return job.BuildFacts(job.RunState{RunID: "pbuild", RepoID: "project", Revision: strings.Repeat("a", 40), PolicyDigest: "policy", ConfigDigest: d.Config, SourceTrust: "owner", Trigger: "local", Pipeline: model.PipelineRef{ID: "pbuild", Name: "release", Kind: "default"}, Selection: model.Selection{Mode: "full", Reason: "selection.full_required", Digest: "digest"}, Deps: []string{"build"}, - Outcomes: []job.Dep{{Name: "build", Status: "passed", Required: true}}, Clean: true, Bound: &d, Now: &d}) + Outcomes: []job.Dep{{Name: "build", Status: status, Required: true}}, Clean: true, Bound: &d, Now: &d}) } type evidenceSigner struct{ priv ed25519.PrivateKey } @@ -44,11 +46,11 @@ func (l evidenceLedger) NodesByKey(context.Context, string) ([]attest.KeyNode, e } func (r *rig) release(id string) error { - f := releaseFacts() + f := releaseFacts("passed") switch id { case "release-no-deps": - f.Evidence.Result = "fail" - f.Evidence.Checks[0].Result = "fail" + // The dep failed: the evidence says so in its result, its check and its counts. + f = releaseFacts("failed") case "untrusted-promotion": f.Evidence.SourceTrust = "untrusted" case "release-signed-remote", "release-revoked-remote": @@ -86,9 +88,13 @@ func (r *rig) release(id string) error { } return v.VerifyEvidence(raw, f.Evidence) } - e := job.Evaluate(f) - if e == nil { - return nil + // The admission path a release attempt takes (run's Dispatch wrapper), with the run's store as audit sink. + st := job.NewStash() + st.Audit = r.st.AppendAudit + queued := false + err := st.Guard(r.ctx, "release-attempt", job.Static(f), func() error { queued = true; return nil }) + if err != nil && queued { + r.t.Error("a refused release attempt was queued") } - return e + return err } diff --git a/free/ci/test/cachesec/signer_test.go b/free/ci/test/cachesec/signer_test.go new file mode 100644 index 000000000..e6e51ca22 --- /dev/null +++ b/free/ci/test/cachesec/signer_test.go @@ -0,0 +1,38 @@ +//go:build cachesec + +package cachesec + +import ( + "context" + "encoding/base64" + + "github.com/nself-org/plugins/free/ci/internal/release/manifest" + cisigning "github.com/nself-org/plugins/free/ci/internal/signing" + "github.com/nself-org/plugins/free/ci/internal/store" +) + +// signerAnswer replays what the isolated signer process writes for a refused request (its stdout and +// exit status: the wire contract internal/signing's client tests pin). +type signerAnswer struct { + out string + code int +} + +func (a signerAnswer) RunStdio(context.Context, cisigning.StdioCall) ([]byte, int, error) { + return []byte(a.out), a.code, nil +} + +// coordinatorSign asks the coordinator's signer client, over the rig store, to sign a manifest of a +// finished run; the signer answers digest_not_committed. It returns the client's refusal. +func (r *rig) coordinatorSign() error { + r.running("release", owner()) + must(r.t, r.st.Transition(r.ctx, "release", "running", "finalizing", store.TransitionDetail{Epoch: 2})) + must(r.t, r.st.Transition(r.ctx, "release", "finalizing", "passed", store.TransitionDetail{Epoch: 3})) + answer := signerAnswer{out: `{"error":{"code":"E717","reason":"` + cisigning.ReasonDigestNotCommitted + `","message":"artifact app is not a committed record of the release attempt"}}`, code: 2} + c := &cisigning.Client{Runner: answer, Src: r.st, Audit: r.st.AppendAudit} + envs, err := c.Sign(r.ctx, "prelease", []cisigning.Item{{PayloadType: manifest.PayloadType, PayloadB64: base64.StdEncoding.EncodeToString([]byte(`{}`))}}) + if len(envs) != 0 { + r.t.Error("a refused signing request returned envelopes") + } + return err +} diff --git a/free/ci/test/cachesec/signing_test.go b/free/ci/test/cachesec/signing_test.go index fd63854c8..277fbfaec 100644 --- a/free/ci/test/cachesec/signing_test.go +++ b/free/ci/test/cachesec/signing_test.go @@ -53,7 +53,7 @@ func (r *rig) manifestKey(id string) error { rel := filepath.Join(dir, "release") must(r.t, os.Mkdir(rel, 0o700)) must(r.t, os.WriteFile(filepath.Join(rel, verify.ManifestFile), b, 0o600)) - res := verify.Release(r.ctx, rel, verify.Options{OperatorKeys: keys, OperatorRevoked: filepath.Join(dir, "none")}) + res := verify.Release(r.ctx, rel, verify.Options{OperatorKeys: keys, OperatorRevoked: filepath.Join(dir, "none"), Audit: r.st.AppendAudit}) if res.Verified { return nil } @@ -88,7 +88,8 @@ func (r *rig) roleRefusal(id string) error { return ask(token(principal, roles.Member)) } -// signUncommitted runs the signer's own refusal case for a digest that is not a committed record. +// signUncommitted runs the signer's own refusal case for a digest that is not a committed record, then +// hands that refusal, as the signer process answers it, to the coordinator's signer client. func (r *rig) signUncommitted() (error, string) { const test = "internal/signing TestSignerRefuses/artifact_digest_is_not_a_record" cmd := exec.Command("go", "test", "-mod=vendor", "-count=1", "-run", `^TestSignerRefuses$/^artifact_digest_is_not_a_record$`, "./internal/signing/") @@ -98,5 +99,5 @@ func (r *rig) signUncommitted() (error, string) { if err != nil { r.t.Fatalf("%s did not pass: %v\n%s", test, err, out) } - return &refusal{Code: "E717", Reason: "digest_not_committed", Msg: "refused by " + test}, "delegated: " + test + " (it asserts the refusal reason; the signer has no audit sink)" + return r.coordinatorSign(), "signer check delegated to " + test + "; the coordinator client received the signer's E717 answer and audited it" } diff --git a/free/ci/test/cachesec/testdata/results.json b/free/ci/test/cachesec/testdata/results.json index beb0387e1..0d554335c 100644 --- a/free/ci/test/cachesec/testdata/results.json +++ b/free/ci/test/cachesec/testdata/results.json @@ -204,14 +204,13 @@ "id": "secret-snapshot", "threat_row": "TM-12", "expected": "refused", - "outcome": "admitted", - "code": "", - "reason": "", + "outcome": "refused", + "code": "E711", + "reason": "secret_match", "audit": "cache.restore.refused", - "audit_found": false, - "match": false, - "platform": "darwin/arm64", - "note": "restore returned a miss (hit=false, 0 bytes restored), not a refusal" + "audit_found": true, + "match": true, + "platform": "darwin/arm64" }, { "id": "cache-secret-raw", @@ -245,8 +244,8 @@ "code": "E712", "reason": "secret_match", "audit": "artifact.collect.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { @@ -257,8 +256,8 @@ "code": "E712", "reason": "secret_match", "audit": "artifact.collect.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { @@ -269,8 +268,8 @@ "code": "E712", "reason": "digest_mismatch", "audit": "artifact.commit.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64", "note": "reason key read from the error text (the error type has no Reason field)" }, @@ -282,8 +281,8 @@ "code": "E712", "reason": "digest_mismatch", "audit": "artifact.commit.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64", "note": "reason key read from the error text (the error type has no Reason field)" }, @@ -295,8 +294,8 @@ "code": "E683", "reason": "dest.cwd", "audit": "artifact.get.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { @@ -307,8 +306,8 @@ "code": "E683", "reason": "dest.not_empty", "audit": "artifact.get.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { @@ -319,8 +318,8 @@ "code": "E683", "reason": "dest.under_git", "audit": "artifact.get.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { @@ -328,25 +327,24 @@ "threat_row": "TM-15", "expected": "refused", "outcome": "refused", - "code": "E710", - "reason": "digest_mismatch", + "code": "E711", + "reason": "blob_invalid", "audit": "cache.restore.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { "id": "partial-local", "threat_row": "TM-15", "expected": "refused", - "outcome": "admitted", - "code": "", - "reason": "", + "outcome": "refused", + "code": "E711", + "reason": "blob_invalid", "audit": "cache.restore.refused", - "audit_found": false, - "match": false, - "platform": "darwin/arm64", - "note": "restore returned a miss (hit=false, 0 bytes restored), not a refusal" + "audit_found": true, + "match": true, + "platform": "darwin/arm64" }, { "id": "corrupt-transit", @@ -356,8 +354,8 @@ "code": "E714", "reason": "digest_mismatch", "audit": "cache.remote.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { @@ -368,8 +366,8 @@ "code": "E714", "reason": "interrupted", "audit": "cache.remote.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { @@ -380,10 +378,9 @@ "code": "E714", "reason": "digest_mismatch", "audit": "cache.restore.refused", - "audit_found": false, - "match": false, - "platform": "darwin/arm64", - "note": "reason key read from the error text (the error type has no Reason field)" + "audit_found": true, + "match": true, + "platform": "darwin/arm64" }, { "id": "partial-readthrough", @@ -393,10 +390,9 @@ "code": "E714", "reason": "digest_mismatch", "audit": "cache.restore.refused", - "audit_found": false, - "match": false, - "platform": "darwin/arm64", - "note": "reason key read from the error text (the error type has no Reason field)" + "audit_found": true, + "match": true, + "platform": "darwin/arm64" }, { "id": "release-no-deps", @@ -404,10 +400,10 @@ "expected": "refused", "outcome": "refused", "code": "E715", - "reason": "deps_invalid", + "reason": "deps_failing", "audit": "release.admission.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { @@ -430,8 +426,8 @@ "code": "E686", "reason": "key_revoked", "audit": "evidence.quarantined", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { @@ -442,8 +438,8 @@ "code": "E719", "reason": "key_purpose", "audit": "release.verify.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { @@ -454,8 +450,8 @@ "code": "E719", "reason": "key_purpose", "audit": "release.verify.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { @@ -466,10 +462,10 @@ "code": "E717", "reason": "digest_not_committed", "audit": "release.sign.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64", - "note": "delegated: internal/signing TestSignerRefuses/artifact_digest_is_not_a_record (it asserts the refusal reason; the signer has no audit sink)" + "note": "signer check delegated to internal/signing TestSignerRefuses/artifact_digest_is_not_a_record; the coordinator client received the signer's E717 answer and audited it" }, { "id": "untrusted-promotion", @@ -479,8 +475,8 @@ "code": "E715", "reason": "trust", "audit": "release.admission.refused", - "audit_found": false, - "match": false, + "audit_found": true, + "match": true, "platform": "darwin/arm64" }, { diff --git a/free/ci/test/cachesec/transfer_test.go b/free/ci/test/cachesec/transfer_test.go index 62f8ce5c7..bda9e28e5 100644 --- a/free/ci/test/cachesec/transfer_test.go +++ b/free/ci/test/cachesec/transfer_test.go @@ -6,7 +6,6 @@ import ( "bytes" "context" "encoding/base64" - "io" "path/filepath" "strings" "time" @@ -19,11 +18,11 @@ import ( "github.com/nself-org/plugins/free/ci/internal/transport" ) -func (r *rig) remoteWrite(id string) error { - // Bind the real coordinator receiver to a durable remote lease. The agent - // carrier permits unordered refs; SSH preserves stream order. +// bindRemote binds the real coordinator receiver (cacheRemote.Handler) to a durable remote lease of +// a fresh attempt. The agent carrier permits unordered refs; SSH preserves stream order. restore, when +// set, is the lease plan's restore blob set. +func (r *rig) bindRemote(unordered bool, restore ...protocol.CacheBlob) (*cacheRemote.Lease, string) { f := owner() - r.running("remote", f) eng := lease.New(r.st, nil, lease.Config{}) // Grant needs a queued attempt, so use a separate attempt from the local rig. must(r.t, r.st.CreatePipeline(r.ctx, store.PipelineRow{ID: "pwire", Revision: strings.Repeat("a", 40), Project: f.Project, Trigger: "local", SourceTrust: f.Trust, PrivacyZone: "local-only", PolicyDigest: "policy", InputDigest: "input", SelectionMode: "full", Status: "queued"})) @@ -33,14 +32,23 @@ func (r *rig) remoteWrite(id string) error { must(r.t, err) ns := namespace(r.t, f) plan := &protocol.LeaseCache{Namespace: base64.RawURLEncoding.EncodeToString(ns.Canonical()), Access: "rw", Save: []protocol.CacheSave{{Spec: "go", Key: key(r.t, f), Dirs: []string{"go/data"}}}} - h := cacheRemote.Handler{Ledger: r.st, Entries: r.entries, Blobs: r.blobs} - l, err := h.Bind(cacheRemote.Binding{AttemptID: "remote-wire", LeaseID: "wire", Node: "node", Epoch: epoch, Facts: f, Plan: plan, Unordered: strings.HasSuffix(id, "agent"), Fence: func(ctx context.Context) error { + if len(restore) > 0 { + plan.Restore = []protocol.CacheRestore{{Spec: "go", Key: key(r.t, f), Dir: "go", Blobs: restore}} + } + h := &cacheRemote.Handler{Ledger: r.st, Entries: r.entries, Blobs: r.blobs} + l, err := h.Bind(cacheRemote.Binding{AttemptID: "remote-wire", LeaseID: "wire", Node: "node", Epoch: epoch, Facts: f, Plan: plan, Unordered: unordered, RestoreWait: 5 * time.Second, Fence: func(ctx context.Context) error { if e := eng.Fence(ctx, "wire", epoch); e != nil { return transfer.ErrStale } return nil }}) must(r.t, err) + return l, plan.Namespace +} + +func (r *rig) remoteWrite(id string) error { + r.running("remote", owner()) + l, _ := r.bindRemote(strings.HasSuffix(id, "agent")) if strings.HasPrefix(id, "ended-") { state, cas, e := r.st.AttemptStateEpoch(r.ctx, "remote-wire") must(r.t, e) @@ -62,12 +70,20 @@ func (r *rig) remoteWrite(id string) error { return err } +// transferBlob sends a corrupted or truncated blob to the coordinator's cache receiver of a live lease. func (r *rig) transferBlob(id string) error { + l, _ := r.bindRemote(false) + state, cas, err := r.st.AttemptStateEpoch(r.ctx, "remote-wire") + must(r.t, err) + must(r.t, r.st.Transition(r.ctx, "remote-wire", state, "running", store.TransitionDetail{Epoch: cas})) realm := cacheRealm(namespace(r.t, owner()).Canonical()) want := []byte("intact payload") - rcv := transfer.Receiver{Store: r.blobs, Realm: realm, Lease: "wire", Limits: transfer.Limits{MaxFile: 1 << 20}, Live: func(context.Context) error { return nil }} - w, err := rcv.Open(r.ctx, transport.BlobHeader{Stream: transfer.StreamArtifact, SHA256: digest(want), Size: int64(len(want))}) + bw, err := l.Blob(r.ctx, transport.BlobHeader{Stream: "cache", SHA256: digest(want), Size: int64(len(want))}) must(r.t, err) + w, ok := bw.(*transfer.Writer) + if !ok { + r.t.Fatalf("the cache receiver handed out a %T, not a transfer writer", bw) + } got := bytes.Clone(want) if id == "partial-transit" { got = got[:len(got)/2] @@ -85,27 +101,6 @@ func (r *rig) transferBlob(id string) error { f.Close() r.t.Error("invalid transfer became readable") } - return err -} - -func (r *rig) readThrough(id string) error { - dir := r.t.TempDir() - c, err := cacheRemote.NewReadThrough(dir, 4<<20) - must(r.t, err) - want := []byte("intact payload") - got := bytes.Clone(want) - if id == "partial-readthrough" { - got = got[:len(got)/2] - } else { - got[0] ^= 1 - } - f, err := c.Open(r.ctx, "realm", digest(want), int64(len(want)), func(_ context.Context, w io.Writer) error { _, e := w.Write(got); return e }) - if f != nil { - f.Close() - r.t.Error("invalid read-through copy was served") - } - if len(tree(r.t, dir)) != 0 { - r.t.Error("read-through published invalid bytes") - } + r.noEntries() return err }