fix: debt batch j36-plugins-debt-d-1427

free/ci/cmd/agent/exec.go

@@ -38,10 +38,11 @@ func init() {
 
 // execSpec is the job a hosted workflow or a test hands to `exec`.
 type execSpec struct {
-	AttemptID     string    `json:"attempt_id"`
-	CoordinatorID string    `json:"coordinator_id"`
-	JobID         string    `json:"job_id"`
-	Job           model.Job `json:"job"`
+	NetworkScope  model.NetworkScope `json:"network_scope,omitempty"`
+	AttemptID     string             `json:"attempt_id"`
+	CoordinatorID string             `json:"coordinator_id"`
+	JobID         string             `json:"job_id"`
+	Job           model.Job          `json:"job"`
 	// Revision is the full commit the job runs against (40 or 64 hex).
 	Revision  string `json:"revision"`
 	SourceDir string `json:"source_dir,omitempty"`
@@ -197,7 +198,10 @@ func execute(ctx context.Context, spec execSpec, in jobInputs) (ciexec.AttemptRe
 	if err != nil {
 		return ciexec.AttemptResult{}, "", err
 	}
-	job := spec.Job
+	job, err := agentJobImage(ctx, spec, cirun.Hooks{}.Resolved().Image)
+	if err != nil {
+		return ciexec.AttemptResult{Attempt: model.Attempt{ID: spec.AttemptID, State: "failed", FailureClass: "infra"}, ExitCode: -1, Started: now, Finished: time.Now(), Err: err}, err.Error(), nil
+	}
 	job.Env = map[string]string{}
 	for k, v := range spec.Job.Env {
 		job.Env[k] = v
@@ -217,7 +221,7 @@ func execute(ctx context.Context, spec execSpec, in jobInputs) (ciexec.AttemptRe
 	}
 	raw := &tailBuffer{max: excerptMax}
 	res := ex.Run(ctx, ciexec.JobSpec{AttemptID: spec.AttemptID, CoordinatorID: agentIncarnation(spec.CoordinatorID), Job: job, Command: job.Run,
-		Secrets: ex.Secrets, SourceDir: spec.SourceDir, Mounts: in.Mounts, Home: home, Slot: in.Slot, Stdin: in.Stdin, Masks: in.Masks, RunnerRoot: in.RunnerRoot, Entrypoint: in.Entrypoint,
+		Secrets: ex.Secrets, SourceDir: spec.SourceDir, Network: spec.NetworkScope, Mounts: in.Mounts, Home: home, Slot: in.Slot, Stdin: in.Stdin, Masks: in.Masks, RunnerRoot: in.RunnerRoot, Entrypoint: in.Entrypoint,
 		CPU: jobCPU(in), MemoryMB: jobMemoryMB(in), BeforeJob: cacheBeforeJob(spec, job.Env), AfterJob: agentAfterJob(spec, in),
 		Output: func(b []byte) {
 			raw.Write(b)

free/ci/cmd/agent/exec_floor_test.go

@@ -18,9 +18,14 @@ import (
 
 	ciexec "github.com/nself-org/plugins/free/ci/internal/exec"
 	"github.com/nself-org/plugins/free/ci/internal/model"
+	cirun "github.com/nself-org/plugins/free/ci/internal/run"
 )
 
 func TestRaisedJobWithoutImageIsRefusedNotRunOnTheHost(t *testing.T) {
+	calls := 0
+	old := cirun.Registered.Image
+	cirun.RegisterImage(missingImageHook{&calls, errors.New("no configured image")})
+	t.Cleanup(func() { cirun.RegisterImage(old) })
 	skipUnixOnly(t)
 	useState(t)
 	dir := t.TempDir()

free/ci/cmd/agent/exec_image.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"context"
+	ciexec "github.com/nself-org/plugins/free/ci/internal/exec"
+	"github.com/nself-org/plugins/free/ci/internal/model"
+	cirun "github.com/nself-org/plugins/free/ci/internal/run"
+)
+
+// A delivery raised to container isolation uses the local run's pinned-image hook.
+func agentJobImage(ctx context.Context, spec execSpec, hook cirun.ContainerImageHook) (model.Job, error) {
+	job := spec.Job
+	if job.Isolation != "container" && job.Isolation != "sandboxed-container" {
+		return job, nil
+	}
+	if job.Container != nil && job.Container.Image != "" {
+		return job, nil
+	}
+	var image string
+	var err error
+	if hook != nil {
+		image, err = hook.Image(ctx, cirun.AttemptInfo{AttemptID: spec.AttemptID, JobID: spec.JobID, Revision: spec.Revision, Job: job, Container: true})
+	}
+	if err != nil || image == "" {
+		return job, &ciexec.Error{Code: "E613", Message: "remote container job needs a pinned toolchain image; configure NSELF_CI_IMAGE on the agent", Err: err}
+	}
+	container := model.ContainerConfig{}
+	if job.Container != nil {
+		container = *job.Container
+	}
+	container.Image = image
+	job.Container = &container
+	return job, nil
+}

free/ci/cmd/agent/exec_image_test.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"github.com/nself-org/plugins/free/ci/internal/agent/session"
+	ciexec "github.com/nself-org/plugins/free/ci/internal/exec"
+	"github.com/nself-org/plugins/free/ci/internal/model"
+	"github.com/nself-org/plugins/free/ci/internal/protocol"
+	cirun "github.com/nself-org/plugins/free/ci/internal/run"
+	"strings"
+	"testing"
+)
+
+type missingImageHook struct {
+	calls *int
+	err   error
+}
+
+func (h missingImageHook) Image(context.Context, cirun.AttemptInfo) (string, error) {
+	*h.calls++
+	return "", h.err
+}
+
+func TestRaisedJobResolvesImageBeforeExecution(t *testing.T) {
+	useState(t)
+	missing := errors.New("test image unavailable")
+	for _, level := range []model.Isolation{"container", "sandboxed-container"} {
+		t.Run(string(level), func(t *testing.T) {
+			calls := 0
+			old := cirun.Registered.Image
+			cirun.RegisterImage(missingImageHook{&calls, missing})
+			t.Cleanup(func() { cirun.RegisterImage(old) })
+			spec := execSpec{AttemptID: "raised", Job: model.Job{Isolation: level}}
+			res, tail, err := execute(context.Background(), spec, jobInputs{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if calls != 1 || !errors.Is(res.Err, missing) || !strings.Contains(tail, "NSELF_CI_IMAGE") {
+				t.Fatalf("image resolution: calls=%d err=%v tail=%q", calls, res.Err, tail)
+			}
+		})
+	}
+}
+
+type deliveredSpecObserver struct {
+	got  *ciexec.JobSpec
+	stop error
+}
+
+func (o deliveredSpecObserver) Prepare(_ context.Context, spec *ciexec.JobSpec) error {
+	*o.got = *spec
+	return o.stop
+}
+
+type pinnedImageHook struct{}
+
+func (pinnedImageHook) Image(context.Context, cirun.AttemptInfo) (string, error) {
+	return "example.invalid/toolchain@sha256:" + strings.Repeat("a", 64), nil
+}
+
+func TestAgentReceivesImageAndLeaseNetwork(t *testing.T) {
+	useState(t)
+	for _, scope := range []model.NetworkScope{"none", "restricted", "internet"} {
+		t.Run(string(scope), func(t *testing.T) {
+			old := cirun.Registered
+			t.Cleanup(func() { cirun.Registered = old })
+			var got ciexec.JobSpec
+			stop := errors.New("stop before runtime")
+			cirun.RegisterIsolation(deliveredSpecObserver{&got, stop})
+			cirun.RegisterImage(pinnedImageHook{})
+			runner := leaseRunner{coordinator: "test"}
+			_, err := runner.Run(context.Background(), session.JobRun{Lease: protocol.Lease{AttemptID: "network", NetworkScope: scope, Checkout: protocol.Checkout{Revision: rev40}, Job: model.Job{Isolation: "container", Run: []string{"true"}}}})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got.Network != scope || got.Job.Container == nil || got.Job.Container.Image == "" || got.Job.Isolation != "container" {
+				t.Fatalf("delivered scope=%s image=%v isolation=%s", got.Network, got.Job.Container, got.Job.Isolation)
+			}
+		})
+	}
+}

free/ci/cmd/agent/serve.go

@@ -147,7 +147,7 @@ func (r leaseRunner) Run(ctx context.Context, j session.JobRun) (session.JobResu
 		// run on an empty tree would report a pass for code it never saw.
 		return session.JobResult{}, errors.New("the lease has a checkout but no hook staged it")
 	}
-	spec := execSpec{AttemptID: l.AttemptID, CoordinatorID: r.coordinator, JobID: "job", Job: l.Job, Revision: l.Checkout.Revision, SourceDir: j.Setup.SourceDir}
+	spec := execSpec{AttemptID: l.AttemptID, CoordinatorID: r.coordinator, JobID: "job", Job: l.Job, NetworkScope: l.NetworkScope, Revision: l.Checkout.Revision, SourceDir: j.Setup.SourceDir}
 	in := jobInputs{Slot: j.Slot, Env: j.Setup.Env, Secrets: j.Setup.Secrets, Mounts: j.Setup.Mounts, Output: j.Output, Tail: j.Tail, Skip: j.Skip}
 	if len(j.Credential) > 0 {
 		if err := wrapCredential(&spec, &in, j); err != nil {

free/ci/internal/artifacts/records/collect.go

@@ -202,6 +202,7 @@ func (c Collector) store(ctx context.Context, root *os.Root, realm cas.Realm, p
 	}
 	limit, reason := streamBudget(c.Policy, c.Usage, streamed)
 	sr := newScanReader(file, c.Scanner, f.rel, limit)
+	defer sr.Close()
 	sr.reason = reason
 	digest, size, err := c.Blobs.Put(ctx, realm, sr)
 	if err != nil {

free/ci/internal/artifacts/records/scan.go

@@ -14,6 +14,7 @@ package records
 import (
 	"errors"
 	"io"
+	"os"
 
 	"github.com/nself-org/plugins/free/ci/internal/authz"
 )
@@ -24,16 +25,17 @@ const (
 )
 
 type scanReader struct {
-	src     io.Reader
-	sc      authz.ValueScanner
-	name    string
-	max     int64
-	reason  string // the refusal when the stream outgrows max
-	read    int64
-	tail    []byte
-	out     []byte
-	err     error
-	matched bool
+	src        io.Reader
+	sc         authz.ValueScanner
+	name       string
+	max        int64
+	reason     string // the refusal when the stream outgrows max
+	read       int64
+	tail       []byte
+	out        []byte
+	err        error
+	compressed *os.File
+	matched    bool
 }
 
 func newScanReader(src io.Reader, sc authz.ValueScanner, name string, max int64) *scanReader {
@@ -69,17 +71,24 @@ func (r *scanReader) fill() {
 		return
 	}
 	if len(chunk) == 0 {
+		if err := r.scanCompressed(nil); err != nil {
+			r.err = err
+		}
 		return
 	}
 	if r.sc == nil {
 		r.err, r.matched = failf(ReasonSecretMatch, "no secret scanner is registered; %s is not stored", r.name), true
 		return
 	}
 	window := append(append(make([]byte, 0, len(r.tail)+len(chunk)), r.tail...), chunk...)
-	if len(r.sc.Scan(window)) != 0 {
+	if !authz.CleanCandidates(r.sc, window) {
 		r.err, r.matched = failf(ReasonSecretMatch, "%s holds a registered secret value; it is not stored", r.name), true
 		return
 	}
+	if err := r.scanCompressed(chunk); err != nil {
+		r.err = err
+		return
+	}
 	keep := min(scanOverlap, len(window))
 	r.tail = append(r.tail[:0], window[len(window)-keep:]...)
 	r.out = chunk

free/ci/internal/artifacts/records/scan_archive.go

@@ -0,0 +1,45 @@
+package records
+
+import (
+	"errors"
+	"github.com/nself-org/plugins/free/ci/internal/authz"
+	"io"
+	"os"
+)
+
+// Keep exactly the compressed bytes streamed to the blob; scan before its EOF commits it.
+func (r *scanReader) scanCompressed(chunk []byte) error {
+	if r.compressed == nil && r.read == int64(len(chunk)) && authz.Compressed(chunk) {
+		f, err := os.CreateTemp("", "nself-artifact-scan-*")
+		if err != nil {
+			return err
+		}
+		r.compressed = f
+	}
+	if r.compressed == nil {
+		return nil
+	}
+	if _, err := r.compressed.Write(chunk); err != nil {
+		return err
+	}
+	if r.err != io.EOF {
+		return nil
+	}
+	budget := int64(8 << 30)
+	err := authz.ScanArchive(r.compressed, r.read, r.sc, &budget)
+	if errors.Is(err, authz.ErrSecretContent) {
+		r.matched = true
+		return failf(ReasonSecretMatch, "%s holds a registered secret value; it is not stored", r.name)
+	}
+	if err != nil {
+		return failf(ReasonUnreadable, "%s could not be scanned", r.name)
+	}
+	return nil
+}
+
+func (r *scanReader) Close() {
+	if r.compressed != nil {
+		_ = r.compressed.Close()
+		_ = os.Remove(r.compressed.Name())
+	}
+}

free/ci/internal/attest/legacy_addr.go

@@ -0,0 +1,44 @@
+package attest
+
+import (
+	"net/netip"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var legacyIPv4 = regexp.MustCompile(`\b(?:0[xX][0-9a-fA-F]+|[0-9]+)(?:\.(?:0[xX][0-9a-fA-F]+|[0-9]+)){0,3}\b`)
+
+// inet-style IPv4 permits one to four components, including hex and octal.
+func parseLegacyIPv4(s string) (netip.Addr, bool) {
+	parts := strings.Split(s, ".")
+	if len(parts) > 4 {
+		return netip.Addr{}, false
+	}
+	var value uint64
+	for i, part := range parts {
+		bits := 8
+		if i == len(parts)-1 {
+			bits = 8 * (5 - len(parts))
+		}
+		base := 10
+		if strings.HasPrefix(part, "0x") || strings.HasPrefix(part, "0X") || len(part) > 1 && part[0] == '0' {
+			base = 0
+		}
+		n, err := strconv.ParseUint(part, base, bits)
+		if err != nil {
+			return netip.Addr{}, false
+		}
+		value = value<<bits | n
+	}
+	return netip.AddrFrom4([4]byte{byte(value >> 24), byte(value >> 16), byte(value >> 8), byte(value)}), true
+}
+
+func (p Profile) scrubLegacyIPv4(s string) string {
+	return legacyIPv4.ReplaceAllStringFunc(s, func(m string) string {
+		if a, ok := parseLegacyIPv4(m); ok && (privateAddr(a) || p.knownAddr(a)) {
+			return redactedIP
+		}
+		return m
+	})
+}

free/ci/internal/attest/profile.go

@@ -210,17 +210,17 @@ func parseLooseAddr(s string) (netip.Addr, bool) {
 	}
 	parts := strings.Split(s, ".")
 	if len(parts) != 4 {
-		return netip.Addr{}, false
+		return parseLegacyIPv4(s)
 	}
 	var b [4]byte
 	for i, part := range parts {
 		if part == "" || len(part) > 3 {
-			return netip.Addr{}, false
+			return parseLegacyIPv4(s)
 		}
 		n := 0
 		for _, c := range part {
 			if c < '0' || c > '9' {
-				return netip.Addr{}, false
+				return parseLegacyIPv4(s)
 			}
 			n = n*10 + int(c-'0')
 		}

free/ci/internal/attest/publish.go

@@ -37,6 +37,11 @@ func PublishWith(ev model.Evidence, p Profile) model.Evidence {
 	out.Runner.Provider = structured.Text(ev.Runner.Provider)
 	out.Runner.Platform = structured.Text(ev.Runner.Platform)
 	out.Pipeline.Name = structured.Text(ev.Pipeline.Name)
+	if ev.Binding.Reason != nil {
+		reason := model.BindingReason(p.Text(string(*ev.Binding.Reason)))
+		out.Binding.Reason = &reason
+	}
+	out.Selection.Reason = model.Reason(p.Text(string(ev.Selection.Reason)))
 	out.PlacementRef = scrubPtr(p, ev.PlacementRef)
 	// every other free string is scrubbed too: a hostile or careless producer can
 	// put a path or an address in any of them
@@ -70,6 +75,12 @@ func PublishWith(ev model.Evidence, p Profile) model.Evidence {
 		c.Tool = p.Text(c.Tool)
 		c.ToolVersion = scrubPtr(p, c.ToolVersion)
 		c.Excerpt = p.Text(c.Excerpt)
+		c.Diagnosis, c.Remediation = p.Text(c.Diagnosis), p.Text(c.Remediation)
+		if c.Flaky != nil {
+			flaky := *c.Flaky
+			flaky.Source = p.Text(flaky.Source)
+			c.Flaky = &flaky
+		}
 		c.LogRef = scrubPtr(p, c.LogRef)
 		out.Checks[i] = c
 	}
@@ -81,6 +92,7 @@ func PublishWith(ev model.Evidence, p Profile) model.Evidence {
 	out.Retries = make([]model.Retry, len(ev.Retries))
 	for i, r := range ev.Retries {
 		r.JobID = p.Text(r.JobID)
+		r.Diagnosis, r.Excerpt = p.Text(r.Diagnosis), p.Text(r.Excerpt)
 		out.Retries[i] = r
 	}
 	reason := "publication profile applied; recompute from the inputs"