fix(ci): lease network scope and evidence context checked end to end (D-1331, D-1299)

.github/wiki/guides/ci/Trust-Security-and-Privacy.md

@@ -149,7 +149,7 @@ A scope is offered only where it is enforced. The probe builds the same network
 ### Known limits
 
 - The exclusion set is the one this host can build for itself. A team coordinator that keeps the set in its own store can replace it with `run.RegisterNetscopeExclusions`; nothing registers one yet.
-- The scheduler reads the proof for the local runner. A remote node's `network.enforced_scopes` is not reported yet, so no untrusted job is placed on a remote node on the strength of it. The agent does not yet pass the lease's network scope into the executor (container jobs there run with `none`), and `netscope.Effective`, which refuses `lan` and `privileged` for an untrusted job, is called by the scheduler's admission rules only.
+- The scheduler reads the proof for the local runner. A remote node's `network.enforced_scopes` is not reported yet, so no untrusted job is placed on a remote node on the strength of it. A remote job's declared scope is checked against the class it was placed at three times: by the scheduler's admission rule, by the coordinator before it delivers the lease, and by the agent before it runs it. An untrusted job that declares `lan` or `privileged` fails there with E679 and never reaches the executor. A job that declares no scope runs as `none`, locally and on a node; the class default is not applied.
 - The proxy sees the destination host and port, not the encrypted content. It does not check the TLS server name, so a shared CDN host on the allowlist can be used for traffic to another site behind it, and uploading to an allowlisted registry is still an exfiltration path. `restricted` limits where a job can talk, not what it says.
 - Linux route discovery reads all routing tables through netlink, including policy routes. A public address that hairpins to a LAN service through the router cannot be learned from those tables. It still needs an observed egress address from the target network in the never-list; automatic public-NAT discovery is not implemented.
 - The sidecar bind-mounts a private staged copy of this process's binary with mode `0755`; the source binary's permissions are unchanged. The copy is removed with the attempt's network resources. Restricted and internet scopes still require Linux.
@@ -475,7 +475,7 @@ nself ci serve admin audit checkpoint verify --pubkey <fingerprint> <path> [--js
 
 `export` (admin, TEAM coordinator only; PERSONAL mode refuses with a reason) seals the log, refuses a damaged chain, signs the head of the newest row with the coordinator's `ci-audit` key (`audit-checkpoint.key`, mode 0600, created on the first export in the data directory) and prints the key's sha256 fingerprint. Copy the checkpoint file and the fingerprint off this host: the key sits in the same data directory as the database, so only a checkpoint kept elsewhere detects a rewrite by someone who can write that directory. `verify` takes the fingerprint you pinned elsewhere; it never trusts the key in the data directory, refuses to run without `--pubkey`, checks the signature, re-verifies the whole chain and then compares the head of the checkpointed row with the live one. It exits 1 when the chain was rewritten (even with every hash recomputed) or rows were removed. Rows added after the checkpoint do not break it.
 
-Handoff for later Tickets: the coordinator calls `run.UseAttestStore(st)` at start (the first attempt also sets it), a node revocation calls `run.RevokeNode`, and the coordinator that receives a result frame builds the signed-remote document with `producer.kind: signed-remote` and `signature: {kind: runner-ed25519, key_id, value}` from the frame; the agent signs the document it sends, so that document must already carry `attempt.id`, `job`, the real `repo_id`, `run_id`, `source_trust`, `policy_digest` and checks for its own job only (the agent's base document does not yet: D-1299; the marker file and the revoke caller are unwired: D-1300).
+Handoff for later Tickets: the coordinator calls `run.UseAttestStore(st)` at start (the first attempt also sets it), a node revocation calls `run.RevokeNode`, and the coordinator that receives a result frame builds the signed-remote document with `producer.kind: signed-remote` and `signature: {kind: runner-ed25519, key_id, value}` from the frame; the agent signs the document it sends, so that document must already carry `attempt.id`, `job`, the real `repo_id`, `run_id`, `source_trust`, `policy_digest` and checks for its own job only. The lease carries those facts as its `evidence` context, read from the coordinator's record of the attempt, and the agent repeats them unchanged (the marker file and the revoke caller are unwired: D-1300).
 
 ## Policy sources and `nself ci policy`
 

free/ci/cmd/agent/evidence_sign_e2e_test.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/nself-org/plugins/free/ci/internal/agent/nodekey"
+	"github.com/nself-org/plugins/free/ci/internal/agent/session"
+	"github.com/nself-org/plugins/free/ci/internal/attest"
+	"github.com/nself-org/plugins/free/ci/internal/model"
+	"github.com/nself-org/plugins/free/ci/internal/protocol"
+	cirun "github.com/nself-org/plugins/free/ci/internal/run"
+	"github.com/nself-org/plugins/free/ci/internal/store"
+)
+
+// signedRig is a coordinator store with one signing-eligible node whose pinned key is the agent's node key, and
+// a source tree at a real commit the attempts run against.
+type signedRig struct {
+	*pullRig
+	key  *nodekey.Key
+	repo string
+	head string
+}
+
+func newSignedRig(t *testing.T) *signedRig {
+	t.Helper()
+	skipUnixOnly(t)
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git not available")
+	}
+	state := useState(t)
+	r := &signedRig{pullRig: newPullRig(t), repo: t.TempDir()}
+	r.enroll(state, "n-1", "https://127.0.0.1:1") // the laptop fixture is signing eligible
+	var err error
+	if r.key, err = nodekey.LoadOrCreate(state); err != nil {
+		t.Fatal(err)
+	}
+	if err = r.st.PKINodeKeyPut(context.Background(), "n-1", r.key.KeyID(), r.key.PublicString()); err != nil {
+		t.Fatal(err)
+	}
+	git(t, r.repo, "init", "-q")
+	if err = os.WriteFile(filepath.Join(r.repo, "f.txt"), []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	git(t, r.repo, "add", "f.txt")
+	git(t, r.repo, "commit", "-q", "-m", "a")
+	r.head = git(t, r.repo, "rev-parse", "HEAD")
+	return r
+}
+
+// attempt records a running attempt of job "build" in project proj-1 at the rig's commit, leased to n-1.
+func (r *signedRig) attempt(n string) string {
+	r.t.Helper()
+	ctx := context.Background()
+	p := store.PipelineRow{ID: "p" + n, Project: "proj-1", Revision: r.head, Trigger: "local", SourceTrust: "owner", PrivacyZone: "local-only", PolicyDigest: "policy-" + n,
+		InputDigest: "i", SelectionMode: "full", Status: "queued"}
+	for _, f := range []func() error{
+		func() error { return r.st.CreatePipeline(ctx, p) },
+		func() error {
+			return r.st.CreateJob(ctx, store.JobRow{ID: "j" + n, PipelineID: p.ID, Name: "build", Kind: "test", Idempotent: true, InfraMax: 1, InputDigest: "i"})
+		},
+		func() error { return r.st.CreateAttempt(ctx, store.AttemptRow{ID: "a" + n, JobID: "j" + n, N: 1}, "") },
+	} {
+		if err := f(); err != nil {
+			r.t.Fatal(err)
+		}
+	}
+	if _, err := r.eng.Grant(ctx, "host", store.Demand{AttemptID: "a" + n, LeaseID: "l" + n, RunnerID: "n-1", CPU: 1, MemMB: 1, CapacityCPU: 100, CapacityMemMB: 100, TTL: r.leaseTTL()}, 1); err != nil {
+		r.t.Fatal(err)
+	}
+	return "a" + n
+}
+
+// run runs the lease through the agent's lease path and returns the frame's evidence bytes (canonical JSON, as the
+// session sends them) and the node's signature over them.
+func (r *signedRig) run(l protocol.Lease) (frame []byte, keyID, sig string) {
+	r.t.Helper()
+	l.Checkout, l.Job = protocol.Checkout{Revision: r.head}, model.Job{Kind: "static", Run: []string{"true"}}
+	res, err := leaseRunner{coordinator: "coord"}.Run(context.Background(), session.JobRun{Lease: l, Setup: session.JobSetup{SourceDir: r.repo}})
+	if err != nil {
+		r.t.Fatal(err)
+	}
+	dec := json.NewDecoder(bytes.NewReader(res.Evidence))
+	dec.UseNumber()
+	var v any
+	if err = dec.Decode(&v); err != nil {
+		r.t.Fatal(err)
+	}
+	if frame, err = json.Marshal(v); err != nil {
+		r.t.Fatal(err)
+	}
+	s := evidenceSigner{key: r.key}
+	raw, err := s.Sign(frame)
+	if err != nil {
+		r.t.Fatal(err)
+	}
+	return frame, s.KeyID(), base64.RawStdEncoding.EncodeToString(raw)
+}
+
+func (r *signedRig) verify(frame []byte, keyID, sig string) attest.RemoteReason {
+	err := attest.VerifyFrame(func() attest.Ledger { return store.AttestLedger{S: r.st} }, frame, keyID, sig)
+	if err == nil {
+		return ""
+	}
+	var re *attest.RemoteError
+	if !errors.As(err, &re) {
+		r.t.Fatalf("VerifyFrame = %v", err)
+	}
+	return re.Reason
+}
+
+// D-1299 end to end: the coordinator puts the attempt's recorded facts into the lease (run.LeaseEvidence, the
+// read the delivery path makes), the agent runs the lease against a real checkout and names them in its
+// evidence, the node key signs the frame bytes, and the coordinator's frame check (attest.VerifyFrame with the
+// real store ledger) accepts it. A lease without the context (the agent's own run id, repo unknown, source
+// untrusted) and a context the node altered are both refused.
+func TestSignedRemoteEvidenceVerifiesAgainstTheCoordinatorRecord(t *testing.T) {
+	r := newSignedRig(t)
+	ctx := context.Background()
+
+	bare := r.attempt("1")
+	if reason := r.verify(r.run(protocol.Lease{AttemptID: bare, Trust: "owner"})); reason != attest.RemoteContextMismatch {
+		t.Fatalf("evidence without the lease's context: reason %q, want %s", reason, attest.RemoteContextMismatch)
+	}
+
+	forged := r.attempt("2")
+	ev, err := cirun.LeaseEvidence(ctx, r.st, forged)
+	if err != nil || ev == nil {
+		t.Fatalf("LeaseEvidence = %v, %v", ev, err)
+	}
+	ev.PolicyDigest = "a-policy-the-node-prefers"
+	if reason := r.verify(r.run(protocol.Lease{AttemptID: forged, Trust: "owner", Evidence: ev})); reason != attest.RemoteContextMismatch {
+		t.Fatalf("evidence with an altered context: reason %q, want %s", reason, attest.RemoteContextMismatch)
+	}
+
+	good := r.attempt("3")
+	if ev, err = cirun.LeaseEvidence(ctx, r.st, good); err != nil || ev == nil {
+		t.Fatalf("LeaseEvidence = %v, %v", ev, err)
+	}
+	frame, keyID, sig := r.run(protocol.Lease{AttemptID: good, Trust: "owner", Evidence: ev})
+	if reason := r.verify(frame, keyID, sig); reason != "" {
+		t.Fatalf("signed evidence for the recorded attempt was refused: %s\n%s", reason, frame)
+	}
+	var doc model.Evidence
+	if err = json.Unmarshal(frame, &doc); err != nil {
+		t.Fatal(err)
+	}
+	if doc.Attempt == nil || doc.Attempt.ID != good || doc.Job == nil || *doc.Job != "build" || doc.Binding.State != "bound" || doc.RepoID != ev.RepoID {
+		t.Fatalf("the signed document does not name the attempt it ran: %s", frame)
+	}
+}

free/ci/cmd/agent/exec.go

@@ -28,6 +28,8 @@ import (
 	"github.com/nself-org/plugins/free/ci/internal/evidence"
 	ciexec "github.com/nself-org/plugins/free/ci/internal/exec"
 	"github.com/nself-org/plugins/free/ci/internal/model"
+	"github.com/nself-org/plugins/free/ci/internal/netscope"
+	"github.com/nself-org/plugins/free/ci/internal/protocol"
 	"github.com/nself-org/plugins/free/ci/internal/report"
 	cirun "github.com/nself-org/plugins/free/ci/internal/run"
 )
@@ -38,14 +40,18 @@ func init() {
 
 // execSpec is the job a hosted workflow or a test hands to `exec`.
 type execSpec struct {
-	NetworkScope  model.NetworkScope `json:"network_scope,omitempty"`
-	AttemptID     string             `json:"attempt_id"`
-	CoordinatorID string             `json:"coordinator_id"`
-	JobID         string             `json:"job_id"`
-	Job           model.Job          `json:"job"`
+	NetworkScope model.NetworkScope `json:"network_scope,omitempty"`
+	// Trust is the class the coordinator admitted the job at (the lease's); with no class the job is untrusted.
+	Trust         model.TrustClass `json:"trust,omitempty"`
+	AttemptID     string           `json:"attempt_id"`
+	CoordinatorID string           `json:"coordinator_id"`
+	JobID         string           `json:"job_id"`
+	Job           model.Job        `json:"job"`
 	// Revision is the full commit the job runs against (40 or 64 hex).
 	Revision  string `json:"revision"`
 	SourceDir string `json:"source_dir,omitempty"`
+	// Evidence is the lease's evidence context: what the coordinator recorded for the attempt (D-1299).
+	Evidence *protocol.LeaseEvidence `json:"evidence,omitempty"`
 }
 
 // jobInputs are what a lease adds to its spec.
@@ -172,6 +178,9 @@ func runSpec(ctx context.Context, spec execSpec, in jobInputs) (session.JobResul
 		} else {
 			unbindIfOtherRevision(ctx, spec, &done)
 		}
+		if spec.Evidence != nil { // the builder names the checkout's own repo id; the signed document names the coordinator's
+			done.RepoID = base.RepoID
+		}
 	} else {
 		unbind(&done, "dirty", "the job ran without a source tree")
 	}
@@ -202,6 +211,12 @@ func execute(ctx context.Context, spec execSpec, in jobInputs) (ciexec.AttemptRe
 	if err != nil {
 		return ciexec.AttemptResult{Attempt: model.Attempt{ID: spec.AttemptID, State: "failed", FailureClass: "infra"}, ExitCode: -1, Started: now, Finished: time.Now(), Err: err}, err.Error(), nil
 	}
+	// The lease's scope is checked again here, with the lease's class: the scheduler's admission rule is not the
+	// only thing between an untrusted job and the LAN (D-1331). An undeclared scope runs as none.
+	scope, err := netscope.RunScope(spec.Trust, spec.NetworkScope)
+	if err != nil {
+		return ciexec.AttemptResult{Attempt: model.Attempt{ID: spec.AttemptID, State: "failed", FailureClass: "infra"}, ExitCode: -1, Started: now, Finished: time.Now(), Err: err}, err.Error(), nil
+	}
 	job.Env = map[string]string{}
 	for k, v := range spec.Job.Env {
 		job.Env[k] = v
@@ -223,7 +238,7 @@ func execute(ctx context.Context, spec execSpec, in jobInputs) (ciexec.AttemptRe
 	}
 	raw := &tailBuffer{max: excerptMax}
 	res := ex.Run(ctx, ciexec.JobSpec{AttemptID: spec.AttemptID, CoordinatorID: agentIncarnation(spec.CoordinatorID), Job: job, Command: job.Run,
-		Secrets: ex.Secrets, SourceDir: spec.SourceDir, Network: spec.NetworkScope, Mounts: in.Mounts, Home: home, Slot: in.Slot, Stdin: in.Stdin, Masks: in.Masks, RunnerRoot: in.RunnerRoot, Entrypoint: in.Entrypoint,
+		Secrets: ex.Secrets, SourceDir: spec.SourceDir, Network: scope, Mounts: in.Mounts, Home: home, Slot: in.Slot, Stdin: in.Stdin, Masks: in.Masks, RunnerRoot: in.RunnerRoot, Entrypoint: in.Entrypoint,
 		CPU: jobCPU(in), MemoryMB: jobMemoryMB(in), BeforeJob: cacheBeforeJob(spec, job.Env), AfterJob: agentAfterJob(spec, in),
 		Output: func(b []byte) {
 			raw.Write(b)

free/ci/cmd/agent/lease_network_test.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/nself-org/plugins/free/ci/internal/agent/session"
+	ciexec "github.com/nself-org/plugins/free/ci/internal/exec"
+	"github.com/nself-org/plugins/free/ci/internal/invariants"
+	"github.com/nself-org/plugins/free/ci/internal/model"
+	"github.com/nself-org/plugins/free/ci/internal/protocol"
+	cirun "github.com/nself-org/plugins/free/ci/internal/run"
+	"github.com/nself-org/plugins/free/ci/internal/trust"
+)
+
+// leaseScopeResult is what the executor saw for one (class, scope) lease: the scope it was handed, or that it
+// was never called because the agent refused the lease first.
+type leaseScopeResult struct {
+	ran     bool
+	network model.NetworkScope
+	state   string
+	excerpt string
+}
+
+// runLeaseScope runs one container lease of class and declared scope through the agent's lease path, up to the
+// isolator (which records the JobSpec and stops), and reports what happened.
+func runLeaseScope(t *testing.T, class model.TrustClass, scope model.NetworkScope) leaseScopeResult {
+	t.Helper()
+	var got ciexec.JobSpec
+	cirun.RegisterIsolation(deliveredSpecObserver{&got, errors.New("stop before runtime")})
+	cirun.RegisterImage(pinnedImageHook{})
+	lease := protocol.Lease{AttemptID: "scope", Trust: class, NetworkScope: scope, Checkout: protocol.Checkout{Revision: rev40},
+		Job: model.Job{Isolation: "container", Run: []string{"true"}}}
+	res, err := leaseRunner{coordinator: "test"}.Run(context.Background(), session.JobRun{Lease: lease})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return leaseScopeResult{ran: got.AttemptID != "", network: got.Network, state: res.Outcome.State, excerpt: string(res.Evidence)}
+}
+
+// TestInvariant_I07 points I07 (untrusted never gets LAN) at the agent's lease path (D-1331): for every world of
+// the invariant catalogue, a lease that declares lan or privileged for an untrusted job never reaches the
+// executor (the attempt fails with E679), any other declared scope reaches it unchanged, and an undeclared scope
+// runs as none. The worlds repeat (class, scope) pairs, so each pair runs once.
+func TestInvariant_I07(t *testing.T) {
+	useState(t)
+	old := cirun.Registered
+	t.Cleanup(func() { cirun.Registered = old })
+	seen := map[string]leaseScopeResult{}
+	invariants.Check(t, func(w invariants.World) error {
+		key := string(w.Trust) + "/" + string(w.Network)
+		r, ok := seen[key]
+		if !ok {
+			r = runLeaseScope(t, w.Trust, w.Network)
+			seen[key] = r
+		}
+		untrusted := trust.DecodeTrust(string(w.Trust)) == "untrusted"
+		lan := w.Network == "lan" || w.Network == "privileged"
+		switch {
+		case untrusted && lan && r.ran:
+			return fmt.Errorf("an untrusted lease declaring %s reached the executor with %q", w.Network, r.network)
+		case untrusted && lan && (r.state != "failed" || !strings.Contains(r.excerpt, "E679")):
+			return fmt.Errorf("an untrusted lease declaring %s ended %q without E679", w.Network, r.state)
+		case untrusted && lan:
+			return nil
+		case !r.ran:
+			return fmt.Errorf("a %s lease declaring %q never reached the executor", w.Trust, w.Network)
+		case w.Network == "" && r.network != "none":
+			return fmt.Errorf("an undeclared scope ran as %q, want none", r.network)
+		case w.Network != "" && r.network != w.Network:
+			return fmt.Errorf("a %s lease declaring %s ran as %q", w.Trust, w.Network, r.network)
+		}
+		return nil
+	})
+}

free/ci/cmd/agent/main.go

@@ -102,19 +102,27 @@ func jobOutcome(spec execSpec, res ciexec.AttemptResult, tail string) cirun.JobO
 // baseEvidence is the run document before the P7-CI-38 builder completes it:
 // one check for the one job, assembled by the run package so the check and
 // verdict logic exists once, with the producer marked self-reported until the
-// result signer (P7-TRUST-20) signs it.
+// result signer (P7-TRUST-20) signs it. A lease's evidence context (D-1299)
+// supplies the run, repository, source trust, policy digest and selection mode
+// the coordinator recorded, so the signed document verifies; without one the
+// agent claims nothing (a run id of its own, repo unknown, source untrusted).
 func baseEvidence(spec execSpec, o cirun.JobOutcome, start, end time.Time) model.Evidence {
 	id := sha256.Sum256([]byte(spec.CoordinatorID + "/" + spec.AttemptID))
-	runID := hex.EncodeToString(id[:8])
-	return cirun.SingleJobEvidence(cirun.SingleJobInput{
-		RunID: runID, Revision: spec.Revision, RepoID: "unknown",
+	runID, repoID, source, mode, policy := hex.EncodeToString(id[:8]), "unknown", "untrusted", model.SelectionMode("full"), ""
+	if c := spec.Evidence; c != nil {
+		runID, repoID, source, mode, policy = c.RunID, c.RepoID, c.SourceTrust, model.SelectionMode(c.SelectionMode), c.PolicyDigest
+	}
+	ev := cirun.SingleJobEvidence(cirun.SingleJobInput{
+		RunID: runID, Revision: spec.Revision, RepoID: repoID,
 		Pipeline: model.PipelineRef{ID: runID, Name: spec.JobID, Kind: "adhoc"}, Trigger: "api",
-		Source:    "untrusted", // the coordinator classifies the source; the agent claims nothing
-		Selection: model.Selection{Mode: "full", Reason: "selection.full_required"},
+		Source:    source,
+		Selection: model.Selection{Mode: mode, Reason: "selection.full_required"},
 		Runner:    model.RunnerRef{ID: "agent", Provider: "agent", Platform: runtime.GOOS + "/" + runtime.GOARCH, Isolation: "process"},
 		Producer:  model.Producer{Kind: "self-reported", Version: "1"},
 		Job:       o, Start: start, End: end,
 	})
+	ev.PolicyDigest = policy
+	return ev
 }
 
 // unbindIfOtherRevision unbinds evidence whose named revision is not the commit