fix(ci): local source trust: rewrites need a source made here; PR refspec globs and grafts (D-1322, D-1431)

.github/wiki/guides/ci/Trust-Security-and-Privacy.md

@@ -289,12 +289,12 @@ A missing fact takes its untrusted value. A first-time contributor's run needs a
 
 **A local run** is classified from git, by provenance and never by author name or email (anyone can write your email into a commit). Git runs with `--no-replace-objects`.
 
-- `owner`: HEAD is on the `origin` default branch; or the reflogs show that every commit that branch does not contain was made here. The oldest reflog entry of a commit decides: `commit`, `commit (amend)`, a rebase step that rewrites a commit (also under `pull --rebase`), a merge commit, `cherry-pick` and `revert` count; `checkout`, `fetch`, `pull` that fast-forwards, `branch: Created from`, `rebase (finish)` and anything that fast-forwards do not. Pushing your own commits changes nothing: you still made them. A stranger's commit you fetched or checked out has an arrival entry, and a branch you fetched back from a peer keeps your `commit` entries.
+- `owner`: HEAD is on the `origin` default branch; or the reflogs show that every commit that branch does not contain was made here. The oldest reflog entry of a commit decides: `commit`, `commit (amend)`, a rebase step that rewrites a commit (also under `pull --rebase`), a merge commit, `cherry-pick` and `revert` count; `checkout`, `fetch`, `pull` that fast-forwards, `branch: Created from`, `rebase (finish)` and anything that fast-forwards do not. Pushing your own commits changes nothing: you still made them. A stranger's commit you fetched or checked out has an arrival entry, and a branch you fetched back from a peer keeps your `commit` entries. A rewrite (a rebase step, `commit (amend)`, `cherry-pick`) counts only when its change has a source made here: its patch (added and removed lines, whitespace and positions ignored) equals that of a commit made here (or one on the default branch), or, for an amend, the commit it amended (its older reflog neighbour with the same parents) counts. A rewrite of someone else's commit never counts, whether or not anything still refers to the original.
 - `collaborator`: HEAD is on another branch of `origin`. Pull-request namespaces (`refs/remotes/origin/pr/*`, `pull/*`, `mr/*` and any refspec that maps something other than `refs/heads/*`) do not count as branches.
 - `untrusted`: anything else, including a stranger's pull request checked out with `gh pr checkout`, an unfinished merge, squash, cherry-pick, revert or rebase (the tree holds content no commit accounts for), and any case where git could not answer.
 - A repository with no remote, a directory that is not a git repository, and a repository without a commit are your own directory: `owner`. A remote that is configured but never fetched is judged by the reflog like a branch with no default.
 
-Limits, so you can judge them: the reflog expires and is read for the last 5000 entries, so an old branch (or more than 500 commits beyond the default branch) may fall to `untrusted`; run `git remote set-head origin -a` so the default branch is known when it is not `main` or `master`. A commit you rebase, amend or cherry-pick is a new commit made here, even when its content came from someone else's branch, so rebasing a stranger's pull request before you test it makes it `owner` (patch-id matching against remote commits is the planned fix, see the debt register). Uncommitted changes count as yours. A repository you extracted from an archive, or cloned from a stranger, is whatever its `origin` says it is. Run `nself ci --trust owner` when you know better, and use an isolated runner for code you did not write.
+Limits, so you can judge them: the reflog expires and is read for the last 5000 entries, so an old branch (or more than 500 commits beyond the default branch) may fall to `untrusted`; run `git remote set-head origin -a` so the default branch is known when it is not `main` or `master`. Rebasing, amending or cherry-picking a stranger's pull request does not make it `owner`: the rewritten commit is reported with `local.rewrite_not_made_here`. Fail-closed costs: a rebase step whose conflicts you resolved, a squash or fixup, your own commits rewritten on another machine, and an amend whose original entry has expired have no source made here and need `--trust owner`. A grafts file (`.git/info/grafts` or `$GIT_GRAFT_FILE`) can rewrite parents, so a checkout that has one is `untrusted` (`local.history_grafted`); a fetch refspec that maps anything but `refs/heads/*` into `refs/remotes/origin/` (a glob anywhere in its destination) is never an origin branch. Content you commit yourself (a `cherry-pick -n` then `commit`, a `merge --squash` then `commit`) counts as yours. Uncommitted changes count as yours. A repository you extracted from an archive, or cloned from a stranger, is whatever its `origin` says it is. Run `nself ci --trust owner` when you know better, and use an isolated runner for code you did not write.
 
 **`--trust owner`** forces `owner` and writes one audit row (`trust.override`) that names the revision, the verdict it replaced and its reasons. If the row cannot be written the run is refused (`E625`). `--trust` with a lower class always works; it never raises a class except to `owner`.
 

free/ci/internal/run/trust_git.go

@@ -7,8 +7,10 @@ package run
 // Constraints: git runs with --no-replace-objects (a `git replace` ref must not rewrite the history
 //          under test); only refs/remotes/origin/ names the default branch; every FETCH_HEAD line counts,
 //          not just the first; an unfinished merge, squash, cherry-pick, revert or rebase is reported;
-//          a git failure is listed in Missing, never read as an answer; the reflog is read from every
-//          log (a linked worktree has a HEAD log of its own) and each commit keeps its OLDEST entry.
+//          a git failure is listed in Missing, never read as an answer; a grafts file (info/grafts or
+//          $GIT_GRAFT_FILE) rewrites parents, so its presence is MissingHistory; the reflog is read from every
+//          log (a linked worktree has a HEAD log of its own) and each commit keeps its OLDEST entry; a
+//          rewritten commit's patch is compared with the commits that arrived (trust_git_patch.go).
 
 import (
 	"context"
@@ -67,6 +69,7 @@ func (h SourceTrust) gather(ctx context.Context, root, rev string) source.GitFac
 	var symref string
 	var refsOK, remotesOK, pathsOK bool
 	var entries []source.ReflogEntry
+	var amends map[string][]string
 	run(func() {
 		out, exit := git("for-each-ref", "--format=%(refname)", "refs/remotes", "refs/pull")
 		refs, refsOK = lines(out), exit == 0
@@ -79,15 +82,15 @@ func (h SourceTrust) gather(ctx context.Context, root, rev string) source.GitFac
 	})
 	run(func() {
 		out, exit := git("rev-parse", "--git-path", "FETCH_HEAD", "--git-path", "SQUASH_MSG", "--git-path", "MERGE_HEAD", "--git-path", "CHERRY_PICK_HEAD",
-			"--git-path", "REVERT_HEAD", "--git-path", "rebase-merge", "--git-path", "rebase-apply")
-		paths, pathsOK = lines(out), exit == 0 && len(lines(out)) == 7
+			"--git-path", "REVERT_HEAD", "--git-path", "rebase-merge", "--git-path", "rebase-apply", "--git-path", "info/grafts")
+		paths, pathsOK = lines(out), exit == 0 && len(lines(out)) == 8
 	})
 	run(func() {
 		out, exit := git("reflog", "show", "--all", "--date=unix", "--format=%H%x09%gd%x09%gs", "-n", "5000")
 		if exit != 0 {
 			miss(source.MissingReflog)
 		}
-		entries = reflogEntries(out)
+		entries, amends = reflogEntries(out), amendPreImages(out)
 	})
 	wg.Wait()
 	if g.NotARepo {
@@ -101,11 +104,14 @@ func (h SourceTrust) gather(ctx context.Context, root, rev string) source.GitFac
 		if !ok {
 			pathsOK = false
 		}
-		for _, m := range paths[1:] {
+		for _, m := range paths[1:7] {
 			if _, err := os.Stat(rootPath(root, m)); err == nil {
 				g.InProgress = true
 			}
 		}
+		if h.grafted(root, paths[7]) {
+			miss(source.MissingHistory)
+		}
 	}
 	if !pathsOK || !refsOK || !remotesOK {
 		miss(source.MissingRemoteRefs)
@@ -154,6 +160,7 @@ func (h SourceTrust) gather(ctx context.Context, root, rev string) source.GitFac
 		run(func() { g.OffDefault = listed("rev-list", "-n", "501", rev) })
 	}
 	wg.Wait()
+	replayedPatches(git, &g, amends)
 	return g
 }
 
@@ -172,7 +179,8 @@ func defaultRef(symref string, refs []string) string {
 }
 
 // excludedRemoteDirs lists the refs/remotes/origin/ prefixes that hold pull requests: the usual names
-// and every destination the origin's fetch refspecs map from something other than refs/heads/*.
+// and the fixed part (up to the first glob) of every destination the origin's fetch refspecs map from
+// something other than refs/heads/* (D-1431: refs/remotes/origin/gh-pr/*/head).
 func excludedRemoteDirs(refspecs []string) []string {
 	var skip []string
 	for _, d := range prDirs {
@@ -181,7 +189,8 @@ func excludedRemoteDirs(refspecs []string) []string {
 	for _, spec := range refspecs {
 		src, dst, ok := strings.Cut(strings.TrimPrefix(spec, "+"), ":")
 		if ok && !strings.HasPrefix(src, "refs/heads/") {
-			skip = append(skip, strings.TrimSuffix(dst, "*"))
+			prefix, _, _ := strings.Cut(dst, "*") // a glob anywhere in the destination: everything below its fixed part
+			skip = append(skip, prefix)
 		}
 	}
 	return skip

free/ci/internal/run/trust_git_patch.go

@@ -0,0 +1,222 @@
+package run
+
+// Purpose: the patch half of local provenance (D-1322): a rewritten commit of a run (oldest reflog entry a
+// rebase step, an amend or a cherry-pick) is created here only when its change has a source made here, so a
+// replay of someone else's change is never counted as created here, even when nothing refers to the
+// original any more (a deleted branch, an overwritten FETCH_HEAD, an expired reflog).
+// Inputs:  the gathered GitFacts (OffDefault, ReflogCreated, DefaultRemoteRef, Head) and the amend pre-images
+//          read from the reflog (amendPreImages).
+// Outputs: GitFacts.ReplayedPatch, or MissingPatches when the comparison could not be made; grafted.
+// Constraints: a source made here is a commit whose oldest entry is a creation other than a rewrite, a
+//          rewrite already counted as made here, or a commit of the default branch that Head does not have;
+//          a rewrite counts when its patch equals a source's, or (an amend) when the commit it amended
+//          counts; a commit with no change lines adds no code and counts. The patch id ignores line numbers,
+//          context and whitespace, so a replay onto a moved base still matches. Known cost, fail closed: a
+//          rebase step whose conflicts you resolved, or a squash, changes the patch and is not counted.
+//          More than maxPatchCommits commits on a side, or a git failure, is MissingPatches.
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io/fs"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+
+	"github.com/nself-org/plugins/free/ci/internal/trust/source"
+)
+
+// maxPatchCommits bounds the commits whose patches are compared on one side.
+const maxPatchCommits = 5000
+
+// patchLog is the log format both sides are read with: one NUL-led line with the commit id and its parents,
+// then its patch.
+var patchLog = []string{"-p", "--no-color", "--full-index", "--no-ext-diff", "--no-textconv", "--no-renames", "--format=%x00%H%x09%P"}
+
+type gitFn func(args ...string) (string, int)
+
+// replayedPatches fills g.ReplayedPatch with the rewritten commits of g.OffDefault that have no source made here.
+func replayedPatches(git gitFn, g *source.GitFacts, amends map[string][]string) {
+	var rewritten, made []string
+	for _, c := range g.OffDefault {
+		if source.Rewrite(g.ReflogCreated[c.SHA]) {
+			rewritten = append(rewritten, c.SHA)
+		}
+	}
+	if len(rewritten) == 0 {
+		return
+	}
+	for sha, subject := range g.ReflogCreated {
+		if source.MadeHere(subject) {
+			made = append(made, sha)
+		}
+	}
+	slices.Sort(made) // a stable argv: map order is random
+	ids, parents, ok := logPatches(git, maxPatchCommits, append([]string{"--ignore-missing", "--no-walk=unsorted"}, made...))
+	for _, sha := range rewritten {
+		if _, have := ids[sha]; !have {
+			ok = false
+		}
+	}
+	if !ok {
+		g.Missing = append(g.Missing, source.MissingPatches)
+		return
+	}
+	own, created := map[string]bool{}, map[string]bool{}
+	for _, sha := range made {
+		if !source.Rewrite(g.ReflogCreated[sha]) {
+			created[sha], own[ids[sha]] = true, true
+		}
+	}
+	settle := func() bool { // true when every rewritten commit of the run counts as made here
+		for changed := true; changed; {
+			changed = false
+			for _, sha := range made {
+				subject, id := g.ReflogCreated[sha], ids[sha]
+				if created[sha] || !source.Rewrite(subject) {
+					continue
+				}
+				if id == "" || own[id] || amendedOwn(subject, sha, amends, parents, created) {
+					created[sha], own[id], changed = true, true, true
+				}
+			}
+		}
+		return !slices.ContainsFunc(rewritten, func(sha string) bool { return !created[sha] })
+	}
+	if !settle() && g.DefaultRemoteRef != "" { // a backport: the default branch's own changes are sources too
+		def, _, fine := logPatches(git, maxPatchCommits, []string{"--no-merges", g.DefaultRemoteRef, "--not", g.Head})
+		if !fine {
+			g.Missing = append(g.Missing, source.MissingPatches)
+			return
+		}
+		for _, id := range def {
+			own[id] = true
+		}
+		settle()
+	}
+	for _, sha := range rewritten {
+		if !created[sha] {
+			if g.ReplayedPatch == nil {
+				g.ReplayedPatch = map[string]bool{}
+			}
+			g.ReplayedPatch[sha] = true
+		}
+	}
+}
+
+func amended(subject string) bool { return strings.HasPrefix(subject, "commit (amend)") }
+
+// amendedOwn: sha is an amend of a commit counted as made here. The pre-image is the older neighbour in the
+// reflog; an amend keeps the parents, so a neighbour with other parents (its own entry was deleted or
+// expired) is not the pre-image and does not count.
+func amendedOwn(subject, sha string, amends map[string][]string, parents map[string]string, created map[string]bool) bool {
+	if !amended(subject) {
+		return false
+	}
+	for _, p := range amends[sha] {
+		if pp, known := parents[p]; known && created[p] && pp == parents[sha] {
+			return true
+		}
+	}
+	return false
+}
+
+// amendPreImages maps each commit an amend made to the candidates for the commit it replaced: the entry just
+// older in each log that records the amend. Each log is listed newest first; the logs are grouped by ref
+// here, whatever order they come in.
+func amendPreImages(out string) map[string][]string {
+	type entry struct{ sha, subject string }
+	logs := map[string][]entry{}
+	for _, l := range strings.Split(out, "\n") {
+		f := strings.SplitN(strings.TrimSpace(l), "\t", 3)
+		if len(f) != 3 || !fullSHA.MatchString(f[0]) {
+			continue
+		}
+		ref := f[1]
+		if at := strings.LastIndex(ref, "@{"); at >= 0 {
+			ref = ref[:at]
+		}
+		logs[ref] = append(logs[ref], entry{f[0], f[2]})
+	}
+	pre := map[string][]string{}
+	for _, es := range logs {
+		for i := 0; i+1 < len(es); i++ {
+			if amended(es[i].subject) {
+				pre[es[i].sha] = append(pre[es[i].sha], es[i+1].sha)
+			}
+		}
+	}
+	return pre
+}
+
+// grafted reports whether a grafts file may rewrite the parents git reports: info/grafts of the repository
+// or $GIT_GRAFT_FILE. A file that cannot be checked counts as present.
+func (h SourceTrust) grafted(root, gitPath string) bool {
+	for _, p := range []string{rootPath(root, gitPath), h.getenv("GIT_GRAFT_FILE")} {
+		if p == "" {
+			continue
+		}
+		if _, err := os.Stat(p); err == nil || !errors.Is(err, fs.ErrNotExist) {
+			return true
+		}
+	}
+	return false
+}
+
+// logPatches runs `git log` with patchLog over args and returns the patch id and the parents of each
+// commit; ok is false when git failed or more than limit commits came back.
+func logPatches(git gitFn, limit int, args []string) (ids, parents map[string]string, ok bool) {
+	out, exit := git(append(append([]string{"log", "-n", strconv.Itoa(limit + 1)}, patchLog...), args...)...)
+	ids, parents = patchIDs(out)
+	return ids, parents, exit == 0 && len(ids) <= limit
+}
+
+// patchIDs splits patchLog output into a patch id per commit ("" for a commit with no change lines) and
+// its parents.
+func patchIDs(out string) (ids, parents map[string]string) {
+	ids, parents = map[string]string{}, map[string]string{}
+	for _, block := range strings.Split(out, "\x00")[1:] {
+		head, diff, _ := strings.Cut(block, "\n")
+		sha, ps, _ := strings.Cut(strings.TrimSpace(head), "\t")
+		if fullSHA.MatchString(sha) {
+			ids[sha], parents[sha] = patchID(diff), strings.TrimSpace(ps)
+		}
+	}
+	return ids, parents
+}
+
+// patchID hashes the changes of one commit the way `git patch-id --stable` treats them, and looser: per file
+// its header lines and its added and removed lines with all whitespace removed (no hunk positions, no index
+// line, no context), the file hashes sorted, so file order and a moved base do not change it.
+func patchID(diff string) string {
+	var files []string
+	var cur []string
+	flush := func() {
+		if len(cur) > 1 {
+			sum := sha256.Sum256([]byte(strings.Join(cur, "\n")))
+			files = append(files, hex.EncodeToString(sum[:]))
+		}
+		cur = nil
+	}
+	for _, l := range strings.Split(diff, "\n") {
+		switch {
+		case strings.HasPrefix(l, "diff --git "):
+			flush()
+			cur = []string{l}
+		case cur == nil, l == "", strings.HasPrefix(l, "index "), strings.HasPrefix(l, "@@"), strings.HasPrefix(l, " "):
+		case strings.HasPrefix(l, "+++ "), strings.HasPrefix(l, "--- "):
+			cur = append(cur, l)
+		default: // + and - lines, mode lines, "Binary files ... differ"
+			cur = append(cur, strings.Join(strings.Fields(l), ""))
+		}
+	}
+	flush()
+	if len(files) == 0 {
+		return ""
+	}
+	slices.Sort(files)
+	sum := sha256.Sum256([]byte(strings.Join(files, "\n")))
+	return hex.EncodeToString(sum[:])
+}

free/ci/internal/trust/source/facts.go

@@ -44,6 +44,8 @@ const (
 	ReasonLocalInProgress  = "local.operation_in_progress"
 	ReasonLocalIncomplete  = "local.facts_incomplete"
 	ReasonLocalOverride    = "local.trust_owner_override"
+	ReasonLocalReplayed    = "local.rewrite_not_made_here"
+	ReasonLocalGrafted     = "local.history_grafted"
 )
 
 // Result is a classification: the class, why, and the side facts the callers need.
@@ -64,6 +66,8 @@ const (
 	MissingRemoteRefs = "remote_refs"
 	MissingCommits    = "commits"
 	MissingReflog     = "reflog"
+	MissingPatches    = "patches"
+	MissingHistory    = "history" // a grafts file may rewrite the parents git reports
 )
 
 // GitFacts are the git facts of the checkout a local run is bound to.
@@ -92,6 +96,11 @@ type GitFacts struct {
 	// ReflogCreated maps a commit to the subject of its OLDEST reflog entry in any log (OldestEntries),
 	// the way it first reached this repository, for example "commit: fix" or "branch: Created from FETCH_HEAD".
 	ReflogCreated map[string]string
+	// ReplayedPatch marks the commits of OffDefault whose oldest entry is a rewrite (Rewrite: a rebase step,
+	// an amend, a cherry-pick) and whose change has no source made here: no commit made here (or on the
+	// default branch) carries its patch, and (an amend) the commit it amended was not made here. Such a
+	// commit replays someone else's change, or one nothing here can account for (D-1322).
+	ReplayedPatch map[string]bool
 	// Missing lists the facts that could not be gathered (the Missing* keys).
 	Missing []string
 }