fix(pivot): give the new rootfs its own tmpfs, sized from real RAM

internal/cli/build.go

@@ -51,12 +51,11 @@ func runBuild(ctx context.Context, cfg *config.Config) error {
 		return nil
 	}
 
-	if err := os.RemoveAll(cfg.WorkDir); err != nil && !os.IsNotExist(err) {
+	// CleanTarget, not RemoveAll: the work dir may be a mount point (the
+	// pivot path mounts a tmpfs there) and unlinking one fails with EBUSY.
+	if err := rootfs.CleanTarget(cfg.WorkDir); err != nil {
 		return fmt.Errorf("clean work dir: %w", err)
 	}
-	if err := os.MkdirAll(cfg.WorkDir, 0o755); err != nil {
-		return fmt.Errorf("create work dir: %w", err)
-	}
 
 	slog.Info("building rootfs", "layers", len(cfg.Layers), "target", cfg.WorkDir)
 	result, err := rootfs.Build(cfg.Layers, cfg.WorkDir)

internal/cli/pivot.go

@@ -20,6 +20,7 @@ import (
 	"github.com/ananthb/xmorph/internal/postpivot"
 	"github.com/ananthb/xmorph/internal/process"
 	"github.com/ananthb/xmorph/internal/rootfs"
+	"github.com/ananthb/xmorph/internal/sysmem"
 	"github.com/ananthb/xmorph/internal/tsnetauth"
 	"github.com/spf13/cobra"
 	"golang.org/x/sys/unix"
@@ -153,12 +154,96 @@ func runPivot(ctx context.Context, cfg *config.Config, stdout interface {
 		return fmt.Errorf("create work dir: %w", err)
 	}
 
+	// Give the new rootfs its own tmpfs, sized explicitly.
+	//
+	// Without this the rootfs is extracted into whatever filesystem WorkDir
+	// happens to land in — by default /run, which is itself a tmpfs capped
+	// at a fraction of RAM (83 MiB on a 415 MiB host). The extract then dies
+	// with ENOSPC well before RAM is exhausted, and the error points at a
+	// random file rather than at the real cause. Our own mount makes the
+	// budget explicit and independent of the host's /run sizing.
+	mem, err := sysmem.Read()
+	if err != nil {
+		return fmt.Errorf("read meminfo: %w", err)
+	}
+	var rootfsBudget uint64
+	if cfg.NoRootfsTmpfs {
+		free, err := pivot.FreeBytes(cfg.WorkDir)
+		if err != nil {
+			return fmt.Errorf("statfs %s: %w", cfg.WorkDir, err)
+		}
+		rootfsBudget = free
+		slog.Info("using --work-dir as-is (no tmpfs)", "dir", cfg.WorkDir, "free_mib", free>>20)
+	} else {
+		mounted, err := pivot.IsMountPoint(cfg.WorkDir)
+		if err != nil {
+			return fmt.Errorf("check %s: %w", cfg.WorkDir, err)
+		}
+		if mounted {
+			// A leftover from an aborted run, or an operator-arranged mount.
+			// Reusing it silently would hide its size, so say so.
+			free, _ := pivot.FreeBytes(cfg.WorkDir)
+			rootfsBudget = free
+			slog.Warn("work dir is already a mount point; reusing it instead of mounting a tmpfs",
+				"dir", cfg.WorkDir, "free_mib", free>>20)
+			// A reused mount may hold a half-extracted rootfs from an
+			// aborted run. Layering a new rootfs over those leftovers is
+			// how you get a system that boots with two distros' worth of
+			// /etc, so empty it first.
+			if err := rootfs.CleanTarget(cfg.WorkDir); err != nil {
+				return fmt.Errorf("clean reused work dir: %w", err)
+			}
+		} else {
+			if cfg.RootfsSize != "" {
+				rootfsBudget, err = config.ParseSize(cfg.RootfsSize, mem.Total)
+				if err != nil {
+					return fmt.Errorf("--rootfs-size: %w", err)
+				}
+			} else {
+				rootfsBudget = mem.RecommendRootfsBytes()
+			}
+			if rootfsBudget == 0 {
+				return fmt.Errorf("no RAM available for a rootfs: %d MiB available, %d MiB total, %d MiB reserved",
+					mem.Available>>20, mem.Total>>20, mem.ReserveBytes()>>20)
+			}
+			// A tmpfs bigger than available RAM is not an error the kernel
+			// will report — it just gets swapped or OOMs mid-extract. Refuse
+			// while the old OS is still alive to say so.
+			if warn, err := mem.HeadroomCheck(rootfsBudget); err != nil {
+				return fmt.Errorf("%w (requested rootfs tmpfs of %d MiB; lower --rootfs-size or free memory first)",
+					err, rootfsBudget>>20)
+			} else if warn {
+				slog.Warn("rootfs tmpfs leaves little headroom; a large rootfs may OOM mid-extract",
+					"size_mib", rootfsBudget>>20, "available_mib", mem.Available>>20)
+			}
+			if err := pivot.MountTmpfs(cfg.WorkDir, rootfsBudget); err != nil {
+				return err
+			}
+			slog.Info("mounted rootfs tmpfs", "dir", cfg.WorkDir, "size_mib", rootfsBudget>>20,
+				"available_mib", mem.Available>>20, "total_mib", mem.Total>>20)
+			// Unmount on any abort before the pivot. unix.Exec never returns
+			// on success, so reaching a return at all means we are staying on
+			// the old OS — where leaving a RAM-backed copy of the rootfs
+			// pinned would be a slow leak on a host we just decided not to
+			// replace. No error check: "we returned" IS the abort signal.
+			defer func() { _ = pivot.UnmountTmpfs(cfg.WorkDir) }()
+		}
+	}
+
 	slog.Info("building rootfs", "layers", len(cfg.Layers), "target", cfg.WorkDir)
 	result, err := rootfs.Build(cfg.Layers, cfg.WorkDir)
 	if err != nil {
-		return fmt.Errorf("build rootfs: %w", err)
+		// ENOSPC here means the budget was too small, which is worth saying
+		// plainly — it is the single most common way this step fails.
+		return fmt.Errorf("build rootfs (tmpfs budget %d MiB — raise --rootfs-size if this was ENOSPC): %w",
+			rootfsBudget>>20, err)
+	}
+	if used, uerr := pivot.UsageBytes(cfg.WorkDir); uerr == nil {
+		slog.Info("rootfs built", "layers", result.LayerCount, "size_mib", used>>20,
+			"budget_mib", rootfsBudget>>20)
+	} else {
+		slog.Info("rootfs built", "layers", result.LayerCount)
 	}
-	slog.Info("rootfs built", "layers", result.LayerCount)
 
 	entrypoint, entryArgs, _ := resolveEntrypoint(cfg, result.Config)
 	slog.Info("entrypoint resolved", "entrypoint", entrypoint, "args", len(entryArgs))
@@ -377,12 +462,11 @@ func runContain(cfg *config.Config) error {
 	}
 
 	slog.Info("building rootfs for --contain", "layers", len(cfg.Layers), "target", cfg.WorkDir)
-	if err := os.RemoveAll(cfg.WorkDir); err != nil && !os.IsNotExist(err) {
+	// CleanTarget, not RemoveAll: the work dir may be a mount point (the
+	// pivot path mounts a tmpfs there) and unlinking one fails with EBUSY.
+	if err := rootfs.CleanTarget(cfg.WorkDir); err != nil {
 		return fmt.Errorf("clean work dir: %w", err)
 	}
-	if err := os.MkdirAll(cfg.WorkDir, 0o755); err != nil {
-		return fmt.Errorf("create work dir: %w", err)
-	}
 
 	result, err := rootfs.Build(cfg.Layers, cfg.WorkDir)
 	if err != nil {

internal/config/config.go

@@ -86,6 +86,21 @@ type Config struct {
 	CacheDir string
 	WorkDir  string
 
+	// RootfsSize is the size of the tmpfs mounted at WorkDir to hold the new
+	// rootfs: a byte count ("512M", "2G") or a percentage of total RAM
+	// ("50%"). Empty means auto — see sysmem.RecommendRootfsBytes.
+	//
+	// This exists because WorkDir defaults to a path under /run, and /run is
+	// itself a size-capped tmpfs (commonly 10-20% of RAM). Without our own
+	// mount the rootfs silently inherits that cap.
+	RootfsSize string
+
+	// NoRootfsTmpfs extracts straight into WorkDir instead of mounting a
+	// tmpfs over it. For the case where the operator has already arranged
+	// the storage — a pre-mounted tmpfs, or a disk that is not the one
+	// being overwritten.
+	NoRootfsTmpfs bool
+
 	// LogDir is an additional on-disk log sink: xmorph mirrors its slog
 	// output to {LogDir}/xmorph.log (alongside stderr/journald and syslog)
 	// and flushes the in-memory buffer there just before pivot_root, so the

internal/config/flags.go

@@ -38,6 +38,8 @@ func bindCommon(fs *pflag.FlagSet, cfg *Config) {
 	fs.BoolVarP(&cfg.Verbose, "verbose", "v", false, "verbose output")
 	fs.BoolVar(&cfg.NoCache, "no-cache", false, "skip build cache, pull fresh")
 	fs.StringVar(&cfg.WorkDir, "work-dir", DefaultWorkDir, "working directory for rootfs extraction")
+	fs.StringVar(&cfg.RootfsSize, "rootfs-size", "", "size of the tmpfs holding the new rootfs (e.g. 512M, 2G, 50%); default: available RAM minus a 10% reserve")
+	fs.BoolVar(&cfg.NoRootfsTmpfs, "no-rootfs-tmpfs", false, "extract into --work-dir as-is instead of mounting a tmpfs over it")
 }
 
 func bindPivotOnly(fs *pflag.FlagSet, cfg *Config) {

internal/config/size.go

@@ -0,0 +1,64 @@
+package config
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// ParseSize turns a human size into bytes. Accepts a plain byte count, an
+// IEC suffix (K/M/G/T, optionally with an "i" and/or a trailing "B" — 512M,
+// 512MiB and 512MB are all 512*1024*1024), or a percentage of total, which
+// is why totalBytes is required.
+//
+// Percentages exist because the useful default here is relative: "half of
+// RAM" is portable across a 415 MiB Pi and a 64 GiB server in a way that
+// "2G" is not.
+func ParseSize(s string, totalBytes uint64) (uint64, error) {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return 0, fmt.Errorf("empty size")
+	}
+
+	if pct, ok := strings.CutSuffix(s, "%"); ok {
+		v, err := strconv.ParseFloat(strings.TrimSpace(pct), 64)
+		if err != nil {
+			return 0, fmt.Errorf("parse percentage %q: %w", s, err)
+		}
+		if v <= 0 || v > 100 {
+			return 0, fmt.Errorf("percentage %q out of range (0, 100]", s)
+		}
+		return uint64(float64(totalBytes) * v / 100), nil
+	}
+
+	// Strip an optional trailing "B"/"iB" so 512MB, 512MiB and 512M agree.
+	u := strings.ToUpper(s)
+	u = strings.TrimSuffix(u, "B")
+	u = strings.TrimSuffix(u, "I")
+
+	mult := uint64(1)
+	if len(u) > 0 {
+		switch u[len(u)-1] {
+		case 'K':
+			mult = 1 << 10
+		case 'M':
+			mult = 1 << 20
+		case 'G':
+			mult = 1 << 30
+		case 'T':
+			mult = 1 << 40
+		}
+		if mult > 1 {
+			u = u[:len(u)-1]
+		}
+	}
+
+	n, err := strconv.ParseUint(strings.TrimSpace(u), 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("parse size %q: %w", s, err)
+	}
+	if n > (1<<64-1)/mult {
+		return 0, fmt.Errorf("size %q overflows", s)
+	}
+	return n * mult, nil
+}

internal/config/size_test.go

@@ -0,0 +1,50 @@
+package config
+
+import "testing"
+
+func TestParseSize(t *testing.T) {
+	const total = 4 << 30 // 4 GiB
+
+	cases := []struct {
+		in   string
+		want uint64
+		bad  bool
+	}{
+		{in: "512", want: 512},
+		{in: "512K", want: 512 << 10},
+		{in: "512M", want: 512 << 20},
+		{in: "2G", want: 2 << 30},
+		{in: "1T", want: 1 << 40},
+		// The three spellings of the same size must agree; operators type
+		// all three and a silent 1000-vs-1024 difference is a bad surprise
+		// when it is the margin between fitting in RAM and an OOM.
+		{in: "512MB", want: 512 << 20},
+		{in: "512MiB", want: 512 << 20},
+		{in: "50%", want: 2 << 30},
+		{in: "100%", want: total},
+		{in: " 256M ", want: 256 << 20},
+		{in: "", bad: true},
+		{in: "0%", bad: true},
+		{in: "101%", bad: true},
+		{in: "-5%", bad: true},
+		{in: "banana", bad: true},
+		{in: "12X", bad: true},
+	}
+
+	for _, c := range cases {
+		got, err := ParseSize(c.in, total)
+		if c.bad {
+			if err == nil {
+				t.Errorf("ParseSize(%q) = %d, want error", c.in, got)
+			}
+			continue
+		}
+		if err != nil {
+			t.Errorf("ParseSize(%q) unexpected error: %v", c.in, err)
+			continue
+		}
+		if got != c.want {
+			t.Errorf("ParseSize(%q) = %d, want %d", c.in, got, c.want)
+		}
+	}
+}

internal/pivot/prepare.go

@@ -41,8 +41,20 @@ func Prepare(opts PrepareOptions) error {
 	// (This is why pivot_root recipes always bind the new root onto itself
 	// before populating it.) pivot_root also requires newRoot to be a mount
 	// point, which this satisfies.
-	if err := EnsureMountPoint(opts.NewRoot); err != nil {
-		return fmt.Errorf("bind new root onto itself: %w", err)
+	//
+	// Skipped when newRoot is already a mount in its own right — the normal
+	// case now that the orchestrator mounts a tmpfs there. The self-bind
+	// exists only to satisfy pivot_root's "new_root must be a mount point"
+	// rule; stacking a second mount to re-satisfy a rule that already holds
+	// just adds a mount to unwind.
+	alreadyMounted, err := IsMountPoint(opts.NewRoot)
+	if err != nil {
+		return fmt.Errorf("check whether new root is a mount point: %w", err)
+	}
+	if !alreadyMounted {
+		if err := EnsureMountPoint(opts.NewRoot); err != nil {
+			return fmt.Errorf("bind new root onto itself: %w", err)
+		}
 	}
 	if err := MakePrivate(opts.NewRoot); err != nil {
 		return fmt.Errorf("make new root private: %w", err)

internal/pivot/tmpfs.go

@@ -0,0 +1,86 @@
+package pivot
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/sys/unix"
+)
+
+// IsMountPoint reports whether path is itself a mount point, by comparing
+// its device number with its parent's. This is the same test `mountpoint(1)`
+// makes, and it is cheaper and more robust than scanning /proc/self/mounts
+// for a string match.
+func IsMountPoint(path string) (bool, error) {
+	var st, parent unix.Stat_t
+	if err := unix.Lstat(path, &st); err != nil {
+		return false, err
+	}
+	if err := unix.Lstat(filepath.Dir(path), &parent); err != nil {
+		return false, err
+	}
+	return st.Dev != parent.Dev, nil
+}
+
+// MountTmpfs mounts a tmpfs of exactly sizeBytes at path, creating path if
+// needed.
+//
+// This exists because the new rootfs must NOT inherit the size cap of
+// whatever filesystem its path happens to sit in. The default work dir is
+// under /run, which on a systemd host is a tmpfs sized to a fraction of RAM
+// (10-20% is typical: 83 MiB on a 415 MiB Raspberry Pi). Extracting a rootfs
+// into it fails with ENOSPC long before RAM is actually exhausted, and the
+// failure surfaces as a confusing mid-extract write error rather than
+// "your rootfs does not fit".
+//
+// Mounting our own tmpfs makes the budget explicit and independent of the
+// host's /run sizing.
+func MountTmpfs(path string, sizeBytes uint64) error {
+	if err := os.MkdirAll(path, 0o755); err != nil {
+		return fmt.Errorf("create %s: %w", path, err)
+	}
+	opts := fmt.Sprintf("size=%d,mode=0755", sizeBytes)
+	if err := unix.Mount("tmpfs", path, "tmpfs", unix.MS_NOSUID|unix.MS_NODEV, opts); err != nil {
+		return fmt.Errorf("mount tmpfs (%s) at %s: %w", opts, path, err)
+	}
+	return nil
+}
+
+// UnmountTmpfs removes a tmpfs previously mounted by MountTmpfs. Used on the
+// abort paths so a failed pivot does not leave a RAM-backed filesystem (and
+// its contents) pinned on a host that is staying on its old OS.
+func UnmountTmpfs(path string) error {
+	return unix.Unmount(path, unix.MNT_DETACH)
+}
+
+// FreeBytes returns the free space available at path.
+func FreeBytes(path string) (uint64, error) {
+	var st unix.Statfs_t
+	if err := unix.Statfs(path, &st); err != nil {
+		return 0, err
+	}
+	return uint64(st.Bavail) * uint64(st.Bsize), nil
+}
+
+// UsageBytes returns the bytes consumed by the tree rooted at path,
+// counting allocated blocks rather than apparent size so it matches what
+// the filesystem actually charges.
+func UsageBytes(path string) (uint64, error) {
+	var total uint64
+	err := filepath.WalkDir(path, func(p string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		var st unix.Stat_t
+		if err := unix.Lstat(p, &st); err != nil {
+			return nil // raced with a write; not worth failing the pivot over
+		}
+		total += uint64(st.Blocks) * 512
+		return nil
+	})
+	return total, err
+}