sandbox: keep the overlay upper on scratch or in RAM

.github/workflows/run-nvx-microvm-tests.yml

@@ -211,6 +211,7 @@ jobs:
           --scenario managed-exec-config
           --scenario sandbox-lifecycle
           --scenario sandbox-proxy-only
+          --scenario sandbox-ram-upper
           --output-dir "build/test-results/public-exec-${{ inputs.backend }}"
 
       - name: Run public managed sandbox acceptance on Windows
@@ -222,6 +223,7 @@ jobs:
           --scenario managed-exec-config
           --scenario sandbox-lifecycle
           --scenario sandbox-proxy-only
+          --scenario sandbox-ram-upper
           --output-dir "build/test-results/public-exec-${{ inputs.backend }}"
 
       - name: Run Ubuntu guest smoke tests on Linux
@@ -526,7 +528,7 @@ jobs:
 
           with open(sys.argv[1], encoding="utf-8") as stream:
               config = json.load(stream)
-          if config["format"] != 7 or config["export"]["owner"] != "caller":
+          if config["format"] != 8 or config["export"]["owner"] != "caller":
               sys.exit(f"managed sandbox did not persist caller ownership: {config}")
           EOF
           if nvx-rs/target/release/nvx-rs sandbox start --state-dir "${state}"; then
@@ -664,7 +666,7 @@ jobs:
               ("/opt/hostedtoolcache", "", "ro", ["nvx-denied"], "directory"),
               ("/results/output.txt", "", "rw", [], "file"),
           ]
-          if config["format"] != 7 or "mounts" in config or shares != expected:
+          if config["format"] != 8 or "mounts" in config or shares != expected:
               sys.exit(f"managed sandbox did not persist every share: {config}")
           EOF
           nvx-rs/target/release/nvx-rs sandbox start --state-dir "${state}"
@@ -800,7 +802,7 @@ jobs:
                   ["nvx-writable"],
               ),
           ]
-          if config["format"] != 7 or "mounts" in config or shares != expected:
+          if config["format"] != 8 or "mounts" in config or shares != expected:
               sys.exit(f"managed sandbox did not persist the access policy: {config}")
           EOF
           nvx-rs/target/release/nvx-rs sandbox start --state-dir "${state}"
@@ -1125,7 +1127,7 @@ jobs:
               $denied = ($child.denied | ForEach-Object { $_.Substring($child.root.Length + 1).Replace("\", "/") }) -join ";"
               "$($bind.target),$($bind.source),$mode,$denied,$kind"
           }) -join " "
-          if ($config.format -ne 7 -or $shares -ne "/workspace,,rw,nvx-denied,directory /config/settings.json,,ro,,file /opt/hostedtoolcache,,ro,nvx-denied,directory /results/output.txt,,rw,,file") {
+          if ($config.format -ne 8 -or $shares -ne "/workspace,,rw,nvx-denied,directory /config/settings.json,,ro,,file /opt/hostedtoolcache,,ro,nvx-denied,directory /results/output.txt,,rw,,file") {
               throw "managed sandbox did not persist every share: $($config | ConvertTo-Json -Compress -Depth 5)"
           }
           try {
@@ -1236,7 +1238,7 @@ jobs:
           function Get-Relative($paths) {
               ($paths | ForEach-Object { $_.Substring($child.root.Length + 1).Replace("\", "/") }) -join ","
           }
-          if ($config.format -ne 7 -or @($config.export.children).Count -ne 1 -or @($config.export.binds).Count -ne 1 -or $bind.target -ne "/workspace" -or $bind.readOnly -or (Get-Relative $child.denied) -ne "nvx-denied" -or (Get-Relative $child.allowed) -ne "nvx-denied/nvx-allowed" -or (Get-Relative $child.write) -ne "nvx-writable") {
+          if ($config.format -ne 8 -or @($config.export.children).Count -ne 1 -or @($config.export.binds).Count -ne 1 -or $bind.target -ne "/workspace" -or $bind.readOnly -or (Get-Relative $child.denied) -ne "nvx-denied" -or (Get-Relative $child.allowed) -ne "nvx-denied/nvx-allowed" -or (Get-Relative $child.write) -ne "nvx-writable") {
               throw "managed sandbox did not persist the access policy: $($config | ConvertTo-Json -Compress -Depth 5)"
           }
           try {

doc/ci.md

@@ -409,18 +409,19 @@ container cgroup, the overlay root over read-only image layers with its upper
 directory on scratch, and that the memory controller kills an allocation of
 twice the limit and the pids controller stops the workload at eight
 processes. The live-share runs below repeat the profile checks. Each
-backend also runs the public `managed-exec-config`, `sandbox-lifecycle`, and
-`sandbox-proxy-only` scenarios of `test-microvm`, which drive
-`nvx-rs sandbox provision`, `start`,
+backend also runs the public `managed-exec-config`, `sandbox-lifecycle`,
+`sandbox-proxy-only`, and `sandbox-ram-upper` scenarios of `test-microvm`,
+which drive `nvx-rs sandbox run`, `provision`, `start`,
 `exec`, `stop`, and `deprovision` as subprocesses (see
 [Run](run.md#managed-lifecycle)): request-scoped working directories and
 environments, timeouts, nonzero exits, stdout and stderr, bounded output and
 outcome reports, warm-guest state across requests and restarts, identity
 refusal, fail-closed lifecycle transitions, a crashed OpenVMM's stale runtime
 state, teardown that leaves no OpenVMM process, control endpoint,
 capability, or runtime record while the layer and scratch artifacts survive,
-and a proxy-only network, whose guest reaches a test proxy on host loopback at
-its gateway and no other destination.
+a proxy-only network, whose guest reaches a test proxy on host loopback at
+its gateway and no other destination, and sandboxes whose overlay upper is a
+tmpfs in guest memory, with no scratch image.
 Their bounded evidence is part of the backend's failure logs. The same
 entrypoint then verifies a live virtio-fs
 share inside the container: a read-write `/workspace` share with a denied

doc/contribute.md

@@ -100,8 +100,9 @@ devices. Add those scenarios to
 `scripts/nvx_tools/microvm_test_scripts` and retain complete failure logs.
 The scenarios that drive the public `nvx-rs sandbox` commands live in
 `scripts/nvx_tools/managed_exec_tests.py`,
-`scripts/nvx_tools/sandbox_lifecycle_tests.py`, and
-`scripts/nvx_tools/sandbox_proxy_tests.py`, and `microvm_tests.py`
+`scripts/nvx_tools/sandbox_lifecycle_tests.py`,
+`scripts/nvx_tools/sandbox_proxy_tests.py`, and
+`scripts/nvx_tools/sandbox_ram_upper_tests.py`, and `microvm_tests.py`
 registers them. `sandbox` and the control protocol belong to `nvx-rs`
 ([microsoft/nvx#359](https://github.com/microsoft/nvx/issues/359)), so the
 scenarios drive them as commands, compile egress policies with

doc/design/1.1-goals.md

@@ -11,6 +11,6 @@ The architecture has the following goals:
 - Capture a running VM into immutable artifacts that can be restored in a new
   process without serializing native host handles.
 - Run exactly one workload per microVM.
-- Assemble the workload filesystem from an immutable image layer, private
-  writable scratch, and optional live host shares.
+- Assemble the workload filesystem from an immutable image layer, a private
+  writable upper on scratch or in guest memory, and optional live host shares.
 - Keep workload policy and host-resource attachment under host control.

doc/design/2.1-machine-model-and-configuration.md

@@ -79,7 +79,9 @@ disks. The roles are the layers of the sandbox root filesystem: a read-only
 A disk without a role is rejected, and a disk-backed topology ends with
 writable scratch. The only exception is the RAM-backed overlay layout, which
 requires exactly one read-only `distro` block and no scratch, keeps the
-guest's overlay upper layer in RAM, and supports neither capture nor restore.
+guest's overlay upper layer in RAM, on a tmpfs that the guest agents mount
+(see [Layered Sandbox Root](4.1-guest-filesystem-and-image-model.md#411-layered-sandbox-root)),
+and supports neither capture nor restore.
 Snapshot capture with blocks requires a read-only `distro` layer followed by
 writable scratch, both backed by cached, regular raw files with nonzero,
 512-byte-aligned geometry. Blockless machines are also snapshot-capable and

doc/design/2.5-device-topology-transport-and-discovery.md

@@ -16,7 +16,7 @@ are resolved.
 | Host filesystem | `fs:microvm0` | `0xd0001000..0xd0001fff` | 6 | Reserved slot; attachment optional |
 | Boot console | `console:microvm-virtio0` | `0xd0002000..0xd0002fff` | 7 | Optional |
 | `distro` block | `blk:sandbox:distro` | `0xd0003000..0xd0003fff` | 4 | Optional, read-only |
-| `scratch` block | `blk:sandbox:scratch` | `0xd0006000..0xd0006fff` | 11 | Writable final role |
+| `scratch` block | `blk:sandbox:scratch` | `0xd0006000..0xd0006fff` | 11 | Optional, writable final role; absent in the RAM-backed overlay layout |
 | Control console | `console:microvm-control0` | `0xd0007000..0xd0007fff` | 3 | Optional |
 
 Placement is explicit rather than allocated sequentially, so an omitted device

doc/design/3.4-block-storage.md

@@ -15,7 +15,8 @@ arbitrary disks.
 
 Roles are unique and appear in fixed order; an omitted `distro` role leaves
 its slot empty. A disk-backed topology ends with scratch, except in the
-RAM-backed overlay layout, which uses exactly one read-only `distro` block.
+RAM-backed overlay layout, which uses exactly one read-only `distro` block and
+whose guest keeps the overlay's upper on a tmpfs in guest memory.
 The role, not Linux enumeration order, defines identity.
 
 ## 3.4.1 Construction and Discovery

doc/design/4-guest-runtime-architecture.md

@@ -17,7 +17,7 @@ isolated workload sandbox.
 ## Table of Contents
 
 - [4.1 Guest Filesystem and Image Model](4.1-guest-filesystem-and-image-model.md):
-  Explains image layers, writable scratch, overlay assembly, and live shares.
+  Explains image layers, the writable upper, overlay assembly, and live shares.
 - [4.2 Guest Agents and Workload Model](4.2-guest-agents-and-workload-model.md):
   Defines the relationship between the guest agents and the workload.
 - [4.3 Isolation and Resource Model](4.3-isolation-and-resource-model.md):

doc/design/4.1-guest-filesystem-and-image-model.md

@@ -9,23 +9,26 @@ As Table 4.1 shows, the guest agents run workloads in one of two roots.
 
 | Root | Construction | Host content |
 | --- | --- | --- |
-| Layered sandbox root | An overlay of one read-only image layer and private writable scratch | Live host shares bound at declared targets |
+| Layered sandbox root | An overlay of one read-only image layer and a private writable upper, on scratch or in guest memory | Live host shares bound at declared targets |
 | Blockless managed root | The guest's own initramfs root, without block devices | Host directories bound by a host mapping table |
 
 The guest kernel provides the facilities both roots need: virtio-blk,
-virtio-fs, EROFS with compression and extended attributes, overlayfs, cgroup
-v2 with the memory and process controllers, namespaces, BPF system calls, and
-cgroup BPF programs.
+virtio-fs, EROFS with compression and extended attributes, tmpfs with
+extended attributes, overlayfs, cgroup v2 with the memory and process
+controllers, namespaces, BPF system calls, and cgroup BPF programs.
 
 ## 4.1.1 Layered Sandbox Root
 
 As Figure 4.1 shows, the layered root combines one immutable EROFS `distro`
-layer with one private ext4 scratch device.
+layer with one private writable upper: either an ext4 scratch device or a
+tmpfs in guest memory.
 
 ```text
 /run/nvx/layers/distro   (EROFS) ----------> lower directory
 /run/nvx/scratch/upper   (ext4) -----------> upper directory
+  or /run/nvx/tmpfs/upper (tmpfs)
 /run/nvx/scratch/work    (same ext4) ------> work directory
+  or /run/nvx/tmpfs/work (same tmpfs)
                                           overlay --> /run/nvx/rootfs
 one-shot: virtio-fs tag microvm (optional share) ---> /run/nvx/rootfs/TARGET
   or an aggregate        ---> /run/nvx/shares, each child bound at
@@ -51,28 +54,31 @@ one contract module defines, with the host's renderer and a parser for guests:
   filesystem UUID or the disk GUID in canonical lowercase text. No token names
   a partition GUID.
 - `nvx_scratch=0xd0006000,ext4` keeps the overlay's upper on the whole-device
-  ext4 `scratch` disk, and `nvx_overlay_upper=ramfs` keeps it in RAM, with no
-  `scratch` block device.
+  ext4 `scratch` disk, and `nvx_overlay_upper=ramfs` keeps it in guest
+  memory, on a tmpfs, with no `scratch` block device.
 
 Guests refuse a token without a value, a repeated token, both uppers, an image
 without an upper or an upper without an image, a RAM upper beside a `scratch`
 block device, and every other form, including the earlier `nvx_layer` form
 without a format, rather than probe the devices. The guest agents described
-here assemble only an EROFS image over scratch: they refuse a GPT image and a
-RAM upper with a message that says so.
+here assemble only an EROFS image, over either upper: they refuse a GPT image
+with a message that says so.
 
 The guest agents assemble the root in this order:
 
 1. Mount runtime tmpfs and cgroup v2, create sibling guest-agent and
    container cgroups, and move the guest agents into the guest-agent cgroup.
 2. Read the root's boot tokens, and resolve each role's MMIO address through
    the kernel's platform-device resources rather than assuming a device name
-   or enumeration order.
+   or enumeration order. For a RAM upper, refuse any device at the `scratch`
+   role's address.
 3. Check any expected sector count, flush cached block-device buffers, mount
    the `distro` layer read-only as EROFS, and check its filesystem UUID when
    the token names one.
-4. Resolve and flush scratch, mount it as ext4 with `rw,nosuid,nodev`, and
-   create its upper and work directories.
+4. Mount the upper's file system, and create the overlay's upper and work
+   directories on it: resolve and flush scratch and mount it as ext4 with
+   `rw,nosuid,nodev`, or mount a tmpfs with `mode=0755,nosuid,nodev` at the
+   kernel's default size, half of the guest's RAM.
 5. Mount the overlay with the `distro` layer as its lower directory,
    metadata-only copy-up, and extended inode numbers.
 6. Verify the workload identity in the assembled root, mount each live share
@@ -83,21 +89,32 @@ The guest agents assemble the root in this order:
    before releasing it; a managed sandbox's agent first maps the shares from
    the export, as their mapping table orders.
 8. Wait for the workload, unmount the shares in reverse order and then the
-   overlay, the `distro` layer, and scratch, and report the workload's status.
+   overlay, the `distro` layer, and the upper, and report the workload's
+   status.
 
 The UUID check detects attachment mistakes; it is not a content-integrity
 proof, and neither the guest nor the VMM verifies layer contents.
 Metadata-only copy-up interprets overlay metadata stored in the `distro`
 layer, so that layer is a trusted input that must not carry untrusted overlay
 attributes. A metadata-only change avoids copying file contents, but a data
-write still copies the whole lower file into scratch, so scratch capacity must
-account for copy-up. Scratch consumes host storage rather than guest RAM and
-can run out of space independently of guest memory pressure.
+write still copies the whole lower file into the upper, so the upper's
+capacity must account for copy-up. The two uppers spend different resources:
+
+- Scratch consumes host storage rather than guest RAM, can run out of space
+  independently of guest memory pressure, and keeps a managed sandbox's
+  changes from one start to the next.
+- A RAM upper consumes guest memory. The memory cgroup of the process that
+  writes a page, normally the workload's, is charged for it, so the
+  workload's memory limit also bounds its files, and since the guest has no
+  swap, those pages stay resident until their files are removed. A write
+  beyond the tmpfs size fails with `ENOSPC`, and nothing the workload writes
+  outlives the virtual machine.
 
 The initramfs remains the guest agents' root; it is not another workload
-layer. The guest agents always keep the overlay upper layer on scratch, so the
-machine's RAM-backed overlay layout, which has no scratch, serves only guests
-that declare it.
+layer. A RAM upper uses the machine's RAM-backed overlay layout, which has
+exactly one read-only `distro` block and no scratch, and which supports
+neither capture nor restore; see
+[Storage Configuration](2.1-machine-model-and-configuration.md#213-storage-configuration).
 
 ## 4.1.2 Live Host Shares
 

doc/design/4.3-isolation-and-resource-model.md

@@ -67,7 +67,10 @@ released. The guest agents' cgroup carries a default memory protection of
 16 MiB, a soft reservation that shields the supervisor from reclaim under
 workload memory pressure but does not guarantee its survival. The workload
 cgroup carries optional memory and process-count limits and is the unit of
-capture freeze, termination, and descendant cleanup.
+capture freeze, termination, and descendant cleanup. When the overlay's upper
+is in guest memory, the files that the workload writes into its root are
+charged to the workload cgroup, so its memory limit also bounds them; see
+[Layered Sandbox Root](4.1-guest-filesystem-and-image-model.md#411-layered-sandbox-root).
 
 In the blockless managed root, each execution joins a dedicated cgroup before
 the workload runs, for termination and descendant accounting. This path does

doc/design/5.1-boot-protocol.md

@@ -83,8 +83,9 @@ Launchers add runtime tokens of their own: the
 [boot tokens of a layered root](4.1-guest-filesystem-and-image-model.md#411-layered-sandbox-root),
 one share token per host-share child, the number of host mappings for a
 blockless managed root, and a request to create the workload account. A caller
-may also declare the RAM-backed overlay layout with exactly one
-`nvx_overlay_upper=ramfs` token; any other form of that token is rejected.
+may also declare the RAM-backed overlay layout, whose overlay upper the guest
+agents keep on a tmpfs, with exactly one `nvx_overlay_upper=ramfs` token; any
+other form of that token is rejected.
 
 Embedded NULs are rejected, and the complete NUL-terminated command line must
 fit in 64 KiB. The effective string and its SHA-256 digest become part of the

doc/design/5.2-readiness-and-runtime-control.md

@@ -57,7 +57,10 @@ Lifecycle transitions fail closed and take effect one at a time per sandbox,
 while executions run beside them. The host identifies the running VMM process
 of each sandbox, reconciles VMM processes with recorded state, and recovers
 interrupted launches. Teardown leaves no VMM process, control endpoint,
-capability, or runtime record, while supplied layers and scratch survive.
+capability, or runtime record, while supplied layers and scratch survive. A
+sandbox's writes to its root therefore persist from one start to the next on
+scratch, but not when its overlay's upper is in guest memory, which each start
+begins empty.
 
 ## 5.2.5 Authority
 

doc/project-structure.md

@@ -144,6 +144,7 @@ nvx/
 |   |   |-- release.py           Release source collection and packaging
 |   |   |-- sandbox_lifecycle_tests.py Managed sandbox lifecycle and security tests
 |   |   |-- sandbox_proxy_tests.py Proxy-only sandbox networking tests
+|   |   |-- sandbox_ram_upper_tests.py Sandboxes whose overlay upper is in RAM
 |   |   |-- time_abi.py          Time ABI harness checks and CPU profile guidance
 |   |   `-- ubuntu.py            Verified Ubuntu rootfs and EROFS preparation
 |   |-- setup/                   Host and Actions runner bootstrap scripts
@@ -222,7 +223,8 @@ the entry script stages Alpine's musl loader and `setpriv` inside the
 workload's root.
 
 The sandbox scripts resolve fixed virtio-blk roles through
-sysfs, assemble EROFS lower layers over ext4 scratch, place the workload in its
+sysfs, assemble EROFS lower layers over the upper that the host declares, an
+ext4 scratch disk or a tmpfs, place the workload in its
 cgroup before release, construct its mount/PID/UTS namespaces, enter its
 filesystem root after dropping capabilities, and retain the agent as the outer
 PID 1. In a managed sandbox, the agent supervises `nvx-managed-agent`, which