From ac82cff93b7ea0d2bf8628fde35f0f8d472d1aa5 Mon Sep 17 00:00:00 2001 From: Young Joon Lee Date: Tue, 20 Oct 2026 08:58:21 +0900 Subject: [PATCH 1/6] feat(tenant): manage long-lived GPU tenant containers Add `rootfiles tenant` (create, apply, status, list, user add, passwd, reset, gpu, image, port, volume resize, restart, logs, import, ops, sync) and an opt-in `tenant` host-prep module. Each tenant gets one container on the nvidia runtime, a size-capped volume on a loop-mounted image mounted before docker, accounts and host keys kept outside the container, temporary passwords that must be changed at first login, recreate with automatic rollback, GPUs in the shared allocation ledger under a tenant: owner, limited operator delegation through one sudoers rule, and a JSONL audit log. Closes #78 Claude-Session: https://claude.ai/code/session_0177Zk7to4EYohgM59v4Fcj7 --- .github/workflows/test.yaml | 15 + AGENTS.md | 7 +- README.md | 22 +- docs/rocky-support.md | 2 +- .../2026-10-20-tenant-containers-design.md | 171 +++++ docs/tenant.md | 96 +++ internal/cli/configcmd.go | 2 +- internal/cli/root.go | 1 + internal/cli/runtime.go | 4 + internal/cli/tenant.go | 595 ++++++++++++++++ internal/config/config.go | 31 + internal/config/distro.go | 2 +- internal/config/validate.go | 8 + internal/module/fleet_sudo.go | 30 +- internal/module/gpu.go | 76 +- internal/module/module.go | 2 + internal/module/module_test.go | 7 +- internal/module/tenant.go | 309 +++++++++ internal/module/tenant_test.go | 116 ++++ internal/tenant/assets.go | 127 ++++ internal/tenant/import.go | 302 ++++++++ internal/tenant/manager.go | 647 ++++++++++++++++++ internal/tenant/ops.go | 66 ++ internal/tenant/spec.go | 305 +++++++++ internal/tenant/tenant_test.go | 317 +++++++++ internal/tenant/volume.go | 237 +++++++ tests/scenarios/tenant-lifecycle.sh | 149 ++++ 27 files changed, 3615 insertions(+), 31 deletions(-) create mode 100644 docs/specs/2026-10-20-tenant-containers-design.md create mode 100644 docs/tenant.md create mode 100644 internal/cli/tenant.go create mode 100644 internal/module/tenant.go create mode 100644 internal/module/tenant_test.go create mode 100644 internal/tenant/assets.go create mode 100644 internal/tenant/import.go create mode 100644 internal/tenant/manager.go create mode 100644 internal/tenant/ops.go create mode 100644 internal/tenant/spec.go create mode 100644 internal/tenant/tenant_test.go create mode 100644 internal/tenant/volume.go create mode 100755 tests/scenarios/tenant-lifecycle.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1eebb11..c1a3a17 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -306,6 +306,21 @@ jobs: chmod +x rootfiles ROOTFILES_TEST_BINARY="$PWD/rootfiles" bash tests/scenarios/fleet.sh + tenant-scenario: + needs: unit + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: rootfiles-binary + path: . + - name: Tenant lifecycle on the runner's Docker and systemd + run: | + chmod +x rootfiles + ROOTFILES_TEST_BINARY="$PWD/rootfiles" bash tests/scenarios/tenant-lifecycle.sh + rocky: needs: unit runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index acba9a2..48e2ee5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,9 +12,10 @@ make test # go test ./... -race ## Architecture - `cmd/rootfiles/` — entry point -- `internal/cli/` — cobra commands (apply, backup, check, config, doctor, gpu, rollback, schedule, status, tunnel, update, user); `update` keeps `upgrade` as an alias. `fleet` is the operator-side controller; `runtime.go` holds the native-host root/lock preflight (native mutating commands are listed in `mutatingCommands`), audit logger and `newRunner` +- `internal/cli/` — cobra commands (apply, backup, check, config, doctor, gpu, rollback, schedule, status, tenant, tunnel, update, user); `update` keeps `upgrade` as an alias. `fleet` is the operator-side controller; `runtime.go` holds the native-host root/lock preflight (native mutating commands are listed in `mutatingCommands`), audit logger and `newRunner` - `internal/config/` — config structs, YAML profile loader (profiles merged as YAML trees, strict keys), `Validate()`, system detector -- `internal/module/` — Module interface + 13 implementations (locale, system, packages, users, ssh, security, docker, nvidia, gpu, cloudflared, storage, network, monitoring), plus `doctor.go` +- `internal/module/` — Module interface + 14 implementations (locale, system, packages, users, ssh, security, docker, nvidia, gpu, cloudflared, storage, network, monitoring, tenant), plus `doctor.go` +- `internal/tenant/` — `rootfiles tenant` specs, containers, volumes, passwords, import and audit; it must not import `internal/module` (module imports it). Design: `docs/specs/2026-10-20-tenant-containers-design.md` - `internal/exec/` — shell runner (dry-run aware; `Runner.Backup` preserves files before WriteFile/Remove/Rename/Symlink), APT and RPM/DNF package-manager backends - `internal/state/` — `/var/lib/rootfiles/state.json` + history, global flock - `internal/ui/` — interactive prompts (Charm huh) + shared output styling: `styles.go` (lipgloss palette), `markers.go` (✓ ✗ → ⚠), `format.go` (WriteHeader/Section/KV/Hint/Bullet). lipgloss auto-detects TTY and honours `NO_COLOR`; all status-style reports must go through these helpers. @@ -35,7 +36,7 @@ type Module interface { - `Check()` reports pending changes without side effects; `CheckResult.Satisfied` gates whether `Apply()` runs. - `Apply()` performs idempotent changes; returns `ApplyResult.Changed` so callers can summarise diffs. -`NewRegistry()` in `internal/module/module.go` wires all 13 implementations. `defaultOrder` in the same file defines execution sequence. These two lists MUST stay in sync — `TestRegistryDefaultOrderSync` in `module_test.go` enforces this. +`NewRegistry()` in `internal/module/module.go` wires all 14 implementations. `defaultOrder` in the same file defines execution sequence. These two lists MUST stay in sync — `TestRegistryDefaultOrderSync` in `module_test.go` enforces this. ## Conventions diff --git a/README.md b/README.md index 627be21..38ce1a4 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,9 @@ Managed hosts must run Linux; macOS is not a supported host OS. The macOS build `rootfiles` is the root half of a two-tool server setup. It prepares a machine with everything that needs root and stops at the user account; [dotfiles](https://github.com/STAIxBWLB/dotfiles) (the `dot` binary) then sets up each user's own environment inside that account, on servers as well as on macOS and Linux workstations. -- **One host, declaratively.** `rootfiles apply` converges a host to a profile or site config through 13 modules. `check` reports drift without changing anything, `doctor` checks live health (SSH access, disks, reboot, GPUs), and every file a run changes is backed up for `rollback`. +- **One host, declaratively.** `rootfiles apply` converges a host to a profile or site config through 14 modules. `check` reports drift without changing anything, `doctor` checks live health (SSH access, disks, reboot, GPUs), and every file a run changes is backed up for `rollback`. - **A fleet over SSH.** `rootfiles fleet` runs `status`, `check` and `doctor` in parallel and `apply`, `update`, `bootstrap` and `schedule` serially from an operator's Mac or Linux machine. Config fingerprints show which configuration each host applied, and `fleet update` flags hosts that still need `apply`. +- **Tenant containers.** `rootfiles tenant` gives outside groups a long-lived SSH container each, with a size-capped volume, accounts and host keys that survive image changes, and GPUs from the shared allocation ledger ([guide](docs/tenant.md)). - **Monitoring.** Node and DCGM GPU exporters on each host, an optional Prometheus, Alertmanager and Grafana hub, and scheduled `check`/`doctor` reports exported as metrics. ``` @@ -146,6 +147,7 @@ All modules are idempotent and support `--dry-run`. | `storage` | RAID/NVMe directory setup, symlinks | | `network` | UFW firewall, port rules (SSH port always allowed) | | `monitoring` | Opt-in Prometheus node exporter and DCGM GPU exporter, plus an optional Prometheus/Alertmanager/Grafana hub that reloads on rule changes | +| `tenant` | Opt-in host prep for `rootfiles tenant`: GPU device guard timer, daily tenant password sync, fstrim, operator sudoers rule, tenant volume mount units | Modules run in this order; `users` precedes `ssh` so declared operators exist before password auth is turned off. @@ -488,6 +490,18 @@ MIG mode and instances (read-only; repartition with `nvidia-smi mig`): rootfiles gpu mig status ``` +### Tenant containers + +Long-lived SSH containers for outside groups, one per tenant, with a size-capped volume and GPUs from the allocation ledger. Full guide: [docs/tenant.md](docs/tenant.md). + +```bash +sudo rootfiles apply --module tenant # once per host (modules.tenant.enabled) +sudo rootfiles tenant create est --image IMAGE --ssh-port 3889 --gpus 2,3 --size 2T --account est:1000 --sudo +sudo rootfiles tenant image est IMAGE2 # recreate; data, passwords, host key and GPUs kept +sudo rootfiles tenant volume resize est 3T # grows online +sudo rootfiles tenant status +``` + ### Cloudflare tunnel + VLAN ```bash @@ -562,10 +576,11 @@ Interactive blueprint: [docs/architecture/rootfiles-v2-rendered.html](docs/archi cmd/rootfiles/ Entry point internal/ cli/ Cobra commands (apply, backup, check, config, doctor, gpu, rollback, - schedule, status, tunnel, update, user) + root/lock preflight + schedule, status, tenant, tunnel, update, user) + root/lock preflight config/ YAML profiles (tree-merged extends), validation, system detector profiles/ Embedded profile YAMLs (go:embed) - module/ 13 modules implementing the Module interface, doctor checks + module/ 14 modules implementing the Module interface, doctor checks + tenant/ Tenant specs, containers, volumes, passwords, import, audit exec/ Shell runner (dry-run aware, per-run file backups), APT wrapper state/ Applied-state record, history, global lock ui/ Interactive prompts (Charm huh) + shared output styling @@ -590,6 +605,7 @@ Every push and pull request runs: | `module` | 2 OS × 9 modules + GPU on DGX mock (isolated module execution) | | `scenario` | 14 E2E flows: dry-run-all-profiles, user backup/restore, user rehome, user list names, user lifecycle, user quota, declarative accounts, tunnel setup/teardown, OS reinstall recovery, system backup, GPU allocation, status, SSH lockout guard, state/rollback/lock | | `fleet-scenario` | Fleet controller against two SSH containers (`tests/scenarios/fleet.sh`): status, serial stop rule, `--skip-unreachable` | +| `tenant-scenario` | Tenant lifecycle on the runner's Docker and systemd (`tests/scenarios/tenant-lifecycle.sh`): temporary password login, recreate, quota and growth, operator delegation, dry run, import | | `rocky` | Rocky 8.9 / 8.10 / 9 images: native core profile, SSH and installer (`tests/run-rocky.sh`) | | `monitoring-hub` | Generated monitoring hub acceptance (`tests/scenarios/monitoring-hub.sh`) | | `release` | GoReleaser on every `v*` tag push, after the tagged commit's main-push Test run succeeded (Test does not run for tags); publishes `docs/releases/.md` as the release body and fails when it is missing or fails `scripts/check-release-notes.sh` (format: `AGENTS.md`, Release notes) | diff --git a/docs/rocky-support.md b/docs/rocky-support.md index 3868fe6..d5495c7 100644 --- a/docs/rocky-support.md +++ b/docs/rocky-support.md @@ -10,7 +10,7 @@ The Rocky profile targets Rocky Linux 8.9, 8.10, and Rocky Linux 9. Other Rocky | `users` | Supported | Administrator membership uses `wheel`; existing homes, keys, passwords, and other groups are preserved. | | `ssh` | Supported | If the main sshd config lacks `/etc/ssh/sshd_config.d/*.conf`, Apply backs it up and adds a global Include at the top while preserving its mode. It validates `sshd -t` and effective settings before reload, restoring the main config and managed drop-in on validation failure. SSH port changes require a compatible SELinux `ssh_port_t` label and unrestricted TCP allowances for both address families in UFW, or an allowance in every active firewalld zone. Source-, family-, zone-limited, or complex deny/rich-rule policies need operator preparation and console verification; rootfiles never opens or broadens firewall policy. | | `security` | Supported with limits | `dnf-automatic-install.timer` applies security updates only from Rocky `baseos` and `appstream`, excludes NVIDIA/CUDA packages, and sets reboot to `never`. Check reads and parses the DNF cache without refreshing it; expired but parseable metadata does not make a correct policy appear drifted, while absent or malformed required metadata reports pending refresh. Apply refreshes metadata and strictly verifies fresh `updateinfo` for both required repositories before writing policy. This Check behavior does not promise current patches. Other enabled repositories without usable `updateinfo` produce an explicit coverage warning on Apply; they are not covered by the security-only guarantee. `chronyd` provides time synchronization. `fail2ban` is rejected unless a later release verifies an explicit EPEL configuration. | -| `docker`, `nvidia`, `gpu`, `cloudflared`, `storage`, `network`, `monitoring` | Unsupported | Rocky capability validation rejects these enabled modules before module execution. No firewall, container, GPU, tunnel, storage, or monitoring vendor stack is changed. | +| `docker`, `nvidia`, `gpu`, `cloudflared`, `storage`, `network`, `monitoring`, `tenant` | Unsupported | Rocky capability validation rejects these enabled modules before module execution, and `rootfiles tenant` refuses to run. No firewall, container, GPU, tunnel, storage, or monitoring vendor stack is changed. | Use `--profile rocky` as the starting point for a Rocky host. Review package names and SSH settings for the target before applying. A reduced configuration does not enable unsupported modules; requesting one produces an explicit capability error. diff --git a/docs/specs/2026-10-20-tenant-containers-design.md b/docs/specs/2026-10-20-tenant-containers-design.md new file mode 100644 index 0000000..53f931a --- /dev/null +++ b/docs/specs/2026-10-20-tenant-containers-design.md @@ -0,0 +1,171 @@ +# Tenant containers design + +Source: STAIxBWLB/rootfiles#78 (the owner authorized implementation and deployment on 2026-10-20). User guide: [docs/tenant.md](../tenant.md). + +## Context + +GPU hosts give external organizations a long-lived Docker container each. Tenants SSH into their container through a published host port. Today this is hand-managed with docker compose files, and it fails in ways rootfiles can prevent. + +- verified 2026-10-20: every container started with `--gpus` on the runc runtime lost GPU access after a `systemctl daemon-reload`. New CUDA processes failed with `Failed to initialize NVML: Unknown Error`. The reload came from an unattended systemd upgrade. A container started with `--runtime=nvidia` kept access. +- verified 2026-10-20: in the OCI spec, `--runtime=nvidia` lists `/dev/nvidia*` under `linux.devices` with cgroup allow rules. The runc + `--gpus` spec has no device entries, because the prestart hook injects the devices. +- verified 2026-10-20: `systemctl set-property --runtime docker-.scope DeviceAllow=...` restores access without a restart. It survives later reloads and goes away with the container. A host-local guard timer built on this healed four containers. +- verified 2026-10-20: one tenant holds about 0.9 TiB in the container writable layer (`/home`, `/tmp`). Recreating the container to change its image or GPUs would lose that data. +- verified 2026-10-20: the tenant user's password and the sshd host keys live only in the container. A new image changes both. +- verified 2026-10-20: the data volume is ext4 without the `quota`/`project` features, and Docker uses overlay2. So `--storage-opt size` is unavailable, and directory project quota needs an unmount. +- verified 2026-10-20: the Docker `local` volume driver rejects `o=loop` (`loop: invalid argument`). A volume on a pre-attached loop device (`losetup` + `--opt device=/dev/loopN`) stopped writes at its 1 GiB size and grew to 2 GiB online with `losetup -c` and `resize2fs`. +- verified 2026-10-20 (kernel 6.8, loop-mounted image bound as a Docker volume, writer running): growing ext4 and xfs from 1 to 2 GiB took effect in the container at once, and the writer kept running. Online shrink failed on both (`On-line shrinking not supported`; xfs experimental shrink returned `Invalid argument`). ext4 shrank offline (unmount, `e2fsck`, `resize2fs`, `truncate`) with data intact. xfs cannot shrink. With `discard`, deleted files returned space to the image at once. +- verified 2026-10-20: no open-source tool fits a single-host Docker setup with long-lived SSH tenant containers, in-container accounts and per-volume quota. Incus needs a runtime change. Backend.AI and Coder are session or workspace platforms without ext4 volume quota. The loopback and xfs volume plugins were last updated in 2019 and 2018. +- rootfiles already has a GPU allocation ledger (`internal/module/gpu.go`), password issuance with `--expire` (`internal/module/users.go`), a precedent for limited sudo (`internal/module/fleet_sudo.go`), systemd unit writing, and a lock and dry-run Runner. + +Implementation starts only on the owner's go-ahead. + +## Decisions + +| Decision | Owner | Limits | Revisit when | +|---|---|---|---| +| Build it inside rootfiles as `rootfiles tenant`, plus a host-prep module, not as a separate tool | AI proposes, user picks | No portal, SSO, reservations or scheduling. One host per invocation, the same as `user` and `gpu assign` | A web portal or self-service sign-up becomes a requirement | +| Quota is per tenant Docker volume, backed by a sparse image file on a loop device. Host user quota is not used | user | Writable layer is unbounded on ext4 hosts and gets a usage warning only | Hosts move to xfs+pquota (then add `--storage-opt size`) | +| GPU containers run with `runtime: nvidia` plus device IDs | AI proposes, user picks | The guard timer stays as a fallback for other containers | A future toolkit or Docker release makes runc + `--gpus` reload-safe | +| The NVIDIA attach method is declared per tenant (`gpu.runtime`, required) and set per container with `--runtime=nvidia`. The daemon `default-runtime` is not changed | user | Accepted value today: `nvidia`. Containers on runc + `--gpus` are reported as drift and converted on the next `apply` | CDI device names (`nvidia.com/gpu=`) become the preferred path in Docker and the toolkit | +| Existing tenants are migrated by deleting data after their external backup and cleanup, then copying the remainder into a new quota volume | user | A tenant's remaining data must fit the host's free space with margin | The cleanup does not get the data under the agreed size | + +## Design + +- **Tenant spec** `/.rootfiles/tenants/.yaml`. It is root-owned and passes the trusted metadata check. Fields: + - `image` + - `gpu.runtime` (required when `gpu.devices` is set; only `nvidia` is accepted) + - `gpu.devices: [uuids]` (an index given at `create` is resolved to the GPU UUID and stored as the UUID, because indices can change) + - `ssh_port` + - `ports` (extra) + - `volume.size` + - `volume.fs` (ext4 by default, xfs optional) + - `users`: name, uid, gid, sudo + - `operators` (host accounts allowed limited actions) + - `extra_binds` (legacy paths) +- **Layout**: + - `/raid/tenants//volume.img` (sparse) + - `/raid/tenants//vol` (loop mount through a generated systemd `.mount` unit that starts at boot before docker) + - `/raid/tenants//system` (root 0700: host keys, password hashes, user list) + - Docker named volume `tenant-` (`type=none,o=bind,device=/raid/tenants//vol`). Its subdirectories `data`, `home` and `tmp` are mounted at `/data`, `/home` and `/tmp`. `system` is mounted read-only at `/etc/tenant-system`. +- **Image-agnostic boot contract**: rootfiles bind-mounts a read-only entrypoint. On every start it: + - creates the users and groups; + - applies the password hashes with `chpasswd -e`; + - restores the host keys; + - starts sshd; + - runs `/data/.tenant/autostart.sh` as the tenant user if it exists; + - then sleeps. + + The only requirement on an image is `openssh-server`. +- **Containers** are created by `docker run`, not compose. Settings: `--runtime=nvidia --gpus device=`, `-e NVIDIA_DRIVER_CAPABILITIES=compute,utility`, `--restart=always`, `--ipc=host`, labels `rootfiles.tenant=`, `rootfiles.tenant.spec=` and `rootfiles.tenant.gpu.runtime=nvidia`. Check compares the label sha with the spec and reports drift. +- **GPU attach check** (Check, and after every create, recreate and restart): + - `HostConfig.Runtime` must be `nvidia`; + - `HostConfig.DeviceRequests` device IDs must equal the spec UUIDs; + - the OCI spec `linux.devices` must list `/dev/nvidiactl`, `/dev/nvidia-uvm` and one `/dev/nvidia` per assigned GPU. + + A mismatch is drift: Check reports it, and `apply` recreates the container. A tenant container on runc + `--gpus` fails the check, because its devices come only from the prestart hook and are lost on `daemon-reload` (verified 2026-10-20). `tenant import` converts such containers. +- **Passwords**: + - `tenant passwd ` generates a password, pipes it to `chpasswd` inside the container, runs `chage -d 0` and stores the hash in `system`. The temporary password is printed once and never logged or put on a command line. + - `tenant reset` is the same flow. + - Before every recreate, and once a day, the in-container shadow hashes are synced back to `system`. A direct bind of `/etc/shadow` is rejected because `passwd` renames the file. +- **Recreate** on an image, GPU or port change: + - sync hashes, then stop; + - rename the old container to `-prev` with restart=no; + - run the new container, then verify GPU (`nvidia-smi -L`), sshd banner and host key fingerprint; + - on failure, roll back automatically. + + Before acting, it prints the old writable-layer size as a loss warning. +- **Volume quota**: + - `tenant volume resize ` grows the volume online (`truncate`, `losetup -c`, then `resize2fs` or `xfs_growfs`). Shrinking while running is refused. `--offline` (ext4 only) stops the container, then runs `e2fsck`, `resize2fs` and `truncate`. + - Check reports used/size per volume and the sum of committed sizes against free space on the backing filesystem, and warns on overcommit. + - A weekly `fstrim` gives deleted space back to the backing file. +- **GPU ledger**: tenant allocations go into the existing `gpu-allocations.json` with a `tenant:` owner. Overlaps with users or other tenants are rejected. +- **Host-prep module** `tenant` (off by default): + - installs the gpu device guard timer (`systemctl set-property --runtime` DeviceAllow heal, every minute); + - writes the volume mount units; + - writes one sudoers rule, `%rootfiles-tenant-ops ALL=(root) NOPASSWD: /usr/local/bin/rootfiles tenant ops *`. + + `tenant ops` checks `SUDO_USER` against the spec's `operators` and allows only `status|logs|gpu-heal|restart|reset`. +- **Audit**: one JSONL line per mutating tenant command in `/var/log/rootfiles-tenant.jsonl` (time, actor, tenant, action, result). +- **Legacy import** `tenant import --from `: + - writes the spec from `docker inspect`; + - copies the host keys and user hashes; + - copies the data the tenant kept after cleanup into the new volume, with `rsync -aHAX` and a size check before the copy; + - copies the writable-layer `/home` and `/tmp` keep-list. + + The old container is kept stopped as a rollback until the operator removes it. +- **Rejected alternatives**: + - A separate tool would re-implement the ledger, passwords, lock, release and tests, and would split GPU allocation across two ledgers. + - Incus would replace Docker images and compose assets, and would add a second runtime next to existing Docker services. + - Direct per-directory ext4 project quota needs a host-wide unmount. + +## Changes + +- `internal/tenant/` (new): spec load/validate, container create/recreate/rollback, entrypoint asset, password sync, volume image and mount units, ledger access, audit. +- `internal/cli/tenant.go` (new): `tenant create|apply|status|list|user add|passwd|reset|gpu|image|port|volume resize|restart|logs|import|ops`. Register the mutating ones in `mutatingCommands`. +- `internal/module/tenant.go` (new): host-prep module (guard timer, mount units, sudoers). Add it to `defaultOrder` and `NewRegistry`, disabled unless `tenant.enabled`. +- `internal/module/gpu.go`: allow a `tenant:` owner in the ledger, with overlap checks across users and tenants. +- `internal/config/config.go`, `validate.go`: `tenant.enabled`, `tenant.root` (default `/raid/tenants`), `tenant.ops_group`. +- `tests/scenarios/`: a tenant lifecycle scenario on the Docker test image (GPU steps stubbed). +- `docs/`: tenant guide, config reference, release notes. + +## Acceptance criteria + +- [ ] AC1: `tenant create` with image, 1 GPU, SSH port, one user and a 10 GiB volume gives a running container. A temporary password logs in over SSH and is forced to change at first login. +- [ ] AC2: changing `image` and running `tenant apply` recreates the container. Volume data, the user's changed password, the host key fingerprint and GPU access all survive. +- [ ] AC3: writes past the volume size fail with ENOSPC. `tenant volume resize` raises the limit while the container runs. +- [ ] AC4: after `systemctl daemon-reload`, `nvidia-smi` in a tenant container still works. +- [ ] AC4a: `tenant status` shows the attach method (`runtime=nvidia`, device UUIDs, devices found in the OCI spec). A container started by hand with runc + `--gpus` is reported as drift, and `apply` recreates it with `--runtime=nvidia`. +- [ ] AC4b: `create` without `gpu.runtime` when `gpu.devices` is set, or with any value other than `nvidia`, fails validation. +- [ ] AC5: a GPU already assigned to a user or another tenant is rejected at `create` and `gpu` time. +- [ ] AC6: an account in the ops group without sudo can run `rootfiles tenant ops status|restart|reset` only for tenants that list it. Any other action or tenant is refused, and every attempt is in the audit log. +- [ ] AC7: no temporary password appears in the audit log, `/var/log/rootfiles.log`, or process arguments. +- [ ] AC8: after a reboot the volume mount units bring every tenant volume back before docker starts its containers. +- [ ] AC9: `tenant import` refuses to start when the data to copy exceeds the new volume size or the free space. A dry-run prints the plan. +- [ ] AC10: `--dry-run` on every mutating tenant command changes nothing. + +## Tasks + +- [ ] T1: promote this spec to `docs/specs/YYYY-MM-DD-tenant-containers-design.md` on the implementation branch + - Done when: the spec file is in the first PR and this issue links it +- [ ] T2: spec, config and ledger owner type + - Done when: `go test ./internal/config ./internal/module -run 'Tenant|GPU'` passes (AC5) +- [ ] T3: volume image, mount unit, resize and overcommit check + - Done when: scenario test fills the volume to ENOSPC, resizes, and survives a unit restart (AC3, AC8) +- [ ] T4: entrypoint contract, create and recreate with rollback + - Done when: scenario covers AC1 and AC2 with a stubbed GPU check +- [ ] T5: passwords and shadow sync + - Done when: AC1, AC2 and AC7 pass, and grep finds no secret in logs or `ps` captures +- [ ] T5a: explicit GPU attach (spec field, UUID resolution, attach check, drift handling) + - Done when: unit tests for validation (AC4b) and an OCI spec fixture check pass; AC4a scenario passes with a stubbed runtime +- [ ] T6: host-prep module (guard timer, sudoers) and `tenant ops` + - Done when: AC4 is verified on a GPU host, and the AC6 scenario passes +- [ ] T7: `tenant import` + - Done when: AC9 dry-run and refusal tests pass + - Needs judgment: the owner confirms each tenant's external backup and cleanup before the real import +- [ ] T8: docs and release notes, then a pilot on one GPU host + - Needs judgment: the owner picks the pilot host and tenant + +## Out of scope + +- Web portal, SSO, reservations, GPU scheduling or fractional GPU. These follow the separate Kubernetes plan. +- Multi-host placement. `fleet` only ships the declarative `tenant` module config. +- A writable-layer size limit on ext4 hosts. +- Single-port SSH routing (for example sshpiper). It is a follow-up issue if the per-tenant ports become a problem. +- Host-specific migration details. They live in the operator workspace (`admin/computing/infra/261020-plan-tenant-container-manager.md`). + + + + +## Implementation notes (changes from the issue) + +- Config lives under `modules.tenant` (`enabled`, `root`, `ops_group`). The default operator group is `rootfiles-tenant-ops`. +- The container mounts the volume subdirectories with `--mount type=bind` instead of a named Docker volume `tenant-`. A bind mount fails when its source is missing, so a container cannot start on the bare mount point of an unmounted volume and write past the quota. The named volume added nothing else. +- The volume image is mounted by a systemd `.mount` unit with `loop,discard,nodev,nosuid`, `Before=docker.service` and `WantedBy=docker.service`. `nosuid,nodev` and a root-only `/` keep host users away from setuid files a tenant leaves in its volume. +- Stored password hashes only seed accounts that have no usable password (a fresh container). A password changed inside a container therefore survives a plain restart even before the daily sync. +- The daily sync is `rootfiles tenant sync`, run by `rootfiles-tenant-sync.timer`. `tenant list` prints tenant names. +- A recreate refuses while `tenant--prev` still exists, so an earlier rollback copy is never discarded silently. +- `tenant ops` refuses every flag and clears `ROOTFILES_*` variables before loading the host's applied config, because the sudoers wildcard would otherwise let an operator point `--config` at a spec they control. +- Import requires the legacy container to be running (accounts, hashes and the overlay merged directory are read from it) and keeps the tenant's passwords by copying the hashes. +- Space return uses the `discard` mount option plus the stock `fstrim.timer`. +- The module installs the GPU device guard verified on dgx02 as `/usr/local/sbin/rootfiles-gpu-guard`. +- AC4 rests on the nvidia runtime behavior verified on 2026-10-20; CI has no GPU, so the tenant scenario covers AC1-AC3 and AC6-AC10 and unit tests cover AC5. diff --git a/docs/tenant.md b/docs/tenant.md new file mode 100644 index 0000000..9d2d022 --- /dev/null +++ b/docs/tenant.md @@ -0,0 +1,96 @@ +# Tenant containers + +`rootfiles tenant` runs long-lived containers for outside groups on a GPU host. Each tenant gets one Docker container with sshd on its own host port, a size-capped volume, accounts whose passwords and host keys survive image changes, and GPUs recorded in the same ledger as `rootfiles gpu assign`. + +## Host preparation + +Enable the module in the site config and apply it once per host: + +```yaml +modules: + tenant: + enabled: true + root: /raid/tenants # default; volume images, mount points, system files + ops_group: rootfiles-tenant-ops # default; members may run `sudo rootfiles tenant ops` +``` + +```bash +sudo rootfiles apply --module tenant +``` + +The module installs: + +- `rootfiles-gpu-guard.timer`: every minute, restores GPU access of running `--gpus` containers whose device filter a `systemctl daemon-reload` rebuilt without their `/dev/nvidia*` nodes (`systemctl set-property --runtime ... DeviceAllow`). Tenant containers use the nvidia runtime and do not need it; it covers the other GPU containers on the host. +- `rootfiles-tenant-sync.timer`: daily `rootfiles tenant sync`, which saves the password hashes of running tenants. +- `fstrim.timer`, so space deleted inside a volume returns to the host. +- `/etc/sudoers.d/rootfiles-tenant-ops`: `% ALL=(root) NOPASSWD: /usr/local/bin/rootfiles tenant ops *`, validated with `visudo`. +- A mount unit per existing tenant volume (rewritten after an OS reinstall). + +GPU tenants need the `nvidia` module (Docker's `nvidia` runtime). + +## Create a tenant + +```bash +sudo rootfiles tenant create est \ + --image registry.local/est:2026.10 --ssh-port 3889 --gpus 2,3 --size 2T \ + --account est:1000 --sudo --operator alice +``` + +This writes `/.rootfiles/tenants/est.yaml`, creates the volume, starts `tenant-est`, checks sshd, the host key and the GPUs, and prints a temporary password per account. The password must be changed at first login. It is printed once and never written to a log or passed on a command line. + +The image needs only `openssh-server` (and `useradd`, `chpasswd`, which Debian, Ubuntu and RHEL images have). rootfiles starts the container with its own read-only entrypoint, `/etc/tenant-system/init.sh`, which on every start creates the accounts, seeds stored password hashes into accounts that have none, restores the host keys, starts sshd and runs `/data/.tenant/autostart.sh` as the first account when that file exists. + +| Inside the container | Host | Limit | +|---|---|---| +| `/data`, `/home`, `/tmp` | `//vol/{data,home,tmp}` on a loop-mounted image | `volume.size` | +| `/etc/tenant-system` (read-only) | `//system`: accounts, password hashes, host keys | root only | +| everything else | container writable layer | none on ext4 hosts; keep data in the volume | +| `extra_binds` | as given | none | + +## Change a tenant + +| Change | Command | Container | +|---|---|---| +| Image | `rootfiles tenant image est IMAGE` | recreated | +| GPUs | `rootfiles tenant gpu est --gpus 0,1` (or `none`) | recreated | +| SSH port | `rootfiles tenant port est 3890` | recreated | +| Account | `rootfiles tenant user add est bob [--sudo]` | kept, account added live | +| Password | `rootfiles tenant passwd est bob` (`reset` is the same) | kept | +| Volume size | `rootfiles tenant volume resize est 3T` | kept (grows online) | +| Anything in the spec | edit the YAML, then `rootfiles tenant apply est` | as needed | + +A recreate saves the current password hashes, prints the size of the old writable layer, renames the old container to `tenant-est-prev` (restart policy `no`), starts the new one and verifies sshd, the stored host key and the GPUs. On failure it removes the new container and restarts the old one. `tenant-est-prev` stays until you remove it; the next recreate refuses while it exists. + +Shrinking a volume needs `--offline` and ext4: it stops the container, checks the filesystem, shrinks it and starts the container again. xfs volumes cannot shrink. + +`rootfiles tenant status` shows each tenant's state, spec drift, volume use and the sum of volume sizes against the free space of the tenant root (sparse images can be overcommitted). `rootfiles tenant status est` adds the writable-layer size. + +## Operators + +Host accounts in `ops_group` that a spec lists under `operators` can, without other sudo rights: + +```bash +sudo rootfiles tenant ops est status|logs|gpu-heal|restart +sudo rootfiles tenant ops est reset USER +``` + +`tenant ops` takes no flags, ignores `ROOTFILES_*` variables and reads only the host's applied configuration. Creating tenants and changing images, GPUs, ports or volumes stay with root. + +## Audit + +Every real run of a changing tenant command appends one JSON line to `/var/log/rootfiles-tenant.jsonl` (time, actor, tenant, action, result). Refused operator attempts are recorded too. `--dry-run` changes nothing and records nothing. + +## Import a hand-made container + +```bash +sudo rootfiles tenant import est --from est-workspace --size 2T \ + --data /raid/data/devcon/est/data --keep /home/est --dry-run +``` + +Import reads the running container: image, published ports (22 becomes the SSH port), GPU IDs, bind mounts (kept as `extra_binds` unless they target `/data`, `/home` or `/tmp`), accounts with a login shell and uid 1000-65533, their sudo membership, password hashes and host keys. It copies `--data` and the `--keep` paths with `rsync -aHAX`, stops the old container with restart policy `no`, and starts the tenant container. Accounts keep their passwords. Before any change it refuses when the data does not fit the volume with 5% headroom or the free space; `--dry-run` prints the plan. Ask the tenant to stop writing first. Remove the old container once the tenant confirms. + +## Limits + +- One host per command; `fleet` only ships the module config. +- The writable layer has no size limit on ext4 hosts. +- Not supported on Rocky Linux. diff --git a/internal/cli/configcmd.go b/internal/cli/configcmd.go index 8f6713e..c55e25e 100644 --- a/internal/cli/configcmd.go +++ b/internal/cli/configcmd.go @@ -44,7 +44,7 @@ func loadForInspection(cmd *cobra.Command) (*config.Config, string, error) { } if sys.OS == "rocky" { names := []string{} - for _, name := range []string{"locale", "system", "packages", "users", "ssh", "security", "docker", "nvidia", "gpu", "cloudflared", "storage", "network", "monitoring"} { + for _, name := range []string{"locale", "system", "packages", "users", "ssh", "security", "docker", "nvidia", "gpu", "cloudflared", "storage", "network", "monitoring", "tenant"} { if cfg.IsModuleEnabled(name) { names = append(names, name) } diff --git a/internal/cli/root.go b/internal/cli/root.go index 9760df4..46d7b1b 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -49,6 +49,7 @@ func NewRootCmd(version, commit string) *cobra.Command { root.AddCommand(newDoctorCmd()) root.AddCommand(newScheduleCmd()) root.AddCommand(newFleetCmd()) + root.AddCommand(newTenantCmd()) markMutating(root) return root diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go index 98fe122..8fbfb87 100644 --- a/internal/cli/runtime.go +++ b/internal/cli/runtime.go @@ -34,6 +34,10 @@ var mutatingCommands = [][]string{ {"gpu", "assign"}, {"gpu", "revoke"}, {"tunnel", "install"}, {"tunnel", "setup"}, {"tunnel", "restart"}, {"tunnel", "update"}, {"tunnel", "uninstall"}, + {"tenant", "create"}, {"tenant", "apply"}, {"tenant", "user", "add"}, + {"tenant", "passwd"}, {"tenant", "reset"}, {"tenant", "gpu"}, {"tenant", "image"}, + {"tenant", "port"}, {"tenant", "volume", "resize"}, {"tenant", "restart"}, + {"tenant", "import"}, {"tenant", "ops"}, {"tenant", "sync"}, } func markMutating(root *cobra.Command) { diff --git a/internal/cli/tenant.go b/internal/cli/tenant.go new file mode 100644 index 0000000..f6f6df2 --- /dev/null +++ b/internal/cli/tenant.go @@ -0,0 +1,595 @@ +package cli + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/STAIxBWLB/rootfiles/internal/config" + "github.com/STAIxBWLB/rootfiles/internal/module" + "github.com/STAIxBWLB/rootfiles/internal/tenant" +) + +func newTenantCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "tenant", + Short: "Manage long-lived tenant containers (volume quota, accounts, GPUs)", + Long: `Each tenant gets one Docker container with SSH on its own host port, a +size-capped volume (/data, /home, /tmp) and accounts whose passwords and +host keys survive image changes. Specs live in /.rootfiles/tenants.`, + } + cmd.AddCommand( + newTenantListCmd(), newTenantStatusCmd(), newTenantCreateCmd(), newTenantApplyCmd(), + newTenantUserCmd(), newTenantPasswdCmd("passwd"), newTenantPasswdCmd("reset"), + newTenantGPUCmd(), newTenantImageCmd(), newTenantPortCmd(), newTenantVolumeCmd(), + newTenantRestartCmd(), newTenantLogsCmd(), newTenantImportCmd(), newTenantOpsCmd(), + newTenantSyncCmd(), + ) + return cmd +} + +// tenantEnv loads the run context and a manager after the tenant checks. +func tenantEnv(cmd *cobra.Command) (*module.RunContext, *tenant.Manager, error) { + rc, err := buildRunContext(cmd) + if err != nil { + return nil, nil, err + } + if config.IsRocky(rc.Config.System) { + return nil, nil, fmt.Errorf("tenant containers are not supported on Rocky Linux") + } + if err := module.RequireTrustedMetadata(rc); err != nil { + return nil, nil, err + } + root := rc.Config.Modules.Tenant.DataRoot() + if err := config.RootOnlyHomeBase("/", root, 0); err != nil { + return nil, nil, fmt.Errorf("refusing tenant root %s: %w", root, err) + } + return rc, module.NewTenantManager(rc, cmd.OutOrStdout()), nil +} + +// tenantDo runs fn for tenant t and records a real run in the audit log. +func tenantDo(cmd *cobra.Command, t, action string, fn func(ctx context.Context, rc *module.RunContext, m *tenant.Manager) error) error { + rc, m, err := tenantEnv(cmd) + if err != nil { + return err + } + err = fn(cmd.Context(), rc, m) + if !rc.DryRun { + recordTenant(t, action, err) + } + return err +} + +func recordTenant(t, action string, err error) { + actor, _ := tenant.Actor() + result := "ok" + if err != nil { + result = "error: " + strings.SplitN(err.Error(), "\n", 2)[0] + } + if aerr := tenant.Audit(actor, t, action, result); aerr != nil { + fmt.Fprintf(os.Stderr, "⚠ tenant audit log: %v\n", aerr) + } +} + +// editTenant loads t, applies edit, saves and converges it. +func editTenant(cmd *cobra.Command, t, action string, edit func(*tenant.Spec) error) error { + return tenantDo(cmd, t, action, func(ctx context.Context, _ *module.RunContext, m *tenant.Manager) error { + s, err := m.Load(t) + if err != nil { + return err + } + if err := edit(s); err != nil { + return err + } + if err := m.SaveSpec(s); err != nil { + return err + } + return m.Apply(ctx, s) + }) +} + +func issuePassword(ctx context.Context, cmd *cobra.Command, m *tenant.Manager, s *tenant.Spec, user string) error { + pw, err := m.Passwd(ctx, s, user) + if err != nil { + return err + } + if pw == "" { + fmt.Fprintf(cmd.OutOrStdout(), "[dry-run] would issue a temporary password for %s@%s\n", user, s.Name) + return nil + } + fmt.Fprintf(cmd.OutOrStdout(), "Temporary password for %s@%s: %s (must be changed at first login; not stored or logged)\n", user, s.Name, pw) + return nil +} + +func newTenantListCmd() *cobra.Command { + return &cobra.Command{ + Use: "list", + Short: "List tenant names", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + _, m, err := tenantEnv(cmd) + if err != nil { + return err + } + names, err := tenant.ListSpecs(m.SpecDir) + for _, n := range names { + fmt.Fprintln(cmd.OutOrStdout(), n) + } + return err + }, + } +} + +func newTenantStatusCmd() *cobra.Command { + return &cobra.Command{ + Use: "status [TENANT]", + Short: "Show tenant containers, spec drift and volume use", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + _, m, err := tenantEnv(cmd) + if err != nil { + return err + } + names := args + if len(names) == 0 { + if names, err = tenant.ListSpecs(m.SpecDir); err != nil { + return err + } + } + return m.Status(cmd.Context(), names, len(args) == 1) + }, + } +} + +func newTenantCreateCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "create TENANT", + Short: "Create a tenant: volume, container and temporary passwords", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + t := args[0] + s := &tenant.Spec{Name: t} + s.Image, _ = cmd.Flags().GetString("image") + s.SSHPort, _ = cmd.Flags().GetInt("ssh-port") + s.Volume.Size, _ = cmd.Flags().GetString("size") + s.Volume.FS, _ = cmd.Flags().GetString("fs") + s.Ports, _ = cmd.Flags().GetStringArray("port") + s.Operators, _ = cmd.Flags().GetStringArray("operator") + s.ExtraBinds, _ = cmd.Flags().GetStringArray("bind") + if g, _ := cmd.Flags().GetString("gpus"); g != "" { + gpus, err := parseGPUList(g) + if err != nil { + return err + } + s.GPUs = gpus + } + sudo, _ := cmd.Flags().GetBool("sudo") + accounts, _ := cmd.Flags().GetStringArray("account") + for i, a := range accounts { + u, err := parseAccount(a, 1000+i, sudo) + if err != nil { + return err + } + s.Users = append(s.Users, u) + } + if err := s.Validate(); err != nil { + return err + } + return tenantDo(cmd, t, "create", func(ctx context.Context, _ *module.RunContext, m *tenant.Manager) error { + if _, err := os.Stat(m.SpecDir + "/" + t + ".yaml"); err == nil { + return fmt.Errorf("tenant %s already exists; edit it and run `rootfiles tenant apply %s`", t, t) + } + if err := m.SaveSpec(s); err != nil { + return err + } + if err := m.Apply(ctx, s); err != nil { + return err + } + for _, u := range s.Users { + if err := issuePassword(ctx, cmd, m, s, u.Name); err != nil { + return err + } + } + return nil + }) + }, + } + f := cmd.Flags() + f.String("image", "", "Container image (needs openssh-server)") + f.Int("ssh-port", 0, "Host port published to the container's sshd") + f.String("size", "", "Volume size, e.g. 500G or 2T") + f.String("fs", "ext4", "Volume filesystem: ext4 (can shrink offline) or xfs") + f.String("gpus", "", "Comma-separated GPU indices") + f.StringArray("account", nil, "Account NAME[:UID[:GID]] (repeatable; uid defaults to 1000, 1001, ...)") + f.Bool("sudo", false, "Give the accounts sudo inside the container") + f.StringArray("port", nil, "Extra published port HOST:CONTAINER[/udp] (repeatable)") + f.StringArray("operator", nil, "Host account allowed `sudo rootfiles tenant ops` on this tenant (repeatable)") + f.StringArray("bind", nil, "Extra bind SRC:DST[:ro|rw] outside the quota (repeatable)") + return cmd +} + +// parseAccount reads NAME[:UID[:GID]]. +func parseAccount(s string, defUID int, sudo bool) (tenant.User, error) { + parts := strings.Split(s, ":") + u := tenant.User{Name: parts[0], UID: defUID, Sudo: sudo} + var err error + if len(parts) > 3 { + return u, fmt.Errorf("account %q: want NAME[:UID[:GID]]", s) + } + if len(parts) > 1 { + if u.UID, err = strconv.Atoi(parts[1]); err != nil { + return u, fmt.Errorf("account %q: bad uid", s) + } + } + u.GID = u.UID + if len(parts) > 2 { + if u.GID, err = strconv.Atoi(parts[2]); err != nil { + return u, fmt.Errorf("account %q: bad gid", s) + } + } + return u, nil +} + +func newTenantApplyCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "apply [TENANT]", + Short: "Converge a tenant to its spec (recreates the container on image, GPU or port changes)", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + all, _ := cmd.Flags().GetBool("all") + if all == (len(args) == 1) { + return fmt.Errorf("give one TENANT or --all") + } + rc, m, err := tenantEnv(cmd) + if err != nil { + return err + } + names := args + if all { + if names, err = tenant.ListSpecs(m.SpecDir); err != nil { + return err + } + } + var failed []string + for _, t := range names { + s, err := m.Load(t) + if err == nil { + err = m.Apply(cmd.Context(), s) + } + if !rc.DryRun { + recordTenant(t, "apply", err) + } + if err != nil { + fmt.Fprintf(os.Stderr, "✗ %s: %v\n", t, err) + failed = append(failed, t) + } + } + if len(failed) > 0 { + return fmt.Errorf("apply failed for: %s", strings.Join(failed, ", ")) + } + return nil + }, + } + cmd.Flags().Bool("all", false, "Apply every tenant") + return cmd +} + +func newTenantUserCmd() *cobra.Command { + userCmd := &cobra.Command{Use: "user", Short: "Manage tenant accounts"} + add := &cobra.Command{ + Use: "add TENANT NAME", + Short: "Add an account and issue its temporary password", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + t, name := args[0], args[1] + uid, _ := cmd.Flags().GetInt("uid") + gid, _ := cmd.Flags().GetInt("gid") + sudo, _ := cmd.Flags().GetBool("sudo") + return tenantDo(cmd, t, "user add "+name, func(ctx context.Context, _ *module.RunContext, m *tenant.Manager) error { + s, err := m.Load(t) + if err != nil { + return err + } + if _, ok := s.User(name); ok { + return fmt.Errorf("tenant %s already has user %s", t, name) + } + if uid == 0 { + uid = 1000 + for _, u := range s.Users { + uid = max(uid, u.UID+1) + } + } + if gid == 0 { + gid = uid + } + s.Users = append(s.Users, tenant.User{Name: name, UID: uid, GID: gid, Sudo: sudo}) + if err := m.SaveSpec(s); err != nil { + return err + } + if err := m.Apply(ctx, s); err != nil { + return err + } + return issuePassword(ctx, cmd, m, s, name) + }) + }, + } + add.Flags().Int("uid", 0, "UID (default: next free from 1000)") + add.Flags().Int("gid", 0, "GID (default: the uid)") + add.Flags().Bool("sudo", false, "Give the account sudo inside the container") + userCmd.AddCommand(add) + return userCmd +} + +func newTenantPasswdCmd(use string) *cobra.Command { + return &cobra.Command{ + Use: use + " TENANT USER", + Short: "Issue a temporary password that must be changed at first login", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + t, user := args[0], args[1] + return tenantDo(cmd, t, use+" "+user, func(ctx context.Context, _ *module.RunContext, m *tenant.Manager) error { + s, err := m.Load(t) + if err != nil { + return err + } + return issuePassword(ctx, cmd, m, s, user) + }) + }, + } +} + +func newTenantGPUCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "gpu TENANT --gpus 0,1|none", + Short: "Change a tenant's GPUs (recreates the container)", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + g, _ := cmd.Flags().GetString("gpus") + var gpus []int + if g != "none" { + var err error + if gpus, err = parseGPUList(g); err != nil { + return err + } + } + return editTenant(cmd, args[0], "gpu "+g, func(s *tenant.Spec) error { + s.GPUs = gpus + return nil + }) + }, + } + cmd.Flags().String("gpus", "", "Comma-separated GPU indices, or none") + _ = cmd.MarkFlagRequired("gpus") + return cmd +} + +func newTenantImageCmd() *cobra.Command { + return &cobra.Command{ + Use: "image TENANT IMAGE", + Short: "Change a tenant's image (recreates the container)", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return editTenant(cmd, args[0], "image "+args[1], func(s *tenant.Spec) error { + s.Image = args[1] + return nil + }) + }, + } +} + +func newTenantPortCmd() *cobra.Command { + return &cobra.Command{ + Use: "port TENANT SSH_PORT", + Short: "Change a tenant's SSH host port (recreates the container)", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + port, err := strconv.Atoi(args[1]) + if err != nil { + return fmt.Errorf("invalid port %q", args[1]) + } + return editTenant(cmd, args[0], "port "+args[1], func(s *tenant.Spec) error { + s.SSHPort = port + return nil + }) + }, + } +} + +func newTenantVolumeCmd() *cobra.Command { + vol := &cobra.Command{Use: "volume", Short: "Manage tenant volumes"} + resize := &cobra.Command{ + Use: "resize TENANT SIZE", + Short: "Grow a volume online, or shrink an ext4 volume with --offline", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + offline, _ := cmd.Flags().GetBool("offline") + return tenantDo(cmd, args[0], "volume resize "+args[1], func(ctx context.Context, _ *module.RunContext, m *tenant.Manager) error { + s, err := m.Load(args[0]) + if err != nil { + return err + } + return m.Resize(ctx, s, args[1], offline) + }) + }, + } + resize.Flags().Bool("offline", false, "Stop the container to shrink an ext4 volume") + vol.AddCommand(resize) + return vol +} + +func newTenantRestartCmd() *cobra.Command { + return &cobra.Command{ + Use: "restart TENANT", + Short: "Restart a tenant container", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return tenantDo(cmd, args[0], "restart", func(ctx context.Context, _ *module.RunContext, m *tenant.Manager) error { + s, err := m.Load(args[0]) + if err != nil { + return err + } + return m.Restart(ctx, s) + }) + }, + } +} + +func newTenantLogsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "logs TENANT", + Short: "Show the container log", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + _, m, err := tenantEnv(cmd) + if err != nil { + return err + } + s, err := m.Load(args[0]) + if err != nil { + return err + } + tail, _ := cmd.Flags().GetInt("tail") + return m.Logs(cmd.Context(), s, tail) + }, + } + cmd.Flags().Int("tail", 200, "Lines to show") + return cmd +} + +func newTenantImportCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "import TENANT --from CONTAINER --size SIZE", + Short: "Move a hand-made container under rootfiles (the old one is kept stopped)", + Long: `Writes the spec from the running container, copies its host keys, password +hashes, --data and --keep paths into a new volume, stops the old container and +starts the tenant container. Ask the tenant to stop writing first. Refuses +before any change when the data does not fit the volume or the free space; +--dry-run prints the plan.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + o := tenant.ImportOptions{Tenant: args[0]} + o.From, _ = cmd.Flags().GetString("from") + o.Size, _ = cmd.Flags().GetString("size") + o.FS, _ = cmd.Flags().GetString("fs") + o.Data, _ = cmd.Flags().GetString("data") + o.Keep, _ = cmd.Flags().GetStringArray("keep") + o.Operators, _ = cmd.Flags().GetStringArray("operator") + if o.From == "" || o.Size == "" { + return fmt.Errorf("--from and --size are required") + } + return tenantDo(cmd, o.Tenant, "import "+o.From, func(ctx context.Context, _ *module.RunContext, m *tenant.Manager) error { + _, err := m.Import(ctx, o) + return err + }) + }, + } + f := cmd.Flags() + f.String("from", "", "Running legacy container") + f.String("size", "", "New volume size") + f.String("fs", "ext4", "Volume filesystem: ext4 or xfs") + f.String("data", "", "Host directory copied into the new /data") + f.StringArray("keep", nil, "Path under /home or /tmp in the old container to copy (repeatable)") + f.StringArray("operator", nil, "Host account allowed `sudo rootfiles tenant ops` (repeatable)") + return cmd +} + +func newTenantOpsCmd() *cobra.Command { + return &cobra.Command{ + Use: "ops TENANT ACTION [USER]", + Short: "Operator actions through sudo: " + strings.Join(tenant.OpsActions, "|"), + Long: `Members of the tenant operator group run this with sudo. It acts only for +tenants that list the caller in operators, takes no flags, and reads only the +host's applied configuration. Every attempt is audited.`, + Args: cobra.RangeArgs(2, 3), + RunE: func(cmd *cobra.Command, args []string) error { + t, action := args[0], args[1] + actor, viaSudo := tenant.Actor() + refuse := func(err error) error { + if aerr := tenant.Audit(actor, t, "ops "+action, "refused: "+err.Error()); aerr != nil { + fmt.Fprintf(os.Stderr, "⚠ tenant audit log: %v\n", aerr) + } + return err + } + // Flags or environment could point the config at files the + // caller controls; operators get the host's applied config only. + var set []string + cmd.Flags().Visit(func(f *pflag.Flag) { set = append(set, "--"+f.Name) }) + if len(set) > 0 { + return refuse(fmt.Errorf("tenant ops takes no flags (got %s)", strings.Join(set, " "))) + } + for _, kv := range os.Environ() { + if k, _, _ := strings.Cut(kv, "="); strings.HasPrefix(k, "ROOTFILES_") { + os.Unsetenv(k) + } + } + _, m, err := tenantEnv(cmd) + if err != nil { + return refuse(err) + } + s, err := m.Load(t) + if err == nil { + err = tenant.AuthorizeOps(s, actor, viaSudo, action) + } + if err == nil && (action == "reset") != (len(args) == 3) { + err = fmt.Errorf("reset takes a USER; other actions take none") + } + if err != nil { + return refuse(err) + } + ctx := cmd.Context() + switch action { + case "status": + err = m.Status(ctx, []string{t}, true) + case "logs": + err = m.Logs(ctx, s, 200) + case "gpu-heal": + err = m.HealGPU(ctx, s) + case "restart": + err = m.Restart(ctx, s) + case "reset": + err = issuePassword(ctx, cmd, m, s, args[2]) + action += " " + args[2] + } + recordTenant(t, "ops "+action, err) + return err + }, + } +} + +func newTenantSyncCmd() *cobra.Command { + return &cobra.Command{ + Use: "sync", + Short: "Save the password hashes of every running tenant (daily timer)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + _, m, err := tenantEnv(cmd) + if err != nil { + return err + } + names, err := tenant.ListSpecs(m.SpecDir) + if err != nil { + return err + } + var failed []string + for _, t := range names { + s, err := m.Load(t) + if err == nil { + err = m.SyncShadow(cmd.Context(), s) + } + if err != nil { + fmt.Fprintf(os.Stderr, "✗ %s: %v\n", t, err) + failed = append(failed, t) + } + } + if len(failed) > 0 { + return fmt.Errorf("password sync failed for: %s", strings.Join(failed, ", ")) + } + return nil + }, + } +} diff --git a/internal/config/config.go b/internal/config/config.go index d936b64..912a58a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -38,6 +38,35 @@ type ModulesConfig struct { Storage StorageConfig `yaml:"storage"` Network NetworkConfig `yaml:"network"` Monitoring MonitoringConfig `yaml:"monitoring"` + // omitempty keeps the effective config, and so its fingerprint, + // unchanged on hosts that never set it. + Tenant TenantConfig `yaml:"tenant,omitempty"` +} + +// TenantConfig prepares a host for `rootfiles tenant` containers. +type TenantConfig struct { + Enabled bool `yaml:"enabled"` + // Root holds each tenant's volume image, mount point and system files. + Root string `yaml:"root,omitempty"` + // OpsGroup members may run `sudo rootfiles tenant ops` for the tenants + // that list them as operators. + OpsGroup string `yaml:"ops_group,omitempty"` +} + +// DataRoot returns the tenant root, /raid/tenants when unset. +func (t TenantConfig) DataRoot() string { + if t.Root == "" { + return "/raid/tenants" + } + return t.Root +} + +// Group returns the operator group, rootfiles-tenant-ops when unset. +func (t TenantConfig) Group() string { + if t.OpsGroup == "" { + return "rootfiles-tenant-ops" + } + return t.OpsGroup } // SystemConfig covers host-level basics. @@ -244,6 +273,8 @@ func (c *Config) IsModuleEnabled(name string) bool { return c.Modules.Storage.Enabled case "network": return c.Modules.Network.Enabled + case "tenant": + return c.Modules.Tenant.Enabled default: return false } diff --git a/internal/config/distro.go b/internal/config/distro.go index f2ae945..9f33459 100644 --- a/internal/config/distro.go +++ b/internal/config/distro.go @@ -64,7 +64,7 @@ func SupportedModules(system *SystemInfo) map[string]bool { return map[string]bool{} } if d.ID != "rocky" { - all := []string{"locale", "system", "packages", "users", "ssh", "security", "docker", "nvidia", "gpu", "cloudflared", "storage", "network", "monitoring"} + all := []string{"locale", "system", "packages", "users", "ssh", "security", "docker", "nvidia", "gpu", "cloudflared", "storage", "network", "monitoring", "tenant"} result := make(map[string]bool, len(all)) for _, name := range all { result[name] = true diff --git a/internal/config/validate.go b/internal/config/validate.go index eb4c637..413cbc2 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -14,6 +14,7 @@ var ( validHostname = regexp.MustCompile(`^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$`) validSize = regexp.MustCompile(`^[1-9][0-9]*[KMGT]$`) validSysctlKey = regexp.MustCompile(`^[a-z0-9_]+(\.[a-zA-Z0-9_*-]+)+$`) + validTenantRoot = regexp.MustCompile(`^/[A-Za-z0-9_./-]+$`) ) // Validate reports semantic errors that YAML decoding cannot catch. All @@ -129,6 +130,13 @@ func (c *Config) Validate() error { add("modules.cloudflared.private_network.interface: %q is not a valid interface name", iface) } + if t := c.Modules.Tenant; t.Root != "" && (!validTenantRoot.MatchString(t.Root) || filepath.Clean(t.Root) != t.Root || t.Root == "/") { + add("modules.tenant.root: %q must be a clean absolute path of letters, digits, '_', '.', '-' and '/'", t.Root) + } + if g := c.Modules.Tenant.OpsGroup; g != "" && (!validAccountName.MatchString(g) || len(g) > 32) { + add("modules.tenant.ops_group: invalid group name %q", g) + } + if strings.ContainsAny(c.Timezone, " ") || strings.HasPrefix(c.Timezone, "/") || strings.Contains(c.Timezone, "..") { add("timezone: %q is not a zone name (e.g. Asia/Seoul)", c.Timezone) } diff --git a/internal/module/fleet_sudo.go b/internal/module/fleet_sudo.go index 2a81456..1ad5393 100644 --- a/internal/module/fleet_sudo.go +++ b/internal/module/fleet_sudo.go @@ -31,10 +31,6 @@ func fleetSudoersPath() string { return filepath.Join(sudoersDir, "rootfiles-fleet") } -func fleetSudoersTempPath() string { - return filepath.Join(sudoersDir, ".rootfiles-fleet.tmp") -} - func fleetSudoersContent(users []string) ([]byte, error) { if len(users) == 0 { return nil, nil @@ -269,33 +265,39 @@ func applyFleetSudo(ctx context.Context, rc *RunContext) (*ApplyResult, error) { } func writeFleetSudoers(ctx context.Context, rc *RunContext, content []byte) error { - tmp := fleetSudoersTempPath() + return installSudoersFile(ctx, rc, fleetSudoersPath(), content, "fleet") +} + +// installSudoersFile installs a sudoers drop-in through a temporary file +// that visudo must accept, so a broken rule never reaches sudoers.d. +func installSudoersFile(ctx context.Context, rc *RunContext, final string, content []byte, what string) error { + tmp := filepath.Join(filepath.Dir(final), "."+filepath.Base(final)+".tmp") if info, err := fleetSudoLstat(tmp); err == nil { if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { - return fmt.Errorf("unsafe temporary fleet sudoers path %s", tmp) + return fmt.Errorf("unsafe temporary %s sudoers path %s", what, tmp) } if err := rc.Runner.Remove(tmp); err != nil { - return fmt.Errorf("removing stale fleet sudoers temp file: %w", err) + return fmt.Errorf("removing stale %s sudoers temp file: %w", what, err) } } else if !os.IsNotExist(err) { - return fmt.Errorf("checking temporary fleet sudoers path: %w", err) + return fmt.Errorf("checking temporary %s sudoers path: %w", what, err) } if err := rc.Runner.WriteFile(tmp, content, 0440); err != nil { - return fmt.Errorf("writing fleet sudoers temporary file: %w", err) + return fmt.Errorf("writing %s sudoers temporary file: %w", what, err) } if rc.DryRun { - return rc.Runner.Rename(tmp, fleetSudoersPath()) + return rc.Runner.Rename(tmp, final) } if !rc.Runner.CommandExists("visudo") { _ = rc.Runner.Remove(tmp) - return fmt.Errorf("visudo is required to validate the fleet sudoers rule") + return fmt.Errorf("visudo is required to validate the %s sudoers rule", what) } if _, err := rc.Runner.Run(ctx, "visudo", "-cf", tmp); err != nil { _ = rc.Runner.Remove(tmp) - return fmt.Errorf("fleet sudoers validation failed: %w", err) + return fmt.Errorf("%s sudoers validation failed: %w", what, err) } - if err := rc.Runner.Rename(tmp, fleetSudoersPath()); err != nil { - return fmt.Errorf("installing fleet sudoers drop-in: %w", err) + if err := rc.Runner.Rename(tmp, final); err != nil { + return fmt.Errorf("installing %s sudoers drop-in: %w", what, err) } return nil } diff --git a/internal/module/gpu.go b/internal/module/gpu.go index 4983f2b..0141dfb 100644 --- a/internal/module/gpu.go +++ b/internal/module/gpu.go @@ -42,12 +42,13 @@ func (m *GPUModule) Check(_ context.Context, rc *RunContext) (*CheckResult, erro return nil, err } db, err := loadGPUDB(rc) - if err != nil || len(db.Allocations) == 0 { + allocs := userAllocations(db) + if err != nil || len(allocs) == 0 { return &CheckResult{Satisfied: true}, nil } var changes []Change - for _, alloc := range db.Allocations { + for _, alloc := range allocs { method := effectiveMethod(rc, alloc.Method) if method == "env" || method == "both" { @@ -100,7 +101,8 @@ func (m *GPUModule) Apply(ctx context.Context, rc *RunContext) (*ApplyResult, er return nil, err } db, err := loadGPUDB(rc) - if err != nil || len(db.Allocations) == 0 { + allocs := userAllocations(db) + if err != nil || len(allocs) == 0 { // Remove wrapper if it exists but no allocations remain wrapperData, _ := rc.Runner.ReadFile(dockerWrapperPath) if len(wrapperData) > 0 { @@ -113,7 +115,7 @@ func (m *GPUModule) Apply(ctx context.Context, rc *RunContext) (*ApplyResult, er var messages, warnings []string changed := false - for _, alloc := range db.Allocations { + for _, alloc := range allocs { method := effectiveMethod(rc, alloc.Method) if method == "env" || method == "both" { @@ -227,6 +229,9 @@ func RevokeGPUs(ctx context.Context, rc *RunContext, username string) error { if err := requireTrustedMetadata(rc); err != nil { return err } + if isTenantOwner(username) { + return fmt.Errorf("%s belongs to a tenant; change it with `rootfiles tenant gpu`", username) + } var method string var allocationsRemaining int if err := withGPUDBLock(rc, func(db *GPUAllocationsDB) error { @@ -244,7 +249,7 @@ func RevokeGPUs(ctx context.Context, rc *RunContext, username string) error { return fmt.Errorf("no GPU allocation found for %s", username) } db.Allocations = remaining - allocationsRemaining = len(remaining) + allocationsRemaining = len(userAllocations(db)) return nil }); err != nil { return err @@ -580,6 +585,67 @@ func removeDockerWrapper(ctx context.Context, rc *RunContext) { } } +// tenantOwnerPrefix marks ledger entries of `rootfiles tenant` containers. +// A colon never appears in an account name, so the owners cannot collide. +const tenantOwnerPrefix = "tenant:" + +func isTenantOwner(owner string) bool { return strings.HasPrefix(owner, tenantOwnerPrefix) } + +// userAllocations are the ledger entries of host accounts. +func userAllocations(db *GPUAllocationsDB) []GPUAllocation { + var out []GPUAllocation + for _, a := range db.Allocations { + if !isTenantOwner(a.Username) { + out = append(out, a) + } + } + return out +} + +// ReserveTenantGPUs records tenant's GPUs in the shared ledger, rejecting +// GPUs held by a user or another tenant. No GPUs drops the tenant's entry. +func ReserveTenantGPUs(ctx context.Context, rc *RunContext, tenant string, gpus []int) error { + if err := requireTrustedMetadata(rc); err != nil { + return err + } + owner := tenantOwnerPrefix + tenant + if len(gpus) == 0 { + db, _ := loadGPUDB(rc) + held := false + for _, a := range db.Allocations { + held = held || a.Username == owner + } + if !held { + return nil + } + } + return withGPUDBLock(rc, func(db *GPUAllocationsDB) error { + if len(gpus) == 0 { + var keep []GPUAllocation + for _, a := range db.Allocations { + if a.Username != owner { + keep = append(keep, a) + } + } + db.Allocations = keep + return nil + } + if total, model := detectGPUInfo(ctx, rc); total > 0 { + db.TotalGPUs, db.GPUModel = total, model + } + if err := validateGPURequest(db, owner, gpus); err != nil { + return err + } + upsertAllocation(db, GPUAllocation{ + Username: owner, + GPUs: gpus, + Method: "tenant", + UpdatedAt: time.Now().UTC().Format(time.RFC3339), + }) + return nil + }) +} + // --- helpers --- // validateGPURequest checks that every requested GPU index is in range and not diff --git a/internal/module/module.go b/internal/module/module.go index cd10d81..2ca993e 100644 --- a/internal/module/module.go +++ b/internal/module/module.go @@ -97,6 +97,7 @@ var defaultOrder = []string{ "storage", "network", "monitoring", + "tenant", // after docker and nvidia: tenant containers need both } // Registry manages module registration and resolution. @@ -120,6 +121,7 @@ func NewRegistry() *Registry { r.Register(NewStorageModule()) r.Register(NewNetworkModule()) r.Register(NewMonitoringModule()) + r.Register(NewTenantModule()) return r } diff --git a/internal/module/module_test.go b/internal/module/module_test.go index 3838a08..afec69d 100644 --- a/internal/module/module_test.go +++ b/internal/module/module_test.go @@ -25,14 +25,15 @@ func TestRegistry_ResolveAll(t *testing.T) { Storage: config.StorageConfig{Enabled: true}, Network: config.NetworkConfig{Enabled: true}, Monitoring: config.MonitoringConfig{Enabled: true}, + Tenant: config.TenantConfig{Enabled: true}, }, } modules := reg.Resolve(cfg, nil) - if len(modules) != 13 { - t.Errorf("expected 13 modules, got %d", len(modules)) + if len(modules) != 14 { + t.Errorf("expected 14 modules, got %d", len(modules)) } // Verify order - expected := []string{"locale", "system", "packages", "users", "ssh", "security", "docker", "nvidia", "gpu", "cloudflared", "storage", "network", "monitoring"} + expected := []string{"locale", "system", "packages", "users", "ssh", "security", "docker", "nvidia", "gpu", "cloudflared", "storage", "network", "monitoring", "tenant"} for i, m := range modules { if m.Name() != expected[i] { t.Errorf("module[%d] = %q, want %q", i, m.Name(), expected[i]) diff --git a/internal/module/tenant.go b/internal/module/tenant.go new file mode 100644 index 0000000..ca46ea3 --- /dev/null +++ b/internal/module/tenant.go @@ -0,0 +1,309 @@ +package module + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/STAIxBWLB/rootfiles/internal/tenant" +) + +// TenantModule prepares a host for `rootfiles tenant` containers: the GPU +// device guard, the daily password sync, fstrim, the operator sudoers rule +// and the volume mount units of existing tenants. +type TenantModule struct{} + +func NewTenantModule() *TenantModule { return &TenantModule{} } +func (m *TenantModule) Name() string { return "tenant" } + +// Overridable in tests. +var ( + tenantUnitDir = "/etc/systemd/system" + tenantGuardPath = tenant.GuardPath +) + +const ( + tenantGuardUnit = "rootfiles-gpu-guard" + tenantSyncUnit = "rootfiles-tenant-sync" +) + +var tenantTimers = []string{tenantGuardUnit + ".timer", tenantSyncUnit + ".timer", "fstrim.timer"} + +// NewTenantManager returns a tenant manager wired to rc's runner, config and +// the shared GPU ledger. +func NewTenantManager(rc *RunContext, out io.Writer) *tenant.Manager { + return &tenant.Manager{ + Runner: rc.Runner, + Root: rc.Config.Modules.Tenant.DataRoot(), + SpecDir: tenant.SpecDir(metaHomeBase(rc)), + UnitDir: tenantUnitDir, + Out: out, + ReserveGPUs: func(ctx context.Context, t string, gpus []int) error { + return ReserveTenantGPUs(ctx, rc, t, gpus) + }, + } +} + +// RequireTrustedMetadata refuses rootfiles metadata, tenant specs included, +// that a user other than root could control. +func RequireTrustedMetadata(rc *RunContext) error { return requireTrustedMetadata(rc) } + +type tenantFile struct { + path, content string + mode os.FileMode +} + +func tenantFiles() []tenantFile { + unit := func(name string) string { return filepath.Join(tenantUnitDir, name) } + return []tenantFile{ + {tenantGuardPath, tenantGuardScript, 0o755}, + {unit(tenantGuardUnit + ".service"), `# Managed by rootfiles +[Unit] +Description=rootfiles: restore GPU access of running containers after a systemd reload +After=docker.service + +[Service] +Type=oneshot +ExecStart=` + tenantGuardPath + ` +`, 0o644}, + {unit(tenantGuardUnit + ".timer"), `# Managed by rootfiles +[Unit] +Description=rootfiles GPU device guard + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min + +[Install] +WantedBy=timers.target +`, 0o644}, + {unit(tenantSyncUnit + ".service"), `# Managed by rootfiles +[Unit] +Description=rootfiles: save tenant password hashes +After=docker.service + +[Service] +Type=oneshot +ExecStart=` + fleetSudoBinaryDefault + ` tenant sync +`, 0o644}, + {unit(tenantSyncUnit + ".timer"), `# Managed by rootfiles +[Unit] +Description=rootfiles daily tenant password sync + +[Timer] +OnCalendar=daily +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target +`, 0o644}, + } +} + +func tenantSudoersPath() string { return filepath.Join(sudoersDir, "rootfiles-tenant-ops") } + +func tenantSudoersContent(group string) []byte { + return []byte("# Managed by rootfiles: operators run `sudo rootfiles tenant ops TENANT ACTION`\n" + + "%" + group + " ALL=(root) NOPASSWD: " + fleetSudoBinaryDefault + " tenant ops *\n") +} + +func (m *TenantModule) Check(ctx context.Context, rc *RunContext) (*CheckResult, error) { + changes, err := tenantChanges(ctx, rc) + if err != nil { + return nil, err + } + return &CheckResult{Satisfied: len(changes) == 0, Changes: changes}, nil +} + +func unitOn(ctx context.Context, rc *RunContext, unit string) bool { + res, err := rc.Runner.Query(ctx, "systemctl", "is-enabled", unit) + return err == nil && strings.TrimSpace(res.Stdout) == "enabled" +} + +func tenantChanges(ctx context.Context, rc *RunContext) ([]Change, error) { + var changes []Change + add := func(desc, cmd string) { changes = append(changes, Change{Description: desc, Command: cmd}) } + for _, f := range tenantFiles() { + if cur, err := rc.Runner.ReadFile(f.path); err != nil || string(cur) != f.content { + add("Write "+filepath.Base(f.path), "write "+f.path) + } + } + for _, t := range tenantTimers { + if !unitOn(ctx, rc, t) { + add("Enable "+t, "systemctl enable --now "+t) + } + } + group := rc.Config.Modules.Tenant.Group() + if _, err := rc.Runner.Query(ctx, "getent", "group", group); err != nil { + add("Create operator group "+group, "groupadd --system "+group) + } + if err := validateFleetSudoBinary(); err != nil { + return nil, err + } + cur, err := rc.Runner.ReadFile(tenantSudoersPath()) + info, statErr := os.Lstat(tenantSudoersPath()) + if err != nil || string(cur) != string(tenantSudoersContent(group)) || statErr != nil || !fleetSudoDropInSafe(info) { + add("Install tenant operator sudoers rule", "write "+tenantSudoersPath()) + } + specs, err := loadTenantSpecs(rc) + if err != nil { + return nil, err + } + mgr := NewTenantManager(rc, io.Discard) + for _, s := range specs { + if mgr.UnitDrift(s) || !unitOn(ctx, rc, mgr.UnitFor(s.Name)) { + add("Install volume mount unit of tenant "+s.Name, "write "+mgr.UnitFor(s.Name)) + } + } + return changes, nil +} + +func loadTenantSpecs(rc *RunContext) ([]*tenant.Spec, error) { + if err := requireTrustedMetadata(rc); err != nil { + return nil, err + } + dir := tenant.SpecDir(metaHomeBase(rc)) + names, err := tenant.ListSpecs(dir) + if err != nil { + return nil, err + } + var specs []*tenant.Spec + for _, n := range names { + s, err := tenant.LoadSpec(dir, n) + if err != nil { + return nil, err + } + specs = append(specs, s) + } + return specs, nil +} + +func (m *TenantModule) Apply(ctx context.Context, rc *RunContext) (*ApplyResult, error) { + res := &ApplyResult{} + note := func(msg string) { res.Changed, res.Messages = true, append(res.Messages, msg) } + reload := false + for _, f := range tenantFiles() { + if cur, err := rc.Runner.ReadFile(f.path); err == nil && string(cur) == f.content { + continue + } + if err := rc.Runner.MkdirAll(filepath.Dir(f.path), 0o755); err != nil { + return nil, err + } + if err := rc.Runner.WriteFile(f.path, []byte(f.content), f.mode); err != nil { + return nil, fmt.Errorf("writing %s: %w", f.path, err) + } + reload = reload || strings.HasPrefix(f.path, tenantUnitDir) + note("wrote " + f.path) + } + specs, err := loadTenantSpecs(rc) + if err != nil { + return nil, err + } + mgr := NewTenantManager(rc, io.Discard) + for _, s := range specs { + if mgr.UnitDrift(s) { + if err := mgr.WriteUnit(s); err != nil { + return nil, err + } + reload = true + note("wrote volume mount unit of tenant " + s.Name) + } + } + if reload { + if w := systemdReload(ctx, rc); w != "" { + res.Warnings = append(res.Warnings, w) + } + } + for _, t := range tenantTimers { + if unitOn(ctx, rc, t) { + continue + } + if _, err := rc.Runner.Run(ctx, "systemctl", "enable", "--now", t); err != nil { + res.Warnings = append(res.Warnings, "enabling "+t+": "+firstLine(err.Error())) + continue + } + note("enabled " + t) + } + for _, s := range specs { + u := mgr.UnitFor(s.Name) + if unitOn(ctx, rc, u) { + continue + } + if _, err := rc.Runner.Run(ctx, "systemctl", "enable", "--now", u); err != nil { + return nil, fmt.Errorf("enabling %s: %w", u, err) + } + note("enabled " + u) + } + group := rc.Config.Modules.Tenant.Group() + if _, err := rc.Runner.Query(ctx, "getent", "group", group); err != nil { + if _, err := rc.Runner.Run(ctx, "groupadd", "--system", group); err != nil { + return nil, fmt.Errorf("creating group %s: %w", group, err) + } + note("created group " + group) + } + if err := validateFleetSudoBinary(); err != nil { + return nil, err + } + want := tenantSudoersContent(group) + cur, err := rc.Runner.ReadFile(tenantSudoersPath()) + info, statErr := os.Lstat(tenantSudoersPath()) + if err != nil || string(cur) != string(want) || statErr != nil || !fleetSudoDropInSafe(info) { + if err := installSudoersFile(ctx, rc, tenantSudoersPath(), want, "tenant operator"); err != nil { + return nil, err + } + note("installed tenant operator sudoers rule for %" + group) + } + if rc.DryRun { + res.Changed = false + } + return res, nil +} + +// tenantGuardScript restores GPU access of running containers whose device +// filter a systemd daemon-reload rebuilt without their /dev/nvidia* nodes +// (containers started with --gpus on runc). Verified on dgx02 2026-10-20. +const tenantGuardScript = `#!/bin/bash +# Managed by rootfiles: GPU device guard. Do not edit. +# Usage: rootfiles-gpu-guard [--dry-run] [container ...] (no args: all running GPU containers) +set -u +DRY=0 +[ "${1:-}" = "--dry-run" ] && { DRY=1; shift; } +log() { echo "$*"; [ -n "${INVOCATION_ID:-}" ] || logger -t rootfiles-gpu-guard -- "$*" 2>/dev/null || true; } + +targets=("$@") +[ ${#targets[@]} -eq 0 ] && mapfile -t targets < <(docker ps -q) + +rc=0 +for c in "${targets[@]}"; do + info=$(docker inspect -f '{{.Name}}|{{.Id}}|{{.HostConfig.Runtime}}|{{json .HostConfig.DeviceRequests}}' "$c" 2>/dev/null) || continue + IFS='|' read -r name id runtime req <<<"$info" + name=${name#/} + case "$req" in *'"gpu"'*) ;; *) continue ;; esac + if docker exec "$c" sh -c ': /dev/null; then + continue + fi + if [ "$runtime" = "nvidia" ]; then + log "WARN $name: GPU access lost on runtime=nvidia (unexpected), not touching" + rc=1; continue + fi + mapfile -t devs < <(docker exec "$c" sh -c 'for d in /dev/nvidia*; do [ -c "$d" ] && echo "$d"; done' 2>/dev/null) + if [ ${#devs[@]} -eq 0 ]; then + log "WARN $name: no /dev/nvidia* nodes inside, needs restart"; rc=1; continue + fi + args=(); for d in "${devs[@]}"; do args+=("DeviceAllow=$d rw"); done + if [ $DRY -eq 1 ]; then + log "DRY $name: GPU access lost, would allow: ${devs[*]}"; rc=1; continue + fi + systemctl set-property --runtime "docker-$id.scope" "${args[@]}" + if docker exec "$c" sh -c ': /dev/null; then + log "HEALED $name: ${devs[*]}" + else + log "FAIL $name: still no GPU access, restart needed (docker restart $name)"; rc=1 + fi +done +exit $rc +` diff --git a/internal/module/tenant_test.go b/internal/module/tenant_test.go new file mode 100644 index 0000000..face365 --- /dev/null +++ b/internal/module/tenant_test.go @@ -0,0 +1,116 @@ +package module + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/STAIxBWLB/rootfiles/internal/config" + "github.com/STAIxBWLB/rootfiles/internal/tenant" +) + +func TestReserveTenantGPUsSharesTheLedger(t *testing.T) { + fakeBin(t, "nvidia-smi") + home := t.TempDir() + rc := newRealRC(t) + rc.Config.Users.HomeBase = home + if err := os.MkdirAll(filepath.Join(home, ".rootfiles"), 0o755); err != nil { + t.Fatal(err) + } + db := GPUAllocationsDB{Version: 1, Allocations: []GPUAllocation{{Username: "alice", GPUs: []int{0, 1}, Method: "env"}}} + data, _ := json.Marshal(db) + if err := os.WriteFile(gpuDBPath(rc), data, 0o644); err != nil { + t.Fatal(err) + } + ctx := context.Background() + + if err := ReserveTenantGPUs(ctx, rc, "est", []int{1, 2}); err == nil || !strings.Contains(err.Error(), "alice") { + t.Errorf("GPU held by a user was given to a tenant: %v", err) + } + if err := ReserveTenantGPUs(ctx, rc, "est", []int{2, 3}); err != nil { + t.Fatal(err) + } + if err := ReserveTenantGPUs(ctx, rc, "pulse", []int{3}); err == nil || !strings.Contains(err.Error(), "tenant:est") { + t.Errorf("GPU held by another tenant was given away: %v", err) + } + if err := ReserveTenantGPUs(ctx, rc, "est", []int{2}); err != nil { + t.Errorf("tenant could not shrink its own GPUs: %v", err) + } + if err := validateGPURequest(mustLoadGPUDB(t, rc), "bob", []int{2}); err == nil { + t.Error("tenant GPU offered to a user") + } + if err := RevokeGPUs(ctx, rc, "tenant:est"); err == nil { + t.Error("gpu revoke removed a tenant entry") + } + if err := ReserveTenantGPUs(ctx, rc, "est", nil); err != nil { + t.Fatal(err) + } + for _, a := range mustLoadGPUDB(t, rc).Allocations { + if isTenantOwner(a.Username) { + t.Errorf("released tenant still in ledger: %+v", a) + } + } +} + +func mustLoadGPUDB(t *testing.T, rc *RunContext) *GPUAllocationsDB { + t.Helper() + db, err := loadGPUDB(rc) + if err != nil { + t.Fatal(err) + } + return db +} + +func TestGPUModuleIgnoresTenantEntries(t *testing.T) { + home := t.TempDir() + rc := testRunContext(t, home, "env") + os.MkdirAll(filepath.Join(home, ".rootfiles"), 0o755) + data, _ := json.Marshal(GPUAllocationsDB{Version: 1, Allocations: []GPUAllocation{{Username: "tenant:est", GPUs: []int{0}, Method: "tenant"}}}) + os.WriteFile(gpuDBPath(rc), data, 0o644) + res, err := NewGPUModule().Check(context.Background(), rc) + if err != nil || !res.Satisfied { + t.Errorf("tenant-only ledger asked for user GPU setup: %+v %v", res, err) + } +} + +func TestTenantModuleCheck(t *testing.T) { + setFleetSudoTestState(t, "") + fakeBin(t, "systemctl", "getent") + dir := t.TempDir() + oldUnit, oldGuard := tenantUnitDir, tenantGuardPath + tenantUnitDir, tenantGuardPath = filepath.Join(dir, "units"), filepath.Join(dir, "guard") + t.Cleanup(func() { tenantUnitDir, tenantGuardPath = oldUnit, oldGuard }) + + home := t.TempDir() + rc := newDryRunRC(t) + rc.Config.Users.HomeBase = home + rc.Config.Modules.Tenant = config.TenantConfig{Enabled: true, Root: "/raid/tenants"} + specDir := tenant.SpecDir(home) + os.MkdirAll(specDir, 0o755) + s := &tenant.Spec{Name: "est", Image: "img", SSHPort: 3889, Volume: tenant.Volume{Size: "10G"}, + Users: []tenant.User{{Name: "est", UID: 1000, GID: 1000}}} + data, _ := s.Marshal() + os.WriteFile(filepath.Join(specDir, "est.yaml"), data, 0o600) + + res, err := NewTenantModule().Check(context.Background(), rc) + if err != nil { + t.Fatal(err) + } + var got []string + for _, c := range res.Changes { + got = append(got, c.Description) + } + all := strings.Join(got, "\n") + for _, want := range []string{"Write rootfiles-gpu-guard", "Write rootfiles-gpu-guard.timer", "Write rootfiles-tenant-sync.timer", + "Enable fstrim.timer", "Install tenant operator sudoers rule", "Install volume mount unit of tenant est"} { + if !strings.Contains(all, want) { + t.Errorf("check lacks %q:\n%s", want, all) + } + } + if !strings.Contains(string(tenantSudoersContent("ops")), "%ops ALL=(root) NOPASSWD: /usr/local/bin/rootfiles tenant ops *") { + t.Errorf("sudoers rule: %s", tenantSudoersContent("ops")) + } +} diff --git a/internal/tenant/assets.go b/internal/tenant/assets.go new file mode 100644 index 0000000..2f7983f --- /dev/null +++ b/internal/tenant/assets.go @@ -0,0 +1,127 @@ +package tenant + +import ( + "fmt" + "path/filepath" + "strings" +) + +// InitScript is the image-agnostic boot contract, bind-mounted read-only at +// /etc/tenant-system/init.sh and run as the container entrypoint. "users" +// re-runs only the account step in a running container. +const InitScript = `#!/bin/sh +# Managed by rootfiles: tenant boot contract. Do not edit. +set -u +SYS=` + SystemMount + ` +log() { echo "tenant-init: $*"; } + +sync_users() { + [ -f "$SYS/users" ] || return 0 + while IFS=: read -r name uid gid sudo; do + [ -n "$name" ] || continue + if ! getent group "$gid" >/dev/null 2>&1; then + groupadd -g "$gid" "$name" 2>/dev/null || groupadd -g "$gid" "g$gid" || log "groupadd $gid failed" + fi + if ! id -u "$name" >/dev/null 2>&1; then + shell=/bin/sh; [ -x /bin/bash ] && shell=/bin/bash + # -o: images such as ubuntu:24.04 already hold uid 1000 for their own user. + useradd -M -d "/home/$name" -s "$shell" -u "$uid" -g "$gid" "$name" 2>/dev/null || + useradd -o -M -d "/home/$name" -s "$shell" -u "$uid" -g "$gid" "$name" || log "useradd $name failed" + fi + if [ ! -d "/home/$name" ]; then + mkdir -p "/home/$name" && cp -a /etc/skel/. "/home/$name/" 2>/dev/null + chown -R "$uid:$gid" "/home/$name"; chmod 750 "/home/$name" + fi + for g in sudo wheel; do + getent group "$g" >/dev/null 2>&1 || continue + if [ "$sudo" = 1 ]; then usermod -aG "$g" "$name"; else gpasswd -d "$name" "$g" >/dev/null 2>&1; fi + done + done < "$SYS/users" + # Stored hashes only seed accounts without a usable password, so a + # password changed in this container survives a restart. + [ -f "$SYS/shadow" ] || return 0 + while IFS=: read -r name hash lastchg; do + [ -n "$name" ] && [ -n "$hash" ] || continue + cur=$(getent shadow "$name" | cut -d: -f2) || continue + case "$cur" in ''|'!'*|'*'*) ;; *) continue ;; esac + printf '%s:%s\n' "$name" "$hash" | chpasswd -e || { log "restoring password of $name failed"; continue; } + [ -n "$lastchg" ] && chage -d "$lastchg" "$name" + done < "$SYS/shadow" +} + +if [ "${1:-boot}" = users ]; then + sync_users + exit 0 +fi + +sync_users +if ls "$SYS"/ssh/ssh_host_*_key >/dev/null 2>&1; then + rm -f /etc/ssh/ssh_host_* + cp "$SYS"/ssh/ssh_host_* /etc/ssh/ + chmod 600 /etc/ssh/ssh_host_*_key + chmod 644 /etc/ssh/ssh_host_*_key.pub +fi +if [ -x /usr/sbin/sshd ]; then + mkdir -p /run/sshd && chmod 755 /run/sshd + /usr/sbin/sshd || log "sshd failed to start" +else + log "openssh-server is not installed in this image" +fi +first=$(head -n 1 "$SYS/users" 2>/dev/null | cut -d: -f1) +if [ -n "$first" ] && [ -f /data/.tenant/autostart.sh ]; then + su -s /bin/sh "$first" -c 'sh /data/.tenant/autostart.sh' >>/tmp/tenant-autostart.log 2>&1 & +fi +trap 'exit 0' TERM INT +while :; do sleep 3600 & wait $!; done +` + +// usersFile renders name:uid:gid:sudo lines for the boot contract. +func usersFile(s *Spec) []byte { + var b strings.Builder + for _, u := range s.Users { + sudo := 0 + if u.Sudo { + sudo = 1 + } + fmt.Fprintf(&b, "%s:%d:%d:%d\n", u.Name, u.UID, u.GID, sudo) + } + return []byte(b.String()) +} + +// UnitName escapes a mount point the way systemd-escape --path does. +func UnitName(where string) string { + p := strings.Trim(filepath.Clean(where), "/") + var b strings.Builder + for i := 0; i < len(p); i++ { + c := p[i] + switch { + case c == '/': + b.WriteByte('-') + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', c == '_', c == '.' && i > 0: + b.WriteByte(c) + default: + fmt.Fprintf(&b, `\x%02x`, c) + } + } + return b.String() + ".mount" +} + +// MountUnit mounts the tenant volume image before docker starts, so a +// container never starts on the bare mount point. +func MountUnit(name string, p Paths, fs string) string { + return `# Managed by rootfiles: tenant ` + name + ` volume +[Unit] +Description=rootfiles tenant ` + name + ` volume +RequiresMountsFor=` + p.Dir + ` +Before=docker.service + +[Mount] +What=` + p.Image + ` +Where=` + p.Mount + ` +Type=` + fs + ` +Options=loop,discard,nodev,nosuid + +[Install] +WantedBy=multi-user.target docker.service +` +} diff --git a/internal/tenant/import.go b/internal/tenant/import.go new file mode 100644 index 0000000..a3b40e0 --- /dev/null +++ b/internal/tenant/import.go @@ -0,0 +1,302 @@ +package tenant + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// ImportOptions describe moving a hand-made container under rootfiles. +type ImportOptions struct { + Tenant, From string + Size, FS string + // Data is a host directory copied into the new /data. + Data string + // Keep lists paths under /home or /tmp in the old container that are + // copied into the volume. + Keep []string + Operators []string +} + +type legacyContainer struct { + Config struct { + Image string + Env []string + } + State struct { + Running bool + } + HostConfig struct { + PortBindings map[string][]struct{ HostPort string } + DeviceRequests []struct { + Driver string + Count int + DeviceIDs []string + } + RestartPolicy struct{ Name string } + } + Mounts []struct { + Type, Source, Destination string + RW bool + } + GraphDriver struct { + Data map[string]string + } +} + +// Import writes a spec from a running legacy container, copies its host +// keys, password hashes and kept data into a new volume, stops the old +// container (kept for rollback) and starts the tenant container. +func (m *Manager) Import(ctx context.Context, o ImportOptions) (*Spec, error) { + if err := CheckName(o.Tenant); err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(m.SpecDir, o.Tenant+".yaml")); err == nil { + return nil, fmt.Errorf("tenant %s already exists", o.Tenant) + } + if m.inspect(ctx, ContainerName(o.Tenant)).Exists { + return nil, fmt.Errorf("container %s already exists", ContainerName(o.Tenant)) + } + if !m.Runner.CommandExists("rsync") { + return nil, fmt.Errorf("rsync is required for import") + } + res, err := m.Runner.Query(ctx, "docker", "inspect", o.From) + if err != nil { + return nil, fmt.Errorf("inspecting %s: %w", o.From, err) + } + var list []legacyContainer + if err := json.Unmarshal([]byte(res.Stdout), &list); err != nil || len(list) != 1 { + return nil, fmt.Errorf("parsing docker inspect %s: %v", o.From, err) + } + old := list[0] + if !old.State.Running { + return nil, fmt.Errorf("%s must be running so its accounts and files can be read; start it and ask the tenant to stop writing", o.From) + } + + s := &Spec{Name: o.Tenant, Image: old.Config.Image, Volume: Volume{Size: o.Size, FS: o.FS}, Operators: o.Operators} + for port, binds := range old.HostConfig.PortBindings { + if len(binds) == 0 || binds[0].HostPort == "" { + continue + } + if port == "22/tcp" { + s.SSHPort, _ = strconv.Atoi(binds[0].HostPort) + continue + } + cport, proto, _ := strings.Cut(port, "/") + p := binds[0].HostPort + ":" + cport + if proto != "" && proto != "tcp" { + p += "/" + proto + } + s.Ports = append(s.Ports, p) + } + sort.Strings(s.Ports) + s.GPUs, err = legacyGPUs(old) + if err != nil { + return nil, err + } + var notes []string + for _, mt := range old.Mounts { + if mt.Type != "bind" || filepath.Clean(mt.Source) == filepath.Clean(o.Data) { + continue + } + mode := "ro" + if mt.RW { + mode = "rw" + } + b := mt.Source + ":" + mt.Destination + ":" + mode + if _, _, _, err := ParseBind(b); err != nil { + notes = append(notes, fmt.Sprintf("bind %s not carried over: %v", b, err)) + continue + } + s.ExtraBinds = append(s.ExtraBinds, b) + } + if s.Users, err = m.legacyUsers(ctx, o.From); err != nil { + return nil, err + } + if err := s.Validate(); err != nil { + return nil, err + } + + // Plan and size check before any change. + merged := old.GraphDriver.Data["MergedDir"] + type copyItem struct{ src, dst string } + var items []copyItem + p := m.Paths(s.Name) + if o.Data != "" { + items = append(items, copyItem{strings.TrimSuffix(o.Data, "/") + "/", filepath.Join(p.Mount, "data") + "/"}) + } + for _, k := range o.Keep { + k = filepath.Clean(k) + if !strings.HasPrefix(k, "/home/") && !strings.HasPrefix(k, "/tmp/") { + return nil, fmt.Errorf("--keep %s: only paths under /home or /tmp can be kept", k) + } + if merged == "" { + return nil, fmt.Errorf("cannot read the writable layer of %s (no overlay MergedDir)", o.From) + } + items = append(items, copyItem{merged + "/." + k, p.Mount + "/"}) + } + var total uint64 + for _, it := range items { + src := strings.Replace(it.src, "/./", "/", 1) + n, err := m.du(ctx, src) + if err != nil { + return nil, err + } + total += n + } + size, _ := s.Volume.Bytes() + free := freeSpace(m.Root) + m.printf("Import plan for tenant %s from %s:\n", s.Name, o.From) + m.printf(" image %s, ssh port %d, gpus %v, users %d, volume %s %s\n", s.Image, s.SSHPort, s.GPUs, len(s.Users), s.Volume.Filesystem(), s.Volume.Size) + for _, it := range items { + m.printf(" copy %s -> %s\n", it.src, it.dst) + } + for _, n := range notes { + m.printf(" ⚠ %s\n", n) + } + m.printf(" data to copy %s, volume %s, free on %s %s\n", human(total), human(size), m.Root, human(free)) + if total > size*95/100 { + return nil, fmt.Errorf("data to copy (%s) does not fit the %s volume with 5%% headroom; clean up first or pass a larger --size", human(total), s.Volume.Size) + } + if total > free { + return nil, fmt.Errorf("data to copy (%s) exceeds the free space on %s (%s)", human(total), m.Root, human(free)) + } + + if err := m.SaveSpec(s); err != nil { + return nil, err + } + if m.ReserveGPUs != nil { + if err := m.ReserveGPUs(ctx, s.Name, s.GPUs); err != nil { + return nil, err + } + } + if err := m.EnsureVolume(ctx, s); err != nil { + return nil, err + } + if err := m.run(ctx, "install", "-d", "-m", "0700", p.System, filepath.Join(p.System, "ssh")); err != nil { + return nil, err + } + for _, kt := range []string{"ed25519", "ecdsa", "rsa"} { + for _, suffix := range []string{"", ".pub"} { + f := "ssh_host_" + kt + "_key" + suffix + // A missing key type is generated by writeSystem. + _ = m.run(ctx, "docker", "cp", o.From+":/etc/ssh/"+f, filepath.Join(p.System, "ssh", f)) + } + } + if sh, err := m.Runner.Query(ctx, "docker", "exec", o.From, "cat", "/etc/shadow"); err == nil { + if err := m.writeIfChanged(filepath.Join(p.System, "shadow"), shadowFile(s, sh.Stdout), 0o600); err != nil { + return nil, err + } + } + if err := m.writeSystem(ctx, s); err != nil { + return nil, err + } + for _, it := range items { + args := []string{"-aHAX", it.src, it.dst} + if strings.Contains(it.src, "/./") { + args = []string{"-aHAX", "--relative", it.src, it.dst} + } + if err := m.run(ctx, "rsync", args...); err != nil { + return nil, fmt.Errorf("copying %s: %w", it.src, err) + } + } + for _, a := range [][]string{{"update", "--restart=no", o.From}, {"stop", o.From}} { + if err := m.run(ctx, "docker", a...); err != nil { + return nil, err + } + } + if err := m.start(ctx, s); err != nil { + _ = m.run(ctx, "docker", "update", "--restart="+orDefault(old.HostConfig.RestartPolicy.Name, "always"), o.From) + _ = m.run(ctx, "docker", "start", o.From) + return nil, fmt.Errorf("%w; %s was restarted, the copied volume and spec are kept for another try", err, o.From) + } + m.printf("✓ tenant %s imported; %s is stopped and kept for rollback until you remove it\n", s.Name, o.From) + return s, nil +} + +func legacyGPUs(c legacyContainer) ([]int, error) { + var ids []string + for _, r := range c.HostConfig.DeviceRequests { + if len(r.DeviceIDs) > 0 { + ids = append(ids, r.DeviceIDs...) + } else if r.Count != 0 { + return nil, fmt.Errorf("the container requests %d GPUs without IDs; set them with `tenant gpu` after import", r.Count) + } + } + if len(ids) == 0 { + for _, e := range c.Config.Env { + if v, ok := strings.CutPrefix(e, "NVIDIA_VISIBLE_DEVICES="); ok && v != "" && v != "all" && v != "none" && v != "void" { + ids = strings.Split(v, ",") + } + } + } + var gpus []int + for _, id := range ids { + for _, part := range strings.Split(id, ",") { + n, err := strconv.Atoi(strings.TrimSpace(part)) + if err != nil { + return nil, fmt.Errorf("GPU id %q is not an index; set GPUs with `tenant gpu` after import", part) + } + gpus = append(gpus, n) + } + } + sort.Ints(gpus) + return gpus, nil +} + +// legacyUsers reads the regular accounts (uid 1000-65533 with a login +// shell) and their sudo/wheel membership from a running container. +func (m *Manager) legacyUsers(ctx context.Context, from string) ([]User, error) { + pw, err := m.Runner.Query(ctx, "docker", "exec", from, "cat", "/etc/passwd") + if err != nil { + return nil, err + } + gr, err := m.Runner.Query(ctx, "docker", "exec", from, "cat", "/etc/group") + if err != nil { + return nil, err + } + sudoers := map[string]bool{} + for _, line := range strings.Split(gr.Stdout, "\n") { + f := strings.Split(line, ":") + if len(f) == 4 && (f[0] == "sudo" || f[0] == "wheel") { + for _, u := range strings.Split(f[3], ",") { + sudoers[u] = true + } + } + } + return parseLegacyUsers(pw.Stdout, sudoers), nil +} + +func parseLegacyUsers(passwd string, sudoers map[string]bool) []User { + var users []User + for _, line := range strings.Split(passwd, "\n") { + f := strings.Split(line, ":") + if len(f) != 7 || strings.HasSuffix(f[6], "nologin") || strings.HasSuffix(f[6], "false") { + continue + } + uid, err1 := strconv.Atoi(f[2]) + gid, err2 := strconv.Atoi(f[3]) + if err1 != nil || err2 != nil || uid < 1000 || uid > 65533 { + continue + } + users = append(users, User{Name: f[0], UID: uid, GID: gid, Sudo: sudoers[f[0]]}) + } + return users +} + +func (m *Manager) du(ctx context.Context, path string) (uint64, error) { + res, err := m.Runner.Query(ctx, "du", "-sb", path) + if err != nil { + return 0, fmt.Errorf("measuring %s: %w", path, err) + } + f := strings.Fields(res.Stdout) + if len(f) == 0 { + return 0, fmt.Errorf("measuring %s: no output", path) + } + return strconv.ParseUint(f[0], 10, 64) +} diff --git a/internal/tenant/manager.go b/internal/tenant/manager.go new file mode 100644 index 0000000..10d3771 --- /dev/null +++ b/internal/tenant/manager.go @@ -0,0 +1,647 @@ +package tenant + +import ( + "bufio" + "context" + "crypto/rand" + "fmt" + "io" + "math/big" + "net" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" + "time" + + "github.com/STAIxBWLB/rootfiles/internal/exec" +) + +// GuardPath is the GPU device guard the host-prep module installs. +const GuardPath = "/usr/local/sbin/rootfiles-gpu-guard" + +// Manager carries out tenant operations on one host. +type Manager struct { + Runner *exec.Runner + Root string // tenant data root, e.g. /raid/tenants + SpecDir string + UnitDir string // systemd unit directory + Out io.Writer + // ReserveGPUs records gpus for tenant in the shared allocation ledger, + // rejecting overlaps; an empty list releases the tenant's entry. + ReserveGPUs func(ctx context.Context, tenant string, gpus []int) error + // WaitTimeout bounds the readiness check of a new container. + WaitTimeout time.Duration +} + +// Paths are a tenant's host paths. +type Paths struct { + Dir, Image, Mount, System string +} + +// Paths returns the host paths of tenant t. +func (m *Manager) Paths(t string) Paths { + dir := filepath.Join(m.Root, t) + return Paths{ + Dir: dir, + Image: filepath.Join(dir, "volume.img"), + Mount: filepath.Join(dir, "vol"), + System: filepath.Join(dir, "system"), + } +} + +func (m *Manager) unitPath(t string) string { + return filepath.Join(m.UnitDir, UnitName(m.Paths(t).Mount)) +} + +func (m *Manager) printf(format string, a ...any) { fmt.Fprintf(m.Out, format, a...) } + +func (m *Manager) run(ctx context.Context, name string, args ...string) error { + _, err := m.Runner.Run(ctx, name, args...) + return err +} + +// SaveSpec writes s to the spec directory. +func (m *Manager) SaveSpec(s *Spec) error { + if err := s.Validate(); err != nil { + return err + } + data, err := s.Marshal() + if err != nil { + return err + } + if err := m.Runner.MkdirAll(m.SpecDir, 0o755); err != nil { + return err + } + path := filepath.Join(m.SpecDir, s.Name+".yaml") + if fi, err := os.Lstat(path); err == nil && !fi.Mode().IsRegular() { + return fmt.Errorf("refusing to write %s: not a regular file", path) + } + return m.Runner.WriteFile(path, data, 0o600) +} + +// Load reads tenant t's spec. +func (m *Manager) Load(t string) (*Spec, error) { return LoadSpec(m.SpecDir, t) } + +// --- container state --- + +type containerState struct { + Exists, Running bool + Digest string + Restart string +} + +func (m *Manager) inspect(ctx context.Context, name string) containerState { + res, err := m.Runner.Query(ctx, "docker", "inspect", "-f", + `{{.State.Running}}|{{index .Config.Labels "rootfiles.tenant.spec"}}|{{.HostConfig.RestartPolicy.Name}}`, name) + if err != nil { + return containerState{} + } + f := strings.Split(strings.TrimSpace(res.Stdout), "|") + st := containerState{Exists: true} + if len(f) == 3 { + st.Running, st.Digest, st.Restart = f[0] == "true", f[1], f[2] + } + return st +} + +// RunArgs is the docker run command line of s. +func (m *Manager) RunArgs(s *Spec) []string { + p := m.Paths(s.Name) + args := []string{"run", "-d", + "--name", ContainerName(s.Name), + "--hostname", s.Name, + "--restart=always", "--init", "--ipc=host", + "--label", "rootfiles.tenant=" + s.Name, + "--label", "rootfiles.tenant.spec=" + s.Digest(), + } + if len(s.GPUs) > 0 { + ids := make([]string, len(s.GPUs)) + for i, g := range s.GPUs { + ids[i] = strconv.Itoa(g) + } + // The nvidia runtime lists the devices in the OCI spec, so a + // systemd daemon-reload cannot revoke them as it does under runc. + args = append(args, "--runtime=nvidia", "--gpus", `"device=`+strings.Join(ids, ",")+`"`) + } + args = append(args, "-p", strconv.Itoa(s.SSHPort)+":22") + for _, port := range s.Ports { + args = append(args, "-p", port) + } + mount := func(src, dst string, ro bool) { + v := "type=bind,src=" + src + ",dst=" + dst + if ro { + v += ",readonly" + } + args = append(args, "--mount", v) + } + // type=bind fails on a missing source, so an unmounted volume (no + // data/home/tmp on the bare mount point) stops the container instead + // of letting it write past the quota. + mount(filepath.Join(p.Mount, "data"), "/data", false) + mount(filepath.Join(p.Mount, "home"), "/home", false) + mount(filepath.Join(p.Mount, "tmp"), "/tmp", false) + mount(p.System, SystemMount, true) + for _, b := range s.ExtraBinds { + src, dst, ro, _ := ParseBind(b) + mount(src, dst, ro) + } + return append(args, "--entrypoint", "/bin/sh", s.Image, SystemMount+"/"+initScript) +} + +// --- apply --- + +// Apply converges tenant s: GPU ledger, volume, system files and container. +func (m *Manager) Apply(ctx context.Context, s *Spec) error { + if err := s.Validate(); err != nil { + return err + } + if m.ReserveGPUs != nil { + if err := m.ReserveGPUs(ctx, s.Name, s.GPUs); err != nil { + return err + } + } + if err := m.EnsureVolume(ctx, s); err != nil { + return err + } + if err := m.writeSystem(ctx, s); err != nil { + return err + } + name := ContainerName(s.Name) + st := m.inspect(ctx, name) + switch { + case !st.Exists: + return m.start(ctx, s) + case st.Digest != s.Digest(): + return m.recreate(ctx, s, st) + case !st.Running: + m.printf("Starting %s\n", name) + return m.run(ctx, "docker", "start", name) + default: + m.printf("%s is up to date; syncing accounts\n", name) + return m.run(ctx, "docker", "exec", name, "/bin/sh", SystemMount+"/"+initScript, "users") + } +} + +func (m *Manager) start(ctx context.Context, s *Spec) error { + name := ContainerName(s.Name) + m.printf("Creating %s (image %s)\n", name, s.Image) + if err := m.run(ctx, "docker", m.RunArgs(s)...); err != nil { + return err + } + if err := m.Verify(ctx, s); err != nil { + _ = m.run(ctx, "docker", "rm", "-f", name) + return fmt.Errorf("new container failed verification and was removed: %w", err) + } + return nil +} + +func (m *Manager) recreate(ctx context.Context, s *Spec, st containerState) error { + name := ContainerName(s.Name) + prev := name + "-prev" + if m.inspect(ctx, prev).Exists { + return fmt.Errorf("%s still holds the previous container; check it and remove it (docker rm %s) before recreating", prev, prev) + } + if st.Running { + if err := m.SyncShadow(ctx, s); err != nil { + return fmt.Errorf("saving passwords before recreate: %w", err) + } + } + if size := m.writableSize(ctx, name); size != "" { + m.printf("⚠ %s writable layer holds %s; files outside /data, /home and /tmp stay in %s, not in the new container\n", name, size, prev) + } + m.printf("Recreating %s (image %s)\n", name, s.Image) + steps := [][]string{ + {"stop", name}, + {"rename", name, prev}, + {"update", "--restart=no", prev}, + } + for _, a := range steps { + if err := m.run(ctx, "docker", a...); err != nil { + return fmt.Errorf("preparing recreate: %w", err) + } + } + err := m.run(ctx, "docker", m.RunArgs(s)...) + if err == nil { + err = m.Verify(ctx, s) + } + if err != nil { + m.printf("✗ new container failed (%v); rolling back\n", firstLine(err.Error())) + _ = m.run(ctx, "docker", "rm", "-f", name) + for _, a := range [][]string{{"rename", prev, name}, {"update", "--restart=" + orDefault(st.Restart, "always"), name}, {"start", name}} { + if rerr := m.run(ctx, "docker", a...); rerr != nil { + return fmt.Errorf("recreate failed (%v) and rollback failed: %w", err, rerr) + } + } + return fmt.Errorf("recreate failed and was rolled back: %w", err) + } + m.printf("Previous container kept as %s; remove it with `docker rm %s` once the tenant confirms\n", prev, prev) + return nil +} + +func (m *Manager) writableSize(ctx context.Context, name string) string { + res, err := m.Runner.Query(ctx, "docker", "ps", "-a", "-s", "--filter", "name=^/"+name+"$", "--format", "{{.Size}}") + if err != nil { + return "" + } + return strings.TrimSpace(res.Stdout) +} + +// Verify waits for sshd with the stored host key and, with GPUs, for +// nvidia-smi to list them. +func (m *Manager) Verify(ctx context.Context, s *Spec) error { + if m.Runner.DryRun { + return nil + } + timeout := m.WaitTimeout + if timeout == 0 { + timeout = 60 * time.Second + } + addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(s.SSHPort)) + deadline := time.Now().Add(timeout) + var lastErr error + for { + if lastErr = sshBanner(addr); lastErr == nil { + break + } + if time.Now().After(deadline) { + return fmt.Errorf("sshd on %s: %w", addr, lastErr) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + if err := m.checkHostKey(ctx, s); err != nil { + return err + } + if len(s.GPUs) > 0 { + res, err := m.Runner.Query(ctx, "docker", "exec", ContainerName(s.Name), "nvidia-smi", "-L") + if err != nil { + return fmt.Errorf("GPU check: %w", err) + } + if n := strings.Count(res.Stdout, "GPU "); n != len(s.GPUs) { + return fmt.Errorf("GPU check: nvidia-smi lists %d GPUs, want %d", n, len(s.GPUs)) + } + } + m.printf("✓ %s: sshd on port %d, host key and GPUs verified\n", ContainerName(s.Name), s.SSHPort) + return nil +} + +func sshBanner(addr string) error { + c, err := net.DialTimeout("tcp", addr, 3*time.Second) + if err != nil { + return err + } + defer c.Close() + _ = c.SetReadDeadline(time.Now().Add(3 * time.Second)) + line, err := bufio.NewReader(c).ReadString('\n') + if err != nil { + return err + } + if !strings.HasPrefix(line, "SSH-") { + return fmt.Errorf("unexpected banner %q", strings.TrimSpace(line)) + } + return nil +} + +func (m *Manager) checkHostKey(ctx context.Context, s *Spec) error { + pub, err := os.ReadFile(filepath.Join(m.Paths(s.Name).System, "ssh", "ssh_host_ed25519_key.pub")) + if err != nil || !m.Runner.CommandExists("ssh-keyscan") { + m.printf("⚠ host key not compared (no stored ed25519 key or no ssh-keyscan)\n") + return nil + } + want := strings.Fields(string(pub)) + res, err := m.Runner.Query(ctx, "ssh-keyscan", "-T", "5", "-t", "ed25519", "-p", strconv.Itoa(s.SSHPort), "127.0.0.1") + if err != nil { + return fmt.Errorf("host key check: %w", err) + } + for _, line := range strings.Split(res.Stdout, "\n") { + f := strings.Fields(line) + if len(f) >= 3 && len(want) >= 2 && f[2] == want[1] { + return nil + } + } + return fmt.Errorf("host key check: sshd does not present the stored ed25519 host key") +} + +// --- system files --- + +func (m *Manager) writeSystem(ctx context.Context, s *Spec) error { + p := m.Paths(s.Name) + if err := m.run(ctx, "install", "-d", "-m", "0700", p.System, filepath.Join(p.System, "ssh")); err != nil { + return err + } + if err := m.writeIfChanged(filepath.Join(p.System, initScript), []byte(InitScript), 0o755); err != nil { + return err + } + if err := m.writeIfChanged(filepath.Join(p.System, "users"), usersFile(s), 0o644); err != nil { + return err + } + for _, kt := range []string{"ed25519", "ecdsa", "rsa"} { + key := filepath.Join(p.System, "ssh", "ssh_host_"+kt+"_key") + if m.Runner.FileExists(key) { + continue + } + if err := m.run(ctx, "ssh-keygen", "-q", "-t", kt, "-N", "", "-C", "tenant-"+s.Name, "-f", key); err != nil { + return fmt.Errorf("generating %s host key: %w", kt, err) + } + } + return nil +} + +func (m *Manager) writeIfChanged(path string, data []byte, perm os.FileMode) error { + if cur, err := os.ReadFile(path); err == nil && string(cur) == string(data) { + return nil + } + if fi, err := os.Lstat(path); err == nil && !fi.Mode().IsRegular() { + return fmt.Errorf("refusing to write %s: not a regular file", path) + } + return m.Runner.WriteFile(path, data, perm) +} + +// --- passwords --- + +const passwordAlphabet = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789" + +func generatePassword(n int) (string, error) { + b := make([]byte, n) + max := big.NewInt(int64(len(passwordAlphabet))) + for i := range b { + idx, err := rand.Int(rand.Reader, max) + if err != nil { + return "", err + } + b[i] = passwordAlphabet[idx.Int64()] + } + return string(b), nil +} + +// Passwd issues a temporary password for user, expired so it must be +// changed at first login, and stores the hash. The password goes to the +// container on stdin only; it is returned once and never logged. +func (m *Manager) Passwd(ctx context.Context, s *Spec, user string) (string, error) { + if _, ok := s.User(user); !ok { + return "", fmt.Errorf("tenant %s has no user %s", s.Name, user) + } + name := ContainerName(s.Name) + if !m.Runner.DryRun && !m.inspect(ctx, name).Running { + return "", fmt.Errorf("%s is not running", name) + } + pw, err := generatePassword(16) + if err != nil { + return "", err + } + if _, err := m.Runner.RunInput(ctx, user+":"+pw+"\n", "docker", "exec", "-i", name, "chpasswd"); err != nil { + return "", err + } + if err := m.run(ctx, "docker", "exec", name, "chage", "-d", "0", user); err != nil { + return "", err + } + if err := m.SyncShadow(ctx, s); err != nil { + return "", err + } + if m.Runner.DryRun { + return "", nil + } + return pw, nil +} + +// SyncShadow copies the spec users' password hashes from the running +// container into the system directory. +func (m *Manager) SyncShadow(ctx context.Context, s *Spec) error { + name := ContainerName(s.Name) + if !m.Runner.DryRun && !m.inspect(ctx, name).Running { + return nil // nothing to read; the stored hashes stay + } + res, err := m.Runner.Query(ctx, "docker", "exec", name, "cat", "/etc/shadow") + if err != nil { + if m.Runner.DryRun { + return nil + } + return fmt.Errorf("reading passwords of %s: %w", name, err) + } + return m.writeIfChanged(filepath.Join(m.Paths(s.Name).System, "shadow"), shadowFile(s, res.Stdout), 0o600) +} + +// shadowFile keeps name:hash:lastchg of the spec users with usable hashes. +func shadowFile(s *Spec, shadow string) []byte { + got := map[string]string{} + for _, line := range strings.Split(shadow, "\n") { + f := strings.Split(line, ":") + if len(f) < 3 { + continue + } + if h := f[1]; h == "" || strings.HasPrefix(h, "!") || strings.HasPrefix(h, "*") { + continue + } + got[f[0]] = f[0] + ":" + f[1] + ":" + f[2] + "\n" + } + var b strings.Builder + for _, u := range s.Users { + b.WriteString(got[u.Name]) + } + return []byte(b.String()) +} + +// --- simple operations --- + +// Restart restarts the tenant container. +func (m *Manager) Restart(ctx context.Context, s *Spec) error { + return m.run(ctx, "docker", "restart", ContainerName(s.Name)) +} + +// Logs prints the container log tail. +func (m *Manager) Logs(ctx context.Context, s *Spec, tail int) error { + res, err := m.Runner.Query(ctx, "docker", "logs", "--tail", strconv.Itoa(tail), ContainerName(s.Name)) + if res != nil { + fmt.Fprint(m.Out, res.Stdout, res.Stderr) + } + return err +} + +// HealGPU runs the device guard for the tenant container. +func (m *Manager) HealGPU(ctx context.Context, s *Spec) error { + if !m.Runner.FileExists(GuardPath) { + return fmt.Errorf("%s is missing; enable the tenant module and run `rootfiles apply --module tenant`", GuardPath) + } + res, err := m.Runner.Run(ctx, GuardPath, ContainerName(s.Name)) + if res != nil { + fmt.Fprint(m.Out, res.Stdout) + } + return err +} + +// --- status --- + +// Usage is a mounted volume's use. +type Usage struct { + Used, Size uint64 + Mounted bool +} + +func (m *Manager) usage(t string) Usage { + p := m.Paths(t) + if !isMountPoint(p.Mount) { + return Usage{} + } + var st syscall.Statfs_t + if err := syscall.Statfs(p.Mount, &st); err != nil { + return Usage{} + } + bs := uint64(st.Bsize) + return Usage{Used: (st.Blocks - st.Bfree) * bs, Size: st.Blocks * bs, Mounted: true} +} + +// isMountPoint reports whether dir is on a different device than its parent. +func isMountPoint(dir string) bool { + a, err1 := os.Stat(dir) + b, err2 := os.Stat(filepath.Dir(dir)) + if err1 != nil || err2 != nil { + return false + } + sa, ok1 := a.Sys().(*syscall.Stat_t) + sb, ok2 := b.Sys().(*syscall.Stat_t) + return ok1 && ok2 && sa.Dev != sb.Dev +} + +// Commitment compares the volume sizes not yet backed by disk blocks with +// the free space of the tenant root. +type Commitment struct { + Committed, Allocated, Free uint64 +} + +// Overcommitted reports whether filling every volume would run out of space. +func (c Commitment) Overcommitted() bool { return c.Committed-c.Allocated > c.Free } + +func (m *Manager) commitment(specs []*Spec) Commitment { + var c Commitment + for _, s := range specs { + n, _ := s.Volume.Bytes() + c.Committed += n + if fi, err := os.Stat(m.Paths(s.Name).Image); err == nil { + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + c.Allocated += uint64(st.Blocks) * 512 + } + } + } + if c.Allocated > c.Committed { + c.Allocated = c.Committed + } + c.Free = freeSpace(m.Root) + return c +} + +// freeSpace is the space available to root on the filesystem holding path, +// or its nearest existing parent. +func freeSpace(path string) uint64 { + for p := filepath.Clean(path); ; p = filepath.Dir(p) { + var st syscall.Statfs_t + if syscall.Statfs(p, &st) == nil { + return uint64(st.Bavail) * uint64(st.Bsize) + } + if p == filepath.Dir(p) { + return 0 + } + } +} + +// Status prints one line per tenant and the volume commitment. +func (m *Manager) Status(ctx context.Context, names []string, detail bool) error { + var specs []*Spec + fmt.Fprintf(m.Out, "%-14s %-9s %-7s %-6s %-8s %-19s %s\n", "TENANT", "STATE", "SSH", "GPUS", "SPEC", "VOLUME", "IMAGE") + for _, n := range names { + s, err := m.Load(n) + if err != nil { + fmt.Fprintf(m.Out, "%-14s error: %v\n", n, err) + continue + } + specs = append(specs, s) + st := m.inspect(ctx, ContainerName(n)) + state := "missing" + if st.Exists { + state = "stopped" + if st.Running { + state = "running" + } + } + drift := "current" + if st.Exists && st.Digest != s.Digest() { + drift = "drift" + } + vol := "unmounted/" + s.Volume.Size + if u := m.usage(n); u.Mounted { + vol = fmt.Sprintf("%s/%s %d%%", human(u.Used), human(u.Size), pct(u.Used, u.Size)) + } + gpus := "-" + if len(s.GPUs) > 0 { + gpus = intList(s.GPUs) + } + fmt.Fprintf(m.Out, "%-14s %-9s %-7d %-6s %-8s %-19s %s\n", n, state, s.SSHPort, gpus, drift, vol, s.Image) + if detail && st.Exists { + if size := m.writableSize(ctx, ContainerName(n)); size != "" { + fmt.Fprintf(m.Out, " writable layer: %s (no quota; keep data in /data, /home or /tmp)\n", size) + } + if m.inspect(ctx, ContainerName(n)+"-prev").Exists { + fmt.Fprintf(m.Out, " previous container %s-prev is kept for rollback\n", ContainerName(n)) + } + } + } + if len(specs) > 0 { + c := m.commitment(specs) + mark := "✓" + if c.Overcommitted() { + mark = "⚠ overcommitted:" + } + fmt.Fprintf(m.Out, "%s volumes commit %s, %s allocated, %s free on %s\n", mark, human(c.Committed), human(c.Allocated), human(c.Free), m.Root) + } + return nil +} + +func pct(a, b uint64) uint64 { + if b == 0 { + return 0 + } + return a * 100 / b +} + +func human(n uint64) string { + units := []string{"B", "K", "M", "G", "T", "P"} + f := float64(n) + i := 0 + for f >= 1024 && i < len(units)-1 { + f /= 1024 + i++ + } + if i == 0 { + return fmt.Sprintf("%dB", n) + } + return strconv.FormatFloat(f, 'f', 1, 64) + units[i] +} + +func intList(v []int) string { + s := append([]int(nil), v...) + sort.Ints(s) + parts := make([]string, len(s)) + for i, g := range s { + parts[i] = strconv.Itoa(g) + } + return strings.Join(parts, ",") +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} + +func orDefault(s, d string) string { + if s == "" { + return d + } + return s +} diff --git a/internal/tenant/ops.go b/internal/tenant/ops.go new file mode 100644 index 0000000..fc1886e --- /dev/null +++ b/internal/tenant/ops.go @@ -0,0 +1,66 @@ +package tenant + +import ( + "encoding/json" + "fmt" + "os" + "os/user" + "strings" + "time" +) + +// OpsActions are what a tenant's operators may run through +// `sudo rootfiles tenant ops`. +var OpsActions = []string{"status", "logs", "gpu-heal", "restart", "reset"} + +// AuthorizeOps allows root, and an account sudo identified as one of the +// tenant's operators, to run an operator action. +func AuthorizeOps(s *Spec, actor string, viaSudo bool, action string) error { + allowed := false + for _, a := range OpsActions { + allowed = allowed || a == action + } + if !allowed { + return fmt.Errorf("action %q is not an operator action (allowed: %s)", action, strings.Join(OpsActions, ", ")) + } + if viaSudo && !s.HasOperator(actor) { + return fmt.Errorf("%s is not an operator of tenant %s", actor, s.Name) + } + return nil +} + +// Actor names who runs the command: the sudo caller, else the current user. +func Actor() (name string, viaSudo bool) { + if u := os.Getenv("SUDO_USER"); u != "" && os.Geteuid() == 0 && u != "root" { + return u, true + } + if u, err := user.Current(); err == nil { + return u.Username, false + } + return fmt.Sprintf("uid:%d", os.Getuid()), false +} + +// AuditPath receives one JSON line per mutating tenant command. +var AuditPath = "/var/log/rootfiles-tenant.jsonl" + +// Audit appends a record. It never carries secrets: callers pass the +// action name and an error summary only. +func Audit(actor, tenant, action, result string) error { + line, err := json.Marshal(struct { + Time string `json:"time"` + Actor string `json:"actor"` + Tenant string `json:"tenant"` + Action string `json:"action"` + Result string `json:"result"` + }{time.Now().UTC().Format(time.RFC3339), actor, tenant, action, result}) + if err != nil { + return err + } + f, err := os.OpenFile(AuditPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o640) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write(append(line, '\n')) + return err +} diff --git a/internal/tenant/spec.go b/internal/tenant/spec.go new file mode 100644 index 0000000..8314ee1 --- /dev/null +++ b/internal/tenant/spec.go @@ -0,0 +1,305 @@ +// Package tenant manages long-lived tenant containers: one Docker container +// per tenant, a size-capped volume on a loop-mounted image, accounts and +// host keys kept outside the container, and GPU devices from the shared +// rootfiles allocation ledger. +package tenant + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/STAIxBWLB/rootfiles/internal/config" +) + +var ( + validName = regexp.MustCompile(`^[a-z][a-z0-9-]{0,30}$`) + validAccount = regexp.MustCompile(`^[a-z_][a-z0-9_.-]{0,31}$`) + validPort = regexp.MustCompile(`^([0-9]{1,5}):([0-9]{1,5})(/(tcp|udp))?$`) +) + +// MinVolumeSize keeps mkfs well clear of its own minimums. +const MinVolumeSize = 64 << 20 + +// Paths inside the container that rootfiles owns. +const ( + SystemMount = "/etc/tenant-system" + initScript = "init.sh" +) + +var reservedTargets = []string{"/data", "/home", "/tmp", SystemMount} + +// Spec is one tenant, stored as /.rootfiles/tenants/.yaml. +type Spec struct { + Name string `yaml:"name"` + Image string `yaml:"image"` + GPUs []int `yaml:"gpus,omitempty"` + SSHPort int `yaml:"ssh_port"` + Ports []string `yaml:"ports,omitempty"` + Volume Volume `yaml:"volume"` + Users []User `yaml:"users"` + Operators []string `yaml:"operators,omitempty"` + ExtraBinds []string `yaml:"extra_binds,omitempty"` +} + +// Volume is the tenant's size-capped volume. +type Volume struct { + Size string `yaml:"size"` + FS string `yaml:"fs,omitempty"` // ext4 (default) or xfs +} + +// User is an account created inside the container on every start. +type User struct { + Name string `yaml:"name"` + UID int `yaml:"uid"` + GID int `yaml:"gid"` + Sudo bool `yaml:"sudo,omitempty"` +} + +// Filesystem returns the volume filesystem, ext4 when unset. +func (v Volume) Filesystem() string { + if v.FS == "" { + return "ext4" + } + return v.FS +} + +// Bytes returns the parsed volume size. +func (v Volume) Bytes() (uint64, error) { + n, err := config.ParseSize(v.Size) + if err != nil { + return 0, err + } + return n, nil +} + +// ContainerName is the Docker container of tenant t. +func ContainerName(t string) string { return "tenant-" + t } + +// CheckName validates a tenant name. +func CheckName(name string) error { + if !validName.MatchString(name) { + return fmt.Errorf("invalid tenant name %q (lowercase letter, then lowercase letters, digits or '-', at most 31)", name) + } + return nil +} + +// Validate reports every problem in the spec at once. +func (s *Spec) Validate() error { + var errs []string + add := func(format string, a ...any) { errs = append(errs, fmt.Sprintf(format, a...)) } + + if err := CheckName(s.Name); err != nil { + add("%v", err) + } + if s.Image == "" || strings.ContainsAny(s.Image, " \t\n") || strings.HasPrefix(s.Image, "-") { + add("image: %q is not a valid image reference", s.Image) + } + seenGPU := map[int]bool{} + for _, g := range s.GPUs { + if g < 0 || seenGPU[g] { + add("gpus: invalid or duplicate index %d", g) + } + seenGPU[g] = true + } + if s.SSHPort < 1 || s.SSHPort > 65535 { + add("ssh_port: %d out of range", s.SSHPort) + } + hostPorts := map[string]bool{strconv.Itoa(s.SSHPort) + "/tcp": true} + for _, p := range s.Ports { + m := validPort.FindStringSubmatch(p) + if m == nil { + add("ports: %q (want HOST:CONTAINER[/tcp|/udp])", p) + continue + } + hp, _ := strconv.Atoi(m[1]) + cp, _ := strconv.Atoi(m[2]) + if hp < 1 || hp > 65535 || cp < 1 || cp > 65535 { + add("ports: %q out of range", p) + } + proto := m[4] + if proto == "" { + proto = "tcp" + } + key := m[1] + "/" + proto + if hostPorts[key] { + add("ports: host port %s used twice", key) + } + hostPorts[key] = true + } + size, err := s.Volume.Bytes() + switch { + case err != nil || s.Volume.Size == "": + add("volume.size: %q (want e.g. 500G, 2T)", s.Volume.Size) + case size < MinVolumeSize: + add("volume.size: %s is below the 64M minimum", s.Volume.Size) + } + if fs := s.Volume.Filesystem(); fs != "ext4" && fs != "xfs" { + add("volume.fs: %q (want ext4 or xfs)", fs) + } + if len(s.Users) == 0 { + add("users: at least one user is required") + } + seenUser, seenUID := map[string]bool{}, map[int]bool{} + for _, u := range s.Users { + if !validAccount.MatchString(u.Name) || u.Name == "root" { + add("users: invalid name %q", u.Name) + } + if u.UID < 1000 || u.UID > 65533 || u.GID < 1000 || u.GID > 65533 { + add("users: %s uid/gid must be 1000-65533", u.Name) + } + if seenUser[u.Name] || seenUID[u.UID] { + add("users: %s or uid %d listed twice", u.Name, u.UID) + } + seenUser[u.Name], seenUID[u.UID] = true, true + } + for _, o := range s.Operators { + if !validAccount.MatchString(o) || o == "root" { + add("operators: invalid account %q", o) + } + } + for _, b := range s.ExtraBinds { + if _, _, _, err := ParseBind(b); err != nil { + add("extra_binds: %v", err) + } + } + if len(errs) > 0 { + return fmt.Errorf("tenant %s: %s", s.Name, strings.Join(errs, "; ")) + } + return nil +} + +// ParseBind reads SRC:DST[:ro|rw] with absolute paths outside the targets +// rootfiles mounts itself. +func ParseBind(b string) (src, dst string, readOnly bool, err error) { + parts := strings.Split(b, ":") + if len(parts) < 2 || len(parts) > 3 { + return "", "", false, fmt.Errorf("%q (want SRC:DST[:ro|rw])", b) + } + src, dst = parts[0], parts[1] + if len(parts) == 3 { + switch parts[2] { + case "ro": + readOnly = true + case "rw": + default: + return "", "", false, fmt.Errorf("%q: mode must be ro or rw", b) + } + } + if !filepath.IsAbs(src) || !filepath.IsAbs(dst) || strings.ContainsAny(b, ", \n") { + return "", "", false, fmt.Errorf("%q: paths must be absolute without commas or spaces", b) + } + dst = filepath.Clean(dst) + for _, r := range reservedTargets { + if dst == r || strings.HasPrefix(dst, r+"/") || dst == "/" { + return "", "", false, fmt.Errorf("%q: %s is managed by rootfiles", b, dst) + } + } + return filepath.Clean(src), dst, readOnly, nil +} + +// Digest identifies the settings that only a new container can change. A +// running container whose label differs is out of date. +func (s *Spec) Digest() string { + gpus := append([]int(nil), s.GPUs...) + sort.Ints(gpus) + b, _ := json.Marshal(struct { + Name, Image string + GPUs []int + SSH int + Ports []string + Binds []string + }{s.Name, s.Image, gpus, s.SSHPort, s.Ports, s.ExtraBinds}) + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:])[:16] +} + +// User returns the named user. +func (s *Spec) User(name string) (User, bool) { + for _, u := range s.Users { + if u.Name == name { + return u, true + } + } + return User{}, false +} + +// HasOperator reports whether account may run `tenant ops` on s. +func (s *Spec) HasOperator(account string) bool { + for _, o := range s.Operators { + if o == account { + return true + } + } + return false +} + +// Marshal renders the spec file. +func (s *Spec) Marshal() ([]byte, error) { + b, err := yaml.Marshal(s) + if err != nil { + return nil, err + } + return append([]byte("# Managed by rootfiles: edit, then run `rootfiles tenant apply "+s.Name+"`\n"), b...), nil +} + +// SpecDir is where tenant specs live under a home base. +func SpecDir(homeBase string) string { + return filepath.Join(homeBase, ".rootfiles", "tenants") +} + +// LoadSpec reads and validates tenant name from dir. +func LoadSpec(dir, name string) (*Spec, error) { + if err := CheckName(name); err != nil { + return nil, err + } + data, err := os.ReadFile(filepath.Join(dir, name+".yaml")) + if os.IsNotExist(err) { + return nil, fmt.Errorf("tenant %s not found (no %s)", name, filepath.Join(dir, name+".yaml")) + } + if err != nil { + return nil, err + } + var s Spec + dec := yaml.NewDecoder(strings.NewReader(string(data))) + dec.KnownFields(true) + if err := dec.Decode(&s); err != nil { + return nil, fmt.Errorf("parsing tenant %s: %w", name, err) + } + if s.Name != name { + return nil, fmt.Errorf("tenant file %s.yaml names tenant %q", name, s.Name) + } + if err := s.Validate(); err != nil { + return nil, err + } + return &s, nil +} + +// ListSpecs returns every tenant name in dir, sorted. +func ListSpecs(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var names []string + for _, e := range entries { + n, ok := strings.CutSuffix(e.Name(), ".yaml") + if ok && e.Type().IsRegular() && CheckName(n) == nil { + names = append(names, n) + } + } + sort.Strings(names) + return names, nil +} diff --git a/internal/tenant/tenant_test.go b/internal/tenant/tenant_test.go new file mode 100644 index 0000000..749d9ef --- /dev/null +++ b/internal/tenant/tenant_test.go @@ -0,0 +1,317 @@ +package tenant + +import ( + "bytes" + "context" + "log/slog" + "os" + osexec "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/STAIxBWLB/rootfiles/internal/exec" +) + +func validSpec() *Spec { + return &Spec{ + Name: "est", + Image: "registry.local/est:1", + GPUs: []int{2, 3}, + SSHPort: 3889, + Ports: []string{"8888:8888", "9000:9000/udp"}, + Volume: Volume{Size: "10G"}, + Users: []User{{Name: "est", UID: 1000, GID: 1000, Sudo: true}}, + } +} + +func TestValidate(t *testing.T) { + if err := validSpec().Validate(); err != nil { + t.Fatalf("valid spec rejected: %v", err) + } + cases := map[string]func(*Spec){ + "name": func(s *Spec) { s.Name = "Bad_Name" }, + "image": func(s *Spec) { s.Image = "--privileged" }, + "gpu dup": func(s *Spec) { s.GPUs = []int{1, 1} }, + "ssh port": func(s *Spec) { s.SSHPort = 0 }, + "port clash": func(s *Spec) { s.Ports = []string{"3889:22"} }, + "port format": func(s *Spec) { s.Ports = []string{"80"} }, + "size": func(s *Spec) { s.Volume.Size = "10 GB" }, + "tiny": func(s *Spec) { s.Volume.Size = "1M" }, + "fs": func(s *Spec) { s.Volume.FS = "btrfs" }, + "no users": func(s *Spec) { s.Users = nil }, + "root user": func(s *Spec) { s.Users[0].Name = "root" }, + "system uid": func(s *Spec) { s.Users[0].UID = 0 }, + "dup uid": func(s *Spec) { s.Users = append(s.Users, User{Name: "b", UID: 1000, GID: 1000}) }, + "operator": func(s *Spec) { s.Operators = []string{"bad name"} }, + "bind target": func(s *Spec) { s.ExtraBinds = []string{"/raid/x:/home/x"} }, + "bind system": func(s *Spec) { s.ExtraBinds = []string{"/raid/x:/etc/tenant-system/k"} }, + "bind rel": func(s *Spec) { s.ExtraBinds = []string{"raid:/legacy"} }, + "bind comma": func(s *Spec) { s.ExtraBinds = []string{"/a,readonly:/legacy"} }, + } + for name, mutate := range cases { + s := validSpec() + mutate(s) + if err := s.Validate(); err == nil { + t.Errorf("%s: invalid spec accepted", name) + } + } + s := validSpec() + s.ExtraBinds = []string{"/raid/data/est:/legacy:ro"} + if err := s.Validate(); err != nil { + t.Errorf("legacy bind rejected: %v", err) + } +} + +func TestDigestCoversContainerSettingsOnly(t *testing.T) { + a, b := validSpec(), validSpec() + b.Users = append(b.Users, User{Name: "b", UID: 1001, GID: 1001}) + b.Volume.Size = "20G" + b.Operators = []string{"alice"} + b.GPUs = []int{3, 2} + if a.Digest() != b.Digest() { + t.Error("users, volume size, operators or GPU order changed the container digest") + } + for _, mutate := range []func(*Spec){ + func(s *Spec) { s.Image = "registry.local/est:2" }, + func(s *Spec) { s.GPUs = []int{2} }, + func(s *Spec) { s.SSHPort = 2222 }, + func(s *Spec) { s.ExtraBinds = []string{"/a:/b"} }, + } { + c := validSpec() + mutate(c) + if c.Digest() == a.Digest() { + t.Errorf("container change kept the digest: %+v", c) + } + } +} + +func TestUnitName(t *testing.T) { + cases := map[string]string{ + "/raid/tenants/est/vol": "raid-tenants-est-vol.mount", + "/raid/tenants/est-2/vol": `raid-tenants-est\x2d2-vol.mount`, + "/srv/my.tenants/a_b/vol/": "srv-my.tenants-a_b-vol.mount", + } + for in, want := range cases { + if got := UnitName(in); got != want { + t.Errorf("UnitName(%q) = %q, want %q", in, got, want) + } + } + if out, err := osexec.Command("systemd-escape", "--path", "--suffix=mount", "/raid/tenants/est-2/vol").Output(); err == nil { + if got := strings.TrimSpace(string(out)); got != UnitName("/raid/tenants/est-2/vol") { + t.Errorf("systemd-escape says %q", got) + } + } +} + +func TestMountUnitOrdersBeforeDocker(t *testing.T) { + m := &Manager{Root: "/raid/tenants"} + u := MountUnit("est", m.Paths("est"), "ext4") + for _, want := range []string{"What=/raid/tenants/est/volume.img", "Where=/raid/tenants/est/vol", "Before=docker.service", + "WantedBy=multi-user.target docker.service", "Options=loop,discard,nodev,nosuid", "RequiresMountsFor=/raid/tenants/est"} { + if !strings.Contains(u, want) { + t.Errorf("mount unit lacks %q:\n%s", want, u) + } + } +} + +func TestRunArgs(t *testing.T) { + m := &Manager{Root: "/raid/tenants"} + s := validSpec() + s.ExtraBinds = []string{"/raid/data/est:/legacy:ro"} + args := strings.Join(m.RunArgs(s), " ") + for _, want := range []string{ + "--name tenant-est", "--restart=always", "--init", "--ipc=host", + "--runtime=nvidia --gpus \"device=2,3\"", + "-p 3889:22", "-p 8888:8888", "-p 9000:9000/udp", + "--mount type=bind,src=/raid/tenants/est/vol/data,dst=/data", + "--mount type=bind,src=/raid/tenants/est/vol/home,dst=/home", + "--mount type=bind,src=/raid/tenants/est/vol/tmp,dst=/tmp", + "--mount type=bind,src=/raid/tenants/est/system,dst=/etc/tenant-system,readonly", + "--mount type=bind,src=/raid/data/est,dst=/legacy,readonly", + "--label rootfiles.tenant.spec=" + s.Digest(), + "--entrypoint /bin/sh registry.local/est:1 /etc/tenant-system/init.sh", + } { + if !strings.Contains(args, want) { + t.Errorf("docker run lacks %q:\n%s", want, args) + } + } + s.GPUs = nil + if args := strings.Join(m.RunArgs(s), " "); strings.Contains(args, "gpus") || strings.Contains(args, "runtime") { + t.Errorf("GPU flags without GPUs: %s", args) + } +} + +func TestShadowFileKeepsUsableHashesOfSpecUsers(t *testing.T) { + s := validSpec() + s.Users = append(s.Users, User{Name: "new", UID: 1001, GID: 1001}) + shadow := "root:*:19000:0:99999:7:::\nest:$y$abc:20000:0:99999:7:::\nnew:!:20000:0:99999:7:::\nother:$6$x:1:::::\n" + if got := string(shadowFile(s, shadow)); got != "est:$y$abc:20000\n" { + t.Errorf("shadowFile = %q", got) + } +} + +func TestInitScriptSyntax(t *testing.T) { + if _, err := osexec.LookPath("sh"); err != nil { + t.Skip("no sh") + } + cmd := osexec.Command("sh", "-n") + cmd.Stdin = strings.NewReader(InitScript) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("init.sh: %v\n%s", err, out) + } +} + +func TestAuthorizeOps(t *testing.T) { + s := validSpec() + s.Operators = []string{"alice"} + if err := AuthorizeOps(s, "alice", true, "restart"); err != nil { + t.Errorf("listed operator refused: %v", err) + } + if err := AuthorizeOps(s, "root", false, "status"); err != nil { + t.Errorf("root refused: %v", err) + } + if AuthorizeOps(s, "bob", true, "status") == nil { + t.Error("unlisted account allowed") + } + for _, action := range []string{"image", "gpu", "create", "volume", ""} { + if AuthorizeOps(s, "alice", true, action) == nil { + t.Errorf("operator allowed %q", action) + } + } +} + +func TestLoadSpecRejectsUnknownFieldsAndRename(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) { + if err := os.WriteFile(filepath.Join(dir, name+".yaml"), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + data, _ := validSpec().Marshal() + write("est", string(data)) + if s, err := LoadSpec(dir, "est"); err != nil || s.Image != "registry.local/est:1" { + t.Fatalf("round trip: %v %+v", err, s) + } + write("other", string(data)) + if _, err := LoadSpec(dir, "other"); err == nil { + t.Error("file name and tenant name mismatch accepted") + } + write("typo", strings.Replace(string(data), "name: est", "name: typo\nimgae: x", 1)) + if _, err := LoadSpec(dir, "typo"); err == nil { + t.Error("unknown field accepted") + } + if names, _ := ListSpecs(dir); strings.Join(names, ",") != "est,other,typo" { + t.Errorf("ListSpecs = %v", names) + } +} + +func TestParseLegacy(t *testing.T) { + users := parseLegacyUsers("root:x:0:0::/root:/bin/bash\nsshd:x:110:65534::/run/sshd:/usr/sbin/nologin\nest:x:1000:1000::/home/est:/bin/bash\nsvc:x:1001:1001::/:/bin/false\n", map[string]bool{"est": true}) + if len(users) != 1 || users[0] != (User{Name: "est", UID: 1000, GID: 1000, Sudo: true}) { + t.Errorf("parseLegacyUsers = %+v", users) + } + var c legacyContainer + c.HostConfig.DeviceRequests = append(c.HostConfig.DeviceRequests, struct { + Driver string + Count int + DeviceIDs []string + }{DeviceIDs: []string{"3", "1"}}) + if g, err := legacyGPUs(c); err != nil || intList(g) != "1,3" { + t.Errorf("legacyGPUs = %v %v", g, err) + } + c.HostConfig.DeviceRequests[0].DeviceIDs, c.HostConfig.DeviceRequests[0].Count = nil, -1 + if _, err := legacyGPUs(c); err == nil { + t.Error("--gpus all accepted without IDs") + } +} + +// fakeDocker stands in for docker: inspect reports a running container, +// exec cat /etc/shadow prints a hash, and every argv and stdin is logged. +func fakeDocker(t *testing.T) (argsLog, stdinLog string) { + t.Helper() + dir := t.TempDir() + argsLog, stdinLog = filepath.Join(dir, "args"), filepath.Join(dir, "stdin") + script := `#!/bin/sh +echo "$*" >> ` + argsLog + ` +case "$1 $2" in + "inspect -f") [ -n "$FAKE_INSPECT" ] || exit 1; echo "$FAKE_INSPECT" ;; + "exec -i") cat >> ` + stdinLog + ` ;; + "exec tenant-est") [ "$3" = cat ] && echo 'est:$y$new:0:0:99999:7:::' ;; +esac +exit 0 +` + if err := os.WriteFile(filepath.Join(dir, "docker"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+":"+os.Getenv("PATH")) + return argsLog, stdinLog +} + +func TestPasswdNeverLogsPassword(t *testing.T) { + argsLog, stdinLog := fakeDocker(t) + t.Setenv("FAKE_INSPECT", "true|x|always") + var logs bytes.Buffer + runner := exec.NewRunner(false, slog.New(slog.NewTextHandler(&logs, nil))) + root := t.TempDir() + m := &Manager{Runner: runner, Root: root, Out: &bytes.Buffer{}} + s := validSpec() + if err := os.MkdirAll(m.Paths("est").System, 0o700); err != nil { + t.Fatal(err) + } + pw, err := m.Passwd(context.Background(), s, "est") + if err != nil || len(pw) != 16 { + t.Fatalf("Passwd = %q, %v", pw, err) + } + args, _ := os.ReadFile(argsLog) + stdin, _ := os.ReadFile(stdinLog) + if strings.Contains(string(args), pw) || strings.Contains(logs.String(), pw) { + t.Error("temporary password leaked into argv or the log") + } + if string(stdin) != "est:"+pw+"\n" { + t.Errorf("chpasswd stdin = %q", stdin) + } + if !strings.Contains(string(args), "exec tenant-est chage -d 0 est") { + t.Errorf("password not expired:\n%s", args) + } + shadow, _ := os.ReadFile(filepath.Join(m.Paths("est").System, "shadow")) + if string(shadow) != "est:$y$new:0\n" { + t.Errorf("stored hash = %q", shadow) + } + if _, err := m.Passwd(context.Background(), s, "nobody"); err == nil { + t.Error("unknown user accepted") + } +} + +func TestDryRunApplyChangesNothing(t *testing.T) { + argsLog, _ := fakeDocker(t) + t.Setenv("FAKE_INSPECT", "") + runner := exec.NewRunner(true, slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil))) + root := filepath.Join(t.TempDir(), "tenants") + specDir := filepath.Join(t.TempDir(), "specs") + m := &Manager{Runner: runner, Root: root, SpecDir: specDir, UnitDir: t.TempDir(), Out: &bytes.Buffer{}} + s := validSpec() + if err := m.SaveSpec(s); err != nil { + t.Fatal(err) + } + if err := m.Apply(context.Background(), s); err != nil { + t.Fatal(err) + } + if err := m.Resize(context.Background(), s, "20G", false); err == nil { + t.Error("resize without a volume image succeeded") + } + for _, p := range []string{root, specDir} { + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Errorf("dry run created %s", p) + } + } + if units, _ := os.ReadDir(m.UnitDir); len(units) != 0 { + t.Errorf("dry run wrote units: %v", units) + } + args, _ := os.ReadFile(argsLog) + for _, line := range strings.Split(strings.TrimSpace(string(args)), "\n") { + if !strings.HasPrefix(line, "inspect ") { + t.Errorf("dry run ran docker %s", line) + } + } +} diff --git a/internal/tenant/volume.go b/internal/tenant/volume.go new file mode 100644 index 0000000..e958c46 --- /dev/null +++ b/internal/tenant/volume.go @@ -0,0 +1,237 @@ +package tenant + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +// EnsureVolume creates the tenant volume image, its mount unit and the +// data/home/tmp directories, and grows the image to the spec size. +func (m *Manager) EnsureVolume(ctx context.Context, s *Spec) error { + p := m.Paths(s.Name) + size, err := s.Volume.Bytes() + if err != nil { + return err + } + if err := m.run(ctx, "install", "-d", "-m", "0755", m.Root); err != nil { + return err + } + // 0700: host users must not reach setuid files a tenant leaves in its volume. + if err := m.run(ctx, "install", "-d", "-m", "0700", p.Dir); err != nil { + return err + } + if err := m.run(ctx, "install", "-d", "-m", "0755", p.Mount); err != nil { + return err + } + fresh := !m.Runner.FileExists(p.Image) + if fresh { + m.printf("Creating %s volume of %s for %s\n", s.Volume.Filesystem(), s.Volume.Size, s.Name) + if err := m.run(ctx, "truncate", "-s", strconv.FormatUint(size, 10), p.Image); err != nil { + return err + } + mkfs := []string{"mkfs.ext4", "-q", "-F", "-m", "0", p.Image} + if s.Volume.Filesystem() == "xfs" { + mkfs = []string{"mkfs.xfs", "-q", p.Image} + } + if err := m.run(ctx, mkfs[0], mkfs[1:]...); err != nil { + return err + } + } + if err := m.ensureMountUnit(ctx, s); err != nil { + return err + } + if !m.Runner.DryRun && !isMountPoint(p.Mount) { + return fmt.Errorf("%s is not mounted; check `systemctl status %s`", p.Mount, UnitName(p.Mount)) + } + uid, gid := strconv.Itoa(s.Users[0].UID), strconv.Itoa(s.Users[0].GID) + for _, d := range [][]string{ + {"-m", "0755", "-o", uid, "-g", gid, filepath.Join(p.Mount, "data")}, + {"-m", "0755", filepath.Join(p.Mount, "home")}, + {"-m", "1777", filepath.Join(p.Mount, "tmp")}, + } { + if !fresh && m.Runner.FileExists(d[len(d)-1]) { + continue // keep what the tenant changed + } + if err := m.run(ctx, "install", append([]string{"-d"}, d...)...); err != nil { + return err + } + } + if fresh { + return nil + } + cur, err := fileSize(p.Image) + if err != nil { + return err + } + switch { + case size > cur: + return m.grow(ctx, s, size) + case size < cur: + m.printf("⚠ %s volume is %s, larger than the spec's %s; shrink it with `rootfiles tenant volume resize %s %s --offline`\n", + s.Name, human(cur), s.Volume.Size, s.Name, s.Volume.Size) + } + return nil +} + +func (m *Manager) ensureMountUnit(ctx context.Context, s *Spec) error { + p := m.Paths(s.Name) + unit := UnitName(p.Mount) + changed, err := m.writeUnit(s) + if err != nil { + return err + } + if changed { + if err := m.run(ctx, "systemctl", "daemon-reload"); err != nil { + return err + } + } + if res, err := m.Runner.Query(ctx, "systemctl", "is-enabled", unit); err != nil || strings.TrimSpace(res.Stdout) != "enabled" { + if err := m.run(ctx, "systemctl", "enable", unit); err != nil { + return err + } + } + if m.Runner.DryRun || !isMountPoint(p.Mount) { + return m.run(ctx, "systemctl", "start", unit) + } + return nil +} + +// writeUnit writes the mount unit when it differs and reports a change. +func (m *Manager) writeUnit(s *Spec) (bool, error) { + want := MountUnit(s.Name, m.Paths(s.Name), s.Volume.Filesystem()) + path := m.unitPath(s.Name) + if cur, err := os.ReadFile(path); err == nil && string(cur) == want { + return false, nil + } + return true, m.writeIfChanged(path, []byte(want), 0o644) +} + +// UnitDrift reports whether tenant s's mount unit file is missing or stale. +func (m *Manager) UnitDrift(s *Spec) bool { + cur, err := os.ReadFile(m.unitPath(s.Name)) + return err != nil || string(cur) != MountUnit(s.Name, m.Paths(s.Name), s.Volume.Filesystem()) +} + +// WriteUnit writes tenant s's mount unit file; the caller reloads systemd. +func (m *Manager) WriteUnit(s *Spec) error { + _, err := m.writeUnit(s) + return err +} + +// UnitFor returns the mount unit name of tenant t. +func (m *Manager) UnitFor(t string) string { return UnitName(m.Paths(t).Mount) } + +func fileSize(path string) (uint64, error) { + fi, err := os.Stat(path) + if err != nil { + return 0, err + } + return uint64(fi.Size()), nil +} + +// loopDevice returns the loop device backing the tenant image. +func (m *Manager) loopDevice(ctx context.Context, p Paths) (string, error) { + res, err := m.Runner.Query(ctx, "losetup", "-j", p.Image) + if err != nil { + return "", err + } + dev, _, ok := strings.Cut(strings.TrimSpace(res.Stdout), ":") + if !ok || !strings.HasPrefix(dev, "/dev/loop") { + return "", fmt.Errorf("no loop device backs %s; is %s mounted?", p.Image, p.Mount) + } + return dev, nil +} + +// grow enlarges a mounted volume without stopping the container. +func (m *Manager) grow(ctx context.Context, s *Spec, size uint64) error { + p := m.Paths(s.Name) + dev := "/dev/loopN" + if !m.Runner.DryRun { + d, err := m.loopDevice(ctx, p) + if err != nil { + return err + } + dev = d + } + m.printf("Growing %s volume to %s online\n", s.Name, human(size)) + if err := m.run(ctx, "truncate", "-s", strconv.FormatUint(size, 10), p.Image); err != nil { + return err + } + if err := m.run(ctx, "losetup", "-c", dev); err != nil { + return err + } + if s.Volume.Filesystem() == "xfs" { + return m.run(ctx, "xfs_growfs", p.Mount) + } + return m.run(ctx, "resize2fs", dev) +} + +// Resize sets the volume size. Growing works while the container runs; +// shrinking needs offline (ext4 only), which stops the container. +func (m *Manager) Resize(ctx context.Context, s *Spec, newSize string, offline bool) error { + s.Volume.Size = newSize + if err := s.Validate(); err != nil { + return err + } + size, _ := s.Volume.Bytes() + p := m.Paths(s.Name) + cur, err := fileSize(p.Image) + if err != nil { + return fmt.Errorf("tenant %s has no volume image: %w", s.Name, err) + } + switch { + case size == cur: + m.printf("%s volume is already %s\n", s.Name, human(size)) + case size > cur: + if err := m.grow(ctx, s, size); err != nil { + return err + } + case !offline: + return fmt.Errorf("shrinking a mounted volume is not supported; rerun with --offline to stop %s and shrink it", ContainerName(s.Name)) + case s.Volume.Filesystem() != "ext4": + return fmt.Errorf("%s volumes cannot shrink", s.Volume.Filesystem()) + default: + if err := m.shrink(ctx, s, size); err != nil { + return err + } + } + return m.SaveSpec(s) +} + +func (m *Manager) shrink(ctx context.Context, s *Spec, size uint64) error { + p := m.Paths(s.Name) + name := ContainerName(s.Name) + unit := UnitName(p.Mount) + running := m.inspect(ctx, name).Running + m.printf("Shrinking %s volume to %s offline\n", s.Name, human(size)) + if running { + if err := m.run(ctx, "docker", "stop", name); err != nil { + return err + } + } + if err := m.run(ctx, "systemctl", "stop", unit); err != nil { + return err + } + // e2fsck exits 1 after correcting errors; the image is then consistent. + if res, err := m.Runner.Run(ctx, "e2fsck", "-f", "-p", p.Image); err != nil && (res == nil || res.ExitCode != 1) { + return fmt.Errorf("checking %s before shrink (volume left unmounted): %w", p.Image, err) + } + if err := m.run(ctx, "resize2fs", p.Image, strconv.FormatUint(size/1024, 10)+"K"); err != nil { + _ = m.run(ctx, "systemctl", "start", unit) + return fmt.Errorf("shrinking filesystem (data too large for %s?): %w", s.Volume.Size, err) + } + if err := m.run(ctx, "truncate", "-s", strconv.FormatUint(size, 10), p.Image); err != nil { + return err + } + if err := m.run(ctx, "systemctl", "start", unit); err != nil { + return err + } + if running { + return m.run(ctx, "docker", "start", name) + } + return nil +} diff --git a/tests/scenarios/tenant-lifecycle.sh b/tests/scenarios/tenant-lifecycle.sh new file mode 100755 index 0000000..3d692ec --- /dev/null +++ b/tests/scenarios/tenant-lifecycle.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Host-run acceptance scenario for `rootfiles tenant` on a disposable Docker + +# systemd host (the CI runner): host prep, create, SSH with a temporary +# password, recreate on an image change, volume quota and online growth, +# mount after a unit restart, operator delegation, dry run and import. +# GPU steps (runtime=nvidia, reload safety) need a GPU host and are not run. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "$REPO_ROOT/tests/integration/assert.sh" + +BIN="${ROOTFILES_TEST_BINARY:?ROOTFILES_TEST_BINARY must name a Linux rootfiles binary}" +sudo install -m 0755 -o root -g root "$BIN" /usr/local/bin/rootfiles +RF="sudo /usr/local/bin/rootfiles" +command -v sshpass >/dev/null || sudo apt-get install -y -qq sshpass >/dev/null + +TMP="$(mktemp -d)" +KH="$TMP/known_hosts" +cleanup() { + docker rm -f tenant-t1 tenant-t1-prev tenant-t2 tenant-t3 legacy >/dev/null 2>&1 || true +} +trap cleanup EXIT + +check() { # check DESCRIPTION COMMAND... + local desc=$1; shift + if "$@"; then pass "$desc"; else fail "$desc"; fi +} +ssh_as() { # ssh_as USER PASSWORD PORT COMMAND + sshpass -p "$2" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile="$KH" -o PreferredAuthentications=password \ + -o PubkeyAuthentication=no -o ConnectTimeout=10 -p "$3" "$1@127.0.0.1" "$4" +} +temp_password() { sed -n 's/^Temporary password for [^:]*: \([^ ]*\) .*/\1/p' | head -n1; } + +echo "=== Scenario: tenant-lifecycle ===" + +docker build -q -t rootfiles-tenant-test:1 - >/dev/null <<'EOF' +FROM ubuntu:24.04 +RUN apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq openssh-server >/dev/null && rm -rf /var/lib/apt/lists/* +EOF +docker build -q -t rootfiles-tenant-test:2 - >/dev/null <<'EOF' +FROM rootfiles-tenant-test:1 +RUN touch /image-v2 +EOF + +sudo mkdir -p /etc/rootfiles-test +sudo tee /etc/rootfiles-test/tenant.yaml >/dev/null <<'EOF' +extends: minimal +users: + home_base: /srv/home +modules: + tenant: + enabled: true + root: /srv/tenants + ops_group: tenant-ops +EOF + +echo "--- host prep ---" +$RF apply --config /etc/rootfiles-test/tenant.yaml --module tenant --yes +check "guard timer enabled" systemctl is-enabled --quiet rootfiles-gpu-guard.timer +check "sync timer enabled" systemctl is-enabled --quiet rootfiles-tenant-sync.timer +check "operator group created" getent group tenant-ops +check "operator sudoers rule" sudo grep -q '^%tenant-ops ALL=(root) NOPASSWD: /usr/local/bin/rootfiles tenant ops \*$' /etc/sudoers.d/rootfiles-tenant-ops +check "tenant module satisfied after apply" $RF check --module tenant + +echo "--- AC10: dry run changes nothing ---" +$RF tenant create t9 --image rootfiles-tenant-test:1 --ssh-port 2209 --size 64M --account bob:2009 --dry-run >/dev/null +check "dry run wrote no spec" sudo test ! -e /srv/home/.rootfiles/tenants/t9.yaml +check "dry run made no volume" sudo test ! -e /srv/tenants/t9 +check "dry run started no container" bash -c '! docker inspect tenant-t9 >/dev/null 2>&1' + +echo "--- AC1: create and first login ---" +out=$($RF tenant create t1 --image rootfiles-tenant-test:1 --ssh-port 2201 --size 100M --account alice:2000 --sudo --operator opsuser) +echo "$out" +PW=$(temp_password <<<"$out") +check "temporary password issued" test -n "$PW" +check "container running" bash -c '[ "$(docker inspect -f {{.State.Running}} tenant-t1)" = true ]' +check "volume mounted" mountpoint -q /srv/tenants/t1/vol +check "mount unit enabled" systemctl is-enabled --quiet srv-tenants-t1-vol.mount +login=$(ssh_as alice "$PW" 2201 true 2>&1 || true) +echo "$login" | tail -n 3 +check "temporary password accepted and change forced" grep -qiE 'expired|change.*password' <<<"$login" + +echo "--- AC7: no password in logs ---" +check "password not in rootfiles.log" bash -c "! sudo grep -qF '$PW' /var/log/rootfiles.log" +check "password not in tenant audit log" bash -c "! sudo grep -qF '$PW' /var/log/rootfiles-tenant.jsonl" +check "audit records create" sudo grep -q '"action":"create","result":"ok"' /var/log/rootfiles-tenant.jsonl + +echo "--- AC2: image change keeps data, password, host key ---" +docker exec -i tenant-t1 chpasswd <<<"alice:Changed-Pass-42" +check "changed password logs in" ssh_as alice Changed-Pass-42 2201 'echo kept > /data/hello && id -u' +key_before=$(ssh-keyscan -t ed25519 -p 2201 127.0.0.1 2>/dev/null | awk '{print $3}') +$RF tenant image t1 rootfiles-tenant-test:2 +check "new image running" docker exec tenant-t1 test -f /image-v2 +check "previous container kept" docker inspect tenant-t1-prev +key_after=$(ssh-keyscan -t ed25519 -p 2201 127.0.0.1 2>/dev/null | awk '{print $3}') +check "host key survived" test -n "$key_before" -a "$key_before" = "$key_after" +check "changed password survived" ssh_as alice Changed-Pass-42 2201 'test "$(cat /data/hello)" = kept' +check "recreate refused while -prev exists" bash -c "! $RF tenant port t1 2211 >/dev/null 2>&1" +docker rm tenant-t1-prev >/dev/null +# Back to the running container's settings: nothing to recreate. +check "apply is idempotent" bash -c "$RF tenant port t1 2201 | grep -q 'up to date'" + +echo "--- AC3: quota and online growth ---" +started=$(docker inspect -f '{{.State.StartedAt}}' tenant-t1) +check "writes past the quota fail" bash -c '! docker exec tenant-t1 dd if=/dev/zero of=/data/fill bs=1M count=150 2>/dev/null' +docker exec tenant-t1 rm -f /data/fill +$RF tenant volume resize t1 300M +check "grown volume takes the write" docker exec tenant-t1 dd if=/dev/zero of=/data/fill bs=1M count=150 status=none +check "container kept running through growth" test "$started" = "$(docker inspect -f '{{.State.StartedAt}}' tenant-t1)" +check "online shrink refused" bash -c "! $RF tenant volume resize t1 200M >/dev/null 2>&1" +docker exec tenant-t1 rm -f /data/fill + +echo "--- AC8: volume comes back after a unit restart ---" +check "mount unit ordered before docker" bash -c 'systemctl show -p Before srv-tenants-t1-vol.mount | grep -q docker.service' +docker stop tenant-t1 >/dev/null +sudo systemctl restart srv-tenants-t1-vol.mount +docker start tenant-t1 >/dev/null +check "data intact after remount" docker exec tenant-t1 test -f /data/hello + +echo "--- AC6: operator delegation ---" +$RF tenant create t2 --image rootfiles-tenant-test:1 --ssh-port 2202 --size 64M --account carol:2100 >/dev/null +sudo useradd -m opsuser +sudo usermod -aG tenant-ops opsuser +ops() { sudo -u opsuser sudo -n /usr/local/bin/rootfiles tenant ops "$@"; } +check "operator status" ops t1 status +check "operator restart" ops t1 restart +check "operator reset" bash -c "sudo -u opsuser sudo -n /usr/local/bin/rootfiles tenant ops t1 reset alice | grep -q 'Temporary password'" +check "operator refused on unlisted tenant" bash -c '! sudo -u opsuser sudo -n /usr/local/bin/rootfiles tenant ops t2 status 2>/dev/null' +check "operator refused other actions" bash -c '! sudo -u opsuser sudo -n /usr/local/bin/rootfiles tenant ops t1 image 2>/dev/null' +check "operator refused flags" bash -c '! sudo -u opsuser sudo -n /usr/local/bin/rootfiles tenant ops t1 status --config /tmp/x 2>/dev/null' +check "operator cannot run other commands" bash -c '! sudo -u opsuser sudo -n /usr/local/bin/rootfiles tenant create x 2>/dev/null' +check "refusals audited" bash -c 'sudo grep -c "\"actor\":\"opsuser\".*refused" /var/log/rootfiles-tenant.jsonl | grep -q "^3$"' + +echo "--- AC9: import ---" +sudo mkdir -p /srv/legacy-data +sudo dd if=/dev/zero of=/srv/legacy-data/blob bs=1M count=200 status=none +docker run -d --name legacy -p 2203:22 -v /srv/legacy-data:/data --entrypoint sleep rootfiles-tenant-test:1 infinity >/dev/null +docker exec legacy sh -c 'mkdir -p /home/ubuntu && echo keep > /home/ubuntu/keep.txt' +check "import refused when data exceeds the volume" bash -c "! $RF tenant import t3 --from legacy --size 128M --data /srv/legacy-data" +check "refused import changed nothing" sudo test ! -e /srv/home/.rootfiles/tenants/t3.yaml +check "dry-run import prints the plan" bash -c "$RF tenant import t3 --from legacy --size 512M --data /srv/legacy-data --keep /home/ubuntu --dry-run | grep -q 'Import plan'" +check "dry-run import changed nothing" bash -c '[ "$(docker inspect -f {{.State.Running}} legacy)" = true ] && ! sudo test -e /srv/tenants/t3' +$RF tenant import t3 --from legacy --size 512M --data /srv/legacy-data --keep /home/ubuntu +check "imported data" docker exec tenant-t3 test -s /data/blob +check "imported writable-layer path" docker exec tenant-t3 test -f /home/ubuntu/keep.txt +check "legacy container kept stopped" bash -c '[ "$(docker inspect -f {{.State.Running}}/{{.HostConfig.RestartPolicy.Name}} legacy)" = false/no ]' + +$RF tenant status +report From abc370ea624bbc6b617afe93a9309c0b6d5f1469 Mon Sep 17 00:00:00 2001 From: Young Joon Lee Date: Tue, 20 Oct 2026 08:58:56 +0900 Subject: [PATCH 2/6] docs(release): v0.16.0 release notes Claude-Session: https://claude.ai/code/session_0177Zk7to4EYohgM59v4Fcj7 --- docs/releases/v0.16.0.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/releases/v0.16.0.md diff --git a/docs/releases/v0.16.0.md b/docs/releases/v0.16.0.md new file mode 100644 index 0000000..b476d05 --- /dev/null +++ b/docs/releases/v0.16.0.md @@ -0,0 +1,13 @@ +GPU hosts can now run long-lived SSH containers for outside groups with `rootfiles tenant`, each with a size-capped volume, accounts that survive image changes and GPUs from the shared allocation ledger. + +### New +- `rootfiles tenant create` starts a tenant container with its own SSH port, GPUs on the NVIDIA runtime (kept after a `systemctl daemon-reload`) and a volume for `/data`, `/home` and `/tmp` capped at `--size`. Each account gets a temporary password that must be changed at first login; it is printed once and never logged. (#79) +- `rootfiles tenant image`, `gpu` and `port` recreate the container and keep the volume, the accounts' passwords, the SSH host key and GPU access. A failed check of the new container rolls back to the old one, which is otherwise kept as `tenant--prev`. (#79) +- `rootfiles tenant volume resize` grows a volume while the container runs; `--offline` shrinks an ext4 volume. `rootfiles tenant status` shows volume use, spec drift and whether the volume sizes exceed the free space. (#79) +- `rootfiles tenant user add`, `passwd`/`reset`, `restart`, `logs`, `apply`, `list` and `sync` manage accounts and containers; every change is recorded in `/var/log/rootfiles-tenant.jsonl`. (#79) +- Host accounts in the operator group can run `sudo rootfiles tenant ops status|logs|gpu-heal|restart|reset` for the tenants that list them, without other sudo rights. (#79) +- `rootfiles tenant import` moves a hand-made container under rootfiles: it copies its accounts, passwords, host keys and the data you name into a new volume, keeps the old container stopped, and refuses before any change when the data does not fit. (#79) +- The opt-in `tenant` module (`modules.tenant.enabled`) installs a GPU device guard that restores GPU access of `--gpus` containers after a `systemctl daemon-reload`, a daily password sync, `fstrim.timer`, the operator sudoers rule and the tenant volume mount units. See `docs/tenant.md`. (#79) +- `rootfiles gpu assign` now also refuses GPUs that a tenant holds. (#79) + +**Full changelog**: https://github.com/STAIxBWLB/rootfiles/compare/v0.15.1...v0.16.0 From a2758b289ef8d2c3b54acd8f82fb28845617cfc3 Mon Sep 17 00:00:00 2001 From: Young Joon Lee Date: Tue, 20 Oct 2026 09:00:26 +0900 Subject: [PATCH 3/6] fix(tenant): contain import copies and undo partial recreate and shrink Claude-Session: https://claude.ai/code/session_0177Zk7to4EYohgM59v4Fcj7 --- internal/tenant/import.go | 17 ++++++++++++----- internal/tenant/manager.go | 11 +++++++++-- internal/tenant/volume.go | 4 +++- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/internal/tenant/import.go b/internal/tenant/import.go index a3b40e0..15efceb 100644 --- a/internal/tenant/import.go +++ b/internal/tenant/import.go @@ -138,6 +138,12 @@ func (m *Manager) Import(ctx context.Context, o ImportOptions) (*Spec, error) { if merged == "" { return nil, fmt.Errorf("cannot read the writable layer of %s (no overlay MergedDir)", o.From) } + // A symlink inside the container (say /home -> /) would resolve + // against the host's root here, so the path must stay in the layer. + base, _ := filepath.EvalSymlinks(merged) // docker's data root may itself be a symlink + if real, err := filepath.EvalSymlinks(filepath.Join(merged, k)); err != nil || !strings.HasPrefix(real, base+"/") { + return nil, fmt.Errorf("--keep %s: not a path inside %s's filesystem (%v)", k, o.From, err) + } items = append(items, copyItem{merged + "/." + k, p.Mount + "/"}) } var total uint64 @@ -167,14 +173,14 @@ func (m *Manager) Import(ctx context.Context, o ImportOptions) (*Spec, error) { return nil, fmt.Errorf("data to copy (%s) exceeds the free space on %s (%s)", human(total), m.Root, human(free)) } - if err := m.SaveSpec(s); err != nil { - return nil, err - } if m.ReserveGPUs != nil { if err := m.ReserveGPUs(ctx, s.Name, s.GPUs); err != nil { return nil, err } } + if err := m.SaveSpec(s); err != nil { + return nil, err + } if err := m.EnsureVolume(ctx, s); err != nil { return nil, err } @@ -184,8 +190,9 @@ func (m *Manager) Import(ctx context.Context, o ImportOptions) (*Spec, error) { for _, kt := range []string{"ed25519", "ecdsa", "rsa"} { for _, suffix := range []string{"", ".pub"} { f := "ssh_host_" + kt + "_key" + suffix - // A missing key type is generated by writeSystem. - _ = m.run(ctx, "docker", "cp", o.From+":/etc/ssh/"+f, filepath.Join(p.System, "ssh", f)) + // A missing key type is generated by writeSystem. -L copies the + // content, never a symlink that would point into the host. + _ = m.run(ctx, "docker", "cp", "-L", o.From+":/etc/ssh/"+f, filepath.Join(p.System, "ssh", f)) } } if sh, err := m.Runner.Query(ctx, "docker", "exec", o.From, "cat", "/etc/shadow"); err == nil { diff --git a/internal/tenant/manager.go b/internal/tenant/manager.go index 10d3771..e7dc073 100644 --- a/internal/tenant/manager.go +++ b/internal/tenant/manager.go @@ -218,9 +218,16 @@ func (m *Manager) recreate(ctx context.Context, s *Spec, st containerState) erro {"rename", name, prev}, {"update", "--restart=no", prev}, } - for _, a := range steps { + for i, a := range steps { if err := m.run(ctx, "docker", a...); err != nil { - return fmt.Errorf("preparing recreate: %w", err) + // Undo what ran: the rename, then the stop. + if i == 2 { + _ = m.run(ctx, "docker", "rename", prev, name) + } + if i > 0 { + _ = m.run(ctx, "docker", "start", name) + } + return fmt.Errorf("preparing recreate (old container restarted): %w", err) } } err := m.run(ctx, "docker", m.RunArgs(s)...) diff --git a/internal/tenant/volume.go b/internal/tenant/volume.go index e958c46..a2c42fe 100644 --- a/internal/tenant/volume.go +++ b/internal/tenant/volume.go @@ -221,7 +221,9 @@ func (m *Manager) shrink(ctx context.Context, s *Spec, size uint64) error { return fmt.Errorf("checking %s before shrink (volume left unmounted): %w", p.Image, err) } if err := m.run(ctx, "resize2fs", p.Image, strconv.FormatUint(size/1024, 10)+"K"); err != nil { - _ = m.run(ctx, "systemctl", "start", unit) + if m.run(ctx, "systemctl", "start", unit) == nil && running { + _ = m.run(ctx, "docker", "start", name) + } return fmt.Errorf("shrinking filesystem (data too large for %s?): %w", s.Volume.Size, err) } if err := m.run(ctx, "truncate", "-s", strconv.FormatUint(size, 10), p.Image); err != nil { From 7739f1f4bcd8274acab3ff192514eed60acf137a Mon Sep 17 00:00:00 2001 From: Young Joon Lee Date: Tue, 20 Oct 2026 09:05:41 +0900 Subject: [PATCH 4/6] test(tenant): make the runner's /usr/local/bin root-only for the sudoers check Claude-Session: https://claude.ai/code/session_0177Zk7to4EYohgM59v4Fcj7 --- tests/scenarios/tenant-lifecycle.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/scenarios/tenant-lifecycle.sh b/tests/scenarios/tenant-lifecycle.sh index 3d692ec..c92616d 100755 --- a/tests/scenarios/tenant-lifecycle.sh +++ b/tests/scenarios/tenant-lifecycle.sh @@ -10,6 +10,9 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" source "$REPO_ROOT/tests/integration/assert.sh" BIN="${ROOTFILES_TEST_BINARY:?ROOTFILES_TEST_BINARY must name a Linux rootfiles binary}" +# The operator sudoers rule needs a binary only root can replace; hosted +# runners ship a group-writable /usr/local/bin. +sudo chmod go-w /usr/local /usr/local/bin sudo install -m 0755 -o root -g root "$BIN" /usr/local/bin/rootfiles RF="sudo /usr/local/bin/rootfiles" command -v sshpass >/dev/null || sudo apt-get install -y -qq sshpass >/dev/null From c8618646df693f5befbc589eb45b89da159481cd Mon Sep 17 00:00:00 2001 From: Young Joon Lee Date: Tue, 20 Oct 2026 09:11:32 +0900 Subject: [PATCH 5/6] test(tenant): read the root-only mount point with sudo; name verified GPUs Claude-Session: https://claude.ai/code/session_0177Zk7to4EYohgM59v4Fcj7 --- internal/tenant/manager.go | 6 +++++- tests/scenarios/tenant-lifecycle.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/tenant/manager.go b/internal/tenant/manager.go index e7dc073..5888586 100644 --- a/internal/tenant/manager.go +++ b/internal/tenant/manager.go @@ -294,7 +294,11 @@ func (m *Manager) Verify(ctx context.Context, s *Spec) error { return fmt.Errorf("GPU check: nvidia-smi lists %d GPUs, want %d", n, len(s.GPUs)) } } - m.printf("✓ %s: sshd on port %d, host key and GPUs verified\n", ContainerName(s.Name), s.SSHPort) + what := "sshd and host key" + if len(s.GPUs) > 0 { + what += " and GPUs " + intList(s.GPUs) + } + m.printf("✓ %s: %s verified on port %d\n", ContainerName(s.Name), what, s.SSHPort) return nil } diff --git a/tests/scenarios/tenant-lifecycle.sh b/tests/scenarios/tenant-lifecycle.sh index c92616d..c219019 100755 --- a/tests/scenarios/tenant-lifecycle.sh +++ b/tests/scenarios/tenant-lifecycle.sh @@ -77,7 +77,7 @@ echo "$out" PW=$(temp_password <<<"$out") check "temporary password issued" test -n "$PW" check "container running" bash -c '[ "$(docker inspect -f {{.State.Running}} tenant-t1)" = true ]' -check "volume mounted" mountpoint -q /srv/tenants/t1/vol +check "volume mounted" sudo mountpoint -q /srv/tenants/t1/vol check "mount unit enabled" systemctl is-enabled --quiet srv-tenants-t1-vol.mount login=$(ssh_as alice "$PW" 2201 true 2>&1 || true) echo "$login" | tail -n 3 From 5912e19a95d95a7d6ff26b2dd34a6a349027bf1f Mon Sep 17 00:00:00 2001 From: Young Joon Lee Date: Tue, 20 Oct 2026 09:23:12 +0900 Subject: [PATCH 6/6] fix(tenant): restore the GPU ledger on failure, keep passwords across images, reserve -prev Claude-Session: https://claude.ai/code/session_0177Zk7to4EYohgM59v4Fcj7 --- .../2026-10-20-tenant-containers-design.md | 3 +- docs/tenant.md | 2 +- internal/tenant/assets.go | 9 +++-- internal/tenant/manager.go | 31 +++++++++++++++-- internal/tenant/spec.go | 3 ++ internal/tenant/tenant_test.go | 33 +++++++++++++++++-- 6 files changed, 71 insertions(+), 10 deletions(-) diff --git a/docs/specs/2026-10-20-tenant-containers-design.md b/docs/specs/2026-10-20-tenant-containers-design.md index 53f931a..df45acf 100644 --- a/docs/specs/2026-10-20-tenant-containers-design.md +++ b/docs/specs/2026-10-20-tenant-containers-design.md @@ -161,7 +161,8 @@ Implementation starts only on the owner's go-ahead. - Config lives under `modules.tenant` (`enabled`, `root`, `ops_group`). The default operator group is `rootfiles-tenant-ops`. - The container mounts the volume subdirectories with `--mount type=bind` instead of a named Docker volume `tenant-`. A bind mount fails when its source is missing, so a container cannot start on the bare mount point of an unmounted volume and write past the quota. The named volume added nothing else. - The volume image is mounted by a systemd `.mount` unit with `loop,discard,nodev,nosuid`, `Before=docker.service` and `WantedBy=docker.service`. `nosuid,nodev` and a root-only `/` keep host users away from setuid files a tenant leaves in its volume. -- Stored password hashes only seed accounts that have no usable password (a fresh container). A password changed inside a container therefore survives a plain restart even before the daily sync. +- On a container's first boot the stored password hashes replace whatever the image ships. On later boots they only seed accounts without a usable password, so a password changed inside a container survives a plain restart even before the daily sync. +- Tenant names cannot end in `-prev`, the suffix of rollback containers. Containers carry a `rootfiles.tenant.gpus` label; a failed apply returns the GPU ledger to the GPUs of the container left running. - The daily sync is `rootfiles tenant sync`, run by `rootfiles-tenant-sync.timer`. `tenant list` prints tenant names. - A recreate refuses while `tenant--prev` still exists, so an earlier rollback copy is never discarded silently. - `tenant ops` refuses every flag and clears `ROOTFILES_*` variables before loading the host's applied config, because the sudoers wildcard would otherwise let an operator point `--config` at a spec they control. diff --git a/docs/tenant.md b/docs/tenant.md index 9d2d022..37106dc 100644 --- a/docs/tenant.md +++ b/docs/tenant.md @@ -38,7 +38,7 @@ sudo rootfiles tenant create est \ This writes `/.rootfiles/tenants/est.yaml`, creates the volume, starts `tenant-est`, checks sshd, the host key and the GPUs, and prints a temporary password per account. The password must be changed at first login. It is printed once and never written to a log or passed on a command line. -The image needs only `openssh-server` (and `useradd`, `chpasswd`, which Debian, Ubuntu and RHEL images have). rootfiles starts the container with its own read-only entrypoint, `/etc/tenant-system/init.sh`, which on every start creates the accounts, seeds stored password hashes into accounts that have none, restores the host keys, starts sshd and runs `/data/.tenant/autostart.sh` as the first account when that file exists. +The image needs only `openssh-server` (and `useradd`, `chpasswd`, which Debian, Ubuntu and RHEL images have). rootfiles starts the container with its own read-only entrypoint, `/etc/tenant-system/init.sh`, which on every start creates the accounts, applies the stored password hashes (on a new container to every account, later only to accounts without a usable password), restores the host keys, starts sshd and runs `/data/.tenant/autostart.sh` as the first account when that file exists. | Inside the container | Host | Limit | |---|---|---| diff --git a/internal/tenant/assets.go b/internal/tenant/assets.go index 2f7983f..be90c7c 100644 --- a/internal/tenant/assets.go +++ b/internal/tenant/assets.go @@ -13,6 +13,7 @@ const InitScript = `#!/bin/sh # Managed by rootfiles: tenant boot contract. Do not edit. set -u SYS=` + SystemMount + ` +BOOTED=/var/lib/rootfiles-tenant-booted log() { echo "tenant-init: $*"; } sync_users() { @@ -37,13 +38,14 @@ sync_users() { if [ "$sudo" = 1 ]; then usermod -aG "$g" "$name"; else gpasswd -d "$name" "$g" >/dev/null 2>&1; fi done done < "$SYS/users" - # Stored hashes only seed accounts without a usable password, so a - # password changed in this container survives a restart. + # On a container's first boot the stored hashes replace whatever the + # image ships; later they only seed accounts without a usable password, + # so a password changed in this container survives a restart. [ -f "$SYS/shadow" ] || return 0 while IFS=: read -r name hash lastchg; do [ -n "$name" ] && [ -n "$hash" ] || continue cur=$(getent shadow "$name" | cut -d: -f2) || continue - case "$cur" in ''|'!'*|'*'*) ;; *) continue ;; esac + [ -e "$BOOTED" ] && case "$cur" in ''|'!'*|'*'*) ;; *) continue ;; esac printf '%s:%s\n' "$name" "$hash" | chpasswd -e || { log "restoring password of $name failed"; continue; } [ -n "$lastchg" ] && chage -d "$lastchg" "$name" done < "$SYS/shadow" @@ -55,6 +57,7 @@ if [ "${1:-boot}" = users ]; then fi sync_users +mkdir -p /var/lib && : > "$BOOTED" if ls "$SYS"/ssh/ssh_host_*_key >/dev/null 2>&1; then rm -f /etc/ssh/ssh_host_* cp "$SYS"/ssh/ssh_host_* /etc/ssh/ diff --git a/internal/tenant/manager.go b/internal/tenant/manager.go index 5888586..dbfe0ff 100644 --- a/internal/tenant/manager.go +++ b/internal/tenant/manager.go @@ -91,18 +91,19 @@ type containerState struct { Exists, Running bool Digest string Restart string + GPUs string // the GPUs the container was started with } func (m *Manager) inspect(ctx context.Context, name string) containerState { res, err := m.Runner.Query(ctx, "docker", "inspect", "-f", - `{{.State.Running}}|{{index .Config.Labels "rootfiles.tenant.spec"}}|{{.HostConfig.RestartPolicy.Name}}`, name) + `{{.State.Running}}|{{index .Config.Labels "rootfiles.tenant.spec"}}|{{.HostConfig.RestartPolicy.Name}}|{{index .Config.Labels "rootfiles.tenant.gpus"}}`, name) if err != nil { return containerState{} } f := strings.Split(strings.TrimSpace(res.Stdout), "|") st := containerState{Exists: true} - if len(f) == 3 { - st.Running, st.Digest, st.Restart = f[0] == "true", f[1], f[2] + if len(f) == 4 { + st.Running, st.Digest, st.Restart, st.GPUs = f[0] == "true", f[1], f[2], f[3] } return st } @@ -116,6 +117,7 @@ func (m *Manager) RunArgs(s *Spec) []string { "--restart=always", "--init", "--ipc=host", "--label", "rootfiles.tenant=" + s.Name, "--label", "rootfiles.tenant.spec=" + s.Digest(), + "--label", "rootfiles.tenant.gpus=" + intList(s.GPUs), } if len(s.GPUs) > 0 { ids := make([]string, len(s.GPUs)) @@ -154,6 +156,8 @@ func (m *Manager) RunArgs(s *Spec) []string { // --- apply --- // Apply converges tenant s: GPU ledger, volume, system files and container. +// When it fails, the ledger goes back to the GPUs of whatever container is +// left running, so GPUs a rolled-back container still uses stay reserved. func (m *Manager) Apply(ctx context.Context, s *Spec) error { if err := s.Validate(); err != nil { return err @@ -163,6 +167,27 @@ func (m *Manager) Apply(ctx context.Context, s *Spec) error { return err } } + err := m.converge(ctx, s) + if err != nil && m.ReserveGPUs != nil && !m.Runner.DryRun { + left := m.inspect(ctx, ContainerName(s.Name)) + if rerr := m.ReserveGPUs(ctx, s.Name, parseInts(left.GPUs)); rerr != nil { + err = fmt.Errorf("%w; restoring the GPU ledger also failed: %v", err, rerr) + } + } + return err +} + +func parseInts(s string) []int { + var out []int + for _, f := range strings.Split(s, ",") { + if n, err := strconv.Atoi(strings.TrimSpace(f)); err == nil { + out = append(out, n) + } + } + return out +} + +func (m *Manager) converge(ctx context.Context, s *Spec) error { if err := m.EnsureVolume(ctx, s); err != nil { return err } diff --git a/internal/tenant/spec.go b/internal/tenant/spec.go index 8314ee1..e46f8bb 100644 --- a/internal/tenant/spec.go +++ b/internal/tenant/spec.go @@ -87,6 +87,9 @@ func ContainerName(t string) string { return "tenant-" + t } // CheckName validates a tenant name. func CheckName(name string) error { + if strings.HasSuffix(name, "-prev") { + return fmt.Errorf("tenant name %q: the -prev suffix names rollback containers", name) + } if !validName.MatchString(name) { return fmt.Errorf("invalid tenant name %q (lowercase letter, then lowercase letters, digits or '-', at most 31)", name) } diff --git a/internal/tenant/tenant_test.go b/internal/tenant/tenant_test.go index 749d9ef..863b820 100644 --- a/internal/tenant/tenant_test.go +++ b/internal/tenant/tenant_test.go @@ -31,6 +31,7 @@ func TestValidate(t *testing.T) { } cases := map[string]func(*Spec){ "name": func(s *Spec) { s.Name = "Bad_Name" }, + "prev suffix": func(s *Spec) { s.Name = "est-prev" }, "image": func(s *Spec) { s.Image = "--privileged" }, "gpu dup": func(s *Spec) { s.GPUs = []int{1, 1} }, "ssh port": func(s *Spec) { s.SSHPort = 0 }, @@ -137,7 +138,7 @@ func TestRunArgs(t *testing.T) { } } s.GPUs = nil - if args := strings.Join(m.RunArgs(s), " "); strings.Contains(args, "gpus") || strings.Contains(args, "runtime") { + if args := strings.Join(m.RunArgs(s), " "); strings.Contains(args, "--gpus") || strings.Contains(args, "--runtime") { t.Errorf("GPU flags without GPUs: %s", args) } } @@ -250,7 +251,7 @@ exit 0 func TestPasswdNeverLogsPassword(t *testing.T) { argsLog, stdinLog := fakeDocker(t) - t.Setenv("FAKE_INSPECT", "true|x|always") + t.Setenv("FAKE_INSPECT", "true|x|always|") var logs bytes.Buffer runner := exec.NewRunner(false, slog.New(slog.NewTextHandler(&logs, nil))) root := t.TempDir() @@ -315,3 +316,31 @@ func TestDryRunApplyChangesNothing(t *testing.T) { } } } + +func TestFailedApplyRestoresGPULedger(t *testing.T) { + fakeDocker(t) + t.Setenv("FAKE_INSPECT", "") // no container is left after the failure + dir := t.TempDir() + for _, n := range []string{"install", "truncate", "mkfs.ext4", "systemctl", "ssh-keygen"} { + if err := os.WriteFile(filepath.Join(dir, n), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", dir+":"+os.Getenv("PATH")) + var calls [][]int + m := &Manager{ + Runner: exec.NewRunner(false, slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil))), + Root: t.TempDir(), SpecDir: t.TempDir(), UnitDir: t.TempDir(), Out: &bytes.Buffer{}, + ReserveGPUs: func(_ context.Context, _ string, gpus []int) error { + calls = append(calls, gpus) + return nil + }, + } + // The fake mount never happens, so Apply fails after reserving. + if err := m.Apply(context.Background(), validSpec()); err == nil { + t.Fatal("apply on an unmounted volume succeeded") + } + if len(calls) != 2 || intList(calls[0]) != "2,3" || len(calls[1]) != 0 { + t.Errorf("ledger calls = %v, want [[2 3] []]", calls) + } +}