diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 0135120..aba20cf 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -17,7 +17,7 @@ uv run poe env:configure # Install pre-commit hooks **Testing:** ```bash -uv run poe test # Run full pytest suite (builds 7 container images, 23 tests) +uv run poe test # Run full pytest suite (builds 9 container images) uv run poe test tests/test_boot.py::test_execlineb_installed # Single test ``` @@ -35,36 +35,48 @@ All commands require: Ignity supervises container processes through three discrete stages: -**Stage 1 (`/etc/s6/boot/stage1`):** Setup -- Creates s6 supervision tree directories -- Initializes `/run/ignity/{envs,init,finalize,services,services-state}` +**Stage 1 (`/etc/s6/boot/stage1`):** Setup (PID 1) - Runs preboot (user/permission setup) if needed -- Delegates to stage 2 - -**Stage 2 (`/etc/s6/boot/stage2`):** Service & CMD Execution -- `stage2-envs`: Loads `/etc/ignity/envs/*` files in order (inheritance pattern) -- `stage2-init`: Executes `/etc/ignity/init/*` scripts in order -- `stage2-services`: Starts s6-supervised services from `/etc/ignity/services/` -- Optionally runs CMD after services healthy (configurable delay via `IGNITY_CMD_WAIT_FOR_SERVICES`) +- Validates `IGNITY_MODE` (`full` | `init` | `services`), fails fast otherwise +- Initializes `/run/ignity/{init,finalize,services,services-state}` +- `stage1-envs`: Builds non-secret defaults in `/run/ignity/envs-defaults` from `/etc/ignity/envs/*` (in order, container env wins, container env is never written to disk). They are applied with `s6-envdir` before `s6-svscan`, so every process inherits them +- Forks stage 2, then runs `s6-svscan` on `/run/ignity/services-state` + +**Stage 2 (`/etc/s6/boot/stage2`):** Service & CMD Execution (depends on `IGNITY_MODE`) +- `stage2-copy`: Copies `/etc/ignity/{init,finalize,services}` to `/run/ignity`, only what the mode needs +- `stage2-init`: Executes the `/run/ignity/init/*` copies of `/etc/ignity/init/*` in order (skipped in `services` mode) +- `stage2-cmd`: Runs the CMD, dropping to `USERMAP_UID:USERMAP_GID` via `s6-applyuidgid` only if PID 1 is root and the mapping isn't 0:0 (supplementary groups set to the gid) +- `stage2-services`: Copies `/run/ignity/services/*` into the live scandir `/run/ignity/services-state` and starts them under s6 (skipped in `init` mode) +- Optionally runs CMD after services healthy (configurable delay via `IGNITY_CMD_WAIT_FOR_SERVICES`); skipped in `init` mode +- Writes the exit status to `/run/ignity/stage2-exited` for stage 3; a failure (or `init` mode completing) stops the supervision tree + +| `IGNITY_MODE` | Runs | Use | +|---|---|---| +| `full` (default) | init -> services -> CMD | Docker, Swarm, single container | +| `init` | init only, then exit (non-zero if a script failed) | Kubernetes init container | +| `services` | services -> CMD (+ finalize) | Kubernetes main container | + +Containers of a pod share files through a user-mounted volume. No env is handed over between them, so secrets given to the init container stay there. **Stage 3 (`/etc/s6/boot/stage3`):** Shutdown - Brings down all services gracefully -- Runs `/etc/ignity/finalize/*` scripts in parallel -- Kills remaining processes, exits +- Runs the `/run/ignity/finalize/*` copies of `/etc/ignity/finalize/*` in parallel +- Kills remaining processes, exits with the code stored in `/run/ignity/stage2-exited` (default 0) ## Key Components **Boot Stages:** - `/init` — Entry point (execlineb wrapper calling stage1) - `/etc/s6/boot/stage{1,2,3}` — Core boot logic -- `/etc/s6/boot/stage2-{envs,init,services}` — Stage 2 sub-scripts +- `/etc/s6/boot/stage1-envs`, `/etc/s6/boot/stage2-{copy,init,services,cmd}` — Boot sub-scripts - `/etc/s6/services/.s6-svscan/{crash,finish}` — s6 supervision control **Utilities:** - `/usr/bin/preboot` — Non-root user setup (UID/GID mapping) -- `/usr/bin/fix-perms` — Apply permission fixes from spec files +- `/usr/bin/gen-accounts` — With `IGNITY_ACCOUNTS_DIR` set (called by preboot): writes passwd/group copies completed with USER there instead of useradd, for read-only roots; the main container mounts them over /etc/passwd and /etc/group (subPath) +- `/usr/bin/fix-perms` — Apply permission fixes from spec files (never follows symlinks: `chown -h`, symlinks skipped for `chmod`, top-level path resolved once) - `/usr/bin/load-envfile` — Load environment variables from files -- `/usr/bin/with-env` — Inject environment variables into command execution +- `/usr/bin/with-env` — POSIX sh. Passthrough by default; `-k ` narrows the inherited env (PATH always kept), `-d ` loads an envdir (repeatable, applied after `-k`, in order) for that process only (scoped secrets) - `/usr/bin/with-retries` — Retry command execution with backoff **Installation:** @@ -89,13 +101,13 @@ Scripts use numbered prefixes (00-, 01-, etc.) for deterministic ordering. Conve **Framework:** pytest + Podman integration tests **Structure:** -- 7 test modules in `/tests/`: `test_boot.py`, `test_init.py`, `test_envs.py`, `test_perms.py`, `test_services.py`, `test_preboot.py`, `test_finalize.py` -- Each module builds its own container image dynamically from `tests/fixtures//Dockerfile.tpl` +- 9 test modules in `/tests/`: `test_boot.py`, `test_init.py`, `test_envs.py`, `test_perms.py`, `test_services.py`, `test_preboot.py`, `test_finalize.py`, `test_modes.py`, `test_least_privilege.py` +- Each module sets `IMAGE_KIND`; the shared `image` fixture (conftest) builds it from `tests/fixtures//`: its own `Containerfile.tpl` if present, else ignity + `rootfs/` (see `DEFAULT_CONTAINERFILE`). Extra images can still be built with `build_image()` - Test fixtures include rootfs overlays with pre-configured services/scripts -- conftest.py provides `build_image(kind)` and `run_in_container(image, cmd)` helpers +- conftest.py provides the `image` fixture, `build_image(kind)`, `run_in_container(image, cmd, extra_args)`, `env_args(**env)` (`-e` arguments) and `READ_ONLY` (read-only root arguments) **Test Flow:** -1. `build_image()` renders Dockerfile.tpl (substitutes `{{DOCKER_BASE_IMAGE}}`), builds with random tag via podman +1. `build_image()` renders Containerfile.tpl (substitutes `{{DOCKER_BASE_IMAGE}}`), builds with random tag via podman 2. `run_in_container()` executes shell command inside container with `/init` entrypoint 3. Tests assert on exit codes and stdout content @@ -115,7 +127,7 @@ Follow conventional commits: `type: description` (types: build, ci, docs, feat, **Key Files:** - `pyproject.toml` — Project manifest, poe tasks, uv config - `tests/conftest.py` — Pytest helpers, immutable tuples, type annotations -- `tests/fixtures/` — Dockerfile templates and rootfs overlays (not actual test code) +- `tests/fixtures/` — Containerfile templates and rootfs overlays (not actual test code) - `.github/workflows/ci.yml` — Runs `uv run poe test` on PR/push **Environment Variables (Runtime):** @@ -123,9 +135,11 @@ Follow conventional commits: `type: description` (types: build, ci, docs, feat, - `IGNITY_CMD_WAIT_FOR_SERVICES_MAXTIME` — Wait timeout in milliseconds (default 5000) - `IGNITY_KILL_GRACETIME` — Shutdown grace period in ms (default 3000) - `IGNITY_KILL_FINALIZE_MAXTIME` — Max time for finalize scripts in ms (default 5000) -- `IGNITY_SERVICES_GRACETIME` — Service shutdown grace period in ms (default 5000) +- `IGNITY_SERVICES_GRACETIME` — Service shutdown grace period in ms (default 16000) - `IGNITY_SKIP_PERMS` — Skip permission fixing (0/1) -- `USER`, `USERMAP_UID`, `USERMAP_GID` — Non-root user mapping +- `IGNITY_ACCOUNTS_DIR` — Write account files there instead of creating USER in /etc (read-only root filesystem, see `gen-accounts`) +- `IGNITY_MODE` — `full` (default) | `init` (init scripts only, then exit) | `services` (skip init scripts); for Kubernetes init containers. No env is handed between containers, share files through a volume +- `USER`, `USERMAP_UID`, `USERMAP_GID` — Non-root user mapping; also the uid/gid the CMD drops to (`stage2-cmd`) when PID 1 is root and the mapping isn't 0:0 ## Language & Implementation Details diff --git a/README.md b/README.md index a07d4d5..f67bcd2 100644 --- a/README.md +++ b/README.md @@ -93,15 +93,18 @@ uv run poe package ### Available configuration parameters | Parameter | Description | Value (Current) | -| -------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | --------------- | +|----------------------------------------|-----------------------------------------------------------------------------------------------------------------------|-----------------| | `IGNITY_CMD_WAIT_FOR_SERVICES_MAXTIME` | The maximum time (in milliseconds) the services could take to bring up before proceding to CMD executing | 5000 | | `IGNITY_CMD_WAIT_FOR_SERVICES` | In order to proceed executing CMD overlay will wait until services are up | 0 | | `IGNITY_KILL_FINALIZE_MAXTIME` | The maximum time (in milliseconds) a script in `/etc/ignity/finalize` could take before sending a `KILL` signal to it | 5000 | | `IGNITY_KILL_GRACETIME` | How long (in milliseconds) `ignity` should wait to reap zombies before sending a `KILL` signal | 3000 | -| `IGNITY_SERVICES_GRACETIME` | How long (in milliseconds) `ignity` should wait services down before sending a down signal | 5000 | +| `IGNITY_MODE` | Which phases the container runs: `full`, `init` or `services` (see Kubernetes init containers) | full | +| `IGNITY_ACCOUNTS_DIR` | Where to write passwd/group files completed with `USER`, instead of creating it in `/etc` (read-only root filesystem) | | +| `IGNITY_SERVICES_GRACETIME` | How long (in milliseconds) `ignity` should wait services down before sending a down signal | 16000 | +| `IGNITY_SYNC_DISKS` | Call `s6-sync` before and after sending the termination signals (0/1) | 1 | | `USER` | User name to map with uid and gid | root | -| `USERMAP_GID` | User gid to map on files | 0 | -| `USERMAP_UID` | User uid to map on files | 0 | +| `USERMAP_GID` | User gid to map on files, and group of the `CMD` (see "How to run the CMD as a non root user") | 0 | +| `USERMAP_UID` | User uid to map on files, and user of the `CMD` (see "How to run the CMD as a non root user") | 0 | ### How to integrate ignity in your project from source @@ -127,20 +130,24 @@ ENTRYPOINT [ "/init" ] - Using `CMD` is a really convenient way to take advantage of `ignity`. - Your `CMD` can be given at build-time in the Dockerfile, or at runtime on the command line, either way is fine. -- It will be run under the s6 supervisor, and when it fails or exits, the container will exit. You can even run interactive programs under the s6 supervisor ! +- It will be run under the s6 supervisor, and when it fails or exits, the container will exit. You can even run + interactive programs under the s6 supervisor ! ### How to load env files - Sometimes it's interesting to load env files to define default variables. - You can simply put any env files in `/etc/ignity/envs` and they will be loaded during runtime in order. - They will be overrided by container environments variables. +- They are inherited by every process (init scripts, services, finalize scripts and `CMD`), so they are meant for non + secret values. ### How to supervise a service - Creating a supervised service cannot be easier - Just create a service directory with the name of your service into `/etc/ignity/services` - Put a `run` file into it, this is the file in which you'll put your long-lived process execution -- You're done! If you want to know more about s6 supervision of servicedirs take a look to [`servicedir`](https://skarnet.org/software/s6/servicedir.html) documentation. +- You're done! If you want to know more about s6 supervision of servicedirs take a look to [ + `servicedir`](https://skarnet.org/software/s6/servicedir.html) documentation. ### How to run hook on service exit @@ -170,11 +177,14 @@ s6-svscanctl -t /run/ignity/services-state ### How to start a service on-demand - By default, all services are started at runtime. -- You can delay or disable automatic start by creating a `down` file in the [service directory](https://skarnet.org/software/s6/servicedir.html). +- You can delay or disable automatic start by creating a `down` file in + the [service directory](https://skarnet.org/software/s6/servicedir.html). - This file can be created at build time in `/etc/ignity/services/myapp/down`. - Or at runtime by creating the file in `/run/ignity/services/myapp/down` using init scripts. -- Note that after first services start the service directory is `/run/ignity/services-state` instead of `/run/ignity/services`. -- Then you can start it manually by removing the file in `/run/ignity/services-state/myapp/down` and by calling `s6-svc -u /run/ignity/services-state/myapp`. +- Note that after first services start the service directory is `/run/ignity/services-state` instead of + `/run/ignity/services`. +- Then you can start it manually by removing the file in `/run/ignity/services-state/myapp/down` and by calling + `s6-svc -u /run/ignity/services-state/myapp`. ### How to execute initialization and/or finalization tasks @@ -186,21 +196,123 @@ s6-svscanctl -t /run/ignity/services-state ### How to set container environment variables -- If you want your custom script to have container environments available just make use of `with-env` helper, which will push all of those into your execution environment, for example: +- The container environment, and the defaults from `/etc/ignity/envs`, are inherited by every process `ignity` starts, + so a plain script can use them: `/etc/ignity/init/01-example`: ```sh -#!/usr/bin/with-env sh +#!/bin/sh echo $MYENV ``` -- This script will output whatever the `MYENV` enviroment variable contains. -- This helper is only here for custom environments variables pushed in `/run/ignity/envs` directory. +- This script will output whatever the `MYENV` environment variable contains. +- The container environment is never copied to disk by `ignity`. +- The `with-env` helper is still available. Without option it simply runs the given program, so existing + `#!/usr/bin/with-env sh` scripts keep working. + +### How to scope secrets to a single process + +- Everything exported in the container environment is inherited by all processes of the container. +- To give a secret to one process only, provide it as a file (for example a mounted secret) and load it with `with-env`, + which can also narrow what the process inherits: + +```text +with-env [-k patterns] [-d dir]... [--] prog [args...] +``` + +- `-k patterns`: keep only the inherited variables matching one of the comma separated names or globs + (`HOME,LANG,APP_*`) and drop the others. `PATH` is always kept. An empty list (`-k ''`) keeps `PATH` only. Quote the + patterns so your shell doesn't expand them. +- `-d dir`: load an envdir, which can be repeated. Each file of the directory is a variable: the file name is the + variable name and the content is its value, verbatim (newlines inside the value are kept, which suits multi-line keys, + only the final newline is removed). An empty file unsets the variable. +- Narrowing is applied first and the directories after it, in the order given. A variable of a directory therefore wins + over an inherited one, and a later directory wins over an earlier one. +- Without option, `with-env` runs the program as is. +- `with-env` is a POSIX `sh` script, so it needs `/bin/sh` in the image, and `awk` when `-k` is used. + +`/etc/ignity/services/myapp/run`: + +```sh +#!/bin/sh +# myapp only gets a few container variables plus its own secrets +exec with-env -k 'HOME,LANG,TZ,APP_*' -d /run/secrets/myapp myapp +``` + +- Give each service its own directory: `-d` loads every file it contains. +- Files have their own constraints: rotation needs a restart of the process, and file mode and ownership have to match + the user running it. + +#### What this does and does not protect + +- This limits what a process, and everything it spawns, inherits. It is not a boundary against other processes of the + container. +- `/proc//environ` is readable by processes running with the same uid, and by root. When services drop privileges + to another uid, they cannot read the environment of PID 1 (root), and `allowPrivilegeEscalation: false` stops them + from getting it back. Services running with the same uid as PID 1 can read it. +- Dropping the uid does not clear the environment: a service still starts with the full container environment unless you + narrow it with `-k`. +- The `CMD` runs as `USERMAP_UID:USERMAP_GID` (see below), but it still starts with the full environment unless you + narrow it. Init and finalize scripts run with the uid of PID 1 (root), and the full environment, unless they drop + privileges themselves. + +### How to use ignity with Kubernetes init containers + +- `ignity` can run the same image twice in a pod: first as an init container, then as the main container. +- `IGNITY_MODE` selects what a container does: + +| Mode | What it runs | Exit | +|------------|-------------------------------------------------------------------------|-----------------------------------------------------------| +| `full` | `init` scripts, services, then `CMD` (default, same as before) | When `CMD` exits or on signal | +| `init` | Only `init` scripts. Services, `finalize` scripts and `CMD` are skipped | As soon as scripts are done, with a failure if one failed | +| `services` | Services and `finalize` scripts, then `CMD`. `init` scripts are skipped | Same as `full` | + +- Results of the init phase are shared through a volume that you mount in both containers. `ignity` does not hand over + any environment variable between containers. +- This is also the way to keep secrets out of the main container: only give them to the init container (environment or + secret volume) and leave them out of the main one. + +```yaml +spec: + volumes: + - name: shared + emptyDir: + medium: Memory + initContainers: + - name: init + image: myapp + command: [ "/init" ] + env: + - name: IGNITY_MODE + value: init + - name: DB_ADMIN_PASSWORD # only visible here + valueFrom: + secretKeyRef: { name: db, key: admin-password } + volumeMounts: + - { name: shared, mountPath: /shared } + containers: + - name: app + image: myapp + command: [ "/init" ] + args: [ "myapp" ] + env: + - name: IGNITY_MODE + value: services + - name: IGNITY_SKIP_PERMS # already applied by the init container + value: "1" + volumeMounts: + - { name: shared, mountPath: /shared } +``` + +- `perms` fixes run on every container that doesn't set `IGNITY_SKIP_PERMS=1`. +- If the init phase produces a secret for the main container, write it as a file on the shared volume with a restrictive + mode, and remove it once consumed. A memory backed `emptyDir` keeps it off the node disk. ### How to run a container with non root user -- To run a container with a non root user, you have to define files and directories permissions in `perms` and then define the following in the Dockerfile +- To run a container with a non root user, you have to define files and directories permissions in `perms` and then + define the following in the Dockerfile ```bash ENV \ @@ -211,21 +323,202 @@ RUN preboot USER ${USERMAP_UID}:${USERMAP_GID} ``` -- This will correct permissions at build time instead of runtime and will let the container start properly with a non root user. +- This will correct permissions at build time instead of runtime and will let the container start properly with a non + root user. + +### How to run ignity as root (PID 1) with the least privileges + +- This is the setup to use when the `CMD` or the services run with their own uid, which a non root PID 1 can't do. +- PID 1 stays root but only keeps the capabilities `ignity` needs: + +| Capability | Needed for | +|----------------|----------------------------------------------------------------------------------------------| +| `CHOWN` | `fix-perms` changing owners | +| `FOWNER` | `fix-perms` changing modes of files that belong to another uid (the one it just handed over) | +| `DAC_OVERRIDE` | `fix-perms` reading what it just handed over (a `0700` directory of another uid) | +| `SETUID` | The `CMD` (`USERMAP_UID`) and services dropping privileges | +| `SETGID` | Same, including supplementary groups | +| `KILL` | Sending signals to processes running as another uid: stopping services, shutdown (without it the container never stops) | + +- Nothing else is used: no `NET_RAW`, `NET_BIND_SERVICE`, `SYS_CHROOT`, `MKNOD`, `SETFCAP`, `SETPCAP`, `AUDIT_WRITE`... + Add `FSETID` only if your `perms` files set setuid or setgid bits. +- This list comes from reading the scripts and is checked by `tests/test_least_privilege.py`. It doesn't cover what your + own init scripts and services do. +- A process that drops privileges loses its capabilities (the `CMD` ends up with none), so they only concern PID 1, the + init and finalize scripts. +- `allowPrivilegeEscalation: false` is fine: nothing relies on setuid binaries. + +**Read-only root filesystem** + +- See "How to run a container in read-only mode": `/run/ignity` is the only writable mount needed, and the user has to + exist at build time or be generated by an init container. + +**Network** + +- Services don't need any capability to connect (HTTP, TCP...) or to listen on a port of 1024 and above. +- Listening on a port below 1024 needs `NET_BIND_SERVICE`, but a service that dropped privileges doesn't keep the + capabilities of PID 1, so adding it to the container doesn't help. Lower the limit for the pod instead with the + `net.ipv4.ip_unprivileged_port_start` sysctl (`securityContext.sysctls`), or listen on a high port behind a Service. + +```yaml +spec: + volumes: + - name: run + emptyDir: + medium: Memory + containers: + - name: app + image: myapp + command: [ "/init" ] + args: [ "myapp" ] + env: + - { name: USERMAP_UID, value: "1000" } + - { name: USERMAP_GID, value: "1000" } + securityContext: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ "ALL" ] + add: [ "CHOWN", "FOWNER", "DAC_OVERRIDE", "SETUID", "SETGID", "KILL" ] + volumeMounts: + # emptyDir mounts allow exec by default + - { name: run, mountPath: /run/ignity } +``` + +- Root and these capabilities are allowed by the Pod Security `baseline` level (all of them are in its default list), + not by `restricted`, which requires `runAsNonRoot`: see "How to run ignity with a non root PID 1" there. + +### How to create the user with a read-only root filesystem + +- `preboot` creates `USER` in `/etc/passwd` and `/etc/group` when it is missing, which a read-only root filesystem + doesn't allow. +- Set `IGNITY_ACCOUNTS_DIR` and it writes copies of the two files completed with the user in that directory instead + (`/etc` is left alone). Mount them over `/etc/passwd` and `/etc/group` in the containers that need the account. +- This is meant for the init container: the mounts exist when a container starts, so the files have to be written by a + container that ran before. +- You only need this if something has to resolve the account (`whoami`, `getpwuid`, `HOME`...). `ignity` itself works + with numeric ids, and `USERMAP_UID` / `USERMAP_GID` don't need an account. + +```yaml +spec: + volumes: + - name: accounts + emptyDir: { } + - name: run-init + emptyDir: { medium: Memory } + - name: run + emptyDir: { medium: Memory } + initContainers: + - name: init + image: myapp + command: [ "/init" ] + env: + - { name: IGNITY_MODE, value: init } + - { name: IGNITY_ACCOUNTS_DIR, value: /accounts } + - { name: USER, value: app } + - { name: USERMAP_UID, value: "1000" } + - { name: USERMAP_GID, value: "1000" } + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: { drop: [ "ALL" ] } + volumeMounts: + - { name: accounts, mountPath: /accounts } + - { name: run-init, mountPath: /run/ignity } + containers: + - name: app + image: myapp + command: [ "/init" ] + args: [ "myapp" ] + env: + - { name: USER, value: app } + - { name: USERMAP_UID, value: "1000" } + - { name: USERMAP_GID, value: "1000" } + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: [ "ALL" ] + add: [ "SETUID", "SETGID", "KILL" ] + volumeMounts: + - { name: run, mountPath: /run/ignity } + - { name: accounts, mountPath: /etc/passwd, subPath: passwd, readOnly: true } + - { name: accounts, mountPath: /etc/group, subPath: group, readOnly: true } +``` + +- The main container finds the account in `/etc/passwd`, so `preboot` has nothing to create there. It doesn't need + `IGNITY_ACCOUNTS_DIR`. +- Don't set `IGNITY_SKIP_PERMS=1` on the init container: it skips the whole `preboot`, accounts included. +- What it writes: the existing entries, plus `USER` (home `/home/USER`, shell `/bin/false`, no home directory created, + like `useradd -M`) and its group. Nothing is added when the uid, the gid or the name is already used, and there is no + `shadow` file. +- `USER` must be a plain name (letters, digits, `_` and `-`), and `USERMAP_UID` and `USERMAP_GID` numbers: the values + end up in colon separated files, anything else makes `preboot` fail. +- Needs `getent` in the image (Debian and Ubuntu have it). + +### How to run ignity with a non root PID 1 + +- This is the exception, not the recommended setup: prefer the root PID 1 with the least privileges above, which keeps + the `CMD` and the services able to run with their own uid. +- It is only worth it when the cluster enforces Pod Security `restricted` (`runAsNonRoot: true`), or when a single uid + for the whole container is fine. +- What changes with a non root PID 1: + - everything runs with the same uid: the `CMD` and the services can't drop to another one, and can read each other's + `/proc//environ`. Narrow what they inherit with `with-env -k` and keep secrets out of the container's + environment; + - `preboot` can't create the user nor apply `perms`: set `IGNITY_SKIP_PERMS=1`, create the user at build time and + fix the ownership of the image paths at build time too; + - Kubernetes sets the uid and gid (`runAsUser`, `runAsGroup`) and the group of the volumes (`fsGroup`, how far it is + honored depends on the volume type and the CSI driver). +- `/run/ignity` still needs its own writable mount, see "How to run a container in read-only mode". +- This setup isn't covered by the tests. + +### How to run the CMD as a non root user + +- By default `USERMAP_UID` and `USERMAP_GID` are `0`, so the `CMD` runs as root, like PID 1. +- If PID 1 is root and either of them is not `0`, the `CMD` runs as `USERMAP_UID:USERMAP_GID`. Supplementary groups are + replaced by `USERMAP_GID`, the `CMD` doesn't keep the groups of root. +- Only the `CMD` is affected. Init scripts, finalize scripts and services keep running as PID 1, and services drop + privileges themselves in their `run` script if they need it. +- The environment is not changed: `HOME`, `USER` and the other variables stay as they are, so set them in the container + if the `CMD` needs them. +- If PID 1 isn't root already (for example `USER` is set in the Dockerfile, see above), it can't change uid: the `CMD` + simply runs as that user. +- The uid and gid are used as is, no user account is required. Files the `CMD` has to write must belong to that uid, + which is what `/etc/ignity/perms` is for. +- Because it isn't the uid of PID 1 anymore, the `CMD` can't read `/proc/1/environ`. + +```yaml +env: + - { name: USERMAP_UID, value: "1000" } + - { name: USERMAP_GID, value: "1000" } +``` ### How to run a container in read-only mode -- To run the container in read-only mode, you will have to mount a tmpfs at `/run/ingity`. -- If you want to run it with a non root user, you will also need to match uid able to write, execute and read on this location. +- The only place `ignity` writes to is `/run/ignity`: mount a writable tmpfs (or a Kubernetes `emptyDir`) there, with + `exec` because init scripts and services are executed from it. +- If PID 1 runs as a non root user, the uid also needs to be able to write, execute and read on this location. +- Use one mount per container: `/run/ignity` is private runtime state, `ignity` wipes it on start. ```yaml tmpfs: - "/run/ignity:exec,mode=1777,uid=" ``` +- Create `USER` at build time (`RUN preboot`, see above): at runtime `preboot` only creates it when it is missing, which + needs a writable `/etc`. On Kubernetes you can generate it in an init container instead, see "How to create the user + with a read-only root filesystem". +- Apply `perms` only on paths that are writable (volumes), or at build time and set `IGNITY_SKIP_PERMS=1`. On a + read-only path `chown` fails and `preboot` aborts the boot. +- Whatever your own services write (`/tmp`, logs, data) needs its own writable volumes: `ignity` doesn't write anywhere + else. + ### How to fix ownership & permissions -- Sometimes it's interesting to fix ownership & permissions before proceeding because, for example, you have mounted/mapped a host folder inside your container. +- Sometimes it's interesting to fix ownership & permissions before proceeding because, for example, you have + mounted/mapped a host folder inside your container. - Ignity provides a way to tackle this issue using files in `/etc/ignity/perms`. - The pattern format followed by fix-perms files: @@ -242,10 +535,13 @@ path recurse account fmode dmode - You can use variables `{{USERMAP_UID}}` and `{{USERMAP_GID}}` in those files. - They will be replaced at runtime or build time based on case. - You can also skip the permission phase by using `IGNITY_SKIP_PERMS=1` +- Symlinks are never followed: a symlink found below a path gets its own owner changed, but what it points to is left + alone, and its mode is ignored. The path itself can be a symlink, it is resolved once. ### How to drop privileges -- When it comes to executing a service, no matter it's a service or a logging service, a very good practice is to drop privileges before executing it. +- When it comes to executing a service, no matter it's a service or a logging service, a very good practice is to drop + privileges before executing it. - `s6` already includes utilities to do exactly these kind of things: In `execline`: @@ -263,22 +559,35 @@ In `sh`: exec s6-setuidgid daemon myservice ``` -- If you want to know more about these utilities, please take a look to: [`s6-setuidgid`](http://skarnet.org/software/s6/s6-setuidgid.html), [`s6-envuidgid`](http://skarnet.org/software/s6/s6-envuidgid.html) and [`s6-applyuidgid`](http://skarnet.org/software/s6/s6-applyuidgid.html). +- If you want to know more about these utilities, please take a look to: [`s6-setuidgid`](http://skarnet.org/software/s6/s6-setuidgid.html), + [`s6-envuidgid`](http://skarnet.org/software/s6/s6-envuidgid.html) and [`s6-applyuidgid`](http://skarnet.org/software/s6/s6-applyuidgid.html). ### How to allow high inheritance -- It's convenient to prefix every scripts in `envs`, `perms`, `init` and `finalize` by number (two chars) to ensure execution order. +- It's convenient to prefix every scripts in `envs`, `perms`, `init` and `finalize` by number (two chars) to ensure + execution order. - A common pattern is to dedicated 10 number by docker image layer to allow logic evolution. ### How to customize `ignity` behaviour -It is possible somehow to tweak `ignity` behaviour by providing an already predefined set of environment variables to the execution context: - -- `IGNITY_CMD_WAIT_FOR_SERVICES_MAXTIME` (default = 5000): The maximum time (in milliseconds) the services could take to bring up before proceding to CMD executing. -- `IGNITY_CMD_WAIT_FOR_SERVICES` (default = 0): In order to proceed executing CMD overlay will wait until services are up. Be aware that up doesn't mean ready. Depending if `notification-fd` was found inside the servicedir overlay will use `s6-svwait -U` or `s6-svwait -u` as the waiting statement. -- `IGNITY_KILL_FINALIZE_MAXTIME` (default = 5000): The maximum time (in milliseconds) a script in `/etc/ignity/finalize` could take before sending a `KILL` signal to it. Take into account that this parameter will be used per each script execution, it's not a max time for the whole set of scripts. -- `IGNITY_KILL_GRACETIME` (default = 3000): How long (in milliseconds) `ignity` should wait to reap zombies before sending a `KILL` signal. -- `IGNITY_SERVICES_GRACETIME` (default = 5000): How long (in milliseconds) `ignity` should wait to reap zombies before sending a down signal. +It is possible somehow to tweak `ignity` behaviour by providing an already predefined set of environment variables to +the execution context: + +- `IGNITY_CMD_WAIT_FOR_SERVICES_MAXTIME` (default = 5000): The maximum time (in milliseconds) the services could take to + bring up before proceding to CMD executing. +- `IGNITY_CMD_WAIT_FOR_SERVICES` (default = 0): In order to proceed executing CMD overlay will wait until services are + up. Be aware that up doesn't mean ready. Depending if `notification-fd` was found inside the servicedir overlay will + use `s6-svwait -U` or `s6-svwait -u` as the waiting statement. +- `IGNITY_KILL_FINALIZE_MAXTIME` (default = 5000): The maximum time (in milliseconds) a script in `/etc/ignity/finalize` + could take before sending a `KILL` signal to it. Take into account that this parameter will be used per each script + execution, it's not a max time for the whole set of scripts. +- `IGNITY_KILL_GRACETIME` (default = 3000): How long (in milliseconds) `ignity` should wait to reap zombies before + sending a `KILL` signal. +- `IGNITY_SERVICES_GRACETIME` (default = 16000): How long (in milliseconds) `ignity` should wait services down before + sending a down signal. +- `IGNITY_SYNC_DISKS` (default = 1): Call `s6-sync` before and after sending the termination signals. +- `IGNITY_MODE` (default = full): `full` runs everything, `init` only runs the `init` scripts and exits, `services` + skips the `init` scripts. See "How to use ignity with Kubernetes init containers". ## :heart: Contributing diff --git a/src/etc/s6/boot/stage1 b/src/etc/s6/boot/stage1 index 1a451b3..4895c97 100755 --- a/src/etc/s6/boot/stage1 +++ b/src/etc/s6/boot/stage1 @@ -3,16 +3,24 @@ # Use preboot to setup user preboot +# IGNITY_MODE selects which phases this container runs: full (default), init or services +importas -D full IGNITY_MODE IGNITY_MODE +if { + ifelse { eltest ${IGNITY_MODE} = full -o ${IGNITY_MODE} = init -o ${IGNITY_MODE} = services } { exit 0 } + fdmove -c 1 2 + foreground { s6-echo "[supervisor/boot] Invalid IGNITY_MODE '${IGNITY_MODE}' (expected: full, init or services)" } + exit 1 +} + ## For consistency cleanup ignity run directory if { elglob -0 g "/run/ignity/*" forx -o 0 d { ${g} } importas -ui d d - s6-rmrf -- ${d} + s6-rmrf ${d} } # Setup ignity context -if { s6-mkdir -pm 0755 -- /run/ignity/envs } if { s6-mkdir -pm 0755 -- /run/ignity/init } if { s6-mkdir -pm 0755 -- /run/ignity/finalize } if { s6-mkdir -pm 0755 -- /run/ignity/services } @@ -21,6 +29,12 @@ if { s6-mkdir -pm 0755 -- /run/ignity/services-state } # Init the scandir with our base services if { s6-hiercopy /etc/s6/services /run/ignity/services-state } +# Build the default environment from /etc/ignity/envs files +if { /etc/s6/boot/stage1-envs } + +# Make those defaults part of the environment inherited by everything below +s6-envdir -f /run/ignity/envs-defaults + ## Fork the "stage2" script background { /etc/s6/boot/stage2 $@ } unexport ! diff --git a/src/etc/s6/boot/stage1-envs b/src/etc/s6/boot/stage1-envs new file mode 100755 index 0000000..dfe99bf --- /dev/null +++ b/src/etc/s6/boot/stage1-envs @@ -0,0 +1,36 @@ +#!/bin/execlineb -S0 + +# Build /run/ignity/envs-defaults from the /etc/ignity/envs files, in order (later files win). +# Only the variables defined by those files are stored. The container environment is never +# copied to disk, and any variable already defined in it is dropped so that it always wins. + +if { s6-mkdir -pm 0700 -- /run/ignity/envs-defaults } + +if { + if -t { eltest -d /etc/ignity/envs } + if { s6-echo "[supervisor/env] Loading environment files" } + if { + elglob -s -0 g "/etc/ignity/envs/*" + forx -o 0 d { ${g} } + importas -ui d d + backtick -n i { s6-basename "${d}" } + importas -ui file i + # Parse each file in an empty environment so that only its own variables get dumped + redirfd -r 0 /etc/ignity/envs/${file} + emptyenv -p + load-envfile + if { s6-dumpenv "/run/ignity/envs-defaults" } + } + # PATH is only there because of the parser, never a default + if { s6-rmrf /run/ignity/envs-defaults/PATH } + if { + elglob -s -0 g "/run/ignity/envs-defaults/*" + forx -o 0 d { ${g} } + importas -ui d d + backtick -n i { s6-basename "${d}" } + importas -ui i i + # Variable already set in the container environment: it wins over the file + foreground { importas -ui v ${i} s6-rmrf /run/ignity/envs-defaults/${i} } + } + s6-echo -- "[supervisor/env] Environment files loaded" +} diff --git a/src/etc/s6/boot/stage2 b/src/etc/s6/boot/stage2 index 62b8139..83a5044 100755 --- a/src/etc/s6/boot/stage2 +++ b/src/etc/s6/boot/stage2 @@ -1,50 +1,50 @@ #!/bin/execlineb -S0 # This file is executed (not as process 1!) as soon as s6-svscan -# starts, with the original stdin/out/err, but NOT the original environment. +# starts, with the original stdin/out/err and the environment of stage 1. # Purpose of this file: to perform all the one-time initialization tasks. +# IGNITY_MODE (validated in stage 1): +# - full: init scripts, services, then CMD +# - init: init scripts only, then exit (e.g. a Kubernetes init container) +# - services: services, then CMD (init scripts are skipped) +importas -D full IGNITY_MODE IGNITY_MODE + foreground { if { foreground { - # Copy user provided files to /run/ignity. + # Copy user provided files to /run/ignity, only the ones this mode needs. if { - foreground { - forx i { "init" "finalize" "services" } - importas -u i i - if { eltest -d /etc/ignity/${i} } - # although s6-hiercopy is prefered, and until it doesn't support 'follow symlinks' - # option, there is no clean way to allow symlinks between user provided runcoms. - if { s6-hiercopy /etc/ignity/${i} /run/ignity/${i} } - } - importas -u ? ? - exit ${?} + if { ifelse { eltest ${IGNITY_MODE} = services } { exit 0 } /etc/s6/boot/stage2-copy init } + if { ifelse { eltest ${IGNITY_MODE} = init } { exit 0 } /etc/s6/boot/stage2-copy finalize } + ifelse { eltest ${IGNITY_MODE} = init } { exit 0 } /etc/s6/boot/stage2-copy services } - # envs: apply user-provided ownership & permission fixes - /etc/s6/boot/stage2-envs - # init: one-time init scripts - /etc/s6/boot/stage2-init - # services: long-lived processes to be supervised - /etc/s6/boot/stage2-services + ifelse { eltest ${IGNITY_MODE} = init } { /etc/s6/boot/stage2-init } + ifelse { eltest ${IGNITY_MODE} = services } { /etc/s6/boot/stage2-services } + /etc/s6/boot/stage2-init /etc/s6/boot/stage2-services } importas -u ? ? - foreground { redirfd -w 1 /run/ignity/envs/IGNITY_STAGE2_EXITED s6-echo -n -- "${?}" } + foreground { redirfd -w 1 /run/ignity/stage2-exited s6-echo -n -- "${?}" } exit ${?} } + # Nothing is left to supervise in init mode, stop the supervision tree. + # A failure of the init scripts is handled below, the same way. + ifelse { eltest ${IGNITY_MODE} = init } { s6-svscanctl -t /run/ignity/services-state } + # The init is complete, If the user has a given CMD, run it now, then kill everything when it exits. if -t { eltest $# -ne 0 } - foreground { with-env s6-setsid -gq -d 0 -- $@ } + foreground { s6-setsid -gq -d 0 -- /etc/s6/boot/stage2-cmd $@ } importas -u ? ? foreground { s6-echo -- "[supervisor/cmd] ${1} exited ${?}" } # Make CMD exit code available in stage3 - foreground { redirfd -w 1 /run/ignity/envs/IGNITY_STAGE2_EXITED s6-echo -n -- "${?}" } + foreground { redirfd -w 1 /run/ignity/stage2-exited s6-echo -n -- "${?}" } # Stop supervision tree foreground { s6-svscanctl -tb /run/ignity/services-state } diff --git a/src/etc/s6/boot/stage2-cmd b/src/etc/s6/boot/stage2-cmd new file mode 100755 index 0000000..1781715 --- /dev/null +++ b/src/etc/s6/boot/stage2-cmd @@ -0,0 +1,16 @@ +#!/bin/execlineb -S0 + +# Run the CMD, dropping privileges to USERMAP_UID:USERMAP_GID when it makes sense: +# - they are both 0 (default): nothing to drop, the CMD runs as root like before +# - PID 1 is not root (e.g. USER set in the Dockerfile): it can't change uid, the CMD runs as is +importas -D 0 USERMAP_UID USERMAP_UID +importas -D 0 USERMAP_GID USERMAP_GID + +backtick -n EUID { id -u } +importas -u EUID EUID + +ifelse { eltest ${EUID} -ne 0 } { $@ } +ifelse { eltest ${USERMAP_UID} -eq 0 -a ${USERMAP_GID} -eq 0 } { $@ } + +# Supplementary groups are explicitly set, otherwise the CMD would keep the ones of root +s6-applyuidgid -u ${USERMAP_UID} -g ${USERMAP_GID} -G ${USERMAP_GID} $@ diff --git a/src/etc/s6/boot/stage2-copy b/src/etc/s6/boot/stage2-copy new file mode 100755 index 0000000..1772cb2 --- /dev/null +++ b/src/etc/s6/boot/stage2-copy @@ -0,0 +1,7 @@ +#!/bin/execlineb -S1 + +# Copy user provided files of /etc/ignity/ to /run/ignity/. +# Although s6-hiercopy is prefered, and until it doesn't support 'follow symlinks' +# option, there is no clean way to allow symlinks between user provided runcoms. +if -t { eltest -d /etc/ignity/${1} } +s6-hiercopy /etc/ignity/${1} /run/ignity/${1} diff --git a/src/etc/s6/boot/stage2-envs b/src/etc/s6/boot/stage2-envs deleted file mode 100755 index 095c43c..0000000 --- a/src/etc/s6/boot/stage2-envs +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/execlineb -S0 - -if { s6-mkdir -pm 0755 -- /run/ignity/envs-container } -if { s6-dumpenv "/run/ignity/envs-container" } - -if { - if -t { eltest -d /etc/ignity/envs } - if { s6-echo "[supervisor/env] Loading environment files" } - if { - elglob -s -0 g "/etc/ignity/envs/*" - forx -o 0 d { ${g} } - importas -ui d d - backtick -n i { s6-basename "${d}" } - importas -ui file i - redirfd -r 0 /etc/ignity/envs/${file} load-envfile - if { s6-dumpenv "/run/ignity/envs" } - } - if { s6-echo -- "[supervisor/env] Environment files loaded" } -} - -if { s6-hiercopy "/run/ignity/envs-container" "/run/ignity/envs" } -if { s6-rmrf "/run/ignity/envs-container" } - -$@ diff --git a/src/etc/s6/boot/stage3 b/src/etc/s6/boot/stage3 index f7d3c48..c2af4ef 100755 --- a/src/etc/s6/boot/stage3 +++ b/src/etc/s6/boot/stage3 @@ -3,8 +3,9 @@ # This is the shutdown script, running as process 1. cd / -# Merge environments from our custom stage into current context -with-env +# Use CMD exit code (written by stage2) defaulting to zero if not present. +backtick -D 0 STAGE2_EXITED { redirfd -r 0 /run/ignity/stage2-exited s6-cat } +importas -u STAGE2_EXITED STAGE2_EXITED # This env decides if we should call s6-sync or not (default no) importas -D 1 -u IGNITY_SYNC_DISKS IGNITY_SYNC_DISKS @@ -15,8 +16,10 @@ foreground { /etc/s6/boot/stage3-finalize } # Wait service down foreground { s6-echo "[supervisor/finalize] Waiting for services" } foreground { - if { eltest -f /run/ignity/services-state } + if { eltest -d /run/ignity/services-state } importas -D 16000 -u IGNITY_SERVICES_GRACETIME IGNITY_SERVICES_GRACETIME + # s6-svwait -t 0 waits forever: a grace time of 0 means do not wait + if { eltest ${IGNITY_SERVICES_GRACETIME} -gt 0 } elglob SERVICES /run/ignity/services-state/* foreground { s6-svwait -D -t ${IGNITY_SERVICES_GRACETIME} ${SERVICES} } s6-sleep -m 200 @@ -53,5 +56,4 @@ foreground { } # Use CMD exit code defaulting to zero if not present. -importas -u -D 0 IGNITY_STAGE2_EXITED IGNITY_STAGE2_EXITED -exit ${IGNITY_STAGE2_EXITED} +exit ${STAGE2_EXITED} diff --git a/src/usr/bin/fix-perms b/src/usr/bin/fix-perms index 48adf6a..a7926da 100755 --- a/src/usr/bin/fix-perms +++ b/src/usr/bin/fix-perms @@ -6,12 +6,12 @@ importas -D 0 -u USERMAP_UID USERMAP_UID importas -D 0 -u USERMAP_GID USERMAP_GID -# Process each line -forstdin -N -d"\n" -- line +# Process each line. Without -o 0 a failing line would be ignored and fix-perms would exit 0 +forstdin -o 0 -N -d"\n" -- line importas -d"\n" -n -u line line -# Test each line isn't empty -if { eltest -n ${line} } +# Skip empty lines (if -t: a failed test is not an error) +if -t { eltest -n ${line} } # Split each line multidefine -0 -Cd" \t" -- ${line} { path uidgid fmode dmode } @@ -23,17 +23,23 @@ importas -u UID_GID UID_GID multidefine -Cd":" -- ${UID_GID} { uid gid } -# Check path exists -if { eltest -e ${path} } +# Skip paths that do not exist +if -t { eltest -e ${path} } -# Fix owners -pipeline { find "${path}" ( -not -gid "${gid}" -or -not -uid "${uid}" ) -print0 } -if { xargs -0 -r -n256 chown "${uid}:${gid}" } +# Resolve the configured path once, it may legitimately be a symlink (e.g. a volume mount point). +# From there nothing is followed: a symlink found below is a file like another one, never a way to +# reach something outside of the path (this runs as root, and the content may be written by others). +backtick -n real { realpath ${path} } +importas -u real real -# Fix files permissions -pipeline { find "${path}" -not -type d -not -perm "${fmode}" -print0 } +# Fix owners (-h: change the symlink itself, not its target) +pipeline { find -P "${real}" ( -not -gid "${gid}" -or -not -uid "${uid}" ) -print0 } +if { xargs -0 -r -n256 chown -h "${uid}:${gid}" } + +# Fix files permissions (symlinks have no mode of their own, chmod would change their target) +pipeline { find -P "${real}" -not -type d -not -type l -not -perm "${fmode}" -print0 } if { xargs -0 -r -n256 chmod "${fmode}" } # Fix directories permissions -pipeline { find "${path}" -type d -not -perm "${dmode}" -print0 } +pipeline { find -P "${real}" -type d -not -perm "${dmode}" -print0 } if { xargs -0 -r -n256 chmod "${dmode}" } diff --git a/src/usr/bin/gen-accounts b/src/usr/bin/gen-accounts new file mode 100755 index 0000000..7193d7e --- /dev/null +++ b/src/usr/bin/gen-accounts @@ -0,0 +1,55 @@ +#!/bin/execlineb -P + +# +# gen-accounts +# +# For a read-only root filesystem, where the user can't be created in /etc: write a copy of +# /etc/passwd and /etc/group completed with USER (uid USERMAP_UID, gid USERMAP_GID) in +# IGNITY_ACCOUNTS_DIR. Run it in an init container, then mount the two files over /etc/passwd and +# /etc/group (subPath) in the containers that need the account. +# +# Called by preboot when IGNITY_ACCOUNTS_DIR is set and USER doesn't exist. +# + +/bin/export PATH /usr/bin:/bin + +importas -u IGNITY_ACCOUNTS_DIR IGNITY_ACCOUNTS_DIR +importas -D root -u USER USER +importas -D 0 -u USERMAP_UID USERMAP_UID +importas -D 0 -u USERMAP_GID USERMAP_GID + +# The values end up in colon separated files, don't let them add anything else to them. +# grep -z makes the whole value a single record, so a newline can't hide a second line. +if { + ifelse { pipeline { s6-echo -n -- ${USER} } grep -Eqzx "[A-Za-z_][A-Za-z0-9_-]*" } { exit 0 } + fdmove -c 1 2 + foreground { s6-echo -- "[supervisor/accounts] Invalid USER" } + exit 1 +} +if { + ifelse { eltest ${USERMAP_UID} -ge 0 -a ${USERMAP_GID} -ge 0 } { exit 0 } + fdmove -c 1 2 + foreground { s6-echo -- "[supervisor/accounts] USERMAP_UID and USERMAP_GID must be numbers" } + exit 1 +} + +if { s6-mkdir -pm 0755 -- ${IGNITY_ACCOUNTS_DIR} } +if { cp /etc/passwd ${IGNITY_ACCOUNTS_DIR}/passwd } +if { cp /etc/group ${IGNITY_ACCOUNTS_DIR}/group } + +# Like groupadd does not, never create a second account for an existing gid or name +if { + ifelse { redirfd -w 1 /dev/null getent group ${USERMAP_GID} } { exit 0 } + ifelse { redirfd -w 1 /dev/null getent group ${USER} } { exit 0 } + redirfd -a 1 ${IGNITY_ACCOUNTS_DIR}/group + s6-echo -- "${USER}:x:${USERMAP_GID}:" +} + +# Same thing for the uid. No home directory is created, like useradd -M +if { + ifelse { redirfd -w 1 /dev/null getent passwd ${USERMAP_UID} } { exit 0 } + redirfd -a 1 ${IGNITY_ACCOUNTS_DIR}/passwd + s6-echo -- "${USER}:x:${USERMAP_UID}:${USERMAP_GID}::/home/${USER}:/bin/false" +} + +s6-echo -- "[supervisor/accounts] Account files written in ${IGNITY_ACCOUNTS_DIR}" diff --git a/src/usr/bin/preboot b/src/usr/bin/preboot index 51ed6d2..0237f64 100755 --- a/src/usr/bin/preboot +++ b/src/usr/bin/preboot @@ -10,13 +10,24 @@ importas -D 0 USERMAP_GID USERMAP_GID # This env decides if we should fix permissions or not (default no) importas -D 0 IGNITY_SKIP_PERMS IGNITY_SKIP_PERMS +# This env, when set, is where to write the account files instead of creating the user in /etc, +# which a read-only root filesystem doesn't allow (see gen-accounts) +importas -D "" IGNITY_ACCOUNTS_DIR IGNITY_ACCOUNTS_DIR + ifelse { eltest ${IGNITY_SKIP_PERMS} -eq 1 } { $@ } +if { + ifelse { eltest -z ${IGNITY_ACCOUNTS_DIR} } { exit 0 } + ifelse { redirfd -a 2 /dev/null redirfd -a 1 /dev/null id -u "${USER}" } { exit 0 } + gen-accounts +} + foreground { + ifelse { eltest -n ${IGNITY_ACCOUNTS_DIR} } { exit 0 } if -n { redirfd -a 2 /dev/null redirfd -a 1 /dev/null id -u "${USER}" } if { groupadd -g "${USERMAP_GID}" "${USER}" } if { useradd -M -u "${USERMAP_UID}" -g "${USERMAP_GID}" -s /bin/false "${USER}" } diff --git a/src/usr/bin/with-env b/src/usr/bin/with-env index 3fd9caa..4f91c87 100755 --- a/src/usr/bin/with-env +++ b/src/usr/bin/with-env @@ -1,2 +1,88 @@ -#!/bin/execlineb -S0 -s6-envdir -f /run/ignity/envs $@ +#!/bin/sh + +# +# with-env [-k patterns] [-d dir]... [--] prog [args...] +# +# The container environment is inherited by every process, so without option prog just runs as is. +# +# -k patterns Narrow the inherited environment: keep only the variables matching one of the +# comma separated names or globs (e.g. 'HOME,LANG,APP_*'), drop all the others. +# PATH is always kept. An empty list keeps PATH only. Can be repeated. +# -d dir Load the files of an envdir (one file per variable: the file name is the variable +# name, the content is the value, verbatim). Can be repeated. Applied after narrowing, +# in the order given, so a variable of an envdir wins over an inherited one and +# an empty file unsets the variable. +# +# This is the way to scope secrets provided as files to a single process. +# +usage() { + echo "usage: with-env [-k patterns] [-d dir]... [--] prog [args...]" >&2 + exit 64 +} + +narrow=0 +keep="" +dirs_rev="" +while [ $# -gt 0 ]; do + case $1 in + -k) + [ $# -ge 2 ] || usage + narrow=1 + keep="${keep:+${keep},}$2" + shift 2 + ;; + -d) + [ $# -ge 2 ] || usage + # Prepend so that the list is reversed: it is consumed below by prepending to the command + dirs_rev="$2${dirs_rev:+ +}${dirs_rev}" + shift 2 + ;; + --) + shift + break + ;; + -*) + usage + ;; + *) + break + ;; + esac +done +[ $# -ge 1 ] || usage + +# Chain one s6-envdir per directory, the first directory given being loaded first +if [ -n "${dirs_rev}" ]; then + while IFS= read -r dir; do + set -- s6-envdir -f "${dir}" "$@" + done </ has no Containerfile.tpl of its own: ignity plus the kind's rootfs +DEFAULT_CONTAINERFILE: Final[str] = """\ +FROM {{DOCKER_BASE_IMAGE}} +COPY src/ / +RUN bash /usr/src/install-ignity.sh +""" + + +def env_args(**env: str | int) -> tuple[str, ...]: + """Turn keyword arguments into `-e NAME=value` container arguments.""" + return tuple(arg for name, value in env.items() for arg in ("-e", f"{name}={value}")) + def build_image(kind: str) -> str: """Build a container image for a test suite kind.""" tag = secrets.token_hex(16) image_name = f"ignity-test-{kind}-{tag}" - dockerfile_tpl = ROOT / "tests" / "fixtures" / kind / "Dockerfile.tpl" + fixture_dir = ROOT / "tests" / "fixtures" / kind + containerfile_tpl = fixture_dir / "Containerfile.tpl" + if containerfile_tpl.exists(): + template = containerfile_tpl.read_text() + else: + template = DEFAULT_CONTAINERFILE + if (fixture_dir / "rootfs").is_dir(): + template += f"COPY tests/fixtures/{kind}/rootfs/ /\n" base_image = os.environ.get("DOCKER_BASE_IMAGE", "debian:bookworm-slim") - rendered = dockerfile_tpl.read_text().replace( - "{{DOCKER_BASE_IMAGE}}", base_image - ) + rendered = template.replace("{{DOCKER_BASE_IMAGE}}", base_image) with tempfile.NamedTemporaryFile( - mode="w", delete=False, suffix=".Dockerfile" + mode="w", delete=False, suffix=".Containerfile" ) as f: f.write(rendered) tmp_path = f.name @@ -42,7 +65,10 @@ def build_image(kind: str) -> str: def run_in_container( - image: str, command: str, extra_args: list[str] | None = None + image: str, + command: str, + extra_args: Sequence[str] | None = None, + timeout: float = 120, ) -> subprocess.CompletedProcess: """Run a command in a container and return the result.""" args = [ @@ -59,4 +85,26 @@ def run_in_container( image, "bash", ] - return subprocess.run(args, input=command, capture_output=True, text=True) + # A container stuck in its shutdown must fail the test, not hang the suite + return subprocess.run( + args, input=command, capture_output=True, text=True, timeout=timeout + ) + + +def image_fixture(kind: str | None = None, name: str = "image"): + """Module-scoped fixture named `name` giving an image built from tests/fixtures//. + + Without kind, the kind is the IMAGE_KIND of the test module. + """ + + @pytest.fixture(scope="module", name=name) + def fixture(request): + image_name = build_image(kind or request.module.IMAGE_KIND) + yield image_name + subprocess.run(["podman", "rmi", "-f", image_name], check=False) + + return fixture + + +# Image of the test module, from its IMAGE_KIND +image = image_fixture() diff --git a/tests/fixtures/boot/Dockerfile.tpl b/tests/fixtures/boot/Dockerfile.tpl deleted file mode 100644 index e57d108..0000000 --- a/tests/fixtures/boot/Dockerfile.tpl +++ /dev/null @@ -1,6 +0,0 @@ -# Base image -FROM {{DOCKER_BASE_IMAGE}} - -# Install ignity -COPY src/ / -RUN bash /usr/src/install-ignity.sh diff --git a/tests/fixtures/capabilities/Containerfile.tpl b/tests/fixtures/capabilities/Containerfile.tpl new file mode 100644 index 0000000..662c62d --- /dev/null +++ b/tests/fixtures/capabilities/Containerfile.tpl @@ -0,0 +1,16 @@ +# Base image +FROM {{DOCKER_BASE_IMAGE}} + +# Install ignity +COPY src/ / +RUN bash /usr/src/install-ignity.sh +COPY tests/fixtures/capabilities/rootfs/ / + +# /data/sub already belongs to another uid and can't be entered by anyone else: fix-perms +# can only fix what is inside with CAP_DAC_OVERRIDE +RUN mkdir -p /data/sub \ + && touch /data/file /data/sub/inner \ + && chmod 644 /data/file /data/sub/inner \ + && chmod 755 /data \ + && chown -R 1234:1234 /data/sub \ + && chmod 700 /data/sub diff --git a/tests/fixtures/capabilities/rootfs/etc/ignity/perms/10-data b/tests/fixtures/capabilities/rootfs/etc/ignity/perms/10-data new file mode 100644 index 0000000..62e45e4 --- /dev/null +++ b/tests/fixtures/capabilities/rootfs/etc/ignity/perms/10-data @@ -0,0 +1 @@ +/data 1234:1234 0600 0700 \ No newline at end of file diff --git a/tests/fixtures/capabilities/rootfs/etc/ignity/services/dropped/run b/tests/fixtures/capabilities/rootfs/etc/ignity/services/dropped/run new file mode 100755 index 0000000..1e2873d --- /dev/null +++ b/tests/fixtures/capabilities/rootfs/etc/ignity/services/dropped/run @@ -0,0 +1,5 @@ +#!/bin/sh +# A service dropping privileges, like the ones that run as their own uid. +# `wait` returns as soon as the trap fires, a foreground `sleep` would delay it by up to a second +# and the shutdown (IGNITY_KILL_GRACETIME=0 in the tests) would kill the service first. +exec s6-setuidgid nobody sh -c 'trap "echo dropped service got TERM; exit 0" TERM; echo dropped service started; while :; do sleep 1 & wait $!; done' diff --git a/tests/fixtures/envs/Dockerfile.tpl b/tests/fixtures/envs/Dockerfile.tpl deleted file mode 100644 index 0d69e12..0000000 --- a/tests/fixtures/envs/Dockerfile.tpl +++ /dev/null @@ -1,7 +0,0 @@ -# Base image -FROM {{DOCKER_BASE_IMAGE}} - -# Install ignity -COPY src/ / -RUN bash /usr/src/install-ignity.sh -COPY tests/fixtures/envs/rootfs/ / diff --git a/tests/fixtures/finalize/Dockerfile.tpl b/tests/fixtures/finalize/Dockerfile.tpl deleted file mode 100644 index a49a410..0000000 --- a/tests/fixtures/finalize/Dockerfile.tpl +++ /dev/null @@ -1,7 +0,0 @@ -# Base image -FROM {{DOCKER_BASE_IMAGE}} - -# Install ignity -COPY src/ / -RUN bash /usr/src/install-ignity.sh -COPY tests/fixtures/finalize/rootfs/ / diff --git a/tests/fixtures/init/Dockerfile.tpl b/tests/fixtures/init/Dockerfile.tpl deleted file mode 100644 index 2eae0c9..0000000 --- a/tests/fixtures/init/Dockerfile.tpl +++ /dev/null @@ -1,7 +0,0 @@ -# Base image -FROM {{DOCKER_BASE_IMAGE}} - -# Install ignity -COPY src/ / -RUN bash /usr/src/install-ignity.sh -COPY tests/fixtures/init/rootfs/ / diff --git a/tests/fixtures/perms/Dockerfile.tpl b/tests/fixtures/perms/Dockerfile.tpl deleted file mode 100644 index 543f70b..0000000 --- a/tests/fixtures/perms/Dockerfile.tpl +++ /dev/null @@ -1,7 +0,0 @@ -# Base image -FROM {{DOCKER_BASE_IMAGE}} - -# Install ignity -COPY src/ / -RUN bash /usr/src/install-ignity.sh -COPY tests/fixtures/perms/rootfs/ / diff --git a/tests/fixtures/perms/rootfs/etc/ignity/perms/13-symlinks b/tests/fixtures/perms/rootfs/etc/ignity/perms/13-symlinks new file mode 100644 index 0000000..26b3d5c --- /dev/null +++ b/tests/fixtures/perms/rootfs/etc/ignity/perms/13-symlinks @@ -0,0 +1,2 @@ +/var/www/symlink-dir 1234:1234 0600 0700 +/var/www/top-link 0:0 0600 0700 \ No newline at end of file diff --git a/tests/fixtures/perms/rootfs/var/www/outside-file b/tests/fixtures/perms/rootfs/var/www/outside-file new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/perms/rootfs/var/www/real-dir/inner b/tests/fixtures/perms/rootfs/var/www/real-dir/inner new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/perms/rootfs/var/www/symlink-dir/escape b/tests/fixtures/perms/rootfs/var/www/symlink-dir/escape new file mode 120000 index 0000000..d6dbcd0 --- /dev/null +++ b/tests/fixtures/perms/rootfs/var/www/symlink-dir/escape @@ -0,0 +1 @@ +/var/www/outside-file \ No newline at end of file diff --git a/tests/fixtures/perms/rootfs/var/www/symlink-dir/regular b/tests/fixtures/perms/rootfs/var/www/symlink-dir/regular new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/perms/rootfs/var/www/top-link b/tests/fixtures/perms/rootfs/var/www/top-link new file mode 120000 index 0000000..3c6fb03 --- /dev/null +++ b/tests/fixtures/perms/rootfs/var/www/top-link @@ -0,0 +1 @@ +/var/www/real-dir \ No newline at end of file diff --git a/tests/fixtures/preboot/Dockerfile.tpl b/tests/fixtures/preboot/Containerfile.tpl similarity index 100% rename from tests/fixtures/preboot/Dockerfile.tpl rename to tests/fixtures/preboot/Containerfile.tpl diff --git a/tests/fixtures/services/Dockerfile.tpl b/tests/fixtures/services/Dockerfile.tpl deleted file mode 100644 index 65191f3..0000000 --- a/tests/fixtures/services/Dockerfile.tpl +++ /dev/null @@ -1,7 +0,0 @@ -# Base image -FROM {{DOCKER_BASE_IMAGE}} - -# Install ignity -COPY src/ / -RUN bash /usr/src/install-ignity.sh -COPY tests/fixtures/services/rootfs/ / diff --git a/tests/fixtures/split/rootfs/etc/ignity/finalize/01-print-msg b/tests/fixtures/split/rootfs/etc/ignity/finalize/01-print-msg new file mode 100755 index 0000000..839c139 --- /dev/null +++ b/tests/fixtures/split/rootfs/etc/ignity/finalize/01-print-msg @@ -0,0 +1,2 @@ +#!/usr/bin/with-env bash +echo "split finalize ran" diff --git a/tests/fixtures/split/rootfs/etc/ignity/init/01-write-marker b/tests/fixtures/split/rootfs/etc/ignity/init/01-write-marker new file mode 100755 index 0000000..be8c4b0 --- /dev/null +++ b/tests/fixtures/split/rootfs/etc/ignity/init/01-write-marker @@ -0,0 +1,4 @@ +#!/usr/bin/with-env bash +mkdir -p /shared +echo -n "from-init" > /shared/marker +echo "run" >> /shared/runs diff --git a/tests/fixtures/split/rootfs/etc/ignity/init/02-maybe-fail b/tests/fixtures/split/rootfs/etc/ignity/init/02-maybe-fail new file mode 100755 index 0000000..6d63880 --- /dev/null +++ b/tests/fixtures/split/rootfs/etc/ignity/init/02-maybe-fail @@ -0,0 +1,3 @@ +#!/usr/bin/with-env bash +[ "${INIT_FAIL:-0}" = "1" ] && exit 1 +exit 0 diff --git a/tests/fixtures/split/rootfs/etc/ignity/services/dropped/run b/tests/fixtures/split/rootfs/etc/ignity/services/dropped/run new file mode 100755 index 0000000..1e2873d --- /dev/null +++ b/tests/fixtures/split/rootfs/etc/ignity/services/dropped/run @@ -0,0 +1,5 @@ +#!/bin/sh +# A service dropping privileges, like the ones that run as their own uid. +# `wait` returns as soon as the trap fires, a foreground `sleep` would delay it by up to a second +# and the shutdown (IGNITY_KILL_GRACETIME=0 in the tests) would kill the service first. +exec s6-setuidgid nobody sh -c 'trap "echo dropped service got TERM; exit 0" TERM; echo dropped service started; while :; do sleep 1 & wait $!; done' diff --git a/tests/fixtures/split/rootfs/etc/ignity/services/svc/run b/tests/fixtures/split/rootfs/etc/ignity/services/svc/run new file mode 100755 index 0000000..ffc151a --- /dev/null +++ b/tests/fixtures/split/rootfs/etc/ignity/services/svc/run @@ -0,0 +1,3 @@ +#!/usr/bin/with-env bash +echo "split service started" +sleep infinity diff --git a/tests/test_boot.py b/tests/test_boot.py index 8db6fda..51a1888 100644 --- a/tests/test_boot.py +++ b/tests/test_boot.py @@ -1,17 +1,8 @@ """Tests for boot stage installation.""" -import subprocess +from conftest import READ_ONLY, env_args, run_in_container -import pytest - -from conftest import build_image, run_in_container - - -@pytest.fixture(scope="module") -def image(): - name = build_image("boot") - yield name - subprocess.run(["podman", "rmi", "-f", name], check=False) +IMAGE_KIND = "boot" def test_execlineb_installed(image): @@ -32,3 +23,138 @@ def test_with_env_installed(image): def test_with_retries_installed(image): assert run_in_container(image, "command -v with-retries").returncode == 0 + + +def test_cmd_runs_as_root_by_default(image): + result = run_in_container(image, "echo UID=$(id -u) GID=$(id -g)") + assert "UID=0 GID=0" in result.stdout + + +def test_cmd_drops_privileges_to_usermap(image): + result = run_in_container( + image, + "echo UID=$(id -u) GID=$(id -g) GROUPS=$(id -G)", + extra_args=env_args(USERMAP_UID=1000, USERMAP_GID=1000), + ) + assert result.returncode == 0 + assert "UID=1000 GID=1000 GROUPS=1000" in result.stdout + + +def test_cmd_dropped_cannot_read_pid1_environ(image): + result = run_in_container( + image, + "cat /proc/1/environ >/dev/null 2>&1; echo RC=$?", + extra_args=[ + "-e", "USERMAP_UID=1000", "-e", "USERMAP_GID=1000", "-e", "MYSECRET=x", + ], + ) + assert "RC=0" not in result.stdout + assert "RC=" in result.stdout + + +def test_cmd_exit_code_kept_when_dropping_privileges(image): + result = run_in_container( + image, + "exit 7", + extra_args=env_args(USERMAP_UID=1000, USERMAP_GID=1000), + ) + assert result.returncode == 7 + + +_ACCOUNT = env_args(USER="exploit", USERMAP_UID=1000, USERMAP_GID=1000) + + +def test_accounts_written_instead_of_creating_user_on_read_only_root(image): + result = run_in_container( + image, + "cat /accounts/passwd /accounts/group; echo IN_ETC=$(grep -c exploit /etc/passwd)", + [*READ_ONLY, "--tmpfs=/accounts", "-e", "IGNITY_ACCOUNTS_DIR=/accounts", *_ACCOUNT], + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "exploit:x:1000:1000::/home/exploit:/bin/false" in result.stdout + assert "\nexploit:x:1000:\n" in result.stdout + # The existing accounts are kept and /etc itself is untouched + assert "root:x:0:0:" in result.stdout + assert "IN_ETC=0" in result.stdout + + +def test_accounts_of_init_container_mounted_in_main_container(image, tmp_path): + accounts = ["-v", f"{tmp_path}:/accounts"] + init = run_in_container( + image, + "true", + [ + *READ_ONLY, *accounts, "-e", "IGNITY_MODE=init", + "-e", "IGNITY_ACCOUNTS_DIR=/accounts", *_ACCOUNT, + ], + ) + assert init.returncode == 0, init.stdout + init.stderr + assert (tmp_path / "passwd").is_file() + assert (tmp_path / "group").is_file() + + # The main container has a read-only root and nothing creates the user there + main = run_in_container( + image, + "echo NAME=$(id -un) GROUP=$(id -gn) UID=$(id -u)", + [ + *READ_ONLY, + "-v", f"{tmp_path}/passwd:/etc/passwd:ro", + "-v", f"{tmp_path}/group:/etc/group:ro", + *_ACCOUNT, + ], + ) + assert main.returncode == 0, main.stdout + main.stderr + assert "NAME=exploit GROUP=exploit UID=1000" in main.stdout + + +def test_accounts_reject_invalid_user_name(image): + result = run_in_container( + image, + "true", + [ + *READ_ONLY, "--tmpfs=/accounts", "-e", "IGNITY_ACCOUNTS_DIR=/accounts", + "-e", "USER=bad:0:0::/:/bin/sh", "-e", "USERMAP_UID=1000", "-e", "USERMAP_GID=1000", + ], + ) + assert result.returncode != 0 + assert "Invalid USER" in result.stdout + result.stderr + + +def test_accounts_reject_name_with_newline(image): + result = run_in_container( + image, + "true", + [ + *READ_ONLY, "--tmpfs=/accounts", "-e", "IGNITY_ACCOUNTS_DIR=/accounts", + "-e", "USER=ok\nroot2:x:0:0::/:/bin/sh", + "-e", "USERMAP_UID=1000", "-e", "USERMAP_GID=1000", + ], + ) + assert result.returncode != 0 + assert "Invalid USER" in result.stdout + result.stderr + + +def test_accounts_reject_non_numeric_ids(image): + result = run_in_container( + image, + "true", + [ + *READ_ONLY, "--tmpfs=/accounts", "-e", "IGNITY_ACCOUNTS_DIR=/accounts", + "-e", "USER=exploit", "-e", "USERMAP_UID=1000:0", "-e", "USERMAP_GID=1000", + ], + ) + assert result.returncode != 0 + + +def test_accounts_never_duplicate_an_existing_id(image): + result = run_in_container( + image, + "echo OTHER=$(grep -c '^other:' /accounts/passwd /accounts/group | tr '\\n' ' ')", + [ + *READ_ONLY, "--tmpfs=/accounts", "-e", "IGNITY_ACCOUNTS_DIR=/accounts", + "-e", "USER=other", "-e", "USERMAP_UID=0", "-e", "USERMAP_GID=0", + ], + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "/accounts/passwd:0" in result.stdout + assert "/accounts/group:0" in result.stdout diff --git a/tests/test_envs.py b/tests/test_envs.py index 54d7247..769f49e 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -1,17 +1,8 @@ """Tests for environment variable loading.""" -import subprocess +from conftest import run_in_container -import pytest - -from conftest import build_image, run_in_container - - -@pytest.fixture(scope="module") -def image(): - name = build_image("envs") - yield name - subprocess.run(["podman", "rmi", "-f", name], check=False) +IMAGE_KIND = "envs" def test_envfile_var_present(image): @@ -36,3 +27,118 @@ def test_envfile_var_inherit_order(image): run_in_container(image, 'with-env env | grep "INHERIT=foobar"').returncode == 0 ) + + +def test_env_file_var_inherited_without_with_env(image): + assert run_in_container(image, 'env | grep "FOOBAR=test"').returncode == 0 + + +def test_container_env_wins_without_with_env(image): + assert ( + run_in_container( + image, 'env | grep "FOOBAR=override"', extra_args=["-e", "FOOBAR=override"] + ).returncode + == 0 + ) + + +def test_container_env_not_written_to_disk(image): + result = run_in_container( + image, + "! grep -rq topsecret /run/ignity", + extra_args=["-e", "MYSECRET=topsecret"], + ) + assert result.returncode == 0 + + +def test_with_env_dir_scopes_variables_to_one_process(image): + command = ( + "mkdir -p /tmp/sec && printf scoped > /tmp/sec/SCOPED_VAR && " + 'with-env -d /tmp/sec env | grep -q "SCOPED_VAR=scoped" && ' + "! env | grep -q SCOPED_VAR" + ) + assert run_in_container(image, command).returncode == 0 + + +def test_with_env_keep_narrows_inherited_variables(image): + result = run_in_container( + image, + "with-env -k 'KEEP_*,HOME' env", + extra_args=["-e", "KEEP_A=1", "-e", "DROP_A=2"], + ) + assert result.returncode == 0 + assert "KEEP_A=1" in result.stdout + assert "DROP_A" not in result.stdout + assert "PATH=" in result.stdout + + +def test_with_env_empty_keep_list_keeps_only_path(image): + result = run_in_container( + image, "with-env -k '' env", extra_args=["-e", "DROP_A=2"] + ) + assert result.returncode == 0 + assert "DROP_A" not in result.stdout + assert "PATH=" in result.stdout + + +def test_with_env_dir_overrides_inherited_variable(image): + command = ( + "mkdir -p /tmp/sec && printf scoped > /tmp/sec/MYVAR && " + 'with-env -d /tmp/sec env | grep -q "MYVAR=scoped"' + ) + assert ( + run_in_container(image, command, extra_args=["-e", "MYVAR=inherited"]).returncode + == 0 + ) + + +def test_with_env_keep_then_dir_merges_secrets_with_narrowed_env(image): + command = ( + "mkdir -p /tmp/sec && printf s3cr3t > /tmp/sec/DB_PASSWORD && " + "with-env -k 'KEEP_*' -d /tmp/sec env" + ) + result = run_in_container( + image, command, extra_args=["-e", "KEEP_A=1", "-e", "DROP_A=2"] + ) + assert result.returncode == 0 + assert "KEEP_A=1" in result.stdout + assert "DB_PASSWORD=s3cr3t" in result.stdout + assert "DROP_A" not in result.stdout + + +def test_with_env_multiple_dirs_applied_in_order(image): + command = ( + "mkdir -p /tmp/a /tmp/b && printf one > /tmp/a/V && printf two > /tmp/b/V && " + 'with-env -d /tmp/a -d /tmp/b env | grep -q "^V=two$" && ' + 'with-env -d /tmp/b -d /tmp/a env | grep -q "^V=one$"' + ) + assert run_in_container(image, command).returncode == 0 + + +def test_with_env_empty_file_unsets_variable(image): + command = ( + "mkdir -p /tmp/sec && : > /tmp/sec/MYVAR && " + "! with-env -d /tmp/sec env | grep -q '^MYVAR='" + ) + assert ( + run_in_container(image, command, extra_args=["-e", "MYVAR=inherited"]).returncode + == 0 + ) + + +def test_with_env_usage_errors(image): + result = run_in_container( + image, "with-env -z env; echo RC=$?; with-env -k X; echo RC=$?" + ) + assert result.stdout.count("RC=64") == 2 + + +def test_unprivileged_process_cannot_read_pid1_environ(image): + command = ( + 'cat /proc/1/environ | tr "\\0" "\\n" | grep -q MYSECRET && echo ROOT_SEES; ' + 's6-setuidgid nobody cat /proc/1/environ >/dev/null 2>&1; echo NOBODY_RC=$?' + ) + result = run_in_container(image, command, extra_args=["-e", "MYSECRET=topsecret"]) + assert "ROOT_SEES" in result.stdout + assert "NOBODY_RC=0" not in result.stdout + assert "NOBODY_RC=" in result.stdout diff --git a/tests/test_finalize.py b/tests/test_finalize.py index aed83c8..74713dc 100644 --- a/tests/test_finalize.py +++ b/tests/test_finalize.py @@ -1,17 +1,8 @@ """Tests for finalize scripts execution.""" -import subprocess +from conftest import run_in_container -import pytest - -from conftest import build_image, run_in_container - - -@pytest.fixture(scope="module") -def image(): - name = build_image("finalize") - yield name - subprocess.run(["podman", "rmi", "-f", name], check=False) +IMAGE_KIND = "finalize" def test_finalize_scripts_ran(image): diff --git a/tests/test_init.py b/tests/test_init.py index b486591..07dd740 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1,17 +1,8 @@ """Tests for init scripts execution.""" -import subprocess +from conftest import run_in_container -import pytest - -from conftest import build_image, run_in_container - - -@pytest.fixture(scope="module") -def image(): - name = build_image("init") - yield name - subprocess.run(["podman", "rmi", "-f", name], check=False) +IMAGE_KIND = "init" def test_init_scripts_ran(image): diff --git a/tests/test_least_privilege.py b/tests/test_least_privilege.py new file mode 100644 index 0000000..192ffaf --- /dev/null +++ b/tests/test_least_privilege.py @@ -0,0 +1,131 @@ +"""Tests that ignity works with the least privileges: few capabilities and a read-only root.""" + +import subprocess + +import pytest + +from conftest import READ_ONLY, env_args, image_fixture, run_in_container + +IMAGE_KIND = "split" + +# What ignity needs when PID 1 is root: +# - CHOWN, FOWNER, DAC_OVERRIDE: fix-perms (chown, chmod, then traverse what it just handed over) +# - SETUID, SETGID: dropping privileges (CMD with USERMAP_*, services) +# - KILL: signalling processes running as another uid (stop services, shutdown) +MINIMAL = ( + "--cap-drop=ALL", + "--cap-add=CHOWN", + "--cap-add=FOWNER", + "--cap-add=DAC_OVERRIDE", + "--cap-add=SETUID", + "--cap-add=SETGID", + "--cap-add=KILL", + "--security-opt=no-new-privileges", + *READ_ONLY, + # Written by the init script of the fixture + "--tmpfs=/shared", + *env_args(USERMAP_UID=1000, USERMAP_GID=1000, IGNITY_SERVICES_GRACETIME=3000), +) + + +@pytest.fixture(scope="module") +def result(image): + # Let the services start and install their traps before the CMD exits + return run_in_container( + image, + "sleep 2; echo UID=$(id -u); grep -E '^Cap(Eff|Prm)' /proc/self/status", + MINIMAL, + ) + + +def test_boots_with_minimal_capabilities_and_read_only_root(result): + assert result.returncode == 0, result.stdout + result.stderr + assert "split service started" in result.stdout + assert "split finalize ran" in result.stdout + + +def test_cmd_dropped_has_no_capabilities(result): + assert "UID=1000" in result.stdout + assert result.stdout.count("0000000000000000") == 2 + + +def test_service_dropping_privileges_works(result): + assert "dropped service started" in result.stdout + + +def test_services_of_another_uid_are_stopped_gracefully(result): + assert "dropped service got TERM" in result.stdout + + +# --- Each capability is really needed ---------------------------------------------------------- +# +# Remove one capability from the minimal set and check that what depends on it breaks. If one of +# these tests starts failing, the capability isn't needed anymore (or the scenario no longer +# exercises it) and the list in the README has to be revisited. + +CAPABILITIES = ("CHOWN", "FOWNER", "DAC_OVERRIDE", "SETUID", "SETGID", "KILL") + +# /data is handed over to 1234:1234 by fix-perms, and the CMD runs as that uid to look at it +COMMAND = ( + "sleep 2; echo UID=$(id -u); " + "echo FILE=$(stat -c '%u:%a' /data/file); " + "echo INNER=$(stat -c '%u:%a' /data/sub/inner)" +) + + +def _args(without: str | None = None) -> tuple[str, ...]: + caps = tuple(f"--cap-add={cap}" for cap in CAPABILITIES if cap != without) + return ( + "--cap-drop=ALL", + *caps, + "--security-opt=no-new-privileges", + # Not read-only: /data is part of the image and has to be fixed by fix-perms + *env_args(USERMAP_UID=1234, USERMAP_GID=1234, IGNITY_SERVICES_GRACETIME=3000), + ) + + +caps_image = image_fixture("capabilities", name="caps_image") + + +def test_control_everything_works_with_the_minimal_set(caps_image): + result = run_in_container(caps_image, COMMAND, _args()) + assert result.returncode == 0, result.stdout + result.stderr + assert "UID=1234" in result.stdout + assert "FILE=1234:600" in result.stdout + assert "INNER=1234:600" in result.stdout + assert "dropped service got TERM" in result.stdout + + +@pytest.mark.parametrize("missing", ["CHOWN", "FOWNER"]) +def test_perms_fail_without(caps_image, missing): + # fix-perms fails, which aborts the boot + result = run_in_container(caps_image, COMMAND, _args(missing)) + assert result.returncode != 0, result.stdout + result.stderr + assert "UID=" not in result.stdout + + +def test_perms_cant_enter_directories_of_other_uids_without_dac_override(caps_image): + result = run_in_container(caps_image, COMMAND, _args("DAC_OVERRIDE")) + assert result.returncode == 0, result.stdout + result.stderr + # What is directly in /data is fixed, what is in /data/sub can't be reached + assert "FILE=1234:600" in result.stdout + assert "INNER=1234:644" in result.stdout + + +@pytest.mark.parametrize("missing", ["SETUID", "SETGID"]) +def test_dropping_privileges_fails_without(caps_image, missing): + result = run_in_container(caps_image, COMMAND, _args(missing)) + assert result.returncode != 0 + assert "UID=1234" not in result.stdout + + +def test_shutdown_is_stuck_without_kill(caps_image): + # Nothing can signal the service of another uid, so PID 1 waits for it for ever + # (the container is left running when the client gives up, remove it) + with pytest.raises(subprocess.TimeoutExpired): + run_in_container(caps_image, COMMAND, _args("KILL"), timeout=30) + stuck = subprocess.run( + ["podman", "ps", "-aq", "--filter", f"ancestor={caps_image}"], + capture_output=True, text=True, + ).stdout.split() + subprocess.run(["podman", "rm", "-f", *stuck], check=False, capture_output=True) diff --git a/tests/test_modes.py b/tests/test_modes.py new file mode 100644 index 0000000..46e547b --- /dev/null +++ b/tests/test_modes.py @@ -0,0 +1,76 @@ +"""Tests for IGNITY_MODE (init container / services container split).""" + +import secrets +import subprocess + +import pytest + +from conftest import env_args, run_in_container + +IMAGE_KIND = "split" + +INIT = env_args(IGNITY_MODE="init") +SERVICES = env_args(IGNITY_MODE="services") + + +@pytest.fixture +def shared(): + """A named volume shared between the init and the services containers.""" + name = f"ignity-shared-{secrets.token_hex(8)}" + subprocess.run(["podman", "volume", "create", name], check=True, capture_output=True) + yield ("-v", f"{name}:/shared") + subprocess.run(["podman", "volume", "rm", "-f", name], check=False, capture_output=True) + + +def test_init_results_shared_with_services(image, shared): + assert run_in_container(image, "true", [*INIT, *shared]).returncode == 0 + result = run_in_container(image, "cat /shared/marker", [*SERVICES, *shared]) + assert result.returncode == 0 + assert "from-init" in result.stdout + + +def test_services_mode_skips_init_scripts(image, shared): + run_in_container(image, "true", [*INIT, *shared]) + result = run_in_container( + image, 'echo "RUNS=$(wc -l < /shared/runs | tr -d " ")"', [*SERVICES, *shared] + ) + assert "RUNS=1" in result.stdout + + +def test_init_mode_does_not_start_services_nor_finalize(image, shared): + result = run_in_container(image, "true", [*INIT, *shared]) + assert result.returncode == 0 + assert "split service started" not in result.stdout + assert "split finalize ran" not in result.stdout + + +def test_services_mode_runs_services_and_finalize(image, shared): + result = run_in_container(image, "true", [*SERVICES, *shared]) + assert result.returncode == 0 + assert "split finalize ran" in result.stdout + + +def test_init_failure_fails_the_container(image, shared): + result = run_in_container(image, "true", [*INIT, *shared, "-e", "INIT_FAIL=1"]) + assert result.returncode != 0 + + +def test_secret_of_init_container_not_in_services_container(image, shared): + run_in_container(image, "true", [*INIT, *shared, "-e", "SECRET=topsecret"]) + result = run_in_container( + image, "env; grep -r topsecret /run/ignity /shared", [*SERVICES, *shared] + ) + assert "topsecret" not in result.stdout + + +def test_full_mode_unchanged(image): + result = run_in_container(image, "cat /shared/marker") + assert result.returncode == 0 + assert "from-init" in result.stdout + assert "split finalize ran" in result.stdout + + +def test_invalid_mode_fails(image): + result = run_in_container(image, "true", ["-e", "IGNITY_MODE=bogus"]) + assert result.returncode != 0 + assert "Invalid IGNITY_MODE" in result.stdout + result.stderr diff --git a/tests/test_perms.py b/tests/test_perms.py index 3720fdf..684c8df 100644 --- a/tests/test_perms.py +++ b/tests/test_perms.py @@ -1,26 +1,10 @@ """Tests for permission and ownership handling.""" -import subprocess +from conftest import env_args, run_in_container -import pytest +IMAGE_KIND = "perms" -from conftest import build_image, run_in_container - -_USERMAP_ARGS = [ - "-e", - "USERMAP_UID=1000", - "-e", - "USERMAP_GID=1000", - "-e", - "USER=exploit", -] - - -@pytest.fixture(scope="module") -def image(): - name = build_image("perms") - yield name - subprocess.run(["podman", "rmi", "-f", name], check=False) +_USERMAP_ARGS = env_args(USERMAP_UID=1000, USERMAP_GID=1000, USER="exploit") def test_static_uid_gid_permissions(image): @@ -84,3 +68,26 @@ def test_execution_perms_stage(image): image, "test -d /etc/ignity", extra_args=["-e", "IGNITY_SKIP_PERMS=0"] ) assert "Applying ownership & permissions fixes" in result.stdout + + +def test_symlink_below_path_is_not_followed(image): + # The symlink itself is fixed, what it points to (outside of the path) must be left alone + result = run_in_container( + image, + "echo LINK=$(stat -c '%u:%g' /var/www/symlink-dir/escape); " + "echo TARGET=$(stat -c '%u:%g:%a' /var/www/outside-file); " + "echo REGULAR=$(stat -c '%u:%g:%a' /var/www/symlink-dir/regular)", + ) + assert "LINK=1234:1234" in result.stdout + assert "TARGET=0:0:644" in result.stdout + assert "REGULAR=1234:1234:600" in result.stdout + + +def test_symlinked_path_is_resolved(image): + result = run_in_container( + image, + "echo INNER=$(stat -c '%a' /var/www/real-dir/inner); " + "echo DIR=$(stat -c '%a' /var/www/real-dir)", + ) + assert "INNER=600" in result.stdout + assert "DIR=700" in result.stdout diff --git a/tests/test_preboot.py b/tests/test_preboot.py index 2f1664a..2008011 100644 --- a/tests/test_preboot.py +++ b/tests/test_preboot.py @@ -1,18 +1,15 @@ """Tests for preboot stage non-root user setup.""" -import subprocess +from conftest import run_in_container -import pytest - -from conftest import build_image, run_in_container - - -@pytest.fixture(scope="module") -def image(): - name = build_image("preboot") - yield name - subprocess.run(["podman", "rmi", "-f", name], check=False) +IMAGE_KIND = "preboot" def test_preboot_allows_non_root_user(image): assert run_in_container(image, "test -d /etc/ignity").returncode == 0 + + +def test_cmd_keeps_uid_when_pid1_is_not_root(image): + result = run_in_container(image, "echo UID=$(id -u) GID=$(id -g)") + assert result.returncode == 0 + assert "UID=1000 GID=1000" in result.stdout diff --git a/tests/test_services.py b/tests/test_services.py index ad9d5ad..9d43127 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -1,17 +1,8 @@ """Tests for s6 services management.""" -import subprocess +from conftest import run_in_container -import pytest - -from conftest import build_image, run_in_container - - -@pytest.fixture(scope="module") -def image(): - name = build_image("services") - yield name - subprocess.run(["podman", "rmi", "-f", name], check=False) +IMAGE_KIND = "services" def test_default_service_ran(image):