feat(monitoring): manage the exporter firewall guard and keep exporters up across docker restarts

.github/workflows/test.yaml

@@ -321,6 +321,21 @@ jobs:
           chmod +x rootfiles
           ROOTFILES_TEST_BINARY="$PWD/rootfiles" bash tests/scenarios/tenant-lifecycle.sh
 
+  exporter-firewall-scenario:
+    needs: unit
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: rootfiles-binary
+          path: .
+      - name: Exporter firewall guard on the runner's Docker, iptables and systemd
+        run: |
+          chmod +x rootfiles
+          ROOTFILES_TEST_BINARY="$PWD/rootfiles" bash tests/scenarios/exporter-firewall.sh
+
   rocky:
     needs: unit
     runs-on: ubuntu-latest

README.md

@@ -650,10 +650,15 @@ modules:
     allow_from:
       - 10.0.0.5/32
     perimeter_firewall: false
+    exporter_firewall: false
 ```
 
 DCGM exporter requires an already working NVIDIA GPU, driver, Docker daemon, and NVIDIA container runtime. A missing prerequisite shows as `blocked` in `rootfiles check` and fails `apply --dry-run`; `apply` still sets up node exporter and the rest of the module, then fails with the DCGM error and the fix (for a missing runtime: `sudo nvidia-ctk runtime configure --runtime=docker && sudo systemctl reload docker`). If UFW is active, `allow_from` only applies when its incoming default is deny/reject and there are no broader allow rules for either exporter port. rootfiles reports conflicts and leaves existing firewall rules unchanged. It does not enable UFW.
 
+`exporter_firewall: true` installs an iptables guard that admits the exporter ports only from `allow_from`, for hosts without UFW whose segment is not closed by a perimeter firewall. It needs an IPv4 `listen_address`, exactly one IPv4 `/32` in `allow_from`, the default ports 9100 and 9400, Docker running (the guard filters DCGM in `DOCKER-USER`) and python3; an active firewalld is reported as blocked. rootfiles writes `/usr/local/sbin/rootfiles-exporter-firewall` and `/etc/rootfiles/exporter-firewall.conf` and runs the helper's `setup`, which adds owner-marked chains (`RFILES_EXPORTER_IN` on INPUT, `RFILES_DCGM_DU` on DOCKER-USER), `rootfiles-exporter-firewall.service` and drop-ins that reapply the rules when Docker or UFW starts. An exporter starts only while its rules are present (`ExecStartPre` runs the helper's `verify-host`/`verify-docker`); stopping the guard unit or Docker leaves the rules and node exporter in place. On a hub host the hub's Compose bridge is admitted so its Prometheus can scrape the host. A guard installed by hand with the same helper is adopted without rule changes. Setting the option back to `false` leaves the guard in place; `rootfiles doctor` reports it as unmanaged, and `sudo rootfiles-exporter-firewall rollback` removes it.
+
+The DCGM exporter and monitoring hub units are wanted by `docker.service`, so they start again whenever Docker starts.
+
 The default DCGM image is pinned to `nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04` for the fleet's older R535 driver baseline. Blackwell and newer driver combinations have not been qualified with that old DCGM release. Select a pinned `dcgm_exporter_image` only after checking NVIDIA's supported GPU, driver, and paired DCGM/exporter versions and validating the endpoint on that host. NVIDIA documents that driver 580.126.16 requires DCGM 4.3.x or newer, so the default image must not be assumed compatible with R580 hosts. The suggested 4.6.1-4.8.4 distroless override needs real GPU validation before rollout.
 
 The managed collector file selects only temperature, utilization, framebuffer used/free, XID, and volatile double-bit ECC fields shared by the legacy and newer image versions. ECC series may be absent on GPUs or configurations that do not expose ECC counters. The DCGM 3.3.8 XID gauge reports the last observed XID and may stay stale after recovery.
@@ -738,7 +743,7 @@ hosts:
     config: sites/gpu01.yaml
 ```
 
-For an exporter host with existing Docker and NVIDIA runtime, enable only monitoring and bind the exporters to its private address and allow the hub address through the host firewall. If a perimeter firewall already restricts the segment, set `perimeter_firewall: true` instead of adding host firewall rules.
+For an exporter host with existing Docker and NVIDIA runtime, enable only monitoring and bind the exporters to its private address and allow the hub address through the host firewall. If a perimeter firewall already restricts the segment, set `perimeter_firewall: true` instead of adding host firewall rules; without UFW, `exporter_firewall: true` restricts the exporters on the host itself.
 
 ```yaml
 modules:

docs/releases/v0.18.0.md

@@ -0,0 +1,13 @@
+rootfiles can now manage the exporter firewall guard, and node exporter, the DCGM exporter and the monitoring hub stay up when Docker restarts.
+
+### New
+- `modules.monitoring.exporter_firewall: true` installs and checks an iptables guard that admits the exporter ports only from `allow_from` (one IPv4 /32, default ports). A guard installed by hand with the same `rootfiles-exporter-firewall` helper is adopted without changing its rules. `rootfiles doctor` counts the exporters as covered only while the guard's check passes, and names a guard that rootfiles does not manage. (#84)
+
+### Fixed
+- After Docker stopped and started again (a package upgrade or a manual restart), node exporter on hosts with the exporter firewall guard stayed down until the next `rootfiles apply`, and the DCGM exporter and the monitoring hub did not come back by themselves. Exporters now keep running while the guard unit or Docker stops, refuse to start only while their firewall rules are missing, and the DCGM exporter and hub start again with Docker. (#84)
+
+### Action needed
+- On hosts with a hand-installed exporter firewall guard, set `exporter_firewall: true` under `modules.monitoring` (and `perimeter_firewall: false` unless a network firewall closes the segment), then run `sudo rootfiles apply --module monitoring` or `rootfiles fleet apply`. The apply replaces the guard's systemd drop-ins and restarts the DCGM exporter once; the firewall rules stay in place. (#84)
+- Hosts with the DCGM exporter or the monitoring hub need one `rootfiles apply` after updating so their units start with Docker. (#84)
+
+**Full changelog**: https://github.com/STAIxBWLB/rootfiles/compare/v0.17.1...v0.18.0

internal/config/config.go

@@ -137,13 +137,16 @@ type MonitoringConfig struct {
 	// NodeExporter installs prometheus-node-exporter; rootfiles' scheduled
 	// report also publishes rootfiles_* metrics through its textfile
 	// collector.
-	NodeExporter      bool                `yaml:"node_exporter"`
-	DCGMExporter      bool                `yaml:"dcgm_exporter"`
-	NodeExporterPort  int                 `yaml:"node_exporter_port,omitempty"`
-	DCGMExporterPort  int                 `yaml:"dcgm_exporter_port,omitempty"`
-	ListenAddress     string              `yaml:"listen_address,omitempty"`
-	AllowFrom         []string            `yaml:"allow_from,omitempty"`
-	PerimeterFirewall bool                `yaml:"perimeter_firewall"`
+	NodeExporter      bool     `yaml:"node_exporter"`
+	DCGMExporter      bool     `yaml:"dcgm_exporter"`
+	NodeExporterPort  int      `yaml:"node_exporter_port,omitempty"`
+	DCGMExporterPort  int      `yaml:"dcgm_exporter_port,omitempty"`
+	ListenAddress     string   `yaml:"listen_address,omitempty"`
+	AllowFrom         []string `yaml:"allow_from,omitempty"`
+	PerimeterFirewall bool     `yaml:"perimeter_firewall"`
+	// ExporterFirewall installs the iptables guard that admits exporter
+	// traffic only from allow_from (IPv4, one /32, default ports).
+	ExporterFirewall  bool                `yaml:"exporter_firewall"`
 	DCGMExporterImage string              `yaml:"dcgm_exporter_image,omitempty"`
 	Hub               MonitoringHubConfig `yaml:"hub"`
 }

internal/config/monitoring_firewall_test.go

@@ -0,0 +1,42 @@
+package config
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestValidateExporterFirewall(t *testing.T) {
+	valid := MonitoringConfig{Enabled: true, NodeExporter: true, ListenAddress: "192.0.2.5", AllowFrom: []string{"198.51.100.200/32"}, ExporterFirewall: true}
+	errFor := func(m MonitoringConfig) string {
+		var msgs []string
+		c := &Config{}
+		c.Modules.Monitoring = m
+		c.validateMonitoring(func(f string, a ...any) { msgs = append(msgs, f) })
+		return strings.Join(msgs, "; ")
+	}
+	if got := errFor(valid); got != "" {
+		t.Fatalf("valid config rejected: %s", got)
+	}
+	for name, mutate := range map[string]func(*MonitoringConfig){
+		"no listen_address":   func(m *MonitoringConfig) { m.ListenAddress = "" },
+		"IPv6 listen_address": func(m *MonitoringConfig) { m.ListenAddress = "2001:db8::5" },
+		"two sources":         func(m *MonitoringConfig) { m.AllowFrom = append(m.AllowFrom, "198.51.100.201/32") },
+		"no source":           func(m *MonitoringConfig) { m.AllowFrom = nil },
+		"subnet source":       func(m *MonitoringConfig) { m.AllowFrom = []string{"198.51.100.0/24"} },
+		"IPv6 source":         func(m *MonitoringConfig) { m.AllowFrom = []string{"2001:db8::1/128"} },
+		"no exporter":         func(m *MonitoringConfig) { m.NodeExporter = false },
+		"custom port":         func(m *MonitoringConfig) { m.NodeExporterPort = 9101 },
+	} {
+		m := valid
+		m.AllowFrom = append([]string(nil), valid.AllowFrom...)
+		mutate(&m)
+		if got := errFor(m); !strings.Contains(got, "exporter_firewall") {
+			t.Errorf("%s: want an exporter_firewall error, got %q", name, got)
+		}
+	}
+	off := valid
+	off.ExporterFirewall, off.AllowFrom = false, []string{"198.51.100.0/24", "198.51.100.201/32"}
+	if got := errFor(off); got != "" {
+		t.Fatalf("exporter_firewall off must not restrict allow_from: %s", got)
+	}
+}

internal/config/monitoring_validate.go

@@ -115,6 +115,22 @@ func (c *Config) validateMonitoring(add func(string, ...any)) {
 	if m.DCGMExporter {
 		image("dcgm_exporter_image", m.ExporterImage())
 	}
+	if m.ExporterFirewall {
+		addr, err := netip.ParseAddr(m.ListenAddress)
+		switch {
+		case err != nil || !addr.Is4():
+			add("modules.monitoring.exporter_firewall: listen_address must be an IPv4 address")
+		case !m.NodeExporter && !m.DCGMExporter:
+			add("modules.monitoring.exporter_firewall: enable node_exporter or dcgm_exporter")
+		case m.NodePort() != 9100 || m.DCGMPort() != 9400:
+			add("modules.monitoring.exporter_firewall: requires the default exporter ports 9100 and 9400")
+		}
+		if len(m.AllowFrom) != 1 {
+			add("modules.monitoring.exporter_firewall: allow_from must list exactly one address")
+		} else if p, err := netip.ParsePrefix(m.AllowFrom[0]); err != nil || !p.Addr().Is4() || p.Bits() != 32 || p.String() != m.AllowFrom[0] {
+			add("modules.monitoring.exporter_firewall: allow_from must be one IPv4 /32 such as 192.0.2.10/32")
+		}
+	}
 	if m.Hub.Enabled {
 		h := m.Hub.WithDefaults()
 		ip("hub.listen_address", h.ListenAddress)