fix(drivers): restart a driver that stays offline

.changeset/driver-restart-watchdog.md

@@ -0,0 +1,5 @@
+---
+"ftw": patch
+---
+
+Restart a driver that stays offline or fails to start, instead of leaving it down until someone restarts it by hand.

go/cmd/ftw/main.go

@@ -746,6 +746,8 @@ func main() {
 			continue
 		}
 		if err := reg.Add(ctx, d); err != nil {
+			// The restart watchdog retries this start. A failed boot is not
+			// terminal.
 			slog.Warn("failed to spawn driver", "name", d.Name, "err", err)
 		}
 	}
@@ -791,6 +793,35 @@ func main() {
 	capMu := &sync.RWMutex{}
 	cfgMu := &sync.RWMutex{}
 	modelsMu := &sync.Mutex{}
+	// Restart a driver that stays offline, or that never finished starting.
+	// The control-loop watchdog still hands a stale driver its default mode;
+	// this one replaces the process when that is not enough. Disabled and
+	// observe-only drivers are left alone, and a cloud login is not repeated
+	// just because the stream went quiet. Stop it before ShutdownAll, which
+	// was deferred above: a cancelled watchdog must not start a driver the
+	// shutdown has already removed.
+	watchdogCtx, stopRestartWatchdog := context.WithCancel(ctx)
+	watchdogDone := make(chan struct{})
+	go func() {
+		defer close(watchdogDone)
+		reg.RunRestartWatchdog(watchdogCtx, func() ([]config.Driver, time.Duration) {
+			// Copy under the lock, then drop it. MaintainDrivers reads this
+			// again after it takes the driver's lifecycle gate and skips the
+			// restart when the driver is now disabled, observe-only, or its
+			// config has changed. Holding cfgMu across the restart would
+			// block a settings save for the whole reconnect.
+			cfgMu.RLock()
+			defer cfgMu.RUnlock()
+			timeout := time.Duration(cfg.Site.WatchdogTimeoutS) * time.Second
+			return config.WithBatterySoCBounds(cfg.ModbusControlDrivers(), cfg.Batteries), timeout
+		})
+	}()
+	// Registered after ShutdownAll, so this runs first and the watchdog
+	// has left the lifecycle gates before drivers are removed.
+	defer func() {
+		stopRestartWatchdog()
+		<-watchdogDone
+	}()
 
 	// Pre-declare services that the hot-reload Applier needs to touch.
 	// The Applier closure captures these by reference; they're assigned

go/internal/api/api.go

@@ -735,6 +735,36 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
 
 // ---- /api/status ----
 
+// driverAutoRestart is the watchdog's view of one driver: how many times
+// core has restarted it, why, and when the next try is due. Empty when the
+// watchdog has nothing to say.
+func (s *Server) driverAutoRestart(name string) map[string]any {
+	if s == nil || s.deps == nil || s.deps.Registry == nil || name == "" {
+		return nil
+	}
+	st, ok := s.deps.Registry.RestartStatus(name)
+	if !ok {
+		return nil
+	}
+	out := map[string]any{}
+	if st.Restarts > 0 {
+		out["auto_restarts"] = st.Restarts
+	}
+	if st.LastReason != "" {
+		out["auto_restart_reason"] = st.LastReason
+	}
+	if st.LastError != "" {
+		out["auto_restart_error"] = st.LastError
+	}
+	if !st.NextAttempt.IsZero() {
+		out["next_auto_restart_ms"] = st.NextAttempt.UnixMilli()
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return out
+}
+
 func statusDriverOnline(tel *telemetry.Store, name string) bool {
 	if tel == nil || name == "" {
 		return false
@@ -1078,6 +1108,9 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) {
 				}
 			}
 		}
+		for k, v := range s.driverAutoRestart(name) {
+			d[k] = v
+		}
 		drivers[name] = d
 	}
 	// Merge config drivers that aren't in the registry so the UI can
@@ -1090,19 +1123,22 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) {
 		if _, ok := drivers[dc.Name]; ok {
 			continue
 		}
+		// Configured but not running — spawn probably failed. Show it as
+		// offline so the user sees it and can restart it.
+		row := map[string]any{
+			"status":      "offline",
+			"not_running": true,
+		}
 		if dc.Disabled {
-			drivers[dc.Name] = map[string]any{
+			row = map[string]any{
 				"status":   "disabled",
 				"disabled": true,
 			}
-		} else {
-			// Configured but not running — spawn probably failed. Show
-			// as offline so the user sees it and can Restart.
-			drivers[dc.Name] = map[string]any{
-				"status":      "offline",
-				"not_running": true,
-			}
 		}
+		for k, v := range s.driverAutoRestart(dc.Name) {
+			row[k] = v
+		}
+		drivers[dc.Name] = row
 	}
 	s.deps.CfgMu.RUnlock()
 

go/internal/api/api_test.go

@@ -170,6 +170,60 @@ func TestHandlePostConfigDoesNotPersistEVPasswordOnInvalidConfig(t *testing.T) {
 	}
 }
 
+func TestHandleStatusReportsAutomaticDriverRestarts(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "fail.lua")
+	src := "function driver_init(config) error('start failed') end\nfunction driver_poll() return 60000 end\n"
+	if err := os.WriteFile(path, []byte(src), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	tel := telemetry.NewStore()
+	reg := drivers.NewRegistry(tel)
+	cfg := config.Driver{Name: "pixii", Lua: path}
+	now := time.Now()
+	reg.MaintainDrivers(context.Background(), now, []config.Driver{cfg}, time.Minute)
+
+	srv := New(&Deps{
+		Tel:      tel,
+		Ctrl:     &control.State{},
+		CtrlMu:   &sync.Mutex{},
+		CapMu:    &sync.RWMutex{},
+		CfgMu:    &sync.RWMutex{},
+		Cfg:      &config.Config{Drivers: []config.Driver{cfg}},
+		Registry: reg,
+	})
+	body := func() map[string]map[string]any {
+		t.Helper()
+		req := httptest.NewRequest(http.MethodGet, "/api/status", nil)
+		rr := httptest.NewRecorder()
+		srv.Handler().ServeHTTP(rr, req)
+		if rr.Code != http.StatusOK {
+			t.Fatalf("status = %d, body: %s", rr.Code, rr.Body.String())
+		}
+		var resp struct {
+			Drivers map[string]map[string]any `json:"drivers"`
+		}
+		if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
+			t.Fatalf("decode: %v", err)
+		}
+		return resp.Drivers
+	}
+
+	pending := body()["pixii"]
+	if pending["not_running"] != true {
+		t.Fatalf("failed driver = %#v", pending)
+	}
+	next, _ := pending["next_auto_restart_ms"].(float64)
+	if int64(next) != now.Add(30*time.Second).UnixMilli() {
+		t.Fatalf("next retry = %v, want %d", pending["next_auto_restart_ms"], now.Add(30*time.Second).UnixMilli())
+	}
+
+	reg.MaintainDrivers(context.Background(), now.Add(30*time.Second), []config.Driver{cfg}, time.Minute)
+	retried := body()["pixii"]
+	if retried["auto_restarts"] != float64(1) || retried["auto_restart_reason"] != "start_failed" || retried["auto_restart_error"] == "" {
+		t.Fatalf("after retry = %#v", retried)
+	}
+}
+
 func TestHandleStatusIgnoresOfflineDERInLiveBalance(t *testing.T) {
 	tel := telemetry.NewStore()
 	ctrl := &control.State{SiteMeterDriver: "site"}

go/internal/drivers/registry.go

@@ -115,6 +115,11 @@ type Registry struct {
 	lifecycleHook    func(name string)
 	lifecycleMu      sync.Mutex
 	lifecycleGates   map[string]*lifecycleGate
+
+	// restartMu guards the automatic-restart backoff. It is never held
+	// while a driver is started or stopped.
+	restartMu sync.Mutex
+	restarts  map[string]*driverRestartState
 }
 
 // lifecycleGate serializes every lifecycle transition for one driver name.