release reagent v1.4.10

availableVersions.json

@@ -1,6 +1,6 @@
 {
-  "production": "1.4.9",
-  "test": "1.4.9",
-  "local": "1.4.9",
-  "all": "1.4.9"
+  "production": "1.4.10",
+  "test": "1.4.10",
+  "local": "1.4.10",
+  "all": "1.4.10"
 }

src/apps/app_manager.go

@@ -236,6 +236,7 @@ func (am *AppManager) syncPortState(payload common.TransitionPayload, app *commo
 	}
 
 	newPorts := make([]common.PortForwardRule, 0)
+	liveRules := make([]common.PortForwardRule, 0, len(portRules))
 
 	// The earliest any tunnel of this app needs another look: one that failed
 	// to come up in this pass is retried by the agent itself (see "Retry
@@ -273,20 +274,17 @@ func (am *AppManager) syncPortState(payload common.TransitionPayload, app *commo
 			dialPort, dialPortKnown, reservationFailure = am.resolveTunnelHostPort(payload, portRule)
 		}
 
-		// A development run tunnels its web ports only: what it serves there
-		// is what a developer inspects in the browser. Its tcp/udp ports are
-		// published on the device's network (computePortBindings) but take no
-		// port from the tunnel server's shared pool.
-		devNoTunnel := payload.Stage == common.DEV && !isWebProtocol(portRule.Protocol)
+		// A development run tunnels every port of its template while it runs.
+		// Its tcp/udp ports have no switch: their tunnels come and go with the
+		// run, so a stale Active flag on such a rule does not hold them off.
+		active := portRule.Active || (payload.Stage == common.DEV && !isWebProtocol(portRule.Protocol))
 		// A DEV label is four characters longer than the PROD one, and an app
 		// name of 40 characters can push it past what a DNS label may hold:
-		// such a tunnel could never be reached, so it is not opened.
-		devLabelTooLong := payload.Stage == common.DEV && !devNoTunnel && len(subdomain) > tunnel.MaxLabelLength
+		// such a tunnel could never be reached, so it is not opened. Only an
+		// http(s) tunnel is reached by its label; a tcp/udp one by its port.
+		devLabelTooLong := payload.Stage == common.DEV && isWebProtocol(portRule.Protocol) && len(subdomain) > tunnel.MaxLabelLength
 
-		if devNoTunnel {
-			log.Debug().Str("app", payload.AppName).Uint64("port", portRule.Port).Str("protocol", portRule.Protocol).
-				Msg("No tunnel for a tcp/udp port of a development run")
-		} else if devLabelTooLong {
+		if devLabelTooLong {
 			if portRule.Active && appRunning && tunnelsAvailable {
 				log.Warn().Str("app", payload.AppName).Str("subdomain", subdomain).
 					Msg("No tunnel for a port of a development run: its label exceeds 63 characters")
@@ -297,7 +295,7 @@ func (am *AppManager) syncPortState(payload common.TransitionPayload, app *commo
 				// Said again at the next run.
 				am.forgetLabelTooLong(tunnelID)
 			}
-		} else if portRule.Active {
+		} else if active {
 			if tunnelsAvailable && appRunning {
 				if !dialPortKnown {
 					// Fresh app that has not been started under managed ports
@@ -440,6 +438,9 @@ func (am *AppManager) syncPortState(payload common.TransitionPayload, app *commo
 			// inactive rule on every sync, forever.
 			if am.tunnelManager.Get(tunnelID) == nil && !am.tunnelInConfigFile(tunnelID) {
 				newPorts = append(newPorts, portRule)
+				liveRule := portRule
+				liveRule.RemotePort, liveRule.CloudRemotePort = 0, 0
+				liveRules = append(liveRules, liveRule)
 				continue
 			}
 
@@ -491,6 +492,13 @@ func (am *AppManager) syncPortState(payload common.TransitionPayload, app *commo
 		}
 		newPorts = append(newPorts, newPort)
 
+		// The remote port a development run announces is that of the tunnel
+		// standing right now, not the one its rule remembers from an earlier
+		// run (kept so the next run asks for the same port again).
+		liveRule := newPort
+		liveRule.RemotePort = newConfig.RemotePort
+		liveRule.CloudRemotePort = 0
+		liveRules = append(liveRules, liveRule)
 	}
 
 	// A pass that looked at the tunnels and found none needing another look
@@ -524,8 +532,11 @@ func (am *AppManager) syncPortState(payload common.TransitionPayload, app *commo
 
 	// Live-update the bind-mounted env files ({NAME}.txt / {NAME}_CLOUD.txt)
 	// so running containers see fresh tunnel ports without a restart. A
-	// development run has no remote ports (remotePortEnvVars).
-	if payload.Stage != common.DEV {
+	// development run's tcp/udp tunnels come up only once its container runs,
+	// so the files are how it learns their ports.
+	if payload.Stage == common.DEV {
+		refreshRemotePortEnvFiles(globalConfig, payload.Stage, payload.AppName, liveRules)
+	} else {
 		refreshRemotePortEnvFiles(globalConfig, payload.Stage, payload.AppName, newPorts)
 	}
 

src/apps/dev_tunnels_test.go

@@ -1,20 +1,24 @@
 package apps
 
 import (
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	"reagent/common"
+	"reagent/config"
 	"reagent/tunnel"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
-// Development runs get a private tunnel for each http(s) port of their port
-// template while the container runs. The label carries a -dev marker, so the
-// PROD install of the same app on the same device keeps its own tunnel.
+// Development runs get a tunnel for each port of their port template while the
+// container runs: a private one for an http(s) port, a remote port for a
+// tcp/udp one. The label carries a -dev marker, so the PROD install of the
+// same app on the same device keeps its own tunnel.
 
 func devTunnelID(t *testing.T, deviceKey int, app string, port uint64, protocol string) (string, string) {
 	t.Helper()
@@ -32,7 +36,23 @@ func readBackConfig(t *testing.T, deviceKey int, app string, port uint64, protoc
 	return tunnel.TunnelConfig{Name: name, Subdomain: sub, AppName: gotApp, DeclaredPort: declared, Stage: gotStage, Protocol: tunnel.Protocol(protocol), LocalPort: 41000 + port}
 }
 
-func TestSyncPortStateDevTunnelsWebPortsOnly(t *testing.T) {
+// devEnvFile reads /data/env/<name>.txt of a development run as its container
+// sees it.
+func devEnvFile(t *testing.T, cfg *config.Config, app, name string) (string, bool) {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join(appEnvFilesHostDir(cfg, common.DEV, app), name+".txt"))
+	if err != nil {
+		require.True(t, os.IsNotExist(err))
+		return "", false
+	}
+	return string(data), true
+}
+
+// A running development run gets a tunnel for every port of its template. Its
+// tcp/udp ports have no switch: a rule flagged inactive is tunneled all the
+// same. Their remote ports reach the container through its env files, and
+// stay on the rules so the next run asks for the same ports again.
+func TestSyncPortStateDevTunnelsAllPorts(t *testing.T) {
 	am, _, mockTunnel, appStore, _, cfg := amHarness(t)
 	mockTunnel.EXPECT().TunnelCapable().Return(true).Maybe()
 
@@ -42,27 +62,40 @@ func TestSyncPortStateDevTunnelsWebPortsOnly(t *testing.T) {
 	payload := amPayload(60, "devweb", common.RUNNING, common.DEV)
 	payload.Ports, _ = tunnel.PortForwardRuleToInterface([]common.PortForwardRule{
 		{RuleName: "ui", Port: 8080, Protocol: "http", Active: true},
-		{RuleName: "mqtt", Port: 1883, Protocol: "tcp", Active: true},
+		{RuleName: "mqtt", Port: 1883, Protocol: "tcp", Active: true, RemotePortEnvironment: "MQTT_PORT"},
+		{RuleName: "media", Port: 10000, Protocol: "udp", Active: false},
 	})
 
-	// The launch path published both on host ports.
+	// The launch path published all three on host ports.
 	_, err := am.hostPorts.RecoverOrReserve(hostPortKey{Stage: common.DEV, AppKey: 60, Protocol: "tcp", Port: 8080}, 41800)
 	require.NoError(t, err)
 	_, err = am.hostPorts.RecoverOrReserve(hostPortKey{Stage: common.DEV, AppKey: 60, Protocol: "tcp", Port: 1883}, 41801)
 	require.NoError(t, err)
+	_, err = am.hostPorts.RecoverOrReserve(hostPortKey{Stage: common.DEV, AppKey: 60, Protocol: "udp", Port: 10000}, 41802)
+	require.NoError(t, err)
 
 	sub, id := devTunnelID(t, cfg.ReswarmConfig.DeviceKey, "devweb", 8080, "http")
 	assert.Equal(t, "1-devweb-8080-dev", sub)
+	tcpSub, tcpID := devTunnelID(t, cfg.ReswarmConfig.DeviceKey, "devweb", 1883, "tcp")
+	assert.Equal(t, "1-devweb-1883-dev-tcp", tcpID)
+	udpSub, udpID := devTunnelID(t, cfg.ReswarmConfig.DeviceKey, "devweb", 10000, "udp")
 
 	mockTunnel.EXPECT().Get(id).Return(nil).Once()
-	// Exactly one add: the http port. The tcp port gets no tunnel.
+	mockTunnel.EXPECT().Get(tcpID).Return(nil).Once()
+	mockTunnel.EXPECT().Get(udpID).Return(nil).Once()
+	added := map[string]tunnel.TunnelConfig{}
 	mockTunnel.EXPECT().AddTunnel(mock.Anything).RunAndReturn(func(conf tunnel.TunnelConfig) (tunnel.TunnelConfig, error) {
-		assert.Equal(t, sub, conf.Subdomain)
-		assert.Equal(t, tunnel.HTTP, conf.Protocol)
 		assert.Equal(t, common.DEV, conf.Stage)
-		assert.Equal(t, uint64(41800), conf.LocalPort, "frpc dials the DEV container's host port")
+		assert.Zero(t, conf.RemotePort, "a first run asks for any free port")
+		added[conf.Subdomain] = conf
+		switch conf.Protocol {
+		case tunnel.TCP:
+			conf.RemotePort = 30123
+		case tunnel.UDP:
+			conf.RemotePort = 30124
+		}
 		return conf, nil
-	}).Once()
+	}).Times(3)
 
 	var saved []interface{}
 	mockTunnel.EXPECT().SaveRemotePorts(mock.Anything).RunAndReturn(func(p common.TransitionPayload) error {
@@ -74,11 +107,103 @@ func TestSyncPortStateDevTunnelsWebPortsOnly(t *testing.T) {
 
 	require.NoError(t, am.syncPortState(payload, app))
 
+	require.Contains(t, added, sub)
+	assert.Equal(t, uint64(41800), added[sub].LocalPort, "frpc dials the DEV container's host port")
+	require.Contains(t, added, tcpSub)
+	assert.Equal(t, uint64(41801), added[tcpSub].LocalPort)
+	require.Contains(t, added, udpSub, "a DEV udp port has no switch: an inactive flag does not hold it off")
+
 	rules := spsRules(t, saved)
-	require.Len(t, rules, 2, "the tcp rule stays in the written-back list")
+	require.Len(t, rules, 3)
 	assert.Equal(t, uint64(41800), rules[0].HostPort, "the DEV row learns its host ports")
-	assert.Equal(t, uint64(41801), rules[1].HostPort)
-	assert.Zero(t, rules[1].RemotePort, "a DEV tcp port takes no tunnel port")
+	assert.Zero(t, rules[0].RemotePort)
+	assert.Equal(t, uint64(30123), rules[1].RemotePort, "the DEV row keeps its tunnel ports for the next run")
+	assert.Equal(t, uint64(30124), rules[2].RemotePort)
+
+	for name, want := range map[string]string{"REMOTE_PORT_FOR_1883": "30123", "MQTT_PORT": "30123", "REMOTE_PORT_FOR_10000": "30124"} {
+		got, ok := devEnvFile(t, cfg, "devweb", name)
+		assert.True(t, ok, name)
+		assert.Equal(t, want, got, name)
+	}
+	_, ok := devEnvFile(t, cfg, "devweb", "REMOTE_PORT_FOR_8080")
+	assert.False(t, ok, "an http tunnel has no remote port")
+}
+
+// Once a development run stops, its tcp/udp tunnel goes and so does the port
+// announced to the container. The rule keeps the port, and the next run asks
+// frps for it again.
+func TestSyncPortStateDevTcpTunnelFollowsRun(t *testing.T) {
+	am, _, mockTunnel, appStore, _, cfg := amHarness(t)
+	mockTunnel.EXPECT().TunnelCapable().Return(true).Maybe()
+
+	app := amSeed(t, appStore, 65, "devmedia", common.PRESENT, common.DEV)
+	app.RequestedState = common.PRESENT
+
+	payload := amPayload(65, "devmedia", common.PRESENT, common.DEV)
+	payload.Ports = spsPorts(t, common.PortForwardRule{RuleName: "media", Port: 10000, Protocol: "udp", Active: true, HostPort: 41850, RemotePort: 30200})
+	_, err := am.hostPorts.RecoverOrReserve(hostPortKey{Stage: common.DEV, AppKey: 65, Protocol: "udp", Port: 10000}, 41850)
+	require.NoError(t, err)
+
+	// A file from the last run, whose tunnel is gone.
+	envDir := appEnvFilesHostDir(cfg, common.DEV, "devmedia")
+	require.NoError(t, ensureEnvDir(envDir))
+	require.NoError(t, writeEnvValue(envDir, "REMOTE_PORT_FOR_10000", "30200"))
+
+	sub, id := devTunnelID(t, cfg.ReswarmConfig.DeviceKey, "devmedia", 10000, "udp")
+
+	// Stopped: the tunnel is removed, and so is the announced port. Nothing
+	// changed on the rule, so nothing is saved.
+	mockTunnel.EXPECT().Get(id).Return(&tunnel.Tunnel{Config: tunnel.TunnelConfig{Subdomain: sub, RemotePort: 30200}}).Once()
+	mockTunnel.EXPECT().RemoveTunnel(mock.MatchedBy(func(c tunnel.TunnelConfig) bool {
+		return c.Subdomain == sub && c.Protocol == tunnel.UDP && c.Stage == common.DEV
+	})).Return(nil).Once()
+	mockTunnel.EXPECT().GetState().Return([]tunnel.TunnelState{}, nil).Once()
+	require.NoError(t, am.syncPortState(payload, app))
+	_, ok := devEnvFile(t, cfg, "devmedia", "REMOTE_PORT_FOR_10000")
+	assert.False(t, ok, "no tunnel, no announced port")
+
+	// Running again: the same port is asked for, and announced once granted.
+	app.CurrentState = common.RUNNING
+	app.RequestedState = common.RUNNING
+	mockTunnel.EXPECT().Get(id).Return(nil).Once()
+	mockTunnel.EXPECT().AddTunnel(mock.Anything).RunAndReturn(func(conf tunnel.TunnelConfig) (tunnel.TunnelConfig, error) {
+		assert.Equal(t, uint64(30200), conf.RemotePort, "the next run asks for the port of the last")
+		assert.False(t, conf.Reserved, "a fallback port is fine for a development run")
+		return conf, nil
+	}).Once()
+	mockTunnel.EXPECT().GetState().Return([]tunnel.TunnelState{}, nil).Once()
+	require.NoError(t, am.syncPortState(payload, app))
+	got, ok := devEnvFile(t, cfg, "devmedia", "REMOTE_PORT_FOR_10000")
+	assert.True(t, ok)
+	assert.Equal(t, "30200", got)
+}
+
+// Only an http(s) tunnel is reached by its label: a tcp/udp port of an app
+// whose DEV label exceeds a DNS label still gets its tunnel.
+func TestSyncPortStateDevLongNameTcpTunnel(t *testing.T) {
+	am, _, mockTunnel, appStore, _, cfg := amHarness(t)
+	mockTunnel.EXPECT().TunnelCapable().Return(true).Maybe()
+
+	name := strings.Repeat("b", 53)
+	app := amSeed(t, appStore, 66, name, common.RUNNING, common.DEV)
+	app.RequestedState = common.RUNNING
+
+	payload := amPayload(66, name, common.RUNNING, common.DEV)
+	payload.Ports = spsPorts(t, common.PortForwardRule{RuleName: "media", Port: 10001, Protocol: "tcp", Active: true, HostPort: 41860})
+	_, err := am.hostPorts.RecoverOrReserve(hostPortKey{Stage: common.DEV, AppKey: 66, Protocol: "tcp", Port: 10001}, 41860)
+	require.NoError(t, err)
+
+	sub, id := devTunnelID(t, cfg.ReswarmConfig.DeviceKey, name, 10001, "tcp")
+	require.Greater(t, len(sub), tunnel.MaxLabelLength)
+
+	mockTunnel.EXPECT().Get(id).Return(nil).Once()
+	mockTunnel.EXPECT().AddTunnel(mock.Anything).RunAndReturn(func(conf tunnel.TunnelConfig) (tunnel.TunnelConfig, error) {
+		conf.RemotePort = 30300
+		return conf, nil
+	}).Once()
+	mockTunnel.EXPECT().SaveRemotePorts(mock.Anything).Return(nil).Once()
+	mockTunnel.EXPECT().GetState().Return([]tunnel.TunnelState{}, nil).Once()
+	require.NoError(t, am.syncPortState(payload, app))
 }
 
 // A development run is not restarted: once it stopped or crashed, its tunnel
@@ -262,17 +387,19 @@ func TestSyncAllPortStatesRebuildsRunningDevTunnel(t *testing.T) {
 	am.SyncAllPortStates()
 }
 
-// A development run announces no remote ports: the PROD install's tunnel of
-// the same port is not its own.
+// A development run's container gets no remote-port variable, even while a
+// tunnel of its own stands (it belongs to the run before): a variable could
+// not be withdrawn once that tunnel went. Its ports reach it through the env
+// files, which follow the live tunnel (TestSyncPortStateDevTunnelsAllPorts).
 func TestRemotePortEnvVarsNoneForDev(t *testing.T) {
 	am, _, mockTunnel, _, _, cfg := amHarness(t)
-	rules := []common.PortForwardRule{{Port: 1883, Protocol: "tcp", Active: true, CloudRemotePort: 30999}}
+	rules := []common.PortForwardRule{{Port: 1883, Protocol: "tcp", Active: true}}
+	dk := uint64(cfg.ReswarmConfig.DeviceKey)
 
 	// No Get for DEV (strict mock).
 	assert.Nil(t, am.StateMachine.remotePortEnvVars(cfg, "envapp", common.DEV, rules))
 
-	prodSub := tunnel.CreateSubdomain(tunnel.TCP, uint64(cfg.ReswarmConfig.DeviceKey), "envapp", 1883, common.PROD)
-	mockTunnel.EXPECT().Get(tunnel.CreateTunnelID(prodSub, "tcp")).
-		Return(&tunnel.Tunnel{Config: tunnel.TunnelConfig{RemotePort: 30111}}).Once()
-	assert.Contains(t, am.StateMachine.remotePortEnvVars(cfg, "envapp", common.PROD, rules), "REMOTE_PORT_FOR_1883=30111")
+	prodID := tunnel.CreateTunnelID(tunnel.CreateSubdomain(tunnel.TCP, dk, "envapp", 1883, common.PROD), "tcp")
+	mockTunnel.EXPECT().Get(prodID).Return(&tunnel.Tunnel{Config: tunnel.TunnelConfig{RemotePort: 30111}}).Once()
+	assert.Equal(t, []string{"REMOTE_PORT_FOR_1883=30111"}, am.StateMachine.remotePortEnvVars(cfg, "envapp", common.PROD, rules))
 }

src/apps/env_files.go

@@ -82,6 +82,11 @@ func writeEnvValue(envDir string, name string, value string) error {
 // allocating a cloud port — so running containers see fresh values within
 // seconds, without a restart. A cloud port that disappeared (forwarding off)
 // removes the file, flipping SDK URL composition back to the LAN fallback.
+//
+// A development run's tunnels stand only while it runs, and its rules carry
+// the remote port of the tunnel standing right now (0: none). Its {NAME}.txt
+// is removed when there is none, so an SDK never announces the address of a
+// tunnel that is gone or not yet up.
 func refreshRemotePortEnvFiles(cfg *config.Config, stage common.Stage, appName string, rules []common.PortForwardRule) {
 	envDir := appEnvFilesHostDir(cfg, stage, appName)
 
@@ -113,6 +118,8 @@ func refreshRemotePortEnvFiles(cfg *config.Config, stage common.Stage, appName s
 		for _, name := range names {
 			if rule.RemotePort > 0 {
 				writeEnvFile(envDir, name, fmt.Sprintf("%d", rule.RemotePort))
+			} else if stage == common.DEV {
+				removeEnvFile(envDir, name)
 			}
 
 			cloudName := name + "_CLOUD"

src/apps/env_files_test.go

@@ -66,6 +66,30 @@ func TestRefreshRemotePortEnvFiles(t *testing.T) {
 	assert.True(t, ok)
 }
 
+// A development run announces only the tunnel standing right now: a rule
+// without one removes its port file. A PROD install keeps its file, as its
+// rule's port stays its address.
+func TestRefreshRemotePortEnvFilesDevRemovesGonePort(t *testing.T) {
+	cfg := envFilesTestConfig(t)
+	read := func(stage common.Stage, name string) bool {
+		_, err := os.Stat(filepath.Join(appEnvFilesHostDir(cfg, stage, "MyApp"), name+".txt"))
+		return err == nil
+	}
+	up := []common.PortForwardRule{{RuleName: "media", Port: 10000, Protocol: "udp", RemotePortEnvironment: "MEDIA_PORT", RemotePort: 30400}}
+	gone := []common.PortForwardRule{{RuleName: "media", Port: 10000, Protocol: "udp", RemotePortEnvironment: "MEDIA_PORT"}}
+
+	for _, stage := range []common.Stage{common.DEV, common.PROD} {
+		refreshRemotePortEnvFiles(cfg, stage, "MyApp", up)
+		assert.True(t, read(stage, "REMOTE_PORT_FOR_10000"), stage)
+		assert.True(t, read(stage, "MEDIA_PORT"), stage)
+		refreshRemotePortEnvFiles(cfg, stage, "MyApp", gone)
+	}
+	assert.False(t, read(common.DEV, "REMOTE_PORT_FOR_10000"))
+	assert.False(t, read(common.DEV, "MEDIA_PORT"))
+	assert.True(t, read(common.PROD, "REMOTE_PORT_FOR_10000"))
+	assert.True(t, read(common.PROD, "MEDIA_PORT"))
+}
+
 // Apps without remote-port env rules must not even get an env dir created.
 func TestRefreshRemotePortEnvFilesNoRelevantRules(t *testing.T) {
 	cfg := envFilesTestConfig(t)

src/apps/run_app.go

@@ -881,9 +881,13 @@ func buildDefaultEnvironmentVariables(config *config.Config, payload common.Tran
 // payload); a rule's custom remote_port_environment name is emitted alongside
 // for backward compatibility.
 //
-// A development run gets none: its only tunnels are http(s) ones, which have
-// no remote port, and the PROD install's tunnel of the same port is not its
-// own.
+// A development run gets none. Its tunnels come up only after its container
+// runs and go when it stops, so a tunnel standing when the container is
+// created belongs to the run before, and a variable can never be withdrawn:
+// once the tunnel went, the SDKs (file first) would fall back to it. It
+// learns its tcp/udp tunnels' ports from the env files alone, which follow
+// the live tunnel (refreshRemotePortEnvFiles). The PROD install's tunnel of
+// the same port is not its own either.
 func (sm *StateMachine) remotePortEnvVars(config *config.Config, appName string, stage common.Stage, portRules []common.PortForwardRule) []string {
 	if stage == common.DEV {
 		return nil