Opt-in retries for HTTP origin connections

cmd/cloudflared/tunnel/cmd.go

@@ -998,6 +998,12 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
+		altsrc.NewDurationFlag(&cli.DurationFlag{
+			Name:    ingress.ProxyConnectRetryTimeoutFlag,
+			Usage:   "Total time to retry refused HTTP origin connections or missing Unix sockets (0 disables retries). Applies to --url or --unix-socket; for ingress rules, set originRequest.connectRetryTimeout.",
+			EnvVars: []string{"TUNNEL_PROXY_CONNECT_RETRY_TIMEOUT"},
+			Hidden:  shouldHide,
+		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   ingress.ProxyTLSTimeoutFlag,
 			Usage:  legacyTunnelFlag("HTTP proxy timeout for completing a TLS handshake"),

cmd/cloudflared/tunnel/origin_retry_test.go

@@ -0,0 +1,62 @@
+package tunnel
+
+import (
+	"testing"
+	"time"
+
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ingress"
+)
+
+func TestOriginConnectRetryFlag(t *testing.T) {
+	testCases := []struct {
+		name     string
+		env      string
+		flag     string
+		expected time.Duration
+		wantErr  bool
+	}{
+		{name: "disabled by default"},
+		{name: "environment", env: "500ms", expected: 500 * time.Millisecond},
+		{name: "flag", flag: "500ms", expected: 500 * time.Millisecond},
+		{name: "flag overrides environment", env: "500ms", flag: "2s", expected: 2 * time.Second},
+		{name: "flag disables environment", env: "500ms", flag: "0s"},
+		{name: "negative flag", flag: "-1s", wantErr: true},
+		{name: "negative environment", env: "-1s", wantErr: true},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Setenv("TUNNEL_PROXY_CONNECT_RETRY_TIMEOUT", tc.env)
+			for _, origin := range [][]string{{"--url", "http://localhost:8000"}, {"--unix-socket", "/tmp/app.sock"}} {
+				t.Run(origin[0], func(t *testing.T) {
+					app := cli.NewApp()
+					app.Flags = configureProxyFlags(false)
+					app.Action = func(c *cli.Context) error {
+						log := zerolog.Nop()
+						rules, err := ingress.ParseIngressFromConfigAndCLI(&config.Configuration{}, c, &log)
+						if err != nil {
+							return err
+						}
+						require.Len(t, rules.Rules, 1)
+						require.Equal(t, tc.expected, rules.Rules[0].Config.ConnectRetryTimeout.Duration)
+						return nil
+					}
+					args := append([]string{"cloudflared"}, origin...)
+					if tc.flag != "" {
+						args = append(args, "--proxy-connect-retry-timeout", tc.flag)
+					}
+					err := app.Run(args)
+					if tc.wantErr {
+						require.ErrorContains(t, err, "connectRetryTimeout must not be negative")
+						return
+					}
+					require.NoError(t, err)
+				})
+			}
+		})
+	}
+}

config/configuration.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"math"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -191,10 +192,12 @@ type UnvalidatedIngressRule struct {
 // config.
 // Note:
 // - To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
-// - To specify a time.Duration in json, use int64 of the nanoseconds
+// - To specify a time.Duration in JSON, use seconds (e.g. 0.5 or 3).
 type OriginRequestConfig struct {
 	// HTTP proxy timeout for establishing a new connection
 	ConnectTimeout *CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
+	// Total time to retry refused HTTP origin connections or missing Unix sockets. Zero disables retries.
+	ConnectRetryTimeout *CustomDuration `yaml:"connectRetryTimeout" json:"connectRetryTimeout,omitempty"`
 	// HTTP proxy timeout for completing a TLS handshake
 	TLSTimeout *CustomDuration `yaml:"tlsTimeout" json:"tlsTimeout,omitempty"`
 	// HTTP proxy TCP keepalive duration
@@ -428,11 +431,7 @@ func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSe
 	return &configuration, warnings, nil
 }
 
-// A CustomDuration is a Duration that has custom serialization for JSON.
-// JSON in Javascript assumes that int fields are 32 bits and Duration fields are deserialized assuming that numbers
-// are in nanoseconds, which in 32bit integers limits to just 2 seconds.
-// This type assumes that when serializing/deserializing from JSON, that the number is in seconds, while it maintains
-// the YAML serde assumptions.
+// A duration encoded as seconds in JSON and as a Go duration string in YAML.
 type CustomDuration struct {
 	time.Duration
 }
@@ -442,12 +441,16 @@ func (s CustomDuration) MarshalJSON() ([]byte, error) {
 }
 
 func (s *CustomDuration) UnmarshalJSON(data []byte) error {
-	seconds, err := strconv.ParseInt(string(data), 10, 64)
+	seconds, err := strconv.ParseFloat(string(data), 64)
 	if err != nil {
 		return err
 	}
 
-	s.Duration = time.Duration(seconds * int64(time.Second))
+	nanoseconds := math.Round(seconds * float64(time.Second))
+	if math.IsNaN(nanoseconds) || nanoseconds >= float64(math.MaxInt64) || nanoseconds < float64(math.MinInt64) {
+		return fmt.Errorf("duration %s seconds is out of range", data)
+	}
+	s.Duration = time.Duration(nanoseconds)
 	return nil
 }
 

config/duration_test.go

@@ -0,0 +1,51 @@
+package config
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCustomDurationJSON(t *testing.T) {
+	t.Parallel()
+	testCases := []struct {
+		json     string
+		duration time.Duration
+	}{
+		{"0", 0},
+		{"0.000000001", time.Nanosecond},
+		{"0.5", 500 * time.Millisecond},
+		{"1.001", 1001 * time.Millisecond},
+		{"-0.5", -500 * time.Millisecond},
+		{"1", time.Second},
+		{"30", 30 * time.Second},
+		{"3600", time.Hour},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.json, func(t *testing.T) {
+			t.Parallel()
+			var decoded CustomDuration
+			err := json.Unmarshal([]byte(tc.json), &decoded)
+			require.NoError(t, err)
+			require.Equal(t, tc.duration, decoded.Duration)
+
+			data, err := json.Marshal(CustomDuration{Duration: tc.duration})
+			require.NoError(t, err)
+			require.JSONEq(t, tc.json, string(data))
+		})
+	}
+}
+
+func TestCustomDurationRejectsInvalidJSON(t *testing.T) {
+	t.Parallel()
+	for _, data := range []string{`null`, `"1s"`, `1e100`, `-1e100`, `9223372037`, `-9223372037`} {
+		t.Run(data, func(t *testing.T) {
+			t.Parallel()
+			var decoded CustomDuration
+			err := json.Unmarshal([]byte(data), &decoded)
+			require.Error(t, err)
+		})
+	}
+}

ingress/config.go

@@ -26,6 +26,7 @@ const (
 	SSHServerFlag                 = "ssh-server"
 	Socks5Flag                    = "socks5"
 	ProxyConnectTimeoutFlag       = "proxy-connect-timeout"
+	ProxyConnectRetryTimeoutFlag  = "proxy-connect-retry-timeout"
 	ProxyTLSTimeoutFlag           = "proxy-tls-timeout"
 	ProxyTCPKeepAliveFlag         = "proxy-tcp-keepalive"
 	ProxyNoHappyEyeballsFlag      = "proxy-no-happy-eyeballs"
@@ -121,6 +122,7 @@ func (rc *RemoteConfig) UnmarshalJSON(b []byte) error {
 
 func originRequestFromSingleRule(c *cli.Context) OriginRequestConfig {
 	var connectTimeout = defaultHTTPConnectTimeout
+	var connectRetryTimeout config.CustomDuration
 	var tlsTimeout = defaultTLSTimeout
 	var tcpKeepAlive = defaultTCPKeepAlive
 	var noHappyEyeballs bool
@@ -140,6 +142,9 @@ func originRequestFromSingleRule(c *cli.Context) OriginRequestConfig {
 	if flag := ProxyConnectTimeoutFlag; c.IsSet(flag) {
 		connectTimeout = config.CustomDuration{Duration: c.Duration(flag)}
 	}
+	if flag := ProxyConnectRetryTimeoutFlag; c.IsSet(flag) {
+		connectRetryTimeout = config.CustomDuration{Duration: c.Duration(flag)}
+	}
 	if flag := ProxyTLSTimeoutFlag; c.IsSet(flag) {
 		tlsTimeout = config.CustomDuration{Duration: c.Duration(flag)}
 	}
@@ -193,6 +198,7 @@ func originRequestFromSingleRule(c *cli.Context) OriginRequestConfig {
 
 	return OriginRequestConfig{
 		ConnectTimeout:         connectTimeout,
+		ConnectRetryTimeout:    connectRetryTimeout,
 		TLSTimeout:             tlsTimeout,
 		TCPKeepAlive:           tcpKeepAlive,
 		NoHappyEyeballs:        noHappyEyeballs,
@@ -224,6 +230,9 @@ func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
 	if c.ConnectTimeout != nil {
 		out.ConnectTimeout = *c.ConnectTimeout
 	}
+	if c.ConnectRetryTimeout != nil {
+		out.ConnectRetryTimeout = *c.ConnectRetryTimeout
+	}
 	if c.TLSTimeout != nil {
 		out.TLSTimeout = *c.TLSTimeout
 	}
@@ -292,6 +301,8 @@ func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
 type OriginRequestConfig struct {
 	// HTTP proxy timeout for establishing a new connection
 	ConnectTimeout config.CustomDuration `yaml:"connectTimeout" json:"connectTimeout"`
+	// Total time to retry refused HTTP origin connections or missing Unix sockets. Zero disables retries.
+	ConnectRetryTimeout config.CustomDuration `yaml:"connectRetryTimeout" json:"connectRetryTimeout,omitzero"`
 	// HTTP proxy timeout for completing a TLS handshake
 	TLSTimeout config.CustomDuration `yaml:"tlsTimeout" json:"tlsTimeout"`
 	// HTTP proxy TCP keepalive duration
@@ -341,6 +352,12 @@ func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRe
 	}
 }
 
+func (defaults *OriginRequestConfig) setConnectRetryTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.ConnectRetryTimeout; val != nil {
+		defaults.ConnectRetryTimeout = *val
+	}
+}
+
 func (defaults *OriginRequestConfig) setTLSTimeout(overrides config.OriginRequestConfig) {
 	if val := overrides.TLSTimeout; val != nil {
 		defaults.TLSTimeout = *val
@@ -467,6 +484,7 @@ func (defaults *OriginRequestConfig) setAccess(overrides config.OriginRequestCon
 func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfig) OriginRequestConfig {
 	cfg := defaults
 	cfg.setConnectTimeout(overrides)
+	cfg.setConnectRetryTimeout(overrides)
 	cfg.setTLSTimeout(overrides)
 	cfg.setNoHappyEyeballs(overrides)
 	cfg.setKeepAliveConnections(overrides)
@@ -491,6 +509,7 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
 
 func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig {
 	var connectTimeout *config.CustomDuration
+	var connectRetryTimeout *config.CustomDuration
 	var tlsTimeout *config.CustomDuration
 	var tcpKeepAlive *config.CustomDuration
 	var keepAliveConnections *int
@@ -501,6 +520,9 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 	if c.ConnectTimeout != defaultHTTPConnectTimeout {
 		connectTimeout = &c.ConnectTimeout
 	}
+	if c.ConnectRetryTimeout.Duration != 0 {
+		connectRetryTimeout = &c.ConnectRetryTimeout
+	}
 	if c.TLSTimeout != defaultTLSTimeout {
 		tlsTimeout = &c.TLSTimeout
 	}
@@ -522,6 +544,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 
 	return config.OriginRequestConfig{
 		ConnectTimeout:         connectTimeout,
+		ConnectRetryTimeout:    connectRetryTimeout,
 		TLSTimeout:             tlsTimeout,
 		TCPKeepAlive:           tcpKeepAlive,
 		NoHappyEyeballs:        defaultBoolToNil(c.NoHappyEyeballs),
@@ -544,7 +567,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 }
 
 func convertToRawIPRules(ipRules []ipaccess.Rule) []config.IngressIPRule {
-	result := make([]config.IngressIPRule, 0)
+	result := make([]config.IngressIPRule, 0, len(ipRules))
 	for _, r := range ipRules {
 		cidr := r.StringCIDR()
 

ingress/ingress.go

@@ -20,12 +20,13 @@ import (
 )
 
 var (
-	ErrNoIngressRules             = errors.New("The config file doesn't contain any ingress rules")
-	ErrNoIngressRulesCLI          = errors.New("No ingress rules were defined in provided config (if any) nor from the cli, cloudflared will return 503 for all incoming HTTP requests")
-	errLastRuleNotCatchAll        = errors.New("The last ingress rule must match all URLs (i.e. it should not have a hostname or path filter)")
-	errBadWildcard                = errors.New("Hostname patterns can have at most one wildcard character (\"*\") and it can only be used for subdomains, e.g. \"*.example.com\"")
-	errHostnameContainsPort       = errors.New("Hostname cannot contain a port")
-	ErrURLIncompatibleWithIngress = errors.New("You can't set the --url flag (or $TUNNEL_URL) when using multiple-origin ingress rules")
+	ErrNoIngressRules              = errors.New("The config file doesn't contain any ingress rules")
+	ErrNoIngressRulesCLI           = errors.New("No ingress rules were defined in provided config (if any) nor from the cli, cloudflared will return 503 for all incoming HTTP requests")
+	errLastRuleNotCatchAll         = errors.New("The last ingress rule must match all URLs (i.e. it should not have a hostname or path filter)")
+	errBadWildcard                 = errors.New("Hostname patterns can have at most one wildcard character (\"*\") and it can only be used for subdomains, e.g. \"*.example.com\"")
+	errHostnameContainsPort        = errors.New("Hostname cannot contain a port")
+	errNegativeConnectRetryTimeout = errors.New("connectRetryTimeout must not be negative")
+	ErrURLIncompatibleWithIngress  = errors.New("You can't set the --url flag (or $TUNNEL_URL) when using multiple-origin ingress rules")
 )
 
 const (
@@ -178,6 +179,9 @@ func parseCLIIngress(c *cli.Context, allowURLFromArgs bool) (Ingress, error) {
 
 	// Construct an Ingress with the single rule.
 	defaults := originRequestFromSingleRule(c)
+	if defaults.ConnectRetryTimeout.Duration < 0 {
+		return Ingress{}, errNegativeConnectRetryTimeout
+	}
 	ing := Ingress{
 		Rules: []Rule{
 			{
@@ -286,9 +290,15 @@ func validateAccessConfiguration(cfg *config.AccessConfig) error {
 }
 
 func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginRequestConfig) (Ingress, error) {
+	if defaults.ConnectRetryTimeout.Duration < 0 {
+		return Ingress{}, errNegativeConnectRetryTimeout
+	}
 	rules := make([]Rule, len(ingress))
 	for i, r := range ingress {
 		cfg := setConfig(defaults, r.OriginRequest)
+		if cfg.ConnectRetryTimeout.Duration < 0 {
+			return Ingress{}, fmt.Errorf("ingress rule %d: %w", i+1, errNegativeConnectRetryTimeout)
+		}
 		var service OriginService
 
 		if prefix := "unix:"; strings.HasPrefix(r.Service, prefix) {