proxies: raise the upstream response-header timeout to 30m and make it configurable

services/lmstudio-proxy/README.md

@@ -29,6 +29,7 @@ lmstudio-proxy [flags]
 | `--ignore-persisted-port` | `false` | Use `--port` even when a prior runtime port was saved |
 | `--ipc` | *(empty — use stdio)* | Path to a Unix domain socket or Windows named pipe for IPC |
 | `--cluster-dir` | *(empty)* | Cluster trust directory (`node.crt`/`node.key` plus trusted pins). Enables the LAN mTLS inference ingress while this node is a cluster member; empty means no ingress and no peer candidates. |
+| `--response-header-timeout` | *(`$NVPAIR_PROXY_RESPONSE_HEADER_TIMEOUT`, else `30m`)* | How long to wait for an upstream's first response header byte (Go duration; `0` waits indefinitely). The clock starts when the request is sent, so for a **non-streaming** completion it covers queueing and the whole generation — the engine sends nothing until the answer is complete. Streaming requests are unaffected. Applies to every inference transport, including the mTLS ingress hop on a peer, so raise it on every node a request may traverse. Unreachable hosts are still cut off by the 10 s dial timeout. |
 | `--log-level` | *(`$NVPAIR_LOG_LEVEL`, else `info`)* | Initial log level: `debug`, `info`, `warn`, or `error`. Changeable at runtime with `log/set-level`. |
 | `--version` | | Print version and exit |
 

services/lmstudio-proxy/main.go

@@ -16,6 +16,7 @@ import (
 
 	"nvpair-shared/applog"
 	"nvpair-shared/clustertrust"
+	"nvpair-shared/envflag"
 )
 
 func main() {
@@ -25,6 +26,8 @@ func main() {
 	clusterDir := flag.String("cluster-dir", "", "cluster trust directory (node.crt/key + trusted pins); enables the LAN mTLS inference ingress when this node is clustered")
 	showVersion := flag.Bool("version", false, "print version and exit")
 	resolveLevel := applog.RegisterFlag(nil, slog.LevelInfo)
+	resolveResponseHeaderTimeout := envflag.RegisterDuration(nil, "response-header-timeout", "NVPAIR_PROXY_RESPONSE_HEADER_TIMEOUT",
+		defaultResponseHeaderTimeout, "how long to wait for an upstream's first response header byte, e.g. 30m or 0 to wait indefinitely; a non-streaming completion sends none until it is fully generated")
 	flag.Parse()
 
 	if *showVersion {
@@ -73,6 +76,11 @@ func main() {
 	codec := NewCodec(transport)
 	disc := NewDiscovery()
 	proxy := NewProxy(codec, disc, effectivePort)
+	responseHeaderTimeout, err := resolveResponseHeaderTimeout()
+	if err != nil {
+		log.Fatalf("invalid response header timeout: %v", err)
+	}
+	proxy.responseHeaderTimeout = responseHeaderTimeout
 	// Open a live view of this node's cluster mTLS trust fabric. While unclustered
 	// the proxy serves only the loopback plaintext personality; once this node is
 	// a member the same listener also serves the pin-gated LAN mTLS ingress, and

services/lmstudio-proxy/proxy.go

@@ -372,6 +372,13 @@ type Proxy struct {
 	transportMu    sync.Mutex
 	plainTransport *http.Transport
 	peerTransports map[string]*http.Transport
+	// responseHeaderTimeout is the ResponseHeaderTimeout applied to every
+	// inference transport (plain, manual and mTLS peer alike); 0 disables it.
+	// Defaults to defaultResponseHeaderTimeout; main overrides it from
+	// --response-header-timeout / $NVPAIR_PROXY_RESPONSE_HEADER_TIMEOUT
+	// before serving. Transports are built lazily, so it must be final by
+	// the first forwarded request.
+	responseHeaderTimeout time.Duration
 
 	// nextRequestID is a monotonic counter for tagging RequestStarted /
 	// RequestEvent pairs. Atomic add returns the new value, so request
@@ -400,6 +407,8 @@ func NewProxy(codec *Codec, discovery *Discovery, port int) *Proxy {
 		targets:   reach.NewChooser(),
 		runID:     newRunID(),
 		activity:  nodeactivity.NewReporter(activityReportInterval),
+
+		responseHeaderTimeout: defaultResponseHeaderTimeout,
 	}
 }
 
@@ -512,7 +521,6 @@ func (p *Proxy) Run(ctx context.Context) error {
 const (
 	proxyDialTimeout     = 10 * time.Second
 	proxyKeepAlive       = 30 * time.Second
-	proxyResponseTimeout = 120 * time.Second
 	proxyMaxIdleConns    = 50
 	proxyIdleConnTimeout = 90 * time.Second
 	// Inbound http.Server limits — keep IdleTimeout aligned with client
@@ -522,6 +530,20 @@ const (
 	maxModelListBytes      = 16 << 20
 )
 
+// defaultResponseHeaderTimeout bounds how long the proxy waits for the
+// FIRST response header byte from an upstream engine or peer. It starts
+// when the request is sent, so for a non-streaming completion it covers
+// queueing, prefill and the whole generation: the engine sends nothing
+// until the answer is complete. The old 120s cut off any non-streaming
+// request that queued behind other jobs or ran a long reasoning model,
+// and the peer's ingress hop applied the same limit again. Unreachable
+// hosts are still caught by proxyDialTimeout, dead clients by
+// idleClientWriteTimeout, and a client that gives up cancels the upstream
+// request through its context; this deadline only remains as a backstop
+// for an engine that accepted the connection and never answers.
+// Overridable per install: see Proxy.responseHeaderTimeout.
+const defaultResponseHeaderTimeout = 30 * time.Minute
+
 // idleClientWriteTimeout bounds how long a single write of streamed response
 // bytes to the client may block. A killed client can leave a half-open socket
 // whose kernel send buffer fills and never drains; without this deadline the
@@ -588,7 +610,7 @@ func (p *Proxy) serveHTTP(ctx context.Context, ln net.Listener) {
 	slog.Info("proxy timeouts configured",
 		"dial_timeout", proxyDialTimeout,
 		"keep_alive", proxyKeepAlive,
-		"response_header_timeout", proxyResponseTimeout,
+		"response_header_timeout", p.responseHeaderTimeout,
 		"max_idle_conns", proxyMaxIdleConns,
 		"idle_conn_timeout", proxyIdleConnTimeout,
 	)
@@ -713,13 +735,13 @@ func (p *Proxy) candidateTransport(c candidate) *http.Transport {
 	return p.peerHTTPTransport(c.peerUUID)
 }
 
-func newProxyTransport(tlsCfg *tls.Config) *http.Transport {
+func newProxyTransport(tlsCfg *tls.Config, responseHeaderTimeout time.Duration) *http.Transport {
 	tr := &http.Transport{
 		DialContext: (&net.Dialer{
 			Timeout:   proxyDialTimeout,
 			KeepAlive: proxyKeepAlive,
 		}).DialContext,
-		ResponseHeaderTimeout: proxyResponseTimeout,
+		ResponseHeaderTimeout: responseHeaderTimeout,
 		MaxIdleConns:          proxyMaxIdleConns,
 		MaxIdleConnsPerHost:   proxyMaxIdleConns,
 		IdleConnTimeout:       proxyIdleConnTimeout,
@@ -734,7 +756,7 @@ func (p *Proxy) plainHTTPTransport() *http.Transport {
 	p.transportMu.Lock()
 	defer p.transportMu.Unlock()
 	if p.plainTransport == nil {
-		p.plainTransport = newProxyTransport(nil)
+		p.plainTransport = newProxyTransport(nil, p.responseHeaderTimeout)
 	}
 	return p.plainTransport
 }
@@ -750,13 +772,13 @@ func (p *Proxy) peerHTTPTransport(peerUUID string) *http.Transport {
 		delete(p.peerTransports, peerUUID)
 	}
 	if p.mesh == nil {
-		return newProxyTransport(nil)
+		return newProxyTransport(nil, p.responseHeaderTimeout)
 	}
 	cfg, ok := p.mesh.ClientTLSConfig(peerUUID)
 	if !ok {
-		return newProxyTransport(nil)
+		return newProxyTransport(nil, p.responseHeaderTimeout)
 	}
-	tr := newProxyTransport(cfg)
+	tr := newProxyTransport(cfg, p.responseHeaderTimeout)
 	if p.peerTransports == nil {
 		p.peerTransports = make(map[string]*http.Transport)
 	}

services/lmstudio-proxy/response_header_timeout_test.go

@@ -0,0 +1,69 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+// The configured header timeout must reach every inference transport, and 0
+// must mean "no deadline" rather than Go's default of none-set-yet.
+func TestResponseHeaderTimeoutReachesEveryTransport(t *testing.T) {
+	p := testProxy(NewDiscovery(), 1235)
+	if p.responseHeaderTimeout != defaultResponseHeaderTimeout {
+		t.Fatalf("default = %v, want %v", p.responseHeaderTimeout, defaultResponseHeaderTimeout)
+	}
+	for _, want := range []time.Duration{45 * time.Minute, 0} {
+		p := testProxy(NewDiscovery(), 1235)
+		p.responseHeaderTimeout = want
+		if got := p.plainHTTPTransport().ResponseHeaderTimeout; got != want {
+			t.Errorf("plain transport ResponseHeaderTimeout = %v, want %v", got, want)
+		}
+		// Unclustered, a peer lookup falls back to a plain transport built the same way.
+		if got := p.peerHTTPTransport("no-such-peer").ResponseHeaderTimeout; got != want {
+			t.Errorf("peer transport ResponseHeaderTimeout = %v, want %v", got, want)
+		}
+	}
+}
+
+// A non-streaming completion sends no header until generation ends. With a
+// deadline shorter than that silence the request must fail as an upstream
+// error; with a longer one the same request must succeed. This is the
+// behaviour the 120s default broke for queued or long-running jobs.
+func TestResponseHeaderTimeoutBoundsSilentUpstream(t *testing.T) {
+	const silence = 500 * time.Millisecond
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(silence)
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"choices":[]}`))
+	}))
+	defer upstream.Close()
+
+	for _, tc := range []struct {
+		name    string
+		timeout time.Duration
+		status  int
+	}{
+		{"shorter than the silence", silence / 3, http.StatusBadGateway},
+		{"longer than the silence", 10 * silence, http.StatusOK},
+		{"disabled", 0, http.StatusOK},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			disc := NewDiscovery()
+			disc.AddManual(nodeForModel(t, "serving-node", upstream.URL, "qwen"))
+			p := testProxy(disc, 1235)
+			p.responseHeaderTimeout = tc.timeout
+
+			rec := httptest.NewRecorder()
+			p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"qwen"}`)))
+			if rec.Code != tc.status {
+				t.Fatalf("status = %d (%s), want %d", rec.Code, strings.TrimSpace(rec.Body.String()), tc.status)
+			}
+		})
+	}
+}

services/ollama-proxy/README.md

@@ -28,6 +28,7 @@ ollama-proxy [flags]
 | `--ignore-persisted-port` | `false` | Use `--port` even when `proxy-port.json` contains a saved port (used by broker-managed startup) |
 | `--ipc` | *(empty — use stdio)* | Path to a Unix domain socket or Windows named pipe for IPC |
 | `--cluster-dir` | *(empty)* | Cluster trust directory (`node.crt`/`node.key` plus trusted pins). Enables the LAN mTLS inference ingress while this node is a cluster member; empty means no ingress and no peer candidates. |
+| `--response-header-timeout` | *(`$NVPAIR_PROXY_RESPONSE_HEADER_TIMEOUT`, else `30m`)* | How long to wait for an upstream's first response header byte (Go duration; `0` waits indefinitely). The clock starts when the request is sent, so for a **non-streaming** completion it covers queueing and the whole generation — the engine sends nothing until the answer is complete. Streaming requests are unaffected. Applies to every inference transport, including the mTLS ingress hop on a peer, so raise it on every node a request may traverse. Unreachable hosts are still cut off by the 10 s dial timeout. |
 | `--log-level` | *(`$NVPAIR_LOG_LEVEL`, else `info`)* | Initial log level: `debug`, `info`, `warn`, or `error`. Changeable at runtime with `log/set-level`. |
 | `--version` | | Print version and exit |
 

services/ollama-proxy/main.go

@@ -17,6 +17,7 @@ import (
 
 	"nvpair-shared/applog"
 	"nvpair-shared/clustertrust"
+	"nvpair-shared/envflag"
 )
 
 type aliasAddressFlags []string
@@ -36,6 +37,8 @@ func main() {
 	clusterDir := flag.String("cluster-dir", "", "cluster trust directory (node.crt/key + trusted pins); enables the LAN mTLS inference ingress when this node is clustered")
 	showVersion := flag.Bool("version", false, "print version and exit")
 	resolveLevel := applog.RegisterFlag(nil, slog.LevelInfo)
+	resolveResponseHeaderTimeout := envflag.RegisterDuration(nil, "response-header-timeout", "NVPAIR_PROXY_RESPONSE_HEADER_TIMEOUT",
+		defaultResponseHeaderTimeout, "how long to wait for an upstream's first response header byte, e.g. 30m or 0 to wait indefinitely; a non-streaming completion sends none until it is fully generated")
 	flag.Parse()
 
 	if *showVersion {
@@ -84,6 +87,11 @@ func main() {
 	codec := NewCodec(transport)
 	disc := NewDiscovery()
 	proxy := NewProxy(codec, disc, effectivePort)
+	responseHeaderTimeout, err := resolveResponseHeaderTimeout()
+	if err != nil {
+		log.Fatalf("invalid response header timeout: %v", err)
+	}
+	proxy.responseHeaderTimeout = responseHeaderTimeout
 	for _, aliasAddress := range aliasAddresses {
 		if err := proxy.setLoopbackAlias(aliasAddress); err != nil {
 			log.Fatalf("invalid alias address: %v", err)

services/ollama-proxy/proxy.go

@@ -381,6 +381,13 @@ type Proxy struct {
 	transportMu    sync.Mutex
 	plainTransport *http.Transport
 	peerTransports map[string]*http.Transport
+	// responseHeaderTimeout is the ResponseHeaderTimeout applied to every
+	// inference transport (plain, manual and mTLS peer alike); 0 disables it.
+	// Defaults to defaultResponseHeaderTimeout; main overrides it from
+	// --response-header-timeout / $NVPAIR_PROXY_RESPONSE_HEADER_TIMEOUT
+	// before serving. Transports are built lazily, so it must be final by
+	// the first forwarded request.
+	responseHeaderTimeout time.Duration
 
 	// nextRequestID is a monotonic counter for tagging RequestStarted /
 	// RequestEvent pairs. Atomic add returns the new value, so request
@@ -409,6 +416,8 @@ func NewProxy(codec *Codec, discovery *Discovery, port int) *Proxy {
 		targets:   reach.NewChooser(),
 		runID:     newRunID(),
 		activity:  nodeactivity.NewReporter(activityReportInterval),
+
+		responseHeaderTimeout: defaultResponseHeaderTimeout,
 	}
 }
 
@@ -528,7 +537,6 @@ func (p *Proxy) Run(ctx context.Context) error {
 const (
 	proxyDialTimeout     = 10 * time.Second
 	proxyKeepAlive       = 30 * time.Second
-	proxyResponseTimeout = 120 * time.Second
 	proxyMaxIdleConns    = 50
 	proxyIdleConnTimeout = 90 * time.Second
 	// Inbound http.Server limits — keep IdleTimeout aligned with client
@@ -538,6 +546,20 @@ const (
 	maxModelListBytes      = 16 << 20
 )
 
+// defaultResponseHeaderTimeout bounds how long the proxy waits for the
+// FIRST response header byte from an upstream engine or peer. It starts
+// when the request is sent, so for a non-streaming completion it covers
+// queueing, prefill and the whole generation: the engine sends nothing
+// until the answer is complete. The old 120s cut off any non-streaming
+// request that queued behind other jobs or ran a long reasoning model,
+// and the peer's ingress hop applied the same limit again. Unreachable
+// hosts are still caught by proxyDialTimeout, dead clients by
+// idleClientWriteTimeout, and a client that gives up cancels the upstream
+// request through its context; this deadline only remains as a backstop
+// for an engine that accepted the connection and never answers.
+// Overridable per install: see Proxy.responseHeaderTimeout.
+const defaultResponseHeaderTimeout = 30 * time.Minute
+
 // idleClientWriteTimeout bounds how long a single write of streamed response
 // bytes to the client may block. A killed client can leave a half-open socket
 // whose kernel send buffer fills and never drains; without this deadline the
@@ -604,7 +626,7 @@ func (p *Proxy) serveHTTP(ctx context.Context, ln net.Listener) {
 	slog.Info("proxy timeouts configured",
 		"dial_timeout", proxyDialTimeout,
 		"keep_alive", proxyKeepAlive,
-		"response_header_timeout", proxyResponseTimeout,
+		"response_header_timeout", p.responseHeaderTimeout,
 		"max_idle_conns", proxyMaxIdleConns,
 		"idle_conn_timeout", proxyIdleConnTimeout,
 	)
@@ -867,13 +889,13 @@ func (p *Proxy) candidateTransport(c candidate) *http.Transport {
 	return p.peerHTTPTransport(c.peerUUID)
 }
 
-func newProxyTransport(tlsCfg *tls.Config) *http.Transport {
+func newProxyTransport(tlsCfg *tls.Config, responseHeaderTimeout time.Duration) *http.Transport {
 	tr := &http.Transport{
 		DialContext: (&net.Dialer{
 			Timeout:   proxyDialTimeout,
 			KeepAlive: proxyKeepAlive,
 		}).DialContext,
-		ResponseHeaderTimeout: proxyResponseTimeout,
+		ResponseHeaderTimeout: responseHeaderTimeout,
 		MaxIdleConns:          proxyMaxIdleConns,
 		MaxIdleConnsPerHost:   proxyMaxIdleConns,
 		IdleConnTimeout:       proxyIdleConnTimeout,
@@ -888,7 +910,7 @@ func (p *Proxy) plainHTTPTransport() *http.Transport {
 	p.transportMu.Lock()
 	defer p.transportMu.Unlock()
 	if p.plainTransport == nil {
-		p.plainTransport = newProxyTransport(nil)
+		p.plainTransport = newProxyTransport(nil, p.responseHeaderTimeout)
 	}
 	return p.plainTransport
 }
@@ -904,13 +926,13 @@ func (p *Proxy) peerHTTPTransport(peerUUID string) *http.Transport {
 		delete(p.peerTransports, peerUUID)
 	}
 	if p.mesh == nil {
-		return newProxyTransport(nil)
+		return newProxyTransport(nil, p.responseHeaderTimeout)
 	}
 	cfg, ok := p.mesh.ClientTLSConfig(peerUUID)
 	if !ok {
-		return newProxyTransport(nil)
+		return newProxyTransport(nil, p.responseHeaderTimeout)
 	}
-	tr := newProxyTransport(cfg)
+	tr := newProxyTransport(cfg, p.responseHeaderTimeout)
 	if p.peerTransports == nil {
 		p.peerTransports = make(map[string]*http.Transport)
 	}