From 068b91e2afbc3b9cf3a9447f208ee027ab73aecb Mon Sep 17 00:00:00 2001 From: Benedikt Franke Date: Thu, 27 Aug 2026 12:27:59 +0200 Subject: [PATCH 1/2] fix(nginx): re-resolve upstreams so a container restart cannot break ingest nginx resolves the service names in an upstream block once at startup and keeps the address for the life of the worker process. Docker does not guarantee a container the same address across starts, so any restart can leave nginx proxying to a container that no longer owns the old address. Ingest then answers 502 on every request while the web UI keeps working, and nothing recovers until someone restarts nginx by hand. The "resolve" parameter makes nginx re-resolve per DNS TTL, bounded by "valid". That is two lookups per 10s against Docker's in-process embedded DNS regardless of request volume, so the cost does not scale with load. Co-authored-by: Ilias Aaguida --- nginx.conf | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/nginx.conf b/nginx.conf index f15cb8cb51c..2ebf14a0624 100644 --- a/nginx.conf +++ b/nginx.conf @@ -65,13 +65,26 @@ http { proxy_read_timeout 90s; proxy_send_timeout 5s; + # Without "resolve", nginx looks the service names up once at startup and keeps that address + # for the life of the worker process. Docker does not guarantee a container the same address + # across starts, so nginx can end up proxying to a container that no longer owns it and + # answer every request with 502 until someone restarts nginx by hand. + # "resolve" needs a resolver, and 127.0.0.11 is Docker's embedded DNS, which is the only + # thing on a Compose network that knows the service names. The nginx image itself assumes it, + # see /docker-entrypoint.d/15-local-resolvers.envsh. + # There are no AAAA records worth waiting for on a Compose network. + resolver 127.0.0.11 valid=10s ipv6=off; + upstream relay { - server relay:3000; + # "resolve" requires the upstream group in shared memory, hence "zone". + zone relay 64k; + server relay:3000 resolve; keepalive 2; } upstream sentry { - server web:9000; + zone sentry 64k; + server web:9000 resolve; keepalive 2; } From d4697bde5f0c327d91c11ed9b93888c51d0cd985 Mon Sep 17 00:00:00 2001 From: Benedikt Franke Date: Thu, 27 Aug 2026 13:03:26 +0200 Subject: [PATCH 2/2] fix(nginx): read the resolver address from the container engine Pinning 127.0.0.11 assumed Docker's embedded DNS. Podman answers on the network gateway instead, so "resolve" would have failed every lookup there and nginx would have returned 502 for every request. The nginx image already derives the address from /etc/resolv.conf into NGINX_LOCAL_RESOLVERS, which reaches the config through envsubst only, so nginx.conf is now mounted as a template and rendered over the image default. NGINX_ENVSUBST_FILTER limits substitution to that one variable so the nginx runtime variables in the config survive. --- docker-compose.yml | 10 +++++++++- nginx.conf | 9 +++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 82c5deee853..1a6762c33f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -601,11 +601,19 @@ services: ports: - "$SENTRY_BIND:80/tcp" image: "nginx:1.31.4-alpine" + environment: + # nginx.conf needs the container engine's DNS address, which differs per engine. + # The image reads it from /etc/resolv.conf into NGINX_LOCAL_RESOLVERS and substitutes + # it into templates, so nginx.conf is mounted as one and rendered over the image default. + # The filter keeps envsubst away from the nginx runtime variables in the config. + NGINX_ENTRYPOINT_LOCAL_RESOLVERS: 1 + NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx + NGINX_ENVSUBST_FILTER: "^NGINX_LOCAL_RESOLVERS$$" volumes: - type: bind read_only: true source: ./nginx.conf - target: /etc/nginx/nginx.conf + target: /etc/nginx/templates/nginx.conf.template - sentry-nginx-cache:/var/cache/nginx - sentry-nginx-www:/var/www healthcheck: diff --git a/nginx.conf b/nginx.conf index 2ebf14a0624..3aa2c10049f 100644 --- a/nginx.conf +++ b/nginx.conf @@ -69,11 +69,12 @@ http { # for the life of the worker process. Docker does not guarantee a container the same address # across starts, so nginx can end up proxying to a container that no longer owns it and # answer every request with 502 until someone restarts nginx by hand. - # "resolve" needs a resolver, and 127.0.0.11 is Docker's embedded DNS, which is the only - # thing on a Compose network that knows the service names. The nginx image itself assumes it, - # see /docker-entrypoint.d/15-local-resolvers.envsh. + # "resolve" needs a resolver, and the container engine's own DNS is the only thing on a Compose + # network that knows the service names. Its address differs per engine -- 127.0.0.11 on Docker, + # the network gateway on Podman -- so it is read from /etc/resolv.conf at startup by + # /docker-entrypoint.d/15-local-resolvers.envsh, which is why this file is a template. # There are no AAAA records worth waiting for on a Compose network. - resolver 127.0.0.11 valid=10s ipv6=off; + resolver ${NGINX_LOCAL_RESOLVERS} valid=10s ipv6=off; upstream relay { # "resolve" requires the upstream group in shared memory, hence "zone".