From 704e00416addcd13774b6767f42656a507f59142 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 15:53:13 -0400 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=A7=AD=20fix:=20Relay=20Loopback=20Re?= =?UTF-8?q?solvers=20Into=20KVM=20Guests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The KVM launcher forwards the runner's /etc/resolv.conf into the libkrun guest. Under Docker Compose that names the embedded resolver 127.0.0.11, but the TSI guest kernel keeps loopback traffic on its own loopback device, so every service-name lookup in the guest failed. Generated-file uploads and session marker reads to egress_gateway were dropped with ConnectionRefused or FailedToOpenSocket, while connections by IP worked. When the runner has a loopback nameserver, relay it over UDP and TCP port 53 from the runner's own IPv4 address with socat and hand the guest that address instead. Further loopback nameservers are dropped, routable resolvers such as Kubernetes cluster DNS are forwarded unchanged, and startup fails if the relay cannot listen or the runner has no routable address. --- README.md | 8 +++++ launcher/entrypoint.sh | 63 ++++++++++++++++++++++++++++++++++++- tests/kvm_guest_dns.sh | 70 ++++++++++++++++++++++++++++++++++++------ 3 files changed, 131 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index db4038c4..4f7f6991 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,14 @@ read-only root disk does not need modification at boot. Rebuild the runner image to pick up this layout change. A missing resolver handoff fails startup rather than leaving the guest with an unrelated public DNS server. +The guest kernel keeps loopback traffic on its own loopback device instead of +proxying it through TSI, so it cannot reach a loopback resolver such as Docker's +embedded `127.0.0.11`. The launcher entrypoint relays the first loopback +nameserver over UDP and TCP port 53 from the runner's own IPv4 address with +`socat`, and forwards that address to the guest in its place; further loopback +nameservers are dropped, and startup fails if the relay cannot listen. Routable +nameservers, including Kubernetes cluster DNS, are forwarded unchanged. + libkrun delivers the guest environment on the kernel command line, which only carries single-line printable ASCII and is capped at 2048 bytes by the guest kernel. The launcher entrypoint therefore forwards only the `nameserver`, diff --git a/launcher/entrypoint.sh b/launcher/entrypoint.sh index 304858e4..8bc42c56 100644 --- a/launcher/entrypoint.sh +++ b/launcher/entrypoint.sh @@ -12,7 +12,22 @@ set -e # joined by a separator that api/src/guest-dns.sh expands back into lines. RESOLV_FIELD_SEPARATOR='|' +# The guest kernel keeps loopback traffic on its own loopback device instead of +# proxying it through TSI, so a loopback resolver such as Docker's embedded +# 127.0.0.11 never answers inside the guest. Relay it from this container's own +# address, which the guest reaches through TSI, and hand the guest that address. +RESOLVER_RELAY_READY_ATTEMPTS=50 + +loopback_nameserver() { + awk '{ sub(/\r$/, "") } $1 == "nameserver" && $2 ~ /^127\./ { print $2; exit }' "$1" +} + +runner_address() { + getent ahostsv4 "$(cat /proc/sys/kernel/hostname)" | awk '$1 !~ /^127\./ { print $1; exit }' +} + encode_resolv_conf() { + local loopback="${1:-}" relay="${2:-}" local LC_ALL=C local line words encoded='' while IFS= read -r line || [ -n "$line" ]; do @@ -21,6 +36,10 @@ encode_resolv_conf() { continue fi read -ra words <<< "$line" + if [ "${words[0]}" = nameserver ] && [[ "${words[1]}" == 127.* ]]; then + [ "${words[1]}" = "$loopback" ] || continue + words[1]="$relay" + fi line="${words[*]}" if [[ "$line" == *[!' '-'~']* || "$line" == *[\"$RESOLV_FIELD_SEPARATOR]* ]]; then echo "ERROR: runner /etc/resolv.conf line cannot cross the kernel command line: $line" >&2 @@ -31,13 +50,55 @@ encode_resolv_conf() { printf '%s' "$encoded" } -SANDBOX_RESOLV_CONF="$(encode_resolv_conf < /etc/resolv.conf)" +proc_net_address() { + local a b c d + IFS=. read -r a b c d <<< "$1" + printf '%02X%02X%02X%02X:0035' "$d" "$c" "$b" "$a" +} + +start_resolver_relay() { + local nameserver="$1" address="$2" local_address udp_pid tcp_pid attempt + socat -T 5 "UDP4-RECVFROM:53,bind=$address,fork" "UDP4-SENDTO:$nameserver:53" & + udp_pid=$! + socat "TCP4-LISTEN:53,bind=$address,reuseaddr,fork" "TCP4:$nameserver:53" & + tcp_pid=$! + local_address="$(proc_net_address "$address")" + for ((attempt = 0; attempt < RESOLVER_RELAY_READY_ATTEMPTS; attempt++)); do + if grep -q " $local_address " /proc/net/udp && grep -q " $local_address 00000000:0000 0A " /proc/net/tcp; then + echo "Relaying guest DNS from $address:53 to loopback resolver $nameserver" + return 0 + fi + if ! kill -0 "$udp_pid" "$tcp_pid" 2>/dev/null; then + break + fi + sleep 0.1 + done + kill "$udp_pid" "$tcp_pid" 2>/dev/null || true + echo "ERROR: guest DNS relay could not listen on $address:53 for loopback resolver $nameserver" >&2 + return 1 +} + +LOOPBACK_NAMESERVER="$(loopback_nameserver /etc/resolv.conf)" +RESOLVER_RELAY_ADDRESS='' +if [ -n "$LOOPBACK_NAMESERVER" ]; then + RESOLVER_RELAY_ADDRESS="$(runner_address)" + if [ -z "$RESOLVER_RELAY_ADDRESS" ]; then + echo "ERROR: runner has no non-loopback IPv4 address to relay loopback resolver $LOOPBACK_NAMESERVER" >&2 + exit 1 + fi +fi + +SANDBOX_RESOLV_CONF="$(encode_resolv_conf "$LOOPBACK_NAMESERVER" "$RESOLVER_RELAY_ADDRESS" < /etc/resolv.conf)" export SANDBOX_RESOLV_CONF if [[ "$RESOLV_FIELD_SEPARATOR$SANDBOX_RESOLV_CONF" != *"${RESOLV_FIELD_SEPARATOR}nameserver "[!\#]* ]]; then echo 'ERROR: runner /etc/resolv.conf has no nameserver' >&2 exit 1 fi +if [ -n "$LOOPBACK_NAMESERVER" ]; then + start_resolver_relay "$LOOPBACK_NAMESERVER" "$RESOLVER_RELAY_ADDRESS" +fi + if [ "${LAUNCHER_FILTER_VSOCK_ENOTCONN:-true}" = "true" ]; then # libkrun can emit this benign TSI/vsock teardown line after the guest has # already closed its side of the socket. It contains the word "error", so diff --git a/tests/kvm_guest_dns.sh b/tests/kvm_guest_dns.sh index 0e038a12..21192125 100755 --- a/tests/kvm_guest_dns.sh +++ b/tests/kvm_guest_dns.sh @@ -80,31 +80,58 @@ printf '%s\n' "$SANDBOX_RESOLV_CONF" > "$TEST_RESOLVER_OUTPUT" STUB cat > "$TEST_DIR/bin/getent" <<'STUB' #!/usr/bin/env bash -printf '192.0.2.99 stale-address\n' +[[ -z "${TEST_RUNNER_ADDRESS-192.0.2.99}" ]] || printf '%s stale-address\n' "${TEST_RUNNER_ADDRESS-192.0.2.99}" STUB -chmod +x "$TEST_DIR/bin/launcher" "$TEST_DIR/bin/getent" +# The relay stub records its endpoints; readiness comes from the fake /proc/net. +cat > "$TEST_DIR/bin/socat" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$TEST_SOCAT_CALLS" +STUB +chmod +x "$TEST_DIR/bin/launcher" "$TEST_DIR/bin/getent" "$TEST_DIR/bin/socat" +# 192.0.2.99:53 bound for UDP and listening for TCP, in /proc/net byte order. +mkdir "$TEST_DIR/proc-net-bound" "$TEST_DIR/proc-net-unbound" +printf ' sl local_address rem_address st\n 0: 630200C0:0035 00000000:0000 07 00000000:00000000\n' > "$TEST_DIR/proc-net-bound/udp" +printf ' sl local_address rem_address st\n 0: 630200C0:0035 00000000:0000 0A 00000000:00000000\n' > "$TEST_DIR/proc-net-bound/tcp" +printf ' sl local_address rem_address st\n' | tee "$TEST_DIR/proc-net-unbound/udp" > "$TEST_DIR/proc-net-unbound/tcp" # libkrun appends every guest environment entry to the kernel command line, # which linux-loader limits to single-line printable ASCII. Docker's generated # resolv.conf is multi-line with comments, so the entrypoint must forward only # the resolver directives, joined by the separator guest-dns.sh expands. run_entrypoint() { - local resolv_conf="$1" + local resolv_conf="$1" proc_net="${2:-$TEST_DIR/proc-net-bound}" sed -e "s|/usr/local/bin/launcher|$TEST_DIR/bin/launcher|g" \ -e "s|/etc/resolv.conf|$resolv_conf|g" \ + -e "s|/proc/net/|$proc_net/|g" \ + -e 's|RESOLVER_RELAY_READY_ATTEMPTS=50|RESOLVER_RELAY_READY_ATTEMPTS=3|' \ "$ROOT/launcher/entrypoint.sh" > "$TEST_DIR/entrypoint.sh" - rm -f "$TEST_DIR/forwarded" + rm -f "$TEST_DIR/forwarded" "$TEST_DIR/socat-calls" PATH="$TEST_DIR/bin:$PATH" \ EGRESS_GATEWAY_URL='https://egress_gateway:3190/base' \ FILE_SERVER_URL='http://[::1]:3000/base' \ SANDBOX_FORWARD_TARGET='tool_call_server:3033' \ LAUNCHER_FILTER_VSOCK_ENOTCONN=false \ TEST_RESOLVER_OUTPUT="$TEST_DIR/forwarded" \ + TEST_SOCAT_CALLS="$TEST_DIR/socat-calls" \ bash "$TEST_DIR/entrypoint.sh" } # Independent reference for the expected handoff: directives only, whitespace -# collapsed, joined by the separator. +# collapsed, joined by the separator. The first loopback nameserver becomes the +# relay address; other loopback nameservers are unreachable and dropped. encoded_reference() { - LC_ALL=C awk '/^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]]/ { sub(/\r$/, ""); $1 = $1; print }' "$1" | paste -sd '|' - + LC_ALL=C awk -v relay=192.0.2.99 '/^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]]/ { + sub(/\r$/, ""); $1 = $1 + if ($1 == "nameserver" && $2 ~ /^127\./) { if (loop == "") loop = $2; if ($2 != loop) next; $2 = relay } + print + }' "$1" | paste -sd '|' - +} +# The relays start in the background, so wait for both before comparing. +assert_relayed() { + local attempt + for ((attempt = 0; attempt < 50; attempt++)); do + [[ -e "$TEST_DIR/socat-calls" && "$(wc -l < "$TEST_DIR/socat-calls")" -ge 2 ]] && break + sleep 0.1 + done + [[ "$(LC_ALL=C sort "$TEST_DIR/socat-calls")" == "$(printf -- '-T 5 UDP4-RECVFROM:53,bind=192.0.2.99,fork UDP4-SENDTO:%s:53\nTCP4-LISTEN:53,bind=192.0.2.99,reuseaddr,fork TCP4:%s:53' "$1" "$1")" ]] } cat > "$TEST_DIR/docker-resolv.conf" <<'RESOLV' # Generated by Docker Engine. @@ -119,24 +146,49 @@ options ndots:0 # Overrides: [] # Option ndots from: internal RESOLV +# The guest keeps loopback traffic local, so Docker's embedded resolver is +# relayed from the runner address and the guest gets that address instead. run_entrypoint "$TEST_DIR/docker-resolv.conf" -[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 127.0.0.11|options ndots:0' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|options ndots:0' ]] [[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/docker-resolv.conf")" ]] +assert_relayed 127.0.0.11 [[ "$(wc -l < "$TEST_DIR/forwarded")" == 1 ]] if LC_ALL=C grep -q '[^ -~]' "$TEST_DIR/forwarded"; then echo 'forwarded resolver contains bytes the kernel command line rejects' >&2; exit 1 fi -# The guest restores exactly the directives Docker supplied. +# The guest restores the Docker directives with the relayed nameserver. rm -rf "$TEST_DIR/guest/run/codeapi-resolver" SANDBOX_RESOLV_CONF="$(cat "$TEST_DIR/forwarded")" configure_guest_dns "$TEST_DIR/guest" -printf 'nameserver 127.0.0.11\noptions ndots:0\n' > "$TEST_DIR/expected" +printf 'nameserver 192.0.2.99\noptions ndots:0\n' > "$TEST_DIR/expected" cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" +# Only one loopback resolver can be relayed; other loopback entries would time +# out in the guest, while routable nameservers keep their order. +printf 'nameserver 127.0.0.53\r\nnameserver 10.0.0.2\nnameserver\t127.0.0.11\nsearch example.internal\n' > "$TEST_DIR/multi-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/multi-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 10.0.0.2|search example.internal' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/multi-loopback-resolv.conf")" ]] +assert_relayed 127.0.0.53 + +# A relay that never listens, or a runner without a routable address, fails +# startup instead of booting a guest that cannot resolve service names. +if run_entrypoint "$TEST_DIR/docker-resolv.conf" "$TEST_DIR/proc-net-unbound" 2> "$TEST_DIR/entrypoint-error"; then + echo 'started launcher without a listening resolver relay' >&2; exit 1 +fi +grep -q 'guest DNS relay could not listen on 192.0.2.99:53' "$TEST_DIR/entrypoint-error" +[[ ! -e "$TEST_DIR/forwarded" ]] +if TEST_RUNNER_ADDRESS='' run_entrypoint "$TEST_DIR/docker-resolv.conf" 2> "$TEST_DIR/entrypoint-error"; then + echo 'started launcher without a resolver relay address' >&2; exit 1 +fi +grep -q 'no non-loopback IPv4 address to relay loopback resolver 127.0.0.11' "$TEST_DIR/entrypoint-error" +[[ ! -e "$TEST_DIR/forwarded" && ! -e "$TEST_DIR/socat-calls" ]] + # Kubernetes resolvers: tabs, CRLF, trailing spaces, and unknown keywords are # normalized away; search domains and options survive intact. printf 'nameserver\t10.96.0.10 \r\nsearch tenant.svc.cluster.local svc.cluster.local cluster.local\n; resolver comment\nlookup file bind\noptions ndots:5\n' > "$TEST_DIR/k8s-resolv.conf" # leak-check:allow run_entrypoint "$TEST_DIR/k8s-resolv.conf" [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 10.96.0.10|search tenant.svc.cluster.local svc.cluster.local cluster.local|options ndots:5' ]] # leak-check:allow +[[ ! -e "$TEST_DIR/socat-calls" ]] # The runner's own resolver must round-trip through the same reference. run_entrypoint /etc/resolv.conf From 594752412ad7b250e84c8d5b293b33d87474562c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 16:01:56 -0400 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=A7=AD=20fix:=20Bound=20the=20Guest?= =?UTF-8?q?=20DNS=20Relay=20and=20Relay=20IPv6=20Loopback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give both relays an idle timeout and cap concurrent TCP children. socat 1.8 fails every UDP-RECVFROM child given max-children, so the UDP relay relies on its short idle timeout. Treat a `::1` nameserver as loopback too, relaying it over IPv6 from the same runner address instead of forwarding an address the guest would only answer locally. --- README.md | 10 ++++++---- launcher/entrypoint.sh | 27 +++++++++++++++++++++++---- tests/kvm_guest_dns.sh | 25 +++++++++++++++++++------ 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 4f7f6991..cbf54059 100644 --- a/README.md +++ b/README.md @@ -248,10 +248,12 @@ rather than leaving the guest with an unrelated public DNS server. The guest kernel keeps loopback traffic on its own loopback device instead of proxying it through TSI, so it cannot reach a loopback resolver such as Docker's embedded `127.0.0.11`. The launcher entrypoint relays the first loopback -nameserver over UDP and TCP port 53 from the runner's own IPv4 address with -`socat`, and forwards that address to the guest in its place; further loopback -nameservers are dropped, and startup fails if the relay cannot listen. Routable -nameservers, including Kubernetes cluster DNS, are forwarded unchanged. +nameserver (`127.0.0.0/8` or `::1`) over UDP and TCP port 53 from the runner's +own IPv4 address with `socat`, and forwards that address to the guest in its +place; further loopback nameservers are dropped, and startup fails if the relay +cannot listen. Relay children exit when idle, and the TCP relay caps its +concurrent children. Routable nameservers, including Kubernetes cluster DNS, are forwarded +unchanged. libkrun delivers the guest environment on the kernel command line, which only carries single-line printable ASCII and is capped at 2048 bytes by the guest diff --git a/launcher/entrypoint.sh b/launcher/entrypoint.sh index 8bc42c56..0d8ae61e 100644 --- a/launcher/entrypoint.sh +++ b/launcher/entrypoint.sh @@ -16,10 +16,22 @@ RESOLV_FIELD_SEPARATOR='|' # proxying it through TSI, so a loopback resolver such as Docker's embedded # 127.0.0.11 never answers inside the guest. Relay it from this container's own # address, which the guest reaches through TSI, and hand the guest that address. +# The relay forks per datagram and per connection, so bound how long an idle +# child may live, and how many TCP children may run at once. socat 1.8 fails +# every UDP-RECVFROM child given max-children, so the UDP relay relies on its +# short idle timeout. Only the runner network reaches the relay: NsJail jobs in +# the guest run in their own network namespace. RESOLVER_RELAY_READY_ATTEMPTS=50 +RESOLVER_RELAY_MAX_CHILDREN=64 +RESOLVER_RELAY_UDP_IDLE_SECONDS=5 +RESOLVER_RELAY_TCP_IDLE_SECONDS=30 + +is_loopback_nameserver() { + [[ "$1" == 127.* || "$1" == ::1 ]] +} loopback_nameserver() { - awk '{ sub(/\r$/, "") } $1 == "nameserver" && $2 ~ /^127\./ { print $2; exit }' "$1" + awk '{ sub(/\r$/, "") } $1 == "nameserver" && ($2 ~ /^127\./ || $2 == "::1") { print $2; exit }' "$1" } runner_address() { @@ -36,7 +48,7 @@ encode_resolv_conf() { continue fi read -ra words <<< "$line" - if [ "${words[0]}" = nameserver ] && [[ "${words[1]}" == 127.* ]]; then + if [ "${words[0]}" = nameserver ] && is_loopback_nameserver "${words[1]}"; then [ "${words[1]}" = "$loopback" ] || continue words[1]="$relay" fi @@ -58,9 +70,16 @@ proc_net_address() { start_resolver_relay() { local nameserver="$1" address="$2" local_address udp_pid tcp_pid attempt - socat -T 5 "UDP4-RECVFROM:53,bind=$address,fork" "UDP4-SENDTO:$nameserver:53" & + local udp_target="UDP4-SENDTO:$nameserver:53" tcp_target="TCP4:$nameserver:53" + if [ "$nameserver" = ::1 ]; then + udp_target='UDP6-SENDTO:[::1]:53' + tcp_target='TCP6:[::1]:53' + fi + socat -T "$RESOLVER_RELAY_UDP_IDLE_SECONDS" \ + "UDP4-RECVFROM:53,bind=$address,fork" "$udp_target" & udp_pid=$! - socat "TCP4-LISTEN:53,bind=$address,reuseaddr,fork" "TCP4:$nameserver:53" & + socat -T "$RESOLVER_RELAY_TCP_IDLE_SECONDS" \ + "TCP4-LISTEN:53,bind=$address,reuseaddr,fork,max-children=$RESOLVER_RELAY_MAX_CHILDREN" "$tcp_target" & tcp_pid=$! local_address="$(proc_net_address "$address")" for ((attempt = 0; attempt < RESOLVER_RELAY_READY_ATTEMPTS; attempt++)); do diff --git a/tests/kvm_guest_dns.sh b/tests/kvm_guest_dns.sh index 21192125..b57a7684 100755 --- a/tests/kvm_guest_dns.sh +++ b/tests/kvm_guest_dns.sh @@ -120,18 +120,19 @@ run_entrypoint() { encoded_reference() { LC_ALL=C awk -v relay=192.0.2.99 '/^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]]/ { sub(/\r$/, ""); $1 = $1 - if ($1 == "nameserver" && $2 ~ /^127\./) { if (loop == "") loop = $2; if ($2 != loop) next; $2 = relay } + if ($1 == "nameserver" && ($2 ~ /^127\./ || $2 == "::1")) { if (loop == "") loop = $2; if ($2 != loop) next; $2 = relay } print }' "$1" | paste -sd '|' - } -# The relays start in the background, so wait for both before comparing. +# The relays start in the background, so wait for both before comparing. Both +# bound idle children; the TCP listener also caps concurrent children. assert_relayed() { - local attempt + local udp_target="$1" tcp_target="$2" attempt for ((attempt = 0; attempt < 50; attempt++)); do [[ -e "$TEST_DIR/socat-calls" && "$(wc -l < "$TEST_DIR/socat-calls")" -ge 2 ]] && break sleep 0.1 done - [[ "$(LC_ALL=C sort "$TEST_DIR/socat-calls")" == "$(printf -- '-T 5 UDP4-RECVFROM:53,bind=192.0.2.99,fork UDP4-SENDTO:%s:53\nTCP4-LISTEN:53,bind=192.0.2.99,reuseaddr,fork TCP4:%s:53' "$1" "$1")" ]] + [[ "$(LC_ALL=C sort "$TEST_DIR/socat-calls")" == "$(printf -- '-T 30 TCP4-LISTEN:53,bind=192.0.2.99,reuseaddr,fork,max-children=64 %s\n-T 5 UDP4-RECVFROM:53,bind=192.0.2.99,fork %s' "$tcp_target" "$udp_target")" ]] } cat > "$TEST_DIR/docker-resolv.conf" <<'RESOLV' # Generated by Docker Engine. @@ -151,7 +152,7 @@ RESOLV run_entrypoint "$TEST_DIR/docker-resolv.conf" [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|options ndots:0' ]] [[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/docker-resolv.conf")" ]] -assert_relayed 127.0.0.11 +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 [[ "$(wc -l < "$TEST_DIR/forwarded")" == 1 ]] if LC_ALL=C grep -q '[^ -~]' "$TEST_DIR/forwarded"; then echo 'forwarded resolver contains bytes the kernel command line rejects' >&2; exit 1 @@ -168,7 +169,19 @@ printf 'nameserver 127.0.0.53\r\nnameserver 10.0.0.2\nnameserver\t127.0.0.11\nse run_entrypoint "$TEST_DIR/multi-loopback-resolv.conf" [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 10.0.0.2|search example.internal' ]] [[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/multi-loopback-resolv.conf")" ]] -assert_relayed 127.0.0.53 +assert_relayed UDP4-SENDTO:127.0.0.53:53 TCP4:127.0.0.53:53 + +# An IPv6 loopback resolver is just as unreachable from the guest; the IPv4 +# relay forwards to it over IPv6. +printf 'nameserver ::1\noptions ndots:0\n' > "$TEST_DIR/ipv6-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/ipv6-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|options ndots:0' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/ipv6-loopback-resolv.conf")" ]] +assert_relayed 'UDP6-SENDTO:[::1]:53' 'TCP6:[::1]:53' +printf 'nameserver 127.0.0.11\nnameserver ::1\n' > "$TEST_DIR/mixed-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/mixed-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99' ]] +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 # A relay that never listens, or a runner without a routable address, fails # startup instead of booting a guest that cannot resolve service names. From 96fadf24a6ce45094553f7165b4e24a6595504ae Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 16:08:04 -0400 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=A7=AD=20fix:=20Recognize=20Every=20L?= =?UTF-8?q?oopback=20Resolver=20Spelling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Classify nameservers as loopback by value rather than by the canonical `::1` string: zero-padded or compressed IPv6 loopback and IPv4-mapped 127.0.0.0/8 are relayed too, forwarding over IPv6 to the address exactly as written. Routable IPv6 resolvers stay untouched. --- README.md | 3 ++- launcher/entrypoint.sh | 20 +++++++++++++++----- tests/kvm_guest_dns.sh | 16 +++++++++++++++- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index cbf54059..0675c70c 100644 --- a/README.md +++ b/README.md @@ -248,7 +248,8 @@ rather than leaving the guest with an unrelated public DNS server. The guest kernel keeps loopback traffic on its own loopback device instead of proxying it through TSI, so it cannot reach a loopback resolver such as Docker's embedded `127.0.0.11`. The launcher entrypoint relays the first loopback -nameserver (`127.0.0.0/8` or `::1`) over UDP and TCP port 53 from the runner's +nameserver (`127.0.0.0/8`, `::1` in any spelling, or IPv4-mapped +`127.0.0.0/8`) over UDP and TCP port 53 from the runner's own IPv4 address with `socat`, and forwards that address to the guest in its place; further loopback nameservers are dropped, and startup fails if the relay cannot listen. Relay children exit when idle, and the TCP relay caps its diff --git a/launcher/entrypoint.sh b/launcher/entrypoint.sh index 0d8ae61e..f363e900 100644 --- a/launcher/entrypoint.sh +++ b/launcher/entrypoint.sh @@ -26,12 +26,22 @@ RESOLVER_RELAY_MAX_CHILDREN=64 RESOLVER_RELAY_UDP_IDLE_SECONDS=5 RESOLVER_RELAY_TCP_IDLE_SECONDS=30 +# IPv4 127.0.0.0/8, IPv6 ::1 in any zero-padded or compressed spelling, and +# IPv4-mapped 127.0.0.0/8. is_loopback_nameserver() { - [[ "$1" == 127.* || "$1" == ::1 ]] + local address="${1,,}" + [[ "$address" == 127.* || "$address" =~ ^[0:]*:0*1$ || "$address" == ::ffff:127.* ]] } loopback_nameserver() { - awk '{ sub(/\r$/, "") } $1 == "nameserver" && ($2 ~ /^127\./ || $2 == "::1") { print $2; exit }' "$1" + local line words + while IFS= read -r line || [ -n "$line" ]; do + read -ra words <<< "${line%$'\r'}" + if [ "${words[0]:-}" = nameserver ] && is_loopback_nameserver "${words[1]:-}"; then + printf '%s' "${words[1]}" + return 0 + fi + done < "$1" } runner_address() { @@ -71,9 +81,9 @@ proc_net_address() { start_resolver_relay() { local nameserver="$1" address="$2" local_address udp_pid tcp_pid attempt local udp_target="UDP4-SENDTO:$nameserver:53" tcp_target="TCP4:$nameserver:53" - if [ "$nameserver" = ::1 ]; then - udp_target='UDP6-SENDTO:[::1]:53' - tcp_target='TCP6:[::1]:53' + if [[ "$nameserver" == *:* ]]; then + udp_target="UDP6-SENDTO:[$nameserver]:53" + tcp_target="TCP6:[$nameserver]:53" fi socat -T "$RESOLVER_RELAY_UDP_IDLE_SECONDS" \ "UDP4-RECVFROM:53,bind=$address,fork" "$udp_target" & diff --git a/tests/kvm_guest_dns.sh b/tests/kvm_guest_dns.sh index b57a7684..3394b87b 100755 --- a/tests/kvm_guest_dns.sh +++ b/tests/kvm_guest_dns.sh @@ -120,7 +120,8 @@ run_entrypoint() { encoded_reference() { LC_ALL=C awk -v relay=192.0.2.99 '/^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]]/ { sub(/\r$/, ""); $1 = $1 - if ($1 == "nameserver" && ($2 ~ /^127\./ || $2 == "::1")) { if (loop == "") loop = $2; if ($2 != loop) next; $2 = relay } + address = tolower($2) + if ($1 == "nameserver" && (address ~ /^127\./ || address ~ /^[0:]*:0*1$/ || address ~ /^::ffff:127\./)) { if (loop == "") loop = $2; if ($2 != loop) next; $2 = relay } print }' "$1" | paste -sd '|' - } @@ -182,6 +183,19 @@ printf 'nameserver 127.0.0.11\nnameserver ::1\n' > "$TEST_DIR/mixed-loopback-res run_entrypoint "$TEST_DIR/mixed-loopback-resolv.conf" [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99' ]] assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +# Every spelling the guest would canonicalize to loopback is relayed too. +for spelling in 0:0:0:0:0:0:0:1 0000::0001 ::FFFF:127.0.0.11; do + printf 'nameserver %s\nnameserver 10.0.0.2\n' "$spelling" > "$TEST_DIR/spelled-loopback-resolv.conf" + run_entrypoint "$TEST_DIR/spelled-loopback-resolv.conf" + [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 10.0.0.2' ]] + [[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/spelled-loopback-resolv.conf")" ]] + assert_relayed "UDP6-SENDTO:[$spelling]:53" "TCP6:[$spelling]:53" +done +# Routable IPv6 resolvers are not mistaken for loopback. +printf 'nameserver 2001:db8::1\nnameserver fd00::11\n' > "$TEST_DIR/ipv6-resolv.conf" +run_entrypoint "$TEST_DIR/ipv6-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 2001:db8::1|nameserver fd00::11' ]] +[[ ! -e "$TEST_DIR/socat-calls" ]] # A relay that never listens, or a runner without a routable address, fails # startup instead of booting a guest that cannot resolve service names. From 582be7c286bb22454932f29635cba605ceba533b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 16:16:11 -0400 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=A7=AD=20fix:=20Parse=20Nameserver=20?= =?UTF-8?q?Addresses=20Before=20Loopback=20Detection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace string matching with real parsing. IPv6 addresses are expanded to their eight groups (compressed zeros, dotted IPv4 tails), and IPv4 addresses are read in the inet_aton(3) forms glibc accepts in resolv.conf. Then ::1 and IPv4-mapped or plain 127.0.0.0/8 are relayed in any spelling. Mapped addresses are relayed over IPv4 to their canonical form. Spellings of the same resolver collapse into one relayed entry, and addresses glibc rejects are forwarded unchanged. --- README.md | 4 +- launcher/entrypoint.sh | 99 ++++++++++++++++++++++++++++++++++++------ tests/kvm_guest_dns.sh | 36 +++++++++++---- 3 files changed, 115 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 0675c70c..94ad3b9c 100644 --- a/README.md +++ b/README.md @@ -248,8 +248,8 @@ rather than leaving the guest with an unrelated public DNS server. The guest kernel keeps loopback traffic on its own loopback device instead of proxying it through TSI, so it cannot reach a loopback resolver such as Docker's embedded `127.0.0.11`. The launcher entrypoint relays the first loopback -nameserver (`127.0.0.0/8`, `::1` in any spelling, or IPv4-mapped -`127.0.0.0/8`) over UDP and TCP port 53 from the runner's +nameserver (`127.0.0.0/8`, `::1` or IPv4-mapped `127.0.0.0/8`, in any spelling +glibc accepts) over UDP and TCP port 53 from the runner's own IPv4 address with `socat`, and forwards that address to the guest in its place; further loopback nameservers are dropped, and startup fails if the relay cannot listen. Relay children exit when idle, and the TCP relay caps its diff --git a/launcher/entrypoint.sh b/launcher/entrypoint.sh index f363e900..47fba9e6 100644 --- a/launcher/entrypoint.sh +++ b/launcher/entrypoint.sh @@ -26,19 +26,92 @@ RESOLVER_RELAY_MAX_CHILDREN=64 RESOLVER_RELAY_UDP_IDLE_SECONDS=5 RESOLVER_RELAY_TCP_IDLE_SECONDS=30 -# IPv4 127.0.0.0/8, IPv6 ::1 in any zero-padded or compressed spelling, and -# IPv4-mapped 127.0.0.0/8. -is_loopback_nameserver() { - local address="${1,,}" - [[ "$address" == 127.* || "$address" =~ ^[0:]*:0*1$ || "$address" == ::ffff:127.* ]] +# One inet_aton(3) part: hex with 0x, octal with a leading 0, else decimal. +inet_aton_part() { + case "$1" in + 0x*) [[ "$1" =~ ^0x[0-9a-f]+$ ]] && echo $((16#${1#0x})) ;; + 0*) [[ "$1" =~ ^0[0-7]*$ ]] && echo $((8#$1)) ;; + *) [[ "$1" =~ ^[0-9]+$ ]] && echo $((10#$1)) ;; + esac +} + +# Numeric value of an IPv4 address in the inet_aton(3) forms glibc accepts in +# resolv.conf: one to four parts, the last filling the remaining bytes. +ipv4_value() { + local parts part value=0 index last + IFS=. read -ra parts <<< "$1" + [[ "$1" != *. ]] && (( ${#parts[@]} >= 1 && ${#parts[@]} <= 4 )) || return 1 + last=$(( ${#parts[@]} - 1 )) + for ((index = 0; index < last; index++)); do + part="$(inet_aton_part "${parts[index]}")" || return 1 + (( part <= 255 )) || return 1 + value=$(( value | part << (24 - 8 * index) )) + done + part="$(inet_aton_part "${parts[last]}")" || return 1 + (( part < 1 << (32 - 8 * last) )) || return 1 + echo $(( value | part )) +} + +# The eight 16-bit groups of an IPv6 address in decimal, accepting compressed +# zeros and a dotted IPv4 tail. +ipv6_groups() { + local address="$1" head tail octets octet group groups=() tail_groups=() fill + if [[ "$address" =~ ^(.*:)([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + address="${BASH_REMATCH[1]}" + IFS=. read -ra octets <<< "${BASH_REMATCH[2]}" + for octet in "${octets[@]}"; do + [[ "$octet" =~ ^(0|[1-9][0-9]{0,2})$ ]] && (( octet <= 255 )) || return 1 + done + address+="$(printf '%x:%x' $(( octets[0] << 8 | octets[1] )) $(( octets[2] << 8 | octets[3] )))" + fi + [[ "$address" =~ ^[0-9a-f:]+$ && "$address" != :[!:]* && "$address" != *[!:]: && "$address" != *:::* ]] || return 1 + if [[ "$address" == *::* ]]; then + head="${address%%::*}" + tail="${address#*::}" + [[ "$tail" != *::* ]] || return 1 + [ -z "$head" ] || IFS=: read -ra groups <<< "$head" + [ -z "$tail" ] || IFS=: read -ra tail_groups <<< "$tail" + fill=$(( 8 - ${#groups[@]} - ${#tail_groups[@]} )) + (( fill >= 1 )) || return 1 + for ((; fill > 0; fill--)); do groups+=(0); done + groups+=("${tail_groups[@]}") + else + IFS=: read -ra groups <<< "$address" + fi + (( ${#groups[@]} == 8 )) || return 1 + for group in "${groups[@]}"; do + [[ "$group" =~ ^[0-9a-f]{1,4}$ ]] || return 1 + printf '%d ' $((16#$group)) + done +} + +# Canonical form of a loopback nameserver in any spelling glibc accepts: IPv4 +# 127.0.0.0/8, IPv6 ::1, or IPv4-mapped 127.0.0.0/8 (relayed as IPv4). Fails for +# every other address. +canonical_loopback() { + local address="${1,,}" text groups value + address="${address%%\%*}" + if [[ "$address" == *:* ]]; then + text="$(ipv6_groups "$address")" || return 1 + read -ra groups <<< "$text" + if [ "${groups[*]}" = '0 0 0 0 0 0 0 1' ]; then + echo ::1 + return 0 + fi + [ "${groups[*]:0:6}" = '0 0 0 0 0 65535' ] || return 1 + value=$(( groups[6] << 16 | groups[7] )) + else + value="$(ipv4_value "$address")" || return 1 + fi + (( value >> 24 == 127 )) || return 1 + printf '%d.%d.%d.%d' $(( value >> 24 )) $(( value >> 16 & 255 )) $(( value >> 8 & 255 )) $(( value & 255 )) } loopback_nameserver() { local line words while IFS= read -r line || [ -n "$line" ]; do read -ra words <<< "${line%$'\r'}" - if [ "${words[0]:-}" = nameserver ] && is_loopback_nameserver "${words[1]:-}"; then - printf '%s' "${words[1]}" + if [ "${words[0]:-}" = nameserver ] && canonical_loopback "${words[1]:-}"; then return 0 fi done < "$1" @@ -51,15 +124,15 @@ runner_address() { encode_resolv_conf() { local loopback="${1:-}" relay="${2:-}" local LC_ALL=C - local line words encoded='' + local line words canonical encoded='' while IFS= read -r line || [ -n "$line" ]; do line="${line%$'\r'}" if [[ ! "$line" =~ ^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]] ]]; then continue fi read -ra words <<< "$line" - if [ "${words[0]}" = nameserver ] && is_loopback_nameserver "${words[1]}"; then - [ "${words[1]}" = "$loopback" ] || continue + if [ "${words[0]}" = nameserver ] && canonical="$(canonical_loopback "${words[1]}")"; then + [ "$canonical" = "$loopback" ] || continue words[1]="$relay" fi line="${words[*]}" @@ -81,9 +154,9 @@ proc_net_address() { start_resolver_relay() { local nameserver="$1" address="$2" local_address udp_pid tcp_pid attempt local udp_target="UDP4-SENDTO:$nameserver:53" tcp_target="TCP4:$nameserver:53" - if [[ "$nameserver" == *:* ]]; then - udp_target="UDP6-SENDTO:[$nameserver]:53" - tcp_target="TCP6:[$nameserver]:53" + if [ "$nameserver" = ::1 ]; then + udp_target='UDP6-SENDTO:[::1]:53' + tcp_target='TCP6:[::1]:53' fi socat -T "$RESOLVER_RELAY_UDP_IDLE_SECONDS" \ "UDP4-RECVFROM:53,bind=$address,fork" "$udp_target" & diff --git a/tests/kvm_guest_dns.sh b/tests/kvm_guest_dns.sh index 3394b87b..4ea99afb 100755 --- a/tests/kvm_guest_dns.sh +++ b/tests/kvm_guest_dns.sh @@ -183,18 +183,36 @@ printf 'nameserver 127.0.0.11\nnameserver ::1\n' > "$TEST_DIR/mixed-loopback-res run_entrypoint "$TEST_DIR/mixed-loopback-resolv.conf" [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99' ]] assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 -# Every spelling the guest would canonicalize to loopback is relayed too. -for spelling in 0:0:0:0:0:0:0:1 0000::0001 ::FFFF:127.0.0.11; do +# Every spelling glibc accepts for a loopback resolver is relayed to its +# canonical address: compressed or zero-padded ::1, IPv4-mapped 127/8 in any +# IPv6 form, and inet_aton(3) IPv4 forms. +while read -r spelling udp_target tcp_target; do printf 'nameserver %s\nnameserver 10.0.0.2\n' "$spelling" > "$TEST_DIR/spelled-loopback-resolv.conf" run_entrypoint "$TEST_DIR/spelled-loopback-resolv.conf" [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 10.0.0.2' ]] - [[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/spelled-loopback-resolv.conf")" ]] - assert_relayed "UDP6-SENDTO:[$spelling]:53" "TCP6:[$spelling]:53" -done -# Routable IPv6 resolvers are not mistaken for loopback. -printf 'nameserver 2001:db8::1\nnameserver fd00::11\n' > "$TEST_DIR/ipv6-resolv.conf" -run_entrypoint "$TEST_DIR/ipv6-resolv.conf" -[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 2001:db8::1|nameserver fd00::11' ]] + assert_relayed "$udp_target" "$tcp_target" +done <<'SPELLINGS' +0:0:0:0:0:0:0:1 UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +0000::0001 UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +::1%lo UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +::FFFF:127.0.0.11 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0:0:0:0:0:ffff:127.0.0.11 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +::ffff:7f00:b UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +127.1 UDP4-SENDTO:127.0.0.1:53 TCP4:127.0.0.1:53 +2130706443 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0x7f.0.0.0xb UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0177.0.0.013 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +SPELLINGS +# Spellings of the same loopback resolver collapse into one relayed entry. +printf 'nameserver 127.0.0.11\nnameserver ::ffff:7f00:b\nnameserver 127.0.0.53\n' > "$TEST_DIR/same-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/same-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 192.0.2.99' ]] +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +# Routable resolvers and addresses glibc rejects are forwarded unchanged, without +# a relay: 0127 is octal 87, and ::7f00:1 is not IPv4-mapped. +printf 'nameserver 2001:db8::1\nnameserver fd00::11\nnameserver ::ffff:10.0.0.2\nnameserver ::7f00:1\nnameserver 0127.0.0.1\nnameserver 127.0.0.256\nnameserver 1:2:3:4:5:6:7:8:\nnameserver 127..1\n' > "$TEST_DIR/routable-resolv.conf" +run_entrypoint "$TEST_DIR/routable-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 2001:db8::1|nameserver fd00::11|nameserver ::ffff:10.0.0.2|nameserver ::7f00:1|nameserver 0127.0.0.1|nameserver 127.0.0.256|nameserver 1:2:3:4:5:6:7:8:|nameserver 127..1' ]] [[ ! -e "$TEST_DIR/socat-calls" ]] # A relay that never listens, or a runner without a routable address, fails