diff --git a/deploy/containerlab/docs/baseline.md b/deploy/containerlab/docs/baseline.md index 224d4056..4e53ae5a 100644 --- a/deploy/containerlab/docs/baseline.md +++ b/deploy/containerlab/docs/baseline.md @@ -88,21 +88,23 @@ The IPv4 check captures on `remote-host` itself and stops every tcpdump there, so `gateway-ipv4` has no `remote-host` capture point. The check's own client-side assertions stand in for it. -### Intermittent TCP loss in the gateway checks - -Both lab ingress checks lose a TCP flow now and then, with or without these -captures: in three back-to-back runs of `verify-gateway-ingress-ipv4.sh --node -dfw-worker2` with no capture running, one answered 6 of 8 TCP flows. The -captures show that the lost flow is delivered. The request reaches the -backend over the gateway, and the backend acknowledges it. Then the `ns60` -responder (`socat ... fork SYSTEM:`) closes without writing a reply, 1 to -2 ms after the client's FIN, and that close returns to the client over the -DSR path. UDP flows are never lost. - -So each gateway scenario's exact flow counts and the check's exit status go -to `volatile.tsv`, and `results.json` records only whether each protocol had +### Gateway flow counts + +Each gateway scenario's exact flow counts and the check's exit status go to +`volatile.tsv`, and `results.json` records only whether each protocol had answered flows. +The baseline was recorded while the `ns60` responder still lost flows: in +three back-to-back runs of `verify-gateway-ingress-ipv4.sh --node dfw-worker2` +with no capture running, one answered 6 of 8 TCP flows. The captures showed +each lost request reaching its backend, which then closed without a reply. +The responder ran `socat ... fork SYSTEM:"echo ..."`, and socat wrote the +client's request into `echo` after `echo` had exited. When that write failed +before socat relayed the reply, socat exited and closed the flow. UDP had the +same fault, less often. Since #1452 the responder discards the request +(`SYSTEM:...!!/dev/null`), and both ingress checks fail on any unanswered +flow. + ## Capture points | Point | Where | diff --git a/deploy/containerlab/resources/tenants/ns60/base/backend.yaml b/deploy/containerlab/resources/tenants/ns60/base/backend.yaml index 8cd1c8bf..84c6ec28 100644 --- a/deploy/containerlab/resources/tenants/ns60/base/backend.yaml +++ b/deploy/containerlab/resources/tenants/ns60/base/backend.yaml @@ -69,13 +69,20 @@ spec: fieldPath: metadata.name # socat sets SOCAT_PEERADDR to the client's address for each # connection or datagram. + # + # SYSTEM:...!!/dev/null sends the reply command's output to the + # client and the client's bytes to /dev/null. With a plain SYSTEM + # address socat writes the client's request to the command, which + # has usually exited by then without reading it. That write fails + # with EPIPE (or the next read with ECONNRESET), and when it comes + # first socat exits and closes the flow without sending the reply. command: - sh - -c - | reply='echo "$POD_NAME $SOCAT_PEERADDR"' - socat TCP6-LISTEN:80,reuseaddr,fork SYSTEM:"$reply" & - exec socat UDP6-RECVFROM:80,reuseaddr,fork SYSTEM:"$reply" + socat TCP6-LISTEN:80,reuseaddr,fork SYSTEM:"$reply"'!!'/dev/null & + exec socat UDP6-RECVFROM:80,reuseaddr,fork SYSTEM:"$reply"'!!'/dev/null --- apiVersion: apps/v1 kind: Deployment @@ -122,10 +129,17 @@ spec: fieldPath: metadata.name # socat sets SOCAT_PEERADDR to the client's address for each # connection or datagram. + # + # SYSTEM:...!!/dev/null sends the reply command's output to the + # client and the client's bytes to /dev/null. With a plain SYSTEM + # address socat writes the client's request to the command, which + # has usually exited by then without reading it. That write fails + # with EPIPE (or the next read with ECONNRESET), and when it comes + # first socat exits and closes the flow without sending the reply. command: - sh - -c - | reply='echo "$POD_NAME $SOCAT_PEERADDR"' - socat TCP6-LISTEN:80,reuseaddr,fork SYSTEM:"$reply" & - exec socat UDP6-RECVFROM:80,reuseaddr,fork SYSTEM:"$reply" + socat TCP6-LISTEN:80,reuseaddr,fork SYSTEM:"$reply"'!!'/dev/null & + exec socat UDP6-RECVFROM:80,reuseaddr,fork SYSTEM:"$reply"'!!'/dev/null diff --git a/deploy/containerlab/scripts/baseline-observe.sh b/deploy/containerlab/scripts/baseline-observe.sh index d3fe9251..a275cb0d 100755 --- a/deploy/containerlab/scripts/baseline-observe.sh +++ b/deploy/containerlab/scripts/baseline-observe.sh @@ -244,11 +244,10 @@ gateway_scenario() { local rc=0 LAB=${LAB} "${SCRIPT_DIR}/${script}" --node dfw-worker2 >"${OUT}/logs/${sc}.log" 2>&1 || rc=$? cap_stop_all - # The lab's ingress checks intermittently lose a TCP flow whose request - # reached a backend: the ns60 responder closes without writing a reply - # (docs/baseline.md). So the exact flow counts and the check's exit status - # are per-run values; the stable result is that each protocol had answered - # flows, which shows delivery in both directions. + # The exact flow counts and the check's exit status are per-run values; the + # stable result is that each protocol had answered flows, which shows + # delivery in both directions. The reference was recorded while the ns60 + # responder still lost flows, before #1452 (docs/baseline.md). volatile "${sc}.check_exit" "${rc}" local proto answered for proto in tcp udp; do diff --git a/deploy/containerlab/scripts/verify-gateway-backend-health.sh b/deploy/containerlab/scripts/verify-gateway-backend-health.sh index 4575cd8b..bd3c8869 100755 --- a/deploy/containerlab/scripts/verify-gateway-backend-health.sh +++ b/deploy/containerlab/scripts/verify-gateway-backend-health.sh @@ -73,10 +73,12 @@ listening() { stop_listener() { kc -n ns60 exec deploy/backend-a -- sh -c 'pkill -f "TCP6-LISTE[N]"' } +# start_listener restarts the same responder resources/tenants/ns60/base/ +# backend.yaml runs, including its '!!'/dev/null (see the comment there). start_listener() { kc -n ns60 exec -i deploy/backend-a -- sh -s <<'EOF' reply='echo "$POD_NAME $SOCAT_PEERADDR"' -setsid socat TCP6-LISTEN:80,reuseaddr,fork SYSTEM:"$reply" /dev/null 2>&1 & +setsid socat TCP6-LISTEN:80,reuseaddr,fork SYSTEM:"$reply"'!!'/dev/null /dev/null 2>&1 & EOF }