From 502ea18f802c06d8e76753dc96289f033f66e40c Mon Sep 17 00:00:00 2001 From: gonzalo-zenrows Date: Fri, 16 Oct 2026 18:59:29 +0200 Subject: [PATCH 1/6] feat(client): retry only uncharged failures, with full-jitter backoff and a time budget A 429, 502, 503 or 504 from the API is never charged and means "try again", so the client now retries those (and network errors before any response) by default, waiting the server's Retry-After or a full-jitter backoff instead of a fixed doubling delay. A 422 is no longer retried: the API no longer sends a retryable 422. A timeout or abort is never retried, because the server may still finish and charge the request. - retries defaults to 2 (was 0); retries: 0 turns retries off - new retryMaxWait (seconds, default 30) caps one backoff wait - new retryBudget (seconds, default 90): no retry starts past it - Retry-After in whole seconds is honoured, capped at 30 s --- README.md | 7 +- src/index.ts | 52 +++--- src/retry.ts | 83 +++++++++ tests/index.test.ts | 8 +- tests/no-data-extracted.test.ts | 4 +- tests/retries.test.ts | 295 ++++++++++++++++++++++++-------- tests/retry-count.test.ts | 25 ++- 7 files changed, 364 insertions(+), 110 deletions(-) create mode 100644 src/retry.ts diff --git a/README.md b/README.md index bcc8991..7282a22 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,12 @@ Start using the API by [creating your API Key](https://app.zenrows.com/register? The SDK uses the official [fetch api](https://nodejs.org/dist/latest-v18.x/docs/api/globals.html) for HTTP requests. The client's response will be a [`Response`](https://nodejs.org/dist/latest-v18.x/docs/api/globals.html#response). -It also uses [fetch-retry](https://github.com/jonbern/fetch-retry) to automatically retry failed requests (network errors and status codes 422, 503 and 504, except Extract's `422 no_data_extracted`). Retries are not active by default; you need to specify the number of retries, as shown below. It already includes an exponential back-off retry delay between failed requests. +It also uses [fetch-retry](https://github.com/jonbern/fetch-retry) to retry failed requests: a network error before any response, and status codes 429, 502, 503 and 504, which are never charged. No other status is retried, `422` included. A timeout or abort is never retried, because the server may still finish the request and charge it. + +- `retries`: retries after the first try. Default `2`; set `retries: 0` to turn retries off. +- Between tries the client waits the `Retry-After` the server sends (at most 30 seconds), or else a random time between 0.25 seconds and `min(retryMaxWait, 2^attempt)` seconds (full jitter). +- `retryMaxWait`: the longest one random wait may be, in seconds. Default `30`. +- `retryBudget`: no retry starts once the time spent plus the next wait would pass this, in seconds. Default `90`. > `client.fetch()` is the primary method for the main page-scraping product. `client.get()` still works and is kept as a deprecated alias — new code should call `fetch()`. diff --git a/src/index.ts b/src/index.ts index 80de054..1a2288e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,12 @@ import fetchRetry from "fetch-retry"; import packageJson from "../package.json" with { type: "json" }; import { ZenRowsBatchClient } from "./batch/client.js"; import { type CrawlClientConfig, ZenRowsCrawlClient } from "./crawl/client.js"; +import { + DEFAULT_RETRIES, + DEFAULT_RETRY_BUDGET_SECONDS, + DEFAULT_RETRY_MAX_WAIT_SECONDS, + retryPolicy, +} from "./retry.js"; export * from "./batch/client.js"; export * from "./batch/errors.js"; @@ -56,7 +62,7 @@ async function isExtractDomainNotReady(response: Response): Promise { /** True when a response is Extract's 422 `no_data_extracted`: Extract knows this kind * of page but found no data on it. The request is not charged and the client never - * retries it, since another try returns the same answer. Reads via `.clone()`, so the + * retries it (no 422 is retried), since another try returns the same answer. Reads via `.clone()`, so the * original response body is left unconsumed for the caller. */ export async function isNoDataExtracted(response: Response): Promise { if (response.status !== 422) return false; @@ -70,7 +76,16 @@ export async function isNoDataExtracted(response: Response): Promise { interface ClientConfig { concurrency?: number; + /** + * Retries for `fetch()`, `extract()` and `post()` after a 429, 502, 503 or 504 (never + * charged) or a network error before any response. Default 2; 0 turns retries off. A timeout + * or abort is never retried: the server may still finish the request and charge it. + */ retries?: number; + /** The longest one backoff wait may be, in seconds. Default 30. A `Retry-After` is honoured up to 30 s. */ + retryMaxWait?: number; + /** No retry starts once the time spent plus the next wait would pass this, in seconds. Default 90. */ + retryBudget?: number; /** Config for `client.crawl`: its own `baseURL`, `retries` and `timeout`. */ crawl?: CrawlClientConfig; } @@ -114,31 +129,11 @@ export class ZenRows { this.clientConfig = clientConfig; this.batch = new ZenRowsBatchClient(apiKey); this.crawl = new ZenRowsCrawlClient(apiKey, clientConfig.crawl); - const retries = this.clientConfig.retries ?? 0; this.queue = fastq.promise(this, this.worker, this.clientConfig.concurrency ?? 5); - this.fetchWithRetry = fetchRetry(fetch, { - retryDelay: (attempt) => 2 ** attempt * 1000, - // retryOn: [422, 503, 504], - retryOn: async (attempt, error, response) => { - if (attempt >= retries) { - return false; - } - - if (error !== null || response?.status === 503 || response?.status === 504) { - return true; - } - - // A 422 is usually a failure to get the target's content and worth another - // try, except Extract's `no_data_extracted`, which is a final answer. - if (response?.status === 422) { - return !(await isNoDataExtracted(response)); - } - - return false; - }, - }); + // The retry policy is per request (see `worker`), so this only wraps fetch. + this.fetchWithRetry = fetchRetry(fetch); } /** @@ -292,7 +287,16 @@ export class ZenRows { } } - const response = await this.fetchWithRetry(`${API_URL}?${params.toString()}`, fetchOptions); + const { retryOn, retryDelay } = retryPolicy({ + retries: this.clientConfig.retries ?? DEFAULT_RETRIES, + retryMaxWait: this.clientConfig.retryMaxWait ?? DEFAULT_RETRY_MAX_WAIT_SECONDS, + retryBudget: this.clientConfig.retryBudget ?? DEFAULT_RETRY_BUDGET_SECONDS, + }); + const response = await this.fetchWithRetry(`${API_URL}?${params.toString()}`, { + ...fetchOptions, + retryOn, + retryDelay, + }); return response; } diff --git a/src/retry.ts b/src/retry.ts new file mode 100644 index 0000000..9cb738a --- /dev/null +++ b/src/retry.ts @@ -0,0 +1,83 @@ +// Retry policy for `fetch()`, `extract()` and `post()`. The same policy ships in every ZenRows +// client: retry only answers that say "try again" and are never charged, back off with full +// jitter, and give up once a total time budget is spent. + +/** Statuses retried: rate limited, or the gateway or its upstream could not answer. Never charged. */ +export const RETRYABLE_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]); + +export const DEFAULT_RETRIES = 2; +/** The longest one backoff wait may be, in seconds. */ +export const DEFAULT_RETRY_MAX_WAIT_SECONDS = 30; +/** No retry starts once the request plus the next wait would pass this, in seconds. */ +export const DEFAULT_RETRY_BUDGET_SECONDS = 90; + +const BACKOFF_MIN_MS = 250; +const BACKOFF_BASE_MS = 1000; +// A Retry-After above this waits only this long, so a large server value cannot hang a call. +const RETRY_AFTER_CAP_MS = 30_000; + +export interface RetryPolicyOptions { + /** Retries after the first try. 0 turns retries off. */ + retries: number; + /** The longest one backoff wait may be, in seconds. */ + retryMaxWait: number; + /** Total time a request and its retries may take before no retry starts, in seconds. */ + retryBudget: number; + /** Clock in milliseconds; tests replace it. */ + now?: () => number; + /** Uniform random number in [0, 1); tests replace it. */ + random?: () => number; +} + +/** Milliseconds from a `Retry-After` in whole seconds, capped. Undefined otherwise (an HTTP date too). */ +function retryAfterMs(response: Response | null): number | undefined { + const raw = response?.headers.get("Retry-After")?.trim(); + if (!raw || !/^\d+$/.test(raw)) return undefined; + return Math.min(Number(raw) * 1000, RETRY_AFTER_CAP_MS); +} + +/** Full jitter for retry `attempt` (0-based): uniform(0.25 s, min(maxWait, 1 s * 2^attempt)). */ +function backoffMs(attempt: number, maxWaitMs: number, random: () => number): number { + const ceiling = Math.max(BACKOFF_MIN_MS, Math.min(maxWaitMs, BACKOFF_BASE_MS * 2 ** attempt)); + return BACKOFF_MIN_MS + random() * (ceiling - BACKOFF_MIN_MS); +} + +/** A timeout or a caller's abort: the server may still be running the request and charge it. */ +function isAbort(error: Error): boolean { + return error.name === "AbortError" || error.name === "TimeoutError"; +} + +/** + * The `retryOn` and `retryDelay` pair for one request through fetch-retry. Build one per + * request: it remembers when the request started and the wait it chose in `retryOn`, which + * fetch-retry then reads through `retryDelay`. + */ +export function retryPolicy(options: RetryPolicyOptions) { + const now = options.now ?? Date.now; + const random = options.random ?? Math.random; + const maxWaitMs = options.retryMaxWait * 1000; + const budgetMs = options.retryBudget * 1000; + const startedAt = now(); + let nextDelayMs = 0; + + return { + retryOn(attempt: number, error: Error | null, response: Response | null): boolean { + if (attempt >= options.retries) return false; + if (error) { + if (isAbort(error)) return false; + } else if (!response || !RETRYABLE_STATUSES.has(response.status)) { + return false; + } + + const delay = retryAfterMs(response) ?? backoffMs(attempt, maxWaitMs, random); + if (now() - startedAt + delay > budgetMs) return false; + nextDelayMs = delay; + // The retried answer is dropped; free its connection instead of waiting for GC. + response?.body?.cancel().catch(() => {}); + return true; + }, + retryDelay(): number { + return nextDelayMs; + }, + }; +} diff --git a/tests/index.test.ts b/tests/index.test.ts index c76d83b..d4cd397 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -25,12 +25,12 @@ describe("ZenRows Client Get", () => { expect(clientSpy).toHaveBeenCalledWith( `${apiUrl}?url=${encodeURIComponent(url)}&apikey=${apiKey}`, - { + expect.objectContaining({ method: "GET", headers: { "User-Agent": `zenrows/${packageJson.version} node`, }, - }, + }), ); expect(response.status).toBe(200); }); @@ -64,12 +64,12 @@ describe("ZenRows Client Get", () => { expect(clientSpy).toHaveBeenCalledWith( `${apiUrl}?url=${encodeURIComponent(url)}&apikey=${apiKey}&custom_headers=true`, - { + expect.objectContaining({ method: "GET", headers: { "User-Agent": "test", }, - }, + }), ); }); diff --git a/tests/no-data-extracted.test.ts b/tests/no-data-extracted.test.ts index c5ecde8..8070f85 100644 --- a/tests/no-data-extracted.test.ts +++ b/tests/no-data-extracted.test.ts @@ -23,7 +23,7 @@ describe("422 no_data_extracted", () => { expect(await response.json()).toEqual(noData); }); - test("any other 422 is still retried", async () => { + test("any other 422 is not retried either", async () => { let fetchCount = 0; server.use( http.get("https://api.zenrows.com/v1/", () => { @@ -36,7 +36,7 @@ describe("422 no_data_extracted", () => { const response = await client.fetch("https://example.com"); expect(response.status).toBe(422); - expect(fetchCount).toBe(2); + expect(fetchCount).toBe(1); }); test("isNoDataExtracted matches only a 422 with that code, in any case", async () => { diff --git a/tests/retries.test.ts b/tests/retries.test.ts index 0f3782d..d739c1b 100644 --- a/tests/retries.test.ts +++ b/tests/retries.test.ts @@ -1,82 +1,233 @@ import { http, HttpResponse } from "msw"; -import { type Mock, beforeEach, describe, expect, test, vi } from "vitest"; //TODO(Nestor): Try to use globals instead of importing +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { ZenRows } from "../src"; +import { retryPolicy } from "../src/retry"; import { server } from "./_setup"; -describe("ZenRows Client with Retries", () => { - const apiKey = "API_KEY"; - const url = "https://example.com"; - const apiUrl = "https://api.zenrows.com/v1/"; - let client: ZenRows; +const apiUrl = "https://api.zenrows.com/v1/"; +const url = "https://example.com"; - beforeEach(() => { - client = new ZenRows(apiKey, { retries: 2 }); - }); - - test( - "should call fetch 3 times (initial plus 2 retries)", - { timeout: Number.POSITIVE_INFINITY }, - async () => { - const clientSpy = vi.spyOn(client, "fetchWithRetry"); - server.use( - http.get("https://api.zenrows.com/v1/", () => { - return HttpResponse.error(); - }), - ); - - try { - await client.get(url); - } catch (error) { - return; - } - - expect(clientSpy).toHaveBeenCalledTimes(3); - }, - ); +/** A policy on a fake clock, so budget checks can be driven by hand. */ +function policyAt( + clock: { ms: number }, + overrides: Partial[0]> = {}, +) { + return retryPolicy({ + retries: 10, + retryMaxWait: 30, + retryBudget: 90, + now: () => clock.ms, + random: () => 0.5, + ...overrides, + }); +} - test( - "should throw error with status code 500", - { timeout: Number.POSITIVE_INFINITY }, - async () => { - const clientSpy = vi.spyOn(client, "get"); - server.use( - http.get("https://api.zenrows.com/v1/", () => { - return new HttpResponse(null, { status: 503 }); - }), - ); - - try { - const response = await client.get(url); - expect(clientSpy).toHaveBeenCalledTimes(3); - expect(response.status).toBe(503); - } catch (error) { - return; - } - }, - ); +const status = (code: number, headers: Record = {}) => + new Response(null, { status: code, headers }); + +describe("retryPolicy", () => { + test("retries 429, 502, 503 and 504", () => { + for (const code of [429, 502, 503, 504]) { + expect(policyAt({ ms: 0 }).retryOn(0, null, status(code))).toBe(true); + } + }); + + test("never retries other statuses, 422 included", () => { + for (const code of [200, 400, 401, 402, 403, 404, 409, 422, 500]) { + expect(policyAt({ ms: 0 }).retryOn(0, null, status(code))).toBe(false); + } + }); + + test("retries a network error, never an abort or a timeout", () => { + expect(policyAt({ ms: 0 }).retryOn(0, new TypeError("fetch failed"), null)).toBe(true); + const abort = new DOMException("The operation was aborted.", "AbortError"); + const timeout = new DOMException("The operation timed out.", "TimeoutError"); + expect(policyAt({ ms: 0 }).retryOn(0, abort, null)).toBe(false); + expect(policyAt({ ms: 0 }).retryOn(0, timeout, null)).toBe(false); + }); + + test("stops after `retries` retries", () => { + const policy = policyAt({ ms: 0 }, { retries: 2 }); + expect(policy.retryOn(0, null, status(503))).toBe(true); + expect(policy.retryOn(1, null, status(503))).toBe(true); + expect(policy.retryOn(2, null, status(503))).toBe(false); + }); - test( - "should fail the first attempt and succeed the second", - { timeout: Number.POSITIVE_INFINITY }, - async () => { - server.use( - http.get( - "https://api.zenrows.com/v1/", - () => { - return new HttpResponse(null, { status: 503 }); - }, - { - once: true, - }, - ), - ); - - try { - const response = await client.get(url); - expect(response).toBe(200); - } catch (error) { - return; - } + test("full jitter stays within [0.25 s, min(retryMaxWait, 1 s * 2^attempt)]", () => { + const ceilings = [1000, 2000, 4000, 8000, 16000, 30000, 30000]; + ceilings.forEach((ceiling, attempt) => { + const low = policyAt({ ms: 0 }, { random: () => 0 }); + const high = policyAt({ ms: 0 }, { random: () => 0.999999 }); + low.retryOn(attempt, null, status(503)); + high.retryOn(attempt, null, status(503)); + expect(low.retryDelay()).toBe(250); + expect(high.retryDelay()).toBeGreaterThan(ceiling - 1); + expect(high.retryDelay()).toBeLessThanOrEqual(ceiling); + }); + }); + + test("retryMaxWait caps the jitter ceiling", () => { + const policy = policyAt({ ms: 0 }, { retryMaxWait: 2, random: () => 0.999999 }); + policy.retryOn(5, null, status(503)); + expect(policy.retryDelay()).toBeLessThanOrEqual(2000); + }); + + test("honours a Retry-After in seconds, capped at 30 s", () => { + const short = policyAt({ ms: 0 }); + short.retryOn(0, null, status(503, { "Retry-After": "7" })); + expect(short.retryDelay()).toBe(7000); + + const long = policyAt({ ms: 0 }); + long.retryOn(0, null, status(429, { "Retry-After": "3600" })); + expect(long.retryDelay()).toBe(30_000); + }); + + test.each(["Wed, 21 Oct 2015 07:28:00 GMT", "1.5", "-3", "soon"])( + "falls back to jitter when Retry-After is %s", + (value) => { + const policy = policyAt({ ms: 0 }, { random: () => 0 }); + policy.retryOn(0, null, status(503, { "Retry-After": value })); + expect(policy.retryDelay()).toBe(250); }, ); + + test("starts no retry that would pass the budget", () => { + const clock = { ms: 0 }; + const policy = policyAt(clock, { retryBudget: 10 }); + clock.ms = 3000; + expect(policy.retryOn(0, null, status(503, { "Retry-After": "7" }))).toBe(true); + clock.ms = 3001; + expect(policy.retryOn(1, null, status(503, { "Retry-After": "7" }))).toBe(false); + }); +}); + +describe("ZenRows client retries", () => { + beforeEach(() => { + vi.useFakeTimers({ toFake: ["setTimeout", "Date"] }); + vi.spyOn(Math, "random").mockReturnValue(0); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + test("a 503 is retried after a jittered wait, then succeeds", async () => { + let calls = 0; + server.use( + http.get(apiUrl, () => { + calls += 1; + return calls === 1 ? new HttpResponse(null, { status: 503 }) : HttpResponse.json({ ok: 1 }); + }), + ); + + const pending = new ZenRows("API_KEY").fetch(url); + await vi.advanceTimersByTimeAsync(249); + expect(calls).toBe(1); + await vi.advanceTimersByTimeAsync(1); + const response = await pending; + + expect(response.status).toBe(200); + expect(calls).toBe(2); + }); + + test("a network error is retried", async () => { + let calls = 0; + server.use( + http.get(apiUrl, () => { + calls += 1; + return calls === 1 ? HttpResponse.error() : HttpResponse.json({ ok: 1 }); + }), + ); + + const pending = new ZenRows("API_KEY").fetch(url); + await vi.advanceTimersByTimeAsync(250); + + expect((await pending).status).toBe(200); + expect(calls).toBe(2); + }); + + test("waits the Retry-After the server sends", async () => { + let calls = 0; + server.use( + http.get(apiUrl, () => { + calls += 1; + return calls === 1 + ? new HttpResponse(null, { status: 429, headers: { "Retry-After": "5" } }) + : HttpResponse.json({ ok: 1 }); + }), + ); + + const pending = new ZenRows("API_KEY").fetch(url); + await vi.advanceTimersByTimeAsync(4999); + expect(calls).toBe(1); + await vi.advanceTimersByTimeAsync(1); + + expect((await pending).status).toBe(200); + expect(calls).toBe(2); + }); + + test("returns the last answer once the budget would be passed", async () => { + let calls = 0; + server.use( + http.get(apiUrl, () => { + calls += 1; + return new HttpResponse(null, { status: 503, headers: { "Retry-After": "20" } }); + }), + ); + + const pending = new ZenRows("API_KEY", { retries: 5, retryBudget: 45 }).fetch(url); + await vi.advanceTimersByTimeAsync(60_000); + const response = await pending; + + // Waits 20 s, then 20 s more (40 s spent); a third wait would end at 60 s, past 45 s. + expect(response.status).toBe(503); + expect(calls).toBe(3); + }); + + test.each([400, 422])("a %i is not retried", async (code) => { + let calls = 0; + server.use( + http.get(apiUrl, () => { + calls += 1; + return HttpResponse.json({ code: "RESP001" }, { status: code }); + }), + ); + + const response = await new ZenRows("API_KEY", { retries: 3 }).fetch(url); + + expect(response.status).toBe(code); + expect(calls).toBe(1); + }); + + test("an abort is not retried", async () => { + const fetchSpy = vi.fn(async () => { + throw new DOMException("The operation was aborted.", "AbortError"); + }); + vi.stubGlobal("fetch", fetchSpy); + + await expect(new ZenRows("API_KEY", { retries: 3 }).fetch(url)).rejects.toMatchObject({ + name: "AbortError", + }); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + + test("the extract() autoparse fallback is unchanged: one fallback request, no retries", async () => { + const urls: URL[] = []; + server.use( + http.get(apiUrl, ({ request }) => { + urls.push(new URL(request.url)); + return urls.length === 1 + ? HttpResponse.json({ code: "REQS007" }, { status: 403 }) + : HttpResponse.json([{ found: "via autoparse" }]); + }), + ); + + const response = await new ZenRows("API_KEY").extract(url); + + expect(response.status).toBe(200); + expect(urls).toHaveLength(2); + expect(urls[1]?.searchParams.get("autoparse")).toBe("true"); + }); }); diff --git a/tests/retry-count.test.ts b/tests/retry-count.test.ts index 61fc964..51b0227 100644 --- a/tests/retry-count.test.ts +++ b/tests/retry-count.test.ts @@ -1,29 +1,40 @@ import { http, HttpResponse } from "msw"; -import { describe, expect, test } from "vitest"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { ZenRows } from "../src"; import { server } from "./_setup"; -describe("retryOn off-by-one fix", () => { +describe("retry count", () => { + beforeEach(() => { + vi.useFakeTimers({ toFake: ["setTimeout", "Date"] }); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + test.each([ + { retries: undefined, expectedCalls: 3 }, { retries: 0, expectedCalls: 1 }, { retries: 1, expectedCalls: 2 }, { retries: 2, expectedCalls: 3 }, ])( - "retries: $retries should make exactly $expectedCalls fetch call(s) on 422", + "retries: $retries makes exactly $expectedCalls call(s) on a 503", async ({ retries, expectedCalls }) => { let fetchCount = 0; server.use( http.get("https://api.zenrows.com/v1/", () => { fetchCount++; - return new HttpResponse("Could Not Get Content", { status: 422 }); + return new HttpResponse("Service Unavailable", { status: 503 }); }), ); - const client = new ZenRows("API_KEY", { retries }); - const response = await client.get("https://example.com"); + const client = new ZenRows("API_KEY", retries === undefined ? {} : { retries }); + const pending = client.get("https://example.com"); + await vi.advanceTimersByTimeAsync(10_000); + const response = await pending; - expect(response.status).toBe(422); + expect(response.status).toBe(503); expect(fetchCount).toBe(expectedCalls); }, ); From 25a5a0800531bcc27d27299f3e53f00e68d448bd Mon Sep 17 00:00:00 2001 From: gonzalo-zenrows Date: Mon, 19 Oct 2026 22:44:53 +0200 Subject: [PATCH 2/6] fix(client): retry only requests that are safe to send again The first version of this change retried requests that the API may already have run and charged, and it dropped a retry that callers relied on. 1. A failed fetch is retried only when error.cause.code shows the request never reached the API: connection refused, DNS failure, unreachable network or host, or a connect timeout. A reset or close after the request was sent, a header timeout, an abort and an invalid header value are not. 2. post() is retried only when its headers carry an Idempotency-Key and its body can be sent again. The API forwards the body to the target, so a blind replay can make the target act twice. Without a key it is sent once. 3. A 422 is retried again, except no_data_extracted. A 500 stays final. 4. Retry-After must be whole seconds and is clamped to 0.25 s to 30 s, so a value of 0 waits 250 ms. The 30 s cap does not depend on retryMaxWait. 5. The extract() fallback to autoparse shares the first request's budget. 6. The constructor rejects an invalid retries, retryMaxWait or retryBudget with a RangeError instead of running with NaN or a negative value. Tests run the failure cases against a real HTTP server through Node's fetch, check the shared conformance vectors, and cover body cancellation, the Retry-After cap and post() with and without an Idempotency-Key. --- README.md | 44 +++- src/index.ts | 55 +++-- src/retry.ts | 145 +++++++++++-- tests/no-data-extracted.test.ts | 4 +- tests/retries.test.ts | 358 +++++++++++++++++++++++++------- tests/retry-network.test.ts | 217 +++++++++++++++++++ 6 files changed, 696 insertions(+), 127 deletions(-) create mode 100644 tests/retry-network.test.ts diff --git a/README.md b/README.md index 7282a22..b229f7f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ SDK to access [Zenrows](https://www.zenrows.com/) API directly from Node.js. Zen - [Installation](#installation) - [Usage](#usage) + - [Retries](#retries) - [Adaptive Stealth Mode](#adaptive-stealth-mode) - [POST Requests](#post-requests) - [Extract](#extract) @@ -45,12 +46,7 @@ Start using the API by [creating your API Key](https://app.zenrows.com/register? The SDK uses the official [fetch api](https://nodejs.org/dist/latest-v18.x/docs/api/globals.html) for HTTP requests. The client's response will be a [`Response`](https://nodejs.org/dist/latest-v18.x/docs/api/globals.html#response). -It also uses [fetch-retry](https://github.com/jonbern/fetch-retry) to retry failed requests: a network error before any response, and status codes 429, 502, 503 and 504, which are never charged. No other status is retried, `422` included. A timeout or abort is never retried, because the server may still finish the request and charge it. - -- `retries`: retries after the first try. Default `2`; set `retries: 0` to turn retries off. -- Between tries the client waits the `Retry-After` the server sends (at most 30 seconds), or else a random time between 0.25 seconds and `min(retryMaxWait, 2^attempt)` seconds (full jitter). -- `retryMaxWait`: the longest one random wait may be, in seconds. Default `30`. -- `retryBudget`: no retry starts once the time spent plus the next wait would pass this, in seconds. Default `90`. +The client retries failed requests by default. See [Retries](#retries) for what it retries and how long it waits. > `client.fetch()` is the primary method for the main page-scraping product. `client.get()` still works and is kept as a deprecated alias — new code should call `fetch()`. @@ -132,6 +128,40 @@ You can also pass optional parameters and headers; the list above is a reference Sending headers to the target URL will overwrite our defaults. Be careful when doing it and contact us if there is any problem. +### Retries + +`fetch()`, `extract()` and `post()` retry failed requests through [fetch-retry](https://github.com/jonbern/fetch-retry). `client.batch` and `client.crawl` keep their own retry rules, described in their sections. + +What the client retries: + +1. Status codes 429, 502, 503 and 504. The API does not charge for them. +2. A `422`, except one with `code: "no_data_extracted"`. That one is a final answer (see [Extract](#extract)). +3. A failure before the request reached the API: connection refused, a name that does not resolve, an unreachable network or host, or a connect timeout. + +What the client never retries: + +1. Any other status, `500` included. +2. A connection reset or close after the request was sent, a response timeout, or an abort. The API may still finish that request and charge it. +3. An error in the request itself, such as an invalid header value. + +Which calls the client retries: + +1. `fetch()`, `get()` and `extract()`: always. +2. `post()`: only when its `headers` carry an `Idempotency-Key`. Without one, the client sends a `post()` once, because the API forwards the body to the target, and the target may act on it twice. The client sends the key to the target as a custom header, like any other header you pass. + +How long the client waits between tries: + +1. When the API sends `Retry-After` in whole seconds, the client waits that long, at least 0.25 seconds and at most 30 seconds. `retryMaxWait` does not change this cap. The client ignores any other `Retry-After` format, such as an HTTP date. +2. Otherwise it waits a random time between 0.25 seconds and `max(0.25, min(retryMaxWait, 2^attempt))` seconds (full jitter). `attempt` counts from 0, so the first retry waits 0.25 to 1 second, the second 0.25 to 2 seconds. + +Options, checked when you build the client (an invalid value throws a `RangeError`): + +1. `retries`: retries after the first try, an integer of 0 or more. Default `2`. Set `retries: 0` to send every request once. +2. `retryMaxWait`: the longest one random wait may be, in seconds, greater than 0. Default `30`. +3. `retryBudget`: in seconds, greater than 0. Default `90`: no retry starts after 90 s. The client counts from the start of the first try, and includes the wait before the retry. The budget does not cut a running try short, so a call can take longer than the budget by one try. The `extract()` fallback to `autoparse` shares the first request's budget. + +The client takes no `AbortSignal`, so you cannot cancel a call while it waits between tries. To bound a call's time, lower `retries` or `retryBudget`. + ### Adaptive Stealth Mode Set `mode: "auto"` to let Zenrows pick the request configuration for you — it starts with the cheapest viable setup and escalates to `js_render`/`premium_proxy` only when the target needs it, billing only for the configuration that succeeds. @@ -146,6 +176,8 @@ Compatible with `proxy_country`, `js_instructions`, and `custom_headers`. The SDK also offers POST requests by calling the `client.post` method. It can receive a new parameter `data` that represents the data sent in, for example, a form. +The client sends a `post()` once, unless you pass an `Idempotency-Key` header. See [Retries](#retries). + ```javascript const { ZenRows } = require("zenrows"); diff --git a/src/index.ts b/src/index.ts index 1a2288e..ae575d8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,9 @@ import { DEFAULT_RETRIES, DEFAULT_RETRY_BUDGET_SECONDS, DEFAULT_RETRY_MAX_WAIT_SECONDS, + isRetryEligible, retryPolicy, + validateRetryOptions, } from "./retry.js"; export * from "./batch/client.js"; @@ -20,6 +22,7 @@ export type * from "./batch/types.js"; export * from "./crawl/client.js"; export { CRAWL_NOT_ENABLED_CODE, ZenRowsCrawlError } from "./crawl/errors.js"; export type * from "./crawl/types.js"; +export { isNoDataExtracted } from "./retry.js"; const API_URL = "https://api.zenrows.com/v1/"; @@ -60,31 +63,19 @@ async function isExtractDomainNotReady(response: Response): Promise { } } -/** True when a response is Extract's 422 `no_data_extracted`: Extract knows this kind - * of page but found no data on it. The request is not charged and the client never - * retries it (no 422 is retried), since another try returns the same answer. Reads via `.clone()`, so the - * original response body is left unconsumed for the caller. */ -export async function isNoDataExtracted(response: Response): Promise { - if (response.status !== 422) return false; - try { - const body = (await response.clone().json()) as { code?: unknown }; - return typeof body?.code === "string" && body.code.toLowerCase() === "no_data_extracted"; - } catch { - return false; - } -} - interface ClientConfig { concurrency?: number; /** - * Retries for `fetch()`, `extract()` and `post()` after a 429, 502, 503 or 504 (never - * charged) or a network error before any response. Default 2; 0 turns retries off. A timeout - * or abort is never retried: the server may still finish the request and charge it. + * Retries for `fetch()`, `extract()` and `post()` after a 429, 502, 503 or 504, a 422 other + * than `no_data_extracted`, or a failure before the request reached the server. An integer of + * 0 or more; default 2; 0 turns retries off. A reset after the request was sent, a timeout or + * an abort is never retried: the server may still finish the request and charge it. `post()` + * is retried only when its headers carry an `Idempotency-Key`. */ retries?: number; - /** The longest one backoff wait may be, in seconds. Default 30. A `Retry-After` is honoured up to 30 s. */ + /** The longest one random wait may be, in seconds (greater than 0). Default 30. It never caps a `Retry-After`. */ retryMaxWait?: number; - /** No retry starts once the time spent plus the next wait would pass this, in seconds. Default 90. */ + /** No retry starts after this many seconds since the first try started (greater than 0). Default 90. */ retryBudget?: number; /** Config for `client.crawl`: its own `baseURL`, `retries` and `timeout`. */ crawl?: CrawlClientConfig; @@ -114,6 +105,11 @@ interface Headers { [x: string]: string; } +/** When a logical call's first try started. Shared by the requests of one call (see `extract()`). */ +interface CallClock { + startedAt?: number; +} + export class ZenRows { readonly apiKey: string; readonly clientConfig: ClientConfig; @@ -125,6 +121,7 @@ export class ZenRows { readonly crawl: ZenRowsCrawlClient; constructor(apiKey: string, clientConfig: ClientConfig = {}) { + validateRetryOptions(clientConfig); this.apiKey = apiKey; this.clientConfig = clientConfig; this.batch = new ZenRowsBatchClient(apiKey); @@ -196,10 +193,18 @@ export class ZenRows { ...rest } = config ?? {}; const mode = extractWireMode(rest, adaptiveStealth); - const response = await this.fetch(url, { ...rest, extract, mode }, opts); + const { headers = {} } = opts; + // One retry budget for the whole call: the fallback reuses the first request's start time. + const clock: CallClock = {}; + const response = await this.queue.push({ + url, + config: { ...rest, extract, mode }, + headers, + clock, + }); if (extract === "auto" && fallbackToAutoparse && (await isExtractDomainNotReady(response))) { - return this.fetch(url, { ...rest, autoparse: true, mode }, opts); + return this.queue.push({ url, config: { ...rest, autoparse: true, mode }, headers, clock }); } return response; @@ -245,12 +250,14 @@ export class ZenRows { config, headers, data, + clock = {}, }: { url: string; method?: HttpMethods; config?: ZenRowsConfig; headers: Headers; data?: unknown; + clock?: CallClock; }): Promise { const params = new URLSearchParams({ url, @@ -287,10 +294,14 @@ export class ZenRows { } } + // A request that is not safe to send twice (a POST without an Idempotency-Key) is sent once. + const eligible = isRetryEligible(method, finalHeaders, fetchOptions.body); + clock.startedAt ??= Date.now(); const { retryOn, retryDelay } = retryPolicy({ - retries: this.clientConfig.retries ?? DEFAULT_RETRIES, + retries: eligible ? this.clientConfig.retries ?? DEFAULT_RETRIES : 0, retryMaxWait: this.clientConfig.retryMaxWait ?? DEFAULT_RETRY_MAX_WAIT_SECONDS, retryBudget: this.clientConfig.retryBudget ?? DEFAULT_RETRY_BUDGET_SECONDS, + startedAt: clock.startedAt, }); const response = await this.fetchWithRetry(`${API_URL}?${params.toString()}`, { ...fetchOptions, diff --git a/src/retry.ts b/src/retry.ts index 9cb738a..92199af 100644 --- a/src/retry.ts +++ b/src/retry.ts @@ -1,55 +1,152 @@ // Retry policy for `fetch()`, `extract()` and `post()`. The same policy ships in every ZenRows -// client: retry only answers that say "try again" and are never charged, back off with full -// jitter, and give up once a total time budget is spent. +// client: retry only answers that say "try again" and are never charged, retry only requests +// that are safe to send twice, back off with full jitter, and start no retry after the budget. /** Statuses retried: rate limited, or the gateway or its upstream could not answer. Never charged. */ export const RETRYABLE_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]); +/** + * `error.cause.code` values that mean the request never reached the server: the connection was + * refused, the name did not resolve, the network or host was unreachable, or the connect timed out. + * Anything else (a reset or close after the request was written, a header or body timeout, an + * abort, a bad header value) is not retried: the server may have run the request already. + */ +const PRE_SEND_ERROR_CODES: ReadonlySet = new Set([ + "ECONNREFUSED", + "ENOTFOUND", + "EAI_AGAIN", + "ENETUNREACH", + "EHOSTUNREACH", + "UND_ERR_CONNECT_TIMEOUT", +]); + +/** Methods that are always safe to send again. Others need an `Idempotency-Key` header. */ +const IDEMPOTENT_METHODS: ReadonlySet = new Set(["GET", "HEAD", "OPTIONS"]); + export const DEFAULT_RETRIES = 2; /** The longest one backoff wait may be, in seconds. */ export const DEFAULT_RETRY_MAX_WAIT_SECONDS = 30; -/** No retry starts once the request plus the next wait would pass this, in seconds. */ +/** No retry starts once the time spent plus the next wait would pass this, in seconds. */ export const DEFAULT_RETRY_BUDGET_SECONDS = 90; const BACKOFF_MIN_MS = 250; const BACKOFF_BASE_MS = 1000; -// A Retry-After above this waits only this long, so a large server value cannot hang a call. -const RETRY_AFTER_CAP_MS = 30_000; +// A Retry-After is clamped to this range. The cap is fixed and does not depend on retryMaxWait. +const RETRY_AFTER_MIN_MS = 250; +const RETRY_AFTER_MAX_MS = 30_000; export interface RetryPolicyOptions { /** Retries after the first try. 0 turns retries off. */ retries: number; - /** The longest one backoff wait may be, in seconds. */ + /** The longest one jitter wait may be, in seconds. It never caps a `Retry-After`. */ retryMaxWait: number; - /** Total time a request and its retries may take before no retry starts, in seconds. */ + /** No retry starts once the time spent plus the next wait would pass this, in seconds. */ retryBudget: number; + /** + * When the logical call's first try started, in `now()` milliseconds. A fallback request that + * belongs to the same call passes the first request's start, so it gets no fresh budget. + */ + startedAt?: number; /** Clock in milliseconds; tests replace it. */ now?: () => number; /** Uniform random number in [0, 1); tests replace it. */ random?: () => number; } -/** Milliseconds from a `Retry-After` in whole seconds, capped. Undefined otherwise (an HTTP date too). */ -function retryAfterMs(response: Response | null): number | undefined { +/** Throws a `RangeError` for a retry option the policy cannot honour, instead of clamping it. */ +export function validateRetryOptions(options: { + retries?: unknown; + retryMaxWait?: unknown; + retryBudget?: unknown; +}): void { + const { retries, retryMaxWait, retryBudget } = options; + if (retries !== undefined && !(Number.isInteger(retries) && (retries as number) >= 0)) { + throw new RangeError(`retries must be an integer of 0 or more, got ${String(retries)}`); + } + for (const [name, value] of [ + ["retryMaxWait", retryMaxWait], + ["retryBudget", retryBudget], + ] as const) { + if (value !== undefined && !(Number.isFinite(value) && (value as number) > 0)) { + throw new RangeError( + `${name} must be a finite number of seconds greater than 0, got ${String(value)}`, + ); + } + } +} + +/** True when a response is Extract's 422 `no_data_extracted`: Extract knows this kind + * of page but found no data on it. The request is not charged and the client never + * retries it, since another try returns the same answer. Reads via `.clone()`, so the + * original response body is left unconsumed for the caller. */ +export async function isNoDataExtracted(response: Response): Promise { + if (response.status !== 422) return false; + try { + const body = (await response.clone().json()) as { code?: unknown }; + return typeof body?.code === "string" && body.code.toLowerCase() === "no_data_extracted"; + } catch { + return false; + } +} + +/** True when the body can be sent again unchanged. A one-shot stream cannot. */ +function isReplayableBody(body: unknown): boolean { + return ( + body === undefined || + body === null || + typeof body === "string" || + body instanceof ArrayBuffer || + ArrayBuffer.isView(body) || + body instanceof URLSearchParams || + body instanceof Blob + ); +} + +/** + * True when a request is safe to send more than once. GET, HEAD and OPTIONS always are. Any + * other method is only with an `Idempotency-Key` header and a body that can be sent again. + */ +export function isRetryEligible( + method: string, + headers: Record, + body: unknown, +): boolean { + if (IDEMPOTENT_METHODS.has(method.toUpperCase())) return true; + const hasIdempotencyKey = Object.entries(headers).some( + ([name, value]) => name.toLowerCase() === "idempotency-key" && value.trim() !== "", + ); + return hasIdempotencyKey && isReplayableBody(body); +} + +/** True when fetch failed before the request was written to the server (see `PRE_SEND_ERROR_CODES`). */ +function isPreSendFailure(error: unknown): boolean { + const code = (error as { cause?: { code?: unknown } } | null)?.cause?.code; + return typeof code === "string" && PRE_SEND_ERROR_CODES.has(code); +} + +/** True when the answer says "try again": 429, 502, 503, 504, or a 422 other than `no_data_extracted`. */ +async function isRetryableResponse(response: Response): Promise { + if (RETRYABLE_STATUSES.has(response.status)) return true; + if (response.status === 422) return !(await isNoDataExtracted(response)); + return false; +} + +/** Milliseconds from a `Retry-After` in whole seconds, clamped to [0.25 s, 30 s]. Undefined otherwise (an HTTP date too). */ +export function retryAfterMs(response: Response | null): number | undefined { const raw = response?.headers.get("Retry-After")?.trim(); if (!raw || !/^\d+$/.test(raw)) return undefined; - return Math.min(Number(raw) * 1000, RETRY_AFTER_CAP_MS); + return Math.min(Math.max(Number(raw) * 1000, RETRY_AFTER_MIN_MS), RETRY_AFTER_MAX_MS); } -/** Full jitter for retry `attempt` (0-based): uniform(0.25 s, min(maxWait, 1 s * 2^attempt)). */ -function backoffMs(attempt: number, maxWaitMs: number, random: () => number): number { +/** Full jitter for retry `attempt` (0-based): uniform(0.25 s, max(0.25 s, min(maxWait, 1 s * 2^attempt))). */ +export function backoffMs(attempt: number, maxWaitMs: number, random: () => number): number { const ceiling = Math.max(BACKOFF_MIN_MS, Math.min(maxWaitMs, BACKOFF_BASE_MS * 2 ** attempt)); return BACKOFF_MIN_MS + random() * (ceiling - BACKOFF_MIN_MS); } -/** A timeout or a caller's abort: the server may still be running the request and charge it. */ -function isAbort(error: Error): boolean { - return error.name === "AbortError" || error.name === "TimeoutError"; -} - /** * The `retryOn` and `retryDelay` pair for one request through fetch-retry. Build one per - * request: it remembers when the request started and the wait it chose in `retryOn`, which + * request: it remembers when the call started and the wait it chose in `retryOn`, which * fetch-retry then reads through `retryDelay`. */ export function retryPolicy(options: RetryPolicyOptions) { @@ -57,15 +154,19 @@ export function retryPolicy(options: RetryPolicyOptions) { const random = options.random ?? Math.random; const maxWaitMs = options.retryMaxWait * 1000; const budgetMs = options.retryBudget * 1000; - const startedAt = now(); + const startedAt = options.startedAt ?? now(); let nextDelayMs = 0; return { - retryOn(attempt: number, error: Error | null, response: Response | null): boolean { + async retryOn( + attempt: number, + error: Error | null, + response: Response | null, + ): Promise { if (attempt >= options.retries) return false; if (error) { - if (isAbort(error)) return false; - } else if (!response || !RETRYABLE_STATUSES.has(response.status)) { + if (!isPreSendFailure(error)) return false; + } else if (!response || !(await isRetryableResponse(response))) { return false; } diff --git a/tests/no-data-extracted.test.ts b/tests/no-data-extracted.test.ts index 8070f85..c5ecde8 100644 --- a/tests/no-data-extracted.test.ts +++ b/tests/no-data-extracted.test.ts @@ -23,7 +23,7 @@ describe("422 no_data_extracted", () => { expect(await response.json()).toEqual(noData); }); - test("any other 422 is not retried either", async () => { + test("any other 422 is still retried", async () => { let fetchCount = 0; server.use( http.get("https://api.zenrows.com/v1/", () => { @@ -36,7 +36,7 @@ describe("422 no_data_extracted", () => { const response = await client.fetch("https://example.com"); expect(response.status).toBe(422); - expect(fetchCount).toBe(1); + expect(fetchCount).toBe(2); }); test("isNoDataExtracted matches only a 422 with that code, in any case", async () => { diff --git a/tests/retries.test.ts b/tests/retries.test.ts index d739c1b..86e0977 100644 --- a/tests/retries.test.ts +++ b/tests/retries.test.ts @@ -1,7 +1,7 @@ import { http, HttpResponse } from "msw"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { ZenRows } from "../src"; -import { retryPolicy } from "../src/retry"; +import { backoffMs, isRetryEligible, retryAfterMs, retryPolicy } from "../src/retry"; import { server } from "./_setup"; const apiUrl = "https://api.zenrows.com/v1/"; @@ -25,79 +25,172 @@ function policyAt( const status = (code: number, headers: Record = {}) => new Response(null, { status: code, headers }); -describe("retryPolicy", () => { - test("retries 429, 502, 503 and 504", () => { - for (const code of [429, 502, 503, 504]) { - expect(policyAt({ ms: 0 }).retryOn(0, null, status(code))).toBe(true); - } +/** The error undici throws: `TypeError('fetch failed')` with the cause carrying the code. */ +const fetchFailed = (code: string) => + new TypeError("fetch failed", { cause: Object.assign(new Error(code), { code }) }); + +/** The highest value `Math.random()` can return. */ +const RANDOM_MAX = 1 - Number.EPSILON; + +describe("conformance vectors", () => { + test.each([ + ["5", 5000], + ["0", 250], + [" 7 ", 7000], + ["99999", 30_000], + ])("Retry-After %j waits %i ms", (value, expected) => { + expect(retryAfterMs(status(503, { "Retry-After": value }))).toBe(expected); + const policy = policyAt({ ms: 0 }); + return policy.retryOn(0, null, status(503, { "Retry-After": value })).then((retried) => { + expect(retried).toBe(true); + expect(policy.retryDelay()).toBe(expected); + }); + }); + + test.each(["1.5", "-1", "1e3", "0x10", "", "abc", "Wed, 21 Oct 2026 07:28:00 GMT"])( + "Retry-After %j is ignored and the wait is jitter", + async (value) => { + expect(retryAfterMs(status(503, { "Retry-After": value }))).toBeUndefined(); + const policy = policyAt({ ms: 0 }, { random: () => 0 }); + await policy.retryOn(0, null, status(503, { "Retry-After": value })); + expect(policy.retryDelay()).toBe(250); + }, + ); + + test.each([ + { attempt: 0, maxWaitMs: 30_000, high: 1000 }, + { attempt: 1, maxWaitMs: 30_000, high: 2000 }, + { attempt: 2, maxWaitMs: 30_000, high: 4000 }, + { attempt: 5, maxWaitMs: 30_000, high: 30_000 }, + { attempt: 9, maxWaitMs: 30_000, high: 30_000 }, + { attempt: 4, maxWaitMs: 5000, high: 5000 }, + { attempt: 3, maxWaitMs: 100, high: 250 }, + ])( + "jitter for attempt $attempt with retryMaxWait $maxWaitMs ms is [250, $high] ms", + ({ attempt, maxWaitMs, high }) => { + expect(backoffMs(attempt, maxWaitMs, () => 0)).toBe(250); + expect(backoffMs(attempt, maxWaitMs, () => RANDOM_MAX)).toBeCloseTo(high, 6); + }, + ); + + test.each([429, 502, 503, 504])("a %i is retried", async (code) => { + expect(await policyAt({ ms: 0 }).retryOn(0, null, status(code))).toBe(true); + }); + + test.each([200, 400, 401, 402, 403, 404, 409, 500])("a %i is not retried", async (code) => { + expect(await policyAt({ ms: 0 }).retryOn(0, null, status(code))).toBe(false); }); - test("never retries other statuses, 422 included", () => { - for (const code of [200, 400, 401, 402, 403, 404, 409, 422, 500]) { - expect(policyAt({ ms: 0 }).retryOn(0, null, status(code))).toBe(false); + test("a 422 is retried unless its code is no_data_extracted", async () => { + const noData = Response.json({ code: "no_data_extracted" }, { status: 422 }); + expect(await policyAt({ ms: 0 }).retryOn(0, null, noData)).toBe(false); + expect(await noData.json()).toEqual({ code: "no_data_extracted" }); + + const other = Response.json({ code: "RESP001" }, { status: 422 }); + expect(await policyAt({ ms: 0 }).retryOn(0, null, other)).toBe(true); + const noBody = status(422); + expect(await policyAt({ ms: 0 }).retryOn(0, null, noBody)).toBe(true); + }); + + test.each(["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ENETUNREACH", "EHOSTUNREACH"])( + "a failure before the request was sent (%s) is retried", + async (code) => { + expect(await policyAt({ ms: 0 }).retryOn(0, fetchFailed(code), null)).toBe(true); + }, + ); + + test("a connect timeout is retried", async () => { + const error = fetchFailed("UND_ERR_CONNECT_TIMEOUT"); + expect(await policyAt({ ms: 0 }).retryOn(0, error, null)).toBe(true); + }); + + test.each([ + ["a socket closed after the request was sent", fetchFailed("UND_ERR_SOCKET")], + ["a connection reset", fetchFailed("ECONNRESET")], + ["a response-header timeout", fetchFailed("UND_ERR_HEADERS_TIMEOUT")], + ["a body timeout", fetchFailed("UND_ERR_BODY_TIMEOUT")], + ["an abort", new DOMException("The operation was aborted.", "AbortError")], + ["a timeout signal", new DOMException("The operation timed out.", "TimeoutError")], + ["a TypeError with no cause", new TypeError('Headers.append: "a\\nb" is an invalid value.')], + ])("%s is not retried", async (_label, error) => { + expect(await policyAt({ ms: 0 }).retryOn(0, error, null)).toBe(false); + }); + + test("a GET, HEAD or OPTIONS is always eligible", () => { + for (const method of ["GET", "HEAD", "OPTIONS", "get"]) { + expect(isRetryEligible(method, {}, undefined)).toBe(true); } }); - test("retries a network error, never an abort or a timeout", () => { - expect(policyAt({ ms: 0 }).retryOn(0, new TypeError("fetch failed"), null)).toBe(true); - const abort = new DOMException("The operation was aborted.", "AbortError"); - const timeout = new DOMException("The operation timed out.", "TimeoutError"); - expect(policyAt({ ms: 0 }).retryOn(0, abort, null)).toBe(false); - expect(policyAt({ ms: 0 }).retryOn(0, timeout, null)).toBe(false); + test("a POST without an Idempotency-Key is not eligible", () => { + expect(isRetryEligible("POST", {}, "{}")).toBe(false); + expect(isRetryEligible("POST", { "Idempotency-Key": " " }, "{}")).toBe(false); }); - test("stops after `retries` retries", () => { + test("a POST with an Idempotency-Key is eligible only with a body it can send again", () => { + const key = { "idempotency-key": "k1" }; + for (const body of [undefined, "{}", new Uint8Array([1]), new URLSearchParams("a=1")]) { + expect(isRetryEligible("POST", key, body)).toBe(true); + } + expect(isRetryEligible("POST", key, new ReadableStream())).toBe(false); + expect(isRetryEligible("PUT", { "Idempotency-Key": "k1" }, "{}")).toBe(true); + }); +}); + +describe("retryPolicy", () => { + test("stops after `retries` retries", async () => { const policy = policyAt({ ms: 0 }, { retries: 2 }); - expect(policy.retryOn(0, null, status(503))).toBe(true); - expect(policy.retryOn(1, null, status(503))).toBe(true); - expect(policy.retryOn(2, null, status(503))).toBe(false); - }); - - test("full jitter stays within [0.25 s, min(retryMaxWait, 1 s * 2^attempt)]", () => { - const ceilings = [1000, 2000, 4000, 8000, 16000, 30000, 30000]; - ceilings.forEach((ceiling, attempt) => { - const low = policyAt({ ms: 0 }, { random: () => 0 }); - const high = policyAt({ ms: 0 }, { random: () => 0.999999 }); - low.retryOn(attempt, null, status(503)); - high.retryOn(attempt, null, status(503)); - expect(low.retryDelay()).toBe(250); - expect(high.retryDelay()).toBeGreaterThan(ceiling - 1); - expect(high.retryDelay()).toBeLessThanOrEqual(ceiling); - }); + expect(await policy.retryOn(0, null, status(503))).toBe(true); + expect(await policy.retryOn(1, null, status(503))).toBe(true); + expect(await policy.retryOn(2, null, status(503))).toBe(false); }); - test("retryMaxWait caps the jitter ceiling", () => { - const policy = policyAt({ ms: 0 }, { retryMaxWait: 2, random: () => 0.999999 }); - policy.retryOn(5, null, status(503)); + test("retryMaxWait caps the jitter ceiling", async () => { + const policy = policyAt({ ms: 0 }, { retryMaxWait: 2, random: () => RANDOM_MAX }); + await policy.retryOn(5, null, status(503)); expect(policy.retryDelay()).toBeLessThanOrEqual(2000); }); - test("honours a Retry-After in seconds, capped at 30 s", () => { - const short = policyAt({ ms: 0 }); - short.retryOn(0, null, status(503, { "Retry-After": "7" })); - expect(short.retryDelay()).toBe(7000); - - const long = policyAt({ ms: 0 }); - long.retryOn(0, null, status(429, { "Retry-After": "3600" })); - expect(long.retryDelay()).toBe(30_000); + test("retryMaxWait never caps a Retry-After", async () => { + const policy = policyAt({ ms: 0 }, { retryMaxWait: 5 }); + await policy.retryOn(0, null, status(503, { "Retry-After": "20" })); + expect(policy.retryDelay()).toBe(20_000); }); - test.each(["Wed, 21 Oct 2015 07:28:00 GMT", "1.5", "-3", "soon"])( - "falls back to jitter when Retry-After is %s", - (value) => { - const policy = policyAt({ ms: 0 }, { random: () => 0 }); - policy.retryOn(0, null, status(503, { "Retry-After": value })); - expect(policy.retryDelay()).toBe(250); - }, - ); - - test("starts no retry that would pass the budget", () => { + test("starts no retry that would pass the budget", async () => { const clock = { ms: 0 }; const policy = policyAt(clock, { retryBudget: 10 }); clock.ms = 3000; - expect(policy.retryOn(0, null, status(503, { "Retry-After": "7" }))).toBe(true); + expect(await policy.retryOn(0, null, status(503, { "Retry-After": "7" }))).toBe(true); clock.ms = 3001; - expect(policy.retryOn(1, null, status(503, { "Retry-After": "7" }))).toBe(false); + expect(await policy.retryOn(1, null, status(503, { "Retry-After": "7" }))).toBe(false); + }); + + test("a start time passed in counts against the budget", async () => { + const clock = { ms: 8000 }; + const policy = policyAt(clock, { retryBudget: 10, startedAt: 0 }); + expect(await policy.retryOn(0, null, status(503, { "Retry-After": "3" }))).toBe(false); + }); + + test("cancels the body of a response it retries, and leaves a final one unread", async () => { + const cancelled: string[] = []; + const response = (name: string, code: number) => + new Response( + new ReadableStream({ + cancel() { + cancelled.push(name); + }, + }), + { status: code }, + ); + + const policy = policyAt({ ms: 0 }, { retries: 1 }); + expect(await policy.retryOn(0, null, response("retried", 503))).toBe(true); + expect(await policy.retryOn(1, null, response("last", 503))).toBe(false); + expect(await policy.retryOn(0, null, response("final", 400))).toBe(false); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(cancelled).toEqual(["retried"]); }); }); @@ -132,35 +225,39 @@ describe("ZenRows client retries", () => { expect(calls).toBe(2); }); - test("a network error is retried", async () => { + test("waits the Retry-After the server sends", async () => { let calls = 0; server.use( http.get(apiUrl, () => { calls += 1; - return calls === 1 ? HttpResponse.error() : HttpResponse.json({ ok: 1 }); + return calls === 1 + ? new HttpResponse(null, { status: 429, headers: { "Retry-After": "5" } }) + : HttpResponse.json({ ok: 1 }); }), ); const pending = new ZenRows("API_KEY").fetch(url); - await vi.advanceTimersByTimeAsync(250); + await vi.advanceTimersByTimeAsync(4999); + expect(calls).toBe(1); + await vi.advanceTimersByTimeAsync(1); expect((await pending).status).toBe(200); expect(calls).toBe(2); }); - test("waits the Retry-After the server sends", async () => { + test("waits a Retry-After of 20 s in full, even with retryMaxWait 5", async () => { let calls = 0; server.use( http.get(apiUrl, () => { calls += 1; return calls === 1 - ? new HttpResponse(null, { status: 429, headers: { "Retry-After": "5" } }) + ? new HttpResponse(null, { status: 503, headers: { "Retry-After": "20" } }) : HttpResponse.json({ ok: 1 }); }), ); - const pending = new ZenRows("API_KEY").fetch(url); - await vi.advanceTimersByTimeAsync(4999); + const pending = new ZenRows("API_KEY", { retryMaxWait: 5 }).fetch(url); + await vi.advanceTimersByTimeAsync(19_999); expect(calls).toBe(1); await vi.advanceTimersByTimeAsync(1); @@ -186,34 +283,119 @@ describe("ZenRows client retries", () => { expect(calls).toBe(3); }); - test.each([400, 422])("a %i is not retried", async (code) => { + test("a 400 is not retried", async () => { let calls = 0; server.use( http.get(apiUrl, () => { calls += 1; - return HttpResponse.json({ code: "RESP001" }, { status: code }); + return HttpResponse.json({ code: "REQS001" }, { status: 400 }); }), ); const response = await new ZenRows("API_KEY", { retries: 3 }).fetch(url); - expect(response.status).toBe(code); + expect(response.status).toBe(400); expect(calls).toBe(1); }); - test("an abort is not retried", async () => { - const fetchSpy = vi.fn(async () => { - throw new DOMException("The operation was aborted.", "AbortError"); - }); - vi.stubGlobal("fetch", fetchSpy); + test("a 422 other than no_data_extracted is retried, and the last one comes back readable", async () => { + let calls = 0; + server.use( + http.get(apiUrl, () => { + calls += 1; + return HttpResponse.json({ code: "RESP001" }, { status: 422 }); + }), + ); - await expect(new ZenRows("API_KEY", { retries: 3 }).fetch(url)).rejects.toMatchObject({ - name: "AbortError", - }); - expect(fetchSpy).toHaveBeenCalledTimes(1); + const pending = new ZenRows("API_KEY").fetch(url); + await vi.advanceTimersByTimeAsync(1000); + const response = await pending; + + expect(calls).toBe(3); + expect(response.status).toBe(422); + expect(await response.json()).toEqual({ code: "RESP001" }); + }); + + test("a 500 is not retried", async () => { + let calls = 0; + server.use( + http.get(apiUrl, () => { + calls += 1; + return new HttpResponse(null, { status: 500 }); + }), + ); + + const response = await new ZenRows("API_KEY", { retries: 3 }).fetch(url); + + expect(response.status).toBe(500); + expect(calls).toBe(1); + }); + + test("post() without an Idempotency-Key is sent once, even on a 503", async () => { + const bodies: string[] = []; + server.use( + http.post(apiUrl, async ({ request }) => { + bodies.push(await request.text()); + return new HttpResponse(null, { status: 503 }); + }), + ); + + const pending = new ZenRows("API_KEY", { retries: 3 }).post(url, {}, { data: { order: 1 } }); + await vi.advanceTimersByTimeAsync(10_000); + + expect((await pending).status).toBe(503); + expect(bodies).toEqual(['{"order":1}']); + }); + + test("post() with an Idempotency-Key is retried with the same body", async () => { + const bodies: string[] = []; + const keys: (string | null)[] = []; + server.use( + http.post(apiUrl, async ({ request }) => { + bodies.push(await request.text()); + keys.push(request.headers.get("Idempotency-Key")); + return bodies.length === 1 + ? new HttpResponse(null, { status: 503 }) + : HttpResponse.json({ ok: 1 }); + }), + ); + + const pending = new ZenRows("API_KEY").post( + url, + {}, + { headers: { "Idempotency-Key": "order-key" }, data: { order: 1 } }, + ); + await vi.advanceTimersByTimeAsync(250); + + expect((await pending).status).toBe(200); + expect(bodies).toEqual(['{"order":1}', '{"order":1}']); + expect(keys).toEqual(["order-key", "order-key"]); }); - test("the extract() autoparse fallback is unchanged: one fallback request, no retries", async () => { + test("the extract() autoparse fallback shares the first request's budget", async () => { + const urls: URL[] = []; + server.use( + http.get(apiUrl, ({ request }) => { + const requestUrl = new URL(request.url); + urls.push(requestUrl); + if (requestUrl.searchParams.get("autoparse") !== "true" && urls.length > 1) { + return HttpResponse.json({ code: "REQS007" }, { status: 403 }); + } + return new HttpResponse(null, { status: 503, headers: { "Retry-After": "20" } }); + }), + ); + + const pending = new ZenRows("API_KEY", { retries: 5, retryBudget: 45 }).extract(url); + await vi.advanceTimersByTimeAsync(120_000); + const response = await pending; + + // First request: 503, wait 20 s, 403 (20 s spent). Fallback: 503, wait 20 s (40 s spent), + // 503; another wait would end at 60 s, past 45 s. A fresh budget would allow one more try. + expect(response.status).toBe(503); + expect(urls.map((u) => u.searchParams.get("autoparse"))).toEqual([null, null, "true", "true"]); + }); + + test("the extract() autoparse fallback is one request when it succeeds", async () => { const urls: URL[] = []; server.use( http.get(apiUrl, ({ request }) => { @@ -231,3 +413,29 @@ describe("ZenRows client retries", () => { expect(urls[1]?.searchParams.get("autoparse")).toBe("true"); }); }); + +describe("retry options are validated", () => { + test.each([-1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, "2"])( + "retries %s is rejected", + (retries) => { + expect(() => new ZenRows("API_KEY", { retries: retries as number })).toThrow( + /retries must be an integer of 0 or more/, + ); + }, + ); + + test.each(["retryMaxWait", "retryBudget"] as const)("%s must be finite and above 0", (name) => { + for (const value of [0, -1, Number.NaN, Number.POSITIVE_INFINITY, "5"]) { + expect(() => new ZenRows("API_KEY", { [name]: value as number })).toThrow( + new RegExp(`${name} must be a finite number of seconds greater than 0`), + ); + } + }); + + test("valid values and defaults are accepted", () => { + expect(() => new ZenRows("API_KEY")).not.toThrow(); + expect( + () => new ZenRows("API_KEY", { retries: 0, retryMaxWait: 0.1, retryBudget: 1 }), + ).not.toThrow(); + }); +}); diff --git a/tests/retry-network.test.ts b/tests/retry-network.test.ts new file mode 100644 index 0000000..f01a057 --- /dev/null +++ b/tests/retry-network.test.ts @@ -0,0 +1,217 @@ +// Retry decisions against a real node:http server through Node's real fetch, so each failure +// reaches the policy in the exact shape undici produces. The request mocks are off in this file. +import http from "node:http"; +import type { AddressInfo, Socket } from "node:net"; +import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest"; +import { ZenRows } from "../src"; +import { server as mockServer } from "./_setup"; + +const API_URL = "https://api.zenrows.com/v1/"; + +let realFetch: typeof fetch; + +beforeAll(() => { + // The shared setup turns the request mocks on; turn them off here so fetch is Node's own. + mockServer.close(); + realFetch = globalThis.fetch; +}); + +afterAll(() => { + // Leave the mocks as the shared setup expects to find them when it closes them. + mockServer.listen(); +}); + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +interface TestServer { + origin: string; + /** Requests the server read in full, by method. */ + requests: string[]; + /** Sockets the server still holds open. */ + sockets: Set; + close: () => Promise; +} + +async function startServer( + handle: (req: http.IncomingMessage, res: http.ServerResponse) => void, +): Promise { + const requests: string[] = []; + const sockets = new Set(); + const httpServer = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + requests.push(req.method ?? ""); + handle(req, res); + }); + }); + httpServer.on("connection", (socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + }); + await new Promise((resolve) => httpServer.listen(0, "127.0.0.1", resolve)); + const { port } = httpServer.address() as AddressInfo; + return { + origin: `http://127.0.0.1:${port}`, + requests, + sockets, + close: () => + new Promise((resolve) => { + for (const socket of sockets) socket.destroy(); + httpServer.close(() => resolve()); + }), + }; +} + +/** A port nothing listens on: bind one, then close it. */ +async function closedPortOrigin(): Promise { + const probe = http.createServer(); + await new Promise((resolve) => probe.listen(0, "127.0.0.1", resolve)); + const { port } = probe.address() as AddressInfo; + await new Promise((resolve) => probe.close(() => resolve())); + return `http://127.0.0.1:${port}`; +} + +/** + * Sends the client's API calls to `origin` through real fetch, with extra fetch options, and + * counts the calls. Must run before the client is built, since the client wraps fetch then. + */ +function routeApiTo(origin: string, extra: Record = {}) { + const calls = { count: 0 }; + vi.stubGlobal("fetch", (input: string | URL, init?: RequestInit) => { + calls.count += 1; + return realFetch(String(input).replace(API_URL, `${origin}/v1/`), { ...init, ...extra }); + }); + return calls; +} + +/** undici's Agent, taken from Node's own global dispatcher: Node bundles undici but does not export it. */ +async function undiciAgent(origin: string): Promise object> { + await (await realFetch(`${origin}/warm-up`)).arrayBuffer(); + const globals = globalThis as unknown as Record; + const dispatcher = + globals[Symbol.for("undici.globalDispatcher.2")] ?? + globals[Symbol.for("undici.globalDispatcher.1")]; + if (!dispatcher) throw new Error("Node's fetch exposes no global dispatcher"); + return dispatcher.constructor as new ( + options: object, + ) => object; +} + +const fastRandom = () => vi.spyOn(Math, "random").mockReturnValue(0); + +describe("failures before the request was sent are retried", () => { + test("a refused connection is retried `retries` times, then throws", async () => { + fastRandom(); + const calls = routeApiTo(await closedPortOrigin()); + + const error = await new ZenRows("API_KEY").fetch("https://example.com").catch((e) => e); + + expect(error).toBeInstanceOf(TypeError); + expect(error.cause.code).toBe("ECONNREFUSED"); + expect(calls.count).toBe(3); + }); + + test("a refused connection is retried for a post() with an Idempotency-Key", async () => { + fastRandom(); + const calls = routeApiTo(await closedPortOrigin()); + + const error = await new ZenRows("API_KEY", { retries: 1 }) + .post("https://example.com", {}, { headers: { "Idempotency-Key": "k1" }, data: { a: 1 } }) + .catch((e) => e); + + expect(error.cause.code).toBe("ECONNREFUSED"); + expect(calls.count).toBe(2); + }); +}); + +describe("failures after the request was sent are not retried", () => { + test("a socket closed after the server read the request", async () => { + fastRandom(); + const server = await startServer((req) => req.socket.destroy()); + routeApiTo(server.origin); + + const error = await new ZenRows("API_KEY").fetch("https://example.com").catch((e) => e); + await server.close(); + + expect(error).toBeInstanceOf(TypeError); + expect(error.cause.code).toBe("UND_ERR_SOCKET"); + expect(server.requests).toEqual(["GET"]); + }); + + test("a socket closed after a post() with an Idempotency-Key was read", async () => { + fastRandom(); + const server = await startServer((req) => req.socket.destroy()); + routeApiTo(server.origin); + + const error = await new ZenRows("API_KEY") + .post("https://example.com", {}, { headers: { "Idempotency-Key": "k1" }, data: { a: 1 } }) + .catch((e) => e); + await server.close(); + + expect(error.cause.code).toBe("UND_ERR_SOCKET"); + expect(server.requests).toEqual(["POST"]); + }); + + test("headers that arrive after undici's headersTimeout", async () => { + fastRandom(); + const server = await startServer((req, res) => { + if (req.url === "/warm-up") return res.end(); + setTimeout(() => res.end("late"), 1000).unref(); + }); + const Agent = await undiciAgent(server.origin); + server.requests.length = 0; + routeApiTo(server.origin, { dispatcher: new Agent({ headersTimeout: 200 }) }); + + const error = await new ZenRows("API_KEY").fetch("https://example.com").catch((e) => e); + await server.close(); + + expect(error).toBeInstanceOf(TypeError); + expect(error.cause.code).toBe("UND_ERR_HEADERS_TIMEOUT"); + expect(server.requests).toEqual(["GET"]); + }); + + test("an invalid header value throws at once, with no retry", async () => { + fastRandom(); + const server = await startServer((_req, res) => res.end("ok")); + const calls = routeApiTo(server.origin); + + const error = await new ZenRows("API_KEY") + .fetch("https://example.com", {}, { headers: { "X-Bad": "a\nb" } }) + .catch((e) => e); + await server.close(); + + expect(error).toBeInstanceOf(TypeError); + expect(error.cause).toBeUndefined(); + expect(calls.count).toBe(1); + expect(server.requests).toEqual([]); + }); +}); + +describe("retried answers", () => { + test("free their connection: no socket stays open after retried 503s with large bodies", async () => { + fastRandom(); + const largeBody = "x".repeat(2 * 1024 * 1024); + const server = await startServer((_req, res) => { + res.writeHead(503, { "Content-Type": "text/plain" }); + res.end(largeBody); + }); + routeApiTo(server.origin); + + const client = new ZenRows("API_KEY"); + const responses = await Promise.all( + Array.from({ length: 5 }, () => client.fetch("https://example.com")), + ); + for (const response of responses) await response.body?.cancel(); + for (let i = 0; i < 50 && server.sockets.size > 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + const open = server.sockets.size; + await server.close(); + + expect(server.requests).toHaveLength(15); + expect(open).toBe(0); + }); +}); From f72cf4475e3334614c288bfb3e2b31ec6024fc5c Mon Sep 17 00:00:00 2001 From: gonzalo-zenrows Date: Mon, 19 Oct 2026 23:10:06 +0200 Subject: [PATCH 3/6] test(client): count held answers, not open sockets, so the check is stable across Node versions The connection test counted open server sockets after retried 503s. On Node 20 and 22 that count is not a leak signal. There, undici drains a cancelled body and keeps the socket for its keep-alive window, about 4 s, even when the server sends Connection: close. It also buffers a 2 MB body in full, so an unread one frees its socket too. The test failed on CI with 5 open sockets, while the client had released every answer. The server now counts answers that have neither finished nor lost their connection, and the body is 8 MB, which undici does not buffer on its own. With body cancellation that count reaches 0 within about 25 ms on Node 20, 22 and 26. Without it, the 10 retried answers stay held. --- tests/retry-network.test.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/retry-network.test.ts b/tests/retry-network.test.ts index f01a057..fe71b33 100644 --- a/tests/retry-network.test.ts +++ b/tests/retry-network.test.ts @@ -191,11 +191,21 @@ describe("failures after the request was sent are not retried", () => { }); describe("retried answers", () => { - test("free their connection: no socket stays open after retried 503s with large bodies", async () => { + // An answer the client drops without reading holds its connection until garbage collection. + // The server counts answers that have neither finished nor lost their connection. That count + // does not depend on the Node version, unlike open sockets. Node 20 and 22 drain a cancelled + // body and keep the socket for undici's keep-alive window, and buffer a small body in full. + test("release their connection: no answer stays held after retried 503s with large bodies", async () => { fastRandom(); - const largeBody = "x".repeat(2 * 1024 * 1024); + // Larger than undici buffers on its own, so an unread body really holds the connection. + const largeBody = Buffer.alloc(8 * 1024 * 1024, "x"); + let held = 0; const server = await startServer((_req, res) => { - res.writeHead(503, { "Content-Type": "text/plain" }); + held += 1; + res.on("close", () => { + held -= 1; + }); + res.writeHead(503, { "Content-Type": "text/plain", "Content-Length": largeBody.length }); res.end(largeBody); }); routeApiTo(server.origin); @@ -205,13 +215,14 @@ describe("retried answers", () => { Array.from({ length: 5 }, () => client.fetch("https://example.com")), ); for (const response of responses) await response.body?.cancel(); - for (let i = 0; i < 50 && server.sockets.size > 0; i++) { + const deadline = Date.now() + 2000; + while (held > 0 && Date.now() < deadline) { await new Promise((resolve) => setTimeout(resolve, 20)); } - const open = server.sockets.size; + const stillHeld = held; await server.close(); expect(server.requests).toHaveLength(15); - expect(open).toBe(0); + expect(stillHeld).toBe(0); }); }); From 4bfa92b2929f09e3202a4094b50a231bc27510a5 Mon Sep 17 00:00:00 2001 From: gonzalo-zenrows Date: Tue, 20 Oct 2026 17:22:05 +0200 Subject: [PATCH 4/6] fix(client): never retry post(), even with an Idempotency-Key The API does not dedupe Idempotency-Key on Fetch or Extract. It only forwards the header to the target site. So the key proves nothing, and a retried post() can make the target act twice. 1. Only GET, HEAD and OPTIONS are retried. A post() goes out once, whatever its headers, the status or the failure, a refused connection included. 2. The body guard now applies to every method: a body that is not none, a string or bytes is never retried. A stream, a reader or a generator counts as one-shot. 3. The README says post() is never retried, and that a caller who needs a retry writes it. Tests send post() with and without an Idempotency-Key on a 503, a 429, a 422 and a refused connection, and check one request each time. They also check that fetch() still retries and that a GET with a stream body is not eligible. --- README.md | 10 ++-- src/index.ts | 14 ++--- src/retry.ts | 34 +++++------ tests/retries.test.ts | 112 +++++++++++++++++++++++------------- tests/retry-network.test.ts | 13 +++-- 5 files changed, 107 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index 3fcf91d..ee03060 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ Sending headers to the target URL will overwrite our defaults. Be careful when d ### Retries -`fetch()`, `extract()` and `post()` retry failed requests through [fetch-retry](https://github.com/jonbern/fetch-retry). `client.batch` and `client.crawl` keep their own retry rules, described in their sections. `prepareDomain()` and `preparedDomainStatus()` never retry: they send one request and reject on an error answer (see [Preparing a domain](#preparing-a-domain)). +`fetch()`, `get()` and `extract()` retry failed requests through [fetch-retry](https://github.com/jonbern/fetch-retry). `post()` never retries. `client.batch` and `client.crawl` keep their own retry rules, described in their sections. `prepareDomain()` and `preparedDomainStatus()` never retry: they send one request and reject on an error answer (see [Preparing a domain](#preparing-a-domain)). What the client retries: @@ -151,7 +151,9 @@ What the client never retries: Which calls the client retries: 1. `fetch()`, `get()` and `extract()`: always. -2. `post()`: only when its `headers` carry an `Idempotency-Key`. Without one, the client sends a `post()` once, because the API forwards the body to the target, and the target may act on it twice. The client sends the key to the target as a custom header, like any other header you pass. +2. `post()`: never. The client sends a `post()` once, whatever its headers, the status or the failure. The API forwards the body to the target, and the target may act on it twice. +3. The API does not dedupe a `post()` on an `Idempotency-Key` header, so the client does not retry on one. +4. If you need to retry a `post()`, write the retry yourself. How long the client waits between tries: @@ -180,7 +182,7 @@ Compatible with `proxy_country`, `js_instructions`, and `custom_headers`. The SDK also offers POST requests by calling the `client.post` method. It can receive a new parameter `data` that represents the data sent in, for example, a form. -The client sends a `post()` once, unless you pass an `Idempotency-Key` header. See [Retries](#retries). +The client sends a `post()` once and never retries it, whatever `retries` says. See [Retries](#retries). ```javascript const { ZenRows } = require("zenrows"); @@ -189,7 +191,7 @@ const apiKey = "YOUR-API-KEY"; const url = "https://httpbin.org/anything"; (async () => { - const client = new ZenRows(apiKey, { retries: 1 }); + const client = new ZenRows(apiKey); const response = await client.post( url, diff --git a/src/index.ts b/src/index.ts index 7c387c8..4b43a4d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -73,11 +73,11 @@ function warnFallbackDeprecated(): void { interface ClientConfig { concurrency?: number; /** - * Retries for `fetch()`, `extract()` and `post()` after a 429, 502, 503 or 504, a 422 other - * than `no_data_extracted`, or a failure before the request reached the server. An integer of - * 0 or more; default 2; 0 turns retries off. A reset after the request was sent, a timeout or - * an abort is never retried: the server may still finish the request and charge it. `post()` - * is retried only when its headers carry an `Idempotency-Key`. + * Retries for `fetch()` and `extract()` after a 429, 502, 503 or 504, a 422 other than + * `no_data_extracted`, or a failure before the request reached the server. An integer of 0 or + * more; default 2; 0 turns retries off. A reset after the request was sent, a timeout or an + * abort is never retried: the server may still finish the request and charge it. `post()` is + * never retried, whatever its headers: the API does not dedupe on `Idempotency-Key`. */ retries?: number; /** The longest one random wait may be, in seconds (greater than 0). Default 30. It never caps a `Retry-After`. */ @@ -341,8 +341,8 @@ export class ZenRows { } } - // A request that is not safe to send twice (a POST without an Idempotency-Key) is sent once. - const eligible = isRetryEligible(method, finalHeaders, fetchOptions.body); + // A POST, or a body that cannot be sent again, goes out once: it is not safe to send twice. + const eligible = isRetryEligible(method, fetchOptions.body); const { retryOn, retryDelay } = retryPolicy({ retries: eligible ? this.clientConfig.retries ?? DEFAULT_RETRIES : 0, retryMaxWait: this.clientConfig.retryMaxWait ?? DEFAULT_RETRY_MAX_WAIT_SECONDS, diff --git a/src/retry.ts b/src/retry.ts index 02d83f5..005bd3b 100644 --- a/src/retry.ts +++ b/src/retry.ts @@ -1,6 +1,7 @@ -// Retry policy for `fetch()`, `extract()` and `post()`. The same policy ships in every ZenRows -// client: retry only answers that say "try again" and are never charged, retry only requests -// that are safe to send twice, back off with full jitter, and start no retry after the budget. +// Retry policy for `fetch()` and `extract()`. `post()` goes through it too, but is never retried. +// The same policy ships in every ZenRows client: retry only answers that say "try again" and are +// never charged, retry only requests that are safe to send twice, back off with full jitter, and +// start no retry after the budget. /** Statuses retried: rate limited, or the gateway or its upstream could not answer. Never charged. */ export const RETRYABLE_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]); @@ -20,7 +21,10 @@ const PRE_SEND_ERROR_CODES: ReadonlySet = new Set([ "UND_ERR_CONNECT_TIMEOUT", ]); -/** Methods that are always safe to send again. Others need an `Idempotency-Key` header. */ +/** + * The only methods the client retries. A POST, PUT, PATCH or DELETE is sent once, whatever its + * headers: the API does not dedupe on `Idempotency-Key`. It only forwards it to the target. + */ const IDEMPOTENT_METHODS: ReadonlySet = new Set(["GET", "HEAD", "OPTIONS"]); export const DEFAULT_RETRIES = 2; @@ -84,33 +88,23 @@ export async function isNoDataExtracted(response: Response): Promise { } } -/** True when the body can be sent again unchanged. A one-shot stream cannot. */ +/** True when the body is none, a string or bytes, so it can be sent again. A stream cannot. */ function isReplayableBody(body: unknown): boolean { return ( body === undefined || body === null || typeof body === "string" || body instanceof ArrayBuffer || - ArrayBuffer.isView(body) || - body instanceof URLSearchParams || - body instanceof Blob + ArrayBuffer.isView(body) ); } /** - * True when a request is safe to send more than once. GET, HEAD and OPTIONS always are. Any - * other method is only with an `Idempotency-Key` header and a body that can be sent again. + * True when a request is safe to send more than once: a GET, HEAD or OPTIONS whose body is none, + * a string or bytes. Any other method, or a body such as a stream, is sent once. */ -export function isRetryEligible( - method: string, - headers: Record, - body: unknown, -): boolean { - if (IDEMPOTENT_METHODS.has(method.toUpperCase())) return true; - const hasIdempotencyKey = Object.entries(headers).some( - ([name, value]) => name.toLowerCase() === "idempotency-key" && value.trim() !== "", - ); - return hasIdempotencyKey && isReplayableBody(body); +export function isRetryEligible(method: string, body: unknown): boolean { + return IDEMPOTENT_METHODS.has(method.toUpperCase()) && isReplayableBody(body); } /** True when fetch failed before the request was written to the server (see `PRE_SEND_ERROR_CODES`). */ diff --git a/tests/retries.test.ts b/tests/retries.test.ts index df347fc..05bf1c8 100644 --- a/tests/retries.test.ts +++ b/tests/retries.test.ts @@ -1,3 +1,4 @@ +import { Readable } from "node:stream"; import { http, HttpResponse } from "msw"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { ZenRows, ZenRowsPreparedDomainError } from "../src"; @@ -116,24 +117,44 @@ describe("conformance vectors", () => { expect(await policyAt({ ms: 0 }).retryOn(0, error, null)).toBe(false); }); - test("a GET, HEAD or OPTIONS is always eligible", () => { + test("a GET, HEAD or OPTIONS with no body, a string or bytes is eligible", () => { + const bodies = [ + undefined, + null, + "", + "{}", + new Uint8Array([1]), + Buffer.from("a"), + new ArrayBuffer(1), + ]; for (const method of ["GET", "HEAD", "OPTIONS", "get"]) { - expect(isRetryEligible(method, {}, undefined)).toBe(true); + for (const body of bodies) { + expect(isRetryEligible(method, body)).toBe(true); + } } }); - test("a POST without an Idempotency-Key is not eligible", () => { - expect(isRetryEligible("POST", {}, "{}")).toBe(false); - expect(isRetryEligible("POST", { "Idempotency-Key": " " }, "{}")).toBe(false); + test("a POST, PUT, PATCH or DELETE is never eligible, whatever its body", () => { + for (const method of ["POST", "PUT", "PATCH", "DELETE", "post"]) { + for (const body of [undefined, "{}", new Uint8Array([1])]) { + expect(isRetryEligible(method, body)).toBe(false); + } + } }); - test("a POST with an Idempotency-Key is eligible only with a body it can send again", () => { - const key = { "idempotency-key": "k1" }; - for (const body of [undefined, "{}", new Uint8Array([1]), new URLSearchParams("a=1")]) { - expect(isRetryEligible("POST", key, body)).toBe(true); + test("a GET with a one-shot body (a stream, a reader or a generator) is not eligible", () => { + async function* generator() { + yield "a"; + } + const oneShot = [ + new ReadableStream(), + new ReadableStream().getReader(), + Readable.from(["a"]), + generator(), + ]; + for (const body of oneShot) { + expect(isRetryEligible("GET", body)).toBe(false); } - expect(isRetryEligible("POST", key, new ReadableStream())).toBe(false); - expect(isRetryEligible("PUT", { "Idempotency-Key": "k1" }, "{}")).toBe(true); }); }); @@ -325,45 +346,54 @@ describe("ZenRows client retries", () => { expect(calls).toBe(1); }); - test("post() without an Idempotency-Key is sent once, even on a 503", async () => { - const bodies: string[] = []; - server.use( - http.post(apiUrl, async ({ request }) => { - bodies.push(await request.text()); - return new HttpResponse(null, { status: 503 }); - }), - ); + const retriedAnswers = [ + [503, () => new HttpResponse(null, { status: 503 })], + [429, () => new HttpResponse(null, { status: 429 })], + [422, () => HttpResponse.json({ code: "RESP001" }, { status: 422 })], + ] as const; + + describe.each([ + ["without an Idempotency-Key", {}], + ["with an Idempotency-Key", { "Idempotency-Key": "order-key" }], + ])("post() %s", (_label, headers: Record) => { + test.each(retriedAnswers)( + "is sent once on a %i, even with retries: 3", + async (code, answer) => { + const bodies: string[] = []; + server.use( + http.post(apiUrl, async ({ request }) => { + bodies.push(await request.text()); + return answer(); + }), + ); - const pending = new ZenRows("API_KEY", { retries: 3 }).post(url, {}, { data: { order: 1 } }); - await vi.advanceTimersByTimeAsync(10_000); + const pending = new ZenRows("API_KEY", { retries: 3 }).post( + url, + {}, + { headers, data: { order: 1 } }, + ); + await vi.advanceTimersByTimeAsync(60_000); - expect((await pending).status).toBe(503); - expect(bodies).toEqual(['{"order":1}']); + expect((await pending).status).toBe(code); + expect(bodies).toEqual(['{"order":1}']); + }, + ); }); - test("post() with an Idempotency-Key is retried with the same body", async () => { - const bodies: string[] = []; - const keys: (string | null)[] = []; + test.each(retriedAnswers)("fetch() is still retried on a %i", async (code, answer) => { + let calls = 0; server.use( - http.post(apiUrl, async ({ request }) => { - bodies.push(await request.text()); - keys.push(request.headers.get("Idempotency-Key")); - return bodies.length === 1 - ? new HttpResponse(null, { status: 503 }) - : HttpResponse.json({ ok: 1 }); + http.get(apiUrl, () => { + calls += 1; + return answer(); }), ); - const pending = new ZenRows("API_KEY").post( - url, - {}, - { headers: { "Idempotency-Key": "order-key" }, data: { order: 1 } }, - ); - await vi.advanceTimersByTimeAsync(250); + const pending = new ZenRows("API_KEY", { retries: 3 }).fetch(url); + await vi.advanceTimersByTimeAsync(60_000); - expect((await pending).status).toBe(200); - expect(bodies).toEqual(['{"order":1}', '{"order":1}']); - expect(keys).toEqual(["order-key", "order-key"]); + expect((await pending).status).toBe(code); + expect(calls).toBe(4); }); }); diff --git a/tests/retry-network.test.ts b/tests/retry-network.test.ts index fe71b33..c4713df 100644 --- a/tests/retry-network.test.ts +++ b/tests/retry-network.test.ts @@ -113,17 +113,22 @@ describe("failures before the request was sent are retried", () => { expect(error.cause.code).toBe("ECONNREFUSED"); expect(calls.count).toBe(3); }); +}); - test("a refused connection is retried for a post() with an Idempotency-Key", async () => { +describe("post() is sent once, even on a refused connection", () => { + test.each([ + ["without an Idempotency-Key", {}], + ["with an Idempotency-Key", { "Idempotency-Key": "k1" }], + ])("%s", async (_label, headers: Record) => { fastRandom(); const calls = routeApiTo(await closedPortOrigin()); - const error = await new ZenRows("API_KEY", { retries: 1 }) - .post("https://example.com", {}, { headers: { "Idempotency-Key": "k1" }, data: { a: 1 } }) + const error = await new ZenRows("API_KEY", { retries: 3 }) + .post("https://example.com", {}, { headers, data: { a: 1 } }) .catch((e) => e); expect(error.cause.code).toBe("ECONNREFUSED"); - expect(calls.count).toBe(2); + expect(calls.count).toBe(1); }); }); From 52c229f102e9f1583775c3d1c7585f39a5962fe6 Mon Sep 17 00:00:00 2001 From: gonzalo-zenrows Date: Tue, 20 Oct 2026 17:49:41 +0200 Subject: [PATCH 5/6] fix(client): never retry a status that comes from the target site With original_status the API returns the target site's own status. With allowed_status_codes it passes the listed statuses through from the target. Such a status is the site's answer, not the API saying "try again", so another try only asks the same site the same question. 1. When the request sets original_status (true, "true", 1 or "1"), no status is retried. 2. When it sets allowed_status_codes (a comma-separated string or an array), a listed status is not retried. Other statuses keep the usual rule. 3. A failure before the request reached the API, such as a refused connection, is still retried. 4. The rule lives in one place, targetStatusRule in retry.ts. worker() passes the request params to retryPolicy, which applies it in retryOn. 5. The README lists the rule under what the client never retries. Tests cover each accepted form of both params on the policy, fetch() and extract() on a fake clock, and a real node:http server for a 503 sent once and a refused connection that is still retried. --- README.md | 1 + src/index.ts | 6 +- src/retry.ts | 43 ++++++++++++- tests/retries.test.ts | 125 ++++++++++++++++++++++++++++++++++++ tests/retry-network.test.ts | 31 +++++++++ 5 files changed, 203 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ee03060..9991395 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,7 @@ What the client never retries: 1. Any other status, `500` included. 2. A connection reset or close after the request was sent, a response timeout, or an abort. The API may still finish that request and charge it. 3. An error in the request itself, such as an invalid header value. +4. Any status when you set `original_status`, and a status you list in `allowed_status_codes`. That status comes from the target site, not the API. A refused connection is still retried. Which calls the client retries: diff --git a/src/index.ts b/src/index.ts index 4b43a4d..b00dea4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -77,7 +77,9 @@ interface ClientConfig { * `no_data_extracted`, or a failure before the request reached the server. An integer of 0 or * more; default 2; 0 turns retries off. A reset after the request was sent, a timeout or an * abort is never retried: the server may still finish the request and charge it. `post()` is - * never retried, whatever its headers: the API does not dedupe on `Idempotency-Key`. + * never retried, whatever its headers: the API does not dedupe on `Idempotency-Key`. No status + * is retried when the request sets `original_status`, and no status listed in + * `allowed_status_codes` is: that status comes from the target site. */ retries?: number; /** The longest one random wait may be, in seconds (greater than 0). Default 30. It never caps a `Retry-After`. */ @@ -345,6 +347,8 @@ export class ZenRows { const eligible = isRetryEligible(method, fetchOptions.body); const { retryOn, retryDelay } = retryPolicy({ retries: eligible ? this.clientConfig.retries ?? DEFAULT_RETRIES : 0, + // A status the target site sent (original_status, allowed_status_codes) is not retried. + params: config, retryMaxWait: this.clientConfig.retryMaxWait ?? DEFAULT_RETRY_MAX_WAIT_SECONDS, retryBudget: this.clientConfig.retryBudget ?? DEFAULT_RETRY_BUDGET_SECONDS, }); diff --git a/src/retry.ts b/src/retry.ts index 005bd3b..e401d8a 100644 --- a/src/retry.ts +++ b/src/retry.ts @@ -1,7 +1,7 @@ // Retry policy for `fetch()` and `extract()`. `post()` goes through it too, but is never retried. // The same policy ships in every ZenRows client: retry only answers that say "try again" and are // never charged, retry only requests that are safe to send twice, back off with full jitter, and -// start no retry after the budget. +// start no retry after the budget. A status the target site sent is never retried. /** Statuses retried: rate limited, or the gateway or its upstream could not answer. Never charged. */ export const RETRYABLE_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]); @@ -39,9 +39,43 @@ const BACKOFF_BASE_MS = 1000; const RETRY_AFTER_MIN_MS = 250; const RETRY_AFTER_MAX_MS = 30_000; +/** + * The request params that make a status the target site's own answer, not the API's. The client + * reads them in the form it sends them: `String(value)`, so an array goes out comma-separated. + */ +export interface TargetStatusParams { + /** When on (`true`, `"true"`, `1` or `"1"`), the API returns the target's status: none is retried. */ + original_status?: unknown; + /** Statuses, comma-separated or as an array, that the API passes through from the target. */ + allowed_status_codes?: unknown; +} + +/** + * Builds the test for "this status came from the target site", from the request params. A + * status from the target is never retried: another try asks the same site the same question. + */ +export function targetStatusRule(params: TargetStatusParams = {}): (status: number) => boolean { + const originalStatus = params.original_status; + if (originalStatus !== undefined && originalStatus !== null) { + const wire = String(originalStatus).trim().toLowerCase(); + if (wire === "true" || wire === "1") return () => true; + } + const allowed = new Set(); + const listed = params.allowed_status_codes; + if (listed !== undefined && listed !== null) { + for (const part of String(listed).split(",")) { + const code = part.trim(); + if (/^\d+$/.test(code)) allowed.add(Number(code)); + } + } + return (status) => allowed.has(status); +} + export interface RetryPolicyOptions { /** Retries after the first try. 0 turns retries off. */ retries: number; + /** The request params `original_status` and `allowed_status_codes` (see `targetStatusRule`). */ + params?: TargetStatusParams; /** The longest one jitter wait may be, in seconds. It never caps a `Retry-After`. */ retryMaxWait: number; /** No retry starts once the time spent plus the next wait would pass this, in seconds. */ @@ -143,6 +177,7 @@ export function retryPolicy(options: RetryPolicyOptions) { const random = options.random ?? Math.random; const maxWaitMs = options.retryMaxWait * 1000; const budgetMs = options.retryBudget * 1000; + const isTargetStatus = targetStatusRule(options.params); const startedAt = now(); let nextDelayMs = 0; @@ -155,7 +190,11 @@ export function retryPolicy(options: RetryPolicyOptions) { if (attempt >= options.retries) return false; if (error) { if (!isPreSendFailure(error)) return false; - } else if (!response || !(await isRetryableResponse(response))) { + } else if ( + !response || + isTargetStatus(response.status) || + !(await isRetryableResponse(response)) + ) { return false; } diff --git a/tests/retries.test.ts b/tests/retries.test.ts index 05bf1c8..703c6da 100644 --- a/tests/retries.test.ts +++ b/tests/retries.test.ts @@ -158,6 +158,58 @@ describe("conformance vectors", () => { }); }); +describe("a status from the target site is not retried", () => { + const retriedCodes = [429, 502, 503, 504, 422]; + + test.each([true, "true", "TRUE", " true ", 1, "1"])( + "original_status %j: no status is retried", + async (on) => { + for (const code of retriedCodes) { + const policy = policyAt({ ms: 0 }, { params: { original_status: on } }); + expect(await policy.retryOn(0, null, status(code))).toBe(false); + } + }, + ); + + test.each([false, "false", 0, "0", "", null, undefined])( + "original_status %j: the usual statuses are retried", + async (off) => { + for (const code of retriedCodes) { + const policy = policyAt({ ms: 0 }, { params: { original_status: off } }); + expect(await policy.retryOn(0, null, status(code))).toBe(true); + } + }, + ); + + test.each([ + ["the string '503'", "503"], + ["a list with spaces", "404, 503"], + ["an array of strings", ["404", "503"]], + ["an array of numbers", [404, 503]], + ["a number", 503], + ])("allowed_status_codes as %s: a 503 is final, a 502 is retried", async (_label, listed) => { + const params = { allowed_status_codes: listed }; + expect(await policyAt({ ms: 0 }, { params }).retryOn(0, null, status(503))).toBe(false); + expect(await policyAt({ ms: 0 }, { params }).retryOn(0, null, status(502))).toBe(true); + }); + + test("allowed_status_codes '422,429': a listed 422 or 429 is final", async () => { + const params = { allowed_status_codes: "422,429" }; + const other = Response.json({ code: "RESP001" }, { status: 422 }); + expect(await policyAt({ ms: 0 }, { params }).retryOn(0, null, other)).toBe(false); + expect(await policyAt({ ms: 0 }, { params }).retryOn(0, null, status(429))).toBe(false); + expect(await policyAt({ ms: 0 }, { params }).retryOn(0, null, status(503))).toBe(true); + }); + + test.each([ + ["original_status", { original_status: true }], + ["allowed_status_codes", { allowed_status_codes: "503" }], + ])("a refused connection is still retried with %s", async (_label, params) => { + const policy = policyAt({ ms: 0 }, { params }); + expect(await policy.retryOn(0, fetchFailed("ECONNREFUSED"), null)).toBe(true); + }); +}); + describe("retryPolicy", () => { test("stops after `retries` retries", async () => { const policy = policyAt({ ms: 0 }, { retries: 2 }); @@ -395,6 +447,79 @@ describe("ZenRows client retries", () => { expect((await pending).status).toBe(code); expect(calls).toBe(4); }); + + describe("a status from the target site", () => { + /** Answers every GET with `answer()` and records the query string of each request. */ + function recordGets(answer: () => Response) { + const queries: URLSearchParams[] = []; + server.use( + http.get(apiUrl, ({ request }) => { + queries.push(new URL(request.url).searchParams); + return answer(); + }), + ); + return queries; + } + + test.each(retriedAnswers)( + "fetch() with original_status: true is sent once on a %i", + async (code, answer) => { + const queries = recordGets(answer); + + const pending = new ZenRows("API_KEY", { retries: 3 }).fetch(url, { + original_status: true, + }); + await vi.advanceTimersByTimeAsync(60_000); + + expect((await pending).status).toBe(code); + expect(queries).toHaveLength(1); + expect(queries[0]?.get("original_status")).toBe("true"); + }, + ); + + test.each([ + ["false", { original_status: false }], + ["absent", {}], + ])("fetch() with original_status %s is retried on a 503", async (_label, config) => { + const queries = recordGets(() => new HttpResponse(null, { status: 503 })); + + const pending = new ZenRows("API_KEY", { retries: 3 }).fetch(url, config); + await vi.advanceTimersByTimeAsync(60_000); + + expect((await pending).status).toBe(503); + expect(queries).toHaveLength(4); + }); + + test("extract() with original_status: true is sent once on a 503", async () => { + const queries = recordGets(() => new HttpResponse(null, { status: 503 })); + + const pending = new ZenRows("API_KEY", { retries: 3 }).extract(url, { + original_status: true, + }); + await vi.advanceTimersByTimeAsync(60_000); + + expect((await pending).status).toBe(503); + expect(queries).toHaveLength(1); + }); + + test("allowed_status_codes '503': a 503 is sent once, a 502 is retried", async () => { + const client = new ZenRows("API_KEY", { retries: 3 }); + const config = { allowed_status_codes: "503" }; + + const once = recordGets(() => new HttpResponse(null, { status: 503 })); + const first = client.fetch(url, config); + await vi.advanceTimersByTimeAsync(60_000); + expect((await first).status).toBe(503); + expect(once).toHaveLength(1); + expect(once[0]?.get("allowed_status_codes")).toBe("503"); + + const retried = recordGets(() => new HttpResponse(null, { status: 502 })); + const second = client.fetch(url, config); + await vi.advanceTimersByTimeAsync(60_000); + expect((await second).status).toBe(502); + expect(retried).toHaveLength(4); + }); + }); }); describe("prepared-domain calls are never retried", () => { diff --git a/tests/retry-network.test.ts b/tests/retry-network.test.ts index c4713df..ec9525e 100644 --- a/tests/retry-network.test.ts +++ b/tests/retry-network.test.ts @@ -115,6 +115,37 @@ describe("failures before the request was sent are retried", () => { }); }); +describe("original_status: true", () => { + test("a refused connection is still retried", async () => { + fastRandom(); + const calls = routeApiTo(await closedPortOrigin()); + + const error = await new ZenRows("API_KEY") + .fetch("https://example.com", { original_status: true }) + .catch((e) => e); + + expect(error.cause.code).toBe("ECONNREFUSED"); + expect(calls.count).toBe(3); + }); + + test("a 503 from the server is sent once", async () => { + fastRandom(); + const server = await startServer((_req, res) => { + res.writeHead(503); + res.end(); + }); + routeApiTo(server.origin); + + const response = await new ZenRows("API_KEY").fetch("https://example.com", { + original_status: true, + }); + await server.close(); + + expect(response.status).toBe(503); + expect(server.requests).toEqual(["GET"]); + }); +}); + describe("post() is sent once, even on a refused connection", () => { test.each([ ["without an Idempotency-Key", {}], From 37b0183a9f5424135bdcccacf3387fa8df67cbcf Mon Sep 17 00:00:00 2001 From: gonzalo-zenrows Date: Tue, 20 Oct 2026 22:24:35 +0200 Subject: [PATCH 6/6] fix(client): retry a connect failure on every address, and a connect timeout With happy eyeballs, Node tries each address family of a host, such as ::1 and 127.0.0.1 for localhost. When every try fails, the fetch error's cause is an AggregateError with one error per try. Node copies the first try's code onto the aggregate. So a refused IPv6 try and a reset IPv4 try looked like a plain refused connection, and the client retried it. 1. For an AggregateError, the inner errors decide. The client retries only when every inner error is a connect failure. An empty one is not retried. 2. A connect failure is a code from the existing list, or ETIMEDOUT from the connect syscall. The list itself is unchanged. 3. A single ETIMEDOUT from connect is now retried too. An ETIMEDOUT from any other syscall is a read timeout and is never retried. 4. The README lists the connect timeout under what the client retries. Unit tests cover each mix of inner errors, with and without a code on the aggregate, and plain ETIMEDOUT from connect, from read and with no syscall. A real-socket test sends fetch() to localhost on a closed port and checks three calls with retries set to 2. It runs when localhost resolves to both address families. --- README.md | 1 + src/retry.ts | 28 ++++++++++++++++-- tests/retries.test.ts | 57 +++++++++++++++++++++++++++++++++++++ tests/retry-network.test.ts | 29 +++++++++++++++++++ 4 files changed, 112 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9991395..3b8e8cb 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,7 @@ What the client retries: 1. Status codes 429, 502, 503 and 504. The API does not charge for them. 2. A `422`, except one with `code: "no_data_extracted"`. That one is a final answer (see [Extract](#extract)). 3. A failure before the request reached the API: connection refused, a name that does not resolve, an unreachable network or host, or a connect timeout. +4. A connection that timed out while connecting. When Node tries several addresses, every try must fail this way or as in item 3. What the client never retries: diff --git a/src/retry.ts b/src/retry.ts index e401d8a..54b0c66 100644 --- a/src/retry.ts +++ b/src/retry.ts @@ -141,10 +141,32 @@ export function isRetryEligible(method: string, body: unknown): boolean { return IDEMPOTENT_METHODS.has(method.toUpperCase()) && isReplayableBody(body); } -/** True when fetch failed before the request was written to the server (see `PRE_SEND_ERROR_CODES`). */ +interface ErrorCause { + code?: unknown; + syscall?: unknown; + errors?: unknown; +} + +/** True when one error says the connection was never made: a listed code, or a connect timeout. */ +function isConnectFailure(cause: ErrorCause | null | undefined): boolean { + const code = cause?.code; + if (typeof code !== "string") return false; + return PRE_SEND_ERROR_CODES.has(code) || (code === "ETIMEDOUT" && cause?.syscall === "connect"); +} + +/** + * True when fetch failed before the request was written to the server (see `PRE_SEND_ERROR_CODES`). + * With happy eyeballs, Node tries each address family and throws an `AggregateError` with one + * error per try. Its own `code` is the first try's, so the inner errors decide: every one must + * be a connect failure. An `ETIMEDOUT` from any syscall but `connect` is a read timeout. + */ function isPreSendFailure(error: unknown): boolean { - const code = (error as { cause?: { code?: unknown } } | null)?.cause?.code; - return typeof code === "string" && PRE_SEND_ERROR_CODES.has(code); + const cause = (error as { cause?: ErrorCause } | null)?.cause; + const inner = cause?.errors; + if (Array.isArray(inner)) { + return inner.length > 0 && inner.every((each) => isConnectFailure(each as ErrorCause)); + } + return isConnectFailure(cause); } /** True when the answer says "try again": 429, 502, 503, 504, or a 422 other than `no_data_extracted`. */ diff --git a/tests/retries.test.ts b/tests/retries.test.ts index 703c6da..9f1ba40 100644 --- a/tests/retries.test.ts +++ b/tests/retries.test.ts @@ -30,6 +30,25 @@ const status = (code: number, headers: Record = {}) => const fetchFailed = (code: string) => new TypeError("fetch failed", { cause: Object.assign(new Error(code), { code }) }); +/** A Node system error, as a socket raises it: a `code` and the `syscall` that failed. */ +const systemError = (code: string, syscall?: string) => + Object.assign(new Error(`${syscall ?? "?"} ${code}`), { code, syscall }); + +/** + * `fetch failed` caused by an `AggregateError`, as Node throws it when every address it tried + * failed. Two shapes: Node copies the first inner error's `code` onto the aggregate, and a + * runtime that does not leaves it off. The policy must judge both by the inner errors alone. + */ +const aggregateFailed = (errors: Error[]) => { + const withCode = Object.assign(new AggregateError(errors), { + code: (errors[0] as { code?: string } | undefined)?.code, + }); + return [ + new TypeError("fetch failed", { cause: withCode }), + new TypeError("fetch failed", { cause: new AggregateError(errors) }), + ]; +}; + /** The highest value `Math.random()` can return. */ const RANDOM_MAX = 1 - Number.EPSILON; @@ -105,6 +124,44 @@ describe("conformance vectors", () => { expect(await policyAt({ ms: 0 }).retryOn(0, error, null)).toBe(true); }); + test("a plain connect timeout (ETIMEDOUT from connect) is retried", async () => { + const error = new TypeError("fetch failed", { cause: systemError("ETIMEDOUT", "connect") }); + expect(await policyAt({ ms: 0 }).retryOn(0, error, null)).toBe(true); + }); + + test.each([ + ["from read", "read"], + ["with no syscall", undefined], + ])("a plain ETIMEDOUT %s is not retried", async (_label, syscall) => { + const error = new TypeError("fetch failed", { cause: systemError("ETIMEDOUT", syscall) }); + expect(await policyAt({ ms: 0 }).retryOn(0, error, null)).toBe(false); + }); + + describe("a happy-eyeballs AggregateError", () => { + const refused = () => systemError("ECONNREFUSED", "connect"); + const connectTimeout = () => systemError("ETIMEDOUT", "connect"); + + test.each([ + ["refused on every address", [refused(), refused()]], + ["a connect timeout on every address", [connectTimeout(), connectTimeout()]], + ["refused on one address, a connect timeout on the other", [refused(), connectTimeout()]], + ])("%s is retried", async (_label, errors) => { + for (const error of aggregateFailed(errors)) { + expect(await policyAt({ ms: 0 }).retryOn(0, error, null)).toBe(true); + } + }); + + test.each([ + ["a read timeout on one address", [systemError("ETIMEDOUT", "read"), refused()]], + ["a reset on one address", [refused(), systemError("ECONNRESET", "read")]], + ["no inner error", []], + ])("%s is not retried", async (_label, errors) => { + for (const error of aggregateFailed(errors)) { + expect(await policyAt({ ms: 0 }).retryOn(0, error, null)).toBe(false); + } + }); + }); + test.each([ ["a socket closed after the request was sent", fetchFailed("UND_ERR_SOCKET")], ["a connection reset", fetchFailed("ECONNRESET")], diff --git a/tests/retry-network.test.ts b/tests/retry-network.test.ts index ec9525e..7949ab3 100644 --- a/tests/retry-network.test.ts +++ b/tests/retry-network.test.ts @@ -1,5 +1,6 @@ // Retry decisions against a real node:http server through Node's real fetch, so each failure // reaches the policy in the exact shape undici produces. The request mocks are off in this file. +import { lookup } from "node:dns/promises"; import http from "node:http"; import type { AddressInfo, Socket } from "node:net"; import { afterAll, afterEach, beforeAll, describe, expect, test, vi } from "vitest"; @@ -102,6 +103,12 @@ async function undiciAgent(origin: string): Promise obj const fastRandom = () => vi.spyOn(Math, "random").mockReturnValue(0); +/** True when `localhost` resolves to an IPv4 and an IPv6 address, as Node's fetch looks it up. */ +const localhostHasBothFamilies = await lookup("localhost", { all: true }).then( + (addresses) => new Set(addresses.map((each) => each.family)).size === 2, + () => false, +); + describe("failures before the request was sent are retried", () => { test("a refused connection is retried `retries` times, then throws", async () => { fastRandom(); @@ -113,6 +120,28 @@ describe("failures before the request was sent are retried", () => { expect(error.cause.code).toBe("ECONNREFUSED"); expect(calls.count).toBe(3); }); + + // `localhost` resolves to ::1 and 127.0.0.1 on most hosts. Node then tries both (happy + // eyeballs) and the cause is an AggregateError with one ECONNREFUSED per address. + test.skipIf(!localhostHasBothFamilies)( + "a refused connection to localhost, on both address families, is retried", + async () => { + fastRandom(); + const port = new URL(await closedPortOrigin()).port; + const calls = routeApiTo(`http://localhost:${port}`); + + const error = await new ZenRows("API_KEY", { retries: 2 }) + .fetch("https://example.com") + .catch((e) => e); + + expect(error).toBeInstanceOf(TypeError); + expect(error.cause).toBeInstanceOf(AggregateError); + const innerCodes = error.cause.errors.map((inner: { code: string }) => inner.code); + expect(innerCodes.length).toBeGreaterThanOrEqual(2); + expect(new Set(innerCodes)).toEqual(new Set(["ECONNREFUSED"])); + expect(calls.count).toBe(3); + }, + ); }); describe("original_status: true", () => {