From 66fb8548babdb5c9ff124cd0706b979ae17ea51c Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Wed, 14 Oct 2026 14:28:30 +0000 Subject: [PATCH 1/8] feat(crawl): return each page as Markdown, and read the discovery strategies a crawl ran with `outputFormat` takes `markdown` as well as `html`: the crawl returns each kept URL's page converted to Markdown, read with `content()` or `download()` as HTML already was. A crawl reports `discovery`, the strategies it finds URLs with on each page it walks, and a create can ask for them. `links` follows the links in the page's HTML, and is what a crawl runs with when the create leaves it out. The field stays optional, so the type parses an older answer too. `content(..., { download: true })` asks for the page as a file to save rather than one to show. The text is the same; the server sends a file name built from the content id. `CrawlErrorCode` gains `api_key_cap_reached`, which a crawl ends with when the key that created it reaches one of its credit caps. --- acceptance/crawl/surface/test.mjs | 36 +++++++++++++++++++++++++++++++ src/crawl/client.ts | 25 ++++++++++++++++----- src/crawl/types.ts | 12 +++++++++-- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index 615cc8e..cf252b1 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -624,6 +624,40 @@ await liveCheck( }; }, ); +await liveCheck( + "22", + "Discovery", + 'create({ discovery: ["links"] }) is echoed back, and a create without it reads ["links"]', + async () => { + const given = await create({ ...base, discovery: ["links"] }); + const leftOut = await create({ ...base, maxItems: 1 }); + for (const c of [given, leftOut]) await client.stop(c.crawl_id); + const seen = [given.discovery, leftOut.discovery]; + const asked = JSON.stringify(seen[0]) === JSON.stringify(["links"]); + const defaulted = JSON.stringify(seen[1]) === JSON.stringify(["links"]); + return { + status: passIf(asked && defaulted), + actual: JSON.stringify({ 'discovery: ["links"]': seen[0], "left out": seen[1] }), + }; + }, +); +await liveCheck( + "23", + "Content as a file", + "content(..., { download: true }) returns the same page as content() does", + async () => { + if (!B) return { status: SKIP, actual: "no html crawl to read a page from" }; + const page = await client.get(B.crawl_id, { limit: 100 }); + const fetched = page.results.find((r) => r.content_url); + if (!fetched) return { status: SKIP, actual: "no result had been fetched" }; + const plain = await client.content(B.crawl_id, fetched); + const asFile = await client.content(B.crawl_id, fetched, { download: true }); + return { + status: passIf(Boolean(plain) && plain === asFile), + actual: `same ${plain.length} chars with and without download: true`, + }; + }, +); await check("15", "Bad key", "any call with the key `invalid` → 401", async () => { const bad = new ZenRowsCrawlClient("invalid", { baseURL: API_BASE, retries: 0 }); const r = await errOf(bad.get("c_does_not_exist")); @@ -717,11 +751,13 @@ H.naming({ include_patterns: "includePatterns", exclude_patterns: "excludePatterns", output_format: "outputFormat", + discovery: "discovery", idempotency_key: "idempotencyKey", cursor: "cursor", limit: "limit", timeout: "timeout", content_id: "contentId", + download: "download", }, }); record("17", "Naming", SKIP, "the API's names in this surface's casing", "computed by the report"); diff --git a/src/crawl/client.ts b/src/crawl/client.ts index 08149cf..8b0efff 100644 --- a/src/crawl/client.ts +++ b/src/crawl/client.ts @@ -2,6 +2,7 @@ import { BatchTransport, DEFAULT_RETRIES } from "../batch/transport.js"; import { ZenRowsCrawlError, crawlError } from "./errors.js"; import type { Crawl, + CrawlDiscoveryStrategy, CrawlDownload, CrawlExportLine, CrawlList, @@ -53,8 +54,13 @@ export interface CreateCrawlParams { includePatterns?: string[]; /** Drop a URL that contains any of these substrings, even if it matches an include pattern. */ excludePatterns?: string[]; - /** Also fetch each kept URL's page. Absent means URLs only. */ + /** Also fetch each kept URL's page, in this format. Absent means URLs only. */ outputFormat?: CrawlOutputFormat; + /** + * How the crawl finds the URLs it follows on each page it walks. `["links"]` follows the + * links in the page's HTML, and is what a crawl runs with when you leave this out. + */ + discovery?: CrawlDiscoveryStrategy[]; } export interface WaitForCrawlOptions { @@ -135,6 +141,7 @@ export class ZenRowsCrawlClient { include_patterns: params.includePatterns, exclude_patterns: params.excludePatterns, output_format: params.outputFormat, + discovery: params.discovery, }; return this.transport.requestJson("POST", "/crawls", { body, @@ -203,14 +210,22 @@ export class ZenRowsCrawlClient { } /** - * One kept URL's page as fetched (HTML). Pass the content id, a `content_url`, or the - * `CrawlResult` itself (its `content_url` is present once `content_status` is `fetched`). - * Throws `TypeError` when no content id can be read. + * One kept URL's page, in the crawl's output format. Pass the content id, a `content_url`, + * or the `CrawlResult` itself (its `content_url` is present once `content_status` is + * `fetched`). Throws `TypeError` when no content id can be read. + * + * With `download: true` the server sends the page as a file to save, named after its + * content id, instead of one to show. The text is the same either way. */ - async content(crawlId: string, contentId: string | CrawlResult): Promise { + async content( + crawlId: string, + contentId: string | CrawlResult, + options: { download?: boolean } = {}, + ): Promise { return this.transport.requestText( "GET", `/crawls/${encodeURIComponent(crawlId)}/contents/${encodeURIComponent(contentIdOf(contentId))}`, + options.download ? { query: { download: true } } : {}, ); } diff --git a/src/crawl/types.ts b/src/crawl/types.ts index 600e0ab..1e8ab16 100644 --- a/src/crawl/types.ts +++ b/src/crawl/types.ts @@ -7,14 +7,17 @@ export type CrawlStopReason = "max_items" | "max_pages" | "user"; /** Why a crawl failed (`status: failed`). */ export type CrawlErrorCode = | "insufficient_credits" + | "api_key_cap_reached" | "seed_unreachable" | "domain_not_allowed" | "no_items_found" | "internal_error"; /** Where a kept URL's page stands, when the crawl has an `output_format`. */ export type CrawlContentStatus = "pending" | "fetched" | "failed"; -/** Return each kept URL's page as fetched. Absent means URLs only. */ -export type CrawlOutputFormat = "html"; +/** Return each kept URL's page, in this format. Absent means URLs only. */ +export type CrawlOutputFormat = "html" | "markdown"; +/** How a crawl finds the URLs it follows on each page it walks. */ +export type CrawlDiscoveryStrategy = "links"; export interface CrawlCoverage { pages_fetched: number; @@ -43,6 +46,11 @@ export interface Crawl { include_patterns?: string[]; exclude_patterns?: string[]; output_format?: Extensible; + /** + * The strategies the crawl finds URLs with. `["links"]` when the create left it out. + * Absent when the server does not send the field. + */ + discovery?: Extensible[]; duplicates_removed?: number; /** Absent while the crawl runs. */ finished_at?: string; From b2953d03d312bab605cfe5c6f8635a9181c52a9d Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 08:27:55 +0000 Subject: [PATCH 2/8] chore(crawl): record the contract this surface follows The lock names the API description this surface was synced to, by its blob id, so a later drift check compares against it instead of reporting the surface as never synced. Two operations in that description are not served to callers and no surface exposes them; the lock lists them as pending, so a change to either does not read as drift here. --- acceptance/crawl/contract.lock | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 acceptance/crawl/contract.lock diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock new file mode 100644 index 0000000..435df9b --- /dev/null +++ b/acceptance/crawl/contract.lock @@ -0,0 +1,3 @@ +1adcbc2fd87e3f508317f3ce24c3d48573aec657 +pending GET /healthz +pending POST /internal/v1/crawls/{crawl_id}/stop From d10510e871c8b041ee1fae808645464d7bc2c88f Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 12:54:34 +0000 Subject: [PATCH 3/8] docs(crawl): say that a crawl out of credits ends stopped, with error set The Crawl API now ends a crawl that runs out of credits, or whose API key hits a credit cap, as `stopped` with `error.code` `insufficient_credits` or `api_key_cap_reached` and no `stop_reason`, instead of `failed`. The type comments, the `wait` comment and the README said `error` belongs to a `failed` crawl and `stopped` to `stop()`. They now describe both cases, the README example handles the credit stop, and a unit test pins that `wait` resolves such a crawl with its error and no stop reason. --- README.md | 4 +++- src/crawl/client.ts | 5 +++-- src/crawl/types.ts | 17 ++++++++++++++--- tests/crawl-client.test.ts | 15 +++++++++++++++ 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 21328a2..81481ea 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,8 @@ const apiKey = "YOUR-API-KEY"; const ended = await client.crawl.wait(crawl.crawl_id); // completed, stopped, failed, or running on timeout if (ended.status === "failed") throw new Error(ended.error?.detail); + // Out of credits or at the key's cap: stopped, with error set. What it fetched stays readable. + if (ended.status === "stopped" && ended.error) console.warn(ended.error.detail); for await (const result of client.crawl.results(crawl.crawl_id)) { console.log(result.url); @@ -330,7 +332,7 @@ Errors throw a `ZenRowsCrawlError` with `status`, `code`, `detail` and `retryAft `content()` takes a content id, a `content_url` or a result, and throws a `TypeError` when it cannot read a content id from it. Requests retry network errors and 502/503/504 (and 429 on reads) when it is safe; a `create()` is retried only with an `idempotencyKey`, and a `stop()` as described above. -`outputFormat` takes `"html"`; without it a crawl returns URLs only. `maxPages` and `maxItems` default to 100 each. A crawl created without `depth` has no `depth` field; one created with it echoes it. Response enum fields (`status`, `stop_reason`, `error.code`, `content_status`) are `Extensible<...>`: the known values plus any `string`. A failed crawl carries `error.code` and `error.detail`. A download line's `content` is a string, or an object for a crawl that returns JSON. +`outputFormat` takes `"html"`; without it a crawl returns URLs only. `maxPages` and `maxItems` default to 100 each. A crawl created without `depth` has no `depth` field; one created with it echoes it. Response enum fields (`status`, `stop_reason`, `error.code`, `content_status`) are `Extensible<...>`: the known values plus any `string`. A failed crawl carries `error.code` and `error.detail`. So does a crawl that ran out of credits, or whose API key hit a credit cap: it ends `stopped`, with `error.code` `insufficient_credits` or `api_key_cap_reached` and no `stop_reason`, and the pages it fetched before that stay readable. A download line's `content` is a string, or an object for a crawl that returns JSON. The crawl client (`ZenRowsCrawlClient`) also works standalone. Both forms take the same options: diff --git a/src/crawl/client.ts b/src/crawl/client.ts index 8b0efff..a79fcc7 100644 --- a/src/crawl/client.ts +++ b/src/crawl/client.ts @@ -247,8 +247,9 @@ export class ZenRowsCrawlClient { /** * Poll until the crawl ends (any status but `running`) or `timeout` seconds pass, and return - * it, without results. A `failed` crawl resolves too: read `error`. On timeout the crawl is - * returned with status `running` and keeps running. + * it, without results. A `failed` crawl resolves too: read `error`. So does a crawl that + * `stopped` because credits ran out or the API key hit a credit cap; `error` says which. On + * timeout the crawl is returned with status `running` and keeps running. */ async wait(crawlId: string, options: WaitForCrawlOptions = {}): Promise { const deadline = Date.now() + (options.timeout ?? DEFAULT_WAIT_TIMEOUT_SECONDS) * 1000; diff --git a/src/crawl/types.ts b/src/crawl/types.ts index 1e8ab16..541c2ed 100644 --- a/src/crawl/types.ts +++ b/src/crawl/types.ts @@ -2,9 +2,16 @@ import type { Extensible } from "../batch/types.js"; /** `running` until the crawl ends; the other three are terminal. */ export type CrawlStatus = "running" | "completed" | "stopped" | "failed"; -/** What ended a crawl early: a limit (`status: completed`) or `stop()` (`status: stopped`). */ +/** + * What ended a crawl early: a limit (`status: completed`) or `stop()` (`status: stopped`). Absent on a + * crawl that stopped because the account ran out of credits or the API key hit a credit cap: that one + * carries `error` instead. + */ export type CrawlStopReason = "max_items" | "max_pages" | "user"; -/** Why a crawl failed (`status: failed`). */ +/** + * Why a crawl failed (`status: failed`), or why it stopped on credits: `insufficient_credits` and + * `api_key_cap_reached` come with `status: stopped`, and the pages fetched before that stay readable. + */ export type CrawlErrorCode = | "insufficient_credits" | "api_key_cap_reached" @@ -25,7 +32,11 @@ export interface CrawlCoverage { items_found: number; } -/** Why a crawl failed. Part of the crawl, not an error response. */ +/** + * Why a crawl failed, or why it stopped because the account ran out of credits or the API key hit a + * credit cap. Present when `status` is `failed`, and when it is `stopped` with `code` + * `insufficient_credits` or `api_key_cap_reached`. Part of the crawl, not an error response. + */ export interface CrawlRunError { code: Extensible; detail: string; diff --git a/tests/crawl-client.test.ts b/tests/crawl-client.test.ts index fb28141..b01715c 100644 --- a/tests/crawl-client.test.ts +++ b/tests/crawl-client.test.ts @@ -678,6 +678,21 @@ describe("ZenRowsCrawlClient — wait", () => { expect(ended.error?.code).toBe("seed_unreachable"); }); + test.each(["insufficient_credits", "api_key_cap_reached"])( + "resolves on a crawl stopped on credits (%s), with its error", + async (code) => { + server.use( + http.get(`${BASE}/crawls/c_1`, () => + HttpResponse.json(crawl({ status: "stopped", error: { code, detail: "x" } })), + ), + ); + const ended = await new ZenRowsCrawlClient("API_KEY").wait("c_1"); + expect(ended.status).toBe("stopped"); + expect(ended.stop_reason).toBeUndefined(); + expect(ended.error?.code).toBe(code); + }, + ); + test("returns the running crawl once the timeout runs out", async () => { let polls = 0; server.use( From bd55de38b1bf8a488c5a87dc191b1f73251f6aab Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 17:42:31 +0000 Subject: [PATCH 4/8] docs(crawl): say that patterns filter what a crawl returns, not what it opens The Crawl API now treats include and exclude patterns as a filter on what a crawl returns. The crawl still opens on-site pages that fail them, within `depth` and `maxPages`, to follow their links to pages that match, and never returns those pages. Every page it opens counts toward `maxPages` and is billed. The JSDoc for `maxPages`, `includePatterns` and `excludePatterns`, and the README, now say so, and that a crawl whose patterns match nothing ends `completed` with no results. --- README.md | 2 +- src/crawl/client.ts | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 81481ea..fdea8fc 100644 --- a/README.md +++ b/README.md @@ -274,7 +274,7 @@ const batch = new ZenRowsBatchClient(apiKey, { baseURL: "https://async.api.zenro ### Crawl -[Crawl](https://docs.zenrows.com/crawl/introduction) (new) takes one start URL and returns the URLs behind it, and is reachable via `client.crawl`. It follows the links on each page up to `depth` hops; without `depth` it has no depth limit and follows links until `maxPages` or `maxItems` stops it (each defaults to 100), or no link is left to open. A crawl stays on the start URL's registrable domain; subdomains count. A crawl runs asynchronously: `create()` returns at once, `wait()` polls until it ends, and `results()` reads the URLs it kept. With `outputFormat: "html"` it also fetches each kept page, which you read with `content()` or all at once with `download()`. +[Crawl](https://docs.zenrows.com/crawl/introduction) (new) takes one start URL and returns the URLs behind it, and is reachable via `client.crawl`. It follows the links on each page up to `depth` hops; without `depth` it has no depth limit and follows links until `maxPages` or `maxItems` stops it (each defaults to 100), or no link is left to open. `includePatterns` and `excludePatterns` filter what a crawl returns, not what it opens: it still opens on-site pages that fail them to reach pages that match, and every page it opens counts toward `maxPages` and is billed. A crawl stays on the start URL's registrable domain; subdomains count. A crawl runs asynchronously: `create()` returns at once, `wait()` polls until it ends, and `results()` reads the URLs it kept. With `outputFormat: "html"` it also fetches each kept page, which you read with `content()` or all at once with `download()`. Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. diff --git a/src/crawl/client.ts b/src/crawl/client.ts index a79fcc7..d9dbf99 100644 --- a/src/crawl/client.ts +++ b/src/crawl/client.ts @@ -48,11 +48,22 @@ export interface CreateCrawlParams { depth?: number; /** Stop once this many URLs are kept, 1–100000. Server default 100. */ maxItems?: number; - /** Stop once this many pages are fetched, 1–100000; bounds the cost. Server default 100. */ + /** + * Stop once this many pages are fetched, 1–100000; bounds the cost. Server default 100. Every + * page the crawl opens counts and is billed, returned or not. + */ maxPages?: number; - /** Keep a URL only if it contains at least one of these substrings. */ + /** + * Keep a URL only if it contains at least one of these substrings. Patterns filter what the + * crawl returns, not what it opens: it still opens on-site pages that fail them, to reach pages + * that match, and those fetches count toward `maxPages` and are billed. A crawl whose patterns + * match nothing ends `completed` with no results. + */ includePatterns?: string[]; - /** Drop a URL that contains any of these substrings, even if it matches an include pattern. */ + /** + * Drop a URL that contains any of these substrings, even if it matches an include pattern. The + * crawl may still open it to follow its links. + */ excludePatterns?: string[]; /** Also fetch each kept URL's page, in this format. Absent means URLs only. */ outputFormat?: CrawlOutputFormat; From 3c26ebb1964ccc1b8ab926ff2579dfff294e411d Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Thu, 15 Oct 2026 17:42:31 +0000 Subject: [PATCH 5/8] test(crawl): expect a crawl whose patterns match nothing to end completed Acceptance check 19 asserted that an include pattern matching nothing ends the crawl `failed` with `no_items_found`. The Crawl API now ends such a crawl `completed` with 0 results and no `error`, so the check asserts that, and reads the crawl's results to confirm none came back. Check 21 still reads the defaults from the same crawl. --- acceptance/crawl/README.md | 2 +- acceptance/crawl/surface/test.mjs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index c61773c..ad098de 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -78,7 +78,7 @@ and signal checks. | 16 | Errors carry the server code | SDKs: `code` holds it, empty when the body has none; MCP, CLI: `server_code` | | 17 | Naming | the API's operation and parameter names, in this surface's casing | | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | -| 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | +| 19 | Patterns match nothing | an include pattern that matches nothing ends `completed` with 0 results and no `error`; SDKs return it; MCP and CLI say `ok: true` (CLI: exit 0) | | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index cf252b1..a7bc457 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -693,21 +693,21 @@ await check( ); await cap.close(); -// ---------- 19-21: a failed crawl, results after a stop, the documented defaults ---------- +// ---------- 19-21: patterns that match nothing, results after a stop, the documented defaults ---------- await liveCheck( "19", - "Failed crawl", - "a crawl that keeps nothing ends failed (no_items_found); wait returns it, not raises", + "Patterns match nothing", + "a crawl whose patterns match nothing ends completed with 0 results and no error", async () => { F = await create({ url: CRAWL_URL, depth: 1, includePatterns: [H.NO_MATCH] }); const w = await client.wait(F.crawl_id); + const urls = []; + for await (const x of client.results(F.crawl_id)) urls.push(x.url); const ok = - w.status === "failed" && - w.error?.code === "no_items_found" && - typeof w.error?.detail === "string"; + w.status === "completed" && !w.error && w.coverage.items_found === 0 && urls.length === 0; return { status: passIf(ok), - actual: `${F.crawl_id}: ${w.status}; error ${JSON.stringify(w.error)}; returned, no exception`, + actual: `${F.crawl_id}: ${w.status}; items_found ${w.coverage.items_found}, ${urls.length} results; error ${JSON.stringify(w.error ?? null)}`, evidence: JSON.stringify(w), }; }, From 0414e96e25bbac7e5bdf07e6d328ad3cec86153f Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 16 Oct 2026 11:23:30 +0000 Subject: [PATCH 6/8] docs(crawl): say that patterns limit what a crawl opens, and that 0 results is not a failure A URL that fails includePatterns or excludePatterns is neither returned nor opened, so the docs no longer say the crawl opens such pages or bills them. A crawl whose pages held nothing new to follow ends completed with no results and no error. seed_unreachable covers an empty start page, and no_items_found means Extract found no data on the start page. --- README.md | 4 ++-- src/crawl/client.ts | 16 ++++------------ src/crawl/types.ts | 4 ++++ 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9859080..bcc8991 100644 --- a/README.md +++ b/README.md @@ -274,7 +274,7 @@ const batch = new ZenRowsBatchClient(apiKey, { baseURL: "https://async.api.zenro ### Crawl -[Crawl](https://docs.zenrows.com/crawl/introduction) (new) takes one start URL and returns the URLs behind it, and is reachable via `client.crawl`. It follows the links on each page up to `depth` hops; without `depth` it has no depth limit and follows links until `maxPages` or `maxItems` stops it (each defaults to 100), or no link is left to open. `includePatterns` and `excludePatterns` filter what a crawl returns, not what it opens: it still opens on-site pages that fail them to reach pages that match, and every page it opens counts toward `maxPages` and is billed. A crawl stays on the start URL's registrable domain; subdomains count. A crawl runs asynchronously: `create()` returns at once, `wait()` polls until it ends, and `results()` reads the URLs it kept. With `outputFormat: "html"` it also fetches each kept page, which you read with `content()` or all at once with `download()`. +[Crawl](https://docs.zenrows.com/crawl/introduction) (new) takes one start URL and returns the URLs behind it, and is reachable via `client.crawl`. It follows the links on each page up to `depth` hops; without `depth` it has no depth limit and follows links until `maxPages` or `maxItems` stops it (each defaults to 100), or no link is left to open. A crawl stays on the start URL's registrable domain; subdomains count. A crawl runs asynchronously: `create()` returns at once, `wait()` polls until it ends, and `results()` reads the URLs it kept. With `outputFormat: "html"` it also fetches each kept page, which you read with `content()` or all at once with `download()`. Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. @@ -332,7 +332,7 @@ Errors throw a `ZenRowsCrawlError` with `status`, `code`, `detail` and `retryAft `content()` takes a content id, a `content_url` or a result, and throws a `TypeError` when it cannot read a content id from it. Requests retry network errors and 502/503/504 (and 429 on reads) when it is safe; a `create()` is retried only with an `idempotencyKey`, and a `stop()` as described above. -`outputFormat` takes `"html"`; without it a crawl returns URLs only. `maxPages` and `maxItems` default to 100 each. A crawl created without `depth` has no `depth` field; one created with it echoes it. Response enum fields (`status`, `stop_reason`, `error.code`, `content_status`) are `Extensible<...>`: the known values plus any `string`. A failed crawl carries `error.code` and `error.detail`. So does a crawl that ran out of credits, or whose API key hit a credit cap: it ends `stopped`, with `error.code` `insufficient_credits` or `api_key_cap_reached` and no `stop_reason`, and the pages it fetched before that stay readable. A download line's `content` is a string, or an object for a crawl that returns JSON. +`outputFormat` takes `"html"`; without it a crawl returns URLs only. `maxPages` and `maxItems` default to 100 each. A crawl created without `depth` has no `depth` field; one created with it echoes it. Response enum fields (`status`, `stop_reason`, `error.code`, `content_status`) are `Extensible<...>`: the known values plus any `string`. A crawl whose pages held nothing new to follow (no link, only links to other sites, or only links the patterns reject) ends `completed` with no results and no `error`: check the patterns against the URLs the start page links to. A failed crawl carries `error.code` and `error.detail`. So does a crawl that ran out of credits, or whose API key hit a credit cap: it ends `stopped`, with `error.code` `insufficient_credits` or `api_key_cap_reached` and no `stop_reason`, and the pages it fetched before that stay readable. A download line's `content` is a string, or an object for a crawl that returns JSON. The crawl client (`ZenRowsCrawlClient`) also works standalone. Both forms take the same options: diff --git a/src/crawl/client.ts b/src/crawl/client.ts index d9dbf99..1ebeaf0 100644 --- a/src/crawl/client.ts +++ b/src/crawl/client.ts @@ -48,22 +48,14 @@ export interface CreateCrawlParams { depth?: number; /** Stop once this many URLs are kept, 1–100000. Server default 100. */ maxItems?: number; - /** - * Stop once this many pages are fetched, 1–100000; bounds the cost. Server default 100. Every - * page the crawl opens counts and is billed, returned or not. - */ + /** Stop once this many pages are fetched, 1–100000; bounds the cost. Server default 100. */ maxPages?: number; /** - * Keep a URL only if it contains at least one of these substrings. Patterns filter what the - * crawl returns, not what it opens: it still opens on-site pages that fail them, to reach pages - * that match, and those fetches count toward `maxPages` and are billed. A crawl whose patterns - * match nothing ends `completed` with no results. + * Keep a URL only if it contains at least one of these substrings. A crawl whose patterns reject + * every link ends `completed` with no results and no `error`. */ includePatterns?: string[]; - /** - * Drop a URL that contains any of these substrings, even if it matches an include pattern. The - * crawl may still open it to follow its links. - */ + /** Drop a URL that contains any of these substrings, even if it matches an include pattern. */ excludePatterns?: string[]; /** Also fetch each kept URL's page, in this format. Absent means URLs only. */ outputFormat?: CrawlOutputFormat; diff --git a/src/crawl/types.ts b/src/crawl/types.ts index 541c2ed..5b8e05b 100644 --- a/src/crawl/types.ts +++ b/src/crawl/types.ts @@ -11,6 +11,10 @@ export type CrawlStopReason = "max_items" | "max_pages" | "user"; /** * Why a crawl failed (`status: failed`), or why it stopped on credits: `insufficient_credits` and * `api_key_cap_reached` come with `status: stopped`, and the pages fetched before that stay readable. + * `seed_unreachable` covers a start page that could not be fetched, was blocked, or came back empty, + * with no visible text and no link. `no_items_found` means that, in a crawl that runs Extract, Extract + * read the start page and found no data on it. A crawl that returns nothing for any other reason is not + * a failure: it ends `completed` with no results. */ export type CrawlErrorCode = | "insufficient_credits" From b607aeb1225c6cc295cf26a254b6818fbe9433ae Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 16 Oct 2026 11:25:04 +0000 Subject: [PATCH 7/8] docs(crawl): describe no_items_found and seed_unreachable without product names no_items_found now reads as the crawl finding no data it could use on the start page, with the advice to start from a listing or product page. --- src/crawl/types.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crawl/types.ts b/src/crawl/types.ts index 5b8e05b..ebc5c0a 100644 --- a/src/crawl/types.ts +++ b/src/crawl/types.ts @@ -11,10 +11,10 @@ export type CrawlStopReason = "max_items" | "max_pages" | "user"; /** * Why a crawl failed (`status: failed`), or why it stopped on credits: `insufficient_credits` and * `api_key_cap_reached` come with `status: stopped`, and the pages fetched before that stay readable. - * `seed_unreachable` covers a start page that could not be fetched, was blocked, or came back empty, - * with no visible text and no link. `no_items_found` means that, in a crawl that runs Extract, Extract - * read the start page and found no data on it. A crawl that returns nothing for any other reason is not - * a failure: it ends `completed` with no results. + * `seed_unreachable`: the start page could not be reached, was blocked, or came back empty, with no + * visible text and no link. `no_items_found`: the crawl read the start page but found no data it could + * use, so it had nothing to follow; start from a listing or product page. A crawl that returns nothing + * for any other reason is not a failure: it ends `completed` with no results. */ export type CrawlErrorCode = | "insufficient_credits" From 6ed83f5ba7eb1c55c374ec8d4fe738d52e9e6ac2 Mon Sep 17 00:00:00 2001 From: "zenrows-dev-pool[bot]" Date: Fri, 16 Oct 2026 11:28:46 +0000 Subject: [PATCH 8/8] test(crawl): name acceptance check 19 in the report as the patterns that match nothing The check expects a completed crawl with 0 results, so the report no longer calls it a failed crawl. --- acceptance/crawl/surface/harness.mjs | 2 +- acceptance/crawl/tools/report.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/crawl/surface/harness.mjs b/acceptance/crawl/surface/harness.mjs index 763d581..f0b3b5d 100644 --- a/acceptance/crawl/surface/harness.mjs +++ b/acceptance/crawl/surface/harness.mjs @@ -10,7 +10,7 @@ export const HAS_KEY = KEY.length > 0; export const CRAWL_URL = process.env.CRAWL_URL; export const INCLUDE = process.env.CRAWL_INCLUDE; export const API_BASE = process.env.CRAWL_API_BASE || "https://api.zenrows.com/v1"; -/** An include pattern no URL contains: the crawl keeps nothing and ends failed (check 19). */ +/** An include pattern no URL contains: the crawl keeps nothing and completes (check 19). */ export const NO_MATCH = "/crawl-acceptance-matches-nothing/"; /** The default every client documents for max_pages and max_items (check 21). */ export const DOCUMENTED_DEFAULT = 100; diff --git a/acceptance/crawl/tools/report.py b/acceptance/crawl/tools/report.py index 94488f8..83e01e2 100644 --- a/acceptance/crawl/tools/report.py +++ b/acceptance/crawl/tools/report.py @@ -18,7 +18,7 @@ "1": "Create", "2": "Create with html", "3": "Idempotency", "4": "Wait", "5": "Wait runs out", "6": "Get", "7": "List", "8": "Results", "9": "Content", "10": "Download", "11": "Stop", "12": "Not found", "13": "Invalid input", "14": "Gated option", "15": "Bad key", "16": "Errors carry the server code", - "17": "Naming", "18": "Copy", "19": "Failed crawl", "20": "Results after stop", "21": "Documented defaults", + "17": "Naming", "18": "Copy", "19": "Patterns match nothing", "20": "Results after stop", "21": "Documented defaults", } # The API's operation and parameter names. Each surface uses them in its own casing. CANON_OPS = ["create", "get", "list", "results", "content", "download", "stop", "wait"]