diff --git a/README.md b/README.md index c1b21a9..bcc8991 100644 --- a/README.md +++ b/README.md @@ -332,7 +332,7 @@ Errors throw a `ZenRowsCrawlError` with `status`, `code`, `detail` and `retryAft `content()` takes a content id, a `content_url` or a result, and throws a `TypeError` when it cannot read a content id from it. Requests retry network errors and 502/503/504 (and 429 on reads) when it is safe; a `create()` is retried only with an `idempotencyKey`, and a `stop()` as described above. -`outputFormat` takes `"html"`; without it a crawl returns URLs only. `maxPages` and `maxItems` default to 100 each. A crawl created without `depth` has no `depth` field; one created with it echoes it. Response enum fields (`status`, `stop_reason`, `error.code`, `content_status`) are `Extensible<...>`: the known values plus any `string`. A failed crawl carries `error.code` and `error.detail`. So does a crawl that ran out of credits, or whose API key hit a credit cap: it ends `stopped`, with `error.code` `insufficient_credits` or `api_key_cap_reached` and no `stop_reason`, and the pages it fetched before that stay readable. A download line's `content` is a string, or an object for a crawl that returns JSON. +`outputFormat` takes `"html"`; without it a crawl returns URLs only. `maxPages` and `maxItems` default to 100 each. A crawl created without `depth` has no `depth` field; one created with it echoes it. Response enum fields (`status`, `stop_reason`, `error.code`, `content_status`) are `Extensible<...>`: the known values plus any `string`. A crawl whose pages held nothing new to follow (no link, only links to other sites, or only links the patterns reject) ends `completed` with no results and no `error`: check the patterns against the URLs the start page links to. A failed crawl carries `error.code` and `error.detail`. So does a crawl that ran out of credits, or whose API key hit a credit cap: it ends `stopped`, with `error.code` `insufficient_credits` or `api_key_cap_reached` and no `stop_reason`, and the pages it fetched before that stay readable. A download line's `content` is a string, or an object for a crawl that returns JSON. The crawl client (`ZenRowsCrawlClient`) also works standalone. Both forms take the same options: diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md index c61773c..ad098de 100644 --- a/acceptance/crawl/README.md +++ b/acceptance/crawl/README.md @@ -78,7 +78,7 @@ and signal checks. | 16 | Errors carry the server code | SDKs: `code` holds it, empty when the body has none; MCP, CLI: `server_code` | | 17 | Naming | the API's operation and parameter names, in this surface's casing | | 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | -| 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | +| 19 | Patterns match nothing | an include pattern that matches nothing ends `completed` with 0 results and no `error`; SDKs return it; MCP and CLI say `ok: true` (CLI: exit 0) | | 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | | 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | diff --git a/acceptance/crawl/contract.lock b/acceptance/crawl/contract.lock new file mode 100644 index 0000000..435df9b --- /dev/null +++ b/acceptance/crawl/contract.lock @@ -0,0 +1,3 @@ +1adcbc2fd87e3f508317f3ce24c3d48573aec657 +pending GET /healthz +pending POST /internal/v1/crawls/{crawl_id}/stop diff --git a/acceptance/crawl/surface/harness.mjs b/acceptance/crawl/surface/harness.mjs index 763d581..f0b3b5d 100644 --- a/acceptance/crawl/surface/harness.mjs +++ b/acceptance/crawl/surface/harness.mjs @@ -10,7 +10,7 @@ export const HAS_KEY = KEY.length > 0; export const CRAWL_URL = process.env.CRAWL_URL; export const INCLUDE = process.env.CRAWL_INCLUDE; export const API_BASE = process.env.CRAWL_API_BASE || "https://api.zenrows.com/v1"; -/** An include pattern no URL contains: the crawl keeps nothing and ends failed (check 19). */ +/** An include pattern no URL contains: the crawl keeps nothing and completes (check 19). */ export const NO_MATCH = "/crawl-acceptance-matches-nothing/"; /** The default every client documents for max_pages and max_items (check 21). */ export const DOCUMENTED_DEFAULT = 100; diff --git a/acceptance/crawl/surface/test.mjs b/acceptance/crawl/surface/test.mjs index 42fc8f7..9bc00d0 100644 --- a/acceptance/crawl/surface/test.mjs +++ b/acceptance/crawl/surface/test.mjs @@ -702,21 +702,21 @@ await check( ); await cap.close(); -// ---------- 19-21: a failed crawl, results after a stop, the documented defaults ---------- +// ---------- 19-21: patterns that match nothing, results after a stop, the documented defaults ---------- await liveCheck( "19", - "Failed crawl", - "a crawl that keeps nothing ends failed (no_items_found); wait returns it, not raises", + "Patterns match nothing", + "a crawl whose patterns match nothing ends completed with 0 results and no error", async () => { F = await create({ url: CRAWL_URL, depth: 1, includePatterns: [H.NO_MATCH] }); const w = await client.wait(F.crawl_id); + const urls = []; + for await (const x of client.results(F.crawl_id)) urls.push(x.url); const ok = - w.status === "failed" && - w.error?.code === "no_items_found" && - typeof w.error?.detail === "string"; + w.status === "completed" && !w.error && w.coverage.items_found === 0 && urls.length === 0; return { status: passIf(ok), - actual: `${F.crawl_id}: ${w.status}; error ${JSON.stringify(w.error)}; returned, no exception`, + actual: `${F.crawl_id}: ${w.status}; items_found ${w.coverage.items_found}, ${urls.length} results; error ${JSON.stringify(w.error ?? null)}`, evidence: JSON.stringify(w), }; }, diff --git a/acceptance/crawl/tools/report.py b/acceptance/crawl/tools/report.py index 94488f8..83e01e2 100644 --- a/acceptance/crawl/tools/report.py +++ b/acceptance/crawl/tools/report.py @@ -18,7 +18,7 @@ "1": "Create", "2": "Create with html", "3": "Idempotency", "4": "Wait", "5": "Wait runs out", "6": "Get", "7": "List", "8": "Results", "9": "Content", "10": "Download", "11": "Stop", "12": "Not found", "13": "Invalid input", "14": "Gated option", "15": "Bad key", "16": "Errors carry the server code", - "17": "Naming", "18": "Copy", "19": "Failed crawl", "20": "Results after stop", "21": "Documented defaults", + "17": "Naming", "18": "Copy", "19": "Patterns match nothing", "20": "Results after stop", "21": "Documented defaults", } # The API's operation and parameter names. Each surface uses them in its own casing. CANON_OPS = ["create", "get", "list", "results", "content", "download", "stop", "wait"] diff --git a/src/crawl/client.ts b/src/crawl/client.ts index a79fcc7..1ebeaf0 100644 --- a/src/crawl/client.ts +++ b/src/crawl/client.ts @@ -50,7 +50,10 @@ export interface CreateCrawlParams { maxItems?: number; /** Stop once this many pages are fetched, 1–100000; bounds the cost. Server default 100. */ maxPages?: number; - /** Keep a URL only if it contains at least one of these substrings. */ + /** + * Keep a URL only if it contains at least one of these substrings. A crawl whose patterns reject + * every link ends `completed` with no results and no `error`. + */ includePatterns?: string[]; /** Drop a URL that contains any of these substrings, even if it matches an include pattern. */ excludePatterns?: string[]; diff --git a/src/crawl/types.ts b/src/crawl/types.ts index 541c2ed..ebc5c0a 100644 --- a/src/crawl/types.ts +++ b/src/crawl/types.ts @@ -11,6 +11,10 @@ export type CrawlStopReason = "max_items" | "max_pages" | "user"; /** * Why a crawl failed (`status: failed`), or why it stopped on credits: `insufficient_credits` and * `api_key_cap_reached` come with `status: stopped`, and the pages fetched before that stay readable. + * `seed_unreachable`: the start page could not be reached, was blocked, or came back empty, with no + * visible text and no link. `no_items_found`: the crawl read the start page but found no data it could + * use, so it had nothing to follow; start from a listing or product page. A crawl that returns nothing + * for any other reason is not a failure: it ends `completed` with no results. */ export type CrawlErrorCode = | "insufficient_credits"