diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f9c55e..bbe76b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Unreleased +- chore: harden document fetching. **Breaking for some URLs:** `get_document_tool` and `batch_get_documents_tool` now accept only `https://docs.mapbox.com` URLs (plus `MAPBOX_DOCS_HOST_OVERRIDE` when set); bare `mapbox.com` and `docs.tilestream.net` (without the override) are rejected. The shared HTTP pipeline no longer follows redirects; document fetches follow up to 3 redirects, each only to an allowed docs URL. `access_token` detection is now case-insensitive (#45) - docs: fix two stale tool descriptions left over from the Algolia-to-`llms.txt` search migration. `search_mapbox_docs_tool`'s README entry still said it used "Algolia full-text search" — that was removed a while back (see the "Replace Algolia search with self-contained llms.txt search" entry below) and it now searches the self-hosted `llms.txt` index. Separately, `get_document_tool`'s own description (the text shown to the LLM, not just docs) referenced a tool named `get_latest_mapbox_docs_tool`, which doesn't exist — it should have said `search_mapbox_docs_tool`, and was actively misdirecting tool-chaining behavior. - fix: startup/shutdown/error logging messages now actually reach clients. The `McpServer` never declared the `logging` capability, so every `sendLoggingMessage` call (including the existing "server started"/"shutting down"/"Fatal error" messages) was a silent no-op regardless of when it was sent. Declared `logging: {}` in the server's capabilities. - feat: the server now identifies which MCP client connected to it. `server.server.getClientVersion()` (populated from the `clientInfo` sent in the client's `initialize` request) is now logged on connect, e.g. `Client identified as: claude-ai v1.0.0`. Reading it required moving the read to a `server.server.oninitialized` callback rather than right after `server.connect()`, since `getClientVersion()` is only populated once the client's `initialize` request has actually been processed, which is not guaranteed by the time `connect()`'s promise resolves. Also recorded as `mcp.client.name`/`mcp.client.version` on every subsequent tool-execution trace span (`withToolSpan`), so OTel-backed traces can be filtered or grouped by client. Matches the equivalent changes landing in `@mapbox/mcp-server` and `@mapbox/mcp-devkit-server`. diff --git a/CLAUDE.md b/CLAUDE.md index c965a99..2f21e4d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,6 +71,7 @@ export class MyTool extends BaseTool< - **Never patch `global.fetch`** — always use the injected `httpRequest` - The shared `httpRequest` from `httpPipeline.ts` adds User-Agent headers, 1-hour caching, and retry logic - Pass `httpRequest` to tool/resource constructors for testability +- The shared pipeline does **not** follow redirects (`NoRedirectPolicy` sets `redirect: 'manual'`), so a tool calling any host gets the raw 3xx response. Docs fetches follow redirects themselves in `docFetcher.ts`, only to URLs that pass `isDocsUrl()` (`docUrl.ts`) ### Resource Architecture @@ -123,6 +124,6 @@ npm run inspect:dev # MCP Inspector against source (tsx, no build needed) - **No global patching**: use `httpRequest` from the pipeline - **Dependency injection**: tools and resources receive `httpRequest` in constructor -- **No real network calls in tests**: mock `httpRequest` with `vi.fn()` +- **No real network calls in tests**: mock `httpRequest` with `vi.fn()`. The one exception is `test/utils/httpPipeline.test.ts`, which uses loopback-only `node:http` servers to verify real `fetch` redirect behavior - **Strict types**: avoid `any` - **No access token env var**: docs tools are token-free; if a token is needed, it's an input field diff --git a/src/tools/batch-get-documents-tool/BatchGetDocumentsTool.input.schema.ts b/src/tools/batch-get-documents-tool/BatchGetDocumentsTool.input.schema.ts index b95b56c..69aac52 100644 --- a/src/tools/batch-get-documents-tool/BatchGetDocumentsTool.input.schema.ts +++ b/src/tools/batch-get-documents-tool/BatchGetDocumentsTool.input.schema.ts @@ -9,7 +9,7 @@ export const BatchGetDocumentsSchema = z.object({ .min(1) .max(20) .describe( - 'Array of Mapbox documentation page URLs to fetch (max 20). All must be mapbox.com URLs.' + 'Array of Mapbox documentation page URLs to fetch (max 20). All must be https://docs.mapbox.com URLs.' ) }); diff --git a/src/tools/batch-get-documents-tool/BatchGetDocumentsTool.ts b/src/tools/batch-get-documents-tool/BatchGetDocumentsTool.ts index f5fd6c9..057ede7 100644 --- a/src/tools/batch-get-documents-tool/BatchGetDocumentsTool.ts +++ b/src/tools/batch-get-documents-tool/BatchGetDocumentsTool.ts @@ -4,6 +4,7 @@ import { CallToolResult } from '@modelcontextprotocol/sdk/types.js'; import { docCache, normalizeCacheKey } from '../../utils/docCache.js'; import { fetchDocContent } from '../../utils/docFetcher.js'; +import { hasAccessToken, isDocsUrl } from '../../utils/docUrl.js'; import type { HttpRequest } from '../../utils/types.js'; import { BaseTool } from '../BaseTool.js'; import { @@ -11,33 +12,6 @@ import { BatchGetDocumentsInput } from './BatchGetDocumentsTool.input.schema.js'; -// Explicit allowlist of hostnames this docs tool is permitted to fetch. -// api.mapbox.com is intentionally excluded — it is a live API that requires -// auth tokens, not a documentation host. Allowing it would let callers poison -// the shared cache with token-authorized private responses under no-token keys. -const ALLOWED_DOC_HOSTNAMES = new Set([ - 'docs.mapbox.com', - 'mapbox.com', - 'docs.tilestream.net' -]); - -function isMapboxUrl(url: string): boolean { - try { - const { hostname } = new URL(url); - return ALLOWED_DOC_HOSTNAMES.has(hostname); - } catch { - return false; - } -} - -function hasAccessToken(url: string): boolean { - try { - return new URL(url).searchParams.has('access_token'); - } catch { - return false; - } -} - type DocumentResult = | { url: string; content: string } | { url: string; error: string }; @@ -66,13 +40,13 @@ export class BatchGetDocumentsTool extends BaseTool< protected async execute( input: BatchGetDocumentsInput ): Promise { - const invalidUrls = input.urls.filter((url) => !isMapboxUrl(url)); + const invalidUrls = input.urls.filter((url) => !isDocsUrl(url)); if (invalidUrls.length > 0) { return { content: [ { type: 'text', - text: `Invalid URLs: only mapbox.com documentation URLs are supported. Invalid: ${invalidUrls.join(', ')}` + text: `Invalid URLs: only https://docs.mapbox.com documentation URLs are supported. Invalid: ${invalidUrls.join(', ')}` } ], isError: true diff --git a/src/tools/get-document-tool/GetDocumentTool.input.schema.ts b/src/tools/get-document-tool/GetDocumentTool.input.schema.ts index b336d9e..8d34c54 100644 --- a/src/tools/get-document-tool/GetDocumentTool.input.schema.ts +++ b/src/tools/get-document-tool/GetDocumentTool.input.schema.ts @@ -8,7 +8,7 @@ export const GetDocumentSchema = z.object({ .string() .url() .describe( - 'URL of a Mapbox documentation page to fetch. Must be a mapbox.com URL (e.g. https://docs.mapbox.com/api/search/geocoding/).' + 'URL of a Mapbox documentation page to fetch. Must be an https://docs.mapbox.com URL (e.g. https://docs.mapbox.com/api/search/geocoding/).' ) }); diff --git a/src/tools/get-document-tool/GetDocumentTool.ts b/src/tools/get-document-tool/GetDocumentTool.ts index 37e5fcc..83958f8 100644 --- a/src/tools/get-document-tool/GetDocumentTool.ts +++ b/src/tools/get-document-tool/GetDocumentTool.ts @@ -4,6 +4,7 @@ import { CallToolResult } from '@modelcontextprotocol/sdk/types.js'; import { docCache } from '../../utils/docCache.js'; import { fetchDocContent } from '../../utils/docFetcher.js'; +import { hasAccessToken, isDocsUrl } from '../../utils/docUrl.js'; import type { HttpRequest } from '../../utils/types.js'; import { BaseTool } from '../BaseTool.js'; import { @@ -11,33 +12,6 @@ import { GetDocumentInput } from './GetDocumentTool.input.schema.js'; -// Explicit allowlist of hostnames this docs tool is permitted to fetch. -// api.mapbox.com is intentionally excluded — it is a live API that requires -// auth tokens, not a documentation host. Allowing it would let callers poison -// the shared cache with token-authorized private responses under no-token keys. -const ALLOWED_DOC_HOSTNAMES = new Set([ - 'docs.mapbox.com', - 'mapbox.com', - 'docs.tilestream.net' -]); - -function isMapboxUrl(url: string): boolean { - try { - const { hostname } = new URL(url); - return ALLOWED_DOC_HOSTNAMES.has(hostname); - } catch { - return false; - } -} - -function hasAccessToken(url: string): boolean { - try { - return new URL(url).searchParams.has('access_token'); - } catch { - return false; - } -} - export class GetDocumentTool extends BaseTool { name = 'get_document_tool'; description = @@ -58,12 +32,12 @@ export class GetDocumentTool extends BaseTool { } protected async execute(input: GetDocumentInput): Promise { - if (!isMapboxUrl(input.url)) { + if (!isDocsUrl(input.url)) { return { content: [ { type: 'text', - text: `Invalid URL: only mapbox.com documentation URLs are supported. Received: ${input.url}` + text: `Invalid URL: only https://docs.mapbox.com documentation URLs are supported. Received: ${input.url}` } ], isError: true diff --git a/src/utils/docFetcher.ts b/src/utils/docFetcher.ts index 60eb54c..4887ccd 100644 --- a/src/utils/docFetcher.ts +++ b/src/utils/docFetcher.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import { docCache, MAX_ENTRY_BYTES, readBodyWithLimit } from './docCache.js'; +import { hasAccessToken, isDocsUrl } from './docUrl.js'; import type { HttpRequest } from './types.js'; const DOCS_HOSTNAME = 'docs.mapbox.com'; @@ -61,6 +62,106 @@ function applyHostOverride(url: string): string { } } +/** + * docs.mapbox.com serves HTML pages at a trailing-slash path and answers the + * slash-less form with a redirect. Add the slash to extension-less paths so + * the fallback fetch gets the page directly. + */ +function withTrailingSlash(url: string): string { + try { + const parsed = new URL(url); + if (parsed.hostname !== getDocsHostname()) return url; + const lastSegment = parsed.pathname.split('/').pop() ?? ''; + if (lastSegment !== '' && !lastSegment.includes('.')) { + parsed.pathname += '/'; + } + return parsed.toString(); + } catch { + return url; + } +} + +const MAX_REDIRECTS = 3; +const FOLLOWED_REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); + +class RedirectError extends Error {} + +/** + * Resolve the Location of a redirect response against the URL that was + * requested. Returns null unless the status is one we follow and the target is + * a different, allowed documentation URL without an access_token. + */ +function redirectTarget(response: Response, current: string): string | null { + if (!FOLLOWED_REDIRECT_STATUSES.has(response.status)) return null; + const location = response.headers.get('location'); + if (!location?.trim()) return null; + try { + const next = new URL(location, current).href; + if (next === new URL(current).href) return null; + if (!isDocsUrl(next) || hasAccessToken(next)) return null; + return next; + } catch { + return null; + } +} + +/** + * Map a URL on the effective docs host back to docs.mapbox.com. Needed because + * toMarkdownUrl only matches docs.mapbox.com and applies the host override + * itself. + */ +function toCanonicalDocsUrl(url: string): string { + try { + const parsed = new URL(url); + if (parsed.hostname === getDocsHostname()) { + parsed.hostname = DOCS_HOSTNAME; + } + return parsed.toString(); + } catch { + return url; + } +} + +/** + * GET a URL, following up to MAX_REDIRECTS redirects. Every hop is validated + * with isDocsUrl() before it is requested, so a redirect can only lead to + * another documentation URL. Throws RedirectError otherwise. + * + * `beforeFollow` is called with each validated hop before it is requested. If + * it returns content, that content is returned and the hop is not requested. + */ +async function fetchFollowingDocsRedirects( + url: string, + httpRequest: HttpRequest +): Promise<{ response: Response; finalUrl: string }>; +async function fetchFollowingDocsRedirects( + url: string, + httpRequest: HttpRequest, + beforeFollow: (next: string) => Promise +): Promise<{ response: Response; finalUrl: string } | { content: string }>; +async function fetchFollowingDocsRedirects( + url: string, + httpRequest: HttpRequest, + beforeFollow?: (next: string) => Promise +): Promise<{ response: Response; finalUrl: string } | { content: string }> { + let current = url; + for (let hops = 0; ; hops++) { + const response = await httpRequest(current, {}); + if (response.status < 300 || response.status >= 400) { + return { response, finalUrl: current }; + } + void response.body?.cancel(); + if (hops >= MAX_REDIRECTS) throw new RedirectError('Too many redirects'); + const next = redirectTarget(response, current); + if (next === null) { + throw new RedirectError(`${response.status} Redirect not followed`); + } + const early = await beforeFollow?.(next); + if (early != null) return { content: early }; + current = next; + } +} + /** * Fetch a URL and cache the result. Intended for fetching llms.txt index * files which are small, change infrequently, and are shared across multiple @@ -73,11 +174,22 @@ export async function fetchCachedText( const cached = docCache.get(url); if (cached !== null) return cached; - const response = await httpRequest(url, {}); + const { response, finalUrl } = await fetchFollowingDocsRedirects( + url, + httpRequest + ); if (!response.ok) { throw new Error(`Failed to fetch ${url}: ${response.statusText}`); } + // A redirected index must still be text, not the HTML page it moved to. + if ( + finalUrl !== url && + response.headers.get('content-type')?.toLowerCase().includes('text/html') + ) { + void response.body?.cancel(); + throw new Error(`Unexpected content type for ${url}`); + } const content = await response.text(); if (Buffer.byteLength(content, 'utf8') > MAX_ENTRY_BYTES) { @@ -90,7 +202,9 @@ export async function fetchCachedText( /** * Fetch a Mapbox documentation page, preferring the .md variant for * docs.mapbox.com URLs when available. Falls back to the original URL - * if the .md endpoint returns a non-2xx response. + * if the .md endpoint returns a non-2xx response. Redirects are followed only + * to other allowed documentation URLs (e.g. a page that has moved); if the + * fallback ends up on a different URL, its .md variant is tried first. * * Set MAPBOX_DOCS_HOST_OVERRIDE=docs.tilestream.net to redirect all * docs.mapbox.com fetches to the staging environment. @@ -99,18 +213,52 @@ export async function fetchDocContent( url: string, httpRequest: HttpRequest ): Promise { - const mdUrl = toMarkdownUrl(url); + const tryMarkdown = async (mdUrl: string): Promise => { + try { + const result = await fetchFollowingDocsRedirects(mdUrl, httpRequest); + if (result.response.ok) { + return await readBodyWithLimit(result.response, MAX_ENTRY_BYTES); + } + void result.response.body?.cancel(); + } catch (error) { + if (!(error instanceof RedirectError)) throw error; + } + return null; + }; + const mdUrl = toMarkdownUrl(url); if (mdUrl) { - const mdResponse = await httpRequest(mdUrl, {}); - if (mdResponse.ok) { - return readBodyWithLimit(mdResponse, MAX_ENTRY_BYTES); - } + const markdown = await tryMarkdown(mdUrl); + if (markdown !== null) return markdown; } - const fetchUrl = applyHostOverride(url); - const response = await httpRequest(fetchUrl, {}); + // For a moved page, try the .md variant of the new location before + // requesting its HTML. `next` has already passed isDocsUrl(); the derived + // .md URL is checked again before it is requested. A single request, no + // redirects: keeps the total request count bounded by MAX_REDIRECTS. + const probed = new Set(mdUrl ? [mdUrl] : []); + const tryMovedMarkdown = async (next: string): Promise => { + const movedMdUrl = toMarkdownUrl(toCanonicalDocsUrl(next)); + if (!movedMdUrl || !isDocsUrl(movedMdUrl) || hasAccessToken(movedMdUrl)) { + return null; + } + if (probed.has(movedMdUrl)) return null; + probed.add(movedMdUrl); + const response = await httpRequest(movedMdUrl, {}); + if (response.ok) return readBodyWithLimit(response, MAX_ENTRY_BYTES); + void response.body?.cancel(); + return null; + }; + + const fetchUrl = withTrailingSlash(applyHostOverride(url)); + const result = await fetchFollowingDocsRedirects( + fetchUrl, + httpRequest, + tryMovedMarkdown + ); + if ('content' in result) return result.content; + const { response } = result; if (!response.ok) { throw new Error(`${response.status} ${response.statusText}`); } diff --git a/src/utils/docUrl.ts b/src/utils/docUrl.ts new file mode 100644 index 0000000..15b5482 --- /dev/null +++ b/src/utils/docUrl.ts @@ -0,0 +1,44 @@ +// Copyright (c) Mapbox, Inc. +// Licensed under the MIT License. + +const DOCS_HOSTNAME = 'docs.mapbox.com'; + +// Only documentation hosts are allowed. Bare mapbox.com and api.mapbox.com are +// intentionally excluded: neither is a documentation host, and api.mapbox.com +// is an authenticated API. +function allowedHostnames(): Set { + const hosts = new Set([DOCS_HOSTNAME]); + const override = process.env.MAPBOX_DOCS_HOST_OVERRIDE; + if (override) hosts.add(override); + return hosts; +} + +/** + * True only for plain https URLs on an allowed documentation host, with no + * explicit port and no embedded credentials. + */ +export function isDocsUrl(url: string): boolean { + try { + const parsed = new URL(url); + return ( + parsed.protocol === 'https:' && + parsed.port === '' && + parsed.username === '' && + parsed.password === '' && + allowedHostnames().has(parsed.hostname) + ); + } catch { + return false; + } +} + +export function hasAccessToken(url: string): boolean { + try { + for (const key of new URL(url).searchParams.keys()) { + if (key.toLowerCase() === 'access_token') return true; + } + return false; + } catch { + return false; + } +} diff --git a/src/utils/httpPipeline.ts b/src/utils/httpPipeline.ts index e1285b1..a7c5c1d 100644 --- a/src/utils/httpPipeline.ts +++ b/src/utils/httpPipeline.ts @@ -161,12 +161,36 @@ export class RetryPolicy implements HttpPolicy { } } +/** + * Stops fetch from following redirects. URLs are validated against the docs + * host allowlist before the request, and a redirect target would skip that + * check. With 'manual', fetch returns the 3xx response itself and the target + * is never requested. The override is applied after the caller's options so it + * cannot be turned back on per request. + */ +export class NoRedirectPolicy implements HttpPolicy { + id: string; + + constructor(id?: string) { + this.id = id ?? createRandomId('no-redirect-'); + } + + async handle( + input: string | URL | Request, + init: RequestInit, + next: HttpRequest + ): Promise { + return next(input, { ...init, redirect: 'manual' }); + } +} + const pipeline = new HttpPipeline(); const versionInfo = getVersionInfo(); pipeline.usePolicy( UserAgentPolicy.fromVersionInfo(versionInfo, 'system-user-agent-policy') ); pipeline.usePolicy(new RetryPolicy(3, 200, 2000, 'system-retry-policy')); +pipeline.usePolicy(new NoRedirectPolicy('system-no-redirect-policy')); export const httpRequest = pipeline.execute.bind(pipeline); export const systemHttpPipeline = pipeline; diff --git a/test/tools/batch-get-documents-tool/BatchGetDocumentsTool.test.ts b/test/tools/batch-get-documents-tool/BatchGetDocumentsTool.test.ts index 7e17420..b6ebdab 100644 --- a/test/tools/batch-get-documents-tool/BatchGetDocumentsTool.test.ts +++ b/test/tools/batch-get-documents-tool/BatchGetDocumentsTool.test.ts @@ -118,13 +118,41 @@ describe('BatchGetDocumentsTool', () => { const tool = new BatchGetDocumentsTool({ httpRequest }); const result = await tool.run({ - urls: ['https://evil.com/page'] + urls: ['https://example.com/page'] }); expect(result.isError).toBe(true); expect(httpRequest).not.toHaveBeenCalled(); }); + it('rejects bare mapbox.com URLs without making a request', async () => { + const httpRequest = vi.fn(); + const tool = new BatchGetDocumentsTool({ httpRequest }); + + const result = await tool.run({ + urls: ['https://mapbox.com/about'] + }); + + expect(result.isError).toBe(true); + expect(httpRequest).not.toHaveBeenCalled(); + }); + + it('reports a per-URL error when a page answers with a redirect', async () => { + const httpRequest = vi.fn().mockResolvedValue( + new Response(null, { + status: 302, + headers: { location: 'https://example.com/other' } + }) + ); + const tool = new BatchGetDocumentsTool({ httpRequest }); + + const result = await tool.run({ urls: ['https://docs.mapbox.com/page'] }); + + const parsed = JSON.parse((result.content[0] as { text: string }).text); + expect(parsed[0].error).toMatch(/Redirect not followed/); + expect(docCache.get('https://docs.mapbox.com/page')).toBeNull(); + }); + it('rejects api.mapbox.com URLs', async () => { const httpRequest = vi.fn(); const tool = new BatchGetDocumentsTool({ httpRequest }); @@ -152,21 +180,18 @@ describe('BatchGetDocumentsTool', () => { expect(httpRequest).not.toHaveBeenCalled(); }); - it('blocks cache poisoning: tokenized URL cannot prime cache for no-token URL', async () => { - // Even if somehow both URLs passed validation (they do not), this test - // documents the expected behavior: private data must not leak. - // In practice the access_token check above prevents this entirely. + it('does not cache entries for api.mapbox.com URLs with or without a token', async () => { + // api.mapbox.com URLs are rejected up front, so nothing is fetched or cached. const httpRequest = vi.fn().mockResolvedValue(makeResponse('private')); const tool = new BatchGetDocumentsTool({ httpRequest }); - // Attempt the poisoning using an api.mapbox.com URL — must be rejected - const poisonResult = await tool.run({ + const result = await tool.run({ urls: [ 'https://api.mapbox.com/styles/v1/owner/id?access_token=secret', 'https://api.mapbox.com/styles/v1/owner/id' ] }); - expect(poisonResult.isError).toBe(true); + expect(result.isError).toBe(true); expect(httpRequest).not.toHaveBeenCalled(); expect(docCache.size).toBe(0); }); diff --git a/test/tools/get-document-tool/GetDocumentTool.test.ts b/test/tools/get-document-tool/GetDocumentTool.test.ts index fb16498..ed16e66 100644 --- a/test/tools/get-document-tool/GetDocumentTool.test.ts +++ b/test/tools/get-document-tool/GetDocumentTool.test.ts @@ -25,7 +25,7 @@ describe('GetDocumentTool', () => { const httpRequest = vi.fn(); const tool = new GetDocumentTool({ httpRequest }); - const result = await tool.run({ url: 'https://evil.com/page' }); + const result = await tool.run({ url: 'https://example.com/page' }); expect(result.isError).toBe(true); expect(httpRequest).not.toHaveBeenCalled(); @@ -58,6 +58,21 @@ describe('GetDocumentTool', () => { expect(httpRequest).not.toHaveBeenCalled(); }); + it.each([ + 'https://mapbox.com/about', + 'http://docs.mapbox.com/page', + 'https://docs.mapbox.com:8080/page', + 'https://192.0.2.1/' + ])('rejects %s without making a request', async (url) => { + const httpRequest = vi.fn(); + const tool = new GetDocumentTool({ httpRequest }); + + const result = await tool.run({ url }); + + expect(result.isError).toBe(true); + expect(httpRequest).not.toHaveBeenCalled(); + }); + it('allows docs.mapbox.com URLs', async () => { const httpRequest = vi.fn().mockResolvedValue(makeResponse('content')); const tool = new GetDocumentTool({ httpRequest }); @@ -84,6 +99,26 @@ describe('GetDocumentTool', () => { }); }); + describe('redirects', () => { + it('returns an error and does not cache when the server answers with a redirect', async () => { + const httpRequest = vi.fn().mockResolvedValue( + new Response(null, { + status: 302, + headers: { location: 'https://example.com/other' } + }) + ); + const tool = new GetDocumentTool({ httpRequest }); + + const result = await tool.run({ url: 'https://docs.mapbox.com/page' }); + + expect(result.isError).toBe(true); + expect((result.content[0] as { text: string }).text).toMatch( + /Redirect not followed/ + ); + expect(docCache.get('https://docs.mapbox.com/page')).toBeNull(); + }); + }); + describe('HTTP errors', () => { it('returns an error on non-ok response', async () => { const httpRequest = vi diff --git a/test/utils/docFetcher.test.ts b/test/utils/docFetcher.test.ts index fadf8f6..16618d0 100644 --- a/test/utils/docFetcher.test.ts +++ b/test/utils/docFetcher.test.ts @@ -157,7 +157,7 @@ describe('fetchDocContent', () => { ); expect(httpRequest).toHaveBeenNthCalledWith( 2, - 'https://docs.mapbox.com/accounts/guides', + 'https://docs.mapbox.com/accounts/guides/', {} ); expect(content).toBe(html); @@ -204,3 +204,298 @@ describe('fetchDocContent', () => { ).rejects.toThrow('404 Not Found'); }); }); + +describe('fallback URL', () => { + it('adds a trailing slash to extension-less paths on the fallback fetch', async () => { + const httpRequest = vi + .fn() + .mockResolvedValueOnce(makeResponse('', 404)) + .mockResolvedValueOnce(makeResponse('html page')); + + await fetchDocContent('https://docs.mapbox.com/api/search', httpRequest); + + expect(httpRequest.mock.calls[1][0]).toBe( + 'https://docs.mapbox.com/api/search/' + ); + }); + + it('leaves paths with a file extension unchanged', async () => { + const httpRequest = vi.fn().mockResolvedValue(makeResponse('text')); + + await fetchDocContent('https://docs.mapbox.com/llms.txt', httpRequest); + + expect(httpRequest.mock.calls[0][0]).toBe( + 'https://docs.mapbox.com/llms.txt' + ); + }); +}); + +describe('redirect handling', () => { + function redirectResponse(location: string): Response { + return new Response(null, { status: 302, headers: { location } }); + } + + it('fetchDocContent falls back from a redirecting .md URL, then errors on a redirecting fallback', async () => { + const httpRequest = vi + .fn() + .mockResolvedValue(redirectResponse('https://example.com/other')); + + await expect( + fetchDocContent('https://docs.mapbox.com/page', httpRequest) + ).rejects.toThrow('302 Redirect not followed'); + + expect(httpRequest.mock.calls.map((c) => c[0])).toEqual([ + 'https://docs.mapbox.com/page.md', + 'https://docs.mapbox.com/page/' + ]); + }); + + it('fetchDocContent does not include the Location header in the error', async () => { + const httpRequest = vi + .fn() + .mockResolvedValue(redirectResponse('https://other.example/target')); + + await expect( + fetchDocContent('https://docs.mapbox.com/page', httpRequest) + ).rejects.not.toThrow(/other\.example/); + }); + + it('fetchCachedText rejects a redirect and does not cache it', async () => { + const httpRequest = vi + .fn() + .mockResolvedValue(redirectResponse('https://example.com/other')); + + await expect( + fetchCachedText('https://docs.mapbox.com/llms.txt', httpRequest) + ).rejects.toThrow('Redirect not followed'); + expect(docCache.get('https://docs.mapbox.com/llms.txt')).toBeNull(); + }); +}); + +describe('following allowed redirects', () => { + function redirect(status: number, location?: string): Response { + const headers: Record = {}; + if (location !== undefined) headers.location = location; + return new Response(null, { status, headers }); + } + + function routes(table: Record Response>) { + return vi.fn(async (input: string | URL | Request) => { + const handler = table[String(input)]; + return handler ? handler() : makeResponse('', 404); + }); + } + + it('follows a moved page and prefers the .md variant of the new location', async () => { + const httpRequest = routes({ + 'https://docs.mapbox.com/help/getting-started/attribution/': () => + redirect(301, '/help/dive-deeper/attribution/'), + 'https://docs.mapbox.com/help/dive-deeper/attribution/': () => + makeResponse('attribution'), + 'https://docs.mapbox.com/help/dive-deeper/attribution.md': () => + makeResponse('# Attribution') + }); + + const content = await fetchDocContent( + 'https://docs.mapbox.com/help/getting-started/attribution/', + httpRequest + ); + + expect(content).toBe('# Attribution'); + expect(httpRequest.mock.calls.map((c) => c[0])).toEqual([ + 'https://docs.mapbox.com/help/getting-started/attribution.md', + 'https://docs.mapbox.com/help/getting-started/attribution/', + 'https://docs.mapbox.com/help/dive-deeper/attribution.md' + ]); + }); + + it('returns the HTML of the new location when it has no .md variant', async () => { + const httpRequest = routes({ + 'https://docs.mapbox.com/old/': () => + redirect(301, 'https://docs.mapbox.com/new/'), + 'https://docs.mapbox.com/new/': () => makeResponse('new') + }); + + await expect( + fetchDocContent('https://docs.mapbox.com/old/', httpRequest) + ).resolves.toBe('new'); + }); + + it.each([301, 302, 303, 307, 308])('follows a %i', async (status) => { + const httpRequest = routes({ + 'https://docs.mapbox.com/a/': () => redirect(status, '/b/'), + 'https://docs.mapbox.com/b/': () => makeResponse('b') + }); + + await expect( + fetchDocContent('https://docs.mapbox.com/a/', httpRequest) + ).resolves.toBe('b'); + }); + + it.each([300, 304, 305])('does not follow a %i', async (status) => { + const httpRequest = routes({ + 'https://docs.mapbox.com/a/': () => redirect(status, '/b/'), + 'https://docs.mapbox.com/b/': () => makeResponse('b') + }); + + await expect( + fetchDocContent('https://docs.mapbox.com/a/', httpRequest) + ).rejects.toThrow('Redirect not followed'); + expect(httpRequest.mock.calls.map((c) => c[0])).not.toContain( + 'https://docs.mapbox.com/b/' + ); + }); + + it.each([ + ['another host', 'https://example.com/x'], + ['http scheme', 'http://docs.mapbox.com/x'], + ['explicit port', 'https://docs.mapbox.com:8443/x'], + ['userinfo trick', 'https://docs.mapbox.com@example.com/x'], + ['protocol-relative host', '//example.com/x'], + ['backslash host', '\\\\example.com/x'], + ['bare mapbox.com', 'https://mapbox.com/x'], + ['access_token', 'https://docs.mapbox.com/x?access_token=pk.secret'], + [ + 'uppercase ACCESS_TOKEN', + 'https://docs.mapbox.com/x?ACCESS_TOKEN=pk.secret' + ], + ['empty Location', ''], + ['same URL', 'https://docs.mapbox.com/a/'] + ])( + 'rejects a redirect to %s without requesting it', + async (_name, location) => { + const httpRequest = routes({ + 'https://docs.mapbox.com/a/': () => redirect(302, location) + }); + + await expect( + fetchDocContent('https://docs.mapbox.com/a/', httpRequest) + ).rejects.toThrow('302 Redirect not followed'); + + for (const call of httpRequest.mock.calls) { + expect(String(call[0])).toMatch( + /^https:\/\/docs\.mapbox\.com\/a(\.md|\/)$/ + ); + } + } + ); + + it('rejects a redirect with no Location header', async () => { + const httpRequest = routes({ + 'https://docs.mapbox.com/a/': () => redirect(302) + }); + + await expect( + fetchDocContent('https://docs.mapbox.com/a/', httpRequest) + ).rejects.toThrow('302 Redirect not followed'); + }); + + it('stops after 3 redirects', async () => { + let n = 0; + const httpRequest = vi.fn(async () => redirect(302, `/hop${++n}/`)); + + await expect( + fetchDocContent('https://docs.mapbox.com/start/', httpRequest) + ).rejects.toThrow('Too many redirects'); + + // .md attempt: 1 request plus 3 hops. Fallback: 1 request plus 3 hops, + // and one .md probe per validated hop. + expect(httpRequest).toHaveBeenCalledTimes(11); + }); + + it('follows a redirect to the staging host only when it is the override host', async () => { + const table = { + 'https://docs.mapbox.com/a/': () => + redirect(302, 'https://docs.tilestream.net/b/'), + 'https://docs.tilestream.net/b/': () => makeResponse('staging') + }; + + await expect( + fetchDocContent('https://docs.mapbox.com/a/', routes(table)) + ).rejects.toThrow('302 Redirect not followed'); + + process.env.MAPBOX_DOCS_HOST_OVERRIDE = 'docs.tilestream.net'; + const overridden = routes({ + 'https://docs.tilestream.net/a/': () => + redirect(302, 'https://docs.tilestream.net/b/'), + 'https://docs.tilestream.net/b/': () => makeResponse('staging') + }); + await expect( + fetchDocContent('https://docs.mapbox.com/a/', overridden) + ).resolves.toBe('staging'); + }); + + it('prefers the .md of a moved page on the staging host', async () => { + process.env.MAPBOX_DOCS_HOST_OVERRIDE = 'docs.tilestream.net'; + const httpRequest = routes({ + 'https://docs.tilestream.net/old/': () => + redirect(301, 'https://docs.tilestream.net/new/'), + 'https://docs.tilestream.net/new.md': () => makeResponse('# New') + }); + + await expect( + fetchDocContent('https://docs.mapbox.com/old/', httpRequest) + ).resolves.toBe('# New'); + expect(httpRequest.mock.calls.map((c) => c[0])).not.toContain( + 'https://docs.tilestream.net/new/' + ); + }); + + it('does not request the .md of a redirect target that is not allowed', async () => { + const httpRequest = routes({ + 'https://docs.mapbox.com/old/': () => + redirect(301, 'https://docs.mapbox.com/new/?access_token=pk.x'), + 'https://docs.mapbox.com/new.md?access_token=pk.x': () => + makeResponse('# leaked') + }); + + await expect( + fetchDocContent('https://docs.mapbox.com/old/', httpRequest) + ).rejects.toThrow('301 Redirect not followed'); + expect(httpRequest.mock.calls.map((c) => c[0])).toEqual([ + 'https://docs.mapbox.com/old.md', + 'https://docs.mapbox.com/old/' + ]); + }); + + it('fetchCachedText follows a redirect to an allowed docs URL', async () => { + const httpRequest = routes({ + 'https://docs.mapbox.com/llms.txt': () => redirect(301, '/docs/llms.txt'), + 'https://docs.mapbox.com/docs/llms.txt': () => makeResponse('index') + }); + + await expect( + fetchCachedText('https://docs.mapbox.com/llms.txt', httpRequest) + ).resolves.toBe('index'); + }); + + it('fetchCachedText rejects a redirect that lands on an HTML page and does not cache it', async () => { + const httpRequest = routes({ + 'https://docs.mapbox.com/x/llms.txt': () => redirect(301, '/x/'), + 'https://docs.mapbox.com/x/': () => + new Response('', { + status: 200, + headers: { 'content-type': 'Text/HTML; charset=utf-8' } + }) + }); + + await expect( + fetchCachedText('https://docs.mapbox.com/x/llms.txt', httpRequest) + ).rejects.toThrow('Unexpected content type'); + expect(docCache.get('https://docs.mapbox.com/x/llms.txt')).toBeNull(); + }); + + it('does not probe the same .md URL twice', async () => { + const httpRequest = routes({ + 'https://docs.mapbox.com/foo/': () => + redirect(301, 'https://docs.mapbox.com/foo'), + 'https://docs.mapbox.com/foo': () => makeResponse('foo') + }); + + await fetchDocContent('https://docs.mapbox.com/foo/', httpRequest); + const urls = httpRequest.mock.calls.map((c) => c[0]); + expect( + urls.filter((u) => u === 'https://docs.mapbox.com/foo.md') + ).toHaveLength(1); + }); +}); diff --git a/test/utils/docUrl.test.ts b/test/utils/docUrl.test.ts new file mode 100644 index 0000000..4b9fd4c --- /dev/null +++ b/test/utils/docUrl.test.ts @@ -0,0 +1,59 @@ +// Copyright (c) Mapbox, Inc. +// Licensed under the MIT License. + +import { describe, it, expect, afterEach } from 'vitest'; +import { isDocsUrl, hasAccessToken } from '../../src/utils/docUrl.js'; + +afterEach(() => { + delete process.env.MAPBOX_DOCS_HOST_OVERRIDE; +}); + +describe('isDocsUrl', () => { + it.each([ + 'https://docs.mapbox.com/', + 'https://docs.mapbox.com/api/overview/', + 'https://docs.mapbox.com/llms.txt?x=1' + ])('accepts %s', (url) => { + expect(isDocsUrl(url)).toBe(true); + }); + + it.each([ + 'http://docs.mapbox.com/', + 'https://mapbox.com/about', + 'https://www.mapbox.com/', + 'https://api.mapbox.com/styles/v1/a/b', + 'https://docs.mapbox.com:8080/', + 'https://user:pass@docs.mapbox.com/', + 'https://docs.mapbox.com.example.com/', + 'https://docs.mapbox.com./', + 'https://example.com/docs.mapbox.com', + 'https://192.0.2.1/', + 'https://example.com/other', + 'https://docs.tilestream.net/', + 'file:///tmp/x', + 'not a url' + ])('rejects %s', (url) => { + expect(isDocsUrl(url)).toBe(false); + }); + + it('accepts the staging host only when MAPBOX_DOCS_HOST_OVERRIDE is set', () => { + expect(isDocsUrl('https://docs.tilestream.net/')).toBe(false); + process.env.MAPBOX_DOCS_HOST_OVERRIDE = 'docs.tilestream.net'; + expect(isDocsUrl('https://docs.tilestream.net/')).toBe(true); + }); +}); + +describe('hasAccessToken', () => { + it('detects access_token in the query string', () => { + expect(hasAccessToken('https://docs.mapbox.com/p?access_token=pk.x')).toBe( + true + ); + expect(hasAccessToken('https://docs.mapbox.com/p?a=1')).toBe(false); + expect(hasAccessToken('https://docs.mapbox.com/p?ACCESS_TOKEN=x')).toBe( + true + ); + expect(hasAccessToken('https://docs.mapbox.com/p?Access_Token=x')).toBe( + true + ); + }); +}); diff --git a/test/utils/httpPipeline.test.ts b/test/utils/httpPipeline.test.ts new file mode 100644 index 0000000..79b94e1 --- /dev/null +++ b/test/utils/httpPipeline.test.ts @@ -0,0 +1,109 @@ +// Copyright (c) Mapbox, Inc. +// Licensed under the MIT License. + +import { createServer, type Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { + HttpPipeline, + NoRedirectPolicy, + RetryPolicy, + systemHttpPipeline +} from '../../src/utils/httpPipeline.js'; + +function listen( + handler: Parameters[1] +): Promise<{ server: Server; port: number }> { + return new Promise((resolve) => { + const server = createServer(handler); + server.listen(0, '127.0.0.1', () => + resolve({ server, port: (server.address() as AddressInfo).port }) + ); + }); +} + +// Deliberate exception to "no real network calls in tests": these use real +// `fetch` against loopback (127.0.0.1) servers, the only way to prove that +// `redirect: 'manual'` is honoured by the runtime. Nothing leaves the machine. +describe('NoRedirectPolicy', () => { + const servers: Server[] = []; + afterEach(() => { + servers.splice(0).forEach((s) => s.close()); + }); + + it('does not request the redirect target (real fetch, two local servers)', async () => { + const target = vi.fn(); + const second = await listen((_req, res) => { + target(); + res.end('target'); + }); + const first = await listen((_req, res) => { + res.writeHead(302, { location: `http://127.0.0.1:${second.port}/` }); + res.end(); + }); + servers.push(first.server, second.server); + + const pipeline = new HttpPipeline(); + pipeline.usePolicy(new NoRedirectPolicy()); + const response = await pipeline.execute(`http://127.0.0.1:${first.port}/`); + + expect(response.status).toBe(302); + expect(target).not.toHaveBeenCalled(); + }); + + it('overrides redirect: follow on a Request input', async () => { + const impl = vi.fn().mockResolvedValue(new Response('ok')); + const pipeline = new HttpPipeline(impl); + pipeline.usePolicy(new NoRedirectPolicy()); + + await pipeline.execute( + new Request('https://docs.mapbox.com/', { redirect: 'follow' }), + {} + ); + + expect(impl.mock.calls[0][1]).toMatchObject({ redirect: 'manual' }); + }); + + it('sets redirect: manual when the caller sets nothing', async () => { + const impl = vi.fn().mockResolvedValue(new Response('ok')); + const pipeline = new HttpPipeline(impl); + pipeline.usePolicy(new NoRedirectPolicy()); + + await pipeline.execute('https://docs.mapbox.com/', {}); + + expect(impl.mock.calls[0][1]).toMatchObject({ redirect: 'manual' }); + }); + + it('overrides a caller-supplied redirect: follow', async () => { + const impl = vi.fn().mockResolvedValue(new Response('ok')); + const pipeline = new HttpPipeline(impl); + pipeline.usePolicy(new NoRedirectPolicy()); + + await pipeline.execute('https://docs.mapbox.com/', { redirect: 'follow' }); + + expect(impl.mock.calls[0][1]).toMatchObject({ redirect: 'manual' }); + }); + + it('returns a 3xx response without a second request or retry', async () => { + const impl = vi.fn().mockResolvedValue( + new Response(null, { + status: 302, + headers: { location: 'https://example.com/other' } + }) + ); + const pipeline = new HttpPipeline(impl); + pipeline.usePolicy(new RetryPolicy(3, 1, 1)); + pipeline.usePolicy(new NoRedirectPolicy()); + + const response = await pipeline.execute('https://docs.mapbox.com/', {}); + + expect(response.status).toBe(302); + expect(impl).toHaveBeenCalledTimes(1); + }); + + it('is registered on the shared system pipeline', () => { + expect( + systemHttpPipeline.findPolicyById('system-no-redirect-policy') + ).toBeInstanceOf(NoRedirectPolicy); + }); +});