From 1f671e978ea028250b8fb8047753c0be17049671 Mon Sep 17 00:00:00 2001 From: Fili <126945152+filiksyos@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:33:04 +0300 Subject: [PATCH 1/2] Restore Library code cards by removing exact-count bottlenecks --- app/library/page.tsx | 6 +- components/library-page.tsx | 20 +++-- docs/library-code-counts.md | 33 +++++++++ lib/library-query.ts | 84 ++++++++++++++------- lib/library-types.ts | 4 + tests/library-browse.test.ts | 138 +++++++++++++++++++++++++++++++++++ 6 files changed, 252 insertions(+), 33 deletions(-) create mode 100644 docs/library-code-counts.md create mode 100644 tests/library-browse.test.ts diff --git a/app/library/page.tsx b/app/library/page.tsx index d02aac8..4875d1b 100644 --- a/app/library/page.tsx +++ b/app/library/page.tsx @@ -35,7 +35,7 @@ const getCachedInitialLibrary = unstable_cache( if (!supabase) throw new Error("Database unavailable."); return fetchInitialLibrary(supabase, INITIAL_LIMIT); }, - ["library-initial-newest-v4"], + ["library-initial-newest-v5"], { revalidate: 60 } ); @@ -57,7 +57,7 @@ export default async function LibraryRoute() { description: "Browse reverse-engineered coding agent prompts from GitHub repositories and live websites.", url: "https://gitreverse.com/library", - numberOfItems: initialTotal, + numberOfItems: initial.totalIsEstimate ? undefined : initialTotal, hasPart: initialData.slice(0, 10).map((entry) => ({ "@type": "TechArticle", name: entry.title, @@ -71,6 +71,8 @@ export default async function LibraryRoute() { diff --git a/components/library-page.tsx b/components/library-page.tsx index eb662f4..24a316f 100644 --- a/components/library-page.tsx +++ b/components/library-page.tsx @@ -32,17 +32,23 @@ function relativeTime(iso: string): string { type LibraryPageProps = { initialData: LibraryEntry[]; initialTotal: number; + initialTotalIsEstimate?: boolean; + initialHasMore?: boolean; initialUnavailableSources?: LibraryEntryKind[]; initialError?: string; }; export function LibraryPage({ - initialData, initialTotal, initialUnavailableSources = [], initialError, + initialData, initialTotal, initialTotalIsEstimate = false, + initialHasMore = initialData.length < initialTotal, + initialUnavailableSources = [], initialError, }: LibraryPageProps) { const [search, setSearch] = useState(""); const [sort, setSort] = useState("newest"); const [entries, setEntries] = useState(initialData); const [total, setTotal] = useState(initialTotal); + const [totalIsEstimate, setTotalIsEstimate] = useState(initialTotalIsEstimate); + const [hasMore, setHasMore] = useState(initialHasMore); const [error, setError] = useState(initialError); const [unavailableSources, setUnavailableSources] = useState(initialUnavailableSources); const requestRef = useRef(0); @@ -91,6 +97,8 @@ export function LibraryPage({ setEntries(json.data); } setTotal(json.total); + setTotalIsEstimate(json.totalIsEstimate ?? false); + setHasMore(json.hasMore ?? false); setUnavailableSources(json.unavailableSources ?? []); setPage(pageVal); } catch { @@ -99,6 +107,8 @@ export function LibraryPage({ if (!append) { setEntries([]); setTotal(0); + setTotalIsEstimate(false); + setHasMore(false); setUnavailableSources([]); } } @@ -135,8 +145,6 @@ export function LibraryPage({ } } - const hasMore = entries.length < total; - return (
@@ -148,7 +156,7 @@ export function LibraryPage({
- {error ? "Prompt library" : `${total.toLocaleString()}+ prompts`} + {error ? "Prompt library" : `${totalIsEstimate ? "~" : ""}${total.toLocaleString()} prompts`}
@@ -286,7 +294,7 @@ export function LibraryPage({ {search && !error ? (

- {total.toLocaleString()} + {totalIsEstimate ? "About " : ""}{total.toLocaleString()} {" "} result{total !== 1 ? "s" : ""}{unavailableSources.length > 0 ? " from available sources" : ""}

@@ -350,7 +358,7 @@ export function LibraryPage({ Loading… ) : ( - <>Load {Math.min(PAGE_SIZE, total - entries.length)} more ↓ + <>Load {totalIsEstimate ? "" : `${Math.min(PAGE_SIZE, total - entries.length)} `}more ↓ )}
diff --git a/docs/library-code-counts.md b/docs/library-code-counts.md new file mode 100644 index 0000000..0a675e1 --- /dev/null +++ b/docs/library-code-counts.md @@ -0,0 +1,33 @@ +# Library code-source timeout repair + +## Diagnosis (2026-10-29) + +Blank production Library browse returned only website cards with `unavailableSources: ["code"]`; code-only browse returned HTTP 503. PostgreSQL logs showed repeated SQLSTATE `57014` cancellations for code browse's combined page plus exact-count statement. Code rows, the cached-at/trending/FTS indexes, the existing security-invoker view, and its SELECT visibility were present. + +Read-only anonymous-role probes using the existing 3-second statement limit isolated the count dependency. Count-free PostgREST-shaped pages passed: newest ~17 ms, oldest ~28 ms, trending ~39 ms, accounting FTS ~172 ms, and no-match FTS ~150 ms. Combined newest page/count probes took ~2.5–2.9 seconds and production requests timed out. Separate exact trending and FTS counts also timed out, so merely splitting row/count requests is not a sufficient repair. Runtime varies; this is not evidence of a deterministic prompt-hydration or parallel-query issue. + +## Change + +- Code browse and keyword fallback use PostgREST `count: "planned"`, avoiding a full exact-count scan. This uses the same public view, publishable client, RLS, fields, filters, ordering, and 96-row per-source fetch bound. +- Responses label estimated totals with `totalIsEstimate`. The UI shows `~` / `About`, and structured data omits an estimated `numberOfItems`. +- Returned rows are authoritative: stale nonzero estimates cannot skip search fallback on an empty response, and estimates cannot report fewer results than the rows already fetched. +- `hasMore` follows the fetched merged window instead of an estimate. It stops on a partial page and at 96 displayed results. The prior cap was per source, which could expose more mixed entries, but merging deeper pages without fetching additional source rows is not reliable. This change makes the safe global stopping boundary explicit instead of repeatedly loading an exhausted window. +- Existing partial-source and true-outage reporting remains intact; the initial cached result key is bumped. + +[PostgREST count documentation](https://postgrest.org/en/stable/references/api/pagination_count.html) describes the planned-count tradeoff. Counts are intentionally approximate; cards are real returned rows. + +## Scope and rollout + +No migration, function, index, grants, RLS, secrets, embeddings provider, or timeout setting is changed. This is independent of draft PR #168 and does not apply its denied/pending RPC migration. Some keyword metadata queries can still time out even without exact counts; this does not claim to solve all keyword-query planning issues. + +The code must still be reviewed, merged and deployed with owner authorization before production behavior changes. After deployment, check blank Library (both sources), code-only newest/oldest/trending, pagination, and an existing keyword result. Verify the deployed commit and absence of code-source errors; don't infer deployment from a green test run. + +## Verification + +- Credential-free full suite: `node --import tsx --test tests/*.test.ts` (72 tests, including 34 Library tests). +- TypeScript and focused ESLint pass; production Next build passes without production credentials (local-only `VIEWS_IP_SALT` placeholder). +- Independent review found no blockers. +- Read-only production SQL shape checks pass as noted above; these are not deployed REST/UI verification. +- Repository-wide lint retains one pre-existing error in `lib/use-partner-preview.ts` and two pre-existing accessibility warnings in `components/reverse-prompt-home.tsx`. +- Local API and server-rendered HTML smoke checks passed with a credential-free mock database that rejects exact code counts; code cards, approximate labels, and `hasMore` were present without an outage warning. +- Local interactive Chromium QA was blocked by the executor's IPC/sandbox startup limitation; cloud browser cannot access this executor's loopback server. Do not count the attempted browser suite as passed. diff --git a/lib/library-query.ts b/lib/library-query.ts index 520c9e0..736acf0 100644 --- a/lib/library-query.ts +++ b/lib/library-query.ts @@ -116,15 +116,17 @@ async function fetchCodeBrowse( supabase: SupabaseClient, sort: SortOption, fetchLimit: number -): Promise<{ rows: PromptRow[]; total: number }> { +): Promise { + // Exact code-cache counts regularly exceed anon's 3s timeout, discarding + // otherwise fast indexed pages. Planner estimates avoid the full count scan. const { data, count, error } = await applyCodeSort( - supabase.from(CODE_TABLE).select(CODE_COLUMNS, { count: "exact" }), + supabase.from(CODE_TABLE).select(CODE_COLUMNS, { count: "planned" }), sort, false ).range(0, fetchLimit - 1); if (error) throw new Error(error.message); - return { rows: (data ?? []) as PromptRow[], total: count ?? 0 }; + return countedCodeRows(data as PromptRow[] | null, count); } async function fetchWebsiteBrowse( @@ -141,6 +143,21 @@ async function fetchWebsiteBrowse( return { rows: (data ?? []) as WebsiteRow[], total: count ?? 0 }; } +type CountedCodeRows = { + rows: PromptRow[]; + total: number; + totalIsEstimate?: true; +}; + +function countedCodeRows(data: PromptRow[] | null, count: number | null): CountedCodeRows { + const rows = data ?? []; + // Estimates can be stale or overestimate zero-match predicates. Returned rows + // are the authority for availability and the minimum total, never the estimate. + return rows.length === 0 + ? { rows, total: 0 } + : { rows, total: Math.max(rows.length, count ?? 0), totalIsEstimate: true }; +} + type FtsStrategy = "fts-plain" | "fts-or" | "ilike-and" | "ilike-or"; async function fetchCodeSearch( @@ -148,13 +165,13 @@ async function fetchCodeSearch( search: string, sort: SortOption, fetchLimit: number -): Promise<{ rows: PromptRow[]; total: number; strategy: FtsStrategy }> { +): Promise { const words = searchWords(search); const runQuery = (strategy?: FtsStrategy) => { let query = supabase .from(CODE_TABLE) - .select(CODE_COLUMNS, { count: "exact" }); + .select(CODE_COLUMNS, { count: "planned" }); if (words.length > 0 && strategy) { switch (strategy) { @@ -196,14 +213,13 @@ async function fetchCodeSearch( if (words.length > 0) strategies.push("ilike-and"); if (words.length > 1) strategies.push("ilike-or"); - let emptyResult: { rows: PromptRow[]; total: number; strategy: FtsStrategy } | undefined; + let emptyResult: (CountedCodeRows & { strategy: FtsStrategy }) | undefined; for (const strategy of strategies) { try { const res = await runQuery(words.length > 0 ? strategy : undefined); if (res.error) throw new Error(res.error.message); const result = { - rows: (res.data ?? []) as PromptRow[], - total: res.count ?? 0, + ...countedCodeRows(res.data as PromptRow[] | null, res.count), strategy, }; if (result.rows.length > 0 || result.total > 0) return result; @@ -376,6 +392,26 @@ function mergeFetchLimit(page: number, limit: number): number { return Math.min(MAX_FETCH_LIMIT, (page + 1) * limit); } +function pageResult( + entries: LibraryEntry[], + page: number, + limit: number, + total: number, + totalIsEstimate = false +): LibraryResult { + const through = (page + 1) * limit; + return { + data: paginateLibraryEntries(entries, page, limit), + total, + ...(totalIsEstimate ? { totalIsEstimate: true } : {}), + // An estimate must never hide a full page, nor keep an exhausted/96-row + // merge window loading forever. A full estimated page permits one more try. + hasMore: through < MAX_FETCH_LIMIT && (totalIsEstimate + ? entries.length >= through + : total > through), + }; +} + function logSourceFailure( source: string, operation: string, @@ -416,7 +452,7 @@ async function searchCodeSource(opts: { search: string; sort: SortOption; useHybrid: boolean; -}, fetchLimit: number): Promise<{ rows: PromptRow[]; total: number; strategy: string }> { +}, fetchLimit: number): Promise { if (opts.useHybrid) { try { // Reuse the same vector for rows and count rather than paying for it twice. @@ -446,10 +482,7 @@ export async function browseLibrary(opts: { if (kind === "code") { const code = await fetchCodeBrowse(opts.supabase, opts.sort, fetchLimit); const merged = mergeBrowse(code.rows, [], opts.sort); - return { - data: paginateLibraryEntries(merged, opts.page, opts.limit), - total: code.total, - }; + return pageResult(merged, opts.page, opts.limit, code.total, code.totalIsEstimate); } if (kind === "website") { @@ -459,10 +492,7 @@ export async function browseLibrary(opts: { fetchLimit ); const merged = mergeBrowse([], website.rows, opts.sort); - return { - data: paginateLibraryEntries(merged, opts.page, opts.limit), - total: website.total, - }; + return pageResult(merged, opts.page, opts.limit, website.total); } const [code, website] = await Promise.all([ @@ -481,8 +511,11 @@ export async function browseLibrary(opts: { opts.sort ); return { - data: paginateLibraryEntries(merged, opts.page, opts.limit), - total: (code.ok ? code.value.total : 0) + (website.ok ? website.value.total : 0), + ...pageResult( + merged, opts.page, opts.limit, + (code.ok ? code.value.total : 0) + (website.ok ? website.value.total : 0), + code.ok && code.value.totalIsEstimate + ), ...(unavailable.length > 0 ? { unavailableSources: unavailable } : {}), }; } @@ -508,8 +541,7 @@ export async function searchLibrary(opts: { ); const merged = mergeSearch([], website.rows, opts.search); return { - data: paginateLibraryEntries(merged, opts.page, opts.limit), - total: website.total, + ...pageResult(merged, opts.page, opts.limit, website.total), strategy: "website-metadata", }; } @@ -517,8 +549,7 @@ export async function searchLibrary(opts: { if (kind === "code") { const code = await searchCodeSource(opts, fetchLimit); return { - data: paginateLibraryEntries(mergeSearch(code.rows, [], opts.search), opts.page, opts.limit), - total: code.total, + ...pageResult(mergeSearch(code.rows, [], opts.search), opts.page, opts.limit, code.total, code.totalIsEstimate), strategy: code.strategy, }; } @@ -537,8 +568,11 @@ export async function searchLibrary(opts: { opts.search ); return { - data: paginateLibraryEntries(merged, opts.page, opts.limit), - total: (code.ok ? code.value.total : 0) + (website.ok ? website.value.total : 0), + ...pageResult( + merged, opts.page, opts.limit, + (code.ok ? code.value.total : 0) + (website.ok ? website.value.total : 0), + code.ok && code.value.totalIsEstimate + ), strategy: code.ok ? code.value.strategy : "website-metadata", ...(unavailable.length > 0 ? { unavailableSources: unavailable } : {}), }; diff --git a/lib/library-types.ts b/lib/library-types.ts index d3463ef..cb99639 100644 --- a/lib/library-types.ts +++ b/lib/library-types.ts @@ -24,6 +24,10 @@ export type LibraryResult = { data: LibraryEntry[]; /** Count for the available sources only when unavailableSources is present. */ total: number; + /** Code keyword/browse totals use planner estimates to avoid full-count timeouts. */ + totalIsEstimate?: boolean; + /** Whether another page can be read within the bounded merge window. */ + hasMore?: boolean; unavailableSources?: LibraryEntryKind[]; }; diff --git a/tests/library-browse.test.ts b/tests/library-browse.test.ts new file mode 100644 index 0000000..83b4905 --- /dev/null +++ b/tests/library-browse.test.ts @@ -0,0 +1,138 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createClient } from "@supabase/supabase-js"; +import { browseLibrary, fetchInitialLibrary, searchLibrary } from "../lib/library-query"; + +const codeRow = { + id: 1, owner: "example", repo: "code", title: "Code prompt", + prompt: "Build an application", views: 10, cached_at: "2026-10-29T00:00:00Z", +}; +const websiteRow = { + slug: "website", target_url: "https://website.example", prompt: "Build a website", + cached_at: "2026-10-28T00:00:00Z", +}; + +function rows(data: unknown[], total = data.length) { + return Response.json(data, { + headers: { "content-range": `0-${Math.max(0, data.length - 1)}/${total}` }, + }); +} +function failure() { + return Response.json({ code: "57014", message: "statement timeout" }, { status: 500 }); +} +function client(handler: (url: URL, init?: RequestInit) => Response) { + return createClient("https://database.example", "test-publishable-key", { + auth: { persistSession: false, autoRefreshToken: false }, + global: { fetch: async (input, init) => handler(new URL(String(input)), init) }, + }); +} + +for (const sort of ["newest", "oldest", "trending"] as const) { + test(`${sort} code browse avoids the exact-count timeout`, async () => { + let calls = 0; + const supabase = client((url, init) => { + calls++; + assert.ok(url.pathname.endsWith("/library_code_entries")); + // Reproduces the production failure: exact count + page times out. + if (new Headers(init?.headers).get("prefer")?.includes("count=exact")) return failure(); + assert.match(new Headers(init?.headers).get("prefer") ?? "", /count=planned/); + assert.equal(url.searchParams.get("select"), "id,owner,repo,prompt,cached_at,views,title"); + assert.equal(url.searchParams.get("limit"), "24"); + assert.equal(url.searchParams.get("order"), sort === "trending" + ? "views.desc,cached_at.desc" : `cached_at.${sort === "oldest" ? "asc" : "desc"}`); + assert.equal(url.searchParams.has("cached_at"), sort === "trending"); + return rows([codeRow], 77107); + }); + const result = await browseLibrary({ supabase, sort, page: 0, limit: 24, kind: "code" }); + assert.equal(result.data[0].kind, "code"); + assert.equal(result.total, 77107); + assert.equal(result.totalIsEstimate, true); + assert.equal(result.unavailableSources, undefined); + assert.equal(calls, 1); + }); +} + +test("initial Library browse returns both sources despite the exact-count timeout", async () => { + const supabase = client((url, init) => { + if (url.pathname.endsWith("/library_website_entries")) return rows([websiteRow], 12); + if (new Headers(init?.headers).get("prefer")?.includes("count=exact")) return failure(); + return rows([codeRow], 77107); + }); + const result = await fetchInitialLibrary(supabase, 24); + assert.deepEqual(result.data.map(({ kind }) => kind), ["code", "website"]); + assert.equal(result.total, 77119); + assert.equal(result.totalIsEstimate, true); + assert.equal(result.unavailableSources, undefined); +}); + +test("keyword code search also avoids exact counts without a new RPC", async () => { + const supabase = client((url, init) => { + assert.ok(url.pathname.endsWith("/library_code_entries")); + assert.match(new Headers(init?.headers).get("prefer") ?? "", /count=planned/); + assert.equal(url.searchParams.get("search_vector"), "plfts(english).accounting"); + return rows([codeRow], 80); + }); + const result = await searchLibrary({ supabase, search: "accounting", sort: "newest", page: 0, limit: 24, kind: "code", useHybrid: false }); + assert.equal(result.data[0].kind, "code"); + assert.equal(result.totalIsEstimate, true); + assert.equal(result.strategy, "fts-plain"); +}); + +test("a stale nonzero FTS estimate cannot skip metadata fallback on empty rows", async () => { + const supabase = client((url) => url.searchParams.has("search_vector") + ? rows([], 100) : rows([codeRow], 1)); + const result = await searchLibrary({ supabase, search: "accounting", sort: "newest", page: 0, limit: 24, kind: "code", useHybrid: false }); + assert.equal(result.strategy, "ilike-and"); + assert.equal(result.data.length, 1); +}); + +for (const estimate of [0, 2, 100000]) { + test(`pagination follows returned rows, not the estimate ${estimate}`, async () => { + const data = Array.from({ length: 25 }, (_, i) => ({ ...codeRow, id: i + 1 })); + const supabase = client((url) => rows(data.slice(0, Number(url.searchParams.get("limit"))), estimate)); + const first = await browseLibrary({ supabase, sort: "newest", page: 0, limit: 24, kind: "code" }); + assert.equal(first.data.length, 24); + assert.ok(first.total >= 24); + assert.equal(first.hasMore, true); + const last = await browseLibrary({ supabase, sort: "newest", page: 1, limit: 24, kind: "code" }); + assert.equal(last.data.length, 1); + assert.equal(last.hasMore, false); + }); +} + +test("mixed-source pagination keeps cards already fetched after the current page", async () => { + const supabase = client((url) => url.pathname.endsWith("/library_code_entries") + ? rows(Array.from({ length: 20 }, (_, i) => ({ ...codeRow, id: i + 1 })), 1) + : rows(Array.from({ length: 20 }, (_, i) => ({ ...websiteRow, slug: String(i) })), 20)); + const first = await browseLibrary({ supabase, sort: "newest", page: 0, limit: 24 }); + assert.equal(first.data.length, 24); + assert.equal(first.hasMore, true); + const last = await browseLibrary({ supabase, sort: "newest", page: 1, limit: 24 }); + assert.equal(last.data.length, 16); + assert.equal(last.hasMore, false); +}); + +test("browse stops loading when the existing 96-row merge window is exhausted", async () => { + const supabase = client((url) => { + assert.equal(url.searchParams.get("limit"), "96"); + return rows(Array.from({ length: 96 }, (_, i) => ({ ...codeRow, id: i + 1 })), 100000); + }); + const result = await browseLibrary({ supabase, sort: "newest", page: 3, limit: 24, kind: "code" }); + assert.equal(result.data.length, 24); + assert.equal(result.hasMore, false); +}); + +test("empty pages ignore stale estimates and do not advertise more results", async () => { + const result = await browseLibrary({ supabase: client(() => rows([], 1000)), sort: "newest", page: 0, limit: 24, kind: "code" }); + assert.equal(result.total, 0); + assert.equal(result.totalIsEstimate, undefined); + assert.equal(result.hasMore, false); +}); + +test("actual row-query failures still report an unavailable source", async () => { + const supabase = client((url) => url.pathname.endsWith("/library_code_entries") ? failure() : rows([websiteRow])); + const result = await browseLibrary({ supabase, sort: "newest", page: 0, limit: 24 }); + assert.deepEqual(result.unavailableSources, ["code"]); + assert.equal(result.totalIsEstimate, undefined); + await assert.rejects(browseLibrary({ supabase, sort: "newest", page: 0, limit: 24, kind: "code" })); +}); From 89c99151db12da2a4207e56a1f5e9c3808ad7990 Mon Sep 17 00:00:00 2001 From: Fili <126945152+filiksyos@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:35:20 +0300 Subject: [PATCH 2/2] Restrict generated cache writes to trusted server paths --- app/api/admin/backfill-embeddings/route.ts | 4 +- app/api/admin/backfill-titles/route.ts | 4 +- app/api/increment-views/route.ts | 4 +- app/api/reverse-prompt/route.ts | 4 +- app/api/reverse-website/route.ts | 20 +++++- docs/cache-write-security.md | 31 ++++++++ lib/supabase.ts | 2 + lib/website-reverse-engine.ts | 3 + lib/website-reverse-storage.ts | 34 +++++---- package.json | 7 +- pnpm-lock.yaml | 16 +++++ .../20261106092711_restrict_cache_writes.sql | 47 +++++++++++++ tests/cache-security-routes.test.ts | 70 +++++++++++++++++++ tests/cache-security-sql.test.ts | 60 ++++++++++++++++ 14 files changed, 282 insertions(+), 24 deletions(-) create mode 100644 docs/cache-write-security.md create mode 100644 supabase/migrations/20261106092711_restrict_cache_writes.sql create mode 100644 tests/cache-security-routes.test.ts create mode 100644 tests/cache-security-sql.test.ts diff --git a/app/api/admin/backfill-embeddings/route.ts b/app/api/admin/backfill-embeddings/route.ts index ac88afa..0e60b75 100644 --- a/app/api/admin/backfill-embeddings/route.ts +++ b/app/api/admin/backfill-embeddings/route.ts @@ -1,5 +1,5 @@ import { NextRequest, NextResponse } from "next/server"; -import { getSupabase } from "@/lib/supabase"; +import { getSupabaseAdmin } from "@/lib/supabase"; import { backfillPromptEmbeddings } from "@/lib/prompt-cache-embedding"; export const dynamic = "force-dynamic"; @@ -20,7 +20,7 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: "Unauthorized." }, { status: 401 }); } - const supabase = getSupabase(); + const supabase = getSupabaseAdmin(); if (!supabase) { return NextResponse.json({ error: "Database unavailable." }, { status: 503 }); } diff --git a/app/api/admin/backfill-titles/route.ts b/app/api/admin/backfill-titles/route.ts index 981c783..48c3798 100644 --- a/app/api/admin/backfill-titles/route.ts +++ b/app/api/admin/backfill-titles/route.ts @@ -1,5 +1,5 @@ import { NextRequest, NextResponse } from "next/server"; -import { getSupabase } from "@/lib/supabase"; +import { getSupabaseAdmin } from "@/lib/supabase"; import { backfillPromptTitles } from "@/lib/prompt-cache-title"; export const dynamic = "force-dynamic"; @@ -20,7 +20,7 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: "Unauthorized." }, { status: 401 }); } - const supabase = getSupabase(); + const supabase = getSupabaseAdmin(); if (!supabase) { return NextResponse.json({ error: "Database unavailable." }, { status: 503 }); } diff --git a/app/api/increment-views/route.ts b/app/api/increment-views/route.ts index 82dc81a..414732d 100644 --- a/app/api/increment-views/route.ts +++ b/app/api/increment-views/route.ts @@ -3,7 +3,7 @@ import { isValidGitHubRepoPath, normalizeRepoSegment, } from "@/lib/parse-github-repo"; -import { getSupabase } from "@/lib/supabase"; +import { getSupabaseAdmin } from "@/lib/supabase"; import { hashVisitorIp, isDefaultIpHashSaltInProduction, @@ -54,7 +54,7 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: "Invalid owner or repo." }, { status: 400 }); } - const supabase = getSupabase(); + const supabase = getSupabaseAdmin(); if (!supabase) { return NextResponse.json({ error: "Database unavailable." }, { status: 503 }); } diff --git a/app/api/reverse-prompt/route.ts b/app/api/reverse-prompt/route.ts index 1361ffd..76c2ead 100644 --- a/app/api/reverse-prompt/route.ts +++ b/app/api/reverse-prompt/route.ts @@ -3,7 +3,7 @@ import { SYSTEM_PROMPT } from "@/lib/system-prompt"; import { getFileTree, getReadme, getRepoMeta } from "@/lib/github-client"; import { formatAsFilteredTree } from "@/lib/file-tree-formatter"; import { parseGitHubRepoInput } from "@/lib/parse-github-repo"; -import { getSupabase } from "@/lib/supabase"; +import { getSupabase, getSupabaseAdmin } from "@/lib/supabase"; import { buildAzureChatCompletionsBody, buildAzureOpenAiUrl, @@ -512,7 +512,7 @@ export async function POST(request: NextRequest) { ); } - const sb = getSupabase(); + const sb = getSupabaseAdmin(); if (sb) { void sb .from("prompt_cache") diff --git a/app/api/reverse-website/route.ts b/app/api/reverse-website/route.ts index e5ffe28..eb5821e 100644 --- a/app/api/reverse-website/route.ts +++ b/app/api/reverse-website/route.ts @@ -1,5 +1,5 @@ import { NextRequest, NextResponse } from "next/server"; -import { isValidWebsiteSlug, parseWebsiteInput } from "@/lib/parse-website-input"; +import { isValidWebsiteSlug, parseWebsiteInput, urlToSlug } from "@/lib/parse-website-input"; import { ensureWebsiteReversed, type WebsiteReverseResult, @@ -114,6 +114,20 @@ export async function POST(request: NextRequest) { return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); } + if (!body || typeof body !== "object" || Array.isArray(body) || + (body.siteSlug !== undefined && typeof body.siteSlug !== "string") || + (body.targetUrl !== undefined && typeof body.targetUrl !== "string")) { + return NextResponse.json({ error: "Invalid request." }, { status: 400 }); + } + + // Refreshes overwrite shared generated content and require explicit admin access. + if (body.force === true) { + const secret = process.env.ADMIN_SECRET?.trim(); + if (!secret || request.headers.get("authorization") !== `Bearer ${secret}`) { + return NextResponse.json({ error: "Unauthorized refresh." }, { status: 403 }); + } + } + const siteSlug = body.siteSlug?.trim().toLowerCase(); if (!siteSlug || !isValidWebsiteSlug(siteSlug)) { return NextResponse.json( @@ -148,6 +162,10 @@ export async function POST(request: NextRequest) { ); } + if (urlToSlug(parsed.hostname) !== siteSlug) { + return NextResponse.json({ error: "Website slug does not match URL." }, { status: 400 }); + } + const timer = new Promise((_, reject) => setTimeout(() => reject(new Error("__timeout__")), ROUTE_TIMEOUT_MS) ); diff --git a/docs/cache-write-security.md b/docs/cache-write-security.md new file mode 100644 index 0000000..64870e8 --- /dev/null +++ b/docs/cache-write-security.md @@ -0,0 +1,31 @@ +# Cache write authorization + +Public Supabase keys are intentionally public. Generated cache content must be writable only by trusted server code, not by anonymous or ordinary signed-in API clients. + +## Release order + +1. Confirm production has `SUPABASE_URL`, `SUPABASE_PUBLISHABLE_KEY`, and server-only `SUPABASE_SERVICE_ROLE_KEY`. Never print or expose credential values. +2. Deploy this code before applying `20261106092711_restrict_cache_writes.sql`. It keeps public reads on the publishable client and moves cache generation, title/embedding backfills, and view counting to the existing privileged server client. +3. Apply the migration using the authorized production database management path. It removes non-SELECT policies and write grants (including column grants) on eight generated caches and `view_log`, and limits the three view-count RPCs to the service role. Missing legacy tables/functions are skipped. +4. Verify catalog privileges/policies, public library reads, admin backfill authorization, and normal generation/view counting. Do not poison a production row as a security test. Use a staging fixture for write-denial tests. + +## Authorization boundaries + +- `lib/supabase.ts` is marked `server-only`; it must never be imported into browser code. +- The reverse-prompt endpoint accepts a repository identifier, not arbitrary cached prompt text. Writes contain server-generated output. +- Website keys must match the URL-derived slug. Forced refreshes overwrite shared content and now require `ADMIN_SECRET` authorization. Regular users can still generate cache misses and read existing entries. +- Website storage binds an existing slug to its exact URL, including collision cases, and uses insert rather than upsert for new entries. A racing insert fails instead of overwriting a different URL. No credentials or arbitrary supplied prompt contents are accepted from the browser. +- Admin title/embedding endpoints retain their existing authorization checks before obtaining privileged access. +- This does not change JWT verification, billing, or user-history ownership policies. + +## Verification + +`pnpm test:cache-security` runs actual PostgreSQL role/RLS behavior in isolated PGlite databases, without production connectivity. It checks anonymous/authenticated write denials, column-grant removal, direct view-RPC denials, public reads, service-role writes, idempotence, and absent legacy objects. Route tests reject untrusted slug/refresh requests and verify privileged website writes with a local fetch stub. + +Run the full suite with `node --conditions=react-server --import tsx --test tests/*.test.ts`, plus TypeScript and targeted lint. The React server condition is required when testing server-only imports outside Next.js. + +## Rollback and limitations + +Do not restore insecure public write policies to undo an application problem. Keep the migrated database locked down and forward-fix server writes; reverting to an older application that uses anonymous writes would lose cache persistence and view updates. Preserve the previous deployment for inspection but do not promote it after migration without equivalent privileged writers. + +This closes direct API cache poisoning; it does not establish whether historical cache rows were modified. Review historical integrity separately. It also does not claim to solve adversarial source content or all existing SQL/security advisories. Legacy feature writers absent from current main must use server credentials before being reintroduced. diff --git a/lib/supabase.ts b/lib/supabase.ts index 878e447..362aa4a 100644 --- a/lib/supabase.ts +++ b/lib/supabase.ts @@ -1,3 +1,5 @@ +import "server-only"; + import { createClient, type SupabaseClient } from "@supabase/supabase-js"; let cached: SupabaseClient | null = null; diff --git a/lib/website-reverse-engine.ts b/lib/website-reverse-engine.ts index 7a8fa11..849b466 100644 --- a/lib/website-reverse-engine.ts +++ b/lib/website-reverse-engine.ts @@ -140,6 +140,9 @@ export async function ensureWebsiteReversed(opts: { if (!force) { const cached = await readWebsiteReverse(slug); if (cached) { + if (cached.meta.targetUrl !== targetUrl) { + return { ok: false, error: "Website slug is already bound to a different URL.", status: 409 }; + } const prompt = appendDesignSystemLink(cached.meta.prompt, slug); if (prompt !== cached.meta.prompt) { // Heal stale design-system URLs in the cache (api → /designs, etc.) diff --git a/lib/website-reverse-storage.ts b/lib/website-reverse-storage.ts index a68d664..257e80f 100644 --- a/lib/website-reverse-storage.ts +++ b/lib/website-reverse-storage.ts @@ -1,4 +1,4 @@ -import { getSupabase } from "@/lib/supabase"; +import { getSupabase, getSupabaseAdmin } from "@/lib/supabase"; export type WebsiteReverseMeta = { targetUrl: string; @@ -52,23 +52,31 @@ export async function writeWebsiteReverse(opts: { designMd: string; prompt: string; }): Promise { - const supabase = getSupabase(); + const supabase = getSupabaseAdmin(); if (!supabase) { throw new Error( - "Supabase not configured. Set SUPABASE_URL and SUPABASE_PUBLISHABLE_KEY." + "Supabase not configured. Set SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY." ); } - const { error } = await supabase.from("website_reverse_cache").upsert( - { - slug: opts.slug, - target_url: opts.targetUrl, - design_md: opts.designMd, - prompt: opts.prompt, - cached_at: new Date().toISOString(), - }, - { onConflict: "slug" } - ); + const { data: existing, error: readError } = await supabase + .from("website_reverse_cache").select("target_url").eq("slug", opts.slug).maybeSingle(); + if (readError) throw new Error(readError.message); + if (existing && existing.target_url !== opts.targetUrl) { + throw new Error("Website slug is already bound to a different URL."); + } + const row = { + slug: opts.slug, + target_url: opts.targetUrl, + design_md: opts.designMd, + prompt: opts.prompt, + cached_at: new Date().toISOString(), + }; + // An insert race must fail rather than overwrite another URL's cache entry. + const { error } = existing + ? await supabase.from("website_reverse_cache").update(row) + .eq("slug", opts.slug).eq("target_url", opts.targetUrl) + : await supabase.from("website_reverse_cache").insert(row); if (error) { throw new Error(error.message); diff --git a/package.json b/package.json index cce43b4..0786867 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "lint": "eslint", "test:titles": "node --env-file=.env.local scripts/test-titles.mjs", "db:migrate:title": "node --env-file=.env.local scripts/apply-title-migration.mjs", - "test:library": "node --import tsx --test tests/library-*.test.ts" + "test:library": "node --import tsx --test tests/library-*.test.ts", + "test:cache-security": "node --conditions=react-server --import tsx --test tests/cache-security*.test.ts" }, "dependencies": { "@supabase/supabase-js": "^2.101.1", @@ -20,6 +21,7 @@ "react": "19.2.4", "react-dom": "19.2.4", "react-markdown": "^10.1.0", + "server-only": "0.0.1", "stripe": "^17.7.0", "undici": "^8.1.0", "zod": "^4.4.3" @@ -33,6 +35,7 @@ "eslint-config-next": "16.2.1", "tailwindcss": "^4", "tsx": "4.23.15", - "typescript": "^5" + "typescript": "^5", + "@electric-sql/pglite": "0.5.15" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 662d072..23c262b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: react-markdown: specifier: ^10.1.0 version: 10.1.0(@types/react@19.2.14)(react@19.2.4) + server-only: + specifier: 0.0.1 + version: 0.0.1 stripe: specifier: ^17.7.0 version: 17.7.0 @@ -39,6 +42,9 @@ importers: specifier: ^4.4.3 version: 4.4.3 devDependencies: + '@electric-sql/pglite': + specifier: 0.5.15 + version: 0.5.15 '@tailwindcss/postcss': specifier: ^4 version: 4.2.2 @@ -69,6 +75,12 @@ importers: packages: + '@electric-sql/pglite@0.5.15': + resolution: {integrity: sha512-ERZWgaRTxueGV4cvL1tfpAOzFCWTXoUtbCBqnf2FhHDVs/so9eAKaanvrwWF9Dntx7BYsmFQujVQ0/B4S8Mmiw==} + + server-only@0.0.1: + resolution: {integrity: sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==} + '@alloc/quick-lru@5.2.0': resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} @@ -2452,6 +2464,10 @@ packages: snapshots: + '@electric-sql/pglite@0.5.15': {} + + server-only@0.0.1: {} + '@alloc/quick-lru@5.2.0': {} '@babel/code-frame@7.29.0': diff --git a/supabase/migrations/20261106092711_restrict_cache_writes.sql b/supabase/migrations/20261106092711_restrict_cache_writes.sql new file mode 100644 index 0000000..beca70d --- /dev/null +++ b/supabase/migrations/20261106092711_restrict_cache_writes.sql @@ -0,0 +1,47 @@ +-- Deploy server-only writers BEFORE applying this migration. +-- Public reads remain unchanged. Generated content is never client-writable. +DO $migration$ +DECLARE + table_name text; + policy_row record; + column_row record; + function_row record; +BEGIN + FOREACH table_name IN ARRAY ARRAY[ + 'prompt_cache', 'custom_prompt_cache', 'deep_report_cache', + 'website_reverse_cache', 'website_swap_cache', 'profile_reverse_cache', + 'game_reverse_cache', 'object3d_reverse_cache', 'view_log' + ] LOOP + -- Some legacy products are not present in every installation. + IF to_regclass(format('public.%I', table_name)) IS NULL THEN CONTINUE; END IF; + EXECUTE format('ALTER TABLE public.%I ENABLE ROW LEVEL SECURITY', table_name); + FOR policy_row IN SELECT policyname FROM pg_policies + WHERE schemaname = 'public' AND tablename = table_name AND cmd <> 'SELECT' + LOOP + EXECUTE format('DROP POLICY %I ON public.%I', policy_row.policyname, table_name); + END LOOP; + EXECUTE format('REVOKE INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER ON TABLE public.%I FROM PUBLIC, anon, authenticated', table_name); + -- Table REVOKE does not remove independently granted column privileges. + FOR column_row IN SELECT attname FROM pg_attribute + WHERE attrelid = to_regclass(format('public.%I', table_name)) + AND attnum > 0 AND NOT attisdropped + LOOP + EXECUTE format('REVOKE INSERT (%I), UPDATE (%I), REFERENCES (%I) ON TABLE public.%I FROM PUBLIC, anon, authenticated', + column_row.attname, column_row.attname, column_row.attname, table_name); + END LOOP; + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public.%I TO service_role', table_name); + END LOOP; + + -- Invoker view counting also now goes through the validated server endpoint. + -- Definer variants must not remain a public write bypass. + FOR function_row IN + SELECT p.oid::regprocedure AS signature FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'public' AND p.proname IN + ('increment_views', 'increment_profile_views', 'increment_object3d_views') + LOOP + EXECUTE format('REVOKE EXECUTE ON FUNCTION %s FROM PUBLIC, anon, authenticated', function_row.signature); + EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO service_role', function_row.signature); + END LOOP; +END; +$migration$; diff --git a/tests/cache-security-routes.test.ts b/tests/cache-security-routes.test.ts new file mode 100644 index 0000000..34d76d5 --- /dev/null +++ b/tests/cache-security-routes.test.ts @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { NextRequest } from "next/server"; +import { POST } from "../app/api/reverse-website/route"; + +const request = (body: unknown) => new NextRequest("https://gitreverse.com/api/reverse-website", { + method: "POST", body: JSON.stringify(body), headers: { "Content-Type": "application/json" }, +}); +test("website endpoint rejects arbitrary cache keys and unauthorized refreshes before generation", async () => { + assert.equal((await POST(request({ siteSlug: "victim-com", targetUrl: "https://attacker.com" }))).status, 400); + assert.equal((await POST(request({ siteSlug: "victim-com", targetUrl: "https://victim.com", force: true }))).status, 403); + for (const body of [null, [], { siteSlug: 1 }, { siteSlug: "victim-com", targetUrl: 1 }]) { + assert.equal((await POST(request(body))).status, 400); + } +}); +test("trusted writes use server-only credentials; public reads retain publishable access", () => { + const client = readFileSync("lib/supabase.ts", "utf8"); + assert.match(client, /import "server-only"/); + const promptRoute = readFileSync("app/api/reverse-prompt/route.ts", "utf8"); + assert.match(promptRoute, /const supabase = getSupabase\(\)/); + assert.match(promptRoute, /const sb = getSupabaseAdmin\(\)/); + for (const path of ["app/api/increment-views/route.ts", "app/api/admin/backfill-titles/route.ts", "app/api/admin/backfill-embeddings/route.ts"]) { + const content = readFileSync(path, "utf8"); + assert.match(content, /getSupabaseAdmin\(\)/); + assert.doesNotMatch(content, /getSupabase\(\)/); + } + const website = readFileSync("lib/website-reverse-storage.ts", "utf8"); + const writer = website.slice(website.indexOf("export async function writeWebsiteReverse")); + assert.match(writer, /getSupabaseAdmin\(\)/); + assert.doesNotMatch(writer, /\.upsert\(/); + assert.match(writer, /existing.target_url !== opts.targetUrl/); + assert.match(writer, /\.eq\("target_url", opts.targetUrl\)/); +}); + +test("website storage uses privileged credentials and refuses URL takeover without writing", async () => { + const { writeWebsiteReverse } = await import("../lib/website-reverse-storage"); + const before = { url: process.env.SUPABASE_URL, key: process.env.SUPABASE_SERVICE_ROLE_KEY }; + process.env.SUPABASE_URL = "https://cache-security-test.invalid"; + process.env.SUPABASE_SERVICE_ROLE_KEY = "test-server-only-key"; + const originalFetch = globalThis.fetch; + let existing: { target_url: string } | null = null; + const writes: string[] = []; + globalThis.fetch = async (input, init) => { + const request = new Request(input, init); + assert.equal(new URL(request.url).hostname, "cache-security-test.invalid"); + assert.equal(request.headers.get("apikey"), "test-server-only-key"); + if (request.method === "GET") return Response.json(existing ? [existing] : []); + writes.push(request.method); + assert.notEqual(request.headers.get("prefer")?.includes("resolution=merge-duplicates"), true); + if (request.method === "PATCH") { + assert.equal(new URL(request.url).searchParams.get("target_url"), "eq.https://example.com/"); + } + return new Response(null, { status: 204 }); + }; + try { + const row = { slug: "example-com", targetUrl: "https://example.com/", prompt: "generated", designMd: "generated design" }; + await writeWebsiteReverse(row); + assert.deepEqual(writes, ["POST"]); + existing = { target_url: row.targetUrl }; + await writeWebsiteReverse(row); + assert.deepEqual(writes, ["POST", "PATCH"]); + await assert.rejects(writeWebsiteReverse({ ...row, targetUrl: "https://example-com.attacker.test/" }), /different URL/); + assert.deepEqual(writes, ["POST", "PATCH"]); + } finally { + globalThis.fetch = originalFetch; + if (before.url === undefined) delete process.env.SUPABASE_URL; else process.env.SUPABASE_URL = before.url; + if (before.key === undefined) delete process.env.SUPABASE_SERVICE_ROLE_KEY; else process.env.SUPABASE_SERVICE_ROLE_KEY = before.key; + } +}); diff --git a/tests/cache-security-sql.test.ts b/tests/cache-security-sql.test.ts new file mode 100644 index 0000000..00a4bc4 --- /dev/null +++ b/tests/cache-security-sql.test.ts @@ -0,0 +1,60 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { PGlite } from "@electric-sql/pglite"; + +const migration = readFileSync("supabase/migrations/20261106092711_restrict_cache_writes.sql", "utf8"); +const tables = ["prompt_cache", "custom_prompt_cache", "deep_report_cache", "website_reverse_cache", "website_swap_cache", "profile_reverse_cache", "game_reverse_cache", "object3d_reverse_cache", "view_log"]; + +test("migration blocks anonymous and signed-in writes but retains reads and trusted writes", async () => { + const db = new PGlite(); + try { + await db.exec(`CREATE ROLE anon; CREATE ROLE authenticated; CREATE ROLE service_role BYPASSRLS; + GRANT USAGE ON SCHEMA public TO anon, authenticated, service_role;`); + for (const table of tables) { + await db.exec(`CREATE TABLE public.${table} (id integer PRIMARY KEY, prompt text); + INSERT INTO public.${table} VALUES (1, 'trusted'); + ALTER TABLE public.${table} ENABLE ROW LEVEL SECURITY; + GRANT ALL ON public.${table} TO anon, authenticated; + GRANT INSERT (prompt), UPDATE (prompt) ON public.${table} TO PUBLIC; + CREATE POLICY read_public ON public.${table} FOR SELECT TO anon, authenticated USING (true); + CREATE POLICY insert_public ON public.${table} FOR INSERT TO anon, authenticated WITH CHECK (true); + CREATE POLICY update_public ON public.${table} FOR UPDATE TO anon, authenticated USING (true) WITH CHECK (true);`); + } + await db.exec(`CREATE FUNCTION public.increment_views() RETURNS void LANGUAGE sql AS $$ UPDATE public.prompt_cache SET prompt = 'view'; $$; + CREATE FUNCTION public.increment_profile_views() RETURNS void LANGUAGE sql SECURITY DEFINER AS $$ UPDATE public.profile_reverse_cache SET prompt = 'view'; $$; + CREATE FUNCTION public.increment_object3d_views() RETURNS void LANGUAGE sql SECURITY DEFINER AS $$ UPDATE public.object3d_reverse_cache SET prompt = 'view'; $$;`); + await db.exec(migration); + await db.exec(migration); // idempotence + for (const role of ["anon", "authenticated"]) { + await db.exec(`SET ROLE ${role}`); + for (const table of tables) { + const { rows } = await db.query(`SELECT prompt FROM public.${table}`); + assert.deepEqual(rows, [{ prompt: "trusted" }]); + for (const sql of [ + `INSERT INTO public.${table} VALUES (2, 'poison')`, + `UPDATE public.${table} SET prompt = 'poison' WHERE id = 1`, + `DELETE FROM public.${table} WHERE id = 1`, + `TRUNCATE public.${table}`, + ]) await assert.rejects(db.exec(sql), /permission denied/); + } + for (const name of ["increment_views", "increment_profile_views", "increment_object3d_views"]) { + await assert.rejects(db.exec(`SELECT public.${name}()`), /permission denied/); + } + await db.exec("RESET ROLE"); + } + await db.exec("SET ROLE service_role"); + for (const table of tables) { + await db.exec(`INSERT INTO public.${table} VALUES (2, 'new'); UPDATE public.${table} SET prompt = 'updated' WHERE id = 2; DELETE FROM public.${table} WHERE id = 2;`); + } + await db.exec("SELECT public.increment_views(); SELECT public.increment_profile_views(); SELECT public.increment_object3d_views(); RESET ROLE;"); + } finally { await db.close(); } +}); + +test("migration tolerates installations without legacy caches or view RPCs", async () => { + const db = new PGlite(); + try { + await db.exec("CREATE ROLE anon; CREATE ROLE authenticated; CREATE ROLE service_role BYPASSRLS;"); + await db.exec(migration); + } finally { await db.close(); } +});