From 9134af84c72f2b6458cf9ef01839f807711610b7 Mon Sep 17 00:00:00 2001 From: Fili <126945152+filiksyos@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:52:01 +0300 Subject: [PATCH] Use an ID-first keyword search RPC to avoid sorted vector scans --- .github/workflows/library-search.yml | 1 + lib/library-query.ts | 22 +++- .../20261006134747_library_keyword_search.sql | 119 ++++++++++++++++++ tests/library-query.test.ts | 9 +- tests/library-rpc.test.ts | 78 ++++++++++++ 5 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 supabase/migrations/20261006134747_library_keyword_search.sql create mode 100644 tests/library-rpc.test.ts diff --git a/.github/workflows/library-search.yml b/.github/workflows/library-search.yml index b044812..6ec92ce 100644 --- a/.github/workflows/library-search.yml +++ b/.github/workflows/library-search.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - 'lib/library-*.ts' + - 'supabase/migrations/*library*.sql' - 'lib/embeddings.ts' - 'lib/azure-openai.ts' - 'app/api/library/**' diff --git a/lib/library-query.ts b/lib/library-query.ts index 520c9e0..7bbde04 100644 --- a/lib/library-query.ts +++ b/lib/library-query.ts @@ -151,7 +151,7 @@ async function fetchCodeSearch( ): Promise<{ rows: PromptRow[]; total: number; strategy: FtsStrategy }> { const words = searchWords(search); - const runQuery = (strategy?: FtsStrategy) => { + const runLegacyQuery = (strategy?: FtsStrategy) => { let query = supabase .from(CODE_TABLE) .select(CODE_COLUMNS, { count: "exact" }); @@ -191,6 +191,26 @@ async function fetchCodeSearch( return applyCodeSort(query, sort, words.length > 0).range(0, fetchLimit - 1); }; + const runQuery = async (strategy?: FtsStrategy) => { + const { data, error } = await supabase.rpc("library_keyword_search", { + query_text: search, + query_words: words, + search_strategy: strategy ?? "fts-plain", + sort_mode: sort === "newest" || sort === "oldest" ? sort : "trending", + match_count: fetchLimit, + }); + // Allow application and database rollouts in either order. Other RPC errors + // remain failures and must not silently fall back to the expensive query plan. + if (error?.code === "PGRST202") { + return runLegacyQuery(strategy); + } + if (error) return { data: null, count: null, error }; + if (!data || !Array.isArray(data.rows) || typeof data.total !== "number") { + throw new Error("Keyword search returned an invalid response."); + } + return { data: data.rows, count: data.total, error: null }; + }; + const strategies: FtsStrategy[] = ["fts-plain"]; if (words.length > 1) strategies.push("fts-or"); if (words.length > 0) strategies.push("ilike-and"); diff --git a/supabase/migrations/20261006134747_library_keyword_search.sql b/supabase/migrations/20261006134747_library_keyword_search.sql new file mode 100644 index 0000000..1bf73d9 --- /dev/null +++ b/supabase/migrations/20261006134747_library_keyword_search.sql @@ -0,0 +1,119 @@ +-- Preserves library_code_entries visibility, existing RLS, text-search semantics, +-- exact totals, newest/oldest/trending ordering, and the app's 96-row merge cap. +-- query_words must be the existing TypeScript searchWords(query_text) result. +-- The caller retains the current strategy fallback order. +CREATE OR REPLACE FUNCTION public.library_keyword_search( + query_text text, + query_words text[], + search_strategy text, + sort_mode text, + match_count integer +) +RETURNS jsonb +LANGUAGE plpgsql +STABLE +SECURITY INVOKER +SET search_path = '' +AS $function$ +DECLARE + v_query tsquery; + v_words text[] := COALESCE(query_words, ARRAY[]::text[]); + v_predicate text; + v_order text; + v_since timestamptz; + v_limit integer := LEAST(96, GREATEST(1, COALESCE(match_count, 24))); + v_result jsonb; +BEGIN + -- Only fixed, developer-authored SQL fragments are interpolated below. + -- All user input is passed as EXECUTE ... USING parameters. + CASE search_strategy + WHEN 'fts-plain' THEN + v_query := pg_catalog.plainto_tsquery('pg_catalog.english'::regconfig, query_text); + v_predicate := 'c.search_vector @@ $1'; + WHEN 'fts-or' THEN + v_query := pg_catalog.websearch_to_tsquery( + 'pg_catalog.english'::regconfig, pg_catalog.array_to_string(v_words, ' OR ') + ); + v_predicate := 'c.search_vector @@ $1'; + WHEN 'ilike-and', 'ilike-or' THEN + -- Generate one ordinary OR clause per term. Terms remain bound parameters; + -- only integer array indexes are interpolated. This also leaves the clauses + -- usable by optional future trigram indexes, unlike a correlated unnest. + SELECT pg_catalog.string_agg( + pg_catalog.format($predicate$ + ( + c.owner ILIKE ('%%' || pg_catalog.replace(($2::text[])[%s], '*', '%%') || '%%') + OR c.repo ILIKE ('%%' || pg_catalog.replace(($2::text[])[%s], '*', '%%') || '%%') + OR c.title ILIKE ('%%' || pg_catalog.replace(($2::text[])[%s], '*', '%%') || '%%') + ) + $predicate$, words.i, words.i, words.i), + CASE WHEN search_strategy = 'ilike-and' THEN ' AND ' ELSE ' OR ' END + ORDER BY words.i + ) INTO v_predicate + FROM pg_catalog.generate_subscripts(v_words, 1) AS words(i); + ELSE + RAISE EXCEPTION 'Unsupported library search strategy' USING ERRCODE = '22023'; + END CASE; + + -- The existing code adds no search filter when searchWords is empty. + IF pg_catalog.cardinality(v_words) = 0 THEN + v_predicate := 'true'; + END IF; + + CASE sort_mode + WHEN 'oldest' THEN + v_order := 'm.cached_at ASC, m.views DESC, m.id DESC'; + WHEN 'newest' THEN + v_order := 'm.cached_at DESC, m.views DESC, m.id DESC'; + WHEN 'trending' THEN + v_order := 'm.views DESC, m.cached_at DESC, m.id DESC'; + v_since := pg_catalog.now() - INTERVAL '30 days'; + v_predicate := '(' || v_predicate || ') AND c.cached_at >= $3'; + ELSE + RAISE EXCEPTION 'Unsupported library sort mode' USING ERRCODE = '22023'; + END CASE; + + -- Dynamic SQL deliberately gets a fresh plan for each query. This avoids a + -- generic CASE/OR strategy predicate hiding the FTS condition from the planner. + EXECUTE pg_catalog.format($query$ + WITH matched AS MATERIALIZED ( + SELECT c.id, c.cached_at, c.views + FROM public.library_code_entries AS c + WHERE %s + ), + page AS MATERIALIZED ( + SELECT m.id, pg_catalog.row_number() OVER (ORDER BY %s) AS ordinal + FROM matched AS m + ORDER BY %s + LIMIT $4 + ) + SELECT pg_catalog.jsonb_build_object( + 'total', (SELECT pg_catalog.count(*) FROM matched), + 'rows', COALESCE(( + SELECT pg_catalog.jsonb_agg( + pg_catalog.jsonb_build_object( + 'id', c.id, + 'owner', c.owner, + 'repo', c.repo, + 'prompt', pg_catalog.left(c.prompt, 180), + 'cached_at', c.cached_at, + 'views', c.views, + 'title', c.title + ) ORDER BY p.ordinal + ) + FROM page AS p + JOIN public.library_code_entries AS c ON c.id = p.id + ), '[]'::jsonb) + ) + $query$, v_predicate, v_order, v_order) + INTO v_result + USING v_query, v_words, v_since, v_limit; + + RETURN v_result; +END; +$function$; + +-- Restrict this new endpoint to existing library audiences. SECURITY INVOKER and +-- the existing security-invoker view continue to enforce caller permissions/RLS. +REVOKE ALL ON FUNCTION public.library_keyword_search(text, text[], text, text, integer) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION public.library_keyword_search(text, text[], text, text, integer) TO anon, authenticated; diff --git a/tests/library-query.test.ts b/tests/library-query.test.ts index abf7e6b..96418a3 100644 --- a/tests/library-query.test.ts +++ b/tests/library-query.test.ts @@ -27,7 +27,14 @@ function failure(message = "statement timeout") { function client(handler: (url: URL, init?: RequestInit) => Response | Promise) { return createClient("https://database.example", "test-publishable-key", { auth: { persistSession: false, autoRefreshToken: false }, - global: { fetch: async (input, init) => handler(new URL(String(input)), init) }, + global: { fetch: async (input, init) => { + const url = new URL(String(input)); + // Existing behavior remains covered for database/application rollout skew. + if (url.pathname.endsWith("/rpc/library_keyword_search")) { + return Response.json({ code: "PGRST202", message: "Function not deployed" }, { status: 404 }); + } + return handler(url, init); + } }, }); } diff --git a/tests/library-rpc.test.ts b/tests/library-rpc.test.ts new file mode 100644 index 0000000..d3c70ae --- /dev/null +++ b/tests/library-rpc.test.ts @@ -0,0 +1,78 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createClient } from "@supabase/supabase-js"; +import { searchLibrary } from "../lib/library-query"; + +const row = { id: 1, owner: "example", repo: "accounting", title: "Accounting", prompt: "Preview", cached_at: "2026-09-01T00:00:00Z", views: 1 }; +const options = { search: "accounting", sort: "newest" as const, page: 0, limit: 24, useHybrid: false, kind: "code" as const }; +function client(handler: (params: Record) => Response) { + return createClient("https://database.example", "test-key", { + auth: { persistSession: false, autoRefreshToken: false }, + global: { fetch: async (input, init) => { + assert.ok(String(input).endsWith("/rpc/library_keyword_search"), "must use the optimized RPC, not the legacy query"); + return handler(JSON.parse(String(init?.body))); + } }, + }); +} + +for (const search of ["accounting,", "a(b)", 'a"b', String.raw`a\b`, "first, second(test)"]) { + test(`passes search terms as RPC data: ${JSON.stringify(search)}`, async () => { + let calls = 0; + const supabase = client((params) => { + calls++; + assert.deepEqual(params, { + query_text: search, query_words: search.split(/\s+/u), search_strategy: "fts-plain", + sort_mode: "newest", match_count: 24, + }); + return Response.json({ rows: [row], total: 17 }); + }); + const result = await searchLibrary({ ...options, search, supabase }); + assert.equal(calls, 1); + assert.equal(result.total, 17); + assert.equal(result.data[0].id, 1); + }); +} + +test("RPC fallback preserves strategy order and supplies original search words", async () => { + const strategies: unknown[] = []; + const supabase = client((params) => { + strategies.push(params.search_strategy); + assert.deepEqual(params.query_words, ["first,", 'second"']); + return Response.json({ rows: params.search_strategy === "ilike-or" ? [row] : [], total: params.search_strategy === "ilike-or" ? 1 : 0 }); + }); + const result = await searchLibrary({ ...options, search: 'first, second"', supabase }); + assert.deepEqual(strategies, ["fts-plain", "fts-or", "ilike-and", "ilike-or"]); + assert.equal(result.strategy, "ilike-or"); +}); + +test("RPC FTS failure can recover via the metadata strategy", async () => { + const supabase = client((params) => params.search_strategy === "fts-plain" + ? Response.json({ code: "57014", message: "timeout" }, { status: 500 }) + : Response.json({ rows: [row], total: 1 })); + const result = await searchLibrary({ ...options, supabase }); + assert.equal(result.strategy, "ilike-and"); +}); + +test("RPC errors do not silently run the expensive legacy queries", async () => { + const supabase = client(() => Response.json({ code: "57014", message: "timeout" }, { status: 500 })); + await assert.rejects(searchLibrary({ ...options, supabase }), /temporarily unavailable/); +}); + +test("an invalid RPC response is a source failure, not a successful empty search", async () => { + await assert.rejects(searchLibrary({ ...options, supabase: client(() => Response.json(null)) }), /temporarily unavailable/); +}); + +test("empty RPC results remain a genuine no-match search", async () => { + const result = await searchLibrary({ ...options, supabase: client(() => Response.json({ rows: [], total: 0 })) }); + assert.equal(result.total, 0); + assert.equal(result.unavailableSources, undefined); +}); + +test("RPC fetch limits stay capped and match the selected sort", async () => { + const supabase = client((params) => { + assert.equal(params.match_count, 96); + assert.equal(params.sort_mode, "oldest"); + return Response.json({ rows: [row], total: 100 }); + }); + await searchLibrary({ ...options, supabase, sort: "oldest", page: 10 }); +});