fix(pgvector,supabase): negated ranges, zero vectors, and metric detection

.changeset/postgres-query-correctness.md

@@ -0,0 +1,5 @@
+---
+"vecstore-sdk": patch
+---
+
+Fix three pgvector and Supabase query defects. `not` around a range filter, such as `not(gt("year", 2000))`, now matches records that have no `year`, as the docs describe; the range comparison evaluated to SQL `NULL` for a missing field, so those records were skipped by queries and by `delete({ filter })`. A record with a zero vector, whose cosine score is NaN, now ranks last instead of first when Postgres reads an HNSW-indexed table without the index. The metric is now read from the `embedding` column's operator class only, so a table whose name contains an operator class name, such as `x_vector_cosine_ops`, keeps its own metric. Re-run `sql/supabase.sql` to get the new `vecstore_filter_sql`, `vecstore_metric`, and `vecstore_query`.

apps/web/content/docs/core/filters.mdx

@@ -49,7 +49,8 @@ The filter `and(eq("genre", "drama"), gt("year", 2000))` compiles to:
 
 ```sql title="pgvector and Supabase"
 ((metadata @> $1::jsonb OR metadata @> $2::jsonb)
-  AND (jsonb_typeof((metadata->$3::text)) = 'number'
+  AND ((metadata->$3::text) IS NOT NULL
+    AND jsonb_typeof((metadata->$3::text)) = 'number'
     AND (metadata->$3::text) > $4::jsonb))
 -- $1 = '{"genre":"drama"}', $2 = '{"genre":["drama"]}', $3 = 'year', $4 = '2000'
 ```

apps/web/lib/landing-content.ts

@@ -171,11 +171,13 @@ export const demoExamples: readonly DemoExample[] = [
     output: {
       pgvector: `((metadata @> $1::jsonb
   OR metadata @> $2::jsonb)
-  AND (jsonb_typeof(
-    (metadata->$3::text))
-    = 'number'
+  AND ((metadata->$3::text)
+    IS NOT NULL
+    AND jsonb_typeof(
+      (metadata->$3::text))
+      = 'number'
     AND (metadata->$3::text)
-    > $4::jsonb))
+      > $4::jsonb))
 
 $1 = '{"genre":"drama"}'
 $2 = '{"genre":["drama"]}'
@@ -203,7 +205,8 @@ $4 = '2000'`,
   @year:[(2000 +inf])`,
       supabase: `((metadata @> '{"genre": "drama"}'::jsonb
   OR metadata @> '{"genre": ["drama"]}'::jsonb)
-  AND (jsonb_typeof((metadata -> 'year'::text))
+  AND ((metadata -> 'year'::text) IS NOT NULL
+    AND jsonb_typeof((metadata -> 'year'::text))
         = 'number'
     AND (metadata -> 'year'::text) > '2000'::jsonb))`,
       upstash: `(genre = 'drama' AND year > 2000)`,
@@ -222,10 +225,12 @@ $4 = '2000'`,
     id: "range",
     label: "Range",
     output: {
-      pgvector: `((jsonb_typeof((metadata->$1::text))
+      pgvector: `(((metadata->$1::text) IS NOT NULL
+    AND jsonb_typeof((metadata->$1::text))
         = 'number'
     AND (metadata->$1::text) >= $2::jsonb)
-  AND (jsonb_typeof((metadata->$3::text))
+  AND ((metadata->$3::text) IS NOT NULL
+    AND jsonb_typeof((metadata->$3::text))
         = 'number'
     AND (metadata->$3::text) <= $4::jsonb))
 
@@ -251,12 +256,14 @@ $3 = 'price'   $4 = '100'`,
 }`,
       redis: `(@price:[10 +inf]
   @price:[-inf 100])`,
-      supabase: `((jsonb_typeof((metadata -> 'price'::text))
-      = 'number'
-  AND (metadata -> 'price'::text) >= '10'::jsonb)
-  AND (jsonb_typeof((metadata -> 'price'::text))
-      = 'number'
-  AND (metadata -> 'price'::text) <= '100'::jsonb))`,
+      supabase: `(((metadata -> 'price'::text) IS NOT NULL
+    AND jsonb_typeof((metadata -> 'price'::text))
+        = 'number'
+    AND (metadata -> 'price'::text) >= '10'::jsonb)
+  AND ((metadata -> 'price'::text) IS NOT NULL
+    AND jsonb_typeof((metadata -> 'price'::text))
+        = 'number'
+    AND (metadata -> 'price'::text) <= '100'::jsonb))`,
       upstash: `(price >= 10 AND price <= 100)`,
       vectorize: `{
   "price": { "$gte": 10, "$lte": 100 }

packages/vecstore-sdk/sql/supabase.sql

@@ -104,7 +104,7 @@ begin
       when 'lt' then '<'
       else '<='
     end;
-    return '(jsonb_typeof(' || path || ') = ''number'' AND ' || path || ' '
+    return '(' || path || ' IS NOT NULL AND jsonb_typeof(' || path || ') = ''number'' AND ' || path || ' '
       || operator || ' ' || quote_literal(match_filter -> 'value') || '::jsonb)';
   end if;
 
@@ -135,17 +135,17 @@ as $$
   select coalesce(
     (
       select case
-        when indexdef like '%vector_cosine_ops%' then 'cosine'
-        when indexdef like '%vector_ip_ops%' then 'dot'
-        when indexdef like '%vector_l2_ops%' then 'euclidean'
+        when strpos(indexdef, '(embedding vector_cosine_ops)') > 0 then 'cosine'
+        when strpos(indexdef, '(embedding vector_ip_ops)') > 0 then 'dot'
+        when strpos(indexdef, '(embedding vector_l2_ops)') > 0 then 'euclidean'
       end
       from pg_indexes
       where tablename = index_name
         and schemaname = coalesce(index_schema, current_schema())
         and (
-          indexdef like '%vector_cosine_ops%'
-          or indexdef like '%vector_ip_ops%'
-          or indexdef like '%vector_l2_ops%'
+          strpos(indexdef, '(embedding vector_cosine_ops)') > 0
+          or strpos(indexdef, '(embedding vector_ip_ops)') > 0
+          or strpos(indexdef, '(embedding vector_l2_ops)') > 0
         )
       limit 1
     ),
@@ -284,7 +284,6 @@ declare
     when 'dot' then '-(' || distance || ')'
     else '1 - (' || distance || ')'
   end;
-  score_order text := case metric when 'euclidean' then 'asc' else 'desc' end;
   where_sql text := 'namespace = ' || quote_literal(match_namespace);
 begin
   if match_filter is not null then
@@ -309,16 +308,16 @@ begin
 
   return query execute format(
     'select hits.id, hits.metadata, hits.embedding, hits.score from (
-       select id, metadata, %s as embedding, %s as score
+       select id, metadata, %s as embedding, %s as score, %s as distance
        from %s where %s order by %s limit %s
-     ) as hits order by hits.score %s',
+     ) as hits order by hits.distance',
     case when include_vector then 'embedding::text' else 'null::text' end,
     score_sql,
+    distance,
     vecstore_table(index_name, index_schema),
     where_sql,
     distance,
-    match_count,
-    score_order
+    match_count
   );
 end;
 $$;

packages/vecstore-sdk/src/filter/pgvector.ts

@@ -49,7 +49,7 @@ const compileNode = (filter: Filter, column: string, param: Param): string => {
       const path = fieldPath(filter.field);
       const operator = RANGE_OPERATORS[filter.kind];
       const value = `${param(JSON.stringify(filter.value))}::jsonb`;
-      return `(jsonb_typeof(${path}) = 'number' AND ${path} ${operator} ${value})`;
+      return `(${path} IS NOT NULL AND jsonb_typeof(${path}) = 'number' AND ${path} ${operator} ${value})`;
     }
     case "in": {
       return containsAny(column, filter.field, filter.values, param);

packages/vecstore-sdk/src/pgvector/index.ts

@@ -74,12 +74,6 @@ const DISTANCE_OPERATORS: Record<Metric, string> = {
   euclidean: "<->",
 };
 
-const SCORE_ORDERS: Record<Metric, string> = {
-  cosine: "DESC",
-  dot: "DESC",
-  euclidean: "ASC",
-};
-
 const HNSW_METHOD = "USING hnsw";
 const ITERATIVE_SCAN_MINOR = 8;
 const ITERATIVE_SCAN_SETTING =
@@ -116,16 +110,19 @@ const extensionVersion = (row: IndexDefinitionRow): string =>
   "extversion" in row && isString(row.extversion) ? row.extversion : "";
 
 const OPCLASS_METRICS: readonly (readonly [string, Metric])[] = [
-  [OPCLASSES.cosine, "cosine"],
-  [OPCLASSES.dot, "dot"],
-  [OPCLASSES.euclidean, "euclidean"],
+  [`(embedding ${OPCLASSES.cosine})`, "cosine"],
+  [`(embedding ${OPCLASSES.dot})`, "dot"],
+  [`(embedding ${OPCLASSES.euclidean})`, "euclidean"],
 ];
 
 const metricFromIndexDef = (definition: string): Metric | undefined =>
   OPCLASS_METRICS.find(([opclass]) => definition.includes(opclass))?.[1];
 
+const distanceExpression = (metric: Metric, vectorParam: string): string =>
+  `embedding ${DISTANCE_OPERATORS[metric]} ${vectorParam}::vector`;
+
 const scoreExpression = (metric: Metric, vectorParam: string): string => {
-  const distance = `embedding ${DISTANCE_OPERATORS[metric]} ${vectorParam}::vector`;
+  const distance = distanceExpression(metric, vectorParam);
   switch (metric) {
     case "cosine": {
       return `1 - (${distance})`;
@@ -187,8 +184,8 @@ const resolveProfile = async (
   return DEFAULT_PROFILE;
 };
 
-const withIterativeScan = (search: string, metric: Metric): string =>
-  `SELECT hits.* FROM (SELECT ${ITERATIVE_SCAN_SETTING} AS scan) AS tuning CROSS JOIN LATERAL (${search}) AS hits ORDER BY hits.score ${SCORE_ORDERS[metric]}`;
+const withIterativeScan = (search: string): string =>
+  `SELECT hits.* FROM (SELECT ${ITERATIVE_SCAN_SETTING} AS scan) AS tuning CROSS JOIN LATERAL (${search}) AS hits ORDER BY hits.distance`;
 
 const searchRows = async (
   context: TableContext,
@@ -313,10 +310,15 @@ const createIndex = (
         const vectorColumn = query.includeVector
           ? "embedding::text AS embedding,"
           : "";
-        const search = `SELECT id, metadata, ${vectorColumn} ${scoreExpression(metric, "$2")} AS score FROM ${table} WHERE ${where} ORDER BY embedding ${DISTANCE_OPERATORS[metric]} $2::vector LIMIT $${params.length}`;
+        const distance = distanceExpression(metric, "$2");
+        const score = `${scoreExpression(metric, "$2")} AS score`;
+        const columns = iterativeScan
+          ? `${score}, ${distance} AS distance`
+          : score;
+        const search = `SELECT id, metadata, ${vectorColumn} ${columns} FROM ${table} WHERE ${where} ORDER BY ${distance} LIMIT $${params.length}`;
         const rows = await searchRows(
           context,
-          iterativeScan ? withIterativeScan(search, metric) : search,
+          iterativeScan ? withIterativeScan(search) : search,
           params
         );
         const includeMetadata = query.includeMetadata ?? true;

packages/vecstore-sdk/test/adapters/pgvector.test.ts

@@ -251,7 +251,7 @@ describe(createPgvectorStore, () => {
         "2000",
         3,
       ],
-      text: `SELECT id, metadata, embedding::text AS embedding, -(embedding <#> $2::vector) AS score FROM "docs" WHERE namespace = $1 AND ((metadata @> $3::jsonb OR metadata @> $4::jsonb) AND (jsonb_typeof((metadata->$5::text)) = 'number' AND (metadata->$5::text) > $6::jsonb)) ORDER BY embedding <#> $2::vector LIMIT $7`,
+      text: `SELECT id, metadata, embedding::text AS embedding, -(embedding <#> $2::vector) AS score FROM "docs" WHERE namespace = $1 AND ((metadata @> $3::jsonb OR metadata @> $4::jsonb) AND ((metadata->$5::text) IS NOT NULL AND jsonb_typeof((metadata->$5::text)) = 'number' AND (metadata->$5::text) > $6::jsonb)) ORDER BY embedding <#> $2::vector LIMIT $7`,
     });
     await store.index("docs").query({ topK: 1, vector: [0, 0] });
     expect(calls).toHaveLength(3);
@@ -299,7 +299,7 @@ describe(createPgvectorStore, () => {
       .index("docs")
       .query({ topK: 3, vector: [1, 2] });
     expect(calls[1]?.text).toBe(
-      `SELECT hits.* FROM (SELECT set_config('hnsw.iterative_scan', CASE WHEN current_setting('hnsw.iterative_scan', true) IN ('strict_order', 'relaxed_order') THEN current_setting('hnsw.iterative_scan', true) ELSE 'strict_order' END, true) AS scan) AS tuning CROSS JOIN LATERAL (SELECT id, metadata,  embedding <-> $2::vector AS score FROM "docs" WHERE namespace = $1 AND tuning.scan IS NOT NULL ORDER BY embedding <-> $2::vector LIMIT $3) AS hits ORDER BY hits.score ASC`
+      `SELECT hits.* FROM (SELECT set_config('hnsw.iterative_scan', CASE WHEN current_setting('hnsw.iterative_scan', true) IN ('strict_order', 'relaxed_order') THEN current_setting('hnsw.iterative_scan', true) ELSE 'strict_order' END, true) AS scan) AS tuning CROSS JOIN LATERAL (SELECT id, metadata,  embedding <-> $2::vector AS score, embedding <-> $2::vector AS distance FROM "docs" WHERE namespace = $1 AND tuning.scan IS NOT NULL ORDER BY embedding <-> $2::vector LIMIT $3) AS hits ORDER BY hits.distance`
     );
   });
 

packages/vecstore-sdk/test/filter/pgvector.test.ts

@@ -31,10 +31,10 @@ describe(compilePgvectorFilter, () => {
     });
   });
 
-  test("range operators guard on the JSON type and compare as jsonb", () => {
+  test("range operators guard on a present JSON number and compare as jsonb", () => {
     expect(compilePgvectorFilter(gt("year", 2000))).toStrictEqual({
       params: ["year", "2000"],
-      text: "(jsonb_typeof((metadata->$1::text)) = 'number' AND (metadata->$1::text) > $2::jsonb)",
+      text: "((metadata->$1::text) IS NOT NULL AND jsonb_typeof((metadata->$1::text)) = 'number' AND (metadata->$1::text) > $2::jsonb)",
     });
     expect(compilePgvectorFilter(gte("year", 2000)).text).toContain(" >= ");
     expect(compilePgvectorFilter(lt("year", 2000)).text).toContain(" < ");
@@ -89,7 +89,7 @@ describe(compilePgvectorFilter, () => {
       })
     ).toStrictEqual({
       params: ['{"a":1}', '{"a":[1]}', "b", "2"],
-      text: "((t.meta @> $4::jsonb OR t.meta @> $5::jsonb) AND (jsonb_typeof((t.meta->$6::text)) = 'number' AND (t.meta->$6::text) > $7::jsonb))",
+      text: "((t.meta @> $4::jsonb OR t.meta @> $5::jsonb) AND ((t.meta->$6::text) IS NOT NULL AND jsonb_typeof((t.meta->$6::text)) = 'number' AND (t.meta->$6::text) > $7::jsonb))",
     });
   });
 });

packages/vecstore-sdk/test/live/conformance.ts

@@ -159,6 +159,32 @@ export const containsCases: LiveCase[] = [
   ],
 ];
 
+export const negatedRangeCases: LiveCase[] = [
+  [
+    "not around a range keeps a record that lacks the field",
+    async ({ index }) => {
+      const matches = unwrap(
+        await index.query({
+          filter: not(gt("year", 2000)),
+          topK: 10,
+          vector: vectors.a,
+        })
+      );
+      expect(idsOf(matches).toSorted()).toStrictEqual(["doc-a", "doc-d"]);
+    },
+  ],
+  [
+    "delete by not around a range removes a record that lacks the field",
+    async ({ index }) => {
+      unwrap(await index.delete({ filter: not(gt("year", 2000)) }));
+      const found = unwrap(
+        await index.fetch(["doc-a", "doc-b", "doc-c", "doc-d"])
+      );
+      expect(idsOf(found)).toStrictEqual(["doc-b", "doc-c"]);
+    },
+  ],
+];
+
 export const liveCases: LiveCase[] = [
   [
     "listIndexes includes the new index",

packages/vecstore-sdk/test/postgres/filter-differential.test.ts

@@ -41,6 +41,13 @@ const corpus: [string, Filter, string[] | undefined][] = [
   ["nin on a list field", notIn("tags", ["y"]), ["c", "d"]],
   ["exists", exists("tags"), ["a", "b", "d"]],
   ["not exists", not(exists("tags")), ["c"]],
+  ["not gt matches a missing field", not(gt("year", 2000)), ["a", "c", "d"]],
+  ["not lte matches a missing field", not(lte("year", 1999)), ["b", "c"]],
+  [
+    "not around an or of a range matches a missing field",
+    not(or(gt("year", 2000), eq("genre", "comedy"))),
+    ["a", "c", "d"],
+  ],
   ["and", and(eq("genre", "drama"), gt("year", 1990)), ["a"]],
   ["or", or(eq("genre", "horror"), isIn("tags", ["y"])), ["a", "b", "c"]],
   ["not in", not(isIn("tags", ["x"])), ["b", "c", "d"]],

packages/vecstore-sdk/test/postgres/pglite.ts

@@ -54,6 +54,17 @@ export const installSupabaseSql = async (db: PGlite): Promise<void> => {
 export const FORCE_INDEX_SCAN =
   "set enable_seqscan = off; set enable_bitmapscan = off; set enable_sort = off";
 
+export const AVOID_INDEX_SCAN = "set enable_indexscan = off";
+
+export const zeroVectorRecords: VectorRecord[] = [
+  { id: "zero", vector: [0, 0] },
+  { id: "x", vector: [1, 0] },
+  { id: "y", vector: [0, 1] },
+  { id: "nx", vector: [-1, 0] },
+];
+
+export const NEAREST_FIRST_ZERO_LAST = ["x", "y", "nx", "zero"];
+
 const SPREAD = 7;
 const RARE_EVERY = 10;
 const RECORDS_PER_NAMESPACE = 200;