Key globs with many stars take seconds to match

.bumpy/glob-backtracking.md

@@ -0,0 +1,5 @@
+---
+"varlock": patch
+---
+
+Key globs with many `*` (in `pick`/`omit`, `--filter` and `@auditIgnoreKeys`) no longer take seconds or minutes to match

packages/varlock/src/cli/commands/audit.command.ts

@@ -5,7 +5,7 @@ import ansis from 'ansis';
 import { FileBasedDataSource } from '../../env-graph';
 import { CliExitError } from '../helpers/exit-error';
 import { parseRegexLikeString } from '../../env-graph/lib/resolver';
-import { globToRegExp } from '../../env-graph/lib/glob';
+import { compileGlob, type GlobMatcher } from '../../env-graph/lib/glob';
 import { loadVarlockEnvGraph } from '../../lib/load-graph';
 import { checkForNoEnvFiles, checkForSchemaErrors } from '../helpers/error-checks';
 import { type TypedGunshiCommandFn } from '../helpers/gunshi-type-utils';
@@ -212,9 +212,9 @@ async function getCustomAuditIgnorePaths(envGraph: any): Promise<Array<string>>
  * scanner doesn't parse strings, so `"set process.env.FOO first"` in a help message
  * counts as a reference to FOO.
  */
-async function getCustomAuditIgnoreKeys(envGraph: any): Promise<Array<RegExp>> {
+async function getCustomAuditIgnoreKeys(envGraph: any): Promise<Array<GlobMatcher>> {
   const patterns = await getRootDecStringArgs(envGraph, 'auditIgnoreKeys');
-  return patterns.map((pattern) => globToRegExp(pattern));
+  return patterns.map((pattern) => compileGlob(pattern));
 }
 
 export const commandFn: TypedGunshiCommandFn<typeof commandSpec> = async (ctx) => {

packages/varlock/src/env-graph/lib/glob.ts

@@ -1,13 +1,38 @@
+export type GlobMatcher = { test: (str: string) => boolean };
+
 /**
- * Compile a simple glob pattern into an anchored regex. Supports `*` (any run of
+ * Compile a simple glob pattern into a matcher for a whole string. Supports `*` (any run of
  * characters) and `?` (single character); every other character matches literally.
- * Used by key filters (`@setValuesBulk` and `@import` pick/omit). Matching is
- * case-sensitive, since env keys are case-sensitive.
+ * Used by key filters (`@setValuesBulk` and `@import` pick/omit, `--filter`) and
+ * `@auditIgnoreKeys`. Matching is case-sensitive, since env keys are case-sensitive.
+ *
+ * This is a loop and not a regex on purpose: `^.*A.*A.*A$` backtracks polynomially in the
+ * number of stars, and a pattern with 8 stars took about a minute against one 60-character key.
  */
-export function globToRegExp(pattern: string): RegExp {
-  const escaped = pattern
-    .replace(/[.+^${}()|[\]\\]/g, '\\$&') // escape regex specials (leaving * and ?)
-    .replace(/\*/g, '.*')
-    .replace(/\?/g, '.');
-  return new RegExp(`^${escaped}$`);
+export function compileGlob(pattern: string): GlobMatcher {
+  return {
+    test(str: string) {
+      // greedy matching, and on a mismatch go back to the most recent star only
+      let p = 0;
+      let s = 0;
+      let starP = -1;
+      let starS = 0;
+      while (s < str.length) {
+        if (pattern[p] === '*') {
+          starP = p++;
+          starS = s;
+        } else if (p < pattern.length && (pattern[p] === '?' || pattern[p] === str[s])) {
+          p++;
+          s++;
+        } else if (starP !== -1) {
+          p = starP + 1;
+          s = ++starS;
+        } else {
+          return false;
+        }
+      }
+      while (pattern[p] === '*') p++;
+      return p === pattern.length;
+    },
+  };
 }

packages/varlock/src/env-graph/lib/item-filter.ts

@@ -1,5 +1,5 @@
 import { SchemaError } from './errors';
-import { globToRegExp } from './glob';
+import { compileGlob, type GlobMatcher } from './glob';
 
 /**
  * Shared item-selection language used by the CLI `--filter` flag (`varlock load`/`run`) and the
@@ -51,12 +51,12 @@ export type ItemFilterOptions = {
   tagSelectorsUnsupportedTip?: string;
 };
 
-type FilterToken = | { negate: boolean, kind: 'key', regex: RegExp }
+type FilterToken = | { negate: boolean, kind: 'key', glob: GlobMatcher }
   | { negate: boolean, kind: 'decorator', name: string }
   | { negate: boolean, kind: 'tag', tag: string };
 
 function tokenMatches(item: FilterableItem, token: FilterToken): boolean {
-  if (token.kind === 'key') return token.regex.test(item.key);
+  if (token.kind === 'key') return token.glob.test(item.key);
   if (token.kind === 'decorator') return DECORATOR_PREDICATES[token.name](item);
   return item.tags.includes(token.tag);
 }
@@ -132,7 +132,7 @@ export class ParsedItemFilter {
         }
         return { negate, kind: 'tag' as const, tag };
       }
-      return { negate, kind: 'key' as const, regex: globToRegExp(token) };
+      return { negate, kind: 'key' as const, glob: compileGlob(token) };
     });
     this.usesDecoratorSelector = this.tokens.some((t) => t.kind === 'decorator');
     this.usesTagSelector = this.tokens.some((t) => t.kind === 'tag');

packages/varlock/src/env-graph/test/glob.test.ts

@@ -0,0 +1,30 @@
+import { describe, test, expect } from 'vitest';
+import { compileGlob } from '../lib/glob';
+
+describe('key globs', () => {
+  test.each([
+    ['API_*', 'API_KEY', true],
+    ['API_*', 'XAPI_KEY', false],
+    ['*_URL', 'DB_URL', true],
+    ['A?C', 'ABC', true],
+    ['A?C', 'AC', false],
+    ['*', '', true],
+    ['', '', true],
+    ['', 'A', false],
+    ['A*B*C', 'AXXBYYC', true],
+    ['A*B*C', 'AXXBYY', false],
+    ['A.B', 'AXB', false], // regex specials are literal
+    ['A.B', 'A.B', true],
+    ['*', 'A*B', true],
+  ])('%s against %s: %s', (pattern, key, expected) => {
+    expect(compileGlob(pattern).test(key)).toBe(expected);
+  });
+
+  test('many stars against a long key do not block', () => {
+    // as a regex (`^.*A.*A.*A.*A.*A.*A.*A.*B$`) this takes about 2 seconds
+    const pattern = `${'*A'.repeat(7)}*B`;
+    const t0 = performance.now();
+    expect(compileGlob(pattern).test('A'.repeat(50))).toBe(false);
+    expect(performance.now() - t0).toBeLessThan(100);
+  });
+});