diff --git a/.bumpy/glob-backtracking.md b/.bumpy/glob-backtracking.md new file mode 100644 index 000000000..8ebfaf9e7 --- /dev/null +++ b/.bumpy/glob-backtracking.md @@ -0,0 +1,5 @@ +--- +"varlock": patch +--- + +Key globs with many `*` (in `pick`/`omit`, `--filter` and `@auditIgnoreKeys`) no longer take seconds or minutes to match diff --git a/packages/varlock/src/cli/commands/audit.command.ts b/packages/varlock/src/cli/commands/audit.command.ts index 2c0bb4ac4..12c8beda2 100644 --- a/packages/varlock/src/cli/commands/audit.command.ts +++ b/packages/varlock/src/cli/commands/audit.command.ts @@ -5,7 +5,7 @@ import ansis from 'ansis'; import { FileBasedDataSource } from '../../env-graph'; import { CliExitError } from '../helpers/exit-error'; import { parseRegexLikeString } from '../../env-graph/lib/resolver'; -import { globToRegExp } from '../../env-graph/lib/glob'; +import { compileGlob, type GlobMatcher } from '../../env-graph/lib/glob'; import { loadVarlockEnvGraph } from '../../lib/load-graph'; import { checkForNoEnvFiles, checkForSchemaErrors } from '../helpers/error-checks'; import { type TypedGunshiCommandFn } from '../helpers/gunshi-type-utils'; @@ -212,9 +212,9 @@ async function getCustomAuditIgnorePaths(envGraph: any): Promise> * scanner doesn't parse strings, so `"set process.env.FOO first"` in a help message * counts as a reference to FOO. */ -async function getCustomAuditIgnoreKeys(envGraph: any): Promise> { +async function getCustomAuditIgnoreKeys(envGraph: any): Promise> { const patterns = await getRootDecStringArgs(envGraph, 'auditIgnoreKeys'); - return patterns.map((pattern) => globToRegExp(pattern)); + return patterns.map((pattern) => compileGlob(pattern)); } export const commandFn: TypedGunshiCommandFn = async (ctx) => { diff --git a/packages/varlock/src/env-graph/lib/glob.ts b/packages/varlock/src/env-graph/lib/glob.ts index 3c495cde4..b2ff806bb 100644 --- a/packages/varlock/src/env-graph/lib/glob.ts +++ b/packages/varlock/src/env-graph/lib/glob.ts @@ -1,13 +1,38 @@ +export type GlobMatcher = { test: (str: string) => boolean }; + /** - * Compile a simple glob pattern into an anchored regex. Supports `*` (any run of + * Compile a simple glob pattern into a matcher for a whole string. Supports `*` (any run of * characters) and `?` (single character); every other character matches literally. - * Used by key filters (`@setValuesBulk` and `@import` pick/omit). Matching is - * case-sensitive, since env keys are case-sensitive. + * Used by key filters (`@setValuesBulk` and `@import` pick/omit, `--filter`) and + * `@auditIgnoreKeys`. Matching is case-sensitive, since env keys are case-sensitive. + * + * This is a loop and not a regex on purpose: `^.*A.*A.*A$` backtracks polynomially in the + * number of stars, and a pattern with 8 stars took about a minute against one 60-character key. */ -export function globToRegExp(pattern: string): RegExp { - const escaped = pattern - .replace(/[.+^${}()|[\]\\]/g, '\\$&') // escape regex specials (leaving * and ?) - .replace(/\*/g, '.*') - .replace(/\?/g, '.'); - return new RegExp(`^${escaped}$`); +export function compileGlob(pattern: string): GlobMatcher { + return { + test(str: string) { + // greedy matching, and on a mismatch go back to the most recent star only + let p = 0; + let s = 0; + let starP = -1; + let starS = 0; + while (s < str.length) { + if (pattern[p] === '*') { + starP = p++; + starS = s; + } else if (p < pattern.length && (pattern[p] === '?' || pattern[p] === str[s])) { + p++; + s++; + } else if (starP !== -1) { + p = starP + 1; + s = ++starS; + } else { + return false; + } + } + while (pattern[p] === '*') p++; + return p === pattern.length; + }, + }; } diff --git a/packages/varlock/src/env-graph/lib/item-filter.ts b/packages/varlock/src/env-graph/lib/item-filter.ts index f1cf0856c..482a51c95 100644 --- a/packages/varlock/src/env-graph/lib/item-filter.ts +++ b/packages/varlock/src/env-graph/lib/item-filter.ts @@ -1,5 +1,5 @@ import { SchemaError } from './errors'; -import { globToRegExp } from './glob'; +import { compileGlob, type GlobMatcher } from './glob'; /** * Shared item-selection language used by the CLI `--filter` flag (`varlock load`/`run`) and the @@ -51,12 +51,12 @@ export type ItemFilterOptions = { tagSelectorsUnsupportedTip?: string; }; -type FilterToken = | { negate: boolean, kind: 'key', regex: RegExp } +type FilterToken = | { negate: boolean, kind: 'key', glob: GlobMatcher } | { negate: boolean, kind: 'decorator', name: string } | { negate: boolean, kind: 'tag', tag: string }; function tokenMatches(item: FilterableItem, token: FilterToken): boolean { - if (token.kind === 'key') return token.regex.test(item.key); + if (token.kind === 'key') return token.glob.test(item.key); if (token.kind === 'decorator') return DECORATOR_PREDICATES[token.name](item); return item.tags.includes(token.tag); } @@ -132,7 +132,7 @@ export class ParsedItemFilter { } return { negate, kind: 'tag' as const, tag }; } - return { negate, kind: 'key' as const, regex: globToRegExp(token) }; + return { negate, kind: 'key' as const, glob: compileGlob(token) }; }); this.usesDecoratorSelector = this.tokens.some((t) => t.kind === 'decorator'); this.usesTagSelector = this.tokens.some((t) => t.kind === 'tag'); diff --git a/packages/varlock/src/env-graph/test/glob.test.ts b/packages/varlock/src/env-graph/test/glob.test.ts new file mode 100644 index 000000000..bf18ff8db --- /dev/null +++ b/packages/varlock/src/env-graph/test/glob.test.ts @@ -0,0 +1,30 @@ +import { describe, test, expect } from 'vitest'; +import { compileGlob } from '../lib/glob'; + +describe('key globs', () => { + test.each([ + ['API_*', 'API_KEY', true], + ['API_*', 'XAPI_KEY', false], + ['*_URL', 'DB_URL', true], + ['A?C', 'ABC', true], + ['A?C', 'AC', false], + ['*', '', true], + ['', '', true], + ['', 'A', false], + ['A*B*C', 'AXXBYYC', true], + ['A*B*C', 'AXXBYY', false], + ['A.B', 'AXB', false], // regex specials are literal + ['A.B', 'A.B', true], + ['*', 'A*B', true], + ])('%s against %s: %s', (pattern, key, expected) => { + expect(compileGlob(pattern).test(key)).toBe(expected); + }); + + test('many stars against a long key do not block', () => { + // as a regex (`^.*A.*A.*A.*A.*A.*A.*A.*B$`) this takes about 2 seconds + const pattern = `${'*A'.repeat(7)}*B`; + const t0 = performance.now(); + expect(compileGlob(pattern).test('A'.repeat(50))).toBe(false); + expect(performance.now() - t0).toBeLessThan(100); + }); +});