From 55b97b9c2261278d3c92ee1f3000d0a80920432d Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Sun, 5 Jul 2026 09:43:29 -0500 Subject: [PATCH 1/4] fix(reranker): Score passages with the supported tokenizer pair API The released branch was loading transformers 3.8.1 correctly, but the reranker was calling the tokenizer in a shape that ignored the passage. Keep the dependency on the released package version and make the recovery test assert the text_pair call so this does not slip back. --- src/core/reranker.ts | 15 +++++++++++---- src/embeddings/transformers.ts | 2 +- tests/reranker-recovery.test.ts | 34 ++++++++++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/core/reranker.ts b/src/core/reranker.ts index 47de513..2dda09c 100644 --- a/src/core/reranker.ts +++ b/src/core/reranker.ts @@ -21,9 +21,15 @@ const AMBIGUITY_THRESHOLD = 0.08; interface CrossEncoderTokenizer { ( - query: string, - passage: string, - options: { padding: boolean; truncation: boolean; max_length: number } + text: string, + options?: { + text_pair?: string | null; + padding?: boolean | 'max_length'; + truncation?: boolean | null; + max_length?: number | null; + return_tensor?: boolean; + return_token_type_ids?: boolean | null; + } ): unknown; } @@ -152,7 +158,8 @@ async function scorePair(query: string, passage: string): Promise { throw new Error('[reranker] Model not loaded — call ensureModelLoaded() first'); } - const inputs = cachedTokenizer(query, passage, { + const inputs = cachedTokenizer(query, { + text_pair: passage, padding: true, truncation: true, max_length: 512 diff --git a/src/embeddings/transformers.ts b/src/embeddings/transformers.ts index 0a2c581..04db102 100644 --- a/src/embeddings/transformers.ts +++ b/src/embeddings/transformers.ts @@ -70,7 +70,7 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider { const { pipeline } = await import('@huggingface/transformers'); // TS2590: pipeline() resolves AllTasks[T] — a union too complex for TSC to represent. - // Cast to a simpler signature; the actual return type IS FeatureExtractionPipelineType. + // Cast to a simpler feature-extraction signature for both v3 and v4-compatible types. type PipelineFn = ( task: 'feature-extraction', model: string, diff --git a/tests/reranker-recovery.test.ts b/tests/reranker-recovery.test.ts index 7cb77ca..6bf3325 100644 --- a/tests/reranker-recovery.test.ts +++ b/tests/reranker-recovery.test.ts @@ -63,7 +63,12 @@ describe('reranker corruption recovery', () => { await fs.mkdir(modelCacheDir, { recursive: true }); await fs.writeFile(path.join(modelCacheDir, 'model.onnx'), 'corrupt'); - const tokenizer = vi.fn((query: string, passage: string) => ({ query, passage })); + const tokenizer = vi.fn( + (query: string, options?: { text_pair?: string | null }) => ({ + query, + passage: options?.text_pair ?? '' + }) + ); const model = vi.fn(async (inputs: { passage: string }) => { if (inputs.passage.includes('/a.ts')) return { logits: { data: [1] } }; if (inputs.passage.includes('/b.ts')) return { logits: { data: [3] } }; @@ -89,6 +94,33 @@ describe('reranker corruption recovery', () => { const reranked = await rerank('auth token', results); expect(transformersMocks.tokenizerFromPretrained).toHaveBeenCalledTimes(2); expect(transformersMocks.modelFromPretrained).toHaveBeenCalledTimes(2); + expect(tokenizer).toHaveBeenCalledWith( + 'auth token', + expect.objectContaining({ + text_pair: expect.stringContaining('/a.ts'), + padding: true, + truncation: true, + max_length: 512 + }) + ); + expect(tokenizer).toHaveBeenCalledWith( + 'auth token', + expect.objectContaining({ + text_pair: expect.stringContaining('/b.ts'), + padding: true, + truncation: true, + max_length: 512 + }) + ); + expect(tokenizer).toHaveBeenCalledWith( + 'auth token', + expect.objectContaining({ + text_pair: expect.stringContaining('/c.ts'), + padding: true, + truncation: true, + max_length: 512 + }) + ); expect(reranked.map((result) => result.filePath)).toEqual(['/b.ts', '/c.ts', '/a.ts']); expect(getRerankerStatus()).toBe('active'); }); From b51adeb96817ec6d85e17fd4edfd4b5e5a0a91bf Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Sun, 5 Jul 2026 09:43:38 -0500 Subject: [PATCH 2/4] test(release): Keep current version checks from going stale The repo is already on 2.3.0, but the release truth test and manual publish fallback were still pinned to 2.2.0. Derive the assertions from package metadata and normalize macOS temp paths so the full suite checks the real contract instead of local path spelling. --- .github/workflows/publish-npm-on-release.yml | 4 +-- .../contextbench-baseline-schema-gate.test.ts | 25 ++++++++++++++++--- tests/release-truth-surfaces.test.ts | 14 +++++------ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish-npm-on-release.yml b/.github/workflows/publish-npm-on-release.yml index 023126a..4ca7cf4 100644 --- a/.github/workflows/publish-npm-on-release.yml +++ b/.github/workflows/publish-npm-on-release.yml @@ -8,9 +8,9 @@ on: workflow_dispatch: inputs: tag: - description: 'Tag to publish (e.g. v2.2.0)' + description: 'Tag to publish (e.g. v2.3.0)' required: true - default: 'v2.2.0' + default: 'v2.3.0' permissions: contents: read diff --git a/tests/contextbench-baseline-schema-gate.test.ts b/tests/contextbench-baseline-schema-gate.test.ts index ad6fc0a..69b7a93 100644 --- a/tests/contextbench-baseline-schema-gate.test.ts +++ b/tests/contextbench-baseline-schema-gate.test.ts @@ -1,5 +1,12 @@ import { execFileSync } from 'node:child_process'; -import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { + chmodSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync +} from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; @@ -55,6 +62,10 @@ function cleanupSessionRoot(sessionRoot: string): void { } } +function normalizeFilesystemPath(filePath: string): string { + return realpathSync(filePath); +} + function tempSessionRoot(phase: 'phase40' | 'phase41' = 'phase40'): string { return path.join( mkdtempSync(path.join(tmpdir(), `contextbench-${phase}-schema-gate-`)), @@ -398,7 +409,9 @@ describe('ContextBench Phase 40 schema gate', () => { repoCheckoutPath: repoPath, verificationStrict: false }); - expect(readFileSync(cwdCapturePath, 'utf8')).toBe(repoPath); + expect(normalizeFilesystemPath(readFileSync(cwdCapturePath, 'utf8'))).toBe( + normalizeFilesystemPath(repoPath) + ); const stdin = readFileSync(stdinCapturePath, 'utf8'); expect(stdin).toContain('Problem statement:'); expect(stdin).toContain('Fix the failing ContextBench task'); @@ -584,7 +597,9 @@ describe('ContextBench Phase 40 schema gate', () => { ], { encoding: 'utf8', env } ); - expect(readFileSync(cwdPath, 'utf8')).toBe(repoPath); + expect(normalizeFilesystemPath(readFileSync(cwdPath, 'utf8'))).toBe( + normalizeFilesystemPath(repoPath) + ); } execFileSync( 'node', @@ -681,7 +696,9 @@ describe('ContextBench Phase 40 schema gate', () => { (candidate) => candidate.scoring && 'baselineArmId' in candidate.scoring ); expect(row).toMatchObject({ status: 'completed' }); - expect(readFileSync(cwdCapturePath, 'utf8')).toBe(repoPath); + expect(normalizeFilesystemPath(readFileSync(cwdCapturePath, 'utf8'))).toBe( + normalizeFilesystemPath(repoPath) + ); const stdin = readFileSync(stdinCapturePath, 'utf8'); expect(stdin).toContain('Problem statement:'); expect(stdin).toContain('Run the diagnostic arm with materialized task text.'); diff --git a/tests/release-truth-surfaces.test.ts b/tests/release-truth-surfaces.test.ts index c6d8e60..f362ba4 100644 --- a/tests/release-truth-surfaces.test.ts +++ b/tests/release-truth-surfaces.test.ts @@ -73,6 +73,7 @@ function extractMarkdownLinks(markdown: string): string[] { describe('release truth surfaces', () => { const packageJson = readJson('package.json'); const releaseManifest = readJson('.release-please-manifest.json'); + const expectedVersion = packageJson.version; const changelog = readText('CHANGELOG.md'); const readme = readText('README.md'); const workflow = readText('.github/workflows/publish-npm-on-release.yml'); @@ -80,10 +81,9 @@ describe('release truth surfaces', () => { const visualsDoc = readOptionalText('docs/visuals.md'); const packagedPaths = ['README.md', 'LICENSE', ...(packageJson.files ?? [])]; - it('keeps package metadata, release manifest, and changelog on 2.2.0', () => { - expect(packageJson.version).toBe('2.2.0'); - expect(releaseManifest['.']).toBe('2.2.0'); - expect(changelog).toContain('## [2.2.0]'); + it('keeps package metadata, release manifest, and changelog aligned', () => { + expect(releaseManifest['.']).toBe(expectedVersion); + expect(changelog).toContain(`## [${expectedVersion}]`); expect(changelog).not.toContain('## Unreleased'); }); @@ -114,8 +114,8 @@ describe('release truth surfaces', () => { expect(visualsDoc).toContain('Historical snapshot'); }); - it('keeps the manual publish fallback aligned to v2.2.0', () => { - expect(workflow).toContain("description: 'Tag to publish (e.g. v2.2.0)'"); - expect(workflow).toContain("default: 'v2.2.0'"); + it('keeps the manual publish fallback aligned to the package version', () => { + expect(workflow).toContain(`description: 'Tag to publish (e.g. v${expectedVersion})'`); + expect(workflow).toContain(`default: 'v${expectedVersion}'`); }); }); From 6e77a04cd0ed612bd3aadfd9ccfb6b694774f7e9 Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Wed, 15 Jul 2026 14:16:04 -0500 Subject: [PATCH 3/4] Upgrade Transformers.js to v4 Move the local embedding and reranker integrations onto the stable v4 API and ONNX Runtime 1.24.3. This removes the macOS crash dependency while preserving the tokenizer pair fix already covered by this branch. --- package.json | 4 +- pnpm-lock.yaml | 89 ++++++++++++++++++++++++---------- src/core/reranker.ts | 17 +------ src/embeddings/transformers.ts | 6 +-- 4 files changed, 70 insertions(+), 46 deletions(-) diff --git a/package.json b/package.json index 00aeb98..3ec44d1 100644 --- a/package.json +++ b/package.json @@ -128,8 +128,8 @@ "prepare": "husky" }, "dependencies": { + "@huggingface/transformers": "^4.2.0", "@inquirer/prompts": "^3.0.0", - "@huggingface/transformers": "^3.8.1", "@lancedb/lancedb": "^0.4.0", "@modelcontextprotocol/sdk": "^1.27.1", "@typescript-eslint/typescript-estree": "^7.0.0", @@ -172,7 +172,7 @@ "@modelcontextprotocol/sdk>ajv": "8.18.0", "@modelcontextprotocol/sdk>@hono/node-server": "1.19.13", "@modelcontextprotocol/sdk>express-rate-limit": "8.2.2", - "@huggingface/transformers>onnxruntime-node": "1.24.2", + "@huggingface/transformers>onnxruntime-node": "1.24.3", "path-to-regexp": "8.4.0", "brace-expansion": "5.0.5", "micromatch>picomatch": "2.3.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2c6e5ab..b550f49 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,7 +8,7 @@ overrides: '@modelcontextprotocol/sdk>ajv': 8.18.0 '@modelcontextprotocol/sdk>@hono/node-server': 1.19.13 '@modelcontextprotocol/sdk>express-rate-limit': 8.2.2 - '@huggingface/transformers>onnxruntime-node': 1.24.2 + '@huggingface/transformers>onnxruntime-node': 1.24.3 path-to-regexp: 8.4.0 brace-expansion: 5.0.5 micromatch>picomatch: 2.3.2 @@ -24,8 +24,8 @@ importers: .: dependencies: '@huggingface/transformers': - specifier: ^3.8.1 - version: 3.8.1 + specifier: ^4.2.0 + version: 4.2.0 '@inquirer/prompts': specifier: ^3.0.0 version: 3.3.2 @@ -324,12 +324,15 @@ packages: peerDependencies: hono: '>=4.12.12' - '@huggingface/jinja@0.5.5': - resolution: {integrity: sha512-xRlzazC+QZwr6z4ixEqYHo9fgwhTZ3xNSdljlKfUFGZSdlvt166DljRELFUfFytlYOYvo3vTisA/AFOuOAzFQQ==} + '@huggingface/jinja@0.5.9': + resolution: {integrity: sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw==} engines: {node: '>=18'} - '@huggingface/transformers@3.8.1': - resolution: {integrity: sha512-tsTk4zVjImqdqjS8/AOZg2yNLd1z9S5v+7oUPpXaasDRwEDhB+xnglK1k5cad26lL5/ZIaeREgWWy0bs9y9pPA==} + '@huggingface/tokenizers@0.1.3': + resolution: {integrity: sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==} + + '@huggingface/transformers@4.2.0': + resolution: {integrity: sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==} '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} @@ -377,89 +380,105 @@ packages: resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-arm@1.2.4': resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-ppc64@1.2.4': resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-riscv64@1.2.4': resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} cpu: [riscv64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-s390x@1.2.4': resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-x64@1.2.4': resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.2.4': resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.2.4': resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-linux-arm64@0.34.5': resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-linux-arm@0.34.5': resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-linux-ppc64@0.34.5': resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-linux-riscv64@0.34.5': resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [riscv64] os: [linux] + libc: [glibc] '@img/sharp-linux-s390x@0.34.5': resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-linux-x64@0.34.5': resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-linuxmusl-arm64@0.34.5': resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-linuxmusl-x64@0.34.5': resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-wasm32@0.34.5': resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} @@ -552,12 +571,14 @@ packages: engines: {node: '>= 18'} cpu: [arm64] os: [linux] + libc: [glibc] '@lancedb/lancedb-linux-x64-gnu@0.4.20': resolution: {integrity: sha512-TVicKBhT3Q73gtWmMdR/0+r4gBve5gw3KbcrvzHP/5bNpo2sffETbLEMgLjulunu/Jp8IgC921WW6IFJzNFwrQ==} engines: {node: '>= 18'} cpu: [x64] os: [linux] + libc: [glibc] '@lancedb/lancedb-win32-x64-msvc@0.4.20': resolution: {integrity: sha512-7RsTJAx93NbVD//9m0/TpgQmPsTHlCBfGNFD2zkdZ0j7iacURdG9+egz+Os0veTBqMgwnr6VfMf3T7S0wuu99g==} @@ -661,66 +682,79 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} @@ -1956,18 +1990,18 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} - onnxruntime-common@1.22.0-dev.20250409-89f8206ba4: - resolution: {integrity: sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==} + onnxruntime-common@1.24.0-dev.20251116-b39e144322: + resolution: {integrity: sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==} - onnxruntime-common@1.24.2: - resolution: {integrity: sha512-S0FFhJaI05jr1c3HVJ/DuPFB/aYdXmnUBuuQfuvLtcNn7WAfpm2ewSXn1vHs9Wa1l8T8OznhfCEdFv8qCn0/xw==} + onnxruntime-common@1.24.3: + resolution: {integrity: sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==} - onnxruntime-node@1.24.2: - resolution: {integrity: sha512-ZtTkUHNNk+Xpi2Sq2BcFjzc9Vx4n3o5RxYLgRvdrFzCBsGUU1dQRFf4LM9tCc7vFhPSoZqbvzZtkzRMXoDqKNg==} + onnxruntime-node@1.24.3: + resolution: {integrity: sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==} os: [win32, darwin, linux] - onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: - resolution: {integrity: sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==} + onnxruntime-web@1.26.0-dev.20260416-b7804b056c: + resolution: {integrity: sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==} openai@4.104.0: resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} @@ -2732,13 +2766,16 @@ snapshots: dependencies: hono: 4.12.14 - '@huggingface/jinja@0.5.5': {} + '@huggingface/jinja@0.5.9': {} + + '@huggingface/tokenizers@0.1.3': {} - '@huggingface/transformers@3.8.1': + '@huggingface/transformers@4.2.0': dependencies: - '@huggingface/jinja': 0.5.5 - onnxruntime-node: 1.24.2 - onnxruntime-web: 1.22.0-dev.20250409-89f8206ba4 + '@huggingface/jinja': 0.5.9 + '@huggingface/tokenizers': 0.1.3 + onnxruntime-node: 1.24.3 + onnxruntime-web: 1.26.0-dev.20260416-b7804b056c sharp: 0.34.5 '@humanfs/core@0.19.1': {} @@ -4481,22 +4518,22 @@ snapshots: dependencies: wrappy: 1.0.2 - onnxruntime-common@1.22.0-dev.20250409-89f8206ba4: {} + onnxruntime-common@1.24.0-dev.20251116-b39e144322: {} - onnxruntime-common@1.24.2: {} + onnxruntime-common@1.24.3: {} - onnxruntime-node@1.24.2: + onnxruntime-node@1.24.3: dependencies: adm-zip: 0.5.16 global-agent: 3.0.0 - onnxruntime-common: 1.24.2 + onnxruntime-common: 1.24.3 - onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: + onnxruntime-web@1.26.0-dev.20260416-b7804b056c: dependencies: flatbuffers: 25.9.23 guid-typescript: 1.0.9 long: 5.3.2 - onnxruntime-common: 1.22.0-dev.20250409-89f8206ba4 + onnxruntime-common: 1.24.0-dev.20251116-b39e144322 platform: 1.3.6 protobufjs: 7.5.6 diff --git a/src/core/reranker.ts b/src/core/reranker.ts index 2dda09c..9dabeac 100644 --- a/src/core/reranker.ts +++ b/src/core/reranker.ts @@ -9,6 +9,7 @@ */ import type { SearchResult } from '../types/index.js'; +import type { PreTrainedTokenizer } from '@huggingface/transformers'; import os from 'os'; const DEFAULT_RERANKER_MODEL = 'Xenova/ms-marco-MiniLM-L-6-v2'; @@ -19,25 +20,11 @@ const RERANK_TOP_K = 10; /** Trigger reranking when the score gap between #1 and #3 is below this threshold */ const AMBIGUITY_THRESHOLD = 0.08; -interface CrossEncoderTokenizer { - ( - text: string, - options?: { - text_pair?: string | null; - padding?: boolean | 'max_length'; - truncation?: boolean | null; - max_length?: number | null; - return_tensor?: boolean; - return_token_type_ids?: boolean | null; - } - ): unknown; -} - interface CrossEncoderModel { (inputs: unknown): Promise<{ logits: { data: ArrayLike } }>; } -let cachedTokenizer: CrossEncoderTokenizer | null = null; +let cachedTokenizer: PreTrainedTokenizer | null = null; let cachedModel: CrossEncoderModel | null = null; let initPromise: Promise | null = null; diff --git a/src/embeddings/transformers.ts b/src/embeddings/transformers.ts index 04db102..7fc62e8 100644 --- a/src/embeddings/transformers.ts +++ b/src/embeddings/transformers.ts @@ -1,5 +1,5 @@ import { EmbeddingProvider, DEFAULT_MODEL } from './types.js'; -import type { FeatureExtractionPipelineType } from '@huggingface/transformers'; +import type { FeatureExtractionPipeline } from '@huggingface/transformers'; import os from 'os'; /** @@ -43,7 +43,7 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider { readonly modelName: string; readonly dimensions: number; - private pipeline: FeatureExtractionPipelineType | null = null; + private pipeline: FeatureExtractionPipeline | null = null; private ready = false; private initPromise: Promise | null = null; @@ -75,7 +75,7 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider { task: 'feature-extraction', model: string, opts: Record - ) => Promise; + ) => Promise; this.pipeline = await (pipeline as PipelineFn)('feature-extraction', this.modelName, { dtype: 'q8', // Limit ONNX Runtime to half cores by default — prevents system freeze during indexing. From cff0760c5b70fd53d526cfd304407802e72e98df Mon Sep 17 00:00:00 2001 From: Aaron Olin Date: Wed, 15 Jul 2026 14:53:27 -0500 Subject: [PATCH 4/4] Keep test runs independent of shell credentials Remove secret-named variables before Vitest starts and suppress Node loader deprecations in the CLI subprocess contract. This keeps ContextBench fixtures free of operator environment values and preserves meaningful stderr assertions on Node 26. --- scripts/run-vitest.mjs | 7 ++++++- tests/codebase-map.test.ts | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/run-vitest.mjs b/scripts/run-vitest.mjs index c529a5c..6f7d357 100644 --- a/scripts/run-vitest.mjs +++ b/scripts/run-vitest.mjs @@ -11,9 +11,14 @@ const vitestEntrypoint = fileURLToPath( new URL('../node_modules/vitest/vitest.mjs', import.meta.url) ); +const sensitiveEnvName = /TOKEN|KEY|SECRET|PASSWORD|AUTH|OPENAI|ANTHROPIC|CLAUDE/i; +const testEnv = Object.fromEntries( + Object.entries(process.env).filter(([name]) => !sensitiveEnvName.test(name)) +); + const child = spawn(process.execPath, [vitestEntrypoint, 'run', ...vitestArgs], { stdio: 'inherit', - env: process.env + env: testEnv }); child.on('exit', (code, signal) => { diff --git a/tests/codebase-map.test.ts b/tests/codebase-map.test.ts index 662e5b2..c90de78 100644 --- a/tests/codebase-map.test.ts +++ b/tests/codebase-map.test.ts @@ -84,7 +84,7 @@ async function removeTempMapProject(rootPath: string): Promise { } function runMapCli(args: string[], rootPath: string) { - return spawnSync(process.execPath, ['--import', 'tsx', ENTRYPOINT, 'map', ...args], { + return spawnSync(process.execPath, ['--no-deprecation', '--import', 'tsx', ENTRYPOINT, 'map', ...args], { cwd: CURRENT_REPO_ROOT, env: { ...process.env,