diff --git a/src/services/embedding.ts b/src/services/embedding.ts index ced6b17b..bf45acf6 100644 --- a/src/services/embedding.ts +++ b/src/services/embedding.ts @@ -7,6 +7,10 @@ import { prepareOnnxruntimeForTransformers, } from "./onnxruntime-resolve.js"; import { createRuntimeRequire } from "./runtime-require.js"; +import { + getTransformersPackageSpecifier, + resolveTransformersCjsEntry, +} from "./transformers-resolve.js"; const requireFromHere = createRuntimeRequire(import.meta); @@ -194,17 +198,10 @@ let _transformers: { env: HfTransformers["env"]; } | null = null; -function getTransformersPackageSpecifier(): string { - // Keep this non-literal so OpenCode/Bun plugin-loader bundling does not eagerly - // traverse @huggingface/transformers internals during plugin startup. The package - // is only needed for the local embedding backend, and should stay lazy. - return ["@huggingface", "transformers"].join("/"); -} - async function ensureTransformersLoaded(): Promise> { if (_transformers !== null) return _transformers; // Pin onnxruntime-node (+ common) to our direct 1.30.0 stack before transformers - // resolves them (#184 / #210 / #225). Load the CJS export so Module._resolveFilename + // resolves them (#184 / #210 / #225 / #463). Load the CJS export so Module._resolveFilename // shim applies — the ESM entry's static import bypasses it under OpenCode nested installs. // // Critical ordering for OpenCode's Bun --compile host (#210 follow-up): @@ -213,17 +210,16 @@ async function ensureTransformersLoaded(): Promise | null { + try { + return JSON.parse(readFileSync(path, "utf8")) as Record; + } catch { + return null; + } +} + +function asExportPath(value: unknown): string | null { + if (typeof value === "string" && value.length > 0) return value; + if ( + value && + typeof value === "object" && + typeof (value as { default?: unknown }).default === "string" + ) { + return (value as { default: string }).default; + } + return null; +} + +/** Pick the Node CJS entry from a transformers package.json. */ +export function cjsEntryFromPackageJson(pkgJsonPath: string): string { + const pkg = readJson(pkgJsonPath); + if (!pkg) throw new Error(`Unable to read ${pkgJsonPath}`); + const exportsField = pkg.exports as PackageJsonExports | undefined; + const relative = + asExportPath(exportsField?.node?.require) || + asExportPath(exportsField?.require) || + (typeof pkg.main === "string" ? pkg.main : null) || + "./dist/transformers.node.cjs"; + return join(dirname(pkgJsonPath), relative); +} + +function packageJsonCandidates(startDir: string, packageName: string): string[] { + const parts = packageName.split("/"); + const out: string[] = []; + let dir = startDir; + for (let i = 0; i < 16; i++) { + out.push(join(dir, "node_modules", ...parts, "package.json")); + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + return out; +} + +function normalizeStartPath(value: string): string { + if (value.startsWith("file:")) { + try { + return fileURLToPath(value); + } catch { + return value; + } + } + return value; +} + +/** + * Resolve Bun's install layout when Node createRequire cannot see the package. + */ +export function resolveWithBun(specifier: string, fromPath: string): string | null { + const bun = (globalThis as { Bun?: { resolveSync?: (id: string, root: string) => string } }).Bun; + if (typeof bun?.resolveSync !== "function") return null; + try { + const fromDir = fromPath.endsWith(sep) ? fromPath : dirname(fromPath); + return bun.resolveSync(specifier, fromDir); + } catch { + return null; + } +} + +export function resolveTransformersCjsEntry(options: { + requireFn: NodeRequire; + startPaths?: Array; + specifier?: string; +}): string { + const specifier = options.specifier ?? TRANSFORMERS_PACKAGE; + const errors: string[] = []; + + try { + return options.requireFn.resolve(specifier); + } catch (error) { + errors.push(`require.resolve: ${error instanceof Error ? error.message : String(error)}`); + } + + const starts = (options.startPaths ?? []) + .filter((p): p is string => typeof p === "string" && p.trim().length > 0) + .map(normalizeStartPath); + + for (const start of starts) { + const bunResolved = resolveWithBun(specifier, start); + if (bunResolved) { + // Bun may return the ESM entry; normalize to CJS via package.json. + let dir = dirname(bunResolved); + for (let i = 0; i < 6; i++) { + const pkgJson = join(dir, "package.json"); + if (existsSync(pkgJson)) { + const pkg = readJson(pkgJson); + if (pkg?.name === specifier) return cjsEntryFromPackageJson(pkgJson); + } + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + if (bunResolved.endsWith(".cjs") || bunResolved.endsWith(".js")) return bunResolved; + errors.push(`Bun.resolveSync returned non-CJS entry: ${bunResolved}`); + } + + for (const candidate of packageJsonCandidates(dirname(start), specifier)) { + if (!existsSync(candidate)) continue; + try { + return cjsEntryFromPackageJson(candidate); + } catch (error) { + errors.push(`${candidate}: ${error instanceof Error ? error.message : String(error)}`); + } + } + } + + throw new Error( + [ + `Cannot resolve ${specifier} before installing the onnxruntime pin shim (#463).`, + "The package should be installed next to opencode-mem; OpenCode nested installs sometimes hide it from Node createRequire.", + "Fix: clear the OpenCode plugin cache (e.g. ~/.cache/opencode/npm/opencode-mem@* and ~/.cache/opencode/packages/opencode-mem@*) and reinstall, or set embeddingApiUrl + embeddingApiKey for remote embeddings.", + errors.length > 0 ? `Attempts: ${errors.join(" | ")}` : null, + ] + .filter(Boolean) + .join(" "), + { cause: errors[0] } + ); +} + +/** Test helper: expose path→fileURL conversion for Bun resolve roots. */ +export function toFileUrlIfNeeded(path: string): string { + return path.startsWith("file:") ? path : pathToFileURL(path).href; +} diff --git a/tests/transformers-resolve.test.ts b/tests/transformers-resolve.test.ts new file mode 100644 index 00000000..42eba7c4 --- /dev/null +++ b/tests/transformers-resolve.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "bun:test"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + cjsEntryFromPackageJson, + resolveTransformersCjsEntry, + TRANSFORMERS_PACKAGE, +} from "../src/services/transformers-resolve.js"; + +const requireFromHere = createRequire(import.meta.url); + +describe("transformers-resolve (#463)", () => { + it("resolves the real installed CJS entry via require", () => { + const entry = resolveTransformersCjsEntry({ + requireFn: requireFromHere, + startPaths: [import.meta.url], + }); + expect(entry.replaceAll("\\", "/")).toContain("@huggingface/transformers"); + expect(entry.endsWith(".cjs") || entry.endsWith(".js")).toBe(true); + }); + + it("reads CJS export path from package.json", () => { + const dir = mkdtempSync(join(tmpdir(), "transformers-pkg-")); + const pkgJson = join(dir, "package.json"); + writeFileSync( + pkgJson, + JSON.stringify({ + name: TRANSFORMERS_PACKAGE, + main: "./dist/transformers.node.cjs", + exports: { + node: { + require: { default: "./dist/transformers.node.cjs" }, + import: { default: "./dist/transformers.node.mjs" }, + }, + }, + }) + ); + expect(cjsEntryFromPackageJson(pkgJson).replaceAll("\\", "/")).toMatch( + /dist\/transformers\.node\.cjs$/ + ); + }); + + it("finds a hoisted package by walking node_modules when require.resolve fails", () => { + const root = mkdtempSync(join(tmpdir(), "transformers-hoist-")); + const pluginDir = join(root, "node_modules", "opencode-mem", "dist", "services"); + const pkgDir = join(root, "node_modules", "@huggingface", "transformers"); + mkdirSync(pluginDir, { recursive: true }); + mkdirSync(join(pkgDir, "dist"), { recursive: true }); + writeFileSync( + join(pkgDir, "package.json"), + JSON.stringify({ + name: TRANSFORMERS_PACKAGE, + exports: { + node: { require: { default: "./dist/transformers.node.cjs" } }, + }, + }) + ); + writeFileSync(join(pkgDir, "dist", "transformers.node.cjs"), "module.exports = {}"); + + const fakeRequire = ((id: string) => { + throw new Error(`Cannot find module '${id}'`); + }) as NodeRequire; + fakeRequire.resolve = ((_id: string) => { + throw new Error(`Cannot find module '${_id}'`); + }) as NodeRequire["resolve"]; + + const start = join(pluginDir, "embedding.js"); + writeFileSync(start, ""); + const entry = resolveTransformersCjsEntry({ + requireFn: fakeRequire, + startPaths: [start], + }); + expect(entry.replaceAll("\\", "/")).toMatch( + /node_modules\/@huggingface\/transformers\/dist\/transformers\.node\.cjs$/ + ); + }); + + it("mentions cache clear / remote embeddings when unresolved", () => { + const fakeRequire = ((id: string) => { + throw new Error(`Cannot find module '${id}'`); + }) as NodeRequire; + fakeRequire.resolve = ((_id: string) => { + throw new Error(`Cannot find module '${_id}'`); + }) as NodeRequire["resolve"]; + + expect(() => + resolveTransformersCjsEntry({ + requireFn: fakeRequire, + startPaths: [join(tmpdir(), "empty-start", "embedding.js")], + }) + ).toThrow(/#463|embeddingApiUrl|plugin cache/i); + }); +});