fix(embedding): harden transformers resolve for OpenCode nested installs

src/services/embedding.ts

@@ -7,6 +7,10 @@ import {
   prepareOnnxruntimeForTransformers,
 } from "./onnxruntime-resolve.js";
 import { createRuntimeRequire } from "./runtime-require.js";
+import {
+  getTransformersPackageSpecifier,
+  resolveTransformersCjsEntry,
+} from "./transformers-resolve.js";
 
 const requireFromHere = createRuntimeRequire(import.meta);
 
@@ -194,17 +198,10 @@ let _transformers: {
   env: HfTransformers["env"];
 } | null = null;
 
-function getTransformersPackageSpecifier(): string {
-  // Keep this non-literal so OpenCode/Bun plugin-loader bundling does not eagerly
-  // traverse @huggingface/transformers internals during plugin startup. The package
-  // is only needed for the local embedding backend, and should stay lazy.
-  return ["@huggingface", "transformers"].join("/");
-}
-
 async function ensureTransformersLoaded(): Promise<NonNullable<typeof _transformers>> {
   if (_transformers !== null) return _transformers;
   // Pin onnxruntime-node (+ common) to our direct 1.30.0 stack before transformers
-  // resolves them (#184 / #210 / #225). Load the CJS export so Module._resolveFilename
+  // resolves them (#184 / #210 / #225 / #463). Load the CJS export so Module._resolveFilename
   // shim applies — the ESM entry's static import bypasses it under OpenCode nested installs.
   //
   // Critical ordering for OpenCode's Bun --compile host (#210 follow-up):
@@ -213,17 +210,16 @@ async function ensureTransformersLoaded(): Promise<NonNullable<typeof _transform
   // `Cannot find module '@huggingface/transformers' from ''` even though the package
   // exists. Loading via the pre-resolved absolute path keeps the onnx pin shim active
   // for transformers' nested requires without needing a post-shim package resolve.
+  //
+  // #463: OpenCode nested npm/Bun cache layouts can make createRequire(packageName)
+  // fail even when the package is on disk — fall back to Bun.resolveSync + node_modules walk.
   const transformersSpecifier = getTransformersPackageSpecifier();
-  let transformersEntry: string;
-  try {
-    transformersEntry = requireFromHere.resolve(transformersSpecifier);
-  } catch (error) {
-    throw new Error(
-      `Cannot resolve ${transformersSpecifier} before installing the onnxruntime pin shim. ` +
-        `Original error: ${error instanceof Error ? error.message : String(error)}`,
-      { cause: error }
-    );
-  }
+  const meta = import.meta as ImportMeta & { path?: string; dirname?: string; dir?: string };
+  const transformersEntry = resolveTransformersCjsEntry({
+    requireFn: requireFromHere,
+    specifier: transformersSpecifier,
+    startPaths: [meta.url, meta.path, meta.dirname, meta.dir, process.cwd()],
+  });
 
   prepareOnnxruntimeForTransformers();
   const mod = requireFromHere(transformersEntry) as HfTransformers;

src/services/transformers-resolve.ts

@@ -0,0 +1,163 @@
+/**
+ * Resolve the CJS entry of `@huggingface/transformers` for OpenCode hosts (#463).
+ *
+ * OpenCode nested installs (Bun linker / npm cache layouts) sometimes make
+ * Node's createRequire(packageName) fail even when the package exists on disk.
+ * Walk node_modules and prefer Bun.resolveSync, then load the package's
+ * `exports.node.require` entry so the onnxruntime pin shim still applies.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join, sep } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+export const TRANSFORMERS_PACKAGE = ["@huggingface", "transformers"].join("/");
+
+/** Non-literal specifier so plugin bundlers do not eagerly traverse transformers. */
+export function getTransformersPackageSpecifier(): string {
+  return TRANSFORMERS_PACKAGE;
+}
+
+type PackageJsonExports = {
+  node?: {
+    require?: { default?: string } | string;
+    import?: { default?: string } | string;
+  };
+  require?: { default?: string } | string;
+  default?: { default?: string } | string;
+};
+
+function readJson(path: string): Record<string, unknown> | null {
+  try {
+    return JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
+  } catch {
+    return null;
+  }
+}
+
+function asExportPath(value: unknown): string | null {
+  if (typeof value === "string" && value.length > 0) return value;
+  if (
+    value &&
+    typeof value === "object" &&
+    typeof (value as { default?: unknown }).default === "string"
+  ) {
+    return (value as { default: string }).default;
+  }
+  return null;
+}
+
+/** Pick the Node CJS entry from a transformers package.json. */
+export function cjsEntryFromPackageJson(pkgJsonPath: string): string {
+  const pkg = readJson(pkgJsonPath);
+  if (!pkg) throw new Error(`Unable to read ${pkgJsonPath}`);
+  const exportsField = pkg.exports as PackageJsonExports | undefined;
+  const relative =
+    asExportPath(exportsField?.node?.require) ||
+    asExportPath(exportsField?.require) ||
+    (typeof pkg.main === "string" ? pkg.main : null) ||
+    "./dist/transformers.node.cjs";
+  return join(dirname(pkgJsonPath), relative);
+}
+
+function packageJsonCandidates(startDir: string, packageName: string): string[] {
+  const parts = packageName.split("/");
+  const out: string[] = [];
+  let dir = startDir;
+  for (let i = 0; i < 16; i++) {
+    out.push(join(dir, "node_modules", ...parts, "package.json"));
+    const parent = dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  return out;
+}
+
+function normalizeStartPath(value: string): string {
+  if (value.startsWith("file:")) {
+    try {
+      return fileURLToPath(value);
+    } catch {
+      return value;
+    }
+  }
+  return value;
+}
+
+/**
+ * Resolve Bun's install layout when Node createRequire cannot see the package.
+ */
+export function resolveWithBun(specifier: string, fromPath: string): string | null {
+  const bun = (globalThis as { Bun?: { resolveSync?: (id: string, root: string) => string } }).Bun;
+  if (typeof bun?.resolveSync !== "function") return null;
+  try {
+    const fromDir = fromPath.endsWith(sep) ? fromPath : dirname(fromPath);
+    return bun.resolveSync(specifier, fromDir);
+  } catch {
+    return null;
+  }
+}
+
+export function resolveTransformersCjsEntry(options: {
+  requireFn: NodeRequire;
+  startPaths?: Array<string | undefined | null>;
+  specifier?: string;
+}): string {
+  const specifier = options.specifier ?? TRANSFORMERS_PACKAGE;
+  const errors: string[] = [];
+
+  try {
+    return options.requireFn.resolve(specifier);
+  } catch (error) {
+    errors.push(`require.resolve: ${error instanceof Error ? error.message : String(error)}`);
+  }
+
+  const starts = (options.startPaths ?? [])
+    .filter((p): p is string => typeof p === "string" && p.trim().length > 0)
+    .map(normalizeStartPath);
+
+  for (const start of starts) {
+    const bunResolved = resolveWithBun(specifier, start);
+    if (bunResolved) {
+      // Bun may return the ESM entry; normalize to CJS via package.json.
+      let dir = dirname(bunResolved);
+      for (let i = 0; i < 6; i++) {
+        const pkgJson = join(dir, "package.json");
+        if (existsSync(pkgJson)) {
+          const pkg = readJson(pkgJson);
+          if (pkg?.name === specifier) return cjsEntryFromPackageJson(pkgJson);
+        }
+        const parent = dirname(dir);
+        if (parent === dir) break;
+        dir = parent;
+      }
+      if (bunResolved.endsWith(".cjs") || bunResolved.endsWith(".js")) return bunResolved;
+      errors.push(`Bun.resolveSync returned non-CJS entry: ${bunResolved}`);
+    }
+
+    for (const candidate of packageJsonCandidates(dirname(start), specifier)) {
+      if (!existsSync(candidate)) continue;
+      try {
+        return cjsEntryFromPackageJson(candidate);
+      } catch (error) {
+        errors.push(`${candidate}: ${error instanceof Error ? error.message : String(error)}`);
+      }
+    }
+  }
+
+  throw new Error(
+    [
+      `Cannot resolve ${specifier} before installing the onnxruntime pin shim (#463).`,
+      "The package should be installed next to opencode-mem; OpenCode nested installs sometimes hide it from Node createRequire.",
+      "Fix: clear the OpenCode plugin cache (e.g. ~/.cache/opencode/npm/opencode-mem@* and ~/.cache/opencode/packages/opencode-mem@*) and reinstall, or set embeddingApiUrl + embeddingApiKey for remote embeddings.",
+      errors.length > 0 ? `Attempts: ${errors.join(" | ")}` : null,
+    ]
+      .filter(Boolean)
+      .join(" "),
+    { cause: errors[0] }
+  );
+}
+
+/** Test helper: expose path→fileURL conversion for Bun resolve roots. */
+export function toFileUrlIfNeeded(path: string): string {
+  return path.startsWith("file:") ? path : pathToFileURL(path).href;
+}

tests/transformers-resolve.test.ts

@@ -0,0 +1,95 @@
+import { describe, expect, it } from "bun:test";
+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  cjsEntryFromPackageJson,
+  resolveTransformersCjsEntry,
+  TRANSFORMERS_PACKAGE,
+} from "../src/services/transformers-resolve.js";
+
+const requireFromHere = createRequire(import.meta.url);
+
+describe("transformers-resolve (#463)", () => {
+  it("resolves the real installed CJS entry via require", () => {
+    const entry = resolveTransformersCjsEntry({
+      requireFn: requireFromHere,
+      startPaths: [import.meta.url],
+    });
+    expect(entry.replaceAll("\\", "/")).toContain("@huggingface/transformers");
+    expect(entry.endsWith(".cjs") || entry.endsWith(".js")).toBe(true);
+  });
+
+  it("reads CJS export path from package.json", () => {
+    const dir = mkdtempSync(join(tmpdir(), "transformers-pkg-"));
+    const pkgJson = join(dir, "package.json");
+    writeFileSync(
+      pkgJson,
+      JSON.stringify({
+        name: TRANSFORMERS_PACKAGE,
+        main: "./dist/transformers.node.cjs",
+        exports: {
+          node: {
+            require: { default: "./dist/transformers.node.cjs" },
+            import: { default: "./dist/transformers.node.mjs" },
+          },
+        },
+      })
+    );
+    expect(cjsEntryFromPackageJson(pkgJson).replaceAll("\\", "/")).toMatch(
+      /dist\/transformers\.node\.cjs$/
+    );
+  });
+
+  it("finds a hoisted package by walking node_modules when require.resolve fails", () => {
+    const root = mkdtempSync(join(tmpdir(), "transformers-hoist-"));
+    const pluginDir = join(root, "node_modules", "opencode-mem", "dist", "services");
+    const pkgDir = join(root, "node_modules", "@huggingface", "transformers");
+    mkdirSync(pluginDir, { recursive: true });
+    mkdirSync(join(pkgDir, "dist"), { recursive: true });
+    writeFileSync(
+      join(pkgDir, "package.json"),
+      JSON.stringify({
+        name: TRANSFORMERS_PACKAGE,
+        exports: {
+          node: { require: { default: "./dist/transformers.node.cjs" } },
+        },
+      })
+    );
+    writeFileSync(join(pkgDir, "dist", "transformers.node.cjs"), "module.exports = {}");
+
+    const fakeRequire = ((id: string) => {
+      throw new Error(`Cannot find module '${id}'`);
+    }) as NodeRequire;
+    fakeRequire.resolve = ((_id: string) => {
+      throw new Error(`Cannot find module '${_id}'`);
+    }) as NodeRequire["resolve"];
+
+    const start = join(pluginDir, "embedding.js");
+    writeFileSync(start, "");
+    const entry = resolveTransformersCjsEntry({
+      requireFn: fakeRequire,
+      startPaths: [start],
+    });
+    expect(entry.replaceAll("\\", "/")).toMatch(
+      /node_modules\/@huggingface\/transformers\/dist\/transformers\.node\.cjs$/
+    );
+  });
+
+  it("mentions cache clear / remote embeddings when unresolved", () => {
+    const fakeRequire = ((id: string) => {
+      throw new Error(`Cannot find module '${id}'`);
+    }) as NodeRequire;
+    fakeRequire.resolve = ((_id: string) => {
+      throw new Error(`Cannot find module '${_id}'`);
+    }) as NodeRequire["resolve"];
+
+    expect(() =>
+      resolveTransformersCjsEntry({
+        requireFn: fakeRequire,
+        startPaths: [join(tmpdir(), "empty-start", "embedding.js")],
+      })
+    ).toThrow(/#463|embeddingApiUrl|plugin cache/i);
+  });
+});