From 3cf0fb26f9b7c2faabc2a4e040c0d472efe6ef09 Mon Sep 17 00:00:00 2001 From: JayceeB1 <45721154+JayceeB1@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:41:44 +0200 Subject: [PATCH 01/15] fix(storage): guard destructive directory moves and deletion --- electron/main/ipc-handlers.ts | 22 +- electron/main/storage-directory-guard.test.ts | 172 +++++++++++++++ electron/main/storage-directory-guard.ts | 199 ++++++++++++++++++ package.json | 2 +- 4 files changed, 377 insertions(+), 18 deletions(-) create mode 100644 electron/main/storage-directory-guard.test.ts create mode 100644 electron/main/storage-directory-guard.ts diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index e9cac040b..f62439674 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -42,6 +42,7 @@ import { weightStorageHasLocalData, } from './model-sources' import { getSettings, setSettings } from './settings-store' +import { deleteStorageDirectory, moveStorageDirectory } from './storage-directory-guard' import { checkSetupNeeded, markSetupDone, runFullSetup, getVenvPythonExe, ensureSslPatch } from './python-setup' import { logger } from './logger' import { getProcessRunner, getPythonProcessRunner, getExtPythonExe, terminateProcessRunner, terminateAllProcessRunners } from './process-runner' @@ -1091,9 +1092,8 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe ipcMain.handle('fs:moveDirectory', async (_, { src, dest }: { src: string; dest: string }) => { try { - await mkdir(dest, { recursive: true }) - await cp(src, dest, { recursive: true }) - await rmAsync(src, { recursive: true, force: true }) + const userData = app.getPath('userData') + await moveStorageDirectory(src, dest, getSettings(userData), userData, { appDir: app.getAppPath() }) return { success: true } } catch (err) { return { success: false, error: String(err) } @@ -1101,21 +1101,9 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe }) ipcMain.handle('fs:deleteDirectory', async (_, dirPath: string) => { - const userData = app.getPath('userData') - const settings = getSettings(userData) - const allowedRoots = [ - settings.modelsDir, - settings.workspaceDir, - settings.extensionsDir, - join(userData, 'gen-cache'), - ] - const resolved = join(dirPath) - const isAllowed = allowedRoots.some((root) => resolved.startsWith(root)) - if (!isAllowed) { - return { success: false, error: 'Path is outside allowed directories' } - } try { - await rmAsync(resolved, { recursive: true, force: true }) + const userData = app.getPath('userData') + await deleteStorageDirectory(dirPath, getSettings(userData), userData, { appDir: app.getAppPath() }) return { success: true } } catch (err) { return { success: false, error: String(err) } diff --git a/electron/main/storage-directory-guard.test.ts b/electron/main/storage-directory-guard.test.ts new file mode 100644 index 000000000..d40aa3fa8 --- /dev/null +++ b/electron/main/storage-directory-guard.test.ts @@ -0,0 +1,172 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { mkdtemp, mkdir, rm, symlink, writeFile, readFile, access } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + validateStorageDeletion, + validateStorageMove, + assertNoSymlinkAncestors, + assertEmptyDestination, + deleteStorageDirectory, + moveStorageDirectory, +} from './storage-directory-guard' + +const dirs = { + modelsDir: 'E:\\modly\\models', + workspaceDir: 'E:\\modly\\workspace', + workflowsDir: 'E:\\modly\\workflows', + extensionsDir: 'E:\\modly\\extensions', + dependenciesDir: 'E:\\modly\\dependencies', + agentDir: 'E:\\modly\\agent', +} +const home = 'C:\\Users\\Tester' +const userData = 'C:\\Users\\Tester\\AppData\\Roaming\\modly' +const opts = { platform: 'win32' as const, homeDir: home, appDir: 'C:\\Program Files\\Modly\\resources\\app.asar' } + +test('deletion permits an exact configured managed directory', () => { + assert.equal(validateStorageDeletion('e:\\MODLY\\models\\\\', dirs, userData, opts), dirs.modelsDir) + assert.equal(validateStorageDeletion(dirs.workflowsDir, dirs, userData, opts), dirs.workflowsDir) + assert.equal(validateStorageDeletion('E:\\modly\\workspace\\tmp', dirs, userData, opts), 'E:\\modly\\workspace\\tmp') +}) + +test('deletion rejects filesystem roots, shallow folders and relative paths', () => { + for (const bad of ['E:\\\\', 'C:\\\\', 'E:\\Photos', 'models', '.', '']) { + assert.throws(() => validateStorageDeletion(bad, { ...dirs, modelsDir: bad }, userData, opts)) + } + assert.throws(() => validateStorageDeletion('\\\\server\\share\\models', dirs, userData, opts)) +}) + +test('deletion requires an exact configured root or workspace/tmp (not a prefix)', () => { + for (const bad of [ + 'E:\\modly\\models-backup', 'E:\\modly\\models\\elsewhere', + 'E:\\modly\\workspace\\tmp-fake', 'E:\\modly\\workflows2', + 'E:\\elsewhere\\models', + ]) { + assert.throws(() => validateStorageDeletion(bad, dirs, userData, opts), /not an approved/i) + } +}) + +test('deletion rejects home, application and userData ancestors even if configured', () => { + for (const bad of [home, 'C:\\Users', 'C:\\Program Files', 'C:\\Users\\Tester\\AppData', userData]) { + assert.throws(() => validateStorageDeletion(bad, { ...dirs, modelsDir: bad }, userData, opts)) + } +}) + +test('deletion refuses one configured folder enclosing another', () => { + assert.throws(() => validateStorageDeletion(dirs.modelsDir, { + ...dirs, workspaceDir: 'E:\\modly\\models\\my-workspace', + }, userData, opts), /another configured/i) +}) + +test('move allows separate configured source and empty destination path', () => { + assert.deepEqual(validateStorageMove(dirs.modelsDir, 'D:\\modly-assets\\models', dirs, userData, opts), { + src: dirs.modelsDir, dest: 'D:\\modly-assets\\models', + }) +}) + +test('move rejects rogue sources, root, parent, child and overlapping destinations', () => { + const badSources = ['E:\\\\', home, 'E:\\modly\\models-other', 'E:\\modly\\models\\child'] + for (const bad of badSources) { + assert.throws(() => validateStorageMove(bad, 'D:\\modly\\models', dirs, userData, opts)) + } + for (const bad of [ + dirs.modelsDir, 'E:\\modly', 'E:\\modly\\models\\new', 'E:\\modly\\workflows', + 'E:\\\\', 'D:\\Backups', 'E:\\modly\\workspace\\inside', + ]) { + assert.throws(() => validateStorageMove(dirs.modelsDir, bad, dirs, userData, opts)) + } +}) + +test('POSIX root and relative path are rejected', () => { + const linuxDirs = { + modelsDir: '/home/tester/modly/models', + workspaceDir: '/home/tester/modly/workspace', + workflowsDir: '/home/tester/modly/workflows', + extensionsDir: '/home/tester/modly/extensions', + dependenciesDir: '/home/tester/modly/dependencies', + agentDir: '/home/tester/modly/agent', + } + const linuxOpts = { platform: 'linux' as const, homeDir: '/home/tester' } + assert.equal(validateStorageDeletion(linuxDirs.modelsDir, linuxDirs, '/home/tester/.config/modly', linuxOpts), linuxDirs.modelsDir) + assert.throws(() => validateStorageDeletion('/', { ...linuxDirs, modelsDir: '/' }, '/home/tester/.config/modly', linuxOpts)) + assert.throws(() => validateStorageMove('relative', '/tmp/safe/new', linuxDirs, '/home/tester/.config/modly', linuxOpts)) +}) + +test('destination must be empty, missing is fine', async () => { + const base = await mkdtemp(join(tmpdir(), 'modly-storage-')) + try { + await assertEmptyDestination(join(base, 'new')) + await mkdir(join(base, 'empty')) + await assertEmptyDestination(join(base, 'empty')) + await writeFile(join(base, 'empty', 'personal.txt'), 'do not overwrite') + await assert.rejects(() => assertEmptyDestination(join(base, 'empty')), /empty/i) + } finally { + await rm(base, { recursive: true, force: true }) + } +}) + +test('symlinked ancestors and target are refused', { skip: process.platform === 'win32' }, async () => { + const base = await mkdtemp(join(tmpdir(), 'modly-storage-link-')) + try { + await mkdir(join(base, 'real', 'models'), { recursive: true }) + await symlink(join(base, 'real'), join(base, 'alias'), 'dir') + await assertNoSymlinkAncestors(join(base, 'real', 'models')) + await assert.rejects(() => assertNoSymlinkAncestors(join(base, 'alias', 'models')), /symbolic link/i) + await assert.rejects(() => assertNoSymlinkAncestors(join(base, 'alias')), /symbolic link/i) + } finally { + await rm(base, { recursive: true, force: true }) + } +}) + +test('unsafe real deletion refuses a storage root and preserves personal files', async () => { + const base = await mkdtemp(join(tmpdir(), 'modly-real-delete-')) + const managed = join(base, 'models') + const personal = join(base, 'personal.txt') + const unixDirs = { + modelsDir: managed, workspaceDir: join(base, 'workspace'), + workflowsDir: join(base, 'workflows'), extensionsDir: join(base, 'extensions'), + dependenciesDir: join(base, 'dependencies'), agentDir: join(base, 'agent'), + } + const options = { platform: process.platform, homeDir: join(base, 'other-home') } + try { + await mkdir(managed) + await writeFile(personal, 'important') + await assert.rejects(() => deleteStorageDirectory(base, { ...unixDirs, modelsDir: base }, + join(base, 'userData'), options), /ancestor|another configured|approved/i) + assert.equal(await readFile(personal, 'utf8'), 'important') + await deleteStorageDirectory(managed, unixDirs, join(base, 'userData'), options) + await assert.rejects(() => access(managed)) + assert.equal(await readFile(personal, 'utf8'), 'important') + } finally { + await rm(base, { recursive: true, force: true }) + } +}) + +test('real move does not overwrite destination files or erase unrelated data', async () => { + const base = await mkdtemp(join(tmpdir(), 'modly-real-move-')) + const source = join(base, 'models') + const destination = join(base, 'migrated', 'models') + const unixDirs = { + modelsDir: source, workspaceDir: join(base, 'workspace'), + workflowsDir: join(base, 'workflows'), extensionsDir: join(base, 'extensions'), + dependenciesDir: join(base, 'dependencies'), agentDir: join(base, 'agent'), + } + const options = { platform: process.platform, homeDir: join(base, 'other-home') } + try { + await mkdir(source) + await mkdir(destination, { recursive: true }) + await writeFile(join(source, 'asset.glb'), 'model') + await writeFile(join(destination, 'personal.txt'), 'preserve') + await assert.rejects(() => moveStorageDirectory(source, destination, unixDirs, + join(base, 'userData'), options), /empty/i) + assert.equal(await readFile(join(destination, 'personal.txt'), 'utf8'), 'preserve') + assert.equal(await readFile(join(source, 'asset.glb'), 'utf8'), 'model') + await rm(destination, { recursive: true }) + await moveStorageDirectory(source, destination, unixDirs, join(base, 'userData'), options) + assert.equal(await readFile(join(destination, 'asset.glb'), 'utf8'), 'model') + await assert.rejects(() => access(source)) + } finally { + await rm(base, { recursive: true, force: true }) + } +}) diff --git a/electron/main/storage-directory-guard.ts b/electron/main/storage-directory-guard.ts new file mode 100644 index 000000000..fb1e9b914 --- /dev/null +++ b/electron/main/storage-directory-guard.ts @@ -0,0 +1,199 @@ +/** Fail-closed validation for settings-driven directory moves and deletions. */ +import { cp, lstat, mkdir, readdir, rm } from 'node:fs/promises' +import { homedir } from 'node:os' +import path from 'node:path' + +type StorageDirs = { + modelsDir: string + workspaceDir: string + workflowsDir: string + extensionsDir: string + dependenciesDir: string + agentDir: string +} + +type Options = { + platform?: NodeJS.Platform + homeDir?: string + appDir?: string +} + +type PathApi = typeof path.win32 + +function pathApi(options: Options): PathApi { + return options.platform === 'win32' || (!options.platform && process.platform === 'win32') + ? path.win32 + : path.posix +} + +function absoluteDirectory(value: string, api: PathApi): string { + if (typeof value !== 'string' || !value.trim() || value.includes('\0') || !api.isAbsolute(value)) { + throw new Error('Storage path must be a non-empty absolute path') + } + // Drive-relative, extended Windows and UNC paths need separate review. + if (api === path.win32 && !/^[a-z]:[\\/]/i.test(value)) { + throw new Error('Storage operations require a local drive-absolute path') + } + const resolved = api.normalize(value) + if (resolved === api.parse(resolved).root) { + throw new Error('Refusing a filesystem or volume root') + } + // Never recursively delete a bare top-level user folder, even if configured. + const pieces = resolved.slice(api.parse(resolved).root.length).split(api.sep).filter(Boolean) + if (pieces.length < 2) { + throw new Error('Storage directory must be at least two levels below its volume root') + } + return resolved +} + +function equalPath(api: PathApi, a: string, b: string): boolean { + return api.relative(a, b) === '' +} + +function sameOrInside(api: PathApi, parent: string, candidate: string): boolean { + const relative = api.relative(parent, candidate) + return relative === '' || (relative !== '..' && !relative.startsWith('..' + api.sep) && !api.isAbsolute(relative)) +} + +function storageEntries(dirs: StorageDirs): Array<[string, string]> { + return [ + ['models', dirs.modelsDir], + ['workspace', dirs.workspaceDir], + ['workflows', dirs.workflowsDir], + ['extensions', dirs.extensionsDir], + ['dependencies', dirs.dependenciesDir], + ['agent', dirs.agentDir], + ] +} + +function verifySource(source: string, userData: string, options: Options): string { + const api = pathApi(options) + const safe = absoluteDirectory(source, api) + const protectedRoots = [userData, options.homeDir ?? homedir(), options.appDir] + for (const item of protectedRoots) { + if (!item) continue + const protectedPath = api.normalize(item) + if (api.isAbsolute(protectedPath) && sameOrInside(api, safe, protectedPath)) { + throw new Error('Refusing to remove an application, home, or user-data ancestor') + } + } + return safe +} + +function verifyNoOtherStorageInside( + source: string, selectedKey: string, dirs: StorageDirs, options: Options, +): void { + const api = pathApi(options) + for (const [key, value] of storageEntries(dirs)) { + if (key === selectedKey || !value) continue + const other = api.normalize(value) + if (api.isAbsolute(other) && sameOrInside(api, source, other)) { + throw new Error('Refusing to remove another configured storage directory (' + key + ')') + } + } +} + +/** Only exact configured roots, plus the specific workspace/tmp cleanup, are deletable. */ +export function validateStorageDeletion( + directory: string, dirs: StorageDirs, userData: string, options: Options = {}, +): string { + const api = pathApi(options) + const target = absoluteDirectory(directory, api) + const roots: Array<[string, string]> = [ + ['models', dirs.modelsDir], + ['workspace', dirs.workspaceDir], + ['workflows', dirs.workflowsDir], + ['extensions', dirs.extensionsDir], + ['cache', api.join(userData, 'gen-cache')], + ] + const selected = roots.find(([, root]) => root && equalPath(api, api.normalize(root), target)) + if (selected) { + const safe = verifySource(selected[1], userData, options) + verifyNoOtherStorageInside(safe, selected[0], dirs, options) + return safe + } + const workspace = verifySource(dirs.workspaceDir, userData, options) + if (equalPath(api, target, api.join(workspace, 'tmp'))) { + verifyNoOtherStorageInside(target, 'workspace', dirs, options) + return target + } + throw new Error('Path is not an approved Modly storage directory') +} + +/** A folder move may delete its source: restrict source and destination equally. */ +export function validateStorageMove( + source: string, destination: string, dirs: StorageDirs, userData: string, options: Options = {}, +): { src: string; dest: string } { + const api = pathApi(options) + const src = verifySource(source, userData, options) + const selected = storageEntries(dirs).find(([key, val]) => + ['models', 'workspace', 'workflows'].includes(key) && equalPath(api, api.normalize(val), src)) + if (!selected) throw new Error('Source is not a configured Modly storage directory') + verifyNoOtherStorageInside(src, selected[0], dirs, options) + const dest = verifySource(destination, userData, options) + if (sameOrInside(api, src, dest) || sameOrInside(api, dest, src)) { + throw new Error('Source and destination must be separate, non-overlapping directories') + } + // Reject destinations overlapping ANY configured storage root. + for (const [, value] of storageEntries(dirs)) { + if (!value) continue + const other = api.normalize(value) + if (sameOrInside(api, other, dest) || sameOrInside(api, dest, other)) { + throw new Error('Destination overlaps configured storage') + } + } + return { src, dest } +} + +/** Reject symlinks or Windows junctions in existing path ancestors. */ +export async function assertNoSymlinkAncestors(directory: string): Promise { + const api = process.platform === 'win32' ? path.win32 : path.posix + const root = api.parse(directory).root + let current = root + for (const part of directory.slice(root.length).split(api.sep).filter(Boolean)) { + current = api.join(current, part) + try { + const info = await lstat(current) + if (info.isSymbolicLink()) throw new Error('Storage path contains a symbolic link: ' + current) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') break + throw error + } + } +} + +export async function assertEmptyDestination(directory: string): Promise { + try { + if ((await readdir(directory)).length > 0) { + throw new Error('Destination must be empty to avoid overwriting existing files') + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return + throw error + } +} + +/** Main-process IPC handlers must use guarded operations, never raw rm/cp. */ +export async function deleteStorageDirectory( + directory: string, dirs: StorageDirs, userData: string, options: Options = {}, +): Promise { + const safe = validateStorageDeletion(directory, dirs, userData, options) + await assertNoSymlinkAncestors(safe) + await rm(safe, { recursive: true, force: true }) +} + +export async function moveStorageDirectory( + source: string, destination: string, dirs: StorageDirs, userData: string, options: Options = {}, +): Promise { + const paths = validateStorageMove(source, destination, dirs, userData, options) + await assertNoSymlinkAncestors(paths.src) + await assertNoSymlinkAncestors(paths.dest) + await assertEmptyDestination(paths.dest) + await mkdir(paths.dest, { recursive: true }) + // Refuse races that add files to the destination after the emptiness check. + await cp(paths.src, paths.dest, { recursive: true, force: false, errorOnExist: true }) + // Re-validate before deleting the source. Never delete it after a copy error. + validateStorageMove(paths.src, paths.dest, dirs, userData, options) + await assertNoSymlinkAncestors(paths.src) + await rm(paths.src, { recursive: true, force: true }) +} diff --git a/package.json b/package.json index a63727a24..a17e0aa4f 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." From 6481e15c6dd87143fed62d77e91c1a6e769b5154 Mon Sep 17 00:00:00 2001 From: JayceeB1 <45721154+JayceeB1@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:07:59 +0200 Subject: [PATCH 02/15] fix(storage): copy safely into empty destination without EEXIST Confirmed by combined CI regression. Remove only an empty target with rmdir; never remove source on copy failure. --- electron/main/storage-directory-guard.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/electron/main/storage-directory-guard.ts b/electron/main/storage-directory-guard.ts index fb1e9b914..f5a63a8c8 100644 --- a/electron/main/storage-directory-guard.ts +++ b/electron/main/storage-directory-guard.ts @@ -1,5 +1,5 @@ /** Fail-closed validation for settings-driven directory moves and deletions. */ -import { cp, lstat, mkdir, readdir, rm } from 'node:fs/promises' +import { cp, lstat, mkdir, readdir, rm, rmdir } from 'node:fs/promises' import { homedir } from 'node:os' import path from 'node:path' @@ -189,8 +189,16 @@ export async function moveStorageDirectory( await assertNoSymlinkAncestors(paths.src) await assertNoSymlinkAncestors(paths.dest) await assertEmptyDestination(paths.dest) - await mkdir(paths.dest, { recursive: true }) - // Refuse races that add files to the destination after the emptiness check. + // fs.cp with errorOnExist refuses even an EMPTY existing directory. Remove + // only the verified-empty destination, never the source, before copying. + // rmdir itself fails closed if a concurrent writer adds a file. + try { + await rmdir(paths.dest) + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err + } + await mkdir(path.dirname(paths.dest), { recursive: true }) + // With a missing destination, errorOnExist protects against unexpected files. await cp(paths.src, paths.dest, { recursive: true, force: false, errorOnExist: true }) // Re-validate before deleting the source. Never delete it after a copy error. validateStorageMove(paths.src, paths.dest, dirs, userData, options) From 51b798affa4d2db17e9ac3255eaa9a705feb4a83 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 19 Oct 2026 10:07:37 +0200 Subject: [PATCH 03/15] feat(mesh-ops): add read-only inspect_mesh and suggest_fixes ops inspect_mesh diagnoses a mesh without modifying it: watertight, manifold, winding, holes, degenerate and self-intersecting faces, parts, bounds, area, volume, UVs/normals, and a printable verdict. suggest_fixes derives registry op calls (repair, decimate) from that report, and lists the problems no op can fix yet as notes. Topology is checked on a position-merged copy, so UV seams do not read as holes. Self-intersections go through pymeshlab; when they are not checked, printable is null rather than a claim. Normals are read from the file itself, since trimesh computes them on demand. MeshOpResult.file_path is now optional so analysis ops return only their report. Both ops are exposed to the agent and the MCP server. Closes #296 --- api/mcp_server.py | 38 +++ api/routers/agent.py | 89 +++++++ api/routers/optimize.py | 4 + api/services/mesh_ops/builtin.py | 28 ++ api/services/mesh_ops/inspection.py | 358 ++++++++++++++++++++++++++ api/services/mesh_ops/processor.py | 4 + api/services/mesh_ops/types.py | 7 +- api/tests/test_agent_router.py | 69 +++++ api/tests/test_mesh_ops_inspection.py | 249 ++++++++++++++++++ api/tests/test_mesh_ops_registry.py | 4 +- api/tests/test_optimize_mesh_ops.py | 19 ++ 11 files changed, 865 insertions(+), 4 deletions(-) create mode 100644 api/services/mesh_ops/inspection.py create mode 100644 api/tests/test_mesh_ops_inspection.py diff --git a/api/mcp_server.py b/api/mcp_server.py index fa9127cf2..3c7968f6c 100644 --- a/api/mcp_server.py +++ b/api/mcp_server.py @@ -19,6 +19,7 @@ """ import asyncio +import json import mimetypes import httpx from mcp.server import Server @@ -83,6 +84,37 @@ async def list_tools() -> list[Tool]: "required": ["job_id"], }, ), + Tool( + name="modly_inspect_mesh", + description=( + "Diagnose a mesh without modifying it: watertight, manifold, holes, " + "self-intersections, parts, dimensions, volume, UVs/normals, printability." + ), + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + }, + "required": ["path"], + }, + ), + Tool( + name="modly_suggest_fixes", + description="Inspect a mesh and list the operations (with params) that would fix its problems.", + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + }, + "required": ["path"], + }, + ), Tool( name="modly_decimate_mesh", description="Reduce the polygon count of a mesh using quadric edge collapse decimation.", @@ -217,6 +249,12 @@ async def _dispatch(client: httpx.AsyncClient, name: str, args: dict) -> str: parts.append(f"Error: {s['error']}") return " | ".join(parts) + elif name in ("modly_inspect_mesh", "modly_suggest_fixes"): + op = name.removeprefix("modly_") + r = await client.post(f"{API_BASE}/optimize/op/{op}", json={"path": args["path"]}) + r.raise_for_status() + return json.dumps(r.json(), indent=2) + elif name == "modly_decimate_mesh": r = await client.post( f"{API_BASE}/optimize/mesh", diff --git a/api/routers/agent.py b/api/routers/agent.py index 1fb389eb6..6a563b9b0 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -28,6 +28,8 @@ - **list_models** — List all downloaded 3D generation models ready to use. - **unload_models** — Unload all 3D generation models from GPU VRAM to free memory. - **get_mesh_info** — Get info about the current mesh in the 3D viewer (path, triangle count). +- **inspect_mesh(path)** — Diagnose a mesh without modifying it: watertight, manifold, holes, self-intersections, parts, dimensions, volume, UVs/normals, and whether it is 3D-printable. +- **suggest_fixes(path)** — Inspect a mesh and list the operations (with params) that would fix its problems. - **decimate_mesh(path, target_faces)** — Reduce the polygon count of a mesh. - **smooth_mesh(path, iterations)** — Apply Laplacian smoothing to a mesh. - **get_generation_status(job_id)** — Poll the status of an ongoing 3D generation job. @@ -39,6 +41,7 @@ - Always use tools to act on the scene — never just describe what you would do. - If you need the current mesh path, call get_mesh_info first. +- To answer questions about a mesh's quality or printability, call inspect_mesh — never guess. Before repairing a mesh, call suggest_fixes to pick the right operations. - If you need to run a workflow but don't know the ID, call list_workflows first. - To create a workflow, ONLY use extension ids listed under "Available extensions" in the context. Never invent an id. Chain steps so each step's input type matches the previous step's output type. - For a workflow's input, `input_type` MUST be exactly one of: `image`, `text`, or `mesh`. These map to the Image, Text, and Load 3D Mesh nodes. Never invent another input. Pick the one matching the first step's expected input. @@ -72,6 +75,40 @@ "parameters": {"type": "object", "properties": {}}, }, }, + { + "type": "function", + "function": { + "name": "inspect_mesh", + "description": "Diagnose a mesh without modifying it: watertight, manifold, holes, self-intersections, disconnected parts, dimensions, volume, UVs/normals, and whether it is 3D-printable.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file. Use get_mesh_info to obtain it.", + }, + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "suggest_fixes", + "description": "Inspect a mesh and list the operations (with parameters) that would fix its problems. Does not modify the mesh.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file. Use get_mesh_info to obtain it.", + }, + }, + "required": ["path"], + }, + }, + }, { "type": "function", "function": { @@ -259,6 +296,42 @@ def _build_workflow_graph(name: str, description: str, input_type: str, steps: l return {"name": name, "description": description, "nodes": all_nodes, "edges": edges} +def _format_inspection(report: dict) -> str: + """Render an inspect_mesh report as compact text for the model.""" + def yes_no(value) -> str: + return "unknown" if value is None else ("yes" if value else "no") + + dims = " × ".join(f"{d:.4g}" for d in report.get("dimensions", [])) + volume = report.get("volume") + self_hits = report.get("self_intersecting_face_count") + lines = [ + f"Printable: {yes_no(report.get('printable'))}", + f"Faces: {report.get('face_count', '?'):,}, vertices: {report.get('vertex_count', '?'):,}, parts: {report.get('component_count', '?')}", + f"Watertight: {yes_no(report.get('is_watertight'))}, manifold: {yes_no(report.get('is_manifold'))}, consistent winding: {yes_no(report.get('is_winding_consistent'))}", + f"Holes: {report.get('hole_count', 0)} ({report.get('boundary_edge_count', 0)} open edges), non-manifold edges: {report.get('non_manifold_edge_count', 0)}, degenerate faces: {report.get('degenerate_face_count', 0)}", + f"Self-intersecting faces: {'not checked' if self_hits is None else self_hits}", + f"Dimensions: {dims}, surface area: {report.get('surface_area', 0):.4g}, volume: {'n/a (not closed)' if volume is None else f'{volume:.4g}'}", + f"UVs: {yes_no(report.get('has_uvs'))}, normals: {yes_no(report.get('has_normals'))}", + ] + issues = report.get("issues") or [] + if issues: + lines.append("Issues: " + "; ".join(issues)) + return "\n".join(lines) + + +def _format_fixes(data: dict) -> str: + """Render a suggest_fixes result as compact text for the model.""" + fixes = data.get("fixes") or [] + notes = data.get("notes") or [] + if not fixes and not notes: + return "No fixes needed — the mesh has no detected problems." + lines = ["Suggested operations:"] if fixes else ["No operation can fix the detected problems."] + for fix in fixes: + lines.append(f"- {fix['op']} {json.dumps(fix.get('params', {}))}: {fix.get('reason', '')}") + lines.extend(f"Note: {note}" for note in notes) + return "\n".join(lines) + + async def execute_tool(name: str, arguments: dict, context: dict) -> tuple[str, dict | None]: """Execute a tool and return (result_text, action_payload). action_payload carries data the frontend needs to react (e.g. new mesh URL). @@ -289,6 +362,22 @@ async def execute_tool(name: str, arguments: dict, context: dict) -> tuple[str, info += f" ({mesh_triangles:,} triangles)" return info, None + elif name == "inspect_mesh": + r = await client.post( + f"{MODLY_API}/optimize/op/inspect_mesh", + json={"path": arguments["path"]}, + ) + r.raise_for_status() + return _format_inspection(r.json()), None + + elif name == "suggest_fixes": + r = await client.post( + f"{MODLY_API}/optimize/op/suggest_fixes", + json={"path": arguments["path"]}, + ) + r.raise_for_status() + return _format_fixes(r.json()), None + elif name == "decimate_mesh": r = await client.post( f"{MODLY_API}/optimize/mesh", diff --git a/api/routers/optimize.py b/api/routers/optimize.py index c325ff206..bcfb9f6b0 100644 --- a/api/routers/optimize.py +++ b/api/routers/optimize.py @@ -101,6 +101,10 @@ def _run_operation( def _operation_response(result: MeshOpResult) -> dict[str, object]: + if result.file_path is None: + # Analysis ops (inspect_mesh, suggest_fixes) return only their report. + return dict(result.details) + output_path = result.file_path.resolve() try: relative_path = output_path.relative_to(registry.WORKSPACE_DIR.resolve()).as_posix() diff --git a/api/services/mesh_ops/builtin.py b/api/services/mesh_ops/builtin.py index 900bd1933..75574fe41 100644 --- a/api/services/mesh_ops/builtin.py +++ b/api/services/mesh_ops/builtin.py @@ -1,5 +1,6 @@ """Built-in mesh operation definitions.""" +from .inspection import inspect_mesh, suggest_fixes from .operations import decimate_mesh, repair_mesh, smooth_mesh from .types import MeshOp @@ -106,6 +107,19 @@ ) +INSPECT_PARAMS = ( + { + "id": "check_self_intersections", + "label": "Check Self-Intersections", + "type": "boolean", + "default": True, + "tooltip": ( + "Detect intersecting faces. The slowest check on very dense meshes." + ), + }, +) + + BUILTIN_MESH_OPS = ( MeshOp( id="repair", @@ -128,4 +142,18 @@ fn=smooth_mesh, category="optimization", ), + MeshOp( + id="inspect_mesh", + label="Inspect Mesh", + params_schema=INSPECT_PARAMS, + fn=inspect_mesh, + category="analysis", + ), + MeshOp( + id="suggest_fixes", + label="Suggest Fixes", + params_schema=INSPECT_PARAMS, + fn=suggest_fixes, + category="analysis", + ), ) diff --git a/api/services/mesh_ops/inspection.py b/api/services/mesh_ops/inspection.py new file mode 100644 index 000000000..a8dfc3f14 --- /dev/null +++ b/api/services/mesh_ops/inspection.py @@ -0,0 +1,358 @@ +"""Read-only mesh diagnostics: inspection report and derived fix suggestions.""" + +import json +import struct +from pathlib import Path +from typing import Any, Mapping, Optional + +from .types import ( + MeshOpContext, + MeshOpExecutionError, + MeshOpResult, + MeshOpUnavailableError, +) + + +# Above this triangle count, suggest a decimation pass (slicers and the viewer +# both struggle well before the generators' multi-million-face outputs). +HIGH_FACE_COUNT = 500_000 +DECIMATE_TARGET_FACES = 200_000 +# Mirrors the repair op's max_hole_size bounds. +MAX_HOLE_SIZE_BOUNDS = (10, 10000) + + +def _trimesh(): + try: + import trimesh + except ImportError as exc: + raise MeshOpUnavailableError( + "inspect_mesh: trimesh is not available on this system" + ) from exc + return trimesh + + +def _scene_meshes(loaded, trimesh_module) -> list[tuple]: + """Return (geometry, world transform or None) for every non-empty triangle mesh.""" + if not isinstance(loaded, trimesh_module.Scene): + candidates = [(loaded, None)] + else: + candidates = [] + for node in loaded.graph.nodes_geometry: + transform, geometry_name = loaded.graph[node] + candidates.append((loaded.geometry.get(geometry_name), transform)) + return [ + (geometry, transform) + for geometry, transform in candidates + if isinstance(geometry, trimesh_module.Trimesh) and len(geometry.faces) + ] + + +def _has_uvs(geometry, trimesh_module) -> bool: + visual = geometry.visual + if not isinstance(visual, trimesh_module.visual.TextureVisuals): + return False + return visual.uv is not None and len(visual.uv) > 0 + + +def _gltf_has_normals(document: Mapping[str, Any]) -> bool: + primitives = [ + primitive + for mesh in document.get("meshes", []) + for primitive in mesh.get("primitives", []) + # Only triangle primitives (mode 4, the default) are inspected. + if primitive.get("mode", 4) == 4 + ] + return bool(primitives) and all( + "NORMAL" in primitive.get("attributes", {}) for primitive in primitives + ) + + +def _file_has_normals(input_path: Path) -> Optional[bool]: + """Whether the file stores vertex normals; None when the format is unknown. + + trimesh computes normals on demand, so its meshes always have them; the + file itself is the only reliable source. + """ + suffix = input_path.suffix.lower() + try: + if suffix == ".glb": + with open(input_path, "rb") as handle: + header = handle.read(20) + if len(header) < 20 or header[:4] != b"glTF": + return None + chunk_length = struct.unpack_from(" dict[str, Any]: + import numpy as np + + edges, counts = np.unique(merged.edges_sorted, axis=0, return_counts=True) + boundary = edges[counts == 1] + loops = ( + trimesh_module.graph.connected_components(boundary, min_len=1) + if len(boundary) + else [] + ) + return { + "boundary_edge_count": int(len(boundary)), + "non_manifold_edge_count": int(np.count_nonzero(counts > 2)), + "hole_count": int(len(loops)), + # A simple boundary loop has as many edges as vertices. + "largest_hole_edges": int(max((len(loop) for loop in loops), default=0)), + } + + +def _self_intersecting_faces(merged, context: MeshOpContext) -> Optional[int]: + try: + import pymeshlab + except ImportError: + context.log("Self-intersection check skipped: pymeshlab is unavailable") + return None + + mesh_set = pymeshlab.MeshSet() + mesh_set.add_mesh( + pymeshlab.Mesh(vertex_matrix=merged.vertices, face_matrix=merged.faces) + ) + try: + mesh_set.compute_selection_by_self_intersections_per_face() + except Exception as exc: + context.log(f"Self-intersection check skipped: {exc}") + return None + return int(mesh_set.current_mesh().selected_face_number()) + + +def _printable(report: Mapping[str, Any]) -> Optional[bool]: + """True/False when the verdict is certain; None when it hinges on an unchecked test.""" + if not ( + report["is_watertight"] + and report["is_winding_consistent"] + and report["is_manifold"] + ): + return False + if report["self_intersecting_face_count"] is None: + return None + return report["self_intersecting_face_count"] == 0 + + +def _issues(report: Mapping[str, Any]) -> list[str]: + issues = [] + if report["hole_count"]: + issues.append( + f"{report['hole_count']} hole(s) " + f"({report['boundary_edge_count']} open boundary edges)" + ) + if report["non_manifold_edge_count"]: + issues.append( + f"{report['non_manifold_edge_count']} non-manifold edge(s)" + ) + if not report["is_winding_consistent"]: + issues.append("inconsistent face winding (normals flipped on some faces)") + if report["self_intersecting_face_count"]: + issues.append( + f"{report['self_intersecting_face_count']} self-intersecting face(s)" + ) + if report["self_intersecting_face_count"] is None: + issues.append("self-intersections were not checked") + if report["degenerate_face_count"]: + issues.append(f"{report['degenerate_face_count']} degenerate face(s)") + if report["component_count"] > 1: + issues.append(f"{report['component_count']} disconnected parts") + return issues + + +def build_report( + input_path: Path, + context: MeshOpContext, + check_self_intersections: bool = True, +) -> dict[str, Any]: + """Diagnose a mesh file without modifying it; every value is JSON-safe.""" + trimesh = _trimesh() + + context.progress(10, "Loading mesh…") + loaded = trimesh.load(str(input_path), process=False) + parts = _scene_meshes(loaded, trimesh) + if not parts: + raise MeshOpExecutionError( + f"inspect_mesh: {input_path.name} contains no triangle mesh" + ) + + context.progress(30, "Analysing topology…") + merged = _merged_geometry(parts, trimesh) + topology = _edge_topology(merged, trimesh) + is_watertight = bool(merged.is_watertight) + is_winding_consistent = bool(merged.is_winding_consistent) + + self_intersections = None + if check_self_intersections: + context.progress(60, "Checking self-intersections…") + self_intersections = _self_intersecting_faces(merged, context) + + context.progress(90, "Measuring…") + bounds = merged.bounds + report: dict[str, Any] = { + "vertex_count": int(sum(len(geometry.vertices) for geometry, _ in parts)), + "face_count": int(sum(len(geometry.faces) for geometry, _ in parts)), + "component_count": int(merged.body_count), + "is_watertight": is_watertight, + "is_winding_consistent": is_winding_consistent, + "is_manifold": topology["non_manifold_edge_count"] == 0, + **topology, + "degenerate_face_count": int( + len(merged.faces) - int(merged.nondegenerate_faces().sum()) + ), + "self_intersecting_face_count": self_intersections, + "bounds": { + "min": [float(value) for value in bounds[0]], + "max": [float(value) for value in bounds[1]], + }, + "dimensions": [float(value) for value in merged.extents], + "surface_area": float(merged.area), + # The signed volume of an open or mis-wound mesh is meaningless. + "volume": ( + float(abs(merged.volume)) + if is_watertight and is_winding_consistent + else None + ), + "has_uvs": any(_has_uvs(geometry, trimesh) for geometry, _ in parts), + "has_normals": _file_has_normals(input_path), + } + report["printable"] = _printable(report) + report["issues"] = _issues(report) + context.log( + f"Inspected {report['face_count']} faces: " + + ("printable" if report["printable"] else "; ".join(report["issues"])) + ) + context.progress(100, "Done") + return report + + +def fixes_for_report(report: Mapping[str, Any]) -> tuple[list[dict], list[str]]: + """Derive registry op calls from a report, plus notes for unfixable issues.""" + fixes: list[dict] = [] + notes: list[str] = [] + + repair_reasons = [] + if report["hole_count"]: + repair_reasons.append(f"fill {report['hole_count']} hole(s)") + if report["non_manifold_edge_count"]: + repair_reasons.append("fix non-manifold edges") + if report["degenerate_face_count"]: + repair_reasons.append("remove degenerate faces") + if repair_reasons: + low, high = MAX_HOLE_SIZE_BOUNDS + fixes.append( + { + "op": "repair", + "params": { + "remove_duplicates": True, + "remove_degenerate": bool(report["degenerate_face_count"]), + "fix_non_manifold": bool(report["non_manifold_edge_count"]), + "fill_holes": bool(report["hole_count"]), + "max_hole_size": max( + low, min(high, report["largest_hole_edges"] or low) + ), + }, + "reason": "Repair to " + ", ".join(repair_reasons) + ".", + } + ) + if report["largest_hole_edges"] > high: + notes.append( + f"The largest hole has {report['largest_hole_edges']} boundary " + f"edges, above the repair limit of {high}; it may stay open." + ) + + if report["face_count"] > HIGH_FACE_COUNT: + fixes.append( + { + "op": "decimate", + "params": {"target_faces": DECIMATE_TARGET_FACES}, + "reason": ( + f"{report['face_count']} faces is heavy; reduce to " + f"{DECIMATE_TARGET_FACES}." + ), + } + ) + + if not report["is_winding_consistent"]: + notes.append("Inconsistent winding has no automatic fix yet.") + if report["self_intersecting_face_count"]: + notes.append("Self-intersections have no automatic fix yet.") + if report["component_count"] > 1: + notes.append( + f"{report['component_count']} disconnected parts; remove stray " + "fragments manually if they are not intended." + ) + return fixes, notes + + +def inspect_mesh( + input_path: Path, + params: Mapping[str, Any], + context: MeshOpContext, +) -> MeshOpResult: + report = build_report( + input_path, + context, + check_self_intersections=bool(params.get("check_self_intersections", True)), + ) + return MeshOpResult(details=report) + + +def suggest_fixes( + input_path: Path, + params: Mapping[str, Any], + context: MeshOpContext, +) -> MeshOpResult: + report = build_report( + input_path, + context, + check_self_intersections=bool(params.get("check_self_intersections", True)), + ) + fixes, notes = fixes_for_report(report) + return MeshOpResult(details={"report": report, "fixes": fixes, "notes": notes}) diff --git a/api/services/mesh_ops/processor.py b/api/services/mesh_ops/processor.py index 9af04353c..f31c5376d 100644 --- a/api/services/mesh_ops/processor.py +++ b/api/services/mesh_ops/processor.py @@ -56,6 +56,10 @@ def run_processor(operation_id: str, processor_id: str) -> None: data.get("params") or {}, context, ) + if result.file_path is None: + raise ValueError( + f"{processor_id}: {operation_id} produces no mesh file" + ) _emit( { "type": "done", diff --git a/api/services/mesh_ops/types.py b/api/services/mesh_ops/types.py index 5951bfe12..1a1929db6 100644 --- a/api/services/mesh_ops/types.py +++ b/api/services/mesh_ops/types.py @@ -44,9 +44,12 @@ def log(self, message: str) -> None: @dataclass(frozen=True) class MeshOpResult: - """The file produced by an operation and optional JSON-safe measurements.""" + """The file produced by an operation and optional JSON-safe measurements. - file_path: Path + Read-only (analysis) operations produce no file and leave file_path None. + """ + + file_path: Optional[Path] = None details: Mapping[str, Any] = field(default_factory=dict) diff --git a/api/tests/test_agent_router.py b/api/tests/test_agent_router.py index ecd2ed7c1..4de9cc07f 100644 --- a/api/tests/test_agent_router.py +++ b/api/tests/test_agent_router.py @@ -49,5 +49,74 @@ def handler(request: httpx.Request) -> httpx.Response: self.assertIsNone(payload) +class InspectMeshToolTests(unittest.TestCase): + REPORT = { + "printable": False, + "face_count": 11, + "vertex_count": 8, + "component_count": 1, + "is_watertight": False, + "is_manifold": True, + "is_winding_consistent": True, + "hole_count": 1, + "boundary_edge_count": 3, + "non_manifold_edge_count": 0, + "degenerate_face_count": 0, + "self_intersecting_face_count": None, + "dimensions": [1.0, 2.0, 0.5], + "surface_area": 5.5, + "volume": None, + "has_uvs": False, + "has_normals": True, + "issues": ["1 hole(s) (3 open boundary edges)"], + } + + def _run(self, name: str, response: dict) -> tuple[str, list]: + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=response) + + factory = _MockClientFactory(handler) + with mock.patch.object(agent.httpx, "AsyncClient", factory): + text, payload = asyncio.run( + agent.execute_tool(name, {"path": "Default/mesh.glb"}, {}) + ) + self.assertIsNone(payload) + return text, requests + + def test_inspect_mesh_calls_the_registry_op_and_summarises(self) -> None: + text, requests = self._run("inspect_mesh", self.REPORT) + + self.assertEqual(requests[0].url.path, "/optimize/op/inspect_mesh") + self.assertIn("Printable: no", text) + self.assertIn("Watertight: no", text) + self.assertIn("Self-intersecting faces: not checked", text) + self.assertIn("volume: n/a", text) + self.assertIn("Issues: 1 hole(s)", text) + + def test_suggest_fixes_lists_ops_and_notes(self) -> None: + text, requests = self._run( + "suggest_fixes", + { + "report": self.REPORT, + "fixes": [ + {"op": "repair", "params": {"fill_holes": True}, "reason": "Repair to fill 1 hole(s)."} + ], + "notes": ["2 disconnected parts"], + }, + ) + + self.assertEqual(requests[0].url.path, "/optimize/op/suggest_fixes") + self.assertIn('- repair {"fill_holes": true}', text) + self.assertIn("Note: 2 disconnected parts", text) + + def test_suggest_fixes_on_a_clean_mesh(self) -> None: + text, _ = self._run("suggest_fixes", {"report": {}, "fixes": [], "notes": []}) + + self.assertIn("No fixes needed", text) + + if __name__ == "__main__": unittest.main() diff --git a/api/tests/test_mesh_ops_inspection.py b/api/tests/test_mesh_ops_inspection.py new file mode 100644 index 000000000..f1ff4fc4c --- /dev/null +++ b/api/tests/test_mesh_ops_inspection.py @@ -0,0 +1,249 @@ +import hashlib +import importlib.util +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import trimesh + +from services.mesh_ops import MeshOpContext, MeshOpExecutionError, mesh_ops_registry +from services.mesh_ops.inspection import ( + DECIMATE_TARGET_FACES, + HIGH_FACE_COUNT, + fixes_for_report, +) + + +HAS_PYMESHLAB = importlib.util.find_spec("pymeshlab") is not None + + +def _box(offset=(0.0, 0.0, 0.0)) -> trimesh.Trimesh: + mesh = trimesh.creation.box() + mesh.apply_translation(offset) + return mesh + + +class InspectMeshTests(unittest.TestCase): + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.workspace = Path(self._directory.name) + self.context = MeshOpContext( + workspace_dir=self.workspace, + temp_dir=self.workspace, + ) + + def tearDown(self) -> None: + self._directory.cleanup() + + def _export(self, geometry, name="mesh.glb", **kwargs) -> Path: + path = self.workspace / name + geometry.export(str(path), **kwargs) + return path + + def _inspect(self, path: Path, **params) -> dict: + result = mesh_ops_registry.run("inspect_mesh", path, params, self.context) + self.assertIsNone(result.file_path) + return dict(result.details) + + def test_closed_box_is_printable(self) -> None: + path = self._export(_box(), include_normals=True) + + report = self._inspect(path) + + self.assertTrue(report["is_watertight"]) + self.assertTrue(report["is_winding_consistent"]) + self.assertTrue(report["is_manifold"]) + self.assertEqual(report["face_count"], 12) + self.assertEqual(report["component_count"], 1) + self.assertEqual(report["hole_count"], 0) + self.assertAlmostEqual(report["volume"], 1.0, places=6) + self.assertAlmostEqual(report["surface_area"], 6.0, places=6) + np.testing.assert_allclose(report["dimensions"], [1.0, 1.0, 1.0]) + self.assertTrue(report["has_normals"]) + self.assertFalse(report["has_uvs"]) + if HAS_PYMESHLAB: + self.assertEqual(report["self_intersecting_face_count"], 0) + self.assertTrue(report["printable"]) + self.assertEqual(report["issues"], []) + + def test_open_box_reports_its_hole_and_no_volume(self) -> None: + mesh = _box() + mesh.update_faces(np.arange(1, len(mesh.faces))) + path = self._export(mesh) + + report = self._inspect(path, check_self_intersections=False) + + self.assertFalse(report["is_watertight"]) + self.assertFalse(report["printable"]) + self.assertEqual(report["hole_count"], 1) + self.assertEqual(report["boundary_edge_count"], 3) + self.assertEqual(report["largest_hole_edges"], 3) + self.assertIsNone(report["volume"]) + self.assertIsNone(report["self_intersecting_face_count"]) + self.assertTrue(any("hole" in issue for issue in report["issues"])) + self.assertFalse(report["has_normals"]) + + def test_unchecked_self_intersections_leave_printability_unknown(self) -> None: + path = self._export(_box()) + + report = self._inspect(path, check_self_intersections=False) + + self.assertTrue(report["is_watertight"]) + self.assertIsNone(report["printable"]) + self.assertIn("self-intersections were not checked", report["issues"]) + + def test_normals_are_read_from_the_file_format(self) -> None: + cases = { + "normals.obj": (dict(include_normals=True), True), + "plain.obj": (dict(include_normals=False), False), + "normals.ply": (dict(vertex_normal=True), True), + "mesh.stl": ({}, False), + } + for name, (kwargs, expected) in cases.items(): + with self.subTest(name=name): + path = self._export(_box(), name=name, **kwargs) + report = self._inspect(path, check_self_intersections=False) + self.assertIs(report["has_normals"], expected) + + def test_uv_seams_do_not_count_as_holes(self) -> None: + mesh = _box() + mesh.unmerge_vertices() + uv = np.random.default_rng(0).random((len(mesh.vertices), 2)) + mesh.visual = trimesh.visual.TextureVisuals(uv=uv) + path = self._export(mesh) + + report = self._inspect(path, check_self_intersections=False) + + self.assertTrue(report["has_uvs"]) + self.assertEqual(report["vertex_count"], 36) + self.assertTrue(report["is_watertight"]) + self.assertEqual(report["hole_count"], 0) + + def test_scene_parts_are_counted_in_world_space(self) -> None: + scene = trimesh.Scene() + scene.add_geometry(_box(), node_name="a") + scene.add_geometry( + _box(), + node_name="b", + transform=trimesh.transformations.translation_matrix([3.0, 0.0, 0.0]), + ) + path = self._export(scene) + + report = self._inspect(path, check_self_intersections=False) + + self.assertEqual(report["component_count"], 2) + self.assertEqual(report["face_count"], 24) + np.testing.assert_allclose(report["dimensions"], [4.0, 1.0, 1.0]) + self.assertAlmostEqual(report["volume"], 2.0, places=6) + + @unittest.skipUnless(HAS_PYMESHLAB, "pymeshlab is not installed") + def test_overlapping_parts_are_self_intersecting(self) -> None: + mesh = trimesh.util.concatenate([_box(), _box((0.5, 0.3, 0.2))]) + path = self._export(mesh) + + report = self._inspect(path) + + self.assertGreater(report["self_intersecting_face_count"], 0) + self.assertFalse(report["printable"]) + + def test_inspection_never_modifies_the_input(self) -> None: + mesh = _box() + mesh.update_faces(np.arange(1, len(mesh.faces))) + path = self._export(mesh) + before = hashlib.sha256(path.read_bytes()).hexdigest() + mtime = path.stat().st_mtime_ns + + self._inspect(path) + mesh_ops_registry.run("suggest_fixes", path, {}, self.context) + + self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), before) + self.assertEqual(path.stat().st_mtime_ns, mtime) + self.assertEqual([p.name for p in self.workspace.iterdir()], ["mesh.glb"]) + + def test_file_without_triangles_is_rejected(self) -> None: + path = self._export(trimesh.PointCloud(np.zeros((3, 3))), name="cloud.ply") + + with self.assertRaises(MeshOpExecutionError): + self._inspect(path) + + def test_suggest_fixes_proposes_a_repair_for_holes(self) -> None: + mesh = _box() + mesh.update_faces(np.arange(1, len(mesh.faces))) + path = self._export(mesh) + + result = mesh_ops_registry.run( + "suggest_fixes", + path, + {"check_self_intersections": False}, + self.context, + ) + + self.assertIsNone(result.file_path) + self.assertEqual(result.details["report"]["hole_count"], 1) + fixes = result.details["fixes"] + self.assertEqual([fix["op"] for fix in fixes], ["repair"]) + self.assertTrue(fixes[0]["params"]["fill_holes"]) + self.assertEqual(fixes[0]["params"]["max_hole_size"], 10) + + +class FixesForReportTests(unittest.TestCase): + CLEAN = { + "face_count": 12, + "component_count": 1, + "is_winding_consistent": True, + "hole_count": 0, + "largest_hole_edges": 0, + "non_manifold_edge_count": 0, + "degenerate_face_count": 0, + "self_intersecting_face_count": 0, + } + + def test_clean_mesh_needs_nothing(self) -> None: + self.assertEqual(fixes_for_report(self.CLEAN), ([], [])) + + def test_suggested_params_are_valid_for_the_registry(self) -> None: + report = { + **self.CLEAN, + "face_count": HIGH_FACE_COUNT + 1, + "hole_count": 2, + "largest_hole_edges": 50_000, + "non_manifold_edge_count": 4, + "degenerate_face_count": 1, + } + + fixes, notes = fixes_for_report(report) + + self.assertEqual([fix["op"] for fix in fixes], ["repair", "decimate"]) + self.assertEqual(fixes[0]["params"]["max_hole_size"], 10000) + self.assertEqual(fixes[1]["params"], {"target_faces": DECIMATE_TARGET_FACES}) + self.assertTrue(any("repair limit" in note for note in notes)) + repair = mesh_ops_registry.get("repair") + declared = {schema["id"] for schema in repair.params_schema} + self.assertLessEqual(set(fixes[0]["params"]), declared) + + def test_problems_without_an_op_become_notes(self) -> None: + report = { + **self.CLEAN, + "is_winding_consistent": False, + "self_intersecting_face_count": 7, + "component_count": 3, + } + + fixes, notes = fixes_for_report(report) + + self.assertEqual(fixes, []) + self.assertEqual(len(notes), 3) + + +class InspectionRegistryTests(unittest.TestCase): + def test_analysis_ops_are_registered_as_read_only(self) -> None: + described = {op["id"]: op for op in mesh_ops_registry.describe()} + + for op_id in ("inspect_mesh", "suggest_fixes"): + self.assertEqual(described[op_id]["category"], "analysis") + self.assertFalse(described[op_id]["destructive"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_mesh_ops_registry.py b/api/tests/test_mesh_ops_registry.py index 31a777a20..8656f3d51 100644 --- a/api/tests/test_mesh_ops_registry.py +++ b/api/tests/test_mesh_ops_registry.py @@ -19,10 +19,10 @@ def test_builtin_metadata_is_serializable_and_complete(self) -> None: self.assertEqual( [description["id"] for description in descriptions], - ["repair", "decimate", "smooth"], + ["repair", "decimate", "smooth", "inspect_mesh", "suggest_fixes"], ) for description in descriptions: - self.assertIn(description["category"], {"repair", "optimization"}) + self.assertIn(description["category"], {"repair", "optimization", "analysis"}) self.assertFalse(description["destructive"]) self.assertTrue(description["undoable"]) self.assertIsInstance(description["params_schema"], list) diff --git a/api/tests/test_optimize_mesh_ops.py b/api/tests/test_optimize_mesh_ops.py index 6c702ddb6..c2c9d071c 100644 --- a/api/tests/test_optimize_mesh_ops.py +++ b/api/tests/test_optimize_mesh_ops.py @@ -105,6 +105,25 @@ def test_legacy_routes_delegate_with_their_existing_clamps_and_names(self) -> No self.assertEqual(optimize_response["face_count"], 42) self.assertEqual(smooth_response["url"], "/workspace/model_smooth20.glb") + def test_analysis_operation_returns_only_its_report(self) -> None: + class AnalysisRegistry: + def run(self, operation_id, input_path, params, context): + return MeshOpResult(details={"printable": False, "hole_count": 1}) + + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + (workspace / "input.glb").touch() + with ( + patch.object(optimize.registry, "WORKSPACE_DIR", workspace), + patch.object(optimize, "mesh_ops_registry", AnalysisRegistry()), + ): + response = optimize.run_mesh_operation( + "inspect_mesh", + optimize.MeshOpRequest(path="input.glb"), + ) + + self.assertEqual(response, {"printable": False, "hole_count": 1}) + def test_unknown_generic_operation_is_a_404(self) -> None: class MissingRegistry: def run(self, operation_id, input_path, params, context): From 7c273f95ddc555594f3ffe62156b953090535cc6 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 19 Oct 2026 10:41:36 +0200 Subject: [PATCH 04/15] feat(mesh-ops): add semantic transform ops and a one-call arrange endpoint Four registry ops derive their 4x4 matrix from the mesh itself: recenter (centroid to origin), drop_to_ground (lowest point to y = 0), auto_orient (smallest rotation that turns the thinnest PCA axis vertical; meshes without a clear flat side are left alone) and normalize_scale (size in m/cm/mm along the longest side or one axis). apply_transforms composes requested steps in a fixed order (orient, scale, recenter, ground) on one load and writes a single file, so grounding is never undone by a later step. POST /optimize/arrange exposes it, validating each step's params through the registry, and the raw /transform endpoint now shares the same bake helper. The agent and MCP server get an arrange_mesh tool that takes every step at once and reports the new mesh path for chaining. The world-space geometry helpers move to mesh_ops/geometry.py, shared with inspect_mesh, and the registry exposes resolve_params. Closes #297 --- api/mcp_server.py | 41 +++++ api/routers/agent.py | 71 ++++++++ api/routers/optimize.py | 79 ++++++--- api/services/mesh_ops/builtin.py | 68 +++++++ api/services/mesh_ops/geometry.py | 44 +++++ api/services/mesh_ops/inspection.py | 47 +---- api/services/mesh_ops/registry.py | 11 +- api/services/mesh_ops/transforms.py | 195 +++++++++++++++++++++ api/tests/test_agent_router.py | 50 ++++++ api/tests/test_mesh_ops_registry.py | 14 +- api/tests/test_mesh_ops_transforms.py | 214 +++++++++++++++++++++++ api/tests/test_mesh_routers_workspace.py | 34 ++++ 12 files changed, 800 insertions(+), 68 deletions(-) create mode 100644 api/services/mesh_ops/geometry.py create mode 100644 api/services/mesh_ops/transforms.py create mode 100644 api/tests/test_mesh_ops_transforms.py diff --git a/api/mcp_server.py b/api/mcp_server.py index 3c7968f6c..38ed65f1b 100644 --- a/api/mcp_server.py +++ b/api/mcp_server.py @@ -115,6 +115,34 @@ async def list_tools() -> list[Tool]: "required": ["path"], }, ), + Tool( + name="modly_arrange_mesh", + description=( + "Reposition a mesh in one call: auto_orient (lay flat), normalize_scale " + "(resize), recenter (centre on origin), drop_to_ground (rest on ground). " + "Steps always run in that order. Writes a new mesh file." + ), + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + "steps": { + "type": "array", + "items": { + "type": "string", + "enum": ["auto_orient", "normalize_scale", "recenter", "drop_to_ground"], + }, + }, + "size": {"type": "number", "description": "normalize_scale: target size in `unit`."}, + "unit": {"type": "string", "enum": ["m", "cm", "mm"]}, + "dimension": {"type": "string", "enum": ["longest", "x", "y", "z"]}, + }, + "required": ["path", "steps"], + }, + ), Tool( name="modly_decimate_mesh", description="Reduce the polygon count of a mesh using quadric edge collapse decimation.", @@ -255,6 +283,19 @@ async def _dispatch(client: httpx.AsyncClient, name: str, args: dict) -> str: r.raise_for_status() return json.dumps(r.json(), indent=2) + elif name == "modly_arrange_mesh": + scale = {k: args[k] for k in ("size", "unit", "dimension") if args.get(k) is not None} + r = await client.post( + f"{API_BASE}/optimize/arrange", + json={ + "path": args["path"], + "steps": args.get("steps") or [], + "params": {"normalize_scale": scale} if scale else {}, + }, + ) + r.raise_for_status() + return json.dumps(r.json(), indent=2) + elif name == "modly_decimate_mesh": r = await client.post( f"{API_BASE}/optimize/mesh", diff --git a/api/routers/agent.py b/api/routers/agent.py index 6a563b9b0..9d6134d98 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -30,6 +30,7 @@ - **get_mesh_info** — Get info about the current mesh in the 3D viewer (path, triangle count). - **inspect_mesh(path)** — Diagnose a mesh without modifying it: watertight, manifold, holes, self-intersections, parts, dimensions, volume, UVs/normals, and whether it is 3D-printable. - **suggest_fixes(path)** — Inspect a mesh and list the operations (with params) that would fix its problems. +- **arrange_mesh(path, steps, size?, unit?, dimension?)** — Reposition a mesh in one call. Steps: `auto_orient` (lay it flat on its widest side), `normalize_scale` (resize to `size` in `unit`), `recenter` (centre on the origin), `drop_to_ground` (rest it on the ground). Steps always run in that order, whatever order you list them in. - **decimate_mesh(path, target_faces)** — Reduce the polygon count of a mesh. - **smooth_mesh(path, iterations)** — Apply Laplacian smoothing to a mesh. - **get_generation_status(job_id)** — Poll the status of an ongoing 3D generation job. @@ -41,6 +42,7 @@ - Always use tools to act on the scene — never just describe what you would do. - If you need the current mesh path, call get_mesh_info first. +- To center, ground, lay flat or resize a mesh, call arrange_mesh ONCE with every requested step — never one call per step. - To answer questions about a mesh's quality or printability, call inspect_mesh — never guess. Before repairing a mesh, call suggest_fixes to pick the right operations. - If you need to run a workflow but don't know the ID, call list_workflows first. - To create a workflow, ONLY use extension ids listed under "Available extensions" in the context. Never invent an id. Chain steps so each step's input type matches the previous step's output type. @@ -109,6 +111,53 @@ }, }, }, + { + "type": "function", + "function": { + "name": "arrange_mesh", + "description": ( + "Reposition a mesh in a single call: lay it flat, resize it, centre it and/or " + "rest it on the ground. Pass every requested step at once; they always run in " + "the order auto_orient → normalize_scale → recenter → drop_to_ground." + ), + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file. Use get_mesh_info to obtain it.", + }, + "steps": { + "type": "array", + "items": { + "type": "string", + "enum": ["auto_orient", "normalize_scale", "recenter", "drop_to_ground"], + }, + "description": ( + "auto_orient: lay flat on its widest side (a standing figure is laid down). " + "normalize_scale: resize to `size`. recenter: centre on the origin. " + "drop_to_ground: lowest point at height 0." + ), + }, + "size": { + "type": "number", + "description": "normalize_scale only: target size of `dimension`, in `unit`.", + }, + "unit": { + "type": "string", + "enum": ["m", "cm", "mm"], + "description": "normalize_scale only: unit of `size`. Default m.", + }, + "dimension": { + "type": "string", + "enum": ["longest", "x", "y", "z"], + "description": "normalize_scale only: which side gets `size` (y is the height). Default longest.", + }, + }, + "required": ["path", "steps"], + }, + }, + }, { "type": "function", "function": { @@ -378,6 +427,28 @@ async def execute_tool(name: str, arguments: dict, context: dict) -> tuple[str, r.raise_for_status() return _format_fixes(r.json()), None + elif name == "arrange_mesh": + scale_params = { + key: arguments[key] for key in ("size", "unit", "dimension") + if arguments.get(key) is not None + } + r = await client.post( + f"{MODLY_API}/optimize/arrange", + json={ + "path": arguments["path"], + "steps": arguments.get("steps") or [], + "params": {"normalize_scale": scale_params} if scale_params else {}, + }, + ) + r.raise_for_status() + data = r.json() + dims = " × ".join(f"{d:.4g}" for d in data.get("dimensions", [])) + text = ( + f"Applied {' → '.join(data.get('applied', []))}. " + f"New mesh path: {data['path']}. Dimensions (x × y × z, metres): {dims}." + ) + return text, {"type": "mesh_update", "url": data["url"]} + elif name == "decimate_mesh": r = await client.post( f"{MODLY_API}/optimize/mesh", diff --git a/api/routers/optimize.py b/api/routers/optimize.py index bcfb9f6b0..01dbf47ed 100644 --- a/api/routers/optimize.py +++ b/api/routers/optimize.py @@ -2,6 +2,7 @@ import os import tempfile import uuid +from contextlib import contextmanager import numpy as np import trimesh @@ -23,6 +24,7 @@ MeshOpUnavailableError, mesh_ops_registry, ) +from services.mesh_ops.transforms import apply_transforms, transform_file router = APIRouter(tags=["optimize"]) @@ -47,6 +49,12 @@ class TransformRequest(BaseModel): matrix: list[list[float]] # row-major 4x4 world transform +class ArrangeRequest(BaseModel): + path: str # format: "{collection}/{filename}" + steps: list[str] # transform op ids, applied in canonical order + params: dict[str, dict[str, object]] = Field(default_factory=dict) # per step id + + def _resolve_input_path(raw_path: str) -> Path: candidate = Path(raw_path) if candidate.is_absolute(): @@ -75,23 +83,25 @@ def _operation_output_path(input_path: Path, output_name: str) -> Path: return output_dir / output_name -def _run_operation( - operation_id: str, - input_path: Path, - params: dict[str, object], +def _operation_context( output_path: Path | None = None, preserve_visuals: bool = False, -) -> MeshOpResult: - context = MeshOpContext( +) -> MeshOpContext: + return MeshOpContext( workspace_dir=registry.WORKSPACE_DIR, temp_dir=Path(tempfile.gettempdir()), output_path=output_path, preserve_visuals=preserve_visuals, ) + + +@contextmanager +def _mesh_op_http_errors(unknown_status: int = 404): + """Translate mesh-op failures into HTTP errors.""" try: - return mesh_ops_registry.run(operation_id, input_path, params, context) + yield except MeshOpNotFoundError as exc: - raise HTTPException(404, f"Unknown mesh operation: {operation_id}") from exc + raise HTTPException(unknown_status, f"Unknown mesh operation: {exc}") from exc except FileNotFoundError as exc: raise HTTPException(404, str(exc)) from exc except (MeshOpUnavailableError, MeshOpExecutionError) as exc: @@ -100,6 +110,18 @@ def _run_operation( raise HTTPException(400, str(exc)) from exc +def _run_operation( + operation_id: str, + input_path: Path, + params: dict[str, object], + output_path: Path | None = None, + preserve_visuals: bool = False, +) -> MeshOpResult: + context = _operation_context(output_path, preserve_visuals) + with _mesh_op_http_errors(): + return mesh_ops_registry.run(operation_id, input_path, params, context) + + def _operation_response(result: MeshOpResult) -> dict[str, object]: if result.file_path is None: # Analysis ops (inspect_mesh, suggest_fixes) return only their report. @@ -186,23 +208,38 @@ def transform_mesh(body: TransformRequest): if not np.all(np.isfinite(matrix)): raise HTTPException(400, "matrix contains non-finite values") - # Keep the loaded result as-is (Scene when textured/multi-geometry) so - # apply_transform preserves materials and UVs. - loaded = trimesh.load(str(input_path)) - loaded.apply_transform(matrix) + output_path = transform_file(input_path, matrix, _transform_output_path(input_path)) - stem = input_path.stem - output_name = f"{stem}_xf_{uuid.uuid4().hex[:8]}.glb" - workspace = registry.WORKSPACE_DIR.resolve() - output_dir = input_path.parent if registry.is_within_workspace(input_path.resolve()) else workspace / "Workflows" - output_dir.mkdir(parents=True, exist_ok=True) - output_path = output_dir / output_name - loaded.export(str(output_path)) - - rel = output_path.resolve().relative_to(workspace).as_posix() + rel = output_path.resolve().relative_to(registry.WORKSPACE_DIR.resolve()).as_posix() return {"url": f"/workspace/{rel}"} +@router.post("/arrange") +def arrange_mesh(body: ArrangeRequest): + # Semantic transforms (recenter, drop_to_ground, auto_orient, + # normalize_scale) composed into one file, in canonical order. + input_path = _resolve_input_path(body.path) + with _mesh_op_http_errors(unknown_status=400): + steps = [ + (step_id, mesh_ops_registry.resolve_params(step_id, body.params.get(step_id))) + for step_id in body.steps + ] + result = apply_transforms( + input_path, + steps, + _transform_output_path(input_path), + _operation_context(), + ) + return _operation_response(result) + + +def _transform_output_path(input_path: Path) -> Path: + return _operation_output_path( + input_path, + f"{input_path.stem}_xf_{uuid.uuid4().hex[:8]}.glb", + ) + + class ImportByPathRequest(BaseModel): path: str # absolute path on disk diff --git a/api/services/mesh_ops/builtin.py b/api/services/mesh_ops/builtin.py index 75574fe41..67cd73bc7 100644 --- a/api/services/mesh_ops/builtin.py +++ b/api/services/mesh_ops/builtin.py @@ -2,6 +2,7 @@ from .inspection import inspect_mesh, suggest_fixes from .operations import decimate_mesh, repair_mesh, smooth_mesh +from .transforms import auto_orient, drop_to_ground, normalize_scale, recenter from .types import MeshOp @@ -120,6 +121,45 @@ ) +NORMALIZE_SCALE_PARAMS = ( + { + "id": "size", + "label": "Size", + "type": "float", + "default": 1.0, + "min": 0.001, + "max": 100000.0, + "step": 0.1, + "tooltip": "Target size of the measured dimension, in the chosen unit.", + }, + { + "id": "unit", + "label": "Unit", + "type": "select", + "default": "m", + "options": [ + {"value": "m", "label": "Metres"}, + {"value": "cm", "label": "Centimetres"}, + {"value": "mm", "label": "Millimetres"}, + ], + "tooltip": "glTF stores metres: 20 cm is written as 0.2 units.", + }, + { + "id": "dimension", + "label": "Dimension", + "type": "select", + "default": "longest", + "options": [ + {"value": "longest", "label": "Longest side"}, + {"value": "x", "label": "Width (X)"}, + {"value": "y", "label": "Height (Y)"}, + {"value": "z", "label": "Depth (Z)"}, + ], + "tooltip": "Which bounding-box dimension is set to the target size.", + }, +) + + BUILTIN_MESH_OPS = ( MeshOp( id="repair", @@ -156,4 +196,32 @@ fn=suggest_fixes, category="analysis", ), + MeshOp( + id="recenter", + label="Recenter", + params_schema=(), + fn=recenter, + category="transform", + ), + MeshOp( + id="drop_to_ground", + label="Drop to Ground", + params_schema=(), + fn=drop_to_ground, + category="transform", + ), + MeshOp( + id="auto_orient", + label="Lay Flat", + params_schema=(), + fn=auto_orient, + category="transform", + ), + MeshOp( + id="normalize_scale", + label="Normalize Scale", + params_schema=NORMALIZE_SCALE_PARAMS, + fn=normalize_scale, + category="transform", + ), ) diff --git a/api/services/mesh_ops/geometry.py b/api/services/mesh_ops/geometry.py new file mode 100644 index 000000000..3fa3c60f3 --- /dev/null +++ b/api/services/mesh_ops/geometry.py @@ -0,0 +1,44 @@ +"""World-space geometry helpers shared by the analysis and transform ops.""" + + +def scene_meshes(loaded, trimesh_module) -> list[tuple]: + """Return (geometry, world transform or None) for every non-empty triangle mesh.""" + if not isinstance(loaded, trimesh_module.Scene): + candidates = [(loaded, None)] + else: + candidates = [] + for node in loaded.graph.nodes_geometry: + transform, geometry_name = loaded.graph[node] + candidates.append((loaded.geometry.get(geometry_name), transform)) + return [ + (geometry, transform) + for geometry, transform in candidates + if isinstance(geometry, trimesh_module.Trimesh) and len(geometry.faces) + ] + + +def merged_geometry(parts, trimesh_module, process: bool = True): + """Concatenate the parts into one world-space mesh, without visuals. + + With process (the default), vertices are merged by position: UV seams split + vertices, and without the merge every textured mesh would look open along + its seams. Pure measurements (bounds, centroid) can skip it. + """ + import numpy as np + + vertices = [] + faces = [] + offset = 0 + for geometry, transform in parts: + points = np.asarray(geometry.vertices, dtype=float) + if transform is not None: + points = trimesh_module.transformations.transform_points(points, transform) + vertices.append(points) + faces.append(np.asarray(geometry.faces, dtype=np.int64) + offset) + offset += len(points) + + return trimesh_module.Trimesh( + vertices=np.vstack(vertices), + faces=np.vstack(faces), + process=process, + ) diff --git a/api/services/mesh_ops/inspection.py b/api/services/mesh_ops/inspection.py index a8dfc3f14..9f4ebe9e4 100644 --- a/api/services/mesh_ops/inspection.py +++ b/api/services/mesh_ops/inspection.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any, Mapping, Optional +from .geometry import merged_geometry, scene_meshes from .types import ( MeshOpContext, MeshOpExecutionError, @@ -31,22 +32,6 @@ def _trimesh(): return trimesh -def _scene_meshes(loaded, trimesh_module) -> list[tuple]: - """Return (geometry, world transform or None) for every non-empty triangle mesh.""" - if not isinstance(loaded, trimesh_module.Scene): - candidates = [(loaded, None)] - else: - candidates = [] - for node in loaded.graph.nodes_geometry: - transform, geometry_name = loaded.graph[node] - candidates.append((loaded.geometry.get(geometry_name), transform)) - return [ - (geometry, transform) - for geometry, transform in candidates - if isinstance(geometry, trimesh_module.Trimesh) and len(geometry.faces) - ] - - def _has_uvs(geometry, trimesh_module) -> bool: visual = geometry.visual if not isinstance(visual, trimesh_module.visual.TextureVisuals): @@ -103,32 +88,6 @@ def _file_has_normals(input_path: Path) -> Optional[bool]: return None -def _merged_geometry(parts, trimesh_module): - """Concatenate the parts into one position-merged mesh for topology checks. - - Visuals are dropped on purpose: UV seams split vertices, and without merging - by position every textured mesh would look open along its seams. - """ - import numpy as np - - vertices = [] - faces = [] - offset = 0 - for geometry, transform in parts: - points = np.asarray(geometry.vertices, dtype=float) - if transform is not None: - points = trimesh_module.transformations.transform_points(points, transform) - vertices.append(points) - faces.append(np.asarray(geometry.faces, dtype=np.int64) + offset) - offset += len(points) - - return trimesh_module.Trimesh( - vertices=np.vstack(vertices), - faces=np.vstack(faces), - process=True, - ) - - def _edge_topology(merged, trimesh_module) -> dict[str, Any]: import numpy as np @@ -216,14 +175,14 @@ def build_report( context.progress(10, "Loading mesh…") loaded = trimesh.load(str(input_path), process=False) - parts = _scene_meshes(loaded, trimesh) + parts = scene_meshes(loaded, trimesh) if not parts: raise MeshOpExecutionError( f"inspect_mesh: {input_path.name} contains no triangle mesh" ) context.progress(30, "Analysing topology…") - merged = _merged_geometry(parts, trimesh) + merged = merged_geometry(parts, trimesh) topology = _edge_topology(merged, trimesh) is_watertight = bool(merged.is_watertight) is_winding_consistent = bool(merged.is_winding_consistent) diff --git a/api/services/mesh_ops/registry.py b/api/services/mesh_ops/registry.py index 5d7553400..92c226df3 100644 --- a/api/services/mesh_ops/registry.py +++ b/api/services/mesh_ops/registry.py @@ -95,6 +95,15 @@ def run( if not path.is_file(): raise FileNotFoundError(f"Input mesh not found: {path}") + return operation.fn(path, self.resolve_params(operation_id, params), context) + + def resolve_params( + self, + operation_id: str, + params: Optional[Mapping[str, Any]], + ) -> dict[str, Any]: + """Apply an operation's schema defaults, then validate and clamp params.""" + operation = self.get(operation_id) resolved_params = { schema["id"]: deepcopy(schema["default"]) for schema in operation.params_schema @@ -113,4 +122,4 @@ def run( schema, ) - return operation.fn(path, resolved_params, context) + return resolved_params diff --git a/api/services/mesh_ops/transforms.py b/api/services/mesh_ops/transforms.py new file mode 100644 index 000000000..12266f115 --- /dev/null +++ b/api/services/mesh_ops/transforms.py @@ -0,0 +1,195 @@ +"""Semantic transforms: derive a 4x4 matrix from the mesh itself, then bake it.""" + +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .geometry import merged_geometry, scene_meshes +from .operations import _output_path +from .types import ( + MeshOpContext, + MeshOpExecutionError, + MeshOpResult, + MeshOpUnavailableError, +) + + +# glTF's unit is the metre. +UNIT_SCALE = {"m": 1.0, "cm": 0.01, "mm": 0.001} +DIMENSION_AXES = {"x": 0, "y": 1, "z": 2} +# Steps always run in this order, whatever order they were requested in: +# orienting or scaling after grounding would lift the mesh off the ground again. +CANONICAL_ORDER = ("auto_orient", "normalize_scale", "recenter", "drop_to_ground") +# auto_orient only acts when the thinnest PCA extent is clearly thinner than +# the next one; a cube or a sphere has no flat side to lay down on. +FLATNESS_RATIO = 0.8 + + +def _trimesh(): + try: + import trimesh + except ImportError as exc: + raise MeshOpUnavailableError( + "mesh transforms: trimesh is not available on this system" + ) from exc + return trimesh + + +def _about(pivot, matrix, trimesh_module): + """Conjugate a matrix so it acts around pivot instead of the origin.""" + translate = trimesh_module.transformations.translation_matrix + return translate(pivot) @ matrix @ translate(-pivot) + + +def _recenter(mesh, params, trimesh_module): + return trimesh_module.transformations.translation_matrix(-mesh.centroid) + + +def _drop_to_ground(mesh, params, trimesh_module): + return trimesh_module.transformations.translation_matrix( + [0.0, -mesh.bounds[0][1], 0.0] + ) + + +def _auto_orient(mesh, params, trimesh_module): + """Smallest rotation that turns the mesh's thinnest axis (PCA) vertical.""" + import numpy as np + + weights = mesh.area_faces + if weights.sum() <= 0: + return np.eye(4) + centers = mesh.triangles_center + offsets = centers - np.average(centers, axis=0, weights=weights) + covariance = (offsets * weights[:, None]).T @ offsets / weights.sum() + # eigh returns ascending eigenvalues: column 0 is the thinnest direction. + variances, axes = np.linalg.eigh(covariance) + spreads = np.sqrt(np.clip(variances, 0.0, None)) + if spreads[1] <= 0 or spreads[0] >= FLATNESS_RATIO * spreads[1]: + return np.eye(4) + + thin_axis = axes[:, 0] + # Of the two directions along the axis, take the one already pointing up so + # the rotation stays under 90° and the mesh is not flipped over. + if thin_axis[1] < 0: + thin_axis = -thin_axis + rotation = trimesh_module.geometry.align_vectors(thin_axis, [0.0, 1.0, 0.0]) + return _about(mesh.centroid, rotation, trimesh_module) + + +def _normalize_scale(mesh, params, trimesh_module): + target = float(params["size"]) * UNIT_SCALE[params["unit"]] + extents = mesh.extents + dimension = params["dimension"] + current = ( + float(max(extents)) + if dimension == "longest" + else float(extents[DIMENSION_AXES[dimension]]) + ) + if current <= 0: + raise MeshOpExecutionError( + f"normalize_scale: the mesh has no extent along {dimension}" + ) + # Scaling around the origin keeps a recentered or grounded mesh so. + return trimesh_module.transformations.scale_matrix(target / current) + + +STEP_MATRICES = { + "recenter": _recenter, + "drop_to_ground": _drop_to_ground, + "auto_orient": _auto_orient, + "normalize_scale": _normalize_scale, +} + + +def _world_mesh(loaded, trimesh_module): + parts = scene_meshes(loaded, trimesh_module) + if not parts: + raise MeshOpExecutionError("mesh transforms: the file contains no triangle mesh") + # Measurements only: no need to pay for a vertex merge. + return merged_geometry(parts, trimesh_module, process=False) + + +def transform_file(input_path: Path, matrix, output_path: Path) -> Path: + """Bake a 4x4 world transform into a mesh file, keeping materials and UVs.""" + trimesh = _trimesh() + # Keep the loaded result as-is (Scene when textured/multi-geometry) so + # apply_transform preserves materials and UVs. + loaded = trimesh.load(str(input_path)) + loaded.apply_transform(matrix) + output_path.parent.mkdir(parents=True, exist_ok=True) + loaded.export(str(output_path)) + return output_path + + +def apply_transforms( + input_path: Path, + steps: Sequence[tuple[str, Mapping[str, Any]]], + output_path: Path, + context: MeshOpContext, +) -> MeshOpResult: + """Apply semantic steps in canonical order and write a single output file. + + Each step measures the mesh as the previous steps left it, so "lay it flat, + centre it and put it on the ground" yields one consistent result. + """ + trimesh = _trimesh() + unknown = [step_id for step_id, _ in steps if step_id not in STEP_MATRICES] + if unknown: + raise ValueError(f"Unknown transform step(s): {', '.join(unknown)}") + if not steps: + raise ValueError("At least one transform step is required") + requested = dict(steps) + ordered = [step_id for step_id in CANONICAL_ORDER if step_id in requested] + + context.progress(10, "Loading mesh…") + loaded = trimesh.load(str(input_path)) + + for index, step_id in enumerate(ordered): + context.progress(20 + 60 * index // len(ordered), f"Applying {step_id}…") + matrix = STEP_MATRICES[step_id]( + _world_mesh(loaded, trimesh), + requested[step_id], + trimesh, + ) + loaded.apply_transform(matrix) + + context.progress(85, "Exporting…") + output_path.parent.mkdir(parents=True, exist_ok=True) + loaded.export(str(output_path)) + + result = _world_mesh(loaded, trimesh) + context.log(f"Applied {', '.join(ordered)} → {output_path}") + context.progress(100, "Done") + return MeshOpResult( + file_path=output_path, + details={ + "applied": ordered, + "dimensions": [float(value) for value in result.extents], + "bounds": { + "min": [float(value) for value in result.bounds[0]], + "max": [float(value) for value in result.bounds[1]], + }, + }, + ) + + +def _single_step_op(step_id: str): + def run( + input_path: Path, + params: Mapping[str, Any], + context: MeshOpContext, + ) -> MeshOpResult: + return apply_transforms( + input_path, + [(step_id, params)], + _output_path(context, step_id.replace("_", "-")), + context, + ) + + run.__name__ = step_id + return run + + +recenter = _single_step_op("recenter") +drop_to_ground = _single_step_op("drop_to_ground") +auto_orient = _single_step_op("auto_orient") +normalize_scale = _single_step_op("normalize_scale") diff --git a/api/tests/test_agent_router.py b/api/tests/test_agent_router.py index 4de9cc07f..2f21bc0f8 100644 --- a/api/tests/test_agent_router.py +++ b/api/tests/test_agent_router.py @@ -1,4 +1,5 @@ import asyncio +import json import unittest from unittest import mock @@ -118,5 +119,54 @@ def test_suggest_fixes_on_a_clean_mesh(self) -> None: self.assertIn("No fixes needed", text) +class ArrangeMeshToolTests(unittest.TestCase): + def _run(self, arguments: dict) -> tuple[str, object, list]: + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json={ + "path": "Default/mesh_xf_1234abcd.glb", + "url": "/workspace/Default/mesh_xf_1234abcd.glb", + "applied": ["auto_orient", "recenter", "drop_to_ground"], + "dimensions": [0.2, 0.05, 0.1], + }) + + factory = _MockClientFactory(handler) + with mock.patch.object(agent.httpx, "AsyncClient", factory): + text, payload = asyncio.run(agent.execute_tool("arrange_mesh", arguments, {})) + return text, payload, requests + + def test_all_steps_go_in_one_request_and_the_new_path_is_reported(self) -> None: + text, payload, requests = self._run({ + "path": "Default/mesh.glb", + "steps": ["drop_to_ground", "recenter", "auto_orient"], + }) + + self.assertEqual(len(requests), 1) + self.assertEqual(requests[0].url.path, "/optimize/arrange") + self.assertEqual(json.loads(requests[0].content), { + "path": "Default/mesh.glb", + "steps": ["drop_to_ground", "recenter", "auto_orient"], + "params": {}, + }) + # The path lets the model chain another tool on the result. + self.assertIn("New mesh path: Default/mesh_xf_1234abcd.glb", text) + self.assertEqual(payload, {"type": "mesh_update", "url": "/workspace/Default/mesh_xf_1234abcd.glb"}) + + def test_scale_arguments_become_normalize_scale_params(self) -> None: + _, _, requests = self._run({ + "path": "Default/mesh.glb", + "steps": ["normalize_scale"], + "size": 20, + "unit": "cm", + }) + + self.assertEqual( + json.loads(requests[0].content)["params"], + {"normalize_scale": {"size": 20, "unit": "cm"}}, + ) + + if __name__ == "__main__": unittest.main() diff --git a/api/tests/test_mesh_ops_registry.py b/api/tests/test_mesh_ops_registry.py index 8656f3d51..8bde5a366 100644 --- a/api/tests/test_mesh_ops_registry.py +++ b/api/tests/test_mesh_ops_registry.py @@ -19,10 +19,20 @@ def test_builtin_metadata_is_serializable_and_complete(self) -> None: self.assertEqual( [description["id"] for description in descriptions], - ["repair", "decimate", "smooth", "inspect_mesh", "suggest_fixes"], + [ + "repair", + "decimate", + "smooth", + "inspect_mesh", + "suggest_fixes", + "recenter", + "drop_to_ground", + "auto_orient", + "normalize_scale", + ], ) for description in descriptions: - self.assertIn(description["category"], {"repair", "optimization", "analysis"}) + self.assertIn(description["category"], {"repair", "optimization", "analysis", "transform"}) self.assertFalse(description["destructive"]) self.assertTrue(description["undoable"]) self.assertIsInstance(description["params_schema"], list) diff --git a/api/tests/test_mesh_ops_transforms.py b/api/tests/test_mesh_ops_transforms.py new file mode 100644 index 000000000..21407209b --- /dev/null +++ b/api/tests/test_mesh_ops_transforms.py @@ -0,0 +1,214 @@ +import hashlib +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import trimesh + +from services.mesh_ops import MeshOpContext, mesh_ops_registry +from services.mesh_ops.transforms import apply_transforms + + +def _tilted_plate() -> trimesh.Trimesh: + """A 2 × 0.2 × 1 plate, tilted 30° about Z and moved off the origin.""" + plate = trimesh.creation.box(extents=[2.0, 0.2, 1.0]) + plate.apply_transform( + trimesh.transformations.rotation_matrix(np.radians(30), [0, 0, 1]) + ) + plate.apply_translation([3.0, 4.0, 5.0]) + return plate + + +def _world_bounds(path: Path) -> np.ndarray: + return trimesh.load(str(path)).bounds + + +class ApplyTransformsTests(unittest.TestCase): + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.workspace = Path(self._directory.name) + self.context = MeshOpContext( + workspace_dir=self.workspace, + temp_dir=self.workspace, + ) + + def tearDown(self) -> None: + self._directory.cleanup() + + def _export(self, geometry, name="mesh.glb") -> Path: + path = self.workspace / name + geometry.export(str(path)) + return path + + def _apply(self, path: Path, *step_ids: str, **params) -> tuple[Path, dict]: + steps = [ + (step_id, mesh_ops_registry.resolve_params(step_id, params.get(step_id))) + for step_id in step_ids + ] + output = self.workspace / "out" / f"{len(list(self.workspace.rglob('*')))}.glb" + result = apply_transforms(path, steps, output, self.context) + self.assertEqual(result.file_path, output) + return output, dict(result.details) + + def test_ground_center_and_lay_flat_in_one_pass_whatever_the_order(self) -> None: + path = self._export(_tilted_plate()) + + output, details = self._apply(path, "drop_to_ground", "recenter", "auto_orient") + + self.assertEqual(details["applied"], ["auto_orient", "recenter", "drop_to_ground"]) + bounds = _world_bounds(output) + # Laid flat: the 0.2 thickness is now the height, and the long side + # keeps its heading (no spin about the vertical axis). + np.testing.assert_allclose(bounds[1] - bounds[0], [2.0, 0.2, 1.0], atol=1e-5) + self.assertAlmostEqual(bounds[0][1], 0.0, places=5) + np.testing.assert_allclose((bounds[0] + bounds[1])[[0, 2]] / 2, [0, 0], atol=1e-5) + np.testing.assert_allclose(details["dimensions"], [2.0, 0.2, 1.0], atol=1e-5) + + def test_recenter_moves_the_centroid_to_the_origin(self) -> None: + path = self._export(_tilted_plate()) + + output, _ = self._apply(path, "recenter") + + centroid = trimesh.load(str(output), force="mesh").centroid + np.testing.assert_allclose(centroid, [0, 0, 0], atol=1e-5) + + def test_drop_to_ground_only_moves_vertically(self) -> None: + path = self._export(_tilted_plate()) + before = _world_bounds(path) + + output, _ = self._apply(path, "drop_to_ground") + + after = _world_bounds(output) + self.assertAlmostEqual(after[0][1], 0.0, places=5) + np.testing.assert_allclose(after[:, [0, 2]], before[:, [0, 2]], atol=1e-5) + + def test_auto_orient_leaves_a_mesh_without_a_flat_side_alone(self) -> None: + cube = trimesh.creation.box() + cube.apply_transform( + trimesh.transformations.rotation_matrix(np.radians(20), [1, 1, 0]) + ) + path = self._export(cube) + + output, _ = self._apply(path, "auto_orient") + + np.testing.assert_allclose(_world_bounds(output), _world_bounds(path), atol=1e-5) + + def test_auto_orient_does_not_flip_a_mesh_over(self) -> None: + # A shallow bowl (open hemisphere-like dish) opening upwards. + bowl = trimesh.creation.icosphere(subdivisions=3) + bowl.update_faces(bowl.triangles_center[:, 1] < -0.3) + bowl.apply_scale([1.0, 0.3, 1.0]) + bowl.apply_transform( + trimesh.transformations.rotation_matrix(np.radians(10), [1, 0, 0]) + ) + path = self._export(bowl) + + output, _ = self._apply(path, "auto_orient") + + mesh = trimesh.load(str(output), force="mesh") + # Still opening upwards: the rim is above the bottom of the dish. + rim_height = mesh.vertices[:, 1].max() + lowest = mesh.vertices[:, 1].min() + center_height = mesh.vertices[np.argmin(np.linalg.norm(mesh.vertices[:, [0, 2]], axis=1)), 1] + self.assertLess(center_height - lowest, rim_height - center_height) + + def test_normalize_scale_to_real_units(self) -> None: + path = self._export(_tilted_plate()) + + _, longest = self._apply(path, "normalize_scale", normalize_scale={"size": 20, "unit": "cm"}) + _, height = self._apply( + path, + "normalize_scale", + normalize_scale={"size": 5, "unit": "mm", "dimension": "y"}, + ) + + self.assertAlmostEqual(max(longest["dimensions"]), 0.2, places=5) + self.assertAlmostEqual(height["dimensions"][1], 0.005, places=6) + + def test_scaling_keeps_a_grounded_mesh_on_the_ground(self) -> None: + path = self._export(_tilted_plate()) + + output, details = self._apply( + path, + "drop_to_ground", + "normalize_scale", + normalize_scale={"size": 3}, + ) + + self.assertEqual(details["applied"], ["normalize_scale", "drop_to_ground"]) + self.assertAlmostEqual(_world_bounds(output)[0][1], 0.0, places=5) + self.assertAlmostEqual(max(details["dimensions"]), 3.0, places=4) + + def test_scene_node_transforms_are_measured_in_world_space(self) -> None: + scene = trimesh.Scene() + scene.add_geometry( + trimesh.creation.box(), + node_name="raised", + transform=trimesh.transformations.translation_matrix([0.0, 7.0, 0.0]), + ) + path = self._export(scene) + + output, _ = self._apply(path, "drop_to_ground") + + self.assertAlmostEqual(_world_bounds(output)[0][1], 0.0, places=5) + + def test_textures_survive_the_transform(self) -> None: + mesh = trimesh.creation.box() + mesh.unmerge_vertices() + uv = np.random.default_rng(0).random((len(mesh.vertices), 2)) + mesh.visual = trimesh.visual.TextureVisuals(uv=uv) + path = self._export(mesh) + + output, _ = self._apply(path, "drop_to_ground") + + geometry = next(iter(trimesh.load(str(output)).geometry.values())) + self.assertIsInstance(geometry.visual, trimesh.visual.TextureVisuals) + self.assertEqual(len(geometry.visual.uv), len(uv)) + + def test_input_file_is_left_untouched(self) -> None: + path = self._export(_tilted_plate()) + before = hashlib.sha256(path.read_bytes()).hexdigest() + + output, _ = self._apply(path, "auto_orient", "recenter", "drop_to_ground") + + self.assertNotEqual(output, path) + self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), before) + + def test_invalid_steps_are_rejected(self) -> None: + path = self._export(_tilted_plate()) + output = self.workspace / "never.glb" + + with self.assertRaises(ValueError): + apply_transforms(path, [], output, self.context) + with self.assertRaises(ValueError): + apply_transforms(path, [("repair", {})], output, self.context) + with self.assertRaises(ValueError): + mesh_ops_registry.resolve_params("normalize_scale", {"unit": "inch"}) + self.assertFalse(output.exists()) + + def test_registry_ops_write_a_new_file(self) -> None: + path = self._export(_tilted_plate()) + + result = mesh_ops_registry.run("drop_to_ground", path, {}, self.context) + + self.assertTrue(result.file_path.is_file()) + self.assertEqual(result.file_path.parent, self.workspace / "Workflows") + self.assertAlmostEqual(_world_bounds(result.file_path)[0][1], 0.0, places=5) + + +class TransformRegistryTests(unittest.TestCase): + def test_transform_ops_are_registered(self) -> None: + described = {op["id"]: op for op in mesh_ops_registry.describe()} + + for op_id in ("recenter", "drop_to_ground", "auto_orient", "normalize_scale"): + self.assertEqual(described[op_id]["category"], "transform") + self.assertFalse(described[op_id]["destructive"]) + self.assertEqual( + [schema["id"] for schema in described["normalize_scale"]["params_schema"]], + ["size", "unit", "dimension"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_mesh_routers_workspace.py b/api/tests/test_mesh_routers_workspace.py index f03e86dff..8e57da721 100644 --- a/api/tests/test_mesh_routers_workspace.py +++ b/api/tests/test_mesh_routers_workspace.py @@ -69,6 +69,40 @@ def test_transform_writes_its_result_into_the_moved_workspace(self) -> None: written = registry.WORKSPACE_DIR / result["url"].removeprefix("/workspace/") self.assertTrue(written.is_file()) + def test_arrange_composes_steps_into_one_file_in_the_moved_workspace(self) -> None: + result = optimize_router.arrange_mesh( + optimize_router.ArrangeRequest( + path="MyColl/mesh.glb", + steps=["drop_to_ground", "normalize_scale"], + params={"normalize_scale": {"size": 50, "unit": "cm"}}, + ) + ) + self.assertEqual(result["applied"], ["normalize_scale", "drop_to_ground"]) + self.assertTrue(result["path"].startswith("MyColl/mesh_xf_")) + self.assertEqual(result["url"], f"/workspace/{result['path']}") + written = trimesh.load(str(registry.WORKSPACE_DIR / result["path"])) + self.assertAlmostEqual(written.bounds[0][1], 0.0, places=5) + self.assertAlmostEqual(max(written.extents), 0.5, places=5) + + def test_arrange_rejects_unknown_steps_and_bad_params(self) -> None: + for steps, params in ( + (["levitate"], {}), + (["repair"], {}), + ([], {}), + (["normalize_scale"], {"normalize_scale": {"unit": "inch"}}), + ): + with self.subTest(steps=steps), self.assertRaises(HTTPException) as raised: + optimize_router.arrange_mesh( + optimize_router.ArrangeRequest( + path="MyColl/mesh.glb", steps=steps, params=params, + ) + ) + self.assertEqual(raised.exception.status_code, 400) + self.assertEqual( + sorted(p.name for p in (registry.WORKSPACE_DIR / "MyColl").iterdir()), + ["mesh.glb"], + ) + def test_decimate_and_smooth_read_their_input_from_the_moved_workspace(self) -> None: # /optimize/mesh and /optimize/smooth resolve their input through this helper. resolved = optimize_router._resolve_input_path("MyColl/mesh.glb") From 5acb3cf49c9e7963ef8354869bebe3c0ddbeff0f Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 19 Oct 2026 11:23:08 +0200 Subject: [PATCH 05/15] feat(mesh-ops): add a diagnostic-driven repair_mesh composite op repair_mesh diagnoses the mesh, runs only the steps the diagnostic calls for (degenerate faces, non-manifold edges, hole filling without a size limit, face re-orientation), then re-checks, for up to three passes or until a pass makes no progress. A mesh left inside-out is flipped at the end. Each pass is judged on position-merged geometry, exactly as inspect_mesh sees it, so UV seams are not mistaken for holes and a reported success holds once the file is re-inspected. A sound mesh is left alone and no file is written. With fill_holes off, only face-preserving fixes run, since removing faces would open holes. The result reports the before/after topology, the remaining issues, and whether textures were dropped (the output is geometry only). inspect_mesh now flags inside-out meshes, and suggest_fixes proposes repair_mesh, which also makes inconsistent winding fixable. The legacy repair op is unchanged for the mesh-repair workflow node. The generic /optimize/op route now writes its output next to the input under a unique name, like the dedicated edit routes. The agent and MCP server get a repair_mesh tool. Closes #298 --- api/mcp_server.py | 30 ++++ api/routers/agent.py | 63 ++++++++ api/routers/optimize.py | 7 +- api/services/mesh_ops/builtin.py | 23 +++ api/services/mesh_ops/inspection.py | 93 ++++++------ api/services/mesh_ops/repair.py | 200 ++++++++++++++++++++++++++ api/tests/test_agent_router.py | 65 +++++++++ api/tests/test_mesh_ops_inspection.py | 41 ++++-- api/tests/test_mesh_ops_registry.py | 1 + api/tests/test_mesh_ops_repair.py | 181 +++++++++++++++++++++++ api/tests/test_optimize_mesh_ops.py | 13 +- 11 files changed, 653 insertions(+), 64 deletions(-) create mode 100644 api/services/mesh_ops/repair.py create mode 100644 api/tests/test_mesh_ops_repair.py diff --git a/api/mcp_server.py b/api/mcp_server.py index 38ed65f1b..a59ff0092 100644 --- a/api/mcp_server.py +++ b/api/mcp_server.py @@ -115,6 +115,27 @@ async def list_tools() -> list[Tool]: "required": ["path"], }, ), + Tool( + name="modly_repair_mesh", + description=( + "Repair a mesh automatically (holes, non-manifold edges, degenerate faces, " + "flipped normals) until it is closed and manifold. Writes a new, untextured mesh." + ), + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + "fill_holes": { + "type": "boolean", + "description": "Close holes (default true).", + }, + }, + "required": ["path"], + }, + ), Tool( name="modly_arrange_mesh", description=( @@ -283,6 +304,15 @@ async def _dispatch(client: httpx.AsyncClient, name: str, args: dict) -> str: r.raise_for_status() return json.dumps(r.json(), indent=2) + elif name == "modly_repair_mesh": + params = {} if args.get("fill_holes") is None else {"fill_holes": bool(args["fill_holes"])} + r = await client.post( + f"{API_BASE}/optimize/op/repair_mesh", + json={"path": args["path"], "params": params}, + ) + r.raise_for_status() + return json.dumps(r.json(), indent=2) + elif name == "modly_arrange_mesh": scale = {k: args[k] for k in ("size", "unit", "dimension") if args.get(k) is not None} r = await client.post( diff --git a/api/routers/agent.py b/api/routers/agent.py index 9d6134d98..3cf15d442 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -30,6 +30,7 @@ - **get_mesh_info** — Get info about the current mesh in the 3D viewer (path, triangle count). - **inspect_mesh(path)** — Diagnose a mesh without modifying it: watertight, manifold, holes, self-intersections, parts, dimensions, volume, UVs/normals, and whether it is 3D-printable. - **suggest_fixes(path)** — Inspect a mesh and list the operations (with params) that would fix its problems. +- **repair_mesh(path, fill_holes?)** — Repair a mesh automatically: fills holes, fixes non-manifold edges, degenerate faces and flipped normals, re-checking until it is closed and manifold. Writes a new, untextured mesh. - **arrange_mesh(path, steps, size?, unit?, dimension?)** — Reposition a mesh in one call. Steps: `auto_orient` (lay it flat on its widest side), `normalize_scale` (resize to `size` in `unit`), `recenter` (centre on the origin), `drop_to_ground` (rest it on the ground). Steps always run in that order, whatever order you list them in. - **decimate_mesh(path, target_faces)** — Reduce the polygon count of a mesh. - **smooth_mesh(path, iterations)** — Apply Laplacian smoothing to a mesh. @@ -42,6 +43,7 @@ - Always use tools to act on the scene — never just describe what you would do. - If you need the current mesh path, call get_mesh_info first. +- To repair or fix a mesh, call repair_mesh. Report the issues it could not fix, and warn the user when it says the textures were dropped. - To center, ground, lay flat or resize a mesh, call arrange_mesh ONCE with every requested step — never one call per step. - To answer questions about a mesh's quality or printability, call inspect_mesh — never guess. Before repairing a mesh, call suggest_fixes to pick the right operations. - If you need to run a workflow but don't know the ID, call list_workflows first. @@ -111,6 +113,34 @@ }, }, }, + { + "type": "function", + "function": { + "name": "repair_mesh", + "description": ( + "Repair a mesh automatically: fill holes, fix non-manifold edges, degenerate " + "faces and flipped normals, re-checking until it is closed and manifold. " + "Writes a new mesh without textures; does nothing if the mesh is already sound." + ), + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file. Use get_mesh_info to obtain it.", + }, + "fill_holes": { + "type": "boolean", + "description": ( + "Close holes (default true). Set false only for meshes meant to stay open, " + "like terrain: then only flipped faces are fixed." + ), + }, + }, + "required": ["path"], + }, + }, + }, { "type": "function", "function": { @@ -381,6 +411,28 @@ def _format_fixes(data: dict) -> str: return "\n".join(lines) +def _format_repair(data: dict) -> tuple[str, dict | None]: + """Render a repair_mesh result for the model, plus the viewer update.""" + remaining = data.get("remaining_issues") or [] + if not data.get("repaired"): + text = "Nothing to repair: the mesh is already closed and manifold." + if remaining: + text = "Nothing was repaired. Remaining: " + "; ".join(remaining) + "." + return text, None + + steps = ", ".join(data.get("steps") or []) + lines = [ + f"Repaired in {data.get('passes', '?')} pass(es) ({steps}).", + ("Still remaining: " + "; ".join(remaining) + ".") if remaining + else "The mesh is now closed (watertight) and manifold.", + f"New mesh path: {data['path']} ({data.get('face_count', '?'):,} faces).", + ] + if data.get("textures_dropped"): + lines.append("The textures and UVs were dropped: the repaired mesh is geometry only.") + payload = {"type": "mesh_update", "url": data["url"], "face_count": data.get("face_count")} + return "\n".join(lines), payload + + async def execute_tool(name: str, arguments: dict, context: dict) -> tuple[str, dict | None]: """Execute a tool and return (result_text, action_payload). action_payload carries data the frontend needs to react (e.g. new mesh URL). @@ -427,6 +479,17 @@ async def execute_tool(name: str, arguments: dict, context: dict) -> tuple[str, r.raise_for_status() return _format_fixes(r.json()), None + elif name == "repair_mesh": + params = {} + if arguments.get("fill_holes") is not None: + params["fill_holes"] = bool(arguments["fill_holes"]) + r = await client.post( + f"{MODLY_API}/optimize/op/repair_mesh", + json={"path": arguments["path"], "params": params}, + ) + r.raise_for_status() + return _format_repair(r.json()) + elif name == "arrange_mesh": scale_params = { key: arguments[key] for key in ("size", "unit", "dimension") diff --git a/api/routers/optimize.py b/api/routers/optimize.py index 01dbf47ed..a362b3b6d 100644 --- a/api/routers/optimize.py +++ b/api/routers/optimize.py @@ -149,8 +149,13 @@ def list_mesh_operations(): @router.post("/op/{op_name}") def run_mesh_operation(op_name: str, body: MeshOpRequest): input_path = _resolve_input_path(body.path) + # Next to the input, like the dedicated edit routes; analysis ops ignore it. + output_path = _operation_output_path( + input_path, + f"{input_path.stem}_{op_name}_{uuid.uuid4().hex[:8]}.glb", + ) return _operation_response( - _run_operation(op_name, input_path, body.params) + _run_operation(op_name, input_path, body.params, output_path) ) diff --git a/api/services/mesh_ops/builtin.py b/api/services/mesh_ops/builtin.py index 67cd73bc7..f8450acd8 100644 --- a/api/services/mesh_ops/builtin.py +++ b/api/services/mesh_ops/builtin.py @@ -2,6 +2,7 @@ from .inspection import inspect_mesh, suggest_fixes from .operations import decimate_mesh, repair_mesh, smooth_mesh +from .repair import auto_repair_mesh from .transforms import auto_orient, drop_to_ground, normalize_scale, recenter from .types import MeshOp @@ -160,6 +161,21 @@ ) +REPAIR_MESH_PARAMS = ( + { + "id": "fill_holes", + "label": "Fill Holes", + "type": "boolean", + "default": True, + "tooltip": ( + "Close every hole, whatever its size. Turn off for meshes that are " + "meant to stay open, such as terrain: only flipped faces are then " + "fixed, since removing bad faces would open new holes." + ), + }, +) + + BUILTIN_MESH_OPS = ( MeshOp( id="repair", @@ -168,6 +184,13 @@ fn=repair_mesh, category="repair", ), + MeshOp( + id="repair_mesh", + label="Repair Mesh (auto)", + params_schema=REPAIR_MESH_PARAMS, + fn=auto_repair_mesh, + category="repair", + ), MeshOp( id="decimate", label="Optimize Mesh", diff --git a/api/services/mesh_ops/inspection.py b/api/services/mesh_ops/inspection.py index 9f4ebe9e4..fd4ed6b73 100644 --- a/api/services/mesh_ops/inspection.py +++ b/api/services/mesh_ops/inspection.py @@ -18,8 +18,6 @@ # both struggle well before the generators' multi-million-face outputs). HIGH_FACE_COUNT = 500_000 DECIMATE_TARGET_FACES = 200_000 -# Mirrors the repair op's max_hole_size bounds. -MAX_HOLE_SIZE_BOUNDS = (10, 10000) def _trimesh(): @@ -107,6 +105,47 @@ def _edge_topology(merged, trimesh_module) -> dict[str, Any]: } +def topology_summary(merged, trimesh_module) -> dict[str, Any]: + """Topology checks on a position-merged mesh, cheap enough to rerun per pass.""" + topology = _edge_topology(merged, trimesh_module) + is_watertight = bool(merged.is_watertight) + is_winding_consistent = bool(merged.is_winding_consistent) + return { + "is_watertight": is_watertight, + "is_winding_consistent": is_winding_consistent, + "is_manifold": topology["non_manifold_edge_count"] == 0, + # A closed, consistently wound mesh with negative volume has every + # normal pointing inwards. + "is_inside_out": bool( + is_watertight and is_winding_consistent and merged.volume < 0 + ), + **topology, + "degenerate_face_count": int( + len(merged.faces) - int(merged.nondegenerate_faces().sum()) + ), + } + + +def topology_issues(summary: Mapping[str, Any]) -> list[str]: + issues = [] + if summary["hole_count"]: + issues.append( + f"{summary['hole_count']} hole(s) " + f"({summary['boundary_edge_count']} open boundary edges)" + ) + if summary["non_manifold_edge_count"]: + issues.append( + f"{summary['non_manifold_edge_count']} non-manifold edge(s)" + ) + if not summary["is_winding_consistent"]: + issues.append("inconsistent face winding (normals flipped on some faces)") + if summary["is_inside_out"]: + issues.append("inside-out (every normal points inwards)") + if summary["degenerate_face_count"]: + issues.append(f"{summary['degenerate_face_count']} degenerate face(s)") + return issues + + def _self_intersecting_faces(merged, context: MeshOpContext) -> Optional[int]: try: import pymeshlab @@ -132,6 +171,7 @@ def _printable(report: Mapping[str, Any]) -> Optional[bool]: report["is_watertight"] and report["is_winding_consistent"] and report["is_manifold"] + and not report["is_inside_out"] ): return False if report["self_intersecting_face_count"] is None: @@ -140,26 +180,13 @@ def _printable(report: Mapping[str, Any]) -> Optional[bool]: def _issues(report: Mapping[str, Any]) -> list[str]: - issues = [] - if report["hole_count"]: - issues.append( - f"{report['hole_count']} hole(s) " - f"({report['boundary_edge_count']} open boundary edges)" - ) - if report["non_manifold_edge_count"]: - issues.append( - f"{report['non_manifold_edge_count']} non-manifold edge(s)" - ) - if not report["is_winding_consistent"]: - issues.append("inconsistent face winding (normals flipped on some faces)") + issues = topology_issues(report) if report["self_intersecting_face_count"]: issues.append( f"{report['self_intersecting_face_count']} self-intersecting face(s)" ) if report["self_intersecting_face_count"] is None: issues.append("self-intersections were not checked") - if report["degenerate_face_count"]: - issues.append(f"{report['degenerate_face_count']} degenerate face(s)") if report["component_count"] > 1: issues.append(f"{report['component_count']} disconnected parts") return issues @@ -183,9 +210,7 @@ def build_report( context.progress(30, "Analysing topology…") merged = merged_geometry(parts, trimesh) - topology = _edge_topology(merged, trimesh) - is_watertight = bool(merged.is_watertight) - is_winding_consistent = bool(merged.is_winding_consistent) + topology = topology_summary(merged, trimesh) self_intersections = None if check_self_intersections: @@ -198,13 +223,7 @@ def build_report( "vertex_count": int(sum(len(geometry.vertices) for geometry, _ in parts)), "face_count": int(sum(len(geometry.faces) for geometry, _ in parts)), "component_count": int(merged.body_count), - "is_watertight": is_watertight, - "is_winding_consistent": is_winding_consistent, - "is_manifold": topology["non_manifold_edge_count"] == 0, **topology, - "degenerate_face_count": int( - len(merged.faces) - int(merged.nondegenerate_faces().sum()) - ), "self_intersecting_face_count": self_intersections, "bounds": { "min": [float(value) for value in bounds[0]], @@ -215,7 +234,7 @@ def build_report( # The signed volume of an open or mis-wound mesh is meaningless. "volume": ( float(abs(merged.volume)) - if is_watertight and is_winding_consistent + if topology["is_watertight"] and topology["is_winding_consistent"] else None ), "has_uvs": any(_has_uvs(geometry, trimesh) for geometry, _ in parts), @@ -243,28 +262,16 @@ def fixes_for_report(report: Mapping[str, Any]) -> tuple[list[dict], list[str]]: repair_reasons.append("fix non-manifold edges") if report["degenerate_face_count"]: repair_reasons.append("remove degenerate faces") + if not report["is_winding_consistent"] or report["is_inside_out"]: + repair_reasons.append("re-orient the faces") if repair_reasons: - low, high = MAX_HOLE_SIZE_BOUNDS fixes.append( { - "op": "repair", - "params": { - "remove_duplicates": True, - "remove_degenerate": bool(report["degenerate_face_count"]), - "fix_non_manifold": bool(report["non_manifold_edge_count"]), - "fill_holes": bool(report["hole_count"]), - "max_hole_size": max( - low, min(high, report["largest_hole_edges"] or low) - ), - }, + "op": "repair_mesh", + "params": {}, "reason": "Repair to " + ", ".join(repair_reasons) + ".", } ) - if report["largest_hole_edges"] > high: - notes.append( - f"The largest hole has {report['largest_hole_edges']} boundary " - f"edges, above the repair limit of {high}; it may stay open." - ) if report["face_count"] > HIGH_FACE_COUNT: fixes.append( @@ -278,8 +285,6 @@ def fixes_for_report(report: Mapping[str, Any]) -> tuple[list[dict], list[str]]: } ) - if not report["is_winding_consistent"]: - notes.append("Inconsistent winding has no automatic fix yet.") if report["self_intersecting_face_count"]: notes.append("Self-intersections have no automatic fix yet.") if report["component_count"] > 1: diff --git a/api/services/mesh_ops/repair.py b/api/services/mesh_ops/repair.py new file mode 100644 index 000000000..f4d8af281 --- /dev/null +++ b/api/services/mesh_ops/repair.py @@ -0,0 +1,200 @@ +"""Composite, diagnostic-driven repair: loop until the mesh is closed and manifold.""" + +from pathlib import Path +from typing import Any, Mapping + +from .geometry import merged_geometry, scene_meshes +from .inspection import _has_uvs, topology_issues, topology_summary +from .operations import _mesh_libraries, _output_path +from .types import MeshOpContext, MeshOpExecutionError, MeshOpResult + + +# Filling a hole can leave a non-manifold edge behind, and fixing that edge can +# open a new hole, so a single pass is not enough; three settle every case seen +# in practice, and the loop also stops as soon as a pass makes no progress. +MAX_PASSES = 3 + + +def _is_sound(summary: Mapping[str, Any], fill_holes: bool) -> bool: + # Inside-out is left to the final flip: it needs a closed, coherent mesh. + return ( + (summary["is_watertight"] or not fill_holes) + and summary["is_manifold"] + and summary["is_winding_consistent"] + ) + + +def _progress_key(summary: Mapping[str, Any]) -> tuple: + return ( + summary["hole_count"], + summary["boundary_edge_count"], + summary["non_manifold_edge_count"], + summary["is_winding_consistent"], + summary["degenerate_face_count"], + ) + + +def _merged(vertices, faces, trimesh_module): + # Merged by position exactly as inspect_mesh does, so each pass is judged + # by the same diagnostic the user will see afterwards. + return trimesh_module.Trimesh(vertices=vertices, faces=faces, process=True) + + +def _run_pass( + mesh, + summary: Mapping[str, Any], + fill_holes: bool, + context: MeshOpContext, + pymeshlab_module, + trimesh_module, +) -> tuple[Any, list[str]]: + """Run the repair steps the diagnostic calls for; return the mesh and steps. + + Removing degenerate or non-manifold faces opens holes, so those steps only + run when the holes may be filled; otherwise the repair would leave the mesh + more open than it found it. + """ + steps = [] + if fill_holes and summary["degenerate_face_count"]: + # The holes this opens are closed by the fill below. + mesh.update_faces(mesh.nondegenerate_faces()) + mesh.remove_unreferenced_vertices() + steps.append("remove_degenerate") + + mesh_set = pymeshlab_module.MeshSet() + mesh_set.add_mesh( + pymeshlab_module.Mesh(vertex_matrix=mesh.vertices, face_matrix=mesh.faces) + ) + mesh_set.meshing_remove_duplicate_faces() + if fill_holes and not summary["is_manifold"]: + mesh_set.meshing_repair_non_manifold_edges(method=0) + mesh_set.meshing_repair_non_manifold_vertices() + mesh_set.meshing_remove_unreferenced_vertices() + steps.append("fix_non_manifold") + + if fill_holes and (summary["hole_count"] or steps): + # No size limit: a hole's boundary can't have more edges than the mesh. + max_hole_size = 3 * mesh_set.current_mesh().face_number() + 3 + try: + mesh_set.meshing_close_holes( + maxholesize=max_hole_size, + newfaceselected=False, + selfintersection=False, + ) + steps.append("fill_holes") + except Exception as exc: + context.log(f"Hole fill skipped this pass: {exc}") + + current = mesh_set.current_mesh() + mesh = _merged(current.vertex_matrix(), current.face_matrix(), trimesh_module) + if not mesh.is_winding_consistent: + trimesh_module.repair.fix_winding(mesh) + steps.append("reorient_faces") + return mesh, steps + + +def _unchanged(before: Mapping[str, Any], mesh, context: MeshOpContext) -> MeshOpResult: + """Report a mesh the repair leaves as it is, without writing a file.""" + remaining = topology_issues(before) + context.log( + "Nothing repaired" + (f": {'; '.join(remaining)}" if remaining else "") + ) + context.progress(100, "Done") + return MeshOpResult( + details={ + "repaired": False, + "passes": 0, + "steps": [], + "before": before, + "after": before, + "remaining_issues": remaining, + "face_count": int(len(mesh.faces)), + "textures_dropped": False, + } + ) + + +def auto_repair_mesh( + input_path: Path, + params: Mapping[str, Any], + context: MeshOpContext, +) -> MeshOpResult: + """Repair what the diagnostic finds, re-checking after every pass. + + The output is geometry only: hole filling and face removal change the + topology, so the original UVs and textures cannot be carried over. + """ + pymeshlab, trimesh = _mesh_libraries("repair_mesh") + fill_holes = bool(params.get("fill_holes", True)) + + context.progress(10, "Diagnosing…") + loaded = trimesh.load(str(input_path), process=False) + parts = scene_meshes(loaded, trimesh) + if not parts: + raise MeshOpExecutionError( + f"repair_mesh: {input_path.name} contains no triangle mesh" + ) + # Merged by position, so UV seams are not mistaken for holes. + mesh = merged_geometry(parts, trimesh) + before = topology_summary(mesh, trimesh) + has_textures = any(_has_uvs(geometry, trimesh) for geometry, _ in parts) + + if ( + _is_sound(before, fill_holes) + and not before["is_inside_out"] + and not before["degenerate_face_count"] + ): + return _unchanged(before, mesh, context) + + summary = before + steps: list[str] = [] + passes = 0 + for passes in range(1, MAX_PASSES + 1): + context.progress(20 + 20 * (passes - 1), f"Repair pass {passes}…") + previous = summary + mesh, pass_steps = _run_pass( + mesh, summary, fill_holes, context, pymeshlab, trimesh + ) + steps.extend(pass_steps) + summary = topology_summary(mesh, trimesh) + context.log( + f"Pass {passes}: " + + ("; ".join(topology_issues(summary)) or "closed and manifold") + ) + if _is_sound(summary, fill_holes) or _progress_key(summary) == _progress_key(previous): + break + + if summary["is_inside_out"]: + mesh.invert() + steps.append("flip_normals") + summary = topology_summary(mesh, trimesh) + + if not steps: + # Only reachable with fill_holes off: every remaining issue needs a fill. + return _unchanged(before, mesh, context) + if not len(mesh.faces): + raise MeshOpExecutionError("repair_mesh: the repair left no faces") + + context.progress(90, "Exporting…") + output_path = _output_path(context, "repair-mesh") + mesh.export(str(output_path)) + remaining = topology_issues(summary) + context.log( + f"Repaired in {passes} pass(es): " + + ("; ".join(remaining) if remaining else "closed and manifold") + ) + context.progress(100, "Done") + return MeshOpResult( + file_path=output_path, + details={ + "repaired": True, + "passes": passes, + # In first-applied order, without repeats across passes. + "steps": list(dict.fromkeys(steps)), + "before": before, + "after": summary, + "remaining_issues": remaining, + "face_count": int(len(mesh.faces)), + "textures_dropped": has_textures, + }, + ) diff --git a/api/tests/test_agent_router.py b/api/tests/test_agent_router.py index 2f21bc0f8..25a1bf96f 100644 --- a/api/tests/test_agent_router.py +++ b/api/tests/test_agent_router.py @@ -119,6 +119,71 @@ def test_suggest_fixes_on_a_clean_mesh(self) -> None: self.assertIn("No fixes needed", text) +class RepairMeshToolTests(unittest.TestCase): + def _run(self, arguments: dict, response: dict) -> tuple[str, object, list]: + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=response) + + factory = _MockClientFactory(handler) + with mock.patch.object(agent.httpx, "AsyncClient", factory): + text, payload = asyncio.run(agent.execute_tool("repair_mesh", arguments, {})) + return text, payload, requests + + def test_repaired_mesh_updates_the_viewer_and_warns_about_textures(self) -> None: + text, payload, requests = self._run( + {"path": "Default/mesh.glb"}, + { + "path": "Workflows/repair-mesh-1.glb", + "url": "/workspace/Workflows/repair-mesh-1.glb", + "repaired": True, + "passes": 2, + "steps": ["fix_non_manifold", "fill_holes"], + "remaining_issues": [], + "face_count": 1276, + "textures_dropped": True, + }, + ) + + self.assertEqual(requests[0].url.path, "/optimize/op/repair_mesh") + self.assertEqual(json.loads(requests[0].content), {"path": "Default/mesh.glb", "params": {}}) + self.assertIn("now closed (watertight) and manifold", text) + self.assertIn("New mesh path: Workflows/repair-mesh-1.glb", text) + self.assertIn("textures and UVs were dropped", text) + self.assertEqual(payload["type"], "mesh_update") + self.assertEqual(payload["url"], "/workspace/Workflows/repair-mesh-1.glb") + + def test_partial_repair_reports_what_remains(self) -> None: + text, _, requests = self._run( + {"path": "Default/mesh.glb", "fill_holes": False}, + { + "path": "Workflows/repair-mesh-2.glb", + "url": "/workspace/Workflows/repair-mesh-2.glb", + "repaired": True, + "passes": 3, + "steps": ["fix_non_manifold"], + "remaining_issues": ["1 hole(s) (6 open boundary edges)"], + "face_count": 100, + "textures_dropped": False, + }, + ) + + self.assertEqual(json.loads(requests[0].content)["params"], {"fill_holes": False}) + self.assertIn("Still remaining: 1 hole(s)", text) + self.assertNotIn("textures", text) + + def test_sound_mesh_changes_nothing(self) -> None: + text, payload, _ = self._run( + {"path": "Default/mesh.glb"}, + {"repaired": False, "passes": 0, "steps": [], "remaining_issues": []}, + ) + + self.assertIn("Nothing to repair", text) + self.assertIsNone(payload) + + class ArrangeMeshToolTests(unittest.TestCase): def _run(self, arguments: dict) -> tuple[str, object, list]: requests = [] diff --git a/api/tests/test_mesh_ops_inspection.py b/api/tests/test_mesh_ops_inspection.py index f1ff4fc4c..1b364b488 100644 --- a/api/tests/test_mesh_ops_inspection.py +++ b/api/tests/test_mesh_ops_inspection.py @@ -182,9 +182,20 @@ def test_suggest_fixes_proposes_a_repair_for_holes(self) -> None: self.assertIsNone(result.file_path) self.assertEqual(result.details["report"]["hole_count"], 1) fixes = result.details["fixes"] - self.assertEqual([fix["op"] for fix in fixes], ["repair"]) - self.assertTrue(fixes[0]["params"]["fill_holes"]) - self.assertEqual(fixes[0]["params"]["max_hole_size"], 10) + self.assertEqual([fix["op"] for fix in fixes], ["repair_mesh"]) + self.assertIn("fill 1 hole(s)", fixes[0]["reason"]) + + def test_inside_out_mesh_is_reported(self) -> None: + mesh = _box() + mesh.invert() + path = self._export(mesh) + + report = self._inspect(path, check_self_intersections=False) + + self.assertTrue(report["is_watertight"]) + self.assertTrue(report["is_inside_out"]) + self.assertFalse(report["printable"]) + self.assertIn("inside-out (every normal points inwards)", report["issues"]) class FixesForReportTests(unittest.TestCase): @@ -192,6 +203,7 @@ class FixesForReportTests(unittest.TestCase): "face_count": 12, "component_count": 1, "is_winding_consistent": True, + "is_inside_out": False, "hole_count": 0, "largest_hole_edges": 0, "non_manifold_edge_count": 0, @@ -214,18 +226,25 @@ def test_suggested_params_are_valid_for_the_registry(self) -> None: fixes, notes = fixes_for_report(report) - self.assertEqual([fix["op"] for fix in fixes], ["repair", "decimate"]) - self.assertEqual(fixes[0]["params"]["max_hole_size"], 10000) + self.assertEqual([fix["op"] for fix in fixes], ["repair_mesh", "decimate"]) self.assertEqual(fixes[1]["params"], {"target_faces": DECIMATE_TARGET_FACES}) - self.assertTrue(any("repair limit" in note for note in notes)) - repair = mesh_ops_registry.get("repair") - declared = {schema["id"] for schema in repair.params_schema} - self.assertLessEqual(set(fixes[0]["params"]), declared) + self.assertEqual(notes, []) + for fix in fixes: + declared = { + schema["id"] for schema in mesh_ops_registry.get(fix["op"]).params_schema + } + self.assertLessEqual(set(fix["params"]), declared) + + def test_flipped_or_inside_out_faces_call_for_a_repair(self) -> None: + for flaw in ({"is_winding_consistent": False}, {"is_inside_out": True}): + with self.subTest(flaw=flaw): + fixes, _ = fixes_for_report({**self.CLEAN, **flaw}) + self.assertEqual([fix["op"] for fix in fixes], ["repair_mesh"]) + self.assertIn("re-orient", fixes[0]["reason"]) def test_problems_without_an_op_become_notes(self) -> None: report = { **self.CLEAN, - "is_winding_consistent": False, "self_intersecting_face_count": 7, "component_count": 3, } @@ -233,7 +252,7 @@ def test_problems_without_an_op_become_notes(self) -> None: fixes, notes = fixes_for_report(report) self.assertEqual(fixes, []) - self.assertEqual(len(notes), 3) + self.assertEqual(len(notes), 2) class InspectionRegistryTests(unittest.TestCase): diff --git a/api/tests/test_mesh_ops_registry.py b/api/tests/test_mesh_ops_registry.py index 8bde5a366..333c7d89a 100644 --- a/api/tests/test_mesh_ops_registry.py +++ b/api/tests/test_mesh_ops_registry.py @@ -21,6 +21,7 @@ def test_builtin_metadata_is_serializable_and_complete(self) -> None: [description["id"] for description in descriptions], [ "repair", + "repair_mesh", "decimate", "smooth", "inspect_mesh", diff --git a/api/tests/test_mesh_ops_repair.py b/api/tests/test_mesh_ops_repair.py new file mode 100644 index 000000000..0b1f543bf --- /dev/null +++ b/api/tests/test_mesh_ops_repair.py @@ -0,0 +1,181 @@ +import hashlib +import importlib.util +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import trimesh + +from services.mesh_ops import MeshOpContext, MeshOpExecutionError, mesh_ops_registry + + +HAS_PYMESHLAB = importlib.util.find_spec("pymeshlab") is not None + + +def _holed_sphere_with_fin() -> trimesh.Trimesh: + """A sphere missing a patch of faces, with a fin making one edge non-manifold.""" + sphere = trimesh.creation.icosphere(subdivisions=3) + sphere.update_faces(np.arange(20, len(sphere.faces))) + edge = sphere.edges_unique[100] + return trimesh.Trimesh( + vertices=np.vstack([sphere.vertices, [[0.0, 2.0, 0.0]]]), + faces=np.vstack([sphere.faces, [[edge[0], edge[1], len(sphere.vertices)]]]), + process=False, + ) + + +def _half_open_sphere() -> trimesh.Trimesh: + sphere = trimesh.creation.icosphere(subdivisions=4) + sphere.update_faces(sphere.triangles_center[:, 1] > -0.5) + return sphere + + +@unittest.skipUnless(HAS_PYMESHLAB, "pymeshlab is not installed") +class RepairMeshTests(unittest.TestCase): + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.workspace = Path(self._directory.name) + self.context = MeshOpContext( + workspace_dir=self.workspace, + temp_dir=self.workspace, + ) + + def tearDown(self) -> None: + self._directory.cleanup() + + def _export(self, geometry, name="mesh.glb") -> Path: + path = self.workspace / name + geometry.export(str(path)) + return path + + def _repair(self, geometry, **params): + return mesh_ops_registry.run( + "repair_mesh", self._export(geometry), params, self.context + ) + + def _assert_closed_and_manifold(self, path: Path) -> dict: + # Judged by the same diagnostic the user and the agent see. + report = mesh_ops_registry.run( + "inspect_mesh", path, {"check_self_intersections": False}, self.context + ).details + self.assertTrue(report["is_watertight"], report["issues"]) + self.assertTrue(report["is_manifold"], report["issues"]) + self.assertTrue(report["is_winding_consistent"], report["issues"]) + self.assertFalse(report["is_inside_out"], report["issues"]) + return report + + def test_holed_non_manifold_mesh_becomes_watertight_and_manifold(self) -> None: + result = self._repair(_holed_sphere_with_fin()) + + details = result.details + self.assertTrue(details["repaired"]) + self.assertFalse(details["before"]["is_watertight"]) + self.assertFalse(details["before"]["is_manifold"]) + self.assertIn("fix_non_manifold", details["steps"]) + self.assertIn("fill_holes", details["steps"]) + self.assertLessEqual(details["passes"], 3) + self.assertTrue(details["after"]["is_watertight"]) + self._assert_closed_and_manifold(result.file_path) + + def test_large_hole_is_closed_without_a_size_limit(self) -> None: + result = self._repair(_half_open_sphere()) + + self.assertEqual(result.details["steps"], ["fill_holes"]) + self.assertEqual(result.details["remaining_issues"], []) + self._assert_closed_and_manifold(result.file_path) + + def test_flipped_faces_are_reoriented(self) -> None: + sphere = trimesh.creation.icosphere(subdivisions=3) + faces = sphere.faces.copy() + faces[:50] = faces[:50, ::-1] + + result = self._repair(trimesh.Trimesh(sphere.vertices, faces, process=False)) + + self.assertIn("reorient_faces", result.details["steps"]) + report = self._assert_closed_and_manifold(result.file_path) + self.assertGreater(report["volume"], 0) + + def test_inside_out_mesh_is_turned_the_right_way(self) -> None: + sphere = trimesh.creation.icosphere(subdivisions=3) + sphere.invert() + + result = self._repair(sphere) + + self.assertTrue(result.details["before"]["is_inside_out"]) + self.assertEqual(result.details["steps"], ["flip_normals"]) + self._assert_closed_and_manifold(result.file_path) + + def test_sound_mesh_is_left_alone(self) -> None: + result = self._repair(trimesh.creation.box()) + + self.assertIsNone(result.file_path) + self.assertFalse(result.details["repaired"]) + self.assertEqual(result.details["remaining_issues"], []) + self.assertEqual(sorted(p.name for p in self.workspace.iterdir()), ["mesh.glb"]) + + def test_holes_are_kept_when_filling_is_off(self) -> None: + result = self._repair(_half_open_sphere(), fill_holes=False) + + self.assertFalse(result.details["repaired"]) + self.assertTrue( + any("hole" in issue for issue in result.details["remaining_issues"]) + ) + + def test_without_hole_filling_no_face_is_removed(self) -> None: + # Fixing the fin would delete faces and open a hole that may not be + # filled, so the mesh is left as it is and the issues reported. + result = self._repair(_holed_sphere_with_fin(), fill_holes=False) + + self.assertFalse(result.details["repaired"]) + self.assertIsNone(result.file_path) + remaining = result.details["remaining_issues"] + self.assertTrue(any("non-manifold" in issue for issue in remaining)) + self.assertTrue(any("hole" in issue for issue in remaining)) + + def test_without_hole_filling_flipped_faces_are_still_fixed(self) -> None: + mesh = _half_open_sphere() + faces = mesh.faces.copy() + faces[:40] = faces[:40, ::-1] + + result = self._repair( + trimesh.Trimesh(mesh.vertices, faces, process=False), fill_holes=False + ) + + self.assertEqual(result.details["steps"], ["reorient_faces"]) + self.assertTrue(result.details["after"]["is_winding_consistent"]) + self.assertEqual(result.details["face_count"], len(faces)) + self.assertEqual(result.details["after"]["hole_count"], 1) + + def test_textured_input_reports_that_textures_are_dropped(self) -> None: + mesh = _half_open_sphere() + mesh.unmerge_vertices() + uv = np.random.default_rng(0).random((len(mesh.vertices), 2)) + mesh.visual = trimesh.visual.TextureVisuals(uv=uv) + + result = self._repair(mesh) + + # UV seams were merged, so only the real hole was filled. + self.assertEqual(result.details["before"]["hole_count"], 1) + self.assertTrue(result.details["textures_dropped"]) + self._assert_closed_and_manifold(result.file_path) + + def test_input_file_is_left_untouched(self) -> None: + path = self._export(_holed_sphere_with_fin()) + before = hashlib.sha256(path.read_bytes()).hexdigest() + + result = mesh_ops_registry.run("repair_mesh", path, {}, self.context) + + self.assertNotEqual(result.file_path, path) + self.assertEqual(result.file_path.parent, self.workspace / "Workflows") + self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), before) + + def test_file_without_triangles_is_rejected(self) -> None: + path = self._export(trimesh.PointCloud(np.zeros((3, 3))), name="cloud.ply") + + with self.assertRaises(MeshOpExecutionError): + mesh_ops_registry.run("repair_mesh", path, {}, self.context) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_optimize_mesh_ops.py b/api/tests/test_optimize_mesh_ops.py index c2c9d071c..5436da007 100644 --- a/api/tests/test_optimize_mesh_ops.py +++ b/api/tests/test_optimize_mesh_ops.py @@ -52,14 +52,11 @@ def test_generic_list_and_run_routes_use_the_shared_registry(self) -> None: ) self.assertEqual(descriptions[0]["id"], "repair") - self.assertEqual( - response, - { - "path": "Workflows/output.glb", - "url": "/workspace/Workflows/output.glb", - "face_count": 42, - }, - ) + # Written next to the input under a unique name, never over it. + self.assertRegex(response["path"], r"^input_repair_[0-9a-f]{8}\.glb$") + self.assertEqual(response["url"], f"/workspace/{response['path']}") + self.assertEqual(response["face_count"], 42) + self.assertEqual(registry.calls[0][3].output_path.parent, workspace) self.assertEqual(registry.calls[0][0:3], ( "repair", input_path, From 95e911726a1e6b29082c7bab5a6452059fa6c4b4 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 19 Oct 2026 11:50:55 +0200 Subject: [PATCH 06/15] fix(deps): declare scipy and keep mesh ops off undeclared networkx The shipped venv installs only api/requirements.txt, which never pulls scipy: trimesh treats it as optional. inspect_mesh, suggest_fixes and repair_mesh need it for connected parts, hole loops and vertex merging, so they failed outside a dev environment, where torch brings it in. repair_mesh re-oriented faces with trimesh.repair.fix_winding, which needs networkx, also undeclared; it now uses pymeshlab, and skips the step for a pass while the mesh is still non-manifold. A guard test runs every mesh-op family with networkx unimportable, including the re-orientation path, and checks scipy stays declared. --- api/requirements.txt | 3 + api/services/mesh_ops/repair.py | 17 +++++- api/tests/test_mesh_ops_dependencies.py | 75 +++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 api/tests/test_mesh_ops_dependencies.py diff --git a/api/requirements.txt b/api/requirements.txt index 975f19ca3..0462a51ba 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -10,6 +10,9 @@ mcp>=1.0.0 # Mesh processing (optimize + export endpoints) trimesh>=4.5.0 pymeshlab>=2023.12 +# trimesh's graph engine: connected parts, hole loops and vertex merging in +# inspect_mesh / repair_mesh. Optional for trimesh, so never pulled in by it. +scipy>=1.10 # Model downloads (HuggingFace) huggingface_hub>=0.27.0 diff --git a/api/services/mesh_ops/repair.py b/api/services/mesh_ops/repair.py index f4d8af281..d964b310a 100644 --- a/api/services/mesh_ops/repair.py +++ b/api/services/mesh_ops/repair.py @@ -88,8 +88,21 @@ def _run_pass( current = mesh_set.current_mesh() mesh = _merged(current.vertex_matrix(), current.face_matrix(), trimesh_module) if not mesh.is_winding_consistent: - trimesh_module.repair.fix_winding(mesh) - steps.append("reorient_faces") + # pymeshlab rather than trimesh.repair.fix_winding, which needs + # networkx: not one of Modly's dependencies. + mesh_set = pymeshlab_module.MeshSet() + mesh_set.add_mesh( + pymeshlab_module.Mesh(vertex_matrix=mesh.vertices, face_matrix=mesh.faces) + ) + try: + mesh_set.meshing_re_orient_faces_coherently() + except Exception as exc: + # Orientation needs a manifold mesh; the next pass fixes that first. + context.log(f"Face re-orientation skipped this pass: {exc}") + else: + current = mesh_set.current_mesh() + mesh = _merged(current.vertex_matrix(), current.face_matrix(), trimesh_module) + steps.append("reorient_faces") return mesh, steps diff --git a/api/tests/test_mesh_ops_dependencies.py b/api/tests/test_mesh_ops_dependencies.py new file mode 100644 index 000000000..3b0d4517d --- /dev/null +++ b/api/tests/test_mesh_ops_dependencies.py @@ -0,0 +1,75 @@ +import re +import subprocess +import sys +import unittest +from pathlib import Path + + +API_DIR = Path(__file__).resolve().parents[1] + +# Runs every mesh-op family on a holed box with networkx unimportable: it is +# not in requirements.txt, so the shipped venv never has it, while a dev venv +# often does (torch pulls it in) and would hide the gap. +_SCRIPT = r""" +import sys, tempfile +sys.modules["networkx"] = None +from pathlib import Path +import numpy as np, trimesh +from services.mesh_ops import MeshOpContext, mesh_ops_registry as registry + +directory = Path(tempfile.mkdtemp()) +context = MeshOpContext(directory, directory) +box = trimesh.creation.box() +box.update_faces(np.arange(1, len(box.faces))) +path = directory / "open.glb" +box.export(path) + +registry.run("inspect_mesh", path, {}, context) +registry.run("suggest_fixes", path, {}, context) +repaired = registry.run("repair_mesh", path, {}, context) +assert repaired.details["after"]["is_watertight"], repaired.details + +# Flipped faces take the re-orientation path. +sphere = trimesh.creation.icosphere(subdivisions=2) +faces = sphere.faces.copy() +faces[:20] = faces[:20, ::-1] +flipped = directory / "flipped.glb" +trimesh.Trimesh(sphere.vertices, faces, process=False).export(flipped) +reoriented = registry.run("repair_mesh", flipped, {}, context) +assert "reorient_faces" in reoriented.details["steps"], reoriented.details +for op in ("recenter", "drop_to_ground", "auto_orient", "normalize_scale"): + registry.run(op, path, {}, context) +print("ok") +""" + + +class MeshOpDependencyTests(unittest.TestCase): + def test_mesh_ops_run_without_undeclared_networkx(self) -> None: + result = subprocess.run( + [sys.executable, "-c", _SCRIPT], + cwd=API_DIR, + capture_output=True, + text=True, + timeout=300, + ) + + self.assertEqual(result.returncode, 0, result.stderr[-2000:]) + self.assertEqual(result.stdout.strip().splitlines()[-1], "ok") + + def test_trimesh_graph_engine_is_a_declared_dependency(self) -> None: + # inspect_mesh and repair_mesh need scipy through trimesh, which does + # not depend on it; a dev environment usually has it anyway. + requirements = (API_DIR / "requirements.txt").read_text(encoding="utf-8") + declared = { + re.split(r"[<>=\[;\s]", line.strip(), maxsplit=1)[0].lower() + for line in requirements.splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + + self.assertIn("scipy", declared) + self.assertIn("trimesh", declared) + self.assertIn("pymeshlab", declared) + + +if __name__ == "__main__": + unittest.main() From 16e497b20e3713a8843a43c9b745f501d808e152 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Tue, 20 Oct 2026 11:23:26 +0200 Subject: [PATCH 07/15] feat(agent): stream chat responses as typed SSE events agent_chat now returns a StreamingResponse emitting token, thinking, tool_start, tool_done and done events. ChatPanel consumes the stream so text appears progressively and tool calls show an in-progress state. Closes #300 --- api/routers/agent.py | 326 +++++++++++++++++--- api/tests/test_agent_streaming.py | 293 ++++++++++++++++++ package.json | 2 +- src/areas/generate/components/ChatPanel.tsx | 162 ++++++---- src/shared/utils/agentStream.test.ts | 57 ++++ src/shared/utils/agentStream.ts | 75 +++++ 6 files changed, 808 insertions(+), 107 deletions(-) create mode 100644 api/tests/test_agent_streaming.py create mode 100644 src/shared/utils/agentStream.test.ts create mode 100644 src/shared/utils/agentStream.ts diff --git a/api/routers/agent.py b/api/routers/agent.py index 3cf15d442..e2cb56efb 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -4,12 +4,12 @@ """ import asyncio import json -import re import uuid from typing import Optional import httpx from fastapi import APIRouter +from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from services import llm_server @@ -634,19 +634,6 @@ class AgentChatResponse(BaseModel): thinking: str | None = None -def _extract_thinking(msg: dict) -> tuple[str, str | None]: - """Return (clean_content, thinking_text). Handles llama-server's - reasoning_content field and inline tags.""" - content = msg.get("content") or "" - thinking = msg.get("reasoning_content") or None - if not thinking: - match = re.search(r"(.*?)", content, re.DOTALL) - if match: - thinking = match.group(1).strip() - content = (content[: match.start()] + content[match.end() :]).strip() - return content, thinking - - def _auth_headers(base_url: str, api_key: Optional[str]) -> dict: headers: dict[str, str] = {} if api_key: @@ -741,8 +728,198 @@ def _drop_images(messages: list[dict]) -> bool: return dropped -@router.post("/chat", response_model=AgentChatResponse) -async def agent_chat(request: AgentChatRequest): +class _ThinkSplitter: + """Route inline … text to the reasoning, the rest to the + answer, holding back a tag a chunk boundary has cut in two.""" + + _OPEN, _CLOSE = "", "" + + def __init__(self) -> None: + self._buffer = "" + self._inside = False + + def _held(self, *tags: str) -> int: + """Length of the buffer's tail that may be the start of a tag.""" + return max( + next((n for n in range(len(tag) - 1, 0, -1) if self._buffer.endswith(tag[:n])), 0) + for tag in tags + ) + + def _cut(self, index: int, tag: str, kind: str, parts: list) -> None: + if index: + parts.append((kind, self._buffer[:index])) + self._buffer = self._buffer[index + len(tag):] + + def feed(self, text: str) -> list[tuple[str, str]]: + self._buffer += text + parts: list[tuple[str, str]] = [] + while self._buffer: + close_at = self._buffer.find(self._CLOSE) + if self._inside: + if close_at >= 0: + self._cut(close_at, self._CLOSE, "thinking", parts) + self._inside = False + continue + keep = self._held(self._CLOSE) + kind = "thinking" + else: + open_at = self._buffer.find(self._OPEN) + if close_at >= 0 and (open_at < 0 or close_at < open_at): + # A chat template that opens in the prompt leaves the + # model only the closing tag: what came before it was reasoning. + self._cut(close_at, self._CLOSE, "thinking", parts) + continue + if open_at >= 0: + self._cut(open_at, self._OPEN, "token", parts) + self._inside = True + continue + keep = self._held(self._OPEN, self._CLOSE) + kind = "token" + if len(self._buffer) > keep: + parts.append((kind, self._buffer[: len(self._buffer) - keep])) + self._buffer = self._buffer[len(self._buffer) - keep:] + break + return parts + + def flush(self) -> list[tuple[str, str]]: + rest, self._buffer = self._buffer, "" + return [("thinking" if self._inside else "token", rest)] if rest else [] + + +class _Round: + """One LLM call of the tool loop, assembled from its streamed deltas.""" + + def __init__(self, answered_before: bool, thought_before: bool) -> None: + self.raw = "" # content as the model sent it, tags included + self.answer = "" + self.thinking = "" + self.tool_calls: dict[int, dict] = {} + self._splitter = _ThinkSplitter() + self._answered_before = answered_before + self._thought_before = thought_before + + def _text(self, kind: str, text: str) -> dict | None: + # Each round's text is stripped at its start, as the non-streamed + # answer was, and set apart from what earlier rounds already showed. + if kind == "token": + if not self.answer: + text = text.lstrip() + if not text: + return None + self.answer = text + return {"type": "token", "text": ("\n\n" if self._answered_before else "") + text} + self.answer += text + return {"type": "token", "text": text} + if not self.thinking: + text = text.lstrip() + if not text: + return None + self.thinking = text + return {"type": "thinking", "text": ("\n\n---\n\n" if self._thought_before else "") + text} + self.thinking += text + return {"type": "thinking", "text": text} + + def apply(self, delta: dict) -> list[dict]: + events = [] + # llama-server sends reasoning_content; Ollama and OpenRouter, reasoning. + reasoning = delta.get("reasoning_content") or delta.get("reasoning") + if isinstance(reasoning, str) and reasoning: + events.append(self._text("thinking", reasoning)) + content = delta.get("content") + if isinstance(content, str) and content: + self.raw += content + events.extend(self._text(kind, text) for kind, text in self._splitter.feed(content)) + for position, fragment in enumerate(delta.get("tool_calls") or []): + # Streamed tool calls arrive in fragments keyed by index: the id and + # name come first, the JSON arguments in pieces after them. + call = self.tool_calls.setdefault( + fragment.get("index", position), + {"id": "", "type": "function", "function": {"name": "", "arguments": ""}}, + ) + call["id"] = fragment.get("id") or call["id"] + function = fragment.get("function") or {} + call["function"]["name"] += function.get("name") or "" + arguments = function.get("arguments") + if isinstance(arguments, dict): + arguments = json.dumps(arguments) + call["function"]["arguments"] += arguments or "" + return [e for e in events if e] + + def finish(self) -> list[dict]: + return [e for e in (self._text(k, t) for k, t in self._splitter.flush()) if e] + + def calls(self) -> list[dict]: + calls = [self.tool_calls[i] for i in sorted(self.tool_calls)] + for call in calls: + # The tool result must answer the call's id; a few servers omit it. + call["id"] = call["id"] or f"call_{uuid.uuid4().hex[:8]}" + return calls + + +async def _read_round(response: httpx.Response, round_: _Round): + """Yield the answer and reasoning of one LLM call as they stream in.""" + if "text/event-stream" not in response.headers.get("content-type", ""): + # A server that ignored "stream": true answers in one JSON body. + message = json.loads(await response.aread())["choices"][0]["message"] + for event in round_.apply(message): + yield event + else: + async for line in response.aiter_lines(): + if not line.startswith("data:"): + continue + data = line[len("data:"):].strip() + if data == "[DONE]": + break + try: + choices = json.loads(data).get("choices") or [] + except ValueError: + continue + if choices: + for event in round_.apply(choices[0].get("delta") or {}): + yield event + for event in round_.finish(): + yield event + + +def _done(message: str, actions: list[ActionDone], thinking: list[str], error: bool = False) -> dict: + return { + "type": "done", + "message": message, + "actions": [a.model_dump() for a in actions], + "thinking": "\n\n---\n\n".join(thinking) if thinking else None, + # The message reports a failure rather than answering: the chat shows + # it after whatever text already streamed. + "error": error, + } + + +def _release_claimed(claim: asyncio.Future) -> None: + if not claim.cancelled() and claim.exception() is None: + claim.result().release() + + +async def _claim_slot(model: str, spec: dict): + """Hold a local LLM slot, without leaking it if the caller is cancelled. + + A streamed chat is cancelled when its client leaves, possibly while the + model is still loading. The loading thread can't be stopped and claims the + slot anyway; nobody would release it, and a held slot is never reaped. + """ + claim = asyncio.ensure_future(asyncio.to_thread(llama_pool.ensure, model, spec, True)) + try: + return await asyncio.shield(claim) + except asyncio.CancelledError: + claim.add_done_callback(_release_claimed) + raise + + +async def _chat_events(request: AgentChatRequest): + """Run the tool loop, yielding its events as they happen. + + token / thinking — answer and reasoning text, in pieces + tool_start / tool_done — around each tool call + done — always last: the final answer, every action and the full reasoning + """ messages: list[dict] = [{"role": "system", "content": SYSTEM_PROMPT}] # Inject scene context so the LLM knows current state @@ -785,9 +962,10 @@ async def agent_chat(request: AgentChatRequest): spec = llm_server.resolve_model(request.model) # hold=True: the slot stays claimed for the whole loop, so neither the # idle reaper nor a model loading elsewhere can evict it mid-answer. - slot = await asyncio.to_thread(llama_pool.ensure, request.model, spec, True) + slot = await _claim_slot(request.model, spec) except Exception as e: - return AgentChatResponse(message=f"Could not start the local LLM: {e}") + yield _done(f"Could not start the local LLM: {e}", [], [], error=True) + return base_url = slot.base_url headers: dict = {} send_images = spec["vision"] @@ -796,7 +974,8 @@ async def agent_chat(request: AgentChatRequest): extra["chat_template_kwargs"] = {"enable_thinking": False} else: if not request.provider.base_url: - return AgentChatResponse(message="No provider URL configured. Set one in Settings → Agent.") + yield _done("No provider URL configured. Set one in Settings → Agent.", [], [], error=True) + return base_url = request.provider.base_url.rstrip("/") headers = _auth_headers(base_url, request.provider.api_key) @@ -805,52 +984,70 @@ async def agent_chat(request: AgentChatRequest): actions_done: list[ActionDone] = [] all_thinking: list[str] = [] - final: AgentChatResponse | None = None + final: str | None = None + error: str | None = None + answered = False try: async with httpx.AsyncClient(timeout=120.0) as client: - for _ in range(10): # max tool-call rounds - r = await client.post( - f"{base_url}/chat/completions", - headers=headers, - json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": False, **extra}, - ) - # An external model's vision support is unknown up front, and a - # text-only one rejects image parts with a 400 that failed the - # whole turn. Retry once with the images described as omitted. - if r.status_code == 400 and request.provider.type != "local" and _drop_images(messages): - r = await client.post( + + async def post() -> httpx.Response: + return await client.send( + client.build_request( + "POST", f"{base_url}/chat/completions", headers=headers, - json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": False, **extra}, - ) - - if r.status_code != 200: - return AgentChatResponse(message=f"LLM error ({r.status_code}): {_error_detail(r)}") + json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": True, **extra}, + ), + stream=True, + ) - msg = r.json()["choices"][0]["message"] - tool_calls = msg.get("tool_calls") or [] - assistant: dict = {"role": "assistant", "content": msg.get("content") or ""} + for _ in range(10): # max tool-call rounds + round_ = _Round(answered, bool(all_thinking)) + r = await post() + try: + # An external model's vision support is unknown up front, and a + # text-only one rejects image parts with a 400 that failed the + # whole turn. Retry once with the images described as omitted. + if r.status_code == 400 and request.provider.type != "local" and _drop_images(messages): + await r.aclose() + r = await post() + + if r.status_code != 200: + await r.aread() + error = f"LLM error ({r.status_code}): {_error_detail(r)}" + break + + async for event in _read_round(r, round_): + yield event + finally: + await r.aclose() + + answered = answered or bool(round_.answer) + if round_.thinking: + all_thinking.append(round_.thinking) + tool_calls = round_.calls() + assistant: dict = {"role": "assistant", "content": round_.raw} if tool_calls: assistant["tool_calls"] = tool_calls messages.append(assistant) - clean_content, thinking_text = _extract_thinking(msg) - if thinking_text: - all_thinking.append(thinking_text) - if not tool_calls: - final = AgentChatResponse(message=clean_content, actions=actions_done) + final = round_.answer.strip() break for tc in tool_calls: fn = tc["function"] - result_text, payload = await execute_tool(fn["name"], _tool_arguments(fn.get("arguments")), request.context) + arguments = _tool_arguments(fn.get("arguments")) + yield {"type": "tool_start", "id": tc["id"], "tool": fn["name"], "arguments": arguments} + result_text, payload = await execute_tool(fn["name"], arguments, request.context) actions_done.append(ActionDone(tool=fn["name"], result=result_text, payload=payload)) - messages.append({"role": "tool", "tool_call_id": tc.get("id", ""), "content": result_text}) + yield {"type": "tool_done", "id": tc["id"], "tool": fn["name"], "result": result_text, "payload": payload} + messages.append({"role": "tool", "tool_call_id": tc["id"], "content": result_text}) except httpx.HTTPError as e: - return AgentChatResponse(message=f"Could not reach the LLM at {base_url}: {e}", actions=actions_done) + error = f"Could not reach the LLM at {base_url}: {e}" finally: + # Also runs when the client disconnects mid-answer. if slot is not None: slot.release() @@ -858,7 +1055,34 @@ async def agent_chat(request: AgentChatRequest): # still holding ours would keep the agent's own model on the GPU. await _unload_llm_after_workflow(request, actions_done) - if final is None: - final = AgentChatResponse(message="Reached maximum tool iterations.", actions=actions_done) - final.thinking = "\n\n---\n\n".join(all_thinking) if all_thinking else None - return final + if error is not None: + yield _done(error, actions_done, all_thinking, error=True) + elif final is None: + yield _done("Reached maximum tool iterations.", actions_done, all_thinking, error=True) + else: + yield _done(final, actions_done, all_thinking) + + +@router.post("/chat", response_model=AgentChatResponse) +async def agent_chat(request: AgentChatRequest): + """The whole turn at once, for callers that don't read a stream.""" + done: dict = {} + async for event in _chat_events(request): + if event["type"] == "done": + done = event + return AgentChatResponse(message=done["message"], actions=done["actions"], thinking=done["thinking"]) + + +@router.post("/chat/stream") +async def agent_chat_stream(request: AgentChatRequest): + """The same turn as Server-Sent Events, one JSON event per `data:` line.""" + + async def body(): + async for event in _chat_events(request): + yield f"data: {json.dumps(event)}\n\n" + + return StreamingResponse( + body(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) diff --git a/api/tests/test_agent_streaming.py b/api/tests/test_agent_streaming.py new file mode 100644 index 000000000..123e93c2c --- /dev/null +++ b/api/tests/test_agent_streaming.py @@ -0,0 +1,293 @@ +import asyncio +import json +import threading +import unittest +from unittest import mock + +import httpx + +import routers.agent as agent + + +def _sse(*deltas: dict) -> httpx.Response: + """A streamed /chat/completions answer, one chunk per delta.""" + lines = [f"data: {json.dumps({'choices': [{'delta': d}]})}\n\n" for d in deltas] + lines.append("data: [DONE]\n\n") + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content="".join(lines).encode(), + ) + + +def _call(index: int, **fields) -> dict: + return {"tool_calls": [{"index": index, **fields}]} + + +_WORKFLOWS = {"workflows": [{"id": "wf1", "name": "My Workflow"}]} + + +class _Scripted: + """Patches httpx.AsyncClient to serve scripted responses in order, recording + every request body.""" + + def __init__(self, *responses: httpx.Response) -> None: + self._responses = list(responses) + self.bodies: list[dict] = [] + self._real = httpx.AsyncClient + + def _handler(self, request: httpx.Request) -> httpx.Response: + self.bodies.append(json.loads(request.content)) + return self._responses.pop(0) + + def __call__(self, *args, **kwargs): + kwargs["transport"] = httpx.MockTransport(self._handler) + return self._real(*args, **kwargs) + + +def _external(context=None) -> agent.AgentChatRequest: + return agent.AgentChatRequest( + messages=[agent.ChatMessage(role="user", content="hi")], + model="gpt-test", + provider=agent.ProviderConfig(type="external", base_url="https://llm.test/v1", api_key="k"), + context=context or {}, + ) + + +def _events(scripted: _Scripted, request: agent.AgentChatRequest) -> list[dict]: + async def collect(): + return [event async for event in agent._chat_events(request)] + + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + return asyncio.run(collect()) + + +def _text(events: list[dict], kind: str) -> str: + return "".join(e["text"] for e in events if e["type"] == kind) + + +class ChatEventTests(unittest.TestCase): + def test_answer_streams_token_by_token_then_done(self) -> None: + scripted = _Scripted(_sse({"content": "Hel"}, {"content": "lo"}, {"content": " there"})) + + events = _events(scripted, _external()) + + self.assertEqual([e["type"] for e in events], ["token", "token", "token", "done"]) + self.assertEqual(_text(events, "token"), "Hello there") + self.assertEqual(events[-1]["message"], "Hello there") + self.assertFalse(events[-1]["error"]) + self.assertTrue(scripted.bodies[0]["stream"]) + + def test_reasoning_field_streams_as_thinking(self) -> None: + events = _events( + _Scripted(_sse({"reasoning_content": "Let me "}, {"reasoning_content": "think."}, {"content": "Hi"})), + _external(), + ) + + self.assertEqual(_text(events, "thinking"), "Let me think.") + self.assertEqual(_text(events, "token"), "Hi") + self.assertEqual(events[-1]["thinking"], "Let me think.") + + def test_inline_think_tags_cut_across_chunks_are_split_out(self) -> None: + events = _events( + _Scripted(_sse({"content": "plan\n\nHello "})), + _external(), + ) + + self.assertEqual(_text(events, "thinking"), "plan") + # A "<" that turns out not to open a tag is still part of the answer. + self.assertEqual(_text(events, "token"), "Hello ") + self.assertEqual(events[-1]["message"], "Hello ") + + def test_fragmented_tool_call_is_reassembled_and_reported_around_its_run(self) -> None: + scripted = _Scripted( + _sse( + {"content": "Starting it."}, + _call(0, id="call_1", type="function", function={"name": "run_workflow", "arguments": ""}), + _call(0, function={"arguments": '{"workflow_'}), + _call(0, function={"arguments": 'id": "wf1"}'}), + ), + _sse({"content": "Done."}), + ) + + events = _events(scripted, _external(_WORKFLOWS)) + + types = [e["type"] for e in events] + self.assertEqual(types, ["token", "tool_start", "tool_done", "token", "done"]) + start, finish = events[1], events[2] + self.assertEqual((start["id"], start["tool"], start["arguments"]), ("call_1", "run_workflow", {"workflow_id": "wf1"})) + self.assertEqual(finish["payload"]["type"], "run_workflow") + # Rounds are set apart in the streamed text; the final answer is the last one. + self.assertEqual(_text(events, "token"), "Starting it.\n\nDone.") + self.assertEqual(events[-1]["message"], "Done.") + self.assertEqual([a["tool"] for a in events[-1]["actions"]], ["run_workflow"]) + + replay = scripted.bodies[1]["messages"] + assistant = next(m for m in replay if m["role"] == "assistant") + self.assertEqual(assistant["tool_calls"][0]["function"]["arguments"], '{"workflow_id": "wf1"}') + self.assertEqual(replay[-1], {"role": "tool", "tool_call_id": "call_1", "content": finish["result"]}) + + def test_tool_call_without_an_id_still_gets_its_result_matched(self) -> None: + scripted = _Scripted( + _sse(_call(0, function={"name": "run_workflow", "arguments": '{"workflow_id": "wf1"}'})), + _sse({"content": "ok"}), + ) + + events = _events(scripted, _external(_WORKFLOWS)) + + call_id = events[0]["id"] + self.assertTrue(call_id) + tool_message = scripted.bodies[1]["messages"][-1] + self.assertEqual(tool_message["tool_call_id"], call_id) + + def test_reasoning_of_successive_rounds_is_separated(self) -> None: + scripted = _Scripted( + _sse({"reasoning_content": "first"}, _call(0, id="c", function={"name": "list_workflows", "arguments": "{}"})), + _sse({"reasoning_content": "second"}, {"content": "ok"}), + ) + + with mock.patch.object(agent, "execute_tool", mock.AsyncMock(return_value=("none", None))): + events = _events(scripted, _external()) + + self.assertEqual(_text(events, "thinking"), "first\n\n---\n\nsecond") + self.assertEqual(events[-1]["thinking"], "first\n\n---\n\nsecond") + + def test_provider_error_ends_with_an_error_done(self) -> None: + events = _events( + _Scripted(httpx.Response(401, json={"error": {"message": "Incorrect API key provided."}})), + _external(), + ) + + self.assertEqual(len(events), 1) + self.assertTrue(events[0]["error"]) + self.assertEqual(events[0]["message"], "LLM error (401): Incorrect API key provided.") + + def test_tool_loop_limit_is_reported_as_an_error(self) -> None: + rounds = [ + _sse(_call(0, id=f"c{i}", function={"name": "list_workflows", "arguments": "{}"})) + for i in range(10) + ] + with mock.patch.object(agent, "execute_tool", mock.AsyncMock(return_value=("none", None))): + events = _events(_Scripted(*rounds), _external()) + + self.assertTrue(events[-1]["error"]) + self.assertEqual(events[-1]["message"], "Reached maximum tool iterations.") + self.assertEqual(len(events[-1]["actions"]), 10) + + def test_local_slot_is_released_when_the_client_disconnects(self) -> None: + class Slot: + base_url = "http://127.0.0.1:8791/v1" + released = 0 + + def release(self) -> None: + self.released += 1 + + slot = Slot() + request = agent.AgentChatRequest( + messages=[agent.ChatMessage(role="user", content="hi")], model="qwen3-4b" + ) + + async def first_event_then_disconnect(): + stream = agent._chat_events(request) + first = await stream.__anext__() + await stream.aclose() + return first + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_sse({"content": "a"}, {"content": "b"}))), \ + mock.patch.object(agent.llm_server, "resolve_model", return_value={"vision": False}), \ + mock.patch.object(agent.llama_pool, "ensure", return_value=slot), \ + mock.patch.object(agent.llama_pool, "unload_all"): + first = asyncio.run(first_event_then_disconnect()) + + self.assertEqual(first, {"type": "token", "text": "a"}) + self.assertEqual(slot.released, 1) + + def test_slot_claimed_after_the_client_left_during_loading_is_released(self) -> None: + class Slot: + base_url = "http://127.0.0.1:8791/v1" + released = 0 + + def release(self) -> None: + self.released += 1 + + slot = Slot() + loading, loaded = threading.Event(), threading.Event() + + def ensure(*_args): + loading.set() + loaded.wait(5) + return slot + + request = agent.AgentChatRequest( + messages=[agent.ChatMessage(role="user", content="hi")], model="qwen3-4b" + ) + + async def leave_while_loading(): + turn = asyncio.ensure_future(agent._chat_events(request).__anext__()) + await asyncio.to_thread(loading.wait, 5) + turn.cancel() + with self.assertRaises(asyncio.CancelledError): + await turn + loaded.set() # the model finishes loading after the client left + for _ in range(200): + if slot.released: + break + await asyncio.sleep(0.01) + + with mock.patch.object(agent.llm_server, "resolve_model", return_value={"vision": False}), \ + mock.patch.object(agent.llama_pool, "ensure", ensure): + asyncio.run(leave_while_loading()) + + self.assertEqual(slot.released, 1) + + +class StreamRouteTests(unittest.TestCase): + def test_stream_route_sends_one_json_event_per_data_line(self) -> None: + async def read_body(): + response = await agent.agent_chat_stream(_external()) + self.assertEqual(response.media_type, "text/event-stream") + return "".join([chunk async for chunk in response.body_iterator]) + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_sse({"content": "Hi"}))): + body = asyncio.run(read_body()) + + blocks = [b for b in body.split("\n\n") if b] + self.assertTrue(all(b.startswith("data: ") for b in blocks)) + events = [json.loads(b[len("data: "):]) for b in blocks] + self.assertEqual([e["type"] for e in events], ["token", "done"]) + + def test_json_route_returns_the_same_turn_at_once(self) -> None: + scripted = _Scripted( + _sse(_call(0, id="call_1", function={"name": "run_workflow", "arguments": '{"workflow_id": "wf1"}'})), + _sse({"reasoning_content": "hm"}, {"content": "Running."}), + ) + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + response = asyncio.run(agent.agent_chat(_external(_WORKFLOWS))) + + self.assertEqual(response.message, "Running.") + self.assertEqual(response.thinking, "hm") + self.assertEqual([a.tool for a in response.actions], ["run_workflow"]) + + +class ThinkSplitterTests(unittest.TestCase): + def test_partial_tag_is_held_until_it_resolves(self) -> None: + splitter = agent._ThinkSplitter() + + self.assertEqual(splitter.feed("a<"), [("token", "a")]) + self.assertEqual(splitter.feed("think>b"), [("thinking", "b")]) + self.assertEqual(splitter.feed(" None: + # The chat template opened in the prompt, so only its end streams. + splitter = agent._ThinkSplitter() + self.assertEqual(splitter.feed("plan\n\nAnswer"), [("thinking", "plan"), ("token", "\n\nAnswer")]) + + # Reasoning that streamed before the tag was already shown; the tag is not. + splitter = agent._ThinkSplitter() + parts = splitter.feed("planAnswer") + splitter.flush() + self.assertEqual(parts, [("token", "plan"), ("token", "Answer")]) + + +if __name__ == "__main__": + unittest.main() diff --git a/package.json b/package.json index a63727a24..e8da0f9d5 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/components/ChatPanel.tsx b/src/areas/generate/components/ChatPanel.tsx index 053e780a9..d6db103e2 100644 --- a/src/areas/generate/components/ChatPanel.tsx +++ b/src/areas/generate/components/ChatPanel.tsx @@ -6,11 +6,13 @@ import { useWorkflowsStore } from '@shared/stores/workflowsStore' import { useExtensionsStore } from '@shared/stores/extensionsStore' import { useWorkflowRunStore } from '@areas/workflows/workflowRunStore' import { buildAllWorkflowExtensions } from '@areas/workflows/mockExtensions' +import { readAgentEvents } from '@shared/utils/agentStream' // ─── Types ──────────────────────────────────────────────────────────────────── import type { ThinkingMode } from '@shared/stores/agentStore' import type { Workflow } from '@shared/types/electron.d' +import type { AgentAction, AgentActionPayload } from '@shared/utils/agentStream' interface Message { id: string @@ -19,19 +21,12 @@ interface Message { thinking?: string imageDataUrls?: string[] actions?: ActionDone[] + streaming?: boolean } -interface ActionDone { - tool: string - result: string - payload?: { - type: string - url?: string - face_count?: number - workflow_id?: string - workflow_name?: string - workflow?: Omit - } | null +interface ActionDone extends AgentAction { + id?: string + running?: boolean } // ─── Constants ──────────────────────────────────────────────────────────────── @@ -86,15 +81,23 @@ const TOOL_LABELS: Record = { function ActionsCard({ actions, onUndo }: { actions: ActionDone[]; onUndo?: () => void }): JSX.Element { const [expanded, setExpanded] = useState(false) const meshActions = actions.filter((a) => a.payload?.type === 'mesh_update') - const canUndo = meshActions.length > 0 && !!onUndo + const running = actions.find((a) => a.running) + const canUndo = meshActions.length > 0 && !!onUndo && !running return (
{/* Header */}
- - {actions.length} action{actions.length > 1 ? 's' : ''} performed - + {running ? ( + + + Running {running.tool.replace(/_/g, ' ')}… + + ) : ( + + {actions.length} action{actions.length > 1 ? 's' : ''} performed + + )}
{canUndo && ( + )} + > + {engineInstall && } + {engineError &&

{engineError}

} + + )} {engineInstalled === null && ( From cd16feb0355c71c72b09889c7a7b0f9177ca03be Mon Sep 17 00:00:00 2001 From: Lorchie Date: Mon, 26 Oct 2026 16:48:02 +0100 Subject: [PATCH 10/15] test(mesh-ops): fix two tests failing on a clean dev checkout - test_auto_orient_does_not_flip_a_mesh_over: trimesh's update_faces keeps the cut-away vertices, so the exported bowl still carried the whole sphere's vertices and the "center" was measured on the orphan top cap. Drop unreferenced vertices; auto_orient itself was correct. - test_mesh_ops_run_without_undeclared_networkx runs repair_mesh, which needs pymeshlab: skip it when pymeshlab is missing, like the other repair/inspection tests. --- api/tests/test_mesh_ops_dependencies.py | 3 +++ api/tests/test_mesh_ops_transforms.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/api/tests/test_mesh_ops_dependencies.py b/api/tests/test_mesh_ops_dependencies.py index 3b0d4517d..96db84105 100644 --- a/api/tests/test_mesh_ops_dependencies.py +++ b/api/tests/test_mesh_ops_dependencies.py @@ -1,3 +1,4 @@ +import importlib.util import re import subprocess import sys @@ -6,6 +7,7 @@ API_DIR = Path(__file__).resolve().parents[1] +HAS_PYMESHLAB = importlib.util.find_spec("pymeshlab") is not None # Runs every mesh-op family on a holed box with networkx unimportable: it is # not in requirements.txt, so the shipped venv never has it, while a dev venv @@ -44,6 +46,7 @@ class MeshOpDependencyTests(unittest.TestCase): + @unittest.skipUnless(HAS_PYMESHLAB, "pymeshlab is not installed") def test_mesh_ops_run_without_undeclared_networkx(self) -> None: result = subprocess.run( [sys.executable, "-c", _SCRIPT], diff --git a/api/tests/test_mesh_ops_transforms.py b/api/tests/test_mesh_ops_transforms.py index 21407209b..8059a4233 100644 --- a/api/tests/test_mesh_ops_transforms.py +++ b/api/tests/test_mesh_ops_transforms.py @@ -98,6 +98,9 @@ def test_auto_orient_does_not_flip_a_mesh_over(self) -> None: # A shallow bowl (open hemisphere-like dish) opening upwards. bowl = trimesh.creation.icosphere(subdivisions=3) bowl.update_faces(bowl.triangles_center[:, 1] < -0.3) + # update_faces keeps the cut-away vertices, which would still be + # exported and measured below as the top of a whole sphere. + bowl.remove_unreferenced_vertices() bowl.apply_scale([1.0, 0.3, 1.0]) bowl.apply_transform( trimesh.transformations.rotation_matrix(np.radians(10), [1, 0, 0]) From 86a23dc0b8d6b40da16f341e2c5bd457af110b18 Mon Sep 17 00:00:00 2001 From: ahgsql Date: Fri, 30 Oct 2026 23:28:43 +0300 Subject: [PATCH 11/15] Keep TripoSG and the texture model loaded together switch_model unloads the previous model on every switch. When you run TripoSG and then Trellis2 Texture Mesh on the same object, both models get reloaded for every single asset, which cost me 20-27 s per switch on a 3090 Ti. This keeps the two loaded side by side. Any other switch, and the LLM eviction, works exactly as before. Two assets (cart + soldier, 3000 faces, warm): 197 s -> 110 s. --- api/services/generator_registry.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/services/generator_registry.py b/api/services/generator_registry.py index 2792030e9..523512493 100644 --- a/api/services/generator_registry.py +++ b/api/services/generator_registry.py @@ -38,6 +38,10 @@ weight_group_sources_are_downloaded, ) +# Model pairs that stay loaded together when the active model switches between them (a mesh -> texture +# pipeline would otherwise unload and reload both models on every asset). +_KEEP_RESIDENT_PAIRS = [{"triposg/generate", "trellis2/refine"}] + # ------------------------------------------------------------------ # # Global paths # ------------------------------------------------------------------ # @@ -904,7 +908,10 @@ def switch_model(self, model_id: str) -> None: llama_pool.unload_all() if model_id != self._active_id: - if self._active_id in self._generators: + # Keep the geometry model and the texture model resident together instead of unloading + # the previous one on every switch (see _KEEP_RESIDENT_PAIRS). + if (self._active_id in self._generators + and {self._active_id, model_id} not in _KEEP_RESIDENT_PAIRS): self._generators[self._active_id].unload() self._active_id = model_id From 7f13d137bc3e1964f57ba6709035e021eefadb70 Mon Sep 17 00:00:00 2001 From: ahgsql Date: Fri, 30 Oct 2026 23:28:43 +0300 Subject: [PATCH 12/15] modly-cli: start the API with the same setup as the desktop app `dev serve-api` / `dev ensure-server --start` ignored dependenciesDir from settings.json. If the dependencies live on another drive (mine are on F:), the venv isn't found and it falls back to whatever Python runs the CLI, which doesn't have the backend deps. It now checks dependenciesDir first, and passes the env vars the Electron bridge sets: PYTHONIOENCODING=utf-8, MODLY_LLM_DIR (agentDir) and MODLY_NODE_EXECUTABLE (Modly.exe) for the installed app. Added two tests; the existing ones still pass. --- tools/modly-cli/agent.py | 19 ++++++++++++++++--- tools/modly-cli/test_agent.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/tools/modly-cli/agent.py b/tools/modly-cli/agent.py index e424aa25b..09f35755c 100644 --- a/tools/modly-cli/agent.py +++ b/tools/modly-cli/agent.py @@ -283,11 +283,16 @@ def _default_api_dir() -> Path | None: return None -def _default_python(api_dir: Path) -> Path | None: +def _default_python(api_dir: Path, settings: dict[str, Any] | None = None) -> Path | None: candidates = [ api_dir / ".venv" / "Scripts" / "python.exe", api_dir / ".venv" / "bin" / "python", ] + # The desktop app can keep its dependencies on another drive (Settings > dependenciesDir). + deps_dir = (settings or {}).get("dependenciesDir") + if deps_dir: + candidates.append(Path(deps_dir) / "venv" / "Scripts" / "python.exe") + candidates.append(Path(deps_dir) / "venv" / "bin" / "python") for appdata in _windows_env_paths("APPDATA"): candidates.append(appdata / "Modly" / "dependencies" / "venv" / "Scripts" / "python.exe") candidates.append(Path(sys.executable)) @@ -317,11 +322,11 @@ def _resolve_serve_config(args: argparse.Namespace) -> tuple[Path, Path, dict[st if not api_dir or not (api_dir / "main.py").exists(): raise ModlyCliError("Could not find Modly api directory; pass --api-dir") - python = Path(args.python).expanduser().resolve() if getattr(args, "python", None) else _default_python(api_dir) + settings = _load_modly_settings() + python = Path(args.python).expanduser().resolve() if getattr(args, "python", None) else _default_python(api_dir, settings) if not python or not python.exists(): raise ModlyCliError("Could not find Modly Python environment; pass --python") - settings = _load_modly_settings() env = os.environ.copy() hf_token = getattr(args, "hf_token", None) or settings.get("hfToken") or os.environ.get("HF_TOKEN", "") env.update({ @@ -332,7 +337,15 @@ def _resolve_serve_config(args: argparse.Namespace) -> tuple[Path, Path, dict[st "SELECTED_MODEL_ID": getattr(args, "model", None) or os.environ.get("SELECTED_MODEL_ID", ""), "HUGGING_FACE_HUB_TOKEN": hf_token, "HF_TOKEN": hf_token, + # Same as the Electron PythonBridge: UTF-8 logs, the local LLM folder, and Modly.exe as the Node + # runtime of the mesh optimizer (installed app only; otherwise mesh_ops falls back to `node`). + "PYTHONIOENCODING": "utf-8", }) + if settings.get("agentDir") and "MODLY_LLM_DIR" not in os.environ: + env["MODLY_LLM_DIR"] = settings["agentDir"] + modly_exe = api_dir.parent.parent / ("Modly.exe" if os.name == "nt" else "modly") + if modly_exe.exists() and "MODLY_NODE_EXECUTABLE" not in os.environ: + env["MODLY_NODE_EXECUTABLE"] = str(modly_exe) cmd = [str(python), "-m", "uvicorn", "main:app", "--host", args.host, "--port", str(args.port)] base_url = f"http://{args.host}:{args.port}" return api_dir, python, env, cmd, base_url diff --git a/tools/modly-cli/test_agent.py b/tools/modly-cli/test_agent.py index 9537ec1af..432492bab 100644 --- a/tools/modly-cli/test_agent.py +++ b/tools/modly-cli/test_agent.py @@ -483,6 +483,35 @@ def test_malformed_timeout_environment_does_not_break_module_import(self) -> Non self.assertEqual(module.DEFAULT_TIMEOUT_SECONDS, 1800) self.assertEqual(module.DEFAULT_POLL_SECONDS, 2.0) + def test_default_python_uses_settings_dependencies_dir(self) -> None: + with tempfile.TemporaryDirectory() as td: + api_dir = Path(td) / "api" + api_dir.mkdir() + deps_python = Path(td) / "deps" / "venv" / "Scripts" / "python.exe" + deps_python.parent.mkdir(parents=True) + deps_python.write_text("# python") + with patch.object(agent, "_windows_env_paths", return_value=[]): + found = agent._default_python(api_dir, {"dependenciesDir": str(Path(td) / "deps")}) + self.assertEqual(found, deps_python) + + def test_resolve_serve_config_matches_desktop_env(self) -> None: + with tempfile.TemporaryDirectory() as td: + api_dir = Path(td) / "resources" / "api" + api_dir.mkdir(parents=True) + (api_dir / "main.py").write_text("# api") + (Path(td) / ("Modly.exe" if os.name == "nt" else "modly")).write_text("# exe") + python = Path(td) / "python.exe" + python.write_text("# python") + args = SimpleNamespace(api_dir=str(api_dir), python=str(python), host="127.0.0.1", port=8765, + models_dir=None, workspace_dir=None, extensions_dir=None, model=None, hf_token=None) + clean = {k: v for k, v in os.environ.items() if k not in ("MODLY_LLM_DIR", "MODLY_NODE_EXECUTABLE")} + with patch.dict(os.environ, clean, clear=True), \ + patch.object(agent, "_load_modly_settings", return_value={"agentDir": "D:/modly/agent"}): + _api_dir, _python, env, _cmd, _url = agent._resolve_serve_config(args) + self.assertEqual(env["PYTHONIOENCODING"], "utf-8") + self.assertEqual(env["MODLY_LLM_DIR"], "D:/modly/agent") + self.assertTrue(env["MODLY_NODE_EXECUTABLE"].lower().endswith(("modly.exe", "modly"))) + def test_resolve_serve_config_explicit_paths(self) -> None: with tempfile.TemporaryDirectory() as td: api_dir = Path(td) / "api" From 4bf4795319bf6bb045be18179219656ca8b75f92 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Sat, 31 Oct 2026 22:11:25 +0100 Subject: [PATCH 13/15] fix(registry): unload stale models when leaving a resident pair, gate pair on VRAM switch_model only unloaded the previous active model. After TripoSG -> Trellis2 refine -> any other model, TripoSG was no longer active but stayed loaded beside the new model until unload_all. switch_model now unloads every loaded model except the target and its resident partners. Keeping both models resident also assumed a card with room for both (~10-11 GB peak). Each pair now declares a minimum total VRAM; below it, or when no NVIDIA GPU answers, switching behaves as before. The pair is set to 19.5 GiB (24 GB cards), the configuration the speedup was measured on. Manifests can't size this: TripoSG declares no vram_gb. --- api/services/generator_registry.py | 33 +++++++++---- api/tests/test_generator_registry.py | 70 ++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 8 deletions(-) diff --git a/api/services/generator_registry.py b/api/services/generator_registry.py index 523512493..40735ce19 100644 --- a/api/services/generator_registry.py +++ b/api/services/generator_registry.py @@ -39,8 +39,11 @@ ) # Model pairs that stay loaded together when the active model switches between them (a mesh -> texture -# pipeline would otherwise unload and reload both models on every asset). -_KEEP_RESIDENT_PAIRS = [{"triposg/generate", "trellis2/refine"}] +# pipeline would otherwise unload and reload both models on every asset), with the minimum total VRAM +# (GiB, as detect_vram_gb() reports it) for keeping both resident. Manifests can't size this: TripoSG +# declares no `vram_gb`. TripoSG + Trellis2 refine peak around 10-11 GB, so only 24 GB cards +# (23.9 GiB reported) qualify; smaller cards keep the unload-on-switch behavior. +_KEEP_RESIDENT_PAIRS = [({"triposg/generate", "trellis2/refine"}, 19.5)] # ------------------------------------------------------------------ # # Global paths @@ -888,7 +891,7 @@ def get_manifest(self, model_id: str) -> dict: return self._manifests[model_id] def switch_model(self, model_id: str) -> None: - """Switches the active model. Unloads the previous one if different.""" + """Switches the active model. Unloads every other model unless it is a resident partner.""" with self._lifecycle_lock: self._assert_not_quarantined(model_id) if model_id not in self._generators: @@ -908,13 +911,27 @@ def switch_model(self, model_id: str) -> None: llama_pool.unload_all() if model_id != self._active_id: - # Keep the geometry model and the texture model resident together instead of unloading - # the previous one on every switch (see _KEEP_RESIDENT_PAIRS). - if (self._active_id in self._generators - and {self._active_id, model_id} not in _KEEP_RESIDENT_PAIRS): - self._generators[self._active_id].unload() + # Unload everything except the target and its resident partners, not just the + # previous active model: after A -> B (pair) -> C, A is no longer active but is + # still loaded, and would otherwise sit in VRAM beside C. + keep = {model_id} | self._resident_partners(model_id) + for other_id, gen in self._generators.items(): + if other_id not in keep and gen.is_loaded(): + gen.unload() self._active_id = model_id + def _resident_partners(self, model_id: str) -> Set[str]: + """Models allowed to stay loaded beside ``model_id`` (see _KEEP_RESIDENT_PAIRS).""" + from services.llm_server import detect_vram_gb + partners: Set[str] = set() + for pair, min_vram_gb in _KEEP_RESIDENT_PAIRS: + if model_id not in pair: + continue + # detect_vram_gb() is 0.0 when no NVIDIA GPU answers, which never passes. + if detect_vram_gb() >= min_vram_gb: + partners |= pair - {model_id} + return partners + # ------------------------------------------------------------------ # # Status # ------------------------------------------------------------------ # diff --git a/api/tests/test_generator_registry.py b/api/tests/test_generator_registry.py index b21253b7c..53c38cd7a 100644 --- a/api/tests/test_generator_registry.py +++ b/api/tests/test_generator_registry.py @@ -7,6 +7,7 @@ import threading import unittest from pathlib import Path +from unittest import mock import services.generator_registry as registry_module from services.extension_process import ExtensionProcess @@ -892,5 +893,74 @@ def read_status() -> None: self.assertFalse(results["model"]["loaded"]) +class _FakeGenerator: + def __init__(self) -> None: + self.loaded = True + + def is_loaded(self) -> bool: + return self.loaded + + def unload(self) -> None: + self.loaded = False + + +class KeepResidentPairTests(unittest.TestCase): + GEOMETRY = "triposg/generate" + TEXTURE = "trellis2/refine" + OTHER = "other/generate" + + def setUp(self) -> None: + self.registry = GeneratorRegistry() + self.gens = {model_id: _FakeGenerator() for model_id in (self.GEOMETRY, self.TEXTURE, self.OTHER)} + self.registry._generators = dict(self.gens) + self.registry._active_id = self.GEOMETRY + self.gens[self.TEXTURE].loaded = False + self.gens[self.OTHER].loaded = False + patches = [ + mock.patch("services.llm_server.llama_pool.unload_all"), + mock.patch("services.llm_server.detect_vram_gb", return_value=23.9), + ] + for patch in patches: + patch.start() + self.addCleanup(patch.stop) + + def _switch_and_load(self, model_id: str) -> None: + self.registry.switch_model(model_id) + self.gens[model_id].loaded = True + + def _loaded(self) -> set: + return {model_id for model_id, gen in self.gens.items() if gen.loaded} + + def test_pair_stays_loaded_when_switching_inside_it(self) -> None: + self._switch_and_load(self.TEXTURE) + self._switch_and_load(self.GEOMETRY) + + self.assertEqual(self._loaded(), {self.GEOMETRY, self.TEXTURE}) + + def test_leaving_the_pair_unloads_both_models(self) -> None: + self._switch_and_load(self.TEXTURE) + self._switch_and_load(self.OTHER) + + self.assertEqual(self._loaded(), {self.OTHER}) + + def test_pair_is_not_kept_when_vram_is_too_small(self) -> None: + with mock.patch("services.llm_server.detect_vram_gb", return_value=11.9): + self._switch_and_load(self.TEXTURE) + + self.assertEqual(self._loaded(), {self.TEXTURE}) + + def test_pair_is_not_kept_when_vram_is_unknown(self) -> None: + with mock.patch("services.llm_server.detect_vram_gb", return_value=0.0): + self._switch_and_load(self.TEXTURE) + + self.assertEqual(self._loaded(), {self.TEXTURE}) + + def test_pair_is_not_kept_on_a_16_gb_card(self) -> None: + with mock.patch("services.llm_server.detect_vram_gb", return_value=15.9): + self._switch_and_load(self.TEXTURE) + + self.assertEqual(self._loaded(), {self.TEXTURE}) + + if __name__ == "__main__": unittest.main() From 2fcc0fb058eb99838ed362e35c4c4ca5386727e5 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Sat, 31 Oct 2026 22:32:34 +0100 Subject: [PATCH 14/15] modly-cli: drop the caller's Python env and prefer the app venv, like the desktop bridge The started API inherited PYTHONHOME, PYTHONPATH, VIRTUAL_ENV, CONDA_PREFIX and the PIP_* variables of whatever shell ran the CLI. From a conda env or an activated venv, these point Modly's venv interpreter at foreign site-packages, or keep it from starting at all. They are now removed, the same list Electron's cleanPythonEnv() strips. _default_python also tried the api's dev .venv before the app's venv, the reverse of the PythonBridge. The app's venv (dependenciesDir, then the default dependencies folder) now comes first, the dev .venv second. --- tools/modly-cli/agent.py | 21 +++++++++++++++------ tools/modly-cli/test_agent.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/tools/modly-cli/agent.py b/tools/modly-cli/agent.py index 09f35755c..b382272be 100644 --- a/tools/modly-cli/agent.py +++ b/tools/modly-cli/agent.py @@ -284,10 +284,8 @@ def _default_api_dir() -> Path | None: def _default_python(api_dir: Path, settings: dict[str, Any] | None = None) -> Path | None: - candidates = [ - api_dir / ".venv" / "Scripts" / "python.exe", - api_dir / ".venv" / "bin" / "python", - ] + # Same order as the Electron PythonBridge: the app's venv first, the api's dev .venv as fallback. + candidates: list[Path] = [] # The desktop app can keep its dependencies on another drive (Settings > dependenciesDir). deps_dir = (settings or {}).get("dependenciesDir") if deps_dir: @@ -295,6 +293,8 @@ def _default_python(api_dir: Path, settings: dict[str, Any] | None = None) -> Pa candidates.append(Path(deps_dir) / "venv" / "bin" / "python") for appdata in _windows_env_paths("APPDATA"): candidates.append(appdata / "Modly" / "dependencies" / "venv" / "Scripts" / "python.exe") + candidates.append(api_dir / ".venv" / "Scripts" / "python.exe") + candidates.append(api_dir / ".venv" / "bin" / "python") candidates.append(Path(sys.executable)) for candidate in candidates: if candidate.exists(): @@ -317,6 +317,13 @@ def _load_modly_settings() -> dict[str, Any]: return {} +_FOREIGN_PYTHON_ENV = ( + "PYTHONHOME", "PYTHONPATH", "PYTHONSTARTUP", "PYTHONUSERBASE", + "PIP_USER", "PIP_TARGET", "PIP_PREFIX", "PIP_REQUIRE_VIRTUALENV", + "VIRTUAL_ENV", "CONDA_PREFIX", +) + + def _resolve_serve_config(args: argparse.Namespace) -> tuple[Path, Path, dict[str, str], list[str], str]: api_dir = Path(args.api_dir).expanduser().resolve() if getattr(args, "api_dir", None) else _default_api_dir() if not api_dir or not (api_dir / "main.py").exists(): @@ -327,8 +334,10 @@ def _resolve_serve_config(args: argparse.Namespace) -> tuple[Path, Path, dict[st if not python or not python.exists(): raise ModlyCliError("Could not find Modly Python environment; pass --python") - env = os.environ.copy() - hf_token = getattr(args, "hf_token", None) or settings.get("hfToken") or os.environ.get("HF_TOKEN", "") + # Like Electron's cleanPythonEnv(): variables from the caller's own Python setup (a conda env, + # an activated venv) would point Modly's venv interpreter at foreign site-packages. + env = {k: v for k, v in os.environ.items() if k not in _FOREIGN_PYTHON_ENV} + hf_token =getattr(args, "hf_token", None) or settings.get("hfToken") or os.environ.get("HF_TOKEN", "") env.update({ "PYTHONUNBUFFERED": "1", "MODELS_DIR": getattr(args, "models_dir", None) or settings.get("modelsDir") or str(Path.home() / ".modly" / "models"), diff --git a/tools/modly-cli/test_agent.py b/tools/modly-cli/test_agent.py index 432492bab..990bfbb0b 100644 --- a/tools/modly-cli/test_agent.py +++ b/tools/modly-cli/test_agent.py @@ -494,6 +494,39 @@ def test_default_python_uses_settings_dependencies_dir(self) -> None: found = agent._default_python(api_dir, {"dependenciesDir": str(Path(td) / "deps")}) self.assertEqual(found, deps_python) + def test_default_python_prefers_app_venv_over_api_dev_venv(self) -> None: + with tempfile.TemporaryDirectory() as td: + api_dir = Path(td) / "api" + dev_python = api_dir / ".venv" / "Scripts" / "python.exe" + dev_python.parent.mkdir(parents=True) + dev_python.write_text("# python") + deps_python = Path(td) / "deps" / "venv" / "Scripts" / "python.exe" + deps_python.parent.mkdir(parents=True) + deps_python.write_text("# python") + with patch.object(agent, "_windows_env_paths", return_value=[]): + found = agent._default_python(api_dir, {"dependenciesDir": str(Path(td) / "deps")}) + fallback = agent._default_python(api_dir, {}) + self.assertEqual(found, deps_python) + self.assertEqual(fallback, dev_python) + + def test_resolve_serve_config_drops_callers_python_env(self) -> None: + with tempfile.TemporaryDirectory() as td: + api_dir = Path(td) / "api" + api_dir.mkdir() + (api_dir / "main.py").write_text("# api") + python = Path(td) / "python.exe" + python.write_text("# python") + args = SimpleNamespace(api_dir=str(api_dir), python=str(python), host="127.0.0.1", port=8765, + models_dir=None, workspace_dir=None, extensions_dir=None, model=None, hf_token=None) + foreign = {"PYTHONHOME": "C:/conda", "PYTHONPATH": "C:/elsewhere", "VIRTUAL_ENV": "C:/venv", + "CONDA_PREFIX": "C:/conda", "PIP_USER": "1"} + with patch.dict(os.environ, foreign), \ + patch.object(agent, "_load_modly_settings", return_value={}): + _api_dir, _python, env, _cmd, _url = agent._resolve_serve_config(args) + for key in foreign: + self.assertNotIn(key, env) + self.assertIn("PATH", env) + def test_resolve_serve_config_matches_desktop_env(self) -> None: with tempfile.TemporaryDirectory() as td: api_dir = Path(td) / "resources" / "api" From 5ee7dea126d95172e211619dc95018088ad9e214 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Sun, 1 Nov 2026 10:41:26 +0100 Subject: [PATCH 15/15] fix(storage): stage moves, allow data-drive folders, surface guard errors - Copy into a unique sibling staging folder, then rename onto the verified-empty destination; a failed copy is discarded so the source stays intact and a retry does not hit "Destination must be empty". - Drop the post-copy re-validation: it re-ran a pure check on the same inputs. - Windows: accept a top-level folder on a data drive (D:\Models) and refuse the well-known system folders instead; POSIX keeps the two-level minimum. - Path API follows options.platform everywhere, including the symlink check. - Return the guard's message over IPC and show it in the move/delete modal. - Tests: resolve tmpdir() through realpath (macOS /var symlink), run the junction test on Windows, cover empty existing destinations and failed copies. --- electron/main/ipc-handlers.ts | 4 +- electron/main/storage-directory-guard.test.ts | 64 ++++++++++++++--- electron/main/storage-directory-guard.ts | 68 ++++++++++++------- .../settings/components/StorageSection.tsx | 20 +++++- 4 files changed, 119 insertions(+), 37 deletions(-) diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index f62439674..564d32dd8 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -1096,7 +1096,7 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe await moveStorageDirectory(src, dest, getSettings(userData), userData, { appDir: app.getAppPath() }) return { success: true } } catch (err) { - return { success: false, error: String(err) } + return { success: false, error: err instanceof Error ? err.message : String(err) } } }) @@ -1106,7 +1106,7 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe await deleteStorageDirectory(dirPath, getSettings(userData), userData, { appDir: app.getAppPath() }) return { success: true } } catch (err) { - return { success: false, error: String(err) } + return { success: false, error: err instanceof Error ? err.message : String(err) } } }) diff --git a/electron/main/storage-directory-guard.test.ts b/electron/main/storage-directory-guard.test.ts index d40aa3fa8..ea3abf21c 100644 --- a/electron/main/storage-directory-guard.test.ts +++ b/electron/main/storage-directory-guard.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { mkdtemp, mkdir, rm, symlink, writeFile, readFile, access } from 'node:fs/promises' +import { mkdtemp, mkdir, rm, symlink, writeFile, readFile, access, readdir, realpath, chmod } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { @@ -24,6 +24,12 @@ const home = 'C:\\Users\\Tester' const userData = 'C:\\Users\\Tester\\AppData\\Roaming\\modly' const opts = { platform: 'win32' as const, homeDir: home, appDir: 'C:\\Program Files\\Modly\\resources\\app.asar' } +// realpath: on macOS tmpdir() lives under /var, itself a symlink to /private/var, +// which the symlink-ancestor guard would (rightly) refuse. +async function tempBase(prefix: string): Promise { + return mkdtemp(join(await realpath(tmpdir()), prefix)) +} + test('deletion permits an exact configured managed directory', () => { assert.equal(validateStorageDeletion('e:\\MODLY\\models\\\\', dirs, userData, opts), dirs.modelsDir) assert.equal(validateStorageDeletion(dirs.workflowsDir, dirs, userData, opts), dirs.workflowsDir) @@ -31,7 +37,7 @@ test('deletion permits an exact configured managed directory', () => { }) test('deletion rejects filesystem roots, shallow folders and relative paths', () => { - for (const bad of ['E:\\\\', 'C:\\\\', 'E:\\Photos', 'models', '.', '']) { + for (const bad of ['E:\\\\', 'C:\\\\', 'C:\\Windows', 'D:\\program files', 'models', '.', '']) { assert.throws(() => validateStorageDeletion(bad, { ...dirs, modelsDir: bad }, userData, opts)) } assert.throws(() => validateStorageDeletion('\\\\server\\share\\models', dirs, userData, opts)) @@ -65,6 +71,12 @@ test('move allows separate configured source and empty destination path', () => }) }) +test('a top-level folder on a Windows data drive is a valid destination', () => { + assert.deepEqual(validateStorageMove(dirs.modelsDir, 'D:\\Models', dirs, userData, opts), { + src: dirs.modelsDir, dest: 'D:\\Models', + }) +}) + test('move rejects rogue sources, root, parent, child and overlapping destinations', () => { const badSources = ['E:\\\\', home, 'E:\\modly\\models-other', 'E:\\modly\\models\\child'] for (const bad of badSources) { @@ -72,7 +84,7 @@ test('move rejects rogue sources, root, parent, child and overlapping destinatio } for (const bad of [ dirs.modelsDir, 'E:\\modly', 'E:\\modly\\models\\new', 'E:\\modly\\workflows', - 'E:\\\\', 'D:\\Backups', 'E:\\modly\\workspace\\inside', + 'E:\\\\', 'D:\\Users', 'E:\\modly\\workspace\\inside', ]) { assert.throws(() => validateStorageMove(dirs.modelsDir, bad, dirs, userData, opts)) } @@ -90,11 +102,12 @@ test('POSIX root and relative path are rejected', () => { const linuxOpts = { platform: 'linux' as const, homeDir: '/home/tester' } assert.equal(validateStorageDeletion(linuxDirs.modelsDir, linuxDirs, '/home/tester/.config/modly', linuxOpts), linuxDirs.modelsDir) assert.throws(() => validateStorageDeletion('/', { ...linuxDirs, modelsDir: '/' }, '/home/tester/.config/modly', linuxOpts)) + assert.throws(() => validateStorageMove(linuxDirs.modelsDir, '/mnt', linuxDirs, '/home/tester/.config/modly', linuxOpts)) assert.throws(() => validateStorageMove('relative', '/tmp/safe/new', linuxDirs, '/home/tester/.config/modly', linuxOpts)) }) test('destination must be empty, missing is fine', async () => { - const base = await mkdtemp(join(tmpdir(), 'modly-storage-')) + const base = await tempBase('modly-storage-') try { await assertEmptyDestination(join(base, 'new')) await mkdir(join(base, 'empty')) @@ -106,11 +119,11 @@ test('destination must be empty, missing is fine', async () => { } }) -test('symlinked ancestors and target are refused', { skip: process.platform === 'win32' }, async () => { - const base = await mkdtemp(join(tmpdir(), 'modly-storage-link-')) +test('symlinked ancestors, Windows junctions and target are refused', async () => { + const base = await tempBase('modly-storage-link-') try { await mkdir(join(base, 'real', 'models'), { recursive: true }) - await symlink(join(base, 'real'), join(base, 'alias'), 'dir') + await symlink(join(base, 'real'), join(base, 'alias'), process.platform === 'win32' ? 'junction' : 'dir') await assertNoSymlinkAncestors(join(base, 'real', 'models')) await assert.rejects(() => assertNoSymlinkAncestors(join(base, 'alias', 'models')), /symbolic link/i) await assert.rejects(() => assertNoSymlinkAncestors(join(base, 'alias')), /symbolic link/i) @@ -120,7 +133,7 @@ test('symlinked ancestors and target are refused', { skip: process.platform === }) test('unsafe real deletion refuses a storage root and preserves personal files', async () => { - const base = await mkdtemp(join(tmpdir(), 'modly-real-delete-')) + const base = await tempBase('modly-real-delete-') const managed = join(base, 'models') const personal = join(base, 'personal.txt') const unixDirs = { @@ -144,7 +157,7 @@ test('unsafe real deletion refuses a storage root and preserves personal files', }) test('real move does not overwrite destination files or erase unrelated data', async () => { - const base = await mkdtemp(join(tmpdir(), 'modly-real-move-')) + const base = await tempBase('modly-real-move-') const source = join(base, 'models') const destination = join(base, 'migrated', 'models') const unixDirs = { @@ -162,11 +175,42 @@ test('real move does not overwrite destination files or erase unrelated data', a join(base, 'userData'), options), /empty/i) assert.equal(await readFile(join(destination, 'personal.txt'), 'utf8'), 'preserve') assert.equal(await readFile(join(source, 'asset.glb'), 'utf8'), 'model') - await rm(destination, { recursive: true }) + // The folder picker usually hands back an existing, empty folder. + await rm(join(destination, 'personal.txt')) await moveStorageDirectory(source, destination, unixDirs, join(base, 'userData'), options) assert.equal(await readFile(join(destination, 'asset.glb'), 'utf8'), 'model') await assert.rejects(() => access(source)) + assert.deepEqual(await readdir(join(base, 'migrated')), ['models']) + } finally { + await rm(base, { recursive: true, force: true }) + } +}) + +test('a failed copy keeps the source and leaves nothing behind at the destination', { + // chmod cannot make a file unreadable on Windows, nor for root. + skip: process.platform === 'win32' || process.getuid?.() === 0, +}, async () => { + const base = await tempBase('modly-failed-move-') + const source = join(base, 'models') + const destination = join(base, 'migrated', 'models') + const unixDirs = { + modelsDir: source, workspaceDir: join(base, 'workspace'), + workflowsDir: join(base, 'workflows'), extensionsDir: join(base, 'extensions'), + dependenciesDir: join(base, 'dependencies'), agentDir: join(base, 'agent'), + } + const options = { platform: process.platform, homeDir: join(base, 'other-home') } + try { + await mkdir(source) + await writeFile(join(source, 'asset.glb'), 'model') + await writeFile(join(source, 'locked.bin'), 'unreadable') + await chmod(join(source, 'locked.bin'), 0o000) + await assert.rejects(() => moveStorageDirectory(source, destination, unixDirs, + join(base, 'userData'), options)) + assert.equal(await readFile(join(source, 'asset.glb'), 'utf8'), 'model') + // No half-copied destination and no staging leftover: a retry starts clean. + assert.deepEqual(await readdir(join(base, 'migrated')), []) } finally { + await chmod(join(source, 'locked.bin'), 0o644).catch(() => {}) await rm(base, { recursive: true, force: true }) } }) diff --git a/electron/main/storage-directory-guard.ts b/electron/main/storage-directory-guard.ts index f5a63a8c8..bff010255 100644 --- a/electron/main/storage-directory-guard.ts +++ b/electron/main/storage-directory-guard.ts @@ -1,8 +1,15 @@ /** Fail-closed validation for settings-driven directory moves and deletions. */ -import { cp, lstat, mkdir, readdir, rm, rmdir } from 'node:fs/promises' +import { randomUUID } from 'node:crypto' +import { cp, lstat, mkdir, readdir, rename, rm, rmdir } from 'node:fs/promises' import { homedir } from 'node:os' import path from 'node:path' +/** Top-level Windows folders owned by the OS, refused on any drive. */ +const WINDOWS_SYSTEM_DIRS = new Set([ + 'windows', 'program files', 'program files (x86)', 'programdata', 'users', + 'perflogs', 'recovery', '$recycle.bin', 'system volume information', +]) + type StorageDirs = { modelsDir: string workspaceDir: string @@ -38,10 +45,15 @@ function absoluteDirectory(value: string, api: PathApi): string { if (resolved === api.parse(resolved).root) { throw new Error('Refusing a filesystem or volume root') } - // Never recursively delete a bare top-level user folder, even if configured. const pieces = resolved.slice(api.parse(resolved).root.length).split(api.sep).filter(Boolean) - if (pieces.length < 2) { - throw new Error('Storage directory must be at least two levels below its volume root') + // POSIX top-level folders (/home, /usr, /mnt, /Volumes…) all belong to the system. + if (api === path.posix && pieces.length < 2) { + throw new Error('Storage directory must be at least two levels below the filesystem root') + } + // On Windows a top-level folder on a data drive (D:\Models) is a normal choice; + // only the well-known system folders are refused. + if (api === path.win32 && pieces.length === 1 && WINDOWS_SYSTEM_DIRS.has(pieces[0].toLowerCase())) { + throw new Error('Refusing a Windows system folder') } return resolved } @@ -146,8 +158,8 @@ export function validateStorageMove( } /** Reject symlinks or Windows junctions in existing path ancestors. */ -export async function assertNoSymlinkAncestors(directory: string): Promise { - const api = process.platform === 'win32' ? path.win32 : path.posix +export async function assertNoSymlinkAncestors(directory: string, options: Options = {}): Promise { + const api = pathApi(options) const root = api.parse(directory).root let current = root for (const part of directory.slice(root.length).split(api.sep).filter(Boolean)) { @@ -178,30 +190,40 @@ export async function deleteStorageDirectory( directory: string, dirs: StorageDirs, userData: string, options: Options = {}, ): Promise { const safe = validateStorageDeletion(directory, dirs, userData, options) - await assertNoSymlinkAncestors(safe) + await assertNoSymlinkAncestors(safe, options) await rm(safe, { recursive: true, force: true }) } export async function moveStorageDirectory( source: string, destination: string, dirs: StorageDirs, userData: string, options: Options = {}, ): Promise { - const paths = validateStorageMove(source, destination, dirs, userData, options) - await assertNoSymlinkAncestors(paths.src) - await assertNoSymlinkAncestors(paths.dest) - await assertEmptyDestination(paths.dest) - // fs.cp with errorOnExist refuses even an EMPTY existing directory. Remove - // only the verified-empty destination, never the source, before copying. - // rmdir itself fails closed if a concurrent writer adds a file. + const api = pathApi(options) + const { src, dest } = validateStorageMove(source, destination, dirs, userData, options) + await assertNoSymlinkAncestors(src, options) + await assertNoSymlinkAncestors(dest, options) + await assertEmptyDestination(dest) + + const parent = api.dirname(dest) + await mkdir(parent, { recursive: true }) + // Copy into a fresh sibling folder that only this call writes to: a failed + // copy can then be discarded without touching the source or the destination, + // and a retry does not trip over a half-filled destination. + const staging = api.join(parent, '.' + api.basename(dest) + '.modly-move-' + randomUUID()) try { - await rmdir(paths.dest) + await cp(src, staging, { recursive: true, force: false, errorOnExist: true }) + // Remove the verified-empty destination; rmdir fails closed if a file appeared. + try { + await rmdir(dest) + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err + } + await rename(staging, dest) } catch (err) { - if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err + await rm(staging, { recursive: true, force: true }) + throw err } - await mkdir(path.dirname(paths.dest), { recursive: true }) - // With a missing destination, errorOnExist protects against unexpected files. - await cp(paths.src, paths.dest, { recursive: true, force: false, errorOnExist: true }) - // Re-validate before deleting the source. Never delete it after a copy error. - validateStorageMove(paths.src, paths.dest, dirs, userData, options) - await assertNoSymlinkAncestors(paths.src) - await rm(paths.src, { recursive: true, force: true }) + + // The source is deleted only once its complete copy sits at the destination. + await assertNoSymlinkAncestors(src, options) + await rm(src, { recursive: true, force: true }) } diff --git a/src/areas/settings/components/StorageSection.tsx b/src/areas/settings/components/StorageSection.tsx index 7c949adc8..c15a2d870 100644 --- a/src/areas/settings/components/StorageSection.tsx +++ b/src/areas/settings/components/StorageSection.tsx @@ -3,7 +3,7 @@ import { Section, Card, Row, PathRow } from '@shared/ui' // ─── MoveFolderModal ────────────────────────────────────────────────────────── -function MoveFolderModal({ title, currentDir, items, itemLabel, moveLabel, moveDesc, deleteLabel, deleteDesc, status, onCancel, onMove, onDelete }: { +function MoveFolderModal({ title, currentDir, items, itemLabel, moveLabel, moveDesc, deleteLabel, deleteDesc, status, error, onCancel, onMove, onDelete }: { title: string currentDir: string items: string[] @@ -13,6 +13,7 @@ function MoveFolderModal({ title, currentDir, items, itemLabel, moveLabel, moveD deleteLabel: string deleteDesc: string status: 'idle' | 'busy' | 'error' + error?: string | null onCancel: () => void onMove: () => void onDelete: () => void @@ -34,7 +35,7 @@ function MoveFolderModal({ title, currentDir, items, itemLabel, moveLabel, moveD
{status === 'error' && ( -

Something went wrong. Please try again.

+

{error || 'Something went wrong. Please try again.'}

)}