From 361ea3cc4eef1c1898ccd8ba3b48a480895706a3 Mon Sep 17 00:00:00 2001 From: securitybrahh Date: Tue, 8 Sep 2026 02:16:27 -0400 Subject: [PATCH 001/108] vram indicator on linux and windows --- electron/main/ipc-handlers.ts | 23 +++++ electron/preload/electron-api.ts | 2 + .../components/ApplicationSection.tsx | 8 +- .../components/layout/MemoryIndicator.tsx | 84 +++++++++++++------ src/shared/stores/appStore.ts | 5 ++ 5 files changed, 96 insertions(+), 26 deletions(-) diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 005f1f786..71294cd45 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -629,6 +629,29 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe return { total, used: total - free, available: free } }) + // GPU memory (VRAM) — NVIDIA only via nvidia-smi + ipcMain.handle('system:gpuMemory', async () => { + if (process.platform === 'darwin' && process.arch === 'arm64') { + // Apple Silicon: unified memory, no separate VRAM + return null + } + try { + const { stdout } = await pExecFile('nvidia-smi', [ + '--query-gpu=memory.total,memory.used', + '--format=csv,noheader,nounits' + ]) + const line = stdout.trim().split('\n')[0].trim() + const [totalMiB, usedMiB] = line.split(',').map(s => parseInt(s.trim(), 10)) + if (isNaN(totalMiB) || isNaN(usedMiB) || totalMiB === 0) return null + const total = totalMiB * 1024 * 1024 + const used = usedMiB * 1024 * 1024 + const available = total - used + return { total, used, available } + } catch { + return null + } + }) + ipcMain.handle('app:info', () => ({ version: app.getVersion(), userData: app.getPath('userData'), diff --git a/electron/preload/electron-api.ts b/electron/preload/electron-api.ts index 89fa69ecc..b45525492 100644 --- a/electron/preload/electron-api.ts +++ b/electron/preload/electron-api.ts @@ -47,6 +47,8 @@ export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFra system: { memory: (): Promise<{ total: number; used: number; available: number }> => ipcRenderer.invoke('system:memory') as Promise<{ total: number; used: number; available: number }>, + gpuMemory: (): Promise<{ total: number; used: number; available: number } | null> => + ipcRenderer.invoke('system:gpuMemory') as Promise<{ total: number; used: number; available: number } | null>, }, // Python / FastAPI bridge diff --git a/src/areas/settings/components/ApplicationSection.tsx b/src/areas/settings/components/ApplicationSection.tsx index 63984cb73..462bb618b 100644 --- a/src/areas/settings/components/ApplicationSection.tsx +++ b/src/areas/settings/components/ApplicationSection.tsx @@ -2,7 +2,7 @@ import { useAppStore } from '@shared/stores/appStore' import { Section, Card, Row, Toggle } from '@shared/ui' export function ApplicationSection(): JSX.Element { - const { showRamIndicator, setShowRamIndicator } = useAppStore() + const { showRamIndicator, setShowRamIndicator, showVramIndicator, setShowVramIndicator } = useAppStore() return (
@@ -13,6 +13,12 @@ export function ApplicationSection(): JSX.Element { > + + +
) diff --git a/src/shared/components/layout/MemoryIndicator.tsx b/src/shared/components/layout/MemoryIndicator.tsx index fe09eac12..f30f8978f 100644 --- a/src/shared/components/layout/MemoryIndicator.tsx +++ b/src/shared/components/layout/MemoryIndicator.tsx @@ -1,4 +1,5 @@ import { useEffect, useState } from 'react' +import { useAppStore } from '@shared/stores/appStore' const GB = 1024 ** 3 @@ -6,15 +7,31 @@ function fmtGB(bytes: number): string { return (bytes / GB).toFixed(1) } +function getBarColors(pct: number): { barColor: string; textColor: string } { + if (pct >= 90) return { barColor: 'bg-red-500', textColor: 'text-red-300' } + if (pct >= 75) return { barColor: 'bg-amber-500', textColor: 'text-amber-300' } + return { barColor: 'bg-emerald-500', textColor: 'text-zinc-300' } +} + export default function MemoryIndicator(): JSX.Element | null { - const [mem, setMem] = useState<{ total: number; used: number; available: number } | null>(null) + const [ram, setRam] = useState<{ total: number; used: number; available: number } | null>(null) + const [vram, setVram] = useState<{ total: number; used: number; available: number } | null>(null) + const platform = useAppStore(s => s.platform) + const isMac = platform === 'darwin' + const showVramIndicator = useAppStore(s => s.showVramIndicator) useEffect(() => { let cancelled = false const tick = async () => { try { - const next = await window.electron.system.memory() - if (!cancelled) setMem(next) + const [ramNext, vramNext] = await Promise.all([ + window.electron.system.memory(), + isMac ? Promise.resolve(null) : window.electron.system.gpuMemory(), + ]) + if (!cancelled) { + setRam(ramNext) + setVram(vramNext) + } } catch { // Renderer should not break if memory sampling fails. } @@ -25,42 +42,59 @@ export default function MemoryIndicator(): JSX.Element | null { cancelled = true clearInterval(id) } - }, []) + }, [isMac]) - if (!mem) return null + if (!ram) return null - const pct = mem.total > 0 ? Math.min(100, Math.round((mem.used / mem.total) * 100)) : 0 + const ramPct = ram.total > 0 ? Math.min(100, Math.round((ram.used / ram.total) * 100)) : 0 + const ramColors = getBarColors(ramPct) - let barColor = 'bg-emerald-500' - let textColor = 'text-zinc-300' - if (pct >= 90) { - barColor = 'bg-red-500' - textColor = 'text-red-300' - } else if (pct >= 75) { - barColor = 'bg-amber-500' - textColor = 'text-amber-300' - } + const ramTooltip = + `RAM:\n` + + ` Used: ${fmtGB(ram.used)} GB\n` + + ` Available: ${fmtGB(ram.available)} GB\n` + + ` Total: ${fmtGB(ram.total)} GB` - const tooltip = - `Used: ${fmtGB(mem.used)} GB\n` + - `Available: ${fmtGB(mem.available)} GB\n` + - `Total: ${fmtGB(mem.total)} GB` + const vramBar = showVramIndicator && !isMac && vram && vram.total > 0 ? (() => { + const vramPct = Math.min(100, Math.round((vram.used / vram.total) * 100)) + const vramColors = getBarColors(vramPct) + const vramTooltip = + `VRAM:\n` + + ` Used: ${fmtGB(vram.used)} GB\n` + + ` Available: ${fmtGB(vram.available)} GB\n` + + ` Total: ${fmtGB(vram.total)} GB` + return ( +
+ VRAM +
+
+
+ + {fmtGB(vram.used)} / {fmtGB(vram.total)} GB + +
+ ) + })() : null return (
RAM
- - {fmtGB(mem.used)} / {fmtGB(mem.total)} GB + + {fmtGB(ram.used)} / {fmtGB(ram.total)} GB + {vramBar}
) -} +} \ No newline at end of file diff --git a/src/shared/stores/appStore.ts b/src/shared/stores/appStore.ts index d06a52112..05281eb28 100644 --- a/src/shared/stores/appStore.ts +++ b/src/shared/stores/appStore.ts @@ -137,6 +137,8 @@ interface AppState { // UI preferences showRamIndicator: boolean setShowRamIndicator: (v: boolean) => void + showVramIndicator: boolean + setShowVramIndicator: (v: boolean) => void // Accessibility useAtkinsonFont: boolean @@ -241,6 +243,8 @@ export const useAppStore = create()( showRamIndicator: true, setShowRamIndicator: (v) => set({ showRamIndicator: v }), + showVramIndicator: true, + setShowVramIndicator: (v) => set({ showVramIndicator: v }), useAtkinsonFont: false, setUseAtkinsonFont: (v) => set({ useAtkinsonFont: v }), @@ -301,6 +305,7 @@ export const useAppStore = create()( partialize: (state) => ({ generationOptions: state.generationOptions, showRamIndicator: state.showRamIndicator, + showVramIndicator: state.showVramIndicator, useAtkinsonFont: state.useAtkinsonFont, uiScale: state.uiScale, lightSettings: state.lightSettings, From dd0da3d67dc15ac20db8a4a55cc81bfd0f4ea1bf Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:02:43 +0900 Subject: [PATCH 002/108] test(generation): guard that cancelling an ended job leaves the worker alone Port the CancelEndedJobTests from #341 onto the _job_generators binding introduced in 10cdc15. Ended jobs/runs are driven through _run_generation so the test covers the binding being released, then cancelled via /generate/cancel and /workflow-runs/{id}/cancel; running jobs/runs bound in _job_generators must still have their worker stopped. --- api/tests/test_workflow_runs_lifecycle.py | 159 ++++++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/api/tests/test_workflow_runs_lifecycle.py b/api/tests/test_workflow_runs_lifecycle.py index 474335739..e8aa355c9 100644 --- a/api/tests/test_workflow_runs_lifecycle.py +++ b/api/tests/test_workflow_runs_lifecycle.py @@ -200,5 +200,164 @@ async def run_cross_route_jobs(): self.assertEqual(fake_registry.active_id, "demo/generate") +class _FakeProc: + """Stands in for a loaded extension worker's subprocess.""" + + def __init__(self) -> None: + self.killed = False + + def poll(self): + return -9 if self.killed else None + + def kill(self) -> None: + self.killed = True + + +class _FakeWorker: + """A loaded generator with a live subprocess, as _run_generation binds it.""" + + def __init__(self) -> None: + self._proc = _FakeProc() + self._loaded = True + self.outputs_dir = None + + def generate(self, image_bytes, params, progress_cb, cancel_event=None): + output = Path(self.outputs_dir) / "result.glb" + output.write_bytes(b"glb") + return output + + +class _FailingWorker(_FakeWorker): + def generate(self, image_bytes, params, progress_cb, cancel_event=None): + raise RuntimeError("inference failed") + + +class _RegistryWithLoadedWorker: + """Keeps one worker loaded and active, the state the worker is left in + after a job ends normally.""" + + def __init__(self, worker: _FakeWorker) -> None: + self.worker = worker + self._generators = {"ext/model": worker} + self._active_id = "ext/model" + + def assert_weight_variant_installed(self, params, model_id) -> None: + pass + + def get_generator(self, model_id: str): + return self.worker + + def model_status(self, model_id: str) -> dict: + return {"name": model_id, "downloaded": True, "loaded": True} + + def activate_ready_generator(self, model_id: str): + return self.worker + + +class CancelEndedJobTests(unittest.TestCase): + """cancel_job kills the subprocess bound to the job in _job_generators so + inference stops at once. Once the job has ended, that binding is gone and + the worker holds the warm model (or another job's generation), so a late + cancel of the ended job/run must leave the worker alone.""" + + def setUp(self) -> None: + self._prev_generation_registry = generation.generator_registry + self._prev_runs_registry = workflow_runs.generator_registry + self._prev_workspace = registry_module.WORKSPACE_DIR + self._tmp = tempfile.TemporaryDirectory() + registry_module.WORKSPACE_DIR = Path(self._tmp.name) + _clear_job_stores() + + def tearDown(self) -> None: + generation.generator_registry = self._prev_generation_registry + workflow_runs.generator_registry = self._prev_runs_registry + registry_module.WORKSPACE_DIR = self._prev_workspace + self._tmp.cleanup() + _clear_job_stores() + + def _use_worker(self, worker: _FakeWorker) -> _FakeWorker: + registry = _RegistryWithLoadedWorker(worker) + generation.generator_registry = registry + workflow_runs.generator_registry = registry + return worker + + def _file_job(self, job_id: str, status: str = "pending") -> None: + generation._jobs[job_id] = JobStatus(job_id=job_id, status=status, progress=0) + generation._cancel_events[job_id] = threading.Event() + + def _run_to_end(self, job_id: str) -> None: + self._file_job(job_id) + asyncio.run(generation._run_generation(job_id, b"img", {}, "Default", "mesh", "ext/model")) + # The job's generator binding is released as soon as it ends. + self.assertNotIn(job_id, generation._job_generators) + + def _assert_worker_untouched(self, worker: _FakeWorker, proc: _FakeProc) -> None: + self.assertFalse(proc.killed) + self.assertIs(worker._proc, proc) + self.assertTrue(worker._loaded) + + def _assert_worker_stopped(self, worker: _FakeWorker, proc: _FakeProc) -> None: + self.assertTrue(proc.killed) + self.assertIsNone(worker._proc) + self.assertFalse(worker._loaded) + + def test_cancelling_a_finished_job_leaves_the_active_worker_alone(self) -> None: + worker = self._use_worker(_FakeWorker()) + proc = worker._proc + self._run_to_end("finished-job") + self.assertEqual(generation._jobs["finished-job"].status, "done") + + asyncio.run(generation.cancel_job("finished-job")) + + self.assertEqual(generation._jobs["finished-job"].status, "done") + self._assert_worker_untouched(worker, proc) + + def test_cancelling_a_failed_run_leaves_the_active_worker_alone(self) -> None: + worker = self._use_worker(_FailingWorker()) + proc = worker._proc + self._run_to_end("failed-run") + self.assertEqual(generation._jobs["failed-run"].status, "error") + + asyncio.run(workflow_runs.cancel_run("failed-run")) + + self.assertEqual(generation._jobs["failed-run"].status, "error") + self._assert_worker_untouched(worker, proc) + + def test_cancelling_an_ended_job_leaves_another_jobs_worker_alone(self) -> None: + worker = self._use_worker(_FakeWorker()) + proc = worker._proc + self._run_to_end("finished-job") + # A later job now owns the same worker. + self._file_job("running-job", "running") + generation._job_generators["running-job"] = worker + + asyncio.run(generation.cancel_job("finished-job")) + + self.assertEqual(generation._jobs["running-job"].status, "running") + self._assert_worker_untouched(worker, proc) + + def test_cancelling_a_running_job_still_stops_the_worker(self) -> None: + worker = self._use_worker(_FakeWorker()) + proc = worker._proc + self._file_job("running-job", "running") + generation._job_generators["running-job"] = worker + + asyncio.run(generation.cancel_job("running-job")) + + self.assertEqual(generation._jobs["running-job"].status, "cancelled") + self._assert_worker_stopped(worker, proc) + + def test_cancelling_a_running_run_still_stops_the_worker(self) -> None: + worker = self._use_worker(_FakeWorker()) + proc = worker._proc + self._file_job("running-run", "running") + generation._job_generators["running-run"] = worker + + asyncio.run(workflow_runs.cancel_run("running-run")) + + self.assertEqual(generation._jobs["running-run"].status, "cancelled") + self._assert_worker_stopped(worker, proc) + + if __name__ == "__main__": unittest.main() From 3cf0fb26f9b7c2faabc2a4e040c0d472efe6ef09 Mon Sep 17 00:00:00 2001 From: JayceeB1 <45721154+JayceeB1@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:41:44 +0200 Subject: [PATCH 003/108] fix(storage): guard destructive directory moves and deletion --- electron/main/ipc-handlers.ts | 22 +- electron/main/storage-directory-guard.test.ts | 172 +++++++++++++++ electron/main/storage-directory-guard.ts | 199 ++++++++++++++++++ package.json | 2 +- 4 files changed, 377 insertions(+), 18 deletions(-) create mode 100644 electron/main/storage-directory-guard.test.ts create mode 100644 electron/main/storage-directory-guard.ts diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index e9cac040b..f62439674 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -42,6 +42,7 @@ import { weightStorageHasLocalData, } from './model-sources' import { getSettings, setSettings } from './settings-store' +import { deleteStorageDirectory, moveStorageDirectory } from './storage-directory-guard' import { checkSetupNeeded, markSetupDone, runFullSetup, getVenvPythonExe, ensureSslPatch } from './python-setup' import { logger } from './logger' import { getProcessRunner, getPythonProcessRunner, getExtPythonExe, terminateProcessRunner, terminateAllProcessRunners } from './process-runner' @@ -1091,9 +1092,8 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe ipcMain.handle('fs:moveDirectory', async (_, { src, dest }: { src: string; dest: string }) => { try { - await mkdir(dest, { recursive: true }) - await cp(src, dest, { recursive: true }) - await rmAsync(src, { recursive: true, force: true }) + const userData = app.getPath('userData') + await moveStorageDirectory(src, dest, getSettings(userData), userData, { appDir: app.getAppPath() }) return { success: true } } catch (err) { return { success: false, error: String(err) } @@ -1101,21 +1101,9 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe }) ipcMain.handle('fs:deleteDirectory', async (_, dirPath: string) => { - const userData = app.getPath('userData') - const settings = getSettings(userData) - const allowedRoots = [ - settings.modelsDir, - settings.workspaceDir, - settings.extensionsDir, - join(userData, 'gen-cache'), - ] - const resolved = join(dirPath) - const isAllowed = allowedRoots.some((root) => resolved.startsWith(root)) - if (!isAllowed) { - return { success: false, error: 'Path is outside allowed directories' } - } try { - await rmAsync(resolved, { recursive: true, force: true }) + const userData = app.getPath('userData') + await deleteStorageDirectory(dirPath, getSettings(userData), userData, { appDir: app.getAppPath() }) return { success: true } } catch (err) { return { success: false, error: String(err) } diff --git a/electron/main/storage-directory-guard.test.ts b/electron/main/storage-directory-guard.test.ts new file mode 100644 index 000000000..d40aa3fa8 --- /dev/null +++ b/electron/main/storage-directory-guard.test.ts @@ -0,0 +1,172 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { mkdtemp, mkdir, rm, symlink, writeFile, readFile, access } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + validateStorageDeletion, + validateStorageMove, + assertNoSymlinkAncestors, + assertEmptyDestination, + deleteStorageDirectory, + moveStorageDirectory, +} from './storage-directory-guard' + +const dirs = { + modelsDir: 'E:\\modly\\models', + workspaceDir: 'E:\\modly\\workspace', + workflowsDir: 'E:\\modly\\workflows', + extensionsDir: 'E:\\modly\\extensions', + dependenciesDir: 'E:\\modly\\dependencies', + agentDir: 'E:\\modly\\agent', +} +const home = 'C:\\Users\\Tester' +const userData = 'C:\\Users\\Tester\\AppData\\Roaming\\modly' +const opts = { platform: 'win32' as const, homeDir: home, appDir: 'C:\\Program Files\\Modly\\resources\\app.asar' } + +test('deletion permits an exact configured managed directory', () => { + assert.equal(validateStorageDeletion('e:\\MODLY\\models\\\\', dirs, userData, opts), dirs.modelsDir) + assert.equal(validateStorageDeletion(dirs.workflowsDir, dirs, userData, opts), dirs.workflowsDir) + assert.equal(validateStorageDeletion('E:\\modly\\workspace\\tmp', dirs, userData, opts), 'E:\\modly\\workspace\\tmp') +}) + +test('deletion rejects filesystem roots, shallow folders and relative paths', () => { + for (const bad of ['E:\\\\', 'C:\\\\', 'E:\\Photos', 'models', '.', '']) { + assert.throws(() => validateStorageDeletion(bad, { ...dirs, modelsDir: bad }, userData, opts)) + } + assert.throws(() => validateStorageDeletion('\\\\server\\share\\models', dirs, userData, opts)) +}) + +test('deletion requires an exact configured root or workspace/tmp (not a prefix)', () => { + for (const bad of [ + 'E:\\modly\\models-backup', 'E:\\modly\\models\\elsewhere', + 'E:\\modly\\workspace\\tmp-fake', 'E:\\modly\\workflows2', + 'E:\\elsewhere\\models', + ]) { + assert.throws(() => validateStorageDeletion(bad, dirs, userData, opts), /not an approved/i) + } +}) + +test('deletion rejects home, application and userData ancestors even if configured', () => { + for (const bad of [home, 'C:\\Users', 'C:\\Program Files', 'C:\\Users\\Tester\\AppData', userData]) { + assert.throws(() => validateStorageDeletion(bad, { ...dirs, modelsDir: bad }, userData, opts)) + } +}) + +test('deletion refuses one configured folder enclosing another', () => { + assert.throws(() => validateStorageDeletion(dirs.modelsDir, { + ...dirs, workspaceDir: 'E:\\modly\\models\\my-workspace', + }, userData, opts), /another configured/i) +}) + +test('move allows separate configured source and empty destination path', () => { + assert.deepEqual(validateStorageMove(dirs.modelsDir, 'D:\\modly-assets\\models', dirs, userData, opts), { + src: dirs.modelsDir, dest: 'D:\\modly-assets\\models', + }) +}) + +test('move rejects rogue sources, root, parent, child and overlapping destinations', () => { + const badSources = ['E:\\\\', home, 'E:\\modly\\models-other', 'E:\\modly\\models\\child'] + for (const bad of badSources) { + assert.throws(() => validateStorageMove(bad, 'D:\\modly\\models', dirs, userData, opts)) + } + for (const bad of [ + dirs.modelsDir, 'E:\\modly', 'E:\\modly\\models\\new', 'E:\\modly\\workflows', + 'E:\\\\', 'D:\\Backups', 'E:\\modly\\workspace\\inside', + ]) { + assert.throws(() => validateStorageMove(dirs.modelsDir, bad, dirs, userData, opts)) + } +}) + +test('POSIX root and relative path are rejected', () => { + const linuxDirs = { + modelsDir: '/home/tester/modly/models', + workspaceDir: '/home/tester/modly/workspace', + workflowsDir: '/home/tester/modly/workflows', + extensionsDir: '/home/tester/modly/extensions', + dependenciesDir: '/home/tester/modly/dependencies', + agentDir: '/home/tester/modly/agent', + } + const linuxOpts = { platform: 'linux' as const, homeDir: '/home/tester' } + assert.equal(validateStorageDeletion(linuxDirs.modelsDir, linuxDirs, '/home/tester/.config/modly', linuxOpts), linuxDirs.modelsDir) + assert.throws(() => validateStorageDeletion('/', { ...linuxDirs, modelsDir: '/' }, '/home/tester/.config/modly', linuxOpts)) + assert.throws(() => validateStorageMove('relative', '/tmp/safe/new', linuxDirs, '/home/tester/.config/modly', linuxOpts)) +}) + +test('destination must be empty, missing is fine', async () => { + const base = await mkdtemp(join(tmpdir(), 'modly-storage-')) + try { + await assertEmptyDestination(join(base, 'new')) + await mkdir(join(base, 'empty')) + await assertEmptyDestination(join(base, 'empty')) + await writeFile(join(base, 'empty', 'personal.txt'), 'do not overwrite') + await assert.rejects(() => assertEmptyDestination(join(base, 'empty')), /empty/i) + } finally { + await rm(base, { recursive: true, force: true }) + } +}) + +test('symlinked ancestors and target are refused', { skip: process.platform === 'win32' }, async () => { + const base = await mkdtemp(join(tmpdir(), 'modly-storage-link-')) + try { + await mkdir(join(base, 'real', 'models'), { recursive: true }) + await symlink(join(base, 'real'), join(base, 'alias'), 'dir') + await assertNoSymlinkAncestors(join(base, 'real', 'models')) + await assert.rejects(() => assertNoSymlinkAncestors(join(base, 'alias', 'models')), /symbolic link/i) + await assert.rejects(() => assertNoSymlinkAncestors(join(base, 'alias')), /symbolic link/i) + } finally { + await rm(base, { recursive: true, force: true }) + } +}) + +test('unsafe real deletion refuses a storage root and preserves personal files', async () => { + const base = await mkdtemp(join(tmpdir(), 'modly-real-delete-')) + const managed = join(base, 'models') + const personal = join(base, 'personal.txt') + const unixDirs = { + modelsDir: managed, workspaceDir: join(base, 'workspace'), + workflowsDir: join(base, 'workflows'), extensionsDir: join(base, 'extensions'), + dependenciesDir: join(base, 'dependencies'), agentDir: join(base, 'agent'), + } + const options = { platform: process.platform, homeDir: join(base, 'other-home') } + try { + await mkdir(managed) + await writeFile(personal, 'important') + await assert.rejects(() => deleteStorageDirectory(base, { ...unixDirs, modelsDir: base }, + join(base, 'userData'), options), /ancestor|another configured|approved/i) + assert.equal(await readFile(personal, 'utf8'), 'important') + await deleteStorageDirectory(managed, unixDirs, join(base, 'userData'), options) + await assert.rejects(() => access(managed)) + assert.equal(await readFile(personal, 'utf8'), 'important') + } finally { + await rm(base, { recursive: true, force: true }) + } +}) + +test('real move does not overwrite destination files or erase unrelated data', async () => { + const base = await mkdtemp(join(tmpdir(), 'modly-real-move-')) + const source = join(base, 'models') + const destination = join(base, 'migrated', 'models') + const unixDirs = { + modelsDir: source, workspaceDir: join(base, 'workspace'), + workflowsDir: join(base, 'workflows'), extensionsDir: join(base, 'extensions'), + dependenciesDir: join(base, 'dependencies'), agentDir: join(base, 'agent'), + } + const options = { platform: process.platform, homeDir: join(base, 'other-home') } + try { + await mkdir(source) + await mkdir(destination, { recursive: true }) + await writeFile(join(source, 'asset.glb'), 'model') + await writeFile(join(destination, 'personal.txt'), 'preserve') + await assert.rejects(() => moveStorageDirectory(source, destination, unixDirs, + join(base, 'userData'), options), /empty/i) + assert.equal(await readFile(join(destination, 'personal.txt'), 'utf8'), 'preserve') + assert.equal(await readFile(join(source, 'asset.glb'), 'utf8'), 'model') + await rm(destination, { recursive: true }) + await moveStorageDirectory(source, destination, unixDirs, join(base, 'userData'), options) + assert.equal(await readFile(join(destination, 'asset.glb'), 'utf8'), 'model') + await assert.rejects(() => access(source)) + } finally { + await rm(base, { recursive: true, force: true }) + } +}) diff --git a/electron/main/storage-directory-guard.ts b/electron/main/storage-directory-guard.ts new file mode 100644 index 000000000..fb1e9b914 --- /dev/null +++ b/electron/main/storage-directory-guard.ts @@ -0,0 +1,199 @@ +/** Fail-closed validation for settings-driven directory moves and deletions. */ +import { cp, lstat, mkdir, readdir, rm } from 'node:fs/promises' +import { homedir } from 'node:os' +import path from 'node:path' + +type StorageDirs = { + modelsDir: string + workspaceDir: string + workflowsDir: string + extensionsDir: string + dependenciesDir: string + agentDir: string +} + +type Options = { + platform?: NodeJS.Platform + homeDir?: string + appDir?: string +} + +type PathApi = typeof path.win32 + +function pathApi(options: Options): PathApi { + return options.platform === 'win32' || (!options.platform && process.platform === 'win32') + ? path.win32 + : path.posix +} + +function absoluteDirectory(value: string, api: PathApi): string { + if (typeof value !== 'string' || !value.trim() || value.includes('\0') || !api.isAbsolute(value)) { + throw new Error('Storage path must be a non-empty absolute path') + } + // Drive-relative, extended Windows and UNC paths need separate review. + if (api === path.win32 && !/^[a-z]:[\\/]/i.test(value)) { + throw new Error('Storage operations require a local drive-absolute path') + } + const resolved = api.normalize(value) + if (resolved === api.parse(resolved).root) { + throw new Error('Refusing a filesystem or volume root') + } + // Never recursively delete a bare top-level user folder, even if configured. + const pieces = resolved.slice(api.parse(resolved).root.length).split(api.sep).filter(Boolean) + if (pieces.length < 2) { + throw new Error('Storage directory must be at least two levels below its volume root') + } + return resolved +} + +function equalPath(api: PathApi, a: string, b: string): boolean { + return api.relative(a, b) === '' +} + +function sameOrInside(api: PathApi, parent: string, candidate: string): boolean { + const relative = api.relative(parent, candidate) + return relative === '' || (relative !== '..' && !relative.startsWith('..' + api.sep) && !api.isAbsolute(relative)) +} + +function storageEntries(dirs: StorageDirs): Array<[string, string]> { + return [ + ['models', dirs.modelsDir], + ['workspace', dirs.workspaceDir], + ['workflows', dirs.workflowsDir], + ['extensions', dirs.extensionsDir], + ['dependencies', dirs.dependenciesDir], + ['agent', dirs.agentDir], + ] +} + +function verifySource(source: string, userData: string, options: Options): string { + const api = pathApi(options) + const safe = absoluteDirectory(source, api) + const protectedRoots = [userData, options.homeDir ?? homedir(), options.appDir] + for (const item of protectedRoots) { + if (!item) continue + const protectedPath = api.normalize(item) + if (api.isAbsolute(protectedPath) && sameOrInside(api, safe, protectedPath)) { + throw new Error('Refusing to remove an application, home, or user-data ancestor') + } + } + return safe +} + +function verifyNoOtherStorageInside( + source: string, selectedKey: string, dirs: StorageDirs, options: Options, +): void { + const api = pathApi(options) + for (const [key, value] of storageEntries(dirs)) { + if (key === selectedKey || !value) continue + const other = api.normalize(value) + if (api.isAbsolute(other) && sameOrInside(api, source, other)) { + throw new Error('Refusing to remove another configured storage directory (' + key + ')') + } + } +} + +/** Only exact configured roots, plus the specific workspace/tmp cleanup, are deletable. */ +export function validateStorageDeletion( + directory: string, dirs: StorageDirs, userData: string, options: Options = {}, +): string { + const api = pathApi(options) + const target = absoluteDirectory(directory, api) + const roots: Array<[string, string]> = [ + ['models', dirs.modelsDir], + ['workspace', dirs.workspaceDir], + ['workflows', dirs.workflowsDir], + ['extensions', dirs.extensionsDir], + ['cache', api.join(userData, 'gen-cache')], + ] + const selected = roots.find(([, root]) => root && equalPath(api, api.normalize(root), target)) + if (selected) { + const safe = verifySource(selected[1], userData, options) + verifyNoOtherStorageInside(safe, selected[0], dirs, options) + return safe + } + const workspace = verifySource(dirs.workspaceDir, userData, options) + if (equalPath(api, target, api.join(workspace, 'tmp'))) { + verifyNoOtherStorageInside(target, 'workspace', dirs, options) + return target + } + throw new Error('Path is not an approved Modly storage directory') +} + +/** A folder move may delete its source: restrict source and destination equally. */ +export function validateStorageMove( + source: string, destination: string, dirs: StorageDirs, userData: string, options: Options = {}, +): { src: string; dest: string } { + const api = pathApi(options) + const src = verifySource(source, userData, options) + const selected = storageEntries(dirs).find(([key, val]) => + ['models', 'workspace', 'workflows'].includes(key) && equalPath(api, api.normalize(val), src)) + if (!selected) throw new Error('Source is not a configured Modly storage directory') + verifyNoOtherStorageInside(src, selected[0], dirs, options) + const dest = verifySource(destination, userData, options) + if (sameOrInside(api, src, dest) || sameOrInside(api, dest, src)) { + throw new Error('Source and destination must be separate, non-overlapping directories') + } + // Reject destinations overlapping ANY configured storage root. + for (const [, value] of storageEntries(dirs)) { + if (!value) continue + const other = api.normalize(value) + if (sameOrInside(api, other, dest) || sameOrInside(api, dest, other)) { + throw new Error('Destination overlaps configured storage') + } + } + return { src, dest } +} + +/** Reject symlinks or Windows junctions in existing path ancestors. */ +export async function assertNoSymlinkAncestors(directory: string): Promise { + const api = process.platform === 'win32' ? path.win32 : path.posix + const root = api.parse(directory).root + let current = root + for (const part of directory.slice(root.length).split(api.sep).filter(Boolean)) { + current = api.join(current, part) + try { + const info = await lstat(current) + if (info.isSymbolicLink()) throw new Error('Storage path contains a symbolic link: ' + current) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') break + throw error + } + } +} + +export async function assertEmptyDestination(directory: string): Promise { + try { + if ((await readdir(directory)).length > 0) { + throw new Error('Destination must be empty to avoid overwriting existing files') + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return + throw error + } +} + +/** Main-process IPC handlers must use guarded operations, never raw rm/cp. */ +export async function deleteStorageDirectory( + directory: string, dirs: StorageDirs, userData: string, options: Options = {}, +): Promise { + const safe = validateStorageDeletion(directory, dirs, userData, options) + await assertNoSymlinkAncestors(safe) + await rm(safe, { recursive: true, force: true }) +} + +export async function moveStorageDirectory( + source: string, destination: string, dirs: StorageDirs, userData: string, options: Options = {}, +): Promise { + const paths = validateStorageMove(source, destination, dirs, userData, options) + await assertNoSymlinkAncestors(paths.src) + await assertNoSymlinkAncestors(paths.dest) + await assertEmptyDestination(paths.dest) + await mkdir(paths.dest, { recursive: true }) + // Refuse races that add files to the destination after the emptiness check. + await cp(paths.src, paths.dest, { recursive: true, force: false, errorOnExist: true }) + // Re-validate before deleting the source. Never delete it after a copy error. + validateStorageMove(paths.src, paths.dest, dirs, userData, options) + await assertNoSymlinkAncestors(paths.src) + await rm(paths.src, { recursive: true, force: true }) +} diff --git a/package.json b/package.json index a63727a24..a17e0aa4f 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." From 6481e15c6dd87143fed62d77e91c1a6e769b5154 Mon Sep 17 00:00:00 2001 From: JayceeB1 <45721154+JayceeB1@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:07:59 +0200 Subject: [PATCH 004/108] fix(storage): copy safely into empty destination without EEXIST Confirmed by combined CI regression. Remove only an empty target with rmdir; never remove source on copy failure. --- electron/main/storage-directory-guard.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/electron/main/storage-directory-guard.ts b/electron/main/storage-directory-guard.ts index fb1e9b914..f5a63a8c8 100644 --- a/electron/main/storage-directory-guard.ts +++ b/electron/main/storage-directory-guard.ts @@ -1,5 +1,5 @@ /** Fail-closed validation for settings-driven directory moves and deletions. */ -import { cp, lstat, mkdir, readdir, rm } from 'node:fs/promises' +import { cp, lstat, mkdir, readdir, rm, rmdir } from 'node:fs/promises' import { homedir } from 'node:os' import path from 'node:path' @@ -189,8 +189,16 @@ export async function moveStorageDirectory( await assertNoSymlinkAncestors(paths.src) await assertNoSymlinkAncestors(paths.dest) await assertEmptyDestination(paths.dest) - await mkdir(paths.dest, { recursive: true }) - // Refuse races that add files to the destination after the emptiness check. + // fs.cp with errorOnExist refuses even an EMPTY existing directory. Remove + // only the verified-empty destination, never the source, before copying. + // rmdir itself fails closed if a concurrent writer adds a file. + try { + await rmdir(paths.dest) + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err + } + await mkdir(path.dirname(paths.dest), { recursive: true }) + // With a missing destination, errorOnExist protects against unexpected files. await cp(paths.src, paths.dest, { recursive: true, force: false, errorOnExist: true }) // Re-validate before deleting the source. Never delete it after a copy error. validateStorageMove(paths.src, paths.dest, dirs, userData, options) From 2180a12b337046bedb429dee72458ea779b68b6f Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Tue, 20 Oct 2026 11:23:26 +0200 Subject: [PATCH 005/108] feat(agent): stream chat responses as typed SSE events agent_chat now returns a StreamingResponse emitting token, thinking, tool_start, tool_done and done events. ChatPanel consumes the stream so text appears progressively and tool calls show an in-progress state. Closes #300 --- api/routers/agent.py | 326 +++++++++++++++++--- api/tests/test_agent_streaming.py | 293 ++++++++++++++++++ package.json | 2 +- src/areas/generate/components/ChatPanel.tsx | 162 ++++++---- src/shared/utils/agentStream.test.ts | 57 ++++ src/shared/utils/agentStream.ts | 75 +++++ 6 files changed, 808 insertions(+), 107 deletions(-) create mode 100644 api/tests/test_agent_streaming.py create mode 100644 src/shared/utils/agentStream.test.ts create mode 100644 src/shared/utils/agentStream.ts diff --git a/api/routers/agent.py b/api/routers/agent.py index 3cf15d442..e2cb56efb 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -4,12 +4,12 @@ """ import asyncio import json -import re import uuid from typing import Optional import httpx from fastapi import APIRouter +from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from services import llm_server @@ -634,19 +634,6 @@ class AgentChatResponse(BaseModel): thinking: str | None = None -def _extract_thinking(msg: dict) -> tuple[str, str | None]: - """Return (clean_content, thinking_text). Handles llama-server's - reasoning_content field and inline tags.""" - content = msg.get("content") or "" - thinking = msg.get("reasoning_content") or None - if not thinking: - match = re.search(r"(.*?)", content, re.DOTALL) - if match: - thinking = match.group(1).strip() - content = (content[: match.start()] + content[match.end() :]).strip() - return content, thinking - - def _auth_headers(base_url: str, api_key: Optional[str]) -> dict: headers: dict[str, str] = {} if api_key: @@ -741,8 +728,198 @@ def _drop_images(messages: list[dict]) -> bool: return dropped -@router.post("/chat", response_model=AgentChatResponse) -async def agent_chat(request: AgentChatRequest): +class _ThinkSplitter: + """Route inline … text to the reasoning, the rest to the + answer, holding back a tag a chunk boundary has cut in two.""" + + _OPEN, _CLOSE = "", "" + + def __init__(self) -> None: + self._buffer = "" + self._inside = False + + def _held(self, *tags: str) -> int: + """Length of the buffer's tail that may be the start of a tag.""" + return max( + next((n for n in range(len(tag) - 1, 0, -1) if self._buffer.endswith(tag[:n])), 0) + for tag in tags + ) + + def _cut(self, index: int, tag: str, kind: str, parts: list) -> None: + if index: + parts.append((kind, self._buffer[:index])) + self._buffer = self._buffer[index + len(tag):] + + def feed(self, text: str) -> list[tuple[str, str]]: + self._buffer += text + parts: list[tuple[str, str]] = [] + while self._buffer: + close_at = self._buffer.find(self._CLOSE) + if self._inside: + if close_at >= 0: + self._cut(close_at, self._CLOSE, "thinking", parts) + self._inside = False + continue + keep = self._held(self._CLOSE) + kind = "thinking" + else: + open_at = self._buffer.find(self._OPEN) + if close_at >= 0 and (open_at < 0 or close_at < open_at): + # A chat template that opens in the prompt leaves the + # model only the closing tag: what came before it was reasoning. + self._cut(close_at, self._CLOSE, "thinking", parts) + continue + if open_at >= 0: + self._cut(open_at, self._OPEN, "token", parts) + self._inside = True + continue + keep = self._held(self._OPEN, self._CLOSE) + kind = "token" + if len(self._buffer) > keep: + parts.append((kind, self._buffer[: len(self._buffer) - keep])) + self._buffer = self._buffer[len(self._buffer) - keep:] + break + return parts + + def flush(self) -> list[tuple[str, str]]: + rest, self._buffer = self._buffer, "" + return [("thinking" if self._inside else "token", rest)] if rest else [] + + +class _Round: + """One LLM call of the tool loop, assembled from its streamed deltas.""" + + def __init__(self, answered_before: bool, thought_before: bool) -> None: + self.raw = "" # content as the model sent it, tags included + self.answer = "" + self.thinking = "" + self.tool_calls: dict[int, dict] = {} + self._splitter = _ThinkSplitter() + self._answered_before = answered_before + self._thought_before = thought_before + + def _text(self, kind: str, text: str) -> dict | None: + # Each round's text is stripped at its start, as the non-streamed + # answer was, and set apart from what earlier rounds already showed. + if kind == "token": + if not self.answer: + text = text.lstrip() + if not text: + return None + self.answer = text + return {"type": "token", "text": ("\n\n" if self._answered_before else "") + text} + self.answer += text + return {"type": "token", "text": text} + if not self.thinking: + text = text.lstrip() + if not text: + return None + self.thinking = text + return {"type": "thinking", "text": ("\n\n---\n\n" if self._thought_before else "") + text} + self.thinking += text + return {"type": "thinking", "text": text} + + def apply(self, delta: dict) -> list[dict]: + events = [] + # llama-server sends reasoning_content; Ollama and OpenRouter, reasoning. + reasoning = delta.get("reasoning_content") or delta.get("reasoning") + if isinstance(reasoning, str) and reasoning: + events.append(self._text("thinking", reasoning)) + content = delta.get("content") + if isinstance(content, str) and content: + self.raw += content + events.extend(self._text(kind, text) for kind, text in self._splitter.feed(content)) + for position, fragment in enumerate(delta.get("tool_calls") or []): + # Streamed tool calls arrive in fragments keyed by index: the id and + # name come first, the JSON arguments in pieces after them. + call = self.tool_calls.setdefault( + fragment.get("index", position), + {"id": "", "type": "function", "function": {"name": "", "arguments": ""}}, + ) + call["id"] = fragment.get("id") or call["id"] + function = fragment.get("function") or {} + call["function"]["name"] += function.get("name") or "" + arguments = function.get("arguments") + if isinstance(arguments, dict): + arguments = json.dumps(arguments) + call["function"]["arguments"] += arguments or "" + return [e for e in events if e] + + def finish(self) -> list[dict]: + return [e for e in (self._text(k, t) for k, t in self._splitter.flush()) if e] + + def calls(self) -> list[dict]: + calls = [self.tool_calls[i] for i in sorted(self.tool_calls)] + for call in calls: + # The tool result must answer the call's id; a few servers omit it. + call["id"] = call["id"] or f"call_{uuid.uuid4().hex[:8]}" + return calls + + +async def _read_round(response: httpx.Response, round_: _Round): + """Yield the answer and reasoning of one LLM call as they stream in.""" + if "text/event-stream" not in response.headers.get("content-type", ""): + # A server that ignored "stream": true answers in one JSON body. + message = json.loads(await response.aread())["choices"][0]["message"] + for event in round_.apply(message): + yield event + else: + async for line in response.aiter_lines(): + if not line.startswith("data:"): + continue + data = line[len("data:"):].strip() + if data == "[DONE]": + break + try: + choices = json.loads(data).get("choices") or [] + except ValueError: + continue + if choices: + for event in round_.apply(choices[0].get("delta") or {}): + yield event + for event in round_.finish(): + yield event + + +def _done(message: str, actions: list[ActionDone], thinking: list[str], error: bool = False) -> dict: + return { + "type": "done", + "message": message, + "actions": [a.model_dump() for a in actions], + "thinking": "\n\n---\n\n".join(thinking) if thinking else None, + # The message reports a failure rather than answering: the chat shows + # it after whatever text already streamed. + "error": error, + } + + +def _release_claimed(claim: asyncio.Future) -> None: + if not claim.cancelled() and claim.exception() is None: + claim.result().release() + + +async def _claim_slot(model: str, spec: dict): + """Hold a local LLM slot, without leaking it if the caller is cancelled. + + A streamed chat is cancelled when its client leaves, possibly while the + model is still loading. The loading thread can't be stopped and claims the + slot anyway; nobody would release it, and a held slot is never reaped. + """ + claim = asyncio.ensure_future(asyncio.to_thread(llama_pool.ensure, model, spec, True)) + try: + return await asyncio.shield(claim) + except asyncio.CancelledError: + claim.add_done_callback(_release_claimed) + raise + + +async def _chat_events(request: AgentChatRequest): + """Run the tool loop, yielding its events as they happen. + + token / thinking — answer and reasoning text, in pieces + tool_start / tool_done — around each tool call + done — always last: the final answer, every action and the full reasoning + """ messages: list[dict] = [{"role": "system", "content": SYSTEM_PROMPT}] # Inject scene context so the LLM knows current state @@ -785,9 +962,10 @@ async def agent_chat(request: AgentChatRequest): spec = llm_server.resolve_model(request.model) # hold=True: the slot stays claimed for the whole loop, so neither the # idle reaper nor a model loading elsewhere can evict it mid-answer. - slot = await asyncio.to_thread(llama_pool.ensure, request.model, spec, True) + slot = await _claim_slot(request.model, spec) except Exception as e: - return AgentChatResponse(message=f"Could not start the local LLM: {e}") + yield _done(f"Could not start the local LLM: {e}", [], [], error=True) + return base_url = slot.base_url headers: dict = {} send_images = spec["vision"] @@ -796,7 +974,8 @@ async def agent_chat(request: AgentChatRequest): extra["chat_template_kwargs"] = {"enable_thinking": False} else: if not request.provider.base_url: - return AgentChatResponse(message="No provider URL configured. Set one in Settings → Agent.") + yield _done("No provider URL configured. Set one in Settings → Agent.", [], [], error=True) + return base_url = request.provider.base_url.rstrip("/") headers = _auth_headers(base_url, request.provider.api_key) @@ -805,52 +984,70 @@ async def agent_chat(request: AgentChatRequest): actions_done: list[ActionDone] = [] all_thinking: list[str] = [] - final: AgentChatResponse | None = None + final: str | None = None + error: str | None = None + answered = False try: async with httpx.AsyncClient(timeout=120.0) as client: - for _ in range(10): # max tool-call rounds - r = await client.post( - f"{base_url}/chat/completions", - headers=headers, - json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": False, **extra}, - ) - # An external model's vision support is unknown up front, and a - # text-only one rejects image parts with a 400 that failed the - # whole turn. Retry once with the images described as omitted. - if r.status_code == 400 and request.provider.type != "local" and _drop_images(messages): - r = await client.post( + + async def post() -> httpx.Response: + return await client.send( + client.build_request( + "POST", f"{base_url}/chat/completions", headers=headers, - json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": False, **extra}, - ) - - if r.status_code != 200: - return AgentChatResponse(message=f"LLM error ({r.status_code}): {_error_detail(r)}") + json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": True, **extra}, + ), + stream=True, + ) - msg = r.json()["choices"][0]["message"] - tool_calls = msg.get("tool_calls") or [] - assistant: dict = {"role": "assistant", "content": msg.get("content") or ""} + for _ in range(10): # max tool-call rounds + round_ = _Round(answered, bool(all_thinking)) + r = await post() + try: + # An external model's vision support is unknown up front, and a + # text-only one rejects image parts with a 400 that failed the + # whole turn. Retry once with the images described as omitted. + if r.status_code == 400 and request.provider.type != "local" and _drop_images(messages): + await r.aclose() + r = await post() + + if r.status_code != 200: + await r.aread() + error = f"LLM error ({r.status_code}): {_error_detail(r)}" + break + + async for event in _read_round(r, round_): + yield event + finally: + await r.aclose() + + answered = answered or bool(round_.answer) + if round_.thinking: + all_thinking.append(round_.thinking) + tool_calls = round_.calls() + assistant: dict = {"role": "assistant", "content": round_.raw} if tool_calls: assistant["tool_calls"] = tool_calls messages.append(assistant) - clean_content, thinking_text = _extract_thinking(msg) - if thinking_text: - all_thinking.append(thinking_text) - if not tool_calls: - final = AgentChatResponse(message=clean_content, actions=actions_done) + final = round_.answer.strip() break for tc in tool_calls: fn = tc["function"] - result_text, payload = await execute_tool(fn["name"], _tool_arguments(fn.get("arguments")), request.context) + arguments = _tool_arguments(fn.get("arguments")) + yield {"type": "tool_start", "id": tc["id"], "tool": fn["name"], "arguments": arguments} + result_text, payload = await execute_tool(fn["name"], arguments, request.context) actions_done.append(ActionDone(tool=fn["name"], result=result_text, payload=payload)) - messages.append({"role": "tool", "tool_call_id": tc.get("id", ""), "content": result_text}) + yield {"type": "tool_done", "id": tc["id"], "tool": fn["name"], "result": result_text, "payload": payload} + messages.append({"role": "tool", "tool_call_id": tc["id"], "content": result_text}) except httpx.HTTPError as e: - return AgentChatResponse(message=f"Could not reach the LLM at {base_url}: {e}", actions=actions_done) + error = f"Could not reach the LLM at {base_url}: {e}" finally: + # Also runs when the client disconnects mid-answer. if slot is not None: slot.release() @@ -858,7 +1055,34 @@ async def agent_chat(request: AgentChatRequest): # still holding ours would keep the agent's own model on the GPU. await _unload_llm_after_workflow(request, actions_done) - if final is None: - final = AgentChatResponse(message="Reached maximum tool iterations.", actions=actions_done) - final.thinking = "\n\n---\n\n".join(all_thinking) if all_thinking else None - return final + if error is not None: + yield _done(error, actions_done, all_thinking, error=True) + elif final is None: + yield _done("Reached maximum tool iterations.", actions_done, all_thinking, error=True) + else: + yield _done(final, actions_done, all_thinking) + + +@router.post("/chat", response_model=AgentChatResponse) +async def agent_chat(request: AgentChatRequest): + """The whole turn at once, for callers that don't read a stream.""" + done: dict = {} + async for event in _chat_events(request): + if event["type"] == "done": + done = event + return AgentChatResponse(message=done["message"], actions=done["actions"], thinking=done["thinking"]) + + +@router.post("/chat/stream") +async def agent_chat_stream(request: AgentChatRequest): + """The same turn as Server-Sent Events, one JSON event per `data:` line.""" + + async def body(): + async for event in _chat_events(request): + yield f"data: {json.dumps(event)}\n\n" + + return StreamingResponse( + body(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) diff --git a/api/tests/test_agent_streaming.py b/api/tests/test_agent_streaming.py new file mode 100644 index 000000000..123e93c2c --- /dev/null +++ b/api/tests/test_agent_streaming.py @@ -0,0 +1,293 @@ +import asyncio +import json +import threading +import unittest +from unittest import mock + +import httpx + +import routers.agent as agent + + +def _sse(*deltas: dict) -> httpx.Response: + """A streamed /chat/completions answer, one chunk per delta.""" + lines = [f"data: {json.dumps({'choices': [{'delta': d}]})}\n\n" for d in deltas] + lines.append("data: [DONE]\n\n") + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content="".join(lines).encode(), + ) + + +def _call(index: int, **fields) -> dict: + return {"tool_calls": [{"index": index, **fields}]} + + +_WORKFLOWS = {"workflows": [{"id": "wf1", "name": "My Workflow"}]} + + +class _Scripted: + """Patches httpx.AsyncClient to serve scripted responses in order, recording + every request body.""" + + def __init__(self, *responses: httpx.Response) -> None: + self._responses = list(responses) + self.bodies: list[dict] = [] + self._real = httpx.AsyncClient + + def _handler(self, request: httpx.Request) -> httpx.Response: + self.bodies.append(json.loads(request.content)) + return self._responses.pop(0) + + def __call__(self, *args, **kwargs): + kwargs["transport"] = httpx.MockTransport(self._handler) + return self._real(*args, **kwargs) + + +def _external(context=None) -> agent.AgentChatRequest: + return agent.AgentChatRequest( + messages=[agent.ChatMessage(role="user", content="hi")], + model="gpt-test", + provider=agent.ProviderConfig(type="external", base_url="https://llm.test/v1", api_key="k"), + context=context or {}, + ) + + +def _events(scripted: _Scripted, request: agent.AgentChatRequest) -> list[dict]: + async def collect(): + return [event async for event in agent._chat_events(request)] + + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + return asyncio.run(collect()) + + +def _text(events: list[dict], kind: str) -> str: + return "".join(e["text"] for e in events if e["type"] == kind) + + +class ChatEventTests(unittest.TestCase): + def test_answer_streams_token_by_token_then_done(self) -> None: + scripted = _Scripted(_sse({"content": "Hel"}, {"content": "lo"}, {"content": " there"})) + + events = _events(scripted, _external()) + + self.assertEqual([e["type"] for e in events], ["token", "token", "token", "done"]) + self.assertEqual(_text(events, "token"), "Hello there") + self.assertEqual(events[-1]["message"], "Hello there") + self.assertFalse(events[-1]["error"]) + self.assertTrue(scripted.bodies[0]["stream"]) + + def test_reasoning_field_streams_as_thinking(self) -> None: + events = _events( + _Scripted(_sse({"reasoning_content": "Let me "}, {"reasoning_content": "think."}, {"content": "Hi"})), + _external(), + ) + + self.assertEqual(_text(events, "thinking"), "Let me think.") + self.assertEqual(_text(events, "token"), "Hi") + self.assertEqual(events[-1]["thinking"], "Let me think.") + + def test_inline_think_tags_cut_across_chunks_are_split_out(self) -> None: + events = _events( + _Scripted(_sse({"content": "plan\n\nHello "})), + _external(), + ) + + self.assertEqual(_text(events, "thinking"), "plan") + # A "<" that turns out not to open a tag is still part of the answer. + self.assertEqual(_text(events, "token"), "Hello ") + self.assertEqual(events[-1]["message"], "Hello ") + + def test_fragmented_tool_call_is_reassembled_and_reported_around_its_run(self) -> None: + scripted = _Scripted( + _sse( + {"content": "Starting it."}, + _call(0, id="call_1", type="function", function={"name": "run_workflow", "arguments": ""}), + _call(0, function={"arguments": '{"workflow_'}), + _call(0, function={"arguments": 'id": "wf1"}'}), + ), + _sse({"content": "Done."}), + ) + + events = _events(scripted, _external(_WORKFLOWS)) + + types = [e["type"] for e in events] + self.assertEqual(types, ["token", "tool_start", "tool_done", "token", "done"]) + start, finish = events[1], events[2] + self.assertEqual((start["id"], start["tool"], start["arguments"]), ("call_1", "run_workflow", {"workflow_id": "wf1"})) + self.assertEqual(finish["payload"]["type"], "run_workflow") + # Rounds are set apart in the streamed text; the final answer is the last one. + self.assertEqual(_text(events, "token"), "Starting it.\n\nDone.") + self.assertEqual(events[-1]["message"], "Done.") + self.assertEqual([a["tool"] for a in events[-1]["actions"]], ["run_workflow"]) + + replay = scripted.bodies[1]["messages"] + assistant = next(m for m in replay if m["role"] == "assistant") + self.assertEqual(assistant["tool_calls"][0]["function"]["arguments"], '{"workflow_id": "wf1"}') + self.assertEqual(replay[-1], {"role": "tool", "tool_call_id": "call_1", "content": finish["result"]}) + + def test_tool_call_without_an_id_still_gets_its_result_matched(self) -> None: + scripted = _Scripted( + _sse(_call(0, function={"name": "run_workflow", "arguments": '{"workflow_id": "wf1"}'})), + _sse({"content": "ok"}), + ) + + events = _events(scripted, _external(_WORKFLOWS)) + + call_id = events[0]["id"] + self.assertTrue(call_id) + tool_message = scripted.bodies[1]["messages"][-1] + self.assertEqual(tool_message["tool_call_id"], call_id) + + def test_reasoning_of_successive_rounds_is_separated(self) -> None: + scripted = _Scripted( + _sse({"reasoning_content": "first"}, _call(0, id="c", function={"name": "list_workflows", "arguments": "{}"})), + _sse({"reasoning_content": "second"}, {"content": "ok"}), + ) + + with mock.patch.object(agent, "execute_tool", mock.AsyncMock(return_value=("none", None))): + events = _events(scripted, _external()) + + self.assertEqual(_text(events, "thinking"), "first\n\n---\n\nsecond") + self.assertEqual(events[-1]["thinking"], "first\n\n---\n\nsecond") + + def test_provider_error_ends_with_an_error_done(self) -> None: + events = _events( + _Scripted(httpx.Response(401, json={"error": {"message": "Incorrect API key provided."}})), + _external(), + ) + + self.assertEqual(len(events), 1) + self.assertTrue(events[0]["error"]) + self.assertEqual(events[0]["message"], "LLM error (401): Incorrect API key provided.") + + def test_tool_loop_limit_is_reported_as_an_error(self) -> None: + rounds = [ + _sse(_call(0, id=f"c{i}", function={"name": "list_workflows", "arguments": "{}"})) + for i in range(10) + ] + with mock.patch.object(agent, "execute_tool", mock.AsyncMock(return_value=("none", None))): + events = _events(_Scripted(*rounds), _external()) + + self.assertTrue(events[-1]["error"]) + self.assertEqual(events[-1]["message"], "Reached maximum tool iterations.") + self.assertEqual(len(events[-1]["actions"]), 10) + + def test_local_slot_is_released_when_the_client_disconnects(self) -> None: + class Slot: + base_url = "http://127.0.0.1:8791/v1" + released = 0 + + def release(self) -> None: + self.released += 1 + + slot = Slot() + request = agent.AgentChatRequest( + messages=[agent.ChatMessage(role="user", content="hi")], model="qwen3-4b" + ) + + async def first_event_then_disconnect(): + stream = agent._chat_events(request) + first = await stream.__anext__() + await stream.aclose() + return first + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_sse({"content": "a"}, {"content": "b"}))), \ + mock.patch.object(agent.llm_server, "resolve_model", return_value={"vision": False}), \ + mock.patch.object(agent.llama_pool, "ensure", return_value=slot), \ + mock.patch.object(agent.llama_pool, "unload_all"): + first = asyncio.run(first_event_then_disconnect()) + + self.assertEqual(first, {"type": "token", "text": "a"}) + self.assertEqual(slot.released, 1) + + def test_slot_claimed_after_the_client_left_during_loading_is_released(self) -> None: + class Slot: + base_url = "http://127.0.0.1:8791/v1" + released = 0 + + def release(self) -> None: + self.released += 1 + + slot = Slot() + loading, loaded = threading.Event(), threading.Event() + + def ensure(*_args): + loading.set() + loaded.wait(5) + return slot + + request = agent.AgentChatRequest( + messages=[agent.ChatMessage(role="user", content="hi")], model="qwen3-4b" + ) + + async def leave_while_loading(): + turn = asyncio.ensure_future(agent._chat_events(request).__anext__()) + await asyncio.to_thread(loading.wait, 5) + turn.cancel() + with self.assertRaises(asyncio.CancelledError): + await turn + loaded.set() # the model finishes loading after the client left + for _ in range(200): + if slot.released: + break + await asyncio.sleep(0.01) + + with mock.patch.object(agent.llm_server, "resolve_model", return_value={"vision": False}), \ + mock.patch.object(agent.llama_pool, "ensure", ensure): + asyncio.run(leave_while_loading()) + + self.assertEqual(slot.released, 1) + + +class StreamRouteTests(unittest.TestCase): + def test_stream_route_sends_one_json_event_per_data_line(self) -> None: + async def read_body(): + response = await agent.agent_chat_stream(_external()) + self.assertEqual(response.media_type, "text/event-stream") + return "".join([chunk async for chunk in response.body_iterator]) + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_sse({"content": "Hi"}))): + body = asyncio.run(read_body()) + + blocks = [b for b in body.split("\n\n") if b] + self.assertTrue(all(b.startswith("data: ") for b in blocks)) + events = [json.loads(b[len("data: "):]) for b in blocks] + self.assertEqual([e["type"] for e in events], ["token", "done"]) + + def test_json_route_returns_the_same_turn_at_once(self) -> None: + scripted = _Scripted( + _sse(_call(0, id="call_1", function={"name": "run_workflow", "arguments": '{"workflow_id": "wf1"}'})), + _sse({"reasoning_content": "hm"}, {"content": "Running."}), + ) + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + response = asyncio.run(agent.agent_chat(_external(_WORKFLOWS))) + + self.assertEqual(response.message, "Running.") + self.assertEqual(response.thinking, "hm") + self.assertEqual([a.tool for a in response.actions], ["run_workflow"]) + + +class ThinkSplitterTests(unittest.TestCase): + def test_partial_tag_is_held_until_it_resolves(self) -> None: + splitter = agent._ThinkSplitter() + + self.assertEqual(splitter.feed("a<"), [("token", "a")]) + self.assertEqual(splitter.feed("think>b"), [("thinking", "b")]) + self.assertEqual(splitter.feed(" None: + # The chat template opened in the prompt, so only its end streams. + splitter = agent._ThinkSplitter() + self.assertEqual(splitter.feed("plan\n\nAnswer"), [("thinking", "plan"), ("token", "\n\nAnswer")]) + + # Reasoning that streamed before the tag was already shown; the tag is not. + splitter = agent._ThinkSplitter() + parts = splitter.feed("planAnswer") + splitter.flush() + self.assertEqual(parts, [("token", "plan"), ("token", "Answer")]) + + +if __name__ == "__main__": + unittest.main() diff --git a/package.json b/package.json index a63727a24..e8da0f9d5 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/components/ChatPanel.tsx b/src/areas/generate/components/ChatPanel.tsx index 053e780a9..d6db103e2 100644 --- a/src/areas/generate/components/ChatPanel.tsx +++ b/src/areas/generate/components/ChatPanel.tsx @@ -6,11 +6,13 @@ import { useWorkflowsStore } from '@shared/stores/workflowsStore' import { useExtensionsStore } from '@shared/stores/extensionsStore' import { useWorkflowRunStore } from '@areas/workflows/workflowRunStore' import { buildAllWorkflowExtensions } from '@areas/workflows/mockExtensions' +import { readAgentEvents } from '@shared/utils/agentStream' // ─── Types ──────────────────────────────────────────────────────────────────── import type { ThinkingMode } from '@shared/stores/agentStore' import type { Workflow } from '@shared/types/electron.d' +import type { AgentAction, AgentActionPayload } from '@shared/utils/agentStream' interface Message { id: string @@ -19,19 +21,12 @@ interface Message { thinking?: string imageDataUrls?: string[] actions?: ActionDone[] + streaming?: boolean } -interface ActionDone { - tool: string - result: string - payload?: { - type: string - url?: string - face_count?: number - workflow_id?: string - workflow_name?: string - workflow?: Omit - } | null +interface ActionDone extends AgentAction { + id?: string + running?: boolean } // ─── Constants ──────────────────────────────────────────────────────────────── @@ -86,15 +81,23 @@ const TOOL_LABELS: Record = { function ActionsCard({ actions, onUndo }: { actions: ActionDone[]; onUndo?: () => void }): JSX.Element { const [expanded, setExpanded] = useState(false) const meshActions = actions.filter((a) => a.payload?.type === 'mesh_update') - const canUndo = meshActions.length > 0 && !!onUndo + const running = actions.find((a) => a.running) + const canUndo = meshActions.length > 0 && !!onUndo && !running return (
{/* Header */}
- - {actions.length} action{actions.length > 1 ? 's' : ''} performed - + {running ? ( + + + Running {running.tool.replace(/_/g, ' ')}… + + ) : ( + + {actions.length} action{actions.length > 1 ? 's' : ''} performed + + )}
{canUndo && ( + )} + > + {engineInstall && } + {engineError &&

{engineError}

} + + )} {engineInstalled === null && ( From cd16feb0355c71c72b09889c7a7b0f9177ca03be Mon Sep 17 00:00:00 2001 From: Lorchie Date: Mon, 26 Oct 2026 16:48:02 +0100 Subject: [PATCH 007/108] test(mesh-ops): fix two tests failing on a clean dev checkout - test_auto_orient_does_not_flip_a_mesh_over: trimesh's update_faces keeps the cut-away vertices, so the exported bowl still carried the whole sphere's vertices and the "center" was measured on the orphan top cap. Drop unreferenced vertices; auto_orient itself was correct. - test_mesh_ops_run_without_undeclared_networkx runs repair_mesh, which needs pymeshlab: skip it when pymeshlab is missing, like the other repair/inspection tests. --- api/tests/test_mesh_ops_dependencies.py | 3 +++ api/tests/test_mesh_ops_transforms.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/api/tests/test_mesh_ops_dependencies.py b/api/tests/test_mesh_ops_dependencies.py index 3b0d4517d..96db84105 100644 --- a/api/tests/test_mesh_ops_dependencies.py +++ b/api/tests/test_mesh_ops_dependencies.py @@ -1,3 +1,4 @@ +import importlib.util import re import subprocess import sys @@ -6,6 +7,7 @@ API_DIR = Path(__file__).resolve().parents[1] +HAS_PYMESHLAB = importlib.util.find_spec("pymeshlab") is not None # Runs every mesh-op family on a holed box with networkx unimportable: it is # not in requirements.txt, so the shipped venv never has it, while a dev venv @@ -44,6 +46,7 @@ class MeshOpDependencyTests(unittest.TestCase): + @unittest.skipUnless(HAS_PYMESHLAB, "pymeshlab is not installed") def test_mesh_ops_run_without_undeclared_networkx(self) -> None: result = subprocess.run( [sys.executable, "-c", _SCRIPT], diff --git a/api/tests/test_mesh_ops_transforms.py b/api/tests/test_mesh_ops_transforms.py index 21407209b..8059a4233 100644 --- a/api/tests/test_mesh_ops_transforms.py +++ b/api/tests/test_mesh_ops_transforms.py @@ -98,6 +98,9 @@ def test_auto_orient_does_not_flip_a_mesh_over(self) -> None: # A shallow bowl (open hemisphere-like dish) opening upwards. bowl = trimesh.creation.icosphere(subdivisions=3) bowl.update_faces(bowl.triangles_center[:, 1] < -0.3) + # update_faces keeps the cut-away vertices, which would still be + # exported and measured below as the top of a whole sphere. + bowl.remove_unreferenced_vertices() bowl.apply_scale([1.0, 0.3, 1.0]) bowl.apply_transform( trimesh.transformations.rotation_matrix(np.radians(10), [1, 0, 0]) From 86a23dc0b8d6b40da16f341e2c5bd457af110b18 Mon Sep 17 00:00:00 2001 From: ahgsql Date: Fri, 30 Oct 2026 23:28:43 +0300 Subject: [PATCH 008/108] Keep TripoSG and the texture model loaded together switch_model unloads the previous model on every switch. When you run TripoSG and then Trellis2 Texture Mesh on the same object, both models get reloaded for every single asset, which cost me 20-27 s per switch on a 3090 Ti. This keeps the two loaded side by side. Any other switch, and the LLM eviction, works exactly as before. Two assets (cart + soldier, 3000 faces, warm): 197 s -> 110 s. --- api/services/generator_registry.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/services/generator_registry.py b/api/services/generator_registry.py index 2792030e9..523512493 100644 --- a/api/services/generator_registry.py +++ b/api/services/generator_registry.py @@ -38,6 +38,10 @@ weight_group_sources_are_downloaded, ) +# Model pairs that stay loaded together when the active model switches between them (a mesh -> texture +# pipeline would otherwise unload and reload both models on every asset). +_KEEP_RESIDENT_PAIRS = [{"triposg/generate", "trellis2/refine"}] + # ------------------------------------------------------------------ # # Global paths # ------------------------------------------------------------------ # @@ -904,7 +908,10 @@ def switch_model(self, model_id: str) -> None: llama_pool.unload_all() if model_id != self._active_id: - if self._active_id in self._generators: + # Keep the geometry model and the texture model resident together instead of unloading + # the previous one on every switch (see _KEEP_RESIDENT_PAIRS). + if (self._active_id in self._generators + and {self._active_id, model_id} not in _KEEP_RESIDENT_PAIRS): self._generators[self._active_id].unload() self._active_id = model_id From 7f13d137bc3e1964f57ba6709035e021eefadb70 Mon Sep 17 00:00:00 2001 From: ahgsql Date: Fri, 30 Oct 2026 23:28:43 +0300 Subject: [PATCH 009/108] modly-cli: start the API with the same setup as the desktop app `dev serve-api` / `dev ensure-server --start` ignored dependenciesDir from settings.json. If the dependencies live on another drive (mine are on F:), the venv isn't found and it falls back to whatever Python runs the CLI, which doesn't have the backend deps. It now checks dependenciesDir first, and passes the env vars the Electron bridge sets: PYTHONIOENCODING=utf-8, MODLY_LLM_DIR (agentDir) and MODLY_NODE_EXECUTABLE (Modly.exe) for the installed app. Added two tests; the existing ones still pass. --- tools/modly-cli/agent.py | 19 ++++++++++++++++--- tools/modly-cli/test_agent.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/tools/modly-cli/agent.py b/tools/modly-cli/agent.py index e424aa25b..09f35755c 100644 --- a/tools/modly-cli/agent.py +++ b/tools/modly-cli/agent.py @@ -283,11 +283,16 @@ def _default_api_dir() -> Path | None: return None -def _default_python(api_dir: Path) -> Path | None: +def _default_python(api_dir: Path, settings: dict[str, Any] | None = None) -> Path | None: candidates = [ api_dir / ".venv" / "Scripts" / "python.exe", api_dir / ".venv" / "bin" / "python", ] + # The desktop app can keep its dependencies on another drive (Settings > dependenciesDir). + deps_dir = (settings or {}).get("dependenciesDir") + if deps_dir: + candidates.append(Path(deps_dir) / "venv" / "Scripts" / "python.exe") + candidates.append(Path(deps_dir) / "venv" / "bin" / "python") for appdata in _windows_env_paths("APPDATA"): candidates.append(appdata / "Modly" / "dependencies" / "venv" / "Scripts" / "python.exe") candidates.append(Path(sys.executable)) @@ -317,11 +322,11 @@ def _resolve_serve_config(args: argparse.Namespace) -> tuple[Path, Path, dict[st if not api_dir or not (api_dir / "main.py").exists(): raise ModlyCliError("Could not find Modly api directory; pass --api-dir") - python = Path(args.python).expanduser().resolve() if getattr(args, "python", None) else _default_python(api_dir) + settings = _load_modly_settings() + python = Path(args.python).expanduser().resolve() if getattr(args, "python", None) else _default_python(api_dir, settings) if not python or not python.exists(): raise ModlyCliError("Could not find Modly Python environment; pass --python") - settings = _load_modly_settings() env = os.environ.copy() hf_token = getattr(args, "hf_token", None) or settings.get("hfToken") or os.environ.get("HF_TOKEN", "") env.update({ @@ -332,7 +337,15 @@ def _resolve_serve_config(args: argparse.Namespace) -> tuple[Path, Path, dict[st "SELECTED_MODEL_ID": getattr(args, "model", None) or os.environ.get("SELECTED_MODEL_ID", ""), "HUGGING_FACE_HUB_TOKEN": hf_token, "HF_TOKEN": hf_token, + # Same as the Electron PythonBridge: UTF-8 logs, the local LLM folder, and Modly.exe as the Node + # runtime of the mesh optimizer (installed app only; otherwise mesh_ops falls back to `node`). + "PYTHONIOENCODING": "utf-8", }) + if settings.get("agentDir") and "MODLY_LLM_DIR" not in os.environ: + env["MODLY_LLM_DIR"] = settings["agentDir"] + modly_exe = api_dir.parent.parent / ("Modly.exe" if os.name == "nt" else "modly") + if modly_exe.exists() and "MODLY_NODE_EXECUTABLE" not in os.environ: + env["MODLY_NODE_EXECUTABLE"] = str(modly_exe) cmd = [str(python), "-m", "uvicorn", "main:app", "--host", args.host, "--port", str(args.port)] base_url = f"http://{args.host}:{args.port}" return api_dir, python, env, cmd, base_url diff --git a/tools/modly-cli/test_agent.py b/tools/modly-cli/test_agent.py index 9537ec1af..432492bab 100644 --- a/tools/modly-cli/test_agent.py +++ b/tools/modly-cli/test_agent.py @@ -483,6 +483,35 @@ def test_malformed_timeout_environment_does_not_break_module_import(self) -> Non self.assertEqual(module.DEFAULT_TIMEOUT_SECONDS, 1800) self.assertEqual(module.DEFAULT_POLL_SECONDS, 2.0) + def test_default_python_uses_settings_dependencies_dir(self) -> None: + with tempfile.TemporaryDirectory() as td: + api_dir = Path(td) / "api" + api_dir.mkdir() + deps_python = Path(td) / "deps" / "venv" / "Scripts" / "python.exe" + deps_python.parent.mkdir(parents=True) + deps_python.write_text("# python") + with patch.object(agent, "_windows_env_paths", return_value=[]): + found = agent._default_python(api_dir, {"dependenciesDir": str(Path(td) / "deps")}) + self.assertEqual(found, deps_python) + + def test_resolve_serve_config_matches_desktop_env(self) -> None: + with tempfile.TemporaryDirectory() as td: + api_dir = Path(td) / "resources" / "api" + api_dir.mkdir(parents=True) + (api_dir / "main.py").write_text("# api") + (Path(td) / ("Modly.exe" if os.name == "nt" else "modly")).write_text("# exe") + python = Path(td) / "python.exe" + python.write_text("# python") + args = SimpleNamespace(api_dir=str(api_dir), python=str(python), host="127.0.0.1", port=8765, + models_dir=None, workspace_dir=None, extensions_dir=None, model=None, hf_token=None) + clean = {k: v for k, v in os.environ.items() if k not in ("MODLY_LLM_DIR", "MODLY_NODE_EXECUTABLE")} + with patch.dict(os.environ, clean, clear=True), \ + patch.object(agent, "_load_modly_settings", return_value={"agentDir": "D:/modly/agent"}): + _api_dir, _python, env, _cmd, _url = agent._resolve_serve_config(args) + self.assertEqual(env["PYTHONIOENCODING"], "utf-8") + self.assertEqual(env["MODLY_LLM_DIR"], "D:/modly/agent") + self.assertTrue(env["MODLY_NODE_EXECUTABLE"].lower().endswith(("modly.exe", "modly"))) + def test_resolve_serve_config_explicit_paths(self) -> None: with tempfile.TemporaryDirectory() as td: api_dir = Path(td) / "api" From 4bf4795319bf6bb045be18179219656ca8b75f92 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Sat, 31 Oct 2026 22:11:25 +0100 Subject: [PATCH 010/108] fix(registry): unload stale models when leaving a resident pair, gate pair on VRAM switch_model only unloaded the previous active model. After TripoSG -> Trellis2 refine -> any other model, TripoSG was no longer active but stayed loaded beside the new model until unload_all. switch_model now unloads every loaded model except the target and its resident partners. Keeping both models resident also assumed a card with room for both (~10-11 GB peak). Each pair now declares a minimum total VRAM; below it, or when no NVIDIA GPU answers, switching behaves as before. The pair is set to 19.5 GiB (24 GB cards), the configuration the speedup was measured on. Manifests can't size this: TripoSG declares no vram_gb. --- api/services/generator_registry.py | 33 +++++++++---- api/tests/test_generator_registry.py | 70 ++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 8 deletions(-) diff --git a/api/services/generator_registry.py b/api/services/generator_registry.py index 523512493..40735ce19 100644 --- a/api/services/generator_registry.py +++ b/api/services/generator_registry.py @@ -39,8 +39,11 @@ ) # Model pairs that stay loaded together when the active model switches between them (a mesh -> texture -# pipeline would otherwise unload and reload both models on every asset). -_KEEP_RESIDENT_PAIRS = [{"triposg/generate", "trellis2/refine"}] +# pipeline would otherwise unload and reload both models on every asset), with the minimum total VRAM +# (GiB, as detect_vram_gb() reports it) for keeping both resident. Manifests can't size this: TripoSG +# declares no `vram_gb`. TripoSG + Trellis2 refine peak around 10-11 GB, so only 24 GB cards +# (23.9 GiB reported) qualify; smaller cards keep the unload-on-switch behavior. +_KEEP_RESIDENT_PAIRS = [({"triposg/generate", "trellis2/refine"}, 19.5)] # ------------------------------------------------------------------ # # Global paths @@ -888,7 +891,7 @@ def get_manifest(self, model_id: str) -> dict: return self._manifests[model_id] def switch_model(self, model_id: str) -> None: - """Switches the active model. Unloads the previous one if different.""" + """Switches the active model. Unloads every other model unless it is a resident partner.""" with self._lifecycle_lock: self._assert_not_quarantined(model_id) if model_id not in self._generators: @@ -908,13 +911,27 @@ def switch_model(self, model_id: str) -> None: llama_pool.unload_all() if model_id != self._active_id: - # Keep the geometry model and the texture model resident together instead of unloading - # the previous one on every switch (see _KEEP_RESIDENT_PAIRS). - if (self._active_id in self._generators - and {self._active_id, model_id} not in _KEEP_RESIDENT_PAIRS): - self._generators[self._active_id].unload() + # Unload everything except the target and its resident partners, not just the + # previous active model: after A -> B (pair) -> C, A is no longer active but is + # still loaded, and would otherwise sit in VRAM beside C. + keep = {model_id} | self._resident_partners(model_id) + for other_id, gen in self._generators.items(): + if other_id not in keep and gen.is_loaded(): + gen.unload() self._active_id = model_id + def _resident_partners(self, model_id: str) -> Set[str]: + """Models allowed to stay loaded beside ``model_id`` (see _KEEP_RESIDENT_PAIRS).""" + from services.llm_server import detect_vram_gb + partners: Set[str] = set() + for pair, min_vram_gb in _KEEP_RESIDENT_PAIRS: + if model_id not in pair: + continue + # detect_vram_gb() is 0.0 when no NVIDIA GPU answers, which never passes. + if detect_vram_gb() >= min_vram_gb: + partners |= pair - {model_id} + return partners + # ------------------------------------------------------------------ # # Status # ------------------------------------------------------------------ # diff --git a/api/tests/test_generator_registry.py b/api/tests/test_generator_registry.py index b21253b7c..53c38cd7a 100644 --- a/api/tests/test_generator_registry.py +++ b/api/tests/test_generator_registry.py @@ -7,6 +7,7 @@ import threading import unittest from pathlib import Path +from unittest import mock import services.generator_registry as registry_module from services.extension_process import ExtensionProcess @@ -892,5 +893,74 @@ def read_status() -> None: self.assertFalse(results["model"]["loaded"]) +class _FakeGenerator: + def __init__(self) -> None: + self.loaded = True + + def is_loaded(self) -> bool: + return self.loaded + + def unload(self) -> None: + self.loaded = False + + +class KeepResidentPairTests(unittest.TestCase): + GEOMETRY = "triposg/generate" + TEXTURE = "trellis2/refine" + OTHER = "other/generate" + + def setUp(self) -> None: + self.registry = GeneratorRegistry() + self.gens = {model_id: _FakeGenerator() for model_id in (self.GEOMETRY, self.TEXTURE, self.OTHER)} + self.registry._generators = dict(self.gens) + self.registry._active_id = self.GEOMETRY + self.gens[self.TEXTURE].loaded = False + self.gens[self.OTHER].loaded = False + patches = [ + mock.patch("services.llm_server.llama_pool.unload_all"), + mock.patch("services.llm_server.detect_vram_gb", return_value=23.9), + ] + for patch in patches: + patch.start() + self.addCleanup(patch.stop) + + def _switch_and_load(self, model_id: str) -> None: + self.registry.switch_model(model_id) + self.gens[model_id].loaded = True + + def _loaded(self) -> set: + return {model_id for model_id, gen in self.gens.items() if gen.loaded} + + def test_pair_stays_loaded_when_switching_inside_it(self) -> None: + self._switch_and_load(self.TEXTURE) + self._switch_and_load(self.GEOMETRY) + + self.assertEqual(self._loaded(), {self.GEOMETRY, self.TEXTURE}) + + def test_leaving_the_pair_unloads_both_models(self) -> None: + self._switch_and_load(self.TEXTURE) + self._switch_and_load(self.OTHER) + + self.assertEqual(self._loaded(), {self.OTHER}) + + def test_pair_is_not_kept_when_vram_is_too_small(self) -> None: + with mock.patch("services.llm_server.detect_vram_gb", return_value=11.9): + self._switch_and_load(self.TEXTURE) + + self.assertEqual(self._loaded(), {self.TEXTURE}) + + def test_pair_is_not_kept_when_vram_is_unknown(self) -> None: + with mock.patch("services.llm_server.detect_vram_gb", return_value=0.0): + self._switch_and_load(self.TEXTURE) + + self.assertEqual(self._loaded(), {self.TEXTURE}) + + def test_pair_is_not_kept_on_a_16_gb_card(self) -> None: + with mock.patch("services.llm_server.detect_vram_gb", return_value=15.9): + self._switch_and_load(self.TEXTURE) + + self.assertEqual(self._loaded(), {self.TEXTURE}) + + if __name__ == "__main__": unittest.main() From 2fcc0fb058eb99838ed362e35c4c4ca5386727e5 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Sat, 31 Oct 2026 22:32:34 +0100 Subject: [PATCH 011/108] modly-cli: drop the caller's Python env and prefer the app venv, like the desktop bridge The started API inherited PYTHONHOME, PYTHONPATH, VIRTUAL_ENV, CONDA_PREFIX and the PIP_* variables of whatever shell ran the CLI. From a conda env or an activated venv, these point Modly's venv interpreter at foreign site-packages, or keep it from starting at all. They are now removed, the same list Electron's cleanPythonEnv() strips. _default_python also tried the api's dev .venv before the app's venv, the reverse of the PythonBridge. The app's venv (dependenciesDir, then the default dependencies folder) now comes first, the dev .venv second. --- tools/modly-cli/agent.py | 21 +++++++++++++++------ tools/modly-cli/test_agent.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/tools/modly-cli/agent.py b/tools/modly-cli/agent.py index 09f35755c..b382272be 100644 --- a/tools/modly-cli/agent.py +++ b/tools/modly-cli/agent.py @@ -284,10 +284,8 @@ def _default_api_dir() -> Path | None: def _default_python(api_dir: Path, settings: dict[str, Any] | None = None) -> Path | None: - candidates = [ - api_dir / ".venv" / "Scripts" / "python.exe", - api_dir / ".venv" / "bin" / "python", - ] + # Same order as the Electron PythonBridge: the app's venv first, the api's dev .venv as fallback. + candidates: list[Path] = [] # The desktop app can keep its dependencies on another drive (Settings > dependenciesDir). deps_dir = (settings or {}).get("dependenciesDir") if deps_dir: @@ -295,6 +293,8 @@ def _default_python(api_dir: Path, settings: dict[str, Any] | None = None) -> Pa candidates.append(Path(deps_dir) / "venv" / "bin" / "python") for appdata in _windows_env_paths("APPDATA"): candidates.append(appdata / "Modly" / "dependencies" / "venv" / "Scripts" / "python.exe") + candidates.append(api_dir / ".venv" / "Scripts" / "python.exe") + candidates.append(api_dir / ".venv" / "bin" / "python") candidates.append(Path(sys.executable)) for candidate in candidates: if candidate.exists(): @@ -317,6 +317,13 @@ def _load_modly_settings() -> dict[str, Any]: return {} +_FOREIGN_PYTHON_ENV = ( + "PYTHONHOME", "PYTHONPATH", "PYTHONSTARTUP", "PYTHONUSERBASE", + "PIP_USER", "PIP_TARGET", "PIP_PREFIX", "PIP_REQUIRE_VIRTUALENV", + "VIRTUAL_ENV", "CONDA_PREFIX", +) + + def _resolve_serve_config(args: argparse.Namespace) -> tuple[Path, Path, dict[str, str], list[str], str]: api_dir = Path(args.api_dir).expanduser().resolve() if getattr(args, "api_dir", None) else _default_api_dir() if not api_dir or not (api_dir / "main.py").exists(): @@ -327,8 +334,10 @@ def _resolve_serve_config(args: argparse.Namespace) -> tuple[Path, Path, dict[st if not python or not python.exists(): raise ModlyCliError("Could not find Modly Python environment; pass --python") - env = os.environ.copy() - hf_token = getattr(args, "hf_token", None) or settings.get("hfToken") or os.environ.get("HF_TOKEN", "") + # Like Electron's cleanPythonEnv(): variables from the caller's own Python setup (a conda env, + # an activated venv) would point Modly's venv interpreter at foreign site-packages. + env = {k: v for k, v in os.environ.items() if k not in _FOREIGN_PYTHON_ENV} + hf_token =getattr(args, "hf_token", None) or settings.get("hfToken") or os.environ.get("HF_TOKEN", "") env.update({ "PYTHONUNBUFFERED": "1", "MODELS_DIR": getattr(args, "models_dir", None) or settings.get("modelsDir") or str(Path.home() / ".modly" / "models"), diff --git a/tools/modly-cli/test_agent.py b/tools/modly-cli/test_agent.py index 432492bab..990bfbb0b 100644 --- a/tools/modly-cli/test_agent.py +++ b/tools/modly-cli/test_agent.py @@ -494,6 +494,39 @@ def test_default_python_uses_settings_dependencies_dir(self) -> None: found = agent._default_python(api_dir, {"dependenciesDir": str(Path(td) / "deps")}) self.assertEqual(found, deps_python) + def test_default_python_prefers_app_venv_over_api_dev_venv(self) -> None: + with tempfile.TemporaryDirectory() as td: + api_dir = Path(td) / "api" + dev_python = api_dir / ".venv" / "Scripts" / "python.exe" + dev_python.parent.mkdir(parents=True) + dev_python.write_text("# python") + deps_python = Path(td) / "deps" / "venv" / "Scripts" / "python.exe" + deps_python.parent.mkdir(parents=True) + deps_python.write_text("# python") + with patch.object(agent, "_windows_env_paths", return_value=[]): + found = agent._default_python(api_dir, {"dependenciesDir": str(Path(td) / "deps")}) + fallback = agent._default_python(api_dir, {}) + self.assertEqual(found, deps_python) + self.assertEqual(fallback, dev_python) + + def test_resolve_serve_config_drops_callers_python_env(self) -> None: + with tempfile.TemporaryDirectory() as td: + api_dir = Path(td) / "api" + api_dir.mkdir() + (api_dir / "main.py").write_text("# api") + python = Path(td) / "python.exe" + python.write_text("# python") + args = SimpleNamespace(api_dir=str(api_dir), python=str(python), host="127.0.0.1", port=8765, + models_dir=None, workspace_dir=None, extensions_dir=None, model=None, hf_token=None) + foreign = {"PYTHONHOME": "C:/conda", "PYTHONPATH": "C:/elsewhere", "VIRTUAL_ENV": "C:/venv", + "CONDA_PREFIX": "C:/conda", "PIP_USER": "1"} + with patch.dict(os.environ, foreign), \ + patch.object(agent, "_load_modly_settings", return_value={}): + _api_dir, _python, env, _cmd, _url = agent._resolve_serve_config(args) + for key in foreign: + self.assertNotIn(key, env) + self.assertIn("PATH", env) + def test_resolve_serve_config_matches_desktop_env(self) -> None: with tempfile.TemporaryDirectory() as td: api_dir = Path(td) / "resources" / "api" From 5ee7dea126d95172e211619dc95018088ad9e214 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Sun, 1 Nov 2026 10:41:26 +0100 Subject: [PATCH 012/108] fix(storage): stage moves, allow data-drive folders, surface guard errors - Copy into a unique sibling staging folder, then rename onto the verified-empty destination; a failed copy is discarded so the source stays intact and a retry does not hit "Destination must be empty". - Drop the post-copy re-validation: it re-ran a pure check on the same inputs. - Windows: accept a top-level folder on a data drive (D:\Models) and refuse the well-known system folders instead; POSIX keeps the two-level minimum. - Path API follows options.platform everywhere, including the symlink check. - Return the guard's message over IPC and show it in the move/delete modal. - Tests: resolve tmpdir() through realpath (macOS /var symlink), run the junction test on Windows, cover empty existing destinations and failed copies. --- electron/main/ipc-handlers.ts | 4 +- electron/main/storage-directory-guard.test.ts | 64 ++++++++++++++--- electron/main/storage-directory-guard.ts | 68 ++++++++++++------- .../settings/components/StorageSection.tsx | 20 +++++- 4 files changed, 119 insertions(+), 37 deletions(-) diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index f62439674..564d32dd8 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -1096,7 +1096,7 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe await moveStorageDirectory(src, dest, getSettings(userData), userData, { appDir: app.getAppPath() }) return { success: true } } catch (err) { - return { success: false, error: String(err) } + return { success: false, error: err instanceof Error ? err.message : String(err) } } }) @@ -1106,7 +1106,7 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe await deleteStorageDirectory(dirPath, getSettings(userData), userData, { appDir: app.getAppPath() }) return { success: true } } catch (err) { - return { success: false, error: String(err) } + return { success: false, error: err instanceof Error ? err.message : String(err) } } }) diff --git a/electron/main/storage-directory-guard.test.ts b/electron/main/storage-directory-guard.test.ts index d40aa3fa8..ea3abf21c 100644 --- a/electron/main/storage-directory-guard.test.ts +++ b/electron/main/storage-directory-guard.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { mkdtemp, mkdir, rm, symlink, writeFile, readFile, access } from 'node:fs/promises' +import { mkdtemp, mkdir, rm, symlink, writeFile, readFile, access, readdir, realpath, chmod } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { @@ -24,6 +24,12 @@ const home = 'C:\\Users\\Tester' const userData = 'C:\\Users\\Tester\\AppData\\Roaming\\modly' const opts = { platform: 'win32' as const, homeDir: home, appDir: 'C:\\Program Files\\Modly\\resources\\app.asar' } +// realpath: on macOS tmpdir() lives under /var, itself a symlink to /private/var, +// which the symlink-ancestor guard would (rightly) refuse. +async function tempBase(prefix: string): Promise { + return mkdtemp(join(await realpath(tmpdir()), prefix)) +} + test('deletion permits an exact configured managed directory', () => { assert.equal(validateStorageDeletion('e:\\MODLY\\models\\\\', dirs, userData, opts), dirs.modelsDir) assert.equal(validateStorageDeletion(dirs.workflowsDir, dirs, userData, opts), dirs.workflowsDir) @@ -31,7 +37,7 @@ test('deletion permits an exact configured managed directory', () => { }) test('deletion rejects filesystem roots, shallow folders and relative paths', () => { - for (const bad of ['E:\\\\', 'C:\\\\', 'E:\\Photos', 'models', '.', '']) { + for (const bad of ['E:\\\\', 'C:\\\\', 'C:\\Windows', 'D:\\program files', 'models', '.', '']) { assert.throws(() => validateStorageDeletion(bad, { ...dirs, modelsDir: bad }, userData, opts)) } assert.throws(() => validateStorageDeletion('\\\\server\\share\\models', dirs, userData, opts)) @@ -65,6 +71,12 @@ test('move allows separate configured source and empty destination path', () => }) }) +test('a top-level folder on a Windows data drive is a valid destination', () => { + assert.deepEqual(validateStorageMove(dirs.modelsDir, 'D:\\Models', dirs, userData, opts), { + src: dirs.modelsDir, dest: 'D:\\Models', + }) +}) + test('move rejects rogue sources, root, parent, child and overlapping destinations', () => { const badSources = ['E:\\\\', home, 'E:\\modly\\models-other', 'E:\\modly\\models\\child'] for (const bad of badSources) { @@ -72,7 +84,7 @@ test('move rejects rogue sources, root, parent, child and overlapping destinatio } for (const bad of [ dirs.modelsDir, 'E:\\modly', 'E:\\modly\\models\\new', 'E:\\modly\\workflows', - 'E:\\\\', 'D:\\Backups', 'E:\\modly\\workspace\\inside', + 'E:\\\\', 'D:\\Users', 'E:\\modly\\workspace\\inside', ]) { assert.throws(() => validateStorageMove(dirs.modelsDir, bad, dirs, userData, opts)) } @@ -90,11 +102,12 @@ test('POSIX root and relative path are rejected', () => { const linuxOpts = { platform: 'linux' as const, homeDir: '/home/tester' } assert.equal(validateStorageDeletion(linuxDirs.modelsDir, linuxDirs, '/home/tester/.config/modly', linuxOpts), linuxDirs.modelsDir) assert.throws(() => validateStorageDeletion('/', { ...linuxDirs, modelsDir: '/' }, '/home/tester/.config/modly', linuxOpts)) + assert.throws(() => validateStorageMove(linuxDirs.modelsDir, '/mnt', linuxDirs, '/home/tester/.config/modly', linuxOpts)) assert.throws(() => validateStorageMove('relative', '/tmp/safe/new', linuxDirs, '/home/tester/.config/modly', linuxOpts)) }) test('destination must be empty, missing is fine', async () => { - const base = await mkdtemp(join(tmpdir(), 'modly-storage-')) + const base = await tempBase('modly-storage-') try { await assertEmptyDestination(join(base, 'new')) await mkdir(join(base, 'empty')) @@ -106,11 +119,11 @@ test('destination must be empty, missing is fine', async () => { } }) -test('symlinked ancestors and target are refused', { skip: process.platform === 'win32' }, async () => { - const base = await mkdtemp(join(tmpdir(), 'modly-storage-link-')) +test('symlinked ancestors, Windows junctions and target are refused', async () => { + const base = await tempBase('modly-storage-link-') try { await mkdir(join(base, 'real', 'models'), { recursive: true }) - await symlink(join(base, 'real'), join(base, 'alias'), 'dir') + await symlink(join(base, 'real'), join(base, 'alias'), process.platform === 'win32' ? 'junction' : 'dir') await assertNoSymlinkAncestors(join(base, 'real', 'models')) await assert.rejects(() => assertNoSymlinkAncestors(join(base, 'alias', 'models')), /symbolic link/i) await assert.rejects(() => assertNoSymlinkAncestors(join(base, 'alias')), /symbolic link/i) @@ -120,7 +133,7 @@ test('symlinked ancestors and target are refused', { skip: process.platform === }) test('unsafe real deletion refuses a storage root and preserves personal files', async () => { - const base = await mkdtemp(join(tmpdir(), 'modly-real-delete-')) + const base = await tempBase('modly-real-delete-') const managed = join(base, 'models') const personal = join(base, 'personal.txt') const unixDirs = { @@ -144,7 +157,7 @@ test('unsafe real deletion refuses a storage root and preserves personal files', }) test('real move does not overwrite destination files or erase unrelated data', async () => { - const base = await mkdtemp(join(tmpdir(), 'modly-real-move-')) + const base = await tempBase('modly-real-move-') const source = join(base, 'models') const destination = join(base, 'migrated', 'models') const unixDirs = { @@ -162,11 +175,42 @@ test('real move does not overwrite destination files or erase unrelated data', a join(base, 'userData'), options), /empty/i) assert.equal(await readFile(join(destination, 'personal.txt'), 'utf8'), 'preserve') assert.equal(await readFile(join(source, 'asset.glb'), 'utf8'), 'model') - await rm(destination, { recursive: true }) + // The folder picker usually hands back an existing, empty folder. + await rm(join(destination, 'personal.txt')) await moveStorageDirectory(source, destination, unixDirs, join(base, 'userData'), options) assert.equal(await readFile(join(destination, 'asset.glb'), 'utf8'), 'model') await assert.rejects(() => access(source)) + assert.deepEqual(await readdir(join(base, 'migrated')), ['models']) + } finally { + await rm(base, { recursive: true, force: true }) + } +}) + +test('a failed copy keeps the source and leaves nothing behind at the destination', { + // chmod cannot make a file unreadable on Windows, nor for root. + skip: process.platform === 'win32' || process.getuid?.() === 0, +}, async () => { + const base = await tempBase('modly-failed-move-') + const source = join(base, 'models') + const destination = join(base, 'migrated', 'models') + const unixDirs = { + modelsDir: source, workspaceDir: join(base, 'workspace'), + workflowsDir: join(base, 'workflows'), extensionsDir: join(base, 'extensions'), + dependenciesDir: join(base, 'dependencies'), agentDir: join(base, 'agent'), + } + const options = { platform: process.platform, homeDir: join(base, 'other-home') } + try { + await mkdir(source) + await writeFile(join(source, 'asset.glb'), 'model') + await writeFile(join(source, 'locked.bin'), 'unreadable') + await chmod(join(source, 'locked.bin'), 0o000) + await assert.rejects(() => moveStorageDirectory(source, destination, unixDirs, + join(base, 'userData'), options)) + assert.equal(await readFile(join(source, 'asset.glb'), 'utf8'), 'model') + // No half-copied destination and no staging leftover: a retry starts clean. + assert.deepEqual(await readdir(join(base, 'migrated')), []) } finally { + await chmod(join(source, 'locked.bin'), 0o644).catch(() => {}) await rm(base, { recursive: true, force: true }) } }) diff --git a/electron/main/storage-directory-guard.ts b/electron/main/storage-directory-guard.ts index f5a63a8c8..bff010255 100644 --- a/electron/main/storage-directory-guard.ts +++ b/electron/main/storage-directory-guard.ts @@ -1,8 +1,15 @@ /** Fail-closed validation for settings-driven directory moves and deletions. */ -import { cp, lstat, mkdir, readdir, rm, rmdir } from 'node:fs/promises' +import { randomUUID } from 'node:crypto' +import { cp, lstat, mkdir, readdir, rename, rm, rmdir } from 'node:fs/promises' import { homedir } from 'node:os' import path from 'node:path' +/** Top-level Windows folders owned by the OS, refused on any drive. */ +const WINDOWS_SYSTEM_DIRS = new Set([ + 'windows', 'program files', 'program files (x86)', 'programdata', 'users', + 'perflogs', 'recovery', '$recycle.bin', 'system volume information', +]) + type StorageDirs = { modelsDir: string workspaceDir: string @@ -38,10 +45,15 @@ function absoluteDirectory(value: string, api: PathApi): string { if (resolved === api.parse(resolved).root) { throw new Error('Refusing a filesystem or volume root') } - // Never recursively delete a bare top-level user folder, even if configured. const pieces = resolved.slice(api.parse(resolved).root.length).split(api.sep).filter(Boolean) - if (pieces.length < 2) { - throw new Error('Storage directory must be at least two levels below its volume root') + // POSIX top-level folders (/home, /usr, /mnt, /Volumes…) all belong to the system. + if (api === path.posix && pieces.length < 2) { + throw new Error('Storage directory must be at least two levels below the filesystem root') + } + // On Windows a top-level folder on a data drive (D:\Models) is a normal choice; + // only the well-known system folders are refused. + if (api === path.win32 && pieces.length === 1 && WINDOWS_SYSTEM_DIRS.has(pieces[0].toLowerCase())) { + throw new Error('Refusing a Windows system folder') } return resolved } @@ -146,8 +158,8 @@ export function validateStorageMove( } /** Reject symlinks or Windows junctions in existing path ancestors. */ -export async function assertNoSymlinkAncestors(directory: string): Promise { - const api = process.platform === 'win32' ? path.win32 : path.posix +export async function assertNoSymlinkAncestors(directory: string, options: Options = {}): Promise { + const api = pathApi(options) const root = api.parse(directory).root let current = root for (const part of directory.slice(root.length).split(api.sep).filter(Boolean)) { @@ -178,30 +190,40 @@ export async function deleteStorageDirectory( directory: string, dirs: StorageDirs, userData: string, options: Options = {}, ): Promise { const safe = validateStorageDeletion(directory, dirs, userData, options) - await assertNoSymlinkAncestors(safe) + await assertNoSymlinkAncestors(safe, options) await rm(safe, { recursive: true, force: true }) } export async function moveStorageDirectory( source: string, destination: string, dirs: StorageDirs, userData: string, options: Options = {}, ): Promise { - const paths = validateStorageMove(source, destination, dirs, userData, options) - await assertNoSymlinkAncestors(paths.src) - await assertNoSymlinkAncestors(paths.dest) - await assertEmptyDestination(paths.dest) - // fs.cp with errorOnExist refuses even an EMPTY existing directory. Remove - // only the verified-empty destination, never the source, before copying. - // rmdir itself fails closed if a concurrent writer adds a file. + const api = pathApi(options) + const { src, dest } = validateStorageMove(source, destination, dirs, userData, options) + await assertNoSymlinkAncestors(src, options) + await assertNoSymlinkAncestors(dest, options) + await assertEmptyDestination(dest) + + const parent = api.dirname(dest) + await mkdir(parent, { recursive: true }) + // Copy into a fresh sibling folder that only this call writes to: a failed + // copy can then be discarded without touching the source or the destination, + // and a retry does not trip over a half-filled destination. + const staging = api.join(parent, '.' + api.basename(dest) + '.modly-move-' + randomUUID()) try { - await rmdir(paths.dest) + await cp(src, staging, { recursive: true, force: false, errorOnExist: true }) + // Remove the verified-empty destination; rmdir fails closed if a file appeared. + try { + await rmdir(dest) + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err + } + await rename(staging, dest) } catch (err) { - if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err + await rm(staging, { recursive: true, force: true }) + throw err } - await mkdir(path.dirname(paths.dest), { recursive: true }) - // With a missing destination, errorOnExist protects against unexpected files. - await cp(paths.src, paths.dest, { recursive: true, force: false, errorOnExist: true }) - // Re-validate before deleting the source. Never delete it after a copy error. - validateStorageMove(paths.src, paths.dest, dirs, userData, options) - await assertNoSymlinkAncestors(paths.src) - await rm(paths.src, { recursive: true, force: true }) + + // The source is deleted only once its complete copy sits at the destination. + await assertNoSymlinkAncestors(src, options) + await rm(src, { recursive: true, force: true }) } diff --git a/src/areas/settings/components/StorageSection.tsx b/src/areas/settings/components/StorageSection.tsx index 7c949adc8..c15a2d870 100644 --- a/src/areas/settings/components/StorageSection.tsx +++ b/src/areas/settings/components/StorageSection.tsx @@ -3,7 +3,7 @@ import { Section, Card, Row, PathRow } from '@shared/ui' // ─── MoveFolderModal ────────────────────────────────────────────────────────── -function MoveFolderModal({ title, currentDir, items, itemLabel, moveLabel, moveDesc, deleteLabel, deleteDesc, status, onCancel, onMove, onDelete }: { +function MoveFolderModal({ title, currentDir, items, itemLabel, moveLabel, moveDesc, deleteLabel, deleteDesc, status, error, onCancel, onMove, onDelete }: { title: string currentDir: string items: string[] @@ -13,6 +13,7 @@ function MoveFolderModal({ title, currentDir, items, itemLabel, moveLabel, moveD deleteLabel: string deleteDesc: string status: 'idle' | 'busy' | 'error' + error?: string | null onCancel: () => void onMove: () => void onDelete: () => void @@ -34,7 +35,7 @@ function MoveFolderModal({ title, currentDir, items, itemLabel, moveLabel, moveD
{status === 'error' && ( -

Something went wrong. Please try again.

+

{error || 'Something went wrong. Please try again.'}

)}
@@ -77,6 +84,9 @@ export function StorageSection(): JSX.Element { const [workspaceDir, setWorkspaceDir] = useState('') const [workflowsDir, setWorkflowsDir] = useState('') const [cacheStatus, setCacheStatus] = useState<'idle' | 'clearing' | 'done' | 'error'>('idle') + const [storageError, setStorageError] = useState(null) + // Items of the current folder Modly did not create (one modal at a time). + const [otherItems, setOtherItems] = useState(0) const [pendingModelsDir, setPendingModelsDir] = useState(null) const [existingModels, setExistingModels] = useState([]) @@ -102,73 +112,98 @@ export function StorageSection(): JSX.Element { }, []) async function applyModelsDir(path: string) { - setModelsDir(path) await window.electron.settings.set({ modelsDir: path }) await window.electron.api.updatePaths({ modelsDir: path }) + setModelsDir(path) } async function applyWorkspaceDir(path: string) { - setWorkspaceDir(path) await window.electron.settings.set({ workspaceDir: path }) await window.electron.api.updatePaths({ workspaceDir: path }) + setWorkspaceDir(path) } async function applyWorkflowsDir(path: string) { - setWorkflowsDir(path) await window.electron.settings.set({ workflowsDir: path }) + setWorkflowsDir(path) + } + + /** Refuses a drive root or system folder before anything is changed. */ + async function checkLocation(dir: string): Promise { + const result = await window.electron.fs.checkStorageLocation(dir) + setStorageError(result.ok ? null : `${dir}: ${result.error ?? 'not a valid storage folder'}`) + return result.ok + } + + async function applyOrReport(apply: () => Promise) { + try { + await apply() + } catch (err) { + setStorageError(err instanceof Error ? err.message : String(err)) + } } async function handleBrowseModels() { const newPath = await window.electron.fs.selectDirectory(modelsDir) if (!newPath || newPath === modelsDir) return + if (!(await checkLocation(newPath))) return - const models = await window.electron.fs.listDir(modelsDir) - if (models.length > 0) { - setExistingModels(models) + // Only what Modly created there is offered for moving or deleting. + const { modly, other } = await window.electron.fs.storageEntries('models') + if (modly.length > 0) { + setExistingModels(modly) + setOtherItems(other.length) setPendingModelsDir(newPath) return } - await applyModelsDir(newPath) + await applyOrReport(() => applyModelsDir(newPath)) } async function handleBrowseWorkspace() { const newPath = await window.electron.fs.selectDirectory(workspaceDir) if (!newPath || newPath === workspaceDir) return + if (!(await checkLocation(newPath))) return - const items = await window.electron.fs.listDir(workspaceDir) - if (items.length > 0) { - setExistingWorkspaces(items) + // Only what Modly created there is offered for moving or deleting. + const { modly, other } = await window.electron.fs.storageEntries('workspace') + if (modly.length > 0) { + setExistingWorkspaces(modly) + setOtherItems(other.length) setPendingWorkspaceDir(newPath) return } - await applyWorkspaceDir(newPath) + await applyOrReport(() => applyWorkspaceDir(newPath)) } async function handleBrowseWorkflows() { const newPath = await window.electron.fs.selectDirectory(workflowsDir) if (!newPath || newPath === workflowsDir) return + if (!(await checkLocation(newPath))) return - const items = await window.electron.fs.listDir(workflowsDir) - if (items.length > 0) { - setExistingWorkflows(items) + // Only what Modly created there is offered for moving or deleting. + const { modly, other } = await window.electron.fs.storageEntries('workflows') + if (modly.length > 0) { + setExistingWorkflows(modly) + setOtherItems(other.length) setPendingWorkflowsDir(newPath) return } - await applyWorkflowsDir(newPath) + await applyOrReport(() => applyWorkflowsDir(newPath)) } async function handleMoveModels() { if (!pendingModelsDir) return setModelsActionStatus('busy') const result = await window.electron.fs.moveDirectory({ src: modelsDir, dest: pendingModelsDir }) - if (result.success) { + try { + if (!result.success) throw new Error(result.error ?? 'Something went wrong. Please try again.') await applyModelsDir(pendingModelsDir) closeModelsModal() - } else { - setModelsActionError(result.error ?? null) + } catch (err) { + setModelsActionError(err instanceof Error ? err.message : String(err)) setModelsActionStatus('error') } } @@ -177,11 +212,12 @@ export function StorageSection(): JSX.Element { if (!pendingModelsDir) return setModelsActionStatus('busy') const result = await window.electron.fs.deleteDirectory(modelsDir) - if (result.success) { + try { + if (!result.success) throw new Error(result.error ?? 'Something went wrong. Please try again.') await applyModelsDir(pendingModelsDir) closeModelsModal() - } else { - setModelsActionError(result.error ?? null) + } catch (err) { + setModelsActionError(err instanceof Error ? err.message : String(err)) setModelsActionStatus('error') } } @@ -197,11 +233,12 @@ export function StorageSection(): JSX.Element { if (!pendingWorkspaceDir) return setWorkspaceActionStatus('busy') const result = await window.electron.fs.moveDirectory({ src: workspaceDir, dest: pendingWorkspaceDir }) - if (result.success) { + try { + if (!result.success) throw new Error(result.error ?? 'Something went wrong. Please try again.') await applyWorkspaceDir(pendingWorkspaceDir) closeWorkspaceModal() - } else { - setWorkspaceActionError(result.error ?? null) + } catch (err) { + setWorkspaceActionError(err instanceof Error ? err.message : String(err)) setWorkspaceActionStatus('error') } } @@ -210,11 +247,12 @@ export function StorageSection(): JSX.Element { if (!pendingWorkspaceDir) return setWorkspaceActionStatus('busy') const result = await window.electron.fs.deleteDirectory(workspaceDir) - if (result.success) { + try { + if (!result.success) throw new Error(result.error ?? 'Something went wrong. Please try again.') await applyWorkspaceDir(pendingWorkspaceDir) closeWorkspaceModal() - } else { - setWorkspaceActionError(result.error ?? null) + } catch (err) { + setWorkspaceActionError(err instanceof Error ? err.message : String(err)) setWorkspaceActionStatus('error') } } @@ -230,11 +268,12 @@ export function StorageSection(): JSX.Element { if (!pendingWorkflowsDir) return setWorkflowsActionStatus('busy') const result = await window.electron.fs.moveDirectory({ src: workflowsDir, dest: pendingWorkflowsDir }) - if (result.success) { + try { + if (!result.success) throw new Error(result.error ?? 'Something went wrong. Please try again.') await applyWorkflowsDir(pendingWorkflowsDir) closeWorkflowsModal() - } else { - setWorkflowsActionError(result.error ?? null) + } catch (err) { + setWorkflowsActionError(err instanceof Error ? err.message : String(err)) setWorkflowsActionStatus('error') } } @@ -243,11 +282,12 @@ export function StorageSection(): JSX.Element { if (!pendingWorkflowsDir) return setWorkflowsActionStatus('busy') const result = await window.electron.fs.deleteDirectory(workflowsDir) - if (result.success) { + try { + if (!result.success) throw new Error(result.error ?? 'Something went wrong. Please try again.') await applyWorkflowsDir(pendingWorkflowsDir) closeWorkflowsModal() - } else { - setWorkflowsActionError(result.error ?? null) + } catch (err) { + setWorkflowsActionError(err instanceof Error ? err.message : String(err)) setWorkflowsActionStatus('error') } } @@ -290,6 +330,7 @@ export function StorageSection(): JSX.Element { value={workflowsDir} onBrowse={handleBrowseWorkflows} /> + {storageError &&

{storageError}

} @@ -318,6 +359,7 @@ export function StorageSection(): JSX.Element { title="Change models folder" currentDir={modelsDir} items={existingModels} + otherCount={otherItems} itemLabel="model" moveLabel="Move to new folder" moveDesc="Transfer all models to the new location." @@ -336,11 +378,12 @@ export function StorageSection(): JSX.Element { title="Change workspace folder" currentDir={workspaceDir} items={existingWorkspaces} + otherCount={otherItems} itemLabel="item" moveLabel="Move to new folder" - moveDesc="Transfer all workspace files to the new location." + moveDesc="Transfer Modly's workspace files to the new location." deleteLabel="Delete workspace files" - deleteDesc="Remove all files from the current folder." + deleteDesc="Remove Modly's files from the current folder." status={workspaceActionStatus} error={workspaceActionError} onCancel={closeWorkspaceModal} @@ -354,6 +397,7 @@ export function StorageSection(): JSX.Element { title="Change workflows folder" currentDir={workflowsDir} items={existingWorkflows} + otherCount={otherItems} itemLabel="workflow" moveLabel="Move to new folder" moveDesc="Transfer all workflow files to the new location." diff --git a/src/shared/types/electron.d.ts b/src/shared/types/electron.d.ts index c171b6648..2be5a5dc1 100644 --- a/src/shared/types/electron.d.ts +++ b/src/shared/types/electron.d.ts @@ -208,8 +208,12 @@ declare global { listDir: (dirPath: string) => Promise listFiles: (dirPath: string, extensions?: string[]) => Promise selectTextFile: () => Promise - moveDirectory: (args: { src: string; dest: string }) => Promise<{ success: boolean; error?: string }> - deleteDirectory: (dirPath: string) => Promise<{ success: boolean; error?: string }> + moveDirectory: (args: { src: string; dest: string }) => Promise<{ success: boolean; error?: string; kept?: string[] }> + deleteDirectory: (dirPath: string) => Promise<{ success: boolean; error?: string; kept?: string[] }> + /** Entries of a storage folder Modly may move/delete, and the ones it leaves alone. */ + storageEntries: (kind: 'models' | 'workspace' | 'workflows') => Promise<{ modly: string[]; other: string[] }> + /** Refuses a drive root, system folder or an ancestor of home/app data as storage. */ + checkStorageLocation: (dirPath: string) => Promise<{ ok: boolean; error?: string }> readScreenshotDataUrl: (filename: string) => Promise } settings: { From dde2b8025784ce59198340e1edacf04317118c84 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Sun, 8 Nov 2026 21:12:20 +0100 Subject: [PATCH 019/108] fix(extensions): say why an extension subprocess died "Subprocess died unexpectedly" was all users got when a model crashed while loading or generating (#369). The cause (CUDA out of memory, a missing DLL, the OS killing the process for lack of RAM) was only on the child's stderr, which went to the logs. ExtensionProcess now keeps the last stderr lines of each subprocess and, on death, waits for its exit code and reports both: the phase (starting, loading, generation), the exit code (NTSTATUS in hex, or the signal), a likely cause for the common failures, and the last relevant lines with progress bars removed. A process stopped by Modly itself (Free Memory) is reported as such, not as a crash. --- api/services/extension_process.py | 65 +++++++++++-- api/services/subprocess_death.py | 126 +++++++++++++++++++++++++ api/tests/test_subprocess_death.py | 142 +++++++++++++++++++++++++++++ 3 files changed, 324 insertions(+), 9 deletions(-) create mode 100644 api/services/subprocess_death.py create mode 100644 api/tests/test_subprocess_death.py diff --git a/api/services/extension_process.py b/api/services/extension_process.py index 5eaed4b87..e6be67bd8 100644 --- a/api/services/extension_process.py +++ b/api/services/extension_process.py @@ -17,11 +17,16 @@ import sys import threading import uuid +from collections import deque from pathlib import Path from typing import Callable, Optional +from services.subprocess_death import describe_death + _RUNNER_PATH = Path(__file__).parent.parent / "runner.py" _MISSING_MODULE_RE = re.compile(r"No module named ['\"]([^'\"]+)['\"]") +# Raw stderr lines kept per subprocess, to explain a crash (#369). +_STDERR_TAIL = 400 _AUTO_REPAIR_PACKAGE_MAP = { "PIL": "Pillow", } @@ -51,6 +56,9 @@ def __init__(self, ext_dir: Path, manifest: dict) -> None: self._queue: queue.Queue = queue.Queue() self._lock: threading.Lock = threading.Lock() self._loaded: bool = False + self._stderr_tail: deque = deque(maxlen=_STDERR_TAIL) + self._stderr_thread: Optional[threading.Thread] = None + self._stopped_by_modly: bool = False # Mirrors BaseGenerator attributes used by the registry self.hf_repo = manifest.get("hf_repo", "") @@ -123,6 +131,7 @@ def _start(self) -> None: # older reader thread cannot poison startup for the new process. run_queue: queue.Queue = queue.Queue() self._queue = run_queue + self._stopped_by_modly = False self._proc = subprocess.Popen( [str(python), str(_RUNNER_PATH)], stdin=subprocess.PIPE, @@ -146,11 +155,16 @@ def _start(self) -> None: reader.start() # Background thread: forward stderr to our stderr - stderr_fwd = threading.Thread(target=self._stderr_loop, args=(self._proc,), daemon=True) + # (and keep its last lines to explain a crash). + self._stderr_tail = deque(maxlen=_STDERR_TAIL) + stderr_fwd = threading.Thread( + target=self._stderr_loop, args=(self._proc, self._stderr_tail), daemon=True, + ) + self._stderr_thread = stderr_fwd stderr_fwd.start() # Wait for ready — runner sends params_schema in this message - msg = self._recv(timeout=None) + msg = self._recv(timeout=None, phase="while starting") if msg.get("type") == "ready": # Override params_schema with what the generator class actually declares if msg.get("params_schema"): @@ -223,7 +237,7 @@ def _read_loop(self, proc: subprocess.Popen, msg_queue: queue.Queue) -> None: finally: msg_queue.put(None) # sentinel: process is done - def _stderr_loop(self, proc: subprocess.Popen) -> None: + def _stderr_loop(self, proc: subprocess.Popen, tail: Optional[deque] = None) -> None: """Forward subprocess stderr to the main process stderr, emitting one line every time we see EITHER '\\n' or '\\r'. tqdm writes live progress updates with '\\r' only, so a newline-only iterator would @@ -233,20 +247,29 @@ def _stderr_loop(self, proc: subprocess.Popen) -> None: No per-line extension-id prefix: the HUD log pane is a single truncated line, and eating 20 characters with "[modly-hy3d2-mac] " hides the tail of the tqdm bar the user actually wants to read. + + Each line also goes into `tail`, so a crash can quote it. """ stream = proc.stderr if stream is None: return buf = [] + + def emit() -> None: + line = ''.join(buf) + print(line, file=sys.stderr, flush=True) + if tail is not None: + tail.append(line) + while True: ch = stream.read(1) if not ch: if buf: - print(''.join(buf), file=sys.stderr, flush=True) + emit() return if ch in ("\r", "\n"): if buf: - print(''.join(buf), file=sys.stderr, flush=True) + emit() buf = [] else: buf.append(ch) @@ -256,15 +279,37 @@ def _send(self, msg: dict) -> None: self._proc.stdin.write(json.dumps(msg) + "\n") self._proc.stdin.flush() - def _recv(self, timeout: float | None = 120.0) -> dict: + def _recv(self, timeout: float | None = 120.0, phase: str = "unexpectedly") -> dict: try: msg = self._queue.get(timeout=timeout) except queue.Empty: raise TimeoutError(f"[{self.MODEL_ID}] No response from subprocess after {timeout}s") if msg is None: - raise RuntimeError(f"[{self.MODEL_ID}] Subprocess died unexpectedly") + raise self._death_error(phase) return msg + def _death_error(self, phase: str) -> RuntimeError: + """Why the subprocess died, with its exit code and last stderr lines. + + stdout closing is how we notice the death; the process may need a moment + to finish exiting, and the stderr thread to drain what is left. + """ + self._loaded = False + if self._stopped_by_modly: + # stop() killed it (Free Memory / unload): not a crash, and its + # SIGKILL must not be reported as the OOM killer. + return RuntimeError(f"[{self.MODEL_ID}] Subprocess was stopped by Modly {phase}") + returncode = None + proc = self._proc + if proc is not None: + try: + returncode = proc.wait(timeout=5.0) + except Exception: + returncode = proc.poll() + if self._stderr_thread is not None: + self._stderr_thread.join(timeout=2.0) + return RuntimeError(describe_death(self.MODEL_ID, phase, returncode, list(self._stderr_tail))) + def _ensure_started(self) -> None: if self._proc is None or self._proc.poll() is not None: self._start() @@ -285,7 +330,8 @@ def load(self) -> None: self._ensure_started() self._send({"action": "load"}) - msg = self._recv(timeout=None) # model load can be arbitrarily slow + # Model load can be arbitrarily slow. + msg = self._recv(timeout=None, phase="while loading the model") if msg.get('type') in ['loaded', 'ready']: self._loaded = True elif msg.get("type") == "error": @@ -370,7 +416,7 @@ def _receive_generation( continue if msg is None: - raise RuntimeError(f"[{self.MODEL_ID}] Subprocess died during generation") + raise self._death_error("during generation") t = msg.get("type") @@ -447,6 +493,7 @@ def stop(self) -> None: proc = self._proc self._loaded = False if proc and proc.poll() is None: + self._stopped_by_modly = True try: proc.kill() proc.wait(timeout=5) diff --git a/api/services/subprocess_death.py b/api/services/subprocess_death.py new file mode 100644 index 000000000..92bc65ba1 --- /dev/null +++ b/api/services/subprocess_death.py @@ -0,0 +1,126 @@ +""" +Explains why an extension subprocess died (#369). + +"Subprocess died unexpectedly" told users nothing: the real cause (CUDA out of +memory, a missing DLL, the OS killing the process) was only on the child's +stderr, which went to the logs and never into the error. This turns the exit +code and the last stderr lines into a message a user can act on. +""" +import re +from typing import Iterable, Optional + +# Lines kept in the message, and the longest line we quote. +TAIL_LINES = 20 +MAX_LINE = 300 + +# tqdm progress bars: "Loading: 45%|████▌ | 9/20 [00:03<00:04, 2.51it/s]" +_PROGRESS_RE = re.compile(r"\d+%\|.*\||\|\s*\d+/\d+\s*\[") + +# Checked in order: the first matching pattern wins. +_STDERR_CAUSES: list[tuple[re.Pattern, str]] = [ + (re.compile(r"CUDA out of memory|OutOfMemoryError|CUBLAS_STATUS_ALLOC_FAILED|MPS backend out of memory", re.I), + "the GPU ran out of memory (VRAM). Close other GPU-heavy apps, lower the " + "resolution or quality settings, or pick a lighter model."), + (re.compile(r"paging file is too small|os error 1455|error code 1455", re.I), + "Windows ran out of virtual memory while loading the model. Increase the " + "paging file size (System > Advanced system settings > Performance > " + "Virtual memory) or free up RAM."), + (re.compile(r"DefaultCPUAllocator: (?:can't allocate|not enough) memory|\bMemoryError\b", re.I), + "the computer ran out of RAM. Close other applications or pick a lighter model."), + (re.compile(r"no kernel image is available|not compatible with the current PyTorch installation", re.I), + "the PyTorch build installed for this extension does not support your GPU. " + "Try repairing the extension, or check that the extension supports your GPU."), + (re.compile(r"Found no NVIDIA driver|CUDA driver version is insufficient|NVIDIA driver on your system is too old", re.I), + "the NVIDIA driver is missing or too old. Update your GPU driver."), + (re.compile(r"DLL load failed", re.I), + "a system library (DLL) could not be loaded. Installing the latest " + "Microsoft Visual C++ Redistributable and repairing the extension usually fixes it."), + (re.compile(r"No module named ['\"]([^'\"]+)['\"]"), + "a Python package is missing from the extension ({0}). Repair the extension " + "from the Models page to reinstall its dependencies."), + (re.compile(r"Unable to load weights from checkpoint|HeaderTooLarge|Error while deserializing header|incomplete metadata", re.I), + "the model files look corrupted or incomplete (often an interrupted download). " + "Delete the model and download it again."), +] + +# Windows NTSTATUS codes, as Python reports them (unsigned). +_WINDOWS_CODES: dict[int, str] = { + 0xC0000005: ("access violation: the extension crashed in native code " + "(GPU driver, CUDA or a compiled library)"), + 0xC0000409: ("the extension aborted in native code (often a GPU driver or " + "CUDA library problem)"), + 0xC0000135: "a required DLL was not found", + 0xC000001D: "illegal instruction: a library was built for a newer CPU than this one", + 0xC00000FD: "stack overflow in native code", + 0xC000012D: "Windows ran out of virtual memory; increase the paging file size or free up RAM", + 0xC0000017: "Windows ran out of memory", +} + +# POSIX: Popen reports death by signal N as -N. +_SIGNALS: dict[int, str] = { + 9: ("the process was killed (SIGKILL). On Linux this is usually the " + "out-of-memory killer: the computer ran out of RAM"), + 11: "segmentation fault: the extension crashed in native code (GPU driver, CUDA or a compiled library)", + 6: "the extension aborted (SIGABRT), usually from a native library", + 4: "illegal instruction: a library was built for a newer CPU than this one", + 7: "bus error in native code", +} + + +def describe_exit_code(returncode: Optional[int]) -> Optional[str]: + """The exit code as people search for it: a signal, an NTSTATUS in hex, or + the plain number. likely_cause() explains it.""" + if returncode is None: + return None + unsigned = returncode & 0xFFFFFFFF + if unsigned in _WINDOWS_CODES: + return f"0x{unsigned:08X}" + if returncode < 0: + return f"signal {-returncode}" + return str(returncode) + + +def relevant_lines(lines: Iterable[str]) -> list[str]: + """The last stderr lines worth showing: no progress bars, no blanks.""" + kept = [ + line.rstrip() for line in lines + if line.strip() and not _PROGRESS_RE.search(line) + ] + return [ + line if len(line) <= MAX_LINE else line[:MAX_LINE] + "…" + for line in kept[-TAIL_LINES:] + ] + + +def likely_cause(lines: list[str], returncode: Optional[int]) -> Optional[str]: + """The most probable cause, from stderr first, then from the exit code.""" + text = "\n".join(lines) + for pattern, cause in _STDERR_CAUSES: + match = pattern.search(text) + if match: + return cause.format(*match.groups()) + if returncode is not None: + if returncode < 0 and -returncode in _SIGNALS: + return _SIGNALS[-returncode] + "." + unsigned = returncode & 0xFFFFFFFF + if unsigned in _WINDOWS_CODES: + return _WINDOWS_CODES[unsigned] + "." + return None + + +def describe_death(model_id: str, phase: str, returncode: Optional[int], stderr_lines: Iterable[str]) -> str: + """The error message for an extension subprocess that died during `phase`.""" + tail = relevant_lines(stderr_lines) + exit_text = describe_exit_code(returncode) + head = f"[{model_id}] Subprocess died {phase}" + if exit_text: + head += f" (exit code {exit_text})" + parts = [head + "."] + cause = likely_cause(tail, returncode) + if cause: + parts.append(f"Likely cause: {cause}") + if tail: + parts.append("Last output from the extension:\n" + "\n".join(f" {line}" for line in tail)) + else: + parts.append("The extension wrote nothing to its error output before stopping.") + return "\n\n".join(parts) diff --git a/api/tests/test_subprocess_death.py b/api/tests/test_subprocess_death.py new file mode 100644 index 000000000..5b2bedd8c --- /dev/null +++ b/api/tests/test_subprocess_death.py @@ -0,0 +1,142 @@ +"""#369: a dead extension subprocess must say why it died.""" +import os +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path +from unittest.mock import patch + +from services import extension_process +from services.extension_process import ExtensionProcess +from services.subprocess_death import describe_death, describe_exit_code, relevant_lines + + +class DescribeDeathTests(unittest.TestCase): + def test_cuda_oom_is_named_and_quoted(self) -> None: + msg = describe_death("triposg/generate", "while loading the model", 1, [ + "Loading pipeline components...: 40%|████ | 2/5 [00:01<00:02, 1.20it/s]", + "torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.00 GiB", + ]) + self.assertIn("[triposg/generate] Subprocess died while loading the model (exit code 1).", msg) + self.assertIn("GPU ran out of memory (VRAM)", msg) + self.assertIn("Tried to allocate 2.00 GiB", msg) + self.assertNotIn("it/s", msg) # progress bars are noise + + def test_windows_access_violation_without_stderr(self) -> None: + msg = describe_death("x", "during generation", 0xC0000005, []) + self.assertIn("(exit code 0xC0000005).", msg) + self.assertIn("Likely cause: access violation", msg) + self.assertEqual(msg.count("access violation"), 1) + self.assertIn("wrote nothing", msg) + + def test_signed_windows_code_is_recognized(self) -> None: + self.assertEqual(describe_exit_code(-1073741819), "0xC0000005") + self.assertEqual(describe_exit_code(-9), "signal 9") + self.assertEqual(describe_exit_code(3), "3") + + def test_sigkill_points_at_the_oom_killer(self) -> None: + msg = describe_death("x", "while loading the model", -9, ["Loading weights"]) + self.assertIn("signal 9", msg) + self.assertIn("out-of-memory killer", msg) + + def test_stderr_cause_beats_exit_code(self) -> None: + msg = describe_death("x", "while starting", 0xC0000005, ["ImportError: DLL load failed while importing _C"]) + self.assertIn("Likely cause: a system library (DLL)", msg) + + def test_missing_module_names_the_module(self) -> None: + msg = describe_death("x", "while starting", 1, ["ModuleNotFoundError: No module named 'xatlas'"]) + self.assertIn("(xatlas)", msg) + self.assertIn("Repair the extension", msg) + + def test_windows_paging_file(self) -> None: + msg = describe_death("x", "while loading the model", 1, + ["OSError: [WinError 1455] The paging file is too small for this operation to complete"]) + self.assertIn("virtual memory", msg) + + def test_unknown_cause_still_quotes_the_output(self) -> None: + msg = describe_death("x", "during generation", 1, ["something odd happened"]) + self.assertNotIn("Likely cause", msg) + self.assertIn(" something odd happened", msg) + + def test_tail_is_bounded(self) -> None: + lines = relevant_lines([f"line {i}" for i in range(100)] + ["x" * 1000]) + self.assertEqual(len(lines), 20) + self.assertEqual(lines[0], "line 81") + self.assertLess(len(lines[-1]), 400) + + +# A stand-in for runner.py: says ready, then dies the way it is told to. +_FAKE_RUNNER = textwrap.dedent(""" + import json, os, signal, sys + mode = os.environ["FAKE_DEATH"] + print(json.dumps({"type": "ready"}), flush=True) + sys.stdin.readline() # the load request + if mode == "oom": + sys.stderr.write("Loading checkpoint shards: 50%|##### | 1/2 [00:01<00:01]\\r") + sys.stderr.write("torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 4.00 GiB\\n") + sys.stderr.flush() + os._exit(1) + if mode == "sigkill": + os.kill(os.getpid(), signal.SIGKILL) + if mode == "silent": + os._exit(3) +""") + + +class RealSubprocessDeathTests(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + runner = Path(self.tmp.name) / "runner.py" + runner.write_text(_FAKE_RUNNER, encoding="utf-8") + patches = [ + patch.object(extension_process, "_venv_python", lambda _dir: Path(sys.executable)), + patch.object(extension_process, "_RUNNER_PATH", runner), + ] + for p in patches: + p.start() + self.addCleanup(p.stop) + self.addCleanup(self.tmp.cleanup) + + def _load_with(self, mode: str) -> tuple[ExtensionProcess, str]: + proc = ExtensionProcess(ext_dir=Path(self.tmp.name), manifest={"id": "fake/generate"}) + with patch.dict(os.environ, {"FAKE_DEATH": mode}), \ + patch("sys.stderr", new=open(os.devnull, "w", encoding="utf-8")): + with self.assertRaises(RuntimeError) as ctx: + proc.load() + sys.stderr.close() + self.addCleanup(lambda: proc._proc and proc._proc.poll() is None and proc._proc.kill()) + return proc, str(ctx.exception) + + def test_crash_during_load_reports_its_stderr(self) -> None: + proc, msg = self._load_with("oom") + self.assertIn("Subprocess died while loading the model (exit code 1)", msg) + self.assertIn("GPU ran out of memory", msg) + self.assertIn("Tried to allocate 4.00 GiB", msg) + self.assertFalse(proc.is_loaded()) + + def test_silent_crash_reports_the_exit_code(self) -> None: + _, msg = self._load_with("silent") + self.assertIn("(exit code 3)", msg) + self.assertIn("wrote nothing", msg) + + @unittest.skipIf(sys.platform == "win32", "no SIGKILL on Windows") + def test_killed_process_is_reported_as_killed(self) -> None: + _, msg = self._load_with("sigkill") + self.assertIn("signal 9", msg) + + def test_stop_by_modly_is_not_reported_as_a_crash(self) -> None: + proc = ExtensionProcess(ext_dir=Path(self.tmp.name), manifest={"id": "fake/generate"}) + with patch.dict(os.environ, {"FAKE_DEATH": "never"}): + proc._start() + proc.stop() + proc._proc = None + proc._queue.put(None) + with self.assertRaises(RuntimeError) as ctx: + proc._recv(timeout=1.0, phase="during generation") + self.assertIn("stopped by Modly", str(ctx.exception)) + self.assertNotIn("killer", str(ctx.exception)) + + +if __name__ == "__main__": + unittest.main() From 4cd298cbfce861924ab32d86cb3969c1acf54b9b Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Sun, 8 Nov 2026 21:34:49 +0100 Subject: [PATCH 020/108] fix(extensions): tie a crash report to its own process Review follow-ups on the subprocess death message: - Each subprocess lifetime (process, stderr thread, stderr tail) is kept together, so a death is explained with its own exit code and output even if a new worker has started meanwhile. - "Stopped by Modly" is tied to that process, set by both stop() and the cancel hard-kill, and cleared when the kill fails, so a later real crash is still reported. - A worker that closed stdout but keeps running is killed instead of being reused. - A clear stderr cause (out of memory, driver...) wins, then the exit code, then weak causes (missing module, DLL load failure) that are often only an earlier warning, e.g. xformers' triton import on Windows. - Causes are matched on full lines; only the quoted text is cut. - The missing-module pattern is shared with auto-repair. --- api/services/extension_process.py | 82 +++++++++++++++++++---------- api/services/subprocess_death.py | 84 +++++++++++++++++++----------- api/tests/test_subprocess_death.py | 78 ++++++++++++++++++++++++--- 3 files changed, 179 insertions(+), 65 deletions(-) diff --git a/api/services/extension_process.py b/api/services/extension_process.py index e6be67bd8..020fa6c83 100644 --- a/api/services/extension_process.py +++ b/api/services/extension_process.py @@ -12,7 +12,6 @@ import os import platform import queue -import re import subprocess import sys import threading @@ -21,10 +20,10 @@ from pathlib import Path from typing import Callable, Optional -from services.subprocess_death import describe_death +from services.subprocess_death import MISSING_MODULE_RE, describe_death _RUNNER_PATH = Path(__file__).parent.parent / "runner.py" -_MISSING_MODULE_RE = re.compile(r"No module named ['\"]([^'\"]+)['\"]") +_MISSING_MODULE_RE = MISSING_MODULE_RE # Raw stderr lines kept per subprocess, to explain a crash (#369). _STDERR_TAIL = 400 _AUTO_REPAIR_PACKAGE_MAP = { @@ -39,6 +38,18 @@ def _venv_python(ext_dir: Path) -> Path: return ext_dir / "venv" / "bin" / "python" +class _Run: + """One subprocess lifetime: what is needed to explain its death.""" + + def __init__(self, proc: subprocess.Popen, stderr_thread: threading.Thread, stderr_tail: deque) -> None: + self.proc = proc + self.stderr_thread = stderr_thread + self.stderr_tail = stderr_tail + # Set when Modly kills it on purpose: not a crash, and its SIGKILL + # must not be reported as the OOM killer. + self.stopped_by_modly = False + + class ExtensionProcess: """ Wraps an extension subprocess. Presents the same interface as a @@ -56,9 +67,7 @@ def __init__(self, ext_dir: Path, manifest: dict) -> None: self._queue: queue.Queue = queue.Queue() self._lock: threading.Lock = threading.Lock() self._loaded: bool = False - self._stderr_tail: deque = deque(maxlen=_STDERR_TAIL) - self._stderr_thread: Optional[threading.Thread] = None - self._stopped_by_modly: bool = False + self._run: Optional[_Run] = None # Mirrors BaseGenerator attributes used by the registry self.hf_repo = manifest.get("hf_repo", "") @@ -131,7 +140,6 @@ def _start(self) -> None: # older reader thread cannot poison startup for the new process. run_queue: queue.Queue = queue.Queue() self._queue = run_queue - self._stopped_by_modly = False self._proc = subprocess.Popen( [str(python), str(_RUNNER_PATH)], stdin=subprocess.PIPE, @@ -156,11 +164,11 @@ def _start(self) -> None: # Background thread: forward stderr to our stderr # (and keep its last lines to explain a crash). - self._stderr_tail = deque(maxlen=_STDERR_TAIL) + stderr_tail: deque = deque(maxlen=_STDERR_TAIL) stderr_fwd = threading.Thread( - target=self._stderr_loop, args=(self._proc, self._stderr_tail), daemon=True, + target=self._stderr_loop, args=(self._proc, stderr_tail), daemon=True, ) - self._stderr_thread = stderr_fwd + self._run = _Run(self._proc, stderr_fwd, stderr_tail) stderr_fwd.start() # Wait for ready — runner sends params_schema in this message @@ -280,35 +288,48 @@ def _send(self, msg: dict) -> None: self._proc.stdin.flush() def _recv(self, timeout: float | None = 120.0, phase: str = "unexpectedly") -> dict: + run = self._run try: msg = self._queue.get(timeout=timeout) except queue.Empty: raise TimeoutError(f"[{self.MODEL_ID}] No response from subprocess after {timeout}s") if msg is None: - raise self._death_error(phase) + raise self._death_error(phase, run) return msg - def _death_error(self, phase: str) -> RuntimeError: - """Why the subprocess died, with its exit code and last stderr lines. + def _death_error(self, phase: str, run: Optional[_Run]) -> RuntimeError: + """Why the subprocess of `run` died, with its exit code and last stderr lines. stdout closing is how we notice the death; the process may need a moment to finish exiting, and the stderr thread to drain what is left. """ self._loaded = False - if self._stopped_by_modly: - # stop() killed it (Free Memory / unload): not a crash, and its - # SIGKILL must not be reported as the OOM killer. + if run is None: + return RuntimeError(describe_death(self.MODEL_ID, phase, None, [])) + if run.stopped_by_modly: return RuntimeError(f"[{self.MODEL_ID}] Subprocess was stopped by Modly {phase}") - returncode = None - proc = self._proc - if proc is not None: + try: + returncode = run.proc.wait(timeout=5.0) + except subprocess.TimeoutExpired: + # stdout closed but the process lives on: it can no longer answer, + # so kill it rather than leave a worker _ensure_started would reuse. + returncode = None try: - returncode = proc.wait(timeout=5.0) + run.proc.kill() + run.proc.wait(timeout=5.0) except Exception: - returncode = proc.poll() - if self._stderr_thread is not None: - self._stderr_thread.join(timeout=2.0) - return RuntimeError(describe_death(self.MODEL_ID, phase, returncode, list(self._stderr_tail))) + pass + run.stderr_thread.join(timeout=2.0) + return RuntimeError(describe_death(self.MODEL_ID, phase, returncode, list(run.stderr_tail))) + + def _mark_stopped_by_modly(self, proc: Optional[subprocess.Popen]) -> Optional[_Run]: + """Flag `proc` as killed on purpose, before the kill: the reader may see + stdout close before kill() returns.""" + run = self._run + if run is not None and proc is not None and run.proc is proc: + run.stopped_by_modly = True + return run + return None def _ensure_started(self) -> None: if self._proc is None or self._proc.poll() is not None: @@ -343,7 +364,7 @@ def unload(self) -> None: if self._proc and self._proc.poll() is None: try: self._send({"action": "unload"}) - self._recv(timeout=30.0) + self._recv(timeout=30.0, phase="while unloading") except Exception: pass self._loaded = False @@ -375,6 +396,7 @@ def _receive_generation( # native call (octree decode, marching cubes, etc.) that ignores stdin. CANCEL_GRACE_SECONDS = 3.0 + run = self._run cancel_sent_at: Optional[float] = None while True: # Check for cancellation @@ -396,6 +418,7 @@ def _receive_generation( # generation forces a fresh load. try: if self._proc and self._proc.poll() is None: + self._mark_stopped_by_modly(self._proc) self._proc.kill() self._proc.wait(timeout=5.0) except Exception: @@ -416,7 +439,7 @@ def _receive_generation( continue if msg is None: - raise self._death_error("during generation") + raise self._death_error("during generation", run) t = msg.get("type") @@ -493,11 +516,14 @@ def stop(self) -> None: proc = self._proc self._loaded = False if proc and proc.poll() is None: - self._stopped_by_modly = True + marked = self._mark_stopped_by_modly(proc) try: proc.kill() proc.wait(timeout=5) except Exception as exc: + # Still alive: a later death would be a real crash. + if marked is not None: + marked.stopped_by_modly = False # Keep the process reference so callers can verify that the # worker may still be alive and refuse to mutate its venv. self._proc = proc @@ -506,6 +532,8 @@ def stop(self) -> None: ) from exc if proc.poll() is None: self._proc = proc + if marked is not None: + marked.stopped_by_modly = False raise RuntimeError( f"[{self.MODEL_ID}] Extension subprocess is still running" ) diff --git a/api/services/subprocess_death.py b/api/services/subprocess_death.py index 92bc65ba1..c55735fc9 100644 --- a/api/services/subprocess_death.py +++ b/api/services/subprocess_death.py @@ -16,31 +16,36 @@ # tqdm progress bars: "Loading: 45%|████▌ | 9/20 [00:03<00:04, 2.51it/s]" _PROGRESS_RE = re.compile(r"\d+%\|.*\||\|\s*\d+/\d+\s*\[") -# Checked in order: the first matching pattern wins. -_STDERR_CAUSES: list[tuple[re.Pattern, str]] = [ +# Shared with ExtensionProcess's auto-repair. +MISSING_MODULE_RE = re.compile(r"No module named ['\"]([^'\"]+)['\"]") + +# (pattern, cause, weak). Weak causes are often printed as harmless warnings by +# optional imports (xformers and triton on Windows), so they never override a +# native crash exit code. +_STDERR_CAUSES: list[tuple[re.Pattern, str, bool]] = [ (re.compile(r"CUDA out of memory|OutOfMemoryError|CUBLAS_STATUS_ALLOC_FAILED|MPS backend out of memory", re.I), "the GPU ran out of memory (VRAM). Close other GPU-heavy apps, lower the " - "resolution or quality settings, or pick a lighter model."), + "resolution or quality settings, or pick a lighter model.", False), (re.compile(r"paging file is too small|os error 1455|error code 1455", re.I), "Windows ran out of virtual memory while loading the model. Increase the " "paging file size (System > Advanced system settings > Performance > " - "Virtual memory) or free up RAM."), + "Virtual memory) or free up RAM.", False), (re.compile(r"DefaultCPUAllocator: (?:can't allocate|not enough) memory|\bMemoryError\b", re.I), - "the computer ran out of RAM. Close other applications or pick a lighter model."), + "the computer ran out of RAM. Close other applications or pick a lighter model.", False), (re.compile(r"no kernel image is available|not compatible with the current PyTorch installation", re.I), "the PyTorch build installed for this extension does not support your GPU. " - "Try repairing the extension, or check that the extension supports your GPU."), + "Try repairing the extension, or check that the extension supports your GPU.", False), (re.compile(r"Found no NVIDIA driver|CUDA driver version is insufficient|NVIDIA driver on your system is too old", re.I), - "the NVIDIA driver is missing or too old. Update your GPU driver."), + "the NVIDIA driver is missing or too old. Update your GPU driver.", False), (re.compile(r"DLL load failed", re.I), "a system library (DLL) could not be loaded. Installing the latest " - "Microsoft Visual C++ Redistributable and repairing the extension usually fixes it."), - (re.compile(r"No module named ['\"]([^'\"]+)['\"]"), + "Microsoft Visual C++ Redistributable and repairing the extension usually fixes it.", True), + (MISSING_MODULE_RE, "a Python package is missing from the extension ({0}). Repair the extension " - "from the Models page to reinstall its dependencies."), + "from the Models page to reinstall its dependencies.", True), (re.compile(r"Unable to load weights from checkpoint|HeaderTooLarge|Error while deserializing header|incomplete metadata", re.I), "the model files look corrupted or incomplete (often an interrupted download). " - "Delete the model and download it again."), + "Delete the model and download it again.", False), ] # Windows NTSTATUS codes, as Python reports them (unsigned). @@ -80,43 +85,60 @@ def describe_exit_code(returncode: Optional[int]) -> Optional[str]: return str(returncode) +def _meaningful(lines: Iterable[str]) -> list[str]: + """Stderr lines without progress bars or blanks, full length.""" + return [line.rstrip() for line in lines if line.strip() and not _PROGRESS_RE.search(line)] + + def relevant_lines(lines: Iterable[str]) -> list[str]: - """The last stderr lines worth showing: no progress bars, no blanks.""" - kept = [ - line.rstrip() for line in lines - if line.strip() and not _PROGRESS_RE.search(line) - ] + """The last stderr lines worth showing, each cut to MAX_LINE characters.""" return [ line if len(line) <= MAX_LINE else line[:MAX_LINE] + "…" - for line in kept[-TAIL_LINES:] + for line in _meaningful(lines)[-TAIL_LINES:] ] +def _exit_code_cause(returncode: Optional[int]) -> Optional[str]: + if returncode is None: + return None + if returncode < 0 and -returncode in _SIGNALS: + return _SIGNALS[-returncode] + "." + unsigned = returncode & 0xFFFFFFFF + if unsigned in _WINDOWS_CODES: + return _WINDOWS_CODES[unsigned] + "." + return None + + def likely_cause(lines: list[str], returncode: Optional[int]) -> Optional[str]: - """The most probable cause, from stderr first, then from the exit code.""" + """The most probable cause. Matched on full lines, never on the cut ones. + + A clear stderr cause (out of memory, driver...) comes first, then what the + exit code says, then weak causes that may only be an earlier warning. + """ text = "\n".join(lines) - for pattern, cause in _STDERR_CAUSES: - match = pattern.search(text) - if match: - return cause.format(*match.groups()) - if returncode is not None: - if returncode < 0 and -returncode in _SIGNALS: - return _SIGNALS[-returncode] + "." - unsigned = returncode & 0xFFFFFFFF - if unsigned in _WINDOWS_CODES: - return _WINDOWS_CODES[unsigned] + "." - return None + exit_cause = _exit_code_cause(returncode) + for weak in (False, True): + if weak and exit_cause: + return exit_cause + for pattern, cause, is_weak in _STDERR_CAUSES: + if is_weak != weak: + continue + match = pattern.search(text) + if match: + return cause.format(*match.groups()) + return exit_cause def describe_death(model_id: str, phase: str, returncode: Optional[int], stderr_lines: Iterable[str]) -> str: """The error message for an extension subprocess that died during `phase`.""" - tail = relevant_lines(stderr_lines) + lines = list(stderr_lines) + tail = relevant_lines(lines) exit_text = describe_exit_code(returncode) head = f"[{model_id}] Subprocess died {phase}" if exit_text: head += f" (exit code {exit_text})" parts = [head + "."] - cause = likely_cause(tail, returncode) + cause = likely_cause(_meaningful(lines), returncode) if cause: parts.append(f"Likely cause: {cause}") if tail: diff --git a/api/tests/test_subprocess_death.py b/api/tests/test_subprocess_death.py index 5b2bedd8c..e8c1f4fb6 100644 --- a/api/tests/test_subprocess_death.py +++ b/api/tests/test_subprocess_death.py @@ -1,5 +1,7 @@ """#369: a dead extension subprocess must say why it died.""" +import io import os +import subprocess import sys import tempfile import textwrap @@ -40,8 +42,8 @@ def test_sigkill_points_at_the_oom_killer(self) -> None: self.assertIn("signal 9", msg) self.assertIn("out-of-memory killer", msg) - def test_stderr_cause_beats_exit_code(self) -> None: - msg = describe_death("x", "while starting", 0xC0000005, ["ImportError: DLL load failed while importing _C"]) + def test_dll_failure_is_named_when_python_exited_normally(self) -> None: + msg = describe_death("x", "while starting", 1, ["ImportError: DLL load failed while importing _C"]) self.assertIn("Likely cause: a system library (DLL)", msg) def test_missing_module_names_the_module(self) -> None: @@ -59,6 +61,25 @@ def test_unknown_cause_still_quotes_the_output(self) -> None: self.assertNotIn("Likely cause", msg) self.assertIn(" something odd happened", msg) + def test_earlier_import_warning_does_not_hide_a_native_crash(self) -> None: + # xformers prints this on every Windows import, then works fine. + msg = describe_death("x", "during generation", 0xC0000005, [ + "A matching Triton is not available, some optimizations will not be enabled", + "ModuleNotFoundError: No module named 'triton'", + ]) + self.assertIn("Likely cause: access violation", msg) + self.assertNotIn("Repair the extension", msg) + + def test_clear_stderr_cause_beats_a_native_exit_code(self) -> None: + msg = describe_death("x", "while loading the model", 0xC0000409, + ["torch.OutOfMemoryError: CUDA out of memory."]) + self.assertIn("GPU ran out of memory", msg) + + def test_cause_found_beyond_the_cut(self) -> None: + long_line = "OSError: " + "C:\\models\\x" * 60 + " Error while deserializing header: HeaderTooLarge" + msg = describe_death("x", "while loading the model", 1, [long_line]) + self.assertIn("corrupted or incomplete", msg) + def test_tail_is_bounded(self) -> None: lines = relevant_lines([f"line {i}" for i in range(100)] + ["x" * 1000]) self.assertEqual(len(lines), 20) @@ -84,6 +105,13 @@ def test_tail_is_bounded(self) -> None: """) +def _reap(proc: ExtensionProcess) -> None: + child = proc._proc + if child is not None and child.poll() is None: + child.kill() + child.wait(timeout=10) + + class RealSubprocessDeathTests(unittest.TestCase): def setUp(self) -> None: self.tmp = tempfile.TemporaryDirectory() @@ -100,12 +128,11 @@ def setUp(self) -> None: def _load_with(self, mode: str) -> tuple[ExtensionProcess, str]: proc = ExtensionProcess(ext_dir=Path(self.tmp.name), manifest={"id": "fake/generate"}) - with patch.dict(os.environ, {"FAKE_DEATH": mode}), \ - patch("sys.stderr", new=open(os.devnull, "w", encoding="utf-8")): + # StringIO, not a file: the stderr thread may still print after the test. + with patch.dict(os.environ, {"FAKE_DEATH": mode}), patch("sys.stderr", new=io.StringIO()): with self.assertRaises(RuntimeError) as ctx: proc.load() - sys.stderr.close() - self.addCleanup(lambda: proc._proc and proc._proc.poll() is None and proc._proc.kill()) + self.addCleanup(_reap, proc) return proc, str(ctx.exception) def test_crash_during_load_reports_its_stderr(self) -> None: @@ -130,13 +157,50 @@ def test_stop_by_modly_is_not_reported_as_a_crash(self) -> None: with patch.dict(os.environ, {"FAKE_DEATH": "never"}): proc._start() proc.stop() - proc._proc = None proc._queue.put(None) with self.assertRaises(RuntimeError) as ctx: proc._recv(timeout=1.0, phase="during generation") self.assertIn("stopped by Modly", str(ctx.exception)) self.assertNotIn("killer", str(ctx.exception)) + def test_failed_stop_keeps_reporting_real_crashes(self) -> None: + proc = ExtensionProcess(ext_dir=Path(self.tmp.name), manifest={"id": "fake/generate"}) + with patch.dict(os.environ, {"FAKE_DEATH": "silent"}): + proc._start() + self.addCleanup(_reap, proc) + child = proc._proc + with patch.object(child, "kill", side_effect=PermissionError("denied")): + with self.assertRaises(RuntimeError): + proc.stop() + self.assertFalse(proc._run.stopped_by_modly) + + def test_a_restart_does_not_borrow_the_new_process_output(self) -> None: + proc = ExtensionProcess(ext_dir=Path(self.tmp.name), manifest={"id": "fake/generate"}) + with patch.dict(os.environ, {"FAKE_DEATH": "oom"}), patch("sys.stderr", new=io.StringIO()): + proc._start() + old_run = proc._run + proc._send({"action": "load"}) + old_run.proc.wait(timeout=10) + old_run.stderr_thread.join(timeout=5) + with patch.dict(os.environ, {"FAKE_DEATH": "never"}): + proc._start() # a new worker replaces the dead one + self.addCleanup(proc.stop) + msg = str(proc._death_error("during generation", old_run)) + self.assertIn("(exit code 1)", msg) + self.assertIn("Tried to allocate 4.00 GiB", msg) + + def test_worker_that_closed_stdout_but_lives_on_is_killed(self) -> None: + proc = ExtensionProcess(ext_dir=Path(self.tmp.name), manifest={"id": "fake/generate"}) + with patch.dict(os.environ, {"FAKE_DEATH": "never"}): + proc._start() + self.addCleanup(_reap, proc) + run = proc._run + with patch.object(run.proc, "wait", side_effect=[subprocess.TimeoutExpired("py", 5), None]), \ + patch.object(run.proc, "kill", wraps=run.proc.kill) as kill: + proc._death_error("during generation", run) + kill.assert_called_once() + run.proc.wait(timeout=10) + if __name__ == "__main__": unittest.main() From d74170e9c0d1d3b0dece8a0009006a6a1cc55e64 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Wed, 11 Nov 2026 09:51:57 +0100 Subject: [PATCH 021/108] fix(extensions): accept .git clone URLs and stop blaming the manifest on install errors Installing from GitHub already downloads tarball/HEAD, so the repo's default branch (master or main) was never the problem in #280. Two things made that install look like a manifest problem: - The Models page always rendered the hint 'The repo must contain a manifest.json and a generator.py at its root' right under the red error box, so any failure (for that repo on macOS, its CUDA-only setup.py) read as a missing manifest. The hint now only shows while there is no error. - A pasted clone URL (https://github.com/owner/repo.git) was sent to the API as repos/owner/repo.git/tarball/HEAD, which answers 404. URL parsing moves to parseGitHubRepoUrl, which strips the .git suffix; the manifest source and trust check use the cleaned repo name too. Refs #280 --- .../main/extension-install-utils.test.mjs | 22 +++++++++++++++++++ electron/main/extension-install-utils.ts | 13 +++++++++++ electron/main/ipc-handlers.ts | 7 ++---- src/areas/models/ModelsPage.tsx | 10 ++++++--- 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/electron/main/extension-install-utils.test.mjs b/electron/main/extension-install-utils.test.mjs index ec192d713..fd0eb1a08 100644 --- a/electron/main/extension-install-utils.test.mjs +++ b/electron/main/extension-install-utils.test.mjs @@ -507,3 +507,25 @@ test('validateInstallManifest validates weight variants and keeps them off proce }, files, 'repository'), /weight_variants is supported only for model nodes/) }) +test('parseGitHubRepoUrl resolves owner and repo from the URL forms people paste (#280)', () => { + const mod = loadModule() + const expected = { owner: 'DrHepa', repo: 'modly-trellis-text-extension' } + for (const url of [ + 'https://github.com/DrHepa/modly-trellis-text-extension', + ' https://github.com/DrHepa/modly-trellis-text-extension/ ', + // The "Code > HTTPS" clone URL: the GitHub API answers 404 for ".git" + 'https://github.com/DrHepa/modly-trellis-text-extension.git', + 'https://github.com/DrHepa/modly-trellis-text-extension/tree/master', + 'https://github.com/DrHepa/modly-trellis-text-extension?tab=readme-ov-file', + ]) { + assert.deepEqual(mod.parseGitHubRepoUrl(url), expected, url) + } +}) + +test('parseGitHubRepoUrl rejects URLs that do not name a GitHub repository', () => { + const mod = loadModule() + assert.throws(() => mod.parseGitHubRepoUrl('https://gitlab.com/owner/repo'), /must be a GitHub repository/) + assert.throws(() => mod.parseGitHubRepoUrl('https://github.com/owner'), /expected format/) + assert.throws(() => mod.parseGitHubRepoUrl('https://github.com/owner/.git'), /expected format/) +}) + diff --git a/electron/main/extension-install-utils.ts b/electron/main/extension-install-utils.ts index 2677f1ec5..c8d9a6da8 100644 --- a/electron/main/extension-install-utils.ts +++ b/electron/main/extension-install-utils.ts @@ -58,6 +58,19 @@ export function assertSupportedSceneNodeShape( } } +/** + * Owner and repo of a pasted github.com URL. Accepts the clone URL form + * (".git"), which the GitHub API would otherwise answer with a 404. + */ +export function parseGitHubRepoUrl(input: string): { owner: string; repo: string } { + const parsed = new URL(input.trim()) + if (parsed.hostname !== 'github.com') throw new Error('Invalid URL: must be a GitHub repository (github.com)') + const [owner, rawRepo] = parsed.pathname.split('/').filter(Boolean) + const repo = rawRepo?.replace(/\.git$/, '') + if (!owner || !repo) throw new Error('Invalid URL: expected format https://github.com/owner/repo') + return { owner, repo } +} + export type IncompleteInstallRecoveryAction = | 'none' | 'remove-incomplete' diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index e9cac040b..1ec530c03 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -63,6 +63,7 @@ import { assertCompatibleExtensionUpdateType, expectedModelIds, markExtensionInstallationInterrupted, + parseGitHubRepoUrl, validateExtensionQuarantinePayload, validateExtensionReloadPayload, validateExistingExtensionReplacement, @@ -1501,11 +1502,7 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe try { // 1. Parse and validate GitHub URL - const parsed = new URL(githubUrl.trim()) - if (parsed.hostname !== 'github.com') throw new Error('Invalid URL: must be a GitHub repository (github.com)') - const parts = parsed.pathname.split('/').filter(Boolean) - if (parts.length < 2) throw new Error('Invalid URL: expected format https://github.com/owner/repo') - const [owner, repo] = parts + const { owner, repo } = parseGitHubRepoUrl(githubUrl) emit({ step: 'downloading', percent: 0 }) diff --git a/src/areas/models/ModelsPage.tsx b/src/areas/models/ModelsPage.tsx index 7688474c1..9bfd7445c 100644 --- a/src/areas/models/ModelsPage.tsx +++ b/src/areas/models/ModelsPage.tsx @@ -596,9 +596,13 @@ export default function ModelsPage(): JSX.Element {
)} -

- The repo must contain a manifest.json and a generator.py at its root. -

+ {/* Only a hint before installing: shown under an unrelated error + (e.g. a failed setup.py) it reads as the cause (#280). */} + {!ghErr && ( +

+ The repo must contain a manifest.json and a generator.py at its root. +

+ )}
)} From ea63a9beb4011a7a553e4db5bfbe2dbdf6796e22 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Wed, 11 Nov 2026 09:53:04 +0100 Subject: [PATCH 022/108] fix(workflow-runs): report undeclared params and reject malformed params JSON /workflow-runs/from-image accepted any params and forwarded them, so a caller had no way to tell that a model ignores one (texture_resolution on hunyuan3d-mini-turbo, which hardcodes a 1024 bake and does not declare it). The response now lists undeclared_params: keys the model's params_schema does not declare. They are still forwarded, since a schema is not an exhaustive contract (hunyuan reads the undeclared enable_texture). Malformed or non-object params JSON used to fall back to {} and run on defaults; it is now a 400 naming the field. Refs #384 --- api/routers/workflow_runs.py | 23 ++++++- api/tests/test_workflow_runs_lifecycle.py | 8 ++- api/tests/test_workflow_runs_router.py | 75 ++++++++++++++++++++++- 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/api/routers/workflow_runs.py b/api/routers/workflow_runs.py index ba654a53b..8a210ea2d 100644 --- a/api/routers/workflow_runs.py +++ b/api/routers/workflow_runs.py @@ -45,10 +45,14 @@ async def create_run_from_image( if not image.content_type or not image.content_type.startswith("image/"): raise HTTPException(400, "File must be an image") + # Malformed params used to fall back to {} and run on defaults, dropping every value the + # caller set without a word (#384). Refuse instead, naming the field. try: model_params = json.loads(params) except (json.JSONDecodeError, TypeError): - model_params = {} + raise HTTPException(400, "params must be a JSON object") + if not isinstance(model_params, dict): + raise HTTPException(400, "params must be a JSON object") full_params = { "remesh": "quad", @@ -64,11 +68,24 @@ async def create_run_from_image( collection = sanitize_collection(collection) try: - generator_registry.get_generator(model_id) + generator = generator_registry.get_generator(model_id) output_kind = generator_registry.get_manifest(model_id).get("output", "mesh") except ValueError as e: raise HTTPException(400, str(e)) + # Report the params this model does not declare, so a caller learns that e.g. + # texture_resolution may have no effect on it (#384). Reported, not rejected: a schema + # describes the UI controls, not every key a generator reads (hunyuan3d-mini consumes the + # undeclared enable_texture), so a 400 here would refuse requests that work. Still + # forwarded unchanged, exactly as /generate/from-image does. + declared = {field.get("id") for field in generator.params_schema()} + undeclared_params = [key for key in model_params if key not in declared] + if undeclared_params: + print( + f"[workflow-runs] {model_id} does not declare params {undeclared_params}; " + "the model may ignore them." + ) + job_id = str(uuid.uuid4()) image_bytes = await image.read() @@ -81,7 +98,7 @@ async def create_run_from_image( _run_generation, job_id, image_bytes, full_params, collection, output_kind, model_id ) - return {"run_id": job_id, "status": "pending"} + return {"run_id": job_id, "status": "pending", "undeclared_params": undeclared_params} @router.get("/{run_id}", response_model=WorkflowRunStatus) diff --git a/api/tests/test_workflow_runs_lifecycle.py b/api/tests/test_workflow_runs_lifecycle.py index 474335739..72072e807 100644 --- a/api/tests/test_workflow_runs_lifecycle.py +++ b/api/tests/test_workflow_runs_lifecycle.py @@ -24,6 +24,11 @@ async def read(self) -> bytes: return self._data +class _FakeGenerator: + def params_schema(self) -> list: + return [] + + class _FakeRegistry: """Accepts any model id and exposes the attrs cancel_run pokes at.""" @@ -31,7 +36,7 @@ class _FakeRegistry: _active_id = None def get_generator(self, model_id: str) -> object: - return object() + return _FakeGenerator() def get_manifest(self, model_id: str) -> dict: return {"output": "mesh"} @@ -118,6 +123,7 @@ def __init__(self, model_id): self._loaded = True self._proc = Proc(self) def is_loaded(self): return self.loaded + def params_schema(self): return [] def load(self): self.loaded = True def unload(self): self.loaded = False def generate(self, image_bytes, params, progress_cb, cancel_event=None): diff --git a/api/tests/test_workflow_runs_router.py b/api/tests/test_workflow_runs_router.py index 7537a4844..cb2bb210f 100644 --- a/api/tests/test_workflow_runs_router.py +++ b/api/tests/test_workflow_runs_router.py @@ -18,11 +18,24 @@ async def read(self) -> bytes: return self._data +class _FakeGenerator: + """Declares a params_schema the way BaseGenerator/ExtensionProcess do.""" + + def __init__(self, schema: list | None = None) -> None: + self._schema = schema or [] + + def params_schema(self) -> list: + return self._schema + + class _FakeRegistry: """Accepts any model id so the route reaches the point where it files the run.""" + def __init__(self, schema: list | None = None) -> None: + self._gen = _FakeGenerator(schema) + def get_generator(self, model_id: str) -> object: - return object() + return self._gen def get_manifest(self, model_id: str) -> dict: return {"output": "mesh"} @@ -102,6 +115,7 @@ class _SwitchTrackingRegistry(_FakeRegistry): """Records whether switch_model ran, to prove a rejected request never reaches it.""" def __init__(self) -> None: + super().__init__() self.switched = False def switch_model(self, model_id: str) -> None: @@ -152,5 +166,64 @@ def test_invalid_remesh_is_rejected_before_switching_the_active_model(self) -> N self.assertFalse(registry.switched) +class CreateRunParamsTests(unittest.TestCase): + """#384: params a model does not declare must not be accepted silently.""" + + # Mirrors hunyuan3d-mini-turbo: texture_resolution is not part of its schema. + _SCHEMA = [{"id": "octree_resolution"}, {"id": "seed"}] + + def setUp(self) -> None: + self._previous = workflow_runs.generator_registry + workflow_runs.generator_registry = _FakeRegistry(self._SCHEMA) + + def tearDown(self) -> None: + workflow_runs.generator_registry = self._previous + + def _create(self, params: str) -> tuple[dict, BackgroundTasks]: + background = BackgroundTasks() + response = asyncio.run( + workflow_runs.create_run_from_image( + background, + image=_FakeUpload(), + model_id="hunyuan3d-mini-turbo", + collection="Default", + params=params, + ) + ) + return response, background + + def test_undeclared_params_are_reported_in_the_response(self) -> None: + response, _ = self._create( + '{"remesh":"quad","enable_texture":true,"texture_resolution":2048,"octree_resolution":256}' + ) + self.assertEqual( + response["undeclared_params"], ["remesh", "enable_texture", "texture_resolution"] + ) + + def test_declared_params_are_not_reported(self) -> None: + response, _ = self._create('{"octree_resolution": 256, "seed": 7}') + self.assertEqual(response["undeclared_params"], []) + + def test_undeclared_params_are_still_forwarded_to_the_generator(self) -> None: + # A schema is UI metadata, not an exhaustive contract: hunyuan reads enable_texture + # without declaring it, so dropping or rejecting undeclared keys would break it. + _, background = self._create('{"enable_texture": true, "texture_resolution": 2048}') + forwarded = background.tasks[0].args[2] + self.assertTrue(forwarded["enable_texture"]) + self.assertEqual(forwarded["texture_resolution"], 2048) + + def test_invalid_params_json_is_rejected(self) -> None: + # Previously parsed as {} so every param was dropped without a word. + with self.assertRaises(HTTPException) as ctx: + self._create("{'texture_resolution': 2048}") + self.assertEqual(ctx.exception.status_code, 400) + self.assertIn("params", str(ctx.exception.detail)) + + def test_non_object_params_json_is_rejected(self) -> None: + with self.assertRaises(HTTPException) as ctx: + self._create('[["texture_resolution", 2048]]') + self.assertEqual(ctx.exception.status_code, 400) + + if __name__ == "__main__": unittest.main() From a12b1467fba9ff92256e40a95cad37bba104c08b Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Wed, 11 Nov 2026 10:28:46 +0100 Subject: [PATCH 023/108] feat(agent): stop runaway tool loops with a budget, a per-tool timeout and a structured error The tool loop used to give up after 10 rounds with a bare "Reached maximum tool iterations." Now each turn has a tool budget (calls, default 10) and each call a timeout (seconds, default 120), both request fields set from Settings -> Agent. Hitting either ends the turn with a `stopped` object ({code, tool, limit, message}) on the done event; a timed-out call's tool_done carries the same object. The chat shows it in the actions card with what to do next. A timed-out call stops waiting on its HTTP request; a mesh op the API had already started still finishes in its worker thread, unseen. Closes #302 --- api/routers/agent.py | 76 +++++++++-- api/tests/test_agent_streaming.py | 119 ++++++++++++++++-- src/areas/generate/components/ChatPanel.tsx | 42 +++++-- .../settings/components/AgentSection.tsx | 45 ++++++- src/shared/stores/agentStore.ts | 10 ++ src/shared/utils/agentStream.test.ts | 18 ++- src/shared/utils/agentStream.ts | 31 ++++- 7 files changed, 308 insertions(+), 33 deletions(-) diff --git a/api/routers/agent.py b/api/routers/agent.py index 114fdcb89..ab6a49be6 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -372,11 +372,13 @@ async def _run_agent_op(client: httpx.AsyncClient, op: MeshOp, arguments: dict) return text, payload -async def execute_tool(name: str, arguments: dict, context: dict) -> tuple[str, dict | None]: +async def execute_tool( + name: str, arguments: dict, context: dict, *, timeout: float = 60.0, +) -> tuple[str, dict | None]: """Execute a tool and return (result_text, action_payload). action_payload carries data the frontend needs to react (e.g. new mesh URL). """ - async with httpx.AsyncClient(timeout=60.0) as client: + async with httpx.AsyncClient(timeout=timeout) as client: try: op = _agent_op(name) if op is not None: @@ -516,18 +518,24 @@ class AgentChatRequest(BaseModel): provider: ProviderConfig = ProviderConfig() context: dict = {} thinking: str = "auto" # "auto" | "on" | "off" + # Guardrails of the tool loop: tool calls allowed in one turn, and seconds + # one call may take. The chat sends the values set in Settings → Agent. + tool_budget: int = Field(default=10, ge=1, le=100) + tool_timeout: float = Field(default=120.0, gt=0, le=3600) class ActionDone(BaseModel): tool: str result: str payload: dict | None = None + error: dict | None = None # the stop, when a guardrail cut this call short class AgentChatResponse(BaseModel): message: str actions: list[ActionDone] = [] thinking: str | None = None + stopped: dict | None = None def _auth_headers(base_url: str, api_key: Optional[str]) -> dict: @@ -777,7 +785,10 @@ async def _read_round(response: httpx.Response, round_: _Round): yield event -def _done(message: str, actions: list[ActionDone], thinking: list[str], error: bool = False) -> dict: +def _done( + message: str, actions: list[ActionDone], thinking: list[str], error: bool = False, + stopped: dict | None = None, +) -> dict: return { "type": "done", "message": message, @@ -786,9 +797,32 @@ def _done(message: str, actions: list[ActionDone], thinking: list[str], error: b # The message reports a failure rather than answering: the chat shows # it after whatever text already streamed. "error": error, + # Set when a guardrail ended the tool loop (see _tool_stop). + "stopped": stopped, } +def _tool_stop(code: str, tool: str, limit: float) -> dict: + """Why a guardrail ended the tool loop, and what the user can do about it. + + tool_budget_exhausted — the model asked for a call past the turn's budget + tool_timeout — one call ran past its timeout + """ + if code == "tool_budget_exhausted": + message = ( + f"Stopped after {limit:g} tool calls: the agent kept calling tools without " + f"finishing (it wanted to run {tool} next). Try a more specific request, " + f"or raise the tool budget in Settings → Agent." + ) + else: + message = ( + f"Stopped: {tool} did not finish within {limit:g} s. It may still be " + f"running in the background. Try again on a lighter mesh, or raise the " + f"tool timeout in Settings → Agent." + ) + return {"code": code, "tool": tool, "limit": limit, "message": message} + + def _release_claimed(claim: asyncio.Future) -> None: if not claim.cancelled() and claim.exception() is None: claim.result().release() @@ -882,6 +916,7 @@ async def _chat_events(request: AgentChatRequest): all_thinking: list[str] = [] final: str | None = None error: str | None = None + stopped: dict | None = None answered = False try: @@ -898,7 +933,8 @@ async def post() -> httpx.Response: stream=True, ) - for _ in range(10): # max tool-call rounds + # Bounded: a round either answers or spends at least one tool call. + while stopped is None: round_ = _Round(answered, bool(all_thinking)) r = await post() try: @@ -934,11 +970,29 @@ async def post() -> httpx.Response: for tc in tool_calls: fn = tc["function"] + if len(actions_done) >= request.tool_budget: + stopped = _tool_stop("tool_budget_exhausted", fn["name"], request.tool_budget) + break arguments = _tool_arguments(fn.get("arguments")) yield {"type": "tool_start", "id": tc["id"], "tool": fn["name"], "arguments": arguments} - result_text, payload = await execute_tool(fn["name"], arguments, request.context) - actions_done.append(ActionDone(tool=fn["name"], result=result_text, payload=payload)) - yield {"type": "tool_done", "id": tc["id"], "tool": fn["name"], "result": result_text, "payload": payload} + try: + result_text, payload = await asyncio.wait_for( + execute_tool(fn["name"], arguments, request.context, timeout=request.tool_timeout), + request.tool_timeout, + ) + except asyncio.TimeoutError: + # Cancelling the call drops its HTTP request, but a mesh op + # the API already started runs to its end in a worker + # thread; its result is never shown. + stopped = _tool_stop("tool_timeout", fn["name"], request.tool_timeout) + result_text, payload = stopped["message"], None + actions_done.append(ActionDone(tool=fn["name"], result=result_text, payload=payload, error=stopped)) + yield { + "type": "tool_done", "id": tc["id"], "tool": fn["name"], + "result": result_text, "payload": payload, "error": stopped, + } + if stopped is not None: + break messages.append({"role": "tool", "tool_call_id": tc["id"], "content": result_text}) except httpx.HTTPError as e: error = f"Could not reach the LLM at {base_url}: {e}" @@ -953,8 +1007,8 @@ async def post() -> httpx.Response: if error is not None: yield _done(error, actions_done, all_thinking, error=True) - elif final is None: - yield _done("Reached maximum tool iterations.", actions_done, all_thinking, error=True) + elif stopped is not None: + yield _done(stopped["message"], actions_done, all_thinking, error=True, stopped=stopped) else: yield _done(final, actions_done, all_thinking) @@ -966,7 +1020,9 @@ async def agent_chat(request: AgentChatRequest): async for event in _chat_events(request): if event["type"] == "done": done = event - return AgentChatResponse(message=done["message"], actions=done["actions"], thinking=done["thinking"]) + return AgentChatResponse( + message=done["message"], actions=done["actions"], thinking=done["thinking"], stopped=done["stopped"], + ) @router.post("/chat/stream") diff --git a/api/tests/test_agent_streaming.py b/api/tests/test_agent_streaming.py index 123e93c2c..9927ff42d 100644 --- a/api/tests/test_agent_streaming.py +++ b/api/tests/test_agent_streaming.py @@ -162,17 +162,11 @@ def test_provider_error_ends_with_an_error_done(self) -> None: self.assertTrue(events[0]["error"]) self.assertEqual(events[0]["message"], "LLM error (401): Incorrect API key provided.") - def test_tool_loop_limit_is_reported_as_an_error(self) -> None: - rounds = [ - _sse(_call(0, id=f"c{i}", function={"name": "list_workflows", "arguments": "{}"})) - for i in range(10) - ] - with mock.patch.object(agent, "execute_tool", mock.AsyncMock(return_value=("none", None))): - events = _events(_Scripted(*rounds), _external()) + def test_done_reports_no_stop_when_the_turn_finishes(self) -> None: + events = _events(_Scripted(_sse({"content": "Hi"})), _external()) + + self.assertIsNone(events[-1]["stopped"]) - self.assertTrue(events[-1]["error"]) - self.assertEqual(events[-1]["message"], "Reached maximum tool iterations.") - self.assertEqual(len(events[-1]["actions"]), 10) def test_local_slot_is_released_when_the_client_disconnects(self) -> None: class Slot: @@ -241,6 +235,111 @@ async def leave_while_loading(): self.assertEqual(slot.released, 1) +def _tool_round(i: int, name: str = "list_workflows") -> httpx.Response: + return _sse(_call(0, id=f"c{i}", function={"name": name, "arguments": "{}"})) + + +def _limited(**limits) -> agent.AgentChatRequest: + request = _external() + return request.model_copy(update=limits) + + +class ToolLoopGuardrailTests(unittest.TestCase): + def test_runaway_loop_stops_at_the_default_budget_with_a_structured_error(self) -> None: + scripted = _Scripted(*[_tool_round(i) for i in range(11)]) + with mock.patch.object(agent, "execute_tool", mock.AsyncMock(return_value=("none", None))): + events = _events(scripted, _external()) + + done = events[-1] + self.assertTrue(done["error"]) + self.assertEqual(len(done["actions"]), 10) + # The call over budget is refused, not started. + self.assertEqual(sum(e["type"] == "tool_start" for e in events), 10) + stop = done["stopped"] + self.assertEqual(stop["code"], "tool_budget_exhausted") + self.assertEqual(stop["tool"], "list_workflows") + self.assertEqual(stop["limit"], 10) + self.assertEqual(done["message"], stop["message"]) + self.assertNotIn("Reached maximum tool iterations", stop["message"]) + self.assertIn("10 tool calls", stop["message"]) + self.assertIn("Settings → Agent", stop["message"]) + + def test_tool_budget_is_set_per_request(self) -> None: + scripted = _Scripted(*[_tool_round(i) for i in range(3)]) + with mock.patch.object(agent, "execute_tool", mock.AsyncMock(return_value=("none", None))): + events = _events(scripted, _limited(tool_budget=2)) + + self.assertEqual(len(events[-1]["actions"]), 2) + self.assertEqual(events[-1]["stopped"]["limit"], 2) + self.assertEqual(len(scripted.bodies), 3) + + def test_spending_the_whole_budget_then_answering_is_not_an_error(self) -> None: + scripted = _Scripted(_tool_round(0), _tool_round(1), _sse({"content": "All done."})) + with mock.patch.object(agent, "execute_tool", mock.AsyncMock(return_value=("none", None))): + events = _events(scripted, _limited(tool_budget=2)) + + self.assertFalse(events[-1]["error"]) + self.assertIsNone(events[-1]["stopped"]) + self.assertEqual(events[-1]["message"], "All done.") + + def test_calls_of_one_round_beyond_the_budget_are_not_run(self) -> None: + both = _sse( + _call(0, id="a", function={"name": "list_workflows", "arguments": "{}"}), + _call(1, id="b", function={"name": "list_models", "arguments": "{}"}), + ) + tool = mock.AsyncMock(return_value=("none", None)) + with mock.patch.object(agent, "execute_tool", tool): + events = _events(_Scripted(both), _limited(tool_budget=1)) + + self.assertEqual(tool.await_count, 1) + self.assertEqual(events[-1]["stopped"]["tool"], "list_models") + + def test_a_tool_over_its_timeout_stops_the_loop_with_a_structured_error(self) -> None: + async def slow(*_args, **_kwargs): + await asyncio.sleep(5) + return "late", None + + scripted = _Scripted(_tool_round(0, "repair_mesh"), _sse({"content": "never asked"})) + with mock.patch.object(agent, "execute_tool", slow): + events = _events(scripted, _limited(tool_timeout=0.05)) + + self.assertEqual([e["type"] for e in events], ["tool_start", "tool_done", "done"]) + finish, done = events[1], events[2] + stop = done["stopped"] + self.assertEqual(stop["code"], "tool_timeout") + self.assertEqual(stop["tool"], "repair_mesh") + self.assertEqual(stop["limit"], 0.05) + self.assertIn("Settings → Agent", stop["message"]) + # The timed-out call settles in the card, flagged with the same error. + self.assertEqual(finish["error"], stop) + self.assertIsNone(finish["payload"]) + self.assertEqual(done["actions"][0]["error"], stop) + self.assertTrue(done["error"]) + # The model is not asked to carry on after a timeout. + self.assertEqual(len(scripted.bodies), 1) + + def test_the_tool_timeout_also_bounds_its_http_calls(self) -> None: + tool = mock.AsyncMock(return_value=("none", None)) + with mock.patch.object(agent, "execute_tool", tool): + _events(_Scripted(_tool_round(0), _sse({"content": "ok"})), _limited(tool_timeout=300)) + + self.assertEqual(tool.await_args.kwargs["timeout"], 300) + + def test_limits_have_defaults_and_bounds(self) -> None: + request = _external() + self.assertEqual((request.tool_budget, request.tool_timeout), (10, 120)) + for bad in ({"tool_budget": 0}, {"tool_budget": 101}, {"tool_timeout": 0}, {"tool_timeout": 3601}): + with self.subTest(bad=bad), self.assertRaises(ValueError): + agent.AgentChatRequest(messages=[], **bad) + + def test_json_route_returns_the_stop(self) -> None: + with mock.patch.object(agent, "execute_tool", mock.AsyncMock(return_value=("none", None))), \ + mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_tool_round(0), _tool_round(1))): + response = asyncio.run(agent.agent_chat(_limited(tool_budget=1))) + + self.assertEqual(response.stopped["code"], "tool_budget_exhausted") + + class StreamRouteTests(unittest.TestCase): def test_stream_route_sends_one_json_event_per_data_line(self) -> None: async def read_body(): diff --git a/src/areas/generate/components/ChatPanel.tsx b/src/areas/generate/components/ChatPanel.tsx index 91ff39f8a..60b07e605 100644 --- a/src/areas/generate/components/ChatPanel.tsx +++ b/src/areas/generate/components/ChatPanel.tsx @@ -6,13 +6,13 @@ import { useWorkflowsStore } from '@shared/stores/workflowsStore' import { useExtensionsStore } from '@shared/stores/extensionsStore' import { useWorkflowRunStore } from '@areas/workflows/workflowRunStore' import { buildAllWorkflowExtensions } from '@areas/workflows/mockExtensions' -import { readAgentEvents } from '@shared/utils/agentStream' +import { readAgentEvents, toolStopTitle } from '@shared/utils/agentStream' // ─── Types ──────────────────────────────────────────────────────────────────── import type { ThinkingMode } from '@shared/stores/agentStore' import type { Workflow } from '@shared/types/electron.d' -import type { AgentAction, AgentActionPayload } from '@shared/utils/agentStream' +import type { AgentAction, AgentActionPayload, ToolStop } from '@shared/utils/agentStream' interface Message { id: string @@ -21,6 +21,7 @@ interface Message { thinking?: string imageDataUrls?: string[] actions?: ActionDone[] + stopped?: ToolStop | null streaming?: boolean } @@ -78,7 +79,11 @@ const TOOL_LABELS: Record = { create_workflow: 'Created workflow', } -function ActionsCard({ actions, onUndo }: { actions: ActionDone[]; onUndo?: () => void }): JSX.Element { +function ActionsCard({ actions, stopped, onUndo }: { + actions: ActionDone[] + stopped?: ToolStop | null + onUndo?: () => void +}): JSX.Element { const [expanded, setExpanded] = useState(false) const meshActions = actions.filter((a) => a.payload?.type === 'mesh_update') const running = actions.find((a) => a.running) @@ -129,6 +134,7 @@ function ActionsCard({ actions, onUndo }: { actions: ActionDone[]; onUndo?: () =
{TOOL_LABELS[a.tool] ?? a.tool.replace(/_/g, ' ')} {a.running && in progress…} + {a.error?.code === 'tool_timeout' && timed out} {a.payload?.type === 'mesh_update' && a.payload.face_count && ( {a.payload.face_count.toLocaleString()} faces )} @@ -142,6 +148,21 @@ function ActionsCard({ actions, onUndo }: { actions: ActionDone[]; onUndo?: () = ))}
)} + + {/* Why the tool loop was stopped — shown even when collapsed */} + {stopped && ( +
+ + + + +
+ {toolStopTitle(stopped)} + {stopped.message} +
+
+ )}
) } @@ -247,7 +268,7 @@ function WorkflowProgressCard({ name }: { name: string }): JSX.Element { // ─── Main component ────────────────────────────────────────────────────────── export default function ChatPanel(): JSX.Element { - const { provider, localModel, external, defaultThinking } = useAgentStore() + const { provider, localModel, external, defaultThinking, toolBudget, toolTimeout } = useAgentStore() const { models: llmModels } = useLlmModels() const isLocal = provider === 'local' const externalConfig = external[provider] @@ -383,6 +404,8 @@ export default function ChatPanel(): JSX.Element { : { type: 'external', base_url: providerBaseUrl(provider, externalConfig), api_key: externalConfig?.apiKey ?? '' }, context, thinking: thinkingMode, + tool_budget: toolBudget, + tool_timeout: toolTimeout, }), }) if (!res.ok || !res.body) throw new Error(`API error ${res.status}`) @@ -410,7 +433,7 @@ export default function ChatPanel(): JSX.Element { patch((m) => ({ ...m, actions: (m.actions ?? []).map((a) => (a.id === event.id - ? { ...a, result: event.result, payload: event.payload, running: false } + ? { ...a, result: event.result, payload: event.payload, error: event.error, running: false } : a)), })) await applyToolPayload(event.payload) @@ -419,8 +442,11 @@ export default function ChatPanel(): JSX.Element { finished = true patch((m) => ({ ...m, - // An error comes after whatever text had already streamed. - content: !m.content ? event.message : event.error ? `${m.content}\n\n${event.message}` : m.content, + // An error comes after whatever text had already streamed; a + // tool-loop stop is shown by the actions card instead. + content: event.stopped ? m.content + : !m.content ? event.message : event.error ? `${m.content}\n\n${event.message}` : m.content, + stopped: event.stopped, thinking: event.thinking ?? m.thinking, streaming: false, })) @@ -603,7 +629,7 @@ export default function ChatPanel(): JSX.Element { {msg.thinking && } {msg.content && } {msg.actions && msg.actions.length > 0 && ( - + )} {!msg.streaming && }
diff --git a/src/areas/settings/components/AgentSection.tsx b/src/areas/settings/components/AgentSection.tsx index 428444fe4..fc1949e0f 100644 --- a/src/areas/settings/components/AgentSection.tsx +++ b/src/areas/settings/components/AgentSection.tsx @@ -10,6 +10,7 @@ import { ModelLibraryModal } from '@shared/components/ui/ModelLibraryModal' import type { LlmModel } from '@shared/stores/llmModelsStore' import { formatBytes } from '@shared/utils/format' import { Section, Card, SegmentedControl } from '@shared/ui' +import { TOOL_LIMITS, clampToolLimit } from '@shared/utils/agentStream' // ─── Helpers ────────────────────────────────────────────────────────────────── @@ -82,6 +83,33 @@ function StepTitle({ n, children }: { n: number; children: React.ReactNode }): J ) } +/** A tool-loop limit, saved on blur or Enter so typing isn't clamped mid-number. */ +function ToolLimitInput({ kind, value, onChange, label }: { + kind: keyof typeof TOOL_LIMITS + value: number + onChange: (value: number) => void + label: string +}): JSX.Element { + const [draft, setDraft] = useState(null) // null: showing the saved value + const commit = () => { + if (draft !== null && draft.trim() !== '') onChange(clampToolLimit(kind, Number(draft))) + setDraft(null) + } + return ( + setDraft(e.target.value)} + onBlur={commit} + onKeyDown={(e) => { if (e.key === 'Enter') commit() }} + className="w-24 bg-zinc-800 border border-zinc-700 text-zinc-200 text-xs rounded-lg px-3 py-2 focus:outline-none focus:border-accent/50" + /> + ) +} + const inputCls = 'w-full bg-zinc-800 border border-zinc-700 text-zinc-200 text-xs rounded-lg px-3 py-2 focus:outline-none focus:border-accent/50' const secondaryBtnCls = 'px-2.5 py-1 rounded-md text-[11px] font-medium bg-zinc-800 hover:bg-zinc-700 text-zinc-300 transition-colors disabled:opacity-50' const primaryBtnCls = 'px-3 py-1.5 rounded-lg bg-accent hover:bg-accent-dark text-white text-xs font-medium transition-colors' @@ -121,8 +149,8 @@ const THINKING_OPTIONS: { value: ThinkingMode; label: string; desc: string }[] = export function AgentSection(): JSX.Element { const { - provider, localModel, external, defaultThinking, - setProvider, setExternal, setDefaultThinking, + provider, localModel, external, defaultThinking, toolBudget, toolTimeout, + setProvider, setExternal, setDefaultThinking, setToolBudget, setToolTimeout, } = useAgentStore() const apiUrl = useAppStore((s) => s.apiUrl) @@ -436,6 +464,19 @@ export function AgentSection(): JSX.Element { ))} + + + } + /> + } + /> + {/* ── Right column ── */} diff --git a/src/shared/stores/agentStore.ts b/src/shared/stores/agentStore.ts index 23b2cfc0b..c7030f80b 100644 --- a/src/shared/stores/agentStore.ts +++ b/src/shared/stores/agentStore.ts @@ -1,5 +1,6 @@ import { create } from 'zustand' import { persist, createJSONStorage } from 'zustand/middleware' +import { TOOL_LIMITS, clampToolLimit } from '@shared/utils/agentStream' export type ThinkingMode = 'auto' | 'on' | 'off' @@ -45,11 +46,15 @@ interface AgentSettings { localModel: string // catalog id or custom: external: Partial> defaultThinking: ThinkingMode + toolBudget: number // tool calls per chat turn + toolTimeout: number // seconds per tool call setProvider: (provider: ProviderId) => void setLocalModel: (model: string) => void setExternal: (provider: ProviderId, cfg: ExternalConfig) => void setDefaultThinking: (mode: ThinkingMode) => void + setToolBudget: (budget: number) => void + setToolTimeout: (seconds: number) => void } // ─── Secure persistence ──────────────────────────────────────────────────────── @@ -118,11 +123,16 @@ export const useAgentStore = create()( localModel: DEFAULT_LOCAL_MODEL, external: {}, defaultThinking: 'auto', + // New keys need no migration: persist fills them from these defaults. + toolBudget: TOOL_LIMITS.budget.default, + toolTimeout: TOOL_LIMITS.timeout.default, setProvider: (provider) => set({ provider }), setLocalModel: (model) => set({ localModel: model }), setExternal: (provider, cfg) => set((s) => ({ external: { ...s.external, [provider]: cfg } })), setDefaultThinking: (mode) => set({ defaultThinking: mode }), + setToolBudget: (budget) => set({ toolBudget: clampToolLimit('budget', budget) }), + setToolTimeout: (seconds) => set({ toolTimeout: clampToolLimit('timeout', seconds) }), }), { name: 'modly-agent-settings', diff --git a/src/shared/utils/agentStream.test.ts b/src/shared/utils/agentStream.test.ts index b2fa95159..8eda242dc 100644 --- a/src/shared/utils/agentStream.test.ts +++ b/src/shared/utils/agentStream.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { createSseParser, readAgentEvents } from './agentStream.ts' +import { clampToolLimit, createSseParser, readAgentEvents, TOOL_LIMITS, toolStopTitle } from './agentStream.ts' const line = (event: object): string => `data: ${JSON.stringify(event)}\n\n` @@ -55,3 +55,19 @@ test('a response body is read into events, multi-byte characters intact', async assert.deepEqual(events.map((e) => e.type), ['token', 'done']) assert.deepEqual(events[0], { type: 'token', text: 'é→' }) }) + +test('a tool-loop stop gets a short title for its code', () => { + const stop = { tool: 'repair_mesh', message: 'm' } + + assert.equal(toolStopTitle({ ...stop, code: 'tool_budget_exhausted', limit: 10 }), 'Tool budget reached (10 calls)') + assert.equal(toolStopTitle({ ...stop, code: 'tool_timeout', limit: 120 }), 'repair mesh timed out (120 s)') +}) + +test('tool limits are kept within what the API accepts', () => { + assert.equal(clampToolLimit('budget', 0), TOOL_LIMITS.budget.min) + assert.equal(clampToolLimit('budget', 7.6), 8) + assert.equal(clampToolLimit('budget', 1e6), TOOL_LIMITS.budget.max) + assert.equal(clampToolLimit('timeout', 1), TOOL_LIMITS.timeout.min) + assert.equal(clampToolLimit('timeout', 90), 90) + assert.equal(clampToolLimit('timeout', Number.NaN), TOOL_LIMITS.timeout.default) +}) diff --git a/src/shared/utils/agentStream.ts b/src/shared/utils/agentStream.ts index 14d155430..3772ca5d6 100644 --- a/src/shared/utils/agentStream.ts +++ b/src/shared/utils/agentStream.ts @@ -11,18 +11,45 @@ export interface AgentActionPayload { workflow?: Omit } +/** Why a guardrail ended the tool loop; `message` says what to do next. */ +export interface ToolStop { + code: 'tool_budget_exhausted' | 'tool_timeout' + tool: string + limit: number // tool calls for the budget, seconds for the timeout + message: string +} + export interface AgentAction { tool: string result: string payload?: AgentActionPayload | null + error?: ToolStop | null } export type AgentEvent = | { type: 'token'; text: string } | { type: 'thinking'; text: string } | { type: 'tool_start'; id: string; tool: string; arguments: Record } - | { type: 'tool_done'; id: string; tool: string; result: string; payload: AgentActionPayload | null } - | { type: 'done'; message: string; actions: AgentAction[]; thinking: string | null; error: boolean } + | { type: 'tool_done'; id: string; tool: string; result: string; payload: AgentActionPayload | null; error?: ToolStop | null } + | { type: 'done'; message: string; actions: AgentAction[]; thinking: string | null; error: boolean; stopped?: ToolStop | null } + +/** Bounds of the request's `tool_budget` (calls per turn) and `tool_timeout` (seconds per call). */ +export const TOOL_LIMITS = { + budget: { min: 1, max: 100, default: 10 }, + timeout: { min: 5, max: 3600, default: 120 }, +} as const + +export function clampToolLimit(kind: keyof typeof TOOL_LIMITS, value: number): number { + const { min, max, default: fallback } = TOOL_LIMITS[kind] + if (!Number.isFinite(value)) return fallback + return Math.min(max, Math.max(min, Math.round(value))) +} + +export function toolStopTitle(stop: ToolStop): string { + return stop.code === 'tool_timeout' + ? `${stop.tool.replace(/_/g, ' ')} timed out (${stop.limit} s)` + : `Tool budget reached (${stop.limit} calls)` +} function parseBlock(block: string): AgentEvent | null { const data = block From a39e4210c9a5d263e37ac4fe700939ac1286dc97 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Wed, 11 Nov 2026 10:25:29 +0100 Subject: [PATCH 024/108] feat(agent): ping Ollama and explain why it isn't answering Add GET /agent/ollama/health: checks the configured Ollama server (/api/tags, 3s timeout) and reports ok / unreachable / model_missing / no_model with the model name and pulled models. The chat panel pings it on mount and when the Ollama URL or model changes, and shows a banner with the fix ("ollama serve" or "ollama pull ", copyable) and a Retry button. A failed Ollama turn re-pings and replaces the raw provider error with the same advice. Only shown when Ollama is the active provider. Closes #303 --- api/routers/agent.py | 8 + api/services/ollama_health.py | 53 +++++++ api/tests/test_agent_ollama_health.py | 146 ++++++++++++++++++ package.json | 2 +- src/areas/generate/components/ChatPanel.tsx | 23 +++ .../generate/components/OllamaBanner.tsx | 85 ++++++++++ src/shared/utils/ollamaHealth.test.ts | 59 +++++++ src/shared/utils/ollamaHealth.ts | 74 +++++++++ 8 files changed, 449 insertions(+), 1 deletion(-) create mode 100644 api/services/ollama_health.py create mode 100644 api/tests/test_agent_ollama_health.py create mode 100644 src/areas/generate/components/OllamaBanner.tsx create mode 100644 src/shared/utils/ollamaHealth.test.ts create mode 100644 src/shared/utils/ollamaHealth.ts diff --git a/api/routers/agent.py b/api/routers/agent.py index ab6a49be6..0e61a043b 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -14,6 +14,7 @@ from pydantic import BaseModel, Field from services import llm_server +from services import ollama_health as ollama_health_service from services.llm_server import llama_pool from services.mesh_ops import MeshOp, MeshOpNotFoundError, MeshOpsRegistry, mesh_ops_registry @@ -572,6 +573,13 @@ async def list_external_models(req: ExternalModelsRequest): return {"models": []} +@router.get("/ollama/health") +async def ollama_health(base_url: str = "http://localhost:11434/v1", model: str = ""): + """Is Ollama up, and is `model` pulled? `status` is `ok`, `unreachable`, + `model_missing` or `no_model`; the chat panel turns it into a banner.""" + return await ollama_health_service.check(base_url, model) + + async def _unload_llm_after_workflow(request: AgentChatRequest, actions_done: list[ActionDone]) -> None: """Free the local LLM's VRAM once a workflow has been dispatched, so the workflow gets the full GPU. Best-effort — never fail the chat over it.""" diff --git a/api/services/ollama_health.py b/api/services/ollama_health.py new file mode 100644 index 000000000..9cfa38c3b --- /dev/null +++ b/api/services/ollama_health.py @@ -0,0 +1,53 @@ +""" +Ollama health check for the chat agent: is the server reachable, and is the +configured model pulled? Answers in a structured status the UI turns into an +actionable message ("ollama serve" / "ollama pull "). +""" +import httpx + +# Short: this runs when the chat panel opens, and a down Ollama must not stall it. +TIMEOUT_S = 3.0 + + +def native_base_url(base_url: str) -> str: + """The chat goes through Ollama's OpenAI-compatible API under `/v1`; the + native API (`/api/tags`) lives at the server root.""" + url = base_url.strip().rstrip("/") + if url.endswith("/v1"): + url = url[: -len("/v1")] + return url + + +def _with_tag(name: str) -> str: + # Ollama resolves an untagged name to `:latest`. + return name if ":" in name else f"{name}:latest" + + +async def check(base_url: str, model: str) -> dict: + root = native_base_url(base_url) + model = model.strip() + result: dict = {"status": "ok", "base_url": root, "model": model} + + try: + async with httpx.AsyncClient(timeout=TIMEOUT_S) as client: + r = await client.get(f"{root}/api/tags") + if r.status_code != 200: + return {**result, "status": "unreachable", "detail": f"HTTP {r.status_code} from {root}/api/tags"} + entries = r.json().get("models", []) + except httpx.HTTPError as e: + return {**result, "status": "unreachable", "detail": str(e) or type(e).__name__} + except (ValueError, AttributeError): + return {**result, "status": "unreachable", "detail": f"{root} did not answer like an Ollama server"} + + available = sorted({ + str(m.get("name") or m.get("model")) + for m in entries + if isinstance(m, dict) and (m.get("name") or m.get("model")) + }) + result["available"] = available + + if not model: + return {**result, "status": "no_model"} + if _with_tag(model) not in {_with_tag(n) for n in available}: + return {**result, "status": "model_missing"} + return result diff --git a/api/tests/test_agent_ollama_health.py b/api/tests/test_agent_ollama_health.py new file mode 100644 index 000000000..7052fd63c --- /dev/null +++ b/api/tests/test_agent_ollama_health.py @@ -0,0 +1,146 @@ +import asyncio +import unittest +from unittest import mock + +import httpx + +import routers.agent as agent +from services import ollama_health + + +class _MockClientFactory: + """Real AsyncClients wired to a MockTransport, so the health check talks to a + fake Ollama instead of the network.""" + + def __init__(self, handler) -> None: + self._handler = handler + self._real = httpx.AsyncClient + self.kwargs: list[dict] = [] + + def __call__(self, *args, **kwargs): + self.kwargs.append(dict(kwargs)) + kwargs["transport"] = httpx.MockTransport(self._handler) + return self._real(*args, **kwargs) + + +def _tags(*names: str) -> dict: + return {"models": [{"name": n, "model": n} for n in names]} + + +def _check(handler, base_url: str = "http://localhost:11434/v1", model: str = "qwen2.5:3b"): + factory = _MockClientFactory(handler) + with mock.patch.object(ollama_health.httpx, "AsyncClient", factory): + result = asyncio.run(agent.ollama_health(base_url=base_url, model=model)) + return result, factory + + +class OllamaHealthTests(unittest.TestCase): + def test_ok_when_reachable_and_model_pulled(self) -> None: + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_tags("llama3:8b", "qwen2.5:3b")) + + result, _ = _check(handler) + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["model"], "qwen2.5:3b") + # The native API lives at the root, not under the OpenAI-compatible /v1. + self.assertEqual(str(requests[0].url), "http://localhost:11434/api/tags") + + def test_uses_the_configured_base_url(self) -> None: + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_tags("qwen2.5:3b")) + + result, _ = _check(handler, base_url="http://192.168.1.20:9999/v1/") + + self.assertEqual(result["status"], "ok") + self.assertEqual(str(requests[0].url), "http://192.168.1.20:9999/api/tags") + self.assertEqual(result["base_url"], "http://192.168.1.20:9999") + + def test_model_missing_when_not_pulled(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=_tags("llama3:8b")) + + result, _ = _check(handler) + + self.assertEqual(result["status"], "model_missing") + self.assertEqual(result["model"], "qwen2.5:3b") + self.assertEqual(result["available"], ["llama3:8b"]) + + def test_untagged_model_matches_latest(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=_tags("mistral:latest")) + + result, _ = _check(handler, model="mistral") + + self.assertEqual(result["status"], "ok") + + def test_tagged_model_does_not_match_a_different_tag(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=_tags("qwen2.5:latest")) + + result, _ = _check(handler, model="qwen2.5:3b") + + self.assertEqual(result["status"], "model_missing") + + def test_no_model_configured(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=_tags("llama3:8b")) + + result, _ = _check(handler, model=" ") + + self.assertEqual(result["status"], "no_model") + + def test_unreachable_on_connection_error(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("connection refused", request=request) + + result, _ = _check(handler) + + self.assertEqual(result["status"], "unreachable") + self.assertEqual(result["base_url"], "http://localhost:11434") + self.assertIn("connection refused", result["detail"]) + + def test_unreachable_on_timeout(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("timed out", request=request) + + result, _ = _check(handler) + + self.assertEqual(result["status"], "unreachable") + + def test_unreachable_when_something_else_answers(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(404, text="not found") + + result, _ = _check(handler) + + self.assertEqual(result["status"], "unreachable") + self.assertIn("404", result["detail"]) + + def test_unreachable_on_non_json_body(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text="hello") + + result, _ = _check(handler) + + self.assertEqual(result["status"], "unreachable") + + def test_uses_a_short_timeout(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=_tags("qwen2.5:3b")) + + _, factory = _check(handler) + + timeout = factory.kwargs[0].get("timeout") + self.assertIsNotNone(timeout) + self.assertLessEqual(float(timeout), 5.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/package.json b/package.json index 4c9732c16..315d90d1b 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/components/ChatPanel.tsx b/src/areas/generate/components/ChatPanel.tsx index 60b07e605..cf9ad2bb6 100644 --- a/src/areas/generate/components/ChatPanel.tsx +++ b/src/areas/generate/components/ChatPanel.tsx @@ -7,6 +7,8 @@ import { useExtensionsStore } from '@shared/stores/extensionsStore' import { useWorkflowRunStore } from '@areas/workflows/workflowRunStore' import { buildAllWorkflowExtensions } from '@areas/workflows/mockExtensions' import { readAgentEvents, toolStopTitle } from '@shared/utils/agentStream' +import { adviceText, ollamaAdvice } from '@shared/utils/ollamaHealth' +import OllamaBanner, { useOllamaHealth } from './OllamaBanner' // ─── Types ──────────────────────────────────────────────────────────────────── @@ -302,6 +304,9 @@ export default function ChatPanel(): JSX.Element { const pushMeshUrl = useAppStore((s) => s.pushMeshUrl) const undoMesh = useAppStore((s) => s.undoMesh) + const isOllama = provider === 'ollama' + const ollama = useOllamaHealth(isOllama, apiUrl, providerBaseUrl(provider, externalConfig), model) + const workflows = useWorkflowsStore((s) => s.workflows) const saveWorkflow = useWorkflowsStore((s) => s.save) const setActiveWorkflow = useWorkflowsStore((s) => s.setActive) @@ -374,6 +379,7 @@ export default function ChatPanel(): JSX.Element { setError(null) const id = `a-${Date.now()}` let finished = false + let failure: string | null = null try { const context = { ...buildContext(), ...extraContext } @@ -440,6 +446,8 @@ export default function ChatPanel(): JSX.Element { break case 'done': finished = true + // A guardrail stop means the LLM answered: nothing to ask Ollama. + if (event.error && !event.stopped) failure = event.message patch((m) => ({ ...m, // An error comes after whatever text had already streamed; a @@ -457,6 +465,7 @@ export default function ChatPanel(): JSX.Element { } } if (!finished) setError('The response was interrupted.') + if (failure !== null && isOllama) await explainOllamaFailure(id, failure) } catch (e: unknown) { const msg = e instanceof Error ? e.message : String(e) setError(msg.includes('fetch') ? 'Cannot reach Modly API. Is the backend running?' : msg) @@ -472,6 +481,17 @@ export default function ChatPanel(): JSX.Element { } } + // A failed Ollama turn: re-ping it (refreshing the banner) and, when it's + // down or the model isn't pulled, say so instead of the raw provider error. + async function explainOllamaFailure(id: string, rawError: string) { + const health = await ollama.check() + const advice = health ? ollamaAdvice(health) : null + if (!advice) return + setMessages((prev) => prev.map((m) => (m.id === id + ? { ...m, content: m.content.replace(rawError, adviceText(advice)) } + : m))) + } + async function applyToolPayload(payload: AgentActionPayload | null) { if (payload?.type === 'mesh_update' && payload.url) { updateCurrentJob({ outputUrl: payload.url }) @@ -663,6 +683,9 @@ export default function ChatPanel(): JSX.Element { + {/* Ollama down / model not pulled */} + { void ollama.check() }} /> + {/* Input bar */}
(null) + const [checking, setChecking] = useState(false) + const seq = useRef(0) + + const check = useCallback(async (): Promise => { + if (!enabled) return null + const n = ++seq.current + setChecking(true) + let result: OllamaHealth | null = null + try { + const res = await fetch(ollamaHealthUrl(apiUrl, baseUrl, model)) + result = res.ok ? parseOllamaHealth(await res.json()) : null + } catch { + result = null + } + // A later check (model switched mid-flight) wins. + if (n === seq.current) { + setHealth(result) + setChecking(false) + } + return result + }, [enabled, apiUrl, baseUrl, model]) + + useEffect(() => { void check() }, [check]) + + return { health: enabled ? health : null, checking, check } +} + +interface Props { + health: OllamaHealth | null + checking: boolean + onRetry: () => void +} + +export default function OllamaBanner({ health, checking, onRetry }: Props): JSX.Element | null { + const [copied, setCopied] = useState(false) + const advice = health ? ollamaAdvice(health) : null + if (!advice) return null + + function copy(command: string) { + navigator.clipboard?.writeText(command).then(() => { + setCopied(true) + setTimeout(() => setCopied(false), 1500) + }).catch(() => {}) + } + + return ( +
+
+

{advice.title}

+ +
+

{advice.body}

+ {advice.command && ( +
+ {advice.command} + +
+ )} +
+ ) +} diff --git a/src/shared/utils/ollamaHealth.test.ts b/src/shared/utils/ollamaHealth.test.ts new file mode 100644 index 000000000..5c39d64b8 --- /dev/null +++ b/src/shared/utils/ollamaHealth.test.ts @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { adviceText, ollamaAdvice, ollamaHealthUrl, parseOllamaHealth } from './ollamaHealth.ts' + +const base = { base_url: 'http://localhost:11434', model: 'qwen2.5:3b' } + +test('the health URL carries the configured base URL and model, encoded', () => { + const url = ollamaHealthUrl('http://127.0.0.1:8765', 'http://localhost:11434/v1', 'qwen2.5:3b') + assert.equal( + url, + 'http://127.0.0.1:8765/agent/ollama/health?base_url=http%3A%2F%2Flocalhost%3A11434%2Fv1&model=qwen2.5%3A3b', + ) +}) + +test('ok needs no advice', () => { + assert.equal(ollamaAdvice({ ...base, status: 'ok' }), null) +}) + +test('unreachable tells the user to start Ollama, with the command', () => { + const advice = ollamaAdvice({ ...base, status: 'unreachable', detail: 'connection refused' }) + assert.ok(advice) + assert.match(advice.title, /not running/i) + assert.match(advice.body, /localhost:11434/) + assert.equal(advice.command, 'ollama serve') +}) + +test('model_missing tells the user to pull the configured model', () => { + const advice = ollamaAdvice({ ...base, status: 'model_missing', available: ['llama3:8b'] }) + assert.ok(advice) + assert.match(advice.title, /qwen2\.5:3b/) + assert.equal(advice.command, 'ollama pull qwen2.5:3b') +}) + +test('no_model points at Settings and has no command', () => { + const advice = ollamaAdvice({ ...base, model: '', status: 'no_model' }) + assert.ok(advice) + assert.match(advice.body, /Settings → Agent/) + assert.equal(advice.command, undefined) +}) + +test('adviceText flattens the advice into one plain-text chat line', () => { + assert.equal( + adviceText({ title: 'Ollama is not running', body: 'Start it, or run:', command: 'ollama serve' }), + 'Ollama is not running. Start it, or run: ollama serve', + ) + assert.equal(adviceText({ title: 'No model', body: 'Pick one.' }), 'No model. Pick one.') +}) + +test('a well-formed response parses', () => { + const parsed = parseOllamaHealth({ ...base, status: 'model_missing', available: ['a'], extra: 1 }) + assert.deepEqual(parsed, { ...base, status: 'model_missing', available: ['a'], detail: undefined }) +}) + +test('an unknown status or a non-object is rejected', () => { + assert.equal(parseOllamaHealth({ ...base, status: 'weird' }), null) + assert.equal(parseOllamaHealth(null), null) + assert.equal(parseOllamaHealth('ok'), null) +}) diff --git a/src/shared/utils/ollamaHealth.ts b/src/shared/utils/ollamaHealth.ts new file mode 100644 index 000000000..bab642b85 --- /dev/null +++ b/src/shared/utils/ollamaHealth.ts @@ -0,0 +1,74 @@ +// GET /agent/ollama/health — is Ollama up, and is the configured model pulled? + +export type OllamaStatus = 'ok' | 'unreachable' | 'model_missing' | 'no_model' + +const STATUSES: readonly OllamaStatus[] = ['ok', 'unreachable', 'model_missing', 'no_model'] + +export interface OllamaHealth { + status: OllamaStatus + /** Ollama's server root (no `/v1`). */ + base_url: string + model: string + /** Models pulled in Ollama (absent when it is unreachable). */ + available?: string[] + detail?: string +} + +export interface OllamaAdvice { + title: string + body: string + /** A shell command that fixes it, shown copyable. */ + command?: string +} + +export function ollamaHealthUrl(apiUrl: string, baseUrl: string, model: string): string { + const query = new URLSearchParams({ base_url: baseUrl, model }) + return `${apiUrl}/agent/ollama/health?${query.toString()}` +} + +export function parseOllamaHealth(data: unknown): OllamaHealth | null { + if (!data || typeof data !== 'object') return null + const d = data as Record + if (!STATUSES.includes(d.status as OllamaStatus)) return null + return { + status: d.status as OllamaStatus, + base_url: typeof d.base_url === 'string' ? d.base_url : '', + model: typeof d.model === 'string' ? d.model : '', + ...(Array.isArray(d.available) ? { available: d.available.filter((m): m is string => typeof m === 'string') } : {}), + detail: typeof d.detail === 'string' ? d.detail : undefined, + } +} + +function host(baseUrl: string): string { + return baseUrl.replace(/^https?:\/\//, '') || 'its configured address' +} + +/** The advice as one plain-text line, for a chat message. */ +export function adviceText(advice: OllamaAdvice): string { + return [`${advice.title}.`, advice.body, advice.command].filter(Boolean).join(' ') +} + +/** What to tell the user, or null when Ollama is ready. */ +export function ollamaAdvice(health: OllamaHealth): OllamaAdvice | null { + switch (health.status) { + case 'ok': + return null + case 'unreachable': + return { + title: 'Ollama is not running', + body: `The agent can't reach Ollama at ${host(health.base_url)}. Start the Ollama app, or run:`, + command: 'ollama serve', + } + case 'model_missing': + return { + title: `Model "${health.model}" is not pulled`, + body: 'Ollama is running but doesn\'t have this model yet. Download it with:', + command: `ollama pull ${health.model}`, + } + case 'no_model': + return { + title: 'No Ollama model selected', + body: 'Pick the model to chat with in Settings → Agent.', + } + } +} From 374b8e5613b7e2213be56848d6fda0b304ce635c Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Wed, 11 Nov 2026 10:31:29 +0100 Subject: [PATCH 025/108] fix(agent): only probe local Ollama addresses from the health check The API answers any origin (CORS *), so a web page could call /agent/ollama/health with any base_url and read the result, turning it into a probe of the user's network. Accept only http(s) loopback URLs without credentials (400 otherwise), don't follow redirects, and report generic reasons ("connection refused", "timed out", "not an Ollama server") instead of echoing the upstream status or exception text. Refs #303 --- api/routers/agent.py | 10 ++++-- api/services/ollama_health.py | 37 ++++++++++++++++++--- api/tests/test_agent_ollama_health.py | 46 +++++++++++++++++++++++---- 3 files changed, 78 insertions(+), 15 deletions(-) diff --git a/api/routers/agent.py b/api/routers/agent.py index 0e61a043b..1eabd6b32 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -9,7 +9,7 @@ from typing import Optional import httpx -from fastapi import APIRouter +from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field @@ -576,8 +576,12 @@ async def list_external_models(req: ExternalModelsRequest): @router.get("/ollama/health") async def ollama_health(base_url: str = "http://localhost:11434/v1", model: str = ""): """Is Ollama up, and is `model` pulled? `status` is `ok`, `unreachable`, - `model_missing` or `no_model`; the chat panel turns it into a banner.""" - return await ollama_health_service.check(base_url, model) + `model_missing` or `no_model`; the chat panel turns it into a banner. + Only local addresses are accepted (400 otherwise).""" + try: + return await ollama_health_service.check(base_url, model) + except ValueError as e: + raise HTTPException(400, str(e)) async def _unload_llm_after_workflow(request: AgentChatRequest, actions_done: list[ActionDone]) -> None: diff --git a/api/services/ollama_health.py b/api/services/ollama_health.py index 9cfa38c3b..190608c62 100644 --- a/api/services/ollama_health.py +++ b/api/services/ollama_health.py @@ -3,11 +3,32 @@ configured model pulled? Answers in a structured status the UI turns into an actionable message ("ollama serve" / "ollama pull "). """ +from urllib.parse import urlsplit + import httpx # Short: this runs when the chat panel opens, and a down Ollama must not stall it. TIMEOUT_S = 3.0 +# The API answers any origin (CORS *), so a web page could call this endpoint. +# Only local addresses are probed, and nothing of the upstream answer is echoed, +# so it can't be used to map the user's network. +LOOPBACK_HOSTS = {"localhost", "127.0.0.1", "::1"} + + +def is_local_url(url: str) -> bool: + try: + parts = urlsplit(url) + host = parts.hostname + except ValueError: + return False + return ( + parts.scheme in ("http", "https") + and not parts.username + and not parts.password + and host in LOOPBACK_HOSTS + ) + def native_base_url(base_url: str) -> str: """The chat goes through Ollama's OpenAI-compatible API under `/v1`; the @@ -24,20 +45,26 @@ def _with_tag(name: str) -> str: async def check(base_url: str, model: str) -> dict: + """Raises ValueError when `base_url` is not a local http(s) address.""" root = native_base_url(base_url) + if not is_local_url(root): + raise ValueError("base_url must be a local http(s) address (localhost, 127.0.0.1 or ::1)") model = model.strip() result: dict = {"status": "ok", "base_url": root, "model": model} + not_ollama = {**result, "status": "unreachable", "detail": "not an Ollama server"} try: - async with httpx.AsyncClient(timeout=TIMEOUT_S) as client: + async with httpx.AsyncClient(timeout=TIMEOUT_S, follow_redirects=False) as client: r = await client.get(f"{root}/api/tags") if r.status_code != 200: - return {**result, "status": "unreachable", "detail": f"HTTP {r.status_code} from {root}/api/tags"} + return not_ollama entries = r.json().get("models", []) - except httpx.HTTPError as e: - return {**result, "status": "unreachable", "detail": str(e) or type(e).__name__} + except httpx.TimeoutException: + return {**result, "status": "unreachable", "detail": "timed out"} + except httpx.HTTPError: + return {**result, "status": "unreachable", "detail": "connection refused"} except (ValueError, AttributeError): - return {**result, "status": "unreachable", "detail": f"{root} did not answer like an Ollama server"} + return not_ollama available = sorted({ str(m.get("name") or m.get("model")) diff --git a/api/tests/test_agent_ollama_health.py b/api/tests/test_agent_ollama_health.py index 7052fd63c..8452722b7 100644 --- a/api/tests/test_agent_ollama_health.py +++ b/api/tests/test_agent_ollama_health.py @@ -3,6 +3,7 @@ from unittest import mock import httpx +from fastapi import HTTPException import routers.agent as agent from services import ollama_health @@ -56,11 +57,41 @@ def handler(request: httpx.Request) -> httpx.Response: requests.append(request) return httpx.Response(200, json=_tags("qwen2.5:3b")) - result, _ = _check(handler, base_url="http://192.168.1.20:9999/v1/") + result, _ = _check(handler, base_url="http://127.0.0.1:9999/v1/") self.assertEqual(result["status"], "ok") - self.assertEqual(str(requests[0].url), "http://192.168.1.20:9999/api/tags") - self.assertEqual(result["base_url"], "http://192.168.1.20:9999") + self.assertEqual(str(requests[0].url), "http://127.0.0.1:9999/api/tags") + self.assertEqual(result["base_url"], "http://127.0.0.1:9999") + + def test_rejects_urls_that_are_not_local(self) -> None: + # The API answers any origin (CORS *): a web page must not be able to use + # this endpoint to probe the user's network. + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_tags("qwen2.5:3b")) + + for url in ( + "http://192.168.1.20:11434/v1", + "http://example.com/v1", + "http://user:pass@localhost:11434/v1", + "file:///etc/passwd", + "ftp://localhost/v1", + "http://localhost.evil.com:11434", + ): + with self.assertRaises(HTTPException, msg=url) as ctx: + _check(handler, base_url=url) + self.assertEqual(ctx.exception.status_code, 400, url) + self.assertEqual(requests, []) + + def test_accepts_loopback_hosts(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=_tags("qwen2.5:3b")) + + for url in ("http://localhost:11434/v1", "http://127.0.0.1:11434", "http://[::1]:11434/v1"): + result, _ = _check(handler, base_url=url) + self.assertEqual(result["status"], "ok", url) def test_model_missing_when_not_pulled(self) -> None: def handler(request: httpx.Request) -> httpx.Response: @@ -98,13 +129,13 @@ def handler(request: httpx.Request) -> httpx.Response: def test_unreachable_on_connection_error(self) -> None: def handler(request: httpx.Request) -> httpx.Response: - raise httpx.ConnectError("connection refused", request=request) + raise httpx.ConnectError("[Errno 111] internal detail", request=request) result, _ = _check(handler) self.assertEqual(result["status"], "unreachable") self.assertEqual(result["base_url"], "http://localhost:11434") - self.assertIn("connection refused", result["detail"]) + self.assertEqual(result["detail"], "connection refused") def test_unreachable_on_timeout(self) -> None: def handler(request: httpx.Request) -> httpx.Response: @@ -116,12 +147,13 @@ def handler(request: httpx.Request) -> httpx.Response: def test_unreachable_when_something_else_answers(self) -> None: def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(404, text="not found") + return httpx.Response(404, text="secret upstream body") result, _ = _check(handler) self.assertEqual(result["status"], "unreachable") - self.assertIn("404", result["detail"]) + # Generic: nothing of the upstream answer is echoed back to the caller. + self.assertEqual(result["detail"], "not an Ollama server") def test_unreachable_on_non_json_body(self) -> None: def handler(request: httpx.Request) -> httpx.Response: From 129f4c3ca7ef2b9650d8dca2bd0e0cc0ab7ec2be Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Wed, 11 Nov 2026 11:06:31 +0100 Subject: [PATCH 026/108] feat(notifications): notify on failed workflow branches and name the failed run Generation and full workflow runs already raised an OS notification on error, but a Wait branch started with Continue that failed stayed silent, and every error toast dumped the raw String(err) with no context. - Extract buildRunErrorNotification(): no toast for a user cancellation, otherwise ": ", generic "Error: " prefix stripped, truncated to 120 chars - Notify when a workflow branch (continueRun) fails - Skip the generation error toast if the user cancelled while the last status poll was in flight - Same focus gating and click-to-focus behaviour as completion toasts Closes #293 --- package.json | 2 +- src/areas/workflows/workflowRunStore.ts | 5 +- src/areas/workflows/workflowSceneRun.test.mjs | 2 +- src/shared/hooks/useGeneration.ts | 10 ++-- src/shared/utils/notification.test.ts | 59 +++++++++++++++++++ src/shared/utils/notification.ts | 32 +++++++++- 6 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 src/shared/utils/notification.test.ts diff --git a/package.json b/package.json index 315d90d1b..a636bdd51 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/workflows/workflowRunStore.ts b/src/areas/workflows/workflowRunStore.ts index 1c64c0c58..383ce0c69 100644 --- a/src/areas/workflows/workflowRunStore.ts +++ b/src/areas/workflows/workflowRunStore.ts @@ -3,7 +3,7 @@ import axios, { AxiosInstance } from 'axios' import { useAppStore } from '@shared/stores/appStore' import { getWorkflowExtension } from './mockExtensions' import { hasUnsupportedSceneShape } from './sceneShape' -import { showCompletionNotification, showErrorNotification } from '@shared/utils/notification' +import { showCompletionNotification, showRunErrorNotification } from '@shared/utils/notification' import type { WorkflowExtension } from './mockExtensions' import type { Workflow, WFNode, WFEdge } from '@shared/types/electron.d' import { isBranchStarter, isSceneOutput, resolveDataSource, reachesSceneOutput, nearestUpstreamWaits } from './nodeBehaviors' @@ -973,7 +973,7 @@ export const useWorkflowRunStore = create((set, get) => { if (!_cancel.current) { set((s) => ({ runState: { ...s.runState, status: 'error', error: String(err) }, activeNodeId: null })) useAppStore.getState().updateCurrentJob({ status: 'error', error: String(err) }) - void showErrorNotification(String(err), 'Workflow run failed') + void showRunErrorNotification({ title: 'Workflow run failed', name: workflow.name, error: err }) } } }, @@ -1032,6 +1032,7 @@ export const useWorkflowRunStore = create((set, get) => { for (const d of descendantWaits(waitId, ctx)) { if (newWaitStates[d] === 'blocked') newWaitStates[d] = 'error' } + void showRunErrorNotification({ title: 'Workflow branch failed', name: ctx.workflow.name, error: err }) } const allFinished = ctx.waitIds.every((id) => newWaitStates[id] === 'done' || newWaitStates[id] === 'error') const anyError = ctx.waitIds.some((id) => newWaitStates[id] === 'error') diff --git a/src/areas/workflows/workflowSceneRun.test.mjs b/src/areas/workflows/workflowSceneRun.test.mjs index cb43b9d55..86600a841 100644 --- a/src/areas/workflows/workflowSceneRun.test.mjs +++ b/src/areas/workflows/workflowSceneRun.test.mjs @@ -17,7 +17,7 @@ useAppStore.getState = () => appState `) const axiosStub = stub('axios.ts', `const axios: any = { create: () => (globalThis as any).__sceneClient }; export default axios; export type AxiosInstance = any`) const extStub = stub('ext.ts', `export const getWorkflowExtension = (id: string, all: any[]) => all.find((value) => value.id === id); export type WorkflowExtension = any`) -const notifyStub = stub('notify.ts', `export const showCompletionNotification = async () => {}; export const showErrorNotification = async () => {}`) +const notifyStub = stub('notify.ts', `export const showCompletionNotification = async () => {}; export const showRunErrorNotification = async () => {}`) const aliases = new Map([ ['axios', axiosStub], ['@shared/stores/appStore', appStoreStub], ['./mockExtensions', extStub], ['@shared/utils/notification', notifyStub], diff --git a/src/shared/hooks/useGeneration.ts b/src/shared/hooks/useGeneration.ts index 6f3d3a375..a2b0f9cf3 100644 --- a/src/shared/hooks/useGeneration.ts +++ b/src/shared/hooks/useGeneration.ts @@ -1,7 +1,7 @@ import { useCallback, useRef } from 'react' import { useAppStore } from '@shared/stores/appStore' import { useApi } from './useApi' -import { showCompletionNotification, showErrorNotification } from '@shared/utils/notification' +import { showCompletionNotification, showRunErrorNotification } from '@shared/utils/notification' export function useGeneration() { const { currentJob, setCurrentJob, updateCurrentJob, generationOptions, selectedImageData, pushMeshUrl, clearMeshHistory } = useAppStore() @@ -36,7 +36,7 @@ export function useGeneration() { updateCurrentJob({ status: 'generating', progress: 0 }) - await pollUntilDone(jobId) + await pollUntilDone(jobId, job.modelId) } catch (err) { if (cancelledRef.current) { setCurrentJob(null) @@ -53,14 +53,14 @@ export function useGeneration() { status: 'error', error: errorMessage }) - void showErrorNotification(errorMessage, 'Generation failed') + void showRunErrorNotification({ title: 'Generation failed', name: job.modelId, error: errorMessage }) } }, // eslint-disable-next-line react-hooks/exhaustive-deps -- useApi re-creates its fns each render, so this re-memoizes anyway (values stay fresh) [generateFromImage, pollJobStatus, cancelJob, setCurrentJob, updateCurrentJob] ) - const pollUntilDone = async (jobId: string) => { + const pollUntilDone = async (jobId: string, modelId: string) => { while (true) { await new Promise((r) => setTimeout(r, 1000)) @@ -86,7 +86,7 @@ export function useGeneration() { if (result.status === 'error') { updateCurrentJob({ status: 'error', error: result.error }) - void showErrorNotification(result.error ?? 'Unknown error', 'Generation failed') + void showRunErrorNotification({ title: 'Generation failed', name: modelId, error: result.error, cancelled: cancelledRef.current }) break } diff --git a/src/shared/utils/notification.test.ts b/src/shared/utils/notification.test.ts new file mode 100644 index 000000000..adb954d50 --- /dev/null +++ b/src/shared/utils/notification.test.ts @@ -0,0 +1,59 @@ +import test from 'node:test' +import assert from 'node:assert/strict' + +import { buildRunErrorNotification } from './notification.ts' + +test('a cancelled run never notifies', () => { + assert.equal( + buildRunErrorNotification({ title: 'Generation failed', error: 'aborted', cancelled: true }), + null, + ) +}) + +test('body is the run name followed by the error message', () => { + assert.deepEqual( + buildRunErrorNotification({ title: 'Workflow run failed', name: 'Bake textures', error: 'CUDA out of memory' }), + { title: 'Workflow run failed', body: 'Bake textures: CUDA out of memory' }, + ) +}) + +test('without a name the body is just the error message', () => { + assert.deepEqual( + buildRunErrorNotification({ title: 'Generation failed', error: 'boom' }), + { title: 'Generation failed', body: 'boom' }, + ) +}) + +test('keeps only the first non-empty line of a multi-line error (e.g. a traceback)', () => { + const n = buildRunErrorNotification({ + title: 'Generation failed', + error: '\n RuntimeError: weights missing \nTraceback (most recent call last):\n File "x.py"', + }) + assert.equal(n?.body, 'RuntimeError: weights missing') +}) + +test('strips the generic "Error: " prefix String(err) adds, but keeps specific error types', () => { + assert.equal( + buildRunErrorNotification({ title: 't', error: new Error('Node failed') })?.body, + 'Node failed', + ) + assert.equal( + buildRunErrorNotification({ title: 't', error: String(new Error('Error: nested')) })?.body, + 'nested', + ) + assert.equal( + buildRunErrorNotification({ title: 't', error: String(new TypeError('bad input')) })?.body, + 'TypeError: bad input', + ) +}) + +test('truncates long messages with an ellipsis', () => { + const body = buildRunErrorNotification({ title: 't', error: 'x'.repeat(500) })?.body ?? '' + assert.equal(body.length, 120) + assert.ok(body.endsWith('…')) +}) + +test('falls back to "Unknown error" for an empty or missing error', () => { + assert.equal(buildRunErrorNotification({ title: 't', error: undefined })?.body, 'Unknown error') + assert.equal(buildRunErrorNotification({ title: 't', name: 'triposr', error: ' ' })?.body, 'triposr: Unknown error') +}) diff --git a/src/shared/utils/notification.ts b/src/shared/utils/notification.ts index e51799740..0338dd66c 100644 --- a/src/shared/utils/notification.ts +++ b/src/shared/utils/notification.ts @@ -20,6 +20,34 @@ export function showCompletionNotification(body: string, title = 'Modly'): Promi return notifyIfUnfocused(title, body) } -export function showErrorNotification(body: string, title = 'Modly'): Promise { - return notifyIfUnfocused(title, body) +const MAX_ERROR_BODY = 120 + +export interface RunErrorInput { + /** Notification title, e.g. "Generation failed". */ + title: string + /** What failed — the model id or workflow name — prefixed to the message. */ + name?: string + error: unknown + /** The user stopped the run themselves: nothing to tell them. */ + cancelled?: boolean +} + +/** + * Decide whether a failed run deserves a notification and with what text: none + * for a user cancellation, otherwise the run's name plus the first line of the + * error (tracebacks and `Error: ` prefixes trimmed), short enough for a toast. + */ +export function buildRunErrorNotification(input: RunErrorInput): { title: string; body: string } | null { + if (input.cancelled) return null + const raw = input.error instanceof Error ? input.error.message : String(input.error ?? '') + const firstLine = raw.split(/\r?\n/).map((l) => l.trim()).find(Boolean) ?? '' + const message = firstLine.replace(/^(Error:\s*)+/, '') || 'Unknown error' + const full = input.name ? `${input.name}: ${message}` : message + const body = full.length > MAX_ERROR_BODY ? `${full.slice(0, MAX_ERROR_BODY - 1)}…` : full + return { title: input.title, body } +} + +export function showRunErrorNotification(input: RunErrorInput): Promise { + const notification = buildRunErrorNotification(input) + return notification ? notifyIfUnfocused(notification.title, notification.body) : Promise.resolve() } From 6d3f28d6c4ad6170f83036202dc419e2b66a022c Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Wed, 11 Nov 2026 11:07:11 +0100 Subject: [PATCH 027/108] feat(generate): multi-level mesh undo with a bounded per-model stack Mesh edits never overwrite their input (every op writes a new workspace file), so undo only needs the chain of URLs. Move it into a pure meshHistory module and route every edit through one store action: - applyMeshEdit(url) records the edit from the mesh currently shown, seeding it as the baseline when it was never recorded (workflow outputs, Add to Scene), so the first edit is always undoable. - The stack is capped at 20 undo steps; a new edit drops the redo branch. - setCurrentJob resets the stack, so undo never jumps to another model (import, library open, Add to Scene and workflow runs used to keep it). - Smooth, decimate and agent mesh tools all go through applyMeshEdit. - The undo button shows the number of available steps. Closes #304 --- package.json | 2 +- src/areas/generate/GeneratePage.tsx | 30 ++-- src/areas/generate/components/ChatPanel.tsx | 4 +- src/shared/stores/appStore.ts | 60 +++++--- src/shared/stores/meshHistory.test.ts | 154 ++++++++++++++++++++ src/shared/stores/meshHistory.ts | 61 ++++++++ 6 files changed, 274 insertions(+), 37 deletions(-) create mode 100644 src/shared/stores/meshHistory.test.ts create mode 100644 src/shared/stores/meshHistory.ts diff --git a/package.json b/package.json index a636bdd51..da6236ad2 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/GeneratePage.tsx b/src/areas/generate/GeneratePage.tsx index 76a131199..846392396 100644 --- a/src/areas/generate/GeneratePage.tsx +++ b/src/areas/generate/GeneratePage.tsx @@ -2,6 +2,7 @@ import { useState, useRef, useCallback, useEffect, useMemo } from 'react' import type { ReactNode } from 'react' import { useAppStore, DEFAULT_LIGHT_SETTINGS } from '@shared/stores/appStore' import type { GenerationJob, LightSettings, PointLight } from '@shared/stores/appStore' +import { redoDepth, undoDepth } from '@shared/stores/meshHistory' import { useApi } from '@shared/hooks/useApi' import { ColorPicker } from '@shared/components/ui' import GenerationHUD from './components/GenerationHUD' @@ -691,15 +692,15 @@ export default function GeneratePage(): JSX.Element { const currentJob = useAppStore((s) => s.currentJob) const apiUrl = useAppStore((s) => s.apiUrl) const showError = useAppStore((s) => s.showError) - const updateCurrentJob = useAppStore((s) => s.updateCurrentJob) const setCurrentJob = useAppStore((s) => s.setCurrentJob) const meshStats = useAppStore((s) => s.meshStats) const meshSelected = useAppStore((s) => s.meshSelected) const pushMeshUrl = useAppStore((s) => s.pushMeshUrl) const undoMesh = useAppStore((s) => s.undoMesh) const redoMesh = useAppStore((s) => s.redoMesh) - const canUndo = useAppStore((s) => s.historyIndex > 0) - const canRedo = useAppStore((s) => s.historyIndex < s.meshHistory.length - 1) + const applyMeshEdit = useAppStore((s) => s.applyMeshEdit) + const undoSteps = useAppStore((s) => undoDepth(s.meshHistory)) + const redoSteps = useAppStore((s) => redoDepth(s.meshHistory)) const { optimizeMesh, smoothMesh, importMesh } = useApi() const assetLibraryService = useMemo(() => getDefaultAssetLibraryService(), []) @@ -894,8 +895,7 @@ export default function GeneratePage(): JSX.Element { try { const path = getOptimizePath(currentJob.outputUrl) const { url } = await smoothMesh(path, iterations) - updateCurrentJob({ outputUrl: url }) - pushMeshUrl(url) + applyMeshEdit(url) setOpenPanel(null) } catch (err) { showError(err instanceof Error ? err.message : String(err)) @@ -910,8 +910,7 @@ export default function GeneratePage(): JSX.Element { try { const path = getOptimizePath(currentJob.outputUrl) const { url } = await optimizeMesh(path, targetFaces) - updateCurrentJob({ outputUrl: url }) - pushMeshUrl(url) + applyMeshEdit(url) setOpenPanel(null) } catch (err) { showError(err instanceof Error ? err.message : String(err)) @@ -971,20 +970,25 @@ export default function GeneratePage(): JSX.Element { {/* Undo / Redo */}
- {/* Viewer area */} -
- - +
+ {/* Viewer area */} +
+ + +
+ {editPanelOpen && setEditPanelOpen(false)} />}
diff --git a/src/areas/generate/components/EditPanel.tsx b/src/areas/generate/components/EditPanel.tsx new file mode 100644 index 000000000..3923d60e3 --- /dev/null +++ b/src/areas/generate/components/EditPanel.tsx @@ -0,0 +1,636 @@ +import { useCallback, useEffect, useState } from 'react' +import { useAppStore } from '@shared/stores/appStore' +import { useApi } from '@shared/hooks/useApi' +import { Tooltip } from '@shared/components/ui' +import { + INSPECT_OP_ID, + LOG_SLIDER_STEPS, + apiErrorMessage, + coerceNumber, + defaultParamValues, + fromSliderPosition, + groupMeshOps, + inspectionBadges, + isMeshUpdate, + meshPathFromUrl, + paramControl, + requiresConfirmation, + summarizeDetails, + toSliderPosition, + type BadgeState, + type MeshOpDescriptor, + type MeshOpResponse, + type ParamControl, +} from '../meshEditPanel' + +// Side panel listing every mesh-ops registry op (the same ones the chat agent +// runs), with controls generated from each op's params_schema. Ops that write +// a mesh go through applyMeshEdit, so they land on the undo stack exactly like +// an agent edit; analysis ops show their report inline. + +type Params = Record + +interface OpResult { + /** Mesh URL the op ran on. */ + sourceUrl: string + response?: MeshOpResponse + error?: string +} + +interface Fix { + op: string + params?: Params + reason?: string +} + +function Spinner({ size = 11 }: { size?: number }): JSX.Element { + return ( + + + + ) +} + +function Chevron({ open }: { open: boolean }): JSX.Element { + return ( + + + + ) +} + +// ─── Param controls ─────────────────────────────────────────────────────────── + +const INPUT_CLASS = + 'bg-zinc-800 border border-zinc-700 rounded-lg px-2 py-1 text-xs text-zinc-200 focus:outline-none focus:border-violet-500 transition-colors disabled:opacity-50' + +/** Number text field that commits (clamped) on blur or Enter. */ +function NumberField({ control, value, onChange, disabled }: { + control: Extract + value: number + onChange: (v: number) => void + disabled: boolean +}): JSX.Element { + const [text, setText] = useState(String(value)) + useEffect(() => { setText(String(value)) }, [value]) + + function commit() { + const parsed = Number(text.replace(',', '.')) + if (text.trim() === '' || !Number.isFinite(parsed)) { setText(String(value)); return } + const next = coerceNumber(control, parsed) + setText(String(next)) + if (next !== value) onChange(next) + } + + return ( + setText(e.target.value)} + onBlur={commit} + onKeyDown={(e) => { if (e.key === 'Enter') commit() }} + className={`${INPUT_CLASS} w-20 shrink-0 text-right tabular-nums`} + /> + ) +} + +function ParamField({ control, value, onChange, disabled }: { + control: ParamControl + value: unknown + onChange: (v: unknown) => void + disabled: boolean +}): JSX.Element { + const label = ( +
+ {control.label} + {control.tooltip && ( + + ⓘ + + )} +
+ ) + + switch (control.kind) { + case 'toggle': { + const on = value === true + return ( +
+ {label} + +
+ ) + } + case 'select': + return ( +
+ {label} + +
+ ) + case 'slider': { + const numeric = typeof value === 'number' ? value : control.defaultValue + const log = control.scale === 'log' + return ( +
+ {label} +
+ onChange(fromSliderPosition(control, e.currentTarget.valueAsNumber))} + className="min-w-0 flex-1 accent-violet-500 cursor-pointer disabled:cursor-default" + /> + +
+
+ ) + } + case 'number': + return ( +
+ {label} + +
+ ) + case 'text': + return ( +
+ {label} + onChange(e.target.value)} + className={`${INPUT_CLASS} w-full`} + /> +
+ ) + } +} + +function ParamFields({ op, values, onChange, disabled }: { + op: MeshOpDescriptor + values: Params + onChange: (id: string, value: unknown) => void + disabled: boolean +}): JSX.Element | null { + if (op.params_schema.length === 0) return null + return ( +
+ {op.params_schema.map((schema) => { + const control = paramControl(schema) + return ( + onChange(schema.id, v)} + disabled={disabled} + /> + ) + })} +
+ ) +} + +// ─── Reports ────────────────────────────────────────────────────────────────── + +const BADGE_CLASS: Record = { + ok: 'bg-emerald-500/10 border-emerald-500/30 text-emerald-300', + bad: 'bg-red-500/10 border-red-500/30 text-red-300', + unknown: 'bg-zinc-800 border-zinc-700 text-zinc-400', +} + +const BADGE_MARK: Record = { ok: '✓', bad: '✗', unknown: '?' } + +function fmt(value: unknown, digits = 4): string { + if (typeof value !== 'number') return '—' + return Number.isInteger(value) ? value.toLocaleString('en-US') : Number(value.toPrecision(digits)).toString() +} + +function InspectionReport({ report }: { report: Record }): JSX.Element { + const dims = Array.isArray(report.dimensions) ? report.dimensions.map((d) => fmt(d)).join(' × ') : '—' + const selfHits = report.self_intersecting_face_count + const stats: [string, string][] = [ + ['Faces', fmt(report.face_count)], + ['Vertices', fmt(report.vertex_count)], + ['Parts', fmt(report.component_count)], + ['Holes', `${fmt(report.hole_count)}${report.boundary_edge_count ? ` (${fmt(report.boundary_edge_count)} open edges)` : ''}`], + ['Non-manifold edges', fmt(report.non_manifold_edge_count)], + ['Degenerate faces', fmt(report.degenerate_face_count)], + ['Self-intersecting faces', selfHits === null ? 'not checked' : fmt(selfHits)], + ['Dimensions', dims], + ['Volume', report.volume == null ? 'n/a (not closed)' : fmt(report.volume)], + ['UVs', report.has_uvs === true ? 'yes' : report.has_uvs === false ? 'no' : 'unknown'], + ['Normals', report.has_normals === true ? 'yes' : report.has_normals === false ? 'no' : 'unknown'], + ] + const issues = Array.isArray(report.issues) ? (report.issues as unknown[]).map(String) : [] + + return ( +
+
+ {inspectionBadges(report).map((badge) => ( + + {BADGE_MARK[badge.state]} {badge.label} + + ))} +
+
+ {stats.map(([k, v]) => ( +
+
{k}
+
{v}
+
+ ))} +
+ {issues.length > 0 && ( +
    + {issues.map((issue) =>
  • • {issue}
  • )} +
+ )} +
+ ) +} + +function DetailRows({ details }: { details: Record }): JSX.Element | null { + const rows = summarizeDetails(details) + if (rows.length === 0) return null + return ( +
+ {rows.map(([k, v]) => ( +
+
{k}
+
{v}
+
+ ))} +
+ ) +} + +// ─── Panel ──────────────────────────────────────────────────────────────────── + +export default function EditPanel({ onClose }: { onClose: () => void }): JSX.Element { + const apiUrl = useAppStore((s) => s.apiUrl) + const currentJob = useAppStore((s) => s.currentJob) + const applyMeshEdit = useAppStore((s) => s.applyMeshEdit) + const { listMeshOps, runMeshOp } = useApi() + + const [ops, setOps] = useState(null) + const [loadError, setLoadError] = useState(null) + const [reloadKey, setReloadKey] = useState(0) + const [params, setParams] = useState>({}) + const [expanded, setExpanded] = useState(null) + // Op awaiting in-panel confirmation, with the params it will run with. + const [confirming, setConfirming] = useState<{ opId: string; values: Params } | null>(null) + const [running, setRunning] = useState(null) + const [results, setResults] = useState>({}) + const [inspection, setInspection] = useState<{ sourceUrl: string; report: Record } | null>(null) + + const isGenerating = currentJob?.status === 'uploading' || currentJob?.status === 'generating' + const meshUrl = currentJob?.status === 'done' ? currentJob.outputUrl : undefined + const canRun = !!meshUrl && !isGenerating && running === null + + useEffect(() => { + let cancelled = false + setLoadError(null) + listMeshOps() + .then((list) => { if (!cancelled) setOps(list) }) + .catch((err) => { if (!cancelled) setLoadError(apiErrorMessage(err)) }) + return () => { cancelled = true } + // eslint-disable-next-line react-hooks/exhaustive-deps -- listMeshOps is rebuilt every render; refetch on API change or retry only + }, [apiUrl, reloadKey]) + + // Results and reports belong to one model: drop them when another is loaded. + const jobId = currentJob?.id + useEffect(() => { + setResults({}) + setInspection(null) + setConfirming(null) + }, [jobId]) + + const opById = useCallback((id: string) => ops?.find((o) => o.id === id), [ops]) + + function valuesFor(op: MeshOpDescriptor): Params { + return params[op.id] ?? defaultParamValues(op.params_schema) + } + + function setParam(op: MeshOpDescriptor, id: string, value: unknown) { + setParams((current) => ({ ...current, [op.id]: { ...(current[op.id] ?? defaultParamValues(op.params_schema)), [id]: value } })) + } + + async function run(op: MeshOpDescriptor, values: Params) { + if (!meshUrl || running !== null) return + const sourceUrl = meshUrl + setConfirming(null) + setRunning(op.id) + setResults((current) => { const next = { ...current }; delete next[op.id]; return next }) + try { + const response = await runMeshOp(op.id, meshPathFromUrl(sourceUrl), values) + // The viewer may have moved on (undo, another model) while the op ran: + // applying would silently drop that change, so discard the result. + if (useAppStore.getState().currentJob?.outputUrl !== sourceUrl) { + setResults((current) => ({ ...current, [op.id]: { sourceUrl, error: 'The mesh changed while this ran; the result was discarded.' } })) + return + } + if (isMeshUpdate(response)) applyMeshEdit(response.url) + if (op.id === INSPECT_OP_ID) setInspection({ sourceUrl, report: response }) + else if (response.report && typeof response.report === 'object') { + // suggest_fixes carries a full inspection report too. + setInspection({ sourceUrl, report: response.report as Record }) + } + setResults((current) => ({ ...current, [op.id]: { sourceUrl, response } })) + } catch (err) { + setResults((current) => ({ ...current, [op.id]: { sourceUrl, error: apiErrorMessage(err) } })) + } finally { + setRunning(null) + } + } + + function requestRun(op: MeshOpDescriptor, values: Params = valuesFor(op)) { + if (requiresConfirmation(op)) { setConfirming({ opId: op.id, values }); return } + void run(op, values) + } + + const inspectOp = ops?.find((o) => o.id === INSPECT_OP_ID) + const groups = ops ? groupMeshOps(ops) : [] + + return ( + + ) +} + +function OpResultView({ response, stale, canRun, opById, onRunFix }: { + response: MeshOpResponse + stale: boolean + canRun: boolean + opById: (id: string) => MeshOpDescriptor | undefined + onRunFix: (op: MeshOpDescriptor, params: Params) => void +}): JSX.Element { + if (isMeshUpdate(response)) { + return ( +
+

Applied — undo with Ctrl+Z.

+ +
+ ) + } + + // suggest_fixes: { report, fixes, notes } — the report goes to Mesh health. + if (Array.isArray(response.fixes)) { + const fixes = response.fixes as Fix[] + const notes = Array.isArray(response.notes) ? (response.notes as unknown[]).map(String) : [] + return ( +
+ {stale &&

The mesh changed since these suggestions.

} + {fixes.length === 0 && notes.length === 0 && ( +

No fixes needed — no problems detected.

+ )} + {fixes.map((fix, i) => { + const fixOp = opById(fix.op) + return ( +
+
+

{fixOp?.label ?? fix.op}

+ {fix.reason &&

{fix.reason}

} +
+ {fixOp && ( + + )} +
+ ) + })} + {notes.map((note) =>

{note}

)} +
+ ) + } + + return ( +
+ {stale &&

The mesh changed since this ran.

} + +
+ ) +} diff --git a/src/areas/generate/meshEditPanel.test.ts b/src/areas/generate/meshEditPanel.test.ts new file mode 100644 index 000000000..b4c7f7bf1 --- /dev/null +++ b/src/areas/generate/meshEditPanel.test.ts @@ -0,0 +1,217 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + apiErrorMessage, + defaultParamValues, + fromSliderPosition, + groupMeshOps, + inspectionBadges, + isMeshUpdate, + meshPathFromUrl, + paramControl, + requiresConfirmation, + summarizeDetails, + toSliderPosition, + type MeshOpDescriptor, +} from './meshEditPanel.ts' + +function op(id: string, category: string, extra: Partial = {}): MeshOpDescriptor { + return { id, label: id, params_schema: [], destructive: false, undoable: true, category, ...extra } +} + +// ─── Schema → control ──────────────────────────────────────────────────────── + +test('a bounded int becomes an integer slider with its default', () => { + const control = paramControl({ id: 'iterations', label: 'Iterations', type: 'int', default: 5, min: 1, max: 50 }) + assert.equal(control.kind, 'slider') + assert.ok(control.kind === 'slider') + assert.equal(control.integer, true) + assert.equal(control.step, 1) + assert.equal(control.min, 1) + assert.equal(control.max, 50) + assert.equal(control.scale, 'linear') + assert.equal(control.defaultValue, 5) +}) + +test('a bounded float keeps its declared step', () => { + const control = paramControl({ id: 'lambda_', label: 'Strength', type: 'float', default: 0.5, min: 0.1, max: 1, step: 0.05 }) + assert.ok(control.kind === 'slider') + assert.equal(control.integer, false) + assert.equal(control.step, 0.05) +}) + +test('a range spanning three orders of magnitude uses a log slider', () => { + const faces = paramControl({ id: 'target_faces', type: 'int', default: 10000, min: 100, max: 1000000 }) + assert.ok(faces.kind === 'slider') + assert.equal(faces.scale, 'log') + const size = paramControl({ id: 'size', type: 'float', default: 1, min: 0.001, max: 100000, step: 0.1 }) + assert.ok(size.kind === 'slider') + assert.equal(size.scale, 'log') +}) + +test('a numeric param missing a bound is a plain number input', () => { + const control = paramControl({ id: 'n', type: 'int', default: 3, min: 0 }) + assert.equal(control.kind, 'number') + assert.ok(control.kind === 'number') + assert.equal(control.min, 0) + assert.equal(control.max, undefined) +}) + +test('boolean, select and string params map to toggle, dropdown and text', () => { + assert.equal(paramControl({ id: 'b', type: 'boolean', default: true }).kind, 'toggle') + const select = paramControl({ + id: 'mode', type: 'select', default: 'taubin', + options: [{ value: 'taubin', label: 'Taubin' }, { value: 'laplacian', label: 'Laplacian' }], + }) + assert.ok(select.kind === 'select') + assert.deepEqual(select.options.map((o) => o.value), ['taubin', 'laplacian']) + assert.equal(paramControl({ id: 's', type: 'string', default: 'x' }).kind, 'text') +}) + +test('the label falls back to a humanized id', () => { + assert.equal(paramControl({ id: 'max_hole_size', type: 'int', default: 1 }).label, 'Max hole size') +}) + +test('defaults fall back per type when the schema omits them', () => { + assert.equal(paramControl({ id: 'b', type: 'boolean' }).defaultValue, false) + assert.equal(paramControl({ id: 'n', type: 'int', min: 4, max: 9 }).defaultValue, 4) + assert.equal( + paramControl({ id: 'm', type: 'select', options: [{ value: 'a', label: 'A' }] }).defaultValue, + 'a', + ) +}) + +test('defaultParamValues collects every default by id', () => { + assert.deepEqual( + defaultParamValues([ + { id: 'iterations', type: 'int', default: 5, min: 1, max: 50 }, + { id: 'mode', type: 'select', default: 'taubin', options: [{ value: 'taubin', label: 'T' }] }, + { id: 'fill', type: 'boolean', default: true }, + ]), + { iterations: 5, mode: 'taubin', fill: true }, + ) +}) + +// ─── Slider scale ──────────────────────────────────────────────────────────── + +test('linear slider positions map straight through', () => { + const control = paramControl({ id: 'i', type: 'int', default: 5, min: 1, max: 50 }) + assert.ok(control.kind === 'slider') + assert.equal(toSliderPosition(control, 20), 20) + assert.equal(fromSliderPosition(control, 20), 20) +}) + +test('log slider positions round-trip and stay within bounds', () => { + const control = paramControl({ id: 'target_faces', type: 'int', default: 10000, min: 100, max: 1000000 }) + assert.ok(control.kind === 'slider') + assert.equal(fromSliderPosition(control, toSliderPosition(control, 100)), 100) + assert.equal(fromSliderPosition(control, toSliderPosition(control, 1000000)), 1000000) + assert.equal(fromSliderPosition(control, toSliderPosition(control, 10000)), 10000) + // Integers stay integers. + assert.ok(Number.isInteger(fromSliderPosition(control, 0.37))) +}) + +// ─── Grouping ──────────────────────────────────────────────────────────────── + +test('ops are grouped in the canonical category order with labels', () => { + const groups = groupMeshOps([ + op('decimate', 'topology'), + op('prepare_for_printing', 'presets'), + op('recenter', 'transform'), + op('repair', 'repair'), + ]) + assert.deepEqual(groups.map((g) => g.category), ['repair', 'transform', 'topology', 'presets']) + assert.deepEqual(groups.map((g) => g.label), ['Repair', 'Transform', 'Topology', 'Presets']) +}) + +test('ops keep registry order within a group', () => { + const groups = groupMeshOps([op('repair', 'repair'), op('recenter', 'transform'), op('repair_mesh', 'repair')]) + assert.deepEqual(groups[0].ops.map((o) => o.id), ['repair', 'repair_mesh']) +}) + +test('unknown categories still appear, after the known ones, alphabetically', () => { + const groups = groupMeshOps([ + op('z', 'zeta_tools'), + op('a', 'alpha_tools'), + op('repair', 'repair'), + ]) + assert.deepEqual(groups.map((g) => g.category), ['repair', 'alpha_tools', 'zeta_tools']) + assert.equal(groups[1].label, 'Alpha tools') +}) + +test('inspect_mesh is left out of the groups: it has its own report panel', () => { + const groups = groupMeshOps([op('inspect_mesh', 'analysis'), op('suggest_fixes', 'analysis')]) + assert.deepEqual(groups.map((g) => g.ops.map((o) => o.id)), [['suggest_fixes']]) +}) + +test('the registry optimization category sits next to topology as Optimize', () => { + const groups = groupMeshOps([op('smooth', 'optimization'), op('remesh', 'topology'), op('repair', 'repair')]) + assert.deepEqual(groups.map((g) => g.label), ['Repair', 'Topology', 'Optimize']) +}) + +test('an op without a category falls into Other', () => { + const groups = groupMeshOps([{ ...op('x', ''), category: undefined as unknown as string }]) + assert.equal(groups[0].label, 'Other') +}) + +// ─── Confirmation / results ────────────────────────────────────────────────── + +test('destructive or non-undoable ops need an in-panel confirmation', () => { + assert.equal(requiresConfirmation(op('a', 'repair')), false) + assert.equal(requiresConfirmation(op('a', 'repair', { destructive: true })), true) + assert.equal(requiresConfirmation(op('a', 'repair', { undoable: false })), true) +}) + +test('a response with a url is a mesh update; a report is not', () => { + assert.equal(isMeshUpdate({ url: '/workspace/a.glb', face_count: 3 }), true) + assert.equal(isMeshUpdate({ is_watertight: true }), false) + assert.equal(isMeshUpdate({ path: 'C:/outside.glb' }), false) +}) + +test('inspection badges read the report flags, null meaning unknown', () => { + const badges = inspectionBadges({ + is_watertight: true, + is_manifold: false, + is_winding_consistent: true, + is_inside_out: false, + printable: null, + }) + const byLabel = Object.fromEntries(badges.map((b) => [b.label, b.state])) + assert.equal(byLabel.Watertight, 'ok') + assert.equal(byLabel.Manifold, 'bad') + assert.equal(byLabel['Consistent winding'], 'ok') + // Inside-out is a problem flag: false is good. + assert.equal(byLabel['Normals outward'], 'ok') + assert.equal(byLabel.Printable, 'unknown') +}) + +test('summarizeDetails lists scalar fields and hides paths', () => { + const rows = summarizeDetails({ + url: '/workspace/a.glb', + path: 'a.glb', + face_count: 12000, + repaired: true, + steps: ['fill holes', 'fix normals'], + nested: { a: 1 }, + }) + assert.deepEqual(rows, [ + ['Face count', '12,000'], + ['Repaired', 'yes'], + ['Steps', 'fill holes, fix normals'], + ]) +}) + +// ─── Misc ──────────────────────────────────────────────────────────────────── + +test('meshPathFromUrl strips the workspace prefix and decodes serve-file urls', () => { + assert.equal(meshPathFromUrl('/workspace/col/a.glb'), 'col/a.glb') + assert.equal(meshPathFromUrl('/optimize/serve-file?path=C%3A%2Fm%2Fa.glb'), 'C:/m/a.glb') + assert.equal(meshPathFromUrl('col/a.glb'), 'col/a.glb') +}) + +test('apiErrorMessage prefers the FastAPI detail', () => { + assert.equal(apiErrorMessage({ response: { data: { detail: 'Unknown mesh operation: x' } }, message: 'Request failed' }), 'Unknown mesh operation: x') + assert.equal(apiErrorMessage(new Error('Network Error')), 'Network Error') + assert.equal(apiErrorMessage('boom'), 'boom') +}) diff --git a/src/areas/generate/meshEditPanel.ts b/src/areas/generate/meshEditPanel.ts new file mode 100644 index 000000000..23226703f --- /dev/null +++ b/src/areas/generate/meshEditPanel.ts @@ -0,0 +1,279 @@ +// Pure logic behind the mesh edit panel: maps the mesh-ops registry +// (GET /optimize/ops) to UI controls and groups, and reads op results. +// Kept free of React so it can be unit-tested with node --test. + +/** One entry of an op's params_schema, as served by the backend registry. */ +export interface MeshOpParamSchema { + id: string + label?: string + type?: string + default?: unknown + min?: number + max?: number + step?: number + options?: { value: string | number; label: string }[] + tooltip?: string +} + +/** MeshOp.describe() on the backend. */ +export interface MeshOpDescriptor { + id: string + label: string + params_schema: MeshOpParamSchema[] + destructive: boolean + undoable: boolean + category: string +} + +/** POST /optimize/op/{id}: `url` when a mesh was written, otherwise the op's report. */ +export type MeshOpResponse = Record + +interface ControlBase { + id: string + label: string + tooltip?: string +} + +export type ParamControl = + | (ControlBase & { + kind: 'slider' + min: number + max: number + step: number + integer: boolean + /** 'log' for ranges spanning several orders of magnitude (face counts, sizes). */ + scale: 'linear' | 'log' + defaultValue: number + }) + | (ControlBase & { + kind: 'number' + min?: number + max?: number + step: number + integer: boolean + defaultValue: number + }) + | (ControlBase & { kind: 'toggle'; defaultValue: boolean }) + | (ControlBase & { + kind: 'select' + options: { value: string | number; label: string }[] + defaultValue: string | number + }) + | (ControlBase & { kind: 'text'; defaultValue: string }) + +/** A slider is switched to a log scale above this max/min ratio. */ +const LOG_SCALE_RATIO = 1000 +/** Resolution of a log slider's underlying . */ +export const LOG_SLIDER_STEPS = 1000 + +export function humanize(id: string): string { + const words = id.replace(/[_-]+/g, ' ').trim() + return words ? words.charAt(0).toUpperCase() + words.slice(1) : id +} + +export function paramControl(schema: MeshOpParamSchema): ParamControl { + const base: ControlBase = { + id: schema.id, + label: schema.label || humanize(schema.id), + tooltip: schema.tooltip, + } + const type = schema.type ?? (typeof schema.default === 'boolean' ? 'boolean' : 'float') + + if (type === 'boolean' || type === 'bool') { + return { ...base, kind: 'toggle', defaultValue: schema.default === true } + } + + if (type === 'select') { + const options = schema.options ?? [] + const fallback = options[0]?.value ?? '' + const defaultValue = (schema.default as string | number | undefined) ?? fallback + return { ...base, kind: 'select', options, defaultValue } + } + + if (type === 'int' || type === 'float' || type === 'number') { + const integer = type === 'int' + const step = schema.step ?? (integer ? 1 : 0.01) + const { min, max } = schema + const numericDefault = + typeof schema.default === 'number' ? schema.default : (min ?? 0) + if (min !== undefined && max !== undefined && max > min) { + const scale = min > 0 && max / min >= LOG_SCALE_RATIO ? 'log' : 'linear' + return { ...base, kind: 'slider', min, max, step, integer, scale, defaultValue: numericDefault } + } + return { ...base, kind: 'number', min, max, step, integer, defaultValue: numericDefault } + } + + return { ...base, kind: 'text', defaultValue: schema.default == null ? '' : String(schema.default) } +} + +export function defaultParamValues(schemas: MeshOpParamSchema[]): Record { + return Object.fromEntries(schemas.map((schema) => [schema.id, paramControl(schema).defaultValue])) +} + +// ─── Slider scale ──────────────────────────────────────────────────────────── + +type SliderControl = Extract + +function clamp(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, value)) +} + +/** Value → position of the . */ +export function toSliderPosition(control: SliderControl, value: number): number { + if (control.scale === 'linear') return clamp(value, control.min, control.max) + const v = clamp(value, control.min, control.max) + const ratio = Math.log(v / control.min) / Math.log(control.max / control.min) + return Math.round(ratio * LOG_SLIDER_STEPS) +} + +/** Position of the → value, rounded to something typeable. */ +export function fromSliderPosition(control: SliderControl, position: number): number { + if (control.scale === 'linear') return clamp(position, control.min, control.max) + if (position <= 0) return control.min + if (position >= LOG_SLIDER_STEPS) return control.max + const raw = control.min * Math.pow(control.max / control.min, position / LOG_SLIDER_STEPS) + // Three significant figures: 12,345 → 12,300, 0.004567 → 0.00457. + const magnitude = Math.pow(10, Math.floor(Math.log10(raw)) - 2) + const rounded = Math.round(raw / magnitude) * magnitude + const value = control.integer ? Math.round(rounded) : Number(rounded.toPrecision(3)) + return clamp(value, control.min, control.max) +} + +/** Clamp a typed number to the control's bounds (integers rounded). */ +export function coerceNumber(control: Extract, value: number): number { + let v = control.integer ? Math.round(value) : value + if (control.min !== undefined) v = Math.max(control.min, v) + if (control.max !== undefined) v = Math.min(control.max, v) + return v +} + +// ─── Grouping ──────────────────────────────────────────────────────────────── + +/** Known categories, in display order. Anything else follows, alphabetically. */ +const CATEGORY_ORDER = ['repair', 'transform', 'topology', 'optimization', 'presets', 'analysis'] + +const CATEGORY_LABELS: Record = { + repair: 'Repair', + transform: 'Transform', + topology: 'Topology', + optimization: 'Optimize', + presets: 'Presets', + analysis: 'Analysis', +} + +/** Op shown in the dedicated report panel rather than in a group. */ +export const INSPECT_OP_ID = 'inspect_mesh' + +export interface MeshOpGroup { + category: string + label: string + ops: MeshOpDescriptor[] +} + +export function categoryLabel(category: string): string { + return CATEGORY_LABELS[category] ?? (category ? humanize(category) : 'Other') +} + +export function groupMeshOps(ops: MeshOpDescriptor[]): MeshOpGroup[] { + const byCategory = new Map() + for (const op of ops) { + if (op.id === INSPECT_OP_ID) continue + const category = op.category || '' + const list = byCategory.get(category) ?? [] + list.push(op) + byCategory.set(category, list) + } + + const rank = (category: string): number => { + const index = CATEGORY_ORDER.indexOf(category) + return index === -1 ? CATEGORY_ORDER.length : index + } + + return [...byCategory.entries()] + .sort(([a], [b]) => rank(a) - rank(b) || a.localeCompare(b)) + .map(([category, list]) => ({ category, label: categoryLabel(category), ops: list })) +} + +// ─── Results ───────────────────────────────────────────────────────────────── + +/** Destructive or non-undoable ops are confirmed in the panel before running. */ +export function requiresConfirmation(op: MeshOpDescriptor): boolean { + return op.destructive || !op.undoable +} + +/** True when the op wrote a mesh that should replace the one in the viewer. */ +export function isMeshUpdate(response: MeshOpResponse): response is MeshOpResponse & { url: string } { + return typeof response.url === 'string' && response.url.length > 0 +} + +export type BadgeState = 'ok' | 'bad' | 'unknown' + +export interface InspectionBadge { + label: string + state: BadgeState +} + +function flag(value: unknown, invert = false): BadgeState { + if (typeof value !== 'boolean') return 'unknown' + return value !== invert ? 'ok' : 'bad' +} + +/** ✓/✗ badges of an inspect_mesh report. */ +export function inspectionBadges(report: Record): InspectionBadge[] { + return [ + { label: 'Printable', state: flag(report.printable) }, + { label: 'Watertight', state: flag(report.is_watertight) }, + { label: 'Manifold', state: flag(report.is_manifold) }, + { label: 'Consistent winding', state: flag(report.is_winding_consistent) }, + { label: 'Normals outward', state: flag(report.is_inside_out, true) }, + ] +} + +const HIDDEN_DETAIL_KEYS = new Set(['url', 'path']) + +function formatScalar(value: unknown): string | null { + if (typeof value === 'boolean') return value ? 'yes' : 'no' + if (typeof value === 'number') { + return Number.isInteger(value) ? value.toLocaleString('en-US') : value.toPrecision(4).replace(/\.?0+$/, '') + } + if (typeof value === 'string') return value + return null +} + +/** Label/value rows for the scalar (and scalar-list) fields of an op result. */ +export function summarizeDetails(details: Record): [string, string][] { + const rows: [string, string][] = [] + for (const [key, value] of Object.entries(details)) { + if (HIDDEN_DETAIL_KEYS.has(key) || value == null) continue + if (Array.isArray(value)) { + const parts = value.map(formatScalar) + if (parts.length && parts.every((p) => p !== null)) rows.push([humanize(key), parts.join(', ')]) + continue + } + const text = formatScalar(value) + if (text !== null) rows.push([humanize(key), text]) + } + return rows +} + +// ─── Misc ──────────────────────────────────────────────────────────────────── + +/** Viewer mesh URL → the `path` the optimize routes expect. */ +export function meshPathFromUrl(url: string): string { + if (url.startsWith('/workspace/')) return url.slice('/workspace/'.length) + if (url.startsWith('/optimize/serve-file?path=')) { + return decodeURIComponent(url.split('path=')[1] ?? '') + } + return url +} + +/** Readable message of a failed API call: FastAPI's `detail` when present. */ +export function apiErrorMessage(err: unknown): string { + if (err && typeof err === 'object') { + const detail = (err as { response?: { data?: { detail?: unknown } } }).response?.data?.detail + if (typeof detail === 'string' && detail) return detail + const message = (err as { message?: unknown }).message + if (typeof message === 'string' && message) return message + } + return String(err) +} diff --git a/src/shared/hooks/useApi.ts b/src/shared/hooks/useApi.ts index f013f1c11..d2c7104e1 100644 --- a/src/shared/hooks/useApi.ts +++ b/src/shared/hooks/useApi.ts @@ -1,5 +1,6 @@ import axios from 'axios' import { useAppStore, GenerationOptions } from '@shared/stores/appStore' +import type { MeshOpDescriptor, MeshOpResponse } from '@areas/generate/meshEditPanel' export function useApi() { const apiUrl = useAppStore((s) => s.apiUrl) @@ -129,5 +130,25 @@ export function useApi() { return { url: data.url } } - return { generateFromImage, pollJobStatus, cancelJob, getModelStatus, getAllModelsStatus, downloadModel, optimizeMesh, smoothMesh, importMesh, transformMesh } + // Mesh-ops registry: every op the chat agent can run, with its params schema. + async function listMeshOps(): Promise { + const { data } = await client.get('/optimize/ops') + return data + } + + // Runs one registry op. The response carries `url` when a new mesh was + // written; analysis ops (inspect_mesh, suggest_fixes) return only a report. + async function runMeshOp( + opId: string, + path: string, + params: Record, + ): Promise { + const { data } = await client.post(`/optimize/op/${encodeURIComponent(opId)}`, { + path, + params, + }) + return data + } + + return { generateFromImage, pollJobStatus, cancelJob, getModelStatus, getAllModelsStatus, downloadModel, optimizeMesh, smoothMesh, importMesh, transformMesh, listMeshOps, runMeshOp } } From 27bd96060a7a83b8831df2c4beb4ce0e6b593bb7 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 09:08:58 +0100 Subject: [PATCH 030/108] fix(mesh-ops): turn inside-out faces outwards and create a missing temp dir in presets recompute_normals now fixes the face winding before deriving normals, so a mesh exported inside out comes out with outward normals. The presets also create context.temp_dir when a workflow points it at a folder that does not exist yet. --- api/services/mesh_ops/presets.py | 19 +++++++++++++++---- api/tests/test_mesh_ops_presets.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/api/services/mesh_ops/presets.py b/api/services/mesh_ops/presets.py index f27151888..13d3cfbf3 100644 --- a/api/services/mesh_ops/presets.py +++ b/api/services/mesh_ops/presets.py @@ -34,6 +34,12 @@ def progress_cb(percent: int, label: str) -> None: return replace(context, output_path=output_path, progress_cb=progress_cb) +def _scratch(context: MeshOpContext) -> tempfile.TemporaryDirectory: + """A scratch directory for intermediate files, under a temp_dir that may not exist yet.""" + Path(context.temp_dir).mkdir(parents=True, exist_ok=True) + return tempfile.TemporaryDirectory(dir=str(context.temp_dir)) + + def _scale(size: float, unit: str, dimension: str) -> dict[str, Any]: return {"size": float(size), "unit": unit, "dimension": dimension} @@ -54,7 +60,9 @@ def _arrange_steps(applied, requested: Mapping[str, Mapping[str, Any]]) -> list[ def recompute_normals(input_path: Path, output_path: Path) -> Path: - """Rewrite every vertex normal from the face winding, keeping UVs and materials. + """Orient the faces outwards, then rewrite every vertex normal from them. + + UVs and materials are kept. Decimation moves vertices but keeps their old normals, and generated meshes often ship with none or with inverted ones; area-weighted normals derived @@ -70,6 +78,9 @@ def recompute_normals(input_path: Path, output_path: Path) -> Path: for geometry in geometries: if not isinstance(geometry, trimesh.Trimesh) or not len(geometry.faces): continue + # Make the winding consistent and outward first: normals derived from + # inside-out faces would point inwards too. + trimesh.repair.fix_normals(geometry, multibody=True) triangles = geometry.triangles face_normals = trimesh.util.unitize( np.cross(triangles[:, 1] - triangles[:, 0], triangles[:, 2] - triangles[:, 0]) @@ -98,7 +109,7 @@ def prepare_for_printing( ) -> MeshOpResult: """Repair until watertight, lay flat, scale in millimetres, rest on the plate.""" output_path = _output_path(context, "prepare-for-printing") - with tempfile.TemporaryDirectory(dir=str(context.temp_dir)) as scratch: + with _scratch(context) as scratch: context.log("Preset: prepare for printing") repaired = auto_repair_mesh( input_path, @@ -188,7 +199,7 @@ def prepare_for_game( ) -> MeshOpResult: """Decimate to a poly budget, scale in metres, recompute the normals.""" output_path = _output_path(context, "prepare-for-game") - with tempfile.TemporaryDirectory(dir=str(context.temp_dir)) as scratch_dir: + with _scratch(context) as scratch_dir: scratch = Path(scratch_dir) context.log("Preset: prepare for game") decimated, decimate_step = _decimate(input_path, params["target_faces"], scratch, context) @@ -213,7 +224,7 @@ def prepare_for_web( ) -> MeshOpResult: """Decimate aggressively for a light download, then recompute the normals.""" output_path = _output_path(context, "prepare-for-web") - with tempfile.TemporaryDirectory(dir=str(context.temp_dir)) as scratch_dir: + with _scratch(context) as scratch_dir: context.log("Preset: prepare for web") decimated, decimate_step = _decimate( input_path, params["target_faces"], Path(scratch_dir), context diff --git a/api/tests/test_mesh_ops_presets.py b/api/tests/test_mesh_ops_presets.py index 8c8741512..e48107ed8 100644 --- a/api/tests/test_mesh_ops_presets.py +++ b/api/tests/test_mesh_ops_presets.py @@ -38,6 +38,13 @@ def _sphere_with_inverted_normals(radius: float = 1.5) -> trimesh.Trimesh: return sphere +def _sphere_with_inverted_winding(radius: float = 1.5) -> trimesh.Trimesh: + """Faces wound inside out, as some generators export them.""" + sphere = trimesh.creation.icosphere(subdivisions=3, radius=radius) + sphere.invert() + return sphere + + def _fake_decimate(calls: list): """Stands in for the Node/meshoptimizer backend: copies the mesh as is.""" @@ -211,6 +218,14 @@ def test_web_decimates_aggressively_and_keeps_the_scale(self) -> None: self.assertAlmostEqual(float(extents.max()), 3.0, places=2) self._assert_outward_normals(result.file_path) + def test_web_turns_inside_out_faces_outwards(self) -> None: + path = self._export(_sphere_with_inverted_winding(), include_normals=True) + result = self._run("prepare_for_web", path) + + self._assert_outward_normals(result.file_path) + mesh = trimesh.load(str(result.file_path), force="mesh") + self.assertGreater(mesh.volume, 0, "faces are still wound inside out") + def test_textures_survive_the_web_preset(self) -> None: from PIL import Image From cb7fedf30f5cfa90ba82cada19347ee3627693a8 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 09:13:41 +0100 Subject: [PATCH 031/108] feat(generate): list the steps of composite ops in the edit panel The prepare_for_* presets report one summary per step; the panel now shows them as a numbered list above the detail rows. --- src/areas/generate/components/EditPanel.tsx | 29 ++++++++++++++------- src/areas/generate/meshEditPanel.test.ts | 17 ++++++++++++ src/areas/generate/meshEditPanel.ts | 9 +++++++ 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/src/areas/generate/components/EditPanel.tsx b/src/areas/generate/components/EditPanel.tsx index 3923d60e3..528e6cc5d 100644 --- a/src/areas/generate/components/EditPanel.tsx +++ b/src/areas/generate/components/EditPanel.tsx @@ -15,6 +15,7 @@ import { meshPathFromUrl, paramControl, requiresConfirmation, + stepSummaries, summarizeDetails, toSliderPosition, type BadgeState, @@ -292,17 +293,27 @@ function InspectionReport({ report }: { report: Record }): JSX. } function DetailRows({ details }: { details: Record }): JSX.Element | null { + const steps = stepSummaries(details) const rows = summarizeDetails(details) - if (rows.length === 0) return null + if (steps.length === 0 && rows.length === 0) return null return ( -
- {rows.map(([k, v]) => ( -
-
{k}
-
{v}
-
- ))} -
+
+ {steps.length > 0 && ( +
    + {steps.map((step, i) =>
  1. {i + 1}. {step}
  2. )} +
+ )} + {rows.length > 0 && ( +
+ {rows.map(([k, v]) => ( +
+
{k}
+
{v}
+
+ ))} +
+ )} +
) } diff --git a/src/areas/generate/meshEditPanel.test.ts b/src/areas/generate/meshEditPanel.test.ts index b4c7f7bf1..a78b5655d 100644 --- a/src/areas/generate/meshEditPanel.test.ts +++ b/src/areas/generate/meshEditPanel.test.ts @@ -11,6 +11,7 @@ import { meshPathFromUrl, paramControl, requiresConfirmation, + stepSummaries, summarizeDetails, toSliderPosition, type MeshOpDescriptor, @@ -215,3 +216,19 @@ test('apiErrorMessage prefers the FastAPI detail', () => { assert.equal(apiErrorMessage(new Error('Network Error')), 'Network Error') assert.equal(apiErrorMessage('boom'), 'boom') }) + +test('stepSummaries reads the per-step summaries of a preset result', () => { + assert.deepEqual( + stepSummaries({ + steps: [ + { op: 'decimate', summary: 'Decimated to 5,000 triangles.' }, + { op: 'recompute_normals' }, + 'not a step', + { op: 'normalize_scale', summary: 'Scaled the longest side to 1 m.' }, + ], + }), + ['Decimated to 5,000 triangles.', 'Scaled the longest side to 1 m.'], + ) + assert.deepEqual(stepSummaries({ steps: 'nope' }), []) + assert.deepEqual(stepSummaries({}), []) +}) diff --git a/src/areas/generate/meshEditPanel.ts b/src/areas/generate/meshEditPanel.ts index 23226703f..a987274b1 100644 --- a/src/areas/generate/meshEditPanel.ts +++ b/src/areas/generate/meshEditPanel.ts @@ -256,6 +256,15 @@ export function summarizeDetails(details: Record): [string, str return rows } +/** One line per step of a composite op (the prepare_for_* presets report `steps`). */ +export function stepSummaries(details: Record): string[] { + if (!Array.isArray(details.steps)) return [] + return details.steps.flatMap((step) => { + const summary = step && typeof step === 'object' ? (step as { summary?: unknown }).summary : undefined + return typeof summary === 'string' && summary ? [summary] : [] + }) +} + // ─── Misc ──────────────────────────────────────────────────────────────────── /** Viewer mesh URL → the `path` the optimize routes expect. */ From 1883fcd629f3e6b14c73ff1e86bd634ca3fce13e Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 10:30:12 +0100 Subject: [PATCH 032/108] chore(sculpt): contract for the sculpt core --- src/areas/generate/sculpt/SculptMesh.ts | 78 +++++++++++++++++++++++++ src/areas/generate/sculpt/types.ts | 49 ++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 src/areas/generate/sculpt/SculptMesh.ts create mode 100644 src/areas/generate/sculpt/types.ts diff --git a/src/areas/generate/sculpt/SculptMesh.ts b/src/areas/generate/sculpt/SculptMesh.ts new file mode 100644 index 000000000..ead4f87d1 --- /dev/null +++ b/src/areas/generate/sculpt/SculptMesh.ts @@ -0,0 +1,78 @@ +// Sculpt core: brushes applied in place on a geometry's position/normal arrays. +// CONTRACT STUB — the public surface below is fixed; the implementation is +// filled in separately (see tasks/todo.md, #432/#433). + +import type { + BrushSettings, + DirtyRegion, + SculptMeshOptions, + StrokeRecord, + Vec3, +} from './types' + +export class SculptMesh { + /** The array passed to the constructor, mutated in place. */ + readonly positions: Float32Array + /** The normals passed in, or a newly allocated array when there were none. Mutated in place. */ + readonly normals: Float32Array + readonly vertexCount: number + /** Bounding-box centre at construction: the symmetry planes pass through it. */ + readonly symmetryCenter: Vec3 + + /** + * Builds the weld map (UV-seam duplicates), vertex adjacency and a spatial + * index. Must stay fast on ~1M-triangle meshes (target < 1.5 s). + * `normals` null → normals are computed from the faces. + */ + constructor( + positions: Float32Array, + normals: Float32Array | null, + index: Uint32Array | Uint16Array | null, + options?: SculptMeshOptions, + ) { + void positions; void normals; void index; void options + throw new Error('SculptMesh: not implemented') + } + + /** + * Starts a stroke at a surface point. For grab, the vertices inside the + * radius and their weights are captured here and stay fixed for the stroke. + */ + beginStroke(settings: BrushSettings, point: Vec3, normal: Vec3): DirtyRegion { + void settings; void point; void normal + throw new Error('SculptMesh: not implemented') + } + + /** + * Continues the stroke to a new point. + * - grab: `point` is the cursor projected on the camera-facing plane through + * the stroke's start point (computed by the viewer); the captured vertices + * follow point − previous point, scaled by their weight. + * - other brushes: dabs are spaced along the path (every ~radius/4) from the + * previous point; `point`/`normal` are the latest surface hit. + * Positions, normals of the affected region, and weld duplicates are updated. + */ + strokeTo(point: Vec3, normal: Vec3): DirtyRegion { + void point; void normal + throw new Error('SculptMesh: not implemented') + } + + /** Ends the stroke. Returns its undo record, or null if nothing moved. */ + endStroke(): StrokeRecord | null { + throw new Error('SculptMesh: not implemented') + } + + /** + * Swaps the vertices of `record` with their current state and returns the + * record of what was overwritten, so undo and redo are the same call. + */ + restore(record: StrokeRecord): { region: DirtyRegion; inverse: StrokeRecord } { + void record + throw new Error('SculptMesh: not implemented') + } + + /** True once any stroke or restore changed the mesh since construction. */ + get changed(): boolean { + throw new Error('SculptMesh: not implemented') + } +} diff --git a/src/areas/generate/sculpt/types.ts b/src/areas/generate/sculpt/types.ts new file mode 100644 index 000000000..a302b4d65 --- /dev/null +++ b/src/areas/generate/sculpt/types.ts @@ -0,0 +1,49 @@ +// Contract between the sculpt core (pure geometry, ./SculptMesh.ts) and the +// viewer integration (Viewer3D / EditPanel). No three.js imports here: the core +// works on raw typed arrays so it can be unit-tested with node --test. +// +// Coordinates are always in the LOCAL space of one THREE.Mesh geometry (the +// same space as geometry.attributes.position). The viewer converts world-space +// hits and radii into that space before calling the core. + +export type Vec3 = [number, number, number] + +export type BrushKind = 'grab' | 'smooth' | 'inflate' | 'flatten' + +export type SymmetryAxis = 'x' | 'y' | 'z' + +export interface BrushSettings { + kind: BrushKind + /** Brush radius, in local units. */ + radius: number + /** 0..1. */ + strength: number + /** Inflate → deflate, flatten → pull outwards (Ctrl held). Ignored by grab and smooth. */ + invert: boolean + /** Mirror every dab across these planes through SculptMesh.symmetryCenter. */ + symmetry: SymmetryAxis[] +} + +/** Buffer vertices touched by one call, so the viewer can upload only that range. */ +export interface DirtyRegion { + /** + * Every buffer vertex index whose position or normal changed, including the + * duplicates welded to a moved vertex (UV seams). Empty when nothing moved. + */ + vertices: Uint32Array + /** Smallest and largest touched buffer index (for BufferAttribute.addUpdateRange). */ + minIndex: number + maxIndex: number +} + +/** Undo record of one stroke: swap it back with SculptMesh.restore(). */ +export interface StrokeRecord { + vertices: Uint32Array + positions: Float32Array + normals: Float32Array +} + +export interface SculptMeshOptions { + /** Distance under which two buffer vertices count as one (UV-seam duplicates). Default: 1e-6 × bbox diagonal. */ + weldEpsilon?: number +} From 0c7b8c504bb78ae0d9a132ece0595814bdf474f6 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 10:43:05 +0100 Subject: [PATCH 033/108] feat(optimize): patch-vertices endpoint to write sculpted positions into a GLB POST /optimize/patch-vertices appends new float32 POSITION/NORMAL accessors for the edited primitives to a copy of the GLB and repoints the primitive attributes, leaving existing accessors, textures, materials and extensions untouched. --- api/routers/optimize.py | 32 +++ api/services/mesh_ops/glb_patch.py | 205 +++++++++++++++ api/tests/test_optimize_patch_vertices.py | 299 ++++++++++++++++++++++ 3 files changed, 536 insertions(+) create mode 100644 api/services/mesh_ops/glb_patch.py create mode 100644 api/tests/test_optimize_patch_vertices.py diff --git a/api/routers/optimize.py b/api/routers/optimize.py index a362b3b6d..5fafc3343 100644 --- a/api/routers/optimize.py +++ b/api/routers/optimize.py @@ -24,6 +24,7 @@ MeshOpUnavailableError, mesh_ops_registry, ) +from services.mesh_ops.glb_patch import GlbPatchError, VertexPatch, patch_glb_file from services.mesh_ops.transforms import apply_transforms, transform_file router = APIRouter(tags=["optimize"]) @@ -55,6 +56,18 @@ class ArrangeRequest(BaseModel): params: dict[str, dict[str, object]] = Field(default_factory=dict) # per step id +class VertexPatchBody(BaseModel): + mesh: int # index into gltf.meshes + primitive: int # index into meshes[mesh].primitives + positions: str # base64 little-endian float32, vertexCount * 3 + normals: str | None = None # same format, optional + + +class PatchVerticesRequest(BaseModel): + path: str # format: "{collection}/{filename}.glb" + patches: list[VertexPatchBody] + + def _resolve_input_path(raw_path: str) -> Path: candidate = Path(raw_path) if candidate.is_absolute(): @@ -238,6 +251,25 @@ def arrange_mesh(body: ArrangeRequest): return _operation_response(result) +@router.post("/patch-vertices") +def patch_vertices(body: PatchVerticesRequest): + # Sculpt commit: write the viewer's edited vertex data into a copy of the + # GLB at container level, so textures, materials and extensions survive. + input_path = _resolve_input_path(body.path) + if input_path.suffix.lower() != ".glb": + raise HTTPException(400, "Sculpt patches can only be applied to .glb files") + output_path = _operation_output_path( + input_path, + f"{input_path.stem}_sculpt_{uuid.uuid4().hex[:8]}.glb", + ) + patches = [VertexPatch(**patch.model_dump()) for patch in body.patches] + try: + patch_glb_file(input_path, output_path, patches) + except GlbPatchError as exc: + raise HTTPException(400, str(exc)) from exc + return _operation_response(MeshOpResult(output_path, {})) + + def _transform_output_path(input_path: Path) -> Path: return _operation_output_path( input_path, diff --git a/api/services/mesh_ops/glb_patch.py b/api/services/mesh_ops/glb_patch.py new file mode 100644 index 000000000..56839f4a8 --- /dev/null +++ b/api/services/mesh_ops/glb_patch.py @@ -0,0 +1,205 @@ +"""Write new vertex positions/normals into a copy of a GLB, leaving the rest intact. + +The sculpt tool edits geometry in the viewer and sends back, per glTF primitive, +the new POSITION (and optionally NORMAL) arrays. Re-exporting through trimesh +would lose extensions and rewrite materials, so instead the GLB is patched at +the container level: the new data is appended to the BIN chunk as fresh +bufferViews + accessors and the primitive's attributes are repointed to them. +Existing accessors are never mutated (they may be shared with other primitives +or morph targets); the old data is simply left unreferenced. +""" + +from __future__ import annotations + +import base64 +import binascii +import json +import math +import struct +import sys +from array import array +from dataclasses import dataclass +from pathlib import Path + +_GLB_MAGIC = 0x46546C67 # "glTF" +_CHUNK_JSON = 0x4E4F534A # "JSON" +_CHUNK_BIN = 0x004E4942 # "BIN\0" +_FLOAT = 5126 +_ARRAY_BUFFER = 34962 +_DRACO = "KHR_draco_mesh_compression" + + +class GlbPatchError(ValueError): + """The GLB or a patch is invalid; the message is safe to show to the user.""" + + +@dataclass(frozen=True) +class VertexPatch: + mesh: int + primitive: int + positions: str # base64, little-endian float32, count * 3 + normals: str | None = None # same format, optional + + +def patch_glb_file(input_path: Path, output_path: Path, patches: list[VertexPatch]) -> None: + data = patch_glb_bytes(Path(input_path).read_bytes(), patches) + Path(output_path).write_bytes(data) + + +def patch_glb_bytes(data: bytes, patches: list[VertexPatch]) -> bytes: + if not patches: + raise GlbPatchError("At least one patch is required") + + gltf, binary = _parse_glb(data) + buffers = gltf.setdefault("buffers", []) + if binary is None: + if buffers: + raise GlbPatchError("GLB has no BIN chunk but declares external buffers") + buffers.append({"byteLength": 0}) + binary = b"" + elif not buffers or "uri" in buffers[0]: + raise GlbPatchError("GLB BIN chunk is not bound to buffers[0]") + + # GLB BIN chunk may be padded beyond the declared buffer length; append after it. + out = bytearray(binary) + meshes = gltf.get("meshes") or [] + accessors = gltf.setdefault("accessors", []) + views = gltf.setdefault("bufferViews", []) + + for patch in patches: + primitive = _primitive(meshes, patch.mesh, patch.primitive) + if _DRACO in (primitive.get("extensions") or {}): + raise GlbPatchError( + f"mesh {patch.mesh} primitive {patch.primitive} is Draco-compressed " + f"({_DRACO}); sculpting Draco meshes is not supported" + ) + attributes = primitive.get("attributes") or {} + position_index = attributes.get("POSITION") + if not isinstance(position_index, int) or not 0 <= position_index < len(accessors): + raise GlbPatchError(f"mesh {patch.mesh} primitive {patch.primitive} has no POSITION accessor") + count = int(accessors[position_index].get("count", 0)) + + label = f"mesh {patch.mesh} primitive {patch.primitive}" + positions_raw, positions = _decode(patch.positions, count, f"{label} positions") + normals_raw = None + if patch.normals is not None: + normals_raw, _ = _decode(patch.normals, count, f"{label} normals") + + new_attributes = dict(attributes) + new_attributes["POSITION"] = _append_accessor( + out, views, accessors, positions_raw, count, bounds=_bounds(positions), + ) + if normals_raw is not None: + new_attributes["NORMAL"] = _append_accessor(out, views, accessors, normals_raw, count) + primitive["attributes"] = new_attributes + + buffers[0]["byteLength"] = len(out) + return _build_glb(gltf, bytes(out)) + + +# --------------------------------------------------------------------------- + +def _parse_glb(data: bytes) -> tuple[dict, bytes | None]: + if len(data) < 20: + raise GlbPatchError("Not a valid GLB file (too short)") + magic, version, length = struct.unpack_from(" len(data): + raise GlbPatchError("GLB file is truncated") + + json_len, json_type = struct.unpack_from(" length: + raise GlbPatchError("GLB is missing its JSON chunk") + try: + gltf = json.loads(data[20:20 + json_len].decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + raise GlbPatchError(f"GLB JSON chunk is invalid: {exc}") from exc + if not isinstance(gltf, dict): + raise GlbPatchError("GLB JSON chunk is not an object") + + offset = 20 + json_len + binary = None + if offset + 8 <= length: + bin_len, bin_type = struct.unpack_from(" length: + raise GlbPatchError("GLB BIN chunk is truncated") + binary = data[offset + 8:offset + 8 + bin_len] + return gltf, binary + + +def _build_glb(gltf: dict, binary: bytes) -> bytes: + json_bytes = json.dumps(gltf, separators=(",", ":")).encode("utf-8") + json_bytes += b" " * (-len(json_bytes) % 4) + binary += b"\0" * (-len(binary) % 4) + body = ( + struct.pack(" dict: + if not 0 <= mesh < len(meshes): + raise GlbPatchError(f"mesh {mesh} does not exist (GLB has {len(meshes)} meshes)") + primitives = meshes[mesh].get("primitives") or [] + if not 0 <= primitive < len(primitives): + raise GlbPatchError( + f"primitive {primitive} does not exist in mesh {mesh} " + f"({len(primitives)} primitives)" + ) + return primitives[primitive] + + +def _decode(encoded: str, count: int, label: str) -> tuple[bytes, array]: + try: + raw = base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError) as exc: + raise GlbPatchError(f"{label}: invalid base64 ({exc})") from exc + if len(raw) % 4: + raise GlbPatchError(f"{label}: byte length {len(raw)} is not a multiple of 4") + values = array("f") + values.frombytes(raw) + if sys.byteorder == "big": + values.byteswap() + if len(values) != count * 3: + raise GlbPatchError(f"{label}: got {len(values)} floats, expected {count * 3} ({count} vertices x 3)") + if not all(map(math.isfinite, values)): + raise GlbPatchError(f"{label}: contains non-finite values (NaN or infinity)") + return raw, values + + +def _bounds(values: array) -> tuple[list[float], list[float]]: + if not values: + return [0.0, 0.0, 0.0], [0.0, 0.0, 0.0] + columns = [values[axis::3] for axis in range(3)] + return [min(c) for c in columns], [max(c) for c in columns] + + +def _append_accessor( + out: bytearray, + views: list, + accessors: list, + raw: bytes, + count: int, + bounds: tuple[list[float], list[float]] | None = None, +) -> int: + out.extend(b"\0" * (-len(out) % 4)) + views.append({ + "buffer": 0, + "byteOffset": len(out), + "byteLength": len(raw), + "target": _ARRAY_BUFFER, + }) + out.extend(raw) + accessor: dict = { + "bufferView": len(views) - 1, + "componentType": _FLOAT, + "count": count, + "type": "VEC3", + } + if bounds is not None: + accessor["min"], accessor["max"] = bounds + accessors.append(accessor) + return len(accessors) - 1 diff --git a/api/tests/test_optimize_patch_vertices.py b/api/tests/test_optimize_patch_vertices.py new file mode 100644 index 000000000..187a64c85 --- /dev/null +++ b/api/tests/test_optimize_patch_vertices.py @@ -0,0 +1,299 @@ +import base64 +import json +import struct +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import trimesh +from fastapi import HTTPException +from PIL import Image + +from routers import optimize + + +# --------------------------------------------------------------------------- +# GLB helpers (test-side, independent from the implementation under test) +# --------------------------------------------------------------------------- + +def _read_glb(path: Path) -> tuple[dict, bytes]: + data = path.read_bytes() + magic, version, length = struct.unpack_from(" None: + json_bytes = json.dumps(gltf).encode() + json_bytes += b" " * (-len(json_bytes) % 4) + chunks = struct.pack(" np.ndarray: + accessor = gltf["accessors"][index] + view = gltf["bufferViews"][accessor["bufferView"]] + start = view.get("byteOffset", 0) + accessor.get("byteOffset", 0) + count = accessor["count"] + stride = view.get("byteStride", 12) + raw = np.frombuffer(binary, dtype=np.uint8, count=stride * (count - 1) + 12, offset=start) + rows = np.lib.stride_tricks.as_strided(raw, shape=(count, 12), strides=(stride, 1)) + return np.frombuffer(rows.copy().tobytes(), dtype=" str: + return base64.b64encode(np.asarray(array, dtype=" None: + textured = trimesh.creation.box() + uv = np.random.default_rng(0).random((len(textured.vertices), 2)) + image = Image.new("RGB", (8, 8), (200, 30, 30)) + textured.visual = trimesh.visual.TextureVisuals(uv=uv, image=image) + plain = trimesh.creation.icosphere(subdivisions=1) + scene = trimesh.Scene() + scene.add_geometry(textured, geom_name="textured") + scene.add_geometry(plain, geom_name="plain") + scene.export(str(path)) + + +def _mesh_index(gltf: dict, name: str) -> int: + return next(i for i, m in enumerate(gltf["meshes"]) if m.get("name") == name) + + +def _shared_accessor_glb(path: Path) -> None: + """Two meshes whose single primitives share one POSITION accessor.""" + positions = np.array([[0, 0, 0], [1, 0, 0], [0, 1, 0]], dtype=" None: + self._dir = tempfile.TemporaryDirectory() + self.workspace = Path(self._dir.name) + self.collection = self.workspace / "Default" + self.collection.mkdir() + patcher = patch.object(optimize.registry, "WORKSPACE_DIR", self.workspace) + patcher.start() + self.addCleanup(patcher.stop) + self.addCleanup(self._dir.cleanup) + + def _patch(self, path: str, patches: list[dict]): + return optimize.patch_vertices(optimize.PatchVerticesRequest(path=path, patches=patches)) + + def _assert_400(self, path: str, patches: list[dict], fragment: str) -> None: + with self.assertRaises(HTTPException) as caught: + self._patch(path, patches) + self.assertEqual(caught.exception.status_code, 400) + self.assertIn(fragment, str(caught.exception.detail)) + + # -- success ------------------------------------------------------------ + + def test_patches_positions_and_normals_keeping_texture_and_other_mesh(self) -> None: + source = self.collection / "model.glb" + _textured_two_mesh_scene(source) + gltf, binary = _read_glb(source) + textured = _mesh_index(gltf, "textured") + plain = _mesh_index(gltf, "plain") + primitive = gltf["meshes"][textured]["primitives"][0] + old_positions = _accessor_floats(gltf, binary, primitive["attributes"]["POSITION"]) + plain_prim = gltf["meshes"][plain]["primitives"][0] + plain_positions = _accessor_floats(gltf, binary, plain_prim["attributes"]["POSITION"]) + + new_positions = old_positions * 2.0 + np.array([0.5, -1.0, 3.0], dtype=" None: + source = self.collection / "shared.glb" + _shared_accessor_glb(source) + normals = np.tile(np.array([0, 0, 1], dtype=" None: + source = self.collection / "shared.glb" + _shared_accessor_glb(source) + moved = np.array([[5, 5, 5], [6, 5, 5], [5, 6, 5]], dtype=" None: + source = self.collection / "odd.glb" + _shared_accessor_glb(source) + gltf, binary = _read_glb(source) + # Make the declared buffer length not a multiple of 4 (padding follows). + binary += b"\x01\x02" + gltf["buffers"][0]["byteLength"] = len(binary) + _write_glb(source, gltf, binary) + response = self._patch("Default/odd.glb", [{ + "mesh": 0, "primitive": 0, "positions": _b64(np.ones((3, 3))), + }]) + out_gltf, _ = _read_glb(self.workspace / response["path"]) + for view in out_gltf["bufferViews"][1:]: + self.assertEqual(view["byteOffset"] % 4, 0) + + # -- errors ------------------------------------------------------------- + + def test_rejects_non_glb_input(self) -> None: + (self.collection / "mesh.obj").write_text("v 0 0 0\n") + self._assert_400("Default/mesh.obj", [{ + "mesh": 0, "primitive": 0, "positions": _b64(np.zeros((1, 3))), + }], ".glb") + + def test_rejects_empty_patch_list(self) -> None: + _shared_accessor_glb(self.collection / "shared.glb") + self._assert_400("Default/shared.glb", [], "patch") + + def test_rejects_count_mismatch(self) -> None: + _shared_accessor_glb(self.collection / "shared.glb") + self._assert_400("Default/shared.glb", [{ + "mesh": 0, "primitive": 0, "positions": _b64(np.zeros((4, 3))), + }], "expected 9") + self._assert_400("Default/shared.glb", [{ + "mesh": 0, "primitive": 0, "positions": _b64(np.zeros((3, 3))), + "normals": _b64(np.zeros((2, 3))), + }], "normals") + + def test_rejects_bad_indices(self) -> None: + _shared_accessor_glb(self.collection / "shared.glb") + self._assert_400("Default/shared.glb", [{ + "mesh": 7, "primitive": 0, "positions": _b64(np.zeros((3, 3))), + }], "mesh 7") + self._assert_400("Default/shared.glb", [{ + "mesh": 0, "primitive": 3, "positions": _b64(np.zeros((3, 3))), + }], "primitive 3") + self._assert_400("Default/shared.glb", [{ + "mesh": -1, "primitive": 0, "positions": _b64(np.zeros((3, 3))), + }], "mesh -1") + + def test_rejects_bad_base64(self) -> None: + _shared_accessor_glb(self.collection / "shared.glb") + self._assert_400("Default/shared.glb", [{ + "mesh": 0, "primitive": 0, "positions": "not base64!!", + }], "base64") + + def test_rejects_non_finite_values(self) -> None: + _shared_accessor_glb(self.collection / "shared.glb") + values = np.zeros((3, 3), dtype=" None: + source = self.collection / "draco.glb" + _shared_accessor_glb(source) + gltf, binary = _read_glb(source) + gltf["meshes"][0]["primitives"][0]["extensions"] = { + "KHR_draco_mesh_compression": {"bufferView": 0, "attributes": {"POSITION": 0}}, + } + gltf["extensionsUsed"] = ["KHR_draco_mesh_compression"] + _write_glb(source, gltf, binary) + self._assert_400("Default/draco.glb", [{ + "mesh": 0, "primitive": 0, "positions": _b64(np.zeros((3, 3))), + }], "Draco") + + def test_rejects_corrupt_glb(self) -> None: + (self.collection / "broken.glb").write_bytes(b"glTF garbage") + self._assert_400("Default/broken.glb", [{ + "mesh": 0, "primitive": 0, "positions": _b64(np.zeros((3, 3))), + }], "GLB") + + +if __name__ == "__main__": + unittest.main() From 39321fec6848f27b49c152a7110f26cc3a056601 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 10:46:50 +0100 Subject: [PATCH 034/108] feat(sculpt): brush core with weld map, spatial hash and stroke undo Grab, smooth, inflate and flatten brushes on raw typed arrays, acting on welded vertices so UV-seam duplicates never tear. Integer-keyed hash weld, CSR adjacency, a dense re-bucketing grid for sphere queries, regional normal updates, symmetry with blended mirrored dabs, and first-touch stroke records that restore() swaps for undo/redo. --- package.json | 2 +- .../generate/sculpt/SculptMesh.perf.test.ts | 92 +++ src/areas/generate/sculpt/SculptMesh.test.ts | 394 +++++++++++ src/areas/generate/sculpt/SculptMesh.ts | 633 +++++++++++++++++- src/areas/generate/sculpt/SpatialGrid.ts | 144 ++++ src/areas/generate/sculpt/testMeshes.ts | 119 ++++ src/areas/generate/sculpt/topology.ts | 67 ++ src/areas/generate/sculpt/weld.ts | 126 ++++ 8 files changed, 1564 insertions(+), 13 deletions(-) create mode 100644 src/areas/generate/sculpt/SculptMesh.perf.test.ts create mode 100644 src/areas/generate/sculpt/SculptMesh.test.ts create mode 100644 src/areas/generate/sculpt/SpatialGrid.ts create mode 100644 src/areas/generate/sculpt/testMeshes.ts create mode 100644 src/areas/generate/sculpt/topology.ts create mode 100644 src/areas/generate/sculpt/weld.ts diff --git a/package.json b/package.json index d6117ff35..69337f6fb 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/sculpt/SculptMesh.perf.test.ts b/src/areas/generate/sculpt/SculptMesh.perf.test.ts new file mode 100644 index 000000000..e900b8a12 --- /dev/null +++ b/src/areas/generate/sculpt/SculptMesh.perf.test.ts @@ -0,0 +1,92 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { DAB_SPACING, SculptMesh } from './SculptMesh.ts' +import type { BrushKind, BrushSettings, Vec3 } from './types.ts' +import { seamSphere } from './testMeshes.ts' + +// Targets: constructor < 1.5 s, one dab over ~2k vertices < 4 ms, a grab +// strokeTo over ~10k vertices < 8 ms. Asserted with a 3× margin so slow CI +// machines do not flake; the measured timings are printed. + +function countWithin(positions: Float32Array, p: Vec3, r: number): number { + let c = 0 + for (let v = 0; v < positions.length / 3; v++) { + const d = (positions[v * 3] - p[0]) ** 2 + (positions[v * 3 + 1] - p[1]) ** 2 + (positions[v * 3 + 2] - p[2]) ** 2 + if (d < r * r) c++ + } + return c +} + +function brush(kind: BrushKind, radius: number): BrushSettings { + return { kind, radius, strength: 0.5, invert: false, symmetry: [] } +} + +test('performance on a ~1M-triangle mesh with seam duplicates', () => { + // Warm the JIT on a small mesh first. + const small = seamSphere(40, 80, 8) + const warm = new SculptMesh(small.positions, null, small.index) + for (const kind of ['inflate', 'smooth', 'flatten', 'grab'] as BrushKind[]) { + warm.beginStroke(brush(kind, 0.3), [1, 0, 0], [1, 0, 0]) + for (let i = 1; i <= 20; i++) warm.strokeTo([1, 0.01 * i, 0], [1, 0, 0]) + warm.endStroke() + } + + const big = seamSphere(500, 1000, 8) + const triangles = big.index!.length / 3 + const t0 = performance.now() + const mesh = new SculptMesh(big.positions, null, big.index) + const buildMs = performance.now() - t0 + console.log(`[sculpt perf] ${triangles} triangles, ${mesh.vertexCount} buffer / ${mesh.weldedVertexCount} welded vertices: constructor ${buildMs.toFixed(0)} ms`) + assert.ok(triangles >= 1_000_000) + assert.ok(mesh.weldedVertexCount < mesh.vertexCount) + assert.ok(buildMs < 4500, `constructor took ${buildMs.toFixed(0)} ms`) + + const a = 0.3 + const centre: Vec3 = [Math.cos(a), 0, Math.sin(a)] + const dabRadius = 0.18 + const dabVerts = countWithin(big.positions, centre, dabRadius) + for (const kind of ['inflate', 'smooth', 'flatten'] as BrushKind[]) { + const settings = brush(kind, dabRadius) + const step = dabRadius * DAB_SPACING + mesh.beginStroke(settings, centre, centre) + // Each strokeTo moves along the equator by a chord just over one spacing, + // so it places exactly one dab. + const dTheta = 2 * Math.asin((step * 1.001) / 2) + const times: number[] = [] + let touched = 0 + for (let i = 1; i <= 30; i++) { + const p: Vec3 = [Math.cos(a + dTheta * i), 0, Math.sin(a + dTheta * i)] + const t = performance.now() + const region = mesh.strokeTo(p, p) + times.push(performance.now() - t) + touched = Math.min(touched || Infinity, region.vertices.length) + } + assert.ok(touched > 2000, `every dab should touch ~2k+ vertices (min ${touched})`) + mesh.endStroke() + times.sort((x, y) => x - y) + const median = times[times.length >> 1] + console.log(`[sculpt perf] ${kind} dab over ~${dabVerts} vertices: median ${median.toFixed(2)} ms, max ${times[times.length - 1].toFixed(2)} ms`) + assert.ok(median < 12, `${kind} dab took ${median.toFixed(2)} ms`) + } + + const grabRadius = 0.4 + const grabVerts = countWithin(big.positions, centre, grabRadius) + mesh.beginStroke({ ...brush('grab', grabRadius), strength: 1 }, centre, centre) + const times: number[] = [] + for (let i = 1; i <= 20; i++) { + const t = performance.now() + mesh.strokeTo([centre[0] + 0.01 * i, centre[1], centre[2]], centre) + times.push(performance.now() - t) + } + const record = mesh.endStroke() + times.sort((x, y) => x - y) + const median = times[times.length >> 1] + console.log(`[sculpt perf] grab strokeTo over ~${grabVerts} vertices: median ${median.toFixed(2)} ms, max ${times[times.length - 1].toFixed(2)} ms`) + assert.ok(grabVerts > 10_000) + assert.ok(median < 24, `grab strokeTo took ${median.toFixed(2)} ms`) + + const t1 = performance.now() + mesh.restore(record!) + console.log(`[sculpt perf] restore of the grab stroke: ${(performance.now() - t1).toFixed(2)} ms`) +}) diff --git a/src/areas/generate/sculpt/SculptMesh.test.ts b/src/areas/generate/sculpt/SculptMesh.test.ts new file mode 100644 index 000000000..8a995f371 --- /dev/null +++ b/src/areas/generate/sculpt/SculptMesh.test.ts @@ -0,0 +1,394 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { SculptMesh } from './SculptMesh.ts' +import type { BrushKind, BrushSettings, DirtyRegion, SymmetryAxis, Vec3 } from './types.ts' +import { grid, gridMirror, seamSphere, splitCube, unindex } from './testMeshes.ts' + +function brush(kind: BrushKind, radius: number, extra: Partial = {}): BrushSettings { + return { kind, radius, strength: 1, invert: false, symmetry: [], ...extra } +} + +function vec(arr: Float32Array, v: number): Vec3 { + return [arr[v * 3], arr[v * 3 + 1], arr[v * 3 + 2]] +} + +function nearestVertex(positions: Float32Array, p: Vec3): number { + let best = -1 + let bestD = Infinity + for (let v = 0; v < positions.length / 3; v++) { + const d = (positions[v * 3] - p[0]) ** 2 + (positions[v * 3 + 1] - p[1]) ** 2 + (positions[v * 3 + 2] - p[2]) ** 2 + if (d < bestD) { bestD = d; best = v } + } + return best +} + +function bitsEqual(a: Float32Array, b: Float32Array): boolean { + const ua = new Uint32Array(a.buffer, a.byteOffset, a.length) + const ub = new Uint32Array(b.buffer, b.byteOffset, b.length) + if (ua.length !== ub.length) return false + for (let i = 0; i < ua.length; i++) if (ua[i] !== ub[i]) return false + return true +} + +/** Asserts that every buffer vertex whose position or normal differs is listed in `region`. */ +function assertRegionCovers(region: DirtyRegion, before: { p: Float32Array; n: Float32Array }, mesh: SculptMesh): void { + const listed = new Set(region.vertices) + let changed = 0 + for (let v = 0; v < mesh.vertexCount; v++) { + let diff = false + for (let k = 0; k < 3; k++) { + if (!Object.is(before.p[v * 3 + k], mesh.positions[v * 3 + k])) diff = true + if (!Object.is(before.n[v * 3 + k], mesh.normals[v * 3 + k])) diff = true + } + if (diff) { + changed++ + assert.ok(listed.has(v), `changed vertex ${v} missing from the dirty region`) + } + } + assert.ok(changed > 0, 'the stroke should change something') + let min = Infinity + let max = -Infinity + for (const v of region.vertices) { min = Math.min(min, v); max = Math.max(max, v) } + assert.equal(region.minIndex, min) + assert.equal(region.maxIndex, max) +} + +function snapshot(mesh: SculptMesh): { p: Float32Array; n: Float32Array } { + return { p: mesh.positions.slice(), n: mesh.normals.slice() } +} + +/** All buffer vertices sharing a welded position must hold identical positions and normals. */ +function assertNoTear(mesh: SculptMesh, original: Float32Array): void { + const groups = new Map() + for (let v = 0; v < mesh.vertexCount; v++) { + const key = `${original[v * 3]},${original[v * 3 + 1]},${original[v * 3 + 2]}` + const list = groups.get(key) + if (list) list.push(v) + else groups.set(key, [v]) + } + for (const list of groups.values()) { + for (const v of list.slice(1)) { + assert.deepEqual(vec(mesh.positions, v), vec(mesh.positions, list[0]), `position of duplicate ${v} tore away`) + assert.deepEqual(vec(mesh.normals, v), vec(mesh.normals, list[0]), `normal of duplicate ${v} differs`) + } + } +} + +test('weld map merges the split vertices of a cube', () => { + const cube = splitCube() + const mesh = new SculptMesh(cube.positions, null, cube.index) + assert.equal(mesh.vertexCount, 24) + assert.equal(mesh.weldedVertexCount, 8) + const soup = unindex(splitCube()) + assert.equal(new SculptMesh(soup.positions, null, null).weldedVertexCount, 8) +}) + +test('computed normals are smooth across the split faces of a corner and point outwards', () => { + const cube = splitCube() + const original = cube.positions.slice() + const mesh = new SculptMesh(cube.positions, null, cube.index) + for (let v = 0; v < 24; v++) { + const p = vec(mesh.positions, v) + const n = vec(mesh.normals, v) + assert.ok(Math.abs(Math.hypot(...n) - 1) < 1e-6) + assert.ok((n[0] * p[0] + n[1] * p[1] + n[2] * p[2]) / Math.sqrt(3) > 0.9, `normal of ${v} is not outward`) + } + assertNoTear(mesh, original) + const s = seamSphere(16, 32, 4) + const sphere = new SculptMesh(s.positions, null, s.index) + for (let v = 0; v < sphere.vertexCount; v++) { + const p = vec(sphere.positions, v) + const n = vec(sphere.normals, v) + assert.ok(n[0] * p[0] + n[1] * p[1] + n[2] * p[2] > 0.98, `sphere normal ${v} not radial`) + } +}) + +test('grab on a cube corner moves its three duplicates identically', () => { + const cube = splitCube() + const original = cube.positions.slice() + const mesh = new SculptMesh(cube.positions, null, cube.index) + mesh.beginStroke(brush('grab', 0.5), [1, 1, 1], [1, 1, 1]) + mesh.strokeTo([1.5, 1.25, 1], [1, 1, 1]) + const record = mesh.endStroke() + assert.ok(record) + let moved = 0 + for (let v = 0; v < 24; v++) { + if (original[v * 3] === 1 && original[v * 3 + 1] === 1 && original[v * 3 + 2] === 1) { + assert.deepEqual(vec(mesh.positions, v), [1.5, 1.25, 1]) + moved++ + } else { + assert.deepEqual(vec(mesh.positions, v), vec(original, v)) + } + } + assert.equal(moved, 3) + assertNoTear(mesh, original) + assert.equal(mesh.changed, true) +}) + +test('grab moves the centre by the delta and leaves vertices outside the radius untouched', () => { + const g = grid(40) + const original = g.positions.slice() + const mesh = new SculptMesh(g.positions, null, g.index) + assert.equal(mesh.changed, false) + const centre = nearestVertex(original, [0, 0, 0]) + const region = mesh.beginStroke(brush('grab', 0.3), [0, 0, 0], [0, 0, 1]) + assert.equal(region.vertices.length, 0) + mesh.strokeTo([0, 0, 0.1], [0, 0, 1]) + mesh.strokeTo([0, 0, 0.2], [0, 0, 1]) + mesh.endStroke() + assert.ok(Math.abs(mesh.positions[centre * 3 + 2] - 0.2) < 1e-6) + for (let v = 0; v < mesh.vertexCount; v++) { + const d = Math.hypot(original[v * 3], original[v * 3 + 1]) + if (d >= 0.3) assert.deepEqual(vec(mesh.positions, v), vec(original, v)) + else if (d > 0) { + const z = mesh.positions[v * 3 + 2] + assert.ok(z > 0 && z < 0.2, `vertex at distance ${d} should move partially, z=${z}`) + } + } +}) + +test('grab at half strength moves the centre half the delta', () => { + const g = grid(20) + const mesh = new SculptMesh(g.positions, null, g.index) + const centre = nearestVertex(g.positions, [0, 0, 0]) + mesh.beginStroke(brush('grab', 0.3, { strength: 0.5 }), [0, 0, 0], [0, 0, 1]) + mesh.strokeTo([0, 0, 0.2], [0, 0, 1]) + assert.ok(Math.abs(mesh.positions[centre * 3 + 2] - 0.1) < 1e-6) +}) + +test('smooth reduces a spike', () => { + const g = grid(40) + const centre = nearestVertex(g.positions, [0, 0, 0]) + g.positions[centre * 3 + 2] = 0.5 + const mesh = new SculptMesh(g.positions, null, g.index) + mesh.beginStroke(brush('smooth', 0.3), [0, 0, 0.5], [0, 0, 1]) + mesh.strokeTo([0.02, 0, 0], [0, 0, 1]) + mesh.endStroke() + assert.ok(mesh.positions[centre * 3 + 2] < 0.25, `spike still ${mesh.positions[centre * 3 + 2]}`) +}) + +test('inflate pushes a sphere outwards and invert pulls it in, without tearing seams', () => { + for (const invert of [false, true]) { + const s = seamSphere(32, 64, 4) + const original = s.positions.slice() + const mesh = new SculptMesh(s.positions, null, s.index) + const v = nearestVertex(original, [1, 0, 0]) + mesh.beginStroke(brush('inflate', 0.4, { invert }), [1, 0, 0], [1, 0, 0]) + mesh.strokeTo([1, 0.1, 0], [1, 0, 0]) + mesh.endStroke() + const r = Math.hypot(...vec(mesh.positions, v)) + if (invert) assert.ok(r < 0.99, `deflate should shrink, r=${r}`) + else assert.ok(r > 1.01, `inflate should grow, r=${r}`) + assertNoTear(mesh, original) + } +}) + +test('flatten reduces the spread of distances to the plane, invert increases it', () => { + const height = (x: number, y: number) => 0.05 * Math.sin(12 * x) * Math.cos(12 * y) + const spread = (positions: Float32Array) => { + let sum = 0 + let sq = 0 + let count = 0 + for (let v = 0; v < positions.length / 3; v++) { + if (Math.hypot(positions[v * 3], positions[v * 3 + 1]) > 0.2) continue + const z = positions[v * 3 + 2] + sum += z + sq += z * z + count++ + } + return sq / count - (sum / count) ** 2 + } + for (const invert of [false, true]) { + const g = grid(60, height) + const before = spread(g.positions) + const mesh = new SculptMesh(g.positions, null, g.index) + mesh.beginStroke(brush('flatten', 0.4, { invert, strength: 0.5 }), [0, 0, 0], [0, 0, 1]) + mesh.strokeTo([0.05, 0, 0], [0, 0, 1]) + mesh.endStroke() + const after = spread(mesh.positions) + if (invert) assert.ok(after > before * 1.1, `invert flatten should roughen: ${before} → ${after}`) + else assert.ok(after < before * 0.5, `flatten should flatten: ${before} → ${after}`) + } +}) + +test('symmetry across x gives a mirror-symmetric result for every brush', () => { + const n = 40 + const height = (x: number, y: number) => 0.3 * x * x * y + 0.08 * Math.sin(7 * y) + 0.05 * Math.cos(9 * x * x) + const kinds: BrushKind[] = ['grab', 'smooth', 'inflate', 'flatten'] + for (const kind of kinds) { + const g = grid(n, height) + const mesh = new SculptMesh(g.positions, null, g.index) + assert.ok(Math.abs(mesh.symmetryCenter[0]) < 1e-7) + const settings = brush(kind, 0.45, { symmetry: ['x'] as SymmetryAxis[], strength: 0.8 }) + // Starts near the plane so the mirrored dabs overlap. + mesh.beginStroke(settings, [0.12, 0.05, height(0.12, 0.05)], [0, 0, 1]) + if (kind === 'grab') { + mesh.strokeTo([0.3, 0.1, 0.2], [0, 0, 1]) + } else { + mesh.strokeTo([0.35, 0.3, height(0.35, 0.3)], [0, 0, 1]) + } + mesh.endStroke() + for (let v = 0; v < mesh.vertexCount; v++) { + const m = gridMirror(n, v) + const p = vec(mesh.positions, v) + const q = vec(mesh.positions, m) + assert.ok(Math.abs(p[0] + q[0]) < 1e-5 && Math.abs(p[1] - q[1]) < 1e-5 && Math.abs(p[2] - q[2]) < 1e-5, + `${kind}: vertex ${v} ${p} is not the mirror of ${m} ${q}`) + const nv = vec(mesh.normals, v) + const nm = vec(mesh.normals, m) + assert.ok(Math.abs(nv[0] + nm[0]) < 1e-4 && Math.abs(nv[1] - nm[1]) < 1e-4 && Math.abs(nv[2] - nm[2]) < 1e-4, + `${kind}: normal of ${v} is not mirrored`) + } + assert.ok(mesh.changed, `${kind} should change the mesh`) + } +}) + +test('a dab on the symmetry plane is not applied twice', () => { + const height = (x: number, y: number) => 0.1 * Math.sin(5 * y) + 0.1 * x * x + for (const kind of ['inflate', 'smooth', 'flatten'] as BrushKind[]) { + const a = grid(30, height) + const b = grid(30, height) + const plain = new SculptMesh(a.positions, null, a.index) + const mirrored = new SculptMesh(b.positions, null, b.index) + plain.beginStroke(brush(kind, 0.4), [0, 0.1, height(0, 0.1)], [0, 0, 1]) + mirrored.beginStroke(brush(kind, 0.4, { symmetry: ['x'] }), [0, 0.1, height(0, 0.1)], [0, 0, 1]) + for (let v = 0; v < plain.positions.length; v++) { + assert.ok(Math.abs(plain.positions[v] - mirrored.positions[v]) < 1e-6, `${kind}: component ${v} differs`) + } + } + // Grab on the plane: a delta along the plane is followed once, not twice. + const g = grid(30) + const mesh = new SculptMesh(g.positions, null, g.index) + const centre = nearestVertex(g.positions, [0, 0, 0]) + mesh.beginStroke(brush('grab', 0.3, { symmetry: ['x'] }), [0, 0, 0], [0, 0, 1]) + mesh.strokeTo([0.2, 0.1, 0.2], [0, 0, 1]) + assert.deepEqual(vec(mesh.positions, centre).map((c) => Math.round(c * 1e6) / 1e6), [0, 0.1, 0.2]) +}) + +test('symmetry on several axes mirrors every combination', () => { + const s = seamSphere(24, 48, 2) + const mesh = new SculptMesh(s.positions, null, s.index) + const p: Vec3 = [Math.SQRT1_2, 0.5, 0.5] + const before = s.positions.slice() + mesh.beginStroke(brush('inflate', 0.3, { symmetry: ['x', 'y', 'z'] }), p, p) + for (const sx of [1, -1]) for (const sy of [1, -1]) for (const sz of [1, -1]) { + const v = nearestVertex(before, [p[0] * sx, p[1] * sy, p[2] * sz]) + assert.ok(Math.hypot(...vec(mesh.positions, v)) > 1.005, `octant ${sx},${sy},${sz} not inflated`) + } +}) + +test('undo restores positions and normals bit-exactly and the inverse redoes', () => { + const s = seamSphere(32, 64, 4) + const mesh = new SculptMesh(s.positions, null, s.index) + const before = snapshot(mesh) + mesh.beginStroke(brush('inflate', 0.5, { symmetry: ['x'] }), [1, 0, 0], [1, 0, 0]) + mesh.strokeTo([0.9, 0.43, 0], [1, 0, 0]) + mesh.strokeTo([0.7, 0.7, 0], [1, 0, 0]) + const record = mesh.endStroke() + assert.ok(record) + const after = snapshot(mesh) + assert.ok(!bitsEqual(before.p, after.p)) + + const undo = mesh.restore(record) + assert.ok(bitsEqual(mesh.positions, before.p), 'positions not restored') + assert.ok(bitsEqual(mesh.normals, before.n), 'normals not restored') + assert.ok(undo.region.vertices.length > 0) + + const redo = mesh.restore(undo.inverse) + assert.ok(bitsEqual(mesh.positions, after.p), 'positions not redone') + assert.ok(bitsEqual(mesh.normals, after.n), 'normals not redone') + + mesh.restore(redo.inverse) + assert.ok(bitsEqual(mesh.positions, before.p)) + + // The spatial index follows restore: a grab right after undo still finds the vertices. + const v = nearestVertex(before.p, [1, 0, 0]) + mesh.beginStroke(brush('grab', 0.2), [1, 0, 0], [1, 0, 0]) + mesh.strokeTo([1.2, 0, 0], [1, 0, 0]) + assert.ok(Math.abs(mesh.positions[v * 3] - 1.2) < 1e-6) +}) + +test('grab undo after a far move restores everything', () => { + const g = grid(30) + const mesh = new SculptMesh(g.positions, null, g.index) + const before = snapshot(mesh) + mesh.beginStroke(brush('grab', 0.3), [0, 0, 0], [0, 0, 1]) + mesh.strokeTo([5, 5, 5], [0, 0, 1]) + const record = mesh.endStroke() + assert.ok(record) + mesh.restore(record) + assert.ok(bitsEqual(mesh.positions, before.p)) + assert.ok(bitsEqual(mesh.normals, before.n)) +}) + +test('the spatial index follows vertices moved far by grab', () => { + const g = grid(30) + const mesh = new SculptMesh(g.positions, null, g.index) + const centre = nearestVertex(g.positions, [0, 0, 0]) + mesh.beginStroke(brush('grab', 0.3), [0, 0, 0], [0, 0, 1]) + mesh.strokeTo([0, 0, 3], [0, 0, 1]) + mesh.endStroke() + // Grab again at the new location: the moved centre must be found there. + mesh.beginStroke(brush('grab', 0.1), [0, 0, 3], [0, 0, 1]) + mesh.strokeTo([0, 0, 4], [0, 0, 1]) + mesh.endStroke() + assert.ok(Math.abs(mesh.positions[centre * 3 + 2] - 4) < 1e-5) +}) + +test('the dirty region lists every changed buffer vertex', () => { + const kinds: BrushKind[] = ['grab', 'smooth', 'inflate', 'flatten'] + for (const kind of kinds) { + const s = seamSphere(32, 64, 4) + const mesh = new SculptMesh(s.positions, null, s.index) + let before = snapshot(mesh) + const settings = brush(kind, 0.35, { symmetry: ['z'] }) + const start = mesh.beginStroke(settings, [1, 0, 0], [1, 0, 0]) + if (kind !== 'grab') assertRegionCovers(start, before, mesh) + before = snapshot(mesh) + const region = mesh.strokeTo(kind === 'grab' ? [1.2, 0.1, 0.05] : [0.8, 0.6, 0], [1, 0, 0]) + assertRegionCovers(region, before, mesh) + const record = mesh.endStroke() + assert.ok(record) + before = snapshot(mesh) + const { region: undoRegion } = mesh.restore(record) + assertRegionCovers(undoRegion, before, mesh) + } +}) + +test('a stroke that touches nothing returns no record', () => { + const g = grid(10) + const mesh = new SculptMesh(g.positions, null, g.index) + const region = mesh.beginStroke(brush('inflate', 0.1), [10, 10, 10], [0, 0, 1]) + assert.equal(region.vertices.length, 0) + mesh.strokeTo([10, 10.5, 10], [0, 0, 1]) + assert.equal(mesh.endStroke(), null) + assert.equal(mesh.changed, false) +}) + +test('provided normals are kept until a stroke recomputes them, and are written to', () => { + const g = grid(10) + const normals = new Float32Array(g.positions.length) + for (let v = 0; v < normals.length / 3; v++) normals[v * 3 + 2] = 1 + const mesh = new SculptMesh(g.positions, normals, g.index) + assert.equal(mesh.normals, normals) + assert.equal(mesh.positions, g.positions) + mesh.beginStroke(brush('grab', 0.5), [0, 0, 0], [0, 0, 1]) + mesh.strokeTo([0, 0, 0.3], [0, 0, 1]) + const v = nearestVertex(g.positions, [0.2, 0, 0.2]) + assert.ok(normals[v * 3] > 0.05, 'normal on the slope of the bump should tilt') +}) + +test('Uint16 indices and dab spacing along a long stroke', () => { + const g = grid(30) + const index = Uint16Array.from(g.index!) + const mesh = new SculptMesh(g.positions, null, index) + const before = g.positions.slice() + mesh.beginStroke(brush('inflate', 0.1), [-0.8, 0, 0], [0, 0, 1]) + mesh.strokeTo([0.8, 0, 0], [0, 0, 1]) + // Dabs are spaced along the segment: every vertex on the path rose. + for (const x of [-0.5, -0.2, 0, 0.3, 0.6]) { + const v = nearestVertex(before, [x, 0, 0]) + assert.ok(mesh.positions[v * 3 + 2] > 0.001, `no dab near x=${x}`) + } +}) diff --git a/src/areas/generate/sculpt/SculptMesh.ts b/src/areas/generate/sculpt/SculptMesh.ts index ead4f87d1..558496b6a 100644 --- a/src/areas/generate/sculpt/SculptMesh.ts +++ b/src/areas/generate/sculpt/SculptMesh.ts @@ -1,6 +1,9 @@ // Sculpt core: brushes applied in place on a geometry's position/normal arrays. -// CONTRACT STUB — the public surface below is fixed; the implementation is -// filled in separately (see tasks/todo.md, #432/#433). +// +// Everything works on "welded" vertices: buffer vertices closer than the weld +// epsilon (UV-seam duplicates, triangle-soup corners) form one logical vertex, +// and every write goes to all its duplicates so the surface never tears. +// Pure typed-array code, no three.js, so it runs under node --test. import type { BrushSettings, @@ -9,6 +12,44 @@ import type { StrokeRecord, Vec3, } from './types' +import { SpatialGrid } from './SpatialGrid' +import { type Csr, vertexFaces, vertexNeighbours, weldedTriangles } from './topology' +import { buildWeldMap } from './weld' + +/** Inflate displacement per dab at the brush centre, as a fraction of the radius (× strength). */ +export const INFLATE_RATE = 0.05 +/** Dab spacing along a stroke, as a fraction of the radius. */ +export const DAB_SPACING = 0.25 +/** Upper bound on dabs placed by one strokeTo call (the spacing grows past it). */ +const MAX_DABS_PER_CALL = 256 + +/** Brush falloff from a squared distance: 1 at the centre, smoothly 0 at the radius. */ +export function falloff(dist2: number, radius2: number): number { + if (!(dist2 < radius2)) return 0 + const t = 1 - dist2 / radius2 + return t * t +} + +/** + * Sign triples (±1 per axis) of every mirror combination for the given axes, + * identity first: [] → 1 copy, ['x'] → 2, ['x','y','z'] → 8 (like Blender). + */ +export function mirrorSigns(axes: readonly string[]): Float64Array { + const flags = [axes.includes('x'), axes.includes('y'), axes.includes('z')] + const used = [0, 1, 2].filter((a) => flags[a]) + const total = 1 << used.length + const out = new Float64Array(total * 3).fill(1) + for (let m = 0; m < total; m++) { + used.forEach((axis, bit) => { + if (m & (1 << bit)) out[m * 3 + axis] = -1 + }) + } + return out +} + +function emptyRegion(): DirtyRegion { + return { vertices: new Uint32Array(0), minIndex: 0, maxIndex: -1 } +} export class SculptMesh { /** The array passed to the constructor, mutated in place. */ @@ -18,6 +59,56 @@ export class SculptMesh { readonly vertexCount: number /** Bounding-box centre at construction: the symmetry planes pass through it. */ readonly symmetryCenter: Vec3 + /** Number of logical (welded) vertices the brushes act on. */ + readonly weldedVertexCount: number + + // Topology (welded space). + private readonly weldOf: Uint32Array + private readonly dupStart: Uint32Array + private readonly dupList: Uint32Array + private readonly tri: Uint32Array + private readonly faces: Csr + private readonly adj: Csr + /** Welded positions, kept in sync with the buffer (the representative's value). */ + private readonly wpos: Float32Array + private readonly grid: SpatialGrid + + // Per-dab accumulation over the mirrored copies of a dab. + private readonly accW: Float64Array + private readonly accMax: Float64Array + private readonly accD: Float64Array + private readonly groupMark: Uint32Array + private readonly groupList: Int32Array + private groupId = 0 + private readonly movedList: Int32Array + private readonly ringMark: Uint32Array + private readonly ringList: Int32Array + private ringId = 0 + + // Dirty welded vertices of the current public call. + private readonly callMark: Uint32Array + private readonly callList: Int32Array + private callId = 0 + private callCount = 0 + + // Undo record of the current stroke (first touch of each buffer vertex). + private readonly strokeMark: Uint32Array + private strokeId = 0 + private recVerts = new Uint32Array(0) + private recPos = new Float32Array(0) + private recNrm = new Float32Array(0) + private recCount = 0 + + // Current stroke. + private settings: BrushSettings | null = null + private mirrors: Float64Array = new Float64Array([1, 1, 1]) + private lastX = 0 + private lastY = 0 + private lastZ = 0 + private grabIds = new Int32Array(0) + private grabG = new Float32Array(0) + + private isChanged = false /** * Builds the weld map (UV-seam duplicates), vertex adjacency and a spatial @@ -30,8 +121,59 @@ export class SculptMesh { index: Uint32Array | Uint16Array | null, options?: SculptMeshOptions, ) { - void positions; void normals; void index; void options - throw new Error('SculptMesh: not implemented') + const n = Math.floor(positions.length / 3) + if (normals && normals.length < n * 3) throw new RangeError('SculptMesh: normals shorter than positions') + this.positions = positions + this.vertexCount = n + + const min = [Infinity, Infinity, Infinity] + const max = [-Infinity, -Infinity, -Infinity] + for (let i = 0; i < n * 3; i += 3) { + for (let k = 0; k < 3; k++) { + const v = positions[i + k] + if (v < min[k]) min[k] = v + if (v > max[k]) max[k] = v + } + } + if (n === 0) { min.fill(0); max.fill(0) } + this.symmetryCenter = [(min[0] + max[0]) / 2, (min[1] + max[1]) / 2, (min[2] + max[2]) / 2] + const diag = Math.hypot(max[0] - min[0], max[1] - min[1], max[2] - min[2]) + + const weld = buildWeldMap(positions, n, options?.weldEpsilon ?? diag * 1e-6, [min[0], min[1], min[2]]) + const W = weld.count + this.weldedVertexCount = W + this.weldOf = weld.weldOf + this.dupStart = weld.dupStart + this.dupList = weld.dupList + + this.wpos = new Float32Array(W * 3) + for (let w = 0; w < W; w++) { + const r = this.dupList[this.dupStart[w]] * 3 + this.wpos[w * 3] = positions[r] + this.wpos[w * 3 + 1] = positions[r + 1] + this.wpos[w * 3 + 2] = positions[r + 2] + } + + this.tri = weldedTriangles(index, n, this.weldOf) + this.faces = vertexFaces(this.tri, W) + this.adj = vertexNeighbours(this.tri, this.faces, W) + + this.normals = normals ?? new Float32Array(n * 3) + if (!normals) this.computeAllNormals() + + this.grid = new SpatialGrid(this.wpos, W, min, max, diag / 64) + + this.accW = new Float64Array(W) + this.accMax = new Float64Array(W) + this.accD = new Float64Array(W * 3) + this.groupMark = new Uint32Array(W) + this.groupList = new Int32Array(W) + this.movedList = new Int32Array(W) + this.ringMark = new Uint32Array(W) + this.ringList = new Int32Array(W) + this.callMark = new Uint32Array(W) + this.callList = new Int32Array(W) + this.strokeMark = new Uint32Array(W) } /** @@ -39,8 +181,20 @@ export class SculptMesh { * radius and their weights are captured here and stay fixed for the stroke. */ beginStroke(settings: BrushSettings, point: Vec3, normal: Vec3): DirtyRegion { - void settings; void point; void normal - throw new Error('SculptMesh: not implemented') + this.settings = { ...settings, symmetry: [...settings.symmetry] } + this.mirrors = mirrorSigns(settings.symmetry) + this.strokeId++ + this.recCount = 0 + this.beginCall() + this.lastX = point[0] + this.lastY = point[1] + this.lastZ = point[2] + if (settings.kind === 'grab') { + this.captureGrab(point) + return emptyRegion() + } + this.dab(point[0], point[1], point[2], normal) + return this.takeRegion() } /** @@ -53,13 +207,58 @@ export class SculptMesh { * Positions, normals of the affected region, and weld duplicates are updated. */ strokeTo(point: Vec3, normal: Vec3): DirtyRegion { - void point; void normal - throw new Error('SculptMesh: not implemented') + const s = this.settings + if (!s) return emptyRegion() + this.beginCall() + const dx = point[0] - this.lastX + const dy = point[1] - this.lastY + const dz = point[2] - this.lastZ + if (s.kind === 'grab') { + this.lastX = point[0] + this.lastY = point[1] + this.lastZ = point[2] + this.moveGrab(dx * s.strength, dy * s.strength, dz * s.strength) + return this.takeRegion() + } + const length = Math.hypot(dx, dy, dz) + let spacing = Math.max(s.radius * DAB_SPACING, 1e-12) + if (!(length >= spacing)) return emptyRegion() + let steps = Math.floor(length / spacing) + if (steps > MAX_DABS_PER_CALL) { + steps = MAX_DABS_PER_CALL + spacing = length / steps + } + const ux = dx / length + const uy = dy / length + const uz = dz / length + const x0 = this.lastX + const y0 = this.lastY + const z0 = this.lastZ + for (let i = 1; i <= steps; i++) { + const t = i * spacing + this.dab(x0 + ux * t, y0 + uy * t, z0 + uz * t, normal) + } + this.lastX = x0 + ux * steps * spacing + this.lastY = y0 + uy * steps * spacing + this.lastZ = z0 + uz * steps * spacing + return this.takeRegion() } /** Ends the stroke. Returns its undo record, or null if nothing moved. */ endStroke(): StrokeRecord | null { - throw new Error('SculptMesh: not implemented') + this.settings = null + this.grabIds = new Int32Array(0) + this.grabG = new Float32Array(0) + const count = this.recCount + this.recCount = 0 + // Invalidate the first-touch marks so a later touch records again. + this.strokeId++ + if (count === 0) return null + return { + vertices: this.recVerts.slice(0, count), + positions: this.recPos.slice(0, count * 3), + normals: this.recNrm.slice(0, count * 3), + } } /** @@ -67,12 +266,422 @@ export class SculptMesh { * record of what was overwritten, so undo and redo are the same call. */ restore(record: StrokeRecord): { region: DirtyRegion; inverse: StrokeRecord } { - void record - throw new Error('SculptMesh: not implemented') + this.beginCall() + const { positions, normals, weldOf } = this + const m = record.vertices.length + const inverse: StrokeRecord = { + vertices: record.vertices.slice(), + positions: new Float32Array(m * 3), + normals: new Float32Array(m * 3), + } + let minIndex = Infinity + let maxIndex = -1 + for (let i = 0; i < m; i++) { + const b = record.vertices[i] + if (b < minIndex) minIndex = b + if (b > maxIndex) maxIndex = b + for (let k = 0; k < 3; k++) { + inverse.positions[i * 3 + k] = positions[b * 3 + k] + positions[b * 3 + k] = record.positions[i * 3 + k] + inverse.normals[i * 3 + k] = normals[b * 3 + k] + normals[b * 3 + k] = record.normals[i * 3 + k] + } + this.markDirty(weldOf[b]) + } + for (let i = 0; i < this.callCount; i++) { + const w = this.callList[i] + const r = this.dupList[this.dupStart[w]] * 3 + this.wpos[w * 3] = positions[r] + this.wpos[w * 3 + 1] = positions[r + 1] + this.wpos[w * 3 + 2] = positions[r + 2] + this.grid.update(w) + } + if (m > 0) this.isChanged = true + const region: DirtyRegion = m > 0 + ? { vertices: record.vertices.slice(), minIndex, maxIndex } + : emptyRegion() + return { region, inverse } } /** True once any stroke or restore changed the mesh since construction. */ get changed(): boolean { - throw new Error('SculptMesh: not implemented') + return this.isChanged + } + + // ── Brushes ──────────────────────────────────────────────────────────── + + /** Grab: captures the vertices under the (mirrored) brush and their per-axis follow factors. */ + private captureGrab(point: Vec3): void { + const s = this.settings! + const gid = this.nextGroup() + let count = 0 + const { accW, accMax, accD, groupMark, groupList } = this + const r2 = s.radius * s.radius + const mirrors = this.mirrors + for (let m = 0; m < mirrors.length; m += 3) { + const sx = mirrors[m], sy = mirrors[m + 1], sz = mirrors[m + 2] + const hits = this.grid.query(this.mirrorX(point[0], sx), this.mirrorY(point[1], sy), this.mirrorZ(point[2], sz), s.radius) + const ids = this.grid.resultIds + const dist = this.grid.resultDist2 + for (let i = 0; i < hits; i++) { + const wt = falloff(dist[i], r2) + if (wt <= 0) continue + const w = ids[i] + if (groupMark[w] !== gid) { + groupMark[w] = gid + groupList[count++] = w + accW[w] = 0 + accMax[w] = 0 + accD[w * 3] = accD[w * 3 + 1] = accD[w * 3 + 2] = 0 + } + accW[w] += wt + if (wt > accMax[w]) accMax[w] = wt + accD[w * 3] += wt * sx + accD[w * 3 + 1] += wt * sy + accD[w * 3 + 2] += wt * sz + } + } + // Overlapping mirrored copies are blended (weighted mean scaled by the + // strongest weight), so a vertex near the plane is never pulled twice. + this.grabIds = groupList.slice(0, count) + this.grabG = new Float32Array(count * 3) + for (let i = 0; i < count; i++) { + const w = groupList[i] + const f = accMax[w] / accW[w] + this.grabG[i * 3] = accD[w * 3] * f + this.grabG[i * 3 + 1] = accD[w * 3 + 1] * f + this.grabG[i * 3 + 2] = accD[w * 3 + 2] * f + } + } + + private moveGrab(dx: number, dy: number, dz: number): void { + if (dx === 0 && dy === 0 && dz === 0) return + const { grabIds, grabG } = this + let moved = 0 + for (let i = 0; i < grabIds.length; i++) { + if (this.moveVertex(grabIds[i], grabG[i * 3] * dx, grabG[i * 3 + 1] * dy, grabG[i * 3 + 2] * dz)) { + this.movedList[moved++] = grabIds[i] + } + } + this.finishMove(moved) + } + + /** + * One dab of smooth / inflate / flatten at (cx, cy, cz) and its mirrored + * copies. Displacements are computed from the pre-dab state, then applied. + */ + private dab(cx: number, cy: number, cz: number, hitNormal: Vec3): void { + const s = this.settings! + const r = s.radius + if (!(r > 0) || !(s.strength > 0)) return + const r2 = r * r + const kind = s.kind + const strength = s.strength + const sign = s.invert ? -1 : 1 + const inflateStep = strength * r * INFLATE_RATE * sign + const { accW, accMax, accD, groupMark, groupList, wpos, normals, dupList, dupStart } = this + const adjStart = this.adj.start + const adjList = this.adj.list + const gid = this.nextGroup() + let count = 0 + const mirrors = this.mirrors + + for (let m = 0; m < mirrors.length; m += 3) { + const sx = mirrors[m], sy = mirrors[m + 1], sz = mirrors[m + 2] + const hits = this.grid.query(this.mirrorX(cx, sx), this.mirrorY(cy, sy), this.mirrorZ(cz, sz), r) + const ids = this.grid.resultIds + const dist = this.grid.resultDist2 + + // Flatten: plane through the weighted mean point, along the weighted mean normal. + let px = 0, py = 0, pz = 0, nx = 0, ny = 0, nz = 0 + if (kind === 'flatten') { + let sum = 0 + for (let i = 0; i < hits; i++) { + const wt = falloff(dist[i], r2) + if (wt <= 0) continue + const w = ids[i] + const b = dupList[dupStart[w]] * 3 + sum += wt + px += wt * wpos[w * 3] + py += wt * wpos[w * 3 + 1] + pz += wt * wpos[w * 3 + 2] + nx += wt * normals[b] + ny += wt * normals[b + 1] + nz += wt * normals[b + 2] + } + if (sum <= 0) continue + px /= sum + py /= sum + pz /= sum + let len = Math.hypot(nx, ny, nz) + if (!(len > 1e-12)) { + nx = hitNormal[0] * sx + ny = hitNormal[1] * sy + nz = hitNormal[2] * sz + len = Math.hypot(nx, ny, nz) + if (!(len > 1e-12)) continue + } + nx /= len + ny /= len + nz /= len + } + + for (let i = 0; i < hits; i++) { + const wt = falloff(dist[i], r2) + if (wt <= 0) continue + const w = ids[i] + const o = w * 3 + let dx = 0, dy = 0, dz = 0 + if (kind === 'smooth') { + const a = adjStart[w] + const e = adjStart[w + 1] + if (e > a) { + let ax = 0, ay = 0, az = 0 + for (let k = a; k < e; k++) { + const u = adjList[k] * 3 + ax += wpos[u] + ay += wpos[u + 1] + az += wpos[u + 2] + } + const inv = 1 / (e - a) + dx = (ax * inv - wpos[o]) * strength + dy = (ay * inv - wpos[o + 1]) * strength + dz = (az * inv - wpos[o + 2]) * strength + } + } else if (kind === 'inflate') { + const b = dupList[dupStart[w]] * 3 + dx = normals[b] * inflateStep + dy = normals[b + 1] * inflateStep + dz = normals[b + 2] * inflateStep + } else { + const d = ((wpos[o] - px) * nx + (wpos[o + 1] - py) * ny + (wpos[o + 2] - pz) * nz) * strength * sign + dx = -d * nx + dy = -d * ny + dz = -d * nz + } + if (groupMark[w] !== gid) { + groupMark[w] = gid + groupList[count++] = w + accW[w] = 0 + accMax[w] = 0 + accD[o] = accD[o + 1] = accD[o + 2] = 0 + } + accW[w] += wt + if (wt > accMax[w]) accMax[w] = wt + accD[o] += wt * dx + accD[o + 1] += wt * dy + accD[o + 2] += wt * dz + } + } + + // Blend the mirrored copies: weighted mean of their displacements, scaled + // by the strongest weight. Away from the planes only one copy reaches a + // vertex (plain weight × displacement); on a plane the copies average out + // instead of adding up. + let moved = 0 + for (let i = 0; i < count; i++) { + const w = groupList[i] + const f = accMax[w] / accW[w] + if (this.moveVertex(w, accD[w * 3] * f, accD[w * 3 + 1] * f, accD[w * 3 + 2] * f)) this.movedList[moved++] = w + } + this.finishMove(moved) + } + + private mirrorX(v: number, s: number): number { + return s > 0 ? v : 2 * this.symmetryCenter[0] - v + } + + private mirrorY(v: number, s: number): number { + return s > 0 ? v : 2 * this.symmetryCenter[1] - v + } + + private mirrorZ(v: number, s: number): number { + return s > 0 ? v : 2 * this.symmetryCenter[2] - v + } + + // ── Vertex updates ───────────────────────────────────────────────────── + + /** Moves welded vertex `w` by d (all duplicates). Returns false when the float32 position does not change. */ + private moveVertex(w: number, dx: number, dy: number, dz: number): boolean { + const o = w * 3 + const { wpos } = this + const x = Math.fround(wpos[o] + dx) + const y = Math.fround(wpos[o + 1] + dy) + const z = Math.fround(wpos[o + 2] + dz) + if (x === wpos[o] && y === wpos[o + 1] && z === wpos[o + 2]) return false + this.touch(w) + wpos[o] = x + wpos[o + 1] = y + wpos[o + 2] = z + const { positions, dupList } = this + for (let k = this.dupStart[w]; k < this.dupStart[w + 1]; k++) { + const b = dupList[k] * 3 + positions[b] = x + positions[b + 1] = y + positions[b + 2] = z + } + this.grid.update(w) + this.markDirty(w) + return true + } + + /** After `moved` vertices (movedList) moved: recompute normals of them and their 1-ring. */ + private finishMove(moved: number): void { + if (moved === 0) return + this.isChanged = true + const rid = ++this.ringId + const { ringMark, ringList, movedList } = this + const adjStart = this.adj.start + const adjList = this.adj.list + let count = 0 + for (let i = 0; i < moved; i++) { + const w = movedList[i] + if (ringMark[w] !== rid) { ringMark[w] = rid; ringList[count++] = w } + for (let k = adjStart[w]; k < adjStart[w + 1]; k++) { + const u = adjList[k] + if (ringMark[u] !== rid) { ringMark[u] = rid; ringList[count++] = u } + } + } + for (let i = 0; i < count; i++) this.updateNormal(ringList[i]) + } + + /** Area-weighted smooth normal of welded vertex `w`, written to all its duplicates. */ + private updateNormal(w: number): void { + const { tri, wpos } = this + const fStart = this.faces.start + const fList = this.faces.list + let sx = 0, sy = 0, sz = 0 + for (let k = fStart[w]; k < fStart[w + 1]; k++) { + const f = fList[k] * 3 + const a = tri[f] * 3 + const b = tri[f + 1] * 3 + const c = tri[f + 2] * 3 + const ux = wpos[b] - wpos[a], uy = wpos[b + 1] - wpos[a + 1], uz = wpos[b + 2] - wpos[a + 2] + const vx = wpos[c] - wpos[a], vy = wpos[c + 1] - wpos[a + 1], vz = wpos[c + 2] - wpos[a + 2] + sx += uy * vz - uz * vy + sy += uz * vx - ux * vz + sz += ux * vy - uy * vx + } + const len = Math.hypot(sx, sy, sz) + if (!(len > 0)) return + this.touch(w) + const nx = sx / len, ny = sy / len, nz = sz / len + const { normals, dupList } = this + for (let k = this.dupStart[w]; k < this.dupStart[w + 1]; k++) { + const b = dupList[k] * 3 + normals[b] = nx + normals[b + 1] = ny + normals[b + 2] = nz + } + this.markDirty(w) + } + + private computeAllNormals(): void { + const { tri, wpos, normals, dupList, dupStart } = this + const W = this.weldedVertexCount + const acc = new Float64Array(W * 3) + for (let f = 0; f < tri.length; f += 3) { + const a = tri[f] * 3 + const b = tri[f + 1] * 3 + const c = tri[f + 2] * 3 + const ux = wpos[b] - wpos[a], uy = wpos[b + 1] - wpos[a + 1], uz = wpos[b + 2] - wpos[a + 2] + const vx = wpos[c] - wpos[a], vy = wpos[c + 1] - wpos[a + 1], vz = wpos[c + 2] - wpos[a + 2] + const nx = uy * vz - uz * vy + const ny = uz * vx - ux * vz + const nz = ux * vy - uy * vx + acc[a] += nx; acc[a + 1] += ny; acc[a + 2] += nz + acc[b] += nx; acc[b + 1] += ny; acc[b + 2] += nz + acc[c] += nx; acc[c + 1] += ny; acc[c + 2] += nz + } + for (let w = 0; w < W; w++) { + const o = w * 3 + const len = Math.hypot(acc[o], acc[o + 1], acc[o + 2]) + const inv = len > 0 ? 1 / len : 0 + for (let k = dupStart[w]; k < dupStart[w + 1]; k++) { + const b = dupList[k] * 3 + normals[b] = acc[o] * inv + normals[b + 1] = acc[o + 1] * inv + normals[b + 2] = acc[o + 2] * inv + } + } + } + + // ── Bookkeeping ──────────────────────────────────────────────────────── + + private nextGroup(): number { + return ++this.groupId + } + + private beginCall(): void { + this.callId++ + this.callCount = 0 + } + + private markDirty(w: number): void { + if (this.callMark[w] === this.callId) return + this.callMark[w] = this.callId + this.callList[this.callCount++] = w + } + + /** Records the original state of all duplicates of `w`, the first time the stroke touches it. */ + private touch(w: number): void { + if (this.strokeMark[w] === this.strokeId) return + this.strokeMark[w] = this.strokeId + const start = this.dupStart[w] + const end = this.dupStart[w + 1] + const needed = this.recCount + (end - start) + if (needed > this.recVerts.length) this.growRecord(needed) + const { positions, normals, recVerts, recPos, recNrm, dupList } = this + let c = this.recCount + for (let k = start; k < end; k++) { + const b = dupList[k] + recVerts[c] = b + recPos[c * 3] = positions[b * 3] + recPos[c * 3 + 1] = positions[b * 3 + 1] + recPos[c * 3 + 2] = positions[b * 3 + 2] + recNrm[c * 3] = normals[b * 3] + recNrm[c * 3 + 1] = normals[b * 3 + 1] + recNrm[c * 3 + 2] = normals[b * 3 + 2] + c++ + } + this.recCount = c + } + + private growRecord(needed: number): void { + const size = Math.max(needed, this.recVerts.length * 2, 1024) + const verts = new Uint32Array(size) + verts.set(this.recVerts.subarray(0, this.recCount)) + const pos = new Float32Array(size * 3) + pos.set(this.recPos.subarray(0, this.recCount * 3)) + const nrm = new Float32Array(size * 3) + nrm.set(this.recNrm.subarray(0, this.recCount * 3)) + this.recVerts = verts + this.recPos = pos + this.recNrm = nrm + } + + /** Expands the welded vertices dirtied by the current call to buffer indices. */ + private takeRegion(): DirtyRegion { + const { callList, dupStart, dupList } = this + let total = 0 + for (let i = 0; i < this.callCount; i++) { + const w = callList[i] + total += dupStart[w + 1] - dupStart[w] + } + if (total === 0) return emptyRegion() + const vertices = new Uint32Array(total) + let minIndex = Infinity + let maxIndex = -1 + let c = 0 + for (let i = 0; i < this.callCount; i++) { + const w = callList[i] + for (let k = dupStart[w]; k < dupStart[w + 1]; k++) { + const b = dupList[k] + vertices[c++] = b + if (b < minIndex) minIndex = b + if (b > maxIndex) maxIndex = b + } + } + return { vertices, minIndex, maxIndex } } } diff --git a/src/areas/generate/sculpt/SpatialGrid.ts b/src/areas/generate/sculpt/SpatialGrid.ts new file mode 100644 index 000000000..cd8a48404 --- /dev/null +++ b/src/areas/generate/sculpt/SpatialGrid.ts @@ -0,0 +1,144 @@ +// Uniform dense grid over the welded vertices, for brush sphere queries. +// +// The grid covers the mesh bounding box plus a margin; vertices outside it +// (e.g. dragged far away by grab) are clamped into the border cells, and +// queries clamp their cell range the same way, so a query is always exact +// whatever the radius or how far vertices moved. Cells hold doubly linked +// lists so a moved vertex is re-bucketed in O(1). + +const MAX_CELLS = 1 << 21 + +export class SpatialGrid { + /** Query results: welded vertex ids and squared distances, valid up to the count returned by query(). */ + resultIds: Int32Array + resultDist2: Float64Array + + private readonly pos: Float32Array + private readonly ox: number + private readonly oy: number + private readonly oz: number + private readonly inv: number + private readonly cell: number + private readonly nx: number + private readonly ny: number + private readonly nz: number + private readonly head: Int32Array + private readonly next: Int32Array + private readonly prev: Int32Array + private readonly cellOf: Int32Array + + /** + * @param pos welded positions (xyz per welded vertex), read live by queries. + * @param cellSize target cell edge; grown if the grid would get too large. + */ + constructor(pos: Float32Array, count: number, min: readonly number[], max: readonly number[], cellSize: number) { + this.pos = pos + const ext = Math.max(max[0] - min[0], max[1] - min[1], max[2] - min[2], 0) + let cell = cellSize > 0 && Number.isFinite(cellSize) ? cellSize : ext > 0 ? ext / 64 : 1 + const margin = Math.max(ext * 0.125, cell) + const dims = (c: number): [number, number, number] => [ + Math.max(1, Math.ceil((max[0] - min[0] + 2 * margin) / c)), + Math.max(1, Math.ceil((max[1] - min[1] + 2 * margin) / c)), + Math.max(1, Math.ceil((max[2] - min[2] + 2 * margin) / c)), + ] + let d = dims(cell) + while (d[0] * d[1] * d[2] > MAX_CELLS) { + cell *= 1.25 + d = dims(cell) + } + this.cell = cell + this.inv = 1 / cell + this.ox = min[0] - margin + this.oy = min[1] - margin + this.oz = min[2] - margin + ;[this.nx, this.ny, this.nz] = d + this.head = new Int32Array(d[0] * d[1] * d[2]).fill(-1) + this.next = new Int32Array(count) + this.prev = new Int32Array(count) + this.cellOf = new Int32Array(count) + this.resultIds = new Int32Array(1024) + this.resultDist2 = new Float64Array(1024) + for (let w = 0; w < count; w++) this.link(w, this.cellAt(pos[w * 3], pos[w * 3 + 1], pos[w * 3 + 2])) + } + + private axis(v: number, o: number, n: number): number { + const i = Math.floor((v - o) * this.inv) + // NaN (degenerate input) lands in cell 0. + return i > 0 ? (i < n ? i : n - 1) : 0 + } + + private cellAt(x: number, y: number, z: number): number { + return (this.axis(z, this.oz, this.nz) * this.ny + this.axis(y, this.oy, this.ny)) * this.nx + this.axis(x, this.ox, this.nx) + } + + private link(w: number, c: number): void { + const h = this.head[c] + this.next[w] = h + this.prev[w] = -1 + if (h !== -1) this.prev[h] = w + this.head[c] = w + this.cellOf[w] = c + } + + /** Re-buckets welded vertex `w` after its position changed. */ + update(w: number): void { + const c = this.cellAt(this.pos[w * 3], this.pos[w * 3 + 1], this.pos[w * 3 + 2]) + const old = this.cellOf[w] + if (c === old) return + const p = this.prev[w] + const n = this.next[w] + if (p !== -1) this.next[p] = n + else this.head[old] = n + if (n !== -1) this.prev[n] = p + this.link(w, c) + } + + /** Collects the welded vertices strictly inside the sphere into resultIds/resultDist2; returns their count. */ + query(x: number, y: number, z: number, radius: number): number { + if (!(radius > 0)) return 0 + const r2 = radius * radius + const x0 = this.axis(x - radius, this.ox, this.nx) + const x1 = this.axis(x + radius, this.ox, this.nx) + const y0 = this.axis(y - radius, this.oy, this.ny) + const y1 = this.axis(y + radius, this.oy, this.ny) + const z0 = this.axis(z - radius, this.oz, this.nz) + const z1 = this.axis(z + radius, this.oz, this.nz) + const pos = this.pos + const head = this.head + const next = this.next + let ids = this.resultIds + let dist = this.resultDist2 + let count = 0 + for (let cz = z0; cz <= z1; cz++) { + for (let cy = y0; cy <= y1; cy++) { + const row = (cz * this.ny + cy) * this.nx + for (let cx = x0; cx <= x1; cx++) { + for (let w = head[row + cx]; w !== -1; w = next[w]) { + const dx = pos[w * 3] - x + const dy = pos[w * 3 + 1] - y + const dz = pos[w * 3 + 2] - z + const d2 = dx * dx + dy * dy + dz * dz + if (d2 >= r2) continue + if (count === ids.length) { + const grownIds = new Int32Array(ids.length * 2) + grownIds.set(ids) + const grownDist = new Float64Array(ids.length * 2) + grownDist.set(dist) + ids = this.resultIds = grownIds + dist = this.resultDist2 = grownDist + } + ids[count] = w + dist[count] = d2 + count++ + } + } + } + } + return count + } + + /** Cell edge length actually used. */ + get cellSize(): number { + return this.cell + } +} diff --git a/src/areas/generate/sculpt/testMeshes.ts b/src/areas/generate/sculpt/testMeshes.ts new file mode 100644 index 000000000..571fb0412 --- /dev/null +++ b/src/areas/generate/sculpt/testMeshes.ts @@ -0,0 +1,119 @@ +// Procedural meshes for the sculpt core tests (not a test file itself). + +export interface TestMesh { + positions: Float32Array + index: Uint32Array | null +} + +/** Unit cube with split vertices per face (24 vertices, 12 triangles), like a UV-mapped GLB. */ +export function splitCube(): TestMesh { + const faces: number[][][] = [ + [[1, -1, -1], [1, 1, -1], [1, 1, 1], [1, -1, 1]], + [[-1, -1, 1], [-1, 1, 1], [-1, 1, -1], [-1, -1, -1]], + [[-1, 1, -1], [-1, 1, 1], [1, 1, 1], [1, 1, -1]], + [[-1, -1, 1], [-1, -1, -1], [1, -1, -1], [1, -1, 1]], + [[-1, -1, 1], [1, -1, 1], [1, 1, 1], [-1, 1, 1]], + [[1, -1, -1], [-1, -1, -1], [-1, 1, -1], [1, 1, -1]], + ] + const positions = new Float32Array(24 * 3) + const index = new Uint32Array(36) + faces.forEach((quad, f) => { + quad.forEach((p, k) => positions.set(p, (f * 4 + k) * 3)) + index.set([f * 4, f * 4 + 1, f * 4 + 2, f * 4, f * 4 + 2, f * 4 + 3], f * 6) + }) + return { positions, index } +} + +/** + * Grid in the xy plane over [-1, 1]², z = height(x, y). The triangulation is + * mirrored across x = 0 so the mesh is exactly symmetric (topology included). + */ +export function grid(n: number, height: (x: number, y: number) => number = () => 0): TestMesh { + const side = n + 1 + const positions = new Float32Array(side * side * 3) + for (let j = 0; j < side; j++) { + for (let i = 0; i < side; i++) { + // Build x from the centre outwards so mirrored columns are exact negatives. + const x = (2 * i - n) / n + const y = (2 * j - n) / n + const o = (j * side + i) * 3 + positions[o] = x + positions[o + 1] = y + positions[o + 2] = height(x, y) + } + } + const index = new Uint32Array(n * n * 6) + let t = 0 + for (let j = 0; j < n; j++) { + for (let i = 0; i < n; i++) { + const a = j * side + i + const b = a + 1 + const c = a + side + const d = c + 1 + if (i < n / 2) index.set([a, b, d, a, d, c], t) + else index.set([a, b, c, b, d, c], t) + t += 6 + } + } + return { positions, index } +} + +/** Mirror buffer index of grid vertex `v` across x = 0. */ +export function gridMirror(n: number, v: number): number { + const side = n + 1 + const i = v % side + const j = (v - i) / side + return j * side + (n - i) +} + +/** + * UV sphere (radius 1, centred at the origin) whose longitudes are split into + * `islands` separate vertex sets, so every island border, the 0/2π seam and + * the poles carry duplicated vertices, like a UV-unwrapped GLB. + */ +export function seamSphere(lat: number, lon: number, islands: number): TestMesh { + const per = lon / islands + const islandVerts = (per + 1) * (lat + 1) + const positions = new Float32Array(islands * islandVerts * 3) + const index = new Uint32Array(islands * per * lat * 6) + let t = 0 + for (let k = 0; k < islands; k++) { + const base = k * islandVerts + for (let r = 0; r <= lat; r++) { + const theta = (Math.PI * r) / lat + const st = Math.sin(theta) + const ct = Math.cos(theta) + for (let c = 0; c <= per; c++) { + const phi = (2 * Math.PI * ((k * per + c) % lon)) / lon + const o = (base + r * (per + 1) + c) * 3 + positions[o] = r === 0 || r === lat ? 0 : st * Math.cos(phi) + positions[o + 1] = ct + positions[o + 2] = r === 0 || r === lat ? 0 : st * Math.sin(phi) + } + } + for (let r = 0; r < lat; r++) { + for (let c = 0; c < per; c++) { + const a = base + r * (per + 1) + c + const b = a + 1 + const d = a + per + 1 + const e = d + 1 + index.set([a, b, d, b, e, d], t) + t += 6 + } + } + } + return { positions, index } +} + +/** Turns an indexed mesh into a triangle soup (non-indexed). */ +export function unindex(mesh: TestMesh): TestMesh { + if (!mesh.index) return mesh + const out = new Float32Array(mesh.index.length * 3) + for (let k = 0; k < mesh.index.length; k++) { + const v = mesh.index[k] + out[k * 3] = mesh.positions[v * 3] + out[k * 3 + 1] = mesh.positions[v * 3 + 1] + out[k * 3 + 2] = mesh.positions[v * 3 + 2] + } + return { positions: out, index: null } +} diff --git a/src/areas/generate/sculpt/topology.ts b/src/areas/generate/sculpt/topology.ts new file mode 100644 index 000000000..40232f5a8 --- /dev/null +++ b/src/areas/generate/sculpt/topology.ts @@ -0,0 +1,67 @@ +// Connectivity over welded vertices: welded triangles, vertex → faces and +// vertex → neighbours, all as CSR typed arrays. + +export interface Csr { + start: Uint32Array + list: Uint32Array +} + +/** Triangle corners expressed as welded vertex ids. `index` null → consecutive triples (triangle soup). */ +export function weldedTriangles(index: Uint32Array | Uint16Array | null, vertexCount: number, weldOf: Uint32Array): Uint32Array { + if (index) { + const length = index.length - (index.length % 3) + const tri = new Uint32Array(length) + for (let k = 0; k < length; k++) { + const v = index[k] + if (v >= vertexCount) throw new RangeError(`SculptMesh: index ${v} out of range (${vertexCount} vertices)`) + tri[k] = weldOf[v] + } + return tri + } + const length = vertexCount - (vertexCount % 3) + const tri = new Uint32Array(length) + for (let k = 0; k < length; k++) tri[k] = weldOf[k] + return tri +} + +/** Welded vertex → incident triangles. */ +export function vertexFaces(tri: Uint32Array, count: number): Csr { + const start = new Uint32Array(count + 1) + for (let k = 0; k < tri.length; k++) start[tri[k] + 1]++ + for (let w = 0; w < count; w++) start[w + 1] += start[w] + const fill = start.slice(0, count) + const list = new Uint32Array(tri.length) + for (let k = 0; k < tri.length; k++) list[fill[tri[k]]++] = (k / 3) | 0 + return { start, list } +} + +/** Welded vertex → distinct edge neighbours (from the triangles). */ +export function vertexNeighbours(tri: Uint32Array, faces: Csr, count: number): Csr { + const mark = new Int32Array(count).fill(-1) + const start = new Uint32Array(count + 1) + // Pass 1: count distinct neighbours. + for (let w = 0; w < count; w++) { + let c = 0 + for (let k = faces.start[w]; k < faces.start[w + 1]; k++) { + const f = faces.list[k] * 3 + for (let j = 0; j < 3; j++) { + const u = tri[f + j] + if (u !== w && mark[u] !== w) { mark[u] = w; c++ } + } + } + start[w + 1] = start[w] + c + } + mark.fill(-1) + const list = new Uint32Array(start[count]) + for (let w = 0; w < count; w++) { + let o = start[w] + for (let k = faces.start[w]; k < faces.start[w + 1]; k++) { + const f = faces.list[k] * 3 + for (let j = 0; j < 3; j++) { + const u = tri[f + j] + if (u !== w && mark[u] !== w) { mark[u] = w; list[o++] = u } + } + } + } + return { start, list } +} diff --git a/src/areas/generate/sculpt/weld.ts b/src/areas/generate/sculpt/weld.ts new file mode 100644 index 000000000..cea595d7f --- /dev/null +++ b/src/areas/generate/sculpt/weld.ts @@ -0,0 +1,126 @@ +// Weld map: buffer vertices closer than `epsilon` (UV-seam duplicates, or the +// corners of a triangle soup) become one logical "welded" vertex. +// +// Vertices are bucketed in an integer-keyed open-addressing hash grid with +// cells of 4 × epsilon; a vertex only probes the neighbouring cells it is +// within epsilon of, so almost every vertex costs a single probe. + +export interface WeldMap { + /** Number of welded vertices. */ + count: number + /** Buffer vertex → welded vertex. */ + weldOf: Uint32Array + /** CSR: buffer vertices of welded vertex w are dupList[dupStart[w] .. dupStart[w + 1]). The first one is its representative. */ + dupStart: Uint32Array + dupList: Uint32Array +} + +function hashCell(x: number, y: number, z: number, mask: number): number { + return (Math.imul(x, 73856093) ^ Math.imul(y, 19349663) ^ Math.imul(z, 83492791)) & mask +} + +export function buildWeldMap( + positions: Float32Array, + vertexCount: number, + epsilon: number, + origin: readonly [number, number, number], +): WeldMap { + const n = vertexCount + const eps = epsilon > 0 && Number.isFinite(epsilon) ? epsilon : 0 + const cell = eps > 0 ? eps * 4 : 1 + const inv = 1 / cell + const epsCell = eps * inv + const eps2 = eps * eps + const [ox, oy, oz] = origin + + let cap = 16 + while (cap < n * 2) cap *= 2 + const mask = cap - 1 + const keyX = new Int32Array(cap) + const keyY = new Int32Array(cap) + const keyZ = new Int32Array(cap) + const head = new Int32Array(cap).fill(-1) + // Per welded vertex: next welded vertex in the same cell, and its representative buffer vertex. + const next = new Int32Array(n) + const rep = new Uint32Array(n) + const weldOf = new Uint32Array(n) + let count = 0 + + const find = (x: number, y: number, z: number): number => { + let h = hashCell(x, y, z, mask) + for (;;) { + if (head[h] === -1) return -1 + if (keyX[h] === x && keyY[h] === y && keyZ[h] === z) return h + h = (h + 1) & mask + } + } + + for (let i = 0; i < n; i++) { + const px = positions[i * 3] + const py = positions[i * 3 + 1] + const pz = positions[i * 3 + 2] + const fx = (px - ox) * inv + const fy = (py - oy) * inv + const fz = (pz - oz) * inv + const qx = Math.floor(fx) + const qy = Math.floor(fy) + const qz = Math.floor(fz) + const rx = fx - qx + const ry = fy - qy + const rz = fz - qz + const x0 = rx < epsCell ? qx - 1 : qx + const x1 = 1 - rx <= epsCell ? qx + 1 : qx + const y0 = ry < epsCell ? qy - 1 : qy + const y1 = 1 - ry <= epsCell ? qy + 1 : qy + const z0 = rz < epsCell ? qz - 1 : qz + const z1 = 1 - rz <= epsCell ? qz + 1 : qz + + let found = -1 + search: for (let cx = x0; cx <= x1; cx++) { + for (let cy = y0; cy <= y1; cy++) { + for (let cz = z0; cz <= z1; cz++) { + const slot = find(cx | 0, cy | 0, cz | 0) + if (slot < 0) continue + for (let w = head[slot]; w !== -1; w = next[w]) { + const r = rep[w] * 3 + const dx = positions[r] - px + const dy = positions[r + 1] - py + const dz = positions[r + 2] - pz + if (dx * dx + dy * dy + dz * dz <= eps2) { found = w; break search } + } + } + } + } + + if (found >= 0) { + weldOf[i] = found + continue + } + const w = count++ + rep[w] = i + weldOf[i] = w + // Keys wrap to int32 for tiny epsilons; aliased cells only cost extra distance checks. + const kx = qx | 0 + const ky = qy | 0 + const kz = qz | 0 + let h = hashCell(kx, ky, kz, mask) + while (head[h] !== -1 && !(keyX[h] === kx && keyY[h] === ky && keyZ[h] === kz)) h = (h + 1) & mask + if (head[h] === -1) { + keyX[h] = kx + keyY[h] = ky + keyZ[h] = kz + } + next[w] = head[h] + head[h] = w + } + + const dupStart = new Uint32Array(count + 1) + for (let i = 0; i < n; i++) dupStart[weldOf[i] + 1]++ + for (let w = 0; w < count; w++) dupStart[w + 1] += dupStart[w] + const fill = dupStart.slice(0, count) + const dupList = new Uint32Array(n) + // Increasing i, so the representative (lowest index) comes first. + for (let i = 0; i < n; i++) dupList[fill[weldOf[i]]++] = i + + return { count, weldOf, dupStart, dupList } +} From 2521a1b92c8a102c08886ce9a9434bce6321255a Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 10:51:00 +0100 Subject: [PATCH 035/108] feat(generate): sculpt mode in the viewer and the edit panel Sculpt section in the Edit panel (Grab, Smooth, Inflate, Flatten, radius, strength, X/Y/Z symmetry), stroke input and brush ring in the viewer, local stroke undo/redo, and a commit to POST /optimize/patch-vertices that lands as one global mesh edit. Pending strokes are saved before export, Edit ops, chat messages, global undo/redo, import and library open. --- package.json | 2 +- src/areas/generate/GeneratePage.tsx | 99 +++- src/areas/generate/components/ChatPanel.tsx | 10 +- src/areas/generate/components/EditPanel.tsx | 12 +- .../generate/components/SculptSection.tsx | 199 +++++++ src/areas/generate/components/Viewer3D.tsx | 70 ++- src/areas/generate/sculpt/SculptLayer.tsx | 547 ++++++++++++++++++ src/areas/generate/sculpt/sculptLogic.test.ts | 194 +++++++ src/areas/generate/sculpt/sculptLogic.ts | 231 ++++++++ src/areas/generate/sculpt/sculptStore.ts | 183 ++++++ src/shared/hooks/useApi.ts | 16 +- 11 files changed, 1525 insertions(+), 38 deletions(-) create mode 100644 src/areas/generate/components/SculptSection.tsx create mode 100644 src/areas/generate/sculpt/SculptLayer.tsx create mode 100644 src/areas/generate/sculpt/sculptLogic.test.ts create mode 100644 src/areas/generate/sculpt/sculptLogic.ts create mode 100644 src/areas/generate/sculpt/sculptStore.ts diff --git a/package.json b/package.json index d6117ff35..e89cca40e 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/GeneratePage.tsx b/src/areas/generate/GeneratePage.tsx index 1fdd98fd2..9f77ef102 100644 --- a/src/areas/generate/GeneratePage.tsx +++ b/src/areas/generate/GeneratePage.tsx @@ -10,6 +10,8 @@ import Viewer3D from './components/Viewer3D' import WorkflowPanel from './components/WorkflowPanel' import EditPanel from './components/EditPanel' import { meshPathFromUrl } from './meshEditPanel' +import { commitSculpt, useSculptStore } from './sculpt/sculptStore' +import { sculptKeyAction } from './sculpt/sculptLogic' import { getDefaultAssetLibraryService } from './assetLibraryService' import { buildOrcaSlicerDeepLink, canOpenInOrcaSlicer } from './orcaSlicerLink' import { resolveAssetLibraryOpenTarget, type ProjectedAssetLibraryEntry } from './assetLibraryProjection' @@ -707,15 +709,57 @@ export default function GeneratePage(): JSX.Element { const { optimizeMesh, smoothMesh, importMesh } = useApi() const assetLibraryService = useMemo(() => getDefaultAssetLibraryService(), []) + const sculpting = useSculptStore((s) => s.active) + + // Global mesh undo/redo. Pending sculpt strokes are saved first, so undo + // steps back over them as one edit (in sculpt mode Ctrl+Z is the stroke undo). + const undoMeshEdit = useCallback(async () => { if (await commitSculpt()) undoMesh() }, [undoMesh]) + const redoMeshEdit = useCallback(async () => { if (await commitSculpt()) redoMesh() }, [redoMesh]) + useEffect(() => { const handler = (e: KeyboardEvent) => { if (!e.ctrlKey && !e.metaKey) return - if (e.key === 'z') { e.preventDefault(); if (gizmoUndoRef.current?.()) return; undoMesh() } - if (e.key === 'y') { e.preventDefault(); redoMesh() } + if (useSculptStore.getState().active) return // sculpt shortcuts below + if (e.key === 'z') { e.preventDefault(); if (gizmoUndoRef.current?.()) return; void undoMeshEdit() } + if (e.key === 'y') { e.preventDefault(); void redoMeshEdit() } + } + window.addEventListener('keydown', handler) + return () => window.removeEventListener('keydown', handler) + }, [undoMeshEdit, redoMeshEdit]) + + // Sculpt-mode shortcuts (see sculptKeyAction). Ignored while typing. + useEffect(() => { + const handler = (e: KeyboardEvent) => { + const sculpt = useSculptStore.getState() + if (!sculpt.active) return + const el = document.activeElement as HTMLElement | null + if (el && (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el.isContentEditable)) return + const action = sculptKeyAction(e) + if (!action) return + e.preventDefault() + switch (action.type) { + case 'undo': sculpt.undo(); break + case 'redo': sculpt.redo(); break + case 'exit': void sculpt.exit(); break + case 'radius': sculpt.stepRadius(action.direction); break + case 'brush': sculpt.setBrush(action.kind); break + } } window.addEventListener('keydown', handler) return () => window.removeEventListener('keydown', handler) - }, [undoMesh, redoMesh]) + }, []) + + // Sculpting replaces the transform tools. + useEffect(() => { + if (sculpting) setGizmoMode(null) + }, [sculpting]) + + // Leaving the page leaves sculpt mode; the viewer saves pending strokes as it unmounts. + useEffect(() => () => useSculptStore.setState({ active: false }), []) + + const closeEditPanel = useCallback(async () => { + if (await useSculptStore.getState().exit()) setEditPanelOpen(false) + }, []) const hasModel = currentJob?.status === 'done' && !!currentJob.outputUrl const showOpenInSlicer = hasModel && canOpenInOrcaSlicer(currentJob?.outputUrl) @@ -743,6 +787,7 @@ export default function GeneratePage(): JSX.Element { const handler = (e: KeyboardEvent) => { const el = document.activeElement as HTMLElement | null if (el && (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el.isContentEditable)) return + if (useSculptStore.getState().active) return // W/R/S/Esc belong to sculpt mode if (e.key === 'Escape') { setGizmoMode((m) => (m ? null : m)); return } if (!meshSelected && !selectedPointLightId) return const k = e.key.toLowerCase() @@ -766,14 +811,21 @@ export default function GeneratePage(): JSX.Element { setTimeout(() => setUnloadStatus('idle'), 2000) } - function handleExport(format: 'glb' | 'obj' | 'stl' | 'ply') { - if (!currentJob?.outputUrl) return + /** Saves pending sculpt strokes, then returns the mesh URL to read (or null). */ + async function committedMeshUrl(): Promise { + if (!(await commitSculpt())) return null + return useAppStore.getState().currentJob?.outputUrl ?? null + } + + async function handleExport(format: 'glb' | 'obj' | 'stl' | 'ply') { + const outputUrl = await committedMeshUrl() + if (!outputUrl) return const stem = `modly-${Date.now()}` const link = document.createElement('a') if (format === 'glb') { - link.href = `${apiUrl}${currentJob.outputUrl}` + link.href = `${apiUrl}${outputUrl}` } else { - const path = encodeURIComponent(currentJob.outputUrl.replace('/workspace/', '')) + const path = encodeURIComponent(outputUrl.replace('/workspace/', '')) link.href = `${apiUrl}/optimize/export?path=${path}&format=${format}` } link.download = `${stem}.${format}` @@ -781,9 +833,10 @@ export default function GeneratePage(): JSX.Element { } async function handleOpenInOrcaSlicer() { - if (!currentJob?.outputUrl) return + const outputUrl = await committedMeshUrl() + if (!outputUrl) return try { - const link = buildOrcaSlicerDeepLink(apiUrl, currentJob.outputUrl) + const link = buildOrcaSlicerDeepLink(apiUrl, outputUrl) const result = await window.electron.slicer.open(link) if (!result.success) { // Deliberately not "make sure it is installed": the main process cannot @@ -796,6 +849,8 @@ export default function GeneratePage(): JSX.Element { } async function handleImportMesh() { + // Save the strokes before the model is replaced (else they'd be dropped). + if (!(await commitSculpt())) return const filePath = await window.electron.fs.selectMeshFile() if (!filePath) return setOpenPanel(null) @@ -855,6 +910,7 @@ export default function GeneratePage(): JSX.Element { setLibraryError(describeAssetLibraryOpenability(selectedEntry)) return } + if (!(await commitSculpt())) return setLibraryOpening(true) setLibraryError(null) @@ -883,10 +939,11 @@ export default function GeneratePage(): JSX.Element { } async function handleSmooth(iterations: number) { - if (!currentJob?.outputUrl) return setSmoothing(true) try { - const path = meshPathFromUrl(currentJob.outputUrl) + const outputUrl = await committedMeshUrl() + if (!outputUrl) return + const path = meshPathFromUrl(outputUrl) const { url } = await smoothMesh(path, iterations) applyMeshEdit(url) setOpenPanel(null) @@ -898,10 +955,11 @@ export default function GeneratePage(): JSX.Element { } async function handleDecimate(targetFaces: number) { - if (!currentJob?.outputUrl) return setDecimating(true) try { - const path = meshPathFromUrl(currentJob.outputUrl) + const outputUrl = await committedMeshUrl() + if (!outputUrl) return + const path = meshPathFromUrl(outputUrl) const { url } = await optimizeMesh(path, targetFaces) applyMeshEdit(url) setOpenPanel(null) @@ -962,7 +1020,7 @@ export default function GeneratePage(): JSX.Element { {/* Undo / Redo */} + ) : ( + + )} + + + {!active && block && !block.transient && ( +

{block.reason}

+ )} + + {active && ( +
+
+ {BRUSH_KINDS.map((kind) => ( + + ))} +
+

{BRUSH_LABELS[brush].hint}

+ + + + + +
+ Symmetry +
+ {AXES.map((axis) => { + const on = symmetry.includes(axis) + return ( + + ) + })} +
+
+ +

+ {dirty + ? `${strokes} unsaved stroke${strokes === 1 ? '' : 's'} — saved when you click Done.` + : 'No unsaved strokes.'} +

+ {error &&

{error}

} + +
+ {SHORTCUTS.map(([k, v]) => ( +
+
{k}
+
{v}
+
+ ))} +
+ +

Mesh tools are disabled while sculpting. Click Done to save the strokes and use them.

+
+ )} + + ) +} diff --git a/src/areas/generate/components/Viewer3D.tsx b/src/areas/generate/components/Viewer3D.tsx index a2621cc64..71d4a3219 100644 --- a/src/areas/generate/components/Viewer3D.tsx +++ b/src/areas/generate/components/Viewer3D.tsx @@ -18,6 +18,8 @@ import { useAppStore } from '@shared/stores/appStore' import { ViewerToolbar, type ViewMode } from './ViewerToolbar' import type { LightSettings, PointLight } from '@shared/stores/appStore' import { DEFAULT_LIGHT_SETTINGS } from '@shared/stores/appStore' +import SculptLayer from '../sculpt/SculptLayer' +import { useSculptStore } from '../sculpt/sculptStore' export type GizmoMode = 'translate' | 'rotate' | 'scale' @@ -190,23 +192,32 @@ function ModelLoadError(): JSX.Element { interface MeshModelProps { url: string + /** Workspace URL of the file (currentJob.outputUrl): what sculpt commits patch. */ + outputUrl: string jobId: string viewMode: ViewMode selected: boolean + /** Sculpt mode: clicks paint instead of selecting. */ + sculpting: boolean onStats: (stats: { vertices: number; triangles: number }) => void onSelect: () => void onObject: (obj: THREE.Object3D | null) => void } -function MeshModel({ url, jobId, viewMode, selected, onStats, onSelect, onObject }: MeshModelProps): JSX.Element { - const extension = url.split('?')[0]?.split('.').pop()?.toLowerCase() - const common = { url, jobId, viewMode, selected, onStats, onSelect, onObject } - return extension === 'obj' ? : +function MeshModel(props: MeshModelProps): JSX.Element { + const extension = props.url.split('?')[0]?.split('.').pop()?.toLowerCase() + return extension === 'obj' ? : } function GltfMeshModel(props: MeshModelProps): JSX.Element { - const { scene } = useGLTF(props.url) - return + const { scene, parser } = useGLTF(props.url) + return ( + <> + + {/* Keyed by URL: a new mesh file starts a new sculpt session. */} + + + ) } function ObjMeshModel(props: MeshModelProps): JSX.Element { @@ -218,6 +229,7 @@ function SceneMeshModel({ url, viewMode, selected, + sculpting, onStats, onSelect, onObject, @@ -351,7 +363,7 @@ function SceneMeshModel({ ) @@ -945,6 +957,10 @@ export default function Viewer3D({ const [meshObject, setMeshObject] = useState(null) + const sculpting = useSculptStore((s) => s.active) + const sculptStatus = useSculptStore((s) => s.status) + const sculptError = useSculptStore((s) => s.error) + // Local gizmo-transform history (live TRS), undoable with Ctrl+Z. A snapshot // is taken when a drag starts and committed on release only if it changed. const transformHistory = useRef([]) @@ -969,11 +985,17 @@ export default function Viewer3D({ // Reset view state when model changes useEffect(() => { setSelected(false) - setViewMode('solid') + // A sculpt commit swaps in the patched file: keep the sculptor's view mode. + if (!useSculptStore.getState().active) setViewMode('solid') setStoreMeshStats(null) // eslint-disable-next-line react-hooks/exhaustive-deps -- reset only when the model changes; setters are stable }, [modelUrl]) + // Sculpt mode owns the left button: no selection, so no transform gizmo. + useEffect(() => { + if (sculpting) setSelected(false) + }, [sculpting, setSelected]) + // Clear the shared selection when the viewer unmounts — the store would // otherwise keep it set and flash a stale selection on the next mount. useEffect(() => () => setSelected(false), [setSelected]) @@ -983,7 +1005,7 @@ export default function Viewer3D({ const handler = (e: KeyboardEvent) => { if (e.key !== 'Delete') return if (document.activeElement instanceof HTMLInputElement) return - if (!selected) return + if (!selected || useSculptStore.getState().active) return setCurrentJob(null) setSelected(false) } @@ -1126,9 +1148,11 @@ export default function Viewer3D({ ))} - {selected && meshObject && gizmoMode === 'translate' && ( + {!sculpting && selected && meshObject && gizmoMode === 'translate' && ( )} - {selected && meshObject && gizmoMode === 'rotate' && ( + {!sculpting && selected && meshObject && gizmoMode === 'rotate' && ( )} - {selected && meshObject && gizmoMode === 'scale' && ( + {!sculpting && selected && meshObject && gizmoMode === 'scale' && ( )} @@ -1204,11 +1228,31 @@ export default function Viewer3D({ )} + {/* Sculpt status: preparing / saving / last error */} + {sculpting && (sculptStatus !== 'idle' || sculptError) && ( +
+ {sculptStatus !== 'idle' ? ( +

+ + + + {sculptStatus === 'preparing' ? 'Preparing mesh…' : 'Saving sculpt…'} +

+ ) : ( +

+ {sculptError} +

+ )} +
+ )} + {/* Bottom-right hint */} {modelUrl && (

- {selected + {sculpting + ? <>Drag on the mesh to sculpt • drag outside to orbit • Esc to finish + : selected ? <>Click mesh to select • Delete to remove : 'Drag to rotate \u2022 Scroll to zoom' } diff --git a/src/areas/generate/sculpt/SculptLayer.tsx b/src/areas/generate/sculpt/SculptLayer.tsx new file mode 100644 index 000000000..e8be4c477 --- /dev/null +++ b/src/areas/generate/sculpt/SculptLayer.tsx @@ -0,0 +1,547 @@ +// Viewer side of sculpt mode, mounted next to a glTF model (one instance per +// model URL). Turns left-drags on the mesh into SculptMesh strokes, uploads the +// touched vertex ranges, draws the brush ring, and registers the session that +// commits the strokes back to the mesh file. +// +// Blender-like input: a left press that hits the mesh starts a stroke (orbit is +// suspended for it); a press on empty space, right/middle drag and the wheel +// still drive OrbitControls. + +import { useEffect, useMemo, useRef } from 'react' +import { useFrame, useThree } from '@react-three/fiber' +import * as THREE from 'three' +import { useAppStore } from '@shared/stores/appStore' +import { useApi } from '@shared/hooks/useApi' +import { apiErrorMessage, meshPathFromUrl } from '../meshEditPanel' +import { SculptMesh } from './SculptMesh' +import type { BrushKind, BrushSettings, DirtyRegion, Vec3 } from './types' +import { useSculptStore, type SculptSession, type SculptStrokeEntry } from './sculptStore' +import { + attributeUpdateRange, + buildVertexPatch, + mergeRange, + pushStroke, + redoStroke, + sessionEndOutcome, + toLocalRadius, + undoStroke, + worldRadius, + type IndexRange, + type VertexPatch, +} from './sculptLogic' + +/** GLTFLoader's parser, reduced to what the commit needs. */ +interface GltfParserLike { + associations: Map +} + +interface SculptTarget { + key: string + /** A THREE.Mesh using the geometry: its glTF association names the primitive to patch. */ + mesh: THREE.Mesh + geometry: THREE.BufferGeometry + core: SculptMesh + /** Vertex range touched since the last GPU upload. */ + pending: IndexRange | null +} + +interface Stroke { + target: SculptTarget + mesh: THREE.Mesh + kind: BrushKind + /** Grab: camera-facing plane through the start point (world space). */ + plane: THREE.Plane | null + /** Grab: surface normal at the start point (local space). */ + startNormal: Vec3 +} + +interface Drag { + pointerId: number + stroke: Stroke | null + released: boolean +} + +const RING_COLOR = '#a78bfa' +const Z_AXIS = new THREE.Vector3(0, 0, 1) + +function vec(v: THREE.Vector3): Vec3 { + return [v.x, v.y, v.z] +} + +function isSculptable(obj: THREE.Object3D): obj is THREE.Mesh { + const mesh = obj as THREE.Mesh & { isSkinnedMesh?: boolean; isInstancedMesh?: boolean } + return mesh.isMesh === true + && !mesh.isSkinnedMesh + && !mesh.isInstancedMesh + && !!mesh.geometry?.getAttribute('position') + && !mesh.geometry.morphAttributes?.position?.length +} + +/** + * The geometry's attribute as a plain, non-normalized Float32 vec3 attribute, + * so the sculpt core can mutate its array in place. Interleaved, quantized or + * non-float data is copied into a fresh Float32BufferAttribute first. + */ +function float32Attribute(geometry: THREE.BufferGeometry, name: 'position' | 'normal'): THREE.BufferAttribute | null { + const attr = geometry.getAttribute(name) as THREE.BufferAttribute | THREE.InterleavedBufferAttribute | undefined + if (!attr) return null + const plain = !(attr as THREE.InterleavedBufferAttribute).isInterleavedBufferAttribute + && attr.array instanceof Float32Array + && !attr.normalized + && attr.itemSize === 3 + if (plain) return attr as THREE.BufferAttribute + const out = new Float32Array(attr.count * 3) + for (let i = 0; i < attr.count; i++) { + out[i * 3] = attr.getX(i) + out[i * 3 + 1] = attr.getY(i) + out[i * 3 + 2] = attr.getZ(i) + } + const fresh = new THREE.Float32BufferAttribute(out, 3) + geometry.setAttribute(name, fresh) + return fresh +} + +function indexArray(geometry: THREE.BufferGeometry): Uint32Array | Uint16Array | null { + const array = geometry.index?.array + if (!array) return null + if (array instanceof Uint32Array || array instanceof Uint16Array) return array + return Uint32Array.from(array as ArrayLike) +} + +function createTarget(mesh: THREE.Mesh): SculptTarget { + const geometry = mesh.geometry + const position = float32Attribute(geometry, 'position') + if (!position) throw new Error('This part has no vertex positions.') + const normal = float32Attribute(geometry, 'normal') + const core = new SculptMesh( + position.array as Float32Array, + normal ? normal.array as Float32Array : null, + indexArray(geometry), + ) + if (!normal) geometry.setAttribute('normal', new THREE.BufferAttribute(core.normals, 3)) + return { key: geometry.uuid, mesh, geometry, core, pending: null } +} + +function uploadPending(target: SculptTarget): void { + if (!target.pending) return + const { start, count } = attributeUpdateRange(target.pending, 3) + for (const name of ['position', 'normal'] as const) { + const attr = target.geometry.getAttribute(name) as THREE.BufferAttribute | undefined + if (!attr) continue + attr.addUpdateRange(start, count) + attr.needsUpdate = true + } + target.pending = null +} + +/** BVH and bounds follow the moved vertices (end of a stroke, undo, redo). */ +function refreshSpatialData(target: SculptTarget): void { + const geometry = target.geometry as THREE.BufferGeometry & { boundsTree?: { refit: () => void } } + geometry.boundsTree?.refit() + geometry.computeBoundingBox() + geometry.computeBoundingSphere() +} + +function markDirty(target: SculptTarget, region: DirtyRegion): void { + target.pending = mergeRange(target.pending, { min: region.minIndex, max: region.maxIndex }) +} + +function errorText(err: unknown): string { + return apiErrorMessage(err) +} + +export default function SculptLayer({ + scene, + parser, + outputUrl, +}: { + scene: THREE.Object3D + parser: GltfParserLike + /** Workspace URL of the mesh file (currentJob.outputUrl) these strokes apply to. */ + outputUrl: string +}): JSX.Element { + const active = useSculptStore((s) => s.active) + const camera = useThree((s) => s.camera) + const gl = useThree((s) => s.gl) + const controls = useThree((s) => s.controls) as { enabled: boolean } | null + const { patchMeshVertices } = useApi() + + const targets = useRef(new Map()) + const drag = useRef(null) + const ringRef = useRef(null) + const hoverNdc = useRef(null) + const diagonal = useRef(null) + const activeRef = useRef(active) + activeRef.current = active + + // Latest values for the long-lived DOM listeners and session callbacks. + const live = useRef({ camera, gl, controls, patchMeshVertices }) + live.current = { camera, gl, controls, patchMeshVertices } + + const raycaster = useMemo(() => { + const r = new THREE.Raycaster() as THREE.Raycaster & { firstHitOnly?: boolean } + r.firstHitOnly = true + return r + }, []) + + const meshes = useMemo(() => { + const list: THREE.Mesh[] = [] + scene.traverse((obj) => { if (isSculptable(obj)) list.push(obj) }) + return list + }, [scene]) + + // Shared helpers, rebuilt only with the model. + const api = useMemo(() => { + const ndc = new THREE.Vector2() + + const setRay = (clientX: number, clientY: number): THREE.Ray => { + const rect = live.current.gl.domElement.getBoundingClientRect() + ndc.set(((clientX - rect.left) / rect.width) * 2 - 1, -((clientY - rect.top) / rect.height) * 2 + 1) + raycaster.setFromCamera(ndc, live.current.camera) + return raycaster.ray + } + + const hitTest = (objects: THREE.Object3D[]): THREE.Intersection | null => { + const hits = raycaster.intersectObjects(objects, false) as THREE.Intersection[] + return hits.find((h) => h.object.visible && h.face) ?? null + } + + const modelDiagonal = (): number => { + if (diagonal.current === null) { + diagonal.current = new THREE.Box3().setFromObject(scene).getSize(new THREE.Vector3()).length() + } + return diagonal.current + } + + const localRadius = (mesh: THREE.Mesh, kind: BrushKind): number => { + const { brushes } = useSculptStore.getState() + mesh.updateWorldMatrix(true, false) + return toLocalRadius(worldRadius(brushes[kind].radius, modelDiagonal()), mesh.matrixWorld.getMaxScaleOnAxis()) + } + + const reportError = (prefix: string, err: unknown) => { + console.warn(`[Sculpt] ${prefix}:`, err) + useSculptStore.getState().setError(`${prefix}: ${errorText(err)}`) + } + + const setOrbit = (enabled: boolean) => { + if (live.current.controls) live.current.controls.enabled = enabled + } + + const startStroke = (target: SculptTarget, hit: THREE.Intersection, ev: PointerEvent): Stroke | null => { + const state = useSculptStore.getState() + // Shift held = smooth, whatever the brush (as in Blender). + const kind: BrushKind = ev.shiftKey ? 'smooth' : state.brush + const mesh = hit.object + const settings: BrushSettings = { + kind, + radius: localRadius(mesh, kind), + strength: state.brushes[kind].strength, + invert: ev.ctrlKey || ev.metaKey, + symmetry: [...state.symmetry], + } + const point = mesh.worldToLocal(hit.point.clone()) + const normal = hit.face!.normal.clone() + try { + markDirty(target, target.core.beginStroke(settings, vec(point), vec(normal))) + } catch (err) { + reportError('Sculpt stroke failed', err) + return null + } + let plane: THREE.Plane | null = null + if (kind === 'grab') { + const facing = live.current.camera.getWorldDirection(new THREE.Vector3()) + plane = new THREE.Plane().setFromNormalAndCoplanarPoint(facing, hit.point) + } + return { target, mesh, kind, plane, startNormal: vec(normal) } + } + + const continueStroke = (stroke: Stroke, ev: PointerEvent) => { + const ray = setRay(ev.clientX, ev.clientY) + let point: THREE.Vector3 | null = null + let normal: Vec3 = stroke.startNormal + if (stroke.plane) { + const world = ray.intersectPlane(stroke.plane, new THREE.Vector3()) + if (world) point = stroke.mesh.worldToLocal(world) + } else { + const hit = hitTest([stroke.mesh]) + if (hit) { + point = stroke.mesh.worldToLocal(hit.point.clone()) + normal = vec(hit.face!.normal) + } + } + if (!point) return + try { + markDirty(stroke.target, stroke.target.core.strokeTo(vec(point), normal)) + } catch (err) { + reportError('Sculpt stroke failed', err) + abortStroke() + } + } + + const finishStroke = (stroke: Stroke) => { + let record = null + try { + record = stroke.target.core.endStroke() + } catch (err) { + reportError('Sculpt stroke failed', err) + } + uploadPending(stroke.target) + refreshSpatialData(stroke.target) + diagonal.current = null + if (record) { + const store = useSculptStore.getState() + const entry: SculptStrokeEntry = { meshKey: stroke.target.key, record } + store.setHistory(pushStroke(store.history, entry)) + } + } + + const onMove = (ev: PointerEvent) => { + const d = drag.current + if (!d || ev.pointerId !== d.pointerId || !d.stroke) return + continueStroke(d.stroke, ev) + hoverNdc.current = null + updateRingFromEvent(ev, d.stroke.mesh) + } + + const endDrag = () => { + const d = drag.current + drag.current = null + window.removeEventListener('pointermove', onMove) + window.removeEventListener('pointerup', onUp) + window.removeEventListener('pointercancel', onUp) + if (d?.stroke) finishStroke(d.stroke) + setOrbit(true) + } + + function onUp(ev: PointerEvent) { + const d = drag.current + if (!d || ev.pointerId !== d.pointerId) return + d.released = true + // Still preparing the mesh: the stroke (if any) ends once it starts. + if (useSculptStore.getState().status === 'preparing') return + endDrag() + } + + function abortStroke() { + const d = drag.current + if (d?.stroke) { + try { d.stroke.target.core.endStroke() } catch { /* already reported */ } + uploadPending(d.stroke.target) + d.stroke = null + } + endDrag() + } + + // Ring under the cursor, sized to the brush and lying on the surface. + const placeRing = (hit: THREE.Intersection | null) => { + const ring = ringRef.current + if (!ring) return + if (!hit) { ring.visible = false; return } + const mesh = hit.object + const state = useSculptStore.getState() + const kind = state.brush + const radius = worldRadius(state.brushes[kind].radius, modelDiagonal()) + const normal = hit.face!.normal.clone().applyNormalMatrix(new THREE.Matrix3().getNormalMatrix(mesh.matrixWorld)) + if (normal.lengthSq() < 1e-12) normal.set(0, 1, 0) + ring.position.copy(hit.point).addScaledVector(normal, radius * 0.01) + ring.quaternion.setFromUnitVectors(Z_AXIS, normal.normalize()) + ring.scale.setScalar(Math.max(radius, 1e-6)) + ring.visible = true + } + + function updateRingFromEvent(ev: PointerEvent, mesh: THREE.Mesh) { + setRay(ev.clientX, ev.clientY) + placeRing(hitTest([mesh])) + } + + const onDown = (ev: PointerEvent) => { + if (!activeRef.current || ev.button !== 0 || drag.current) return + const store = useSculptStore.getState() + if (store.status !== 'idle') return + setRay(ev.clientX, ev.clientY) + const hit = hitTest(meshes) + if (!hit) return // empty space: let OrbitControls orbit + + // Runs before OrbitControls' own pointerdown (capture phase), so it never starts. + setOrbit(false) + store.setError(null) + const d: Drag = { pointerId: ev.pointerId, stroke: null, released: false } + drag.current = d + window.addEventListener('pointermove', onMove) + window.addEventListener('pointerup', onUp) + window.addEventListener('pointercancel', onUp) + + const geometryKey = hit.object.geometry.uuid + const existing = targets.current.get(geometryKey) + if (existing) { + d.stroke = startStroke(existing, hit, ev) + if (!d.stroke) endDrag() + return + } + + // First stroke on this part: build its sculpt data (can take ~1 s on big + // meshes). Let React paint "Preparing mesh…" before blocking the thread. + store.setStatus('preparing') + window.setTimeout(() => { + let target: SculptTarget | null = null + try { + target = createTarget(hit.object) + targets.current.set(geometryKey, target) + } catch (err) { + reportError('Could not prepare the mesh for sculpting', err) + } + useSculptStore.getState().setStatus('idle') + if (drag.current !== d) return + if (target && activeRef.current) d.stroke = startStroke(target, hit, ev) + if (!d.stroke || d.released) endDrag() + }, 30) + } + + const onHover = (ev: PointerEvent) => { + if (!activeRef.current || drag.current) return + const rect = live.current.gl.domElement.getBoundingClientRect() + hoverNdc.current = new THREE.Vector2( + ((ev.clientX - rect.left) / rect.width) * 2 - 1, + -((ev.clientY - rect.top) / rect.height) * 2 + 1, + ) + } + + const onLeave = () => { + hoverNdc.current = null + if (ringRef.current && !drag.current) ringRef.current.visible = false + } + + const hover = (point: THREE.Vector2) => { + raycaster.setFromCamera(point, live.current.camera) + placeRing(hitTest(meshes)) + } + + const replay = (entry: SculptStrokeEntry): SculptStrokeEntry => { + const target = targets.current.get(entry.meshKey) + if (!target) throw new Error('The sculpted part is no longer loaded.') + const { region, inverse } = target.core.restore(entry.record) + markDirty(target, region) + uploadPending(target) + refreshSpatialData(target) + return { meshKey: entry.meshKey, record: inverse } + } + + const session: SculptSession = { + async commit() { + // A stroke in progress belongs to this commit. + if (drag.current) endDrag() + // Let "Saving sculpt…" paint before the (synchronous) encoding. + await new Promise((resolve) => window.setTimeout(resolve, 0)) + const patches: VertexPatch[] = [] + for (const target of targets.current.values()) { + if (!target.core.changed) continue + const patch = buildVertexPatch( + parser.associations.get(target.mesh) as { meshes?: number; primitives?: number } | undefined, + target.core.positions, + target.core.normals, + ) + if (!patch) throw new Error('A sculpted part is not a glTF primitive and cannot be written back.') + patches.push(patch) + } + if (patches.length === 0) return + const { url } = await live.current.patchMeshVertices(meshPathFromUrl(outputUrl), patches) + // Another mesh replaced this one while saving: applying would drop it. + if (useAppStore.getState().currentJob?.outputUrl !== outputUrl) { + throw new Error('The mesh changed while saving; the sculpt was not applied.') + } + useAppStore.getState().applyMeshEdit(url) + }, + undo() { + if (drag.current) return + const store = useSculptStore.getState() + try { store.setHistory(undoStroke(store.history, replay)) } catch (err) { reportError('Undo failed', err) } + }, + redo() { + if (drag.current) return + const store = useSculptStore.getState() + try { store.setHistory(redoStroke(store.history, replay)) } catch (err) { reportError('Redo failed', err) } + }, + } + + return { onDown, onHover, onLeave, hover, session, endDrag, flush: () => targets.current.forEach(uploadPending) } + }, [scene, meshes, parser, outputUrl, raycaster]) + + // One session per model: commit (viewer going away) or drop (mesh replaced) + // whatever is still pending when it ends. + useEffect(() => { + const { session } = api + const unregister = useSculptStore.getState().registerSession(session) + const owned = targets.current + return () => { + const store = useSculptStore.getState() + const outcome = sessionEndOutcome( + store.session === session && store.dirty, + outputUrl, + useAppStore.getState().currentJob?.outputUrl, + ) + if (drag.current) api.endDrag() + if (outcome === 'commit') { + session.commit().catch((err) => { + useAppStore.getState().showToast(`Could not save the sculpt: ${errorText(err)}`, 5000) + }) + } else if (outcome === 'discard') { + useAppStore.getState().showToast('Unsaved sculpt strokes were dropped: the mesh was replaced.', 5000) + } + unregister() + if (outcome !== 'commit') owned.clear() + } + }, [api, outputUrl]) + + // Pointer input, capture phase on the canvas: decides stroke vs orbit before + // OrbitControls (listening further up) sees the press. + useEffect(() => { + const el = gl.domElement + el.addEventListener('pointerdown', api.onDown, { capture: true }) + el.addEventListener('pointermove', api.onHover) + el.addEventListener('pointerleave', api.onLeave) + return () => { + el.removeEventListener('pointerdown', api.onDown, { capture: true }) + el.removeEventListener('pointermove', api.onHover) + el.removeEventListener('pointerleave', api.onLeave) + } + }, [gl, api]) + + // Entering / leaving sculpt mode: fresh model size, no stale ring, no stuck stroke. + useEffect(() => { + diagonal.current = null + if (!active) { + if (drag.current) api.endDrag() + hoverNdc.current = null + if (ringRef.current) ringRef.current.visible = false + } + gl.domElement.style.cursor = active ? 'crosshair' : '' + return () => { gl.domElement.style.cursor = '' } + }, [active, api, gl]) + + // At most one GPU upload per frame, and one hover raycast per frame. + useFrame(() => { + api.flush() + if (!activeRef.current || drag.current) return + const point = hoverNdc.current + if (point) { + hoverNdc.current = null + api.hover(point) + } + }) + + return ( + null}> + + + + ) +} diff --git a/src/areas/generate/sculpt/sculptLogic.test.ts b/src/areas/generate/sculpt/sculptLogic.test.ts new file mode 100644 index 000000000..e8c73f69a --- /dev/null +++ b/src/areas/generate/sculpt/sculptLogic.test.ts @@ -0,0 +1,194 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + RADIUS_MAX, + RADIUS_MIN, + SCULPT_MAX_TRIANGLES, + attributeUpdateRange, + buildVertexPatch, + bytesToBase64, + clampRadius, + emptyHistory, + float32ToBase64, + isHistoryDirty, + mergeRange, + pushStroke, + redoStroke, + sculptBlock, + sculptKeyAction, + sessionEndOutcome, + stepRadius, + toLocalRadius, + undoStroke, + worldRadius, + type KeyLike, +} from './sculptLogic.ts' + +// ─── Availability ──────────────────────────────────────────────────────────── + +test('sculptBlock allows a loaded GLB under the triangle limit', () => { + assert.equal(sculptBlock({ meshUrl: '/workspace/a/mesh.glb', generating: false, triangles: 10_000 }), null) + assert.equal(sculptBlock({ meshUrl: '/workspace/a/mesh.glb', generating: false, triangles: SCULPT_MAX_TRIANGLES }), null) +}) + +test('sculptBlock rejects missing models, generation, OBJ and splats', () => { + assert.match(sculptBlock({ meshUrl: undefined, generating: false, triangles: null })!.reason, /Generate, import or open/) + assert.match(sculptBlock({ meshUrl: '/workspace/a/mesh.glb', generating: true, triangles: 10 })!.reason, /Generating/) + assert.match(sculptBlock({ meshUrl: '/workspace/a/mesh.OBJ', generating: false, triangles: 10 })!.reason, /OBJ/) + assert.match(sculptBlock({ meshUrl: '/workspace/a/s.ply', generating: false, triangles: 10 })!.reason, /splat/) +}) + +test('sculptBlock names the triangle count of a mesh that is too big', () => { + const block = sculptBlock({ meshUrl: '/workspace/a/mesh.glb', generating: false, triangles: 3_500_000 }) + assert.equal(block?.transient, false) + assert.equal( + block?.reason, + 'This mesh has too many triangles to sculpt (3,500,000). Decimate it or use Prepare for Game first.', + ) +}) + +test('sculptBlock treats a loading mesh as transient', () => { + const block = sculptBlock({ meshUrl: '/workspace/a/mesh.glb', generating: false, triangles: null }) + assert.equal(block?.transient, true) +}) + +// ─── Radius ────────────────────────────────────────────────────────────────── + +test('radius is clamped and stepped geometrically', () => { + assert.equal(clampRadius(0), RADIUS_MIN) + assert.equal(clampRadius(10), RADIUS_MAX) + assert.equal(clampRadius(Number.NaN), 0.05) + assert.equal(stepRadius(0.05, 1), 0.0625) + assert.equal(stepRadius(0.05, -1), 0.04) + assert.equal(stepRadius(RADIUS_MAX, 1), RADIUS_MAX) + assert.equal(stepRadius(RADIUS_MIN, -1), RADIUS_MIN) +}) + +test('world radius scales with the model and local radius undoes the world scale', () => { + assert.equal(worldRadius(0.1, 2), 0.2) + assert.equal(toLocalRadius(0.2, 4), 0.05) + assert.equal(toLocalRadius(0.2, 0), 0.2) +}) + +// ─── History ───────────────────────────────────────────────────────────────── + +test('undo and redo swap entries through apply and keep the stacks in step', () => { + let h = pushStroke(pushStroke(emptyHistory(), 'a'), 'b') + assert.deepEqual(h, { undo: ['a', 'b'], redo: [], truncated: false }) + assert.equal(isHistoryDirty(h), true) + + const applied: string[] = [] + h = undoStroke(h, (e) => { applied.push(e); return `${e}'` }) + assert.deepEqual(h, { undo: ['a'], redo: ["b'"], truncated: false }) + h = undoStroke(h, (e) => { applied.push(e); return `${e}'` }) + assert.equal(isHistoryDirty(h), false) + assert.equal(undoStroke(h, () => assert.fail('nothing to undo')), h) + + h = redoStroke(h, (e) => { applied.push(e); return e.replace("'", '') }) + assert.deepEqual(h.undo, ['a']) + assert.deepEqual(applied, ['b', 'a', "a'"]) +}) + +test('a new stroke clears the redo stack', () => { + let h = pushStroke(emptyHistory(), 'a') + h = undoStroke(h, (e) => e) + h = pushStroke(h, 'c') + assert.deepEqual(h, { undo: ['c'], redo: [], truncated: false }) +}) + +test('a failing apply leaves the history untouched', () => { + const h = pushStroke(emptyHistory(), 'a') + assert.throws(() => undoStroke(h, () => { throw new Error('boom') }), /boom/) + assert.deepEqual(h.undo, ['a']) +}) + +test('strokes over the limit fall off and keep the mesh dirty', () => { + let h = emptyHistory() + for (let i = 0; i < 4; i++) h = pushStroke(h, i, 3) + assert.deepEqual(h.undo, [1, 2, 3]) + assert.equal(h.truncated, true) + for (let i = 0; i < 3; i++) h = undoStroke(h, (e) => e) + assert.equal(h.undo.length, 0) + assert.equal(isHistoryDirty(h), true) +}) + +// ─── Ranges ────────────────────────────────────────────────────────────────── + +test('dirty ranges merge and map to attribute update ranges', () => { + let r = mergeRange(null, { min: 10, max: 20 }) + r = mergeRange(r, { min: 4, max: 12 }) + assert.deepEqual(r, { min: 4, max: 20 }) + assert.deepEqual(mergeRange(r, { min: 1, max: 0 }), r) // empty region + assert.equal(mergeRange(null, { min: 0, max: -1 }), null) + assert.deepEqual(attributeUpdateRange({ min: 4, max: 20 }, 3), { start: 12, count: 51 }) +}) + +// ─── Encoding ──────────────────────────────────────────────────────────────── + +test('bytesToBase64 matches Buffer on sizes around the chunk boundary', () => { + for (const size of [0, 1, 2, 3, 32_765, 32_766, 32_767, 100_003]) { + const bytes = new Uint8Array(size) + for (let i = 0; i < size; i++) bytes[i] = (i * 31 + 7) & 0xff + assert.equal(bytesToBase64(bytes), Buffer.from(bytes).toString('base64'), `size ${size}`) + } +}) + +test('float32ToBase64 encodes only the view, little-endian', () => { + const backing = new Float32Array([9, 1.5, -2, 9]) + const view = backing.subarray(1, 3) + const expected = Buffer.alloc(8) + expected.writeFloatLE(1.5, 0) + expected.writeFloatLE(-2, 4) + assert.equal(float32ToBase64(view), expected.toString('base64')) +}) + +test('buildVertexPatch maps the glTF association to a patch', () => { + const positions = new Float32Array([1, 2, 3]) + const normals = new Float32Array([0, 0, 1]) + assert.deepEqual(buildVertexPatch({ meshes: 2, primitives: 1 }, positions, normals), { + mesh: 2, + primitive: 1, + positions: float32ToBase64(positions), + normals: float32ToBase64(normals), + }) + assert.equal(buildVertexPatch({ meshes: 0, primitives: 0 }, positions, null)?.normals, undefined) + assert.equal(buildVertexPatch(undefined, positions, normals), null) + assert.equal(buildVertexPatch({ nodes: 3 } as { meshes?: number }, positions, normals), null) +}) + +// ─── Session end ───────────────────────────────────────────────────────────── + +test('sessionEndOutcome saves when the same mesh is shown and drops otherwise', () => { + assert.equal(sessionEndOutcome(false, '/workspace/a.glb', '/workspace/b.glb'), 'none') + assert.equal(sessionEndOutcome(true, '/workspace/a.glb', '/workspace/a.glb'), 'commit') + assert.equal(sessionEndOutcome(true, '/workspace/a.glb', '/workspace/b.glb'), 'discard') + assert.equal(sessionEndOutcome(true, '/workspace/a.glb', undefined), 'discard') +}) + +// ─── Shortcuts ─────────────────────────────────────────────────────────────── + +function key(k: string, mods: Partial = {}): KeyLike { + return { key: k, ctrlKey: false, metaKey: false, shiftKey: false, altKey: false, ...mods } +} + +test('sculptKeyAction maps the sculpt shortcuts', () => { + assert.deepEqual(sculptKeyAction(key('z', { ctrlKey: true })), { type: 'undo' }) + assert.deepEqual(sculptKeyAction(key('Z', { ctrlKey: true, shiftKey: true })), { type: 'redo' }) + assert.deepEqual(sculptKeyAction(key('y', { metaKey: true })), { type: 'redo' }) + assert.deepEqual(sculptKeyAction(key('Escape')), { type: 'exit' }) + assert.deepEqual(sculptKeyAction(key('f')), { type: 'radius', direction: 1 }) + assert.deepEqual(sculptKeyAction(key('F', { shiftKey: true })), { type: 'radius', direction: -1 }) + assert.deepEqual(sculptKeyAction(key('g')), { type: 'brush', kind: 'grab' }) + assert.deepEqual(sculptKeyAction(key('s')), { type: 'brush', kind: 'smooth' }) + assert.deepEqual(sculptKeyAction(key('i')), { type: 'brush', kind: 'inflate' }) + assert.deepEqual(sculptKeyAction(key('t')), { type: 'brush', kind: 'flatten' }) +}) + +test('sculptKeyAction ignores other keys and modified brush keys', () => { + assert.equal(sculptKeyAction(key('w')), null) + assert.equal(sculptKeyAction(key('r')), null) + assert.equal(sculptKeyAction(key('s', { ctrlKey: true })), null) + assert.equal(sculptKeyAction(key('g', { altKey: true })), null) + assert.equal(sculptKeyAction(key('S', { shiftKey: true })), null) +}) diff --git a/src/areas/generate/sculpt/sculptLogic.ts b/src/areas/generate/sculpt/sculptLogic.ts new file mode 100644 index 000000000..57f28ae69 --- /dev/null +++ b/src/areas/generate/sculpt/sculptLogic.ts @@ -0,0 +1,231 @@ +// Pure glue between the sculpt core (SculptMesh), the viewer and the edit +// panel: availability guard, brush radius conversion, stroke history, the +// commit payload encoding and the sculpt-mode shortcuts. No React, no three.js, +// so it can be unit-tested with node --test. + +import type { BrushKind } from './types' + +// ─── Availability ──────────────────────────────────────────────────────────── + +/** Above this many triangles the sculpt core would be too slow / too heavy. */ +export const SCULPT_MAX_TRIANGLES = 3_000_000 + +export interface SculptAvailabilityInput { + /** Mesh URL in the viewer (currentJob.outputUrl when the job is done). */ + meshUrl: string | undefined + generating: boolean + /** Triangle count from the viewer; null while the mesh is still loading. */ + triangles: number | null +} + +export interface SculptBlock { + reason: string + /** + * True while the mesh is merely loading (e.g. the patched file a commit just + * produced): sculpt mode can't start, but an active session isn't ended. + */ + transient: boolean +} + +/** Why sculpt mode can't be used right now, or null when it can. */ +export function sculptBlock({ meshUrl, generating, triangles }: SculptAvailabilityInput): SculptBlock | null { + const block = (reason: string, transient = false): SculptBlock => ({ reason, transient }) + if (generating) return block('Generating… sculpting unlocks once the mesh is ready.') + if (!meshUrl) return block('Generate, import or open a mesh to sculpt it.') + const ext = meshUrl.split('?')[0]?.split('.').pop()?.toLowerCase() + if (ext === 'obj') return block('OBJ models can\'t be sculpted: only GLB meshes can.') + if (ext === 'ply' || ext === 'splat') return block('Gaussian splats can\'t be sculpted.') + if (triangles === null) return block('Loading the mesh…', true) + if (triangles > SCULPT_MAX_TRIANGLES) { + return block(`This mesh has too many triangles to sculpt (${triangles.toLocaleString('en-US')}). Decimate it or use Prepare for Game first.`) + } + return null +} + +// ─── Brush radius ──────────────────────────────────────────────────────────── + +/** Radius is stored as a fraction of the model's bounding-box diagonal. */ +export const RADIUS_MIN = 0.005 +export const RADIUS_MAX = 0.5 +export const RADIUS_DEFAULT = 0.05 +/** F / Shift+F multiply / divide the radius by this. */ +export const RADIUS_STEP = 1.25 + +export function clampRadius(fraction: number): number { + if (!Number.isFinite(fraction)) return RADIUS_DEFAULT + return Math.min(RADIUS_MAX, Math.max(RADIUS_MIN, fraction)) +} + +export function stepRadius(fraction: number, direction: 1 | -1): number { + const next = direction > 0 ? fraction * RADIUS_STEP : fraction / RADIUS_STEP + return clampRadius(Number(next.toPrecision(3))) +} + +/** World-space brush radius for a model whose bbox diagonal is `modelDiagonal`. */ +export function worldRadius(fraction: number, modelDiagonal: number): number { + return clampRadius(fraction) * Math.max(modelDiagonal, 0) +} + +/** + * World radius → local radius of one mesh, whose world matrix scales by + * `maxWorldScale` along its largest axis (Matrix4.getMaxScaleOnAxis). + */ +export function toLocalRadius(radiusWorld: number, maxWorldScale: number): number { + return maxWorldScale > 1e-12 ? radiusWorld / maxWorldScale : radiusWorld +} + +// ─── Stroke history ────────────────────────────────────────────────────────── + +/** Strokes kept for local undo; older ones fall off (and the mesh stays dirty). */ +export const SCULPT_HISTORY_LIMIT = 64 + +export interface SculptHistory { + undo: T[] + redo: T[] + /** True once a stroke fell off the bottom: undoing everything no longer restores the source mesh. */ + truncated: boolean +} + +export function emptyHistory(): SculptHistory { + return { undo: [], redo: [], truncated: false } +} + +export function pushStroke(h: SculptHistory, entry: T, limit = SCULPT_HISTORY_LIMIT): SculptHistory { + const undo = [...h.undo, entry] + const overflow = Math.max(0, undo.length - limit) + return { undo: undo.slice(overflow), redo: [], truncated: h.truncated || overflow > 0 } +} + +/** + * Undo the latest stroke: `apply` swaps it back into the mesh and returns the + * entry that redoes it. Returns `h` unchanged when there is nothing to undo; if + * `apply` throws, the history is left as it was. + */ +export function undoStroke(h: SculptHistory, apply: (entry: T) => T): SculptHistory { + const entry = h.undo[h.undo.length - 1] + if (entry === undefined) return h + const inverse = apply(entry) + return { ...h, undo: h.undo.slice(0, -1), redo: [...h.redo, inverse] } +} + +export function redoStroke(h: SculptHistory, apply: (entry: T) => T): SculptHistory { + const entry = h.redo[h.redo.length - 1] + if (entry === undefined) return h + const inverse = apply(entry) + return { ...h, undo: [...h.undo, inverse], redo: h.redo.slice(0, -1) } +} + +/** Whether the meshes differ from the file on disk (and a commit has work to do). */ +export function isHistoryDirty(h: SculptHistory): boolean { + return h.undo.length > 0 || h.truncated +} + +// ─── Upload ranges ─────────────────────────────────────────────────────────── + +export interface IndexRange { min: number; max: number } + +/** Union of two vertex index ranges; `region` with min > max (nothing moved) is ignored. */ +export function mergeRange(pending: IndexRange | null, region: IndexRange): IndexRange | null { + if (!(region.min <= region.max)) return pending + if (!pending) return { min: region.min, max: region.max } + return { min: Math.min(pending.min, region.min), max: Math.max(pending.max, region.max) } +} + +/** Vertex range → BufferAttribute.addUpdateRange(start, count) arguments. */ +export function attributeUpdateRange(range: IndexRange, itemSize: number): { start: number; count: number } { + return { start: range.min * itemSize, count: (range.max - range.min + 1) * itemSize } +} + +// ─── Commit payload ────────────────────────────────────────────────────────── + +/** Bytes per btoa() call: a multiple of 3 so the chunks concatenate without padding. */ +const BASE64_CHUNK = 3 * 10_922 + +/** Base64 of arbitrary bytes, encoded in slices (no giant binary string, no quadratic concat). */ +export function bytesToBase64(bytes: Uint8Array): string { + const parts: string[] = [] + for (let i = 0; i < bytes.length; i += BASE64_CHUNK) { + const slice = bytes.subarray(i, i + BASE64_CHUNK) + parts.push(btoa(String.fromCharCode.apply(null, slice as unknown as number[]))) + } + return parts.join('') +} + +/** Base64 of a Float32Array's bytes (little-endian on every platform Modly ships on). */ +export function float32ToBase64(values: Float32Array): string { + return bytesToBase64(new Uint8Array(values.buffer, values.byteOffset, values.byteLength)) +} + +/** One entry of POST /optimize/patch-vertices. */ +export interface VertexPatch { + mesh: number + primitive: number + positions: string + normals?: string +} + +/** + * Patch for one glTF primitive, from GLTFLoader's parser.associations entry of + * the sculpted THREE.Mesh. Null when the mesh isn't a glTF primitive. + */ +export function buildVertexPatch( + association: { meshes?: number; primitives?: number } | undefined, + positions: Float32Array, + normals: Float32Array | null, +): VertexPatch | null { + if (!association || typeof association.meshes !== 'number' || typeof association.primitives !== 'number') return null + const patch: VertexPatch = { mesh: association.meshes, primitive: association.primitives, positions: float32ToBase64(positions) } + if (normals) patch.normals = float32ToBase64(normals) + return patch +} + +// ─── Model URL change ──────────────────────────────────────────────────────── + +/** + * What to do with uncommitted strokes when the sculpt session of one model + * goes away: the same mesh still in the viewer (the viewer is unmounting) → + * save it; another mesh replaced it (undo, agent, workflow…) → the strokes no + * longer apply to anything and are dropped. + */ +export function sessionEndOutcome(dirty: boolean, sessionUrl: string, currentUrl: string | undefined): 'none' | 'commit' | 'discard' { + if (!dirty) return 'none' + return currentUrl === sessionUrl ? 'commit' : 'discard' +} + +// ─── Shortcuts ─────────────────────────────────────────────────────────────── + +export type SculptKeyAction = + | { type: 'undo' } + | { type: 'redo' } + | { type: 'exit' } + | { type: 'radius'; direction: 1 | -1 } + | { type: 'brush'; kind: BrushKind } + +const BRUSH_KEYS: Record = { g: 'grab', s: 'smooth', i: 'inflate', t: 'flatten' } + +export interface KeyLike { + key: string + ctrlKey: boolean + metaKey: boolean + shiftKey: boolean + altKey: boolean +} + +/** + * The sculpt-mode shortcut for a keydown, or null when the key isn't one. + * Ctrl+Z / Ctrl+Y (and Ctrl+Shift+Z) are the local stroke undo/redo; Escape + * exits; F / Shift+F grow / shrink the brush; G S I T pick a brush. + */ +export function sculptKeyAction(e: KeyLike): SculptKeyAction | null { + const key = e.key.toLowerCase() + if (e.ctrlKey || e.metaKey) { + if (key === 'z') return e.shiftKey ? { type: 'redo' } : { type: 'undo' } + if (key === 'y') return { type: 'redo' } + return null + } + if (e.altKey) return null + if (key === 'escape') return { type: 'exit' } + if (key === 'f') return { type: 'radius', direction: e.shiftKey ? -1 : 1 } + if (!e.shiftKey && BRUSH_KEYS[key]) return { type: 'brush', kind: BRUSH_KEYS[key] } + return null +} diff --git a/src/areas/generate/sculpt/sculptStore.ts b/src/areas/generate/sculpt/sculptStore.ts new file mode 100644 index 000000000..a2eaf2975 --- /dev/null +++ b/src/areas/generate/sculpt/sculptStore.ts @@ -0,0 +1,183 @@ +// Sculpt mode state shared by the edit panel (settings), the viewer (strokes) +// and the page (shortcuts, commit before export / chat / ops). +// +// The meshes being sculpted live in the viewer: it registers a SculptSession +// that knows how to upload them and how to replay the stroke history. Anyone +// about to read or replace the mesh file awaits `commit()` first. + +import { create } from 'zustand' +import { useAppStore } from '@shared/stores/appStore' +import { apiErrorMessage } from '../meshEditPanel' +import type { BrushKind, StrokeRecord, SymmetryAxis } from './types' +import { + RADIUS_DEFAULT, + clampRadius, + emptyHistory, + isHistoryDirty, + stepRadius, + type SculptHistory, +} from './sculptLogic' + +/** One local undo step: a stroke on the SculptMesh of the geometry `meshKey`. */ +export interface SculptStrokeEntry { + meshKey: string + record: StrokeRecord +} + +/** Registered by the viewer for the model it shows. */ +export interface SculptSession { + /** Writes the sculpted vertices to a new mesh file and shows it (one global undo step). Throws on failure. */ + commit(): Promise + undo(): void + redo(): void +} + +export interface BrushParams { + /** Fraction of the model's bounding-box diagonal. */ + radius: number + /** 0..1. */ + strength: number +} + +export type SculptStatus = 'idle' | 'preparing' | 'saving' + +export const BRUSH_KINDS: BrushKind[] = ['grab', 'smooth', 'inflate', 'flatten'] + +const DEFAULT_BRUSHES: Record = { + grab: { radius: 0.08, strength: 1 }, + smooth: { radius: RADIUS_DEFAULT, strength: 0.5 }, + inflate: { radius: RADIUS_DEFAULT, strength: 0.3 }, + flatten: { radius: RADIUS_DEFAULT, strength: 0.5 }, +} + +interface SculptState { + active: boolean + brush: BrushKind + brushes: Record + symmetry: SymmetryAxis[] + history: SculptHistory + /** Strokes not yet written to the mesh file. */ + dirty: boolean + status: SculptStatus + error: string | null + session: SculptSession | null + + enter: () => void + /** Commits, then leaves sculpt mode. Stays in sculpt mode (false) when saving failed. */ + exit: () => Promise + /** Leaves sculpt mode without saving (the model went away). */ + reset: () => void + setBrush: (kind: BrushKind) => void + setRadius: (fraction: number) => void + stepRadius: (direction: 1 | -1) => void + setStrength: (strength: number) => void + toggleSymmetry: (axis: SymmetryAxis) => void + setHistory: (history: SculptHistory) => void + setStatus: (status: SculptStatus) => void + setError: (error: string | null) => void + /** Returns the unregister function. */ + registerSession: (session: SculptSession) => () => void + /** + * Writes pending strokes to the mesh file. Resolves true when there is + * nothing left to save, false when saving failed (the strokes are kept and + * the error is shown). + */ + commit: () => Promise + undo: () => void + redo: () => void +} + +let inflight: Promise | null = null + +export const useSculptStore = create()((set, get) => ({ + active: false, + brush: 'grab', + brushes: DEFAULT_BRUSHES, + symmetry: [], + history: emptyHistory(), + dirty: false, + status: 'idle', + error: null, + session: null, + + enter: () => set({ active: true, error: null }), + + exit: async () => { + const ok = await get().commit() + if (ok) set({ active: false, error: null }) + return ok + }, + + reset: () => set({ active: false, history: emptyHistory(), dirty: false, status: 'idle', error: null }), + + setBrush: (kind) => set({ brush: kind }), + + setRadius: (fraction) => { + const { brush, brushes } = get() + set({ brushes: { ...brushes, [brush]: { ...brushes[brush], radius: clampRadius(fraction) } } }) + }, + + stepRadius: (direction) => { + const { brush, brushes } = get() + get().setRadius(stepRadius(brushes[brush].radius, direction)) + }, + + setStrength: (strength) => { + const { brush, brushes } = get() + const s = Math.min(1, Math.max(0, Number.isFinite(strength) ? strength : 0)) + set({ brushes: { ...brushes, [brush]: { ...brushes[brush], strength: s } } }) + }, + + toggleSymmetry: (axis) => { + const { symmetry } = get() + set({ symmetry: symmetry.includes(axis) ? symmetry.filter((a) => a !== axis) : [...symmetry, axis].sort() }) + }, + + setHistory: (history) => set({ history, dirty: isHistoryDirty(history) }), + setStatus: (status) => set({ status }), + setError: (error) => set({ error }), + + registerSession: (session) => { + set({ session, history: emptyHistory(), dirty: false, status: 'idle' }) + return () => { + if (get().session === session) set({ session: null, history: emptyHistory(), dirty: false, status: 'idle' }) + } + }, + + commit: () => { + if (inflight) return inflight + const { session, dirty } = get() + if (!dirty) return Promise.resolve(true) + if (!session) return Promise.resolve(true) + set({ status: 'saving', error: null }) + inflight = session.commit() + .then(() => { + set({ history: emptyHistory(), dirty: false, status: 'idle' }) + return true + }, (err: unknown) => { + const message = apiErrorMessage(err) + set({ status: 'idle', error: `Could not save the sculpt: ${message}` }) + useAppStore.getState().showToast(`Could not save the sculpt: ${message}`, 5000) + return false + }) + .finally(() => { inflight = null }) + return inflight + }, + + undo: () => { + const { session, status } = get() + if (status !== 'idle') return + session?.undo() + }, + + redo: () => { + const { session, status } = get() + if (status !== 'idle') return + session?.redo() + }, +})) + +/** Await pending sculpt strokes before reading or replacing the mesh file. */ +export function commitSculpt(): Promise { + return useSculptStore.getState().commit() +} diff --git a/src/shared/hooks/useApi.ts b/src/shared/hooks/useApi.ts index d2c7104e1..4b24b7870 100644 --- a/src/shared/hooks/useApi.ts +++ b/src/shared/hooks/useApi.ts @@ -1,6 +1,7 @@ import axios from 'axios' import { useAppStore, GenerationOptions } from '@shared/stores/appStore' import type { MeshOpDescriptor, MeshOpResponse } from '@areas/generate/meshEditPanel' +import type { VertexPatch } from '@areas/generate/sculpt/sculptLogic' export function useApi() { const apiUrl = useAppStore((s) => s.apiUrl) @@ -150,5 +151,18 @@ export function useApi() { return data } - return { generateFromImage, pollJobStatus, cancelJob, getModelStatus, getAllModelsStatus, downloadModel, optimizeMesh, smoothMesh, importMesh, transformMesh, listMeshOps, runMeshOp } + // Writes sculpted vertex positions/normals into a copy of the mesh file. + // Each patch replaces whole attributes of one glTF primitive (base64 LE float32). + async function patchMeshVertices( + path: string, + patches: VertexPatch[], + ): Promise<{ path: string; url: string }> { + const { data } = await client.post<{ path: string; url: string }>('/optimize/patch-vertices', { + path, + patches, + }) + return data + } + + return { generateFromImage, pollJobStatus, cancelJob, getModelStatus, getAllModelsStatus, downloadModel, optimizeMesh, smoothMesh, importMesh, transformMesh, listMeshOps, runMeshOp, patchMeshVertices } } From 50463fe20028bb2323ee9774a94fa22d75b3d37a Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 11:19:45 +0100 Subject: [PATCH 036/108] fix(deps): declare Pillow so mesh ops keep their textures trimesh decodes glTF images with Pillow but does not depend on it. The setup venv therefore had no Pillow, and every op that reloads and re-exports a mesh (presets, transforms, normals) silently wrote it back without its textures. The dev .venv happened to have it, which hid the bug. Changing requirements.txt also changes its hash, so existing installs re-run setup and get Pillow on the next launch. --- api/requirements.txt | 3 +++ api/tests/test_requirements.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 api/tests/test_requirements.py diff --git a/api/requirements.txt b/api/requirements.txt index 0462a51ba..a6d9199d2 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -13,6 +13,9 @@ pymeshlab>=2023.12 # trimesh's graph engine: connected parts, hole loops and vertex merging in # inspect_mesh / repair_mesh. Optional for trimesh, so never pulled in by it. scipy>=1.10 +# trimesh decodes glTF images with Pillow. Optional for trimesh: without it, +# every op that reloads a mesh silently drops its textures. +Pillow>=10.0 # Model downloads (HuggingFace) huggingface_hub>=0.27.0 diff --git a/api/tests/test_requirements.py b/api/tests/test_requirements.py new file mode 100644 index 000000000..bbd0445fc --- /dev/null +++ b/api/tests/test_requirements.py @@ -0,0 +1,26 @@ +import re +import unittest +from pathlib import Path + + +REQUIREMENTS = Path(__file__).resolve().parents[1] / "requirements.txt" + + +def declared_packages() -> set[str]: + names = set() + for line in REQUIREMENTS.read_text(encoding="utf-8").splitlines(): + line = line.split("#", 1)[0].strip() + if line: + names.add(re.split(r"[\[<>=!~ ;]", line, maxsplit=1)[0].lower()) + return names + + +class RequirementsTest(unittest.TestCase): + def test_pillow_is_declared(self) -> None: + # Without Pillow, trimesh cannot decode glTF images and silently drops + # every texture when an op reloads and re-exports a mesh. + self.assertIn("pillow", declared_packages()) + + +if __name__ == "__main__": + unittest.main() From cca63fd703631b1175e9c68ca91be304fdc18cc2 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 12:42:21 +0100 Subject: [PATCH 037/108] fix(sculpt): repoint every primitive sharing the patched POSITION accessor The viewer loads a shared accessor once, so a stroke moves every primitive using it. Patching only the clicked primitive made the others snap back after the reload and tore the mesh. --- api/services/mesh_ops/glb_patch.py | 28 ++++++++++++++++++---- api/tests/test_optimize_patch_vertices.py | 29 +++++++++++++++++------ 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/api/services/mesh_ops/glb_patch.py b/api/services/mesh_ops/glb_patch.py index 56839f4a8..3d68d5885 100644 --- a/api/services/mesh_ops/glb_patch.py +++ b/api/services/mesh_ops/glb_patch.py @@ -85,13 +85,24 @@ def patch_glb_bytes(data: bytes, patches: list[VertexPatch]) -> bytes: if patch.normals is not None: normals_raw, _ = _decode(patch.normals, count, f"{label} normals") - new_attributes = dict(attributes) - new_attributes["POSITION"] = _append_accessor( + old_normal = attributes.get("NORMAL") + new_position = _append_accessor( out, views, accessors, positions_raw, count, bounds=_bounds(positions), ) + new_normal = None if normals_raw is not None: - new_attributes["NORMAL"] = _append_accessor(out, views, accessors, normals_raw, count) - primitive["attributes"] = new_attributes + new_normal = _append_accessor(out, views, accessors, normals_raw, count) + + # The viewer loads an accessor once and shares it between the primitives + # using it, so the stroke moved all of them: repoint them all, or the + # unpatched ones snap back and the mesh tears. Normals follow only where + # they were shared as well. + for other in _primitives_using(meshes, "POSITION", position_index): + other_attributes = dict(other.get("attributes") or {}) + other_attributes["POSITION"] = new_position + if new_normal is not None and other_attributes.get("NORMAL") == old_normal: + other_attributes["NORMAL"] = new_normal + other["attributes"] = other_attributes buffers[0]["byteLength"] = len(out) return _build_glb(gltf, bytes(out)) @@ -152,6 +163,15 @@ def _primitive(meshes: list, mesh: int, primitive: int) -> dict: return primitives[primitive] +def _primitives_using(meshes: list, attribute: str, accessor: int) -> list[dict]: + return [ + primitive + for mesh in meshes + for primitive in (mesh.get("primitives") or []) + if (primitive.get("attributes") or {}).get(attribute) == accessor + ] + + def _decode(encoded: str, count: int, label: str) -> tuple[bytes, array]: try: raw = base64.b64decode(encoded, validate=True) diff --git a/api/tests/test_optimize_patch_vertices.py b/api/tests/test_optimize_patch_vertices.py index 187a64c85..d9e545bc4 100644 --- a/api/tests/test_optimize_patch_vertices.py +++ b/api/tests/test_optimize_patch_vertices.py @@ -189,7 +189,9 @@ def test_adds_normal_attribute_when_missing(self) -> None: attributes = out_gltf["meshes"][0]["primitives"][0]["attributes"] np.testing.assert_array_equal(_accessor_floats(out_gltf, out_bin, attributes["NORMAL"]), normals) - def test_shared_position_accessor_only_changes_patched_primitive(self) -> None: + def test_shared_position_accessor_moves_every_primitive_using_it(self) -> None: + # The viewer loads one accessor as one BufferAttribute, so a stroke on + # either primitive moves both; the file must agree or the mesh tears. source = self.collection / "shared.glb" _shared_accessor_glb(source) moved = np.array([[5, 5, 5], [6, 5, 5], [5, 6, 5]], dtype=" None: out_gltf, out_bin = _read_glb(self.workspace / response["path"]) first = out_gltf["meshes"][0]["primitives"][0]["attributes"] second = out_gltf["meshes"][1]["primitives"][0]["attributes"] - self.assertEqual(first, {"POSITION": 0}) self.assertNotEqual(second["POSITION"], 0) - self.assertNotIn("NORMAL", second) - np.testing.assert_array_equal( - _accessor_floats(out_gltf, out_bin, 0), - np.array([[0, 0, 0], [1, 0, 0], [0, 1, 0]], dtype=" None: + source = self.collection / "shared.glb" + _shared_accessor_glb(source) + gltf, binary = _read_glb(source) + gltf["meshes"][0]["primitives"][0]["attributes"]["NORMAL"] = 0 # its own (dummy) normals + _write_glb(source, gltf, binary) + moved = np.array([[5, 5, 5], [6, 5, 5], [5, 6, 5]], dtype=" None: source = self.collection / "odd.glb" _shared_accessor_glb(source) From f56701b8b6708f5e1192dfa8645073c6f3d0f6c2 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 13:52:31 +0100 Subject: [PATCH 038/108] docs(agent): design for the global agent side panel --- .../2026-11-12-agent-side-panel-design.md | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 docs/superpowers/specs/2026-11-12-agent-side-panel-design.md diff --git a/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md b/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md new file mode 100644 index 000000000..69bfa8db5 --- /dev/null +++ b/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md @@ -0,0 +1,78 @@ +# Agent side panel: design + +## Intent + +The agent today lives in the "chat" mode of the Generate page's left panel. It loses its conversation whenever that panel or the page changes. The user wants the agent available **all the time**, in a panel on the right that **pushes the whole app**, so that the app can be driven entirely through the agent. + +## Goals + +- **Global panel.** A right-side panel lives in `MainLayout`, next to `

`, and pushes the content rather than overlaying it. + - A TopBar button toggles it, and so does `Ctrl+L` (my call). + - Its width can be resized by dragging (min 320, max 640, default 380; my call). + - Its open state and width are remembered across restarts. +- **Chat moves out of Generate.** + - `WorkflowPanel` loses its basic/chat toggle and keeps basic mode only. + - `ChatPanel` moves to `src/shared/components/agent/` as the panel body, keeping its current behaviour (streaming, thinking, actions card, mesh undo, workflow progress, image attachments). +- **The agent acts on the whole app without navigating.** It works on the page's state through stores, IPC and the API, never by changing `currentPage`, and confirms what it did in the panel. +- **Destructive actions ask first.** The panel shows a Confirm / Cancel card and the agent waits for the answer. This applies to uninstalling an extension, deleting model weights, deleting or overwriting a workflow, and changing a storage folder. +- **Conversations are persisted.** + - Each conversation is a JSON file in `/agent/conversations/{id}.json`. + - The panel lists past conversations (title, date) and offers New, Open and Delete. + - Images attached to messages are stored inline as data URLs (my call; simple, and good enough at this scale). +- **Context everywhere.** Every request sends the current page, the mesh (path, triangles), workflows, installed extensions, downloaded models and storage paths. + +## New agent tools + +**Server-side tools** run in `execute_tool` and call the local API: + +| Tool | Does | +|---|---| +| `generate_from_image(model_id?, params?)` | Starts a generation from the image attached to the message. The frontend runs it, as with `run_workflow`. | +| `list_extensions` | Lists installed extensions (from context). | +| `get_settings` | Returns the storage paths and a few UI options (from context). | + +**Client-side tools** need Electron IPC or renderer stores, so the frontend runs them: + +| Tool | Confirm? | +|---|---| +| `install_extension(github_url)` | no | +| `uninstall_extension(extension_id)` | **yes** | +| `reload_extensions` | no | +| `download_model_weights(extension_id, variant?)` | no | +| `delete_model_weights(extension_id, variant?)` | **yes** | +| `delete_workflow(workflow_id)` | **yes** | +| `open_workflow(workflow_id)` | no (opens it in the workflows store, does not navigate) | +| `set_setting(key, value)`, with key in `uiScale` / `useAtkinsonFont` / `showRamIndicator` | no | +| `set_storage_folder(kind, path)` | **yes** | +| `undo_mesh` / `redo_mesh` | no | +| `navigate(page)` | no; only when the user explicitly asks to go to a page | + +## Client tool bridge + +This is the core mechanism. + +1. The LLM calls a client tool. `_chat_events` does not run it. It yields a `client_tool` event `{id, tool, arguments, confirm}`, registers an `asyncio.Future` under the call id, and awaits it. +2. The panel runs the tool through a frontend registry (`clientTools.ts`, mapping each tool name to an async handler that returns a result string). When `confirm` is set, it first shows the Confirm / Cancel card. Cancel resolves with "The user declined." +3. The panel sends `POST /agent/tool-result {call_id, result, error?}`. The future resolves, the result goes into `messages` as a real `role: tool` message, and the loop continues in the same turn. +4. Timeouts work differently depending on whether the user is involved: + - Execution is bounded by `tool_timeout`. + - Waiting for the user to confirm is not bounded by it, but is capped at 10 minutes. + - If the stream disconnects, the futures are cancelled. +5. Which tools run on the client, and which of those need confirmation, is declared once on the backend in a `CLIENT_TOOLS` dict. The frontend just obeys `confirm`. + +This replaces nothing that exists today. `mesh_update`, `create_workflow` and `run_workflow` payloads keep working. + +## Non-goals + +- No page navigation as a side effect of an action. +- No change to the providers or to Settings > Agent. +- No change to the MCP server. + +## Anti-goals + +- The agent performing a destructive action without an explicit Confirm click. +- Losing the conversation when the page changes. + +## Left to the builder + +Visual details of the panel and the conversation list, as long as they match the existing zinc/violet style. From 70fe74e13f35726310af48e151416251281c057a Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:00:44 +0100 Subject: [PATCH 039/108] feat(agent): client tool bridge and app-wide tools for the side panel Tools the app must run itself (extensions, model weights, workflows, settings, mesh undo/redo, navigation) are declared in CLIENT_TOOLS. The tool loop hands them to the app as a client_tool event and waits for the answer on POST /agent/tool-result; destructive ones ask the user first and wait up to 10 minutes. Adds list_extensions, get_settings and generate_from_image, and renders the page, models, workflows, storage folders and UI options into the system prompt. --- api/routers/agent.py | 286 +++++++++++++++++++-- api/tests/test_agent_client_tools.py | 365 +++++++++++++++++++++++++++ 2 files changed, 626 insertions(+), 25 deletions(-) create mode 100644 api/tests/test_agent_client_tools.py diff --git a/api/routers/agent.py b/api/routers/agent.py index 2bdd04c13..797e40461 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -93,10 +93,21 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: SYSTEM_PROMPT = f"""\ You are Modly's built-in AI assistant, specialized in 3D modeling and workflow automation. You help users generate 3D models from images, optimize meshes, and manage workflows directly inside the Modly application. +You can act on the whole app (extensions, model weights, workflows, settings, generation) from any page, without changing page. ## Available tools - **list_models** — List all downloaded 3D generation models ready to use. +- **generate_from_image(model_id?)** — Generate a 3D model from the image attached to the user's message. Omit model_id to use the model selected in the app. +- **list_extensions** — List installed extensions. +- **get_settings** — Show the storage folders and UI options. +- **install_extension(github_url)** / **uninstall_extension(extension_id)** / **reload_extensions** — Manage extensions. +- **download_model_weights(extension_id, variant?)** / **delete_model_weights(extension_id, variant?)** — Manage model weights. +- **open_workflow(workflow_id)** / **delete_workflow(workflow_id)** — Open a workflow in the editor, or delete it. +- **set_setting(key, value)** — Change `uiScale` (number), `useAtkinsonFont` or `showRamIndicator` (booleans). +- **set_storage_folder(kind, path)** — Move the `models`, `workspace` or `workflows` folder. +- **undo_mesh** / **redo_mesh** — Undo or redo the last change to the current mesh. +- **navigate(page)** — Switch the app to another page. - **unload_models** — Unload all 3D generation models from GPU VRAM to free memory. - **get_mesh_info** — Get info about the current mesh in the 3D viewer (path, triangle count). {registry_tool_lines(mesh_ops_registry)} @@ -117,6 +128,8 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: - If you need to run a workflow but don't know the ID, call list_workflows first. - To create a workflow, ONLY use extension ids listed under "Available extensions" in the context. Never invent an id. Chain steps so each step's input type matches the previous step's output type. - For a workflow's input, `input_type` MUST be exactly one of: `image`, `text`, or `mesh`. These map to the Image, Text, and Load 3D Mesh nodes. Never invent another input. Pick the one matching the first step's expected input. +- uninstall_extension, delete_model_weights, delete_workflow and set_storage_folder ask the user to confirm. If the result says the user declined, acknowledge it and do not retry. +- Never call navigate unless the user explicitly asks to go to a page. Act from where the user is. - After each tool call, give a short one-sentence summary of what was done. - Always reply in the same language the user is writing in. - Be concise. No unnecessary explanations.\ @@ -281,6 +294,124 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: TOOLS += registry_tools(mesh_ops_registry) +def _function(name: str, description: str, properties: dict | None = None, required: list[str] | None = None) -> dict: + parameters: dict = {"type": "object", "properties": properties or {}} + if required: + parameters["required"] = required + return {"type": "function", "function": {"name": name, "description": description, "parameters": parameters}} + + +_EXTENSION_ID = {"type": "string", "description": "Exact extension id, as listed by list_extensions."} +_WORKFLOW_ID = {"type": "string", "description": "Exact workflow id, as listed by list_workflows."} +_VARIANT = {"type": "string", "description": "Optional weights variant, for extensions that offer several. Omit for the default."} + +# Server-side tools that read the request context or hand a run to the app. +TOOLS += [ + _function("list_extensions", "List the installed extensions, with their ids and input→output types."), + _function("get_settings", "Show Modly's storage folders (models, workspace, workflows, extensions) and UI options."), + _function( + "generate_from_image", + "Generate a 3D model from the image the user attached to their message. The app runs the " + "generation and shows its progress. Needs an attached image.", + {"model_id": { + "type": "string", + "description": "Id of a downloaded generation model (see list_models). Omit to use the model selected in the app.", + }}, + ), +] + +# Tools the app runs itself (Electron IPC, renderer stores), mapped to whether +# the user must confirm them first. The frontend obeys `confirm`. +CLIENT_TOOLS: dict[str, bool] = { + "install_extension": False, + "uninstall_extension": True, + "reload_extensions": False, + "download_model_weights": False, + "delete_model_weights": True, + "delete_workflow": True, + "open_workflow": False, + "set_setting": False, + "set_storage_folder": True, + "undo_mesh": False, + "redo_mesh": False, + "navigate": False, +} + +TOOLS += [ + _function( + "install_extension", + "Install an extension from its GitHub repository URL, then load it.", + {"github_url": {"type": "string", "description": "Repository URL, e.g. https://github.com/owner/repo."}}, + ["github_url"], + ), + _function( + "uninstall_extension", + "Uninstall an extension and remove its files. The user is asked to confirm.", + {"extension_id": _EXTENSION_ID}, + ["extension_id"], + ), + _function("reload_extensions", "Reload every installed extension, e.g. after its files changed on disk."), + _function( + "download_model_weights", + "Download the model weights an extension needs. The download continues in the background.", + {"extension_id": _EXTENSION_ID, "variant": _VARIANT}, + ["extension_id"], + ), + _function( + "delete_model_weights", + "Delete an extension's downloaded model weights from disk. The user is asked to confirm.", + {"extension_id": _EXTENSION_ID, "variant": _VARIANT}, + ["extension_id"], + ), + _function( + "delete_workflow", + "Delete a saved workflow. The user is asked to confirm.", + {"workflow_id": _WORKFLOW_ID}, + ["workflow_id"], + ), + _function( + "open_workflow", + "Open a workflow in the workflow editor, without leaving the current page.", + {"workflow_id": _WORKFLOW_ID}, + ["workflow_id"], + ), + _function( + "set_setting", + "Change a UI setting. uiScale is a number (1 = 100%); useAtkinsonFont and showRamIndicator are booleans.", + { + "key": {"type": "string", "enum": ["uiScale", "useAtkinsonFont", "showRamIndicator"]}, + "value": {"type": ["number", "boolean"], "description": "A number for uiScale, a boolean otherwise."}, + }, + ["key", "value"], + ), + _function( + "set_storage_folder", + "Change where Modly stores models, the workspace or workflows. The user is asked to confirm.", + { + "kind": {"type": "string", "enum": ["models", "workspace", "workflows"]}, + "path": {"type": "string", "description": "Absolute path of the new folder."}, + }, + ["kind", "path"], + ), + _function("undo_mesh", "Undo the last change to the current mesh in the viewer."), + _function("redo_mesh", "Redo the last undone change to the current mesh in the viewer."), + _function( + "navigate", + "Switch the app to another page. Use it ONLY when the user explicitly asks to go to a page; " + "every other tool works from any page.", + {"page": {"type": "string", "enum": ["generate", "workflows", "models", "settings"]}}, + ["page"], + ), +] + +# Seconds the app has to run a client tool that needs the user's click. +CONFIRM_TIMEOUT = 600.0 +_NO_ANSWER = "Error: The app did not answer in time." + +# call_id → the turn waiting for the app's answer (see /agent/tool-result). +_pending_client_tools: dict[str, asyncio.Future] = {} + + # Input kinds the agent may pick, mapped to the real Modly source-node types. # Keep this in sync with the node palette in WorkflowsPage.tsx. INPUT_NODES = { @@ -374,11 +505,40 @@ async def _run_agent_op(client: httpx.AsyncClient, op: MeshOp, arguments: dict) return text, payload +def _extension_lines(extensions: list[dict]) -> list[str]: + return [ + f"- {e['id']} ({e.get('input', '?')}→{e.get('output', '?')}): {e.get('name', e['id'])}" + for e in extensions if e.get("id") + ] + + +def _id_name_lines(items: list[dict]) -> list[str]: + return [f"- {i['id']}: {i.get('name') or i['id']}" for i in items if i.get("id")] + + +_STORAGE_LABELS = { + "modelsDir": "Models folder", + "workspaceDir": "Workspace folder", + "workflowsDir": "Workflows folder", + "extensionsDir": "Extensions folder", +} + + +def _settings_lines(context: dict) -> list[str]: + """The storage folders and UI options the app sent, one per line.""" + storage = context.get("storage") or {} + ui = context.get("ui") or {} + lines = [f"{label}: {storage[key]}" for key, label in _STORAGE_LABELS.items() if storage.get(key)] + lines += [f"{key}: {_show(value)}" for key, value in ui.items() if value is not None] + return lines + + async def execute_tool( - name: str, arguments: dict, context: dict, *, timeout: float = 60.0, + name: str, arguments: dict, context: dict, *, timeout: float = 60.0, has_image: bool = False, ) -> tuple[str, dict | None]: """Execute a tool and return (result_text, action_payload). action_payload carries data the frontend needs to react (e.g. new mesh URL). + has_image: the user's last message carries an image (generate_from_image). """ async with httpx.AsyncClient(timeout=timeout) as client: try: @@ -459,6 +619,34 @@ async def execute_tool( payload = {"type": "run_workflow", "workflow_id": workflow_id, "workflow_name": match["name"]} return f"Executing workflow '{match['name']}'…", payload + elif name == "list_extensions": + lines = _extension_lines(context.get("extensions") or []) + if not lines: + return "No extensions installed.", None + return "Installed extensions:\n" + "\n".join(lines), None + + elif name == "get_settings": + lines = _settings_lines(context) + if not lines: + return "No settings were sent by the app.", None + return "\n".join(lines), None + + elif name == "generate_from_image": + if not has_image: + return "No image is attached. Ask the user to attach the image to generate from.", None + model_id = arguments.get("model_id") or None + models = context.get("models") or [] + label = "the selected model" + if model_id and models: + match = next((m for m in models if m.get("id") == model_id), None) + if not match: + ids = ", ".join(m["id"] for m in models if m.get("id")) + return f"Model '{model_id}' is not downloaded. Use one of: {ids}.", None + label = match.get("name") or model_id + elif model_id: + label = model_id + return f"Generating a 3D model with {label}…", {"type": "generate_from_image", "model_id": model_id} + elif name == "create_workflow": steps = arguments.get("steps") or [] if not steps: @@ -861,6 +1049,8 @@ async def _chat_events(request: AgentChatRequest): token / thinking — answer and reasoning text, in pieces tool_start / tool_done — around each tool call + client_tool — between them, for a tool the app runs: the turn waits for + its answer on POST /agent/tool-result with the same call_id done — always last: the final answer, every action and the full reasoning """ messages: list[dict] = [{"role": "system", "content": SYSTEM_PROMPT}] @@ -868,6 +1058,8 @@ async def _chat_events(request: AgentChatRequest): # Inject scene context so the LLM knows current state if request.context: ctx_lines = [] + if request.context.get("page"): + ctx_lines.append(f"Current page: {request.context['page']}") if request.context.get("currentMeshPath"): ctx_lines.append(f"Current mesh path: {request.context['currentMeshPath']}") if request.context.get("meshTriangles"): @@ -878,19 +1070,17 @@ async def _chat_events(request: AgentChatRequest): "content": "Scene context:\n" + "\n".join(ctx_lines), }) - extensions = request.context.get("extensions") or [] - if extensions: - ext_lines = [ - f"- {e['id']} ({e.get('input', '?')}→{e.get('output', '?')}): {e.get('name', e['id'])}" - for e in extensions - ] - messages.append({ - "role": "system", - "content": ( - "Available extensions (use the exact id when creating workflows):\n" - + "\n".join(ext_lines) - ), - }) + sections = [ + ("Available extensions (use the exact id when creating workflows)", + _extension_lines(request.context.get("extensions") or [])), + ("Downloaded generation models (use the exact id with generate_from_image)", + _id_name_lines(request.context.get("models") or [])), + ("Workflows", _id_name_lines(request.context.get("workflows") or [])), + ("Settings", _settings_lines(request.context)), + ] + for title, lines in sections: + if lines: + messages.append({"role": "system", "content": f"{title}:\n" + "\n".join(lines)}) # ONE system message, first. Qwen3.5's chat template raises "System message # must be at the beginning", which llama-server returns as a flat HTTP 400; @@ -924,6 +1114,10 @@ async def _chat_events(request: AgentChatRequest): for m in request.messages: messages.append(_user_entry(m, send_images)) + # The image generate_from_image would use: the frontend takes it from the + # user's last message, as run_workflow does. + last_user = next((m for m in reversed(request.messages) if m.role == "user"), None) + has_image = bool(last_user and last_user.images) actions_done: list[ActionDone] = [] all_thinking: list[str] = [] @@ -988,17 +1182,43 @@ async def post() -> httpx.Response: break arguments = _tool_arguments(fn.get("arguments")) yield {"type": "tool_start", "id": tc["id"], "tool": fn["name"], "arguments": arguments} - try: - result_text, payload = await asyncio.wait_for( - execute_tool(fn["name"], arguments, request.context, timeout=request.tool_timeout), - request.tool_timeout, - ) - except asyncio.TimeoutError: - # Cancelling the call drops its HTTP request, but a mesh op - # the API already started runs to its end in a worker - # thread; its result is never shown. - stopped = _tool_stop("tool_timeout", fn["name"], request.tool_timeout) - result_text, payload = stopped["message"], None + if fn["name"] in CLIENT_TOOLS: + # The app runs it and answers through /agent/tool-result. + confirm = CLIENT_TOOLS[fn["name"]] + call_id = uuid.uuid4().hex + answer = asyncio.get_running_loop().create_future() + _pending_client_tools[call_id] = answer + try: + yield { + "type": "client_tool", "id": tc["id"], "call_id": call_id, + "tool": fn["name"], "arguments": arguments, "confirm": confirm, + } + # A confirmation waits on the user's click, not on the tool. + limit = CONFIRM_TIMEOUT if confirm else request.tool_timeout + try: + result_text = await asyncio.wait_for(answer, limit) + except asyncio.TimeoutError: + result_text = _NO_ANSWER + finally: + # Also runs when the client leaves: nobody will answer. + _pending_client_tools.pop(call_id, None) + answer.cancel() + payload = None + else: + try: + result_text, payload = await asyncio.wait_for( + execute_tool( + fn["name"], arguments, request.context, + timeout=request.tool_timeout, has_image=has_image, + ), + request.tool_timeout, + ) + except asyncio.TimeoutError: + # Cancelling the call drops its HTTP request, but a mesh op + # the API already started runs to its end in a worker + # thread; its result is never shown. + stopped = _tool_stop("tool_timeout", fn["name"], request.tool_timeout) + result_text, payload = stopped["message"], None actions_done.append(ActionDone(tool=fn["name"], result=result_text, payload=payload, error=stopped)) yield { "type": "tool_done", "id": tc["id"], "tool": fn["name"], @@ -1026,6 +1246,22 @@ async def post() -> httpx.Response: yield _done(final, actions_done, all_thinking) +class ToolResultRequest(BaseModel): + call_id: str + result: str + error: bool = False + + +@router.post("/tool-result") +async def agent_tool_result(req: ToolResultRequest): + """The app's answer to a client_tool event; the waiting turn resumes with it.""" + answer = _pending_client_tools.get(req.call_id) + if answer is None or answer.done(): + raise HTTPException(status_code=404, detail="No tool call is waiting for this call_id.") + answer.set_result(f"Error: {req.result}" if req.error else req.result) + return {"ok": True} + + @router.post("/chat", response_model=AgentChatResponse) async def agent_chat(request: AgentChatRequest): """The whole turn at once, for callers that don't read a stream.""" diff --git a/api/tests/test_agent_client_tools.py b/api/tests/test_agent_client_tools.py new file mode 100644 index 000000000..999a9ce01 --- /dev/null +++ b/api/tests/test_agent_client_tools.py @@ -0,0 +1,365 @@ +import asyncio +import json +import unittest +from unittest import mock + +import httpx +from fastapi import HTTPException + +import routers.agent as agent +from tests.test_agent_streaming import _call, _external, _Scripted, _sse + + +def _client_round(name: str, arguments: dict | None = None, call_id: str = "c1") -> httpx.Response: + return _sse(_call(0, id=call_id, function={"name": name, "arguments": json.dumps(arguments or {})})) + + +async def _answer(call_id: str, result: str, error: bool = False) -> dict: + return await agent.agent_tool_result(agent.ToolResultRequest(call_id=call_id, result=result, error=error)) + + +def _run_turn(scripted: _Scripted, request: agent.AgentChatRequest, on_client_tool) -> list[dict]: + """Drive a turn, letting `on_client_tool(event)` answer each client tool + the way the panel would (it may be a coroutine function).""" + + async def collect(): + events = [] + async for event in agent._chat_events(request): + events.append(event) + if event["type"] == "client_tool": + answered = on_client_tool(event) + if asyncio.iscoroutine(answered): + await answered + return events + + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + return asyncio.run(collect()) + + +class ClientToolBridgeTests(unittest.TestCase): + def tearDown(self) -> None: + agent._pending_client_tools.clear() + + def test_client_tool_is_handed_to_the_app_and_its_answer_reaches_the_model(self) -> None: + scripted = _Scripted( + _client_round("install_extension", {"github_url": "https://github.com/a/b"}), + _sse({"content": "Installed."}), + ) + execute = mock.AsyncMock(return_value=("never", None)) + answers = [] + + async def answer(event): + answers.append(await _answer(event["call_id"], "Installed b v1.0.")) + + with mock.patch.object(agent, "execute_tool", execute): + events = _run_turn(scripted, _external(), answer) + + execute.assert_not_awaited() + self.assertEqual([e["type"] for e in events], ["tool_start", "client_tool", "tool_done", "token", "done"]) + handoff = events[1] + self.assertEqual( + {k: v for k, v in handoff.items() if k != "call_id"}, + { + "type": "client_tool", "id": "c1", "tool": "install_extension", + "arguments": {"github_url": "https://github.com/a/b"}, "confirm": False, + }, + ) + self.assertRegex(handoff["call_id"], r"^[0-9a-f]{32}$") + self.assertEqual(answers, [{"ok": True}]) + self.assertEqual(events[2]["result"], "Installed b v1.0.") + self.assertEqual(events[-1]["actions"][0]["tool"], "install_extension") + self.assertEqual( + scripted.bodies[1]["messages"][-1], + {"role": "tool", "tool_call_id": "c1", "content": "Installed b v1.0."}, + ) + self.assertEqual(agent._pending_client_tools, {}) + + def test_answer_while_the_turn_is_waiting(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})) + + def answer_later(event): + async def later(): + await asyncio.sleep(0.01) + await _answer(event["call_id"], "Reloaded.") + asyncio.ensure_future(later()) + + events = _run_turn(scripted, _external(), answer_later) + + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Reloaded.") + self.assertFalse(events[-1]["error"]) + + def test_destructive_tool_asks_for_confirmation_and_a_decline_is_relayed(self) -> None: + scripted = _Scripted( + _client_round("uninstall_extension", {"extension_id": "x"}), + _sse({"content": "Kept it."}), + ) + events = _run_turn(scripted, _external(), lambda e: _answer(e["call_id"], "The user declined.")) + + handoff = next(e for e in events if e["type"] == "client_tool") + self.assertTrue(handoff["confirm"]) + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "The user declined.") + + def test_error_answer_is_prefixed(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})) + events = _run_turn(scripted, _external(), lambda e: _answer(e["call_id"], "disk full", error=True)) + + self.assertEqual(events[2]["result"], "Error: disk full") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: disk full") + + def test_unknown_or_already_answered_call_is_404(self) -> None: + with self.assertRaises(HTTPException) as caught: + asyncio.run(_answer("nope", "x")) + self.assertEqual(caught.exception.status_code, 404) + + seen = [] + + async def answer_twice(event): + await _answer(event["call_id"], "first") + try: + await _answer(event["call_id"], "second") + except HTTPException as e: + seen.append(e.status_code) + + _run_turn(_Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})), _external(), answer_twice) + self.assertEqual(seen, [404]) + + def test_unanswered_tool_times_out_and_the_loop_continues(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "It did not answer."})) + events = _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), lambda e: None) + + finish = next(e for e in events if e["type"] == "tool_done") + self.assertEqual(finish["result"], "Error: The app did not answer in time.") + self.assertIsNone(finish["error"]) + done = events[-1] + self.assertFalse(done["error"]) + self.assertIsNone(done["stopped"]) + self.assertEqual(done["message"], "It did not answer.") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: The app did not answer in time.") + self.assertEqual(agent._pending_client_tools, {}) + + def test_confirm_tools_wait_for_the_confirm_cap_not_the_tool_timeout(self) -> None: + scripted = _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})) + + def answer_after_tool_timeout(event): + async def later(): + await asyncio.sleep(0.15) + await _answer(event["call_id"], "Deleted.") + asyncio.ensure_future(later()) + + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 5.0): + _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), answer_after_tool_timeout) + + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Deleted.") + + def test_confirm_wait_is_capped(self) -> None: + self.assertEqual(agent.CONFIRM_TIMEOUT, 600) + scripted = _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})) + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 0.05): + events = _run_turn(scripted, _external(), lambda e: None) + + self.assertEqual(events[2]["result"], "Error: The app did not answer in time.") + + def test_client_tools_count_against_the_budget(self) -> None: + both = _sse( + _call(0, id="a", function={"name": "reload_extensions", "arguments": "{}"}), + _call(1, id="b", function={"name": "undo_mesh", "arguments": "{}"}), + ) + events = _run_turn( + _Scripted(both), _external().model_copy(update={"tool_budget": 1}), + lambda e: _answer(e["call_id"], "ok"), + ) + + self.assertEqual(sum(e["type"] == "client_tool" for e in events), 1) + self.assertEqual(events[-1]["stopped"]["code"], "tool_budget_exhausted") + self.assertEqual(events[-1]["stopped"]["tool"], "undo_mesh") + + def test_closing_the_stream_while_a_tool_waits_cancels_it(self) -> None: + async def leave_at_handoff(): + stream = agent._chat_events(_external()) + async for event in stream: + if event["type"] == "client_tool": + future = agent._pending_client_tools[event["call_id"]] + await stream.aclose() + return event, future + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_client_round("reload_extensions"))): + event, future = asyncio.run(leave_at_handoff()) + + self.assertTrue(future.cancelled()) + self.assertNotIn(event["call_id"], agent._pending_client_tools) + + def test_cancelling_the_turn_while_it_awaits_the_app_cancels_the_future(self) -> None: + async def cancel_while_waiting(): + stream = agent._chat_events(_external()) + while (event := await stream.__anext__())["type"] != "client_tool": + pass + future = agent._pending_client_tools[event["call_id"]] + waiting = asyncio.ensure_future(stream.__anext__()) + await asyncio.sleep(0.01) + waiting.cancel() + with self.assertRaises(asyncio.CancelledError): + await waiting + return event, future + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_client_round("reload_extensions"))): + event, future = asyncio.run(cancel_while_waiting()) + + self.assertTrue(future.cancelled()) + self.assertNotIn(event["call_id"], agent._pending_client_tools) + + +_CLIENT_TOOL_ARGS = { + "install_extension": ["github_url"], + "uninstall_extension": ["extension_id"], + "reload_extensions": [], + "download_model_weights": ["extension_id", "variant"], + "delete_model_weights": ["extension_id", "variant"], + "delete_workflow": ["workflow_id"], + "open_workflow": ["workflow_id"], + "set_setting": ["key", "value"], + "set_storage_folder": ["kind", "path"], + "undo_mesh": [], + "redo_mesh": [], + "navigate": ["page"], +} + + +class ClientToolDeclarationTests(unittest.TestCase): + def _tool(self, name: str) -> dict: + return next(t["function"] for t in agent.TOOLS if t["function"]["name"] == name) + + def test_every_client_tool_is_declared_with_its_confirmation(self) -> None: + self.assertEqual(set(agent.CLIENT_TOOLS), set(_CLIENT_TOOL_ARGS)) + confirmed = {name for name, confirm in agent.CLIENT_TOOLS.items() if confirm} + self.assertEqual( + confirmed, + {"uninstall_extension", "delete_model_weights", "delete_workflow", "set_storage_folder"}, + ) + + def test_every_client_tool_has_a_schema(self) -> None: + for name, args in _CLIENT_TOOL_ARGS.items(): + with self.subTest(name=name): + fn = self._tool(name) + self.assertTrue(fn["description"]) + self.assertEqual(sorted(fn["parameters"]["properties"]), sorted(args)) + + def test_enums(self) -> None: + props = self._tool("set_setting")["parameters"]["properties"] + self.assertEqual(props["key"]["enum"], ["uiScale", "useAtkinsonFont", "showRamIndicator"]) + props = self._tool("set_storage_folder")["parameters"]["properties"] + self.assertEqual(props["kind"]["enum"], ["models", "workspace", "workflows"]) + nav = self._tool("navigate") + self.assertEqual(nav["parameters"]["properties"]["page"]["enum"], ["generate", "workflows", "models", "settings"]) + self.assertIn("ONLY", nav["description"]) + + def test_tool_names_are_unique(self) -> None: + names = [t["function"]["name"] for t in agent.TOOLS] + self.assertEqual(len(names), len(set(names))) + + def test_system_prompt_lists_the_new_tools(self) -> None: + for name in [*_CLIENT_TOOL_ARGS, "list_extensions", "get_settings", "generate_from_image"]: + with self.subTest(name=name): + self.assertIn(name, agent.SYSTEM_PROMPT) + self.assertIn("declined", agent.SYSTEM_PROMPT) + + +_CONTEXT = { + "extensions": [{"id": "trellis/gen", "name": "Trellis", "input": "image", "output": "mesh"}], + "models": [{"id": "trellis", "name": "Trellis 2"}, {"id": "sf3d", "name": "SF3D"}], + "storage": { + "modelsDir": "D:/models", "workspaceDir": "D:/ws", + "workflowsDir": "D:/wf", "extensionsDir": "D:/ext", + }, + "ui": {"uiScale": 1.1, "useAtkinsonFont": False, "showRamIndicator": True}, +} + + +def _run(name: str, arguments: dict | None = None, context: dict | None = None, **kwargs): + return asyncio.run(agent.execute_tool(name, arguments or {}, _CONTEXT if context is None else context, **kwargs)) + + +class ServerToolTests(unittest.TestCase): + def test_list_extensions(self) -> None: + text, payload = _run("list_extensions") + self.assertIn("- trellis/gen (image→mesh): Trellis", text) + self.assertIsNone(payload) + self.assertIn("No extensions", _run("list_extensions", context={})[0]) + + def test_get_settings(self) -> None: + text, payload = _run("get_settings") + for expected in ("D:/models", "D:/ws", "D:/wf", "D:/ext", "uiScale: 1.1", "useAtkinsonFont: false", "showRamIndicator: true"): + self.assertIn(expected, text) + self.assertIsNone(payload) + self.assertIn("No settings", _run("get_settings", context={})[0]) + + def test_generate_from_image_needs_an_attached_image(self) -> None: + text, payload = _run("generate_from_image") + self.assertIsNone(payload) + self.assertIn("attach", text.lower()) + + def test_generate_from_image_hands_the_run_to_the_app(self) -> None: + text, payload = _run("generate_from_image", {"model_id": "sf3d"}, has_image=True) + self.assertEqual(payload, {"type": "generate_from_image", "model_id": "sf3d"}) + self.assertIn("SF3D", text) + + text, payload = _run("generate_from_image", has_image=True) + self.assertEqual(payload, {"type": "generate_from_image", "model_id": None}) + + def test_generate_from_image_rejects_an_unknown_model(self) -> None: + text, payload = _run("generate_from_image", {"model_id": "ghost"}, has_image=True) + self.assertIsNone(payload) + self.assertIn("ghost", text) + self.assertIn("trellis", text) + + def test_the_turn_tells_the_tool_whether_the_last_user_message_has_an_image(self) -> None: + def turn(messages): + request = _external().model_copy(update={"messages": messages}) + tool = mock.AsyncMock(return_value=("ok", None)) + with mock.patch.object(agent, "execute_tool", tool): + _run_turn(_Scripted(_client_round("generate_from_image"), _sse({"content": "ok"})), request, None) + return tool.await_args.kwargs["has_image"] + + img = "data:image/png;base64,AAAA" + self.assertTrue(turn([agent.ChatMessage(role="user", content="go", images=[img])])) + self.assertFalse(turn([ + agent.ChatMessage(role="user", content="a", images=[img]), + agent.ChatMessage(role="assistant", content="b"), + agent.ChatMessage(role="user", content="now generate"), + ])) + + +class ContextRenderingTests(unittest.TestCase): + def test_context_is_rendered_into_the_single_system_message(self) -> None: + context = { + **_CONTEXT, + "page": "models", + "workflows": [{"id": "wf1", "name": "My Workflow"}], + "currentMeshPath": "Default/mesh.glb", + } + scripted = _Scripted(_sse({"content": "hi"})) + _run_turn(scripted, _external(context), None) + + messages = scripted.bodies[0]["messages"] + self.assertEqual(sum(m["role"] == "system" for m in messages), 1) + system = messages[0]["content"] + for expected in ( + "Current page: models", + "- trellis: Trellis 2", + "- sf3d: SF3D", + "- wf1: My Workflow", + "D:/models", "D:/ws", "D:/wf", "D:/ext", + "uiScale: 1.1", "useAtkinsonFont: false", "showRamIndicator: true", + "Current mesh path: Default/mesh.glb", + "- trellis/gen (image→mesh): Trellis", + ): + with self.subTest(expected=expected): + self.assertIn(expected, system) + + def test_empty_context_adds_nothing(self) -> None: + scripted = _Scripted(_sse({"content": "hi"})) + _run_turn(scripted, _external(), None) + self.assertEqual(scripted.bodies[0]["messages"][0]["content"], agent.SYSTEM_PROMPT) + + +if __name__ == "__main__": + unittest.main() From be528925fde4e07b55675eb559534312966f6599 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:03:44 +0100 Subject: [PATCH 040/108] feat(agent): persist agent conversations as JSON files over IPC Adds window.electron.agent.conversations (list/read/save/remove), one file per conversation in /agent/conversations. Ids are checked against /^[a-z0-9-]{1,64}$/ and writes go through a temp file plus rename. Pure title/sort/settle helpers live in src/shared/utils/agentConversations. --- electron/main/agent-conversations.test.ts | 109 ++++++++++++++++++++ electron/main/agent-conversations.ts | 93 +++++++++++++++++ electron/main/ipc-handlers.ts | 9 ++ electron/preload/electron-api.ts | 15 +++ package.json | 2 +- src/shared/types/electron.d.ts | 11 ++ src/shared/utils/agentConversations.test.ts | 77 ++++++++++++++ src/shared/utils/agentConversations.ts | 81 +++++++++++++++ 8 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 electron/main/agent-conversations.test.ts create mode 100644 electron/main/agent-conversations.ts create mode 100644 src/shared/utils/agentConversations.test.ts create mode 100644 src/shared/utils/agentConversations.ts diff --git a/electron/main/agent-conversations.test.ts b/electron/main/agent-conversations.test.ts new file mode 100644 index 000000000..7d8387778 --- /dev/null +++ b/electron/main/agent-conversations.test.ts @@ -0,0 +1,109 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { mkdtemp, readdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { + createConversationStorage, + isConversation, + isValidConversationId, + sortSummaries, +} from './agent-conversations' + +async function withDir(fn: (dir: string) => Promise): Promise { + const root = await mkdtemp(join(tmpdir(), 'modly-conv-')) + try { + await fn(join(root, 'agent', 'conversations')) + } finally { + await rm(root, { recursive: true, force: true }) + } +} + +const conv = (id: string, updatedAt: number, title = id) => + ({ id, title, createdAt: 1, updatedAt, messages: [{ id: 'u', role: 'user', content: title }] }) + +test('conversation ids: uuids pass, path tricks and odd characters do not', () => { + assert.equal(isValidConversationId('3f2a9c1e-8b7d-4e6f-a5b4-c3d2e1f0a9b8'), true) + assert.equal(isValidConversationId('a'), true) + assert.equal(isValidConversationId('a'.repeat(64)), true) + assert.equal(isValidConversationId('a'.repeat(65)), false) + assert.equal(isValidConversationId(''), false) + assert.equal(isValidConversationId('../secrets'), false) + assert.equal(isValidConversationId('..\\secrets'), false) + assert.equal(isValidConversationId('a/b'), false) + assert.equal(isValidConversationId('ABC'), false) + assert.equal(isValidConversationId('abc.json'), false) + assert.equal(isValidConversationId('abc\n'), false) + assert.equal(isValidConversationId(42), false) +}) + +test('isConversation checks the stored shape', () => { + assert.equal(isConversation(conv('abc-1', 2)), true) + assert.equal(isConversation({ ...conv('abc-1', 2), id: '../x' }), false) + assert.equal(isConversation({ ...conv('abc-1', 2), messages: 'nope' }), false) + assert.equal(isConversation({ ...conv('abc-1', 2), updatedAt: '2' }), false) + assert.equal(isConversation(null), false) +}) + +test('sortSummaries puts the newest first', () => { + const sorted = sortSummaries([conv('a', 10), conv('b', 30), conv('c', 20)]) + assert.deepEqual(sorted.map((s) => s.id), ['b', 'c', 'a']) +}) + +test('list is empty before anything is saved', async () => { + await withDir(async (dir) => { + assert.deepEqual(await createConversationStorage(dir).list(), []) + }) +}) + +test('save, read, list newest first, remove', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('old', 10, 'Old')) + await storage.save(conv('new', 20, 'New')) + assert.deepEqual(await storage.read('old'), conv('old', 10, 'Old')) + assert.deepEqual(await storage.list(), [ + { id: 'new', title: 'New', updatedAt: 20 }, + { id: 'old', title: 'Old', updatedAt: 10 }, + ]) + // No temp file is left behind. + assert.deepEqual((await readdir(dir)).sort(), ['new.json', 'old.json']) + + await storage.remove('old') + assert.equal(await storage.read('old'), null) + assert.deepEqual((await storage.list()).map((s) => s.id), ['new']) + }) +}) + +test('saving again overwrites the file', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('a', 1, 'First')) + await storage.save(conv('a', 2, 'Second')) + assert.equal((await storage.read('a'))?.title, 'Second') + }) +}) + +test('invalid ids and shapes are refused', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await assert.rejects(storage.save({ ...conv('a', 1), id: '../escape' })) + await assert.rejects(storage.save({ id: 'a' })) + await assert.rejects(storage.remove('../escape')) + assert.equal(await storage.read('../escape'), null) + }) +}) + +test('corrupted or mismatched files are skipped', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('good', 5)) + await writeFile(join(dir, 'broken.json'), '{not json', 'utf-8') + await writeFile(join(dir, 'other.json'), JSON.stringify(conv('different', 9)), 'utf-8') + await writeFile(join(dir, 'notes.txt'), 'hi', 'utf-8') + assert.deepEqual((await storage.list()).map((s) => s.id), ['good']) + assert.equal(await storage.read('broken'), null) + assert.equal(await storage.read('other'), null) + }) +}) diff --git a/electron/main/agent-conversations.ts b/electron/main/agent-conversations.ts new file mode 100644 index 000000000..24c130fd2 --- /dev/null +++ b/electron/main/agent-conversations.ts @@ -0,0 +1,93 @@ +// Agent conversations on disk: one JSON file per conversation in +// /agent/conversations/{id}.json. + +import { join } from 'path' +import { mkdir, readFile, readdir, rename, rm, writeFile } from 'fs/promises' + +export interface ConversationSummary { + id: string + title: string + updatedAt: number +} + +export interface Conversation extends ConversationSummary { + createdAt: number + messages: unknown[] +} + +export const CONVERSATION_ID_RE = /^[a-z0-9-]{1,64}$/ + +/** Ids name files on disk: anything else could escape the conversations folder. */ +export function isValidConversationId(id: unknown): id is string { + return typeof id === 'string' && CONVERSATION_ID_RE.test(id) +} + +/** Shape check for a conversation read from disk or received over IPC. */ +export function isConversation(value: unknown): value is Conversation { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false + const c = value as Record + return isValidConversationId(c.id) + && typeof c.title === 'string' + && typeof c.createdAt === 'number' + && typeof c.updatedAt === 'number' + && Array.isArray(c.messages) +} + +/** Newest first; ties keep a stable order by id. */ +export function sortSummaries(list: readonly T[]): T[] { + return [...list].sort((a, b) => (b.updatedAt - a.updatedAt) || a.id.localeCompare(b.id)) +} + +export function createConversationStorage(dir: string) { + const fileFor = (id: string): string => join(dir, `${id}.json`) + + async function readConversation(id: string): Promise { + if (!isValidConversationId(id)) return null + try { + const parsed: unknown = JSON.parse(await readFile(fileFor(id), 'utf-8')) + return isConversation(parsed) && parsed.id === id ? parsed : null + } catch { + return null + } + } + + return { + async list(): Promise { + let files: string[] + try { + files = await readdir(dir) + } catch { + return [] + } + const summaries: ConversationSummary[] = [] + for (const file of files) { + if (!file.endsWith('.json')) continue + const c = await readConversation(file.slice(0, -'.json'.length)) + if (c) summaries.push({ id: c.id, title: c.title, updatedAt: c.updatedAt }) + } + return sortSummaries(summaries) + }, + + read: readConversation, + + async save(conversation: unknown): Promise { + if (!isConversation(conversation)) throw new Error('Invalid conversation') + await mkdir(dir, { recursive: true }) + const target = fileFor(conversation.id) + // Write then rename: a crash mid-write never leaves a truncated file. + const tmp = `${target}.${process.pid}.${Date.now()}.tmp` + await writeFile(tmp, JSON.stringify(conversation), 'utf-8') + try { + await rename(tmp, target) + } catch (err) { + await rm(tmp, { force: true }) + throw err + } + }, + + async remove(id: string): Promise { + if (!isValidConversationId(id)) throw new Error('Invalid conversation id') + await rm(fileFor(id), { force: true }) + }, + } +} diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 33a9b2409..039705780 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -50,6 +50,7 @@ import { checkSetupNeeded, markSetupDone, runFullSetup, getVenvPythonExe, ensure import { logger } from './logger' import { getProcessRunner, getPythonProcessRunner, getExtPythonExe, terminateProcessRunner, terminateAllProcessRunners } from './process-runner' import { getBuiltinExtensionsDir } from './builtin-sync' +import { createConversationStorage } from './agent-conversations' import { spawn, execFile } from 'child_process' import { EXT_INCOMPLETE_MARKER, @@ -1078,6 +1079,14 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe return result.canceled ? null : result.filePaths[0] }) + // Agent conversations: one JSON file each under /agent/conversations. + // Ids are validated in the storage layer, so no path escapes that folder. + const conversations = createConversationStorage(join(app.getPath('userData'), 'agent', 'conversations')) + ipcMain.handle('agent:conversations:list', () => conversations.list()) + ipcMain.handle('agent:conversations:read', (_, id: string) => conversations.read(id)) + ipcMain.handle('agent:conversations:save', (_, conversation: unknown) => conversations.save(conversation)) + ipcMain.handle('agent:conversations:remove', (_, id: string) => conversations.remove(id)) + // Add a local GGUF to the agent's models folder. Picking and copying both // happen here, so the renderer never hands main an arbitrary path to copy. ipcMain.handle('agent:addModel', async (): Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }> => { diff --git a/electron/preload/electron-api.ts b/electron/preload/electron-api.ts index 04b50ca06..d11863beb 100644 --- a/electron/preload/electron-api.ts +++ b/electron/preload/electron-api.ts @@ -5,6 +5,10 @@ import type { AssetLibraryReadRequest, AssetLibraryReadResult, } from '../../src/shared/types/assetLibrary' +import type { + Conversation as AgentConversation, + ConversationSummary as AgentConversationSummary, +} from '../main/agent-conversations' export interface IpcRendererLike { invoke(channel: string, ...args: unknown[]): Promise @@ -120,6 +124,17 @@ export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFra // Opens a file picker and copies the chosen .gguf into the agent's models folder. addModel: (): Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }> => ipcRenderer.invoke('agent:addModel') as Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }>, + // Persisted chat conversations (/agent/conversations/{id}.json). + conversations: { + list: (): Promise => + ipcRenderer.invoke('agent:conversations:list') as Promise, + read: (id: string): Promise => + ipcRenderer.invoke('agent:conversations:read', id) as Promise, + save: (conversation: AgentConversation): Promise => + ipcRenderer.invoke('agent:conversations:save', conversation) as Promise, + remove: (id: string): Promise => + ipcRenderer.invoke('agent:conversations:remove', id) as Promise, + }, }, // Settings diff --git a/package.json b/package.json index 22e207ac9..a8289b931 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/shared/types/electron.d.ts b/src/shared/types/electron.d.ts index 2be5a5dc1..675827f62 100644 --- a/src/shared/types/electron.d.ts +++ b/src/shared/types/electron.d.ts @@ -8,6 +8,7 @@ import type { AssetLibraryReadRequest, AssetLibraryReadResult, } from './assetLibrary' +import type { Conversation, ConversationSummary } from '../utils/agentConversations' // ─── Extension types ────────────────────────────────────────────────────────── @@ -228,6 +229,16 @@ declare global { agent: { /** Opens a file picker and copies the chosen .gguf into the agent's models folder. */ addModel: () => Promise<{ success: boolean; cancelled?: boolean; fileName?: string; error?: string }> + /** Persisted chat conversations, one JSON file each in /agent/conversations. */ + conversations: { + /** Newest first. */ + list: () => Promise + /** null when the id is invalid, missing or the file is corrupted. */ + read: (id: string) => Promise + /** Rejects on an invalid id or shape. */ + save: (conversation: Conversation) => Promise + remove: (id: string) => Promise + } } cache: { clear: () => Promise<{ success: boolean; error?: string }> diff --git a/src/shared/utils/agentConversations.test.ts b/src/shared/utils/agentConversations.test.ts new file mode 100644 index 000000000..8ed04cc6a --- /dev/null +++ b/src/shared/utils/agentConversations.test.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + TITLE_MAX_LENGTH, + UNTITLED, + deriveTitle, + relativeDate, + settleMessages, + sortSummaries, + upsertSummary, +} from './agentConversations' + +test('title is the first user message, collapsed to one line', () => { + const title = deriveTitle([ + { role: 'assistant', content: 'Hello' }, + { role: 'user', content: ' make the\n\nmesh smoother ' }, + { role: 'user', content: 'second' }, + ]) + assert.equal(title, 'make the mesh smoother') +}) + +test('a long first message is cut to the title limit', () => { + const title = deriveTitle([{ role: 'user', content: 'x'.repeat(200) }]) + assert.equal(title.length, TITLE_MAX_LENGTH) + assert.ok(title.endsWith('…')) + assert.equal(deriveTitle([{ role: 'user', content: 'y'.repeat(TITLE_MAX_LENGTH) }]), 'y'.repeat(TITLE_MAX_LENGTH)) +}) + +test('no usable user message gives the placeholder title', () => { + assert.equal(deriveTitle([]), UNTITLED) + assert.equal(deriveTitle([{ role: 'assistant', content: 'hi' }, { role: 'user', content: ' ' }, null]), UNTITLED) +}) + +test('summaries sort newest first without mutating the input', () => { + const list = [ + { id: 'a', title: 'A', updatedAt: 10 }, + { id: 'b', title: 'B', updatedAt: 30 }, + { id: 'c', title: 'C', updatedAt: 20 }, + ] + assert.deepEqual(sortSummaries(list).map((s) => s.id), ['b', 'c', 'a']) + assert.deepEqual(list.map((s) => s.id), ['a', 'b', 'c']) +}) + +test('upsertSummary replaces an entry and moves it to the top', () => { + const list = [ + { id: 'b', title: 'B', updatedAt: 30 }, + { id: 'a', title: 'A', updatedAt: 10 }, + ] + assert.deepEqual(upsertSummary(list, { id: 'a', title: 'A2', updatedAt: 40 }), [ + { id: 'a', title: 'A2', updatedAt: 40 }, + { id: 'b', title: 'B', updatedAt: 30 }, + ]) + assert.equal(upsertSummary(list, { id: 'c', title: 'C', updatedAt: 1 }).length, 3) +}) + +test('settleMessages clears streaming and running flags', () => { + const settled = settleMessages([ + { id: 'u', role: 'user', content: 'hi' }, + { id: 'a', role: 'assistant', content: 'part', streaming: true, actions: [{ id: 't', tool: 'x', running: true }] }, + ]) + assert.deepEqual(settled, [ + { id: 'u', role: 'user', content: 'hi' }, + { id: 'a', role: 'assistant', content: 'part', streaming: false, actions: [{ id: 't', tool: 'x', running: false }] }, + ]) +}) + +test('relativeDate buckets by age', () => { + const now = Date.UTC(2026, 10, 12, 12, 0, 0) + assert.equal(relativeDate(now - 5_000, now), 'just now') + assert.equal(relativeDate(now - 5 * 60_000, now), '5m ago') + assert.equal(relativeDate(now - 3 * 3_600_000, now), '3h ago') + assert.equal(relativeDate(now - 2 * 86_400_000, now), '2d ago') + assert.equal(relativeDate(now + 60_000, now), 'just now') + assert.match(relativeDate(now - 30 * 86_400_000, now), /^Oct \d+$/) + assert.match(relativeDate(Date.UTC(2024, 0, 15), now), /2024/) +}) diff --git a/src/shared/utils/agentConversations.ts b/src/shared/utils/agentConversations.ts new file mode 100644 index 000000000..06c8e3dd0 --- /dev/null +++ b/src/shared/utils/agentConversations.ts @@ -0,0 +1,81 @@ +// Pure renderer-side logic for persisted agent conversations. File storage +// and id validation live in electron/main/agent-conversations.ts. + +export interface ConversationSummary { + id: string + title: string + /** Epoch milliseconds. */ + updatedAt: number +} + +export interface Conversation { + id: string + title: string + /** Epoch milliseconds. */ + createdAt: number + /** Epoch milliseconds. */ + updatedAt: number + /** AgentChat messages; attached images stay inline as data URLs. */ + messages: unknown[] +} + +export const TITLE_MAX_LENGTH = 60 +export const UNTITLED = 'New conversation' + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +/** The first user message, on one line, cut to TITLE_MAX_LENGTH characters. */ +export function deriveTitle(messages: readonly unknown[]): string { + for (const m of messages) { + if (!isRecord(m) || m.role !== 'user' || typeof m.content !== 'string') continue + const text = m.content.replace(/\s+/g, ' ').trim() + if (!text) continue + return text.length > TITLE_MAX_LENGTH ? `${text.slice(0, TITLE_MAX_LENGTH - 1).trimEnd()}…` : text + } + return UNTITLED +} + +/** Newest first; ties keep a stable order by id. */ +export function sortSummaries(list: readonly T[]): T[] { + return [...list].sort((a, b) => (b.updatedAt - a.updatedAt) || a.id.localeCompare(b.id)) +} + +/** Replace (or add) one summary and keep the list sorted. */ +export function upsertSummary(list: readonly ConversationSummary[], summary: ConversationSummary): ConversationSummary[] { + return sortSummaries([...list.filter((s) => s.id !== summary.id), summary]) +} + +/** + * A conversation saved mid-turn holds a streaming message and running tools. + * Nothing is streaming once it is loaded again, so settle them. + */ +export function settleMessages(messages: readonly unknown[]): unknown[] { + return messages.map((m) => { + if (!isRecord(m)) return m + const settled: Record = { ...m } + if (settled.streaming) settled.streaming = false + if (Array.isArray(settled.actions)) { + settled.actions = settled.actions.map((a) => (isRecord(a) && a.running ? { ...a, running: false } : a)) + } + return settled + }) +} + +const MINUTE = 60_000 +const HOUR = 60 * MINUTE +const DAY = 24 * HOUR + +/** "just now", "5m ago", "3h ago", "2d ago", then a short date. */ +export function relativeDate(ts: number, now: number = Date.now()): string { + const diff = Math.max(0, now - ts) + if (diff < MINUTE) return 'just now' + if (diff < HOUR) return `${Math.floor(diff / MINUTE)}m ago` + if (diff < DAY) return `${Math.floor(diff / HOUR)}h ago` + if (diff < 7 * DAY) return `${Math.floor(diff / DAY)}d ago` + const d = new Date(ts) + const opts: Intl.DateTimeFormatOptions = { month: 'short', day: 'numeric' } + if (d.getFullYear() !== new Date(now).getFullYear()) opts.year = 'numeric' + return d.toLocaleDateString('en-US', opts) +} From 4e791fd4ba54b1d387138d9604adebcfc2323089 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:03:44 +0100 Subject: [PATCH 041/108] feat(agent): global right-side agent panel The chat leaves the Generate page's basic/chat toggle and becomes the body of an AgentPanel next to
in MainLayout, so it pushes the app. The panel toggles from a TopBar button or Ctrl/Cmd+L, resizes from its left edge (320-640px) and remembers its open state and width. Messages move to agentConversationStore: they survive page changes and panel close, save to disk after 500ms of quiet, and the latest conversation reopens at startup. The header lists past conversations with open and delete. --- .../generate/components/WorkflowPanel.tsx | 112 +++---- .../components/agent/AgentChat.tsx} | 42 +-- src/shared/components/agent/AgentPanel.tsx | 273 ++++++++++++++++++ src/shared/components/layout/MainLayout.tsx | 5 +- src/shared/components/layout/TopBar.tsx | 24 ++ src/shared/stores/agentConversationStore.ts | 170 +++++++++++ src/shared/stores/agentPanelStore.ts | 43 +++ 7 files changed, 571 insertions(+), 98 deletions(-) rename src/{areas/generate/components/ChatPanel.tsx => shared/components/agent/AgentChat.tsx} (96%) create mode 100644 src/shared/components/agent/AgentPanel.tsx create mode 100644 src/shared/stores/agentConversationStore.ts create mode 100644 src/shared/stores/agentPanelStore.ts diff --git a/src/areas/generate/components/WorkflowPanel.tsx b/src/areas/generate/components/WorkflowPanel.tsx index 14b257815..d1be5e9f7 100644 --- a/src/areas/generate/components/WorkflowPanel.tsx +++ b/src/areas/generate/components/WorkflowPanel.tsx @@ -19,9 +19,6 @@ import type { Workflow, WFNode, WFEdge, ParamSchema } from '@shared/types/electr import { PICKER_LABELS, openParamPicker, resolvePickerIntent } from '@shared/utils/paramPicker' import { FloatInput, IntInput, PickerIcon } from '@shared/components/ui' import { isMissingWeightVariant, withWeightVariantAvailability } from '@shared/utils/weightVariants' -import ChatPanel from './ChatPanel' - -type PanelMode = 'basic' | 'chat' // ─── Constants ──────────────────────────────────────────────────────────────── @@ -610,30 +607,6 @@ function EmbeddedCanvas({ workflow, allExtensions }: { ) } -// ─── Mode toggle ────────────────────────────────────────────────────────────── - -function ModeToggle({ mode, onChange }: { mode: PanelMode; onChange: (m: PanelMode) => void }): JSX.Element { - return ( -
-
- {(['basic', 'chat'] as PanelMode[]).map((m) => ( - - ))} -
-
- ) -} - // ─── Main panel ─────────────────────────────────────────────────────────────── export default function WorkflowPanel() { @@ -642,7 +615,6 @@ export default function WorkflowPanel() { const loadExtensions = useExtensionsStore((s) => s.loadExtensions) const { navigate } = useNavStore() const [selectedId, setSelectedId] = useState(activeId) - const [mode, setMode] = useState('basic') const allExtensions = useMemo( () => buildAllWorkflowExtensions(modelExtensions, processExtensions), @@ -667,56 +639,44 @@ export default function WorkflowPanel() { return (
- {/* Mode toggle */} - - - {mode === 'chat' ? ( - <> -
- - - ) : ( - <> - {/* Header */} -
-

Workflow

-
-
- -
- {selectedId && ( - - )} -
+ {/* Header */} +
+

Workflow

+
+
+
- - {/* Canvas or empty state */} - {workflow ? ( - - - - ) : ( -
-

- No workflows yet.
Create one in the Workflows tab. -

-
+ {selectedId && ( + )} - +
+
+ + {/* Canvas or empty state */} + {workflow ? ( + + + + ) : ( +
+

+ No workflows yet.
Create one in the Workflows tab. +

+
)}
) diff --git a/src/areas/generate/components/ChatPanel.tsx b/src/shared/components/agent/AgentChat.tsx similarity index 96% rename from src/areas/generate/components/ChatPanel.tsx rename to src/shared/components/agent/AgentChat.tsx index 24d76f238..d155c80cb 100644 --- a/src/areas/generate/components/ChatPanel.tsx +++ b/src/shared/components/agent/AgentChat.tsx @@ -8,30 +8,16 @@ import { useWorkflowRunStore } from '@areas/workflows/workflowRunStore' import { buildAllWorkflowExtensions } from '@areas/workflows/mockExtensions' import { readAgentEvents, toolStopTitle } from '@shared/utils/agentStream' import { adviceText, ollamaAdvice } from '@shared/utils/ollamaHealth' -import OllamaBanner, { useOllamaHealth } from './OllamaBanner' -import { commitSculpt } from '../sculpt/sculptStore' +import OllamaBanner, { useOllamaHealth } from '@areas/generate/components/OllamaBanner' +import { commitSculpt } from '@areas/generate/sculpt/sculptStore' +import { useAgentConversationStore } from '@shared/stores/agentConversationStore' // ─── Types ──────────────────────────────────────────────────────────────────── import type { ThinkingMode } from '@shared/stores/agentStore' import type { Workflow } from '@shared/types/electron.d' -import type { AgentAction, AgentActionPayload, ToolStop } from '@shared/utils/agentStream' - -interface Message { - id: string - role: 'user' | 'assistant' - content: string - thinking?: string - imageDataUrls?: string[] - actions?: ActionDone[] - stopped?: ToolStop | null - streaming?: boolean -} - -interface ActionDone extends AgentAction { - id?: string - running?: boolean -} +import type { AgentActionPayload, ToolStop } from '@shared/utils/agentStream' +import type { ActionDone, AgentMessage as Message } from '@shared/stores/agentConversationStore' // ─── Constants ──────────────────────────────────────────────────────────────── @@ -270,13 +256,18 @@ function WorkflowProgressCard({ name }: { name: string }): JSX.Element { // ─── Main component ────────────────────────────────────────────────────────── -export default function ChatPanel(): JSX.Element { +export default function AgentChat(): JSX.Element { const { provider, localModel, external, defaultThinking, toolBudget, toolTimeout } = useAgentStore() const { models: llmModels } = useLlmModels() const isLocal = provider === 'local' const externalConfig = external[provider] - const [messages, setMessages] = useState([]) + // Messages live in the conversation store: they survive page changes and + // panel close, and are saved to disk. + const messages = useAgentConversationStore((s) => s.messages) + const setMessages = useAgentConversationStore((s) => s.setMessages) + const currentId = useAgentConversationStore((s) => s.currentId) + const setBusy = useAgentConversationStore((s) => s.setBusy) const [input, setInput] = useState('') const [isLoading, setIsLoading] = useState(false) const [error, setError] = useState(null) @@ -358,6 +349,15 @@ export default function ChatPanel(): JSX.Element { // eslint-disable-next-line react-hooks/exhaustive-deps -- fire on run-status transition; error/outputUrl read atomically }, [runState.status, pendingWorkflow]) + // The panel locks conversation switching while a turn or its workflow runs. + useEffect(() => { setBusy(isLoading || pendingWorkflow !== null) }, [isLoading, pendingWorkflow, setBusy]) + + // Another conversation was opened: drop the view state of the previous one. + useEffect(() => { + setShowAll(false) + setError(null) + }, [currentId]) + useEffect(() => { // Instant while streaming: a smooth scroll per token never catches up. endRef.current?.scrollIntoView({ behavior: isLoading ? 'auto' : 'smooth' }) diff --git a/src/shared/components/agent/AgentPanel.tsx b/src/shared/components/agent/AgentPanel.tsx new file mode 100644 index 000000000..c3ba61c90 --- /dev/null +++ b/src/shared/components/agent/AgentPanel.tsx @@ -0,0 +1,273 @@ +import { useEffect, useRef, useState } from 'react' +import { + AGENT_PANEL_DEFAULT_WIDTH, + AGENT_PANEL_MAX_WIDTH, + AGENT_PANEL_MIN_WIDTH, + clampPanelWidth, + useAgentPanelStore, +} from '@shared/stores/agentPanelStore' +import { useAgentConversationStore } from '@shared/stores/agentConversationStore' +import { UNTITLED, deriveTitle, relativeDate } from '@shared/utils/agentConversations' +import AgentChat from './AgentChat' + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function isEditable(el: EventTarget | null): boolean { + if (!(el instanceof HTMLElement)) return false + return el.isContentEditable || el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.tagName === 'SELECT' +} + +function IconButton({ label, onClick, pressed, disabled, children }: { + label: string + onClick: () => void + pressed?: boolean + disabled?: boolean + children: React.ReactNode +}): JSX.Element { + return ( + + ) +} + +// ─── Conversation list ──────────────────────────────────────────────────────── + +function ConversationList({ onPicked }: { onPicked: () => void }): JSX.Element { + const list = useAgentConversationStore((s) => s.list) + const currentId = useAgentConversationStore((s) => s.currentId) + const busy = useAgentConversationStore((s) => s.busy) + const open = useAgentConversationStore((s) => s.open) + const remove = useAgentConversationStore((s) => s.remove) + // Delete takes two clicks: the first arms it. + const [armedId, setArmedId] = useState(null) + const now = Date.now() + + if (list.length === 0) { + return ( +
+

No saved conversations yet.

+
+ ) + } + + return ( +
    setArmedId(null)}> + {list.map((c) => { + const active = c.id === currentId + const locked = busy && !active + return ( +
  • + + +
  • + ) + })} +
+ ) +} + +// ─── Panel ──────────────────────────────────────────────────────────────────── + +export default function AgentPanel(): JSX.Element { + const open = useAgentPanelStore((s) => s.open) + const width = useAgentPanelStore((s) => s.width) + const setOpen = useAgentPanelStore((s) => s.setOpen) + const toggle = useAgentPanelStore((s) => s.toggle) + const setWidth = useAgentPanelStore((s) => s.setWidth) + + const messages = useAgentConversationStore((s) => s.messages) + const busy = useAgentConversationStore((s) => s.busy) + const newConversation = useAgentConversationStore((s) => s.newConversation) + + const [showList, setShowList] = useState(false) + const [dragWidth, setDragWidth] = useState(null) + const panelRef = useRef(null) + const dragRef = useRef<{ startX: number; startWidth: number } | null>(null) + + const shownWidth = dragWidth ?? width + const title = messages.length > 0 ? deriveTitle(messages) : UNTITLED + + // Load the saved conversations and reopen the latest; save before quitting. + useEffect(() => { + const { init, flush } = useAgentConversationStore.getState() + void init() + const onUnload = () => { void flush() } + window.addEventListener('beforeunload', onUnload) + return () => window.removeEventListener('beforeunload', onUnload) + }, []) + + // Ctrl+L / Cmd+L toggles the panel, except while typing elsewhere. + useEffect(() => { + const handler = (e: KeyboardEvent) => { + if (!(e.ctrlKey || e.metaKey) || e.shiftKey || e.altKey || e.key.toLowerCase() !== 'l') return + const target = e.target as Node | null + if (isEditable(e.target) && !panelRef.current?.contains(target)) return + e.preventDefault() + toggle() + } + window.addEventListener('keydown', handler) + return () => window.removeEventListener('keydown', handler) + }, [toggle]) + + // Opening the panel puts the cursor in the chat input. + useEffect(() => { + if (!open || showList) return + panelRef.current?.querySelector('textarea')?.focus() + }, [open, showList]) + + // ── Resize ── + function onHandlePointerDown(e: React.PointerEvent) { + if (e.button !== 0) return + e.preventDefault() + e.currentTarget.setPointerCapture(e.pointerId) + dragRef.current = { startX: e.clientX, startWidth: width } + setDragWidth(width) + } + + function onHandlePointerMove(e: React.PointerEvent) { + const drag = dragRef.current + if (!drag) return + // The panel sits on the right: dragging left widens it. + setDragWidth(clampPanelWidth(drag.startWidth + drag.startX - e.clientX)) + } + + function onHandlePointerUp(e: React.PointerEvent) { + if (!dragRef.current) return + e.currentTarget.releasePointerCapture(e.pointerId) + dragRef.current = null + if (dragWidth !== null) setWidth(dragWidth) + setDragWidth(null) + } + + function onHandleKeyDown(e: React.KeyboardEvent) { + const step = e.shiftKey ? 64 : 16 + if (e.key === 'ArrowLeft') { e.preventDefault(); setWidth(width + step) } + if (e.key === 'ArrowRight') { e.preventDefault(); setWidth(width - step) } + } + + return ( + + ) +} diff --git a/src/shared/components/layout/MainLayout.tsx b/src/shared/components/layout/MainLayout.tsx index da5bcd9c5..71a0cccac 100644 --- a/src/shared/components/layout/MainLayout.tsx +++ b/src/shared/components/layout/MainLayout.tsx @@ -1,6 +1,7 @@ import TopBar from './TopBar' import Sidebar from './Sidebar' import Router from '@shared/router/Router' +import AgentPanel from '@shared/components/agent/AgentPanel' export default function MainLayout(): JSX.Element { @@ -11,9 +12,11 @@ export default function MainLayout(): JSX.Element {
-
+
+ +
) diff --git a/src/shared/components/layout/TopBar.tsx b/src/shared/components/layout/TopBar.tsx index c73fd02a0..056ddf8af 100644 --- a/src/shared/components/layout/TopBar.tsx +++ b/src/shared/components/layout/TopBar.tsx @@ -1,10 +1,13 @@ import { useEffect, useState } from 'react' import { useAppStore } from '@shared/stores/appStore' +import { useAgentPanelStore } from '@shared/stores/agentPanelStore' import MemoryIndicator from './MemoryIndicator' export default function TopBar(): JSX.Element { const { patchUpdateReady, platform, showRamIndicator } = useAppStore() const isMac = platform === 'darwin' + const agentOpen = useAgentPanelStore((s) => s.open) + const toggleAgent = useAgentPanelStore((s) => s.toggle) const [isMaximized, setIsMaximized] = useState(false) @@ -45,6 +48,27 @@ export default function TopBar(): JSX.Element { {/* Memory indicator */} {showRamIndicator && } + {/* Agent panel toggle */} + + {/* Patch update badge */} {patchUpdateReady && (
diff --git a/src/shared/stores/agentConversationStore.ts b/src/shared/stores/agentConversationStore.ts new file mode 100644 index 000000000..d52261c1c --- /dev/null +++ b/src/shared/stores/agentConversationStore.ts @@ -0,0 +1,170 @@ +import { create } from 'zustand' +import type { AgentAction, ToolStop } from '@shared/utils/agentStream' +import { + deriveTitle, + settleMessages, + upsertSummary, + type Conversation, + type ConversationSummary, +} from '@shared/utils/agentConversations' + +// ─── Message types (AgentChat) ──────────────────────────────────────────────── + +export interface ActionDone extends AgentAction { + id?: string + running?: boolean +} + +export interface AgentMessage { + id: string + role: 'user' | 'assistant' + content: string + thinking?: string + /** Attached images, inline as data URLs (persisted as-is). */ + imageDataUrls?: string[] + actions?: ActionDone[] + stopped?: ToolStop | null + streaming?: boolean +} + +// ─── Store ──────────────────────────────────────────────────────────────────── + +const SAVE_DEBOUNCE_MS = 500 + +interface AgentConversationState { + /** Id of the conversation on screen; not on disk until it has a message. */ + currentId: string + createdAt: number + messages: AgentMessage[] + /** Saved conversations, newest first. */ + list: ConversationSummary[] + /** A turn is streaming or a workflow it started is running: switching would misroute its output. */ + busy: boolean + + setMessages: (update: AgentMessage[] | ((prev: AgentMessage[]) => AgentMessage[])) => void + setBusy: (busy: boolean) => void + newConversation: () => void + open: (id: string) => Promise + remove: (id: string) => Promise + refreshList: () => Promise + /** Loads the list and opens the most recent conversation. Runs once. */ + init: () => Promise + /** Writes a pending debounced save now. */ + flush: () => Promise +} + +function newId(): string { + return crypto.randomUUID() +} + +let saveTimer: ReturnType | null = null +// Unsaved changes: opening or switching away must not bump an untouched conversation. +let dirty = false +let initPromise: Promise | null = null + +export const useAgentConversationStore = create()((set, get) => { + async function save(): Promise { + if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } + const { currentId, createdAt, messages } = get() + // An empty conversation never reaches the disk. + if (!dirty || messages.length === 0) return + dirty = false + const conversation: Conversation = { + id: currentId, + title: deriveTitle(messages), + createdAt, + updatedAt: Date.now(), + messages, + } + try { + await window.electron.agent.conversations.save(conversation) + set((s) => ({ list: upsertSummary(s.list, { id: conversation.id, title: conversation.title, updatedAt: conversation.updatedAt }) })) + } catch (err) { + console.error('[agent] Failed to save conversation:', err) + } + } + + function scheduleSave(): void { + if (saveTimer) clearTimeout(saveTimer) + saveTimer = setTimeout(() => { void save() }, SAVE_DEBOUNCE_MS) + } + + return { + currentId: newId(), + createdAt: Date.now(), + messages: [], + list: [], + busy: false, + + setMessages: (update) => { + set((s) => ({ messages: typeof update === 'function' ? update(s.messages) : update })) + dirty = true + scheduleSave() + }, + + setBusy: (busy) => set({ busy }), + + newConversation: () => { + if (get().busy) return + void save() + set({ currentId: newId(), createdAt: Date.now(), messages: [] }) + dirty = false + }, + + open: async (id) => { + if (get().busy || id === get().currentId) return + await save() + const conversation = await window.electron.agent.conversations.read(id) + if (!conversation) { + // Gone or corrupted on disk: drop it from the list. + set((s) => ({ list: s.list.filter((c) => c.id !== id) })) + return + } + set({ + currentId: conversation.id, + createdAt: conversation.createdAt, + messages: settleMessages(conversation.messages) as AgentMessage[], + }) + dirty = false + }, + + remove: async (id) => { + const isCurrent = id === get().currentId + if (isCurrent && get().busy) return + if (isCurrent) { + if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } + dirty = false + } + try { + await window.electron.agent.conversations.remove(id) + } catch (err) { + console.error('[agent] Failed to delete conversation:', err) + return + } + set((s) => ({ + list: s.list.filter((c) => c.id !== id), + ...(isCurrent ? { currentId: newId(), createdAt: Date.now(), messages: [] } : {}), + })) + }, + + refreshList: async () => { + try { + set({ list: await window.electron.agent.conversations.list() }) + } catch (err) { + console.error('[agent] Failed to list conversations:', err) + } + }, + + init: () => { + initPromise ??= (async () => { + await get().refreshList() + const latest = get().list[0] + // The user may already have started typing a new conversation. + if (latest && get().messages.length === 0 && !get().busy) await get().open(latest.id) + })() + return initPromise + }, + + flush: save, + } +}) diff --git a/src/shared/stores/agentPanelStore.ts b/src/shared/stores/agentPanelStore.ts new file mode 100644 index 000000000..f82997087 --- /dev/null +++ b/src/shared/stores/agentPanelStore.ts @@ -0,0 +1,43 @@ +import { create } from 'zustand' +import { persist } from 'zustand/middleware' + +export const AGENT_PANEL_MIN_WIDTH = 320 +export const AGENT_PANEL_MAX_WIDTH = 640 +export const AGENT_PANEL_DEFAULT_WIDTH = 380 + +export function clampPanelWidth(width: number): number { + if (!Number.isFinite(width)) return AGENT_PANEL_DEFAULT_WIDTH + return Math.round(Math.min(AGENT_PANEL_MAX_WIDTH, Math.max(AGENT_PANEL_MIN_WIDTH, width))) +} + +interface AgentPanelState { + open: boolean + width: number + setOpen: (open: boolean) => void + toggle: () => void + setWidth: (width: number) => void +} + +export const useAgentPanelStore = create()( + persist( + (set) => ({ + open: false, + width: AGENT_PANEL_DEFAULT_WIDTH, + setOpen: (open) => set({ open }), + toggle: () => set((s) => ({ open: !s.open })), + setWidth: (width) => set({ width: clampPanelWidth(width) }), + }), + { + name: 'modly-agent-panel', + partialize: (s) => ({ open: s.open, width: s.width }), + merge: (persisted, current) => { + const p = (persisted ?? {}) as Partial + return { + ...current, + open: typeof p.open === 'boolean' ? p.open : current.open, + width: typeof p.width === 'number' ? clampPanelWidth(p.width) : current.width, + } + }, + }, + ), +) From df1d2c838125f78510f844c61e0bf2d1787539d1 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:04:58 +0100 Subject: [PATCH 042/108] feat(agent): parse client_tool events and the generate_from_image payload --- src/shared/utils/agentStream.test.ts | 29 ++++++++++++++++++++++++++++ src/shared/utils/agentStream.ts | 19 ++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/shared/utils/agentStream.test.ts b/src/shared/utils/agentStream.test.ts index b2fa95159..0d6876679 100644 --- a/src/shared/utils/agentStream.test.ts +++ b/src/shared/utils/agentStream.test.ts @@ -38,6 +38,35 @@ test('malformed and non-data lines are skipped', () => { ]) }) +test('a client_tool event passes through with its call id, arguments and confirm flag', () => { + const parser = createSseParser() + const event = { + type: 'client_tool', + id: 'call_1', + call_id: 'c-42', + tool: 'uninstall_extension', + arguments: { extension_id: 'triposr' }, + confirm: true, + } + const done = { type: 'tool_done', id: 'call_1', tool: 'uninstall_extension', result: 'ok', payload: null } + + const events = parser.push(line(event) + line(done)) + + assert.deepEqual(events, [event, done]) + const [parsed] = events + assert.ok(parsed.type === 'client_tool' && parsed.call_id === 'c-42' && parsed.confirm) +}) + +test('a generate_from_image payload keeps its model id', () => { + const parser = createSseParser() + const payload = { type: 'generate_from_image', model_id: 'sf3d/main' } + + const [event] = parser.push(line({ type: 'tool_done', id: 't', tool: 'generate_from_image', result: 'queued', payload })) + + assert.ok(event.type === 'tool_done') + assert.deepEqual(event.payload, payload) +}) + test('a response body is read into events, multi-byte characters intact', async () => { const bytes = new TextEncoder().encode(line({ type: 'token', text: 'é→' }) + line({ type: 'done', message: 'é→', actions: [], thinking: null, error: false })) // Split inside the multi-byte "é" so the decoder has to carry it over. diff --git a/src/shared/utils/agentStream.ts b/src/shared/utils/agentStream.ts index 14d155430..82fbf38ee 100644 --- a/src/shared/utils/agentStream.ts +++ b/src/shared/utils/agentStream.ts @@ -9,6 +9,10 @@ export interface AgentActionPayload { workflow_id?: string workflow_name?: string workflow?: Omit + /** `generate_from_image`: model to generate with; the Generate page's selection when absent. */ + model_id?: string + /** `generate_from_image`: model parameters overriding the defaults. */ + params?: Record } export interface AgentAction { @@ -17,11 +21,26 @@ export interface AgentAction { payload?: AgentActionPayload | null } +/** + * A tool the frontend must run (it needs IPC or renderer stores). The server + * waits for `POST /agent/tool-result {call_id, result, error?}`, then sends the + * usual `tool_done` for the same `id`. `confirm` asks the user first. + */ +export interface ClientToolEvent { + type: 'client_tool' + id: string + call_id: string + tool: string + arguments: Record + confirm: boolean +} + export type AgentEvent = | { type: 'token'; text: string } | { type: 'thinking'; text: string } | { type: 'tool_start'; id: string; tool: string; arguments: Record } | { type: 'tool_done'; id: string; tool: string; result: string; payload: AgentActionPayload | null } + | ClientToolEvent | { type: 'done'; message: string; actions: AgentAction[]; thinking: string | null; error: boolean } function parseBlock(block: string): AgentEvent | null { From 28119bb541d214eae958739af5646012701c1c65 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:04:59 +0100 Subject: [PATCH 043/108] refactor(generate): extract the image generation path out of useGeneration runGeneration and createGenerationApi hold the from-image submit and status polling without React, so the agent can start a generation outside a hook. useGeneration and useApi now delegate to them with unchanged behaviour. --- src/shared/hooks/useApi.ts | 45 +------ src/shared/hooks/useGeneration.ts | 88 ++------------ src/shared/utils/generationRunner.ts | 171 +++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 121 deletions(-) create mode 100644 src/shared/utils/generationRunner.ts diff --git a/src/shared/hooks/useApi.ts b/src/shared/hooks/useApi.ts index f013f1c11..93ef6f5bd 100644 --- a/src/shared/hooks/useApi.ts +++ b/src/shared/hooks/useApi.ts @@ -1,48 +1,13 @@ import axios from 'axios' -import { useAppStore, GenerationOptions } from '@shared/stores/appStore' +import { useAppStore } from '@shared/stores/appStore' +import { createGenerationApi } from '../utils/generationRunner' export function useApi() { const apiUrl = useAppStore((s) => s.apiUrl) const client = axios.create({ baseURL: apiUrl }) - async function generateFromImage( - imagePath: string, - options: GenerationOptions, - imageData?: string, - signal?: AbortSignal, - ): Promise<{ jobId: string }> { - // Use provided base64 (drag & drop) or read from disk via IPC - const base64 = imageData ?? await window.electron.fs.readFileBase64(imagePath) - const byteArray = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)) - const blob = new Blob([byteArray], { type: 'image/png' }) - const filename = imagePath.split(/[\\/]/).pop() ?? 'image.png' - - const formData = new FormData() - formData.append('image', blob, filename) - formData.append('model_id', options.modelId) - formData.append('remesh', options.remesh) - formData.append('enable_texture', String(options.enableTexture)) - formData.append('texture_resolution', String(options.textureResolution)) - formData.append('params', JSON.stringify(options.modelParams)) - const { data } = await client.post<{ job_id: string }>('/generate/from-image', formData, { - headers: { 'Content-Type': 'multipart/form-data' }, - signal, - }) - - return { jobId: data.job_id } - } - - async function pollJobStatus(jobId: string): Promise<{ - status: 'pending' | 'running' | 'done' | 'error' | 'cancelled' - progress: number - step?: string - outputUrl?: string - error?: string - }> { - const { data } = await client.get(`/generate/status/${jobId}`) - return { ...data, outputUrl: data.output_url } - } + const { generateFromImage, pollJobStatus, cancelJob } = createGenerationApi(apiUrl) async function getModelStatus(): Promise<{ downloaded: boolean @@ -96,10 +61,6 @@ export function useApi() { return { url: data.url, faceCount: data.face_count } } - async function cancelJob(jobId: string): Promise { - await client.post(`/generate/cancel/${jobId}`).catch(() => {}) - } - async function smoothMesh( path: string, iterations: number, diff --git a/src/shared/hooks/useGeneration.ts b/src/shared/hooks/useGeneration.ts index 6f3d3a375..a0900d989 100644 --- a/src/shared/hooks/useGeneration.ts +++ b/src/shared/hooks/useGeneration.ts @@ -2,6 +2,7 @@ import { useCallback, useRef } from 'react' import { useAppStore } from '@shared/stores/appStore' import { useApi } from './useApi' import { showCompletionNotification, showErrorNotification } from '@shared/utils/notification' +import { runGeneration } from '@shared/utils/generationRunner' export function useGeneration() { const { currentJob, setCurrentJob, updateCurrentJob, generationOptions, selectedImageData, pushMeshUrl, clearMeshHistory } = useAppStore() @@ -13,90 +14,19 @@ export function useGeneration() { async (imagePath: string) => { cancelledRef.current = false abortControllerRef.current = new AbortController() - clearMeshHistory() - const job = { - id: crypto.randomUUID(), - imageFile: imagePath, - status: 'uploading' as const, - progress: 0, - createdAt: Date.now(), - modelId: generationOptions.modelId, - generationOptions, - } - setCurrentJob(job) - - try { - const { jobId } = await generateFromImage(imagePath, generationOptions, selectedImageData ?? undefined, abortControllerRef.current.signal) - - if (cancelledRef.current) { - await cancelJob(jobId) - setCurrentJob(null) - return - } - - updateCurrentJob({ status: 'generating', progress: 0 }) - - await pollUntilDone(jobId) - } catch (err) { - if (cancelledRef.current) { - setCurrentJob(null) - return - } - let errorMessage: string - if (err && typeof err === 'object' && 'response' in err) { - const axiosErr = err as { response?: { data?: { detail?: string } }; message: string } - errorMessage = axiosErr.response?.data?.detail ?? axiosErr.message - } else { - errorMessage = err instanceof Error ? err.message : String(err) - } - updateCurrentJob({ - status: 'error', - error: errorMessage - }) - void showErrorNotification(errorMessage, 'Generation failed') - } + await runGeneration(imagePath, generationOptions, selectedImageData ?? undefined, { + api: { generateFromImage, pollJobStatus, cancelJob }, + store: { setCurrentJob, updateCurrentJob, pushMeshUrl, clearMeshHistory }, + notifyComplete: (body) => showCompletionNotification(body), + notifyError: (body, title) => showErrorNotification(body, title), + isCancelled: () => cancelledRef.current, + signal: abortControllerRef.current.signal, + }) }, // eslint-disable-next-line react-hooks/exhaustive-deps -- useApi re-creates its fns each render, so this re-memoizes anyway (values stay fresh) [generateFromImage, pollJobStatus, cancelJob, setCurrentJob, updateCurrentJob] ) - const pollUntilDone = async (jobId: string) => { - while (true) { - await new Promise((r) => setTimeout(r, 1000)) - - if (cancelledRef.current) { - await cancelJob(jobId) - setCurrentJob(null) - break - } - - const result = await pollJobStatus(jobId) - - if (result.status === 'cancelled') { - setCurrentJob(null) - break - } - - if (result.status === 'done') { - updateCurrentJob({ status: 'done', progress: 100, outputUrl: result.outputUrl, originalOutputUrl: result.outputUrl }) - if (result.outputUrl) pushMeshUrl(result.outputUrl) - void showCompletionNotification('Generation complete') - break - } - - if (result.status === 'error') { - updateCurrentJob({ status: 'error', error: result.error }) - void showErrorNotification(result.error ?? 'Unknown error', 'Generation failed') - break - } - - updateCurrentJob({ - progress: result.progress, - step: result.step, - }) - } - } - const cancelGeneration = useCallback(() => { cancelledRef.current = true abortControllerRef.current?.abort() diff --git a/src/shared/utils/generationRunner.ts b/src/shared/utils/generationRunner.ts new file mode 100644 index 000000000..8b25e2091 --- /dev/null +++ b/src/shared/utils/generationRunner.ts @@ -0,0 +1,171 @@ +import axios from 'axios' +import type { GenerationJob, GenerationOptions } from '../stores/appStore' + +// The single-image generation path (POST /generate/from-image, then poll +// /generate/status), free of React so the agent can start one outside a hook. +// useGeneration and useApi both go through it. + +export interface JobStatus { + status: 'pending' | 'running' | 'done' | 'error' | 'cancelled' + progress: number + step?: string + outputUrl?: string + error?: string +} + +export interface GenerationApi { + generateFromImage: (imagePath: string, options: GenerationOptions, imageData?: string, signal?: AbortSignal) => Promise<{ jobId: string }> + pollJobStatus: (jobId: string) => Promise + cancelJob: (jobId: string) => Promise +} + +export function createGenerationApi(apiUrl: string): GenerationApi { + const client = axios.create({ baseURL: apiUrl }) + + async function generateFromImage( + imagePath: string, + options: GenerationOptions, + imageData?: string, + signal?: AbortSignal, + ): Promise<{ jobId: string }> { + // Use provided base64 (drag & drop) or read from disk via IPC + const base64 = imageData ?? await window.electron.fs.readFileBase64(imagePath) + const byteArray = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)) + const blob = new Blob([byteArray], { type: 'image/png' }) + const filename = imagePath.split(/[\\/]/).pop() ?? 'image.png' + + const formData = new FormData() + formData.append('image', blob, filename) + formData.append('model_id', options.modelId) + formData.append('remesh', options.remesh) + formData.append('enable_texture', String(options.enableTexture)) + formData.append('texture_resolution', String(options.textureResolution)) + formData.append('params', JSON.stringify(options.modelParams)) + const { data } = await client.post<{ job_id: string }>('/generate/from-image', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + signal, + }) + + return { jobId: data.job_id } + } + + async function pollJobStatus(jobId: string): Promise { + const { data } = await client.get(`/generate/status/${jobId}`) + return { ...data, outputUrl: data.output_url } + } + + async function cancelJob(jobId: string): Promise { + await client.post(`/generate/cancel/${jobId}`).catch(() => {}) + } + + return { generateFromImage, pollJobStatus, cancelJob } +} + +export interface GenerationStoreActions { + setCurrentJob: (job: GenerationJob | null) => void + updateCurrentJob: (patch: Partial) => void + pushMeshUrl: (url: string) => void + clearMeshHistory: () => void +} + +export interface GenerationRunDeps { + api: GenerationApi + store: GenerationStoreActions + notifyComplete: (body: string) => void | Promise + notifyError: (body: string, title: string) => void | Promise + isCancelled: () => boolean + signal?: AbortSignal + /** Delay between two status polls; tests pass a fast one. */ + wait?: () => Promise +} + +export type GenerationOutcome = + | { status: 'done'; outputUrl?: string } + | { status: 'error'; error: string } + | { status: 'cancelled' } + +/** Runs one generation into `currentJob`, so the Generate viewer shows it. */ +export async function runGeneration( + imagePath: string, + options: GenerationOptions, + imageData: string | undefined, + deps: GenerationRunDeps, +): Promise { + const { api, store } = deps + const wait = deps.wait ?? (() => new Promise((r) => setTimeout(r, 1000))) + store.clearMeshHistory() + store.setCurrentJob({ + id: crypto.randomUUID(), + imageFile: imagePath, + status: 'uploading', + progress: 0, + createdAt: Date.now(), + modelId: options.modelId, + generationOptions: options, + }) + + try { + const { jobId } = await api.generateFromImage(imagePath, options, imageData, deps.signal) + + if (deps.isCancelled()) { + await api.cancelJob(jobId) + store.setCurrentJob(null) + return { status: 'cancelled' } + } + + store.updateCurrentJob({ status: 'generating', progress: 0 }) + + for (;;) { + await wait() + + if (deps.isCancelled()) { + await api.cancelJob(jobId) + store.setCurrentJob(null) + return { status: 'cancelled' } + } + + const result = await api.pollJobStatus(jobId) + + if (result.status === 'cancelled') { + store.setCurrentJob(null) + return { status: 'cancelled' } + } + + if (result.status === 'done') { + store.updateCurrentJob({ status: 'done', progress: 100, outputUrl: result.outputUrl, originalOutputUrl: result.outputUrl }) + if (result.outputUrl) store.pushMeshUrl(result.outputUrl) + void deps.notifyComplete('Generation complete') + return { status: 'done', outputUrl: result.outputUrl } + } + + if (result.status === 'error') { + store.updateCurrentJob({ status: 'error', error: result.error }) + void deps.notifyError(result.error ?? 'Unknown error', 'Generation failed') + return { status: 'error', error: result.error ?? 'Unknown error' } + } + + store.updateCurrentJob({ + progress: result.progress, + step: result.step, + }) + } + } catch (err) { + if (deps.isCancelled()) { + store.setCurrentJob(null) + return { status: 'cancelled' } + } + let errorMessage: string + if (err && typeof err === 'object' && 'response' in err) { + const axiosErr = err as { response?: { data?: { detail?: string } }; message: string } + errorMessage = axiosErr.response?.data?.detail ?? axiosErr.message + } else { + errorMessage = err instanceof Error ? err.message : String(err) + } + store.updateCurrentJob({ + status: 'error', + error: errorMessage, + }) + void deps.notifyError(errorMessage, 'Generation failed') + return { status: 'error', error: errorMessage } + } +} From 918a3a0951318affe35f76e6b54906fc6782db9f Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:05:00 +0100 Subject: [PATCH 044/108] feat(agent): add client tools, confirm card and agent context clientTools.ts runs the client_tool calls of the agent (extensions, weights, workflows, settings, storage folders, mesh undo/redo, navigation) through injected dependencies, asks for confirmation when required and posts the result to /agent/tool-result. ConfirmCard.tsx renders the Confirm/Cancel card and its promise queue. agentContext.ts builds the request context from the stores. --- package.json | 2 +- .../components/agent/ConfirmCard.test.mjs | 66 ++ src/shared/components/agent/ConfirmCard.tsx | 173 ++++ .../components/agent/agentContext.test.ts | 81 ++ src/shared/components/agent/agentContext.ts | 84 ++ .../components/agent/clientTools.test.ts | 434 ++++++++++ src/shared/components/agent/clientTools.ts | 761 ++++++++++++++++++ 7 files changed, 1600 insertions(+), 1 deletion(-) create mode 100644 src/shared/components/agent/ConfirmCard.test.mjs create mode 100644 src/shared/components/agent/ConfirmCard.tsx create mode 100644 src/shared/components/agent/agentContext.test.ts create mode 100644 src/shared/components/agent/agentContext.ts create mode 100644 src/shared/components/agent/clientTools.test.ts create mode 100644 src/shared/components/agent/clientTools.ts diff --git a/package.json b/package.json index d74c54d74..5a4b69dfe 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/shared/components/agent/ConfirmCard.test.mjs b/src/shared/components/agent/ConfirmCard.test.mjs new file mode 100644 index 000000000..da117e50a --- /dev/null +++ b/src/shared/components/agent/ConfirmCard.test.mjs @@ -0,0 +1,66 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildSync } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +// Bundle ConfirmCard.tsx with its real react/zustand dependencies; its only +// other import is type-only. Rendering is left to the app; this covers the +// confirm queue that turns confirm(description) into a Promise. +function load() { + const outfile = join(mkdtempSync(join(tmpdir(), 'modly-confirm-test-')), 'ConfirmCard.cjs') + const result = buildSync({ + entryPoints: [resolve('src/shared/components/agent/ConfirmCard.tsx')], + bundle: true, + platform: 'node', + format: 'cjs', + jsx: 'automatic', + write: false, + }) + writeFileSync(outfile, result.outputFiles[0].text, 'utf8') + return createRequire(import.meta.url)(outfile) +} + +const { useConfirmQueue, ConfirmCard } = load() +const queue = () => useConfirmQueue.getState() + +test('confirm shows a pending request and resolves true on Confirm', async () => { + const answer = queue().confirm('Uninstall the extension “TripoSR”', { tool: 'uninstall_extension', id: 'e1', callId: 'c1', destructive: true }) + const [request] = queue().requests + assert.equal(request.status, 'pending') + assert.equal(request.callId, 'c1') + assert.equal(request.destructive, true) + + queue().answer(request.key, true) + + assert.equal(await answer, true) + assert.equal(queue().requests[0].status, 'confirmed') + queue().clearAnswered() + assert.deepEqual(queue().requests, []) +}) + +test('Cancel resolves false, and a second answer is ignored', async () => { + const answer = queue().confirm('Delete the workflow “Turntable”') + const { key } = queue().requests[0] + queue().answer(key, false) + queue().answer(key, true) + + assert.equal(await answer, false) + assert.equal(queue().requests[0].status, 'cancelled') + queue().clearAnswered() +}) + +test('cancelAll declines every pending request', async () => { + const a = queue().confirm('one') + const b = queue().confirm('two') + queue().cancelAll() + assert.deepEqual(await Promise.all([a, b]), [false, false]) + assert.deepEqual(queue().requests.map((r) => r.status), ['cancelled', 'cancelled']) + queue().clearAnswered() +}) + +test('ConfirmCard is a component', () => { + assert.equal(typeof ConfirmCard, 'function') +}) diff --git a/src/shared/components/agent/ConfirmCard.tsx b/src/shared/components/agent/ConfirmCard.tsx new file mode 100644 index 000000000..55aeb6a7d --- /dev/null +++ b/src/shared/components/agent/ConfirmCard.tsx @@ -0,0 +1,173 @@ +import { create } from 'zustand' +import type { ConfirmInfo } from './clientTools' + +// ─── Confirm queue ──────────────────────────────────────────────────────────── + +export type ConfirmStatus = 'pending' | 'confirmed' | 'cancelled' + +export interface ConfirmRequest { + /** Unique per request. */ + key: string + description: string + destructive: boolean + status: ConfirmStatus + /** The client_tool event this request belongs to, when it came from one. */ + tool?: string + eventId?: string + callId?: string +} + +interface ConfirmQueueState { + requests: ConfirmRequest[] + /** Shows a card and resolves with the user's answer. Matches runClientTool's `confirm`. */ + confirm: (description: string, info?: ConfirmInfo) => Promise + answer: (key: string, confirmed: boolean) => void + /** Cancels every pending request (stream aborted, panel closed, new conversation). */ + cancelAll: () => void + /** Forgets answered requests. */ + clearAnswered: () => void +} + +// Resolvers stay out of the state: they are not data to render. +const resolvers = new Map void>() +let nextKey = 0 + +export const useConfirmQueue = create((set, get) => ({ + requests: [], + + confirm(description, info) { + const key = `confirm-${++nextKey}` + const request: ConfirmRequest = { + key, + description, + destructive: info?.destructive ?? true, + status: 'pending', + tool: info?.tool, + eventId: info?.id, + callId: info?.callId, + } + return new Promise((resolve) => { + resolvers.set(key, resolve) + set((s) => ({ requests: [...s.requests, request] })) + }) + }, + + answer(key, confirmed) { + const resolve = resolvers.get(key) + if (!resolve) return + resolvers.delete(key) + set((s) => ({ + requests: s.requests.map((r) => (r.key === key ? { ...r, status: confirmed ? 'confirmed' : 'cancelled' } : r)), + })) + resolve(confirmed) + }, + + cancelAll() { + for (const r of get().requests) if (r.status === 'pending') get().answer(r.key, false) + }, + + clearAnswered() { + set((s) => ({ requests: s.requests.filter((r) => r.status === 'pending') })) + }, +})) + +// ─── Card ───────────────────────────────────────────────────────────────────── + +export interface ConfirmCardProps { + description: string + destructive?: boolean + status: ConfirmStatus + onConfirm: () => void + onCancel: () => void +} + +/** The Confirm / Cancel card shown in the chat before a sensitive action. */ +export function ConfirmCard({ description, destructive = true, status, onConfirm, onCancel }: ConfirmCardProps): JSX.Element { + const pending = status === 'pending' + + function handleKeyDown(e: React.KeyboardEvent) { + if (!pending) return + if (e.key === 'Escape') { + e.preventDefault() + onCancel() + } else if (e.key === 'Enter' && !(e.target as HTMLElement).dataset.confirmCancel) { + // Enter on the Cancel button keeps its native meaning. + e.preventDefault() + onConfirm() + } + } + + return ( +
+
+ + + + +

{description}

+
+ +
+ {pending ? ( + <> + + + + ) : ( + + {status === 'confirmed' ? ( + + + + ) : ( + + + + )} + {status === 'confirmed' ? 'Confirmed' : 'Cancelled'} + + )} +
+
+ ) +} + +/** A ConfirmCard bound to one request of the queue. */ +export function QueuedConfirmCard({ request }: { request: ConfirmRequest }): JSX.Element { + const answer = useConfirmQueue((s) => s.answer) + return ( + answer(request.key, true)} + onCancel={() => answer(request.key, false)} + /> + ) +} diff --git a/src/shared/components/agent/agentContext.test.ts b/src/shared/components/agent/agentContext.test.ts new file mode 100644 index 000000000..93ff023e2 --- /dev/null +++ b/src/shared/components/agent/agentContext.test.ts @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { assembleAgentContext, buildAgentContext } from './agentContext' +import type { AgentContextInputs } from './agentContext' +import type { ModelExtension, ProcessExtension } from '../../types/electron.d' +import { useAppStore } from '../../stores/appStore' +import { useNavStore } from '../../stores/navStore' +import { useWorkflowsStore } from '../../stores/workflowsStore' +import { useExtensionsStore } from '../../stores/extensionsStore' + +const node = (id: string, input: string, output: string) => ({ id, name: id.toUpperCase(), input, output, paramsSchema: [] }) + +const modelExt = { + type: 'model', id: 'triposr', name: 'TripoSR', trusted: true, builtin: false, nodes: [node('main', 'image', 'mesh')], +} as unknown as ModelExtension +const processExt = { + type: 'process', id: 'mesh-ops', name: 'Mesh Ops', trusted: true, builtin: true, entry: 'x', nodes: [node('decimate', 'mesh', 'mesh')], +} as unknown as ProcessExtension +const STORAGE = { modelsDir: 'M', workspaceDir: 'W', workflowsDir: 'F', extensionsDir: 'E' } + +const inputs = (over: Partial = {}): AgentContextInputs => ({ + page: 'models', + currentMeshUrl: '/workspace/Default/mesh.glb', + meshTriangles: 1200, + workflows: [{ id: 'wf1', name: 'Turntable' }], + modelExtensions: [modelExt], + processExtensions: [processExt], + models: [{ id: 'triposr/main', name: 'TripoSR' }], + storage: STORAGE, + ui: { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }, + ...over, +}) + +test('the context carries the page, mesh, workflows, extensions, models, storage and UI options', () => { + assert.deepEqual(assembleAgentContext(inputs()), { + page: 'models', + currentMeshPath: 'Default/mesh.glb', + meshTriangles: 1200, + workflows: [{ id: 'wf1', name: 'Turntable' }], + extensions: [ + { id: 'mesh-ops/decimate', name: 'DECIMATE', input: 'mesh', output: 'mesh' }, + { id: 'triposr/main', name: 'MAIN', input: 'image', output: 'mesh' }, + ], + models: [{ id: 'triposr/main', name: 'TripoSR' }], + storage: STORAGE, + ui: { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }, + }) +}) + +test('empty or unreadable parts are left out, as buildContext did', () => { + const ctx = assembleAgentContext(inputs({ + currentMeshUrl: undefined, meshTriangles: null, workflows: [], modelExtensions: [], processExtensions: [], models: null, storage: null, + })) + assert.deepEqual(Object.keys(ctx), ['page', 'ui']) +}) + +test('storage keeps only the four folders, never the HF token', () => { + const ctx = assembleAgentContext(inputs({ storage: { ...STORAGE, hfToken: 'secret' } as typeof STORAGE })) + assert.deepEqual(ctx.storage, STORAGE) +}) + +test('buildAgentContext reads the stores and the async sources', async () => { + useNavStore.setState({ currentPage: 'workflows' }) + useAppStore.setState({ currentJob: null, meshStats: null, uiScale: 'small', useAtkinsonFont: false, showRamIndicator: true }) + useWorkflowsStore.setState({ workflows: [{ id: 'w', name: 'W', description: '', nodes: [], edges: [], createdAt: '', updatedAt: '' }] }) + useExtensionsStore.setState({ modelExtensions: [modelExt], processExtensions: [] }) + + const ctx = await buildAgentContext({ + fetchModels: async () => [{ id: 'triposr/main', name: 'TripoSR' }], + getStorage: async () => { throw new Error('IPC down') }, + }) + + assert.deepEqual(ctx, { + page: 'workflows', + workflows: [{ id: 'w', name: 'W' }], + extensions: [{ id: 'triposr/main', name: 'MAIN', input: 'image', output: 'mesh' }], + models: [{ id: 'triposr/main', name: 'TripoSR' }], + ui: { uiScale: 'small', useAtkinsonFont: false, showRamIndicator: true }, + }) +}) diff --git a/src/shared/components/agent/agentContext.ts b/src/shared/components/agent/agentContext.ts new file mode 100644 index 000000000..a13a06de8 --- /dev/null +++ b/src/shared/components/agent/agentContext.ts @@ -0,0 +1,84 @@ +import type { ModelExtension, ProcessExtension } from '../../types/electron.d' +import type { Workflow } from '../../types/electron.d' +import type { UiScale } from '../../stores/appStore' +import type { Page } from '../../stores/navStore' +import { useAppStore } from '../../stores/appStore' +import { useExtensionsStore } from '../../stores/extensionsStore' +import { useWorkflowsStore } from '../../stores/workflowsStore' +import { useNavStore } from '../../stores/navStore' +import { buildAllWorkflowExtensions } from '../../../areas/workflows/mockExtensions' + +// The `context` object sent with every agent request: what the app shows and +// holds right now, so the agent can act without asking. + +export interface AgentContextInputs { + page: Page + currentMeshUrl?: string | null + meshTriangles?: number | null + workflows: Pick[] + modelExtensions: ModelExtension[] + processExtensions: ProcessExtension[] + /** Downloaded generation models; null when they could not be read. */ + models: { id: string; name: string }[] | null + /** Null when the settings could not be read. */ + storage: { modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string } | null + ui: { uiScale: UiScale; useAtkinsonFont: boolean; showRamIndicator: boolean } +} + +/** Pure assembly of the context from already-read state. */ +export function assembleAgentContext(inputs: AgentContextInputs): Record { + const ctx: Record = { page: inputs.page } + if (inputs.currentMeshUrl) ctx.currentMeshPath = inputs.currentMeshUrl.replace('/workspace/', '') + if (inputs.meshTriangles) ctx.meshTriangles = inputs.meshTriangles + if (inputs.workflows.length > 0) ctx.workflows = inputs.workflows.map((w) => ({ id: w.id, name: w.name })) + const extensions = buildAllWorkflowExtensions(inputs.modelExtensions, inputs.processExtensions) + if (extensions.length > 0) ctx.extensions = extensions.map((e) => ({ + id: e.id, name: e.name, input: e.input, output: e.output, + })) + if (inputs.models) ctx.models = inputs.models.map((m) => ({ id: m.id, name: m.name })) + if (inputs.storage) { + const { modelsDir, workspaceDir, workflowsDir, extensionsDir } = inputs.storage + ctx.storage = { modelsDir, workspaceDir, workflowsDir, extensionsDir } + } + ctx.ui = { uiScale: inputs.ui.uiScale, useAtkinsonFont: inputs.ui.useAtkinsonFont, showRamIndicator: inputs.ui.showRamIndicator } + return ctx +} + +export interface AgentContextSources { + /** GET /model/all, filtered to the downloaded ones, as the Generate options list does. */ + fetchModels: () => Promise<{ id: string; name: string }[]> + getStorage: () => Promise<{ modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string }> +} + +function defaultSources(): AgentContextSources { + return { + async fetchModels() { + const res = await fetch(`${useAppStore.getState().apiUrl}/model/all`) + if (!res.ok) throw new Error(`API error ${res.status}`) + const list = await res.json() as { id: string; name?: string; downloaded: boolean }[] + return list.filter((m) => m.downloaded).map((m) => ({ id: m.id, name: m.name ?? m.id })) + }, + getStorage: () => window.electron.settings.get(), + } +} + +/** Reads the stores, the downloaded models and the storage folders, then assembles the context. */ +export async function buildAgentContext(sources: AgentContextSources = defaultSources()): Promise> { + const app = useAppStore.getState() + const ext = useExtensionsStore.getState() + const [models, storage] = await Promise.all([ + sources.fetchModels().catch(() => null), + sources.getStorage().catch(() => null), + ]) + return assembleAgentContext({ + page: useNavStore.getState().currentPage, + currentMeshUrl: app.currentJob?.outputUrl, + meshTriangles: app.meshStats?.triangles, + workflows: useWorkflowsStore.getState().workflows, + modelExtensions: ext.modelExtensions, + processExtensions: ext.processExtensions, + models, + storage, + ui: { uiScale: app.uiScale, useAtkinsonFont: app.useAtkinsonFont, showRamIndicator: app.showRamIndicator }, + }) +} diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts new file mode 100644 index 000000000..cda280082 --- /dev/null +++ b/src/shared/components/agent/clientTools.test.ts @@ -0,0 +1,434 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + clientTools, + describeClientTool, + isErrorResult, + latestUserImage, + runClientTool, + startGenerationFromImage, +} from './clientTools' +import type { ClientToolDeps, GenerationDeps, StorageKind, ToolResultBody } from './clientTools' +import type { AnyExtension, Workflow } from '../../types/electron.d' +import type { GenerationJob, GenerationOptions, UiScale } from '../../stores/appStore' + +// ─── Fakes ──────────────────────────────────────────────────────────────────── + +const node = (id: string, extra: Record = {}) => ({ + id, name: id.toUpperCase(), input: 'image', output: 'mesh', paramsSchema: [], ...extra, +}) + +function extensions(): AnyExtension[] { + return [ + { + type: 'model', id: 'triposr', name: 'TripoSR', trusted: true, builtin: false, + nodes: [node('main', { hfRepo: 'stabilityai/TripoSR' })], + }, + { + type: 'model', id: 'hunyuan', name: 'Hunyuan3D', trusted: true, builtin: false, + nodes: [ + node('shape', { hfRepo: 'tencent/h', weightVariants: { param: 'v', default: 'full', options: [{ id: 'full', label: 'Full' }, { id: 'mini', label: 'Mini' }] } }), + node('paint', { hfRepo: 'tencent/p' }), + ], + }, + { + type: 'model', id: 'shared', name: 'Shared', trusted: true, builtin: false, + weightGroups: [{ id: 'g', dependentNodeIds: ['a'] }], + nodes: [node('a', { weightGroups: ['g'] })], + }, + { + type: 'process', id: 'mesh-ops', name: 'Mesh Ops', trusted: true, builtin: true, entry: 'x', + nodes: [node('decimate')], + }, + ] as AnyExtension[] +} + +const workflow = (id: string, name: string): Workflow => ({ + id, name, description: '', nodes: [], edges: [], createdAt: '', updatedAt: '', +}) + +interface Fake { + deps: ClientToolDeps + calls: unknown[][] + posted: ToolResultBody[] + ui: { uiScale: UiScale; useAtkinsonFont: boolean; showRamIndicator: boolean } + dirs: { modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string } + history: { index: number; length: number } + dirItems: Record +} + +function fake(overrides: Partial = {}): Fake { + const calls: unknown[][] = [] + const posted: ToolResultBody[] = [] + const rec = (...call: unknown[]) => { calls.push(call) } + const ok = { success: true } + const state = { + ui: { uiScale: 'medium' as UiScale, useAtkinsonFont: false, showRamIndicator: true }, + dirs: { modelsDir: 'C:\\Modly\\models', workspaceDir: 'C:\\Modly\\workspace', workflowsDir: 'C:\\Modly\\workflows', extensionsDir: 'C:\\Modly\\extensions' }, + history: { index: 1, length: 3 }, + dirItems: {} as Record, + } + const list = extensions() + const wfs = [workflow('wf1', 'Turntable'), workflow('wf2', 'Cleanup')] + const deps: ClientToolDeps = { + extensions: { + list: () => list, + installFromGitHub: async (url) => { + rec('install', url) + return { success: true, extensionId: 'sf3d', extension: { type: 'model', id: 'sf3d', name: 'SF3D', trusted: false, builtin: false, nodes: [node('main', { hfRepo: 'x/y' })] } as AnyExtension } + }, + uninstall: async (id) => { rec('uninstall', id); return ok }, + reload: async () => { rec('reload') }, + loadErrors: () => ({ broken: 'bad manifest' }), + }, + weights: { + download: async (id, variant) => { rec('download', id, variant); return ok }, + delete: async (id) => { rec('delete', id); return ok }, + deleteVariant: async (id, variant) => { rec('deleteVariant', id, variant); return ok }, + deleteExtensionWeights: async (id) => { rec('deleteExtensionWeights', id); return ok }, + refresh: async () => { rec('refresh') }, + }, + workflows: { + list: () => wfs, + remove: async (id) => { rec('removeWorkflow', id); return ok }, + open: (id) => { rec('openWorkflow', id) }, + reload: async () => { rec('reloadWorkflows') }, + }, + ui: { + get: () => state.ui, + setUiScale: (v) => { state.ui.uiScale = v }, + setUseAtkinsonFont: (v) => { state.ui.useAtkinsonFont = v }, + setShowRamIndicator: (v) => { state.ui.showRamIndicator = v }, + }, + storage: { + get: async () => state.dirs, + listDir: async (dir) => state.dirItems[dir] ?? [], + moveDirectory: async (args) => { rec('move', args.src, args.dest); return ok }, + apply: async (kind: StorageKind, path: string) => { rec('applyStorage', kind, path) }, + }, + mesh: { + history: () => state.history, + undo: () => { rec('undo') }, + redo: () => { rec('redo') }, + }, + navigate: (page) => { rec('navigate', page) }, + postResult: async (body) => { posted.push(body) }, + downloadSettleMs: 20, + ...overrides, + } + return { deps, calls, posted, ...state } +} + +const run = (tool: string, args: Record, f = fake()) => clientTools[tool].run(args, f.deps) + +// ─── Extensions ─────────────────────────────────────────────────────────────── + +test('install_extension installs from GitHub and names the extension', async () => { + const f = fake() + const result = await run('install_extension', { github_url: 'https://github.com/x/sf3d' }, f) + + assert.deepEqual(f.calls, [['install', 'https://github.com/x/sf3d']]) + assert.match(result, /Installed the extension “SF3D” \(id sf3d\)/) + assert.match(result, /not downloaded yet/) +}) + +test('install_extension refuses a missing or non-GitHub URL without installing', async () => { + const f = fake() + assert.match(await run('install_extension', {}, f), /^Error: github_url is required/) + assert.match(await run('install_extension', { github_url: 'https://evil.example/x' }, f), /^Error: .*github\.com/) + assert.match(await run('install_extension', { github_url: 'not a url' }, f), /^Error: .*not a valid URL/) + assert.deepEqual(f.calls, []) +}) + +test('install_extension reports the installer error', async () => { + const f = fake() + f.deps.extensions.installFromGitHub = async () => ({ success: false, error: 'No manifest.json' }) + assert.equal(await run('install_extension', { github_url: 'https://github.com/x/y' }, f), 'Error: No manifest.json') +}) + +test('uninstall_extension uninstalls by id, describes by name and refuses built-ins and unknown ids', async () => { + const f = fake() + assert.equal(describeClientTool('uninstall_extension', { extension_id: 'triposr' }, f.deps), 'Uninstall the extension “TripoSR”') + assert.match(await run('uninstall_extension', { extension_id: 'triposr' }, f), /^Uninstalled the extension “TripoSR”/) + assert.deepEqual(f.calls, [['uninstall', 'triposr']]) + + assert.match(await run('uninstall_extension', { extension_id: 'mesh-ops' }, f), /^Error: “Mesh Ops” is built into Modly/) + assert.match(await run('uninstall_extension', { extension_id: 'nope' }, f), /^Error: No installed extension has the id “nope”. Installed extensions: triposr, hunyuan, shared, mesh-ops\./) + assert.match(await run('uninstall_extension', {}, f), /^Error: extension_id is required/) + assert.equal(f.calls.length, 1) +}) + +test('reload_extensions reloads and reports the count and load errors', async () => { + const f = fake() + const result = await run('reload_extensions', {}, f) + assert.deepEqual(f.calls, [['reload']]) + assert.equal(result, 'Reloaded the extensions: 4 installed. 1 failed to load: broken (bad manifest).') +}) + +// ─── Weights ────────────────────────────────────────────────────────────────── + +test('download_model_weights downloads every model of the extension in turn', async () => { + const f = fake() + const result = await run('download_model_weights', { extension_id: 'hunyuan' }, f) + assert.deepEqual(f.calls, [['download', 'hunyuan/shape', undefined], ['download', 'hunyuan/paint', undefined], ['refresh']]) + assert.equal(result, 'Downloaded the model weights of “Hunyuan3D”.') +}) + +test('download_model_weights with a variant only targets the models offering it', async () => { + const f = fake() + const result = await run('download_model_weights', { extension_id: 'hunyuan', variant: 'mini' }, f) + assert.deepEqual(f.calls, [['download', 'hunyuan/shape', 'mini'], ['refresh']]) + assert.equal(result, 'Downloaded the model weights of “Hunyuan3D” (variant “mini”).') +}) + +test('download_model_weights validates the extension, model and variant', async () => { + const f = fake() + assert.match(await run('download_model_weights', { extension_id: 'hunyuan', variant: 'huge' }, f), /^Error: “Hunyuan3D” has no weight variant “huge”. Available variants: full, mini\./) + assert.match(await run('download_model_weights', { extension_id: 'triposr', variant: 'x' }, f), /^Error: “TripoSR” does not offer weight variants/) + assert.match(await run('download_model_weights', { extension_id: 'hunyuan/nope' }, f), /^Error: “Hunyuan3D” has no model “nope”/) + assert.match(await run('download_model_weights', { extension_id: 'mesh-ops' }, f), /^Error: “Mesh Ops” is a process extension/) + assert.match(await run('download_model_weights', { extension_id: 'ghost' }, f), /^Error: No installed extension/) + assert.match(await run('download_model_weights', { extension_id: 'triposr', variant: 3 }, f), /^Error: variant must be a string/) + assert.deepEqual(f.calls, []) +}) + +test('download_model_weights reports a long download as started, and an early failure as an error', async () => { + const slow = fake() + slow.deps.weights.download = () => new Promise(() => {}) + assert.match(await run('download_model_weights', { extension_id: 'triposr' }, slow), /^Started downloading the model weights of “TripoSR” \(1 model\)\. The download continues in the background/) + + const broken = fake() + broken.deps.weights.download = async () => ({ success: false, error: 'HF 401' }) + assert.equal(await run('download_model_weights', { extension_id: 'triposr/main' }, broken), 'Error: Downloading “TripoSR / MAIN” failed: triposr/main: HF 401.') +}) + +test('delete_model_weights deletes per model, per variant, or the shared extension weights', async () => { + const f = fake() + assert.equal(await run('delete_model_weights', { extension_id: 'hunyuan' }, f), 'Deleted the downloaded model weights of “Hunyuan3D”.') + assert.equal(await run('delete_model_weights', { extension_id: 'hunyuan', variant: 'full' }, f), 'Deleted the downloaded model weights of “Hunyuan3D” (variant “full”).') + await run('delete_model_weights', { extension_id: 'shared' }, f) + assert.deepEqual(f.calls, [ + ['delete', 'hunyuan/shape'], ['delete', 'hunyuan/paint'], ['refresh'], + ['deleteVariant', 'hunyuan/shape', 'full'], ['refresh'], + ['deleteExtensionWeights', 'shared'], ['refresh'], + ]) + assert.equal(describeClientTool('delete_model_weights', { extension_id: 'hunyuan/shape', variant: 'full' }, f.deps), 'Delete the downloaded model weights of “Hunyuan3D / SHAPE” (variant “full”)') +}) + +test('delete_model_weights reports a failed delete', async () => { + const f = fake() + f.deps.weights.delete = async () => ({ success: false, error: 'file locked' }) + assert.equal(await run('delete_model_weights', { extension_id: 'triposr' }, f), 'Error: Deleting the weights of “TripoSR” failed: file locked.') +}) + +// ─── Workflows ──────────────────────────────────────────────────────────────── + +test('delete_workflow deletes by id and resolves its name', async () => { + const f = fake() + assert.equal(describeClientTool('delete_workflow', { workflow_id: 'wf2' }, f.deps), 'Delete the workflow “Cleanup”') + assert.equal(await run('delete_workflow', { workflow_id: 'wf2' }, f), 'Deleted the workflow “Cleanup”.') + assert.deepEqual(f.calls, [['removeWorkflow', 'wf2']]) + assert.match(await run('delete_workflow', { workflow_id: 'wf9' }, f), /^Error: No workflow has the id “wf9”. Workflows: Turntable \(wf1\), Cleanup \(wf2\)\./) +}) + +test('open_workflow opens the tab without navigating', async () => { + const f = fake() + const result = await run('open_workflow', { workflow_id: 'wf1' }, f) + assert.deepEqual(f.calls, [['openWorkflow', 'wf1']]) + assert.match(result, /“Turntable”.*did not change/) + assert.match(await run('open_workflow', {}, f), /^Error: workflow_id is required/) +}) + +// ─── Settings ───────────────────────────────────────────────────────────────── + +test('set_setting sets each UI option and validates key and value', async () => { + const f = fake() + assert.equal(await run('set_setting', { key: 'uiScale', value: 'large' }, f), 'Set the interface scale to "large".') + assert.equal(await run('set_setting', { key: 'useAtkinsonFont', value: true }, f), 'Set the Atkinson Hyperlegible font to true.') + assert.equal(await run('set_setting', { key: 'showRamIndicator', value: 'false' }, f), 'Set the RAM indicator to false.') + assert.deepEqual(f.ui, { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }) + + assert.match(await run('set_setting', { key: 'uiScale', value: 'huge' }, f), /^Error: uiScale must be one of: small, medium, large, very-large/) + assert.match(await run('set_setting', { key: 'showRamIndicator', value: 1 }, f), /^Error: showRamIndicator must be true or false/) + assert.match(await run('set_setting', { key: 'hfToken', value: 'x' }, f), /^Error: key must be one of/) +}) + +test('set_storage_folder moves existing files, then saves the folder', async () => { + const f = fake() + f.dirItems['C:\\Modly\\models'] = ['triposr', 'hunyuan'] + const result = await run('set_storage_folder', { kind: 'models', path: 'D:\\AI\\models' }, f) + assert.deepEqual(f.calls, [['move', 'C:\\Modly\\models', 'D:\\AI\\models'], ['applyStorage', 'models', 'D:\\AI\\models']]) + assert.equal(result, 'Moved 2 items from “C:\\Modly\\models” and set the models folder to “D:\\AI\\models”.') +}) + +test('set_storage_folder with an empty folder only saves it', async () => { + const f = fake() + assert.equal(await run('set_storage_folder', { kind: 'workflows', path: 'D:/flows' }, f), 'Set the workflows folder to “D:/flows”.') + assert.deepEqual(f.calls, [['applyStorage', 'workflows', 'D:/flows']]) +}) + +test('set_storage_folder refuses invalid locations and keeps everything on a failed move', async () => { + const f = fake() + assert.match(await run('set_storage_folder', { kind: 'cache', path: 'D:/x' }, f), /^Error: kind must be one of/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'models' }, f), /^Error: “models” is not an absolute path/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'c:/modly/models/sub' }, f), /^Error: .*overlaps the current models folder/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'C:\\Modly' }, f), /^Error: .*overlaps the current models folder/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'C:\\Modly\\workspace\\m' }, f), /^Error: .*overlaps another Modly folder/) + assert.match(await run('set_storage_folder', { kind: 'models', path: 'C:/Modly/models/' }, f), /already/) + assert.deepEqual(f.calls, []) + + f.dirItems['C:\\Modly\\workspace'] = ['a.glb'] + f.deps.storage.moveDirectory = async () => ({ success: false, error: 'EACCES' }) + assert.equal(await run('set_storage_folder', { kind: 'workspace', path: 'E:/ws' }, f), 'Error: Could not move the workspace files to “E:/ws”: EACCES. Nothing changed.') + assert.deepEqual(f.calls, []) +}) + +// ─── Mesh, navigation ───────────────────────────────────────────────────────── + +test('undo_mesh and redo_mesh step through the history, and say when there is nothing to do', async () => { + const f = fake() + assert.match(await run('undo_mesh', {}, f), /^Undid/) + assert.match(await run('redo_mesh', {}, f), /^Redid/) + assert.deepEqual(f.calls, [['undo'], ['redo']]) + + f.history.index = 0 + f.history.length = 1 + assert.equal(await run('undo_mesh', {}, f), 'There is no mesh change to undo.') + assert.equal(await run('redo_mesh', {}, f), 'There is no mesh change to redo.') + assert.equal(f.calls.length, 2) +}) + +test('navigate changes page and validates it', async () => { + const f = fake() + assert.equal(await run('navigate', { page: 'models' }, f), 'Opened the models page.') + assert.deepEqual(f.calls, [['navigate', 'models']]) + assert.match(await run('navigate', { page: 'home' }, f), /^Error: page must be one of: generate, workflows, models, settings/) +}) + +test('a throwing dependency comes back as an error text', async () => { + const f = fake() + f.deps.extensions.reload = async () => { throw new Error('IPC gone') } + assert.equal(await run('reload_extensions', {}, f), 'Error: IPC gone') +}) + +// ─── runClientTool ──────────────────────────────────────────────────────────── + +const event = (tool: string, args: Record, confirm = false) => ({ + type: 'client_tool' as const, id: 'evt-1', call_id: 'call-7', tool, arguments: args, confirm, +}) + +test('runClientTool runs the tool and posts the result with its call_id', async () => { + const f = fake() + const outcome = await runClientTool(event('navigate', { page: 'settings' }), f.deps, async () => { throw new Error('not asked') }) + assert.deepEqual(f.posted, [{ call_id: 'call-7', result: 'Opened the settings page.' }]) + assert.deepEqual(outcome, { description: 'Go to the settings page', result: 'Opened the settings page.', error: false, declined: false, posted: true }) +}) + +test('runClientTool asks first when confirm is set, and runs on Confirm', async () => { + const f = fake() + const asked: unknown[] = [] + await runClientTool(event('uninstall_extension', { extension_id: 'triposr' }, true), f.deps, async (description, info) => { + asked.push(description, info) + return true + }) + assert.deepEqual(asked, ['Uninstall the extension “TripoSR”', { tool: 'uninstall_extension', id: 'evt-1', callId: 'call-7', destructive: true }]) + assert.deepEqual(f.calls, [['uninstall', 'triposr']]) + assert.equal(f.posted[0].call_id, 'call-7') +}) + +test('runClientTool reports a declined confirmation without running the tool', async () => { + const f = fake() + const outcome = await runClientTool(event('delete_workflow', { workflow_id: 'wf1' }, true), f.deps, async () => false) + assert.deepEqual(f.calls, []) + assert.deepEqual(f.posted, [{ call_id: 'call-7', result: 'The user declined: Delete the workflow “Turntable”.' }]) + assert.equal(outcome.declined, true) +}) + +test('runClientTool does not ask about a call that cannot run, and flags errors', async () => { + const f = fake() + let asked = false + const outcome = await runClientTool(event('delete_workflow', { workflow_id: 'nope' }, true), f.deps, async () => { asked = true; return true }) + assert.equal(asked, false) + assert.equal(outcome.error, true) + assert.equal(f.posted[0].error, true) + assert.match(f.posted[0].result, /^Error: No workflow has the id “nope”/) +}) + +test('runClientTool answers an unknown tool with an error', async () => { + const f = fake() + const outcome = await runClientTool(event('format_disk', {}), f.deps, async () => true) + assert.match(outcome.result, /^Error: Unknown client tool “format_disk”. Client tools: install_extension,/) + assert.deepEqual(f.posted, [{ call_id: 'call-7', result: outcome.result, error: true }]) +}) + +test('runClientTool survives a failed post', async () => { + const f = fake({ postResult: async () => { throw new Error('offline') } }) + const outcome = await runClientTool(event('reload_extensions', {}), f.deps, async () => true) + assert.equal(outcome.posted, false) + assert.equal(isErrorResult(outcome.result), false) +}) + +// ─── generate_from_image ────────────────────────────────────────────────────── + +const OPTIONS: GenerationOptions = { modelId: 'triposr/main', remesh: 'quad', enableTexture: false, textureResolution: 512, modelParams: { steps: 30 } } + +function generation(statuses: { status: string; progress?: number; outputUrl?: string; error?: string }[]) { + const calls: unknown[][] = [] + let job: GenerationJob | null = null + const deps: GenerationDeps = { + api: { + generateFromImage: async (path, options, data) => { calls.push(['submit', path, options, data]); return { jobId: 'j1' } }, + pollJobStatus: async () => ({ progress: 0, ...statuses.shift()! }) as never, + cancelJob: async () => { calls.push(['cancel']) }, + }, + store: { + setCurrentJob: (j) => { job = j }, + updateCurrentJob: (patch) => { if (job) job = { ...job, ...patch } }, + pushMeshUrl: (url) => { calls.push(['push', url]) }, + clearMeshHistory: () => { calls.push(['clear']) }, + }, + options: () => OPTIONS, + notifyComplete: () => {}, + notifyError: () => {}, + wait: async () => {}, + } + return { deps, calls, job: () => job } +} + +test('startGenerationFromImage generates from the attached image into the current job', async () => { + const g = generation([{ status: 'running', progress: 50 }, { status: 'done', outputUrl: '/workspace/out.glb' }]) + const result = await startGenerationFromImage({ type: 'generate_from_image', model_id: 'sf3d/main' }, 'data:image/jpeg;base64,QUJD', g.deps) + + assert.equal(result, 'Generated a mesh with sf3d/main: /workspace/out.glb.') + const submit = g.calls.find((c) => c[0] === 'submit')! + assert.equal(submit[3], 'QUJD') + assert.deepEqual(submit[2], { ...OPTIONS, modelId: 'sf3d/main', modelParams: {} }) + assert.deepEqual(g.calls.at(-1), ['push', '/workspace/out.glb']) + assert.equal(g.job()?.status, 'done') + assert.equal(g.job()?.outputUrl, '/workspace/out.glb') +}) + +test('startGenerationFromImage keeps the selected model and its params when none is given', async () => { + const g = generation([{ status: 'done', outputUrl: '/workspace/o.glb' }]) + await startGenerationFromImage({ type: 'generate_from_image' }, 'QUJD', g.deps) + assert.deepEqual(g.calls.find((c) => c[0] === 'submit')![2], OPTIONS) +}) + +test('startGenerationFromImage reports a missing image, a wrong payload and a failed job', async () => { + const g = generation([{ status: 'error', error: 'CUDA OOM' }]) + assert.match(await startGenerationFromImage({ type: 'generate_from_image' }, undefined, g.deps), /^Error: No image/) + assert.match(await startGenerationFromImage({ type: 'mesh_update' }, 'QUJD', g.deps), /^Error: Not a generate_from_image payload/) + assert.equal(await startGenerationFromImage({ type: 'generate_from_image' }, 'QUJD', g.deps), 'Error: The generation failed: CUDA OOM') + assert.equal(g.job()?.status, 'error') +}) + +test('latestUserImage takes the most recent user message with an image', () => { + assert.equal(latestUserImage([ + { role: 'user', imageDataUrls: ['data:old'] }, + { role: 'assistant', imageDataUrls: ['data:assistant'] }, + { role: 'user', imageDataUrls: ['data:new', 'data:second'] }, + { role: 'user' }, + ]), 'data:new') + assert.equal(latestUserImage([{ role: 'user' }]), undefined) +}) diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts new file mode 100644 index 000000000..c7ce651f2 --- /dev/null +++ b/src/shared/components/agent/clientTools.ts @@ -0,0 +1,761 @@ +import type { AnyExtension, ExtensionNode, Workflow } from '../../types/electron.d' +import type { AgentActionPayload, ClientToolEvent } from '../../utils/agentStream' +import type { GenerationOptions, UiScale } from '../../stores/appStore' +import type { Page } from '../../stores/navStore' +import type { GenerationApi, GenerationStoreActions } from '../../utils/generationRunner' +import { createGenerationApi, runGeneration } from '../../utils/generationRunner' +import { useAppStore } from '../../stores/appStore' +import { useExtensionsStore } from '../../stores/extensionsStore' +import { useWorkflowsStore } from '../../stores/workflowsStore' +import { useNavStore } from '../../stores/navStore' +import { showCompletionNotification, showErrorNotification } from '../../utils/notification' + +// Tools the agent asks the frontend to run, because they need IPC or the +// renderer stores. Every handler returns the text sent back to the LLM and +// never throws: failures come back as text starting with "Error: ". + +// ─── Dependencies ───────────────────────────────────────────────────────────── + +type OpResult = { success: boolean; error?: string } + +export type StorageKind = 'models' | 'workspace' | 'workflows' + +export interface StorageDirs { + modelsDir: string + workspaceDir: string + workflowsDir: string + extensionsDir: string +} + +export interface UiSettings { + uiScale: UiScale + useAtkinsonFont: boolean + showRamIndicator: boolean +} + +export interface ToolResultBody { + call_id: string + result: string + error?: boolean +} + +export interface ClientToolDeps { + extensions: { + list: () => AnyExtension[] + installFromGitHub: (url: string) => Promise + uninstall: (extensionId: string) => Promise + reload: () => Promise + loadErrors: () => Record + } + weights: { + download: (modelId: string, variantId?: string) => Promise + delete: (modelId: string) => Promise + deleteVariant: (modelId: string, variantId: string) => Promise + deleteExtensionWeights: (extensionId: string) => Promise + /** Re-reads what is installed, so the Models page stays accurate. */ + refresh: () => Promise + } + workflows: { + list: () => Workflow[] + remove: (id: string) => Promise + /** Opens the workflow as the active editor tab, without changing page. */ + open: (id: string) => void + reload: () => Promise + } + ui: { + get: () => UiSettings + setUiScale: (value: UiScale) => void + setUseAtkinsonFont: (value: boolean) => void + setShowRamIndicator: (value: boolean) => void + } + storage: { + get: () => Promise + listDir: (dirPath: string) => Promise + moveDirectory: (args: { src: string; dest: string }) => Promise + /** Saves the new folder, the way Settings > Storage does once files are moved. */ + apply: (kind: StorageKind, path: string) => Promise + } + mesh: { + history: () => { index: number; length: number } + undo: () => void + redo: () => void + } + navigate: (page: Page) => void + postResult: (body: ToolResultBody) => Promise + /** How long a download is watched for an early failure before it is reported as started. */ + downloadSettleMs?: number +} + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +type Args = Record + +const ERROR_PREFIX = 'Error: ' + +const fail = (message: string): string => ERROR_PREFIX + message + +/** True for a result text that reports a failure. */ +export function isErrorResult(result: string): boolean { + return result.startsWith(ERROR_PREFIX) +} + +const quote = (s: string): string => `“${s}”` + +function errorText(err: unknown): string { + return err instanceof Error ? err.message : String(err) +} + +function str(args: Args, key: string): string | null { + const value = args[key] + return typeof value === 'string' && value.trim() ? value.trim() : null +} + +function nodeHasManagedWeights(node: ExtensionNode): boolean { + return Boolean(node.hfRepo || node.hasModelSources || node.weightGroups?.length) +} + +function listOf(items: string[]): string { + return items.length ? items.join(', ') : 'none' +} + +function findExtension(id: string, deps: ClientToolDeps | undefined): AnyExtension | undefined { + return deps?.extensions.list().find((e) => e.id === id.split('/')[0]) +} + +function extensionLabel(id: string, deps: ClientToolDeps | undefined): string { + const ext = findExtension(id, deps) + if (!ext) return quote(id) + const nodeId = id.includes('/') ? id.slice(id.indexOf('/') + 1) : null + const node = nodeId ? ext.nodes.find((n) => n.id === nodeId) : undefined + return quote(node ? `${ext.name} / ${node.name}` : ext.name) +} + +function workflowLabel(id: string, deps: ClientToolDeps | undefined): string { + return quote(deps?.workflows.list().find((w) => w.id === id)?.name ?? id) +} + +function lookupExtension(args: Args, deps: ClientToolDeps): { ext: AnyExtension; id: string } | string { + const id = str(args, 'extension_id') + if (!id) return 'extension_id is required.' + const ext = findExtension(id, deps) + if (!ext) { + const installed = deps.extensions.list().map((e) => e.id) + return `No installed extension has the id ${quote(id)}. Installed extensions: ${listOf(installed)}.` + } + return { ext, id } +} + +function lookupWorkflow(args: Args, deps: ClientToolDeps): Workflow | string { + const id = str(args, 'workflow_id') + if (!id) return 'workflow_id is required.' + const workflows = deps.workflows.list() + const wf = workflows.find((w) => w.id === id) + if (!wf) { + return `No workflow has the id ${quote(id)}. Workflows: ${listOf(workflows.map((w) => `${w.name} (${w.id})`))}.` + } + return wf +} + +interface WeightTarget { + ext: AnyExtension + /** Set when the id named one model ("ext/node") rather than the whole extension. */ + nodeId: string | null + variant: string | null + nodes: { fullId: string; node: ExtensionNode }[] +} + +function lookupWeights(args: Args, deps: ClientToolDeps): WeightTarget | string { + const found = lookupExtension(args, deps) + if (typeof found === 'string') return found + const { ext, id } = found + if (args.variant !== undefined && args.variant !== null && typeof args.variant !== 'string') { + return 'variant must be a string.' + } + const variant = str(args, 'variant') + if (ext.type !== 'model') return `${quote(ext.name)} is a process extension; it has no model weights.` + + const nodeId = id.includes('/') ? id.slice(id.indexOf('/') + 1) : null + let nodes = ext.nodes.filter(nodeHasManagedWeights) + if (nodeId) { + nodes = nodes.filter((n) => n.id === nodeId) + if (!nodes.length) { + const models = ext.nodes.filter(nodeHasManagedWeights).map((n) => `${ext.id}/${n.id}`) + return `${quote(ext.name)} has no model ${quote(nodeId)} with weights. Models with weights: ${listOf(models)}.` + } + } + if (!nodes.length) return `${quote(ext.name)} has no downloadable model weights.` + if (variant) { + const offered = [...new Set(nodes.flatMap((n) => n.weightVariants?.options.map((o) => o.id) ?? []))] + nodes = nodes.filter((n) => n.weightVariants?.options.some((o) => o.id === variant)) + if (!nodes.length) { + return offered.length + ? `${quote(ext.name)} has no weight variant ${quote(variant)}. Available variants: ${listOf(offered)}.` + : `${quote(ext.name)} does not offer weight variants.` + } + } + return { ext, nodeId, variant, nodes: nodes.map((node) => ({ fullId: `${ext.id}/${node.id}`, node })) } +} + +function weightsLabel(args: Args, deps: ClientToolDeps | undefined): string { + const id = typeof args.extension_id === 'string' ? args.extension_id : '?' + const variant = typeof args.variant === 'string' && args.variant ? ` (variant ${quote(args.variant)})` : '' + return `${extensionLabel(id, deps)}${variant}` +} + +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(() => r(null), ms)) + +// ─── Paths ──────────────────────────────────────────────────────────────────── + +const STORAGE_KINDS: StorageKind[] = ['models', 'workspace', 'workflows'] + +const STORAGE_KEY: Record = { + models: 'modelsDir', + workspace: 'workspaceDir', + workflows: 'workflowsDir', +} + +function isAbsolutePath(p: string): boolean { + return /^[A-Za-z]:[\\/]/.test(p) || p.startsWith('\\\\') || p.startsWith('/') +} + +/** Comparable form: forward slashes, no trailing slash, Windows paths lower-cased. */ +function normPath(p: string): string { + const s = p.replace(/\\/g, '/').replace(/\/+$/, '') + return /^[A-Za-z]:/.test(s) || s.startsWith('//') ? s.toLowerCase() : s +} + +function isInside(child: string, parent: string): boolean { + return child === parent || child.startsWith(`${parent}/`) +} + +// ─── Registry ───────────────────────────────────────────────────────────────── + +export interface ClientTool { + /** Destructive: the confirm card shows a red Confirm button. */ + destructive: boolean + /** The human sentence for the confirm card and the action list. */ + describe: (args: Args, deps?: ClientToolDeps) => string + /** Why the arguments cannot be run, or null when they can. */ + check: (args: Args, deps: ClientToolDeps) => string | null + /** Runs the tool; returns the result text for the LLM and never throws. */ + run: (args: Args, deps: ClientToolDeps) => Promise +} + +interface ToolSpec { + destructive?: boolean + describe: (args: Args, deps?: ClientToolDeps) => string + check: (args: Args, deps: ClientToolDeps) => string | null + execute: (args: Args, deps: ClientToolDeps) => Promise +} + +function defineTool(spec: ToolSpec): ClientTool { + return { + destructive: spec.destructive ?? false, + describe: spec.describe, + check: (args, deps) => { + try { + return spec.check(args, deps) + } catch (err) { + return errorText(err) + } + }, + run: async (args, deps) => { + try { + const problem = spec.check(args, deps) + if (problem) return fail(problem) + return await spec.execute(args, deps) + } catch (err) { + return fail(errorText(err)) + } + }, + } +} + +const checked = (value: T | string): string | null => (typeof value === 'string' ? value : null) + +export const clientTools: Record = { + install_extension: defineTool({ + describe: (args) => `Install the extension from ${quote(String(args.github_url ?? '?'))}`, + check: (args) => { + const url = str(args, 'github_url') + if (!url) return 'github_url is required.' + let host: string + try { + host = new URL(url).hostname.toLowerCase() + } catch { + return `${quote(url)} is not a valid URL.` + } + return host === 'github.com' || host === 'www.github.com' ? null : 'github_url must be a github.com repository URL.' + }, + async execute(args, deps) { + const result = await deps.extensions.installFromGitHub(str(args, 'github_url')!) + if (!result.success) return fail(result.error ?? 'Installation failed.') + const ext = result.extension + const id = ext?.id ?? result.extensionId ?? '?' + const needsWeights = ext?.type === 'model' && ext.nodes.some(nodeHasManagedWeights) + return `Installed the extension ${quote(ext?.name ?? id)} (id ${id}).` + + (needsWeights ? ' Its model weights are not downloaded yet; download_model_weights fetches them.' : '') + }, + }), + + uninstall_extension: defineTool({ + destructive: true, + describe: (args, deps) => `Uninstall the extension ${extensionLabel(String(args.extension_id ?? '?'), deps)}`, + check: (args, deps) => { + const found = lookupExtension(args, deps) + if (typeof found === 'string') return found + return found.ext.builtin ? `${quote(found.ext.name)} is built into Modly and cannot be uninstalled.` : null + }, + async execute(args, deps) { + const { ext } = lookupExtension(args, deps) as { ext: AnyExtension } + const result = await deps.extensions.uninstall(ext.id) + if (!result.success) return fail(result.error ?? `Could not uninstall ${quote(ext.name)}.`) + return `Uninstalled the extension ${quote(ext.name)}. Its downloaded model weights, if any, were left on disk.` + }, + }), + + reload_extensions: defineTool({ + describe: () => 'Reload the installed extensions', + check: () => null, + async execute(_args, deps) { + await deps.extensions.reload() + const count = deps.extensions.list().length + const errors = Object.entries(deps.extensions.loadErrors()) + const tail = errors.length + ? ` ${errors.length} failed to load: ${errors.map(([id, err]) => `${id} (${err})`).join('; ')}.` + : '' + return `Reloaded the extensions: ${count} installed.${tail}` + }, + }), + + download_model_weights: defineTool({ + describe: (args, deps) => `Download the model weights of ${weightsLabel(args, deps)}`, + check: (args, deps) => checked(lookupWeights(args, deps)), + async execute(args, deps) { + const target = lookupWeights(args, deps) as WeightTarget + const variant = target.variant ?? undefined + // Downloads run one after the other, as "Install all" does on the Models page. + const queue = (async () => { + const outcomes: { fullId: string; result: OpResult & { paused?: boolean; cancelled?: boolean } }[] = [] + for (const { fullId } of target.nodes) { + let result: OpResult & { paused?: boolean; cancelled?: boolean } + try { + result = await deps.weights.download(fullId, variant) + } catch (err) { + result = { success: false, error: errorText(err) } + } + outcomes.push({ fullId, result }) + } + await deps.weights.refresh().catch(() => {}) + return outcomes + })() + const outcomes = await Promise.race([queue, sleep(deps.downloadSettleMs ?? 1500)]) + const label = weightsLabel(args, deps) + if (!outcomes) { + return `Started downloading the model weights of ${label} (${target.nodes.length} model${target.nodes.length > 1 ? 's' : ''}). ` + + 'The download continues in the background; its progress shows on the Models page.' + } + const failed = outcomes.filter((o) => !o.result.success && !o.result.paused && !o.result.cancelled) + if (failed.length) { + return fail(`Downloading ${label} failed: ${failed.map((o) => `${o.fullId}: ${o.result.error ?? 'unknown error'}`).join('; ')}.`) + } + if (outcomes.some((o) => o.result.paused || o.result.cancelled)) { + return `The download of ${label} was paused or cancelled before it finished.` + } + return `Downloaded the model weights of ${label}.` + }, + }), + + delete_model_weights: defineTool({ + destructive: true, + describe: (args, deps) => `Delete the downloaded model weights of ${weightsLabel(args, deps)}`, + check: (args, deps) => checked(lookupWeights(args, deps)), + async execute(args, deps) { + const target = lookupWeights(args, deps) as WeightTarget + const label = weightsLabel(args, deps) + const results: OpResult[] = [] + if (target.variant) { + for (const { fullId } of target.nodes) results.push(await deps.weights.deleteVariant(fullId, target.variant)) + } else if (!target.nodeId && target.ext.type === 'model' && target.ext.weightGroups?.length) { + // Shared weight groups: the whole-extension delete, as the uninstall dialog does. + results.push(await deps.weights.deleteExtensionWeights(target.ext.id)) + } else { + for (const { fullId } of target.nodes) results.push(await deps.weights.delete(fullId)) + } + await deps.weights.refresh().catch(() => {}) + const errors = results.filter((r) => !r.success).map((r) => r.error ?? 'unknown error') + if (errors.length) return fail(`Deleting the weights of ${label} failed: ${errors.join('; ')}.`) + return `Deleted the downloaded model weights of ${label}.` + }, + }), + + delete_workflow: defineTool({ + destructive: true, + describe: (args, deps) => `Delete the workflow ${workflowLabel(String(args.workflow_id ?? '?'), deps)}`, + check: (args, deps) => checked(lookupWorkflow(args, deps)), + async execute(args, deps) { + const wf = lookupWorkflow(args, deps) as Workflow + const result = await deps.workflows.remove(wf.id) + if (!result.success) return fail(result.error ?? `Could not delete the workflow ${quote(wf.name)}.`) + return `Deleted the workflow ${quote(wf.name)}.` + }, + }), + + open_workflow: defineTool({ + describe: (args, deps) => `Open the workflow ${workflowLabel(String(args.workflow_id ?? '?'), deps)}`, + check: (args, deps) => checked(lookupWorkflow(args, deps)), + async execute(args, deps) { + const wf = lookupWorkflow(args, deps) as Workflow + deps.workflows.open(wf.id) + return `Opened the workflow ${quote(wf.name)} as the active tab of the Workflows editor. The current page did not change.` + }, + }), + + set_setting: defineTool({ + describe: (args) => `Set ${SETTING_LABELS[String(args.key)] ?? String(args.key ?? '?')} to ${JSON.stringify(args.value)}`, + check: (args) => { + const key = args.key + if (typeof key !== 'string' || !(key in SETTING_LABELS)) { + return `key must be one of: ${Object.keys(SETTING_LABELS).join(', ')}.` + } + if (key === 'uiScale') { + return UI_SCALES.includes(args.value as UiScale) ? null : `uiScale must be one of: ${UI_SCALES.join(', ')}.` + } + return toBool(args.value) === null ? `${key} must be true or false.` : null + }, + async execute(args, deps) { + const key = args.key as keyof UiSettings + if (key === 'uiScale') deps.ui.setUiScale(args.value as UiScale) + else if (key === 'useAtkinsonFont') deps.ui.setUseAtkinsonFont(toBool(args.value)!) + else deps.ui.setShowRamIndicator(toBool(args.value)!) + return `Set ${SETTING_LABELS[key]} to ${JSON.stringify(deps.ui.get()[key])}.` + }, + }), + + set_storage_folder: defineTool({ + destructive: true, + describe: (args) => `Change the ${String(args.kind ?? '?')} folder to ${quote(String(args.path ?? '?'))}, moving its current files there`, + check: (args) => { + if (!STORAGE_KINDS.includes(args.kind as StorageKind)) return `kind must be one of: ${STORAGE_KINDS.join(', ')}.` + const path = str(args, 'path') + if (!path) return 'path is required.' + return isAbsolutePath(path) ? null : `${quote(path)} is not an absolute path.` + }, + async execute(args, deps) { + const kind = args.kind as StorageKind + const path = str(args, 'path')! + const dirs = await deps.storage.get() + const current = dirs[STORAGE_KEY[kind]] + const next = normPath(path) + if (current && normPath(current) === next) return `The ${kind} folder is already ${quote(current)}; nothing changed.` + if (current && (isInside(next, normPath(current)) || isInside(normPath(current), next))) { + return fail(`${quote(path)} overlaps the current ${kind} folder ${quote(current)}; choose a folder outside it. Nothing changed.`) + } + for (const [key, dir] of Object.entries(dirs) as [keyof StorageDirs, string][]) { + if (key === STORAGE_KEY[kind] || !dir) continue + if (isInside(next, normPath(dir)) || isInside(normPath(dir), next)) { + return fail(`${quote(path)} overlaps another Modly folder (${quote(dir)}); choose a separate folder. Nothing changed.`) + } + } + + const items = current ? await deps.storage.listDir(current).catch(() => [] as string[]) : [] + if (items.length > 0) { + const moved = await deps.storage.moveDirectory({ src: current, dest: path }) + if (!moved.success) { + return fail(`Could not move the ${kind} files to ${quote(path)}: ${moved.error ?? 'unknown error'}. Nothing changed.`) + } + } + try { + await deps.storage.apply(kind, path) + } catch (err) { + return fail(items.length + ? `Moved the ${kind} files to ${quote(path)}, but saving the new folder failed: ${errorText(err)}. Set it in Settings > Storage.` + : `Saving the new ${kind} folder failed: ${errorText(err)}. Nothing changed.`) + } + return items.length + ? `Moved ${items.length} item${items.length > 1 ? 's' : ''} from ${quote(current)} and set the ${kind} folder to ${quote(path)}.` + : `Set the ${kind} folder to ${quote(path)}.` + }, + }), + + undo_mesh: defineTool({ + describe: () => 'Undo the last mesh change', + check: () => null, + async execute(_args, deps) { + const { index } = deps.mesh.history() + if (index <= 0) return 'There is no mesh change to undo.' + deps.mesh.undo() + return 'Undid the last mesh change; the viewer shows the previous mesh.' + }, + }), + + redo_mesh: defineTool({ + describe: () => 'Redo the mesh change', + check: () => null, + async execute(_args, deps) { + const { index, length } = deps.mesh.history() + if (index >= length - 1) return 'There is no mesh change to redo.' + deps.mesh.redo() + return 'Redid the mesh change; the viewer shows the next mesh.' + }, + }), + + navigate: defineTool({ + describe: (args) => `Go to the ${String(args.page ?? '?')} page`, + check: (args) => (PAGES.includes(args.page as Page) ? null : `page must be one of: ${PAGES.join(', ')}.`), + async execute(args, deps) { + deps.navigate(args.page as Page) + return `Opened the ${String(args.page)} page.` + }, + }), +} + +const SETTING_LABELS: Record = { + uiScale: 'the interface scale', + useAtkinsonFont: 'the Atkinson Hyperlegible font', + showRamIndicator: 'the RAM indicator', +} + +const UI_SCALES: UiScale[] = ['small', 'medium', 'large', 'very-large'] + +const PAGES: Page[] = ['generate', 'workflows', 'models', 'settings'] + +function toBool(value: unknown): boolean | null { + if (typeof value === 'boolean') return value + if (value === 'true') return true + if (value === 'false') return false + return null +} + +/** The sentence for a tool call; falls back to the tool name for an unknown tool. */ +export function describeClientTool(tool: string, args: Args, deps?: ClientToolDeps): string { + const entry = clientTools[tool] + if (!entry) return tool.replace(/_/g, ' ') + try { + return entry.describe(args ?? {}, deps) + } catch { + return tool.replace(/_/g, ' ') + } +} + +// ─── Running a client_tool event ────────────────────────────────────────────── + +export interface ConfirmInfo { + tool: string + /** The `client_tool` event id, which the later `tool_done` carries too. */ + id: string + callId: string + destructive: boolean +} + +export type ConfirmFn = (description: string, info?: ConfirmInfo) => Promise + +export interface ClientToolOutcome { + description: string + result: string + error: boolean + declined: boolean + /** False when the result could not be sent back to the server. */ + posted: boolean +} + +/** + * Runs one `client_tool` event: asks for confirmation when the server says so, + * runs the tool, then posts the result to `/agent/tool-result`. Never throws. + */ +export async function runClientTool( + event: Pick, + deps: ClientToolDeps, + confirm: ConfirmFn, +): Promise { + const tool = clientTools[event.tool] + const args: Args = event.arguments && typeof event.arguments === 'object' && !Array.isArray(event.arguments) + ? event.arguments + : {} + const description = describeClientTool(event.tool, args, deps) + let result: string + let declined = false + + if (!tool) { + result = fail(`Unknown client tool ${quote(event.tool)}. Client tools: ${Object.keys(clientTools).join(', ')}.`) + } else { + const problem = tool.check(args, deps) + if (problem) { + // Nothing to confirm: the call cannot run as given. + result = fail(problem) + } else { + let ok = true + if (event.confirm) { + try { + ok = await confirm(description, { tool: event.tool, id: event.id, callId: event.call_id, destructive: tool.destructive }) + } catch { + ok = false + } + } + if (ok) { + result = await tool.run(args, deps) + } else { + declined = true + result = `The user declined: ${description}.` + } + } + } + + const error = isErrorResult(result) + let posted = true + try { + await deps.postResult({ call_id: event.call_id, result, ...(error ? { error: true } : {}) }) + } catch { + posted = false + } + return { description, result, error, declined, posted } +} + +// ─── generate_from_image ────────────────────────────────────────────────────── + +export interface GenerationDeps { + api: GenerationApi + store: GenerationStoreActions + /** The Generate page's current options (model, remesh, texture, params). */ + options: () => GenerationOptions + notifyComplete: (body: string) => void | Promise + notifyError: (body: string, title: string) => void | Promise + isCancelled?: () => boolean + wait?: () => Promise +} + +/** The first image of the most recent user message that has one. */ +export function latestUserImage(messages: { role: string; imageDataUrls?: string[] }[]): string | undefined { + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] + if (m.role === 'user' && m.imageDataUrls?.length) return m.imageDataUrls[0] + } + return undefined +} + +/** + * Runs a `generate_from_image` payload through the Generate page's own path, + * so the result appears in its viewer. `image` is a data URL or bare base64. + * Resolves once the generation ends, with a sentence describing the outcome. + */ +export async function startGenerationFromImage( + payload: AgentActionPayload | null | undefined, + image: string | undefined, + deps: GenerationDeps, +): Promise { + if (payload?.type !== 'generate_from_image') return fail('Not a generate_from_image payload.') + const base64 = image?.startsWith('data:') ? image.slice(image.indexOf(',') + 1) : image + if (!base64) return fail('No image is attached to the conversation; attach one and ask again.') + + const current = deps.options() + const modelId = payload.model_id || current.modelId + if (!modelId) return fail('No generation model is selected or downloaded.') + const options: GenerationOptions = { + ...current, + modelId, + // Saved params belong to the selected model; another model starts from its defaults. + modelParams: { ...(modelId === current.modelId ? current.modelParams : {}), ...(payload.params ?? {}) }, + } + + try { + const outcome = await runGeneration('agent-image.png', options, base64, { + api: deps.api, + store: deps.store, + notifyComplete: deps.notifyComplete, + notifyError: deps.notifyError, + isCancelled: deps.isCancelled ?? (() => false), + wait: deps.wait, + }) + if (outcome.status === 'done') return `Generated a mesh with ${modelId}${outcome.outputUrl ? `: ${outcome.outputUrl}` : ''}.` + if (outcome.status === 'cancelled') return 'The generation was cancelled.' + return fail(`The generation failed: ${outcome.error}`) + } catch (err) { + return fail(`The generation failed: ${errorText(err)}`) + } +} + +// ─── App wiring ─────────────────────────────────────────────────────────────── + +/** The real dependencies: stores, IPC and the local API. */ +export function createClientToolDeps(): ClientToolDeps { + const ext = () => useExtensionsStore.getState() + const app = () => useAppStore.getState() + const wfs = () => useWorkflowsStore.getState() + return { + extensions: { + list: () => [...ext().modelExtensions, ...ext().processExtensions], + installFromGitHub: (url) => ext().installFromGitHub(url), + uninstall: (id) => ext().uninstall(id), + reload: () => ext().reload(), + loadErrors: () => ext().loadErrors, + }, + weights: { + download: (modelId, variantId) => window.electron.model.download(modelId, variantId), + delete: (modelId) => window.electron.model.delete(modelId), + deleteVariant: (modelId, variantId) => window.electron.model.deleteWeightVariant(modelId, variantId), + deleteExtensionWeights: (extensionId) => window.electron.model.deleteExtensionWeights(extensionId), + refresh: () => ext().refreshInstalledWeightVariants(), + }, + workflows: { + list: () => wfs().workflows, + remove: (id) => wfs().remove(id), + open: (id) => wfs().openWorkflow(id), + reload: () => wfs().load(), + }, + ui: { + get: () => ({ uiScale: app().uiScale, useAtkinsonFont: app().useAtkinsonFont, showRamIndicator: app().showRamIndicator }), + setUiScale: (v) => app().setUiScale(v), + setUseAtkinsonFont: (v) => app().setUseAtkinsonFont(v), + setShowRamIndicator: (v) => app().setShowRamIndicator(v), + }, + storage: { + get: () => window.electron.settings.get(), + listDir: (dirPath) => window.electron.fs.listDir(dirPath), + moveDirectory: (args) => window.electron.fs.moveDirectory(args), + async apply(kind, path) { + // Same calls as Settings > Storage. + if (kind === 'models') { + await window.electron.settings.set({ modelsDir: path }) + await window.electron.api.updatePaths({ modelsDir: path }) + } else if (kind === 'workspace') { + await window.electron.settings.set({ workspaceDir: path }) + await window.electron.api.updatePaths({ workspaceDir: path }) + } else { + await window.electron.settings.set({ workflowsDir: path }) + await wfs().load() + } + }, + }, + mesh: { + history: () => ({ index: app().historyIndex, length: app().meshHistory.length }), + undo: () => app().undoMesh(), + redo: () => app().redoMesh(), + }, + navigate: (page) => useNavStore.getState().navigate(page), + async postResult(body) { + const res = await fetch(`${app().apiUrl}/agent/tool-result`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) + if (!res.ok) throw new Error(`API error ${res.status}`) + }, + } +} + +/** The real dependencies of startGenerationFromImage. */ +export function createGenerationDeps(): GenerationDeps { + const app = () => useAppStore.getState() + return { + api: createGenerationApi(app().apiUrl), + store: { + setCurrentJob: (job) => app().setCurrentJob(job), + updateCurrentJob: (patch) => app().updateCurrentJob(patch), + pushMeshUrl: (url) => app().pushMeshUrl(url), + clearMeshHistory: () => app().clearMeshHistory(), + }, + options: () => app().generationOptions, + notifyComplete: (body) => showCompletionNotification(body), + notifyError: (body, title) => showErrorNotification(body, title), + } +} From e3d3ca530c294c244cdc3ffa6f4deaaf94fb460b Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:16:25 +0100 Subject: [PATCH 045/108] feat(agent): wire client tools, confirm cards and app-wide context into the chat The chat runs client_tool events through the client tool registry (after a Confirm click for destructive ones), starts generate_from_image after the turn, and sends the app-wide context. Undo/redo checks read the mesh history depths. --- src/shared/components/agent/AgentChat.tsx | 42 ++++++++++++------- .../components/agent/clientTools.test.ts | 8 ++-- src/shared/components/agent/clientTools.ts | 12 +++--- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/src/shared/components/agent/AgentChat.tsx b/src/shared/components/agent/AgentChat.tsx index d155c80cb..93b8e9f43 100644 --- a/src/shared/components/agent/AgentChat.tsx +++ b/src/shared/components/agent/AgentChat.tsx @@ -11,6 +11,9 @@ import { adviceText, ollamaAdvice } from '@shared/utils/ollamaHealth' import OllamaBanner, { useOllamaHealth } from '@areas/generate/components/OllamaBanner' import { commitSculpt } from '@areas/generate/sculpt/sculptStore' import { useAgentConversationStore } from '@shared/stores/agentConversationStore' +import { buildAgentContext } from './agentContext' +import { createClientToolDeps, createGenerationDeps, latestUserImage, runClientTool, startGenerationFromImage } from './clientTools' +import { QueuedConfirmCard, useConfirmQueue } from './ConfirmCard' // ─── Types ──────────────────────────────────────────────────────────────────── @@ -295,6 +298,10 @@ export default function AgentChat(): JSX.Element { const applyMeshEdit = useAppStore((s) => s.applyMeshEdit) const undoMesh = useAppStore((s) => s.undoMesh) + const clientDeps = useMemo(() => createClientToolDeps(), []) + const confirm = useConfirmQueue((s) => s.confirm) + const confirmRequests = useConfirmQueue((s) => s.requests) + const isOllama = provider === 'ollama' const ollama = useOllamaHealth(isOllama, apiUrl, providerBaseUrl(provider, externalConfig), model) @@ -363,19 +370,6 @@ export default function AgentChat(): JSX.Element { endRef.current?.scrollIntoView({ behavior: isLoading ? 'auto' : 'smooth' }) }, [messages, isLoading, pendingWorkflow]) - function buildContext(): Record { - const ctx: Record = {} - // Read from the store, not the render: a sculpt commit may just have swapped the mesh. - const { currentJob, meshStats } = useAppStore.getState() - if (currentJob?.outputUrl) ctx.currentMeshPath = currentJob.outputUrl.replace('/workspace/', '') - if (meshStats?.triangles) ctx.meshTriangles = meshStats.triangles - if (workflows.length > 0) ctx.workflows = workflows.map((w) => ({ id: w.id, name: w.name })) - if (allExtensions.length > 0) ctx.extensions = allExtensions.map((e) => ({ - id: e.id, name: e.name, input: e.input, output: e.output, - })) - return ctx - } - async function callAgent(msgs: Message[], extraContext: Record = {}) { setIsLoading(true) setError(null) @@ -383,7 +377,9 @@ export default function AgentChat(): JSX.Element { let finished = false let failure: string | null = null try { - const context = { ...buildContext(), ...extraContext } + // Read at send time: the page, the mesh (a sculpt commit may just have + // swapped it), workflows, extensions, models, folders and UI settings. + const context: Record = { ...(await buildAgentContext()), ...extraContext } // Inject workflow completion as a system hint if present const apiMessages = msgs.map((m) => { @@ -437,6 +433,12 @@ export default function AgentChat(): JSX.Element { case 'tool_start': patch((m) => ({ ...m, actions: [...(m.actions ?? []), { id: event.id, tool: event.tool, result: '', running: true }] })) break + case 'client_tool': + // Runs in the app (stores / IPC), after a Confirm click when asked; + // the server waits for its result, then sends the usual tool_done. + // Not awaited: the stream must keep reading meanwhile. + void runClientTool(event, clientDeps, confirm) + break case 'tool_done': patch((m) => ({ ...m, @@ -463,6 +465,7 @@ export default function AgentChat(): JSX.Element { // Started only now: the local LLM frees its VRAM at the end of the // turn, and the workflow needs the whole GPU. for (const action of event.actions) startWorkflow(action.payload, msgs) + for (const action of event.actions) startGeneration(action.payload, msgs) break } } @@ -479,6 +482,8 @@ export default function AgentChat(): JSX.Element { ? { ...m, streaming: false, actions: m.actions?.map((a) => ({ ...a, running: false })) } : m))) } + // A card still waiting when the turn ends has no one listening any more. + useConfirmQueue.getState().cancelAll() setIsLoading(false) } } @@ -507,6 +512,14 @@ export default function AgentChat(): JSX.Element { } } + // Started after `done`, like workflows: the local LLM frees the GPU first. + function startGeneration(payload: AgentActionPayload | null | undefined, msgs: Message[]) { + if (payload?.type !== 'generate_from_image') return + void startGenerationFromImage(payload, latestUserImage(msgs), createGenerationDeps()).then((result) => { + setMessages((prev) => [...prev, { id: `sys-${Date.now()}`, role: 'assistant', content: result }]) + }) + } + function startWorkflow(payload: AgentActionPayload | null | undefined, msgs: Message[]) { if (payload?.type !== 'run_workflow' || !payload.workflow_id) return const wf = workflows.find((w) => w.id === payload.workflow_id) @@ -685,6 +698,7 @@ export default function AgentChat(): JSX.Element {
)} + {confirmRequests.map((request) => )}
diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index cda280082..30bc265db 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -54,7 +54,7 @@ interface Fake { posted: ToolResultBody[] ui: { uiScale: UiScale; useAtkinsonFont: boolean; showRamIndicator: boolean } dirs: { modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string } - history: { index: number; length: number } + history: { undo: number; redo: number } dirItems: Record } @@ -66,7 +66,7 @@ function fake(overrides: Partial = {}): Fake { const state = { ui: { uiScale: 'medium' as UiScale, useAtkinsonFont: false, showRamIndicator: true }, dirs: { modelsDir: 'C:\\Modly\\models', workspaceDir: 'C:\\Modly\\workspace', workflowsDir: 'C:\\Modly\\workflows', extensionsDir: 'C:\\Modly\\extensions' }, - history: { index: 1, length: 3 }, + history: { undo: 1, redo: 1 }, dirItems: {} as Record, } const list = extensions() @@ -292,8 +292,8 @@ test('undo_mesh and redo_mesh step through the history, and say when there is no assert.match(await run('redo_mesh', {}, f), /^Redid/) assert.deepEqual(f.calls, [['undo'], ['redo']]) - f.history.index = 0 - f.history.length = 1 + f.history.undo = 0 + f.history.redo = 0 assert.equal(await run('undo_mesh', {}, f), 'There is no mesh change to undo.') assert.equal(await run('redo_mesh', {}, f), 'There is no mesh change to redo.') assert.equal(f.calls.length, 2) diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts index fc4221333..e181126e9 100644 --- a/src/shared/components/agent/clientTools.ts +++ b/src/shared/components/agent/clientTools.ts @@ -8,6 +8,7 @@ import { useAppStore } from '../../stores/appStore' import { useExtensionsStore } from '../../stores/extensionsStore' import { useWorkflowsStore } from '../../stores/workflowsStore' import { useNavStore } from '../../stores/navStore' +import { redoDepth, undoDepth } from '../../stores/meshHistory' import { showCompletionNotification, showRunErrorNotification } from '../../utils/notification' // Tools the agent asks the frontend to run, because they need IPC or the @@ -76,7 +77,8 @@ export interface ClientToolDeps { apply: (kind: StorageKind, path: string) => Promise } mesh: { - history: () => { index: number; length: number } + /** Steps available each way on the mesh undo stack. */ + history: () => { undo: number; redo: number } undo: () => void redo: () => void } @@ -482,8 +484,7 @@ export const clientTools: Record = { describe: () => 'Undo the last mesh change', check: () => null, async execute(_args, deps) { - const { index } = deps.mesh.history() - if (index <= 0) return 'There is no mesh change to undo.' + if (deps.mesh.history().undo <= 0) return 'There is no mesh change to undo.' deps.mesh.undo() return 'Undid the last mesh change; the viewer shows the previous mesh.' }, @@ -493,8 +494,7 @@ export const clientTools: Record = { describe: () => 'Redo the mesh change', check: () => null, async execute(_args, deps) { - const { index, length } = deps.mesh.history() - if (index >= length - 1) return 'There is no mesh change to redo.' + if (deps.mesh.history().redo <= 0) return 'There is no mesh change to redo.' deps.mesh.redo() return 'Redid the mesh change; the viewer shows the next mesh.' }, @@ -727,7 +727,7 @@ export function createClientToolDeps(): ClientToolDeps { }, }, mesh: { - history: () => ({ index: app().historyIndex, length: app().meshHistory.length }), + history: () => ({ undo: undoDepth(app().meshHistory), redo: redoDepth(app().meshHistory) }), undo: () => app().undoMesh(), redo: () => app().redoMesh(), }, From 02d77c64dbda46bb292a81c54719fba8e29b6ae9 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:18:36 +0100 Subject: [PATCH 046/108] fix(agent): confirm extension installs, and enforce confirmation in the app install_extension runs code downloaded from GitHub, so it now needs a Confirm click showing the URL. The app also asks for destructive tools and installs whatever the server's confirm flag says, so a backend slip cannot bypass the card. --- api/routers/agent.py | 3 ++- api/tests/test_agent_client_tools.py | 5 +++-- src/shared/components/agent/clientTools.test.ts | 14 ++++++++++++++ src/shared/components/agent/clientTools.ts | 11 ++++++++++- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/api/routers/agent.py b/api/routers/agent.py index 797e40461..884d730d3 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -323,7 +323,8 @@ def _function(name: str, description: str, properties: dict | None = None, requi # Tools the app runs itself (Electron IPC, renderer stores), mapped to whether # the user must confirm them first. The frontend obeys `confirm`. CLIENT_TOOLS: dict[str, bool] = { - "install_extension": False, + # Installing runs code downloaded from GitHub: the user must see the URL and click. + "install_extension": True, "uninstall_extension": True, "reload_extensions": False, "download_model_weights": False, diff --git a/api/tests/test_agent_client_tools.py b/api/tests/test_agent_client_tools.py index 999a9ce01..e27d61773 100644 --- a/api/tests/test_agent_client_tools.py +++ b/api/tests/test_agent_client_tools.py @@ -61,7 +61,7 @@ async def answer(event): {k: v for k, v in handoff.items() if k != "call_id"}, { "type": "client_tool", "id": "c1", "tool": "install_extension", - "arguments": {"github_url": "https://github.com/a/b"}, "confirm": False, + "arguments": {"github_url": "https://github.com/a/b"}, "confirm": True, }, ) self.assertRegex(handoff["call_id"], r"^[0-9a-f]{32}$") @@ -233,7 +233,8 @@ def test_every_client_tool_is_declared_with_its_confirmation(self) -> None: confirmed = {name for name, confirm in agent.CLIENT_TOOLS.items() if confirm} self.assertEqual( confirmed, - {"uninstall_extension", "delete_model_weights", "delete_workflow", "set_storage_folder"}, + # install_extension runs code downloaded from GitHub: never without a click. + {"install_extension", "uninstall_extension", "delete_model_weights", "delete_workflow", "set_storage_folder"}, ) def test_every_client_tool_has_a_schema(self) -> None: diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index 30bc265db..4a6f49a4f 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -345,6 +345,20 @@ test('runClientTool reports a declined confirmation without running the tool', a assert.equal(outcome.declined, true) }) +test('runClientTool asks for code installs and destructive tools even when the event says no confirm', async () => { + for (const [tool, args] of [ + ['install_extension', { github_url: 'https://github.com/a/b' }], + ['delete_workflow', { workflow_id: 'wf1' }], + ] as const) { + const f = fake() + let asked = false + const outcome = await runClientTool(event(tool, { ...args }, false), f.deps, async () => { asked = true; return false }) + assert.equal(asked, true, tool) + assert.equal(outcome.declined, true, tool) + assert.deepEqual(f.calls, [], tool) + } +}) + test('runClientTool does not ask about a call that cannot run, and flags errors', async () => { const f = fake() let asked = false diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts index e181126e9..05f162225 100644 --- a/src/shared/components/agent/clientTools.ts +++ b/src/shared/components/agent/clientTools.ts @@ -235,6 +235,11 @@ function isInside(child: string, parent: string): boolean { export interface ClientTool { /** Destructive: the confirm card shows a red Confirm button. */ destructive: boolean + /** + * Always asks before running, whatever the server's `confirm` says: true for + * destructive tools and for installs (they run code downloaded from GitHub). + */ + requiresConfirm: boolean /** The human sentence for the confirm card and the action list. */ describe: (args: Args, deps?: ClientToolDeps) => string /** Why the arguments cannot be run, or null when they can. */ @@ -245,6 +250,8 @@ export interface ClientTool { interface ToolSpec { destructive?: boolean + /** Ask even when not destructive (see ClientTool.requiresConfirm). */ + requiresConfirm?: boolean describe: (args: Args, deps?: ClientToolDeps) => string check: (args: Args, deps: ClientToolDeps) => string | null execute: (args: Args, deps: ClientToolDeps) => Promise @@ -253,6 +260,7 @@ interface ToolSpec { function defineTool(spec: ToolSpec): ClientTool { return { destructive: spec.destructive ?? false, + requiresConfirm: (spec.destructive ?? false) || (spec.requiresConfirm ?? false), describe: spec.describe, check: (args, deps) => { try { @@ -277,6 +285,7 @@ const checked = (value: T | string): string | null => (typeof value === 'stri export const clientTools: Record = { install_extension: defineTool({ + requiresConfirm: true, describe: (args) => `Install the extension from ${quote(String(args.github_url ?? '?'))}`, check: (args) => { const url = str(args, 'github_url') @@ -585,7 +594,7 @@ export async function runClientTool( result = fail(problem) } else { let ok = true - if (event.confirm) { + if (event.confirm || tool.requiresConfirm) { try { ok = await confirm(description, { tool: event.tool, id: event.id, callId: event.call_id, destructive: tool.destructive }) } catch { From 119ce1ccb456431cb558da57165df76c6ecb29d8 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:23:13 +0100 Subject: [PATCH 047/108] fix(agent): review fixes for the side panel - set_storage_folder moves what Settings would (storageEntries, so a workflows folder of JSON files moves too), refuses what Settings refuses, and reports items left behind. - set_setting declares uiScale by its scale names, matching the app. - A queued confirm card opens the panel; a card the server stopped waiting for is cancelled; answered cards are cleared per turn and per conversation. - An agent-started generation keeps the conversation busy, so its result cannot land in another one. - A failed conversation save stays dirty and blocks switching away; temp files get unique names. --- api/routers/agent.py | 6 ++-- api/tests/test_agent_client_tools.py | 9 ++++-- electron/main/agent-conversations.ts | 3 +- src/shared/components/agent/AgentChat.tsx | 17 ++++++++--- src/shared/components/agent/AgentPanel.tsx | 12 ++++++++ .../components/agent/ConfirmCard.test.mjs | 22 +++++++++++++++ src/shared/components/agent/ConfirmCard.tsx | 13 +++++++++ .../components/agent/clientTools.test.ts | 28 +++++++++++++++++-- src/shared/components/agent/clientTools.ts | 27 +++++++++++++----- src/shared/stores/agentConversationStore.ts | 20 ++++++++----- 10 files changed, 129 insertions(+), 28 deletions(-) diff --git a/api/routers/agent.py b/api/routers/agent.py index 884d730d3..1e067b4a3 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -104,7 +104,7 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: - **install_extension(github_url)** / **uninstall_extension(extension_id)** / **reload_extensions** — Manage extensions. - **download_model_weights(extension_id, variant?)** / **delete_model_weights(extension_id, variant?)** — Manage model weights. - **open_workflow(workflow_id)** / **delete_workflow(workflow_id)** — Open a workflow in the editor, or delete it. -- **set_setting(key, value)** — Change `uiScale` (number), `useAtkinsonFont` or `showRamIndicator` (booleans). +- **set_setting(key, value)** — Change `uiScale` (one of `small`, `medium`, `large`, `very-large`), `useAtkinsonFont` or `showRamIndicator` (booleans). - **set_storage_folder(kind, path)** — Move the `models`, `workspace` or `workflows` folder. - **undo_mesh** / **redo_mesh** — Undo or redo the last change to the current mesh. - **navigate(page)** — Switch the app to another page. @@ -378,10 +378,10 @@ def _function(name: str, description: str, properties: dict | None = None, requi ), _function( "set_setting", - "Change a UI setting. uiScale is a number (1 = 100%); useAtkinsonFont and showRamIndicator are booleans.", + "Change a UI setting. uiScale is one of small, medium, large, very-large; useAtkinsonFont and showRamIndicator are booleans.", { "key": {"type": "string", "enum": ["uiScale", "useAtkinsonFont", "showRamIndicator"]}, - "value": {"type": ["number", "boolean"], "description": "A number for uiScale, a boolean otherwise."}, + "value": {"type": ["string", "boolean"], "description": "For uiScale one of: small, medium, large, very-large. A boolean otherwise."}, }, ["key", "value"], ), diff --git a/api/tests/test_agent_client_tools.py b/api/tests/test_agent_client_tools.py index e27d61773..d6fabfca1 100644 --- a/api/tests/test_agent_client_tools.py +++ b/api/tests/test_agent_client_tools.py @@ -247,6 +247,9 @@ def test_every_client_tool_has_a_schema(self) -> None: def test_enums(self) -> None: props = self._tool("set_setting")["parameters"]["properties"] self.assertEqual(props["key"]["enum"], ["uiScale", "useAtkinsonFont", "showRamIndicator"]) + # The app's scale names, not a number: the frontend accepts only these. + self.assertEqual(props["value"]["type"], ["string", "boolean"]) + self.assertIn("small, medium, large, very-large", props["value"]["description"]) props = self._tool("set_storage_folder")["parameters"]["properties"] self.assertEqual(props["kind"]["enum"], ["models", "workspace", "workflows"]) nav = self._tool("navigate") @@ -271,7 +274,7 @@ def test_system_prompt_lists_the_new_tools(self) -> None: "modelsDir": "D:/models", "workspaceDir": "D:/ws", "workflowsDir": "D:/wf", "extensionsDir": "D:/ext", }, - "ui": {"uiScale": 1.1, "useAtkinsonFont": False, "showRamIndicator": True}, + "ui": {"uiScale": "large", "useAtkinsonFont": False, "showRamIndicator": True}, } @@ -288,7 +291,7 @@ def test_list_extensions(self) -> None: def test_get_settings(self) -> None: text, payload = _run("get_settings") - for expected in ("D:/models", "D:/ws", "D:/wf", "D:/ext", "uiScale: 1.1", "useAtkinsonFont: false", "showRamIndicator: true"): + for expected in ("D:/models", "D:/ws", "D:/wf", "D:/ext", "uiScale: large", "useAtkinsonFont: false", "showRamIndicator: true"): self.assertIn(expected, text) self.assertIsNone(payload) self.assertIn("No settings", _run("get_settings", context={})[0]) @@ -349,7 +352,7 @@ def test_context_is_rendered_into_the_single_system_message(self) -> None: "- sf3d: SF3D", "- wf1: My Workflow", "D:/models", "D:/ws", "D:/wf", "D:/ext", - "uiScale: 1.1", "useAtkinsonFont: false", "showRamIndicator: true", + "uiScale: large", "useAtkinsonFont: false", "showRamIndicator: true", "Current mesh path: Default/mesh.glb", "- trellis/gen (image→mesh): Trellis", ): diff --git a/electron/main/agent-conversations.ts b/electron/main/agent-conversations.ts index 24c130fd2..8a2db16e4 100644 --- a/electron/main/agent-conversations.ts +++ b/electron/main/agent-conversations.ts @@ -2,6 +2,7 @@ // /agent/conversations/{id}.json. import { join } from 'path' +import { randomUUID } from 'crypto' import { mkdir, readFile, readdir, rename, rm, writeFile } from 'fs/promises' export interface ConversationSummary { @@ -75,7 +76,7 @@ export function createConversationStorage(dir: string) { await mkdir(dir, { recursive: true }) const target = fileFor(conversation.id) // Write then rename: a crash mid-write never leaves a truncated file. - const tmp = `${target}.${process.pid}.${Date.now()}.tmp` + const tmp = `${target}.${randomUUID()}.tmp` await writeFile(tmp, JSON.stringify(conversation), 'utf-8') try { await rename(tmp, target) diff --git a/src/shared/components/agent/AgentChat.tsx b/src/shared/components/agent/AgentChat.tsx index 93b8e9f43..6f1f1c03f 100644 --- a/src/shared/components/agent/AgentChat.tsx +++ b/src/shared/components/agent/AgentChat.tsx @@ -282,6 +282,8 @@ export default function AgentChat(): JSX.Element { const localModels = llmModels.filter((m) => m.downloaded && !(m.tags ?? []).some((t) => t === 'code' || t === 'cad')) const [showModelPicker, setShowModelPicker] = useState(false) const [pendingWorkflow, setPendingWorkflow] = useState<{ id: string; name: string } | null>(null) + // Generations the agent started; the conversation stays put until they end. + const [generations, setGenerations] = useState(0) const [attachments, setAttachments] = useState([]) // data URLs const [isDragging, setIsDragging] = useState(false) const [thinkingMode, setThinkingMode] = useState(defaultThinking) @@ -357,12 +359,14 @@ export default function AgentChat(): JSX.Element { }, [runState.status, pendingWorkflow]) // The panel locks conversation switching while a turn or its workflow runs. - useEffect(() => { setBusy(isLoading || pendingWorkflow !== null) }, [isLoading, pendingWorkflow, setBusy]) + useEffect(() => { setBusy(isLoading || pendingWorkflow !== null || generations > 0) }, [isLoading, pendingWorkflow, generations, setBusy]) // Another conversation was opened: drop the view state of the previous one. useEffect(() => { setShowAll(false) setError(null) + useConfirmQueue.getState().cancelAll() + useConfirmQueue.getState().clearAnswered() }, [currentId]) useEffect(() => { @@ -371,6 +375,7 @@ export default function AgentChat(): JSX.Element { }, [messages, isLoading, pendingWorkflow]) async function callAgent(msgs: Message[], extraContext: Record = {}) { + useConfirmQueue.getState().clearAnswered() setIsLoading(true) setError(null) const id = `a-${Date.now()}` @@ -440,6 +445,7 @@ export default function AgentChat(): JSX.Element { void runClientTool(event, clientDeps, confirm) break case 'tool_done': + useConfirmQueue.getState().cancelEvent(event.id) patch((m) => ({ ...m, actions: (m.actions ?? []).map((a) => (a.id === event.id @@ -515,9 +521,12 @@ export default function AgentChat(): JSX.Element { // Started after `done`, like workflows: the local LLM frees the GPU first. function startGeneration(payload: AgentActionPayload | null | undefined, msgs: Message[]) { if (payload?.type !== 'generate_from_image') return - void startGenerationFromImage(payload, latestUserImage(msgs), createGenerationDeps()).then((result) => { - setMessages((prev) => [...prev, { id: `sys-${Date.now()}`, role: 'assistant', content: result }]) - }) + setGenerations((n) => n + 1) + void startGenerationFromImage(payload, latestUserImage(msgs), createGenerationDeps()) + .then((result) => { + setMessages((prev) => [...prev, { id: `sys-${Date.now()}`, role: 'assistant', content: result }]) + }) + .finally(() => setGenerations((n) => n - 1)) } function startWorkflow(payload: AgentActionPayload | null | undefined, msgs: Message[]) { diff --git a/src/shared/components/agent/AgentPanel.tsx b/src/shared/components/agent/AgentPanel.tsx index c3ba61c90..f2d68b158 100644 --- a/src/shared/components/agent/AgentPanel.tsx +++ b/src/shared/components/agent/AgentPanel.tsx @@ -9,6 +9,7 @@ import { import { useAgentConversationStore } from '@shared/stores/agentConversationStore' import { UNTITLED, deriveTitle, relativeDate } from '@shared/utils/agentConversations' import AgentChat from './AgentChat' +import { useConfirmQueue } from './ConfirmCard' // ─── Helpers ────────────────────────────────────────────────────────────────── @@ -137,6 +138,17 @@ export default function AgentPanel(): JSX.Element { return () => window.removeEventListener('beforeunload', onUnload) }, []) + // A confirm card is waiting for a click: show it, even if the panel was + // closed mid-turn or the conversation list covered the chat. + useEffect(() => { + const { setOnQueued } = useConfirmQueue.getState() + setOnQueued(() => { + setOpen(true) + setShowList(false) + }) + return () => setOnQueued(null) + }, [setOpen]) + // Ctrl+L / Cmd+L toggles the panel, except while typing elsewhere. useEffect(() => { const handler = (e: KeyboardEvent) => { diff --git a/src/shared/components/agent/ConfirmCard.test.mjs b/src/shared/components/agent/ConfirmCard.test.mjs index da117e50a..a9553ed95 100644 --- a/src/shared/components/agent/ConfirmCard.test.mjs +++ b/src/shared/components/agent/ConfirmCard.test.mjs @@ -64,3 +64,25 @@ test('cancelAll declines every pending request', async () => { test('ConfirmCard is a component', () => { assert.equal(typeof ConfirmCard, 'function') }) + +test('cancelEvent cancels the pending card of a tool call the server gave up on', async () => { + const answer = queue().confirm('Delete the workflow “Turntable”', { tool: 'delete_workflow', id: 'e9', callId: 'c9', destructive: true }) + const other = queue().confirm('Uninstall the extension “X”', { tool: 'uninstall_extension', id: 'e10', callId: 'c10', destructive: true }) + queue().cancelEvent('e9') + assert.equal(await answer, false) + assert.deepEqual(queue().requests.map((r) => r.status), ['cancelled', 'pending']) + queue().cancelAll() + assert.equal(await other, false) + queue().clearAnswered() +}) + +test('a queued confirm calls onQueued, so the panel can open itself', async () => { + let opened = 0 + queue().setOnQueued(() => { opened++ }) + const answer = queue().confirm('Install the extension from “https://github.com/a/b”', { tool: 'install_extension', id: 'e11', callId: 'c11', destructive: false }) + assert.equal(opened, 1) + queue().cancelAll() + assert.equal(await answer, false) + queue().setOnQueued(null) + queue().clearAnswered() +}) diff --git a/src/shared/components/agent/ConfirmCard.tsx b/src/shared/components/agent/ConfirmCard.tsx index 55aeb6a7d..e406b28f2 100644 --- a/src/shared/components/agent/ConfirmCard.tsx +++ b/src/shared/components/agent/ConfirmCard.tsx @@ -26,6 +26,11 @@ interface ConfirmQueueState { cancelAll: () => void /** Forgets answered requests. */ clearAnswered: () => void + /** Cancels the pending card of one client_tool event (the server stopped waiting for it). */ + cancelEvent: (eventId: string) => void + /** Called whenever a card is queued; the panel uses it to open itself so the card is seen. */ + onQueued: (() => void) | null + setOnQueued: (callback: (() => void) | null) => void } // Resolvers stay out of the state: they are not data to render. @@ -49,6 +54,7 @@ export const useConfirmQueue = create((set, get) => ({ return new Promise((resolve) => { resolvers.set(key, resolve) set((s) => ({ requests: [...s.requests, request] })) + get().onQueued?.() }) }, @@ -69,6 +75,13 @@ export const useConfirmQueue = create((set, get) => ({ clearAnswered() { set((s) => ({ requests: s.requests.filter((r) => r.status === 'pending') })) }, + + cancelEvent(eventId) { + for (const r of get().requests) if (r.status === 'pending' && r.eventId === eventId) get().answer(r.key, false) + }, + + onQueued: null, + setOnQueued: (callback) => set({ onQueued: callback }), })) // ─── Card ───────────────────────────────────────────────────────────────────── diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index 4a6f49a4f..62082de08 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -103,7 +103,8 @@ function fake(overrides: Partial = {}): Fake { }, storage: { get: async () => state.dirs, - listDir: async (dir) => state.dirItems[dir] ?? [], + entries: async (kind: StorageKind) => ({ modly: state.dirItems[kind] ?? [], other: [] }), + checkLocation: async (path: string) => (path.startsWith('C:\\Windows') ? { ok: false, error: 'This is a system folder.' } : { ok: true }), moveDirectory: async (args) => { rec('move', args.src, args.dest); return ok }, apply: async (kind: StorageKind, path: string) => { rec('applyStorage', kind, path) }, }, @@ -256,12 +257,33 @@ test('set_setting sets each UI option and validates key and value', async () => test('set_storage_folder moves existing files, then saves the folder', async () => { const f = fake() - f.dirItems['C:\\Modly\\models'] = ['triposr', 'hunyuan'] + f.dirItems.models = ['triposr', 'hunyuan'] const result = await run('set_storage_folder', { kind: 'models', path: 'D:\\AI\\models' }, f) assert.deepEqual(f.calls, [['move', 'C:\\Modly\\models', 'D:\\AI\\models'], ['applyStorage', 'models', 'D:\\AI\\models']]) assert.equal(result, 'Moved 2 items from “C:\\Modly\\models” and set the models folder to “D:\\AI\\models”.') }) +test('set_storage_folder moves a workflows folder that holds only files', async () => { + const f = fake() + f.dirItems.workflows = ['turntable.json'] + const result = await run('set_storage_folder', { kind: 'workflows', path: 'D:/flows' }, f) + assert.deepEqual(f.calls, [['move', 'C:\\Modly\\workflows', 'D:/flows'], ['applyStorage', 'workflows', 'D:/flows']]) + assert.match(result, /^Moved 1 item from/) +}) + +test('set_storage_folder refuses a location Settings would refuse', async () => { + const f = fake() + assert.equal(await run('set_storage_folder', { kind: 'models', path: 'C:\\Windows\\m' }, f), 'Error: This is a system folder. Nothing changed.') + assert.deepEqual(f.calls, []) +}) + +test('set_storage_folder says which items stayed behind', async () => { + const f = fake() + f.dirItems.models = ['a', 'b'] + f.deps.storage.moveDirectory = async (args) => { f.calls.push(['move', args.src, args.dest]); return { success: true, kept: ['b'] } } + assert.match(await run('set_storage_folder', { kind: 'models', path: 'D:/m' }, f), /^Moved 1 item from .*; 1 item could not be moved and stayed in “C:\\Modly\\models”\.$/) +}) + test('set_storage_folder with an empty folder only saves it', async () => { const f = fake() assert.equal(await run('set_storage_folder', { kind: 'workflows', path: 'D:/flows' }, f), 'Set the workflows folder to “D:/flows”.') @@ -278,7 +300,7 @@ test('set_storage_folder refuses invalid locations and keeps everything on a fai assert.match(await run('set_storage_folder', { kind: 'models', path: 'C:/Modly/models/' }, f), /already/) assert.deepEqual(f.calls, []) - f.dirItems['C:\\Modly\\workspace'] = ['a.glb'] + f.dirItems.workspace = ['a.glb'] f.deps.storage.moveDirectory = async () => ({ success: false, error: 'EACCES' }) assert.equal(await run('set_storage_folder', { kind: 'workspace', path: 'E:/ws' }, f), 'Error: Could not move the workspace files to “E:/ws”: EACCES. Nothing changed.') assert.deepEqual(f.calls, []) diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts index 05f162225..a9bdeeb78 100644 --- a/src/shared/components/agent/clientTools.ts +++ b/src/shared/components/agent/clientTools.ts @@ -71,8 +71,11 @@ export interface ClientToolDeps { } storage: { get: () => Promise - listDir: (dirPath: string) => Promise - moveDirectory: (args: { src: string; dest: string }) => Promise + /** What Modly created in the current folder (moved) and what it did not (left alone), as Settings sees it. */ + entries: (kind: StorageKind) => Promise<{ modly: string[]; other: string[] }> + /** Settings' guard: refuses a drive root, a system folder or an ancestor of home/app data. */ + checkLocation: (path: string) => Promise<{ ok: boolean; error?: string }> + moveDirectory: (args: { src: string; dest: string }) => Promise /** Saves the new folder, the way Settings > Storage does once files are moved. */ apply: (kind: StorageKind, path: string) => Promise } @@ -469,12 +472,18 @@ export const clientTools: Record = { } } - const items = current ? await deps.storage.listDir(current).catch(() => [] as string[]) : [] + const location = await deps.storage.checkLocation(path).catch((err) => ({ ok: false, error: errorText(err) })) + if (!location.ok) return fail(`${location.error ?? `${quote(path)} cannot hold Modly data.`} Nothing changed.`) + + // Files count too: a workflows folder holds only JSON files. + const items = current ? (await deps.storage.entries(kind).catch(() => ({ modly: [] as string[] }))).modly : [] + let kept: string[] = [] if (items.length > 0) { const moved = await deps.storage.moveDirectory({ src: current, dest: path }) if (!moved.success) { return fail(`Could not move the ${kind} files to ${quote(path)}: ${moved.error ?? 'unknown error'}. Nothing changed.`) } + kept = moved.kept ?? [] } try { await deps.storage.apply(kind, path) @@ -483,9 +492,12 @@ export const clientTools: Record = { ? `Moved the ${kind} files to ${quote(path)}, but saving the new folder failed: ${errorText(err)}. Set it in Settings > Storage.` : `Saving the new ${kind} folder failed: ${errorText(err)}. Nothing changed.`) } - return items.length - ? `Moved ${items.length} item${items.length > 1 ? 's' : ''} from ${quote(current)} and set the ${kind} folder to ${quote(path)}.` - : `Set the ${kind} folder to ${quote(path)}.` + if (!items.length) return `Set the ${kind} folder to ${quote(path)}.` + const count = (n: number) => `${n} item${n > 1 ? 's' : ''}` + const movedCount = items.length - kept.length + return kept.length + ? `Moved ${count(movedCount)} from ${quote(current)} and set the ${kind} folder to ${quote(path)}; ${count(kept.length)} could not be moved and stayed in ${quote(current)}.` + : `Moved ${count(movedCount)} from ${quote(current)} and set the ${kind} folder to ${quote(path)}.` }, }), @@ -719,7 +731,8 @@ export function createClientToolDeps(): ClientToolDeps { }, storage: { get: () => window.electron.settings.get(), - listDir: (dirPath) => window.electron.fs.listDir(dirPath), + entries: (kind) => window.electron.fs.storageEntries(kind), + checkLocation: (path) => window.electron.fs.checkStorageLocation(path), moveDirectory: (args) => window.electron.fs.moveDirectory(args), async apply(kind, path) { // Same calls as Settings > Storage. diff --git a/src/shared/stores/agentConversationStore.ts b/src/shared/stores/agentConversationStore.ts index d52261c1c..7ff4e114e 100644 --- a/src/shared/stores/agentConversationStore.ts +++ b/src/shared/stores/agentConversationStore.ts @@ -43,7 +43,7 @@ interface AgentConversationState { setMessages: (update: AgentMessage[] | ((prev: AgentMessage[]) => AgentMessage[])) => void setBusy: (busy: boolean) => void - newConversation: () => void + newConversation: () => Promise open: (id: string) => Promise remove: (id: string) => Promise refreshList: () => Promise @@ -63,11 +63,12 @@ let dirty = false let initPromise: Promise | null = null export const useAgentConversationStore = create()((set, get) => { - async function save(): Promise { + /** False when the write failed: the changes stay unsaved and switching away must wait. */ + async function save(): Promise { if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } const { currentId, createdAt, messages } = get() // An empty conversation never reaches the disk. - if (!dirty || messages.length === 0) return + if (!dirty || messages.length === 0) return true dirty = false const conversation: Conversation = { id: currentId, @@ -79,8 +80,12 @@ export const useAgentConversationStore = create()((set, try { await window.electron.agent.conversations.save(conversation) set((s) => ({ list: upsertSummary(s.list, { id: conversation.id, title: conversation.title, updatedAt: conversation.updatedAt }) })) + return true } catch (err) { console.error('[agent] Failed to save conversation:', err) + // Still unsaved: the next change or switch tries again. + if (get().currentId === conversation.id) dirty = true + return false } } @@ -104,16 +109,17 @@ export const useAgentConversationStore = create()((set, setBusy: (busy) => set({ busy }), - newConversation: () => { + newConversation: async () => { if (get().busy) return - void save() + // Leaving a conversation whose save failed would lose it. + if (!(await save())) return set({ currentId: newId(), createdAt: Date.now(), messages: [] }) dirty = false }, open: async (id) => { if (get().busy || id === get().currentId) return - await save() + if (!(await save())) return const conversation = await window.electron.agent.conversations.read(id) if (!conversation) { // Gone or corrupted on disk: drop it from the list. @@ -165,6 +171,6 @@ export const useAgentConversationStore = create()((set, return initPromise }, - flush: save, + flush: async () => { await save() }, } }) From e580e27d1b25e6a25588e6cab3e910c0bf024163 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:32:06 +0100 Subject: [PATCH 048/108] feat(agent): slide the side panel in and out The panel's slot grows to its width while the panel slides and fades in from the right (220 ms), and the reverse on close. No transition while resizing, none with reduced motion. Closed, it stays mounted but invisible. --- src/shared/components/agent/AgentPanel.tsx | 25 +++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/shared/components/agent/AgentPanel.tsx b/src/shared/components/agent/AgentPanel.tsx index f2d68b158..cca241cf8 100644 --- a/src/shared/components/agent/AgentPanel.tsx +++ b/src/shared/components/agent/AgentPanel.tsx @@ -198,14 +198,28 @@ export default function AgentPanel(): JSX.Element { if (e.key === 'ArrowRight') { e.preventDefault(); setWidth(width - step) } } + const dragging = dragWidth !== null + return ( + // The slot grows from 0 to the panel width, pushing the app aside, while + // the panel slides in from the right. Closed, it stays mounted (a streaming + // turn or a running workflow keeps going) but invisible, so it takes no + // focus once the slide-out has ended. +
+
) } From 362d93940da1ca6e59b706772ee1421302ff8b4c Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 14:34:28 +0100 Subject: [PATCH 049/108] style(agent): Claude-like composer in the side panel A lighter rounded box with the text on top, a plus to attach and the send button inside; the model picker and the thinking mode sit as quiet text controls under it. The Shift+Enter hint moves to the send button's tooltip. --- src/shared/components/agent/AgentChat.tsx | 131 ++++++++++++---------- 1 file changed, 69 insertions(+), 62 deletions(-) diff --git a/src/shared/components/agent/AgentChat.tsx b/src/shared/components/agent/AgentChat.tsx index 6f1f1c03f..6441924d7 100644 --- a/src/shared/components/agent/AgentChat.tsx +++ b/src/shared/components/agent/AgentChat.tsx @@ -715,8 +715,9 @@ export default function AgentChat(): JSX.Element { {/* Ollama down / model not pulled */} { void ollama.check() }} /> - {/* Input bar */} -
+ {/* Input bar: a composer box (text, attach, send) with the model and + thinking controls on a quiet line under it, as in Claude. */} +
{ if (e.target.files) handleFiles(Array.from(e.target.files)); e.target.value = '' }} /> -
+
{/* Attachment previews */} {attachments.length > 0 && (
@@ -734,6 +735,7 @@ export default function AgentChat(): JSX.Element { - {/* Thinking toggle */} + + {/* Send */} - {/* Model selector */} -
- - - {showModelPicker && ( -
- {!isLocal ? ( -

{PROVIDERS[provider].label} — change the model in Settings → Agent.

- ) : localModels.length === 0 ? ( -

No local model downloaded — open Settings → Agent.

- ) : ( - localModels.map((m) => ( - - )) - )} -
- )} -
-
+
+
+ {/* Model and thinking, under the box */} +
+
+ + {showModelPicker && ( +
+ {!isLocal ? ( +

{PROVIDERS[provider].label} — change the model in Settings → Agent.

+ ) : localModels.length === 0 ? ( +

No local model downloaded — open Settings → Agent.

+ ) : ( + localModels.map((m) => ( + + )) + )} +
+ )}
+ + {/* Thinking: auto → on → off */} +
-

Shift+Enter for new line

From a8c44e6ea3d9645b41185022ac229657f1276dd7 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 18:10:43 +0100 Subject: [PATCH 050/108] fix(agent): review fixes for the side panel and conversation saving Agent tools: - generate_from_image refuses while a generation is already running - the tool-result endpoint no longer doubles the "Error: " prefix - a confirmed tool waits for the click (10 min), then gets tool_timeout for its run, via the new POST /agent/tool-confirmed - the non-streaming /agent/chat no longer offers client tools - uninstall_extension rejects "ext/node" ids so the card can't mislead Conversations: - every disk operation goes through one queue: a save in flight is always awaited, and a delete can't race the write of its file - a failed delete keeps the pending changes - init retries after a failure and skips unreadable conversations - rename retries on transient Windows locks; orphan .tmp files are cleaned --- api/routers/agent.py | 95 ++++++++-- api/tests/test_agent_client_tools.py | 96 ++++++++++ .../2026-11-12-agent-side-panel-design.md | 2 +- electron/main/agent-conversations.test.ts | 64 ++++++- electron/main/agent-conversations.ts | 52 +++++- src/shared/components/agent/AgentChat.tsx | 9 +- .../components/agent/clientTools.test.ts | 55 ++++++ src/shared/components/agent/clientTools.ts | 40 +++- src/shared/stores/agentConversationStore.ts | 126 +++++++++---- src/shared/utils/agentConversations.test.ts | 175 ++++++++++++++++++ src/shared/utils/agentConversations.ts | 13 ++ 11 files changed, 665 insertions(+), 62 deletions(-) diff --git a/api/routers/agent.py b/api/routers/agent.py index 1e067b4a3..39e8cb8cb 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -405,12 +405,55 @@ def _function(name: str, description: str, properties: dict | None = None, requi ), ] -# Seconds the app has to run a client tool that needs the user's click. +# Seconds the user has to click Confirm or Cancel on a confirmed client tool. +# Once they confirm (POST /agent/tool-confirmed), the tool gets tool_timeout. CONFIRM_TIMEOUT = 600.0 _NO_ANSWER = "Error: The app did not answer in time." +# Client tools are run by the Modly app; /agent/chat has nobody to run them. +_NEEDS_APP = "Error: This tool needs the Modly app." # call_id → the turn waiting for the app's answer (see /agent/tool-result). _pending_client_tools: dict[str, asyncio.Future] = {} +# call_id → set when the user clicked Confirm (see /agent/tool-confirmed). +_confirmed_client_tools: dict[str, asyncio.Event] = {} + +# What the non-streaming /agent/chat offers: no client tool, nobody would answer. +SERVER_TOOLS = [t for t in TOOLS if t["function"]["name"] not in CLIENT_TOOLS] + + +def _not_finished(tool: str, limit: float) -> str: + return ( + f"Error: {tool} was confirmed but did not finish within {limit:g} s. " + "It may still be running in the app; check before retrying." + ) + + +async def _await_client_answer( + tool: str, answer: asyncio.Future, confirmed: asyncio.Event, confirm: bool, tool_timeout: float, +) -> str: + """The app's answer to one client tool, or why there is none. + + A confirmed tool waits up to CONFIRM_TIMEOUT for the click (or a direct + answer, e.g. a Cancel), then tool_timeout for the run itself. + """ + if confirm: + clicked = asyncio.ensure_future(confirmed.wait()) + try: + await asyncio.wait({answer, clicked}, timeout=CONFIRM_TIMEOUT, return_when=asyncio.FIRST_COMPLETED) + finally: + clicked.cancel() + if answer.done(): + return answer.result() + if not confirmed.is_set(): + return _NO_ANSWER + try: + return await asyncio.wait_for(answer, tool_timeout) + except asyncio.TimeoutError: + return _not_finished(tool, tool_timeout) + try: + return await asyncio.wait_for(answer, tool_timeout) + except asyncio.TimeoutError: + return _NO_ANSWER # Input kinds the agent may pick, mapped to the real Modly source-node types. @@ -1045,9 +1088,12 @@ async def _claim_slot(model: str, spec: dict): raise -async def _chat_events(request: AgentChatRequest): +async def _chat_events(request: AgentChatRequest, client_tools: bool = True): """Run the tool loop, yielding its events as they happen. + client_tools — False when nobody reads the stream to run them (/agent/chat): + they are not offered, and a call to one fails at once. + token / thinking — answer and reasoning text, in pieces tool_start / tool_done — around each tool call client_tool — between them, for a tool the app runs: the turn waits for @@ -1136,7 +1182,10 @@ async def post() -> httpx.Response: "POST", f"{base_url}/chat/completions", headers=headers, - json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": True, **extra}, + json={ + "model": request.model, "messages": messages, + "tools": TOOLS if client_tools else SERVER_TOOLS, "stream": True, **extra, + }, ), stream=True, ) @@ -1183,26 +1232,29 @@ async def post() -> httpx.Response: break arguments = _tool_arguments(fn.get("arguments")) yield {"type": "tool_start", "id": tc["id"], "tool": fn["name"], "arguments": arguments} - if fn["name"] in CLIENT_TOOLS: + if fn["name"] in CLIENT_TOOLS and not client_tools: + # Not offered here, but a model may still name one. + result_text, payload = _NEEDS_APP, None + elif fn["name"] in CLIENT_TOOLS: # The app runs it and answers through /agent/tool-result. confirm = CLIENT_TOOLS[fn["name"]] call_id = uuid.uuid4().hex answer = asyncio.get_running_loop().create_future() + confirmed = asyncio.Event() _pending_client_tools[call_id] = answer + _confirmed_client_tools[call_id] = confirmed try: yield { "type": "client_tool", "id": tc["id"], "call_id": call_id, "tool": fn["name"], "arguments": arguments, "confirm": confirm, } - # A confirmation waits on the user's click, not on the tool. - limit = CONFIRM_TIMEOUT if confirm else request.tool_timeout - try: - result_text = await asyncio.wait_for(answer, limit) - except asyncio.TimeoutError: - result_text = _NO_ANSWER + result_text = await _await_client_answer( + fn["name"], answer, confirmed, confirm, request.tool_timeout, + ) finally: # Also runs when the client leaves: nobody will answer. _pending_client_tools.pop(call_id, None) + _confirmed_client_tools.pop(call_id, None) answer.cancel() payload = None else: @@ -1259,7 +1311,26 @@ async def agent_tool_result(req: ToolResultRequest): answer = _pending_client_tools.get(req.call_id) if answer is None or answer.done(): raise HTTPException(status_code=404, detail="No tool call is waiting for this call_id.") - answer.set_result(f"Error: {req.result}" if req.error else req.result) + # The app's results already start with "Error: "; don't say it twice. + result = req.result + if req.error and not result.startswith("Error:"): + result = f"Error: {result}" + answer.set_result(result) + return {"ok": True} + + +class ToolConfirmedRequest(BaseModel): + call_id: str + + +@router.post("/tool-confirmed") +async def agent_tool_confirmed(req: ToolConfirmedRequest): + """The user clicked Confirm: the turn now waits tool_timeout for the result.""" + confirmed = _confirmed_client_tools.get(req.call_id) + answer = _pending_client_tools.get(req.call_id) + if confirmed is None or answer is None or answer.done(): + raise HTTPException(status_code=404, detail="No tool call is waiting for this call_id.") + confirmed.set() return {"ok": True} @@ -1267,7 +1338,7 @@ async def agent_tool_result(req: ToolResultRequest): async def agent_chat(request: AgentChatRequest): """The whole turn at once, for callers that don't read a stream.""" done: dict = {} - async for event in _chat_events(request): + async for event in _chat_events(request, client_tools=False): if event["type"] == "done": done = event return AgentChatResponse( diff --git a/api/tests/test_agent_client_tools.py b/api/tests/test_agent_client_tools.py index d6fabfca1..cd43d3b54 100644 --- a/api/tests/test_agent_client_tools.py +++ b/api/tests/test_agent_client_tools.py @@ -18,6 +18,10 @@ async def _answer(call_id: str, result: str, error: bool = False) -> dict: return await agent.agent_tool_result(agent.ToolResultRequest(call_id=call_id, result=result, error=error)) +async def _confirmed(call_id: str) -> dict: + return await agent.agent_tool_confirmed(agent.ToolConfirmedRequest(call_id=call_id)) + + def _run_turn(scripted: _Scripted, request: agent.AgentChatRequest, on_client_tool) -> list[dict]: """Drive a turn, letting `on_client_tool(event)` answer each client tool the way the panel would (it may be a coroutine function).""" @@ -39,6 +43,7 @@ async def collect(): class ClientToolBridgeTests(unittest.TestCase): def tearDown(self) -> None: agent._pending_client_tools.clear() + agent._confirmed_client_tools.clear() def test_client_tool_is_handed_to_the_app_and_its_answer_reaches_the_model(self) -> None: scripted = _Scripted( @@ -106,6 +111,13 @@ def test_error_answer_is_prefixed(self) -> None: self.assertEqual(events[2]["result"], "Error: disk full") self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: disk full") + def test_an_error_already_prefixed_by_the_app_is_not_prefixed_again(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "ok"})) + events = _run_turn(scripted, _external(), lambda e: _answer(e["call_id"], "Error: disk full", error=True)) + + self.assertEqual(events[2]["result"], "Error: disk full") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: disk full") + def test_unknown_or_already_answered_call_is_404(self) -> None: with self.assertRaises(HTTPException) as caught: asyncio.run(_answer("nope", "x")) @@ -159,6 +171,63 @@ def test_confirm_wait_is_capped(self) -> None: self.assertEqual(events[2]["result"], "Error: The app did not answer in time.") + def test_after_confirm_the_tool_gets_the_tool_timeout(self) -> None: + scripted = _Scripted(_client_round("set_storage_folder", {"kind": "models", "path": "E:/m"}), _sse({"content": "ok"})) + seen = [] + + async def confirm_then_hang(event): + await asyncio.sleep(0.02) + seen.append(await _confirmed(event["call_id"])) + + def on_tool(event): + asyncio.ensure_future(confirm_then_hang(event)) + + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 5.0): + events = _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), on_tool) + + self.assertEqual(seen, [{"ok": True}]) + result = events[2]["result"] + self.assertTrue(result.startswith("Error: set_storage_folder was confirmed but did not finish within 0.05 s")) + self.assertIn("may still be running", result) + self.assertEqual(agent._pending_client_tools, {}) + self.assertEqual(agent._confirmed_client_tools, {}) + + def test_a_confirmed_tool_answering_within_the_tool_timeout_is_relayed(self) -> None: + scripted = _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})) + + def on_tool(event): + async def click_then_run(): + await asyncio.sleep(0.1) # longer than tool_timeout: the click is not timed by it + await _confirmed(event["call_id"]) + await asyncio.sleep(0.02) + await _answer(event["call_id"], "Deleted.") + asyncio.ensure_future(click_then_run()) + + with mock.patch.object(agent, "CONFIRM_TIMEOUT", 5.0): + _run_turn(scripted, _external().model_copy(update={"tool_timeout": 0.05}), on_tool) + + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Deleted.") + + def test_confirmed_for_an_unknown_or_answered_call_is_404(self) -> None: + with self.assertRaises(HTTPException) as caught: + asyncio.run(_confirmed("nope")) + self.assertEqual(caught.exception.status_code, 404) + + seen = [] + + async def answer_then_confirm(event): + await _answer(event["call_id"], "The user declined.") + try: + await _confirmed(event["call_id"]) + except HTTPException as e: + seen.append(e.status_code) + + _run_turn( + _Scripted(_client_round("delete_workflow", {"workflow_id": "w"}), _sse({"content": "ok"})), + _external(), answer_then_confirm, + ) + self.assertEqual(seen, [404]) + def test_client_tools_count_against_the_budget(self) -> None: both = _sse( _call(0, id="a", function={"name": "reload_extensions", "arguments": "{}"}), @@ -208,6 +277,33 @@ async def cancel_while_waiting(): self.assertNotIn(event["call_id"], agent._pending_client_tools) +class NonStreamingChatTests(unittest.TestCase): + def test_chat_does_not_offer_client_tools(self) -> None: + scripted = _Scripted(_sse({"content": "hi"})) + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + response = asyncio.run(agent.agent_chat(_external())) + + self.assertEqual(response.message, "hi") + offered = {t["function"]["name"] for t in scripted.bodies[0]["tools"]} + self.assertFalse(offered & set(agent.CLIENT_TOOLS)) + self.assertIn("list_extensions", offered) + + def test_chat_answers_a_client_tool_call_at_once(self) -> None: + scripted = _Scripted(_client_round("reload_extensions"), _sse({"content": "Could not."})) + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + response = asyncio.run(asyncio.wait_for(agent.agent_chat(_external()), 5)) + + self.assertEqual(response.actions[0].result, "Error: This tool needs the Modly app.") + self.assertEqual(scripted.bodies[1]["messages"][-1]["content"], "Error: This tool needs the Modly app.") + self.assertEqual(agent._pending_client_tools, {}) + + def test_the_stream_still_offers_client_tools(self) -> None: + scripted = _Scripted(_sse({"content": "hi"})) + _run_turn(scripted, _external(), None) + offered = {t["function"]["name"] for t in scripted.bodies[0]["tools"]} + self.assertTrue(set(agent.CLIENT_TOOLS) <= offered) + + _CLIENT_TOOL_ARGS = { "install_extension": ["github_url"], "uninstall_extension": ["extension_id"], diff --git a/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md b/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md index 69bfa8db5..43a784bc0 100644 --- a/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md +++ b/docs/superpowers/specs/2026-11-12-agent-side-panel-design.md @@ -27,7 +27,7 @@ The agent today lives in the "chat" mode of the Generate page's left panel. It l | Tool | Does | |---|---| -| `generate_from_image(model_id?, params?)` | Starts a generation from the image attached to the message. The frontend runs it, as with `run_workflow`. | +| `generate_from_image(model_id?)` | Starts a generation from the image attached to the message. The frontend runs it, as with `run_workflow`. | | `list_extensions` | Lists installed extensions (from context). | | `get_settings` | Returns the storage paths and a few UI options (from context). | diff --git a/electron/main/agent-conversations.test.ts b/electron/main/agent-conversations.test.ts index 7d8387778..b680039d7 100644 --- a/electron/main/agent-conversations.test.ts +++ b/electron/main/agent-conversations.test.ts @@ -1,10 +1,11 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { mkdtemp, readdir, rm, writeFile } from 'node:fs/promises' +import { mkdtemp, readdir, rename, rm, utimes, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { + ORPHAN_TMP_AGE_MS, createConversationStorage, isConversation, isValidConversationId, @@ -107,3 +108,64 @@ test('corrupted or mismatched files are skipped', async () => { assert.equal(await storage.read('other'), null) }) }) + +const errno = (code: string): NodeJS.ErrnoException => Object.assign(new Error(code), { code }) + +test('a rename locked for a moment (antivirus, indexer) is retried', async () => { + await withDir(async (dir) => { + let calls = 0 + const storage = createConversationStorage(dir, { + retryDelayMs: 1, + rename: async (from, to) => { + calls++ + if (calls < 3) throw errno(calls === 1 ? 'EPERM' : 'EBUSY') + await rename(from, to) + }, + }) + await storage.save(conv('a', 1, 'Locked')) + assert.equal(calls, 3) + assert.equal((await storage.read('a'))?.title, 'Locked') + assert.deepEqual(await readdir(dir), ['a.json']) + }) +}) + +test('a rename still locked after the retries fails and leaves no temp file', async () => { + await withDir(async (dir) => { + let calls = 0 + const storage = createConversationStorage(dir, { + retryDelayMs: 1, + rename: async () => { calls++; throw errno('EACCES') }, + }) + await assert.rejects(storage.save(conv('a', 1)), { code: 'EACCES' }) + assert.equal(calls, 3) + assert.deepEqual(await readdir(dir), []) + }) +}) + +test('other rename errors are not retried', async () => { + await withDir(async (dir) => { + let calls = 0 + const storage = createConversationStorage(dir, { + retryDelayMs: 1, + rename: async () => { calls++; throw errno('ENOSPC') }, + }) + await assert.rejects(storage.save(conv('a', 1)), { code: 'ENOSPC' }) + assert.equal(calls, 1) + }) +}) + +test('list removes orphan temp files, but not ones a write may still own', async () => { + await withDir(async (dir) => { + const storage = createConversationStorage(dir) + await storage.save(conv('a', 1)) + const old = join(dir, 'a.json.1111.tmp') + const fresh = join(dir, 'a.json.2222.tmp') + await writeFile(old, '{', 'utf-8') + await writeFile(fresh, '{', 'utf-8') + const past = (Date.now() - ORPHAN_TMP_AGE_MS - 10_000) / 1000 + await utimes(old, past, past) + + assert.deepEqual((await storage.list()).map((s) => s.id), ['a']) + assert.deepEqual((await readdir(dir)).sort(), ['a.json', 'a.json.2222.tmp']) + }) +}) diff --git a/electron/main/agent-conversations.ts b/electron/main/agent-conversations.ts index 8a2db16e4..65d9011f3 100644 --- a/electron/main/agent-conversations.ts +++ b/electron/main/agent-conversations.ts @@ -3,7 +3,7 @@ import { join } from 'path' import { randomUUID } from 'crypto' -import { mkdir, readFile, readdir, rename, rm, writeFile } from 'fs/promises' +import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'fs/promises' export interface ConversationSummary { id: string @@ -39,7 +39,26 @@ export function sortSummaries(list: readonly T[]) return [...list].sort((a, b) => (b.updatedAt - a.updatedAt) || a.id.localeCompare(b.id)) } -export function createConversationStorage(dir: string) { +/** Windows antivirus or indexers briefly lock a freshly written file. */ +const TRANSIENT_RENAME_CODES = new Set(['EPERM', 'EBUSY', 'EACCES']) +const RENAME_ATTEMPTS = 3 +const RENAME_RETRY_MS = 50 +/** A temp file this old belongs to no write in flight: a crash left it behind. */ +export const ORPHAN_TMP_AGE_MS = 60_000 + +export interface ConversationStorageOptions { + /** Injectable for tests. */ + rename?: (from: string, to: string) => Promise + retryDelayMs?: number + now?: () => number +} + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +export function createConversationStorage(dir: string, options: ConversationStorageOptions = {}) { + const renameFile = options.rename ?? rename + const retryDelayMs = options.retryDelayMs ?? RENAME_RETRY_MS + const now = options.now ?? Date.now const fileFor = (id: string): string => join(dir, `${id}.json`) async function readConversation(id: string): Promise { @@ -52,6 +71,32 @@ export function createConversationStorage(dir: string) { } } + async function renameWithRetry(from: string, to: string): Promise { + for (let attempt = 1; ; attempt++) { + try { + await renameFile(from, to) + return + } catch (err) { + const code = (err as NodeJS.ErrnoException | null)?.code + if (attempt >= RENAME_ATTEMPTS || !code || !TRANSIENT_RENAME_CODES.has(code)) throw err + await sleep(retryDelayMs * attempt) + } + } + } + + /** Best effort: temp files left by a crash between write and rename. */ + async function removeOrphanTmpFiles(files: readonly string[]): Promise { + for (const file of files) { + if (!file.endsWith('.tmp')) continue + const path = join(dir, file) + try { + if (now() - (await stat(path)).mtimeMs >= ORPHAN_TMP_AGE_MS) await rm(path, { force: true }) + } catch { + // Gone already, or locked: the next listing tries again. + } + } + } + return { async list(): Promise { let files: string[] @@ -60,6 +105,7 @@ export function createConversationStorage(dir: string) { } catch { return [] } + await removeOrphanTmpFiles(files) const summaries: ConversationSummary[] = [] for (const file of files) { if (!file.endsWith('.json')) continue @@ -79,7 +125,7 @@ export function createConversationStorage(dir: string) { const tmp = `${target}.${randomUUID()}.tmp` await writeFile(tmp, JSON.stringify(conversation), 'utf-8') try { - await rename(tmp, target) + await renameWithRetry(tmp, target) } catch (err) { await rm(tmp, { force: true }) throw err diff --git a/src/shared/components/agent/AgentChat.tsx b/src/shared/components/agent/AgentChat.tsx index 6441924d7..a7c1bcab5 100644 --- a/src/shared/components/agent/AgentChat.tsx +++ b/src/shared/components/agent/AgentChat.tsx @@ -521,12 +521,15 @@ export default function AgentChat(): JSX.Element { // Started after `done`, like workflows: the local LLM frees the GPU first. function startGeneration(payload: AgentActionPayload | null | undefined, msgs: Message[]) { if (payload?.type !== 'generate_from_image') return - setGenerations((n) => n + 1) - void startGenerationFromImage(payload, latestUserImage(msgs), createGenerationDeps()) + const deps = createGenerationDeps() + // A refused start (one is already running) is not ours to count. + const counted = !deps.isGenerating() + if (counted) setGenerations((n) => n + 1) + void startGenerationFromImage(payload, latestUserImage(msgs), deps) .then((result) => { setMessages((prev) => [...prev, { id: `sys-${Date.now()}`, role: 'assistant', content: result }]) }) - .finally(() => setGenerations((n) => n - 1)) + .finally(() => { if (counted) setGenerations((n) => n - 1) }) } function startWorkflow(payload: AgentActionPayload | null | undefined, msgs: Message[]) { diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index 62082de08..e6cb9ecde 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -5,6 +5,7 @@ import { clientTools, describeClientTool, isErrorResult, + isGenerationRunning, latestUserImage, runClientTool, startGenerationFromImage, @@ -160,6 +161,17 @@ test('uninstall_extension uninstalls by id, describes by name and refuses built- assert.equal(f.calls.length, 1) }) +test('uninstall_extension refuses a model id, which would remove the whole extension', async () => { + const f = fake() + const problem = clientTools.uninstall_extension.check({ extension_id: 'hunyuan/shape' }, f.deps) + assert.match(problem ?? '', /“hunyuan\/shape” names a model; pass the extension id \(“hunyuan”\)/) + let asked = false + const outcome = await runClientTool(event('uninstall_extension', { extension_id: 'hunyuan/shape' }, true), f.deps, async () => { asked = true; return true }) + assert.equal(asked, false) + assert.equal(outcome.error, true) + assert.deepEqual(f.calls, []) +}) + test('reload_extensions reloads and reports the count and load errors', async () => { const f = fake() const result = await run('reload_extensions', {}, f) @@ -359,6 +371,32 @@ test('runClientTool asks first when confirm is set, and runs on Confirm', async assert.equal(f.posted[0].call_id, 'call-7') }) +test('runClientTool tells the server about the Confirm click before running, and not on a decline', async () => { + const order: string[] = [] + const f = fake({ + postConfirmed: async (callId) => { order.push(`confirmed ${callId}`) }, + postResult: async (body) => { order.push(`result ${body.call_id}`) }, + }) + f.deps.extensions.uninstall = async () => { order.push('uninstall'); return { success: true } } + await runClientTool(event('uninstall_extension', { extension_id: 'triposr' }, true), f.deps, async () => true) + assert.deepEqual(order, ['confirmed call-7', 'uninstall', 'result call-7']) + + order.length = 0 + await runClientTool(event('uninstall_extension', { extension_id: 'triposr' }, true), f.deps, async () => false) + assert.deepEqual(order, ['result call-7']) + + order.length = 0 + await runClientTool(event('navigate', { page: 'models' }), f.deps, async () => true) + assert.deepEqual(order, ['result call-7']) +}) + +test('runClientTool still runs the tool when the confirmed notice fails', async () => { + const f = fake({ postConfirmed: async () => { throw new Error('offline') } }) + const outcome = await runClientTool(event('delete_workflow', { workflow_id: 'wf1' }, true), f.deps, async () => true) + assert.deepEqual(f.calls, [['removeWorkflow', 'wf1']]) + assert.equal(outcome.error, false) +}) + test('runClientTool reports a declined confirmation without running the tool', async () => { const f = fake() const outcome = await runClientTool(event('delete_workflow', { workflow_id: 'wf1' }, true), f.deps, async () => false) @@ -413,6 +451,7 @@ function generation(statuses: { status: string; progress?: number; outputUrl?: s const calls: unknown[][] = [] let job: GenerationJob | null = null const deps: GenerationDeps = { + isGenerating: () => isGenerationRunning(job), api: { generateFromImage: async (path, options, data) => { calls.push(['submit', path, options, data]); return { jobId: 'j1' } }, pollJobStatus: async () => ({ progress: 0, ...statuses.shift()! }) as never, @@ -459,6 +498,22 @@ test('startGenerationFromImage reports a missing image, a wrong payload and a fa assert.equal(g.job()?.status, 'error') }) +test('startGenerationFromImage refuses while a generation is running, without touching the current job', async () => { + for (const status of ['uploading', 'generating'] as const) { + const g = generation([{ status: 'done', outputUrl: '/workspace/o.glb' }]) + const running = { id: 'busy', imageFile: 'x.png', status, progress: 40, createdAt: 1 } as GenerationJob + g.deps.store.setCurrentJob(running) + const result = await startGenerationFromImage({ type: 'generate_from_image' }, 'QUJD', g.deps) + assert.equal(result, 'Error: A generation is already running. Wait for it to finish.') + assert.deepEqual(g.calls, [], status) + assert.equal(g.job(), running) + } + for (const status of ['done', 'error', 'idle'] as const) { + assert.equal(isGenerationRunning({ status }), false, status) + } + assert.equal(isGenerationRunning(null), false) +}) + test('latestUserImage takes the most recent user message with an image', () => { assert.equal(latestUserImage([ { role: 'user', imageDataUrls: ['data:old'] }, diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts index a9bdeeb78..526873e64 100644 --- a/src/shared/components/agent/clientTools.ts +++ b/src/shared/components/agent/clientTools.ts @@ -87,6 +87,12 @@ export interface ClientToolDeps { } navigate: (page: Page) => void postResult: (body: ToolResultBody) => Promise + /** + * Tells the server the user clicked Confirm, so it stops waiting on the click + * and gives the tool its own timeout. Optional: without it the server keeps + * waiting under the confirmation cap. + */ + postConfirmed?: (callId: string) => Promise /** How long a download is watched for an early failure before it is reported as started. */ downloadSettleMs?: number } @@ -316,6 +322,12 @@ export const clientTools: Record = { destructive: true, describe: (args, deps) => `Uninstall the extension ${extensionLabel(String(args.extension_id ?? '?'), deps)}`, check: (args, deps) => { + // "ext/node" names one model, but uninstalling removes the whole extension: + // the card must not read as if only that model goes. + const id = str(args, 'extension_id') + if (id?.includes('/')) { + return `${quote(id)} names a model; pass the extension id (${quote(id.split('/')[0])}) to uninstall the whole extension.` + } const found = lookupExtension(args, deps) if (typeof found === 'string') return found return found.ext.builtin ? `${quote(found.ext.name)} is built into Modly and cannot be uninstalled.` : null @@ -606,7 +618,8 @@ export async function runClientTool( result = fail(problem) } else { let ok = true - if (event.confirm || tool.requiresConfirm) { + const asked = event.confirm || tool.requiresConfirm + if (asked) { try { ok = await confirm(description, { tool: event.tool, id: event.id, callId: event.call_id, destructive: tool.destructive }) } catch { @@ -614,6 +627,10 @@ export async function runClientTool( } } if (ok) { + if (asked && deps.postConfirmed) { + // From here the server waits on the tool, not on the click. + await deps.postConfirmed(event.call_id).catch(() => {}) + } result = await tool.run(args, deps) } else { declined = true @@ -635,6 +652,8 @@ export async function runClientTool( // ─── generate_from_image ────────────────────────────────────────────────────── export interface GenerationDeps { + /** True while a generation is uploading or running (the Generate page then disables its upload too). */ + isGenerating: () => boolean api: GenerationApi store: GenerationStoreActions /** The Generate page's current options (model, remesh, texture, params). */ @@ -645,6 +664,13 @@ export interface GenerationDeps { wait?: () => Promise } +export const GENERATION_BUSY = 'A generation is already running. Wait for it to finish.' + +/** Same condition as the Generate page's upload and options lock. */ +export function isGenerationRunning(job: { status: string } | null | undefined): boolean { + return job?.status === 'uploading' || job?.status === 'generating' +} + /** The first image of the most recent user message that has one. */ export function latestUserImage(messages: { role: string; imageDataUrls?: string[] }[]): string | undefined { for (let i = messages.length - 1; i >= 0; i--) { @@ -667,6 +693,9 @@ export async function startGenerationFromImage( if (payload?.type !== 'generate_from_image') return fail('Not a generate_from_image payload.') const base64 = image?.startsWith('data:') ? image.slice(image.indexOf(',') + 1) : image if (!base64) return fail('No image is attached to the conversation; attach one and ask again.') + // Checked synchronously right before runGeneration claims currentJob, so two + // calls cannot both pass it. + if (deps.isGenerating()) return fail(GENERATION_BUSY) const current = deps.options() const modelId = payload.model_id || current.modelId @@ -762,6 +791,14 @@ export function createClientToolDeps(): ClientToolDeps { }) if (!res.ok) throw new Error(`API error ${res.status}`) }, + async postConfirmed(callId) { + const res = await fetch(`${app().apiUrl}/agent/tool-confirmed`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ call_id: callId }), + }) + if (!res.ok) throw new Error(`API error ${res.status}`) + }, } } @@ -769,6 +806,7 @@ export function createClientToolDeps(): ClientToolDeps { export function createGenerationDeps(): GenerationDeps { const app = () => useAppStore.getState() return { + isGenerating: () => isGenerationRunning(app().currentJob), api: createGenerationApi(app().apiUrl), store: { setCurrentJob: (job) => app().setCurrentJob(job), diff --git a/src/shared/stores/agentConversationStore.ts b/src/shared/stores/agentConversationStore.ts index 7ff4e114e..621f5e1a4 100644 --- a/src/shared/stores/agentConversationStore.ts +++ b/src/shared/stores/agentConversationStore.ts @@ -1,12 +1,14 @@ import { create } from 'zustand' import type { AgentAction, ToolStop } from '@shared/utils/agentStream' +// Relative so the store loads under `node --test` (no path aliases there). import { + createSerialQueue, deriveTitle, settleMessages, upsertSummary, type Conversation, type ConversationSummary, -} from '@shared/utils/agentConversations' +} from '../utils/agentConversations' // ─── Message types (AgentChat) ──────────────────────────────────────────────── @@ -30,6 +32,8 @@ export interface AgentMessage { // ─── Store ──────────────────────────────────────────────────────────────────── const SAVE_DEBOUNCE_MS = 500 +/** Unreadable conversations init skips before giving up on reopening one. */ +const MAX_INIT_ATTEMPTS = 5 interface AgentConversationState { /** Id of the conversation on screen; not on disk until it has a message. */ @@ -47,12 +51,14 @@ interface AgentConversationState { open: (id: string) => Promise remove: (id: string) => Promise refreshList: () => Promise - /** Loads the list and opens the most recent conversation. Runs once. */ + /** Loads the list and opens the most recent readable conversation. Runs once, retried if it failed. */ init: () => Promise /** Writes a pending debounced save now. */ flush: () => Promise } +type OpenResult = 'opened' | 'unreadable' | 'skipped' + function newId(): string { return crypto.randomUUID() } @@ -60,16 +66,26 @@ function newId(): string { let saveTimer: ReturnType | null = null // Unsaved changes: opening or switching away must not bump an untouched conversation. let dirty = false +// Bumped on every edit: a save clears `dirty` only if nothing changed while it was writing. +let edits = 0 +// Deleted conversations: a save finishing late must not bring one back into the list. +const removedIds = new Set() let initPromise: Promise | null = null +// Every disk operation runs through this queue, one at a time: a save in +// flight is always awaited, and a delete never races the write of its file. +const enqueue = createSerialQueue() export const useAgentConversationStore = create()((set, get) => { - /** False when the write failed: the changes stay unsaved and switching away must wait. */ - async function save(): Promise { + /** + * Writes the current conversation if it has unsaved changes. Queued tasks only. + * False when the write failed: the changes stay unsaved and switching away must wait. + */ + async function saveNow(): Promise { if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } const { currentId, createdAt, messages } = get() // An empty conversation never reaches the disk. if (!dirty || messages.length === 0) return true - dirty = false + const editsAtStart = edits const conversation: Conversation = { id: currentId, title: deriveTitle(messages), @@ -79,14 +95,20 @@ export const useAgentConversationStore = create()((set, } try { await window.electron.agent.conversations.save(conversation) - set((s) => ({ list: upsertSummary(s.list, { id: conversation.id, title: conversation.title, updatedAt: conversation.updatedAt }) })) - return true } catch (err) { + // Still dirty: the next change or switch tries again. console.error('[agent] Failed to save conversation:', err) - // Still unsaved: the next change or switch tries again. - if (get().currentId === conversation.id) dirty = true return false } + if (get().currentId === conversation.id && edits === editsAtStart) dirty = false + if (!removedIds.has(conversation.id)) { + set((s) => ({ list: upsertSummary(s.list, { id: conversation.id, title: conversation.title, updatedAt: conversation.updatedAt }) })) + } + return true + } + + function save(): Promise { + return enqueue(saveNow) } function scheduleSave(): void { @@ -94,6 +116,32 @@ export const useAgentConversationStore = create()((set, saveTimer = setTimeout(() => { void save() }, SAVE_DEBOUNCE_MS) } + /** Queued tasks only. */ + async function openNow(id: string): Promise { + if (get().busy || id === get().currentId) return 'skipped' + // Leaving a conversation whose save failed would lose it. + if (!(await saveNow())) return 'skipped' + let conversation: Conversation | null + try { + conversation = await window.electron.agent.conversations.read(id) + } catch (err) { + console.error('[agent] Failed to read conversation:', err) + return 'skipped' + } + if (!conversation) { + // Gone or corrupted on disk: drop it from the list. + set((s) => ({ list: s.list.filter((c) => c.id !== id) })) + return 'unreadable' + } + set({ + currentId: conversation.id, + createdAt: conversation.createdAt, + messages: settleMessages(conversation.messages) as AgentMessage[], + }) + dirty = false + return 'opened' + } + return { currentId: newId(), createdAt: Date.now(), @@ -104,70 +152,66 @@ export const useAgentConversationStore = create()((set, setMessages: (update) => { set((s) => ({ messages: typeof update === 'function' ? update(s.messages) : update })) dirty = true + edits++ scheduleSave() }, setBusy: (busy) => set({ busy }), - newConversation: async () => { + newConversation: () => enqueue(async () => { if (get().busy) return // Leaving a conversation whose save failed would lose it. - if (!(await save())) return + if (!(await saveNow())) return set({ currentId: newId(), createdAt: Date.now(), messages: [] }) dirty = false - }, + }), - open: async (id) => { - if (get().busy || id === get().currentId) return - if (!(await save())) return - const conversation = await window.electron.agent.conversations.read(id) - if (!conversation) { - // Gone or corrupted on disk: drop it from the list. - set((s) => ({ list: s.list.filter((c) => c.id !== id) })) - return - } - set({ - currentId: conversation.id, - createdAt: conversation.createdAt, - messages: settleMessages(conversation.messages) as AgentMessage[], - }) - dirty = false - }, + open: async (id) => { await enqueue(() => openNow(id)) }, - remove: async (id) => { + remove: (id) => enqueue(async () => { const isCurrent = id === get().currentId if (isCurrent && get().busy) return - if (isCurrent) { - if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } - dirty = false - } try { await window.electron.agent.conversations.remove(id) } catch (err) { + // Still on disk, and its pending changes still get saved. console.error('[agent] Failed to delete conversation:', err) return } + removedIds.add(id) + if (isCurrent) { + if (saveTimer) { clearTimeout(saveTimer); saveTimer = null } + dirty = false + } set((s) => ({ list: s.list.filter((c) => c.id !== id), ...(isCurrent ? { currentId: newId(), createdAt: Date.now(), messages: [] } : {}), })) - }, + }), - refreshList: async () => { + refreshList: () => enqueue(async () => { try { set({ list: await window.electron.agent.conversations.list() }) } catch (err) { console.error('[agent] Failed to list conversations:', err) } - }, + }), init: () => { initPromise ??= (async () => { - await get().refreshList() - const latest = get().list[0] - // The user may already have started typing a new conversation. - if (latest && get().messages.length === 0 && !get().busy) await get().open(latest.id) - })() + // Throws on failure, so a failed init is forgotten and the next call retries. + const list = await enqueue(() => window.electron.agent.conversations.list()) + set({ list }) + // Open the most recent readable one; unreadable files drop out of the list. + for (const { id } of list.slice(0, MAX_INIT_ATTEMPTS)) { + // The user may already have started typing a new conversation. + if (get().messages.length > 0 || get().busy) return + if ((await enqueue(() => openNow(id))) !== 'unreadable') return + } + })().catch((err: unknown) => { + console.error('[agent] Failed to load conversations:', err) + initPromise = null + }) return initPromise }, diff --git a/src/shared/utils/agentConversations.test.ts b/src/shared/utils/agentConversations.test.ts index 8ed04cc6a..1362f54c9 100644 --- a/src/shared/utils/agentConversations.test.ts +++ b/src/shared/utils/agentConversations.test.ts @@ -4,12 +4,14 @@ import test from 'node:test' import { TITLE_MAX_LENGTH, UNTITLED, + createSerialQueue, deriveTitle, relativeDate, settleMessages, sortSummaries, upsertSummary, } from './agentConversations' +import { useAgentConversationStore } from '../stores/agentConversationStore' test('title is the first user message, collapsed to one line', () => { const title = deriveTitle([ @@ -75,3 +77,176 @@ test('relativeDate buckets by age', () => { assert.match(relativeDate(now - 30 * 86_400_000, now), /^Oct \d+$/) assert.match(relativeDate(Date.UTC(2024, 0, 15), now), /2024/) }) + +test('the serial queue runs tasks one at a time, and a failure does not stop it', async () => { + const enqueue = createSerialQueue() + const log: string[] = [] + const first = deferred() + const a = enqueue(async () => { log.push('a start'); const v = await first.promise; log.push('a end'); return v }) + const b = enqueue(async () => { log.push('b'); throw new Error('b failed') }) + const c = enqueue(async () => { log.push('c'); return 'c' }) + await tick() + assert.deepEqual(log, ['a start']) + first.resolve('a') + assert.equal(await a, 'a') + await assert.rejects(b, /b failed/) + assert.equal(await c, 'c') + assert.deepEqual(log, ['a start', 'a end', 'b', 'c']) +}) + +// ─── Store: disk operations against a fake IPC ─────────────────────────────── + +interface Deferred { promise: Promise; resolve: (v: T) => void; reject: (e: unknown) => void } + +function deferred(): Deferred { + let resolve!: (v: T) => void + let reject!: (e: unknown) => void + const promise = new Promise((res, rej) => { resolve = res; reject = rej }) + return { promise, resolve, reject } +} + +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)) + +type Ipc = { + list: () => Promise + read: (id: string) => Promise + save: (c: { id: string; messages: unknown[] }) => Promise + remove: (id: string) => Promise +} +const ipc: Ipc = { + list: async () => [], + read: async () => null, + save: async () => {}, + remove: async () => {}, +} +const calls: string[] = [] +;(globalThis as Record).window = { + electron: { + agent: { + conversations: { + list: () => { calls.push('list'); return ipc.list() }, + read: (id: string) => { calls.push(`read ${id}`); return ipc.read(id) }, + save: (c: { id: string; messages: unknown[] }) => { calls.push(`save ${c.id}`); return ipc.save(c) }, + remove: (id: string) => { calls.push(`remove ${id}`); return ipc.remove(id) }, + }, + }, + }, +} + +const store = useAgentConversationStore +const msg = (content: string) => ({ id: content, role: 'user' as const, content }) +const stored = (id: string, updatedAt: number) => ({ id, title: id, createdAt: 1, updatedAt, messages: [msg(id)] }) + +function resetIpc(): void { + ipc.list = async () => [] + ipc.read = async () => null + ipc.save = async () => {} + ipc.remove = async () => {} + calls.length = 0 +} + +// Runs first: init is memoized once it succeeds. +test('store: a failed init is retried, and an unreadable latest conversation falls back to the next', async () => { + resetIpc() + const quietError = console.error + console.error = () => {} + try { + ipc.list = async () => { throw new Error('IPC down') } + await store.getState().init() + assert.deepEqual(store.getState().list, []) + + ipc.list = async () => [ + { id: 'broken', title: 'broken', updatedAt: 30 }, + { id: 'good', title: 'good', updatedAt: 20 }, + { id: 'older', title: 'older', updatedAt: 10 }, + ] + ipc.read = async (id) => (id === 'good' ? stored('good', 20) : null) + await store.getState().init() + assert.equal(store.getState().currentId, 'good') + assert.deepEqual(store.getState().list.map((s) => s.id), ['good', 'older']) + assert.deepEqual(calls, ['list', 'list', 'read broken', 'read good']) + + // Memoized now. + await store.getState().init() + assert.equal(calls.filter((c) => c === 'list').length, 2) + } finally { + console.error = quietError + } +}) + +test('store: switching away waits for a save in flight, and stays put when it fails', async () => { + resetIpc() + await store.getState().newConversation() + const id = store.getState().currentId + const quietError = console.error + console.error = () => {} + try { + const first = deferred() + ipc.save = () => first.promise + store.getState().setMessages([msg('hello')]) + const flushing = store.getState().flush() + await tick() + const switching = store.getState().newConversation() + await tick() + // Only the first write has started; the switch is waiting for it. + assert.deepEqual(calls, [`save ${id}`]) + + ipc.save = async () => { throw new Error('disk full') } + first.reject(new Error('disk full')) + await flushing + await switching + // The switch retried the write, which failed too: nothing is lost. + assert.deepEqual(calls, [`save ${id}`, `save ${id}`]) + assert.equal(store.getState().currentId, id) + assert.deepEqual(store.getState().messages, [msg('hello')]) + + ipc.save = async () => {} + await store.getState().newConversation() + assert.notEqual(store.getState().currentId, id) + assert.equal(store.getState().list[0]?.id, id) + } finally { + console.error = quietError + } +}) + +test('store: deleting the current conversation waits for its save and does not come back', async () => { + resetIpc() + await store.getState().newConversation() + const id = store.getState().currentId + const write = deferred() + ipc.save = () => write.promise + store.getState().setMessages([msg('to delete')]) + const flushing = store.getState().flush() + await tick() + const removing = store.getState().remove(id) + await tick() + assert.deepEqual(calls, [`save ${id}`]) + + write.resolve() + await flushing + await removing + assert.deepEqual(calls, [`save ${id}`, `remove ${id}`]) + assert.equal(store.getState().list.some((s) => s.id === id), false) + assert.notEqual(store.getState().currentId, id) + assert.deepEqual(store.getState().messages, []) +}) + +test('store: a failed delete keeps the pending changes', async () => { + resetIpc() + await store.getState().newConversation() + const id = store.getState().currentId + const quietError = console.error + console.error = () => {} + try { + ipc.remove = async () => { throw new Error('locked') } + store.getState().setMessages([msg('keep me')]) + await store.getState().remove(id) + assert.equal(store.getState().currentId, id) + + await store.getState().flush() + assert.deepEqual(calls, [`remove ${id}`, `save ${id}`]) + assert.equal(store.getState().list[0]?.id, id) + } finally { + console.error = quietError + } +}) diff --git a/src/shared/utils/agentConversations.ts b/src/shared/utils/agentConversations.ts index 06c8e3dd0..6f97943bd 100644 --- a/src/shared/utils/agentConversations.ts +++ b/src/shared/utils/agentConversations.ts @@ -47,6 +47,19 @@ export function upsertSummary(list: readonly ConversationSummary[], summary: Con return sortSummaries([...list.filter((s) => s.id !== summary.id), summary]) } +/** + * Runs async tasks one at a time, in call order. A failed task does not stop + * the ones queued after it; its caller still gets the rejection. + */ +export function createSerialQueue(): (task: () => Promise) => Promise { + let tail: Promise = Promise.resolve() + return (task: () => Promise): Promise => { + const run = tail.then(() => task()) + tail = run.catch(() => undefined) + return run + } +} + /** * A conversation saved mid-turn holds a streaming message and running tools. * Nothing is streaming once it is loaded again, so settle them. From 865365328e3c76f942cfe69982d991f25b1cd655 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Thu, 12 Nov 2026 22:03:55 +0100 Subject: [PATCH 051/108] feat(nav): page registry and grouped sidebar One registry (src/shared/pages/pageRegistry.ts) lists every page: id, label, group, icon, lazy component and wrapper classes. The sidebar, the page switch, the nav store's Page type and the agent all read it, so adding a page is one entry. - Sidebar sections Home, Create, Manage, System with separators; empty groups are skipped. - navigate(page, params?) opens a page on a target; openExtension is navigate('models', { extension }). A plain navigate clears old params. - The agent's navigate tool validates against the registry, and the page list reaches the backend in the context instead of a fixed enum. --- api/routers/agent.py | 15 ++- api/tests/test_agent_client_tools.py | 6 +- package.json | 2 +- src/areas/models/ModelsPage.tsx | 8 +- .../components/agent/agentContext.test.ts | 9 +- src/shared/components/agent/agentContext.ts | 5 + .../components/agent/clientTools.test.ts | 6 +- src/shared/components/agent/clientTools.ts | 9 +- src/shared/components/layout/Sidebar.tsx | 107 +++++++----------- src/shared/pages/pageRegistry.test.ts | 61 ++++++++++ src/shared/pages/pageRegistry.ts | 100 ++++++++++++++++ src/shared/router/Router.tsx | 14 ++- src/shared/router/routes.tsx | 19 ---- src/shared/stores/navStore.ts | 25 ++-- 14 files changed, 270 insertions(+), 116 deletions(-) create mode 100644 src/shared/pages/pageRegistry.test.ts create mode 100644 src/shared/pages/pageRegistry.ts delete mode 100644 src/shared/router/routes.tsx diff --git a/api/routers/agent.py b/api/routers/agent.py index 39e8cb8cb..62fdff064 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -400,7 +400,7 @@ def _function(name: str, description: str, properties: dict | None = None, requi "navigate", "Switch the app to another page. Use it ONLY when the user explicitly asks to go to a page; " "every other tool works from any page.", - {"page": {"type": "string", "enum": ["generate", "workflows", "models", "settings"]}}, + {"page": {"type": "string", "description": "A page id from the app context's Pages line."}}, ["page"], ), ] @@ -421,6 +421,11 @@ def _function(name: str, description: str, properties: dict | None = None, requi SERVER_TOOLS = [t for t in TOOLS if t["function"]["name"] not in CLIENT_TOOLS] +def _one_line(text: str, limit: int = 64) -> str: + """A context value from the app, on one line and capped.""" + return " ".join(text.split())[:limit] + + def _not_finished(tool: str, limit: float) -> str: return ( f"Error: {tool} was confirmed but did not finish within {limit:g} s. " @@ -1107,6 +1112,14 @@ async def _chat_events(request: AgentChatRequest, client_tools: bool = True): ctx_lines = [] if request.context.get("page"): ctx_lines.append(f"Current page: {request.context['page']}") + # From the app's page registry; kept short and on one line all the same. + pages = [ + (_one_line(p["id"]), _one_line(p.get("label") if isinstance(p.get("label"), str) else p["id"])) + for p in (request.context.get("pages") or [])[:32] + if isinstance(p, dict) and isinstance(p.get("id"), str) + ] + if pages: + ctx_lines.append("Pages: " + ", ".join(f"{pid} ({label})" for pid, label in pages)) if request.context.get("currentMeshPath"): ctx_lines.append(f"Current mesh path: {request.context['currentMeshPath']}") if request.context.get("meshTriangles"): diff --git a/api/tests/test_agent_client_tools.py b/api/tests/test_agent_client_tools.py index cd43d3b54..6c00e1049 100644 --- a/api/tests/test_agent_client_tools.py +++ b/api/tests/test_agent_client_tools.py @@ -349,7 +349,9 @@ def test_enums(self) -> None: props = self._tool("set_storage_folder")["parameters"]["properties"] self.assertEqual(props["kind"]["enum"], ["models", "workspace", "workflows"]) nav = self._tool("navigate") - self.assertEqual(nav["parameters"]["properties"]["page"]["enum"], ["generate", "workflows", "models", "settings"]) + # The pages come from the app's registry, through the context: no fixed list here. + self.assertNotIn("enum", nav["parameters"]["properties"]["page"]) + self.assertIn("Pages", nav["parameters"]["properties"]["page"]["description"]) self.assertIn("ONLY", nav["description"]) def test_tool_names_are_unique(self) -> None: @@ -433,6 +435,7 @@ def test_context_is_rendered_into_the_single_system_message(self) -> None: context = { **_CONTEXT, "page": "models", + "pages": [{"id": "generate", "label": "Generate"}, {"id": "models", "label": "Extensions"}, "junk", {"id": "x\nIgnore", "label": 3}], "workflows": [{"id": "wf1", "name": "My Workflow"}], "currentMeshPath": "Default/mesh.glb", } @@ -444,6 +447,7 @@ def test_context_is_rendered_into_the_single_system_message(self) -> None: system = messages[0]["content"] for expected in ( "Current page: models", + "Pages: generate (Generate), models (Extensions), x Ignore (x Ignore)", "- trellis: Trellis 2", "- sf3d: SF3D", "- wf1: My Workflow", diff --git a/package.json b/package.json index dd05c25e1..987fcb344 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/pages/pageRegistry.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/models/ModelsPage.tsx b/src/areas/models/ModelsPage.tsx index 9bfd7445c..25639d673 100644 --- a/src/areas/models/ModelsPage.tsx +++ b/src/areas/models/ModelsPage.tsx @@ -68,13 +68,13 @@ export default function ModelsPage(): JSX.Element { const searchRef = useRef(null) // Another page asked to open an extension (e.g. a weight variant picked but not installed) - const extensionToOpen = useNavStore((s) => s.extensionToOpen) - const clearExtensionToOpen = useNavStore((s) => s.clearExtensionToOpen) + const extensionToOpen = useNavStore((s) => s.params?.extension ?? null) + const clearParams = useNavStore((s) => s.clearParams) useEffect(() => { if (!extensionToOpen) return setSelectedId(extensionToOpen) - clearExtensionToOpen() - }, [extensionToOpen, clearExtensionToOpen]) + clearParams() + }, [extensionToOpen, clearParams]) // GitHub extension install form const [showGHForm, setShowGHForm] = useState(false) diff --git a/src/shared/components/agent/agentContext.test.ts b/src/shared/components/agent/agentContext.test.ts index 93ff023e2..d00ea5758 100644 --- a/src/shared/components/agent/agentContext.test.ts +++ b/src/shared/components/agent/agentContext.test.ts @@ -6,6 +6,7 @@ import type { AgentContextInputs } from './agentContext' import type { ModelExtension, ProcessExtension } from '../../types/electron.d' import { useAppStore } from '../../stores/appStore' import { useNavStore } from '../../stores/navStore' +import { PAGES } from '../../pages/pageRegistry' import { useWorkflowsStore } from '../../stores/workflowsStore' import { useExtensionsStore } from '../../stores/extensionsStore' @@ -19,8 +20,11 @@ const processExt = { } as unknown as ProcessExtension const STORAGE = { modelsDir: 'M', workspaceDir: 'W', workflowsDir: 'F', extensionsDir: 'E' } +const PAGE_LIST = [{ id: 'generate', label: 'Generate' }, { id: 'models', label: 'Extensions' }] + const inputs = (over: Partial = {}): AgentContextInputs => ({ page: 'models', + pages: PAGE_LIST, currentMeshUrl: '/workspace/Default/mesh.glb', meshTriangles: 1200, workflows: [{ id: 'wf1', name: 'Turntable' }], @@ -35,6 +39,7 @@ const inputs = (over: Partial = {}): AgentContextInputs => ( test('the context carries the page, mesh, workflows, extensions, models, storage and UI options', () => { assert.deepEqual(assembleAgentContext(inputs()), { page: 'models', + pages: PAGE_LIST, currentMeshPath: 'Default/mesh.glb', meshTriangles: 1200, workflows: [{ id: 'wf1', name: 'Turntable' }], @@ -52,7 +57,7 @@ test('empty or unreadable parts are left out, as buildContext did', () => { const ctx = assembleAgentContext(inputs({ currentMeshUrl: undefined, meshTriangles: null, workflows: [], modelExtensions: [], processExtensions: [], models: null, storage: null, })) - assert.deepEqual(Object.keys(ctx), ['page', 'ui']) + assert.deepEqual(Object.keys(ctx), ['page', 'pages', 'ui']) }) test('storage keeps only the four folders, never the HF token', () => { @@ -73,6 +78,8 @@ test('buildAgentContext reads the stores and the async sources', async () => { assert.deepEqual(ctx, { page: 'workflows', + // Every registered page, by id and sidebar label. + pages: PAGES.map((p) => ({ id: p.id, label: p.label })), workflows: [{ id: 'w', name: 'W' }], extensions: [{ id: 'triposr/main', name: 'MAIN', input: 'image', output: 'mesh' }], models: [{ id: 'triposr/main', name: 'TripoSR' }], diff --git a/src/shared/components/agent/agentContext.ts b/src/shared/components/agent/agentContext.ts index a13a06de8..79c078d93 100644 --- a/src/shared/components/agent/agentContext.ts +++ b/src/shared/components/agent/agentContext.ts @@ -2,6 +2,7 @@ import type { ModelExtension, ProcessExtension } from '../../types/electron.d' import type { Workflow } from '../../types/electron.d' import type { UiScale } from '../../stores/appStore' import type { Page } from '../../stores/navStore' +import { PAGES } from '../../pages/pageRegistry' import { useAppStore } from '../../stores/appStore' import { useExtensionsStore } from '../../stores/extensionsStore' import { useWorkflowsStore } from '../../stores/workflowsStore' @@ -13,6 +14,8 @@ import { buildAllWorkflowExtensions } from '../../../areas/workflows/mockExtensi export interface AgentContextInputs { page: Page + /** Every page the app has, for the navigate tool. */ + pages: readonly { id: string; label: string }[] currentMeshUrl?: string | null meshTriangles?: number | null workflows: Pick[] @@ -28,6 +31,7 @@ export interface AgentContextInputs { /** Pure assembly of the context from already-read state. */ export function assembleAgentContext(inputs: AgentContextInputs): Record { const ctx: Record = { page: inputs.page } + ctx.pages = inputs.pages.map((p) => ({ id: p.id, label: p.label })) if (inputs.currentMeshUrl) ctx.currentMeshPath = inputs.currentMeshUrl.replace('/workspace/', '') if (inputs.meshTriangles) ctx.meshTriangles = inputs.meshTriangles if (inputs.workflows.length > 0) ctx.workflows = inputs.workflows.map((w) => ({ id: w.id, name: w.name })) @@ -72,6 +76,7 @@ export async function buildAgentContext(sources: AgentContextSources = defaultSo ]) return assembleAgentContext({ page: useNavStore.getState().currentPage, + pages: PAGES, currentMeshUrl: app.currentJob?.outputUrl, meshTriangles: app.meshStats?.triangles, workflows: useWorkflowsStore.getState().workflows, diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index e6cb9ecde..3584d0326 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -335,7 +335,7 @@ test('undo_mesh and redo_mesh step through the history, and say when there is no test('navigate changes page and validates it', async () => { const f = fake() - assert.equal(await run('navigate', { page: 'models' }, f), 'Opened the models page.') + assert.equal(await run('navigate', { page: 'models' }, f), 'Opened the Extensions page.') assert.deepEqual(f.calls, [['navigate', 'models']]) assert.match(await run('navigate', { page: 'home' }, f), /^Error: page must be one of: generate, workflows, models, settings/) }) @@ -355,8 +355,8 @@ const event = (tool: string, args: Record, confirm = false) => test('runClientTool runs the tool and posts the result with its call_id', async () => { const f = fake() const outcome = await runClientTool(event('navigate', { page: 'settings' }), f.deps, async () => { throw new Error('not asked') }) - assert.deepEqual(f.posted, [{ call_id: 'call-7', result: 'Opened the settings page.' }]) - assert.deepEqual(outcome, { description: 'Go to the settings page', result: 'Opened the settings page.', error: false, declined: false, posted: true }) + assert.deepEqual(f.posted, [{ call_id: 'call-7', result: 'Opened the Settings page.' }]) + assert.deepEqual(outcome, { description: 'Go to the Settings page', result: 'Opened the Settings page.', error: false, declined: false, posted: true }) }) test('runClientTool asks first when confirm is set, and runs on Confirm', async () => { diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts index 526873e64..7be3c55fc 100644 --- a/src/shared/components/agent/clientTools.ts +++ b/src/shared/components/agent/clientTools.ts @@ -2,6 +2,7 @@ import type { AnyExtension, ExtensionNode, Workflow } from '../../types/electron import type { AgentActionPayload, ClientToolEvent } from '../../utils/agentStream' import type { GenerationOptions, UiScale } from '../../stores/appStore' import type { Page } from '../../stores/navStore' +import { PAGE_IDS, getPage, isPage } from '../../pages/pageRegistry' import type { GenerationApi, GenerationStoreActions } from '../../utils/generationRunner' import { createGenerationApi, runGeneration } from '../../utils/generationRunner' import { useAppStore } from '../../stores/appStore' @@ -534,11 +535,11 @@ export const clientTools: Record = { }), navigate: defineTool({ - describe: (args) => `Go to the ${String(args.page ?? '?')} page`, - check: (args) => (PAGES.includes(args.page as Page) ? null : `page must be one of: ${PAGES.join(', ')}.`), + describe: (args) => `Go to the ${isPage(args.page) ? getPage(args.page).label : String(args.page ?? '?')} page`, + check: (args) => (isPage(args.page) ? null : `page must be one of: ${PAGE_IDS.join(', ')}.`), async execute(args, deps) { deps.navigate(args.page as Page) - return `Opened the ${String(args.page)} page.` + return `Opened the ${getPage(args.page as Page).label} page.` }, }), } @@ -551,8 +552,6 @@ const SETTING_LABELS: Record = { const UI_SCALES: UiScale[] = ['small', 'medium', 'large', 'very-large'] -const PAGES: Page[] = ['generate', 'workflows', 'models', 'settings'] - function toBool(value: unknown): boolean | null { if (typeof value === 'boolean') return value if (value === 'true') return true diff --git a/src/shared/components/layout/Sidebar.tsx b/src/shared/components/layout/Sidebar.tsx index 767b2b017..b8b7740bb 100644 --- a/src/shared/components/layout/Sidebar.tsx +++ b/src/shared/components/layout/Sidebar.tsx @@ -1,76 +1,49 @@ -import { useNavStore, type Page } from '@shared/stores/navStore' +import { createElement, Fragment } from 'react' +import { useNavStore } from '@shared/stores/navStore' +import { pagesByGroup, type IconShape, type Page } from '@shared/pages/pageRegistry' -const NAV_ITEMS: { id: Page; label: string; icon: JSX.Element }[] = [ - { - id: 'generate', - label: 'Generate', - icon: ( - - - - ) - }, - { - id: 'workflows', - label: 'Workflows', - icon: ( - - - - - - - - ) - }, - { - id: 'models', - label: 'Extensions', - icon: ( - - - - - - ) - }, - { - id: 'settings', - label: 'Settings', - icon: ( - - - - - ) - } -] +const SECTIONS = pagesByGroup() + +function PageIcon({ shapes }: { shapes: readonly IconShape[] }): JSX.Element { + return ( + + {shapes.map(([tag, attrs], i) => createElement(tag, { key: i, ...attrs }))} + + ) +} export default function Sidebar(): JSX.Element { - const { currentPage, navigate } = useNavStore() + const currentPage = useNavStore((s) => s.currentPage) + const navigate = useNavStore((s) => s.navigate) return ( ) } diff --git a/src/shared/pages/pageRegistry.test.ts b/src/shared/pages/pageRegistry.test.ts new file mode 100644 index 000000000..97b7f514c --- /dev/null +++ b/src/shared/pages/pageRegistry.test.ts @@ -0,0 +1,61 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { PAGES, PAGE_GROUPS, PAGE_IDS, getPage, isPage, pagesByGroup, type PageDefinition } from './pageRegistry' +import { useNavStore } from '../stores/navStore' + +const page = (id: string, group: PageDefinition['group']): PageDefinition => ({ + id, label: id, group, icon: [], load: async () => ({ default: () => null }), wrapperClass: '', +}) + +test('page ids are unique and every page has a known group, a label and an icon', () => { + assert.equal(new Set(PAGE_IDS).size, PAGES.length) + for (const p of PAGES) { + assert.ok(PAGE_GROUPS.includes(p.group), p.id) + assert.ok(p.label, p.id) + assert.ok(p.icon.length > 0, p.id) + assert.equal(typeof p.load, 'function', p.id) + } +}) + +test('the four existing pages keep their ids, labels and sidebar order', () => { + assert.deepEqual( + pagesByGroup().map((s) => [s.group, s.pages.map((p) => `${p.id}:${p.label}`)]), + [ + ['create', ['generate:Generate', 'workflows:Workflows']], + ['system', ['models:Extensions', 'settings:Settings']], + ], + ) +}) + +test('pagesByGroup follows PAGE_GROUPS order, keeps registry order inside a group, and skips empty groups', () => { + const sections = pagesByGroup([page('b', 'system'), page('a', 'create'), page('h', 'home'), page('c', 'create')]) + assert.deepEqual(sections.map((s) => [s.group, s.pages.map((p) => p.id)]), [ + ['home', ['h']], + ['create', ['a', 'c']], + ['system', ['b']], + ]) +}) + +test('isPage and getPage', () => { + assert.ok(isPage('models')) + assert.ok(!isPage('nope')) + assert.ok(!isPage(undefined)) + assert.equal(getPage('models').label, 'Extensions') +}) + +test('navigate sets the page and its params, and a plain navigate clears stale params', () => { + const nav = useNavStore.getState() + nav.navigate('workflows', { run: 'r1' }) + assert.equal(useNavStore.getState().currentPage, 'workflows') + assert.deepEqual(useNavStore.getState().params, { run: 'r1' }) + nav.navigate('settings') + assert.equal(useNavStore.getState().params, null) +}) + +test('openExtension is navigate to the Extensions page on that extension', () => { + useNavStore.getState().openExtension('hunyuan') + assert.equal(useNavStore.getState().currentPage, 'models') + assert.deepEqual(useNavStore.getState().params, { extension: 'hunyuan' }) + useNavStore.getState().clearParams() + assert.equal(useNavStore.getState().params, null) +}) diff --git a/src/shared/pages/pageRegistry.ts b/src/shared/pages/pageRegistry.ts new file mode 100644 index 000000000..405c1395a --- /dev/null +++ b/src/shared/pages/pageRegistry.ts @@ -0,0 +1,100 @@ +import type { ComponentType } from 'react' + +// The app's pages, in one place. The sidebar, the page switch, the navigation +// store and the agent all read this list: adding a page is one entry here. +// +// Plain data on purpose (no JSX, components loaded on demand), so the stores +// and the agent's tools can import it, and so can the node tests. + +export type PageGroup = 'home' | 'create' | 'manage' | 'system' + +/** Sidebar order; a group with no page is skipped. */ +export const PAGE_GROUPS: readonly PageGroup[] = ['home', 'create', 'manage', 'system'] + +/** One child of a 24×24 stroked SVG icon, as [tag, attributes]. */ +export type IconShape = readonly [ + tag: 'path' | 'rect' | 'circle' | 'line' | 'polyline' | 'polygon', + attrs: Readonly>, +] + +export interface PageDefinition { + id: string + /** Sidebar label and tooltip. */ + label: string + group: PageGroup + icon: readonly IconShape[] + /** The page component, loaded the first time the page opens. */ + load: () => Promise<{ default: ComponentType }> + /** Classes of the element the page renders into. */ + wrapperClass: string +} + +export const PAGES = [ + { + id: 'generate', + label: 'Generate', + group: 'create', + icon: [['path', { d: 'M12 2L2 7l10 5 10-5-10-5zM2 17l10 5 10-5M2 12l10 5 10-5' }]], + load: () => import('@areas/generate/GeneratePage'), + wrapperClass: 'flex flex-1 overflow-hidden', + }, + { + id: 'workflows', + label: 'Workflows', + group: 'create', + icon: [ + ['rect', { x: 3, y: 3, width: 6, height: 5, rx: 1 }], + ['rect', { x: 3, y: 11, width: 6, height: 5, rx: 1 }], + ['rect', { x: 3, y: 19, width: 6, height: 2, rx: 1 }], + ['path', { d: 'M9 5.5h3.5a1 1 0 0 1 1 1v11a1 1 0 0 1-1 1H9' }], + ['rect', { x: 13.5, y: 3, width: 7.5, height: 16, rx: 1 }], + ], + load: () => import('@areas/workflows/WorkflowsPage'), + wrapperClass: 'flex flex-1 overflow-hidden', + }, + { + id: 'models', + label: 'Extensions', + group: 'system', + icon: [ + ['path', { d: 'M21 16V8a2 2 0 0 0-1-1.73l-7-4a2 2 0 0 0-2 0l-7 4A2 2 0 0 0 3 8v8a2 2 0 0 0 1 1.73l7 4a2 2 0 0 0 2 0l7-4A2 2 0 0 0 21 16z' }], + ['polyline', { points: '3.27 6.96 12 12.01 20.73 6.96' }], + ['line', { x1: 12, y1: 22.08, x2: 12, y2: 12 }], + ], + load: () => import('@areas/models/ModelsPage'), + wrapperClass: 'flex-1 overflow-y-auto', + }, + { + id: 'settings', + label: 'Settings', + group: 'system', + icon: [ + ['circle', { cx: 12, cy: 12, r: 3 }], + ['path', { d: 'M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1-2.83 2.83l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-4 0v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83-2.83l.06-.06A1.65 1.65 0 0 0 4.68 15a1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1 0-4h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 2.83-2.83l.06.06A1.65 1.65 0 0 0 9 4.68a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 4 0v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 2.83l-.06.06A1.65 1.65 0 0 0 19.4 9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 0 4h-.09a1.65 1.65 0 0 0-1.51 1z' }], + ], + load: () => import('@areas/settings/SettingsPage'), + wrapperClass: 'flex-1 overflow-hidden', + }, +] as const satisfies readonly PageDefinition[] + +export type Page = (typeof PAGES)[number]['id'] + +export const PAGE_IDS: readonly Page[] = PAGES.map((p) => p.id) + +export function isPage(value: unknown): value is Page { + return typeof value === 'string' && (PAGE_IDS as readonly string[]).includes(value) +} + +export function getPage(id: Page): PageDefinition { + // Every Page id comes from PAGES, so the lookup always finds one. + return PAGES.find((p) => p.id === id) as PageDefinition +} + +/** The sidebar's sections: each group that has pages, in PAGE_GROUPS order. */ +export function pagesByGroup( + pages: readonly PageDefinition[] = PAGES, +): { group: PageGroup; pages: PageDefinition[] }[] { + return PAGE_GROUPS + .map((group) => ({ group, pages: pages.filter((p) => p.group === group) })) + .filter((section) => section.pages.length > 0) +} diff --git a/src/shared/router/Router.tsx b/src/shared/router/Router.tsx index 0b4f31df2..2a44ea6a8 100644 --- a/src/shared/router/Router.tsx +++ b/src/shared/router/Router.tsx @@ -1,15 +1,19 @@ -import { Suspense } from 'react' +import { Suspense, lazy, type ComponentType, type LazyExoticComponent } from 'react' import { useNavStore } from '@shared/stores/navStore' -import { ROUTES } from './routes' +import { PAGES, getPage, type Page, type PageDefinition } from '@shared/pages/pageRegistry' + +// One lazy component per registered page, made once. +const COMPONENTS = {} as Record> +for (const page of PAGES as readonly PageDefinition[]) COMPONENTS[page.id as Page] = lazy(page.load) export default function Router(): JSX.Element { const currentPage = useNavStore((s) => s.currentPage) - const { component: Page, wrapperClass } = ROUTES[currentPage] + const PageComponent = COMPONENTS[currentPage] return ( -
- +
+
) diff --git a/src/shared/router/routes.tsx b/src/shared/router/routes.tsx deleted file mode 100644 index 732920bfe..000000000 --- a/src/shared/router/routes.tsx +++ /dev/null @@ -1,19 +0,0 @@ -import { lazy } from 'react' -import type { Page } from '@shared/stores/navStore' - -const GeneratePage = lazy(() => import('@areas/generate/GeneratePage')) -const WorkflowsPage = lazy(() => import('@areas/workflows/WorkflowsPage')) -const ModelsPage = lazy(() => import('@areas/models/ModelsPage')) -const SettingsPage = lazy(() => import('@areas/settings/SettingsPage')) - -export interface RouteConfig { - component: React.ComponentType - wrapperClass: string -} - -export const ROUTES: Record = { - generate: { component: GeneratePage, wrapperClass: 'flex flex-1 overflow-hidden' }, - workflows: { component: WorkflowsPage, wrapperClass: 'flex flex-1 overflow-hidden' }, - models: { component: ModelsPage, wrapperClass: 'flex-1 overflow-y-auto' }, - settings: { component: SettingsPage, wrapperClass: 'flex-1 overflow-hidden' }, -} diff --git a/src/shared/stores/navStore.ts b/src/shared/stores/navStore.ts index 62afa2f58..6afcdfdac 100644 --- a/src/shared/stores/navStore.ts +++ b/src/shared/stores/navStore.ts @@ -1,19 +1,26 @@ import { create } from 'zustand' +// Relative and type-only, so the store loads under `node --test`. +import type { Page } from '../pages/pageRegistry' -export type Page = 'generate' | 'workflows' | 'models' | 'settings' +export type { Page } + +/** What a page is opened on, e.g. `{ extension: 'hunyuan' }` for the Extensions page. */ +export type PageParams = Readonly> interface NavState { currentPage: Page - extensionToOpen: string | null - navigate: (page: Page) => void + /** Target of the last navigate() call; the page reads it, then clears it. */ + params: PageParams | null + navigate: (page: Page, params?: PageParams) => void + /** Opens the Extensions page on one extension. */ openExtension: (extensionId: string) => void - clearExtensionToOpen: () => void + clearParams: () => void } -export const useNavStore = create((set) => ({ +export const useNavStore = create((set, get) => ({ currentPage: 'generate', - extensionToOpen: null, - navigate: (page) => set({ currentPage: page }), - openExtension: (extensionId) => set({ currentPage: 'models', extensionToOpen: extensionId }), - clearExtensionToOpen: () => set({ extensionToOpen: null }), + params: null, + navigate: (page, params) => set({ currentPage: page, params: params ?? null }), + openExtension: (extensionId) => get().navigate('models', { extension: extensionId }), + clearParams: () => set({ params: null }), })) From 90d678709824506a630bfc805dc32384ee013fda Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Fri, 13 Nov 2026 10:01:30 +0100 Subject: [PATCH 052/108] feat(library): cache asset thumbnails per workspace path and file version Thumbnails live in a Modly-owned folder keyed by a hash of workspace dir + path, with the file's mtime and size in the name, so an edited mesh misses the cache. Library entries carry thumbnailUrl when the current version is cached, and pruning drops thumbnails of deleted or changed meshes. --- electron/main/artifact-registry-service.ts | 25 +- electron/main/asset-thumbnail-cache.test.ts | 151 +++++++++++ electron/main/asset-thumbnail-cache.ts | 262 ++++++++++++++++++++ package.json | 2 +- src/shared/types/assetLibrary.ts | 23 ++ 5 files changed, 456 insertions(+), 7 deletions(-) create mode 100644 electron/main/asset-thumbnail-cache.test.ts create mode 100644 electron/main/asset-thumbnail-cache.ts diff --git a/electron/main/artifact-registry-service.ts b/electron/main/artifact-registry-service.ts index 2acb657fb..9a4a14732 100644 --- a/electron/main/artifact-registry-service.ts +++ b/electron/main/artifact-registry-service.ts @@ -14,6 +14,7 @@ import type { AssetLibrarySourceScope, } from '../../src/shared/types/assetLibrary' import type { ArtifactProvenance } from '../../src/shared/types/artifacts' +import { findCachedThumbnailUrl, pruneThumbnails } from './asset-thumbnail-cache' const WINDOWS_ABSOLUTE_PATH = /^[a-zA-Z]:[\\/]/ const ENCODED_ESCAPE_PATTERN = /%2e|%2f|%5c/i @@ -44,6 +45,8 @@ export interface NormalizedWorkspaceAssetPath { export interface WorkspaceAssetLibraryRequest { workspaceDir: string + /** Modly-owned thumbnail cache; when set, renderable entries carry their cached `thumbnailUrl`. */ + thumbnailsDir?: string } export interface WorkspaceAssetLibraryReadRequest extends WorkspaceAssetLibraryRequest { @@ -67,6 +70,7 @@ export interface IpcMainLike { export interface WorkspaceAssetLibraryIpcDeps { ipcMain: IpcMainLike getWorkspaceDir: () => string + thumbnailsDir?: string } function libraryError(code: AssetLibraryError['code'], message: string): AssetLibraryError { @@ -223,11 +227,14 @@ async function hasRigMetadata(absolutePath: string): Promise { } } -async function buildEntry(workspaceDir: string, workspacePath: string): Promise { +async function buildEntry(workspaceDir: string, workspacePath: string, thumbnailsDir?: string): Promise { const { absolutePath } = normalizeWorkspaceAssetPath(workspaceDir, workspacePath) const stats = await stat(absolutePath) const classification = classifyAssetLibraryCandidate({ workspacePath, hasRigMetadata: await hasRigMetadata(absolutePath) }) const metadata = await readMetadata(workspaceDir, workspacePath, absolutePath) + const thumbnailUrl = thumbnailsDir && classification.previewKind === '3d-model' + ? await findCachedThumbnailUrl({ workspaceDir, thumbnailsDir }, workspacePath, stats) + : undefined const manifestCapability = metadata.manifestWorkspacePath ? manifestCapabilityFor(metadata.manifestWorkspacePath) : undefined return { id: `library:${workspacePath}`, @@ -247,6 +254,7 @@ async function buildEntry(workspaceDir: string, workspacePath: string): Promise< nonOpenableReason: classification.nonOpenableReason, createdAt: (stats.birthtime.getTime() > 0 ? stats.birthtime : stats.mtime).toISOString(), updatedAt: stats.mtime.toISOString(), + thumbnailUrl, } } @@ -278,7 +286,7 @@ async function collectFiles(workspaceDir: string, rootName: typeof ALLOWED_ROOTS export async function listWorkspaceAssetLibrary(request: WorkspaceAssetLibraryRequest): Promise { try { const workspacePaths = (await Promise.all(ALLOWED_ROOTS.map((root) => collectFiles(request.workspaceDir, root)))).flat().sort() - const entries = await Promise.all(workspacePaths.map((workspacePath) => buildEntry(request.workspaceDir, workspacePath))) + const entries = await Promise.all(workspacePaths.map((workspacePath) => buildEntry(request.workspaceDir, workspacePath, request.thumbnailsDir))) return { success: true, entries: entries.filter((entry) => entry.state !== 'unsupported') } } catch (error) { return { success: false, error: libraryError('list-failed', error instanceof Error ? error.message : String(error)) } @@ -304,7 +312,7 @@ export async function readWorkspaceAssetLibraryEntry(request: WorkspaceAssetLibr const normalized = normalizeWorkspaceAssetPath(request.workspaceDir, request.workspacePath) const sourceValidation = await validateRequestedSource(request.workspaceDir, normalized.workspacePath, request.sourceWorkspacePath) if (!sourceValidation.success) return { success: false, error: sourceValidation.error } - const entry = await buildEntry(request.workspaceDir, normalized.workspacePath) + const entry = await buildEntry(request.workspaceDir, normalized.workspacePath, request.thumbnailsDir) return { success: true, entry, preview: await previewEntry(normalized.absolutePath, entry) } } catch (error) { return { success: false, error: libraryError('unsafe-path', error instanceof Error ? error.message : String(error)) } @@ -353,15 +361,20 @@ function readPayloadRequest(payload: unknown): { workspacePath?: string, sourceW } export function registerWorkspaceAssetLibraryIpcHandlers(deps: WorkspaceAssetLibraryIpcDeps): void { - deps.ipcMain.handle('workspace:library:list', async () => listWorkspaceAssetLibrary({ workspaceDir: deps.getWorkspaceDir() })) + deps.ipcMain.handle('workspace:library:list', async () => { + const result = await listWorkspaceAssetLibrary({ workspaceDir: deps.getWorkspaceDir(), thumbnailsDir: deps.thumbnailsDir }) + // Listing is the natural sweep point: drop thumbnails of deleted or changed meshes in the background. + if (deps.thumbnailsDir) void pruneThumbnails(deps.thumbnailsDir).catch(() => undefined) + return result + }) deps.ipcMain.handle('workspace:library:read', async (_event, payload) => { const { workspacePath, sourceWorkspacePath } = readPayloadRequest(payload) if (!workspacePath) return { success: false, error: libraryError('invalid-request', 'workspacePath is required.') } - return readWorkspaceAssetLibraryEntry({ workspaceDir: deps.getWorkspaceDir(), workspacePath, sourceWorkspacePath }) + return readWorkspaceAssetLibraryEntry({ workspaceDir: deps.getWorkspaceDir(), thumbnailsDir: deps.thumbnailsDir, workspacePath, sourceWorkspacePath }) }) deps.ipcMain.handle('workspace:library:open', async (_event, payload) => { const { workspacePath, sourceWorkspacePath } = readPayloadRequest(payload) if (!workspacePath) return { success: false, error: libraryError('invalid-request', 'workspacePath is required.') } - return openWorkspaceAssetLibraryEntry({ workspaceDir: deps.getWorkspaceDir(), workspacePath, sourceWorkspacePath }) + return openWorkspaceAssetLibraryEntry({ workspaceDir: deps.getWorkspaceDir(), thumbnailsDir: deps.thumbnailsDir, workspacePath, sourceWorkspacePath }) }) } diff --git a/electron/main/asset-thumbnail-cache.test.ts b/electron/main/asset-thumbnail-cache.test.ts new file mode 100644 index 000000000..d746d64ab --- /dev/null +++ b/electron/main/asset-thumbnail-cache.test.ts @@ -0,0 +1,151 @@ +import assert from 'node:assert/strict' +import { mkdir, mkdtemp, readdir, rm, utimes, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import test from 'node:test' + +import { + getThumbnailStatus, + normalizeThumbnailSourcePath, + pruneThumbnails, + removeThumbnails, + resolveThumbnailUrl, + saveThumbnail, + thumbnailPathKey, + thumbnailVersion, +} from './asset-thumbnail-cache.ts' +import { listWorkspaceAssetLibrary } from './artifact-registry-service.ts' + +const WEBP = new Uint8Array([...Buffer.from('RIFF'), 4, 0, 0, 0, ...Buffer.from('WEBPVP8 ')]) + +async function withCache(run: (context: { workspaceDir: string, thumbnailsDir: string }) => Promise) { + const root = await mkdtemp(path.join(tmpdir(), 'modly-thumbs-')) + try { + const context = { workspaceDir: path.join(root, 'workspace'), thumbnailsDir: path.join(root, 'thumbnails') } + await mkdir(path.join(context.workspaceDir, 'Workflows'), { recursive: true }) + await run(context) + } finally { + await rm(root, { recursive: true, force: true }) + } +} + +async function writeMesh(workspaceDir: string, workspacePath: string, content = 'glb', mtime = new Date('2026-01-01T00:00:00Z')) { + const absolute = path.join(workspaceDir, ...workspacePath.split('/')) + await mkdir(path.dirname(absolute), { recursive: true }) + await writeFile(absolute, content) + await utimes(absolute, mtime, mtime) +} + +test('cache key depends on workspace dir and path; version on mtime and size', () => { + assert.equal(thumbnailPathKey('/a', 'Workflows/hero.glb'), thumbnailPathKey('/a', 'Workflows/hero.glb')) + assert.notEqual(thumbnailPathKey('/a', 'Workflows/hero.glb'), thumbnailPathKey('/b', 'Workflows/hero.glb')) + assert.notEqual(thumbnailPathKey('/a', 'Workflows/hero.glb'), thumbnailPathKey('/a', 'Workflows/other.glb')) + assert.match(thumbnailPathKey('/a', 'Workflows/hero.glb'), /^[0-9a-f]{32}$/) + assert.equal(thumbnailVersion({ mtimeMs: 1000.7, size: 10 }), thumbnailVersion({ mtimeMs: 1000.2, size: 10 })) + assert.notEqual(thumbnailVersion({ mtimeMs: 1000, size: 10 }), thumbnailVersion({ mtimeMs: 2000, size: 10 })) + assert.notEqual(thumbnailVersion({ mtimeMs: 1000, size: 10 }), thumbnailVersion({ mtimeMs: 1000, size: 11 })) +}) + +test('only safe workspace-relative .glb/.gltf paths are thumbnail sources', () => { + assert.equal(normalizeThumbnailSourcePath('/ws', 'Default\\hero.glb').workspacePath, 'Default/hero.glb') + assert.equal(normalizeThumbnailSourcePath('/ws', 'Workflows/./a/hero.GLTF').workspacePath, 'Workflows/a/hero.GLTF') + assert.throws(() => normalizeThumbnailSourcePath('/ws', '../hero.glb')) + assert.throws(() => normalizeThumbnailSourcePath('/ws', '/tmp/hero.glb')) + assert.throws(() => normalizeThumbnailSourcePath('/ws', 'C:\\hero.glb')) + assert.throws(() => normalizeThumbnailSourcePath('/ws', 'Workflows/%2e%2e/hero.glb')) + assert.throws(() => normalizeThumbnailSourcePath('/ws', 'Workflows/notes.json')) +}) + +test('thumbnail URLs resolve only to cache file names', () => { + const name = `${'a'.repeat(32)}.abc-1f-r1.webp` + assert.equal(resolveThumbnailUrl('/cache', `modly-thumb://thumbnails/${name}`), path.join('/cache', name)) + assert.equal(resolveThumbnailUrl('/cache', 'modly-thumb://thumbnails/../secret.webp'), null) + assert.equal(resolveThumbnailUrl('/cache', `modly-thumb://other/${name}`), null) + assert.equal(resolveThumbnailUrl('/cache', `file:///cache/${name}`), null) + assert.equal(resolveThumbnailUrl('/cache', 'modly-thumb://thumbnails/x.json'), null) +}) + +test('a saved thumbnail is served for its version and goes stale when the file changes', () => withCache(async (context) => { + await writeMesh(context.workspaceDir, 'Workflows/hero.glb') + const before = await getThumbnailStatus(context, 'Workflows/hero.glb') + assert.ok(before.success) + assert.equal(before.thumbnailUrl, undefined) + + const saved = await saveThumbnail(context, { workspacePath: 'Workflows/hero.glb', version: before.version, image: WEBP }) + assert.ok(saved.success) + assert.ok(resolveThumbnailUrl(context.thumbnailsDir, saved.thumbnailUrl)) + const cached = await getThumbnailStatus(context, 'Workflows/hero.glb') + assert.ok(cached.success) + assert.equal(cached.thumbnailUrl, saved.thumbnailUrl) + + await writeMesh(context.workspaceDir, 'Workflows/hero.glb', 'edited glb', new Date('2026-02-01T00:00:00Z')) + const edited = await getThumbnailStatus(context, 'Workflows/hero.glb') + assert.ok(edited.success) + assert.equal(edited.thumbnailUrl, undefined) + assert.notEqual(edited.version, before.version) + + // A render of the old version is refused; the new version replaces the old file. + const stale = await saveThumbnail(context, { workspacePath: 'Workflows/hero.glb', version: before.version, image: WEBP }) + assert.equal(!stale.success && stale.error.code, 'stale') + const resaved = await saveThumbnail(context, { workspacePath: 'Workflows/hero.glb', version: edited.version, image: WEBP }) + assert.ok(resaved.success) + const webps = (await readdir(context.thumbnailsDir)).filter((name) => name.endsWith('.webp')) + assert.equal(webps.length, 1) +})) + +test('save rejects non-WebP payloads and unsafe paths', () => withCache(async (context) => { + await writeMesh(context.workspaceDir, 'Workflows/hero.glb') + const status = await getThumbnailStatus(context, 'Workflows/hero.glb') + assert.ok(status.success) + const png = await saveThumbnail(context, { workspacePath: 'Workflows/hero.glb', version: status.version, image: new Uint8Array([0x89, 0x50, 0x4e, 0x47]) }) + assert.equal(!png.success && png.error.code, 'invalid-image') + const notBytes = await saveThumbnail(context, { workspacePath: 'Workflows/hero.glb', version: status.version, image: 'RIFF' }) + assert.equal(!notBytes.success && notBytes.error.code, 'invalid-image') + const unsafe = await saveThumbnail(context, { workspacePath: '../hero.glb', version: status.version, image: WEBP }) + assert.equal(!unsafe.success && unsafe.error.code, 'unsafe-path') + const missing = await getThumbnailStatus(context, 'Workflows/missing.glb') + assert.equal(!missing.success && missing.error.code, 'not-found') +})) + +test('removeThumbnails and pruning drop thumbnails of deleted assets', () => withCache(async (context) => { + await writeMesh(context.workspaceDir, 'Workflows/kept.glb') + await writeMesh(context.workspaceDir, 'Workflows/deleted.glb') + await writeMesh(context.workspaceDir, 'Default/removed.glb') + for (const workspacePath of ['Workflows/kept.glb', 'Workflows/deleted.glb', 'Default/removed.glb']) { + const status = await getThumbnailStatus(context, workspacePath) + assert.ok(status.success) + assert.ok((await saveThumbnail(context, { workspacePath, version: status.version, image: WEBP })).success) + } + + await removeThumbnails(context, 'Default/removed.glb') + assert.equal((await readdir(context.thumbnailsDir)).length, 4) + + await rm(path.join(context.workspaceDir, 'Workflows', 'deleted.glb')) + await writeFile(path.join(context.thumbnailsDir, `${'b'.repeat(32)}.abc-1-r1.webp`), WEBP) + await pruneThumbnails(context.thumbnailsDir) + const remaining = await readdir(context.thumbnailsDir) + assert.equal(remaining.length, 2) + const kept = await getThumbnailStatus(context, 'Workflows/kept.glb') + assert.ok(kept.success && kept.thumbnailUrl) +})) + +test('library entries expose thumbnailUrl only for cached renders of the current version', () => withCache(async (context) => { + await writeMesh(context.workspaceDir, 'Workflows/hero.glb') + await writeMesh(context.workspaceDir, 'Workflows/other.glb') + await writeFile(path.join(context.workspaceDir, 'Workflows', 'hero.world.json'), '{}') + const status = await getThumbnailStatus(context, 'Workflows/hero.glb') + assert.ok(status.success) + const saved = await saveThumbnail(context, { workspacePath: 'Workflows/hero.glb', version: status.version, image: WEBP }) + assert.ok(saved.success) + + const listed = await listWorkspaceAssetLibrary(context) + assert.ok(listed.success) + const byPath = new Map(listed.entries.map((entry) => [entry.workspacePath, entry])) + assert.equal(byPath.get('Workflows/hero.glb')?.thumbnailUrl, saved.thumbnailUrl) + assert.equal(byPath.get('Workflows/other.glb')?.thumbnailUrl, undefined) + assert.equal(byPath.get('Workflows/hero.world.json')?.thumbnailUrl, undefined) + + const withoutCache = await listWorkspaceAssetLibrary({ workspaceDir: context.workspaceDir }) + assert.ok(withoutCache.success) + assert.ok(withoutCache.entries.every((entry) => entry.thumbnailUrl === undefined)) +})) diff --git a/electron/main/asset-thumbnail-cache.ts b/electron/main/asset-thumbnail-cache.ts new file mode 100644 index 000000000..1c9319631 --- /dev/null +++ b/electron/main/asset-thumbnail-cache.ts @@ -0,0 +1,262 @@ +import { createHash } from 'node:crypto' +import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises' +import { isAbsolute, join, relative, resolve } from 'node:path' + +import type { + AssetThumbnailError, + AssetThumbnailSaveResult, + AssetThumbnailStatusResult, +} from '../../src/shared/types/assetLibrary' + +// Thumbnails are rendered by the renderer (offscreen three.js, same rig as +// Viewer3D) and cached here, in a Modly-owned folder outside the workspace. +// Layout, flat in `thumbnailsDir`: +// ..webp — the render of one file version +// .json — { workspaceDir, workspacePath } so pruning can +// find the source again from a hashed name +// pathKey hashes workspace dir + workspace path; version encodes the file's +// mtime and size, so an edited file misses the cache and is re-rendered. + +export const THUMBNAIL_PROTOCOL = 'modly-thumb' +const THUMBNAIL_HOST = 'thumbnails' +/** Bump when the render rig changes so every cached thumbnail goes stale. */ +export const THUMBNAIL_RENDER_REVISION = 1 +const MAX_THUMBNAIL_BYTES = 1024 * 1024 +const PATH_KEY_PATTERN = /^[0-9a-f]{32}$/ +const THUMBNAIL_FILE_PATTERN = /^([0-9a-f]{32})\.([0-9a-z]+-[0-9a-z]+-r[0-9]+)\.webp$/ +const ENCODED_ESCAPE_PATTERN = /%2e|%2f|%5c/i + +export interface FileVersionStats { + mtimeMs: number + size: number +} + +export interface ThumbnailCacheContext { + workspaceDir: string + thumbnailsDir: string +} + +interface ThumbnailSidecar { + workspaceDir: string + workspacePath: string +} + +function thumbnailError(code: AssetThumbnailError['code'], message: string): AssetThumbnailError { + return { code, message } +} + +/** Validates a workspace-relative .glb/.gltf path and resolves it inside the workspace. */ +export function normalizeThumbnailSourcePath(workspaceDir: string, workspacePath: string): { workspacePath: string, absolutePath: string } { + const normalized = workspacePath.trim().replace(/\\/g, '/') + if (!normalized || ENCODED_ESCAPE_PATTERN.test(normalized)) throw new Error('Thumbnail source path is not a safe workspace path') + if (isAbsolute(normalized) || /^[a-zA-Z]:[\\/]/.test(workspacePath) || workspacePath.startsWith('\\\\')) { + throw new Error('Thumbnail source path must be workspace-relative') + } + const segments = normalized.split('/').filter((segment) => segment && segment !== '.') + if (segments.length === 0 || segments.includes('..')) throw new Error('Thumbnail source path must not contain traversal segments') + const safePath = segments.join('/') + if (!/\.(glb|gltf)$/i.test(safePath)) throw new Error('Only .glb/.gltf meshes have thumbnails') + const root = resolve(workspaceDir) + const absolutePath = resolve(root, ...segments) + const back = relative(root, absolutePath) + if (!back || back.startsWith('..') || isAbsolute(back)) throw new Error('Thumbnail source path escapes the workspace root') + return { workspacePath: safePath, absolutePath } +} + +export function thumbnailPathKey(workspaceDir: string, workspacePath: string): string { + return createHash('sha256').update(`${resolve(workspaceDir)}\n${workspacePath}`).digest('hex').slice(0, 32) +} + +export function thumbnailVersion(stats: FileVersionStats): string { + return `${Math.trunc(stats.mtimeMs).toString(36)}-${stats.size.toString(36)}-r${THUMBNAIL_RENDER_REVISION}` +} + +export function thumbnailFileName(pathKey: string, version: string): string { + return `${pathKey}.${version}.webp` +} + +export function thumbnailUrlFor(fileName: string): string { + return `${THUMBNAIL_PROTOCOL}://${THUMBNAIL_HOST}/${fileName}` +} + +/** Maps a modly-thumb:// URL back to a cached file, or null for anything that is not one. */ +export function resolveThumbnailUrl(thumbnailsDir: string, url: string): string | null { + let parsed: URL + try { + parsed = new URL(url) + } catch { + return null + } + if (parsed.protocol !== `${THUMBNAIL_PROTOCOL}:` || parsed.hostname !== THUMBNAIL_HOST) return null + const fileName = parsed.pathname.replace(/^\//, '') + if (!THUMBNAIL_FILE_PATTERN.test(fileName)) return null + return join(thumbnailsDir, fileName) +} + +export function isWebpImage(bytes: Uint8Array): boolean { + const ascii = (start: number, end: number) => String.fromCharCode(...bytes.subarray(start, end)) + return bytes.length >= 12 && ascii(0, 4) === 'RIFF' && ascii(8, 12) === 'WEBP' +} + +async function exists(path: string): Promise { + try { + await stat(path) + return true + } catch { + return false + } +} + +/** URL of the cached thumbnail for this exact file version, if it was rendered already. */ +export async function findCachedThumbnailUrl(context: ThumbnailCacheContext, workspacePath: string, stats: FileVersionStats): Promise { + const fileName = thumbnailFileName(thumbnailPathKey(context.workspaceDir, workspacePath), thumbnailVersion(stats)) + return await exists(join(context.thumbnailsDir, fileName)) ? thumbnailUrlFor(fileName) : undefined +} + +export async function getThumbnailStatus(context: ThumbnailCacheContext, workspacePath: string): Promise { + let source: { workspacePath: string, absolutePath: string } + try { + source = normalizeThumbnailSourcePath(context.workspaceDir, workspacePath) + } catch (error) { + return { success: false, error: thumbnailError('unsafe-path', error instanceof Error ? error.message : String(error)) } + } + let stats: FileVersionStats + try { + stats = await stat(source.absolutePath) + } catch { + return { success: false, error: thumbnailError('not-found', 'Workspace mesh was not found.') } + } + return { + success: true, + workspacePath: source.workspacePath, + version: thumbnailVersion(stats), + thumbnailUrl: await findCachedThumbnailUrl(context, source.workspacePath, stats), + } +} + +async function cachedFilesFor(thumbnailsDir: string, pathKey: string): Promise { + let names: string[] + try { + names = await readdir(thumbnailsDir) + } catch { + return [] + } + return names.filter((name) => THUMBNAIL_FILE_PATTERN.exec(name)?.[1] === pathKey) +} + +/** + * Stores a render under the version the renderer was given. A file edited + * since then is refused as stale, so a thumbnail never claims a newer version + * than the one it shows; older versions of the same path are dropped. + */ +export async function saveThumbnail(context: ThumbnailCacheContext, request: { workspacePath: string, version: string, image: unknown }): Promise { + if (!(request.image instanceof Uint8Array) || request.image.length > MAX_THUMBNAIL_BYTES || !isWebpImage(request.image)) { + return { success: false, error: thumbnailError('invalid-image', 'Thumbnail must be a WebP image under 1 MB.') } + } + const status = await getThumbnailStatus(context, request.workspacePath) + if (!status.success) return status + if (status.version !== request.version) { + return { success: false, error: thumbnailError('stale', 'Workspace mesh changed while its thumbnail was rendering.') } + } + + const pathKey = thumbnailPathKey(context.workspaceDir, status.workspacePath) + const fileName = thumbnailFileName(pathKey, status.version) + const target = join(context.thumbnailsDir, fileName) + try { + await mkdir(context.thumbnailsDir, { recursive: true }) + const sidecar: ThumbnailSidecar = { workspaceDir: resolve(context.workspaceDir), workspacePath: status.workspacePath } + await writeFile(join(context.thumbnailsDir, `${pathKey}.json`), JSON.stringify(sidecar), 'utf8') + const temp = `${target}.${process.pid}.tmp` + await writeFile(temp, request.image) + await rename(temp, target) + const outdated = (await cachedFilesFor(context.thumbnailsDir, pathKey)).filter((name) => name !== fileName) + await Promise.all(outdated.map((name) => rm(join(context.thumbnailsDir, name), { force: true }))) + } catch (error) { + return { success: false, error: thumbnailError('write-failed', error instanceof Error ? error.message : String(error)) } + } + return { success: true, thumbnailUrl: thumbnailUrlFor(fileName) } +} + +async function removePathKey(thumbnailsDir: string, pathKey: string): Promise { + const files = [...await cachedFilesFor(thumbnailsDir, pathKey), `${pathKey}.json`] + await Promise.all(files.map((name) => rm(join(thumbnailsDir, name), { force: true }))) +} + +/** Drops every cached thumbnail of a workspace path (used when the asset is deleted). */ +export async function removeThumbnails(context: ThumbnailCacheContext, workspacePath: string): Promise { + const normalized = workspacePath.trim().replace(/\\/g, '/').split('/').filter((segment) => segment && segment !== '.').join('/') + await removePathKey(context.thumbnailsDir, thumbnailPathKey(context.workspaceDir, normalized)) +} + +async function readSidecar(path: string): Promise { + try { + const parsed = JSON.parse(await readFile(path, 'utf8')) as Partial + return typeof parsed.workspaceDir === 'string' && typeof parsed.workspacePath === 'string' + ? { workspaceDir: parsed.workspaceDir, workspacePath: parsed.workspacePath } + : null + } catch { + return null + } +} + +/** + * Garbage-collects the cache: removes thumbnails whose source mesh is gone + * (however it was deleted), versions that no longer match the file on disk, + * and renders left without a sidecar. + */ +export async function pruneThumbnails(thumbnailsDir: string): Promise { + let names: string[] + try { + names = await readdir(thumbnailsDir) + } catch { + return + } + const sidecarKeys = new Set(names.filter((name) => name.endsWith('.json')).map((name) => name.slice(0, -'.json'.length)).filter((key) => PATH_KEY_PATTERN.test(key))) + + for (const pathKey of sidecarKeys) { + const sidecar = await readSidecar(join(thumbnailsDir, `${pathKey}.json`)) + let current: string | null = null + if (sidecar) { + try { + const { absolutePath } = normalizeThumbnailSourcePath(sidecar.workspaceDir, sidecar.workspacePath) + current = thumbnailFileName(pathKey, thumbnailVersion(await stat(absolutePath))) + } catch { + current = null + } + } + if (!current) { + await removePathKey(thumbnailsDir, pathKey) + continue + } + const outdated = names.filter((name) => THUMBNAIL_FILE_PATTERN.exec(name)?.[1] === pathKey && name !== current) + await Promise.all(outdated.map((name) => rm(join(thumbnailsDir, name), { force: true }))) + } + + const orphans = names.filter((name) => { + const key = THUMBNAIL_FILE_PATTERN.exec(name)?.[1] + return (key !== undefined && !sidecarKeys.has(key)) || name.endsWith('.tmp') + }) + await Promise.all(orphans.map((name) => rm(join(thumbnailsDir, name), { force: true }))) +} + +export interface AssetThumbnailIpcDeps { + ipcMain: { handle(channel: string, handler: (event: unknown, payload?: unknown) => Promise): void } + getWorkspaceDir: () => string + thumbnailsDir: string +} + +export function registerAssetThumbnailIpcHandlers(deps: AssetThumbnailIpcDeps): void { + const context = (): ThumbnailCacheContext => ({ workspaceDir: deps.getWorkspaceDir(), thumbnailsDir: deps.thumbnailsDir }) + deps.ipcMain.handle('workspace:thumbnails:status', async (_event, payload) => { + const workspacePath = (payload as { workspacePath?: unknown } | undefined)?.workspacePath + if (typeof workspacePath !== 'string') return { success: false, error: thumbnailError('unsafe-path', 'workspacePath is required.') } + return getThumbnailStatus(context(), workspacePath) + }) + deps.ipcMain.handle('workspace:thumbnails:save', async (_event, payload) => { + const values = (payload ?? {}) as { workspacePath?: unknown, version?: unknown, image?: unknown } + if (typeof values.workspacePath !== 'string' || typeof values.version !== 'string') { + return { success: false, error: thumbnailError('unsafe-path', 'workspacePath and version are required.') } + } + return saveThumbnail(context(), { workspacePath: values.workspacePath, version: values.version, image: values.image }) + }) +} diff --git a/package.json b/package.json index dd05c25e1..e017a0b75 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/shared/types/assetLibrary.ts b/src/shared/types/assetLibrary.ts index fb5a602d6..618096e72 100644 --- a/src/shared/types/assetLibrary.ts +++ b/src/shared/types/assetLibrary.ts @@ -43,6 +43,8 @@ export interface AssetLibraryEntry { nonOpenableReason?: string createdAt?: string updatedAt?: string + /** Cached render of the current file version (modly-thumb:// URL); absent until rendered. */ + thumbnailUrl?: string } export interface AssetLibrarySourceLink { @@ -81,3 +83,24 @@ export type AssetLibraryReadResult = export type AssetLibraryOpenResult = | { success: true, entry: AssetLibraryEntry } | { success: false, error: AssetLibraryError } + +export interface AssetThumbnailError { + code: 'unsafe-path' | 'not-found' | 'unsupported' | 'stale' | 'invalid-image' | 'write-failed' + message: string +} + +/** A workspace mesh's thumbnail state: `version` identifies the file version a new render must be saved under. */ +export type AssetThumbnailStatusResult = + | { success: true, workspacePath: string, version: string, thumbnailUrl?: string } + | { success: false, error: AssetThumbnailError } + +export interface AssetThumbnailSaveRequest { + workspacePath: string + version: string + /** Encoded WebP image bytes. */ + image: Uint8Array +} + +export type AssetThumbnailSaveResult = + | { success: true, thumbnailUrl: string } + | { success: false, error: AssetThumbnailError } From 44071c85103cc9c80bca5f09914466dd396aa071 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Fri, 13 Nov 2026 10:01:31 +0100 Subject: [PATCH 053/108] feat(library): serve thumbnails over modly-thumb:// and expose status/save IPC A privileged custom scheme works for both the dev-server and packaged renderer origins, and only resolves file names from the cache. Deleting a job also drops its thumbnail. --- electron/main/asset-thumbnail-protocol.ts | 42 +++++++++++++++++++ electron/main/index.ts | 4 ++ electron/main/ipc-handlers.ts | 12 ++++++ .../preload/artifact-registry-preload.test.ts | 22 ++++++++++ electron/preload/electron-api.ts | 7 ++++ src/index.html | 2 +- src/shared/types/electron.d.ts | 7 ++++ 7 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 electron/main/asset-thumbnail-protocol.ts diff --git a/electron/main/asset-thumbnail-protocol.ts b/electron/main/asset-thumbnail-protocol.ts new file mode 100644 index 000000000..a2c521ee0 --- /dev/null +++ b/electron/main/asset-thumbnail-protocol.ts @@ -0,0 +1,42 @@ +import { app, protocol } from 'electron' +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' + +import { resolveThumbnailUrl, THUMBNAIL_PROTOCOL } from './asset-thumbnail-cache' + +/** Dedicated folder: userData/cache collides with Chromium's own disk cache on Windows. */ +export function getThumbnailsDir(): string { + return join(app.getPath('userData'), 'thumbnails') +} + +/** Must run before `app.whenReady()` resolves. */ +export function registerThumbnailScheme(): void { + protocol.registerSchemesAsPrivileged([ + { scheme: THUMBNAIL_PROTOCOL, privileges: { standard: true, secure: true } }, + ]) +} + +/** + * Serves cached thumbnails as `modly-thumb://thumbnails/.webp`. Works for + * both the dev server (http) and packaged (file://) renderer origins, which a + * plain file:// URL would not. Only names matching the cache's own pattern + * resolve, so the scheme cannot reach anything else on disk. + */ +export function handleThumbnailProtocol(): void { + protocol.handle(THUMBNAIL_PROTOCOL, async (request) => { + const filePath = resolveThumbnailUrl(getThumbnailsDir(), request.url) + if (!filePath) return new Response(null, { status: 400 }) + try { + const body = await readFile(filePath) + return new Response(body, { + headers: { + 'content-type': 'image/webp', + // File names embed the source version, so a URL's content never changes. + 'cache-control': 'public, max-age=31536000, immutable', + }, + }) + } catch { + return new Response(null, { status: 404 }) + } + }) +} diff --git a/electron/main/index.ts b/electron/main/index.ts index 2f3f5b83c..24c686420 100644 --- a/electron/main/index.ts +++ b/electron/main/index.ts @@ -8,6 +8,7 @@ import { initAutoUpdater } from './updater' import { syncBuiltinExtensions } from './builtin-sync' import { reconcileInterruptedExtensionInstalls } from './extension-install-recovery' import { getSettings } from './settings-store' +import { handleThumbnailProtocol, registerThumbnailScheme } from './asset-thumbnail-protocol' let mainWindow: BrowserWindow | null = null let pythonBridge: PythonBridge | null = null @@ -76,6 +77,7 @@ function createWindow(): void { } app.setName('Modly') +registerThumbnailScheme() process.on('uncaughtException', (err) => { if ((err as NodeJS.ErrnoException).code === 'EPIPE') return @@ -101,6 +103,8 @@ app.whenReady().then(async () => { optimizer.watchWindowShortcuts(window) }) + handleThumbnailProtocol() + // Sync built-in extensions from app resources to userData syncBuiltinExtensions() diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 039705780..0632620ff 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -91,6 +91,8 @@ import { validateExtensionDestinationRegistration, } from './extension-install-recovery' import { registerWorkspaceAssetLibraryIpcHandlers } from './artifact-registry-service' +import { getThumbnailsDir } from './asset-thumbnail-protocol' +import { registerAssetThumbnailIpcHandlers, removeThumbnails } from './asset-thumbnail-cache' import { updatesSupported } from './updater' import { ModelWeightOperations } from './model-weight-operations' import { readLocalFileBase64 } from './bounded-file-reader' @@ -995,6 +997,12 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe registerWorkspaceAssetLibraryIpcHandlers({ ipcMain, getWorkspaceDir: () => getSettings(app.getPath('userData')).workspaceDir, + thumbnailsDir: getThumbnailsDir(), + }) + registerAssetThumbnailIpcHandlers({ + ipcMain, + getWorkspaceDir: () => getSettings(app.getPath('userData')).workspaceDir, + thumbnailsDir: getThumbnailsDir(), }) ipcMain.handle('workspace:listCollections', async () => { @@ -1035,6 +1043,10 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe ipcMain.handle('workspace:deleteJob', async (_, { collection, filename }: { collection: string; filename: string }) => { await rmAsync(workspacePath(collection, filename), { force: true }) await rmAsync(workspacePath(collection, filename.replace(/\.glb$/, '.meta.json')), { force: true }) + await removeThumbnails( + { workspaceDir: getSettings(app.getPath('userData')).workspaceDir, thumbnailsDir: getThumbnailsDir() }, + `${collection}/${filename}`, + ).catch(() => undefined) }) // Directory utilities for settings diff --git a/electron/preload/artifact-registry-preload.test.ts b/electron/preload/artifact-registry-preload.test.ts index 9dec46fd8..2f78c2e27 100644 --- a/electron/preload/artifact-registry-preload.test.ts +++ b/electron/preload/artifact-registry-preload.test.ts @@ -44,6 +44,28 @@ test('preload exposes scoped workspace library list/read/open methods', async () ]) }) +test('preload exposes workspace thumbnail status/save methods', async () => { + const calls: Array<{ channel: string, payload?: unknown }> = [] + const api = createElectronApi({ + invoke: async (channel: string, payload?: unknown) => { + calls.push({ channel, payload }) + return { success: true } + }, + send: () => undefined, + on: () => undefined, + removeAllListeners: () => undefined, + }, { setZoomFactor: () => undefined }, { getPathForFile: () => '' }) + const image = new Uint8Array([1, 2, 3]) + + await api.workspace.thumbnails.status('Workflows/hero.glb') + await api.workspace.thumbnails.save({ workspacePath: 'Workflows/hero.glb', version: 'v1', image }) + + assert.deepEqual(calls, [ + { channel: 'workspace:thumbnails:status', payload: { workspacePath: 'Workflows/hero.glb' } }, + { channel: 'workspace:thumbnails:save', payload: { workspacePath: 'Workflows/hero.glb', version: 'v1', image } }, + ]) +}) + test('preload resolves dropped file paths through Electron webUtils', () => { const files: unknown[] = [] const calls: string[] = [] diff --git a/electron/preload/electron-api.ts b/electron/preload/electron-api.ts index d11863beb..364ba965f 100644 --- a/electron/preload/electron-api.ts +++ b/electron/preload/electron-api.ts @@ -4,6 +4,9 @@ import type { AssetLibraryOpenResult, AssetLibraryReadRequest, AssetLibraryReadResult, + AssetThumbnailSaveRequest, + AssetThumbnailSaveResult, + AssetThumbnailStatusResult, } from '../../src/shared/types/assetLibrary' import type { Conversation as AgentConversation, @@ -249,6 +252,10 @@ export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFra read: (request: AssetLibraryReadRequest): Promise => ipcRenderer.invoke('workspace:library:read', request) as Promise, open: (request: AssetLibraryOpenRequest): Promise => ipcRenderer.invoke('workspace:library:open', request) as Promise, }, + thumbnails: { + status: (workspacePath: string): Promise => ipcRenderer.invoke('workspace:thumbnails:status', { workspacePath }) as Promise, + save: (request: AssetThumbnailSaveRequest): Promise => ipcRenderer.invoke('workspace:thumbnails:save', request) as Promise, + }, }, // Extensions diff --git a/src/index.html b/src/index.html index b1c8a532f..03ed408ea 100644 --- a/src/index.html +++ b/src/index.html @@ -3,7 +3,7 @@ - + Modly diff --git a/src/shared/types/electron.d.ts b/src/shared/types/electron.d.ts index 675827f62..57ec8f3ad 100644 --- a/src/shared/types/electron.d.ts +++ b/src/shared/types/electron.d.ts @@ -7,6 +7,9 @@ import type { AssetLibraryOpenResult, AssetLibraryReadRequest, AssetLibraryReadResult, + AssetThumbnailSaveRequest, + AssetThumbnailSaveResult, + AssetThumbnailStatusResult, } from './assetLibrary' import type { Conversation, ConversationSummary } from '../utils/agentConversations' @@ -313,6 +316,10 @@ declare global { read: (request: AssetLibraryReadRequest) => Promise open: (request: AssetLibraryOpenRequest) => Promise } + thumbnails: { + status: (workspacePath: string) => Promise + save: (request: AssetThumbnailSaveRequest) => Promise + } } setup: { check: () => Promise<{ needed: boolean; defaultDataDir: string; platform: string; arch: string }> From 4cf073d36d76465ef9218954af9383e292a9596a Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Fri, 13 Nov 2026 10:01:32 +0100 Subject: [PATCH 054/108] feat(library): render thumbnails offscreen and backfill the library The renderer draws each mesh with Viewer3D's default rig in a single offscreen WebGL context, one at a time, whenever a mesh reaches the viewer (generation, import, edits, library opens) and for library entries still missing one. three.js is loaded lazily so it stays out of the entry chunk. --- package.json | 2 +- src/App.tsx | 3 + src/areas/generate/GeneratePage.tsx | 2 + .../thumbnails/assetThumbnailQueue.test.ts | 113 ++++++++++++++++++ src/shared/thumbnails/assetThumbnailQueue.ts | 87 ++++++++++++++ .../thumbnails/assetThumbnailService.ts | 68 +++++++++++ src/shared/thumbnails/renderMeshThumbnail.ts | 106 ++++++++++++++++ 7 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 src/shared/thumbnails/assetThumbnailQueue.test.ts create mode 100644 src/shared/thumbnails/assetThumbnailQueue.ts create mode 100644 src/shared/thumbnails/assetThumbnailService.ts create mode 100644 src/shared/thumbnails/renderMeshThumbnail.ts diff --git a/package.json b/package.json index e017a0b75..0d8700a5a 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/App.tsx b/src/App.tsx index 2de172243..66f210925 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -5,6 +5,7 @@ import MainLayout from '@shared/components/layout/MainLayout' import { UpdateModal } from '@shared/components/ui/UpdateModal' import { ErrorModal } from '@shared/components/ui/ErrorModal' import { Toast } from '@shared/components/ui/Toast' +import { startAssetThumbnailService } from '@shared/thumbnails/assetThumbnailService' const UI_SCALE_FACTORS: Record = { small: 0.875, medium: 1, large: 1.25, 'very-large': 1.5 } @@ -37,6 +38,8 @@ export default function App(): JSX.Element { window.electron.ui.setZoomFactor(UI_SCALE_FACTORS[uiScale]) }, [useAtkinsonFont, uiScale]) + useEffect(() => startAssetThumbnailService(), []) + useEffect(() => { if (setupStatus === 'done') initApp() // eslint-disable-next-line react-hooks/exhaustive-deps -- react to setup transition only; initApp is stable diff --git a/src/areas/generate/GeneratePage.tsx b/src/areas/generate/GeneratePage.tsx index 9f77ef102..82cbf53c3 100644 --- a/src/areas/generate/GeneratePage.tsx +++ b/src/areas/generate/GeneratePage.tsx @@ -13,6 +13,7 @@ import { meshPathFromUrl } from './meshEditPanel' import { commitSculpt, useSculptStore } from './sculpt/sculptStore' import { sculptKeyAction } from './sculpt/sculptLogic' import { getDefaultAssetLibraryService } from './assetLibraryService' +import { requestLibraryThumbnails } from '@shared/thumbnails/assetThumbnailService' import { buildOrcaSlicerDeepLink, canOpenInOrcaSlicer } from './orcaSlicerLink' import { resolveAssetLibraryOpenTarget, type ProjectedAssetLibraryEntry } from './assetLibraryProjection' import { @@ -886,6 +887,7 @@ export default function GeneratePage(): JSX.Element { return } setLibraryEntries(result.entries) + void requestLibraryThumbnails(result.entries) setLibrarySelectedEntryId((current) => current && result.entries.some((entry) => entry.id === current) ? current : result.entries.find(isAssetLibraryEntryOpenable)?.id ?? result.entries[0]?.id ?? null) diff --git a/src/shared/thumbnails/assetThumbnailQueue.test.ts b/src/shared/thumbnails/assetThumbnailQueue.test.ts new file mode 100644 index 000000000..e7d2619fd --- /dev/null +++ b/src/shared/thumbnails/assetThumbnailQueue.test.ts @@ -0,0 +1,113 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import type { AssetLibraryEntry, AssetThumbnailSaveResult } from '../types/assetLibrary' +import { + createAssetThumbnailQueue, + entriesNeedingThumbnails, + meshUrlForWorkspacePath, + workspacePathFromMeshUrl, +} from './assetThumbnailQueue.ts' + +const IMAGE = new Uint8Array([1, 2, 3]) + +function fakeDeps(options: { cached?: Set, version?: () => string, save?: () => AssetThumbnailSaveResult, renderFails?: boolean } = {}) { + const renders: string[] = [] + const saves: Array<{ workspacePath: string, version: string }> = [] + let active = 0 + let maxActive = 0 + let idle = 0 + const deps = { + status: async (workspacePath: string) => ({ + success: true as const, + workspacePath, + version: options.version?.() ?? 'v1', + thumbnailUrl: options.cached?.has(workspacePath) ? `modly-thumb://thumbnails/${workspacePath}` : undefined, + }), + save: async (request: { workspacePath: string, version: string }) => { + saves.push({ workspacePath: request.workspacePath, version: request.version }) + return options.save?.() ?? { success: true as const, thumbnailUrl: `modly-thumb://thumbnails/${request.workspacePath}` } + }, + render: async (workspacePath: string) => { + active += 1 + maxActive = Math.max(maxActive, active) + renders.push(workspacePath) + await new Promise((resolve) => setTimeout(resolve, 5)) + active -= 1 + if (options.renderFails) throw new Error('bad mesh') + return IMAGE + }, + onIdle: () => { idle += 1 }, + } + return { deps, renders, saves, stats: () => ({ maxActive, idle }) } +} + +test('mesh URLs map to workspace paths for glb/gltf only', () => { + assert.equal(workspacePathFromMeshUrl('/workspace/Default/hero.glb'), 'Default/hero.glb') + assert.equal(workspacePathFromMeshUrl('/workspace/My%20Coll/hero.gltf?v=2'), 'My Coll/hero.gltf') + assert.equal(workspacePathFromMeshUrl('/workspace/Default/part.stl'), null) + assert.equal(workspacePathFromMeshUrl('/optimize/serve-file?path=C%3A%2Fhero.glb'), null) + assert.equal(workspacePathFromMeshUrl('/workspace/%E0%A4%A.glb'), null) + assert.equal(meshUrlForWorkspacePath('http://127.0.0.1:8765/', 'My Coll/hero #1.glb'), 'http://127.0.0.1:8765/workspace/My%20Coll/hero%20%231.glb') +}) + +test('only ready, openable 3D entries without a thumbnail need one', () => { + const base: AssetLibraryEntry = { + id: 'a', workspacePath: 'Workflows/a.glb', displayName: 'a.glb', sourceScope: 'workflows', + state: 'ready', previewKind: '3d-model', warnings: [], openable: true, + } + const entries: AssetLibraryEntry[] = [ + base, + { ...base, id: 'b', thumbnailUrl: 'modly-thumb://thumbnails/b' }, + { ...base, id: 'c', previewKind: 'text', openable: false }, + { ...base, id: 'd', state: 'unsafe' }, + ] + assert.deepEqual(entriesNeedingThumbnails(entries).map((entry) => entry.id), ['a']) +}) + +test('renders serially, dedupes in-flight paths and skips cached versions', async () => { + const { deps, renders, saves, stats } = fakeDeps({ cached: new Set(['Workflows/cached.glb']) }) + const queue = createAssetThumbnailQueue(deps) + const results = await Promise.all([ + queue.ensure('Workflows/a.glb'), + queue.ensure('Workflows/a.glb'), + queue.ensure('Workflows/b.glb'), + queue.ensure('Workflows/cached.glb'), + ]) + assert.deepEqual(results, [ + 'modly-thumb://thumbnails/Workflows/a.glb', + 'modly-thumb://thumbnails/Workflows/a.glb', + 'modly-thumb://thumbnails/Workflows/b.glb', + 'modly-thumb://thumbnails/Workflows/cached.glb', + ]) + assert.deepEqual(renders, ['Workflows/a.glb', 'Workflows/b.glb']) + assert.deepEqual(saves, [{ workspacePath: 'Workflows/a.glb', version: 'v1' }, { workspacePath: 'Workflows/b.glb', version: 'v1' }]) + assert.equal(stats().maxActive, 1) + assert.equal(stats().idle, 1) +}) + +test('a version that failed to render is not retried until the file changes', async () => { + let version = 'v1' + const { deps, renders } = fakeDeps({ renderFails: true, version: () => version }) + const queue = createAssetThumbnailQueue(deps) + assert.equal(await queue.ensure('Workflows/broken.glb'), null) + assert.equal(await queue.ensure('Workflows/broken.glb'), null) + assert.equal(renders.length, 1) + version = 'v2' + await queue.ensure('Workflows/broken.glb') + assert.equal(renders.length, 2) +}) + +test('a stale save is retried on the next request', async () => { + let stale = true + const { deps, renders } = fakeDeps({ + save: () => stale + ? { success: false, error: { code: 'stale', message: 'changed' } } + : { success: true, thumbnailUrl: 'modly-thumb://thumbnails/x' }, + }) + const queue = createAssetThumbnailQueue(deps) + assert.equal(await queue.ensure('Workflows/a.glb'), null) + stale = false + assert.equal(await queue.ensure('Workflows/a.glb'), 'modly-thumb://thumbnails/x') + assert.equal(renders.length, 2) +}) diff --git a/src/shared/thumbnails/assetThumbnailQueue.ts b/src/shared/thumbnails/assetThumbnailQueue.ts new file mode 100644 index 000000000..1d66ed366 --- /dev/null +++ b/src/shared/thumbnails/assetThumbnailQueue.ts @@ -0,0 +1,87 @@ +import type { + AssetLibraryEntry, + AssetThumbnailSaveRequest, + AssetThumbnailSaveResult, + AssetThumbnailStatusResult, +} from '../types/assetLibrary' + +export interface AssetThumbnailQueueDeps { + status: (workspacePath: string) => Promise + save: (request: AssetThumbnailSaveRequest) => Promise + /** Renders the mesh at this workspace path and returns encoded WebP bytes. */ + render: (workspacePath: string) => Promise + /** Called whenever the queue drains, e.g. to release the offscreen GL context. */ + onIdle?: () => void +} + +/** `/workspace/` mesh URLs (what generation, import and edits push) → workspace path, for .glb/.gltf only. */ +export function workspacePathFromMeshUrl(url: string): string | null { + if (!url.startsWith('/workspace/')) return null + let workspacePath: string + try { + workspacePath = decodeURIComponent(url.slice('/workspace/'.length).split('?')[0] ?? '') + } catch { + return null + } + return /\.(glb|gltf)$/i.test(workspacePath) ? workspacePath : null +} + +export function meshUrlForWorkspacePath(apiUrl: string, workspacePath: string): string { + return `${apiUrl.replace(/\/$/, '')}/workspace/${workspacePath.split('/').map(encodeURIComponent).join('/')}` +} + +/** Library entries that can be rendered but have no thumbnail for their current file version yet. */ +export function entriesNeedingThumbnails(entries: AssetLibraryEntry[]): AssetLibraryEntry[] { + return entries.filter((entry) => entry.state === 'ready' && entry.openable && entry.previewKind === '3d-model' && !entry.thumbnailUrl) +} + +/** + * Serial render queue: one offscreen render at a time, each workspace path at + * most once in flight, and a file version that failed to render is not retried + * until the file changes. The main process is the source of truth for whether + * a version is already cached, so a thumbnail is rendered at most once per + * file version even across sessions. + */ +export function createAssetThumbnailQueue(deps: AssetThumbnailQueueDeps) { + const inFlight = new Map>() + const failedVersions = new Set() + let tail: Promise = Promise.resolve() + + async function run(workspacePath: string): Promise { + const status = await deps.status(workspacePath) + if (!status.success) return null + if (status.thumbnailUrl) return status.thumbnailUrl + const versionKey = `${status.workspacePath}@${status.version}` + if (failedVersions.has(versionKey)) return null + try { + const image = await deps.render(status.workspacePath) + const saved = await deps.save({ workspacePath: status.workspacePath, version: status.version, image }) + if (saved.success) return saved.thumbnailUrl + // A stale save means the file changed mid-render: the next request renders the new version. + if (saved.error.code !== 'stale') failedVersions.add(versionKey) + return null + } catch { + failedVersions.add(versionKey) + return null + } + } + + function ensure(workspacePath: string): Promise { + const existing = inFlight.get(workspacePath) + if (existing) return existing + const job = tail + .then(() => run(workspacePath)) + .catch(() => null) + .finally(() => { + inFlight.delete(workspacePath) + if (inFlight.size === 0) deps.onIdle?.() + }) + inFlight.set(workspacePath, job) + tail = job + return job + } + + return { ensure } +} + +export type AssetThumbnailQueue = ReturnType diff --git a/src/shared/thumbnails/assetThumbnailService.ts b/src/shared/thumbnails/assetThumbnailService.ts new file mode 100644 index 000000000..7e4e448ee --- /dev/null +++ b/src/shared/thumbnails/assetThumbnailService.ts @@ -0,0 +1,68 @@ +import { useAppStore } from '@shared/stores/appStore' +import type { AssetLibraryEntry } from '../types/assetLibrary' +import { + createAssetThumbnailQueue, + entriesNeedingThumbnails, + meshUrlForWorkspacePath, + workspacePathFromMeshUrl, + type AssetThumbnailQueue, +} from './assetThumbnailQueue' + +// Pause between renders so a large backfill never monopolises the UI thread. +const RENDER_SPACING_MS = 100 + +let queue: AssetThumbnailQueue | null = null +// three.js is loaded on first render so it stays out of the app's entry chunk. +let renderModule: Promise | null = null +const loadRenderModule = () => (renderModule ??= import('./renderMeshThumbnail')) + +function getQueue(): AssetThumbnailQueue { + if (queue) return queue + const thumbnails = window.electron.workspace.thumbnails + queue = createAssetThumbnailQueue({ + status: (workspacePath) => thumbnails.status(workspacePath), + save: (request) => thumbnails.save(request), + render: async (workspacePath) => { + await new Promise((resolve) => setTimeout(resolve, RENDER_SPACING_MS)) + const { renderMeshThumbnail } = await loadRenderModule() + return renderMeshThumbnail(meshUrlForWorkspacePath(useAppStore.getState().apiUrl, workspacePath)) + }, + onIdle: () => { void renderModule?.then((module) => module.releaseThumbnailRenderer()) }, + }) + return queue +} + +/** Queues thumbnails for library entries that lack one; resolves once they are all processed. */ +export async function requestLibraryThumbnails(entries: AssetLibraryEntry[]): Promise { + await Promise.all(entriesNeedingThumbnails(entries).map((entry) => getQueue().ensure(entry.workspacePath))) +} + +async function backfillLibraryThumbnails(): Promise { + try { + const result = await window.electron.workspace.library.list() + if (result.success) await requestLibraryThumbnails(result.entries) + } catch { + // Backfill is best effort; the next library listing retries. + } +} + +function viewerMeshUrl(state: ReturnType): string | undefined { + return state.currentJob?.status === 'done' ? state.currentJob.outputUrl : undefined +} + +/** + * Renders a thumbnail for every mesh that reaches the viewer (generation, + * import, mesh edits, agent edits, library opens) and backfills the library + * once the backend is up. Meshes already cached cost one status call. Returns + * an unsubscribe function. + */ +export function startAssetThumbnailService(): () => void { + if (useAppStore.getState().backendStatus === 'ready') void backfillLibraryThumbnails() + return useAppStore.subscribe((state, previous) => { + if (state.backendStatus === 'ready' && previous.backendStatus !== 'ready') void backfillLibraryThumbnails() + const url = viewerMeshUrl(state) + if (!url || url === viewerMeshUrl(previous)) return + const workspacePath = workspacePathFromMeshUrl(url) + if (workspacePath) void getQueue().ensure(workspacePath) + }) +} diff --git a/src/shared/thumbnails/renderMeshThumbnail.ts b/src/shared/thumbnails/renderMeshThumbnail.ts new file mode 100644 index 000000000..c56e57d72 --- /dev/null +++ b/src/shared/thumbnails/renderMeshThumbnail.ts @@ -0,0 +1,106 @@ +import * as THREE from 'three' +import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js' +import { DEFAULT_LIGHT_SETTINGS } from '@shared/stores/appStore' + +export const THUMBNAIL_SIZE = 256 + +// Same look as Viewer3D's default view: its background, default light rig, +// fov and three-quarter camera direction, R3F's ACES tone mapping and sRGB +// output, DoubleSide materials. The user's live lighting tweaks are left out +// on purpose so every thumbnail is rendered the same way. +const BACKGROUND = '#18181b' +const CAMERA_FOV = 45 +const CAMERA_DIRECTION = new THREE.Vector3(0, 1.5, 4).normalize() + +let renderer: THREE.WebGLRenderer | null = null + +function getRenderer(): THREE.WebGLRenderer { + if (renderer) return renderer + const canvas = document.createElement('canvas') + renderer = new THREE.WebGLRenderer({ canvas, antialias: true, preserveDrawingBuffer: true }) + renderer.setPixelRatio(1) + renderer.setSize(THUMBNAIL_SIZE, THUMBNAIL_SIZE, false) + renderer.outputColorSpace = THREE.SRGBColorSpace + renderer.toneMapping = THREE.ACESFilmicToneMapping + return renderer +} + +/** Frees the offscreen WebGL context; the next render recreates it. */ +export function releaseThumbnailRenderer(): void { + if (!renderer) return + renderer.dispose() + renderer.forceContextLoss() + renderer = null +} + +function disposeObject(root: THREE.Object3D): void { + root.traverse((child) => { + if (!(child instanceof THREE.Mesh)) return + child.geometry.dispose() + const materials: THREE.Material[] = Array.isArray(child.material) ? child.material : [child.material] + for (const material of materials) { + for (const value of Object.values(material)) { + if (value instanceof THREE.Texture) value.dispose() + } + material.dispose() + } + }) +} + +function buildScene(model: THREE.Object3D): { scene: THREE.Scene, camera: THREE.PerspectiveCamera } { + model.traverse((child) => { + if (!(child instanceof THREE.Mesh)) return + const materials: THREE.Material[] = Array.isArray(child.material) ? child.material : [child.material] + materials.forEach((material) => { material.side = THREE.DoubleSide }) + }) + + // Rest the mesh on the ground at the origin, as Viewer3D does. + const box = new THREE.Box3().setFromObject(model) + const center = box.getCenter(new THREE.Vector3()) + model.position.set(-center.x, -box.min.y, -center.z) + model.updateMatrixWorld(true) + + const scene = new THREE.Scene() + scene.background = new THREE.Color(BACKGROUND) + scene.add(new THREE.AmbientLight(0xffffff, DEFAULT_LIGHT_SETTINGS.ambientIntensity)) + const main = new THREE.DirectionalLight(DEFAULT_LIGHT_SETTINGS.mainColor, DEFAULT_LIGHT_SETTINGS.mainIntensity) + main.position.set(5, 8, 5) + const fill = new THREE.DirectionalLight(DEFAULT_LIGHT_SETTINGS.fillColor, DEFAULT_LIGHT_SETTINGS.fillIntensity) + fill.position.set(-4, 2, -4) + scene.add(main, fill, model) + + // Frame the bounding sphere from Viewer3D's default camera direction. + const sphere = new THREE.Box3().setFromObject(model).getBoundingSphere(new THREE.Sphere()) + const radius = Math.max(sphere.radius, 1e-3) + const distance = (radius / Math.sin(THREE.MathUtils.degToRad(CAMERA_FOV) / 2)) * 1.05 + const camera = new THREE.PerspectiveCamera(CAMERA_FOV, 1, distance / 100, distance * 10) + camera.position.copy(sphere.center).addScaledVector(CAMERA_DIRECTION, distance) + camera.lookAt(sphere.center) + return { scene, camera } +} + +function canvasToWebp(canvas: HTMLCanvasElement): Promise { + return new Promise((resolve, reject) => { + canvas.toBlob((blob) => { + if (!blob) { + reject(new Error('Thumbnail encoding failed')) + return + } + blob.arrayBuffer().then((buffer) => resolve(new Uint8Array(buffer)), reject) + }, 'image/webp', 0.85) + }) +} + +/** Loads a .glb/.gltf from `url` and renders a square WebP thumbnail of it. */ +export async function renderMeshThumbnail(url: string): Promise { + const gltf = await new GLTFLoader().loadAsync(url) + const model = gltf.scene + try { + const gl = getRenderer() + const { scene, camera } = buildScene(model) + gl.render(scene, camera) + return await canvasToWebp(gl.domElement) + } finally { + disposeObject(model) + } +} From 4bb794fc871f0e6f2e7c3f4da7683c99c5f8c84a Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Fri, 13 Nov 2026 10:21:38 +0100 Subject: [PATCH 055/108] feat(library): publish finished thumbnails to a store so open views repaint The render queue reports each thumbnail it ends a request with; the service records it by workspace path in a small zustand store. A tile reads its own key, so a background render repaints that tile without re-listing the library. --- package.json | 2 +- .../thumbnails/assetThumbnailQueue.test.ts | 16 ++++++++++++++ src/shared/thumbnails/assetThumbnailQueue.ts | 6 +++++ .../thumbnails/assetThumbnailService.ts | 2 ++ .../thumbnails/assetThumbnailStore.test.ts | 18 +++++++++++++++ src/shared/thumbnails/assetThumbnailStore.ts | 22 +++++++++++++++++++ 6 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 src/shared/thumbnails/assetThumbnailStore.test.ts create mode 100644 src/shared/thumbnails/assetThumbnailStore.ts diff --git a/package.json b/package.json index 58db5c46e..f3210b09a 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs src/shared/pages/pageRegistry.test.ts", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs src/shared/pages/pageRegistry.test.ts", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/shared/thumbnails/assetThumbnailQueue.test.ts b/src/shared/thumbnails/assetThumbnailQueue.test.ts index e7d2619fd..c39c1240b 100644 --- a/src/shared/thumbnails/assetThumbnailQueue.test.ts +++ b/src/shared/thumbnails/assetThumbnailQueue.test.ts @@ -98,6 +98,22 @@ test('a version that failed to render is not retried until the file changes', as assert.equal(renders.length, 2) }) +test('reports each thumbnail once per processed request, and nothing for failures', async () => { + const { deps } = fakeDeps({ cached: new Set(['Workflows/cached.glb']) }) + const ready: Array<[string, string]> = [] + const queue = createAssetThumbnailQueue({ ...deps, onReady: (path, url) => { ready.push([path, url]) } }) + await Promise.all([queue.ensure('Workflows/a.glb'), queue.ensure('Workflows/a.glb'), queue.ensure('Workflows/cached.glb')]) + assert.deepEqual(ready, [ + ['Workflows/a.glb', 'modly-thumb://thumbnails/Workflows/a.glb'], + ['Workflows/cached.glb', 'modly-thumb://thumbnails/Workflows/cached.glb'], + ]) + + const failing = fakeDeps({ renderFails: true }) + const silent: string[] = [] + await createAssetThumbnailQueue({ ...failing.deps, onReady: (path) => { silent.push(path) } }).ensure('Workflows/broken.glb') + assert.deepEqual(silent, []) +}) + test('a stale save is retried on the next request', async () => { let stale = true const { deps, renders } = fakeDeps({ diff --git a/src/shared/thumbnails/assetThumbnailQueue.ts b/src/shared/thumbnails/assetThumbnailQueue.ts index 1d66ed366..6b791922e 100644 --- a/src/shared/thumbnails/assetThumbnailQueue.ts +++ b/src/shared/thumbnails/assetThumbnailQueue.ts @@ -12,6 +12,8 @@ export interface AssetThumbnailQueueDeps { render: (workspacePath: string) => Promise /** Called whenever the queue drains, e.g. to release the offscreen GL context. */ onIdle?: () => void + /** Called once per processed request that ends with a thumbnail, so open views can show it. */ + onReady?: (workspacePath: string, thumbnailUrl: string) => void } /** `/workspace/` mesh URLs (what generation, import and edits push) → workspace path, for .glb/.gltf only. */ @@ -71,6 +73,10 @@ export function createAssetThumbnailQueue(deps: AssetThumbnailQueueDeps) { if (existing) return existing const job = tail .then(() => run(workspacePath)) + .then((thumbnailUrl) => { + if (thumbnailUrl) deps.onReady?.(workspacePath, thumbnailUrl) + return thumbnailUrl + }) .catch(() => null) .finally(() => { inFlight.delete(workspacePath) diff --git a/src/shared/thumbnails/assetThumbnailService.ts b/src/shared/thumbnails/assetThumbnailService.ts index 7e4e448ee..710e17e72 100644 --- a/src/shared/thumbnails/assetThumbnailService.ts +++ b/src/shared/thumbnails/assetThumbnailService.ts @@ -7,6 +7,7 @@ import { workspacePathFromMeshUrl, type AssetThumbnailQueue, } from './assetThumbnailQueue' +import { useAssetThumbnailStore } from './assetThumbnailStore' // Pause between renders so a large backfill never monopolises the UI thread. const RENDER_SPACING_MS = 100 @@ -28,6 +29,7 @@ function getQueue(): AssetThumbnailQueue { return renderMeshThumbnail(meshUrlForWorkspacePath(useAppStore.getState().apiUrl, workspacePath)) }, onIdle: () => { void renderModule?.then((module) => module.releaseThumbnailRenderer()) }, + onReady: (workspacePath, thumbnailUrl) => useAssetThumbnailStore.getState().setThumbnailUrl(workspacePath, thumbnailUrl), }) return queue } diff --git a/src/shared/thumbnails/assetThumbnailStore.test.ts b/src/shared/thumbnails/assetThumbnailStore.test.ts new file mode 100644 index 000000000..15f7da9fd --- /dev/null +++ b/src/shared/thumbnails/assetThumbnailStore.test.ts @@ -0,0 +1,18 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { useAssetThumbnailStore } from './assetThumbnailStore' + +test('records rendered thumbnails by workspace path and ignores repeats', () => { + const { setThumbnailUrl } = useAssetThumbnailStore.getState() + setThumbnailUrl('Workflows/a.glb', 'modly-thumb://a@v1') + const afterFirst = useAssetThumbnailStore.getState().urls + setThumbnailUrl('Workflows/a.glb', 'modly-thumb://a@v1') + assert.equal(useAssetThumbnailStore.getState().urls, afterFirst) + setThumbnailUrl('Workflows/b.glb', 'modly-thumb://b@v1') + setThumbnailUrl('Workflows/a.glb', 'modly-thumb://a@v2') + assert.deepEqual(useAssetThumbnailStore.getState().urls, { + 'Workflows/a.glb': 'modly-thumb://a@v2', + 'Workflows/b.glb': 'modly-thumb://b@v1', + }) +}) diff --git a/src/shared/thumbnails/assetThumbnailStore.ts b/src/shared/thumbnails/assetThumbnailStore.ts new file mode 100644 index 000000000..d1e90d695 --- /dev/null +++ b/src/shared/thumbnails/assetThumbnailStore.ts @@ -0,0 +1,22 @@ +import { create } from 'zustand' + +// Thumbnails rendered since the library was listed, by workspace path. A tile +// reads its own key, so a finished render repaints that tile only, without +// re-listing the library. + +interface AssetThumbnailState { + urls: Readonly> + setThumbnailUrl: (workspacePath: string, thumbnailUrl: string) => void +} + +export const useAssetThumbnailStore = create((set) => ({ + urls: {}, + setThumbnailUrl: (workspacePath, thumbnailUrl) => set((state) => ( + state.urls[workspacePath] === thumbnailUrl ? state : { urls: { ...state.urls, [workspacePath]: thumbnailUrl } } + )), +})) + +/** The freshest thumbnail for a library entry: a render from this session, else the one it was listed with. */ +export function useAssetThumbnailUrl(entry: { workspacePath: string, thumbnailUrl?: string }): string | undefined { + return useAssetThumbnailStore((state) => state.urls[entry.workspacePath]) ?? entry.thumbnailUrl +} From 5a46e070a6e2158d0e20ddd4b71d4af0c7d65733 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Fri, 13 Nov 2026 10:26:37 +0100 Subject: [PATCH 056/108] feat(library): filtering, sorting and grid windowing for the Library page selectLibraryEntries reuses the Generate picker's visibility, search and sort rules and adds capability and source-scope filters, as one flat list. gridWindow holds the windowing math (columns, visible rows, reveal on keyboard moves). openAssetLibraryEntry factors the picker's open flow so another page can open an entry in Generate. --- package.json | 2 +- src/areas/generate/assetLibraryUi.test.ts | 35 ++++++++++ src/areas/generate/assetLibraryUi.ts | 29 +++++++- src/areas/library/gridWindow.test.ts | 60 +++++++++++++++++ src/areas/library/gridWindow.ts | 79 ++++++++++++++++++++++ src/areas/library/libraryView.test.ts | 64 ++++++++++++++++++ src/areas/library/libraryView.ts | 81 +++++++++++++++++++++++ 7 files changed, 346 insertions(+), 4 deletions(-) create mode 100644 src/areas/library/gridWindow.test.ts create mode 100644 src/areas/library/gridWindow.ts create mode 100644 src/areas/library/libraryView.test.ts create mode 100644 src/areas/library/libraryView.ts diff --git a/package.json b/package.json index f3210b09a..0d3d2bafb 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs src/shared/pages/pageRegistry.test.ts", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs src/shared/pages/pageRegistry.test.ts", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/assetLibraryUi.test.ts b/src/areas/generate/assetLibraryUi.test.ts index e6b9fbca6..98942fd6f 100644 --- a/src/areas/generate/assetLibraryUi.test.ts +++ b/src/areas/generate/assetLibraryUi.test.ts @@ -8,6 +8,7 @@ import { filterAssetLibraryScopeGroups, getDefaultAssetLibraryCollapsedSectionKeys, isAssetLibraryEntryOpenable, + openAssetLibraryEntry, resolveOpenPanelAfterLibrarySelection, toggleAssetLibrarySectionKey, type AssetLibrarySortMode, @@ -123,3 +124,37 @@ test('builds linked-source open requests and import jobs for safe sidecars', () assert.equal(selection?.historyUrl, '/workspace/Workflows/run/hero.glb') assert.equal(selection?.job.outputUrl, '/workspace/Workflows/run/hero.glb') }) + +test('openAssetLibraryEntry opens through the service and builds the viewer job', async () => { + const requests: unknown[] = [] + const service = { + open: async (request: { workspacePath: string, sourceWorkspacePath?: string }) => { + requests.push(request) + return { success: true as const, entry: entry() } + }, + } + const opened = await openAssetLibraryEntry(service, entry(), 42) + assert.deepEqual(requests, [{ workspacePath: 'Workflows/hero.glb' }]) + assert.equal(opened.success, true) + if (!opened.success) throw new Error('expected success') + assert.equal(opened.selection.historyUrl, '/workspace/Workflows/hero.glb') + assert.equal(opened.selection.job.id, 'library-42') +}) + +test('openAssetLibraryEntry refuses unopenable entries without calling the service and relays service errors', async () => { + let calls = 0 + const failing = { + open: async () => { + calls += 1 + return { success: false as const, error: { code: 'not-found' as const, message: 'Gone.' } } + }, + } + const unsafe = entry({ workspacePath: 'C:/outside/hero.glb' }) + assert.deepEqual(await openAssetLibraryEntry(failing, unsafe), { + success: false, + message: describeAssetLibraryOpenability(unsafe), + }) + assert.equal(calls, 0) + assert.deepEqual(await openAssetLibraryEntry(failing, entry()), { success: false, message: 'Gone.' }) + assert.equal(calls, 1) +}) \ No newline at end of file diff --git a/src/areas/generate/assetLibraryUi.ts b/src/areas/generate/assetLibraryUi.ts index 5e83e65a8..a5dae40e3 100644 --- a/src/areas/generate/assetLibraryUi.ts +++ b/src/areas/generate/assetLibraryUi.ts @@ -1,6 +1,7 @@ import type { GenerationJob } from '../../shared/stores/appStore' import type { AssetLibraryOpenRequest } from '../../shared/types/assetLibrary' import { resolveAssetLibraryOpenTarget, type AssetLibraryOpenTarget, type ProjectedAssetLibraryEntry } from './assetLibraryProjection' +import type { ProjectedAssetLibraryOpenResult } from './assetLibraryService' export type GenerateOpenPanel = 'export' | 'decimate' | 'smooth' | 'import' | 'library' | 'light' | null export type AssetLibrarySortMode = 'type' | 'name' | 'date' @@ -24,7 +25,7 @@ export interface AssetLibraryOpenSelection { job: GenerationJob } -const ASSET_LIBRARY_CAPABILITY_SECTIONS = [ +export const ASSET_LIBRARY_CAPABILITY_SECTIONS = [ { capability: 'mesh', label: 'Mesh' }, { capability: 'rigged-mesh', label: 'Rigged mesh' }, { capability: 'animation-motion', label: 'Animations/motions' }, @@ -33,7 +34,7 @@ const ASSET_LIBRARY_CAPABILITY_SECTIONS = [ { capability: 'scene-manifest', label: 'Scene manifests' }, ] as const satisfies ReadonlyArray<{ capability: NonNullable, label: string }> -const ASSET_LIBRARY_SOURCE_SCOPE_SECTIONS = [ +export const ASSET_LIBRARY_SOURCE_SCOPE_SECTIONS = [ { sourceScope: 'workflows', label: 'Workflows' }, { sourceScope: 'exports', label: 'Exports' }, ] as const satisfies ReadonlyArray<{ sourceScope: ProjectedAssetLibraryEntry['sourceScope'], label: string }> @@ -164,11 +165,33 @@ export function createAssetLibraryOpenJob( } } +export type AssetLibraryOpenOutcome = + | { success: true, entry: ProjectedAssetLibraryEntry, selection: AssetLibraryOpenSelection } + | { success: false, message: string } + +/** + * Opens a library entry for the Generate viewer: checks it is openable, asks + * the main process to open it, and builds the viewer job. The caller applies + * `selection` (setCurrentJob + pushMeshUrl), from Generate or another page. + */ +export async function openAssetLibraryEntry( + service: { open: (request: AssetLibraryOpenRequest) => Promise }, + entry: ProjectedAssetLibraryEntry, + now = Date.now(), +): Promise { + if (!isAssetLibraryEntryOpenable(entry)) return { success: false, message: describeAssetLibraryOpenability(entry) } + const result = await service.open(buildAssetLibraryOpenRequest(entry)) + if (!result.success) return { success: false, message: result.error.message } + const selection = createAssetLibraryOpenJob(result.entry, resolveAssetLibraryOpenTarget(result.entry), now) + if (!selection) return { success: false, message: describeAssetLibraryOpenability(result.entry) } + return { success: true, entry: result.entry, selection } +} + export function resolveOpenPanelAfterLibrarySelection(currentPanel: GenerateOpenPanel): GenerateOpenPanel { return currentPanel === 'library' ? 'library' : currentPanel } -function sortAssetLibraryEntries( +export function sortAssetLibraryEntries( entries: ProjectedAssetLibraryEntry[], sortMode: AssetLibrarySortMode, ): ProjectedAssetLibraryEntry[] { diff --git a/src/areas/library/gridWindow.test.ts b/src/areas/library/gridWindow.test.ts new file mode 100644 index 000000000..58d55881e --- /dev/null +++ b/src/areas/library/gridWindow.test.ts @@ -0,0 +1,60 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { computeGridLayout, moveGridIndex, scrollTopToReveal, visibleRowRange } from './gridWindow' + +test('grid columns stretch to fill the width and rows are square tiles plus the label', () => { + const layout = computeGridLayout({ width: 1000, itemCount: 500, gap: 16, minColumnWidth: 160, labelHeight: 40 }) + assert.equal(layout.columns, 5) + assert.equal(layout.columnWidth, 187) + assert.equal(layout.rowHeight, 227) + assert.equal(layout.rowStride, 243) + assert.equal(layout.rowCount, 100) + assert.equal(layout.totalHeight, 100 * 227 + 99 * 16) +}) + +test('grid layout degrades gracefully: one column when narrow, nothing when empty', () => { + const narrow = computeGridLayout({ width: 100, itemCount: 3, gap: 16, minColumnWidth: 160, labelHeight: 40 }) + assert.equal(narrow.columns, 1) + assert.equal(narrow.rowCount, 3) + const empty = computeGridLayout({ width: 0, itemCount: 0, gap: 16, minColumnWidth: 160 }) + assert.equal(empty.rowCount, 0) + assert.equal(empty.totalHeight, 0) +}) + +test('a fixed row height gives a list layout', () => { + const list = computeGridLayout({ width: 800, itemCount: 10, gap: 2, minColumnWidth: Number.MAX_SAFE_INTEGER, rowHeight: 52 }) + assert.equal(list.columns, 1) + assert.equal(list.columnWidth, 800) + assert.equal(list.rowStride, 54) + assert.equal(list.totalHeight, 10 * 52 + 9 * 2) +}) + +test('only rows near the viewport are rendered, whatever the library size', () => { + const layout = { rowStride: 100, rowCount: 1000 } + assert.deepEqual(visibleRowRange(layout, 0, 350, 2), { start: 0, end: 6 }) + assert.deepEqual(visibleRowRange(layout, 5000, 350, 2), { start: 48, end: 56 }) + assert.deepEqual(visibleRowRange(layout, 99_950, 350, 2), { start: 997, end: 1000 }) + assert.deepEqual(visibleRowRange({ rowStride: 100, rowCount: 0 }, 0, 350), { start: 0, end: 0 }) + assert.deepEqual(visibleRowRange(layout, -40, 0, 0), { start: 0, end: 1 }) +}) + +test('scrollTopToReveal scrolls the least needed to show a row', () => { + const layout = { rowStride: 110, rowHeight: 100 } + assert.equal(scrollTopToReveal(layout, 2, 0, 400), null) + assert.equal(scrollTopToReveal(layout, 5, 0, 400), 550 + 100 - 400) + assert.equal(scrollTopToReveal(layout, 1, 500, 400), 110) +}) + +test('arrow keys move through the grid and stay in bounds', () => { + assert.equal(moveGridIndex(0, 'ArrowRight', 4, 10), 1) + assert.equal(moveGridIndex(0, 'ArrowLeft', 4, 10), 0) + assert.equal(moveGridIndex(1, 'ArrowDown', 4, 10), 5) + assert.equal(moveGridIndex(8, 'ArrowDown', 4, 10), 9) + assert.equal(moveGridIndex(5, 'ArrowUp', 4, 10), 1) + assert.equal(moveGridIndex(5, 'End', 4, 10), 9) + assert.equal(moveGridIndex(5, 'Home', 4, 10), 0) + assert.equal(moveGridIndex(-1, 'ArrowDown', 4, 10), 0) + assert.equal(moveGridIndex(3, 'a', 4, 10), null) + assert.equal(moveGridIndex(0, 'ArrowDown', 4, 0), null) +}) diff --git a/src/areas/library/gridWindow.ts b/src/areas/library/gridWindow.ts new file mode 100644 index 000000000..bd5c48ca0 --- /dev/null +++ b/src/areas/library/gridWindow.ts @@ -0,0 +1,79 @@ +// Windowing math for the Library grid: only the rows in (or near) the viewport +// are rendered, so a large library scrolls at a constant cost. + +export interface GridLayoutOptions { + /** Content width available to the rows, in px. */ + width: number + itemCount: number + gap: number + /** Narrowest a column may get; columns stretch to fill the width. */ + minColumnWidth: number + /** Fixed row height; when absent rows are square tiles plus `labelHeight`. */ + rowHeight?: number + labelHeight?: number +} + +export interface GridLayout { + columns: number + columnWidth: number + rowHeight: number + /** Row height plus the gap below it. */ + rowStride: number + rowCount: number + totalHeight: number +} + +export function computeGridLayout(options: GridLayoutOptions): GridLayout { + const width = Math.max(0, options.width) + const gap = Math.max(0, options.gap) + const columns = Math.max(1, Math.floor((width + gap) / (Math.max(1, options.minColumnWidth) + gap))) + const columnWidth = Math.max(0, Math.floor((width - gap * (columns - 1)) / columns)) + const rowHeight = options.rowHeight ?? columnWidth + (options.labelHeight ?? 0) + const rowCount = Math.ceil(Math.max(0, options.itemCount) / columns) + const rowStride = rowHeight + gap + const totalHeight = rowCount === 0 ? 0 : rowCount * rowHeight + (rowCount - 1) * gap + return { columns, columnWidth, rowHeight, rowStride, rowCount, totalHeight } +} + +export interface RowRange { + /** First row to render. */ + start: number + /** One past the last row to render. */ + end: number +} + +/** Rows intersecting the viewport, plus `overscan` rows on each side. */ +export function visibleRowRange(layout: Pick, scrollTop: number, viewportHeight: number, overscan = 2): RowRange { + if (layout.rowCount === 0 || layout.rowStride <= 0) return { start: 0, end: 0 } + const top = Math.max(0, scrollTop) + const first = Math.min(layout.rowCount - 1, Math.floor(top / layout.rowStride)) + const last = Math.ceil((top + Math.max(0, viewportHeight)) / layout.rowStride) + return { + start: Math.max(0, first - overscan), + end: Math.min(layout.rowCount, Math.max(first + 1, last) + overscan), + } +} + +/** The scrollTop that brings a row fully into view, or null when it already is. */ +export function scrollTopToReveal(layout: Pick, row: number, scrollTop: number, viewportHeight: number): number | null { + const top = row * layout.rowStride + const bottom = top + layout.rowHeight + if (top < scrollTop) return top + if (bottom > scrollTop + viewportHeight) return Math.max(0, bottom - viewportHeight) + return null +} + +/** Arrow-key movement through a grid of `count` items laid out in `columns`. */ +export function moveGridIndex(index: number, key: string, columns: number, count: number): number | null { + if (count === 0) return null + if (index < 0) return 0 + const next = key === 'ArrowRight' ? index + 1 + : key === 'ArrowLeft' ? index - 1 + : key === 'ArrowDown' ? index + columns + : key === 'ArrowUp' ? index - columns + : key === 'Home' ? 0 + : key === 'End' ? count - 1 + : null + if (next === null) return null + return Math.min(count - 1, Math.max(0, next)) +} diff --git a/src/areas/library/libraryView.test.ts b/src/areas/library/libraryView.test.ts new file mode 100644 index 000000000..2589ecd59 --- /dev/null +++ b/src/areas/library/libraryView.test.ts @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import type { AssetLibraryEntry } from '../../shared/types/assetLibrary' +import { projectAssetLibraryEntry, type ProjectedAssetLibraryEntry } from '../generate/assetLibraryProjection' +import { + DEFAULT_LIBRARY_QUERY, + capabilityLabel, + libraryCapabilityOptions, + libraryScopeOptions, + selectLibraryEntries, + type LibraryQuery, +} from './libraryView' + +function entry(name: string, overrides: Partial = {}): ProjectedAssetLibraryEntry { + return projectAssetLibraryEntry({ + id: `library:${name}`, + workspacePath: `Workflows/run/${name}`, + displayName: name, + sourceScope: 'workflows', + capability: 'mesh', + state: 'ready', + previewKind: '3d-model', + warnings: [], + openable: true, + ...overrides, + }) +} + +const ENTRIES = [ + entry('zebra.glb', { createdAt: '2026-01-01T00:00:00.000Z' }), + entry('apple.glb', { createdAt: '2026-03-01T00:00:00.000Z' }), + entry('rig.glb', { capability: 'rigged-mesh', createdAt: '2026-02-01T00:00:00.000Z' }), + entry('export.glb', { sourceScope: 'exports', workspacePath: 'Exports/export.glb', createdAt: '2025-12-01T00:00:00.000Z' }), + entry('hidden.glb', { workspacePath: 'Workflows/run/tmp/hidden.glb' }), + entry('unsupported.bin', { state: 'unsupported' }), + entry('loose.glb', { capability: undefined }), +] + +const names = (entries: ProjectedAssetLibraryEntry[]) => entries.map((e) => e.displayName) +const query = (overrides: Partial): LibraryQuery => ({ ...DEFAULT_LIBRARY_QUERY, ...overrides }) + +test('sorts globally by date (newest first) or name, hiding entries the picker hides', () => { + assert.deepEqual(names(selectLibraryEntries(ENTRIES, query({ sort: 'date' }))), ['apple.glb', 'rig.glb', 'zebra.glb', 'export.glb']) + assert.deepEqual(names(selectLibraryEntries(ENTRIES, query({ sort: 'name' }))), ['apple.glb', 'export.glb', 'rig.glb', 'zebra.glb']) +}) + +test('type sort keeps the picker order: scope, then capability, then name', () => { + assert.deepEqual(names(selectLibraryEntries(ENTRIES, query({ sort: 'type' }))), ['apple.glb', 'zebra.glb', 'rig.glb', 'export.glb']) +}) + +test('filters by capability, source scope and search together', () => { + assert.deepEqual(names(selectLibraryEntries(ENTRIES, query({ capability: 'rigged-mesh' }))), ['rig.glb']) + assert.deepEqual(names(selectLibraryEntries(ENTRIES, query({ scope: 'exports' }))), ['export.glb']) + assert.deepEqual(names(selectLibraryEntries(ENTRIES, query({ search: 'APP' }))), ['apple.glb']) + assert.deepEqual(names(selectLibraryEntries(ENTRIES, query({ search: 'apple', capability: 'rigged-mesh' }))), []) +}) + +test('filter options count what each choice would show', () => { + assert.deepEqual(libraryScopeOptions(ENTRIES).map((o) => [o.value, o.count]), [['all', 4], ['workflows', 3], ['exports', 1]]) + assert.deepEqual(libraryCapabilityOptions(ENTRIES).map((o) => [o.value, o.count]), [['all', 4], ['mesh', 3], ['rigged-mesh', 1]]) + assert.equal(capabilityLabel('rigged-mesh'), 'Rigged mesh') + assert.equal(capabilityLabel(undefined), 'Asset') +}) diff --git a/src/areas/library/libraryView.ts b/src/areas/library/libraryView.ts new file mode 100644 index 000000000..0c92235f6 --- /dev/null +++ b/src/areas/library/libraryView.ts @@ -0,0 +1,81 @@ +import type { AssetCapability, AssetLibrarySourceScope } from '../../shared/types/assetLibrary' +import type { ProjectedAssetLibraryEntry } from '../generate/assetLibraryProjection' +import { + ASSET_LIBRARY_CAPABILITY_SECTIONS, + ASSET_LIBRARY_SOURCE_SCOPE_SECTIONS, + filterAssetLibraryScopeGroups, + filterVisibleAssetLibraryEntries, + sortAssetLibraryEntries, + type AssetLibrarySortMode, +} from '../generate/assetLibraryUi' + +// What the Library page shows: the Generate picker's visibility, search and +// sort rules, plus capability and source-scope filters, as one flat list for +// the grid. + +export type LibraryCapabilityFilter = AssetCapability | 'all' +export type LibraryScopeFilter = AssetLibrarySourceScope | 'all' +export type LibraryViewMode = 'grid' | 'list' + +export interface LibraryQuery { + search: string + capability: LibraryCapabilityFilter + scope: LibraryScopeFilter + sort: AssetLibrarySortMode +} + +export interface LibraryFilterOption { + value: T + label: string + count: number +} + +export const DEFAULT_LIBRARY_QUERY: LibraryQuery = { search: '', capability: 'all', scope: 'all', sort: 'date' } + +const CAPABILITY_LABELS = new Map(ASSET_LIBRARY_CAPABILITY_SECTIONS.map((s) => [s.capability, s.label])) + +export function capabilityLabel(capability: AssetCapability | undefined): string { + return (capability && CAPABILITY_LABELS.get(capability)) ?? 'Asset' +} + +/** The entries to show, filtered and sorted. Type sort keeps the picker's scope → capability order. */ +export function selectLibraryEntries(entries: ProjectedAssetLibraryEntry[], query: LibraryQuery): ProjectedAssetLibraryEntry[] { + const inFilters = entries.filter((entry) => ( + (query.scope === 'all' || entry.sourceScope === query.scope) + && (query.capability === 'all' || entry.capability === query.capability) + )) + const matches = filterAssetLibraryScopeGroups(inFilters, query.search, query.sort) + .flatMap((scopeGroup) => scopeGroup.entryGroups.flatMap((group) => group.entries)) + return query.sort === 'type' ? matches : sortAssetLibraryEntries(matches, query.sort) +} + +/** Scope filter options with the number of visible entries in each (ignoring the other filters). */ +export function libraryScopeOptions(entries: ProjectedAssetLibraryEntry[]): LibraryFilterOption[] { + const visible = visibleWithCapability(entries) + return [ + { value: 'all', label: 'All', count: visible.length }, + ...ASSET_LIBRARY_SOURCE_SCOPE_SECTIONS.map((section) => ({ + value: section.sourceScope, + label: section.label, + count: visible.filter((entry) => entry.sourceScope === section.sourceScope).length, + })), + ] +} + +/** Capability filter options: "All", then each capability present in the library. */ +export function libraryCapabilityOptions(entries: ProjectedAssetLibraryEntry[]): LibraryFilterOption[] { + const visible = visibleWithCapability(entries) + const present = ASSET_LIBRARY_CAPABILITY_SECTIONS + .map((section) => ({ + value: section.capability as LibraryCapabilityFilter, + label: section.label, + count: visible.filter((entry) => entry.capability === section.capability).length, + })) + .filter((option) => option.count > 0) + return [{ value: 'all', label: 'All types', count: visible.length }, ...present] +} + +// The picker only lists entries with a known capability; count the same set. +function visibleWithCapability(entries: ProjectedAssetLibraryEntry[]): ProjectedAssetLibraryEntry[] { + return filterVisibleAssetLibraryEntries(entries).filter((entry) => entry.capability !== undefined) +} From be2f22621515d2704595ba2acd8e003d86a7020a Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Fri, 13 Nov 2026 10:26:38 +0100 Subject: [PATCH 057/108] test: run the page registry tests under the extensionless TS loader The test imports ./pageRegistry without an extension, so the plain node --test pass could not resolve it. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0d3d2bafb..74c026e63 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs src/shared/pages/pageRegistry.test.ts", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." From bb435ed525404731916b21c8ec0349bc21c90277 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Fri, 13 Nov 2026 10:27:48 +0100 Subject: [PATCH 058/108] feat(library): Library page with a virtualized thumbnail grid New Library page (group manage) listing workspace assets as square thumbnail tiles or a compact list, with search, source-scope and type filters, sort, and a grid/list toggle. Only the rows near the viewport are mounted, so large workspaces scroll at constant cost. Tiles without a thumbnail show their capability icon and repaint when a background render lands; unopenable entries stay dimmed with their reason. Double-click (or Enter, or the status-bar button) opens the asset in Generate. --- src/areas/library/LibraryGrid.tsx | 300 ++++++++++++++++ src/areas/library/LibraryPage.tsx | 338 ++++++++++++++++++ .../components/agent/clientTools.test.ts | 2 +- src/shared/pages/pageRegistry.test.ts | 3 +- src/shared/pages/pageRegistry.ts | 13 + 5 files changed, 654 insertions(+), 2 deletions(-) create mode 100644 src/areas/library/LibraryGrid.tsx create mode 100644 src/areas/library/LibraryPage.tsx diff --git a/src/areas/library/LibraryGrid.tsx b/src/areas/library/LibraryGrid.tsx new file mode 100644 index 000000000..b6d8f10f2 --- /dev/null +++ b/src/areas/library/LibraryGrid.tsx @@ -0,0 +1,300 @@ +import { createElement, useEffect, useLayoutEffect, useRef, useState, type CSSProperties, type KeyboardEvent, type RefObject } from 'react' +import type { IconShape } from '@shared/pages/pageRegistry' +import type { AssetCapability } from '@shared/types/assetLibrary' +import { useAssetThumbnailUrl } from '@shared/thumbnails/assetThumbnailStore' +import type { ProjectedAssetLibraryEntry } from '@areas/generate/assetLibraryProjection' +import { describeAssetLibraryOpenability, isAssetLibraryEntryOpenable } from '@areas/generate/assetLibraryUi' +import { computeGridLayout, moveGridIndex, scrollTopToReveal, visibleRowRange, type GridLayout } from './gridWindow' +import { capabilityLabel, type LibraryViewMode } from './libraryView' + +// Side padding of the scroll area (matches the page's px-7). +const PADDING_X = 28 +const PADDING_Y = 4 +const GRID = { gap: 16, minColumnWidth: 164, labelHeight: 46 } +const LIST = { gap: 2, rowHeight: 56 } +const OVERSCAN_ROWS = 3 + +const CAPABILITY_ICONS: Record = { + mesh: [ + ['path', { d: 'M21 16V8a2 2 0 0 0-1-1.73l-7-4a2 2 0 0 0-2 0l-7 4A2 2 0 0 0 3 8v8a2 2 0 0 0 1 1.73l7 4a2 2 0 0 0 2 0l7-4A2 2 0 0 0 21 16z' }], + ['polyline', { points: '3.27 6.96 12 12.01 20.73 6.96' }], + ['line', { x1: 12, y1: 22.08, x2: 12, y2: 12 }], + ], + 'rigged-mesh': [ + ['circle', { cx: 12, cy: 4.5, r: 2 }], + ['path', { d: 'M12 6.5v7M12 13.5l-4 7M12 13.5l4 7M5.5 10h13' }], + ], + 'animation-motion': [ + ['circle', { cx: 12, cy: 12, r: 9 }], + ['path', { d: 'M10 8.5v7l6-3.5-6-3.5z' }], + ], + 'landmarks-sidecar': [ + ['path', { d: 'M12 21s-7-6.2-7-11.5a7 7 0 0 1 14 0C19 14.8 12 21 12 21z' }], + ['circle', { cx: 12, cy: 9.5, r: 2.5 }], + ], + 'generated-world': [ + ['circle', { cx: 12, cy: 12, r: 9 }], + ['path', { d: 'M3 12h18M12 3a14 14 0 0 1 0 18M12 3a14 14 0 0 0 0 18' }], + ], + 'scene-manifest': [ + ['path', { d: 'M14 3H6a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V9z' }], + ['polyline', { points: '14 3 14 9 20 9' }], + ['line', { x1: 8, y1: 13, x2: 16, y2: 13 }], + ['line', { x1: 8, y1: 17, x2: 13, y2: 17 }], + ], + unknown: [['rect', { x: 4, y: 4, width: 16, height: 16, rx: 2 }]], +} + +export function CapabilityIcon({ capability, size = 18, className }: { capability?: AssetCapability, size?: number, className?: string }): JSX.Element { + return ( + + ) +} + +const DATE_FORMAT = new Intl.DateTimeFormat(undefined, { day: 'numeric', month: 'short', year: 'numeric' }) + +export function formatEntryDate(entry: ProjectedAssetLibraryEntry): string | null { + const value = entry.createdAt ?? entry.updatedAt + const time = value ? Date.parse(value) : Number.NaN + return Number.isFinite(time) ? DATE_FORMAT.format(time) : null +} + +/** The asset's thumbnail, or its capability icon until one is rendered (or if it fails to load). */ +function Thumbnail({ entry, iconSize }: { entry: ProjectedAssetLibraryEntry, iconSize: number }): JSX.Element { + const url = useAssetThumbnailUrl(entry) + const [failedUrl, setFailedUrl] = useState(null) + if (url && url !== failedUrl) { + return ( + setFailedUrl(url)} + className="h-full w-full object-cover transition-transform duration-300 ease-out group-hover:scale-[1.04]" + /> + ) + } + return ( +
+ +
+ ) +} + +function LockBadge(): JSX.Element { + return ( + + + + ) +} + +function Spinner(): JSX.Element { + return
From d3e6a5b1b9f6df3a6f1529cd89fcc82e5d33e79a Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Fri, 13 Nov 2026 10:04:52 +0100 Subject: [PATCH 060/108] feat(runs): add a persisted run history store One RunRecord per generation, workflow or mesh-op run, kept in a JSON file at the workspace root. Writes are atomic, a corrupt file is moved aside, runs a previous backend left in flight are marked interrupted on load, and the history is capped. The record shape is the one provenance (T-F1) will reuse. --- api/schemas/runs.py | 62 ++++++ api/services/run_history.py | 364 ++++++++++++++++++++++++++++++++++++ 2 files changed, 426 insertions(+) create mode 100644 api/schemas/runs.py create mode 100644 api/services/run_history.py diff --git a/api/schemas/runs.py b/api/schemas/runs.py new file mode 100644 index 000000000..3d95902fc --- /dev/null +++ b/api/schemas/runs.py @@ -0,0 +1,62 @@ +from typing import Any, Literal, Optional +from pydantic import BaseModel, Field, computed_field + +RunKind = Literal["generation", "workflow", "mesh_op"] +# JobStatus' vocabulary plus "interrupted": a run that was still pending/running when +# the backend stopped. It is terminal -- nothing will ever finish it. +RunStatusValue = Literal["pending", "running", "done", "error", "cancelled", "interrupted"] + +TERMINAL_RUN_STATUSES = frozenset({"done", "error", "cancelled", "interrupted"}) + + +class RunStep(BaseModel): + """One timed phase of a run: a generation step label, or a workflow node.""" + name: str + node_id: Optional[str] = None + status: RunStatusValue = "running" + started_at: float # epoch seconds + ended_at: Optional[float] = None + + +class RunRecord(BaseModel): + """What ran, with what, for how long, and what came out of it. + + One record per run, whatever drove it: a backend generation job, a workflow + executed by the renderer, or a mesh operation. This is deliberately the shape + the provenance model (T-F1) is meant to reuse -- an output's provenance is the + record of the run that produced it (model or workflow, inputs, params, + output_path) -- so there is one store of "how was this made", not two. + + Timestamps are epoch seconds. ``output_path`` is workspace-relative (POSIX + separators) when the output lives in the workspace, absolute otherwise. + """ + id: str + kind: RunKind + status: RunStatusValue = "pending" + model_id: Optional[str] = None # generation model, e.g. "triposg/generate" + workflow_id: Optional[str] = None + workflow_name: Optional[str] = None + op_id: Optional[str] = None # mesh operation id, e.g. "decimate" + inputs: dict[str, Any] = Field(default_factory=dict) + params: dict[str, Any] = Field(default_factory=dict) + steps: list[RunStep] = Field(default_factory=list) + error: Optional[str] = None + output_path: Optional[str] = None + created_at: float + started_at: Optional[float] = None + ended_at: Optional[float] = None + + @computed_field # type: ignore[prop-decorator] + @property + def duration_s(self) -> Optional[float]: + start = self.started_at if self.started_at is not None else self.created_at + if self.ended_at is None: + return None + return round(max(0.0, self.ended_at - start), 3) + + +class RunRecordPage(BaseModel): + runs: list[RunRecord] + total: int + limit: int + offset: int diff --git a/api/services/run_history.py b/api/services/run_history.py new file mode 100644 index 000000000..77c7261b6 --- /dev/null +++ b/api/services/run_history.py @@ -0,0 +1,364 @@ +"""Persisted run history (one record per generation, workflow or mesh-op run). + +The live job table in ``routers.generation`` is in memory and purged 30 minutes +after a job ends, so on its own nothing survives a restart. This store keeps a +``RunRecord`` (see ``schemas.runs`` -- the shape the provenance model reuses) per +run in a single JSON file at the workspace root. + +Robustness rules: +- Writes are atomic (temp file + ``os.replace``) so a crash never leaves half a file. +- A missing file is an empty history; a corrupt one is moved aside and replaced. +- Records still pending/running in a file this process did not write them to were + left behind by a backend that stopped mid-run: they become ``interrupted``. +- The history is capped (``max_records``); the oldest finished runs go first. +- I/O failures are logged, never raised: history must not break a generation. + +The store is inert until ``start()`` gives it a path, so importing the routers +(tests, tooling) never writes into a real workspace. +""" +from __future__ import annotations + +import json +import os +import re +import tempfile +import threading +import time +from pathlib import Path +from typing import Any, Callable, Iterable, Optional + +from pydantic import ValidationError + +from schemas.runs import TERMINAL_RUN_STATUSES, RunKind, RunRecord, RunStep + +RUN_HISTORY_FILENAME = ".modly-run-history.json" +DEFAULT_MAX_RECORDS = 500 +MAX_STEPS = 200 +MAX_ERROR_CHARS = 4000 +_FILE_VERSION = 1 +INTERRUPTED_ERROR = "Interrupted: Modly stopped before this run finished" + +# Extensions report progress through the step label ("Sampling 12/50", "Denoising 40%"), +# so counters are folded out before deciding whether a label starts a new step. +_STEP_COUNTERS = re.compile(r"[\d.,%/]+") + + +def _step_key(name: str) -> str: + return _STEP_COUNTERS.sub("#", name).strip() + + +def _truncate_error(error: Optional[str]) -> Optional[str]: + if error is None or len(error) <= MAX_ERROR_CHARS: + return error + # Tracebacks end with the actual exception, so keep the tail. + return "…" + error[-(MAX_ERROR_CHARS - 1):] + + +class RunHistoryStore: + def __init__(self, max_records: int = DEFAULT_MAX_RECORDS) -> None: + self.max_records = max_records + self._lock = threading.RLock() + self._path_provider: Optional[Callable[[], Path]] = None + self._loaded_path: Optional[Path] = None + self._records: dict[str, RunRecord] = {} + # Runs this process is still driving; everything else non-terminal on disk is stale. + self._live: dict[str, RunRecord] = {} + + # ------------------------------------------------------------------ # + # Lifecycle + # ------------------------------------------------------------------ # + + def start(self, path_provider: Callable[[], Path]) -> None: + """Enable persistence. The provider is called per operation, so a relocated + workspace is followed. Loading marks runs a previous backend left behind as + interrupted.""" + with self._lock: + self._path_provider = path_provider + self._loaded_path = None + self._records = {} + self._live = {} + self._sync_path() + + def stop(self) -> None: + """Graceful shutdown: runs still in flight will never finish.""" + with self._lock: + if self._path_provider is None: + return + if self._live: + self._sync_path() + now = time.time() + for record in self._live.values(): + self._interrupt(record, now) + self._records[record.id] = record + self._live = {} + self._persist() + self._path_provider = None + self._loaded_path = None + self._records = {} + + @property + def enabled(self) -> bool: + return self._path_provider is not None + + # ------------------------------------------------------------------ # + # Recording + # ------------------------------------------------------------------ # + + def begin( + self, + run_id: str, + kind: RunKind, + *, + model_id: Optional[str] = None, + op_id: Optional[str] = None, + inputs: Optional[dict[str, Any]] = None, + params: Optional[dict[str, Any]] = None, + ) -> None: + """File a pending run (persisted, so a crash before it starts still shows).""" + if not self.enabled: + return + record = RunRecord( + id=run_id, kind=kind, status="pending", model_id=model_id, op_id=op_id, + inputs=inputs or {}, params=params or {}, + created_at=time.time(), + ) + self.save(record) + + def mark_running(self, run_id: str) -> None: + with self._lock: + record = self._live.get(run_id) + if record is None or record.status in TERMINAL_RUN_STATUSES: + return + record.status = "running" + record.started_at = record.started_at or time.time() + self.save(record) + + def step(self, run_id: str, name: str) -> None: + """Note the current step label. In memory only -- labels can change many times + a second; the steps are written out with the next status change.""" + if not name: + return + with self._lock: + record = self._live.get(run_id) + if record is None: + return + now = time.time() + current = record.steps[-1] if record.steps else None + if current is not None and current.ended_at is None: + if _step_key(current.name) == _step_key(name): + return + if len(record.steps) >= MAX_STEPS: + return # the last step absorbs the rest of the run + current.ended_at = now + current.status = "done" + record.steps.append(RunStep(name=name, started_at=now)) + + def finish( + self, + run_id: str, + status: str, + *, + error: Optional[str] = None, + output_path: Optional[str] = None, + ) -> None: + with self._lock: + record = self._live.get(run_id) + if record is None: + return + now = time.time() + record.status = status # type: ignore[assignment] + record.ended_at = now + if error is not None: + record.error = error + if output_path is not None: + record.output_path = output_path + _close_steps(record, now) + self.save(record) + + def save(self, record: RunRecord) -> None: + """Insert or replace a whole record (used directly for renderer-driven workflow + runs). A non-terminal record is tracked as live by this process.""" + if not self.enabled: + return + with self._lock: + self._sync_path() + record.inputs = _json_safe(record.inputs) + record.params = _json_safe(record.params) + record.error = _truncate_error(record.error) + if len(record.steps) > MAX_STEPS: + record.steps = record.steps[:MAX_STEPS] + if record.status in TERMINAL_RUN_STATUSES: + self._live.pop(record.id, None) + else: + self._live[record.id] = record + self._records[record.id] = record + self._apply_retention() + self._persist() + + # ------------------------------------------------------------------ # + # Reading + # ------------------------------------------------------------------ # + + def get(self, run_id: str) -> Optional[RunRecord]: + if not self.enabled: + return None + with self._lock: + self._sync_path() + record = self._records.get(run_id) + return record.model_copy(deep=True) if record is not None else None + + def list( + self, + *, + statuses: Optional[Iterable[str]] = None, + kinds: Optional[Iterable[str]] = None, + limit: int = 50, + offset: int = 0, + ) -> tuple[list[RunRecord], int]: + """Newest first. Returns (page, total matching).""" + if not self.enabled: + return [], 0 + wanted_status = set(statuses) if statuses else None + wanted_kind = set(kinds) if kinds else None + with self._lock: + self._sync_path() + matching = [ + r for r in self._records.values() + if (wanted_status is None or r.status in wanted_status) + and (wanted_kind is None or r.kind in wanted_kind) + ] + matching.sort(key=lambda r: r.created_at, reverse=True) + page = matching[offset:offset + limit] + return [r.model_copy(deep=True) for r in page], len(matching) + + # ------------------------------------------------------------------ # + # Internals (lock held) + # ------------------------------------------------------------------ # + + def _sync_path(self) -> None: + """(Re)load when the workspace path changed since the last load.""" + assert self._path_provider is not None + path = Path(self._path_provider()) + if path == self._loaded_path: + return + self._loaded_path = path + self._records = self._load(path) + now = time.time() + stale = False + for record in self._records.values(): + if record.status not in TERMINAL_RUN_STATUSES and record.id not in self._live: + self._interrupt(record, now) + stale = True + # Live runs keep being written wherever the workspace now is. + for run_id, record in self._live.items(): + self._records[run_id] = record + if stale: + self._persist() + + @staticmethod + def _interrupt(record: RunRecord, now: float) -> None: + record.status = "interrupted" + record.error = record.error or INTERRUPTED_ERROR + # When it really stopped is unknown; leave ended_at empty rather than invent one. + for step in record.steps: + if step.ended_at is None: + step.status = "interrupted" + + def _load(self, path: Path) -> dict[str, RunRecord]: + try: + raw = path.read_text(encoding="utf-8") + except FileNotFoundError: + return {} + except OSError as exc: + print(f"[RunHistory] Cannot read {path}: {exc}") + return {} + try: + data = json.loads(raw) + entries = data["runs"] if isinstance(data, dict) else None + if not isinstance(entries, list): + raise ValueError("missing 'runs' list") + except (ValueError, KeyError, TypeError) as exc: + self._quarantine(path, exc) + return {} + records: dict[str, RunRecord] = {} + for entry in entries: + try: + record = RunRecord.model_validate(entry) + except ValidationError: + continue # one bad record must not cost the whole history + records[record.id] = record + return records + + @staticmethod + def _quarantine(path: Path, exc: Exception) -> None: + target = path.with_name(f"{path.name}.corrupt-{int(time.time())}") + try: + os.replace(path, target) + print(f"[RunHistory] {path} is unreadable ({exc}); moved to {target.name}") + except OSError as move_exc: + print(f"[RunHistory] {path} is unreadable ({exc}) and could not be moved: {move_exc}") + + def _apply_retention(self) -> None: + excess = len(self._records) - self.max_records + if excess <= 0: + return + finished = sorted( + (r for r in self._records.values() if r.id not in self._live), + key=lambda r: r.created_at, + ) + for record in finished[:excess]: + del self._records[record.id] + + def _persist(self) -> None: + path = self._loaded_path + if path is None: + return + ordered = sorted(self._records.values(), key=lambda r: r.created_at, reverse=True) + payload = { + "version": _FILE_VERSION, + "runs": [r.model_dump(mode="json", exclude={"duration_s"}) for r in ordered], + } + tmp_name: Optional[str] = None + try: + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(prefix=f"{path.name}.", suffix=".tmp", dir=path.parent) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_name, path) + tmp_name = None + except OSError as exc: + print(f"[RunHistory] Cannot write {path}: {exc}") + finally: + if tmp_name is not None: + try: + os.unlink(tmp_name) + except OSError: + pass + + +def _close_steps(record: RunRecord, now: float) -> None: + for step in record.steps: + if step.ended_at is None: + step.ended_at = now + step.status = record.status + + +def _json_safe(value: dict[str, Any]) -> dict[str, Any]: + """Params come from callers; keep only what survives a JSON round trip.""" + try: + return json.loads(json.dumps(value, default=str)) + except (TypeError, ValueError): + return {} + + +def workspace_relative(path: Path, workspace: Path) -> str: + """Workspace-relative POSIX path when inside the workspace, absolute otherwise.""" + try: + return path.resolve().relative_to(workspace.resolve()).as_posix() + except (OSError, ValueError): + return str(path) + + +run_history = RunHistoryStore() From adc686f52c71719a9c146b6947dc1db82deacd48 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Fri, 13 Nov 2026 10:04:53 +0100 Subject: [PATCH 061/108] feat(runs): record generations and mesh ops, list them at GET /workflow-runs Generation jobs are recorded when filed, started and ended (cancel included), with their step timings. Mesh ops that write a mesh or fail are recorded too. GET /workflow-runs pages the history with status/kind filters, PUT /workflow-runs/{id} lets the renderer report workflow runs, and GET /workflow-runs/{id} falls back to the history once a job is purged. --- api/main.py | 10 +- api/routers/generation.py | 37 ++- api/routers/optimize.py | 42 ++- api/routers/workflow_runs.py | 77 ++++- api/tests/test_generation_router.py | 2 + api/tests/test_run_history.py | 368 ++++++++++++++++++++++ api/tests/test_workflow_runs_lifecycle.py | 2 + api/tests/test_workflow_runs_router.py | 2 + 8 files changed, 532 insertions(+), 8 deletions(-) create mode 100644 api/tests/test_run_history.py diff --git a/api/main.py b/api/main.py index 958ba2a02..9bf352304 100644 --- a/api/main.py +++ b/api/main.py @@ -20,8 +20,16 @@ async def lifespan(app: FastAPI): # Startup: initialize the registry (instantiates all adapters) from services.generator_registry import generator_registry generator_registry.initialize() + # Persisted run history at the workspace root; read per call so a relocated + # workspace is followed. Loading marks runs a previous backend left running as + # interrupted. + import services.generator_registry as registry + from services.run_history import RUN_HISTORY_FILENAME, run_history + run_history.start(lambda: registry.WORKSPACE_DIR / RUN_HISTORY_FILENAME) yield - # Shutdown: unload all models and stop the local LLM server + # Shutdown: runs still in flight will never finish + run_history.stop() + # Unload all models and stop the local LLM server generator_registry.unload_all() from services.llm_server import llama_pool llama_pool.unload_all(force=True) diff --git a/api/routers/generation.py b/api/routers/generation.py index 33686cc3e..b1dd63b8c 100644 --- a/api/routers/generation.py +++ b/api/routers/generation.py @@ -17,6 +17,7 @@ import services.generator_registry as registry from services.generator_registry import generator_registry from schemas.generation import GenerateFromArtifactRequest, JobStatus +from services.run_history import run_history from services.artifact_input import ( RESERVED_ARTIFACT_PARAMS, TypedArtifactInput, @@ -56,6 +57,30 @@ def _purge_old_jobs() -> None: _completed_at.pop(jid, None) +def record_job_filed( + job_id: str, + model_id: str, + inputs: dict, + params: dict, + collection: str, +) -> None: + """File the persisted history record of a generation job (see services.run_history).""" + run_history.begin( + job_id, "generation", model_id=model_id, inputs=inputs, + params={**params, "collection": collection}, + ) + + +def _record_job_end(job_id: str) -> None: + """Copy a job's final state into its history record once its worker returned.""" + job = _jobs.get(job_id) + if job is None or job.status not in ("done", "error", "cancelled"): + run_history.finish(job_id, "error", error="Generation ended without a final status") + return + output_path = job.output_url.removeprefix("/workspace/") if job.output_url else None + run_history.finish(job_id, job.status, error=job.error, output_path=output_path) + + def sanitize_collection(collection: str) -> str: """Normalize a caller-supplied collection name into a safe workspace subfolder. @@ -146,6 +171,8 @@ async def generate_from_image( _jobs[job_id] = job _cancel_events[job_id] = threading.Event() + record_job_filed(job_id, model_id, {"image": image.filename}, full_params, collection) + background_tasks.add_task( _run_generation, job_id, image_bytes, full_params, collection, output_kind, model_id ) @@ -172,9 +199,12 @@ async def generate_from_artifact( raise HTTPException(400, str(exc)) from exc params = {k: v for k, v in request.params.items() if k not in RESERVED_ARTIFACT_PARAMS} - params["scene_manifest_path"] = str(artifact.path) collection = sanitize_collection(request.collection) job_id = str(uuid.uuid4()) + record_job_filed( + job_id, request.model_id, {request.input_kind: request.input_path}, params, collection + ) + params["scene_manifest_path"] = str(artifact.path) _purge_old_jobs() _jobs[job_id] = JobStatus(job_id=job_id, status="pending", progress=0) _cancel_events[job_id] = threading.Event() @@ -207,6 +237,7 @@ async def cancel_job(job_id: str): _cancel_events[job_id].set() job.status = "cancelled" _completed_at[job_id] = time.monotonic() + run_history.finish(job_id, "cancelled") # Kill only the subprocess bound to this job. A queued cancellation must not # terminate whichever earlier job currently owns the active generator, and an # unpinned job runs beside the pinned worker, so it can share its generator: @@ -274,6 +305,8 @@ async def _run_generation( job.status = "cancelled" _completed_at[job_id] = time.monotonic() raise + finally: + _record_job_end(job_id) def _run_generation_impl( @@ -289,6 +322,7 @@ def _run_generation_impl( job = _jobs[job_id] job.status = "running" job.step = None + run_history.mark_running(job_id) def progress_cb(pct: int, step: str = "") -> None: # Monotonic: the loading phase walks the bar up on a background thread and @@ -298,6 +332,7 @@ def progress_cb(pct: int, step: str = "") -> None: job.progress = pct if step: job.step = step + run_history.step(job_id, step) try: # Refuse a selected weight variant that is not installed before loading diff --git a/api/routers/optimize.py b/api/routers/optimize.py index 5fafc3343..c6d70a1c0 100644 --- a/api/routers/optimize.py +++ b/api/routers/optimize.py @@ -1,6 +1,7 @@ import hashlib import os import tempfile +import time import uuid from contextlib import contextmanager @@ -26,6 +27,8 @@ ) from services.mesh_ops.glb_patch import GlbPatchError, VertexPatch, patch_glb_file from services.mesh_ops.transforms import apply_transforms, transform_file +from schemas.runs import RunRecord +from services.run_history import run_history, workspace_relative router = APIRouter(tags=["optimize"]) @@ -131,8 +134,43 @@ def _run_operation( preserve_visuals: bool = False, ) -> MeshOpResult: context = _operation_context(output_path, preserve_visuals) - with _mesh_op_http_errors(): - return mesh_ops_registry.run(operation_id, input_path, params, context) + started_at = time.time() + try: + with _mesh_op_http_errors(): + result = mesh_ops_registry.run(operation_id, input_path, params, context) + except HTTPException as exc: + _record_mesh_op(operation_id, input_path, params, started_at, error=str(exc.detail)) + raise + # Analysis ops (inspect_mesh, suggest_fixes) only report: the viewer calls them on + # every selection, and they would crowd real runs out of the capped history. + if result.file_path is not None: + _record_mesh_op(operation_id, input_path, params, started_at, output=result.file_path) + return result + + +def _record_mesh_op( + operation_id: str, + input_path: Path, + params: dict[str, object], + started_at: float, + output: Path | None = None, + error: str | None = None, +) -> None: + """Persist a synchronous mesh op in run history (services.run_history).""" + workspace = registry.WORKSPACE_DIR + run_history.save(RunRecord( + id=str(uuid.uuid4()), + kind="mesh_op", + op_id=operation_id, + status="error" if error is not None else "done", + inputs={"mesh": workspace_relative(input_path, workspace)}, + params=dict(params), + error=error, + output_path=workspace_relative(output, workspace) if output is not None else None, + created_at=started_at, + started_at=started_at, + ended_at=time.time(), + )) def _operation_response(result: MeshOpResult) -> dict[str, object]: diff --git a/api/routers/workflow_runs.py b/api/routers/workflow_runs.py index 8a210ea2d..b928c9ae3 100644 --- a/api/routers/workflow_runs.py +++ b/api/routers/workflow_runs.py @@ -1,9 +1,10 @@ import json import threading +import time import uuid -from typing import Optional -from fastapi import APIRouter, BackgroundTasks, File, Form, HTTPException, UploadFile -from pydantic import BaseModel +from typing import Any, Optional +from fastapi import APIRouter, BackgroundTasks, File, Form, HTTPException, Query, UploadFile +from pydantic import BaseModel, Field from routers.generation import ( VALID_REMESH_MODES, @@ -12,10 +13,13 @@ _purge_old_jobs, _run_generation, cancel_job, + record_job_filed, sanitize_collection, ) from schemas.generation import JobStatus +from schemas.runs import RunRecord, RunRecordPage, RunStatusValue, RunStep, RunKind from services.generator_registry import generator_registry +from services.run_history import run_history router = APIRouter(tags=["workflow-runs"]) @@ -30,6 +34,55 @@ class WorkflowRunStatus(BaseModel): scene_candidate: Optional[dict] = None +class WorkflowRunRecordIn(BaseModel): + """A workflow run's record as the renderer, which executes workflows, reports it.""" + workflow_id: Optional[str] = None + workflow_name: Optional[str] = None + status: RunStatusValue + inputs: dict[str, Any] = Field(default_factory=dict) + params: dict[str, Any] = Field(default_factory=dict) + steps: list[RunStep] = Field(default_factory=list) + error: Optional[str] = None + output_path: Optional[str] = None + created_at: Optional[float] = None + started_at: Optional[float] = None + ended_at: Optional[float] = None + + +MAX_PAGE = 200 + + +@router.get("", response_model=RunRecordPage) +async def list_runs( + status: Optional[list[RunStatusValue]] = Query(None), + kind: Optional[list[RunKind]] = Query(None), + limit: int = Query(50, ge=1, le=MAX_PAGE), + offset: int = Query(0, ge=0), +): + """Persisted run history, newest first. ``status``/``kind`` may repeat to match any.""" + runs, total = run_history.list(statuses=status, kinds=kind, limit=limit, offset=offset) + return RunRecordPage(runs=runs, total=total, limit=limit, offset=offset) + + +@router.put("/{run_id}", response_model=RunRecord) +async def put_workflow_run_record(run_id: str, body: WorkflowRunRecordIn): + """Insert or replace the history record of a renderer-driven workflow run.""" + if not run_id or len(run_id) > 200: + raise HTTPException(400, "Invalid run id") + existing = run_history.get(run_id) + if existing is not None and existing.kind != "workflow": + raise HTTPException(409, f"Run {run_id} is not a workflow run") + record = RunRecord( + id=run_id, + kind="workflow", + created_at=body.created_at + or (existing.created_at if existing is not None else time.time()), + **body.model_dump(exclude={"created_at"}), + ) + run_history.save(record) + return record + + @router.post("/from-image") async def create_run_from_image( background_tasks: BackgroundTasks, @@ -93,6 +146,7 @@ async def create_run_from_image( _jobs[job_id] = JobStatus(job_id=job_id, status="pending", progress=0) _cancel_events[job_id] = threading.Event() + record_job_filed(job_id, model_id, {"image": image.filename}, full_params, collection) background_tasks.add_task( _run_generation, job_id, image_bytes, full_params, collection, output_kind, model_id @@ -105,7 +159,22 @@ async def create_run_from_image( async def get_run(run_id: str): job = _jobs.get(run_id) if not job: - raise HTTPException(404, f"Run {run_id} not found") + # Purged from the live table (or from before a restart): answer from history. + record = run_history.get(run_id) + if record is None: + raise HTTPException(404, f"Run {run_id} not found") + output_url = f"/workspace/{record.output_path}" if ( + record.output_path and record.status == "done" + and not record.output_path.startswith("/") and ":" not in record.output_path + ) else None + return WorkflowRunStatus( + run_id=record.id, + status=record.status, + progress=100 if record.status == "done" else 0, + output_url=output_url, + error=record.error, + scene_candidate={"workspace_path": record.output_path} if output_url else None, + ) scene_candidate = None if job.status == "done" and job.output_url: diff --git a/api/tests/test_generation_router.py b/api/tests/test_generation_router.py index 3d69118c7..784fc2248 100644 --- a/api/tests/test_generation_router.py +++ b/api/tests/test_generation_router.py @@ -26,6 +26,8 @@ def generate(self, image_bytes, params, progress_cb, cancel_event=None) -> Path: class _FakeUpload: """Minimal UploadFile stand-in: an image content-type and readable bytes.""" + filename = "input.png" + def __init__(self, content_type: str = "image/png", data: bytes = b"\x89PNG\r\n") -> None: self.content_type = content_type self._data = data diff --git a/api/tests/test_run_history.py b/api/tests/test_run_history.py new file mode 100644 index 000000000..8c67f95f4 --- /dev/null +++ b/api/tests/test_run_history.py @@ -0,0 +1,368 @@ +import asyncio +import json +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from fastapi import HTTPException + +import routers.generation as generation +import routers.optimize as optimize +import routers.workflow_runs as workflow_runs +import services.run_history as run_history_module +from schemas.generation import JobStatus +from schemas.runs import RunRecord, RunStep +from services.mesh_ops import MeshOpExecutionError, MeshOpResult +from services.run_history import ( + INTERRUPTED_ERROR, + MAX_STEPS, + RUN_HISTORY_FILENAME, + RunHistoryStore, +) + + +def _record(run_id: str, status: str = "done", created_at: float = 1000.0, kind: str = "generation") -> RunRecord: + return RunRecord(id=run_id, kind=kind, status=status, created_at=created_at) + + +class _TempStoreCase(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.workspace = Path(self._tmp.name) + self.path = self.workspace / RUN_HISTORY_FILENAME + self.store = RunHistoryStore() + self.store.start(lambda: self.path) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def reopen(self, max_records: int | None = None) -> RunHistoryStore: + """A fresh store on the same file: what the next app start sees.""" + store = RunHistoryStore(max_records or self.store.max_records) + store.start(lambda: self.path) + return store + + +class RunHistoryStoreTests(_TempStoreCase): + def test_inert_until_started(self) -> None: + store = RunHistoryStore() + store.begin("a", "generation") + store.save(_record("b")) + self.assertEqual(store.list(), ([], 0)) + self.assertIsNone(store.get("b")) + + def test_lifecycle_is_recorded_with_steps_and_duration(self) -> None: + self.store.begin("run-1", "generation", model_id="sf3d", inputs={"image": "a.png"}, params={"seed": 1}) + self.assertEqual(self.store.get("run-1").status, "pending") + self.store.mark_running("run-1") + self.store.step("run-1", "Loading SF3D…") + self.store.step("run-1", "Sampling 1/50") + self.store.step("run-1", "Sampling 2/50") # same step, only the counter moved + self.store.step("run-1", "Exporting") + self.store.finish("run-1", "done", output_path="Default/out.glb") + + record = self.reopen().get("run-1") + self.assertEqual(record.status, "done") + self.assertEqual(record.model_id, "sf3d") + self.assertEqual(record.inputs, {"image": "a.png"}) + self.assertEqual(record.params, {"seed": 1}) + self.assertEqual(record.output_path, "Default/out.glb") + self.assertEqual([s.name for s in record.steps], ["Loading SF3D…", "Sampling 1/50", "Exporting"]) + self.assertTrue(all(s.ended_at is not None and s.status == "done" for s in record.steps)) + self.assertIsNotNone(record.duration_s) + self.assertGreaterEqual(record.duration_s, 0) + + def test_steps_are_capped(self) -> None: + self.store.begin("r", "generation") + for i in range(MAX_STEPS + 20): + self.store.step("r", f"step {chr(65 + i % 26)}{chr(65 + i // 26)}") + self.store.finish("r", "done") + self.assertEqual(len(self.store.get("r").steps), MAX_STEPS) + + def test_error_and_cancel_are_final(self) -> None: + self.store.begin("bad", "generation") + self.store.finish("bad", "error", error="x" * 10_000) + self.store.begin("stop", "generation") + self.store.finish("stop", "cancelled") + self.store.finish("stop", "done") # a late worker return must not overwrite it + + reopened = self.reopen() + bad = reopened.get("bad") + self.assertEqual(bad.status, "error") + self.assertLessEqual(len(bad.error), run_history_module.MAX_ERROR_CHARS) + self.assertEqual(reopened.get("stop").status, "cancelled") + + def test_list_is_newest_first_with_paging(self) -> None: + for i in range(5): + self.store.save(_record(f"r{i}", created_at=1000.0 + i)) + page, total = self.store.list(limit=2, offset=1) + self.assertEqual(total, 5) + self.assertEqual([r.id for r in page], ["r3", "r2"]) + page, total = self.store.list(limit=10, offset=4) + self.assertEqual([r.id for r in page], ["r0"]) + + def test_list_filters_by_status_and_kind(self) -> None: + self.store.save(_record("ok", "done", 1)) + self.store.save(_record("bad", "error", 2)) + self.store.save(_record("stop", "cancelled", 3, kind="workflow")) + page, total = self.store.list(statuses=["error", "cancelled"]) + self.assertEqual((total, [r.id for r in page]), (2, ["stop", "bad"])) + page, total = self.store.list(kinds=["workflow"]) + self.assertEqual([r.id for r in page], ["stop"]) + + def test_retention_cap_drops_oldest_finished_runs(self) -> None: + store = RunHistoryStore(max_records=3) + store.start(lambda: self.path) + store.begin("live", "generation") # oldest, but still running: kept + store.mark_running("live") + for i in range(4): + store.save(_record(f"r{i}", created_at=time.time() + 1 + i)) + ids = {r.id for r in store.list(limit=50)[0]} + self.assertEqual(ids, {"live", "r2", "r3"}) + on_disk = json.loads(self.path.read_text(encoding="utf-8"))["runs"] + self.assertEqual(len(on_disk), 3) + + def test_restart_marks_in_flight_runs_interrupted(self) -> None: + self.store.begin("queued", "generation") + self.store.begin("busy", "generation") + self.store.mark_running("busy") + self.store.step("busy", "Generating") + self.store.save(_record("finished", "done")) + + reopened = self.reopen() # the old process died without stop() + for run_id in ("queued", "busy"): + record = reopened.get(run_id) + self.assertEqual(record.status, "interrupted") + self.assertEqual(record.error, INTERRUPTED_ERROR) + self.assertEqual(reopened.get("finished").status, "done") + # ... and it is written back, so the next reader agrees. + raw = json.loads(self.path.read_text(encoding="utf-8")) + self.assertEqual({r["id"]: r["status"] for r in raw["runs"]}["busy"], "interrupted") + + def test_stop_marks_live_runs_interrupted(self) -> None: + self.store.begin("busy", "generation") + self.store.mark_running("busy") + self.store.stop() + self.assertEqual(self.reopen().get("busy").status, "interrupted") + + def test_missing_file_is_an_empty_history(self) -> None: + self.assertEqual(self.store.list(), ([], 0)) + self.assertFalse(self.path.exists()) + + def test_corrupt_file_is_moved_aside(self) -> None: + self.path.write_text("{not json", encoding="utf-8") + store = self.reopen() + self.assertEqual(store.list(), ([], 0)) + store.save(_record("fresh")) + self.assertEqual(self.reopen().get("fresh").status, "done") + self.assertEqual(len(list(self.workspace.glob(f"{RUN_HISTORY_FILENAME}.corrupt-*"))), 1) + + def test_invalid_entries_are_skipped(self) -> None: + good = _record("good").model_dump(mode="json") + self.path.write_text(json.dumps({"version": 1, "runs": [good, {"id": "x"}, 42]}), encoding="utf-8") + page, total = self.reopen().list() + self.assertEqual([r.id for r in page], ["good"]) + + def test_writes_are_atomic_and_leave_no_temp_files(self) -> None: + for i in range(3): + self.store.save(_record(f"r{i}")) + self.assertEqual(sorted(p.name for p in self.workspace.iterdir()), [RUN_HISTORY_FILENAME]) + + def test_write_failure_does_not_raise(self) -> None: + with patch("services.run_history.os.replace", side_effect=PermissionError("locked")): + self.store.save(_record("r")) # logged, not raised + self.assertEqual(self.store.get("r").status, "done") # still served from memory + + def test_follows_a_relocated_workspace(self) -> None: + other = self.workspace / "moved" / RUN_HISTORY_FILENAME + current = {"path": self.path} + store = RunHistoryStore() + store.start(lambda: current["path"]) + store.begin("live", "generation") + store.mark_running("live") + current["path"] = other + store.finish("live", "done") + reloaded = RunHistoryStore() + reloaded.start(lambda: other) + self.assertEqual(reloaded.get("live").status, "done") + + +class _RouterHistoryCase(_TempStoreCase): + """Points the routers' shared store at a temp workspace for one test.""" + + def setUp(self) -> None: + super().setUp() + patcher = patch.object(run_history_module.run_history, "_path_provider", None) + patcher.start() + self.addCleanup(patcher.stop) + self.shared = run_history_module.run_history + self.shared.start(lambda: self.path) + self.addCleanup(self.shared.stop) + for store in (generation._jobs, generation._cancel_events, generation._cancelled, + generation._completed_at, generation._job_generators): + store.clear() + + +class ListRunsEndpointTests(_RouterHistoryCase): + def test_lists_with_paging_and_status_filter(self) -> None: + for i, status in enumerate(["done", "error", "cancelled", "interrupted", "done"]): + self.shared.save(_record(f"r{i}", status, 1000.0 + i)) + page = asyncio.run(workflow_runs.list_runs(status=["error", "cancelled"], kind=None, limit=1, offset=0)) + self.assertEqual(page.total, 2) + self.assertEqual([r.id for r in page.runs], ["r2"]) + page = asyncio.run(workflow_runs.list_runs(status=None, kind=None, limit=50, offset=0)) + self.assertEqual(page.total, 5) + self.assertEqual(page.runs[0].id, "r4") + + def test_route_is_mounted_at_the_collection_root(self) -> None: + paths = {(route.path, tuple(sorted(route.methods))) for route in workflow_runs.router.routes} + self.assertIn(("", ("GET",)), paths) + self.assertIn(("/{run_id}", ("PUT",)), paths) + + def test_get_run_falls_back_to_history_after_purge(self) -> None: + record = _record("old", "done") + record.output_path = "Default/out.glb" + self.shared.save(record) + status = asyncio.run(workflow_runs.get_run("old")) + self.assertEqual(status.status, "done") + self.assertEqual(status.output_url, "/workspace/Default/out.glb") + with self.assertRaises(HTTPException): + asyncio.run(workflow_runs.get_run("unknown")) + + +class PutWorkflowRunTests(_RouterHistoryCase): + def _put(self, run_id: str, **fields) -> RunRecord: + body = workflow_runs.WorkflowRunRecordIn(**fields) + return asyncio.run(workflow_runs.put_workflow_run_record(run_id, body)) + + def test_upserts_a_workflow_record(self) -> None: + self._put("wf-run", workflow_id="wf", workflow_name="Flow", status="running", + created_at=1000.0, started_at=1000.0, + steps=[RunStep(name="Mesh", node_id="n1", started_at=1000.0)]) + self._put("wf-run", workflow_id="wf", workflow_name="Flow", status="done", + started_at=1000.0, ended_at=1012.5, output_path="Workflows/out.glb", + steps=[RunStep(name="Mesh", node_id="n1", status="done", started_at=1000.0, ended_at=1012.5)]) + record = self.reopen().get("wf-run") + self.assertEqual(record.kind, "workflow") + self.assertEqual(record.status, "done") + self.assertEqual(record.created_at, 1000.0) # kept from the first report + self.assertEqual(record.duration_s, 12.5) + self.assertEqual(record.output_path, "Workflows/out.glb") + + def test_running_workflow_is_interrupted_on_restart(self) -> None: + self._put("wf-run", status="running", created_at=1000.0) + self.assertEqual(self.reopen().get("wf-run").status, "interrupted") + + def test_refuses_to_overwrite_a_backend_run(self) -> None: + self.shared.save(_record("gen")) + with self.assertRaises(HTTPException) as ctx: + self._put("gen", status="done") + self.assertEqual(ctx.exception.status_code, 409) + + +class _Gen: + def __init__(self, fail: bool = False, out_dir: Path | None = None) -> None: + self.fail = fail + self.outputs_dir = out_dir + + def generate(self, image_bytes, params, progress_cb, cancel_event=None): + progress_cb(10, "Generating") + if self.fail: + raise RuntimeError("boom") + out = Path(self.outputs_dir) / "mesh.glb" + out.write_bytes(b"glb") + return str(out) + + +class _Registry: + def __init__(self, gen: _Gen) -> None: + self.gen = gen + + def assert_weight_variant_installed(self, params, model_id) -> None: + pass + + def get_generator(self, model_id): + return self.gen + + def activate_ready_generator(self, model_id): + return self.gen + + def model_status(self, model_id): + return {"loaded": True, "name": model_id, "downloaded": True} + + +class GenerationRecordingTests(_RouterHistoryCase): + def setUp(self) -> None: + super().setUp() + ws = patch.object(generation.registry, "WORKSPACE_DIR", self.workspace) + ws.start() + self.addCleanup(ws.stop) + + def _run(self, job_id: str, gen: _Gen) -> None: + generation._jobs[job_id] = JobStatus(job_id=job_id, status="pending") + generation._cancel_events[job_id] = threading.Event() + generation.record_job_filed(job_id, "demo/model", {"image": "in.png"}, {"seed": 3}, "Default") + with patch.object(generation, "generator_registry", _Registry(gen)): + asyncio.run(generation._run_generation(job_id, b"img", {"seed": 3}, "Default", "mesh", "demo/model")) + + def test_successful_generation_is_recorded_done(self) -> None: + self._run("ok", _Gen()) + record = self.reopen().get("ok") + self.assertEqual(record.status, "done") + self.assertEqual(record.model_id, "demo/model") + self.assertEqual(record.params, {"seed": 3, "collection": "Default"}) + self.assertEqual(record.output_path, "Default/mesh.glb") + self.assertEqual([s.name for s in record.steps], ["Generating"]) + self.assertIsNotNone(record.duration_s) + + def test_failed_generation_is_recorded_with_its_error(self) -> None: + self._run("bad", _Gen(fail=True)) + record = self.reopen().get("bad") + self.assertEqual(record.status, "error") + self.assertIn("boom", record.error) + + def test_cancel_is_recorded_immediately(self) -> None: + generation._jobs["c"] = JobStatus(job_id="c", status="running") + generation.record_job_filed("c", "demo/model", {}, {}, "Default") + asyncio.run(generation.cancel_job("c")) + self.assertEqual(self.reopen().get("c").status, "cancelled") + + +class MeshOpRecordingTests(_RouterHistoryCase): + def setUp(self) -> None: + super().setUp() + ws = patch.object(optimize.registry, "WORKSPACE_DIR", self.workspace) + ws.start() + self.addCleanup(ws.stop) + self.mesh = self.workspace / "Default" / "in.glb" + self.mesh.parent.mkdir() + self.mesh.write_bytes(b"glb") + + def test_mesh_writing_op_is_recorded(self) -> None: + out = self.workspace / "Default" / "out.glb" + with patch.object(optimize.mesh_ops_registry, "run", return_value=MeshOpResult(file_path=out)): + optimize._run_operation("decimate", self.mesh, {"target_faces": 100}, out) + page, _ = self.reopen().list() + self.assertEqual(len(page), 1) + self.assertEqual((page[0].kind, page[0].op_id, page[0].status), ("mesh_op", "decimate", "done")) + self.assertEqual(page[0].inputs, {"mesh": "Default/in.glb"}) + self.assertEqual(page[0].output_path, "Default/out.glb") + + def test_failed_op_is_recorded(self) -> None: + with patch.object(optimize.mesh_ops_registry, "run", side_effect=MeshOpExecutionError("broken")): + with self.assertRaises(HTTPException): + optimize._run_operation("smooth", self.mesh, {}) + page, _ = self.reopen().list() + self.assertEqual((page[0].status, page[0].error), ("error", "broken")) + + def test_report_only_op_is_not_recorded(self) -> None: + with patch.object(optimize.mesh_ops_registry, "run", return_value=MeshOpResult(file_path=None)): + optimize._run_operation("inspect_mesh", self.mesh, {}) + self.assertEqual(self.reopen().list(), ([], 0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_workflow_runs_lifecycle.py b/api/tests/test_workflow_runs_lifecycle.py index cf1f5269b..f80a7590f 100644 --- a/api/tests/test_workflow_runs_lifecycle.py +++ b/api/tests/test_workflow_runs_lifecycle.py @@ -18,6 +18,8 @@ class _FakeUpload: """Minimal UploadFile stand-in: an image content-type and readable bytes.""" + filename = "input.png" + def __init__(self, content_type: str = "image/png", data: bytes = b"\x89PNG\r\n") -> None: self.content_type = content_type self._data = data diff --git a/api/tests/test_workflow_runs_router.py b/api/tests/test_workflow_runs_router.py index cb2bb210f..3515ff532 100644 --- a/api/tests/test_workflow_runs_router.py +++ b/api/tests/test_workflow_runs_router.py @@ -10,6 +10,8 @@ class _FakeUpload: """Minimal stand-in for UploadFile: an image content-type and readable bytes.""" + filename = "input.png" + def __init__(self, content_type: str = "image/png", data: bytes = b"\x89PNG\r\n") -> None: self.content_type = content_type self._data = data From 2afa5407f7319bd828d81bbb913eb4909a42d47b Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Fri, 13 Nov 2026 10:04:55 +0100 Subject: [PATCH 062/108] feat(runs): persist workflow runs from the renderer The workflow runner reports its run record (per-node step timings, final status, output) to PUT /workflow-runs/{id}, chaining writes so the final status always lands last. Cancelled and abandoned runs are recorded as cancelled. --- package.json | 2 +- src/areas/workflows/workflowRunRecord.test.ts | 95 +++++++++++++++++++ src/areas/workflows/workflowRunRecord.ts | 95 +++++++++++++++++++ src/areas/workflows/workflowRunStore.ts | 66 +++++++++++++ src/shared/types/runs.ts | 44 +++++++++ 5 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 src/areas/workflows/workflowRunRecord.test.ts create mode 100644 src/areas/workflows/workflowRunRecord.ts create mode 100644 src/shared/types/runs.ts diff --git a/package.json b/package.json index 74c026e63..f7c15b511 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/areas/workflows/workflowRunRecord.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/workflows/workflowRunRecord.test.ts b/src/areas/workflows/workflowRunRecord.test.ts new file mode 100644 index 000000000..099a608e9 --- /dev/null +++ b/src/areas/workflows/workflowRunRecord.test.ts @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + MAX_RECORD_STEPS, + endRecordStep, + finishWorkflowRunRecord, + newWorkflowRunRecord, + reopenWorkflowRunRecord, + runOutputPath, + startRecordStep, + workflowRunParams, +} from './workflowRunRecord' + +const workflow = { + id: 'wf-1', + name: 'Image to mesh', + nodes: [ + { id: 'img', type: 'imageNode', data: { enabled: true, params: { filePath: 'C:/a.png' } } }, + { id: 'gen', type: 'extensionNode', data: { extensionId: 'triposg/generate', enabled: true, params: { seed: 7 } } }, + { id: 'off', type: 'extensionNode', data: { extensionId: 'mesh/decimate', enabled: false, params: {} } }, + ], +} + +test('a new record is running with the workflow, its inputs and enabled node params', () => { + const record = newWorkflowRunRecord('run-1', workflow, { image: 'C:/a.png' }, 100) + assert.equal(record.status, 'running') + assert.equal(record.workflow_id, 'wf-1') + assert.equal(record.workflow_name, 'Image to mesh') + assert.deepEqual(record.inputs, { image: 'C:/a.png' }) + assert.deepEqual(record.params, { nodes: { gen: { extension_id: 'triposg/generate', params: { seed: 7 } } } }) + assert.equal(record.created_at, 100) + assert.equal(record.started_at, 100) + assert.deepEqual(record.steps, []) +}) + +test('workflowRunParams skips disabled and non-extension nodes', () => { + assert.deepEqual(Object.keys(workflowRunParams(workflow).nodes as object), ['gen']) +}) + +test('steps are timed per node and a new step closes the open one as done', () => { + let record = newWorkflowRunRecord('run-1', workflow, {}, 100) + record = startRecordStep(record, 'TripoSG', 'gen', 101) + record = startRecordStep(record, 'Decimate', 'dec', 110) + assert.deepEqual(record.steps[0], { name: 'TripoSG', node_id: 'gen', status: 'done', started_at: 101, ended_at: 110 }) + assert.deepEqual(record.steps[1], { name: 'Decimate', node_id: 'dec', status: 'running', started_at: 110 }) + record = endRecordStep(record, 'error', 115) + assert.equal(record.steps[1].status, 'error') + assert.equal(record.steps[1].ended_at, 115) + // Ending again is a no-op: the step is already closed. + assert.equal(endRecordStep(record, 'done', 120), record) +}) + +test('steps are capped', () => { + let record = newWorkflowRunRecord('run-1', workflow, {}, 0) + for (let i = 0; i < MAX_RECORD_STEPS + 10; i++) record = startRecordStep(record, `n${i}`, `n${i}`, i) + assert.equal(record.steps.length, MAX_RECORD_STEPS) +}) + +test('finishing closes the open step with the run status and records the outcome', () => { + let record = startRecordStep(newWorkflowRunRecord('run-1', workflow, {}, 100), 'TripoSG', 'gen', 100) + record = finishWorkflowRunRecord(record, 'error', 130, { error: 'Error: boom' }) + assert.equal(record.status, 'error') + assert.equal(record.ended_at, 130) + assert.equal(record.error, 'Error: boom') + assert.equal(record.steps[0].status, 'error') + + const done = finishWorkflowRunRecord(newWorkflowRunRecord('run-2', workflow, {}, 100), 'done', 150, { outputPath: 'Workflows/a.glb' }) + assert.equal(done.output_path, 'Workflows/a.glb') + assert.equal(done.error, null) +}) + +test('a cancelled run keeps its status and partial steps', () => { + let record = startRecordStep(newWorkflowRunRecord('run-1', workflow, {}, 100), 'TripoSG', 'gen', 100) + record = finishWorkflowRunRecord(record, 'cancelled', 105) + assert.equal(record.status, 'cancelled') + assert.equal(record.steps[0].status, 'cancelled') +}) + +test('reopening a finished run makes it running again and clears the end', () => { + const finished = finishWorkflowRunRecord(newWorkflowRunRecord('run-1', workflow, {}, 100), 'error', 120, { error: 'x' }) + const reopened = reopenWorkflowRunRecord(finished) + assert.equal(reopened.status, 'running') + assert.equal(reopened.ended_at, null) + assert.equal(reopened.error, null) + const running = newWorkflowRunRecord('run-2', workflow, {}, 100) + assert.equal(reopenWorkflowRunRecord(running), running) +}) + +test('runOutputPath prefers a workspace-relative path', () => { + assert.equal(runOutputPath('/workspace/Workflows/a 100%.glb'), 'Workflows/a 100%.glb') + assert.equal(runOutputPath(undefined, 'D:/out/a.wav'), 'D:/out/a.wav') + assert.equal(runOutputPath('/optimize/serve-file?path=x', 'D:/x.glb'), 'D:/x.glb') + assert.equal(runOutputPath(), undefined) +}) diff --git a/src/areas/workflows/workflowRunRecord.ts b/src/areas/workflows/workflowRunRecord.ts new file mode 100644 index 000000000..e236e823e --- /dev/null +++ b/src/areas/workflows/workflowRunRecord.ts @@ -0,0 +1,95 @@ +// Pure bookkeeping for a workflow run's history record (see shared/types/runs.ts). +// The renderer executes workflows, so it owns their record and reports it with +// PUT /workflow-runs/{id}; the store wires these helpers to the run lifecycle. +// Every helper returns a new record. `now` is epoch seconds. + +import type { RunRecord, RunStatus } from '../../shared/types/runs' + +export type WorkflowRunRecord = Omit + +/** Matches the backend cap (services/run_history.py MAX_STEPS). */ +export const MAX_RECORD_STEPS = 200 + +const TERMINAL: ReadonlySet = new Set(['done', 'error', 'cancelled', 'interrupted']) + +export function isTerminalRunStatus(status: RunStatus): boolean { + return TERMINAL.has(status) +} + +interface WorkflowLike { + id: string + name: string + nodes: { id: string; type: string; data: { extensionId?: string; enabled?: boolean; params?: Record } }[] +} + +/** The params a run used: each enabled extension node's id, extension and params. */ +export function workflowRunParams(workflow: WorkflowLike): Record { + const nodes: Record = {} + for (const node of workflow.nodes) { + if (node.type !== 'extensionNode' || !node.data.enabled) continue + nodes[node.id] = { extension_id: node.data.extensionId ?? null, params: node.data.params ?? {} } + } + return { nodes } +} + +export function newWorkflowRunRecord( + id: string, + workflow: WorkflowLike, + inputs: Record, + now: number, +): WorkflowRunRecord { + return { + id, + status: 'running', + workflow_id: workflow.id, + workflow_name: workflow.name, + inputs, + params: workflowRunParams(workflow), + steps: [], + created_at: now, + started_at: now, + } +} + +/** Close the open step (if any) with `status`. */ +export function endRecordStep(record: WorkflowRunRecord, status: RunStatus, now: number): WorkflowRunRecord { + const last = record.steps[record.steps.length - 1] + if (!last || last.ended_at != null) return record + return { ...record, steps: [...record.steps.slice(0, -1), { ...last, status, ended_at: now }] } +} + +/** Open a step for a node; a step left open is closed as done first. */ +export function startRecordStep(record: WorkflowRunRecord, name: string, nodeId: string, now: number): WorkflowRunRecord { + const closed = endRecordStep(record, 'done', now) + if (closed.steps.length >= MAX_RECORD_STEPS) return closed + return { ...closed, steps: [...closed.steps, { name, node_id: nodeId, status: 'running', started_at: now }] } +} + +export function finishWorkflowRunRecord( + record: WorkflowRunRecord, + status: RunStatus, + now: number, + outcome: { error?: string; outputPath?: string } = {}, +): WorkflowRunRecord { + const closed = endRecordStep(record, status, now) + return { + ...closed, + status, + ended_at: now, + error: outcome.error ?? closed.error ?? null, + output_path: outcome.outputPath ?? closed.output_path ?? null, + } +} + +/** A finished run the user resumes (re-running a Wait branch) is running again. */ +export function reopenWorkflowRunRecord(record: WorkflowRunRecord): WorkflowRunRecord { + if (!isTerminalRunStatus(record.status)) return record + return { ...record, status: 'running', ended_at: null, error: null } +} + +/** Workspace-relative path of a `/workspace/...` URL, else the absolute path as-is. */ +export function runOutputPath(outputUrl?: string, outputPath?: string): string | undefined { + // toWorkspaceUrl does not percent-encode, so the rest of the URL is the path itself. + if (outputUrl?.startsWith('/workspace/')) return outputUrl.slice('/workspace/'.length) + return outputPath +} diff --git a/src/areas/workflows/workflowRunStore.ts b/src/areas/workflows/workflowRunStore.ts index 383ce0c69..87076f629 100644 --- a/src/areas/workflows/workflowRunStore.ts +++ b/src/areas/workflows/workflowRunStore.ts @@ -9,6 +9,12 @@ import type { Workflow, WFNode, WFEdge } from '@shared/types/electron.d' import { isBranchStarter, isSceneOutput, resolveDataSource, reachesSceneOutput, nearestUpstreamWaits } from './nodeBehaviors' import { assignSlotFilePaths } from './slotInputs' import type { SlotInputType } from './slotInputs' +import { + endRecordStep, finishWorkflowRunRecord, isTerminalRunStatus, newWorkflowRunRecord, + reopenWorkflowRunRecord, runOutputPath, startRecordStep, +} from './workflowRunRecord' +import type { WorkflowRunRecord } from './workflowRunRecord' +import type { RunStatus } from '@shared/types/runs' // ─── Types ──────────────────────────────────────────────────────────────────── @@ -42,6 +48,44 @@ const _pauseRequested = { current: false } // latest values when a body node starts (instead of the snapshot from run start). const _liveParams = { current: new Map>() } +// ─── Run history ────────────────────────────────────────────────────────────── +// The run's persisted record (GET /workflow-runs lists it). Every change is PUT to +// the backend; the PUTs are chained so a slow early write can never land after — +// and overwrite — the final status. + +const _record = { current: null as WorkflowRunRecord | null } +let _recordWrites: Promise = Promise.resolve() + +function nowSeconds(): number { + return Date.now() / 1000 +} + +function updateRecord(fn: (record: WorkflowRunRecord) => WorkflowRunRecord): void { + if (!_record.current) return + const record = fn(_record.current) + _record.current = record + const { id, ...body } = record + const apiUrl = useAppStore.getState().apiUrl + _recordWrites = _recordWrites + .then(() => axios.put(`${apiUrl}/workflow-runs/${encodeURIComponent(id)}`, body)) + .then(() => undefined, (err) => { console.warn('[workflows] Could not save run history:', err) }) +} + +function finishRecord(status: RunStatus, outcome?: { error?: string; outputPath?: string }): void { + if (!_record.current || isTerminalRunStatus(_record.current.status)) return + updateRecord((r) => finishWorkflowRunRecord(r, status, nowSeconds(), outcome)) +} + +function recordStepStart(node: WFNode, allExtensions: WorkflowExtension[]): void { + const name = getWorkflowExtension(node.data.extensionId ?? '', allExtensions)?.name + ?? node.data.extensionId ?? node.type + updateRecord((r) => startRecordStep(r, name, node.id, nowSeconds())) +} + +function recordStepEnd(status: RunStatus): void { + updateRecord((r) => endRecordStep(r, status, nowSeconds())) +} + function flushResume(): void { const fn = _resume.current _resume.current = null @@ -644,6 +688,7 @@ export const useWorkflowRunStore = create((set, get) => { }, })) useAppStore.getState().updateCurrentJob({ status: 'done', progress: 100, outputUrl }) + finishRecord('done', { outputPath: runOutputPath(outputUrl, outputPath) }) void showCompletionNotification('Workflow run complete') } @@ -666,6 +711,16 @@ export const useWorkflowRunStore = create((set, get) => { const appState = useAppStore.getState() const apiUrl = appState.apiUrl + // A previous run still parked on its Wait branches is abandoned by this one. + finishRecord('cancelled') + _record.current = newWorkflowRunRecord( + crypto.randomUUID(), + workflow, + overrideImageData ? { image: '(attached image)' } : appState.selectedImagePath ? { image: appState.selectedImagePath } : {}, + nowSeconds(), + ) + updateRecord((r) => r) + const { preExecExtNodes, branches, waitIds, parentWait, ordered } = identifyBranches(workflow) const branchSteps = waitIds.reduce((acc, w) => acc + (branches.get(w)?.length ?? 0), 0) @@ -680,6 +735,7 @@ export const useWorkflowRunStore = create((set, get) => { const dir = (w.data.params?.dir as string | undefined)?.trim() const fail = (msg: string, step: string): void => { set((s) => ({ runState: { ...s.runState, status: 'error', error: msg, blockStep: step }, activeNodeId: null })) + finishRecord('error', { error: msg }) } if (!dir) { fail('For Each: pick a folder first', 'No folder selected'); return } try { @@ -951,7 +1007,9 @@ export const useWorkflowRunStore = create((set, get) => { activeNodeId: node.id, runState: { ...s.runState, blockIndex: stepsDone, blockProgress: 0, blockStep: 'Starting…' }, })) + recordStepStart(node, allExtensions) await executeExtensionNode(node, ctx, setRunState) + recordStepEnd('done') stepsDone++ const jump = await handleLoopEnd(i) @@ -972,6 +1030,7 @@ export const useWorkflowRunStore = create((set, get) => { } catch (err) { if (!_cancel.current) { set((s) => ({ runState: { ...s.runState, status: 'error', error: String(err) }, activeNodeId: null })) + finishRecord('error', { error: String(err) }) useAppStore.getState().updateCurrentJob({ status: 'error', error: String(err) }) void showRunErrorNotification({ title: 'Workflow run failed', name: workflow.name, error: err }) } @@ -1000,6 +1059,7 @@ export const useWorkflowRunStore = create((set, get) => { const descendants = descendantWaits(waitId, ctx) _cancel.current = false + updateRecord(reopenWorkflowRunRecord) // Reset outputs for this branch's nodes so Retry re-executes cleanly. for (const node of branch) ctx.nodeOutputs.delete(node.id) @@ -1040,6 +1100,7 @@ export const useWorkflowRunStore = create((set, get) => { if (allFinished && !anyError) { finalize(ctx, newWaitStates) } else { + if (allFinished) finishRecord('error', { error: err ?? get().runState.error }) set((s) => ({ activeNodeId: null, runningBranchId: null, @@ -1062,15 +1123,19 @@ export const useWorkflowRunStore = create((set, get) => { activeNodeId: node.id, runState: { ...s.runState, blockIndex: i, blockProgress: 0, blockStep: 'Starting…' }, })) + recordStepStart(node, ctx.allExtensions) await executeExtensionNode(node, ctx, setRunState) + recordStepEnd('done') } finishBranch('done') } catch (err) { + if (!_cancel.current) recordStepEnd('error') finishBranch('error', String(err)) } }, cancel() { + finishRecord('cancelled') _cancel.current = true _pauseRequested.current = false flushResume() // unblock a manual While pause so the run can tear down @@ -1085,6 +1150,7 @@ export const useWorkflowRunStore = create((set, get) => { }, reset() { + finishRecord('cancelled') _ctx.current = null set({ runState: IDLE, activeNodeId: null, activeWorkflowId: null, nodeImageOutputs: {}, waitStates: {}, runningBranchId: null, whileProgress: {}, pausedGroup: [] }) }, diff --git a/src/shared/types/runs.ts b/src/shared/types/runs.ts new file mode 100644 index 000000000..c8fda5b63 --- /dev/null +++ b/src/shared/types/runs.ts @@ -0,0 +1,44 @@ +// Persisted run history -- mirrors api/schemas/runs.py (RunRecord). One record per +// generation, workflow or mesh-op run, listed by GET /workflow-runs. This is the +// shape the provenance model (T-F1) is meant to reuse rather than a second store. +// Timestamps are epoch seconds; output_path is workspace-relative when the output +// lives in the workspace, absolute otherwise. + +export type RunKind = 'generation' | 'workflow' | 'mesh_op' +/** 'interrupted' = still in flight when the backend stopped; terminal. */ +export type RunStatus = 'pending' | 'running' | 'done' | 'error' | 'cancelled' | 'interrupted' + +export interface RunStep { + name: string + node_id?: string | null + status: RunStatus + started_at: number + ended_at?: number | null +} + +export interface RunRecord { + id: string + kind: RunKind + status: RunStatus + model_id?: string | null + workflow_id?: string | null + workflow_name?: string | null + op_id?: string | null + inputs: Record + params: Record + steps: RunStep[] + error?: string | null + output_path?: string | null + created_at: number + started_at?: number | null + ended_at?: number | null + /** Computed by the backend; null while the run has not ended. */ + duration_s?: number | null +} + +export interface RunRecordPage { + runs: RunRecord[] + total: number + limit: number + offset: number +} From 6c2c61c7fbf3267f8b2c52dfe790bcd23a0cff30 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Sat, 14 Nov 2026 19:32:52 +0100 Subject: [PATCH 063/108] fix(workflows): accept object-shaped manifest inputs in preflight A manifest may declare `inputs` as slot objects ({ type, label, required }) instead of type names. The preflight compared those objects against output type strings, so every connection looked missing and the error named the type "text" whatever it was. Slot objects are now reduced to their type when the manifest is parsed, with their labels and optional flags carried as inputLabels and inputsRequired. Preflight only requires types with at least one required slot, and names unknown types as declared. Fixes #241 --- .../main/extension-install-utils.test.mjs | 44 ++++++++++++++++ electron/main/extension-install-utils.ts | 51 ++++++++++++++++++- electron/main/ipc-handlers.ts | 11 ++-- src/areas/workflows/mockExtensions.ts | 3 ++ src/areas/workflows/preflight.test.mjs | 27 ++++++++++ src/areas/workflows/preflight.ts | 15 +++--- src/shared/types/electron.d.ts | 1 + 7 files changed, 139 insertions(+), 13 deletions(-) diff --git a/electron/main/extension-install-utils.test.mjs b/electron/main/extension-install-utils.test.mjs index fd0eb1a08..f8c94df65 100644 --- a/electron/main/extension-install-utils.test.mjs +++ b/electron/main/extension-install-utils.test.mjs @@ -529,3 +529,47 @@ test('parseGitHubRepoUrl rejects URLs that do not name a GitHub repository', () assert.throws(() => mod.parseGitHubRepoUrl('https://github.com/owner/.git'), /expected format/) }) + +test('normalizeNodeInputs reduces slot objects to type names and keeps labels and optionality (#241)', () => { + const mod = loadModule() + // The manifest shape from the report (modly-Codex-image-extension, image-to-image) + assert.deepEqual(mod.normalizeNodeInputs({ + id: 'image-to-image', + input: 'image', + inputs: [ + { name: 'front', label: 'Primary image', type: 'image', required: true }, + { name: 'left', label: 'Image 2', type: 'image', required: false }, + ], + }), { + inputs: ['image', 'image'], + inputLabels: ['Primary image', 'Image 2'], + inputsRequired: [true, false], + }) + + // Plain type names are unchanged, with no derived labels or optionality + assert.deepEqual( + mod.normalizeNodeInputs({ inputs: ['image', 'text'], input_labels: ['Image', 'Prompt'] }), + { inputs: ['image', 'text'], inputLabels: ['Image', 'Prompt'], inputsRequired: undefined }, + ) + // Explicit input_labels win; a slot without a label falls back to its name, then its type + assert.deepEqual( + mod.normalizeNodeInputs({ inputs: ['mesh', { type: 'image', name: 'ref' }] }).inputLabels, + ['mesh', 'ref'], + ) + assert.deepEqual(mod.normalizeNodeInputs({ input: 'image' }), { inputLabels: undefined }) +}) + +test('normalizeNodeInputs rejects slot entries without a type', () => { + const mod = loadModule() + assert.throws(() => mod.normalizeNodeInputs({ id: 'n', inputs: [{ label: 'x' }] }), /n inputs\[0\] must be a type name/) + assert.throws(() => mod.normalizeNodeInputs({ id: 'n', inputs: ['image', null] }), /inputs\[1\]/) +}) + +test('validateInstallManifest sees object-shaped scene slots as scene inputs', () => { + const mod = loadModule() + const files = { hasEntryFile: () => true, hasGeneratorFile: () => true } + assert.throws(() => mod.validateInstallManifest({ + id: 'proc', type: 'process', entry: 'processor.js', + nodes: [{ id: 'run', inputs: [{ type: 'scene' }] }], + }, files, 'repository'), /scene input and output are supported only for model nodes/) +}) diff --git a/electron/main/extension-install-utils.ts b/electron/main/extension-install-utils.ts index c8d9a6da8..7f6d06c5c 100644 --- a/electron/main/extension-install-utils.ts +++ b/electron/main/extension-install-utils.ts @@ -21,6 +21,7 @@ export interface InstallManifest { id?: string input?: unknown inputs?: unknown + input_labels?: unknown output?: unknown hf_repo?: unknown model_sources?: unknown @@ -43,6 +44,54 @@ export interface ExtensionReloadPayload { errors: Record } +export interface NormalizedNodeInputs { + /** One type name per input slot, or undefined for a single-input node. */ + inputs?: string[] + /** Display label per slot: explicit `input_labels`, else the objects' labels. */ + inputLabels?: string[] + /** Set only when at least one slot is optional; true means required. */ + inputsRequired?: boolean[] +} + +/** + * A manifest may declare `inputs` as type names (`["image", "text"]`) or as + * slot objects (`[{ "type": "image", "label": "Front", "required": false }]`). + * Everything downstream compares slots against type names, so objects are + * reduced to their type here and their label/required carried separately. + */ +export function normalizeNodeInputs(node: { id?: unknown; inputs?: unknown; input_labels?: unknown }): NormalizedNodeInputs { + const explicitLabels = Array.isArray(node.input_labels) ? node.input_labels as string[] : undefined + if (!Array.isArray(node.inputs)) return { inputLabels: explicitLabels } + + const inputs: string[] = [] + const labels: (string | undefined)[] = [] + const required: boolean[] = [] + node.inputs.forEach((entry: unknown, i: number) => { + if (typeof entry === 'string') { + inputs.push(entry) + labels.push(undefined) + required.push(true) + return + } + const slot = entry as { type?: unknown; label?: unknown; name?: unknown; required?: unknown } | null + if (!slot || typeof slot !== 'object' || typeof slot.type !== 'string') { + throw new Error(`manifest.json: ${String(node.id ?? 'node')} inputs[${i}] must be a type name or an object with a "type"`) + } + inputs.push(slot.type) + labels.push(typeof slot.label === 'string' ? slot.label : typeof slot.name === 'string' ? slot.name : undefined) + required.push(slot.required !== false) + }) + + const derivedLabels = labels.some((label) => label !== undefined) + ? labels.map((label, i) => label ?? inputs[i]) + : undefined + return { + inputs, + inputLabels: explicitLabels ?? derivedLabels, + inputsRequired: required.every(Boolean) ? undefined : required, + } +} + export function assertSupportedSceneNodeShape( kind: 'model' | 'process', node: { id?: string; input?: unknown; inputs?: unknown; output?: unknown }, @@ -100,7 +149,7 @@ export function validateInstallManifest( validateModelNodeIds(manifest.nodes ?? []) } for (const node of Array.isArray(manifest.nodes) ? manifest.nodes : []) { - const declaredInputs = Array.isArray(node.inputs) ? node.inputs : [node.input ?? 'image'] + const declaredInputs = normalizeNodeInputs(node).inputs ?? [node.input ?? 'image'] const output = node.output ?? 'mesh' assertSupportedSceneNodeShape(isProcess ? 'process' : 'model', node, declaredInputs, output) const usesSharedWeights = weightGroups !== undefined || node.weight_groups !== undefined diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 0632620ff..13c8c7b28 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -74,6 +74,7 @@ import { validateExistingExtensionReplacement, validateInstallManifest, assertSupportedSceneNodeShape, + normalizeNodeInputs, } from './extension-install-utils' import { beginExtensionRegistrationTransaction, @@ -1225,7 +1226,7 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe id: string name?: string input?: string - inputs?: string[] + inputs?: unknown[] // type names, or { type, label?, required? } slot objects input_labels?: string[] output?: string params_schema?: unknown[] @@ -1263,7 +1264,8 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe validateModelNodeIds(parsed.nodes ?? []) } const nodes = (parsed.nodes ?? []).map(n => { - const declaredInputs = Array.isArray(n.inputs) ? n.inputs : [n.input ?? 'image'] + const slots = normalizeNodeInputs(n) + const declaredInputs = slots.inputs ?? [n.input ?? 'image'] const output = n.output ?? 'mesh' assertSupportedSceneNodeShape(parsed.type === 'process' ? 'process' : 'model', n, declaredInputs, output) const usesManagedWeights = weightGroups !== undefined @@ -1295,8 +1297,9 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe id: nodeId, name: n.name ?? n.id, input: n.input ?? 'image' as const, - inputs: n.inputs, - inputLabels: n.input_labels, + inputs: slots.inputs, + inputLabels: slots.inputLabels, + inputsRequired: slots.inputsRequired, output: n.output ?? 'mesh' as const, paramsSchema: n.params_schema ?? parsed.params_schema ?? [], paramDefaults: { ...(parsed.param_defaults ?? {}), ...(n.param_defaults ?? {}) }, diff --git a/src/areas/workflows/mockExtensions.ts b/src/areas/workflows/mockExtensions.ts index 9bdb9a0a6..46de1c99b 100644 --- a/src/areas/workflows/mockExtensions.ts +++ b/src/areas/workflows/mockExtensions.ts @@ -13,6 +13,7 @@ export interface WorkflowExtension { input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene')[] // multi-input; overrides input when set inputLabels?: string[] // display labels per input slot + inputsRequired?: boolean[] // per slot; absent means all required output: 'image' | 'text' | 'mesh' | 'audio' | 'scene' params: ParamSchema[] builtin: boolean @@ -52,6 +53,7 @@ export function buildAllWorkflowExtensions( input: node.input, inputs: node.inputs, inputLabels: node.inputLabels, + inputsRequired: node.inputsRequired, output: node.output, params: applyParamDefaults(node.paramsSchema as ParamSchema[], node.paramDefaults), builtin: ext.builtin, @@ -74,6 +76,7 @@ export function buildAllWorkflowExtensions( input: node.input, inputs: node.inputs, inputLabels: node.inputLabels, + inputsRequired: node.inputsRequired, output: node.output, params: applyParamDefaults(node.paramsSchema as ParamSchema[], node.paramDefaults), builtin: ext.builtin, diff --git a/src/areas/workflows/preflight.test.mjs b/src/areas/workflows/preflight.test.mjs index 3b4df8aa8..fd6d54445 100644 --- a/src/areas/workflows/preflight.test.mjs +++ b/src/areas/workflows/preflight.test.mjs @@ -132,6 +132,33 @@ test('multi-input extension requires every declared input type', () => { assert.ok(issues.some((i) => i.key === 'proc:missing:text')) }) +test('optional input slots are not required, but their type is still accepted (#241)', () => { + const { validateWorkflowPreflight } = loadModule() + const proc = { id: 'proc', type: 'extensionNode', position: { x: 0, y: 0 }, data: { extensionId: 'pack/process-node' } } + // image-to-image: required front image, optional second image + const imageToImage = ext({ inputs: ['image', 'image'], inputsRequired: [true, false] }) + assert.deepEqual( + validateWorkflowPreflight(wf([imageNode('img'), proc], [{ id: 'e1', source: 'img', target: 'proc', targetHandle: 'input-0' }]), [imageToImage]), + [], + ) + + // an optional text prompt next to a required image + const withPrompt = ext({ inputs: ['image', 'text'], inputsRequired: [true, false] }) + assert.deepEqual(validateWorkflowPreflight(wf([imageNode('img'), proc], [{ id: 'e1', source: 'img', target: 'proc' }]), [withPrompt]), []) + const both = validateWorkflowPreflight( + wf([imageNode('img'), textNode('txt'), proc], [{ id: 'e1', source: 'img', target: 'proc' }, { id: 'e2', source: 'txt', target: 'proc' }]), + [withPrompt], + ) + assert.deepEqual(both, []) +}) + +test('missing type names the real type instead of defaulting to text', () => { + const { validateWorkflowPreflight } = loadModule() + const proc = { id: 'proc', type: 'extensionNode', position: { x: 0, y: 0 }, data: { extensionId: 'pack/process-node' } } + const issues = validateWorkflowPreflight(wf([proc]), [ext({ input: 'pointcloud' })]) + assert.match(issues[0].message, /needs an incoming pointcloud connection/) +}) + test('scene input requires a validated Load Scene source and rejects image wiring', () => { const { validateWorkflowPreflight } = loadModule() const model = { id: 'model', type: 'extensionNode', position: { x: 0, y: 0 }, data: { extensionId: 'pack/process-node' } } diff --git a/src/areas/workflows/preflight.ts b/src/areas/workflows/preflight.ts index 068a9277b..695de05e2 100644 --- a/src/areas/workflows/preflight.ts +++ b/src/areas/workflows/preflight.ts @@ -30,11 +30,7 @@ function nodeLabel(node: WFNode, allExtensions: WorkflowExtension[]): string { } function formatType(type: DataType): string { - if (type === 'scene') return 'scene' - if (type === 'mesh') return 'mesh' - if (type === 'image') return 'image' - if (type === 'audio') return 'audio' - return 'text' + return String(type) } function formatRequiredTypes(types: DataType[]): string { @@ -142,7 +138,10 @@ export function validateWorkflowPreflight( } const incomingEdges = workflow.edges.filter((edge) => edge.target === node.id) - const requiredTypes = [...new Set((ext.inputs ?? [ext.input]) as DataType[])] + const slotTypes = (ext.inputs ?? [ext.input]) as DataType[] + const acceptedTypes = [...new Set(slotTypes)] + // A type is required when at least one slot of that type is not optional. + const requiredTypes = [...new Set(slotTypes.filter((_, i) => ext.inputsRequired?.[i] !== false))] for (const requiredType of requiredTypes) { const hasMatchingInput = incomingEdges.some((edge) => outputTypes.get(edge.source) === requiredType) @@ -158,11 +157,11 @@ export function validateWorkflowPreflight( for (const edge of incomingEdges) { const sourceNode = nodeMap.get(edge.source) const sourceType = outputTypes.get(edge.source) - if (!sourceNode || !sourceType || requiredTypes.includes(sourceType)) continue + if (!sourceNode || !sourceType || acceptedTypes.includes(sourceType)) continue pushIssue(issues, { key: `${node.id}:type:${edge.id}`, nodeId: node.id, - message: `${ext.name} expects ${formatRequiredTypes(requiredTypes)}, but ${nodeLabel(sourceNode, allExtensions)} outputs ${formatType(sourceType)}.`, + message: `${ext.name} expects ${formatRequiredTypes(acceptedTypes)}, but ${nodeLabel(sourceNode, allExtensions)} outputs ${formatType(sourceType)}.`, }) } } diff --git a/src/shared/types/electron.d.ts b/src/shared/types/electron.d.ts index 57ec8f3ad..653c09bd3 100644 --- a/src/shared/types/electron.d.ts +++ b/src/shared/types/electron.d.ts @@ -21,6 +21,7 @@ export interface ExtensionNode { input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene')[] // multi-input nodes; overrides input when set inputLabels?: string[] // display labels per input slot (e.g. positive/negative) + inputsRequired?: boolean[] // per input slot; absent means every slot is required output: 'image' | 'text' | 'mesh' | 'audio' | 'scene' paramsSchema: ParamSchema[] paramDefaults?: Record From d21a306edd837684fd8997fd5dfbe036158b1e97 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Sat, 14 Nov 2026 19:35:01 +0100 Subject: [PATCH 064/108] fix(linux): keep AppImage library paths out of extension setup The AppImage launcher points LD_LIBRARY_PATH (and possibly LD_PRELOAD and PATH) into its own mount. Extension setup inherited the raw environment, so Python, pip and the compilers it runs loaded the AppImage's bundled libraries, and in-app install/Repair failed where the same setup.py succeeded from a clean shell. Extension setup now runs with extensionChildEnv(): the Python/pip redirect stripping cleanPythonEnv() already did, plus removal of any path entry under APPDIR. It is a no-op outside an AppImage. The API and process extensions keep their environment, since they relaunch the bundled Electron binary. Refs #236 --- electron/main/child-env.test.mjs | 72 ++++++++++++++++++++++++++++++++ electron/main/child-env.ts | 56 +++++++++++++++++++++++++ electron/main/ipc-handlers.ts | 3 +- electron/main/python-setup.ts | 12 +----- 4 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 electron/main/child-env.test.mjs create mode 100644 electron/main/child-env.ts diff --git a/electron/main/child-env.test.mjs b/electron/main/child-env.test.mjs new file mode 100644 index 000000000..e71b3a70e --- /dev/null +++ b/electron/main/child-env.test.mjs @@ -0,0 +1,72 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildSync } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +function loadModule() { + const outfile = join(mkdtempSync(join(tmpdir(), 'modly-child-env-test-')), 'child-env.cjs') + const require = createRequire(import.meta.url) + const result = buildSync({ + entryPoints: [resolve('electron/main/child-env.ts')], + bundle: true, + platform: 'node', + format: 'cjs', + write: false, + }) + writeFileSync(outfile, result.outputFiles[0].text, 'utf8') + return require(outfile) +} + +test('pythonChildEnv strips vars that redirect to the user\'s Python', () => { + const { pythonChildEnv } = loadModule() + const env = pythonChildEnv({ HOME: '/home/u', PYTHONPATH: '/x', VIRTUAL_ENV: '/venv', PIP_USER: '1', CONDA_PREFIX: '/c' }) + assert.deepEqual(env, { HOME: '/home/u' }) +}) + +test('extensionChildEnv drops AppImage mount entries from library and binary paths (#236)', () => { + const { extensionChildEnv } = loadModule() + const mount = '/tmp/.mount_Modly-AbC123' + const env = extensionChildEnv({ + APPDIR: mount, + APPIMAGE: '/home/u/Modly.AppImage', + LD_LIBRARY_PATH: `${mount}/usr/lib:${mount}:/usr/local/cuda/lib64`, + LD_PRELOAD: `${mount}/usr/lib/libfoo.so /usr/lib/libbar.so`, + PATH: `${mount}/usr/bin:/usr/local/bin:/usr/bin`, + }) + assert.equal(env.LD_LIBRARY_PATH, '/usr/local/cuda/lib64') + assert.equal(env.LD_PRELOAD, '/usr/lib/libbar.so') + assert.equal(env.PATH, '/usr/local/bin:/usr/bin') + // The AppImage identity vars themselves are harmless and kept + assert.equal(env.APPDIR, mount) +}) + +test('extensionChildEnv removes a path var left empty, and keeps look-alike siblings', () => { + const { extensionChildEnv } = loadModule() + const mount = '/tmp/.mount_Modly-AbC123/' + const env = extensionChildEnv({ + APPDIR: mount, + LD_LIBRARY_PATH: '/tmp/.mount_Modly-AbC123/usr/lib', + PATH: '/tmp/.mount_Modly-AbC123-other/bin:/usr/bin', + }) + assert.equal('LD_LIBRARY_PATH' in env, false) + assert.equal(env.PATH, '/tmp/.mount_Modly-AbC123-other/bin:/usr/bin') +}) + +test('extensionChildEnv leaves paths alone outside an AppImage', () => { + const { extensionChildEnv } = loadModule() + const input = { LD_LIBRARY_PATH: '/opt/lib', PATH: '/usr/bin' } + assert.deepEqual(extensionChildEnv(input), input) + // The caller's env object is not mutated + const appEnv = { APPDIR: '/tmp/.mount_x', LD_LIBRARY_PATH: '/tmp/.mount_x/usr/lib' } + extensionChildEnv(appEnv) + assert.equal(appEnv.LD_LIBRARY_PATH, '/tmp/.mount_x/usr/lib') +}) + +test('pythonChildEnv keeps AppImage paths, for processes that relaunch the bundled Electron', () => { + const { pythonChildEnv } = loadModule() + const env = pythonChildEnv({ APPDIR: '/tmp/.mount_x', LD_LIBRARY_PATH: '/tmp/.mount_x/usr/lib', PYTHONPATH: '/x' }) + assert.deepEqual(env, { APPDIR: '/tmp/.mount_x', LD_LIBRARY_PATH: '/tmp/.mount_x/usr/lib' }) +}) diff --git a/electron/main/child-env.ts b/electron/main/child-env.ts new file mode 100644 index 000000000..28cf48a94 --- /dev/null +++ b/electron/main/child-env.ts @@ -0,0 +1,56 @@ +// Environment for the Python processes Modly spawns. Kept free of Electron +// imports so it can be unit-tested directly. + +/** Vars that would redirect imports or installs to the user's own Python. */ +const PYTHON_REDIRECT_VARS = [ + 'PYTHONHOME', 'PYTHONPATH', 'PYTHONSTARTUP', 'PYTHONUSERBASE', + 'PIP_USER', 'PIP_TARGET', 'PIP_PREFIX', 'PIP_REQUIRE_VIRTUALENV', + 'VIRTUAL_ENV', 'CONDA_PREFIX', +] + +/** + * Search-path vars an AppImage launcher points into its own mount. Python and + * the compilers pip runs would load the AppImage's bundled libraries instead of + * the system ones, which breaks venv creation and native builds (#236). + */ +const APPIMAGE_PATH_VARS: Record = { + LD_LIBRARY_PATH: ':', + LD_PRELOAD: ' ', + PATH: ':', +} + +function withoutEntriesUnder(value: string, dir: string, separator: string): string { + const root = dir.replace(/\/+$/, '') + return value + .split(separator === ' ' ? /[\s:]+/ : separator) + .filter((entry) => entry !== '' && entry !== root && !entry.startsWith(`${root}/`)) + .join(separator) +} + +/** The env without vars that redirect to the user's system Python. */ +export function pythonChildEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const clean: NodeJS.ProcessEnv = { ...env } + for (const name of PYTHON_REDIRECT_VARS) delete clean[name] + return clean +} + +/** + * pythonChildEnv, plus no path entries into the AppImage mount. Only for + * extension setup, which runs the stable Python copy, pip and system compilers. + * Not for the API or process extensions: they relaunch the Electron binary that + * lives inside the mount (MODLY_NODE_EXECUTABLE) and may need its libraries. + */ +export function extensionChildEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const clean = pythonChildEnv(env) + // APPDIR is set by the AppImage runtime to its mount (/tmp/.mount_Modly-XXXX). + const appDir = env.APPDIR + if (!appDir) return clean + for (const [name, separator] of Object.entries(APPIMAGE_PATH_VARS)) { + const value = clean[name] + if (value === undefined) continue + const kept = withoutEntriesUnder(value, appDir, separator) + if (kept) clean[name] = kept + else delete clean[name] + } + return clean +} diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 0632620ff..6fb1256c6 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -47,6 +47,7 @@ import { validateStorageLocation, type StorageKind, } from './storage-directory-guard' import { checkSetupNeeded, markSetupDone, runFullSetup, getVenvPythonExe, ensureSslPatch } from './python-setup' +import { extensionChildEnv } from './child-env' import { logger } from './logger' import { getProcessRunner, getPythonProcessRunner, getExtPythonExe, terminateProcessRunner, terminateAllProcessRunners } from './process-runner' import { getBuiltinExtensionsDir } from './builtin-sync' @@ -143,7 +144,7 @@ function runExtensionSetup( const proc = spawn(pythonExe, ['-c', launcher, setupPy, args], { stdio: ['ignore', 'pipe', 'pipe'], env: { - ...process.env, + ...extensionChildEnv(process.env), PIP_CACHE_DIR: pipCacheDir, MODLY_TORCH_FLAVOR: torchFlavor, MODLY_TORCH_INDEX_URL: gpu.torchIndexUrl ?? '', diff --git a/electron/main/python-setup.ts b/electron/main/python-setup.ts index 13e86a251..15daff1c9 100644 --- a/electron/main/python-setup.ts +++ b/electron/main/python-setup.ts @@ -6,6 +6,7 @@ import { spawn, execSync } from 'child_process' import { createHash } from 'crypto' import { getSettings } from './settings-store' import { copyRuntimeTree } from './copy-runtime' +import { pythonChildEnv } from './child-env' const SETUP_VERSION = 3 @@ -36,16 +37,7 @@ function hashRequirements(): string { * Strips vars that could redirect imports or installs to the user's system Python. */ export function cleanPythonEnv(): NodeJS.ProcessEnv { - const { - PYTHONHOME, PYTHONPATH, PYTHONSTARTUP, PYTHONUSERBASE, - PIP_USER, PIP_TARGET, PIP_PREFIX, PIP_REQUIRE_VIRTUALENV, - VIRTUAL_ENV, CONDA_PREFIX, - ...rest - } = process.env - void PYTHONHOME; void PYTHONPATH; void PYTHONSTARTUP; void PYTHONUSERBASE - void PIP_USER; void PIP_TARGET; void PIP_PREFIX; void PIP_REQUIRE_VIRTUALENV - void VIRTUAL_ENV; void CONDA_PREFIX - return rest + return pythonChildEnv(process.env) } // ─── Path helpers ───────────────────────────────────────────────────────────── From 266c0d1d25452b12a13335b711381eff192fe437 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Sat, 14 Nov 2026 21:49:48 +0100 Subject: [PATCH 065/108] feat(home): add a Home page with recent creations and quick start The app now opens on a Home page, first in the sidebar: - the Modly logo above a centred welcome - a drop zone that hands the image to Generate and opens it; Generate's workflow panel puts it in the first image input it shows, as if it had been browsed there - quick actions: open a workflow, import a mesh, browse the library - the six newest openable assets with thumbnails, reopened in Generate Settings > Application gets a Start page choice (Home or Generate) so users can keep the old behaviour; it lives in localStorage and is read by navStore at init. The imported-mesh job moves to a shared helper (meshJob.ts) used by Generate and Home, the setup wizard's logo becomes a shared ModlyLogo component, and LibraryGrid exports its Thumbnail. Closes #416 --- package.json | 2 +- src/areas/generate/GeneratePage.tsx | 15 +- .../generate/components/WorkflowPanel.tsx | 31 ++- src/areas/home/HomePage.tsx | 211 ++++++++++++++++++ src/areas/home/recents.test.ts | 44 ++++ src/areas/home/recents.ts | 38 ++++ src/areas/library/LibraryGrid.tsx | 2 +- .../components/ApplicationSection.tsx | 21 +- src/areas/setup/FirstRunSetup.tsx | 20 +- .../components/agent/clientTools.test.ts | 2 +- src/shared/components/ui/ModlyLogo.tsx | 19 ++ src/shared/pages/pageRegistry.test.ts | 1 + src/shared/pages/pageRegistry.ts | 11 + src/shared/pages/startPage.test.ts | 33 +++ src/shared/pages/startPage.ts | 28 +++ src/shared/stores/navStore.ts | 3 +- src/shared/utils/imageInput.ts | 30 +++ src/shared/utils/meshJob.ts | 21 ++ 18 files changed, 494 insertions(+), 38 deletions(-) create mode 100644 src/areas/home/HomePage.tsx create mode 100644 src/areas/home/recents.test.ts create mode 100644 src/areas/home/recents.ts create mode 100644 src/shared/components/ui/ModlyLogo.tsx create mode 100644 src/shared/pages/startPage.test.ts create mode 100644 src/shared/pages/startPage.ts create mode 100644 src/shared/utils/imageInput.ts create mode 100644 src/shared/utils/meshJob.ts diff --git a/package.json b/package.json index f7c15b511..41881a139 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/areas/workflows/workflowRunRecord.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/shared/pages/startPage.test.ts src/areas/home/recents.test.ts src/areas/workflows/workflowRunRecord.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/GeneratePage.tsx b/src/areas/generate/GeneratePage.tsx index 759fb2bd9..b546ef3a2 100644 --- a/src/areas/generate/GeneratePage.tsx +++ b/src/areas/generate/GeneratePage.tsx @@ -1,7 +1,8 @@ import { useState, useRef, useCallback, useEffect, useMemo } from 'react' import type { ReactNode } from 'react' import { useAppStore, DEFAULT_LIGHT_SETTINGS } from '@shared/stores/appStore' -import type { GenerationJob, LightSettings, PointLight } from '@shared/stores/appStore' +import type { LightSettings, PointLight } from '@shared/stores/appStore' +import { showImportedMesh } from '@shared/utils/meshJob' import { redoDepth, undoDepth } from '@shared/stores/meshHistory' import { useApi } from '@shared/hooks/useApi' import { ColorPicker } from '@shared/components/ui' @@ -872,17 +873,7 @@ export default function GeneratePage(): JSX.Element { setImporting(true) try { const { url } = await importMesh(filePath) - const job: GenerationJob = { - id: `import-${Date.now()}`, - imageFile: '', - status: 'done', - progress: 100, - outputUrl: url, - originalOutputUrl: url, - createdAt: Date.now(), - } - setCurrentJob(job) - pushMeshUrl(url) + showImportedMesh(url) } finally { setImporting(false) } diff --git a/src/areas/generate/components/WorkflowPanel.tsx b/src/areas/generate/components/WorkflowPanel.tsx index d1be5e9f7..493bf42c6 100644 --- a/src/areas/generate/components/WorkflowPanel.tsx +++ b/src/areas/generate/components/WorkflowPanel.tsx @@ -14,6 +14,7 @@ import { useWaitButton } from '@areas/workflows/useWaitButton' import { buildAllWorkflowExtensions, getWorkflowExtension } from '@areas/workflows/mockExtensions' import { validateWorkflowPreflight } from '@areas/workflows/preflight' import { mimeFromPath } from '@areas/workflows/nodes/imageUtils' +import { SUPPORTED_IMAGE_TYPES, useImageHandoff } from '@shared/utils/imageInput' import type { WorkflowExtension } from '@areas/workflows/mockExtensions' import type { Workflow, WFNode, WFEdge, ParamSchema } from '@shared/types/electron.d' import { PICKER_LABELS, openParamPicker, resolvePickerIntent } from '@shared/utils/paramPicker' @@ -28,8 +29,6 @@ const TYPE_COLOR: Record = { text: '#fbbf24', } -const SUPPORTED_IMAGE_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp']) - // ─── Helpers ────────────────────────────────────────────────────────────────── function topoSortNodes(nodes: Workflow['nodes'], edges: Workflow['edges']): WFNode[] { @@ -155,6 +154,12 @@ function WorkflowDropdown({ workflows, value, onChange }: { type PatchFn = (nodeId: string, patch: Record) => void +/** The image node params for a file on disk: its path and an inline preview. */ +async function readImageParams(path: string): Promise<{ filePath: string, preview: string }> { + const base64 = await window.electron.fs.readFileBase64(path) + return { filePath: path, preview: `data:${mimeFromPath(path)};base64,${base64}` } +} + function ImageParamRow({ nodeId, nodes, onPatch }: { nodeId: string; nodes: FlowNode[]; onPatch: PatchFn }) { const node = nodes.find((n) => n.id === nodeId) const data = node?.data as { params: Record } | undefined @@ -166,10 +171,9 @@ function ImageParamRow({ nodeId, nodes, onPatch }: { nodeId: string; nodes: Flow const request = ++loadRequest.current if (!path) return try { - const base64 = await window.electron.fs.readFileBase64(path) + const image = await readImageParams(path) if (request !== loadRequest.current) return - const src = `data:${mimeFromPath(path)};base64,${base64}` - onPatch(nodeId, { params: { ...(data?.params ?? {}), filePath: path, preview: src } }) + onPatch(nodeId, { params: { ...(data?.params ?? {}), ...image } }) } catch { if (request === loadRequest.current) showToast('Unable to load the selected image') } @@ -519,6 +523,23 @@ function EmbeddedCanvas({ workflow, allExtensions }: { && (n.data as { showInGenerate?: boolean }).showInGenerate === true, ) + // An image handed over by another page (Home) goes into the first image input + // shown here, exactly as if it had been browsed in this panel. + const pendingImage = useImageHandoff((s) => s.pendingPath) + useEffect(() => { + const path = useImageHandoff.getState().take() + if (!path) return + const target = paramNodes.find((n) => n.type === 'imageNode') + if (!target) { + showToast('This workflow has no image input. Pick a workflow with one.') + return + } + readImageParams(path) + .then((image) => patchNode(target.id, { params: { ...(target.data.params ?? {}), ...image } })) + .catch(() => showToast('Unable to load the selected image')) + // eslint-disable-next-line react-hooks/exhaustive-deps -- runs when an image is handed over; nodes are read at that point + }, [pendingImage]) + const handleGenerate = useCallback(() => { if (firstPreflightIssue) { showToast(firstPreflightIssue) diff --git a/src/areas/home/HomePage.tsx b/src/areas/home/HomePage.tsx new file mode 100644 index 000000000..f0cf5b26c --- /dev/null +++ b/src/areas/home/HomePage.tsx @@ -0,0 +1,211 @@ +import { useCallback, useEffect, useMemo, useState } from 'react' +import { useAppStore } from '@shared/stores/appStore' +import { useNavStore } from '@shared/stores/navStore' +import { useApi } from '@shared/hooks/useApi' +import { droppedImagePath, useImageHandoff } from '@shared/utils/imageInput' +import { showImportedMesh } from '@shared/utils/meshJob' +import { requestLibraryThumbnails } from '@shared/thumbnails/assetThumbnailService' +import type { ProjectedAssetLibraryEntry } from '@areas/generate/assetLibraryProjection' +import { getDefaultAssetLibraryService } from '@areas/generate/assetLibraryService' +import { openAssetLibraryEntry } from '@areas/generate/assetLibraryUi' +import { Thumbnail } from '@areas/library/LibraryGrid' +import ModlyLogo from '@shared/components/ui/ModlyLogo' +import { formatRelativeDate, selectRecentAssets } from './recents' + +function Icon({ d, size = 16 }: { d: string, size?: number }): JSX.Element { + return ( + + ) +} + +const ICONS = { + upload: 'M12 16V4m0 0-4.5 4.5M12 4l4.5 4.5M4 15v3a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-3', + workflow: 'M4 5h5v5H4zM15 14h5v5h-5zM9 7.5h3.5a2 2 0 0 1 2 2V14', + mesh: 'M12 2 3 7v10l9 5 9-5V7zM3 7l9 5 9-5M12 12v10', + library: 'M4 4h6.5v6.5H4zM13.5 4H20v6.5h-6.5zM4 13.5h6.5V20H4zM13.5 13.5H20V20h-6.5z', +} + +function SectionHeader({ title, action }: { title: string, action?: { label: string, onClick: () => void } }): JSX.Element { + return ( +
+

{title}

+ {action && ( + + )} +
+ ) +} + +function QuickAction({ icon, label, detail, busy, onClick }: { icon: string, label: string, detail: string, busy?: boolean, onClick: () => void }): JSX.Element { + return ( + + ) +} + +function DropZone({ onImage }: { onImage: (path: string) => void }): JSX.Element { + const [dragging, setDragging] = useState(false) + return ( + + ) +} + +function RecentAsset({ entry, opening, onOpen }: { entry: ProjectedAssetLibraryEntry, opening: boolean, onOpen: () => void }): JSX.Element { + const date = formatRelativeDate(entry.updatedAt ?? entry.createdAt) + return ( + + ) +} + +function Empty({ text }: { text: string }): JSX.Element { + return ( +
+

{text}

+
+ ) +} + +export default function HomePage(): JSX.Element { + const navigate = useNavStore((s) => s.navigate) + const showError = useAppStore((s) => s.showError) + const { importMesh } = useApi() + const service = useMemo(() => getDefaultAssetLibraryService(), []) + + const [assets, setAssets] = useState([]) + const [assetsLoaded, setAssetsLoaded] = useState(false) + const [openingId, setOpeningId] = useState(null) + const [importing, setImporting] = useState(false) + + const loadAssets = useCallback(async () => { + try { + const result = await service.list() + if (!result.success) return + const recent = selectRecentAssets(result.entries) + setAssets(recent) + // Render only the thumbnails this page shows. + void requestLibraryThumbnails(recent) + } finally { + setAssetsLoaded(true) + } + }, [service]) + + useEffect(() => { void loadAssets() }, [loadAssets]) + + async function openAsset(entry: ProjectedAssetLibraryEntry) { + if (openingId) return + setOpeningId(entry.id) + try { + const result = await openAssetLibraryEntry(service, entry) + if (!result.success) { + showError(result.message) + return + } + const { setCurrentJob, pushMeshUrl } = useAppStore.getState() + setCurrentJob(result.selection.job) + pushMeshUrl(result.selection.historyUrl) + navigate('generate') + } catch (err) { + showError(err instanceof Error ? err.message : String(err)) + } finally { + setOpeningId(null) + } + } + + async function importFromDisk() { + const filePath = await window.electron.fs.selectMeshFile() + if (!filePath) return + setImporting(true) + try { + const { url } = await importMesh(filePath) + showImportedMesh(url) + navigate('generate') + } catch (err) { + showError(err instanceof Error ? err.message : 'Could not import the mesh.') + } finally { + setImporting(false) + } + } + + const returning = assets.length > 0 + + return ( +
+
+ +

{returning ? 'Welcome back' : 'Welcome to Modly'}

+
+ +
+ { useImageHandoff.getState().offer(path); navigate('generate') }} /> +
+ navigate('workflows')} /> + { void importFromDisk() }} /> + navigate('library')} /> +
+
+ +
+ 0 ? { label: 'See all', onClick: () => navigate('library') } : undefined} /> + {assets.length > 0 ? ( +
+ {assets.map((entry) => ( + { void openAsset(entry) }} /> + ))} +
+ ) : assetsLoaded ? ( + + ) : ( +
+
+ ) +} diff --git a/src/areas/home/recents.test.ts b/src/areas/home/recents.test.ts new file mode 100644 index 000000000..6a1e732c0 --- /dev/null +++ b/src/areas/home/recents.test.ts @@ -0,0 +1,44 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import type { AssetLibraryEntry } from '../../shared/types/assetLibrary' +import { projectAssetLibraryEntry, type ProjectedAssetLibraryEntry } from '../generate/assetLibraryProjection' +import { formatRelativeDate, selectRecentAssets } from './recents' + +function entry(name: string, overrides: Partial = {}): ProjectedAssetLibraryEntry { + return projectAssetLibraryEntry({ + id: `library:${name}`, + workspacePath: `Workflows/run/${name}`, + displayName: name, + sourceScope: 'workflows', + capability: 'mesh', + state: 'ready', + previewKind: '3d-model', + warnings: [], + openable: true, + ...overrides, + }) +} + +test('recent assets are the newest openable ones, newest first, capped', () => { + const entries = [ + entry('old.glb', { createdAt: '2026-01-01T00:00:00.000Z' }), + entry('new.glb', { createdAt: '2026-03-01T00:00:00.000Z' }), + entry('mid.glb', { createdAt: '2026-02-01T00:00:00.000Z' }), + entry('locked.glb', { createdAt: '2026-04-01T00:00:00.000Z', state: 'unsafe', openable: false }), + entry('notes.txt', { createdAt: '2026-05-01T00:00:00.000Z', previewKind: 'text', openable: false }), + ] + assert.deepEqual(selectRecentAssets(entries).map((e) => e.displayName), ['new.glb', 'mid.glb', 'old.glb']) + assert.deepEqual(selectRecentAssets(entries, 2).map((e) => e.displayName), ['new.glb', 'mid.glb']) + assert.deepEqual(selectRecentAssets([]), []) +}) + +test('formatRelativeDate reads ISO dates relative to now', () => { + const now = Date.parse('2026-11-14T12:00:00.000Z') + // English whatever the OS language, like the rest of the UI. + assert.equal(formatRelativeDate('2026-11-14T11:55:00.000Z', now), '5 minutes ago') + assert.equal(formatRelativeDate('2026-11-13T12:00:00.000Z', now), 'yesterday') + assert.equal(formatRelativeDate('2026-11-14T11:59:50.000Z', now), 'this minute') + assert.equal(formatRelativeDate('nope', now), null) + assert.equal(formatRelativeDate(undefined, now), null) +}) diff --git a/src/areas/home/recents.ts b/src/areas/home/recents.ts new file mode 100644 index 000000000..48ed2764c --- /dev/null +++ b/src/areas/home/recents.ts @@ -0,0 +1,38 @@ +import type { ProjectedAssetLibraryEntry } from '../generate/assetLibraryProjection' +import { isAssetLibraryEntryOpenable } from '../generate/assetLibraryUi' +import { DEFAULT_LIBRARY_QUERY, selectLibraryEntries } from '../library/libraryView' + +export const RECENT_ASSET_LIMIT = 6 + +/** The newest assets the user can reopen, by the Library's own visibility and date rules. */ +export function selectRecentAssets( + entries: ProjectedAssetLibraryEntry[], + limit = RECENT_ASSET_LIMIT, +): ProjectedAssetLibraryEntry[] { + return selectLibraryEntries(entries, DEFAULT_LIBRARY_QUERY) + .filter((entry) => isAssetLibraryEntryOpenable(entry)) + .slice(0, limit) +} + +function timeOf(value: string | undefined): number { + const time = value ? Date.parse(value) : Number.NaN + return Number.isFinite(time) ? time : Number.NEGATIVE_INFINITY +} + +// English like the rest of the UI ("yesterday"), not the OS language. +const RELATIVE = new Intl.RelativeTimeFormat('en', { numeric: 'auto' }) +const UNITS: [Intl.RelativeTimeFormatUnit, number][] = [ + ['year', 365 * 24 * 3600], ['month', 30 * 24 * 3600], ['week', 7 * 24 * 3600], + ['day', 24 * 3600], ['hour', 3600], ['minute', 60], +] + +/** "5 minutes ago", "yesterday"… for an ISO date; null when it can't be read. */ +export function formatRelativeDate(value: string | undefined, now = Date.now()): string | null { + const time = timeOf(value) + if (!Number.isFinite(time)) return null + const seconds = Math.round((time - now) / 1000) + for (const [unit, size] of UNITS) { + if (Math.abs(seconds) >= size) return RELATIVE.format(Math.round(seconds / size), unit) + } + return RELATIVE.format(0, 'minute') +} diff --git a/src/areas/library/LibraryGrid.tsx b/src/areas/library/LibraryGrid.tsx index b6d8f10f2..5feba2266 100644 --- a/src/areas/library/LibraryGrid.tsx +++ b/src/areas/library/LibraryGrid.tsx @@ -62,7 +62,7 @@ export function formatEntryDate(entry: ProjectedAssetLibraryEntry): string | nul } /** The asset's thumbnail, or its capability icon until one is rendered (or if it fails to load). */ -function Thumbnail({ entry, iconSize }: { entry: ProjectedAssetLibraryEntry, iconSize: number }): JSX.Element { +export function Thumbnail({ entry, iconSize }: { entry: ProjectedAssetLibraryEntry, iconSize: number }): JSX.Element { const url = useAssetThumbnailUrl(entry) const [failedUrl, setFailedUrl] = useState(null) if (url && url !== failedUrl) { diff --git a/src/areas/settings/components/ApplicationSection.tsx b/src/areas/settings/components/ApplicationSection.tsx index 63984cb73..22583caf3 100644 --- a/src/areas/settings/components/ApplicationSection.tsx +++ b/src/areas/settings/components/ApplicationSection.tsx @@ -1,12 +1,31 @@ +import { useState } from 'react' import { useAppStore } from '@shared/stores/appStore' -import { Section, Card, Row, Toggle } from '@shared/ui' +import { readStartPage, writeStartPage, type StartPage } from '@shared/pages/startPage' +import { Section, Card, Row, Toggle, SegmentedControl } from '@shared/ui' + +const START_PAGE_OPTIONS: { value: StartPage; label: string }[] = [ + { value: 'home', label: 'Home' }, + { value: 'generate', label: 'Generate' }, +] export function ApplicationSection(): JSX.Element { const { showRamIndicator, setShowRamIndicator } = useAppStore() + const [startPage, setStartPage] = useState(readStartPage) + + function changeStartPage(page: StartPage) { + setStartPage(page) + writeStartPage(page) + } return (
+ + + - - - - - - - - - - - - - - +
) } @@ -27,7 +15,7 @@ function ModlyLogo(): JSX.Element { function AppHeader(): JSX.Element { return ( <> - +

Modly

AI-powered 3D mesh generation

diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index b6f29acec..24a31ab45 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -337,7 +337,7 @@ test('navigate changes page and validates it', async () => { const f = fake() assert.equal(await run('navigate', { page: 'models' }, f), 'Opened the Extensions page.') assert.deepEqual(f.calls, [['navigate', 'models']]) - assert.match(await run('navigate', { page: 'home' }, f), /^Error: page must be one of: generate, workflows, library, models, settings/) + assert.match(await run('navigate', { page: 'nope' }, f), /^Error: page must be one of: home, generate, workflows, library, models, settings/) }) test('a throwing dependency comes back as an error text', async () => { diff --git a/src/shared/components/ui/ModlyLogo.tsx b/src/shared/components/ui/ModlyLogo.tsx new file mode 100644 index 000000000..8b62b221f --- /dev/null +++ b/src/shared/components/ui/ModlyLogo.tsx @@ -0,0 +1,19 @@ +/** The Modly mark, in its violet gradient. */ +export default function ModlyLogo({ size = 64, className }: { size?: number, className?: string }): JSX.Element { + return ( + + + + + + + + + + + + + + + ) +} diff --git a/src/shared/pages/pageRegistry.test.ts b/src/shared/pages/pageRegistry.test.ts index ce65055b7..eec0988bb 100644 --- a/src/shared/pages/pageRegistry.test.ts +++ b/src/shared/pages/pageRegistry.test.ts @@ -21,6 +21,7 @@ test('the pages keep their ids, labels and sidebar order', () => { assert.deepEqual( pagesByGroup().map((s) => [s.group, s.pages.map((p) => `${p.id}:${p.label}`)]), [ + ['home', ['home:Home']], ['create', ['generate:Generate', 'workflows:Workflows']], ['manage', ['library:Library']], ['system', ['models:Extensions', 'settings:Settings']], diff --git a/src/shared/pages/pageRegistry.ts b/src/shared/pages/pageRegistry.ts index 6428d0718..c886fb23c 100644 --- a/src/shared/pages/pageRegistry.ts +++ b/src/shared/pages/pageRegistry.ts @@ -30,6 +30,17 @@ export interface PageDefinition { } export const PAGES = [ + { + id: 'home', + label: 'Home', + group: 'home', + icon: [ + ['path', { d: 'M3 10.5 12 3l9 7.5' }], + ['path', { d: 'M5 9v10.5a1 1 0 0 0 1 1h4v-6h4v6h4a1 1 0 0 0 1-1V9' }], + ], + load: () => import('@areas/home/HomePage'), + wrapperClass: 'flex-1 overflow-y-auto', + }, { id: 'generate', label: 'Generate', diff --git a/src/shared/pages/startPage.test.ts b/src/shared/pages/startPage.test.ts new file mode 100644 index 000000000..eb628668d --- /dev/null +++ b/src/shared/pages/startPage.test.ts @@ -0,0 +1,33 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { DEFAULT_START_PAGE, readStartPage, writeStartPage } from './startPage' + +function memoryStorage(initial: Record = {}) { + const data = new Map(Object.entries(initial)) + return { + getItem: (key: string) => data.get(key) ?? null, + setItem: (key: string, value: string) => { data.set(key, value) }, + } +} + +test('the app starts on Home unless the user chose Generate', () => { + assert.equal(DEFAULT_START_PAGE, 'home') + assert.equal(readStartPage(memoryStorage()), 'home') + assert.equal(readStartPage(memoryStorage({ 'modly.startPage': 'generate' })), 'generate') +}) + +test('an unknown or unreadable value falls back to Home', () => { + assert.equal(readStartPage(memoryStorage({ 'modly.startPage': 'settings' })), 'home') + assert.equal(readStartPage(undefined), 'home') + const throwing = { getItem: () => { throw new Error('blocked') }, setItem: () => { throw new Error('blocked') } } + assert.equal(readStartPage(throwing), 'home') + assert.doesNotThrow(() => writeStartPage('generate', throwing)) +}) + +test('writeStartPage round-trips', () => { + const storage = memoryStorage() + writeStartPage('generate', storage) + assert.equal(readStartPage(storage), 'generate') + writeStartPage('home', storage) + assert.equal(readStartPage(storage), 'home') +}) diff --git a/src/shared/pages/startPage.ts b/src/shared/pages/startPage.ts new file mode 100644 index 000000000..23a0f117c --- /dev/null +++ b/src/shared/pages/startPage.ts @@ -0,0 +1,28 @@ +// The page the app opens on. Kept in localStorage by this module rather than in +// appStore, so navStore can read it at init and still load under `node --test`. + +export type StartPage = 'home' | 'generate' + +export const START_PAGES: readonly StartPage[] = ['home', 'generate'] +export const DEFAULT_START_PAGE: StartPage = 'home' + +const STORAGE_KEY = 'modly.startPage' + +type Storage = Pick + +function defaultStorage(): Storage | undefined { + try { return globalThis.localStorage } catch { return undefined } +} + +export function readStartPage(storage: Storage | undefined = defaultStorage()): StartPage { + try { + const value = storage?.getItem(STORAGE_KEY) + return (START_PAGES as readonly string[]).includes(value ?? '') ? value as StartPage : DEFAULT_START_PAGE + } catch { + return DEFAULT_START_PAGE + } +} + +export function writeStartPage(page: StartPage, storage: Storage | undefined = defaultStorage()): void { + try { storage?.setItem(STORAGE_KEY, page) } catch { /* private mode / quota */ } +} diff --git a/src/shared/stores/navStore.ts b/src/shared/stores/navStore.ts index 6afcdfdac..b43f304cc 100644 --- a/src/shared/stores/navStore.ts +++ b/src/shared/stores/navStore.ts @@ -1,6 +1,7 @@ import { create } from 'zustand' // Relative and type-only, so the store loads under `node --test`. import type { Page } from '../pages/pageRegistry' +import { readStartPage } from '../pages/startPage' export type { Page } @@ -18,7 +19,7 @@ interface NavState { } export const useNavStore = create((set, get) => ({ - currentPage: 'generate', + currentPage: readStartPage(), params: null, navigate: (page, params) => set({ currentPage: page, params: params ?? null }), openExtension: (extensionId) => get().navigate('models', { extension: extensionId }), diff --git a/src/shared/utils/imageInput.ts b/src/shared/utils/imageInput.ts new file mode 100644 index 000000000..2aeba8846 --- /dev/null +++ b/src/shared/utils/imageInput.ts @@ -0,0 +1,30 @@ +import { create } from 'zustand' + +// Hands an input image from another page (Home) to Generate. Generate's image +// lives in the selected workflow's image node, which only Generate's panel can +// edit, so the path waits here until that panel picks it up. + +export const SUPPORTED_IMAGE_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp']) + +interface ImageHandoff { + pendingPath: string | null + offer: (path: string) => void + /** Returns the waiting path, if any, and clears it. */ + take: () => string | null +} + +export const useImageHandoff = create((set, get) => ({ + pendingPath: null, + offer: (path) => set({ pendingPath: path }), + take: () => { + const path = get().pendingPath + if (path) set({ pendingPath: null }) + return path + }, +})) + +/** The disk path of a dropped file, or null when it is not a supported image. */ +export function droppedImagePath(file: File | undefined): string | null { + if (!file || !SUPPORTED_IMAGE_TYPES.has(file.type)) return null + return window.electron.fs.getPathForFile(file) || null +} diff --git a/src/shared/utils/meshJob.ts b/src/shared/utils/meshJob.ts new file mode 100644 index 000000000..4cf2329a2 --- /dev/null +++ b/src/shared/utils/meshJob.ts @@ -0,0 +1,21 @@ +import { useAppStore, type GenerationJob } from '@shared/stores/appStore' + +/** The viewer job for a mesh imported from disk (already converted by the API). */ +export function importedMeshJob(url: string, now = Date.now()): GenerationJob { + return { + id: `import-${now}`, + imageFile: '', + status: 'done', + progress: 100, + outputUrl: url, + originalOutputUrl: url, + createdAt: now, + } +} + +/** Shows an imported mesh in Generate's viewer, as its first undo state. */ +export function showImportedMesh(url: string): void { + const { setCurrentJob, pushMeshUrl } = useAppStore.getState() + setCurrentJob(importedMeshJob(url)) + pushMeshUrl(url) +} From b9150fa363c95402e8632290c5b1571e1886ef66 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 16 Nov 2026 09:37:48 +0100 Subject: [PATCH 066/108] fix(workflows): resolve Mesh Exporter output paths against the workspace A relative Output path was resolved against the app's working directory (often Program Files on Windows), so exports failed or landed somewhere unexpected. It now resolves against the workspace folder. The field also accepts a filename: a path ending in the selected format's extension is written as that file (parent folders are created), while a trailing separator or an existing folder still means "export into this folder". A filename whose extension does not match the format is refused instead of silently writing the wrong format. --- src/areas/workflows/meshExporter.test.mjs | 93 +++++++++++++++++++ .../nodes/mesh-exporter/manifest.json | 2 +- .../nodes/mesh-exporter/processor.ts | 13 ++- 3 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 src/areas/workflows/meshExporter.test.mjs diff --git a/src/areas/workflows/meshExporter.test.mjs b/src/areas/workflows/meshExporter.test.mjs new file mode 100644 index 000000000..b5f5ff65a --- /dev/null +++ b/src/areas/workflows/meshExporter.test.mjs @@ -0,0 +1,93 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildSync } from 'esbuild' +import { createRequire } from 'node:module' +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import vm from 'node:vm' + +const require = createRequire(import.meta.url) +const compiled = buildSync({ + entryPoints: [fileURLToPath(new URL('./nodes/mesh-exporter/processor.ts', import.meta.url))], + bundle: true, + platform: 'node', + format: 'cjs', + write: false, + packages: 'external', +}).outputFiles[0].text + +function loadProcessor() { + // Exercise the real destination creation and file writing without needing + // glTF dependencies or a generated mesh for these export-path regressions. + class NodeIO { + registerExtensions() { return this } + async read() { return {} } + async write(filePath) { writeFileSync(filePath, 'exported mesh') } + } + const module = { exports: {} } + vm.runInNewContext(compiled, { + module, + exports: module.exports, + require: (name) => { + if (name === '@gltf-transform/core') return { NodeIO } + if (name === '@gltf-transform/extensions') return { ALL_EXTENSIONS: [] } + return require(name) + }, + Buffer, + }) + return module.exports +} + +for (const scenario of [ + { name: 'empty destination uses workspace Exports', destination: ' ', folder: 'Exports' }, + { name: 'relative directory uses workspace', destination: 'models', folder: 'models' }, + { name: 'relative filename stays a file in workspace', destination: 'Android_Female_Proportions.glb', file: 'Android_Female_Proportions.glb' }, + { name: 'nested filename creates parent directories', destination: 'models/character.glb', file: 'models/character.glb' }, + { name: 'absolute directory is preserved', absolute: true, destination: 'custom-exports', folder: 'custom-exports' }, + { name: 'absolute filename is preserved', absolute: true, destination: 'custom/character.glb', file: 'custom/character.glb' }, + { name: 'existing directory ending in glb remains a directory', destination: 'collection.glb', folder: 'collection.glb', existing: true }, + { name: 'trailing separator explicitly identifies a directory', destination: 'collection.glb/', folder: 'collection.glb' }, +]) { + test(scenario.name, async () => { + const root = mkdtempSync(join(tmpdir(), 'modly-export-')) + const workspaceDir = join(root, 'workspace') + const base = scenario.absolute ? root : workspaceDir + const destination = scenario.absolute ? join(root, scenario.destination) : scenario.destination + if (scenario.existing) mkdirSync(join(base, scenario.folder), { recursive: true }) + try { + const result = await loadProcessor()( + { filePath: join(root, 'input.glb') }, + { output_path: destination }, + { workspaceDir, log() {}, progress() {} }, + ) + if (scenario.file) { + assert.equal(result.filePath, join(base, scenario.file)) + } else { + assert.equal(dirname(result.filePath), join(base, scenario.folder)) + assert.match(result.filePath, /export-\d+\.glb$/) + } + assert.equal(readFileSync(result.filePath, 'utf8'), 'exported mesh') + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) +} + +test('a filename cannot silently use a different export format', async () => { + const root = mkdtempSync(join(tmpdir(), 'modly-export-')) + try { + await assert.rejects( + loadProcessor()( + { filePath: join(root, 'input.glb') }, + { output_path: 'character.stl', export_format: 'glb' }, + { workspaceDir: join(root, 'workspace'), log() {}, progress() {} }, + ), + /output file extension must be \.glb/, + ) + assert.equal(existsSync(join(root, 'workspace')), false) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) diff --git a/src/areas/workflows/nodes/mesh-exporter/manifest.json b/src/areas/workflows/nodes/mesh-exporter/manifest.json index 20eb7a860..ec1dbac41 100644 --- a/src/areas/workflows/nodes/mesh-exporter/manifest.json +++ b/src/areas/workflows/nodes/mesh-exporter/manifest.json @@ -31,7 +31,7 @@ "label": "Output Path", "type": "string", "default": "", - "tooltip": "Destination folder. Leave empty to save in the workspace Exports folder." + "tooltip": "Destination folder or filename with the selected format's extension. Relative paths use the workspace folder. Leave empty to save in the workspace Exports folder." } ] } diff --git a/src/areas/workflows/nodes/mesh-exporter/processor.ts b/src/areas/workflows/nodes/mesh-exporter/processor.ts index 4c1885611..19d819f73 100644 --- a/src/areas/workflows/nodes/mesh-exporter/processor.ts +++ b/src/areas/workflows/nodes/mesh-exporter/processor.ts @@ -202,8 +202,17 @@ const processor = async ( let outPath: string if (outputPath) { - fs.mkdirSync(outputPath, { recursive: true }) - outPath = path.join(outputPath, `export-${Date.now()}${ext}`) + // Relative destinations belong to the configured workspace, never the + // application's working directory (often Program Files on Windows). + const destination = path.resolve(context.workspaceDir, outputPath) + const destinationExt = path.extname(destination).toLowerCase() + const isFile = !/[\\/]$/.test(outputPath) + && Object.values(EXT_MAP).includes(destinationExt) + && !(fs.existsSync(destination) && fs.statSync(destination).isDirectory()) + if (isFile && destinationExt !== ext) { + throw new Error(`mesh-exporter: output file extension must be ${ext} for format "${format}"`) + } + outPath = isFile ? destination : path.join(destination, `export-${Date.now()}${ext}`) } else { const exportsDir = path.join(context.workspaceDir, 'Exports') fs.mkdirSync(exportsDir, { recursive: true }) From a188562d45873327df028f12bde6a1c7d40332e9 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 16 Nov 2026 09:51:52 +0100 Subject: [PATCH 067/108] docs(workflows): say that Mesh Exporter replaces an existing output file --- src/areas/workflows/nodes/mesh-exporter/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/areas/workflows/nodes/mesh-exporter/manifest.json b/src/areas/workflows/nodes/mesh-exporter/manifest.json index ec1dbac41..e603f729f 100644 --- a/src/areas/workflows/nodes/mesh-exporter/manifest.json +++ b/src/areas/workflows/nodes/mesh-exporter/manifest.json @@ -31,7 +31,7 @@ "label": "Output Path", "type": "string", "default": "", - "tooltip": "Destination folder or filename with the selected format's extension. Relative paths use the workspace folder. Leave empty to save in the workspace Exports folder." + "tooltip": "Destination folder or filename with the selected format's extension. Relative paths use the workspace folder. An existing file with that name is replaced. Leave empty to save in the workspace Exports folder." } ] } From 46059728c42e7db07a799e18c18d15b19c7312ba Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 16 Nov 2026 10:27:15 +0100 Subject: [PATCH 068/108] fix(system): make the VRAM indicator cheap, independent and quiet Follow-up to the VRAM indicator from #334: - nvidia-smi only runs while the VRAM indicator is on, with a 3 s timeout, windowsHide, one call at a time, and never again in the session once it turns out not to exist (AMD, Intel, no GPU). macOS never spawns it. - RAM and VRAM are sampled independently, so a slow nvidia-smi during a generation no longer freezes the RAM bar; a missed sample keeps the last value instead of flickering. - Each indicator can be shown on its own; VRAM no longer needs RAM on. - With several GPUs the one with the most memory is shown, named in the tooltip. The setting says it is NVIDIA only and counts every app. - The in-app agent can read and set showVramIndicator. Parsing and the sampler live in electron/main/gpu-memory.ts, with tests. --- api/routers/agent.py | 6 +- api/tests/test_agent_client_tools.py | 8 +- electron/main/gpu-memory.test.ts | 148 +++++++++++++++++ electron/main/gpu-memory.ts | 157 ++++++++++++++++++ electron/main/ipc-handlers.ts | 24 +-- electron/preload/electron-api.ts | 5 +- package.json | 2 +- .../components/ApplicationSection.tsx | 2 +- .../components/agent/agentContext.test.ts | 8 +- src/shared/components/agent/agentContext.ts | 6 +- .../components/agent/clientTools.test.ts | 8 +- src/shared/components/agent/clientTools.ts | 9 +- .../components/layout/MemoryIndicator.tsx | 137 ++++++++------- src/shared/components/layout/TopBar.tsx | 4 +- src/shared/types/electron.d.ts | 2 + 15 files changed, 420 insertions(+), 106 deletions(-) create mode 100644 electron/main/gpu-memory.test.ts create mode 100644 electron/main/gpu-memory.ts diff --git a/api/routers/agent.py b/api/routers/agent.py index 62fdff064..e8b1fd8b0 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -104,7 +104,7 @@ def registry_tool_lines(registry: MeshOpsRegistry) -> str: - **install_extension(github_url)** / **uninstall_extension(extension_id)** / **reload_extensions** — Manage extensions. - **download_model_weights(extension_id, variant?)** / **delete_model_weights(extension_id, variant?)** — Manage model weights. - **open_workflow(workflow_id)** / **delete_workflow(workflow_id)** — Open a workflow in the editor, or delete it. -- **set_setting(key, value)** — Change `uiScale` (one of `small`, `medium`, `large`, `very-large`), `useAtkinsonFont` or `showRamIndicator` (booleans). +- **set_setting(key, value)** — Change `uiScale` (one of `small`, `medium`, `large`, `very-large`), `useAtkinsonFont`, `showRamIndicator` or `showVramIndicator` (booleans). - **set_storage_folder(kind, path)** — Move the `models`, `workspace` or `workflows` folder. - **undo_mesh** / **redo_mesh** — Undo or redo the last change to the current mesh. - **navigate(page)** — Switch the app to another page. @@ -378,9 +378,9 @@ def _function(name: str, description: str, properties: dict | None = None, requi ), _function( "set_setting", - "Change a UI setting. uiScale is one of small, medium, large, very-large; useAtkinsonFont and showRamIndicator are booleans.", + "Change a UI setting. uiScale is one of small, medium, large, very-large; useAtkinsonFont, showRamIndicator and showVramIndicator are booleans.", { - "key": {"type": "string", "enum": ["uiScale", "useAtkinsonFont", "showRamIndicator"]}, + "key": {"type": "string", "enum": ["uiScale", "useAtkinsonFont", "showRamIndicator", "showVramIndicator"]}, "value": {"type": ["string", "boolean"], "description": "For uiScale one of: small, medium, large, very-large. A boolean otherwise."}, }, ["key", "value"], diff --git a/api/tests/test_agent_client_tools.py b/api/tests/test_agent_client_tools.py index 6c00e1049..502b8c70a 100644 --- a/api/tests/test_agent_client_tools.py +++ b/api/tests/test_agent_client_tools.py @@ -342,7 +342,7 @@ def test_every_client_tool_has_a_schema(self) -> None: def test_enums(self) -> None: props = self._tool("set_setting")["parameters"]["properties"] - self.assertEqual(props["key"]["enum"], ["uiScale", "useAtkinsonFont", "showRamIndicator"]) + self.assertEqual(props["key"]["enum"], ["uiScale", "useAtkinsonFont", "showRamIndicator", "showVramIndicator"]) # The app's scale names, not a number: the frontend accepts only these. self.assertEqual(props["value"]["type"], ["string", "boolean"]) self.assertIn("small, medium, large, very-large", props["value"]["description"]) @@ -372,7 +372,7 @@ def test_system_prompt_lists_the_new_tools(self) -> None: "modelsDir": "D:/models", "workspaceDir": "D:/ws", "workflowsDir": "D:/wf", "extensionsDir": "D:/ext", }, - "ui": {"uiScale": "large", "useAtkinsonFont": False, "showRamIndicator": True}, + "ui": {"uiScale": "large", "useAtkinsonFont": False, "showRamIndicator": True, "showVramIndicator": False}, } @@ -389,7 +389,7 @@ def test_list_extensions(self) -> None: def test_get_settings(self) -> None: text, payload = _run("get_settings") - for expected in ("D:/models", "D:/ws", "D:/wf", "D:/ext", "uiScale: large", "useAtkinsonFont: false", "showRamIndicator: true"): + for expected in ("D:/models", "D:/ws", "D:/wf", "D:/ext", "uiScale: large", "useAtkinsonFont: false", "showRamIndicator: true", "showVramIndicator: false"): self.assertIn(expected, text) self.assertIsNone(payload) self.assertIn("No settings", _run("get_settings", context={})[0]) @@ -452,7 +452,7 @@ def test_context_is_rendered_into_the_single_system_message(self) -> None: "- sf3d: SF3D", "- wf1: My Workflow", "D:/models", "D:/ws", "D:/wf", "D:/ext", - "uiScale: large", "useAtkinsonFont: false", "showRamIndicator: true", + "uiScale: large", "useAtkinsonFont: false", "showRamIndicator: true", "showVramIndicator: false", "Current mesh path: Default/mesh.glb", "- trellis/gen (image→mesh): Trellis", ): diff --git a/electron/main/gpu-memory.test.ts b/electron/main/gpu-memory.test.ts new file mode 100644 index 000000000..aeaedae99 --- /dev/null +++ b/electron/main/gpu-memory.test.ts @@ -0,0 +1,148 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +async function load() { + return import(new URL('./gpu-memory.ts', import.meta.url).href) +} + +const MIB = 1024 * 1024 + +test('parses a single GPU and converts MiB to bytes', async () => { + const { parseNvidiaSmiGpus, pickGpu, toGpuMemory } = await load() + const gpus = parseNvidiaSmiGpus('0, NVIDIA GeForce RTX 4090, 24564, 1234\n') + assert.deepEqual(gpus, [{ index: 0, name: 'NVIDIA GeForce RTX 4090', totalMiB: 24564, usedMiB: 1234 }]) + assert.deepEqual(toGpuMemory(pickGpu(gpus)), { + total: 24564 * MIB, + used: 1234 * MIB, + available: (24564 - 1234) * MIB, + name: 'NVIDIA GeForce RTX 4090', + }) +}) + +test('picks the GPU with the most memory on a multi-GPU machine', async () => { + const { parseNvidiaSmiGpus, pickGpu } = await load() + const stdout = [ + '0, NVIDIA GeForce GTX 1650, 4096, 300', + '1, NVIDIA GeForce RTX 3090, 24576, 9000', + '2, NVIDIA GeForce RTX 3060, 12288, 100', + ].join('\r\n') + const gpus = parseNvidiaSmiGpus(stdout) + assert.equal(gpus.length, 3) + assert.equal(pickGpu(gpus).index, 1) + assert.equal(pickGpu(gpus).name, 'NVIDIA GeForce RTX 3090') +}) + +test('keeps the first GPU on a tie', async () => { + const { parseNvidiaSmiGpus, pickGpu } = await load() + const gpus = parseNvidiaSmiGpus('0, A, 8192, 1\n1, B, 8192, 2\n') + assert.equal(pickGpu(gpus).index, 0) +}) + +test('rebuilds names containing commas and collapses extra spaces', async () => { + const { parseNvidiaSmiGpus } = await load() + const [gpu] = parseNvidiaSmiGpus(' 3 , Quadro RTX, 8000 Rev,B , 49152 , 0 ') + assert.deepEqual(gpu, { index: 3, name: 'Quadro RTX, 8000 Rev, B', totalMiB: 49152, usedMiB: 0 }) +}) + +test('skips lines with [N/A] or garbage and keeps the valid ones', async () => { + const { parseNvidiaSmiGpus, pickGpu } = await load() + const gpus = parseNvidiaSmiGpus('0, Tesla, [N/A], [N/A]\nnot a gpu\n1, RTX 4070, 12282, 512\n') + assert.equal(gpus.length, 1) + assert.equal(pickGpu(gpus).name, 'RTX 4070') +}) + +test('empty or error output yields no GPU', async () => { + const { parseNvidiaSmiGpus, pickGpu } = await load() + for (const stdout of ['', '\n\n', 'NVIDIA-SMI has failed because it couldn\'t communicate with the NVIDIA driver.', ', , ,', '0, X, 0, 0']) { + assert.equal(pickGpu(parseNvidiaSmiGpus(stdout)), null, JSON.stringify(stdout)) + } +}) + +test('sampler returns the parsed GPU and passes the timeout', async () => { + const { createGpuMemorySampler, NVIDIA_SMI_ARGS } = await load() + const calls: Array<{ args: string[]; timeoutMs: number }> = [] + const sample = createGpuMemorySampler({ + platform: 'win32', + timeoutMs: 3000, + run: async (args: string[], timeoutMs: number) => { + calls.push({ args, timeoutMs }) + return { kind: 'ok', stdout: '0, RTX, 8192, 2048\n' } + }, + }) + const mem = await sample() + assert.deepEqual(mem, { total: 8192 * MIB, used: 2048 * MIB, available: 6144 * MIB, name: 'RTX' }) + assert.deepEqual(calls, [{ args: NVIDIA_SMI_ARGS, timeoutMs: 3000 }]) +}) + +test('sampler remembers a missing nvidia-smi and stops spawning', async () => { + const { createGpuMemorySampler } = await load() + let runs = 0 + const sample = createGpuMemorySampler({ + platform: 'linux', + run: async () => { runs++; return { kind: 'missing' } }, + }) + assert.equal(await sample(), null) + assert.equal(await sample(), null) + assert.equal(await sample(), null) + assert.equal(runs, 1) +}) + +test('sampler retries after a transient failure (timeout / non-zero exit)', async () => { + const { createGpuMemorySampler } = await load() + const results = [{ kind: 'failed' }, { kind: 'ok', stdout: '0, RTX, 4096, 0' }] + let runs = 0 + const sample = createGpuMemorySampler({ + platform: 'linux', + run: async () => results[runs++], + }) + assert.equal(await sample(), null) + assert.equal((await sample())?.name, 'RTX') + assert.equal(runs, 2) +}) + +test('sampler shares one in-flight run between concurrent calls', async () => { + const { createGpuMemorySampler } = await load() + let runs = 0 + let release: (value: unknown) => void = () => {} + const sample = createGpuMemorySampler({ + platform: 'win32', + run: () => { + runs++ + return new Promise((resolve) => { release = resolve }) + }, + }) + const a = sample() + const b = sample() + assert.equal(a, b) + release({ kind: 'ok', stdout: '0, RTX, 4096, 1024' }) + const [ra, rb] = await Promise.all([a, b]) + assert.equal(runs, 1) + assert.deepEqual(ra, rb) + + // Once settled, the next call starts a fresh run. + const c = sample() + release({ kind: 'failed' }) + assert.equal(await c, null) + assert.equal(runs, 2) +}) + +test('sampler never spawns on macOS', async () => { + const { createGpuMemorySampler } = await load() + let runs = 0 + const sample = createGpuMemorySampler({ + platform: 'darwin', + run: async () => { runs++; return { kind: 'ok', stdout: '0, X, 1024, 0' } }, + }) + assert.equal(await sample(), null) + assert.equal(runs, 0) +}) + +test('sampler swallows a throwing runner', async () => { + const { createGpuMemorySampler } = await load() + const sample = createGpuMemorySampler({ + platform: 'linux', + run: async () => { throw new Error('boom') }, + }) + assert.equal(await sample(), null) +}) + diff --git a/electron/main/gpu-memory.ts b/electron/main/gpu-memory.ts new file mode 100644 index 000000000..18a46a625 --- /dev/null +++ b/electron/main/gpu-memory.ts @@ -0,0 +1,157 @@ +/** + * VRAM sampling for the status-bar indicator (NVIDIA only, via nvidia-smi). + * + * The renderer polls this every couple of seconds, so the sampler: + * - never runs two nvidia-smi processes at once (concurrent calls share one), + * - kills a stalled nvidia-smi after a timeout (the driver can hang while a + * model is generating) and reports null for that sample, + * - stops spawning for the rest of the session once nvidia-smi turns out not + * to exist (AMD / Intel / no GPU), and + * - returns null on macOS without spawning anything. + */ + +import { spawn } from 'child_process' + +export interface NvidiaGpu { + index: number + name: string + totalMiB: number + usedMiB: number +} + +/** Bytes. Mirrors the `system:gpuMemory` IPC contract. */ +export interface GpuMemory { + total: number + used: number + available: number + name: string +} + +const MIB = 1024 * 1024 + +export const NVIDIA_SMI_ARGS = [ + '--query-gpu=index,name,memory.total,memory.used', + '--format=csv,noheader,nounits', +] + +// ─── Pure helpers ───────────────────────────────────────────────────────────── + +/** + * Parses `nvidia-smi --query-gpu=index,name,memory.total,memory.used + * --format=csv,noheader,nounits`, e.g. "0, NVIDIA GeForce RTX 4090, 24564, 1234". + * The name sits between the index and the two numbers, so a name that itself + * contains commas is rebuilt from the middle fields. Lines that do not parse + * (blank, "[N/A]", error text) are skipped. + */ +export function parseNvidiaSmiGpus(stdout: string): NvidiaGpu[] { + const gpus: NvidiaGpu[] = [] + for (const raw of stdout.split(/\r?\n/)) { + const parts = raw.split(',').map((s) => s.trim()) + if (parts.length < 4) continue + const index = Number(parts[0]) + const totalMiB = Number(parts[parts.length - 2]) + const usedMiB = Number(parts[parts.length - 1]) + const name = parts.slice(1, -2).join(', ').replace(/\s+/g, ' ').trim() + if (parts[0] === '' || !Number.isInteger(index)) continue + if (!Number.isFinite(totalMiB) || !Number.isFinite(usedMiB)) continue + if (totalMiB <= 0 || usedMiB < 0) continue + gpus.push({ index, name, totalMiB, usedMiB }) + } + return gpus +} + +/** The GPU with the most memory (where heavy models end up); first one wins ties. */ +export function pickGpu(gpus: NvidiaGpu[]): NvidiaGpu | null { + let best: NvidiaGpu | null = null + for (const gpu of gpus) { + if (!best || gpu.totalMiB > best.totalMiB) best = gpu + } + return best +} + +export function toGpuMemory(gpu: NvidiaGpu): GpuMemory { + const total = gpu.totalMiB * MIB + const used = Math.min(gpu.usedMiB * MIB, total) + return { total, used, available: total - used, name: gpu.name || `GPU ${gpu.index}` } +} + +// ─── Sampler ────────────────────────────────────────────────────────────────── + +/** + * Outcome of one nvidia-smi run. `missing` means the binary is not there and + * never will be this session; `failed` covers timeouts and non-zero exits, + * which may clear up on the next sample. + */ +export type NvidiaSmiResult = + | { kind: 'ok'; stdout: string } + | { kind: 'missing' } + | { kind: 'failed' } + +export type NvidiaSmiRunner = (args: string[], timeoutMs: number) => Promise + +export const runNvidiaSmi: NvidiaSmiRunner = (args, timeoutMs) => + new Promise((resolve) => { + let settled = false + const finish = (result: NvidiaSmiResult): void => { + if (settled) return + settled = true + clearTimeout(timer) + resolve(result) + } + + const proc = spawn('nvidia-smi', args, { + stdio: ['ignore', 'pipe', 'ignore'], + windowsHide: true, + }) + const timer = setTimeout(() => { + proc.kill() + finish({ kind: 'failed' }) + }, timeoutMs) + + let out = '' + proc.stdout?.on('data', (d: Buffer) => { out += d.toString() }) + proc.on('close', (code) => finish(code === 0 ? { kind: 'ok', stdout: out } : { kind: 'failed' })) + proc.on('error', (err: NodeJS.ErrnoException) => + finish(err.code === 'ENOENT' ? { kind: 'missing' } : { kind: 'failed' })) + }) + +export interface GpuMemorySamplerOptions { + run?: NvidiaSmiRunner + platform?: NodeJS.Platform + timeoutMs?: number +} + +export function createGpuMemorySampler(options: GpuMemorySamplerOptions = {}): () => Promise { + const run = options.run ?? runNvidiaSmi + const platform = options.platform ?? process.platform + const timeoutMs = options.timeoutMs ?? 3000 + + let unavailable = platform === 'darwin' + let inFlight: Promise | null = null + + const sampleOnce = async (): Promise => { + try { + const result = await run(NVIDIA_SMI_ARGS, timeoutMs) + if (result.kind === 'missing') { + unavailable = true + return null + } + if (result.kind !== 'ok') return null + const gpu = pickGpu(parseNvidiaSmiGpus(result.stdout)) + return gpu ? toGpuMemory(gpu) : null + } catch { + return null + } + } + + return () => { + if (unavailable) return Promise.resolve(null) + if (!inFlight) { + inFlight = sampleOnce().finally(() => { inFlight = null }) + } + return inFlight + } +} + +/** Session-wide sampler used by the `system:gpuMemory` IPC handler. */ +export const sampleGpuMemory = createGpuMemorySampler() diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 5c17de81a..a43d1efa9 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -64,6 +64,7 @@ import { resolveExtensionPathWithinRoot, } from './extension-path-guard' import { detectGpuInfo, describeGpuInfo, torchFlavorFor, type GpuInfo } from './gpu-detect' +import { sampleGpuMemory } from './gpu-memory' import { SETUP_LAUNCHER_SOURCE } from './setup-launcher' import { assertCompatibleExtensionUpdateType, @@ -915,27 +916,8 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe }) // GPU memory (VRAM) — NVIDIA only via nvidia-smi - ipcMain.handle('system:gpuMemory', async () => { - if (process.platform === 'darwin' && process.arch === 'arm64') { - // Apple Silicon: unified memory, no separate VRAM - return null - } - try { - const { stdout } = await pExecFile('nvidia-smi', [ - '--query-gpu=memory.total,memory.used', - '--format=csv,noheader,nounits' - ]) - const line = stdout.trim().split('\n')[0].trim() - const [totalMiB, usedMiB] = line.split(',').map(s => parseInt(s.trim(), 10)) - if (isNaN(totalMiB) || isNaN(usedMiB) || totalMiB === 0) return null - const total = totalMiB * 1024 * 1024 - const used = usedMiB * 1024 * 1024 - const available = total - used - return { total, used, available } - } catch { - return null - } - }) + // (null on macOS, without NVIDIA, or when a sample times out). + ipcMain.handle('system:gpuMemory', () => sampleGpuMemory()) ipcMain.handle('app:info', () => ({ version: app.getVersion(), diff --git a/electron/preload/electron-api.ts b/electron/preload/electron-api.ts index 7c86fa3c9..9a309e27e 100644 --- a/electron/preload/electron-api.ts +++ b/electron/preload/electron-api.ts @@ -62,8 +62,9 @@ export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFra system: { memory: (): Promise<{ total: number; used: number; available: number }> => ipcRenderer.invoke('system:memory') as Promise<{ total: number; used: number; available: number }>, - gpuMemory: (): Promise<{ total: number; used: number; available: number } | null> => - ipcRenderer.invoke('system:gpuMemory') as Promise<{ total: number; used: number; available: number } | null>, + /** VRAM in bytes of the largest NVIDIA GPU; null without NVIDIA, on macOS, or when a sample timed out. */ + gpuMemory: (): Promise<{ total: number; used: number; available: number; name: string } | null> => + ipcRenderer.invoke('system:gpuMemory') as Promise<{ total: number; used: number; available: number; name: string } | null>, }, // Python / FastAPI bridge diff --git a/package.json b/package.json index 41881a139..0ef0908f1 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/shared/pages/startPage.test.ts src/areas/home/recents.test.ts src/areas/workflows/workflowRunRecord.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/main/gpu-memory.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/shared/pages/startPage.test.ts src/areas/home/recents.test.ts src/areas/workflows/workflowRunRecord.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/settings/components/ApplicationSection.tsx b/src/areas/settings/components/ApplicationSection.tsx index f77ab962c..52fa6a132 100644 --- a/src/areas/settings/components/ApplicationSection.tsx +++ b/src/areas/settings/components/ApplicationSection.tsx @@ -34,7 +34,7 @@ export function ApplicationSection(): JSX.Element { diff --git a/src/shared/components/agent/agentContext.test.ts b/src/shared/components/agent/agentContext.test.ts index d00ea5758..f13031bb1 100644 --- a/src/shared/components/agent/agentContext.test.ts +++ b/src/shared/components/agent/agentContext.test.ts @@ -32,7 +32,7 @@ const inputs = (over: Partial = {}): AgentContextInputs => ( processExtensions: [processExt], models: [{ id: 'triposr/main', name: 'TripoSR' }], storage: STORAGE, - ui: { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }, + ui: { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false, showVramIndicator: true }, ...over, }) @@ -49,7 +49,7 @@ test('the context carries the page, mesh, workflows, extensions, models, storage ], models: [{ id: 'triposr/main', name: 'TripoSR' }], storage: STORAGE, - ui: { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }, + ui: { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false, showVramIndicator: true }, }) }) @@ -67,7 +67,7 @@ test('storage keeps only the four folders, never the HF token', () => { test('buildAgentContext reads the stores and the async sources', async () => { useNavStore.setState({ currentPage: 'workflows' }) - useAppStore.setState({ currentJob: null, meshStats: null, uiScale: 'small', useAtkinsonFont: false, showRamIndicator: true }) + useAppStore.setState({ currentJob: null, meshStats: null, uiScale: 'small', useAtkinsonFont: false, showRamIndicator: true, showVramIndicator: false }) useWorkflowsStore.setState({ workflows: [{ id: 'w', name: 'W', description: '', nodes: [], edges: [], createdAt: '', updatedAt: '' }] }) useExtensionsStore.setState({ modelExtensions: [modelExt], processExtensions: [] }) @@ -83,6 +83,6 @@ test('buildAgentContext reads the stores and the async sources', async () => { workflows: [{ id: 'w', name: 'W' }], extensions: [{ id: 'triposr/main', name: 'MAIN', input: 'image', output: 'mesh' }], models: [{ id: 'triposr/main', name: 'TripoSR' }], - ui: { uiScale: 'small', useAtkinsonFont: false, showRamIndicator: true }, + ui: { uiScale: 'small', useAtkinsonFont: false, showRamIndicator: true, showVramIndicator: false }, }) }) diff --git a/src/shared/components/agent/agentContext.ts b/src/shared/components/agent/agentContext.ts index 79c078d93..ecf81db11 100644 --- a/src/shared/components/agent/agentContext.ts +++ b/src/shared/components/agent/agentContext.ts @@ -25,7 +25,7 @@ export interface AgentContextInputs { models: { id: string; name: string }[] | null /** Null when the settings could not be read. */ storage: { modelsDir: string; workspaceDir: string; workflowsDir: string; extensionsDir: string } | null - ui: { uiScale: UiScale; useAtkinsonFont: boolean; showRamIndicator: boolean } + ui: { uiScale: UiScale; useAtkinsonFont: boolean; showRamIndicator: boolean; showVramIndicator: boolean } } /** Pure assembly of the context from already-read state. */ @@ -44,7 +44,7 @@ export function assembleAgentContext(inputs: AgentContextInputs): Record @@ -65,7 +65,7 @@ function fake(overrides: Partial = {}): Fake { const rec = (...call: unknown[]) => { calls.push(call) } const ok = { success: true } const state = { - ui: { uiScale: 'medium' as UiScale, useAtkinsonFont: false, showRamIndicator: true }, + ui: { uiScale: 'medium' as UiScale, useAtkinsonFont: false, showRamIndicator: true, showVramIndicator: true }, dirs: { modelsDir: 'C:\\Modly\\models', workspaceDir: 'C:\\Modly\\workspace', workflowsDir: 'C:\\Modly\\workflows', extensionsDir: 'C:\\Modly\\extensions' }, history: { undo: 1, redo: 1 }, dirItems: {} as Record, @@ -101,6 +101,7 @@ function fake(overrides: Partial = {}): Fake { setUiScale: (v) => { state.ui.uiScale = v }, setUseAtkinsonFont: (v) => { state.ui.useAtkinsonFont = v }, setShowRamIndicator: (v) => { state.ui.showRamIndicator = v }, + setShowVramIndicator: (v) => { state.ui.showVramIndicator = v }, }, storage: { get: async () => state.dirs, @@ -260,7 +261,8 @@ test('set_setting sets each UI option and validates key and value', async () => assert.equal(await run('set_setting', { key: 'uiScale', value: 'large' }, f), 'Set the interface scale to "large".') assert.equal(await run('set_setting', { key: 'useAtkinsonFont', value: true }, f), 'Set the Atkinson Hyperlegible font to true.') assert.equal(await run('set_setting', { key: 'showRamIndicator', value: 'false' }, f), 'Set the RAM indicator to false.') - assert.deepEqual(f.ui, { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false }) + assert.equal(await run('set_setting', { key: 'showVramIndicator', value: false }, f), 'Set the VRAM indicator to false.') + assert.deepEqual(f.ui, { uiScale: 'large', useAtkinsonFont: true, showRamIndicator: false, showVramIndicator: false }) assert.match(await run('set_setting', { key: 'uiScale', value: 'huge' }, f), /^Error: uiScale must be one of: small, medium, large, very-large/) assert.match(await run('set_setting', { key: 'showRamIndicator', value: 1 }, f), /^Error: showRamIndicator must be true or false/) diff --git a/src/shared/components/agent/clientTools.ts b/src/shared/components/agent/clientTools.ts index 7be3c55fc..c5645af77 100644 --- a/src/shared/components/agent/clientTools.ts +++ b/src/shared/components/agent/clientTools.ts @@ -33,6 +33,7 @@ export interface UiSettings { uiScale: UiScale useAtkinsonFont: boolean showRamIndicator: boolean + showVramIndicator: boolean } export interface ToolResultBody { @@ -69,6 +70,7 @@ export interface ClientToolDeps { setUiScale: (value: UiScale) => void setUseAtkinsonFont: (value: boolean) => void setShowRamIndicator: (value: boolean) => void + setShowVramIndicator: (value: boolean) => void } storage: { get: () => Promise @@ -454,7 +456,8 @@ export const clientTools: Record = { const key = args.key as keyof UiSettings if (key === 'uiScale') deps.ui.setUiScale(args.value as UiScale) else if (key === 'useAtkinsonFont') deps.ui.setUseAtkinsonFont(toBool(args.value)!) - else deps.ui.setShowRamIndicator(toBool(args.value)!) + else if (key === 'showRamIndicator') deps.ui.setShowRamIndicator(toBool(args.value)!) + else deps.ui.setShowVramIndicator(toBool(args.value)!) return `Set ${SETTING_LABELS[key]} to ${JSON.stringify(deps.ui.get()[key])}.` }, }), @@ -548,6 +551,7 @@ const SETTING_LABELS: Record = { uiScale: 'the interface scale', useAtkinsonFont: 'the Atkinson Hyperlegible font', showRamIndicator: 'the RAM indicator', + showVramIndicator: 'the VRAM indicator', } const UI_SCALES: UiScale[] = ['small', 'medium', 'large', 'very-large'] @@ -752,10 +756,11 @@ export function createClientToolDeps(): ClientToolDeps { reload: () => wfs().load(), }, ui: { - get: () => ({ uiScale: app().uiScale, useAtkinsonFont: app().useAtkinsonFont, showRamIndicator: app().showRamIndicator }), + get: () => ({ uiScale: app().uiScale, useAtkinsonFont: app().useAtkinsonFont, showRamIndicator: app().showRamIndicator, showVramIndicator: app().showVramIndicator }), setUiScale: (v) => app().setUiScale(v), setUseAtkinsonFont: (v) => app().setUseAtkinsonFont(v), setShowRamIndicator: (v) => app().setShowRamIndicator(v), + setShowVramIndicator: (v) => app().setShowVramIndicator(v), }, storage: { get: () => window.electron.settings.get(), diff --git a/src/shared/components/layout/MemoryIndicator.tsx b/src/shared/components/layout/MemoryIndicator.tsx index f30f8978f..e200a3899 100644 --- a/src/shared/components/layout/MemoryIndicator.tsx +++ b/src/shared/components/layout/MemoryIndicator.tsx @@ -2,6 +2,14 @@ import { useEffect, useState } from 'react' import { useAppStore } from '@shared/stores/appStore' const GB = 1024 ** 3 +const RAM_INTERVAL_MS = 2000 +const VRAM_INTERVAL_MS = 2000 + +interface MemorySample { + total: number + used: number + available: number +} function fmtGB(bytes: number): string { return (bytes / GB).toFixed(1) @@ -13,88 +21,97 @@ function getBarColors(pct: number): { barColor: string; textColor: string } { return { barColor: 'bg-emerald-500', textColor: 'text-zinc-300' } } -export default function MemoryIndicator(): JSX.Element | null { - const [ram, setRam] = useState<{ total: number; used: number; available: number } | null>(null) - const [vram, setVram] = useState<{ total: number; used: number; available: number } | null>(null) - const platform = useAppStore(s => s.platform) - const isMac = platform === 'darwin' - const showVramIndicator = useAppStore(s => s.showVramIndicator) +/** + * Polls `sample` every `intervalMs` while `enabled`, skipping a tick while the + * previous call is still running. A null or failed sample keeps the last value, + * so a transient miss does not flicker the bar away. Disabling clears the value. + */ +function useSampler(sample: () => Promise, intervalMs: number, enabled: boolean): T | null { + const [value, setValue] = useState(null) useEffect(() => { + if (!enabled) { + setValue(null) + return + } let cancelled = false + let inFlight = false const tick = async () => { + if (inFlight) return + inFlight = true try { - const [ramNext, vramNext] = await Promise.all([ - window.electron.system.memory(), - isMac ? Promise.resolve(null) : window.electron.system.gpuMemory(), - ]) - if (!cancelled) { - setRam(ramNext) - setVram(vramNext) - } + const next = await sample() + if (!cancelled && next) setValue(next) } catch { // Renderer should not break if memory sampling fails. + } finally { + inFlight = false } } tick() - const id = setInterval(tick, 2000) + const id = setInterval(tick, intervalMs) return () => { cancelled = true clearInterval(id) } - }, [isMac]) - - if (!ram) return null - - const ramPct = ram.total > 0 ? Math.min(100, Math.round((ram.used / ram.total) * 100)) : 0 - const ramColors = getBarColors(ramPct) + }, [sample, intervalMs, enabled]) - const ramTooltip = - `RAM:\n` + - ` Used: ${fmtGB(ram.used)} GB\n` + - ` Available: ${fmtGB(ram.available)} GB\n` + - ` Total: ${fmtGB(ram.total)} GB` + return value +} - const vramBar = showVramIndicator && !isMac && vram && vram.total > 0 ? (() => { - const vramPct = Math.min(100, Math.round((vram.used / vram.total) * 100)) - const vramColors = getBarColors(vramPct) - const vramTooltip = - `VRAM:\n` + - ` Used: ${fmtGB(vram.used)} GB\n` + - ` Available: ${fmtGB(vram.available)} GB\n` + - ` Total: ${fmtGB(vram.total)} GB` - return ( -
- VRAM -
-
-
- - {fmtGB(vram.used)} / {fmtGB(vram.total)} GB - -
- ) - })() : null +const sampleRam = () => window.electron.system.memory() +const sampleVram = () => window.electron.system.gpuMemory() +function MemoryBar({ label, sample, tooltip }: { label: string; sample: MemorySample; tooltip: string }): JSX.Element { + const pct = sample.total > 0 ? Math.min(100, Math.round((sample.used / sample.total) * 100)) : 0 + const { barColor, textColor } = getBarColors(pct) return ( -
- RAM +
+ {label}
- - {fmtGB(ram.used)} / {fmtGB(ram.total)} GB + + {fmtGB(sample.used)} / {fmtGB(sample.total)} GB - {vramBar}
) -} \ No newline at end of file +} + +function tooltipFor(heading: string, sample: MemorySample): string { + return ( + `${heading}\n` + + ` Used: ${fmtGB(sample.used)} GB\n` + + ` Available: ${fmtGB(sample.available)} GB\n` + + ` Total: ${fmtGB(sample.total)} GB` + ) +} + +export default function MemoryIndicator(): JSX.Element | null { + const isMac = useAppStore(s => s.platform === 'darwin') + const showRam = useAppStore(s => s.showRamIndicator) + const showVram = useAppStore(s => s.showVramIndicator) && !isMac + + const ram = useSampler(sampleRam, RAM_INTERVAL_MS, showRam) + const vram = useSampler(sampleVram, VRAM_INTERVAL_MS, showVram) + + const ramBar = showRam && ram ? ram : null + const vramBar = showVram && vram && vram.total > 0 ? vram : null + if (!ramBar && !vramBar) return null + + return ( +
+ {ramBar && } + {vramBar && ( + + )} +
+ ) +} diff --git a/src/shared/components/layout/TopBar.tsx b/src/shared/components/layout/TopBar.tsx index 056ddf8af..102b99c02 100644 --- a/src/shared/components/layout/TopBar.tsx +++ b/src/shared/components/layout/TopBar.tsx @@ -4,7 +4,7 @@ import { useAgentPanelStore } from '@shared/stores/agentPanelStore' import MemoryIndicator from './MemoryIndicator' export default function TopBar(): JSX.Element { - const { patchUpdateReady, platform, showRamIndicator } = useAppStore() + const { patchUpdateReady, platform, showRamIndicator, showVramIndicator } = useAppStore() const isMac = platform === 'darwin' const agentOpen = useAgentPanelStore((s) => s.open) const toggleAgent = useAgentPanelStore((s) => s.toggle) @@ -46,7 +46,7 @@ export default function TopBar(): JSX.Element {
{/* Memory indicator */} - {showRamIndicator && } + {(showRamIndicator || showVramIndicator) && } {/* Agent panel toggle */}
)} diff --git a/src/areas/queue/QueuePage.tsx b/src/areas/queue/QueuePage.tsx new file mode 100644 index 000000000..54fdc6874 --- /dev/null +++ b/src/areas/queue/QueuePage.tsx @@ -0,0 +1,462 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import axios from 'axios' +import { useAppStore } from '@shared/stores/appStore' +import { useNavStore } from '@shared/stores/navStore' +import { useExtensionsStore } from '@shared/stores/extensionsStore' +import { useAssetThumbnailUrl } from '@shared/thumbnails/assetThumbnailStore' +import { ensureThumbnail } from '@shared/thumbnails/assetThumbnailService' +import { showImportedMesh } from '@shared/utils/meshJob' +import { useLiveWorkflowRun } from '@areas/workflows/workflowRunStore' +import type { RunRecord, RunRecordPage, RunStatus } from '@shared/types/runs' +import { useInputPreview } from './inputPreview' +import { + buildQueueSections, + formatDuration, + formatStepLabel, + isFinishedStatus, + modelNameResolver, + rowFromRecord, + runElapsedSeconds, + runOpenTarget, + type ModelNameResolver, + type QueueRow, +} from './queueModel' + +const POLL_MS = 2000 +const PAGE_SIZE = 50 +/** The list endpoint's largest page. */ +const MAX_LIMIT = 200 + +const nowSeconds = () => Date.now() / 1000 + +// ─── Data ───────────────────────────────────────────────────────────────────── + +/** + * Top-level run history, refreshed every 2 s while the page is mounted. The + * first MAX_LIMIT runs are polled; older pages ("Load more" past that) are + * fetched once and kept. + */ +function useRunHistory(apiUrl: string) { + const [polled, setPolled] = useState([]) + const [older, setOlder] = useState([]) + const [total, setTotal] = useState(0) + const [limit, setLimit] = useState(PAGE_SIZE) + const [loaded, setLoaded] = useState(false) + const [error, setError] = useState(null) + const [loadingMore, setLoadingMore] = useState(false) + const inFlight = useRef(false) + + const refresh = useCallback(async () => { + if (inFlight.current) return + inFlight.current = true + try { + const { data } = await axios.get(`${apiUrl}/workflow-runs`, { + params: { top_level: true, limit, offset: 0 }, + }) + setPolled(data.runs) + setTotal(data.total) + setError(null) + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + inFlight.current = false + setLoaded(true) + } + }, [apiUrl, limit]) + + useEffect(() => { + void refresh() + const timer = setInterval(() => { void refresh() }, POLL_MS) + return () => clearInterval(timer) + }, [refresh]) + + const loadMore = useCallback(async () => { + if (limit < MAX_LIMIT) { + setLimit((current) => Math.min(MAX_LIMIT, current + PAGE_SIZE)) + return + } + setLoadingMore(true) + try { + const { data } = await axios.get(`${apiUrl}/workflow-runs`, { + params: { top_level: true, limit: MAX_LIMIT, offset: MAX_LIMIT + older.length }, + }) + setOlder((current) => [...current, ...data.runs]) + setTotal(data.total) + } catch (err) { + setError(err instanceof Error ? err.message : String(err)) + } finally { + setLoadingMore(false) + } + }, [apiUrl, limit, older.length]) + + const records = useMemo(() => { + const ids = new Set(polled.map((r) => r.id)) + return [...polled, ...older.filter((r) => !ids.has(r.id))] + }, [polled, older]) + + return { records, hasMore: total > polled.length + older.length, loaded, error, loadingMore, loadMore } +} + +/** A workflow run's child runs (its model generations), re-fetched while it runs. */ +function useChildRuns(apiUrl: string, runId: string, live: boolean) { + const [children, setChildren] = useState(null) + const [error, setError] = useState(null) + useEffect(() => { + let alive = true + const fetchChildren = async () => { + try { + const { data } = await axios.get<{ runs: RunRecord[] }>(`${apiUrl}/workflow-runs/${encodeURIComponent(runId)}/children`) + if (alive) { setChildren(data.runs); setError(null) } + } catch (err) { + if (alive) setError(err instanceof Error ? err.message : String(err)) + } + } + void fetchChildren() + if (!live) return () => { alive = false } + const timer = setInterval(() => { void fetchChildren() }, POLL_MS) + return () => { alive = false; clearInterval(timer) } + }, [apiUrl, runId, live]) + return { children, error } +} + +/** Seconds since the epoch, re-rendering every second while `ticking`. */ +function useNow(ticking: boolean): number { + const [now, setNow] = useState(nowSeconds) + useEffect(() => { + if (!ticking) return + setNow(nowSeconds()) + const timer = setInterval(() => setNow(nowSeconds()), 1000) + return () => clearInterval(timer) + }, [ticking]) + return now +} + +// ─── Pieces ─────────────────────────────────────────────────────────────────── + +function Icon({ d, size = 14 }: { d: string, size?: number }): JSX.Element { + return ( + + ) +} + +const ICONS = { + mesh: 'M12 2 3 7v10l9 5 9-5V7zM3 7l9 5 9-5M12 12v10', + workflow: 'M4 5h5v5H4zM15 14h5v5h-5zM9 7.5h3.5a2 2 0 0 1 2 2V14', + chevron: 'm9 6 6 6-6 6', + queue: 'M4 6h11M4 12h7M4 18h7M17 13v3l2 1.5', +} + +const STATUS_STYLES: Record = { + pending: { label: 'Queued', className: 'bg-zinc-800 text-zinc-300' }, + running: { label: 'Running', className: 'bg-accent/15 text-accent-light' }, + paused: { label: 'Waiting', className: 'bg-amber-500/10 text-amber-300' }, + done: { label: 'Done', className: 'bg-emerald-500/10 text-emerald-300' }, + error: { label: 'Failed', className: 'bg-red-500/10 text-red-300' }, + cancelled: { label: 'Cancelled', className: 'bg-zinc-800 text-zinc-400' }, + interrupted: { label: 'Interrupted', className: 'bg-amber-500/10 text-amber-300' }, +} + +function StatusPill({ status, paused = false }: { status: RunStatus, paused?: boolean }): JSX.Element { + const style = STATUS_STYLES[paused && status === 'running' ? 'paused' : status] + return ( + + {status === 'running' && !paused && + ) +} + +function SectionHeader({ title, count }: { title: string, count: number }): JSX.Element { + return ( +
+

{title}

+ {count} +
+ ) +} + +function Empty({ text }: { text: string }): JSX.Element { + return ( +
+

{text}

+
+ ) +} + +/** The run's input image, else its output mesh's thumbnail, else a kind icon. */ +function RunThumbnail({ row }: { row: QueueRow }): JSX.Element { + const inputUrl = useInputPreview(row.inputImage) + const target = runOpenTarget(row) + const meshPath = !row.inputImage && target.openable ? target.workspacePath : '' + const meshUrl = useAssetThumbnailUrl({ workspacePath: meshPath }) + useEffect(() => { if (meshPath) ensureThumbnail(meshPath) }, [meshPath]) + const src = inputUrl ?? (meshPath ? meshUrl : undefined) + return ( +
+ {src + ? + : } +
+ ) +} + +function ProgressBar({ progress }: { progress: number }): JSX.Element { + // A negative progress means unknown: an indeterminate shimmer. + const known = progress >= 0 + return ( +
+
+
+ ) +} + +function OutputActions({ row, compact = false }: { row: QueueRow, compact?: boolean }): JSX.Element | null { + const navigate = useNavStore((s) => s.navigate) + if (!isFinishedStatus(row.status)) return null + const target = runOpenTarget(row) + const tooltip = target.openable ? undefined : target.reason + const button = 'rounded-lg px-2.5 py-1 text-[11.5px] font-medium transition-colors disabled:cursor-not-allowed' + return ( +
+ + +
+ ) +} + +function stepSummary(row: QueueRow): string { + if (row.current) return formatStepLabel(row.current) + if (row.status === 'pending') return 'Waiting to start' + if (row.paused) return 'Waiting for you to continue a branch' + if (row.status === 'error' && row.error) return row.error + const count = row.steps.length + if (count === 0) return row.status === 'running' ? 'Starting…' : '' + return `${count} ${count === 1 ? 'step' : 'steps'}` +} + +function StepList({ row, now }: { row: QueueRow, now: number }): JSX.Element { + if (row.steps.length === 0) return

No steps recorded.

+ return ( +
    + {row.steps.map((step, index) => ( +
  1. + {index + 1} + {step.name} + + {formatDuration(runElapsedSeconds({ startedAt: step.started_at, endedAt: step.ended_at ?? null }, now))} + + +
  2. + ))} +
+ ) +} + +function ChildRuns({ row, apiUrl, resolveModel, now }: { row: QueueRow, apiUrl: string, resolveModel: ModelNameResolver, now: number }): JSX.Element { + const { children, error } = useChildRuns(apiUrl, row.id, !isFinishedStatus(row.status)) + if (error && !children) return

Could not load the generations: {error}

+ if (!children) return

Loading generations…

+ if (children.length === 0) return

No model generations in this run.

+ return ( +
    + {children.map((record) => { + const child = rowFromRecord(record, resolveModel) + return ( +
  1. + {child.name} + {formatDuration(runElapsedSeconds(child, now))} + + +
  2. + ) + })} +
+ ) +} + +function RunCard({ row, now, apiUrl, resolveModel }: { row: QueueRow, now: number, apiUrl: string, resolveModel: ModelNameResolver }): JSX.Element { + const [expanded, setExpanded] = useState(false) + const expandable = row.kind === 'workflow' + const summary = stepSummary(row) + const failed = row.status === 'error' && !row.current + return ( +
  • +
    setExpanded((v) => !v) : undefined} + > + {expandable ? ( + + ) :
    + + {expanded && ( +
    +
    +

    Steps

    + +
    +
    +

    Generations

    + +
    +
    + )} +
  • + ) +} + +function Section({ title, rows, empty, now, apiUrl, resolveModel, footer }: { + title: string + rows: QueueRow[] + empty: string + now: number + apiUrl: string + resolveModel: ModelNameResolver + footer?: JSX.Element | null +}): JSX.Element { + return ( +
    + + {rows.length > 0 ? ( +
      + {rows.map((row) => )} +
    + ) : } + {footer} +
    + ) +} + +// ─── Page ───────────────────────────────────────────────────────────────────── + +export default function QueuePage(): JSX.Element { + const apiUrl = useAppStore((s) => s.apiUrl) + const navigate = useNavStore((s) => s.navigate) + const modelExtensions = useExtensionsStore((s) => s.modelExtensions) + const live = useLiveWorkflowRun() + const { records, hasMore, loaded, error, loadingMore, loadMore } = useRunHistory(apiUrl) + + const resolveModel = useMemo(() => modelNameResolver(modelExtensions), [modelExtensions]) + const sections = useMemo(() => buildQueueSections(live, records, resolveModel), [live, records, resolveModel]) + const now = useNow(sections.running.length > 0) + + const nothingYet = loaded && sections.running.length + sections.queued.length + sections.history.length === 0 + + return ( +
    +
    +
    +

    Queue

    +

    + {!loaded + ? 'Loading runs…' + : sections.running.length > 0 + ? <>{sections.running.length} running{sections.queued.length > 0 ? <>, {sections.queued.length} queued : null} + : 'Nothing running right now'} +

    +
    + {error && ( +

    + Could not refresh the run history +

    + )} +
    + + {nothingYet ? ( +
    +
    + +
    +

    No runs yet

    +

    Generations and workflow runs show up here while they run, and stay listed once they finish.

    + +
    + ) : ( + <> +
    +
    +
    + +
    + ) : null} + /> + + )} +
    + ) +} diff --git a/src/areas/queue/inputPreview.ts b/src/areas/queue/inputPreview.ts new file mode 100644 index 000000000..f8ab29a03 --- /dev/null +++ b/src/areas/queue/inputPreview.ts @@ -0,0 +1,65 @@ +import { useEffect, useState } from 'react' + +// Small previews of run input images. The file is read once through the fs +// IPC bridge, drawn into a tiny canvas and kept as a data URL in memory, so a +// list of runs costs one read per image per session. Reads run one at a time. + +const PREVIEW_SIZE = 96 +const MAX_CACHED = 200 + +const cache = new Map>() +let tail: Promise = Promise.resolve() + +function mimeFor(path: string): string { + const ext = path.split('.').pop()?.toLowerCase() + if (ext === 'jpg' || ext === 'jpeg') return 'image/jpeg' + if (ext === 'webp') return 'image/webp' + return 'image/png' +} + +function loadImage(src: string): Promise { + return new Promise((resolve, reject) => { + const image = new Image() + image.onload = () => resolve(image) + image.onerror = () => reject(new Error('Could not decode the image')) + image.src = src + }) +} + +async function renderPreview(path: string): Promise { + try { + const base64 = await window.electron.fs.readFileBase64(path) + const image = await loadImage(`data:${mimeFor(path)};base64,${base64}`) + const scale = PREVIEW_SIZE / Math.max(image.naturalWidth, image.naturalHeight, 1) + const canvas = document.createElement('canvas') + canvas.width = Math.max(1, Math.round(image.naturalWidth * Math.min(1, scale))) + canvas.height = Math.max(1, Math.round(image.naturalHeight * Math.min(1, scale))) + canvas.getContext('2d')?.drawImage(image, 0, 0, canvas.width, canvas.height) + return canvas.toDataURL('image/webp', 0.8) + } catch { + return null + } +} + +/** A downscaled data URL of the image at `path`, or null when it cannot be read. */ +export function loadInputPreview(path: string): Promise { + const cached = cache.get(path) + if (cached) return cached + if (cache.size >= MAX_CACHED) cache.delete(cache.keys().next().value as string) + const job = tail.then(() => renderPreview(path)) + tail = job + cache.set(path, job) + return job +} + +export function useInputPreview(path: string | null): string | null { + const [url, setUrl] = useState(null) + useEffect(() => { + setUrl(null) + if (!path) return + let alive = true + void loadInputPreview(path).then((result) => { if (alive) setUrl(result) }) + return () => { alive = false } + }, [path]) + return url +} diff --git a/src/areas/queue/queueModel.test.ts b/src/areas/queue/queueModel.test.ts new file mode 100644 index 000000000..825edc0be --- /dev/null +++ b/src/areas/queue/queueModel.test.ts @@ -0,0 +1,147 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import type { RunRecord } from '../../shared/types/runs' +import type { LiveWorkflowRun } from '../workflows/workflowRunRecord' +import { + buildQueueSections, + formatDuration, + formatStepLabel, + inputImagePath, + modelNameResolver, + runDisplayName, + runElapsedSeconds, + runOpenTarget, +} from './queueModel' + +function record(id: string, overrides: Partial = {}): RunRecord { + return { + id, + kind: 'workflow', + status: 'done', + workflow_name: `wf ${id}`, + inputs: {}, + params: {}, + steps: [], + created_at: 100, + started_at: 100, + ended_at: 160, + ...overrides, + } +} + +function live(overrides: Partial = {}): LiveWorkflowRun { + return { + runId: 'live', + workflowId: 'wf-1', + workflowName: 'Image to mesh', + status: 'running', + paused: false, + startedAt: 500, + endedAt: null, + inputs: { image: 'C:/in/cat.png' }, + steps: [{ name: 'Generate Mesh', node_id: 'gen', status: 'running', started_at: 501 }], + current: { index: 2, total: 4, name: 'Generate Mesh', detail: 'Generating…', progress: 40 }, + outputPath: null, + error: null, + ...overrides, + } +} + +const ids = (rows: { id: string }[]) => rows.map((r) => r.id) + +test('records split into running, queued and history', () => { + const sections = buildQueueSections(null, [ + record('r1', { status: 'running', started_at: 300, ended_at: null }), + record('q2', { status: 'pending', created_at: 250, started_at: null, ended_at: null }), + record('q1', { status: 'pending', created_at: 200, started_at: null, ended_at: null }), + record('d1', { status: 'done', started_at: 150 }), + record('e1', { status: 'error', started_at: 180, error: 'boom' }), + record('c1', { status: 'interrupted', started_at: 120 }), + ]) + assert.deepEqual(ids(sections.running), ['r1']) + // Next up first. + assert.deepEqual(ids(sections.queued), ['q1', 'q2']) + assert.deepEqual(ids(sections.history), ['e1', 'd1', 'c1']) + assert.equal(sections.history[0].error, 'boom') +}) + +test('the live run replaces its own running record, with its live progress', () => { + const sections = buildQueueSections(live(), [ + record('live', { status: 'running', started_at: 500, ended_at: null }), + record('d1'), + ]) + assert.deepEqual(ids(sections.running), ['live']) + assert.equal(sections.running[0].live, true) + assert.equal(sections.running[0].current?.progress, 40) + assert.equal(sections.running[0].inputImage, 'C:/in/cat.png') + assert.deepEqual(ids(sections.history), ['d1']) +}) + +test('a finished live run shows once in history: the record wins, the live copy stands in until it is listed', () => { + const ended = live({ status: 'done', endedAt: 560, current: null, outputPath: 'Workflows/a.glb' }) + const listed = buildQueueSections(ended, [record('live', { started_at: 500, ended_at: 561 }), record('d1')]) + assert.deepEqual(ids(listed.history), ['live', 'd1']) + assert.equal(listed.history[0].live, false) + assert.equal(listed.running.length, 0) + + const notYetListed = buildQueueSections(ended, [record('d1')]) + assert.deepEqual(ids(notYetListed.history), ['live', 'd1']) + assert.equal(notYetListed.history[0].live, true) +}) + +test('child runs never show at the top level', () => { + const sections = buildQueueSections(null, [record('child', { kind: 'generation', parent_run_id: 'p' }), record('p')]) + assert.deepEqual(ids(sections.history), ['p']) +}) + +test('a running record without live progress still names its open step', () => { + const { running } = buildQueueSections(null, [record('g', { + kind: 'generation', status: 'running', ended_at: null, model_id: 'triposg', + steps: [{ name: 'Load', status: 'done', started_at: 1, ended_at: 2 }, { name: 'Mesh', status: 'running', started_at: 2 }], + })]) + assert.equal(running[0].current?.name, 'Mesh') + assert.equal(formatStepLabel(running[0].current!), 'Step 2 · Mesh') +}) + +test('display names', () => { + const resolve = modelNameResolver([{ id: 'hunyuan', name: 'Hunyuan3D', nodes: [{ id: 'mini', name: 'Hunyuan3D Mini' }] }]) + assert.equal(runDisplayName(record('w'), resolve), 'wf w') + assert.equal(runDisplayName(record('w', { workflow_name: ' ' })), 'Untitled workflow') + assert.equal(runDisplayName(record('g', { kind: 'generation', model_id: 'hunyuan/mini' }), resolve), 'Hunyuan3D Mini') + assert.equal(runDisplayName(record('g', { kind: 'generation', model_id: 'hunyuan' }), resolve), 'Hunyuan3D') + assert.equal(runDisplayName(record('g', { kind: 'generation', model_id: 'trip_osr' }), resolve), 'Trip osr') + assert.equal(runDisplayName(record('m', { kind: 'mesh_op', op_id: 'decimate' })), 'Decimate') +}) + +test('input image paths are absolute files only', () => { + assert.equal(inputImagePath({ image: 'C:\\in\\a.png' }), 'C:\\in\\a.png') + assert.equal(inputImagePath({ image: '/home/u/a.png' }), '/home/u/a.png') + assert.equal(inputImagePath({ image: 'a.png' }), null) + assert.equal(inputImagePath({ image: '(attached image)' }), null) + assert.equal(inputImagePath({}), null) +}) + +test('elapsed and duration formatting', () => { + assert.equal(runElapsedSeconds({ startedAt: 100, endedAt: null }, 142.5), 42.5) + assert.equal(runElapsedSeconds({ startedAt: 100, endedAt: 160 }, 999), 60) + assert.equal(runElapsedSeconds({ startedAt: null, endedAt: null }, 999), null) + assert.equal(formatDuration(42.9), '42s') + assert.equal(formatDuration(185), '3m 05s') + assert.equal(formatDuration(3720), '1h 02m') + assert.equal(formatDuration(null), '—') + assert.equal(formatStepLabel({ index: 2, total: 4, name: 'Generate Mesh' }), 'Step 2/4 · Generate Mesh') +}) + +test('open targets: finished workspace meshes only', () => { + assert.deepEqual(runOpenTarget({ status: 'done', outputPath: 'Workflows/run/a.glb' }), { + openable: true, workspacePath: 'Workflows/run/a.glb', url: '/workspace/Workflows/run/a.glb', + }) + assert.equal(runOpenTarget({ status: 'running', outputPath: 'a.glb' }).openable, false) + assert.equal(runOpenTarget({ status: 'done', outputPath: null }).openable, false) + assert.equal(runOpenTarget({ status: 'done', outputPath: 'C:/out/a.glb' }).openable, false) + assert.equal(runOpenTarget({ status: 'done', outputPath: '../a.glb' }).openable, false) + assert.equal(runOpenTarget({ status: 'done', outputPath: 'Workflows/voice.wav' }).openable, false) + assert.equal(runOpenTarget({ status: 'done', outputPath: 'Scenes/s/scene.json' }).openable, false) + assert.equal(runOpenTarget({ status: 'done', outputPath: 'Workflows/a.obj' }).openable, false) +}) diff --git a/src/areas/queue/queueModel.ts b/src/areas/queue/queueModel.ts new file mode 100644 index 000000000..85a519104 --- /dev/null +++ b/src/areas/queue/queueModel.ts @@ -0,0 +1,216 @@ +// Pure model of the Queue page: merges the workflow run this window is +// executing with the persisted run history (GET /workflow-runs) into Running, +// Queued and History sections, and derives what each row shows and opens. +// Timestamps are epoch seconds, as in the run records. + +import type { RunKind, RunRecord, RunStatus, RunStep } from '../../shared/types/runs' +import type { LiveRunCurrentStep, LiveWorkflowRun } from '../workflows/workflowRunRecord' + +export interface QueueRow { + id: string + kind: RunKind + name: string + status: RunStatus + /** Running but waiting on the user (a Wait branch, a paused loop). */ + paused: boolean + /** Absolute path of the input image, when the run had one on disk. */ + inputImage: string | null + createdAt: number + startedAt: number | null + endedAt: number | null + steps: RunStep[] + /** The step running now, with live progress when this window runs it. */ + current: LiveRunCurrentStep | null + outputPath: string | null + error: string | null + /** Fed by this window's runner rather than by the history list. */ + live: boolean +} + +export interface QueueSections { + running: QueueRow[] + queued: QueueRow[] + history: QueueRow[] +} + +/** Resolves a generation's model id to a display name. */ +export type ModelNameResolver = (modelId: string) => string | undefined + +const TERMINAL: ReadonlySet = new Set(['done', 'error', 'cancelled', 'interrupted']) + +export function isFinishedStatus(status: RunStatus): boolean { + return TERMINAL.has(status) +} + +/** Model extensions → a resolver for `model_id` ("ext" or "ext/node"). */ +export function modelNameResolver( + extensions: readonly { id: string, name: string, nodes?: readonly { id: string, name?: string }[] }[], +): ModelNameResolver { + return (modelId) => { + const [extId, nodeId] = modelId.split('/') + const ext = extensions.find((e) => e.id === extId) + if (!ext) return undefined + const node = nodeId ? ext.nodes?.find((n) => n.id === nodeId) : undefined + return node?.name ?? ext.name + } +} + +function prettyId(id: string): string { + const last = id.split('/').pop() ?? id + const words = last.replace(/[_-]+/g, ' ').trim() + return words ? words[0].toUpperCase() + words.slice(1) : id +} + +export function runDisplayName(record: RunRecord, resolveModel?: ModelNameResolver): string { + if (record.kind === 'workflow') return record.workflow_name?.trim() || 'Untitled workflow' + if (record.kind === 'mesh_op') return record.op_id ? prettyId(record.op_id) : 'Mesh operation' + if (!record.model_id) return 'Generation' + return resolveModel?.(record.model_id) ?? prettyId(record.model_id) +} + +const ABSOLUTE_PATH = /^(?:[a-zA-Z]:[\\/]|[\\/])/ + +/** The input image a run read from disk; generations only record a file name. */ +export function inputImagePath(inputs: Record | undefined): string | null { + const image = inputs?.image + return typeof image === 'string' && ABSOLUTE_PATH.test(image) ? image : null +} + +/** The open step of a history record (its progress is unknown here). */ +function recordCurrentStep(record: RunRecord): LiveRunCurrentStep | null { + if (record.status !== 'running') return null + const last = record.steps[record.steps.length - 1] + if (!last || last.ended_at != null) return null + return { index: record.steps.length, total: 0, name: last.name, detail: '', progress: -1 } +} + +export function rowFromRecord(record: RunRecord, resolveModel?: ModelNameResolver): QueueRow { + return { + id: record.id, + kind: record.kind, + name: runDisplayName(record, resolveModel), + status: record.status, + paused: false, + inputImage: inputImagePath(record.inputs), + createdAt: record.created_at, + startedAt: record.started_at ?? null, + endedAt: record.ended_at ?? null, + steps: record.steps ?? [], + current: recordCurrentStep(record), + outputPath: record.output_path ?? null, + error: record.error ?? null, + live: false, + } +} + +export function rowFromLiveRun(run: LiveWorkflowRun): QueueRow { + return { + id: run.runId, + kind: 'workflow', + name: run.workflowName.trim() || 'Untitled workflow', + status: run.status, + paused: run.paused, + inputImage: inputImagePath(run.inputs), + createdAt: run.startedAt, + startedAt: run.startedAt, + endedAt: run.endedAt, + steps: run.steps, + current: run.current, + outputPath: run.outputPath, + error: run.error, + live: true, + } +} + +const newestFirst = (a: QueueRow, b: QueueRow) => (b.startedAt ?? b.createdAt) - (a.startedAt ?? a.createdAt) + +/** + * Splits runs into sections. The live run and its own history record are the + * same run: while it runs the live copy wins (it has step progress the record + * lacks); once it has ended the record wins, and the live copy only stands in + * until the history lists it. Queued runs come oldest first (next up on top); + * Running and History newest first. + */ +export function buildQueueSections( + live: LiveWorkflowRun | null, + records: readonly RunRecord[], + resolveModel?: ModelNameResolver, +): QueueSections { + const running: QueueRow[] = [] + const queued: QueueRow[] = [] + const history: QueueRow[] = [] + const seen = new Set() + + const liveRow = live ? rowFromLiveRun(live) : null + const liveRunning = liveRow !== null && !isFinishedStatus(liveRow.status) + if (liveRow && liveRunning) { + running.push(liveRow) + seen.add(liveRow.id) + } + + for (const record of records) { + if (seen.has(record.id) || record.parent_run_id) continue + seen.add(record.id) + const row = rowFromRecord(record, resolveModel) + if (row.status === 'pending') queued.push(row) + else if (row.status === 'running') running.push(row) + else history.push(row) + } + + if (liveRow && !seen.has(liveRow.id)) history.push(liveRow) + + running.sort(newestFirst) + queued.sort((a, b) => a.createdAt - b.createdAt) + history.sort(newestFirst) + return { running, queued, history } +} + +/** Seconds a run has been going (running) or took (finished); null before it starts. */ +export function runElapsedSeconds(row: Pick, now: number): number | null { + if (row.startedAt == null) return null + const end = row.endedAt ?? now + return Math.max(0, end - row.startedAt) +} + +/** "42s", "3m 05s", "1h 02m". */ +export function formatDuration(seconds: number | null | undefined): string { + if (seconds == null || !Number.isFinite(seconds)) return '—' + const total = Math.max(0, Math.floor(seconds)) + if (total < 60) return `${total}s` + const minutes = Math.floor(total / 60) + if (minutes < 60) return `${minutes}m ${String(total % 60).padStart(2, '0')}s` + const hours = Math.floor(minutes / 60) + return `${hours}h ${String(minutes % 60).padStart(2, '0')}m` +} + +/** "Step 2/4 · Generate Mesh", or "Step 2 · Generate Mesh" when the total is unknown. */ +export function formatStepLabel(step: Pick): string { + const position = step.total >= step.index ? `${step.index}/${step.total}` : `${step.index}` + return `Step ${position} · ${step.name}` +} + +export type RunOpenTarget = + | { openable: true, workspacePath: string, url: string } + | { openable: false, reason: string } + +const MESH_EXTENSIONS = /\.(glb|gltf)$/i +const AUDIO_EXTENSIONS = /\.(wav|mp3|ogg|flac|m4a)$/i +const SCENE_EXTENSIONS = /\.json$/i + +/** + * What a finished run's output opens as. Only workspace meshes (.glb/.gltf) + * open in Generate and show in the Library; output_path is workspace-relative + * for those, absolute when the output lives elsewhere. + */ +export function runOpenTarget(row: Pick): RunOpenTarget { + if (!isFinishedStatus(row.status)) return { openable: false, reason: 'The run has not finished yet.' } + const path = row.outputPath?.trim().replace(/\\/g, '/') + if (!path) return { openable: false, reason: 'This run did not produce an output.' } + if (ABSOLUTE_PATH.test(path) || path.split('/').includes('..')) { + return { openable: false, reason: 'The output is outside the workspace.' } + } + if (AUDIO_EXTENSIONS.test(path)) return { openable: false, reason: 'Audio outputs cannot be opened in the 3D viewer.' } + if (SCENE_EXTENSIONS.test(path)) return { openable: false, reason: 'Scene outputs cannot be opened from the Queue yet.' } + if (!MESH_EXTENSIONS.test(path)) return { openable: false, reason: 'Only .glb and .gltf outputs can be opened.' } + return { openable: true, workspacePath: path, url: `/workspace/${path}` } +} diff --git a/src/areas/workflows/workflowRunRecord.test.ts b/src/areas/workflows/workflowRunRecord.test.ts index 099a608e9..7309c6b02 100644 --- a/src/areas/workflows/workflowRunRecord.test.ts +++ b/src/areas/workflows/workflowRunRecord.test.ts @@ -9,6 +9,8 @@ import { reopenWorkflowRunRecord, runOutputPath, startRecordStep, + toLiveWorkflowRun, + workflowRunInputs, workflowRunParams, } from './workflowRunRecord' @@ -93,3 +95,31 @@ test('runOutputPath prefers a workspace-relative path', () => { assert.equal(runOutputPath('/optimize/serve-file?path=x', 'D:/x.glb'), 'D:/x.glb') assert.equal(runOutputPath(), undefined) }) + +test("inputs record the Image nodes' files, or an attached image", () => { + assert.deepEqual(workflowRunInputs(workflow, false), { image: 'C:/a.png' }) + assert.deepEqual(workflowRunInputs(workflow, true), { image: '(attached image)' }) + const two = { ...workflow, nodes: [...workflow.nodes, { id: 'img2', type: 'imageNode', data: { params: { filePath: 'C:/b.png' } } }] } + assert.deepEqual(workflowRunInputs(two, false), { image: 'C:/a.png', images: ['C:/a.png', 'C:/b.png'] }) + const none = { ...workflow, nodes: [{ id: 'img', type: 'imageNode', data: { params: {} } }] } + assert.deepEqual(workflowRunInputs(none, false), {}) +}) + +test("the live run exposes the record with the current step's progress", () => { + let record = newWorkflowRunRecord('run-1', workflow, { image: 'C:/a.png' }, 100) + record = startRecordStep(record, 'TripoSG', 'gen', 101) + const progress = { status: 'running' as const, blockIndex: 0, blockTotal: 3, blockProgress: 40, blockStep: 'Generating…' } + const run = toLiveWorkflowRun(record, progress) + assert.equal(run.runId, 'run-1') + assert.equal(run.workflowName, 'Image to mesh') + assert.equal(run.startedAt, 100) + assert.equal(run.paused, false) + assert.deepEqual(run.current, { index: 1, total: 3, name: 'TripoSG', detail: 'Generating…', progress: 40 }) + + assert.equal(toLiveWorkflowRun(record, { ...progress, status: 'paused' }).paused, true) + const done = finishWorkflowRunRecord(record, 'done', 130, { outputPath: 'Workflows/a.glb' }) + const ended = toLiveWorkflowRun(done, { ...progress, status: 'done' }) + assert.equal(ended.current, null) + assert.equal(ended.endedAt, 130) + assert.equal(ended.outputPath, 'Workflows/a.glb') +}) diff --git a/src/areas/workflows/workflowRunRecord.ts b/src/areas/workflows/workflowRunRecord.ts index e236e823e..af51731e6 100644 --- a/src/areas/workflows/workflowRunRecord.ts +++ b/src/areas/workflows/workflowRunRecord.ts @@ -93,3 +93,90 @@ export function runOutputPath(outputUrl?: string, outputPath?: string): string | if (outputUrl?.startsWith('/workspace/')) return outputUrl.slice('/workspace/'.length) return outputPath } + +/** What `inputs.image` holds when the run got its image as data, not as a file. */ +export const ATTACHED_IMAGE_INPUT = '(attached image)' + +/** + * A run's inputs: the image file of each Image node (`image` is the first, + * `images` lists them all when there are several). An image handed over as + * data replaces those files, so it is recorded as attached. + */ +export function workflowRunInputs(workflow: WorkflowLike, attachedImage: boolean): Record { + if (attachedImage) return { image: ATTACHED_IMAGE_INPUT } + const images: string[] = [] + for (const node of workflow.nodes) { + if (node.type !== 'imageNode') continue + const filePath = node.data.params?.filePath + if (typeof filePath === 'string' && filePath.trim() !== '' && !images.includes(filePath)) images.push(filePath) + } + if (images.length === 0) return {} + return images.length === 1 ? { image: images[0] } : { image: images[0], images } +} + +/** The step the runner is on, as the Queue page shows it ("Step 2/4 · Generate Mesh"). */ +export interface LiveRunCurrentStep { + /** 1-based position of the step in the current pass (main run or Wait branch). */ + index: number + /** Steps in the current pass; 0 when unknown. */ + total: number + name: string + /** What the step is doing right now, e.g. "Generating…". */ + detail: string + /** 0-100. */ + progress: number +} + +/** The workflow run this window is executing, with its live progress. */ +export interface LiveWorkflowRun { + runId: string + workflowId: string | null + workflowName: string + status: RunStatus + /** Running but waiting on the user (a Wait branch, a paused loop). */ + paused: boolean + startedAt: number + endedAt: number | null + inputs: Record + /** Steps run so far, in order; the open one is `running`. */ + steps: RunRecord['steps'] + current: LiveRunCurrentStep | null + outputPath: string | null + error: string | null +} + +/** The runner's progress fields this needs (WorkflowRunState). */ +interface RunProgress { + status: 'idle' | 'running' | 'paused' | 'done' | 'error' + blockIndex: number + blockTotal: number + blockProgress: number + blockStep: string +} + +export function toLiveWorkflowRun(record: WorkflowRunRecord, progress: RunProgress): LiveWorkflowRun { + const open = record.status === 'running' ? record.steps[record.steps.length - 1] : undefined + const current = open && open.ended_at == null + ? { + index: Math.max(1, progress.blockIndex + 1), + total: Math.max(0, progress.blockTotal), + name: open.name, + detail: progress.blockStep, + progress: Math.max(0, Math.min(100, progress.blockProgress)), + } + : null + return { + runId: record.id, + workflowId: record.workflow_id ?? null, + workflowName: record.workflow_name ?? 'Workflow', + status: record.status, + paused: record.status === 'running' && progress.status === 'paused', + startedAt: record.started_at ?? record.created_at, + endedAt: record.ended_at ?? null, + inputs: record.inputs, + steps: record.steps, + current, + outputPath: record.output_path ?? null, + error: record.error ?? null, + } +} diff --git a/src/areas/workflows/workflowRunStore.ts b/src/areas/workflows/workflowRunStore.ts index 87076f629..e7125fd75 100644 --- a/src/areas/workflows/workflowRunStore.ts +++ b/src/areas/workflows/workflowRunStore.ts @@ -1,3 +1,4 @@ +import { useMemo } from 'react' import { create } from 'zustand' import axios, { AxiosInstance } from 'axios' import { useAppStore } from '@shared/stores/appStore' @@ -11,9 +12,9 @@ import { assignSlotFilePaths } from './slotInputs' import type { SlotInputType } from './slotInputs' import { endRecordStep, finishWorkflowRunRecord, isTerminalRunStatus, newWorkflowRunRecord, - reopenWorkflowRunRecord, runOutputPath, startRecordStep, + reopenWorkflowRunRecord, runOutputPath, startRecordStep, toLiveWorkflowRun, workflowRunInputs, } from './workflowRunRecord' -import type { WorkflowRunRecord } from './workflowRunRecord' +import type { LiveWorkflowRun, WorkflowRunRecord } from './workflowRunRecord' import type { RunStatus } from '@shared/types/runs' // ─── Types ──────────────────────────────────────────────────────────────────── @@ -64,6 +65,7 @@ function updateRecord(fn: (record: WorkflowRunRecord) => WorkflowRunRecord): voi if (!_record.current) return const record = fn(_record.current) _record.current = record + useWorkflowRunStore.setState({ liveRecord: record }) const { id, ...body } = record const apiUrl = useAppStore.getState().apiUrl _recordWrites = _recordWrites @@ -468,6 +470,7 @@ async function executeExtensionNode( input_kind: 'scene', input_path: inputPath, model_id: node.data.extensionId ?? '', collection: 'Workflows', params: { ...effectiveParams, ...extraParams }, + parent_run_id: _record.current?.id ?? null, }) } else { const fd = new FormData() @@ -478,6 +481,8 @@ async function executeExtensionNode( fd.append('enable_texture', 'false') fd.append('texture_resolution', '1024') fd.append('params', JSON.stringify({ ...effectiveParams, ...extraParams })) + // Files this generation under the workflow run (Queue page: the run's steps). + if (_record.current) fd.append('parent_run_id', _record.current.id) submission = await client.post('/generate/from-image', fd, { headers: { 'Content-Type': 'multipart/form-data' } }) } const { data } = submission @@ -599,6 +604,11 @@ function pushBranchSceneMesh(ctx: RunContext, waitId: string): void { interface WorkflowRunStore { runState: WorkflowRunState + /** + * History record of the latest run started in this window (kept after it + * ends, until the next run). Read it through useLiveWorkflowRun(). + */ + liveRecord: WorkflowRunRecord | null activeNodeId: string | null activeWorkflowId: string | null nodeImageOutputs: Record @@ -694,6 +704,7 @@ export const useWorkflowRunStore = create((set, get) => { return { runState: IDLE, + liveRecord: null, activeNodeId: null, activeWorkflowId: null, nodeImageOutputs: {}, @@ -716,7 +727,7 @@ export const useWorkflowRunStore = create((set, get) => { _record.current = newWorkflowRunRecord( crypto.randomUUID(), workflow, - overrideImageData ? { image: '(attached image)' } : appState.selectedImagePath ? { image: appState.selectedImagePath } : {}, + workflowRunInputs(workflow, !!overrideImageData), nowSeconds(), ) updateRecord((r) => r) @@ -1173,3 +1184,13 @@ export const useWorkflowRunStore = create((set, get) => { }, } }) + +/** + * The latest workflow run of this window with its live progress (step list, + * current step, elapsed-time origin), or null before the first run. + */ +export function useLiveWorkflowRun(): LiveWorkflowRun | null { + const record = useWorkflowRunStore((s) => s.liveRecord) + const runState = useWorkflowRunStore((s) => s.runState) + return useMemo(() => (record ? toLiveWorkflowRun(record, runState) : null), [record, runState]) +} diff --git a/src/areas/workflows/workflowSceneRun.test.mjs b/src/areas/workflows/workflowSceneRun.test.mjs index 86600a841..7f5da8622 100644 --- a/src/areas/workflows/workflowSceneRun.test.mjs +++ b/src/areas/workflows/workflowSceneRun.test.mjs @@ -54,6 +54,8 @@ test('scene model uses typed artifact route and preserves scene output', async ( assert.deepEqual(posts[0].body, { input_kind: 'scene', input_path: 'Workflows/input/scene-manifest.json', model_id: 'pixal/world', collection: 'Workflows', params: {}, + // The generation is filed under the workflow run that started it. + parent_run_id: useWorkflowRunStore.getState().liveRecord.id, }) assert.equal(useWorkflowRunStore.getState().runState.outputPath, '/workspace/Workflows/result/scene-manifest.json') assert.equal(useWorkflowRunStore.getState().runState.outputUrl, undefined) diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index 92703792f..0bc4276ad 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -339,7 +339,7 @@ test('navigate changes page and validates it', async () => { const f = fake() assert.equal(await run('navigate', { page: 'models' }, f), 'Opened the Extensions page.') assert.deepEqual(f.calls, [['navigate', 'models']]) - assert.match(await run('navigate', { page: 'nope' }, f), /^Error: page must be one of: home, generate, workflows, library, models, settings/) + assert.match(await run('navigate', { page: 'nope' }, f), /^Error: page must be one of: home, generate, workflows, library, queue, models, settings/) }) test('a throwing dependency comes back as an error text', async () => { diff --git a/src/shared/pages/pageRegistry.test.ts b/src/shared/pages/pageRegistry.test.ts index eec0988bb..2446809d1 100644 --- a/src/shared/pages/pageRegistry.test.ts +++ b/src/shared/pages/pageRegistry.test.ts @@ -23,7 +23,7 @@ test('the pages keep their ids, labels and sidebar order', () => { [ ['home', ['home:Home']], ['create', ['generate:Generate', 'workflows:Workflows']], - ['manage', ['library:Library']], + ['manage', ['library:Library', 'queue:Queue']], ['system', ['models:Extensions', 'settings:Settings']], ], ) diff --git a/src/shared/pages/pageRegistry.ts b/src/shared/pages/pageRegistry.ts index c886fb23c..71602a961 100644 --- a/src/shared/pages/pageRegistry.ts +++ b/src/shared/pages/pageRegistry.ts @@ -76,6 +76,20 @@ export const PAGES = [ load: () => import('@areas/library/LibraryPage'), wrapperClass: 'flex flex-1 overflow-hidden', }, + { + id: 'queue', + label: 'Queue', + group: 'manage', + icon: [ + ['path', { d: 'M4 6h11' }], + ['path', { d: 'M4 12h6' }], + ['path', { d: 'M4 18h6' }], + ['circle', { cx: 17, cy: 15.5, r: 4.5 }], + ['path', { d: 'M17 13.5v2.2l1.4 1.1' }], + ], + load: () => import('@areas/queue/QueuePage'), + wrapperClass: 'flex-1 overflow-y-auto', + }, { id: 'models', label: 'Extensions', diff --git a/src/shared/thumbnails/assetThumbnailService.ts b/src/shared/thumbnails/assetThumbnailService.ts index 710e17e72..8bcb77329 100644 --- a/src/shared/thumbnails/assetThumbnailService.ts +++ b/src/shared/thumbnails/assetThumbnailService.ts @@ -39,6 +39,15 @@ export async function requestLibraryThumbnails(entries: AssetLibraryEntry[]): Pr await Promise.all(entriesNeedingThumbnails(entries).map((entry) => getQueue().ensure(entry.workspacePath))) } +/** + * Queues a thumbnail for one workspace mesh (.glb/.gltf) unless it is cached; + * the result lands in the thumbnail store, keyed by `workspacePath`. + */ +export function ensureThumbnail(workspacePath: string): void { + if (!/\.(glb|gltf)$/i.test(workspacePath)) return + void getQueue().ensure(workspacePath) +} + async function backfillLibraryThumbnails(): Promise { try { const result = await window.electron.workspace.library.list() diff --git a/src/shared/types/runs.ts b/src/shared/types/runs.ts index c8fda5b63..5665a6e36 100644 --- a/src/shared/types/runs.ts +++ b/src/shared/types/runs.ts @@ -34,6 +34,8 @@ export interface RunRecord { ended_at?: number | null /** Computed by the backend; null while the run has not ended. */ duration_s?: number | null + /** The run that launched this one (a workflow's model step); null for top-level runs. */ + parent_run_id?: string | null } export interface RunRecordPage { From fa8aae99eb400d9bce3fcd17af54fc716600f3a1 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 16 Nov 2026 11:25:26 +0100 Subject: [PATCH 070/108] feat(image-studio): add an Image Studio page and paste images into Generate A new Image Studio page (Create, after Generate) to prepare input images: - a canvas on a checkerboard, with a before/after slider when an image was made from another one (ready for the image operations of #421) - a history strip of the session's images, kept across page switches - file picker, drag and drop anywhere, and Ctrl+V paste - "Send to Generate" puts the image in Generate's image input Ctrl+V outside text fields also pastes an image into Generate's image input. Pasted images, and dropped ones without a file path, are saved to /Images/ through a narrow workspace:images:save IPC (PNG, JPEG or WebP checked by signature, 40 MB cap, sanitized unique names, never overwrites, never in tmp/). Picked files keep their own path. Closes #420 --- electron/main/ipc-handlers.ts | 6 + electron/main/workspace-images.test.ts | 123 +++++++ electron/main/workspace-images.ts | 152 ++++++++ electron/preload/electron-api.ts | 7 + package.json | 2 +- .../generate/components/WorkflowPanel.tsx | 32 +- src/areas/image-studio/ImageStudioPage.tsx | 333 ++++++++++++++++++ .../image-studio/imageStudioStore.test.ts | 78 ++++ src/areas/image-studio/imageStudioStore.ts | 116 ++++++ .../components/agent/clientTools.test.ts | 2 +- src/shared/pages/pageRegistry.test.ts | 2 +- src/shared/pages/pageRegistry.ts | 12 + src/shared/types/electron.d.ts | 5 + src/shared/utils/imageInput.test.ts | 25 ++ src/shared/utils/imageInput.ts | 46 +++ 15 files changed, 935 insertions(+), 6 deletions(-) create mode 100644 electron/main/workspace-images.test.ts create mode 100644 electron/main/workspace-images.ts create mode 100644 src/areas/image-studio/ImageStudioPage.tsx create mode 100644 src/areas/image-studio/imageStudioStore.test.ts create mode 100644 src/areas/image-studio/imageStudioStore.ts create mode 100644 src/shared/utils/imageInput.test.ts diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index a43d1efa9..c14132b1a 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -96,6 +96,7 @@ import { import { registerWorkspaceAssetLibraryIpcHandlers } from './artifact-registry-service' import { getThumbnailsDir } from './asset-thumbnail-protocol' import { registerAssetThumbnailIpcHandlers, removeThumbnails } from './asset-thumbnail-cache' +import { registerWorkspaceImageIpcHandlers } from './workspace-images' import { updatesSupported } from './updater' import { ModelWeightOperations } from './model-weight-operations' import { readLocalFileBase64 } from './bounded-file-reader' @@ -1011,6 +1012,11 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe getWorkspaceDir: () => getSettings(app.getPath('userData')).workspaceDir, thumbnailsDir: getThumbnailsDir(), }) + registerWorkspaceImageIpcHandlers({ + ipcMain, + getWorkspaceDir: () => getSettings(app.getPath('userData')).workspaceDir, + showItemInFolder: (path) => shell.showItemInFolder(path), + }) ipcMain.handle('workspace:listCollections', async () => { const base = workspacePath() diff --git a/electron/main/workspace-images.test.ts b/electron/main/workspace-images.test.ts new file mode 100644 index 000000000..e3e7c9d52 --- /dev/null +++ b/electron/main/workspace-images.test.ts @@ -0,0 +1,123 @@ +import assert from 'node:assert/strict' +import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import test from 'node:test' + +import { + MAX_WORKSPACE_IMAGE_BYTES, + isRevealableImagePath, + registerWorkspaceImageIpcHandlers, + sanitizeImageBaseName, + saveWorkspaceImage, + sniffImageMime, + validateWorkspaceImageRequest, + workspaceImageFileName, +} from './workspace-images' + +const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0]) +const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0]) +const WEBP = new Uint8Array([0x52, 0x49, 0x46, 0x46, 1, 0, 0, 0, 0x57, 0x45, 0x42, 0x50, 0]) + +async function withWorkspace(run: (workspaceDir: string) => Promise) { + const workspaceDir = await mkdtemp(path.join(tmpdir(), 'modly-images-')) + try { + await run(workspaceDir) + } finally { + await rm(workspaceDir, { recursive: true, force: true }) + } +} + +test('sniffImageMime recognises PNG, JPEG and WebP signatures only', () => { + assert.equal(sniffImageMime(PNG), 'image/png') + assert.equal(sniffImageMime(JPEG), 'image/jpeg') + assert.equal(sniffImageMime(WEBP), 'image/webp') + assert.equal(sniffImageMime(new TextEncoder().encode('GIF89a')), null) + assert.equal(sniffImageMime(new Uint8Array([0x52, 0x49, 0x46, 0x46, 1, 0, 0, 0, 0x57, 0x41, 0x56, 0x45])), null) + assert.equal(sniffImageMime(new Uint8Array()), null) +}) + +test('sanitizeImageBaseName keeps a short safe stem and never a path or device name', () => { + assert.equal(sanitizeImageBaseName('My photo (1).png'), 'My-photo-1') + assert.equal(sanitizeImageBaseName('../../etc/passwd'), 'passwd') + assert.equal(sanitizeImageBaseName('C:\\Users\\me\\shot.webp'), 'shot') + assert.equal(sanitizeImageBaseName('Café déjà vu.jpg'), 'Cafe-deja-vu') + assert.equal(sanitizeImageBaseName('CON.png'), 'image') + assert.equal(sanitizeImageBaseName('..'), 'image') + assert.equal(sanitizeImageBaseName('.png'), 'png') + assert.equal(sanitizeImageBaseName('***'), 'image') + assert.equal(sanitizeImageBaseName(undefined), 'image') + assert.equal(sanitizeImageBaseName(42), 'image') + assert.ok(sanitizeImageBaseName('a'.repeat(300)).length <= 60) +}) + +test('workspaceImageFileName is time-stamped and suffixed', () => { + assert.equal(workspaceImageFileName('pasted', 'png', new Date(2026, 10, 16, 9, 5, 7), 'a1b2c3'), 'pasted-20261116-090507-a1b2c3.png') +}) + +test('validateWorkspaceImageRequest refuses missing, oversized, unsupported and mislabelled bytes', () => { + assert.ok('error' in validateWorkspaceImageRequest(undefined)) + assert.ok('error' in validateWorkspaceImageRequest({ bytes: 'abc', mime: 'image/png' })) + assert.ok('error' in validateWorkspaceImageRequest({ bytes: new Uint8Array(), mime: 'image/png' })) + assert.ok('error' in validateWorkspaceImageRequest({ bytes: PNG, mime: 'image/gif' })) + assert.ok('error' in validateWorkspaceImageRequest({ bytes: JPEG, mime: 'image/png' })) + const big = new Uint8Array(MAX_WORKSPACE_IMAGE_BYTES + 1) + big.set(PNG) + assert.ok('error' in validateWorkspaceImageRequest({ bytes: big, mime: 'image/png' })) + assert.deepEqual(validateWorkspaceImageRequest({ bytes: JPEG, mime: 'image/jpeg', name: 'x.jpeg' }), { bytes: JPEG, extension: 'jpg', baseName: 'x' }) +}) + +test('saveWorkspaceImage writes into /Images and never overwrites', async () => { + await withWorkspace(async (workspaceDir) => { + const suffixes = ['aaaaaa', 'aaaaaa', 'bbbbbb'] + const deps = { now: () => new Date(2026, 0, 2, 3, 4, 5), randomSuffix: () => suffixes.shift() ?? 'zzzzzz' } + const first = await saveWorkspaceImage(workspaceDir, { bytes: PNG, mime: 'image/png', name: '../evil' }, deps) + assert.deepEqual(first, { success: true, path: path.join(workspaceDir, 'Images', 'evil-20260102-030405-aaaaaa.png') }) + // Same name and suffix: the clash draws another suffix instead of overwriting. + const second = await saveWorkspaceImage(workspaceDir, { bytes: PNG, mime: 'image/png', name: 'evil.png' }, deps) + assert.ok(second.success) + assert.equal(path.basename(second.path), 'evil-20260102-030405-bbbbbb.png') + assert.deepEqual((await readdir(path.join(workspaceDir, 'Images'))).sort(), ['evil-20260102-030405-aaaaaa.png', 'evil-20260102-030405-bbbbbb.png']) + assert.deepEqual(new Uint8Array(await readFile(first.success ? first.path : '')), PNG) + }) +}) + +test('saveWorkspaceImage reports invalid requests without writing', async () => { + await withWorkspace(async (workspaceDir) => { + const result = await saveWorkspaceImage(workspaceDir, { bytes: new TextEncoder().encode(''), mime: 'image/svg+xml' }) + assert.equal(result.success, false) + assert.deepEqual(await readdir(workspaceDir), []) + assert.equal((await saveWorkspaceImage('', { bytes: PNG, mime: 'image/png' })).success, false) + }) +}) + +test('isRevealableImagePath accepts only existing absolute image files', async () => { + await withWorkspace(async (workspaceDir) => { + const saved = await saveWorkspaceImage(workspaceDir, { bytes: PNG, mime: 'image/png' }) + assert.ok(saved.success) + assert.equal(await isRevealableImagePath(saved.path), true) + assert.equal(await isRevealableImagePath(path.join(workspaceDir, 'missing.png')), false) + assert.equal(await isRevealableImagePath(path.join(workspaceDir, 'Images')), false) + assert.equal(await isRevealableImagePath('relative.png'), false) + assert.equal(await isRevealableImagePath(path.join(workspaceDir, 'script.exe')), false) + assert.equal(await isRevealableImagePath(null), false) + }) +}) + +test('the IPC handlers save into the current workspace and reveal only images', async () => { + await withWorkspace(async (workspaceDir) => { + const handlers = new Map Promise>() + const revealed: string[] = [] + registerWorkspaceImageIpcHandlers({ + ipcMain: { handle: (channel, handler) => { handlers.set(channel, handler) } }, + getWorkspaceDir: () => workspaceDir, + showItemInFolder: (p) => { revealed.push(p) }, + }) + const saved = await handlers.get('workspace:images:save')?.({}, { bytes: PNG, mime: 'image/png', name: 'pasted' }) as { success: boolean, path: string } + assert.ok(saved.success) + assert.ok(saved.path.startsWith(path.join(workspaceDir, 'Images') + path.sep)) + assert.deepEqual(await handlers.get('workspace:images:reveal')?.({}, saved.path), { success: true }) + assert.equal((await handlers.get('workspace:images:reveal')?.({}, workspaceDir) as { success: boolean }).success, false) + assert.deepEqual(revealed, [saved.path]) + }) +}) diff --git a/electron/main/workspace-images.ts b/electron/main/workspace-images.ts new file mode 100644 index 000000000..30dd1b5a4 --- /dev/null +++ b/electron/main/workspace-images.ts @@ -0,0 +1,152 @@ +import { randomBytes } from 'node:crypto' +import { mkdir, stat, writeFile } from 'node:fs/promises' +import { extname, isAbsolute, join } from 'node:path' + +import { resolvePathWithinRoot } from './extension-path-guard' + +// Images the renderer hands over as bytes (pasted from the clipboard, dropped +// from a browser) need a file on disk, since every image input in Modly is a +// path. They land in /Images/, never in the workspace's tmp/ +// folder, which is wiped at each workflow run. Images picked from disk are not +// copied: their own path is used. + +export const WORKSPACE_IMAGES_DIR = 'Images' +export const MAX_WORKSPACE_IMAGE_BYTES = 40 * 1024 * 1024 + +const EXTENSION_BY_MIME = { + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'image/webp': 'webp', +} as const + +type ImageMime = keyof typeof EXTENSION_BY_MIME +type ImageExtension = (typeof EXTENSION_BY_MIME)[ImageMime] + +const REVEALABLE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.webp']) +const WINDOWS_RESERVED_NAME = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])$/i +const MAX_BASE_NAME_LENGTH = 60 + +export type WorkspaceImageSaveResult = + | { success: true, path: string } + | { success: false, error: string } + +function hasPrefix(bytes: Uint8Array, prefix: readonly number[], offset = 0): boolean { + return bytes.length >= offset + prefix.length && prefix.every((value, i) => bytes[offset + i] === value) +} + +/** The format the bytes actually are, from their signature; null when not PNG, JPEG or WebP. */ +export function sniffImageMime(bytes: Uint8Array): ImageMime | null { + if (hasPrefix(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) return 'image/png' + if (hasPrefix(bytes, [0xff, 0xd8, 0xff])) return 'image/jpeg' + if (hasPrefix(bytes, [0x52, 0x49, 0x46, 0x46]) && hasPrefix(bytes, [0x57, 0x45, 0x42, 0x50], 8)) return 'image/webp' + return null +} + +/** + * A short, filesystem-safe stem from a user-facing name: no extension, no path, + * only [A-Za-z0-9_-], never a reserved Windows device name. Falls back to "image". + */ +export function sanitizeImageBaseName(name: unknown): string { + if (typeof name !== 'string') return 'image' + const leaf = name.split(/[\\/]/).pop() ?? '' + const stem = leaf.slice(0, leaf.length - extname(leaf).length) || leaf + const safe = stem + .normalize('NFKD') + .replace(/[̀-ͯ]/g, '') + .replace(/[^A-Za-z0-9_-]+/g, '-') + .replace(/-{2,}/g, '-') + .replace(/^[-_]+|[-_]+$/g, '') + .slice(0, MAX_BASE_NAME_LENGTH) + .replace(/[-_]+$/g, '') + if (!safe || WINDOWS_RESERVED_NAME.test(safe)) return 'image' + return safe +} + +function pad(value: number): string { + return String(value).padStart(2, '0') +} + +/** `-YYYYMMDD-HHMMSS-.`: sortable by time, unique by suffix. */ +export function workspaceImageFileName(baseName: string, extension: ImageExtension, now: Date, suffix: string): string { + const stamp = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}` + return `${baseName}-${stamp}-${suffix}.${extension}` +} + +export interface ValidatedImageRequest { + bytes: Uint8Array + extension: ImageExtension + baseName: string +} + +/** + * Checks a save request from the renderer: bytes under the size cap whose + * signature is a supported image and agrees with the declared MIME type. + */ +export function validateWorkspaceImageRequest(payload: unknown): ValidatedImageRequest | { error: string } { + const values = (payload ?? {}) as { bytes?: unknown, name?: unknown, mime?: unknown } + if (!(values.bytes instanceof Uint8Array) || values.bytes.byteLength === 0) return { error: 'Image bytes are required.' } + if (values.bytes.byteLength > MAX_WORKSPACE_IMAGE_BYTES) return { error: 'Image is larger than 40 MB.' } + if (typeof values.mime !== 'string' || !(values.mime in EXTENSION_BY_MIME)) return { error: 'Only PNG, JPEG and WebP images can be saved.' } + const sniffed = sniffImageMime(values.bytes) + if (sniffed !== values.mime) return { error: 'Image content does not match its type.' } + return { bytes: values.bytes, extension: EXTENSION_BY_MIME[sniffed], baseName: sanitizeImageBaseName(values.name) } +} + +export interface SaveWorkspaceImageDeps { + now?: () => Date + randomSuffix?: () => string +} + +/** Writes the image to /Images/ under a fresh name and returns its absolute path. */ +export async function saveWorkspaceImage(workspaceDir: string, payload: unknown, deps: SaveWorkspaceImageDeps = {}): Promise { + const request = validateWorkspaceImageRequest(payload) + if ('error' in request) return { success: false, error: request.error } + if (typeof workspaceDir !== 'string' || !workspaceDir.trim()) return { success: false, error: 'No workspace folder is set.' } + + const now = deps.now ?? (() => new Date()) + const randomSuffix = deps.randomSuffix ?? (() => randomBytes(3).toString('hex')) + try { + const dir = resolvePathWithinRoot(workspaceDir, WORKSPACE_IMAGES_DIR) + await mkdir(dir, { recursive: true }) + for (let attempt = 0; attempt < 5; attempt++) { + const fileName = workspaceImageFileName(request.baseName, request.extension, now(), randomSuffix()) + const target = resolvePathWithinRoot(workspaceDir, join(WORKSPACE_IMAGES_DIR, fileName)) + try { + // 'wx' never overwrites: a clash just draws another suffix. + await writeFile(target, request.bytes, { flag: 'wx' }) + return { success: true, path: target } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error + } + } + return { success: false, error: 'Could not find a free file name for the image.' } + } catch (error) { + return { success: false, error: error instanceof Error ? error.message : String(error) } + } +} + +/** An absolute path to an existing PNG/JPEG/WebP file, safe to reveal in the OS file manager. */ +export async function isRevealableImagePath(path: unknown): Promise { + if (typeof path !== 'string' || !isAbsolute(path)) return false + if (!REVEALABLE_EXTENSIONS.has(extname(path).toLowerCase())) return false + try { + return (await stat(path)).isFile() + } catch { + return false + } +} + +export interface WorkspaceImageIpcDeps { + ipcMain: { handle(channel: string, handler: (event: unknown, payload?: unknown) => Promise): void } + getWorkspaceDir: () => string + showItemInFolder: (path: string) => void +} + +export function registerWorkspaceImageIpcHandlers(deps: WorkspaceImageIpcDeps): void { + deps.ipcMain.handle('workspace:images:save', async (_event, payload) => saveWorkspaceImage(deps.getWorkspaceDir(), payload)) + deps.ipcMain.handle('workspace:images:reveal', async (_event, payload) => { + if (!(await isRevealableImagePath(payload))) return { success: false, error: 'Not an image file.' } + deps.showItemInFolder(payload as string) + return { success: true } + }) +} diff --git a/electron/preload/electron-api.ts b/electron/preload/electron-api.ts index 9a309e27e..6259d8604 100644 --- a/electron/preload/electron-api.ts +++ b/electron/preload/electron-api.ts @@ -259,6 +259,13 @@ export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFra status: (workspacePath: string): Promise => ipcRenderer.invoke('workspace:thumbnails:status', { workspacePath }) as Promise, save: (request: AssetThumbnailSaveRequest): Promise => ipcRenderer.invoke('workspace:thumbnails:save', request) as Promise, }, + // Images handed over as bytes (pasted, dropped from a browser), saved to /Images/. + images: { + save: (request: { bytes: Uint8Array; name?: string; mime: string }): Promise<{ success: true; path: string } | { success: false; error: string }> => + ipcRenderer.invoke('workspace:images:save', request) as Promise<{ success: true; path: string } | { success: false; error: string }>, + reveal: (path: string): Promise<{ success: boolean; error?: string }> => + ipcRenderer.invoke('workspace:images:reveal', path) as Promise<{ success: boolean; error?: string }>, + }, }, // Extensions diff --git a/package.json b/package.json index dbb876925..063a2f9fd 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/main/gpu-memory.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/shared/pages/startPage.test.ts src/areas/home/recents.test.ts src/areas/workflows/workflowRunRecord.test.ts src/areas/queue/queueModel.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/main/gpu-memory.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/shared/pages/startPage.test.ts src/areas/home/recents.test.ts src/areas/workflows/workflowRunRecord.test.ts src/areas/queue/queueModel.test.ts electron/main/workspace-images.test.ts src/areas/image-studio/imageStudioStore.test.ts src/shared/utils/imageInput.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/components/WorkflowPanel.tsx b/src/areas/generate/components/WorkflowPanel.tsx index 493bf42c6..8d0241aa0 100644 --- a/src/areas/generate/components/WorkflowPanel.tsx +++ b/src/areas/generate/components/WorkflowPanel.tsx @@ -14,7 +14,7 @@ import { useWaitButton } from '@areas/workflows/useWaitButton' import { buildAllWorkflowExtensions, getWorkflowExtension } from '@areas/workflows/mockExtensions' import { validateWorkflowPreflight } from '@areas/workflows/preflight' import { mimeFromPath } from '@areas/workflows/nodes/imageUtils' -import { SUPPORTED_IMAGE_TYPES, useImageHandoff } from '@shared/utils/imageInput' +import { SUPPORTED_IMAGE_TYPES, saveImageToWorkspace, takePastedImage, useImageHandoff } from '@shared/utils/imageInput' import type { WorkflowExtension } from '@areas/workflows/mockExtensions' import type { Workflow, WFNode, WFEdge, ParamSchema } from '@shared/types/electron.d' import { PICKER_LABELS, openParamPicker, resolvePickerIntent } from '@shared/utils/paramPicker' @@ -160,7 +160,13 @@ async function readImageParams(path: string): Promise<{ filePath: string, previe return { filePath: path, preview: `data:${mimeFromPath(path)};base64,${base64}` } } -function ImageParamRow({ nodeId, nodes, onPatch }: { nodeId: string; nodes: FlowNode[]; onPatch: PatchFn }) { +function ImageParamRow({ nodeId, nodes, onPatch, pasteTarget = false }: { + nodeId: string + nodes: FlowNode[] + onPatch: PatchFn + /** Takes images pasted with Ctrl+V (only one row of the panel does). */ + pasteTarget?: boolean +}) { const node = nodes.find((n) => n.id === nodeId) const data = node?.data as { params: Record } | undefined const preview = data?.params.preview as string | undefined @@ -183,6 +189,23 @@ function ImageParamRow({ nodeId, nodes, onPatch }: { nodeId: string; nodes: Flow await applyImagePath(await window.electron.fs.selectImage()) }, [applyImagePath]) + // Ctrl+V outside text fields: the clipboard image is saved to the workspace + // (the node needs a file path), then loaded like a browsed one. + const applyRef = useRef(applyImagePath) + applyRef.current = applyImagePath + useEffect(() => { + if (!pasteTarget) return + const onPaste = (event: ClipboardEvent) => { + const file = takePastedImage(event) + if (!file) return + saveImageToWorkspace(file, 'pasted-image') + .then((path) => applyRef.current(path)) + .catch((err: unknown) => showToast(err instanceof Error ? err.message : 'Unable to paste the image')) + } + window.addEventListener('paste', onPaste) + return () => window.removeEventListener('paste', onPaste) + }, [pasteTarget, showToast]) + return (
    Browse image… + {pasteTarget && or paste with Ctrl+V} )}
    @@ -526,6 +550,8 @@ function EmbeddedCanvas({ workflow, allExtensions }: { // An image handed over by another page (Home) goes into the first image input // shown here, exactly as if it had been browsed in this panel. const pendingImage = useImageHandoff((s) => s.pendingPath) + // Ctrl+V goes to the same input. + const pasteTargetId = paramNodes.find((n) => n.type === 'imageNode')?.id useEffect(() => { const path = useImageHandoff.getState().take() if (!path) return @@ -567,7 +593,7 @@ function EmbeddedCanvas({ workflow, allExtensions }: { const isLast = i === paramNodes.length - 1 return (
    - {node.type === 'imageNode' && } + {node.type === 'imageNode' && } {node.type === 'textNode' && } {node.type === 'meshNode' && } {node.type === 'waitNode' && } diff --git a/src/areas/image-studio/ImageStudioPage.tsx b/src/areas/image-studio/ImageStudioPage.tsx new file mode 100644 index 000000000..59cdd5eec --- /dev/null +++ b/src/areas/image-studio/ImageStudioPage.tsx @@ -0,0 +1,333 @@ +import { useCallback, useEffect, useRef, useState, type CSSProperties } from 'react' +import { useAppStore } from '@shared/stores/appStore' +import { useNavStore } from '@shared/stores/navStore' +import { SUPPORTED_IMAGE_TYPES, saveImageToWorkspace, takePastedImage, useImageHandoff } from '@shared/utils/imageInput' +import { mimeFromPath } from '@areas/workflows/nodes/imageUtils' +import { fileNameOf, findByPath, sourceOf, useImageStudioStore, type StudioImage } from './imageStudioStore' + +// ─── Icons ──────────────────────────────────────────────────────────────────── + +function Icon({ d, size = 16 }: { d: string, size?: number }): JSX.Element { + return ( + + ) +} + +const ICONS = { + upload: 'M12 16V4m0 0-4.5 4.5M12 4l4.5 4.5M4 15v3a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-3', + folder: 'M3 7a2 2 0 0 1 2-2h4l2 2h8a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2z', + generate: 'M12 2 3 7v10l9 5 9-5V7zM3 7l9 5 9-5M12 12v10', + close: 'M6 6l12 12M18 6 6 18', +} + +const PASTE_HINT = typeof navigator !== 'undefined' && /mac/i.test(navigator.platform) ? '⌘V' : 'Ctrl+V' + +/** Transparent pixels show as a checkerboard, as in image editors. */ +const CHECKERBOARD: CSSProperties = { + backgroundColor: '#18181b', + backgroundImage: 'repeating-conic-gradient(#27272a 0% 25%, transparent 0% 50%)', + backgroundSize: '20px 20px', +} + +// ─── Loading images ─────────────────────────────────────────────────────────── + +/** An object URL of an image file on disk (the renderer cannot read file:// in dev). */ +async function previewUrlForPath(path: string): Promise { + const base64 = await window.electron.fs.readFileBase64(path) + const bytes = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)) + return URL.createObjectURL(new Blob([bytes], { type: mimeFromPath(path) })) +} + +// ─── Image operations (slot) ────────────────────────────────────────────────── +// Image operations (background removal, … — ticket #421) plug in here. An +// operation reads the selected image and writes a NEW file; the page adds it +// to the history with the selected image as its source, so the canvas shows +// the result before/after against it. The originals are never modified. + +interface ImageOp { + id: string + label: string + icon: string + /** Produces a new image file from `image` and returns its absolute path. */ + run: (image: StudioImage) => Promise +} + +const IMAGE_OPS: readonly ImageOp[] = [] + +function ImageOpsToolbar({ image, onResult }: { image: StudioImage, onResult: (path: string, source: StudioImage) => Promise }): JSX.Element | null { + const showError = useAppStore((s) => s.showError) + const [running, setRunning] = useState(null) + if (IMAGE_OPS.length === 0) return null + + async function run(op: ImageOp) { + setRunning(op.id) + try { + await onResult(await op.run(image), image) + } catch (err) { + showError(err instanceof Error ? err.message : String(err)) + } finally { + setRunning(null) + } + } + + return ( +
    + {IMAGE_OPS.map((op) => ( + + ))} +
    + ) +} + +// ─── Canvas ─────────────────────────────────────────────────────────────────── + +function CompareView({ before, after }: { before: StudioImage, after: StudioImage }): JSX.Element { + const [position, setPosition] = useState(50) + const box = useRef(null) + + const moveTo = (clientX: number) => { + const rect = box.current?.getBoundingClientRect() + if (!rect || rect.width === 0) return + setPosition(Math.min(100, Math.max(0, ((clientX - rect.left) / rect.width) * 100))) + } + + return ( +
    { event.currentTarget.setPointerCapture(event.pointerId); moveTo(event.clientX) }} + onPointerMove={(event) => { if (event.currentTarget.hasPointerCapture(event.pointerId)) moveTo(event.clientX) }} + role="slider" + aria-label="Before / after" + aria-valuemin={0} + aria-valuemax={100} + aria-valuenow={Math.round(position)} + tabIndex={0} + onKeyDown={(event) => { + if (event.key === 'ArrowLeft') setPosition((p) => Math.max(0, p - 5)) + if (event.key === 'ArrowRight') setPosition((p) => Math.min(100, p + 5)) + }} + > + {after.name} + {before.name} +
    + + + +
    + Before + After +
    + ) +} + +function Canvas({ image, source }: { image: StudioImage, source?: StudioImage }): JSX.Element { + return ( +
    + {source + ? + : {image.name}} +
    + ) +} + +function EmptyCanvas({ busy, onBrowse }: { busy: boolean, onBrowse: () => void }): JSX.Element { + return ( + + ) +} + +// ─── History strip ──────────────────────────────────────────────────────────── + +function HistoryStrip({ entries, selectedId, onSelect, onRemove }: { + entries: StudioImage[] + selectedId: string | null + onSelect: (id: string) => void + onRemove: (id: string) => void +}): JSX.Element { + return ( +
    + {entries.map((entry) => { + const selected = entry.id === selectedId + return ( +
    + + {entry.sourceId && ( + edit + )} + +
    + ) + })} +
    + ) +} + +// ─── Page ───────────────────────────────────────────────────────────────────── + +export default function ImageStudioPage(): JSX.Element { + const navigate = useNavStore((s) => s.navigate) + const showError = useAppStore((s) => s.showError) + const showToast = useAppStore((s) => s.showToast) + const entries = useImageStudioStore((s) => s.entries) + const selectedId = useImageStudioStore((s) => s.selectedId) + const { add, select, remove } = useImageStudioStore.getState() + const [busy, setBusy] = useState(false) + const [dragging, setDragging] = useState(false) + + const selected = entries.find((e) => e.id === selectedId) + const source = sourceOf(entries, selected) + + /** An image already on disk: used where it is, not copied. */ + const addPath = useCallback(async (path: string, sourceImage?: StudioImage) => { + const existing = sourceImage ? undefined : findByPath(useImageStudioStore.getState().entries, path) + if (existing) { + select(existing.id) + return + } + const previewUrl = await previewUrlForPath(path) + add({ path, previewUrl, name: fileNameOf(path), sourceId: sourceImage?.id }) + }, [add, select]) + + /** Image bytes with no file behind them (clipboard, browser drag): saved to the workspace first. */ + const addBlob = useCallback(async (file: File, name: string) => { + const path = await saveImageToWorkspace(file, name) + add({ path, previewUrl: URL.createObjectURL(file), name: fileNameOf(path) }) + }, [add]) + + const runImport = useCallback(async (work: () => Promise) => { + setBusy(true) + try { + await work() + } catch (err) { + showError(err instanceof Error ? err.message : 'Could not add the image.') + } finally { + setBusy(false) + } + }, [showError]) + + const browse = useCallback(() => { + void runImport(async () => { + const path = await window.electron.fs.selectImage() + if (path) await addPath(path) + }) + }, [runImport, addPath]) + + const importFile = useCallback((file: File | undefined) => { + if (!file) return + if (!SUPPORTED_IMAGE_TYPES.has(file.type)) { + showToast('Only PNG, JPG and WEBP images are supported') + return + } + const path = window.electron.fs.getPathForFile(file) + void runImport(() => (path ? addPath(path) : addBlob(file, file.name || 'dropped-image'))) + }, [runImport, addPath, addBlob, showToast]) + + // Ctrl+V anywhere on the page, unless the focus is in a text field. + useEffect(() => { + const onPaste = (event: ClipboardEvent) => { + const file = takePastedImage(event) + if (file) void runImport(() => addBlob(file, 'pasted-image')) + } + window.addEventListener('paste', onPaste) + return () => window.removeEventListener('paste', onPaste) + }, [runImport, addBlob]) + + function sendToGenerate(image: StudioImage) { + useImageHandoff.getState().offer(image.path) + navigate('generate') + } + + async function revealInFolder(image: StudioImage) { + const result = await window.electron.workspace.images.reveal(image.path) + if (!result.success) showToast(result.error ?? 'Could not show the file') + } + + return ( +
    { event.preventDefault(); event.dataTransfer.dropEffect = 'copy'; setDragging(true) }} + onDragLeave={(event) => { if (!event.currentTarget.contains(event.relatedTarget as Node | null)) setDragging(false) }} + onDrop={(event) => { + event.preventDefault() + setDragging(false) + importFile(event.dataTransfer.files[0]) + }} + > +
    +
    +

    Image Studio

    +

    + {selected ? selected.name : <>Prepare an image before turning it into 3D · {PASTE_HINT} to paste} +

    +
    + + {selected && } + + + {selected && ( + + )} + +
    + + {selected ? : } + + {entries.length > 0 && ( +
    +

    This session

    + +
    + )} + + {dragging && ( +
    + Drop to add the image +
    + )} +
    + ) +} diff --git a/src/areas/image-studio/imageStudioStore.test.ts b/src/areas/image-studio/imageStudioStore.test.ts new file mode 100644 index 000000000..940d358f8 --- /dev/null +++ b/src/areas/image-studio/imageStudioStore.test.ts @@ -0,0 +1,78 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { + addToHistory, + createImageStudioStore, + fileNameOf, + findByPath, + sourceOf, + withoutEntries, + type StudioHistory, + type StudioImage, +} from './imageStudioStore' + +const image = (id: string, sourceId?: string): StudioImage => ({ + id, path: `C:\\work\\${id}.png`, previewUrl: `blob:${id}`, name: `${id}.png`, sourceId, createdAt: 0, +}) + +const history = (...entries: StudioImage[]): StudioHistory => ({ entries, selectedId: entries[0]?.id ?? null }) + +test('addToHistory puts the image first, selects it and drops the oldest beyond the cap', () => { + const { history: next, dropped } = addToHistory(history(image('b'), image('a')), image('c'), 2) + assert.deepEqual(next.entries.map((e) => e.id), ['c', 'b']) + assert.equal(next.selectedId, 'c') + assert.deepEqual(dropped.map((e) => e.id), ['a']) +}) + +test('withoutEntries moves the selection to the next older image, else the newer one', () => { + const start = { ...history(image('c'), image('b'), image('a')), selectedId: 'b' } + assert.equal(withoutEntries(start, ['b']).history.selectedId, 'a') + assert.equal(withoutEntries({ ...start, selectedId: 'a' }, ['a']).history.selectedId, 'b') + assert.equal(withoutEntries(start, ['a', 'b', 'c']).history.selectedId, null) + assert.equal(withoutEntries(start, ['c']).history.selectedId, 'b') +}) + +test('withoutEntries detaches images made from a removed source and is a no-op for unknown ids', () => { + const start = history(image('cut', 'photo'), image('photo')) + const { history: next, dropped } = withoutEntries(start, ['photo']) + assert.deepEqual(next.entries, [{ ...image('cut'), sourceId: undefined }]) + assert.deepEqual(dropped.map((e) => e.id), ['photo']) + assert.equal(withoutEntries(start, ['nope']).history, start) +}) + +test('sourceOf, findByPath and fileNameOf', () => { + const entries = [image('cut', 'photo'), image('photo'), image('orphan', 'gone')] + assert.equal(sourceOf(entries, entries[0])?.id, 'photo') + assert.equal(sourceOf(entries, entries[1]), undefined) + assert.equal(sourceOf(entries, entries[2]), undefined) + assert.equal(sourceOf(entries, undefined), undefined) + assert.equal(findByPath(entries, 'c:/WORK/photo.png')?.id, 'photo') + assert.equal(findByPath(entries, 'c:/work/other.png'), undefined) + assert.equal(fileNameOf('C:\\a\\b\\pic.webp'), 'pic.webp') + assert.equal(fileNameOf('/x/y.png'), 'y.png') +}) + +test('the store keeps history, selects new images and revokes previews it drops', () => { + const revoked: string[] = [] + const store = createImageStudioStore((url) => revoked.push(url)) + const a = store.getState().add({ path: 'C:/a.png', previewUrl: 'blob:a', name: 'a.png' }) + const b = store.getState().add({ path: 'C:/b.png', previewUrl: 'blob:b', name: 'b.png', sourceId: a.id }) + assert.notEqual(a.id, b.id) + assert.equal(store.getState().selectedId, b.id) + assert.deepEqual(store.getState().entries.map((e) => e.id), [b.id, a.id]) + + store.getState().select(a.id) + assert.equal(store.getState().selectedId, a.id) + store.getState().select('missing') + assert.equal(store.getState().selectedId, a.id) + + store.getState().remove(a.id) + assert.deepEqual(revoked, ['blob:a']) + assert.equal(store.getState().selectedId, b.id) + assert.equal(store.getState().entries[0].sourceId, undefined) + + store.getState().clear() + assert.deepEqual(revoked, ['blob:a', 'blob:b']) + assert.deepEqual(store.getState().entries, []) + assert.equal(store.getState().selectedId, null) +}) diff --git a/src/areas/image-studio/imageStudioStore.ts b/src/areas/image-studio/imageStudioStore.ts new file mode 100644 index 000000000..47c1a4e39 --- /dev/null +++ b/src/areas/image-studio/imageStudioStore.ts @@ -0,0 +1,116 @@ +import { create } from 'zustand' + +// The Image Studio's history: every image of this session, newest first. +// Pages unmount when the user switches away, so it lives here, at module +// level, and survives navigation (not app restarts). +// +// An image made from another one (an image operation, e.g. background +// removal) records it as `sourceId`; the studio compares the two before/after. + +export interface StudioImage { + id: string + /** Absolute path on disk: what Generate and image operations read. */ + path: string + /** URL the renderer displays (an object URL the store owns, or a data URL). */ + previewUrl: string + name: string + /** The image this one was produced from, when it came from an operation. */ + sourceId?: string + createdAt: number +} + +export type NewStudioImage = Omit + +export interface StudioHistory { + entries: StudioImage[] + selectedId: string | null +} + +/** Oldest images beyond this are dropped from the history (not from disk). */ +export const MAX_STUDIO_HISTORY = 60 + +/** Adds an image in front and selects it; entries over `max` fall off the end. */ +export function addToHistory(history: StudioHistory, entry: StudioImage, max = MAX_STUDIO_HISTORY): { history: StudioHistory, dropped: StudioImage[] } { + const all = [entry, ...history.entries] + return withoutEntries({ entries: all, selectedId: entry.id }, all.slice(max).map((e) => e.id)) +} + +/** + * Drops entries from the history. Images made from a dropped one lose their + * source (nothing to compare with any more); the selection moves to the + * neighbour of the removed image. + */ +export function withoutEntries(history: StudioHistory, ids: readonly string[]): { history: StudioHistory, dropped: StudioImage[] } { + const gone = new Set(ids) + const dropped = history.entries.filter((e) => gone.has(e.id)) + if (dropped.length === 0) return { history, dropped } + const entries = history.entries + .filter((e) => !gone.has(e.id)) + .map((e) => (e.sourceId && gone.has(e.sourceId) ? { ...e, sourceId: undefined } : e)) + let selectedId = history.selectedId + if (selectedId && gone.has(selectedId)) { + const index = history.entries.findIndex((e) => e.id === selectedId) + const after = history.entries.slice(index + 1).find((e) => !gone.has(e.id)) + const before = history.entries.slice(0, index).reverse().find((e) => !gone.has(e.id)) + selectedId = (after ?? before)?.id ?? null + } + return { history: { entries, selectedId }, dropped } +} + +export function findByPath(entries: readonly StudioImage[], path: string): StudioImage | undefined { + const key = normalizePath(path) + return entries.find((e) => normalizePath(e.path) === key) +} + +/** The image an entry was made from, if it is still in the history. */ +export function sourceOf(entries: readonly StudioImage[], entry: StudioImage | undefined): StudioImage | undefined { + return entry?.sourceId ? entries.find((e) => e.id === entry.sourceId) : undefined +} + +function normalizePath(path: string): string { + return path.replace(/\\/g, '/').toLowerCase() +} + +export function fileNameOf(path: string): string { + return path.split(/[\\/]/).pop() || path +} + +export interface ImageStudioState extends StudioHistory { + /** Adds an image to the history and selects it. */ + add: (image: NewStudioImage) => StudioImage + select: (id: string) => void + /** Removes an image from the history; the file stays on disk. */ + remove: (id: string) => void + clear: () => void +} + +function revokeObjectUrl(url: string): void { + if (url.startsWith('blob:')) URL.revokeObjectURL(url) +} + +let nextId = 0 + +export function createImageStudioStore(revoke: (url: string) => void = revokeObjectUrl) { + return create((set, get) => { + const apply = ({ history, dropped }: { history: StudioHistory, dropped: StudioImage[] }) => { + set(history) + for (const entry of dropped) revoke(entry.previewUrl) + } + return { + entries: [], + selectedId: null, + add: (image) => { + const entry: StudioImage = { ...image, id: `img-${Date.now().toString(36)}-${(nextId++).toString(36)}`, createdAt: Date.now() } + apply(addToHistory(get(), entry)) + return entry + }, + select: (id) => { + if (get().entries.some((e) => e.id === id)) set({ selectedId: id }) + }, + remove: (id) => apply(withoutEntries(get(), [id])), + clear: () => apply(withoutEntries(get(), get().entries.map((e) => e.id))), + } + }) +} + +export const useImageStudioStore = createImageStudioStore() diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index 0bc4276ad..ef6754bb9 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -339,7 +339,7 @@ test('navigate changes page and validates it', async () => { const f = fake() assert.equal(await run('navigate', { page: 'models' }, f), 'Opened the Extensions page.') assert.deepEqual(f.calls, [['navigate', 'models']]) - assert.match(await run('navigate', { page: 'nope' }, f), /^Error: page must be one of: home, generate, workflows, library, queue, models, settings/) + assert.match(await run('navigate', { page: 'nope' }, f), /^Error: page must be one of: home, generate, image-studio, workflows, library, queue, models, settings/) }) test('a throwing dependency comes back as an error text', async () => { diff --git a/src/shared/pages/pageRegistry.test.ts b/src/shared/pages/pageRegistry.test.ts index 2446809d1..5b970af37 100644 --- a/src/shared/pages/pageRegistry.test.ts +++ b/src/shared/pages/pageRegistry.test.ts @@ -22,7 +22,7 @@ test('the pages keep their ids, labels and sidebar order', () => { pagesByGroup().map((s) => [s.group, s.pages.map((p) => `${p.id}:${p.label}`)]), [ ['home', ['home:Home']], - ['create', ['generate:Generate', 'workflows:Workflows']], + ['create', ['generate:Generate', 'image-studio:Image Studio', 'workflows:Workflows']], ['manage', ['library:Library', 'queue:Queue']], ['system', ['models:Extensions', 'settings:Settings']], ], diff --git a/src/shared/pages/pageRegistry.ts b/src/shared/pages/pageRegistry.ts index 71602a961..726a6f998 100644 --- a/src/shared/pages/pageRegistry.ts +++ b/src/shared/pages/pageRegistry.ts @@ -49,6 +49,18 @@ export const PAGES = [ load: () => import('@areas/generate/GeneratePage'), wrapperClass: 'flex flex-1 overflow-hidden', }, + { + id: 'image-studio', + label: 'Image Studio', + group: 'create', + icon: [ + ['rect', { x: 3, y: 3, width: 18, height: 18, rx: 2 }], + ['circle', { cx: 8.5, cy: 8.5, r: 1.5 }], + ['path', { d: 'M21 15l-5-5L5 21' }], + ], + load: () => import('@areas/image-studio/ImageStudioPage'), + wrapperClass: 'flex flex-1 overflow-hidden', + }, { id: 'workflows', label: 'Workflows', diff --git a/src/shared/types/electron.d.ts b/src/shared/types/electron.d.ts index 49c8915b3..0935069ca 100644 --- a/src/shared/types/electron.d.ts +++ b/src/shared/types/electron.d.ts @@ -323,6 +323,11 @@ declare global { status: (workspacePath: string) => Promise save: (request: AssetThumbnailSaveRequest) => Promise } + /** Saves image bytes (PNG/JPEG/WebP, max 40 MB) to /Images/ and returns the new file's absolute path. */ + images: { + save: (request: { bytes: Uint8Array; name?: string; mime: string }) => Promise<{ success: true; path: string } | { success: false; error: string }> + reveal: (path: string) => Promise<{ success: boolean; error?: string }> + } } setup: { check: () => Promise<{ needed: boolean; defaultDataDir: string; platform: string; arch: string }> diff --git a/src/shared/utils/imageInput.test.ts b/src/shared/utils/imageInput.test.ts new file mode 100644 index 000000000..0874ec19e --- /dev/null +++ b/src/shared/utils/imageInput.test.ts @@ -0,0 +1,25 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { clipboardImage, isTextEntryTarget } from './imageInput' + +const item = (kind: string, type: string, file: unknown = { name: type }) => ({ kind, type, getAsFile: () => file as File }) + +test('clipboardImage returns the first supported image file', () => { + assert.equal(clipboardImage(null), null) + assert.equal(clipboardImage({ items: [item('string', 'text/plain')] }), null) + assert.equal(clipboardImage({ items: [item('file', 'image/gif')] }), null) + const png = { name: 'image.png' } + assert.equal(clipboardImage({ items: [item('string', 'text/html'), item('file', 'image/png', png), item('file', 'image/jpeg')] }), png) +}) + +test('isTextEntryTarget is true only where Ctrl+V should paste text', () => { + assert.equal(isTextEntryTarget(null), false) + assert.equal(isTextEntryTarget({ tagName: 'BODY' }), false) + assert.equal(isTextEntryTarget({ tagName: 'BUTTON' }), false) + assert.equal(isTextEntryTarget({ tagName: 'TEXTAREA' }), true) + assert.equal(isTextEntryTarget({ tagName: 'INPUT', type: 'text' }), true) + assert.equal(isTextEntryTarget({ tagName: 'INPUT', type: 'number' }), true) + assert.equal(isTextEntryTarget({ tagName: 'INPUT', type: 'checkbox' }), false) + assert.equal(isTextEntryTarget({ tagName: 'INPUT', type: 'range' }), false) + assert.equal(isTextEntryTarget({ tagName: 'DIV', isContentEditable: true }), true) +}) diff --git a/src/shared/utils/imageInput.ts b/src/shared/utils/imageInput.ts index 2aeba8846..84f9ba2e0 100644 --- a/src/shared/utils/imageInput.ts +++ b/src/shared/utils/imageInput.ts @@ -28,3 +28,49 @@ export function droppedImagePath(file: File | undefined): string | null { if (!file || !SUPPORTED_IMAGE_TYPES.has(file.type)) return null return window.electron.fs.getPathForFile(file) || null } + +/** Where a supported image dropped or pasted without a disk path is saved: a workspace file. */ +export async function saveImageToWorkspace(blob: Blob, name?: string): Promise { + const bytes = new Uint8Array(await blob.arrayBuffer()) + const result = await window.electron.workspace.images.save({ bytes, name, mime: blob.type }) + if (!result.success) throw new Error(result.error) + return result.path +} + +interface ClipboardItemLike { + kind: string + type: string + getAsFile: () => File | null +} + +/** The first supported image in a paste's clipboard data, or null. */ +export function clipboardImage(data: { items: ArrayLike } | null | undefined): File | null { + if (!data) return null + for (const item of Array.from(data.items)) { + if (item.kind === 'file' && SUPPORTED_IMAGE_TYPES.has(item.type)) return item.getAsFile() + } + return null +} + +/** Whether a paste lands in something that takes text, where Ctrl+V must keep its usual meaning. */ +export function isTextEntryTarget(target: unknown): boolean { + const el = target as { tagName?: unknown, isContentEditable?: unknown, type?: unknown } | null + if (!el || typeof el !== 'object') return false + if (el.isContentEditable === true) return true + const tag = typeof el.tagName === 'string' ? el.tagName.toUpperCase() : '' + if (tag === 'TEXTAREA' || tag === 'SELECT') return true + if (tag !== 'INPUT') return false + const type = typeof el.type === 'string' ? el.type.toLowerCase() : 'text' + return !['button', 'checkbox', 'color', 'file', 'image', 'radio', 'range', 'reset', 'submit'].includes(type) +} + +/** + * The image a paste event carries, unless it was aimed at a text field. Call + * from a window 'paste' listener; it prevents the default when it takes one. + */ +export function takePastedImage(event: ClipboardEvent): File | null { + if (isTextEntryTarget(event.target) || isTextEntryTarget(document.activeElement)) return null + const file = clipboardImage(event.clipboardData) + if (file) event.preventDefault() + return file +} From 1a93db28cfe55bc76c767124f29133f3a13efcac Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 16 Nov 2026 14:07:34 +0100 Subject: [PATCH 071/108] feat(image-studio): remove an image's background on the CPU Image Studio gets a "Remove background" tool. The cut-out lands in the history with the original as its source, so the before/after slider compares them; a Mask toggle shows the matte and an edge threshold hardens soft edges. It needs no extension and no GPU: BiRefNet-lite (MIT, ONNX, pinned revision, checked by size and SHA-256) runs with onnxruntime on the CPU. Both are fetched on first use behind a one-time setup card with progress (~249 MB): onnxruntime 1.23.2 is pip-installed into the backend's own environment on demand rather than added to requirements.txt, which would force every user through a full Python reinstall; 1.23.2 is the last release with Intel Mac wheels. Backend: an image_ops registry modelled on mesh_ops, under /image: GET /image/ops, GET /image/op/{op}/status, POST /image/op/{op}/prepare (SSE, LLM download format) and POST /image/op/{op} (409 until prepared). Outputs go to /Images/ and never overwrite. EditPanel's param fields move to a shared ParamFields component, used by both panels. Closes #421 --- api/main.py | 3 +- api/routers/image.py | 237 ++++++++++++++++ api/services/image_ops/__init__.py | 30 +++ api/services/image_ops/birefnet.py | 90 +++++++ api/services/image_ops/builtin.py | 88 ++++++ api/services/image_ops/outputs.py | 72 +++++ api/services/image_ops/registry.py | 125 +++++++++ api/services/image_ops/requirements.py | 248 +++++++++++++++++ api/services/image_ops/types.py | 101 +++++++ api/tests/test_image_ops_registry.py | 137 ++++++++++ api/tests/test_image_ops_remove_background.py | 253 ++++++++++++++++++ api/tests/test_image_ops_requirements.py | 177 ++++++++++++ api/tests/test_image_ops_router.py | 235 ++++++++++++++++ package.json | 2 +- src/areas/generate/components/EditPanel.tsx | 184 +------------ src/areas/image-studio/ImageOpsToolbar.tsx | 247 +++++++++++++++++ src/areas/image-studio/ImageStudioPage.tsx | 125 +++++---- src/areas/image-studio/imageOps.test.ts | 134 ++++++++++ src/areas/image-studio/imageOps.ts | 150 +++++++++++ src/areas/image-studio/imageOpsApi.ts | 50 ++++ src/areas/image-studio/imageStudioStore.ts | 4 + src/shared/components/ui/ParamFields.tsx | 186 +++++++++++++ src/shared/services/llmDownloads.ts | 9 +- 23 files changed, 2647 insertions(+), 240 deletions(-) create mode 100644 api/routers/image.py create mode 100644 api/services/image_ops/__init__.py create mode 100644 api/services/image_ops/birefnet.py create mode 100644 api/services/image_ops/builtin.py create mode 100644 api/services/image_ops/outputs.py create mode 100644 api/services/image_ops/registry.py create mode 100644 api/services/image_ops/requirements.py create mode 100644 api/services/image_ops/types.py create mode 100644 api/tests/test_image_ops_registry.py create mode 100644 api/tests/test_image_ops_remove_background.py create mode 100644 api/tests/test_image_ops_requirements.py create mode 100644 api/tests/test_image_ops_router.py create mode 100644 src/areas/image-studio/ImageOpsToolbar.tsx create mode 100644 src/areas/image-studio/imageOps.test.ts create mode 100644 src/areas/image-studio/imageOps.ts create mode 100644 src/areas/image-studio/imageOpsApi.ts create mode 100644 src/shared/components/ui/ParamFields.tsx diff --git a/api/main.py b/api/main.py index 9bf352304..0526d2e37 100644 --- a/api/main.py +++ b/api/main.py @@ -12,7 +12,7 @@ from services.stdio_utf8 import ensure_utf8_stdio ensure_utf8_stdio() # must run before any print/logging hits the pipe -from routers import generation, model, optimize, status, settings, extensions, export, workflow_runs, agent, llm +from routers import generation, model, optimize, status, settings, extensions, export, workflow_runs, agent, llm, image @asynccontextmanager @@ -68,6 +68,7 @@ def filter(self, record): app.include_router(workflow_runs.router, prefix="/workflow-runs") app.include_router(agent.router) app.include_router(llm.router, prefix="/llm") +app.include_router(image.router, prefix="/image") # Serve generated files from workspace — dynamic so path changes take effect immediately @app.get("/workspace/{full_path:path}") diff --git a/api/routers/image.py b/api/routers/image.py new file mode 100644 index 000000000..f440032ba --- /dev/null +++ b/api/routers/image.py @@ -0,0 +1,237 @@ +""" +Image operations (background removal, ...) on local files. + +Ops may need things installed on first use (a runtime, model weights): +`GET /image/op/{op}/status` says what is missing, `POST /image/op/{op}/prepare` +installs it while streaming progress in the same SSE format as the LLM +downloads (src/shared/services/llmDownloads.ts), and `POST /image/op/{op}` +runs the op once everything is there. +""" +import asyncio +import json +import threading +from pathlib import Path +from typing import Any, Optional + +from fastapi import APIRouter, HTTPException +from fastapi.responses import StreamingResponse +from pydantic import BaseModel, Field +from starlette.concurrency import run_in_threadpool + +import services.generator_registry as registry +from services.image_ops import ( + ImageOp, + ImageOpContext, + ImageOpExecutionError, + ImageOpInputError, + ImageOpNotFoundError, + ImageOpNotReadyError, + image_ops_registry, +) + +router = APIRouter(tags=["image"]) + +IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"} + + +class ImageOpRequest(BaseModel): + path: str + params: dict[str, Any] = Field(default_factory=dict) + + +def _fmt(data: dict) -> str: + return f"data: {json.dumps(data)}\n\n" + + +def _get_op(op_name: str) -> ImageOp: + try: + return image_ops_registry.get(op_name) + except ImageOpNotFoundError as exc: + raise HTTPException(404, f"Unknown image operation: {op_name}") from exc + + +def _missing_bytes(op: ImageOp) -> Optional[int]: + sizes = [req.download_bytes() for req in op.requirements if not req.is_ready()] + known = [size for size in sizes if size is not None] + return sum(known) if known else None + + +# ─── Catalog / status ───────────────────────────────────────────────────────── + +@router.get("/ops") +def list_image_operations(): + return {"ops": image_ops_registry.describe()} + + +@router.get("/op/{op_name}/status") +def image_operation_status(op_name: str): + op = _get_op(op_name) + missing = op.missing() + state = _prepares.get(op_name) + return { + "ready": not missing, + "missing": missing, + "download_bytes": _missing_bytes(op) if missing else None, + "preparing": bool(state and state.task and not state.task.done()), + } + + +# ─── Prepare (SSE) ──────────────────────────────────────────────────────────── + +class _PrepareState: + """One op's in-flight prepare, independent of any SSE connection, so a + second caller (or a reopened panel) attaches instead of starting a second + pip install / download racing the first.""" + + def __init__(self) -> None: + self.last_msg: dict = {} + self.subscribers: list[asyncio.Queue] = [] + self.task: Optional[asyncio.Task] = None + + +_prepares: dict[str, _PrepareState] = {} +# Two ops sharing a requirement (same runtime) must not install it twice at once. +_install_locks: dict[str, threading.Lock] = {} + + +def _is_terminal(msg: dict) -> bool: + return msg.get("status") == "done" or "error" in msg + + +async def _run_prepare(op: ImageOp, state: _PrepareState) -> None: + loop = asyncio.get_running_loop() + + def _broadcast(msg: dict) -> None: + def _do() -> None: + state.last_msg = msg + for queue in list(state.subscribers): + queue.put_nowait(msg) + loop.call_soon_threadsafe(_do) + + try: + pending = [req for req in op.requirements if not req.is_ready()] + # Weight each step by its download size so the bar is roughly linear in time. + weights = [max(1, req.download_bytes() or 1) for req in pending] + total = sum(weights) or 1 + done_weight = 0 + _broadcast({"percent": 0, "status": "Starting…"}) + + for req, weight in zip(pending, weights): + def _progress(msg: dict, _base: int = done_weight, _weight: int = weight, _kind: str = req.kind) -> None: + step_pct = max(0, min(100, msg.get("percent") or 0)) + overall = min(99, round((_base + _weight * step_pct / 100) / total * 100)) + _broadcast({**msg, "percent": overall, "step": _kind}) + + lock = _install_locks.setdefault(req.id, threading.Lock()) + + def _install(req=req, lock=lock, cb=_progress) -> None: + with lock: + req.install(cb) + + await loop.run_in_executor(None, _install) + done_weight += weight + + _broadcast({"percent": 100, "status": "done"}) + except Exception as exc: + _broadcast({"error": str(exc)}) + + +@router.api_route("/op/{op_name}/prepare", methods=["POST", "GET"]) +async def prepare_image_operation(op_name: str): + """Install what the op is missing, streaming progress. Idempotent: an op + that is already ready gets a single `done` frame; a call while a prepare is + running joins it. GET is accepted too so the renderer's existing + `consumeSse` (a plain fetch) can be reused unchanged.""" + op = _get_op(op_name) + state = _prepares.get(op_name) + running = state is not None and state.task is not None and not state.task.done() + + if not running and not op.missing(): + async def already(): + yield _fmt({"percent": 100, "status": "done"}) + return StreamingResponse(already(), media_type="text/event-stream") + + if not running: + state = _PrepareState() + _prepares[op_name] = state + state.task = asyncio.create_task(_run_prepare(op, state)) + + queue: asyncio.Queue[dict] = asyncio.Queue() + state.subscribers.append(queue) + if state.last_msg: + queue.put_nowait(state.last_msg) # replay current progress on (re)connect + + async def stream(): + try: + while True: + msg = await queue.get() + yield _fmt(msg) + if _is_terminal(msg): + break + finally: + if queue in state.subscribers: + state.subscribers.remove(queue) + + return StreamingResponse(stream(), media_type="text/event-stream") + + +# ─── Run ────────────────────────────────────────────────────────────────────── + +def _resolve_image_path(raw_path: str) -> Path: + if not raw_path or not raw_path.strip(): + raise HTTPException(400, "path is required") + candidate = Path(raw_path) + if candidate.is_absolute(): + resolved = candidate.resolve() + else: + resolved = (registry.WORKSPACE_DIR / raw_path).resolve() + if not registry.is_within_workspace(resolved): + raise HTTPException(400, "Invalid path") + if resolved.suffix.lower() not in IMAGE_EXTENSIONS: + raise HTTPException(400, f"Unsupported image type {resolved.suffix or '(none)'}: use png, jpg, jpeg or webp") + if not resolved.is_file(): + raise HTTPException(404, f"File not found: {raw_path}") + return resolved + + +def _workspace_url(path: Path) -> Optional[str]: + try: + relative = path.resolve().relative_to(registry.WORKSPACE_DIR.resolve()).as_posix() + except ValueError: + return None + return f"/workspace/{relative}" + + +@router.post("/op/{op_name}") +async def run_image_operation(op_name: str, body: ImageOpRequest): + op = _get_op(op_name) + input_path = _resolve_image_path(body.path) + missing = op.missing() + if missing: + raise HTTPException(409, {"code": "not_ready", "missing": missing}) + + context = ImageOpContext(workspace_dir=registry.WORKSPACE_DIR) + try: + # Inference takes seconds on CPU: keep the event loop free. + result = await run_in_threadpool(image_ops_registry.run, op_name, input_path, body.params, context) + except ImageOpNotReadyError as exc: + raise HTTPException(409, {"code": "not_ready", "missing": exc.missing}) from exc + except FileNotFoundError as exc: + raise HTTPException(404, str(exc)) from exc + except (ImageOpInputError, TypeError, ValueError) as exc: + raise HTTPException(400, str(exc)) from exc + except ImageOpExecutionError as exc: + raise HTTPException(500, str(exc)) from exc + + response: dict[str, Any] = { + "path": str(result.path.resolve()), + "mask_path": str(result.mask_path.resolve()) if result.mask_path else None, + } + url = _workspace_url(result.path) + if url: + response["url"] = url + if result.mask_path is not None: + mask_url = _workspace_url(result.mask_path) + if mask_url: + response["mask_url"] = mask_url + return response diff --git a/api/services/image_ops/__init__.py b/api/services/image_ops/__init__.py new file mode 100644 index 000000000..0be994d43 --- /dev/null +++ b/api/services/image_ops/__init__.py @@ -0,0 +1,30 @@ +"""Image operation registry (background removal, ...) used by the /image API.""" + +from .builtin import BUILTIN_IMAGE_OPS +from .registry import ImageOpsRegistry +from .types import ( + ImageOp, + ImageOpContext, + ImageOpExecutionError, + ImageOpInputError, + ImageOpNotFoundError, + ImageOpNotReadyError, + ImageOpResult, + Requirement, +) + + +image_ops_registry = ImageOpsRegistry(BUILTIN_IMAGE_OPS) + +__all__ = [ + "ImageOp", + "ImageOpContext", + "ImageOpExecutionError", + "ImageOpInputError", + "ImageOpNotFoundError", + "ImageOpNotReadyError", + "ImageOpResult", + "ImageOpsRegistry", + "Requirement", + "image_ops_registry", +] diff --git a/api/services/image_ops/birefnet.py b/api/services/image_ops/birefnet.py new file mode 100644 index 000000000..da3ca87b1 --- /dev/null +++ b/api/services/image_ops/birefnet.py @@ -0,0 +1,90 @@ +"""BiRefNet-lite background matting on CPU through onnxruntime. + +onnxruntime is installed on first use (see requirements.py), so it is only +imported inside `_create_onnx_session`, never at module import. +Pre/post-processing follows the model's preprocessor_config.json: +1024x1024 bilinear resize, x/255, ImageNet mean/std; the output is logits. +""" + +import threading +from pathlib import Path +from typing import Any, Callable, Optional + +import numpy as np +from PIL import Image + +INPUT_SIZE = 1024 +_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32) +_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32) + +SessionFactory = Callable[[Path], Any] + + +def _create_onnx_session(model_path: Path) -> Any: + import onnxruntime as ort # lazy: installed on demand + + options = ort.SessionOptions() + options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + return ort.InferenceSession(str(model_path), sess_options=options, providers=["CPUExecutionProvider"]) + + +class SessionCache: + """One InferenceSession per model file, built on first use and reused. + Tests inject a fake factory so they never need onnxruntime or the model.""" + + def __init__(self, factory: SessionFactory = _create_onnx_session) -> None: + self._factory = factory + self._lock = threading.Lock() + self._key: Optional[tuple[str, int]] = None + self._session: Any = None + + def get(self, model_path: Path) -> Any: + stat = model_path.stat() + key = (str(model_path.resolve()), stat.st_mtime_ns) + with self._lock: + if self._session is None or self._key != key: + self._session = self._factory(model_path) + self._key = key + return self._session + + def clear(self) -> None: + with self._lock: + self._session = None + self._key = None + + +session_cache = SessionCache() + + +def preprocess(rgb: Image.Image) -> np.ndarray: + """RGB PIL image -> float32 NCHW tensor (1, 3, 1024, 1024).""" + resized = rgb.resize((INPUT_SIZE, INPUT_SIZE), Image.Resampling.BILINEAR) + array = np.asarray(resized, dtype=np.float32) / 255.0 + array = (array - _MEAN) / _STD + return np.ascontiguousarray(array.transpose(2, 0, 1)[None, ...], dtype=np.float32) + + +def _sigmoid(x: np.ndarray) -> np.ndarray: + return 1.0 / (1.0 + np.exp(-np.clip(x, -60.0, 60.0))) + + +def predict_alpha(session: Any, rgb: Image.Image) -> np.ndarray: + """Soft foreground alpha in [0, 1], float32, at the image's own size (H, W).""" + input_name = session.get_inputs()[0].name + outputs = session.run(None, {input_name: preprocess(rgb)}) + logits = np.asarray(outputs[0], dtype=np.float32) + matte = _sigmoid(logits.reshape(logits.shape[-2], logits.shape[-1])) + resized = Image.fromarray(matte.astype(np.float32)).resize(rgb.size, Image.Resampling.BILINEAR) + return np.clip(np.asarray(resized, dtype=np.float32), 0.0, 1.0) + + +def apply_edge_threshold(alpha: np.ndarray, threshold: float) -> np.ndarray: + """0 keeps the soft alpha; t > 0 hardens: clamp((a - t) / (1 - t), 0, 1).""" + if threshold <= 0: + return np.clip(alpha, 0.0, 1.0) + t = min(float(threshold), 0.999) + return np.clip((alpha - t) / (1.0 - t), 0.0, 1.0) + + +def to_uint8(alpha: np.ndarray) -> np.ndarray: + return np.clip(np.rint(alpha * 255.0), 0, 255).astype(np.uint8) diff --git a/api/services/image_ops/builtin.py b/api/services/image_ops/builtin.py new file mode 100644 index 000000000..d603e6011 --- /dev/null +++ b/api/services/image_ops/builtin.py @@ -0,0 +1,88 @@ +"""Built-in image operations.""" + +from pathlib import Path +from typing import Any, Mapping + +import numpy as np +from PIL import Image, ImageOps, UnidentifiedImageError + +from . import birefnet +from .outputs import png_bytes, sanitize_stem, write_new_files +from .requirements import BIREFNET_LITE, ONNXRUNTIME +from .types import ImageOp, ImageOpContext, ImageOpExecutionError, ImageOpInputError, ImageOpResult + + +def load_image(path: Path) -> Image.Image: + """Open an image upright (EXIF orientation applied), fully decoded.""" + try: + with Image.open(path) as opened: + image = ImageOps.exif_transpose(opened) + image.load() + return image + except (UnidentifiedImageError, OSError, Image.DecompressionBombError) as exc: + raise ImageOpInputError(f"Cannot read image {path.name}: {exc}") from exc + + +def _source_alpha(image: Image.Image) -> np.ndarray | None: + """The input's own transparency in [0, 1], if it has any.""" + if image.mode in ("RGBA", "LA", "PA") or (image.mode == "P" and "transparency" in image.info): + return np.asarray(image.convert("RGBA").getchannel("A"), dtype=np.float32) / 255.0 + return None + + +def remove_background(input_path: Path, params: Mapping[str, Any], context: ImageOpContext) -> ImageOpResult: + image = load_image(input_path) + rgb = image.convert("RGB") + + try: + session = birefnet.session_cache.get(BIREFNET_LITE.path) + alpha = birefnet.predict_alpha(session, rgb) + except Exception as exc: + raise ImageOpExecutionError(f"Background removal failed: {exc}") from exc + + alpha = birefnet.apply_edge_threshold(alpha, float(params.get("edge_threshold", 0.0))) + source_alpha = _source_alpha(image) + if source_alpha is not None: + # Already-transparent pixels stay transparent. + alpha = alpha * source_alpha + mask = Image.fromarray(birefnet.to_uint8(alpha)) # 2D uint8 -> mode "L" + + cutout = rgb.copy() + cutout.putalpha(mask) + + out_path, mask_path = write_new_files( + context.images_dir, + sanitize_stem(input_path.name), + [("nobg", png_bytes(cutout)), ("mask", png_bytes(mask))], + ) + return ImageOpResult( + path=out_path, + mask_path=mask_path, + details={"width": rgb.width, "height": rgb.height}, + ) + + +REMOVE_BACKGROUND = ImageOp( + id="remove_background", + label="Remove background", + params_schema=( + { + "id": "edge_threshold", + "label": "Edge hardness", + "type": "float", + "default": 0.0, + "min": 0.0, + "max": 0.95, + "step": 0.05, + "tooltip": ( + "0 keeps soft, natural edges (hair, fur, blur). " + "Raise it to get crisper edges and drop faint leftovers of the background; " + "too high can eat into thin details of the subject." + ), + }, + ), + fn=remove_background, + requirements=(ONNXRUNTIME, BIREFNET_LITE), +) + +BUILTIN_IMAGE_OPS = (REMOVE_BACKGROUND,) diff --git a/api/services/image_ops/outputs.py b/api/services/image_ops/outputs.py new file mode 100644 index 000000000..870f7057c --- /dev/null +++ b/api/services/image_ops/outputs.py @@ -0,0 +1,72 @@ +"""Output file naming for image ops, matching electron/main/workspace-images.ts: +`--YYYYMMDD-HHMMSS-.png`, sortable by time, never overwriting.""" + +import io +import re +import secrets +import unicodedata +from datetime import datetime +from pathlib import Path +from typing import Callable, Optional, Sequence + +_WINDOWS_RESERVED = re.compile(r"^(con|prn|aux|nul|com[0-9]|lpt[0-9])$", re.IGNORECASE) +_MAX_STEM = 60 + + +def sanitize_stem(name: str) -> str: + """Filesystem-safe stem: only [A-Za-z0-9_-], no reserved Windows name, else "image".""" + leaf = re.split(r"[\\/]", name or "")[-1] + stem = Path(leaf).stem or leaf + ascii_only = unicodedata.normalize("NFKD", stem).encode("ascii", "ignore").decode("ascii") + safe = re.sub(r"[^A-Za-z0-9_-]+", "-", ascii_only) + safe = re.sub(r"-{2,}", "-", safe).strip("-_")[:_MAX_STEM].rstrip("-_") + if not safe or _WINDOWS_RESERVED.match(safe): + return "image" + return safe + + +def output_name(stem: str, tag: str, now: datetime, suffix: str) -> str: + return f"{stem}-{tag}-{now:%Y%m%d-%H%M%S}-{suffix}.png" + + +def write_new_files( + directory: Path, + stem: str, + payloads: Sequence[tuple[str, bytes]], + now: Optional[datetime] = None, + token: Callable[[], str] = lambda: secrets.token_hex(3), + attempts: int = 20, +) -> list[Path]: + """Write each (tag, bytes) as a new file sharing one timestamp and suffix. + + Files are created exclusively ("xb"), so an existing file is never + overwritten: on a name clash every file of the set is removed and the set + is retried with a fresh suffix. + """ + directory.mkdir(parents=True, exist_ok=True) + stamp = now or datetime.now() + for _ in range(attempts): + suffix = token() + written: list[Path] = [] + try: + for tag, data in payloads: + path = directory / output_name(stem, tag, stamp, suffix) + with open(path, "xb") as handle: + written.append(path) + handle.write(data) + return written + except FileExistsError: + for path in written: + path.unlink(missing_ok=True) + continue + except BaseException: + for path in written: + path.unlink(missing_ok=True) + raise + raise FileExistsError(f"Could not find a free output name in {directory}") + + +def png_bytes(image) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() diff --git a/api/services/image_ops/registry.py b/api/services/image_ops/registry.py new file mode 100644 index 000000000..b5bf873e5 --- /dev/null +++ b/api/services/image_ops/registry.py @@ -0,0 +1,125 @@ +"""Registry and dispatcher for image operations. + +Param handling follows services.mesh_ops.registry (same schema format: defaults, +then type checks and clamping to min/max). It is not imported from there: that +package pulls in trimesh and the whole mesh stack at import. +""" + +import re +from copy import deepcopy +from math import isfinite +from pathlib import Path +from typing import Any, Iterable, Iterator, Mapping, Optional + +from .types import ( + ImageOp, + ImageOpContext, + ImageOpNotFoundError, + ImageOpNotReadyError, + ImageOpResult, +) + + +_OP_ID = re.compile(r"^[a-z][a-z0-9_-]*$") + + +def _validate_param(parameter_id: str, value: Any, schema: Mapping[str, Any]) -> Any: + """Validate a supplied param against its schema; clamp numbers to min/max.""" + param_type = schema.get("type") + + if param_type == "boolean": + if not isinstance(value, bool): + raise TypeError(f"Parameter {parameter_id!r} must be a boolean") + return value + + if param_type == "select": + allowed = {option["value"] for option in schema.get("options", [])} + if allowed and value not in allowed: + raise ValueError(f"Parameter {parameter_id!r} must be one of {sorted(allowed)}") + return value + + if param_type in ("int", "float"): + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"Parameter {parameter_id!r} must be numeric") + if isinstance(value, float) and not isfinite(value): + raise ValueError(f"Parameter {parameter_id!r} must be finite") + if param_type == "int": + if isinstance(value, float) and not value.is_integer(): + raise TypeError(f"Parameter {parameter_id!r} must be an integer") + value = int(value) + else: + value = float(value) + if schema.get("min") is not None: + value = max(schema["min"], value) + if schema.get("max") is not None: + value = min(schema["max"], value) + return value + + return value + + +class ImageOpsRegistry: + def __init__(self, operations: Iterable[ImageOp] = ()) -> None: + self._operations: dict[str, ImageOp] = {} + for operation in operations: + self.register(operation) + + def register(self, operation: ImageOp) -> None: + if not _OP_ID.fullmatch(operation.id): + raise ValueError(f"Invalid image operation id: {operation.id!r}") + if operation.id in self._operations: + raise ValueError(f"Duplicate image operation id: {operation.id!r}") + self._operations[operation.id] = operation + + def get(self, operation_id: str) -> ImageOp: + try: + return self._operations[operation_id] + except KeyError as exc: + raise ImageOpNotFoundError(operation_id) from exc + + def __iter__(self) -> Iterator[ImageOp]: + return iter(self._operations.values()) + + def describe(self) -> list[dict[str, Any]]: + return [operation.describe() for operation in self._operations.values()] + + def resolve_params( + self, + operation_id: str, + params: Optional[Mapping[str, Any]], + ) -> dict[str, Any]: + """Apply the schema defaults, then validate and clamp. Unknown keys are rejected.""" + operation = self.get(operation_id) + known = {schema["id"] for schema in operation.params_schema if "id" in schema} + unknown = sorted(set(params or {}) - known) + if unknown: + raise ValueError(f"Unknown parameter(s) for {operation_id}: {', '.join(unknown)}") + + resolved = { + schema["id"]: deepcopy(schema["default"]) + for schema in operation.params_schema + if "id" in schema and "default" in schema + } + resolved.update(params or {}) + for schema in operation.params_schema: + parameter_id = schema.get("id") + if parameter_id in resolved: + resolved[parameter_id] = _validate_param(parameter_id, resolved[parameter_id], schema) + return resolved + + def run( + self, + operation_id: str, + input_path: Path, + params: Optional[Mapping[str, Any]], + context: ImageOpContext, + ) -> ImageOpResult: + operation = self.get(operation_id) + resolved = self.resolve_params(operation_id, params) + missing = operation.missing() + if missing: + raise ImageOpNotReadyError(operation_id, missing) + path = Path(input_path) + if not path.is_file(): + raise FileNotFoundError(f"Input image not found: {path}") + return operation.fn(path, resolved, context) diff --git a/api/services/image_ops/requirements.py b/api/services/image_ops/requirements.py new file mode 100644 index 000000000..5a285f4d1 --- /dev/null +++ b/api/services/image_ops/requirements.py @@ -0,0 +1,248 @@ +"""Things image ops install on first use: a pip package, a model file. + +Neither is in api/requirements.txt on purpose: changing that file sends every +user through a full venv reinstall (electron/main/python-setup.ts), for a +feature many will never open. They are fetched the first time the op is +prepared, into the backend's own environment and MODELS_DIR. +""" + +import hashlib +import importlib +import importlib.util +import os +import subprocess +import sys +import threading +from collections import deque +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Optional + +from .types import ImageOpExecutionError, ProgressCallback + + +def _models_dir() -> Path: + # Read at call time: POST /settings/paths rebinds it when the user moves it. + import services.generator_registry as registry + return registry.MODELS_DIR + + +def _hf_offline() -> bool: + return os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in ("1", "true", "yes", "on") + + +# ─── pip package ────────────────────────────────────────────────────────────── + +@dataclass(frozen=True) +class PipPackageRequirement: + """A Python package pip-installed into the running interpreter's environment.""" + + id: str # pip / import name, e.g. "onnxruntime" + label: str + pip_spec: str # e.g. "onnxruntime==1.23.2" + approx_bytes: int # wheel + dependencies, for the status estimate + kind: str = "runtime" + module: Optional[str] = None # import name when it differs from `id` + + @property + def module_name(self) -> str: + return self.module or self.id + + def is_ready(self) -> bool: + return importlib.util.find_spec(self.module_name) is not None + + def download_bytes(self) -> Optional[int]: + return None if self.is_ready() else self.approx_bytes + + def install(self, progress: ProgressCallback) -> None: + if self.is_ready(): + progress({"percent": 100, "status": f"{self.label} already installed"}) + return + + progress({"percent": 0, "status": f"Installing {self.label}…"}) + command = [ + sys.executable, "-m", "pip", "install", self.pip_spec, + "--disable-pip-version-check", "--no-input", + "--no-warn-script-location", "--progress-bar", "off", + ] + creationflags = getattr(subprocess, "CREATE_NO_WINDOW", 0) if sys.platform == "win32" else 0 + try: + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, + text=True, + encoding="utf-8", + errors="replace", + creationflags=creationflags, + ) + except OSError as exc: + raise ImageOpExecutionError(f"Could not start pip to install {self.label}: {exc}") from exc + + # pip prints no byte counts with the progress bar off: advance on its + # phase lines so the bar moves, and keep the tail for the error message. + tail: deque[str] = deque(maxlen=12) + percent = 0 + assert process.stdout is not None + for raw in process.stdout: + line = raw.strip() + if not line: + continue + tail.append(line) + if line.startswith("Collecting "): + percent = min(60, percent + 8) + elif line.startswith("Downloading "): + percent = min(70, percent + 4) + elif line.startswith("Installing collected packages"): + percent = 80 + else: + continue + progress({"percent": percent, "status": f"Installing {self.label}… {line[:80]}"}) + code = process.wait() + + if code != 0: + detail = " | ".join(line for line in tail if "error" in line.lower()) or " | ".join(list(tail)[-3:]) + hint = "" + if any(token in detail.lower() for token in ("connection", "network", "resolve", "proxy", "timed out", "no matching distribution")): + hint = " Check your internet connection and try again." + raise ImageOpExecutionError( + f"Installing {self.label} failed (pip exited with code {code}).{hint} {detail}".strip() + ) + + importlib.invalidate_caches() + if not self.is_ready(): + raise ImageOpExecutionError(f"{self.label} was installed but cannot be found by Python") + try: + importlib.import_module(self.module_name) + except Exception as exc: # broken wheel, missing system library… + raise ImageOpExecutionError(f"{self.label} was installed but fails to load: {exc}") from exc + progress({"percent": 100, "status": f"{self.label} installed"}) + + +# ─── Hugging Face model file ────────────────────────────────────────────────── + +@dataclass(frozen=True) +class HFFileRequirement: + """One model file downloaded from a pinned Hugging Face revision into + //, verified by size and SHA-256.""" + + id: str # also the folder name under MODELS_DIR + label: str + repo_id: str + filename: str # path inside the repo + revision: str # pinned commit + size: int + sha256: str + local_name: str = "model.onnx" + kind: str = "model" + + @property + def path(self) -> Path: + return _models_dir() / self.id / self.local_name + + def is_ready(self) -> bool: + try: + return self.path.is_file() and self.path.stat().st_size == self.size + except OSError: + return False + + def download_bytes(self) -> Optional[int]: + """Bytes left to fetch; a resumable .part file counts as already fetched.""" + if self.is_ready(): + return None + part = self.path.with_suffix(self.path.suffix + ".part") + try: + done = part.stat().st_size if part.is_file() else 0 + except OSError: + done = 0 + return max(0, self.size - done) + + def install(self, progress: ProgressCallback) -> None: + if self.is_ready(): + progress({"percent": 100, "status": f"{self.label} already downloaded"}) + return + if _hf_offline(): + raise ImageOpExecutionError( + f"{self.label} is not downloaded and HF_HUB_OFFLINE is set; go online once to fetch it" + ) + + from huggingface_hub import hf_hub_url + from routers.model import _download_file_streamed + + target = self.path + # A complete-looking file of the wrong size (interrupted copy, older + # revision) would be returned as-is by the downloader: start clean. + if target.exists(): + target.unlink() + + def _on_progress(msg: dict) -> None: + done = msg.get("bytesDownloaded") or 0 + total = msg.get("totalBytes") or self.size + pct = min(99, round(done / total * 100)) if total else 0 + progress({ + "percent": pct, + "status": f"Downloading {self.label}… {pct}%" + if str(msg.get("status", "")).startswith(("Downloading", "Resuming")) + else str(msg.get("status") or f"Downloading {self.label}…"), + "bytesDownloaded": done, + "totalBytes": total, + }) + + control = {"pause": threading.Event(), "cancel": threading.Event()} + try: + _download_file_streamed( + url=hf_hub_url(repo_id=self.repo_id, filename=self.filename, revision=self.revision), + filename=self.local_name, + dest_dir=str(target.parent), + file_index=1, + total_files=1, + base_percent=0, + progress_cb=_on_progress, + control=control, + ) + except Exception as exc: + # The downloader wraps network failures (after its retries) in + # RuntimeError; a bare OSError here is local (disk full, permissions). + hint = "" if isinstance(exc, OSError) else " Check your internet connection and try again." + raise ImageOpExecutionError(f"Downloading {self.label} failed: {exc}.{hint}") from exc + + progress({"percent": 99, "status": f"Verifying {self.label}…"}) + digest = _sha256(target) + if target.stat().st_size != self.size or digest != self.sha256: + target.unlink(missing_ok=True) + raise ImageOpExecutionError(f"{self.label} download is corrupted (checksum mismatch); try again") + progress({"percent": 100, "status": f"{self.label} ready", "bytesDownloaded": self.size, "totalBytes": self.size}) + + +def _sha256(path: Path, chunk: int = 4 * 1024 * 1024) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(chunk), b""): + h.update(block) + return h.hexdigest() + + +# ─── The concrete requirements ──────────────────────────────────────────────── + +# 1.23.2 is the newest onnxruntime with CPU wheels for every platform Modly runs +# on: win_amd64, macOS arm64 *and* x86_64 (1.24 dropped Intel Macs), manylinux +# x86_64/aarch64, for CPython 3.10–3.13 including the bundled 3.11. +ONNXRUNTIME = PipPackageRequirement( + id="onnxruntime", + label="ONNX Runtime", + pip_spec="onnxruntime==1.23.2", + approx_bytes=25_000_000, +) + +# BiRefNet-lite (MIT), fp32 ONNX export. fp16 halves the download but CPU +# execution of fp16 graphs in onnxruntime is slower and less accurate. +BIREFNET_LITE = HFFileRequirement( + id="birefnet-lite", + label="BiRefNet-lite model", + repo_id="onnx-community/BiRefNet_lite-ONNX", + filename="onnx/model.onnx", + revision="de15b22ba131738a16dff04aab8bdf8dc32e3ac1", + size=224_005_088, + sha256="5600024376f572a557870a5eb0afb1e5961636bef4e1e22132025467d0f03333", +) diff --git a/api/services/image_ops/types.py b/api/services/image_ops/types.py new file mode 100644 index 000000000..026ff03e3 --- /dev/null +++ b/api/services/image_ops/types.py @@ -0,0 +1,101 @@ +"""Shared types for Modly image operations. + +Mirrors services.mesh_ops.types, plus what mesh ops never needed: an image op +can depend on things that are fetched on first use (a Python runtime, model +weights). Those are `Requirement`s; the router reports which are missing and +installs them on demand. +""" + +from copy import deepcopy +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Mapping, Optional, Protocol + + +# One progress frame, in the SSE format the renderer already consumes for LLM +# downloads (src/shared/services/llmDownloads.ts: percent, status, +# bytesDownloaded, totalBytes). Percent is 0..100 within the requirement. +ProgressCallback = Callable[[dict], None] + + +class ImageOpNotFoundError(LookupError): + """Raised when a caller requests an operation that is not registered.""" + + +class ImageOpNotReadyError(RuntimeError): + """Raised when an operation runs before its requirements are installed.""" + + def __init__(self, operation_id: str, missing: list[str]) -> None: + super().__init__(f"{operation_id} is not ready: missing {', '.join(missing)}") + self.operation_id = operation_id + self.missing = missing + + +class ImageOpInputError(ValueError): + """Raised when the input file is not an image the operation can read.""" + + +class ImageOpExecutionError(RuntimeError): + """Raised when an operation, or installing one of its requirements, fails.""" + + +class Requirement(Protocol): + """Something an op needs that is installed on first use.""" + + kind: str # "runtime" | "model" + id: str # e.g. "onnxruntime", "birefnet-lite" + label: str # shown in progress messages + + def is_ready(self) -> bool: ... + + def download_bytes(self) -> Optional[int]: + """Approximate bytes still to fetch (best effort), None when unknown.""" + + def install(self, progress: ProgressCallback) -> None: + """Fetch and install; idempotent. Raises ImageOpExecutionError on failure.""" + + +@dataclass(frozen=True) +class ImageOpContext: + """Where an operation writes its outputs (read from the registry at call time).""" + + workspace_dir: Path + + @property + def images_dir(self) -> Path: + # Same folder Electron saves pasted images to (electron/main/workspace-images.ts). + return self.workspace_dir / "Images" + + +@dataclass(frozen=True) +class ImageOpResult: + path: Path + mask_path: Optional[Path] = None + details: Mapping[str, Any] = field(default_factory=dict) + + +ImageOpFn = Callable[[Path, Mapping[str, Any], ImageOpContext], ImageOpResult] + + +@dataclass(frozen=True) +class ImageOp: + id: str + label: str + params_schema: tuple[Mapping[str, Any], ...] + fn: ImageOpFn + requirements: tuple[Requirement, ...] = () + + def missing(self) -> list[str]: + """Kinds of the requirements not installed yet, in install order.""" + return [req.kind for req in self.requirements if not req.is_ready()] + + def describe(self) -> dict[str, Any]: + return { + "id": self.id, + "label": self.label, + "params_schema": deepcopy(list(self.params_schema)), + "requirements": [ + {"kind": req.kind, "id": req.id, "label": req.label} + for req in self.requirements + ], + } diff --git a/api/tests/test_image_ops_registry.py b/api/tests/test_image_ops_registry.py new file mode 100644 index 000000000..d7874f4b9 --- /dev/null +++ b/api/tests/test_image_ops_registry.py @@ -0,0 +1,137 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from services.image_ops import ( + ImageOp, + ImageOpContext, + ImageOpNotFoundError, + ImageOpNotReadyError, + ImageOpResult, + ImageOpsRegistry, + image_ops_registry, +) + + +class FakeRequirement: + def __init__(self, kind: str, ready: bool, size: int | None = 10, id: str | None = None) -> None: + self.kind = kind + self.id = id or f"fake-{kind}" + self.label = f"Fake {kind}" + self.ready = ready + self.size = size + self.installs = 0 + + def is_ready(self) -> bool: + return self.ready + + def download_bytes(self): + return None if self.ready else self.size + + def install(self, progress) -> None: + self.installs += 1 + progress({"percent": 50, "status": "half"}) + self.ready = True + progress({"percent": 100, "status": f"{self.label} ready"}) + + +SCHEMA = ( + {"id": "strength", "label": "Strength", "type": "float", "default": 0.0, "min": 0.0, "max": 0.95}, + {"id": "count", "label": "Count", "type": "int", "default": 2, "min": 1, "max": 5}, + {"id": "flag", "label": "Flag", "type": "boolean", "default": False}, +) + + +def make_op(requirements=(), fn=None) -> ImageOp: + return ImageOp( + id="example", + label="Example", + params_schema=SCHEMA, + fn=fn or (lambda path, params, ctx: ImageOpResult(path=path, details={"params": params})), + requirements=tuple(requirements), + ) + + +class ImageOpsRegistryTests(unittest.TestCase): + def test_builtin_metadata_is_serializable(self) -> None: + described = image_ops_registry.describe() + self.assertEqual([d["id"] for d in described], ["remove_background"]) + op = described[0] + self.assertEqual(op["label"], "Remove background") + self.assertEqual( + [(r["kind"], r["id"]) for r in op["requirements"]], + [("runtime", "onnxruntime"), ("model", "birefnet-lite")], + ) + (param,) = op["params_schema"] + self.assertEqual( + {k: param[k] for k in ("id", "type", "default", "min", "max")}, + {"id": "edge_threshold", "type": "float", "default": 0.0, "min": 0.0, "max": 0.95}, + ) + self.assertTrue(param["tooltip"]) + json.dumps(described) + + def test_defaults_are_applied_without_mutating_schema(self) -> None: + registry = ImageOpsRegistry([make_op()]) + self.assertEqual(registry.resolve_params("example", None), {"strength": 0.0, "count": 2, "flag": False}) + resolved = registry.resolve_params("example", {"strength": 0.3}) + self.assertEqual(resolved["strength"], 0.3) + self.assertEqual(SCHEMA[0]["default"], 0.0) + + def test_numbers_are_clamped_and_coerced(self) -> None: + registry = ImageOpsRegistry([make_op()]) + self.assertEqual(registry.resolve_params("example", {"strength": 2})["strength"], 0.95) + self.assertIsInstance(registry.resolve_params("example", {"strength": 1})["strength"], float) + self.assertEqual(registry.resolve_params("example", {"strength": -1.0})["strength"], 0.0) + self.assertEqual(registry.resolve_params("example", {"count": 9.0})["count"], 5) + + def test_invalid_params_are_rejected(self) -> None: + registry = ImageOpsRegistry([make_op()]) + for bad in ({"strength": "high"}, {"strength": True}, {"count": 1.5}, {"flag": 1}): + with self.subTest(bad=bad), self.assertRaises(TypeError): + registry.resolve_params("example", bad) + with self.assertRaises(ValueError): + registry.resolve_params("example", {"strength": float("nan")}) + with self.assertRaisesRegex(ValueError, "Unknown parameter"): + registry.resolve_params("example", {"nope": 1}) + + def test_ids_are_validated_and_unique(self) -> None: + registry = ImageOpsRegistry([make_op()]) + with self.assertRaises(ValueError): + registry.register(make_op()) + with self.assertRaises(ValueError): + ImageOpsRegistry([ImageOp(id="Bad Id", label="x", params_schema=(), fn=lambda *a: None)]) + with self.assertRaises(ImageOpNotFoundError): + registry.get("missing") + + def test_missing_lists_unready_requirement_kinds_in_order(self) -> None: + op = make_op([FakeRequirement("runtime", False), FakeRequirement("model", True)]) + self.assertEqual(op.missing(), ["runtime"]) + op = make_op([FakeRequirement("runtime", False), FakeRequirement("model", False)]) + self.assertEqual(op.missing(), ["runtime", "model"]) + + def test_run_refuses_when_not_ready(self) -> None: + calls = [] + op = make_op([FakeRequirement("model", False)], fn=lambda *a: calls.append(a)) + registry = ImageOpsRegistry([op]) + with tempfile.TemporaryDirectory() as tmp: + image = Path(tmp) / "a.png" + image.write_bytes(b"x") + with self.assertRaises(ImageOpNotReadyError) as caught: + registry.run("example", image, {}, ImageOpContext(Path(tmp))) + self.assertEqual(caught.exception.missing, ["model"]) + self.assertEqual(calls, []) + + def test_run_passes_resolved_params(self) -> None: + registry = ImageOpsRegistry([make_op([FakeRequirement("model", True)])]) + with tempfile.TemporaryDirectory() as tmp: + image = Path(tmp) / "a.png" + image.write_bytes(b"x") + result = registry.run("example", image, {"count": 3}, ImageOpContext(Path(tmp))) + self.assertEqual(result.details["params"], {"strength": 0.0, "count": 3, "flag": False}) + with self.assertRaises(FileNotFoundError): + registry.run("example", Path(tmp) / "missing.png", {}, ImageOpContext(Path(tmp))) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_image_ops_remove_background.py b/api/tests/test_image_ops_remove_background.py new file mode 100644 index 000000000..d709842f1 --- /dev/null +++ b/api/tests/test_image_ops_remove_background.py @@ -0,0 +1,253 @@ +import importlib.util +import os +import tempfile +import time +import unittest +from datetime import datetime +from pathlib import Path +from unittest.mock import patch + +import numpy as np +from PIL import Image + +import services.generator_registry as registry +from services.image_ops import ImageOpContext, ImageOpInputError, image_ops_registry +from services.image_ops import birefnet, builtin, outputs +from services.image_ops.requirements import BIREFNET_LITE + +HAS_ONNXRUNTIME = importlib.util.find_spec("onnxruntime") is not None + + +def _open(path) -> Image.Image: + with Image.open(path) as image: + image.load() + return image + + +class _Input: + def __init__(self, name: str) -> None: + self.name = name + + +class FakeSession: + """Stands in for an onnxruntime InferenceSession: the left half of the + frame is foreground (logit +8), the right half background (logit -8).""" + + def __init__(self) -> None: + self.feeds = [] + + def get_inputs(self): + return [_Input("input_image")] + + def run(self, output_names, feeds): + self.feeds.append(feeds) + logits = np.full((1, 1, birefnet.INPUT_SIZE, birefnet.INPUT_SIZE), -8.0, dtype=np.float32) + logits[..., : birefnet.INPUT_SIZE // 2] = 8.0 + return [logits] + + +class AlphaMathTests(unittest.TestCase): + def test_zero_threshold_keeps_soft_alpha(self) -> None: + alpha = np.array([[0.0, 0.25, 0.5, 1.0]], dtype=np.float32) + np.testing.assert_allclose(birefnet.apply_edge_threshold(alpha, 0.0), alpha) + + def test_threshold_hardens(self) -> None: + alpha = np.array([[0.0, 0.25, 0.5, 0.75, 1.0]], dtype=np.float32) + np.testing.assert_allclose(birefnet.apply_edge_threshold(alpha, 0.5), [[0.0, 0.0, 0.0, 0.5, 1.0]]) + np.testing.assert_allclose( + birefnet.apply_edge_threshold(alpha, 0.2), np.clip((alpha - 0.2) / 0.8, 0, 1), rtol=1e-6 + ) + + def test_to_uint8_rounds_and_clamps(self) -> None: + self.assertEqual(birefnet.to_uint8(np.array([-1.0, 0.0, 0.5, 1.0, 2.0])).tolist(), [0, 0, 128, 255, 255]) + + def test_preprocess_normalizes_to_nchw(self) -> None: + tensor = birefnet.preprocess(Image.new("RGB", (8, 6), (255, 0, 128))) + self.assertEqual(tensor.shape, (1, 3, 1024, 1024)) + self.assertEqual(tensor.dtype, np.float32) + np.testing.assert_allclose(tensor[0, :, 0, 0], [(1 - 0.485) / 0.229, (0 - 0.456) / 0.224, (128 / 255 - 0.406) / 0.225], rtol=1e-5) + + def test_predict_alpha_applies_sigmoid_and_resizes_back(self) -> None: + alpha = birefnet.predict_alpha(FakeSession(), Image.new("RGB", (40, 10))) + self.assertEqual(alpha.shape, (10, 40)) + self.assertGreater(alpha[:, :19].min(), 0.99) + self.assertLess(alpha[:, 21:].max(), 0.01) + + +class OutputNamingTests(unittest.TestCase): + def test_sanitize_stem(self) -> None: + self.assertEqual(outputs.sanitize_stem("My Photo (1).JPG"), "My-Photo-1") + self.assertEqual(outputs.sanitize_stem("Éclair.png"), "Eclair") + self.assertEqual(outputs.sanitize_stem("con.png"), "image") + self.assertEqual(outputs.sanitize_stem("!!!.png"), "image") + self.assertEqual(len(outputs.sanitize_stem("x" * 200 + ".png")), 60) + + def test_names_share_stamp_and_suffix(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + now = datetime(2026, 1, 2, 3, 4, 5) + paths = outputs.write_new_files(Path(tmp) / "Images", "cat", [("nobg", b"a"), ("mask", b"b")], now=now, token=lambda: "abcdef") + self.assertEqual([p.name for p in paths], ["cat-nobg-20260102-030405-abcdef.png", "cat-mask-20260102-030405-abcdef.png"]) + self.assertEqual(paths[0].read_bytes(), b"a") + + def test_never_overwrites(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + images = Path(tmp) + now = datetime(2026, 1, 2, 3, 4, 5) + # Only the mask name of the first suffix is taken: the set moves on as a pair. + taken = images / "cat-mask-20260102-030405-aaaaaa.png" + taken.write_bytes(b"keep") + tokens = iter(["aaaaaa", "bbbbbb"]) + paths = outputs.write_new_files(images, "cat", [("nobg", b"a"), ("mask", b"b")], now=now, token=lambda: next(tokens)) + self.assertEqual([p.name for p in paths], ["cat-nobg-20260102-030405-bbbbbb.png", "cat-mask-20260102-030405-bbbbbb.png"]) + self.assertEqual(taken.read_bytes(), b"keep") + self.assertFalse((images / "cat-nobg-20260102-030405-aaaaaa.png").exists()) + + def test_gives_up_after_too_many_clashes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + now = datetime(2026, 1, 2, 3, 4, 5) + (Path(tmp) / "cat-nobg-20260102-030405-aaaaaa.png").write_bytes(b"x") + with self.assertRaises(FileExistsError): + outputs.write_new_files(Path(tmp), "cat", [("nobg", b"a")], now=now, token=lambda: "aaaaaa", attempts=3) + + +class SessionCacheTests(unittest.TestCase): + def test_session_is_built_once_and_rebuilt_when_the_file_changes(self) -> None: + built = [] + cache = birefnet.SessionCache(lambda path: built.append(path) or object()) + with tempfile.TemporaryDirectory() as tmp: + model = Path(tmp) / "model.onnx" + model.write_bytes(b"a") + first = cache.get(model) + self.assertIs(cache.get(model), first) + time.sleep(0.01) + model.write_bytes(b"bb") + os.utime(model, ns=(time.time_ns(), time.time_ns() + 10_000_000)) + self.assertIsNot(cache.get(model), first) + self.assertEqual(len(built), 2) + + +class RemoveBackgroundTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + root = Path(self._tmp.name) + self.workspace = root / "workspace" + self.models = root / "models" + self.session = FakeSession() + self._patches = [ + patch.object(registry, "WORKSPACE_DIR", self.workspace), + patch.object(registry, "MODELS_DIR", self.models), + patch.object(birefnet, "session_cache", birefnet.SessionCache(lambda path: self.session)), + ] + for p in self._patches: + p.start() + # The model "is downloaded" (right size), the runtime is reported present. + BIREFNET_LITE.path.parent.mkdir(parents=True) + with open(BIREFNET_LITE.path, "wb") as f: + f.truncate(BIREFNET_LITE.size) + self._patches.append(patch("importlib.util.find_spec", side_effect=lambda name, *a: object() if name == "onnxruntime" else _real_find_spec(name, *a))) + self._patches[-1].start() + self.context = ImageOpContext(workspace_dir=self.workspace) + + def tearDown(self) -> None: + for p in reversed(self._patches): + p.stop() + self._tmp.cleanup() + + def _photo(self, name: str = "photo.jpg", size=(64, 32), mode="RGB", **save) -> Path: + rng = np.random.default_rng(0) + pixels = rng.integers(0, 255, (size[1], size[0], 3), dtype=np.uint8) + image = Image.fromarray(pixels).convert(mode) + path = self.workspace / name + path.parent.mkdir(parents=True, exist_ok=True) + image.save(path, **save) + return path + + def test_end_to_end_with_fake_session(self) -> None: + photo = self._photo("photo.png") + result = image_ops_registry.run("remove_background", photo, {}, self.context) + + self.assertEqual(result.path.parent, self.workspace / "Images") + self.assertRegex(result.path.name, r"^photo-nobg-\d{8}-\d{6}-[0-9a-f]{6}\.png$") + self.assertRegex(result.mask_path.name, r"^photo-mask-\d{8}-\d{6}-[0-9a-f]{6}\.png$") + out = _open(result.path) + mask = _open(result.mask_path) + self.assertEqual((out.mode, out.size), ("RGBA", (64, 32))) + self.assertEqual((mask.mode, mask.size), ("L", (64, 32))) + alpha = np.asarray(out.getchannel("A")) + self.assertTrue((alpha[:, :30] == 255).all()) + self.assertTrue((alpha[:, 34:] == 0).all()) + np.testing.assert_array_equal(np.asarray(mask), alpha) + # Original pixels are kept untouched under the alpha. + np.testing.assert_array_equal(np.asarray(out.convert("RGB")), np.asarray(_open(photo).convert("RGB"))) + self.assertEqual(self.session.feeds[0]["input_image"].shape, (1, 3, 1024, 1024)) + + def test_two_runs_never_overwrite(self) -> None: + photo = self._photo("photo.png") + first = image_ops_registry.run("remove_background", photo, {}, self.context) + second = image_ops_registry.run("remove_background", photo, {}, self.context) + self.assertNotEqual(first.path, second.path) + self.assertEqual(len(list((self.workspace / "Images").iterdir())), 4) + + def test_edge_threshold_is_applied(self) -> None: + class HalfSession(FakeSession): + def run(self, output_names, feeds): + return [np.zeros((1, 1, 1024, 1024), dtype=np.float32)] # sigmoid(0) = 0.5 everywhere + + birefnet.session_cache = birefnet.SessionCache(lambda path: HalfSession()) + photo = self._photo("photo.png") + soft = image_ops_registry.run("remove_background", photo, {"edge_threshold": 0}, self.context) + hard = image_ops_registry.run("remove_background", photo, {"edge_threshold": 0.6}, self.context) + self.assertTrue((np.asarray(_open(soft.mask_path)) == 128).all()) + self.assertTrue((np.asarray(_open(hard.mask_path)) == 0).all()) + + def test_exif_orientation_is_applied(self) -> None: + exif = Image.Exif() + exif[0x0112] = 6 # rotate 90° CW on display: 64x32 stored -> 32x64 shown + photo = self._photo("phone.jpg", exif=exif.tobytes()) + result = image_ops_registry.run("remove_background", photo, {}, self.context) + self.assertEqual(_open(result.path).size, (32, 64)) + + def test_any_mode_is_accepted_and_existing_transparency_is_kept(self) -> None: + for mode in ("L", "P", "RGBA"): + with self.subTest(mode=mode): + photo = self._photo(f"in-{mode}.png", mode=mode) + result = image_ops_registry.run("remove_background", photo, {}, self.context) + self.assertEqual(_open(result.path).mode, "RGBA") + + rgba = Image.new("RGBA", (64, 32), (200, 10, 10, 0)) # fully transparent input + path = self.workspace / "clear.png" + rgba.save(path) + result = image_ops_registry.run("remove_background", path, {}, self.context) + self.assertTrue((np.asarray(_open(result.path).getchannel("A")) == 0).all()) + + def test_unreadable_image_is_an_input_error(self) -> None: + bogus = self.workspace / "bogus.png" + bogus.parent.mkdir(parents=True, exist_ok=True) + bogus.write_bytes(b"not an image") + with self.assertRaises(ImageOpInputError): + image_ops_registry.run("remove_background", bogus, {}, self.context) + self.assertFalse((self.workspace / "Images").exists()) + + +_real_find_spec = importlib.util.find_spec + + +@unittest.skipUnless(HAS_ONNXRUNTIME, "onnxruntime is not installed") +class RealModelTests(unittest.TestCase): + """Runs the real model when this environment has onnxruntime and the + weights are already downloaded; never downloads anything itself.""" + + def test_real_inference_produces_a_mask(self) -> None: + if not BIREFNET_LITE.is_ready(): + self.skipTest("BiRefNet-lite weights are not downloaded") + session = birefnet.SessionCache().get(BIREFNET_LITE.path) + image = Image.new("RGB", (128, 96), (240, 240, 240)) + image.paste((200, 30, 30), (40, 24, 88, 72)) + alpha = birefnet.predict_alpha(session, image) + self.assertEqual(alpha.shape, (96, 128)) + self.assertGreaterEqual(alpha.min(), 0.0) + self.assertLessEqual(alpha.max(), 1.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_image_ops_requirements.py b/api/tests/test_image_ops_requirements.py new file mode 100644 index 000000000..d12cae022 --- /dev/null +++ b/api/tests/test_image_ops_requirements.py @@ -0,0 +1,177 @@ +import hashlib +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import services.generator_registry as registry +from services.image_ops import ImageOpExecutionError +from services.image_ops import requirements as reqs + + +def _model_req(content: bytes) -> reqs.HFFileRequirement: + return reqs.HFFileRequirement( + id="tiny-model", + label="Tiny model", + repo_id="org/repo", + filename="onnx/model.onnx", + revision="abc", + size=len(content), + sha256=hashlib.sha256(content).hexdigest(), + ) + + +class HFFileRequirementTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.models = Path(self._tmp.name) / "models" + self._patch = patch.object(registry, "MODELS_DIR", self.models) + self._patch.start() + + def tearDown(self) -> None: + self._patch.stop() + self._tmp.cleanup() + + def test_path_follows_models_dir_at_call_time(self) -> None: + req = _model_req(b"weights") + self.assertEqual(req.path, self.models / "tiny-model" / "model.onnx") + moved = Path(self._tmp.name) / "moved" + with patch.object(registry, "MODELS_DIR", moved): + self.assertEqual(req.path, moved / "tiny-model" / "model.onnx") + + def test_ready_only_when_file_has_expected_size(self) -> None: + req = _model_req(b"weights") + self.assertFalse(req.is_ready()) + self.assertEqual(req.download_bytes(), 7) + req.path.parent.mkdir(parents=True) + req.path.write_bytes(b"wei") # truncated + self.assertFalse(req.is_ready()) + req.path.write_bytes(b"weights") + self.assertTrue(req.is_ready()) + self.assertIsNone(req.download_bytes()) + + def test_download_bytes_discounts_a_resumable_part_file(self) -> None: + req = _model_req(b"0123456789") + req.path.parent.mkdir(parents=True) + req.path.with_suffix(".onnx.part").write_bytes(b"0123") + self.assertEqual(req.download_bytes(), 6) + + def _fake_download(self, content: bytes): + calls = [] + + def download(*, url, filename, dest_dir, progress_cb, **_): + calls.append(url) + progress_cb({"status": "Downloading… 50%", "bytesDownloaded": len(content) // 2, "totalBytes": len(content)}) + Path(dest_dir).mkdir(parents=True, exist_ok=True) # as the real downloader does + (Path(dest_dir) / filename).write_bytes(content) + return len(content) + + return download, calls + + def test_install_downloads_pinned_revision_and_verifies(self) -> None: + req = _model_req(b"weights!") + download, calls = self._fake_download(b"weights!") + frames = [] + with patch("routers.model._download_file_streamed", download): + req.install(frames.append) + self.assertTrue(req.is_ready()) + self.assertIn("org/repo", calls[0]) + self.assertIn("/abc/", calls[0]) # pinned revision + self.assertEqual(frames[-1]["percent"], 100) + self.assertEqual(frames[0]["percent"], 50) + self.assertEqual(frames[0]["status"], "Downloading Tiny model… 50%") + + def test_install_rejects_a_corrupted_download(self) -> None: + req = _model_req(b"weights") + download, _ = self._fake_download(b"WEIGHTS") # right size, wrong bytes + with patch("routers.model._download_file_streamed", download): + with self.assertRaisesRegex(ImageOpExecutionError, "checksum"): + req.install(lambda msg: None) + self.assertFalse(req.path.exists()) + + def test_install_reports_network_failure_clearly(self) -> None: + req = _model_req(b"weights") + + def download(**_): + raise RuntimeError("Failed to download model.onnx: ") + + with patch("routers.model._download_file_streamed", download): + with self.assertRaisesRegex(ImageOpExecutionError, "internet connection"): + req.install(lambda msg: None) + + def test_install_is_a_noop_when_ready(self) -> None: + req = _model_req(b"weights") + req.path.parent.mkdir(parents=True) + req.path.write_bytes(b"weights") + with patch("routers.model._download_file_streamed", side_effect=AssertionError("no download")): + req.install(lambda msg: None) + + def test_offline_mode_fails_without_network(self) -> None: + req = _model_req(b"weights") + with patch.dict("os.environ", {"HF_HUB_OFFLINE": "1"}), \ + patch("routers.model._download_file_streamed", side_effect=AssertionError("no download")): + with self.assertRaisesRegex(ImageOpExecutionError, "HF_HUB_OFFLINE"): + req.install(lambda msg: None) + + +class FakePopen: + def __init__(self, lines, code): + self.stdout = iter(line + "\n" for line in lines) + self._code = code + + def wait(self): + return self._code + + +class PipPackageRequirementTests(unittest.TestCase): + def setUp(self) -> None: + self.req = reqs.PipPackageRequirement( + id="definitely_not_installed_pkg", label="Fake runtime", + pip_spec="definitely_not_installed_pkg==1.0", approx_bytes=123, + ) + + def test_not_ready_reports_approximate_size(self) -> None: + self.assertFalse(self.req.is_ready()) + self.assertEqual(self.req.download_bytes(), 123) + + def test_already_installed_skips_pip(self) -> None: + req = reqs.PipPackageRequirement(id="json", label="json", pip_spec="json", approx_bytes=1) + self.assertTrue(req.is_ready()) + self.assertIsNone(req.download_bytes()) + with patch.object(subprocess, "Popen", side_effect=AssertionError("pip must not run")): + req.install(lambda msg: None) + + def test_pip_failure_raises_a_clear_error(self) -> None: + lines = [ + "Collecting definitely_not_installed_pkg==1.0", + "WARNING: Retrying ... NewConnectionError: Failed to establish a new connection", + "ERROR: Could not find a version that satisfies the requirement definitely_not_installed_pkg==1.0", + "ERROR: No matching distribution found for definitely_not_installed_pkg==1.0", + ] + frames = [] + with patch.object(subprocess, "Popen", return_value=FakePopen(lines, 1)) as popen: + with self.assertRaises(ImageOpExecutionError) as caught: + self.req.install(frames.append) + message = str(caught.exception) + self.assertIn("pip exited with code 1", message) + self.assertIn("internet connection", message) + self.assertIn("No matching distribution", message) + command = popen.call_args.args[0] + self.assertEqual(command[1:4], ["-m", "pip", "install"]) + self.assertIn("definitely_not_installed_pkg==1.0", command) + self.assertTrue(any(f["percent"] > 0 for f in frames)) + + def test_success_without_importable_module_is_an_error(self) -> None: + with patch.object(subprocess, "Popen", return_value=FakePopen(["Successfully installed x"], 0)): + with self.assertRaisesRegex(ImageOpExecutionError, "cannot be found"): + self.req.install(lambda msg: None) + + def test_onnxruntime_is_pinned_and_not_in_requirements_txt(self) -> None: + self.assertRegex(reqs.ONNXRUNTIME.pip_spec, r"^onnxruntime==\d+\.\d+\.\d+$") + requirements_txt = (Path(__file__).resolve().parents[1] / "requirements.txt").read_text() + self.assertNotIn("onnxruntime", requirements_txt) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_image_ops_router.py b/api/tests/test_image_ops_router.py new file mode 100644 index 000000000..561a6cbb9 --- /dev/null +++ b/api/tests/test_image_ops_router.py @@ -0,0 +1,235 @@ +import asyncio +import json +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import patch + +import numpy as np +from fastapi import HTTPException +from PIL import Image + +import routers.image as image_router +import services.generator_registry as registry +from services.image_ops import ImageOpExecutionError, ImageOpResult, ImageOpsRegistry +from test_image_ops_registry import FakeRequirement, make_op + + +async def _collect(response) -> list[dict]: + frames = [] + async for chunk in response.body_iterator: + text = chunk.decode() if isinstance(chunk, bytes) else chunk + for part in text.split("\n\n"): + if part.startswith("data: "): + frames.append(json.loads(part[6:])) + return frames + + +class ImageRouterTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.workspace = Path(self._tmp.name) / "workspace" + self.workspace.mkdir() + self.runtime = FakeRequirement("runtime", False, size=100) + self.model = FakeRequirement("model", False, size=300) + self.calls = [] + + def fn(path, params, ctx): + self.calls.append((path, params, ctx)) + out = ctx.images_dir / "out-nobg.png" + mask = ctx.images_dir / "out-mask.png" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(b"png") + mask.write_bytes(b"png") + return ImageOpResult(path=out, mask_path=mask) + + op = make_op([self.runtime, self.model], fn=fn) + self.registry = ImageOpsRegistry([op]) + self._patches = [ + patch.object(registry, "WORKSPACE_DIR", self.workspace), + patch.object(image_router, "image_ops_registry", self.registry), + patch.dict(image_router._prepares, clear=True), + ] + for p in self._patches: + p.start() + self.image = self.workspace / "photo.jpg" + Image.fromarray(np.zeros((4, 4, 3), dtype=np.uint8)).save(self.image) + + def tearDown(self) -> None: + for p in reversed(self._patches): + p.stop() + self._tmp.cleanup() + + def _run(self, path: str, params=None): + return asyncio.run(image_router.run_image_operation("example", image_router.ImageOpRequest(path=path, params=params or {}))) + + def test_ops_lists_the_registry(self) -> None: + body = image_router.list_image_operations() + self.assertEqual([op["id"] for op in body["ops"]], ["example"]) + self.assertEqual([r["kind"] for r in body["ops"][0]["requirements"]], ["runtime", "model"]) + + def test_builtin_op_is_exposed(self) -> None: + from services.image_ops import image_ops_registry + self.assertIn("remove_background", [op["id"] for op in image_ops_registry.describe()]) + + def test_status_reports_missing_and_bytes(self) -> None: + self.assertEqual( + image_router.image_operation_status("example"), + {"ready": False, "missing": ["runtime", "model"], "download_bytes": 400, "preparing": False}, + ) + self.runtime.ready = True + self.assertEqual(image_router.image_operation_status("example")["download_bytes"], 300) + self.model.ready = True + self.assertEqual( + image_router.image_operation_status("example"), + {"ready": True, "missing": [], "download_bytes": None, "preparing": False}, + ) + + def test_unknown_op_is_404(self) -> None: + for call in ( + lambda: image_router.image_operation_status("nope"), + lambda: asyncio.run(image_router.run_image_operation("nope", image_router.ImageOpRequest(path=str(self.image)))), + lambda: asyncio.run(image_router.prepare_image_operation("nope")), + ): + with self.assertRaises(HTTPException) as caught: + call() + self.assertEqual(caught.exception.status_code, 404) + + def test_run_when_not_ready_is_409_with_missing(self) -> None: + self.runtime.ready = True + with self.assertRaises(HTTPException) as caught: + self._run(str(self.image)) + self.assertEqual(caught.exception.status_code, 409) + self.assertEqual(caught.exception.detail, {"code": "not_ready", "missing": ["model"]}) + self.assertEqual(self.calls, []) + + def test_run_rejects_non_images_and_missing_files(self) -> None: + self.runtime.ready = self.model.ready = True + other = self.workspace / "model.glb" + other.write_bytes(b"glb") + cases = [(str(other), 400), ("../outside.png", 400), ("", 400), (str(self.workspace / "gone.png"), 404)] + for path, status in cases: + with self.subTest(path=path), self.assertRaises(HTTPException) as caught: + self._run(path) + self.assertEqual(caught.exception.status_code, status) + self.assertEqual(self.calls, []) + + def test_run_accepts_absolute_and_workspace_relative_paths(self) -> None: + self.runtime.ready = self.model.ready = True + for path in (str(self.image), "photo.jpg"): + with self.subTest(path=path): + body = self._run(path, {"strength": 0.5}) + self.assertEqual(Path(body["path"]), (self.workspace / "Images" / "out-nobg.png").resolve()) + self.assertEqual(Path(body["mask_path"]), (self.workspace / "Images" / "out-mask.png").resolve()) + self.assertEqual(body["url"], "/workspace/Images/out-nobg.png") + path, params, ctx = self.calls[-1] + self.assertEqual(path, self.image.resolve()) + self.assertEqual(params["strength"], 0.5) + self.assertEqual(ctx.workspace_dir, self.workspace) + + def test_run_maps_bad_params_and_failures(self) -> None: + self.runtime.ready = self.model.ready = True + with self.assertRaises(HTTPException) as caught: + self._run(str(self.image), {"strength": "x"}) + self.assertEqual(caught.exception.status_code, 400) + + def boom(*_): + raise ImageOpExecutionError("inference failed") + + self.registry._operations["example"] = make_op([self.runtime, self.model], fn=boom) + with self.assertRaises(HTTPException) as caught: + self._run(str(self.image)) + self.assertEqual(caught.exception.status_code, 500) + + def test_run_happens_off_the_event_loop(self) -> None: + self.runtime.ready = self.model.ready = True + threads = [] + self.registry._operations["example"] = make_op( + [self.runtime, self.model], + fn=lambda p, params, ctx: threads.append(threading.current_thread()) or ImageOpResult(path=p), + ) + self._run(str(self.image)) + self.assertIsNot(threads[0], threading.main_thread()) + + def test_prepare_when_ready_is_a_single_done_frame(self) -> None: + self.runtime.ready = self.model.ready = True + + async def go(): + return await _collect(await image_router.prepare_image_operation("example")) + + self.assertEqual(asyncio.run(go()), [{"percent": 100, "status": "done"}]) + self.assertEqual(self.runtime.installs + self.model.installs, 0) + + def test_prepare_installs_runtime_then_model_with_weighted_progress(self) -> None: + order = [] + for req in (self.runtime, self.model): + original = req.install + req.install = lambda progress, _o=original, _r=req: (order.append(_r.kind), _o(progress)) + + async def go(): + return await _collect(await image_router.prepare_image_operation("example")) + + frames = asyncio.run(go()) + self.assertEqual(order, ["runtime", "model"]) + self.assertEqual(frames[-1], {"percent": 100, "status": "done"}) + percents = [f["percent"] for f in frames] + self.assertEqual(percents, sorted(percents)) + # runtime is 100 of 400 bytes: its half-way frame is 12-13 %. + half_runtime = next(f for f in frames if f.get("step") == "runtime" and f["status"] == "half") + self.assertEqual(half_runtime["percent"], 12) + half_model = next(f for f in frames if f.get("step") == "model" and f["status"] == "half") + self.assertEqual(half_model["percent"], 62) + self.assertEqual(self.registry.get("example").missing(), []) + + def test_prepare_skips_requirements_already_installed(self) -> None: + self.runtime.ready = True + + async def go(): + return await _collect(await image_router.prepare_image_operation("example")) + + asyncio.run(go()) + self.assertEqual((self.runtime.installs, self.model.installs), (0, 1)) + + def test_prepare_failure_is_an_error_frame(self) -> None: + def fail(progress): + raise ImageOpExecutionError("Installing ONNX Runtime failed (pip exited with code 1).") + + self.runtime.install = fail + + async def go(): + return await _collect(await image_router.prepare_image_operation("example")) + + frames = asyncio.run(go()) + self.assertEqual(frames[-1], {"error": "Installing ONNX Runtime failed (pip exited with code 1)."}) + self.assertEqual(self.model.installs, 0) + + def test_a_second_prepare_joins_the_running_one(self) -> None: + gate = threading.Event() + original = self.model.install + + def slow_install(progress): + gate.wait(5) + original(progress) + + self.model.install = slow_install + + async def go(): + first = await image_router.prepare_image_operation("example") + first_frames = asyncio.ensure_future(_collect(first)) + await asyncio.sleep(0.05) + self.assertTrue(image_router.image_operation_status("example")["preparing"]) + second = await image_router.prepare_image_operation("example") + second_frames = asyncio.ensure_future(_collect(second)) + await asyncio.sleep(0.05) + gate.set() + return await first_frames, await second_frames + + first, second = asyncio.run(go()) + self.assertEqual(first[-1]["status"], "done") + self.assertEqual(second[-1]["status"], "done") + self.assertEqual((self.runtime.installs, self.model.installs), (1, 1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/package.json b/package.json index 063a2f9fd..239b30074 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/main/gpu-memory.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/shared/pages/startPage.test.ts src/areas/home/recents.test.ts src/areas/workflows/workflowRunRecord.test.ts src/areas/queue/queueModel.test.ts electron/main/workspace-images.test.ts src/areas/image-studio/imageStudioStore.test.ts src/shared/utils/imageInput.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/main/gpu-memory.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/shared/pages/startPage.test.ts src/areas/home/recents.test.ts src/areas/workflows/workflowRunRecord.test.ts src/areas/queue/queueModel.test.ts electron/main/workspace-images.test.ts src/areas/image-studio/imageStudioStore.test.ts src/areas/image-studio/imageOps.test.ts src/shared/utils/imageInput.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/components/EditPanel.tsx b/src/areas/generate/components/EditPanel.tsx index ed44b8128..2e22c45df 100644 --- a/src/areas/generate/components/EditPanel.tsx +++ b/src/areas/generate/components/EditPanel.tsx @@ -1,27 +1,21 @@ import { useCallback, useEffect, useState } from 'react' import { useAppStore } from '@shared/stores/appStore' import { useApi } from '@shared/hooks/useApi' -import { Tooltip } from '@shared/components/ui' +import { ParamFields } from '@shared/components/ui/ParamFields' import { INSPECT_OP_ID, - LOG_SLIDER_STEPS, apiErrorMessage, - coerceNumber, defaultParamValues, - fromSliderPosition, groupMeshOps, inspectionBadges, isMeshUpdate, meshPathFromUrl, - paramControl, requiresConfirmation, stepSummaries, summarizeDetails, - toSliderPosition, type BadgeState, type MeshOpDescriptor, type MeshOpResponse, - type ParamControl, } from '../meshEditPanel' import { commitSculpt, useSculptStore } from '../sculpt/sculptStore' import SculptSection from './SculptSection' @@ -63,178 +57,6 @@ function Chevron({ open }: { open: boolean }): JSX.Element { ) } -// ─── Param controls ─────────────────────────────────────────────────────────── - -const INPUT_CLASS = - 'bg-zinc-800 border border-zinc-700 rounded-lg px-2 py-1 text-xs text-zinc-200 focus:outline-none focus:border-violet-500 transition-colors disabled:opacity-50' - -/** Number text field that commits (clamped) on blur or Enter. */ -function NumberField({ control, value, onChange, disabled }: { - control: Extract - value: number - onChange: (v: number) => void - disabled: boolean -}): JSX.Element { - const [text, setText] = useState(String(value)) - useEffect(() => { setText(String(value)) }, [value]) - - function commit() { - const parsed = Number(text.replace(',', '.')) - if (text.trim() === '' || !Number.isFinite(parsed)) { setText(String(value)); return } - const next = coerceNumber(control, parsed) - setText(String(next)) - if (next !== value) onChange(next) - } - - return ( - setText(e.target.value)} - onBlur={commit} - onKeyDown={(e) => { if (e.key === 'Enter') commit() }} - className={`${INPUT_CLASS} w-20 shrink-0 text-right tabular-nums`} - /> - ) -} - -function ParamField({ control, value, onChange, disabled }: { - control: ParamControl - value: unknown - onChange: (v: unknown) => void - disabled: boolean -}): JSX.Element { - const label = ( -
    - {control.label} - {control.tooltip && ( - - ⓘ - - )} -
    - ) - - switch (control.kind) { - case 'toggle': { - const on = value === true - return ( -
    - {label} - -
    - ) - } - case 'select': - return ( -
    - {label} - -
    - ) - case 'slider': { - const numeric = typeof value === 'number' ? value : control.defaultValue - const log = control.scale === 'log' - return ( -
    - {label} -
    - onChange(fromSliderPosition(control, e.currentTarget.valueAsNumber))} - className="min-w-0 flex-1 accent-violet-500 cursor-pointer disabled:cursor-default" - /> - -
    -
    - ) - } - case 'number': - return ( -
    - {label} - -
    - ) - case 'text': - return ( -
    - {label} - onChange(e.target.value)} - className={`${INPUT_CLASS} w-full`} - /> -
    - ) - } -} - -function ParamFields({ op, values, onChange, disabled }: { - op: MeshOpDescriptor - values: Params - onChange: (id: string, value: unknown) => void - disabled: boolean -}): JSX.Element | null { - if (op.params_schema.length === 0) return null - return ( -
    - {op.params_schema.map((schema) => { - const control = paramControl(schema) - return ( - onChange(schema.id, v)} - disabled={disabled} - /> - ) - })} -
    - ) -} - // ─── Reports ────────────────────────────────────────────────────────────────── const BADGE_CLASS: Record = { @@ -470,7 +292,7 @@ export default function EditPanel({ onClose }: { onClose: () => void }): JSX.Ele
    setParam(inspectOp, id, v)} disabled={running !== null} @@ -526,7 +348,7 @@ export default function EditPanel({ onClose }: { onClose: () => void }): JSX.Ele {open && hasParams && (
    - setParam(op, id, v)} disabled={running !== null} /> + setParam(op, id, v)} disabled={running !== null} /> {params[op.id] && ( + + {card.phase !== 'hidden' && card.phase !== 'checking' && ( + { void download() }} + onClose={() => { pending.current = null; setCard(HIDDEN_CARD) }} + /> + )} +
    + ) +} + +function SetupCardView({ card, message, onDownload, onClose }: { + card: Exclude + message: string + onDownload: () => void + onClose: () => void +}): JSX.Element { + return ( +
    + {card.phase === 'confirm' && ( + <> +

    {message}

    +
    + + +
    + + )} + {card.phase === 'installing' && ( + <> +

    {stepLabel(card) ?? 'Setting up…'}

    + +

    This happens once. It will run as soon as it is done.

    + + )} + {card.phase === 'error' && ( + <> +

    {card.message}

    +
    + + +
    + + )} +
    + ) +} diff --git a/src/areas/image-studio/ImageStudioPage.tsx b/src/areas/image-studio/ImageStudioPage.tsx index 59cdd5eec..7f0d6eac9 100644 --- a/src/areas/image-studio/ImageStudioPage.tsx +++ b/src/areas/image-studio/ImageStudioPage.tsx @@ -4,6 +4,7 @@ import { useNavStore } from '@shared/stores/navStore' import { SUPPORTED_IMAGE_TYPES, saveImageToWorkspace, takePastedImage, useImageHandoff } from '@shared/utils/imageInput' import { mimeFromPath } from '@areas/workflows/nodes/imageUtils' import { fileNameOf, findByPath, sourceOf, useImageStudioStore, type StudioImage } from './imageStudioStore' +import ImageOpsToolbar, { type ImageOpActivity, type ImageOpResult } from './ImageOpsToolbar' // ─── Icons ──────────────────────────────────────────────────────────────────── @@ -40,51 +41,6 @@ async function previewUrlForPath(path: string): Promise { return URL.createObjectURL(new Blob([bytes], { type: mimeFromPath(path) })) } -// ─── Image operations (slot) ────────────────────────────────────────────────── -// Image operations (background removal, … — ticket #421) plug in here. An -// operation reads the selected image and writes a NEW file; the page adds it -// to the history with the selected image as its source, so the canvas shows -// the result before/after against it. The originals are never modified. - -interface ImageOp { - id: string - label: string - icon: string - /** Produces a new image file from `image` and returns its absolute path. */ - run: (image: StudioImage) => Promise -} - -const IMAGE_OPS: readonly ImageOp[] = [] - -function ImageOpsToolbar({ image, onResult }: { image: StudioImage, onResult: (path: string, source: StudioImage) => Promise }): JSX.Element | null { - const showError = useAppStore((s) => s.showError) - const [running, setRunning] = useState(null) - if (IMAGE_OPS.length === 0) return null - - async function run(op: ImageOp) { - setRunning(op.id) - try { - await onResult(await op.run(image), image) - } catch (err) { - showError(err instanceof Error ? err.message : String(err)) - } finally { - setRunning(null) - } - } - - return ( -
    - {IMAGE_OPS.map((op) => ( - - ))} -
    - ) -} - // ─── Canvas ─────────────────────────────────────────────────────────────────── function CompareView({ before, after }: { before: StudioImage, after: StudioImage }): JSX.Element { @@ -130,12 +86,64 @@ function CompareView({ before, after }: { before: StudioImage, after: StudioImag ) } -function Canvas({ image, source }: { image: StudioImage, source?: StudioImage }): JSX.Element { +/** Object URL of the mask file while `path` is set; revoked when it changes. */ +function useMaskPreview(path: string | undefined): { url: string | null, failed: boolean } { + const [state, setState] = useState<{ path: string, url: string | null, failed: boolean } | null>(null) + useEffect(() => { + if (!path) return + let cancelled = false + let created: string | null = null + previewUrlForPath(path) + .then((url) => { + if (cancelled) { URL.revokeObjectURL(url); return } + created = url + setState({ path, url, failed: false }) + }) + .catch(() => { if (!cancelled) setState({ path, url: null, failed: true }) }) + return () => { + cancelled = true + if (created) URL.revokeObjectURL(created) + } + }, [path]) + return state && state.path === path ? state : { url: null, failed: false } +} + +function Canvas({ image, source, activity }: { image: StudioImage, source?: StudioImage, activity: string | null }): JSX.Element { + const [maskFor, setMaskFor] = useState(null) + const showMask = Boolean(image.maskPath) && maskFor === image.id + const mask = useMaskPreview(showMask ? image.maskPath : undefined) + + let content: JSX.Element + if (showMask) { + content = mask.url + ? {`Mask + : {mask.failed ? 'Could not load the mask' : 'Loading mask…'} + } else if (source) { + content = + } else { + content = {image.name} + } + return (
    - {source - ? - : {image.name}} + {content} + {image.maskPath && ( + + )} + {activity && ( +
    + + + {activity} + +
    + )}
    ) } @@ -205,21 +213,28 @@ export default function ImageStudioPage(): JSX.Element { const { add, select, remove } = useImageStudioStore.getState() const [busy, setBusy] = useState(false) const [dragging, setDragging] = useState(false) + const [activity, setActivity] = useState(null) const selected = entries.find((e) => e.id === selectedId) const source = sourceOf(entries, selected) /** An image already on disk: used where it is, not copied. */ - const addPath = useCallback(async (path: string, sourceImage?: StudioImage) => { - const existing = sourceImage ? undefined : findByPath(useImageStudioStore.getState().entries, path) + const addPath = useCallback(async (path: string) => { + const existing = findByPath(useImageStudioStore.getState().entries, path) if (existing) { select(existing.id) return } const previewUrl = await previewUrlForPath(path) - add({ path, previewUrl, name: fileNameOf(path), sourceId: sourceImage?.id }) + add({ path, previewUrl, name: fileNameOf(path) }) }, [add, select]) + /** An image operation's output: a new entry made from `sourceImage`, selected. */ + const addResult = useCallback(async (result: ImageOpResult, sourceImage: StudioImage) => { + const previewUrl = await previewUrlForPath(result.path) + add({ path: result.path, previewUrl, name: fileNameOf(result.path), sourceId: sourceImage.id, op: result.op, maskPath: result.maskPath }) + }, [add]) + /** Image bytes with no file behind them (clipboard, browser drag): saved to the workspace first. */ const addBlob = useCallback(async (file: File, name: string) => { const path = await saveImageToWorkspace(file, name) @@ -293,7 +308,7 @@ export default function ImageStudioPage(): JSX.Element {

    - {selected && } + {selected && } - {selected ? : } + {selected + ? +: } {entries.length > 0 && (
    diff --git a/src/areas/image-studio/imageOps.test.ts b/src/areas/image-studio/imageOps.test.ts new file mode 100644 index 000000000..4d4ff6bd0 --- /dev/null +++ b/src/areas/image-studio/imageOps.test.ts @@ -0,0 +1,134 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { + HIDDEN_CARD, + applyPrepareEvent, + approxSize, + cardForStatus, + failInstall, + notReadyMissing, + runInputFor, + setupMessage, + startInstall, + stepLabel, + type SetupCard, +} from './imageOps' +import type { StudioImage } from './imageStudioStore' + +const image = (id: string, extra: Partial = {}): StudioImage => ({ + id, path: `C:\\work\\${id}.png`, previewUrl: `blob:${id}`, name: `${id}.png`, createdAt: 0, ...extra, +}) + +const OP = 'remove_background' + +// ─── runInputFor ────────────────────────────────────────────────────────────── + +test('runInputFor runs on the selected image when it is an original', () => { + const photo = image('photo') + assert.equal(runInputFor([photo], photo, OP), photo) +}) + +test('runInputFor re-runs on the source when the selected image is a result of the same op', () => { + const photo = image('photo') + const cut = image('cut', { sourceId: 'photo', op: OP }) + assert.equal(runInputFor([cut, photo], cut, OP), photo) +}) + +test('runInputFor keeps a result of another op as the input', () => { + const photo = image('photo') + const upscaled = image('up', { sourceId: 'photo', op: 'upscale' }) + assert.equal(runInputFor([upscaled, photo], upscaled, OP), upscaled) +}) + +test('runInputFor falls back to the selected image when its source left the history', () => { + const cut = image('cut', { sourceId: 'gone', op: OP }) + assert.equal(runInputFor([cut], cut, OP), cut) +}) + +// ─── Status → card ──────────────────────────────────────────────────────────── + +test('cardForStatus hides the card when the op is ready', () => { + assert.deepEqual(cardForStatus({ ready: true, missing: [], download_bytes: null }), HIDDEN_CARD) +}) + +test('cardForStatus offers the download with the missing steps in install order', () => { + assert.deepEqual( + cardForStatus({ ready: false, missing: ['model', 'runtime'], download_bytes: 176_000_000 }), + { phase: 'confirm', missing: ['runtime', 'model'], downloadBytes: 176_000_000 }, + ) +}) + +test('cardForStatus ignores unknown steps and a missing size', () => { + assert.deepEqual( + cardForStatus({ ready: false, missing: ['model', 'gpu' as never], download_bytes: null }), + { phase: 'confirm', missing: ['model'], downloadBytes: null }, + ) +}) + +test('setupMessage states the size and the CPU note for background removal', () => { + assert.equal( + setupMessage({ id: OP, label: 'Remove background' }, 176_000_000), + 'Background removal needs a one-time download (~180 MB). It runs on your CPU, no GPU needed.', + ) + assert.equal(setupMessage({ id: 'other', label: 'Upscale' }, null), 'Upscale needs a one-time download.') +}) + +test('approxSize rounds to what a person reads', () => { + assert.equal(approxSize(null), null) + assert.equal(approxSize(0), null) + assert.equal(approxSize(3_200_000), '~3 MB') + assert.equal(approxSize(204_000_000), '~200 MB') + assert.equal(approxSize(1_430_000_000), '~1.4 GB') +}) + +// ─── Prepare stream → card ──────────────────────────────────────────────────── + +const confirm: SetupCard = { phase: 'confirm', missing: ['runtime', 'model'], downloadBytes: 200_000_000 } + +test('startInstall begins on the first missing step at 0%', () => { + const card = startInstall(confirm) + assert.equal(card.phase, 'installing') + assert.equal(stepLabel(card), 'Step 1 of 2 · Installing the runtime') + assert.equal(card.phase === 'installing' && card.event.percent, 0) +}) + +test('applyPrepareEvent follows the step the backend reports, and keeps it until the next one', () => { + let card = startInstall(confirm) + card = applyPrepareEvent(card, { step: 'model', percent: 12, status: 'Downloading model', bytesDownloaded: 24, totalBytes: 200 }) + assert.equal(stepLabel(card), 'Step 2 of 2 · Downloading the model') + card = applyPrepareEvent(card, { percent: 50 }) + assert.equal(stepLabel(card), 'Step 2 of 2 · Downloading the model') + assert.equal(card.phase === 'installing' && card.event.percent, 50) +}) + +test('a single missing step is labelled without a counter', () => { + const card = startInstall({ phase: 'confirm', missing: ['model'], downloadBytes: null }) + assert.equal(stepLabel(card), 'Downloading the model') +}) + +test('an error frame turns the card into an error that keeps the setup for Retry', () => { + const card = applyPrepareEvent(startInstall(confirm), { error: 'pip failed' }) + assert.deepEqual(card, { phase: 'error', missing: ['runtime', 'model'], downloadBytes: 200_000_000, message: 'pip failed' }) + const retried = startInstall(card) + assert.equal(retried.phase, 'installing') + assert.equal(stepLabel(retried), 'Step 1 of 2 · Installing the runtime') +}) + +test('a cancelled frame and a broken stream are errors too', () => { + assert.equal(applyPrepareEvent(startInstall(confirm), { cancelled: true }).phase, 'error') + const failed = failInstall(startInstall(confirm), 'HTTP 500') + assert.equal(failed.phase === 'error' && failed.message, 'HTTP 500') +}) + +// ─── 409 not_ready ──────────────────────────────────────────────────────────── + +test('notReadyMissing reads the missing steps of a 409 not_ready', () => { + const err = { response: { status: 409, data: { detail: { code: 'not_ready', missing: ['model'] } } } } + assert.deepEqual(notReadyMissing(err), ['model']) +}) + +test('notReadyMissing ignores other errors', () => { + assert.equal(notReadyMissing(new Error('boom')), null) + assert.equal(notReadyMissing({ response: { status: 409, data: { detail: 'busy' } } }), null) + assert.equal(notReadyMissing({ response: { status: 500, data: { detail: { code: 'not_ready' } } } }), null) +}) diff --git a/src/areas/image-studio/imageOps.ts b/src/areas/image-studio/imageOps.ts new file mode 100644 index 000000000..0dde671f6 --- /dev/null +++ b/src/areas/image-studio/imageOps.ts @@ -0,0 +1,150 @@ +// Pure logic behind the Image Studio's operations (GET /image/ops): which +// image a run reads, the one-time setup card and its download progress. +// Kept free of React so it can be unit-tested with node --test. + +import type { MeshOpParamSchema } from '../generate/meshEditPanel' +import type { SseEvent } from '../../shared/services/llmDownloads' +import { sourceOf, type StudioImage } from './imageStudioStore' + +/** What an op may need installed before its first run. */ +export type SetupStep = 'runtime' | 'model' + +/** One entry of GET /image/ops. */ +export interface ImageOpDescriptor { + id: string + label: string + params_schema: MeshOpParamSchema[] + requirements: string[] +} + +/** GET /image/op/{op}/status */ +export interface ImageOpStatus { + ready: boolean + missing: SetupStep[] + download_bytes: number | null +} + +/** A frame of POST /image/op/{op}/prepare: the LLM download format, plus the step it belongs to. */ +export type PrepareEvent = SseEvent & { step?: SetupStep } + +// ─── Run input ──────────────────────────────────────────────────────────────── + +/** + * The image a run of `opId` reads. Re-running an op on its own result (e.g. + * with another edge threshold) starts again from the original, not from the + * already-processed image. + */ +export function runInputFor(entries: readonly StudioImage[], selected: StudioImage, opId: string): StudioImage { + if (selected.op === opId) return sourceOf(entries, selected) ?? selected + return selected +} + +// ─── Setup card ─────────────────────────────────────────────────────────────── + +export type SetupCard = + | { phase: 'hidden' } + | { phase: 'checking' } + | { phase: 'confirm', missing: SetupStep[], downloadBytes: number | null } + | { phase: 'installing', missing: SetupStep[], downloadBytes: number | null, step: SetupStep | null, event: SseEvent } + | { phase: 'error', missing: SetupStep[], downloadBytes: number | null, message: string } + +export const HIDDEN_CARD: SetupCard = { phase: 'hidden' } + +const STEPS: readonly SetupStep[] = ['runtime', 'model'] + +function knownSteps(missing: unknown): SetupStep[] { + return Array.isArray(missing) ? STEPS.filter((s) => missing.includes(s)) : [] +} + +/** What the card shows for a status: nothing when the op is ready, else the download offer. */ +export function cardForStatus(status: ImageOpStatus): SetupCard { + if (status.ready) return HIDDEN_CARD + const bytes = typeof status.download_bytes === 'number' && status.download_bytes > 0 ? status.download_bytes : null + return { phase: 'confirm', missing: knownSteps(status.missing), downloadBytes: bytes } +} + +function setupOf(card: SetupCard): { missing: SetupStep[], downloadBytes: number | null } { + return 'missing' in card ? { missing: card.missing, downloadBytes: card.downloadBytes } : { missing: [], downloadBytes: null } +} + +/** The card once the download starts (before the first frame arrives). */ +export function startInstall(card: SetupCard): SetupCard { + const setup = setupOf(card) + return { phase: 'installing', ...setup, step: setup.missing[0] ?? null, event: { percent: 0, status: 'Starting…' } } +} + +/** Folds one SSE frame of the prepare stream into the card. */ +export function applyPrepareEvent(card: SetupCard, event: PrepareEvent): SetupCard { + const setup = setupOf(card) + if (event.error) return { phase: 'error', ...setup, message: event.error } + if (event.cancelled) return { phase: 'error', ...setup, message: 'The download was cancelled.' } + const previous = card.phase === 'installing' ? card.step : null + const step = event.step && STEPS.includes(event.step) ? event.step : previous ?? setup.missing[0] ?? null + return { phase: 'installing', ...setup, step, event } +} + +/** The card when the stream failed outright (backend gone, HTTP error). */ +export function failInstall(card: SetupCard, message: string): SetupCard { + return { phase: 'error', ...setupOf(card), message } +} + +const STEP_LABELS: Record = { + runtime: 'Installing the runtime', + model: 'Downloading the model', +} + +/** "Step 1 of 2 · Installing the runtime", or null when the step is unknown. */ +export function stepLabel(card: SetupCard): string | null { + if (card.phase !== 'installing' || !card.step) return null + const label = STEP_LABELS[card.step] + const total = card.missing.length + const index = card.missing.indexOf(card.step) + return total > 1 && index >= 0 ? `Step ${index + 1} of ${total} · ${label}` : label +} + +/** "~200 MB" (rounded to what a person reads), or null when unknown. */ +export function approxSize(bytes: number | null): string | null { + if (bytes == null || bytes <= 0) return null + const mb = bytes / 1e6 + if (mb >= 1000) return `~${(mb / 1000).toFixed(1)} GB` + return `~${Math.max(1, Math.round(mb / 10) * 10 || Math.round(mb))} MB` +} + +// ─── Copy ───────────────────────────────────────────────────────────────────── + +interface OpCopy { + /** Subject of the setup sentence ("Background removal needs…"). */ + noun: string + /** Shown on the button while the op runs. */ + running: string + /** Extra line under the download offer. */ + note?: string +} + +const OP_COPY: Record = { + remove_background: { noun: 'Background removal', running: 'Removing background…', note: 'It runs on your CPU, no GPU needed.' }, +} + +export function opCopy(op: Pick): OpCopy { + return OP_COPY[op.id] ?? { noun: op.label, running: `${op.label}…` } +} + +/** The download offer's sentence. */ +export function setupMessage(op: Pick, downloadBytes: number | null): string { + const copy = opCopy(op) + const size = approxSize(downloadBytes) + const first = `${copy.noun} needs a one-time download${size ? ` (${size})` : ''}.` + return copy.note ? `${first} ${copy.note}` : first +} + +// ─── API errors ─────────────────────────────────────────────────────────────── + +/** The missing steps of a 409 `{detail:{code:'not_ready', missing}}`, or null for any other error. */ +export function notReadyMissing(err: unknown): SetupStep[] | null { + if (!err || typeof err !== 'object') return null + const response = (err as { response?: { status?: number, data?: { detail?: unknown } } }).response + if (response?.status !== 409) return null + const detail = response.data?.detail + if (!detail || typeof detail !== 'object' || (detail as { code?: unknown }).code !== 'not_ready') return null + return knownSteps((detail as { missing?: unknown }).missing) +} diff --git a/src/areas/image-studio/imageOpsApi.ts b/src/areas/image-studio/imageOpsApi.ts new file mode 100644 index 000000000..16a133ba1 --- /dev/null +++ b/src/areas/image-studio/imageOpsApi.ts @@ -0,0 +1,50 @@ +import axios from 'axios' +import { useAppStore } from '@shared/stores/appStore' +import { consumeSse } from '@shared/services/llmDownloads' +import { apiErrorMessage } from '@areas/generate/meshEditPanel' +import { notReadyMissing, type ImageOpDescriptor, type ImageOpStatus, type PrepareEvent, type SetupStep } from './imageOps' + +// Client of the image-ops routes (api/routers/image*.py): list, status, +// one-time setup (SSE) and run. Paths are absolute files in /Images. + +/** POST /image/op/{op} answered 409: the op needs its one-time setup first. */ +export class ImageOpNotReadyError extends Error { + constructor(readonly missing: SetupStep[]) { + super('This operation needs a one-time download first.') + this.name = 'ImageOpNotReadyError' + } +} + +function opUrl(op: string, suffix = ''): string { + return `${useAppStore.getState().apiUrl}/image/op/${encodeURIComponent(op)}${suffix}` +} + +export async function listImageOps(): Promise { + const { data } = await axios.get<{ ops?: ImageOpDescriptor[] }>(`${useAppStore.getState().apiUrl}/image/ops`) + return Array.isArray(data?.ops) ? data.ops : [] +} + +export async function getImageOpStatus(op: string): Promise { + try { + const { data } = await axios.get(opUrl(op, '/status')) + return data + } catch (err) { + throw new Error(apiErrorMessage(err)) + } +} + +/** Streams the install of what `op` is missing; resolves when the stream ends. */ +export function prepareImageOp(op: string, onEvent: (event: PrepareEvent) => void, signal?: AbortSignal): Promise { + return consumeSse(opUrl(op, '/prepare'), onEvent, signal, { method: 'POST' }) +} + +export async function runImageOp(op: string, path: string, params: Record): Promise<{ path: string, maskPath?: string }> { + try { + const { data } = await axios.post<{ path: string, mask_path?: string | null }>(opUrl(op), { path, params }) + return { path: data.path, maskPath: data.mask_path ?? undefined } + } catch (err) { + const missing = notReadyMissing(err) + if (missing) throw new ImageOpNotReadyError(missing) + throw new Error(apiErrorMessage(err)) + } +} diff --git a/src/areas/image-studio/imageStudioStore.ts b/src/areas/image-studio/imageStudioStore.ts index 47c1a4e39..a04b55e55 100644 --- a/src/areas/image-studio/imageStudioStore.ts +++ b/src/areas/image-studio/imageStudioStore.ts @@ -16,6 +16,10 @@ export interface StudioImage { name: string /** The image this one was produced from, when it came from an operation. */ sourceId?: string + /** Id of the image operation that produced it (e.g. background removal). */ + op?: string + /** Absolute path of the mask the operation wrote alongside, if any. */ + maskPath?: string createdAt: number } diff --git a/src/shared/components/ui/ParamFields.tsx b/src/shared/components/ui/ParamFields.tsx new file mode 100644 index 000000000..dbe0b8dfd --- /dev/null +++ b/src/shared/components/ui/ParamFields.tsx @@ -0,0 +1,186 @@ +import { useEffect, useState } from 'react' +import { Tooltip } from './Tooltip' +import { + LOG_SLIDER_STEPS, + coerceNumber, + fromSliderPosition, + paramControl, + toSliderPosition, + type MeshOpParamSchema, + type ParamControl, +} from '@areas/generate/meshEditPanel' + +// Controls generated from an op's params_schema (mesh ops, image ops): the +// same param dict format everywhere, so one renderer. + +const INPUT_CLASS = + 'bg-zinc-800 border border-zinc-700 rounded-lg px-2 py-1 text-xs text-zinc-200 focus:outline-none focus:border-violet-500 transition-colors disabled:opacity-50' + +/** Number text field that commits (clamped) on blur or Enter. */ +function NumberField({ control, value, onChange, disabled }: { + control: Extract + value: number + onChange: (v: number) => void + disabled: boolean +}): JSX.Element { + const [text, setText] = useState(String(value)) + useEffect(() => { setText(String(value)) }, [value]) + + function commit() { + const parsed = Number(text.replace(',', '.')) + if (text.trim() === '' || !Number.isFinite(parsed)) { setText(String(value)); return } + const next = coerceNumber(control, parsed) + setText(String(next)) + if (next !== value) onChange(next) + } + + return ( + setText(e.target.value)} + onBlur={commit} + onKeyDown={(e) => { if (e.key === 'Enter') commit() }} + className={`${INPUT_CLASS} w-20 shrink-0 text-right tabular-nums`} + /> + ) +} + +export function ParamField({ control, value, onChange, disabled }: { + control: ParamControl + value: unknown + onChange: (v: unknown) => void + disabled: boolean +}): JSX.Element { + const label = ( +
    + {control.label} + {control.tooltip && ( + + ⓘ + + )} +
    + ) + + switch (control.kind) { + case 'toggle': { + const on = value === true + return ( +
    + {label} + +
    + ) + } + case 'select': + return ( +
    + {label} + +
    + ) + case 'slider': { + const numeric = typeof value === 'number' ? value : control.defaultValue + const log = control.scale === 'log' + return ( +
    + {label} +
    + onChange(fromSliderPosition(control, e.currentTarget.valueAsNumber))} + className="min-w-0 flex-1 accent-violet-500 cursor-pointer disabled:cursor-default" + /> + +
    +
    + ) + } + case 'number': + return ( +
    + {label} + +
    + ) + case 'text': + return ( +
    + {label} + onChange(e.target.value)} + className={`${INPUT_CLASS} w-full`} + /> +
    + ) + } +} + +/** One control per entry of `schema`; nothing when the op takes no params. */ +export function ParamFields({ schema, values, onChange, disabled, className = 'flex flex-col gap-2.5' }: { + schema: MeshOpParamSchema[] + values: Record + onChange: (id: string, value: unknown) => void + disabled: boolean + className?: string +}): JSX.Element | null { + if (schema.length === 0) return null + return ( +
    + {schema.map((param) => { + const control = paramControl(param) + return ( + onChange(param.id, v)} + disabled={disabled} + /> + ) + })} +
    + ) +} diff --git a/src/shared/services/llmDownloads.ts b/src/shared/services/llmDownloads.ts index 18de4412f..46d58235b 100644 --- a/src/shared/services/llmDownloads.ts +++ b/src/shared/services/llmDownloads.ts @@ -17,9 +17,12 @@ export interface SseEvent { paused?: boolean } -/** Reads one SSE `data: {...}` frame at a time and calls `onEvent` for each. */ -export async function consumeSse(url: string, onEvent: (data: SseEvent) => void, signal?: AbortSignal): Promise { - const res = await fetch(url, { signal }) +/** + * Reads one SSE `data: {...}` frame at a time and calls `onEvent` for each. + * `init` sets the method/body for streams opened with a POST. + */ +export async function consumeSse(url: string, onEvent: (data: SseEvent) => void, signal?: AbortSignal, init?: RequestInit): Promise { + const res = await fetch(url, { ...init, signal }) if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`) const reader = res.body.getReader() const decoder = new TextDecoder() From 771ddd0d0c6c57b812e23bdd6063d6be14df95af Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 16 Nov 2026 14:12:57 +0100 Subject: [PATCH 072/108] fix(image-studio): accept the background-removal setup only as POST A GET could start the one-time install from any page (an image tag is enough); the renderer already uses POST. --- api/routers/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/routers/image.py b/api/routers/image.py index f440032ba..08f4bfd44 100644 --- a/api/routers/image.py +++ b/api/routers/image.py @@ -136,7 +136,7 @@ def _install(req=req, lock=lock, cb=_progress) -> None: _broadcast({"error": str(exc)}) -@router.api_route("/op/{op_name}/prepare", methods=["POST", "GET"]) +@router.post("/op/{op_name}/prepare") async def prepare_image_operation(op_name: str): """Install what the op is missing, streaming progress. Idempotent: an op that is already ready gets a single `done` frame; a call while a prepare is From fd09eabe4cff45e959182ba232410fbd292e83bb Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 16 Nov 2026 14:58:19 +0100 Subject: [PATCH 073/108] refactor(image-studio): show the selected image only, without a before/after slider The canvas shows the cut-out itself; the original stays one click away in the history strip. --- src/areas/image-studio/ImageStudioPage.tsx | 54 ++-------------------- src/areas/image-studio/imageStudioStore.ts | 4 +- 2 files changed, 6 insertions(+), 52 deletions(-) diff --git a/src/areas/image-studio/ImageStudioPage.tsx b/src/areas/image-studio/ImageStudioPage.tsx index 7f0d6eac9..e61721e64 100644 --- a/src/areas/image-studio/ImageStudioPage.tsx +++ b/src/areas/image-studio/ImageStudioPage.tsx @@ -1,9 +1,9 @@ -import { useCallback, useEffect, useRef, useState, type CSSProperties } from 'react' +import { useCallback, useEffect, useState, type CSSProperties } from 'react' import { useAppStore } from '@shared/stores/appStore' import { useNavStore } from '@shared/stores/navStore' import { SUPPORTED_IMAGE_TYPES, saveImageToWorkspace, takePastedImage, useImageHandoff } from '@shared/utils/imageInput' import { mimeFromPath } from '@areas/workflows/nodes/imageUtils' -import { fileNameOf, findByPath, sourceOf, useImageStudioStore, type StudioImage } from './imageStudioStore' +import { fileNameOf, findByPath, useImageStudioStore, type StudioImage } from './imageStudioStore' import ImageOpsToolbar, { type ImageOpActivity, type ImageOpResult } from './ImageOpsToolbar' // ─── Icons ──────────────────────────────────────────────────────────────────── @@ -43,49 +43,6 @@ async function previewUrlForPath(path: string): Promise { // ─── Canvas ─────────────────────────────────────────────────────────────────── -function CompareView({ before, after }: { before: StudioImage, after: StudioImage }): JSX.Element { - const [position, setPosition] = useState(50) - const box = useRef(null) - - const moveTo = (clientX: number) => { - const rect = box.current?.getBoundingClientRect() - if (!rect || rect.width === 0) return - setPosition(Math.min(100, Math.max(0, ((clientX - rect.left) / rect.width) * 100))) - } - - return ( -
    { event.currentTarget.setPointerCapture(event.pointerId); moveTo(event.clientX) }} - onPointerMove={(event) => { if (event.currentTarget.hasPointerCapture(event.pointerId)) moveTo(event.clientX) }} - role="slider" - aria-label="Before / after" - aria-valuemin={0} - aria-valuemax={100} - aria-valuenow={Math.round(position)} - tabIndex={0} - onKeyDown={(event) => { - if (event.key === 'ArrowLeft') setPosition((p) => Math.max(0, p - 5)) - if (event.key === 'ArrowRight') setPosition((p) => Math.min(100, p + 5)) - }} - > - {after.name} - {before.name} -
    - - - -
    - Before - After -
    - ) -} - /** Object URL of the mask file while `path` is set; revoked when it changes. */ function useMaskPreview(path: string | undefined): { url: string | null, failed: boolean } { const [state, setState] = useState<{ path: string, url: string | null, failed: boolean } | null>(null) @@ -108,7 +65,7 @@ function useMaskPreview(path: string | undefined): { url: string | null, failed: return state && state.path === path ? state : { url: null, failed: false } } -function Canvas({ image, source, activity }: { image: StudioImage, source?: StudioImage, activity: string | null }): JSX.Element { +function Canvas({ image, activity }: { image: StudioImage, activity: string | null }): JSX.Element { const [maskFor, setMaskFor] = useState(null) const showMask = Boolean(image.maskPath) && maskFor === image.id const mask = useMaskPreview(showMask ? image.maskPath : undefined) @@ -118,8 +75,6 @@ function Canvas({ image, source, activity }: { image: StudioImage, source?: Stud content = mask.url ? {`Mask : {mask.failed ? 'Could not load the mask' : 'Loading mask…'} - } else if (source) { - content = } else { content = {image.name} } @@ -216,7 +171,6 @@ export default function ImageStudioPage(): JSX.Element { const [activity, setActivity] = useState(null) const selected = entries.find((e) => e.id === selectedId) - const source = sourceOf(entries, selected) /** An image already on disk: used where it is, not copied. */ const addPath = useCallback(async (path: string) => { @@ -330,7 +284,7 @@ export default function ImageStudioPage(): JSX.Element { {selected - ? + ? : } {entries.length > 0 && ( diff --git a/src/areas/image-studio/imageStudioStore.ts b/src/areas/image-studio/imageStudioStore.ts index a04b55e55..9d02c9da2 100644 --- a/src/areas/image-studio/imageStudioStore.ts +++ b/src/areas/image-studio/imageStudioStore.ts @@ -5,7 +5,7 @@ import { create } from 'zustand' // level, and survives navigation (not app restarts). // // An image made from another one (an image operation, e.g. background -// removal) records it as `sourceId`; the studio compares the two before/after. +// removal) records it as `sourceId`, so re-running the op starts from the original. export interface StudioImage { id: string @@ -41,7 +41,7 @@ export function addToHistory(history: StudioHistory, entry: StudioImage, max = M /** * Drops entries from the history. Images made from a dropped one lose their - * source (nothing to compare with any more); the selection moves to the + * source (a re-run then starts from them); the selection moves to the * neighbour of the removed image. */ export function withoutEntries(history: StudioHistory, ids: readonly string[]): { history: StudioHistory, dropped: StudioImage[] } { From a7e2a799969a15fc4905613ec05ffb89cb1455d4 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Tue, 17 Nov 2026 17:44:39 +0100 Subject: [PATCH 074/108] feat(mesh-ops): declare an asset's real-world scale with set_real_scale "This object is 30 cm": the new set_real_scale op rescales the mesh with normalize_scale and marks the GLB as calibrated, in the default scene's extras (modly.real_scale = {unit: "m"}). The declaration travels with the file, and the registry carries it over to the output of any later op that rebuilds the mesh. Exports use it: a calibrated asset's STL is written in millimetres, and the slicer hand-off sends its real size instead of the 50 mm default. Closes #323 --- api/routers/export.py | 11 +- api/routers/optimize.py | 4 + api/services/mesh_ops/builtin.py | 61 ++++++++++ api/services/mesh_ops/real_scale.py | 129 ++++++++++++++++++++ api/services/mesh_ops/registry.py | 5 +- api/tests/test_mesh_ops_real_scale.py | 164 ++++++++++++++++++++++++++ api/tests/test_mesh_ops_registry.py | 1 + 7 files changed, 373 insertions(+), 2 deletions(-) create mode 100644 api/services/mesh_ops/real_scale.py create mode 100644 api/tests/test_mesh_ops_real_scale.py diff --git a/api/routers/export.py b/api/routers/export.py index 854d8bf21..c7d34a213 100644 --- a/api/routers/export.py +++ b/api/routers/export.py @@ -9,6 +9,7 @@ from fastapi.responses import Response, FileResponse from services import imported_sources +from services.mesh_ops.real_scale import MM_PER_M, export_scale, is_calibrated # Import the module (not the name) so WORKSPACE_DIR is read at call time: the # settings endpoint rebinds it when the user moves the workspace. import services.generator_registry as registry @@ -164,7 +165,12 @@ def export_for_slicer(fmt: str, token: str, filename: str): # thing this corrects — lay an upright model on its side. if source_suffix in GLTF_SUFFIXES: mesh.apply_transform(trimesh.transformations.rotation_matrix(math.pi / 2, [1, 0, 0])) - _normalize_print_scale(mesh) + # A calibrated asset is in metres and slicers read millimetres; only an + # uncalibrated one needs the unit-size guess. + if is_calibrated(full_path): + mesh.apply_scale(MM_PER_M) + else: + _normalize_print_scale(mesh) data = mesh.export(file_type=fmt) if isinstance(data, str): @@ -200,6 +206,9 @@ def export_mesh(fmt: str, path: str): mesh = trimesh.util.concatenate(geoms) if len(geoms) > 1 else geoms[0] else: mesh = loaded + scale = export_scale(full_path, fmt) + if scale != 1.0: + mesh.apply_scale(scale) buf = io.BytesIO() if fmt == "stl": diff --git a/api/routers/optimize.py b/api/routers/optimize.py index c6d70a1c0..7b6fadb55 100644 --- a/api/routers/optimize.py +++ b/api/routers/optimize.py @@ -26,6 +26,7 @@ mesh_ops_registry, ) from services.mesh_ops.glb_patch import GlbPatchError, VertexPatch, patch_glb_file +from services.mesh_ops.real_scale import export_scale from services.mesh_ops.transforms import apply_transforms, transform_file from schemas.runs import RunRecord from services.run_history import run_history, workspace_relative @@ -537,6 +538,9 @@ def export_mesh(path: str, format: str): mesh = trimesh.util.concatenate(geoms) if len(geoms) > 1 else geoms[0] else: mesh = loaded + scale = export_scale(input_path, format) + if scale != 1.0: + mesh.apply_scale(scale) data = mesh.export(file_type=format) stem = input_path.stem diff --git a/api/services/mesh_ops/builtin.py b/api/services/mesh_ops/builtin.py index 68bc1530d..4775254bb 100644 --- a/api/services/mesh_ops/builtin.py +++ b/api/services/mesh_ops/builtin.py @@ -4,6 +4,7 @@ from .inspection import inspect_mesh, suggest_fixes from .operations import decimate_mesh, repair_mesh, smooth_mesh from .presets import prepare_for_game, prepare_for_printing, prepare_for_web +from .real_scale import set_real_scale from .repair import auto_repair_mesh from .transforms import auto_orient, drop_to_ground, normalize_scale, recenter from .types import AgentTool, MeshOp @@ -163,6 +164,45 @@ ) +SET_REAL_SCALE_PARAMS = ( + { + "id": "size", + "label": "Real Size", + "type": "float", + "default": 10.0, + "min": 0.001, + "max": 100000.0, + "step": 0.1, + "tooltip": "How big the measured dimension really is, in the chosen unit.", + }, + { + "id": "unit", + "label": "Unit", + "type": "select", + "default": "cm", + "options": [ + {"value": "mm", "label": "Millimetres"}, + {"value": "cm", "label": "Centimetres"}, + {"value": "m", "label": "Metres"}, + ], + "tooltip": "Unit of the real size.", + }, + { + "id": "dimension", + "label": "Dimension", + "type": "select", + "default": "longest", + "options": [ + {"value": "longest", "label": "Longest side"}, + {"value": "x", "label": "Width (X)"}, + {"value": "y", "label": "Height (Y)"}, + {"value": "z", "label": "Depth (Z)"}, + ], + "tooltip": "Which bounding-box dimension has the real size.", + }, +) + + REPAIR_MESH_PARAMS = ( { "id": "fill_holes", @@ -355,6 +395,27 @@ fn=normalize_scale, category="transform", ), + MeshOp( + id="set_real_scale", + label="Set Real Size", + params_schema=SET_REAL_SCALE_PARAMS, + fn=set_real_scale, + category="transform", + agent=AgentTool( + description=( + "Declare how big the object really is (\"this is 30 cm tall\"): rescales the " + "mesh to that size and marks it calibrated, so the Print page and STL " + "exports use its real size." + ), + params={ + "size": "Real size of `dimension`, in `unit`.", + "unit": "mm, cm or m.", + "dimension": DIMENSION_HINT, + }, + required=("size", "unit"), + format_result=format_preset, + ), + ), MeshOp( id="prepare_for_printing", label="Prepare for 3D Printing", diff --git a/api/services/mesh_ops/real_scale.py b/api/services/mesh_ops/real_scale.py new file mode 100644 index 000000000..3d950c300 --- /dev/null +++ b/api/services/mesh_ops/real_scale.py @@ -0,0 +1,129 @@ +"""Real-world scale: declare how big an asset really is, and remember it. + +glTF's unit is the metre, but generators emit meshes at an arbitrary size, so +a GLB's numbers only mean metres once someone has said how big the object is. +That declaration is stored in the file itself, in the default scene's extras: + + scenes[scene].extras.modly.real_scale = {"unit": "m"} + +so it travels with the asset (library, export, re-import) and the viewer reads +it from the loaded scene's userData. Ops that rebuild the mesh drop the extras; +the registry carries the marker over to their output (see carry_real_scale). +""" + +from pathlib import Path +from typing import Any, Mapping, Optional + +from .glb_patch import GlbPatchError, _build_glb, _parse_glb +from .operations import _output_path +from .presets import DIMENSION_NAMES +from .transforms import apply_transforms +from .types import MeshOpContext, MeshOpResult + + +EXTRAS_KEY = "modly" +REAL_SCALE_KEY = "real_scale" +CALIBRATED = {"unit": "m"} +# STL carries no unit and every slicer reads it as millimetres. +MM_PER_M = 1000.0 + + +def _default_scene(gltf: dict) -> Optional[dict]: + scenes = gltf.get("scenes") + if not isinstance(scenes, list) or not scenes: + return None + index = gltf.get("scene", 0) + if not isinstance(index, int) or not 0 <= index < len(scenes): + return None + scene = scenes[index] + return scene if isinstance(scene, dict) else None + + +def read_real_scale(path: Path) -> Optional[dict[str, Any]]: + """The asset's real-scale declaration, or None when it is not calibrated.""" + path = Path(path) + if path.suffix.lower() != ".glb" or not path.is_file(): + return None + try: + gltf, _ = _parse_glb(path.read_bytes()) + except (GlbPatchError, OSError): + return None + scene = _default_scene(gltf) + extras = scene.get("extras") if scene else None + modly = extras.get(EXTRAS_KEY) if isinstance(extras, dict) else None + declaration = modly.get(REAL_SCALE_KEY) if isinstance(modly, dict) else None + if isinstance(declaration, dict) and declaration.get("unit") == "m": + return dict(declaration) + return None + + +def is_calibrated(path: Path) -> bool: + return read_real_scale(path) is not None + + +def write_real_scale(path: Path, declaration: Mapping[str, Any] = CALIBRATED) -> None: + """Stamp the declaration into a GLB in place, leaving everything else intact.""" + path = Path(path) + gltf, binary = _parse_glb(path.read_bytes()) + scene = _default_scene(gltf) + if scene is None: + raise GlbPatchError("GLB has no default scene to hold the real-scale declaration") + extras = scene.get("extras") + if not isinstance(extras, dict): + extras = scene["extras"] = {} + modly = extras.get(EXTRAS_KEY) + if not isinstance(modly, dict): + modly = extras[EXTRAS_KEY] = {} + modly[REAL_SCALE_KEY] = dict(declaration) + path.write_bytes(_build_glb(gltf, binary or b"")) + + +def carry_real_scale(input_path: Path, output_path: Optional[Path]) -> None: + """Keep a calibrated input's declaration on an op's output GLB. + + Every op keeps the coordinates in metres, so a calibrated input yields a + calibrated output, even when the op rebuilt the file and lost its extras. + """ + if output_path is None or Path(output_path).suffix.lower() != ".glb": + return + declaration = read_real_scale(input_path) + if declaration is None or is_calibrated(output_path): + return + try: + write_real_scale(output_path, declaration) + except (GlbPatchError, OSError): + pass + + +def export_scale(path: Path, fmt: str) -> float: + """Factor from the GLB's units to an export format's units. + + A calibrated asset's STL is written in millimetres; everything else keeps + the file's own units (metres for a calibrated GLB, arbitrary otherwise). + """ + return MM_PER_M if fmt.lower() == "stl" and is_calibrated(path) else 1.0 + + +def set_real_scale( + input_path: Path, + params: Mapping[str, Any], + context: MeshOpContext, +) -> MeshOpResult: + """"This object is 30 cm": rescale the mesh to that size and mark it calibrated.""" + output_path = _output_path(context, "set-real-scale") + # normalize_scale already turns "size in a unit along a dimension" into + # metres; the declaration is what makes the result trustworthy downstream. + result = apply_transforms(input_path, [("normalize_scale", params)], output_path, context) + write_real_scale(output_path) + dimensions = [float(value) for value in result.details["dimensions"]] + side = DIMENSION_NAMES[params["dimension"]] + summary = f"Declared the {side} as {params['size']:g} {params['unit']}: the mesh is now at real size." + return MeshOpResult( + file_path=output_path, + details={ + "steps": [{"op": "set_real_scale", **params, "summary": summary}], + "real_scale": dict(CALIBRATED), + "dimensions": dimensions, + "dimensions_mm": [value * MM_PER_M for value in dimensions], + }, + ) diff --git a/api/services/mesh_ops/registry.py b/api/services/mesh_ops/registry.py index 614b54bf4..be29ed6e0 100644 --- a/api/services/mesh_ops/registry.py +++ b/api/services/mesh_ops/registry.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, Iterable, Iterator, Mapping, Optional +from .real_scale import carry_real_scale from .types import MeshOp, MeshOpContext, MeshOpNotFoundError, MeshOpResult @@ -98,7 +99,9 @@ def run( if not path.is_file(): raise FileNotFoundError(f"Input mesh not found: {path}") - return operation.fn(path, self.resolve_params(operation_id, params), context) + result = operation.fn(path, self.resolve_params(operation_id, params), context) + carry_real_scale(path, result.file_path) + return result def resolve_params( self, diff --git a/api/tests/test_mesh_ops_real_scale.py b/api/tests/test_mesh_ops_real_scale.py new file mode 100644 index 000000000..dd95af9b0 --- /dev/null +++ b/api/tests/test_mesh_ops_real_scale.py @@ -0,0 +1,164 @@ +import base64 +import io +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import trimesh + +import routers.export as export_router +import routers.optimize as optimize_router +from services.mesh_ops import MeshOp, MeshOpContext, MeshOpResult, MeshOpsRegistry, mesh_ops_registry +from services.mesh_ops.real_scale import ( + is_calibrated, + read_real_scale, + write_real_scale, +) + + +def _extents(path: Path) -> np.ndarray: + return trimesh.load(str(path), force="mesh").extents + + +def _stl_extents(body: bytes) -> np.ndarray: + return trimesh.load(io.BytesIO(body), file_type="stl").extents + + +class SetRealScaleTests(unittest.TestCase): + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.workspace = Path(self._directory.name) + self.context = MeshOpContext(workspace_dir=self.workspace, temp_dir=self.workspace) + self.mesh = self.workspace / "mesh.glb" + # Generator output: about one unit, no real size. + trimesh.creation.box(extents=[2.0, 1.0, 0.5]).export(str(self.mesh)) + + def tearDown(self) -> None: + self._directory.cleanup() + + def _set(self, path: Path, **params) -> MeshOpResult: + output = self.workspace / "out" / f"{len(list(self.workspace.rglob('*')))}.glb" + context = MeshOpContext(self.workspace, self.workspace, output_path=output) + return mesh_ops_registry.run("set_real_scale", path, params, context) + + def test_a_generated_mesh_is_not_calibrated(self) -> None: + self.assertIsNone(read_real_scale(self.mesh)) + + def test_declaring_a_size_rescales_the_mesh_in_metres_and_marks_it(self) -> None: + result = self._set(self.mesh, size=30, unit="cm") + + np.testing.assert_allclose(_extents(result.file_path), [0.3, 0.15, 0.075], atol=1e-6) + self.assertEqual(read_real_scale(result.file_path), {"unit": "m"}) + np.testing.assert_allclose(result.details["dimensions_mm"], [300, 150, 75], atol=1e-3) + self.assertEqual( + result.details["steps"][0]["summary"], + "Declared the longest side as 30 cm: the mesh is now at real size.", + ) + # The input is left as it was. + self.assertFalse(is_calibrated(self.mesh)) + + def test_a_dimension_other_than_the_longest_can_be_declared(self) -> None: + result = self._set(self.mesh, size=50, unit="mm", dimension="y") + + np.testing.assert_allclose(_extents(result.file_path), [0.1, 0.05, 0.025], atol=1e-6) + + def test_the_declaration_survives_later_ops(self) -> None: + calibrated = self._set(self.mesh, size=30, unit="cm").file_path + + # An op that rebuilds the file and loses its extras, like a repair. + def rebuild(input_path, params, context): + output = self.workspace / "rebuilt.glb" + trimesh.load(str(input_path), force="mesh").export(str(output)) + return MeshOpResult(output) + + registry = MeshOpsRegistry([MeshOp("rebuild", "Rebuild", (), rebuild, "repair")]) + rebuilt = registry.run("rebuild", calibrated, {}, self.context).file_path + self.assertTrue(is_calibrated(rebuilt)) + + # ...but an uncalibrated input never comes out calibrated. + rebuilt = registry.run("rebuild", self.mesh, {}, self.context).file_path + self.assertFalse(is_calibrated(rebuilt)) + + def test_rescaling_a_calibrated_mesh_keeps_it_calibrated(self) -> None: + calibrated = self._set(self.mesh, size=30, unit="cm").file_path + output = self.workspace / "scaled.glb" + context = MeshOpContext(self.workspace, self.workspace, output_path=output) + + mesh_ops_registry.run("normalize_scale", calibrated, {"size": 12, "unit": "cm"}, context) + + self.assertTrue(is_calibrated(output)) + self.assertAlmostEqual(float(max(_extents(output))), 0.12, places=6) + + def test_writing_the_declaration_keeps_the_rest_of_the_file(self) -> None: + scene = trimesh.Scene(trimesh.creation.box(extents=[1.0, 2.0, 3.0])) + scene.metadata["author"] = "someone" + path = self.workspace / "extras.glb" + scene.export(str(path)) + + write_real_scale(path) + + reloaded = trimesh.load(str(path)) + self.assertEqual(reloaded.metadata["author"], "someone") + self.assertEqual(reloaded.metadata["modly"], {"real_scale": {"unit": "m"}}) + np.testing.assert_allclose(_extents(path), [1.0, 2.0, 3.0], atol=1e-6) + + def test_files_that_are_not_glb_are_never_calibrated(self) -> None: + stl = self.workspace / "mesh.stl" + trimesh.creation.box().export(str(stl)) + junk = self.workspace / "junk.glb" + junk.write_bytes(b"not a glb") + + self.assertIsNone(read_real_scale(stl)) + self.assertIsNone(read_real_scale(junk)) + self.assertIsNone(read_real_scale(self.workspace / "missing.glb")) + + def test_the_op_is_offered_in_the_registry(self) -> None: + described = {op["id"]: op for op in mesh_ops_registry.describe()}["set_real_scale"] + + self.assertEqual(described["category"], "transform") + self.assertEqual( + [schema["id"] for schema in described["params_schema"]], + ["size", "unit", "dimension"], + ) + with self.assertRaises(ValueError): + mesh_ops_registry.resolve_params("set_real_scale", {"unit": "inch"}) + + +class RealScaleExportTests(unittest.TestCase): + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.workspace = Path(self._directory.name).resolve() + self._original = export_router.registry.WORKSPACE_DIR + export_router.registry.WORKSPACE_DIR = self.workspace + # 0.3 m tall: a calibrated 30 cm figurine, or 0.3 arbitrary units. + trimesh.creation.box(extents=[0.1, 0.3, 0.2]).export(str(self.workspace / "raw.glb")) + trimesh.creation.box(extents=[0.1, 0.3, 0.2]).export(str(self.workspace / "real.glb")) + write_real_scale(self.workspace / "real.glb") + + def tearDown(self) -> None: + export_router.registry.WORKSPACE_DIR = self._original + self._directory.cleanup() + + def test_a_calibrated_stl_export_is_in_millimetres(self) -> None: + body = export_router.export_mesh("stl", "real.glb").body + np.testing.assert_allclose(_stl_extents(body), [100, 300, 200], atol=1e-3) + + body = optimize_router.export_mesh(path="real.glb", format="stl").body + np.testing.assert_allclose(_stl_extents(body), [100, 300, 200], atol=1e-3) + + def test_an_uncalibrated_export_keeps_its_units(self) -> None: + body = export_router.export_mesh("stl", "raw.glb").body + np.testing.assert_allclose(_stl_extents(body), [0.1, 0.3, 0.2], atol=1e-6) + + def test_the_slicer_gets_the_real_size_instead_of_the_default(self) -> None: + token = base64.urlsafe_b64encode(b"real.glb").decode("ascii").rstrip("=") + + body = export_router.export_for_slicer("stl", token, "model.stl").body + + # Y-up to Z-up, in millimetres, not resized to the 50 mm default. + np.testing.assert_allclose(_stl_extents(body), [100, 200, 300], atol=1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_mesh_ops_registry.py b/api/tests/test_mesh_ops_registry.py index bc2e57334..ce014b93d 100644 --- a/api/tests/test_mesh_ops_registry.py +++ b/api/tests/test_mesh_ops_registry.py @@ -30,6 +30,7 @@ def test_builtin_metadata_is_serializable_and_complete(self) -> None: "drop_to_ground", "auto_orient", "normalize_scale", + "set_real_scale", "prepare_for_printing", "prepare_for_game", "prepare_for_web", From c07c2a3d860cc3a53d8121a201c7a196999b2011 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Tue, 17 Nov 2026 17:53:12 +0100 Subject: [PATCH 075/108] feat(image-studio): add crop and auto-frame image ops Crop trims a percentage off each edge. Auto-frame recenters the subject on its alpha bounding box and pads it to a transparent square at a chosen resolution with a uniform margin, so generators get a framed input instead of cropping it each their own way. --- api/services/image_ops/builtin.py | 88 +++++++++++++++- api/services/image_ops/framing.py | 57 ++++++++++ api/tests/test_image_ops_framing.py | 150 +++++++++++++++++++++++++++ api/tests/test_image_ops_registry.py | 4 +- 4 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 api/services/image_ops/framing.py create mode 100644 api/tests/test_image_ops_framing.py diff --git a/api/services/image_ops/builtin.py b/api/services/image_ops/builtin.py index d603e6011..c96d38fcb 100644 --- a/api/services/image_ops/builtin.py +++ b/api/services/image_ops/builtin.py @@ -6,7 +6,7 @@ import numpy as np from PIL import Image, ImageOps, UnidentifiedImageError -from . import birefnet +from . import birefnet, framing from .outputs import png_bytes, sanitize_stem, write_new_files from .requirements import BIREFNET_LITE, ONNXRUNTIME from .types import ImageOp, ImageOpContext, ImageOpExecutionError, ImageOpInputError, ImageOpResult @@ -85,4 +85,88 @@ def remove_background(input_path: Path, params: Mapping[str, Any], context: Imag requirements=(ONNXRUNTIME, BIREFNET_LITE), ) -BUILTIN_IMAGE_OPS = (REMOVE_BACKGROUND,) + +def _edge_param(param_id: str, label: str) -> dict[str, Any]: + return { + "id": param_id, + "label": label, + "type": "int", + "default": 0, + "min": 0, + "max": 45, + "step": 1, + "tooltip": f"Percentage of the image cut off the {param_id} edge.", + } + + +def crop(input_path: Path, params: Mapping[str, Any], context: ImageOpContext) -> ImageOpResult: + image = load_image(input_path) + image = image.convert("RGBA" if _source_alpha(image) is not None else "RGB") + box = framing.crop_box( + image.width, image.height, + params.get("left", 0), params.get("top", 0), params.get("right", 0), params.get("bottom", 0), + ) + cropped = image.crop(box) + (out_path,) = write_new_files(context.images_dir, sanitize_stem(input_path.name), [("crop", png_bytes(cropped))]) + return ImageOpResult(path=out_path, details={"width": cropped.width, "height": cropped.height, "box": list(box)}) + + +CROP = ImageOp( + id="crop", + label="Crop", + params_schema=( + _edge_param("left", "Left %"), + _edge_param("top", "Top %"), + _edge_param("right", "Right %"), + _edge_param("bottom", "Bottom %"), + ), + fn=crop, +) + + +def auto_frame(input_path: Path, params: Mapping[str, Any], context: ImageOpContext) -> ImageOpResult: + image = load_image(input_path) + alpha = _source_alpha(image) + if alpha is None: + raise ImageOpInputError("Auto-frame needs a transparent background: remove the background first.") + bbox = framing.subject_bbox(birefnet.to_uint8(alpha)) + if bbox is None: + raise ImageOpInputError("Auto-frame found no subject: the image is fully transparent.") + size = int(params.get("size", 1024)) + framed = framing.frame_square(image, bbox, size, float(params.get("margin", 10))) + (out_path,) = write_new_files(context.images_dir, sanitize_stem(input_path.name), [("framed", png_bytes(framed))]) + return ImageOpResult(path=out_path, details={"width": size, "height": size, "subject_box": list(bbox)}) + + +AUTO_FRAME = ImageOp( + id="auto_frame", + label="Auto-frame", + params_schema=( + { + "id": "margin", + "label": "Margin %", + "type": "int", + "default": 10, + "min": 0, + "max": 40, + "step": 1, + "tooltip": "Empty space kept around the subject on every side, as a percentage of the square.", + }, + { + "id": "size", + "label": "Resolution", + "type": "select", + "default": 1024, + "options": [ + {"value": 512, "label": "512 × 512"}, + {"value": 768, "label": "768 × 768"}, + {"value": 1024, "label": "1024 × 1024"}, + {"value": 2048, "label": "2048 × 2048"}, + ], + "tooltip": "Size of the square image written, and sent to Generate as is.", + }, + ), + fn=auto_frame, +) + +BUILTIN_IMAGE_OPS = (CROP, REMOVE_BACKGROUND, AUTO_FRAME) diff --git a/api/services/image_ops/framing.py b/api/services/image_ops/framing.py new file mode 100644 index 000000000..b37a7d6bf --- /dev/null +++ b/api/services/image_ops/framing.py @@ -0,0 +1,57 @@ +"""Crop and auto-frame geometry, kept apart from file I/O so it can be tested on its own. + +Generators resize and crop their input each in their own way; framing the +subject ourselves (centered, square, known margin, known resolution) means the +file sent to Generate is already what the model should see. +""" + +from typing import Optional + +import numpy as np +from PIL import Image + +# Alpha (0..255) at or below this is background: background removal leaves a +# faint haze of near-zero alpha that must not stretch the subject's box. +SUBJECT_ALPHA_THRESHOLD = 12 + +Box = tuple[int, int, int, int] # left, top, right, bottom (right/bottom exclusive) + + +def crop_box(width: int, height: int, left: float, top: float, right: float, bottom: float) -> Box: + """The box left after trimming each edge by a percentage of the image side. + + Always at least 1 px wide and high. + """ + x0 = min(width - 1, round(width * left / 100)) + y0 = min(height - 1, round(height * top / 100)) + x1 = max(x0 + 1, width - round(width * right / 100)) + y1 = max(y0 + 1, height - round(height * bottom / 100)) + return x0, y0, x1, y1 + + +def subject_bbox(alpha: np.ndarray, threshold: int = SUBJECT_ALPHA_THRESHOLD) -> Optional[Box]: + """Bounding box of the pixels whose alpha is above `threshold`, None if there are none.""" + visible = alpha > threshold + rows = np.flatnonzero(visible.any(axis=1)) + cols = np.flatnonzero(visible.any(axis=0)) + if rows.size == 0: + return None + return int(cols[0]), int(rows[0]), int(cols[-1]) + 1, int(rows[-1]) + 1 + + +def frame_square(image: Image.Image, bbox: Box, size: int, margin_percent: float) -> Image.Image: + """The subject in `bbox`, scaled so its longer side spans `size` minus a + margin on each side, centered on a transparent `size` x `size` canvas.""" + margin = round(size * margin_percent / 100) + inner = max(1, size - 2 * margin) + subject = image.convert("RGBA").crop(bbox) + scale = inner / max(subject.width, subject.height) + scaled_size = ( + max(1, min(inner, round(subject.width * scale))), + max(1, min(inner, round(subject.height * scale))), + ) + # Resample premultiplied so transparent pixels don't bleed their color into the edges. + scaled = subject.convert("RGBa").resize(scaled_size, Image.LANCZOS).convert("RGBA") + canvas = Image.new("RGBA", (size, size), (0, 0, 0, 0)) + canvas.paste(scaled, ((size - scaled.width) // 2, (size - scaled.height) // 2)) + return canvas diff --git a/api/tests/test_image_ops_framing.py b/api/tests/test_image_ops_framing.py new file mode 100644 index 000000000..b99fcd0de --- /dev/null +++ b/api/tests/test_image_ops_framing.py @@ -0,0 +1,150 @@ +import tempfile +import unittest +from pathlib import Path + +import numpy as np +from PIL import Image + +from services.image_ops import ImageOpContext, ImageOpInputError, image_ops_registry +from services.image_ops import framing + + +def _open(path) -> Image.Image: + with Image.open(path) as image: + image.load() + return image + + +def _visible_box(image: Image.Image) -> tuple[int, int, int, int]: + return framing.subject_bbox(np.asarray(image.getchannel("A"))) + + +class CropBoxTests(unittest.TestCase): + def test_trims_each_edge_by_its_percentage(self) -> None: + self.assertEqual(framing.crop_box(200, 100, 10, 20, 25, 0), (20, 20, 150, 100)) + + def test_no_trim_keeps_the_whole_image(self) -> None: + self.assertEqual(framing.crop_box(64, 32, 0, 0, 0, 0), (0, 0, 64, 32)) + + def test_never_empty(self) -> None: + self.assertEqual(framing.crop_box(1, 1, 45, 45, 45, 45), (0, 0, 1, 1)) + x0, y0, x1, y1 = framing.crop_box(3, 3, 60, 60, 60, 60) + self.assertGreater(x1, x0) + self.assertGreater(y1, y0) + + +class SubjectBoxTests(unittest.TestCase): + def test_box_of_the_visible_pixels(self) -> None: + alpha = np.zeros((10, 20), dtype=np.uint8) + alpha[2:5, 3:9] = 255 + self.assertEqual(framing.subject_bbox(alpha), (3, 2, 9, 5)) + + def test_faint_haze_is_ignored(self) -> None: + alpha = np.full((10, 20), framing.SUBJECT_ALPHA_THRESHOLD, dtype=np.uint8) + alpha[4, 4] = 200 + self.assertEqual(framing.subject_bbox(alpha), (4, 4, 5, 5)) + + def test_none_when_fully_transparent(self) -> None: + self.assertIsNone(framing.subject_bbox(np.zeros((4, 4), dtype=np.uint8))) + + +class FrameSquareTests(unittest.TestCase): + def test_off_center_subject_is_centered_with_a_uniform_margin(self) -> None: + image = Image.new("RGBA", (300, 200), (0, 0, 0, 0)) + image.paste((200, 30, 30, 255), (200, 20, 280, 60)) # 80x40, near the top-right corner + framed = framing.frame_square(image, (200, 20, 280, 60), 100, 10) + self.assertEqual((framed.mode, framed.size), ("RGBA", (100, 100))) + # The longer side spans the square minus 10 px on each side; the other is centered. + self.assertEqual(_visible_box(framed), (10, 30, 90, 70)) + + def test_tall_subject_keeps_its_aspect_ratio(self) -> None: + image = Image.new("RGBA", (50, 100), (0, 0, 0, 0)) + image.paste((0, 0, 255, 255), (0, 0, 25, 100)) + framed = framing.frame_square(image, (0, 0, 25, 100), 200, 0) + self.assertEqual(_visible_box(framed), (75, 0, 125, 200)) + + def test_transparent_edges_do_not_darken(self) -> None: + image = Image.new("RGBA", (40, 40), (0, 0, 0, 0)) + image.paste((255, 255, 255, 255), (10, 10, 30, 30)) + framed = framing.frame_square(image, (0, 0, 40, 40), 97, 0) + rgba = np.asarray(framed) + partly = (rgba[..., 3] >= 32) & (rgba[..., 3] < 255) + self.assertTrue(partly.any()) + self.assertGreater(rgba[partly][:, :3].min(), 240) + + +class FramingOpsTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.workspace = Path(self._tmp.name) / "workspace" + self.workspace.mkdir() + self.context = ImageOpContext(workspace_dir=self.workspace) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def _save(self, image: Image.Image, name: str) -> Path: + path = self.workspace / name + image.save(path) + return path + + def test_ops_need_no_download(self) -> None: + for op_id in ("crop", "auto_frame"): + with self.subTest(op=op_id): + self.assertEqual(image_ops_registry.get(op_id).missing(), []) + + def test_crop_writes_a_new_image(self) -> None: + pixels = np.arange(40 * 20 * 3, dtype=np.uint32).reshape(20, 40, 3).astype(np.uint8) + photo = self._save(Image.fromarray(pixels), "photo.png") + result = image_ops_registry.run("crop", photo, {"left": 25, "bottom": 40}, self.context) + self.assertRegex(result.path.name, r"^photo-crop-\d{8}-\d{6}-[0-9a-f]{6}\.png$") + self.assertIsNone(result.mask_path) + out = _open(result.path) + self.assertEqual((out.mode, out.size), ("RGB", (30, 12))) + np.testing.assert_array_equal(np.asarray(out), pixels[:12, 10:]) + self.assertEqual(_open(photo).size, (40, 20)) + + def test_crop_keeps_transparency(self) -> None: + photo = self._save(Image.new("RGBA", (10, 10), (1, 2, 3, 0)), "cut.png") + result = image_ops_registry.run("crop", photo, {"top": 20}, self.context) + self.assertEqual(_open(result.path).mode, "RGBA") + + def test_crop_edges_are_clamped(self) -> None: + self.assertEqual(image_ops_registry.resolve_params("crop", {"left": 90})["left"], 45) + + def test_auto_frame_centers_an_off_center_cutout(self) -> None: + cutout = Image.new("RGBA", (400, 300), (0, 0, 0, 0)) + cutout.paste((30, 200, 30, 255), (10, 200, 110, 290)) # 100x90, bottom-left + path = self._save(cutout, "cat-nobg.png") + result = image_ops_registry.run("auto_frame", path, {"margin": 10, "size": 512}, self.context) + self.assertRegex(result.path.name, r"^cat-nobg-framed-\d{8}-\d{6}-[0-9a-f]{6}\.png$") + framed = _open(result.path) + self.assertEqual((framed.mode, framed.size), ("RGBA", (512, 512))) + left, top, right, bottom = _visible_box(framed) + margin = round(512 * 0.10) + self.assertEqual((left, 512 - right), (margin, margin)) + self.assertLessEqual(abs(top - (512 - bottom)), 1) + + def test_auto_frame_defaults_to_1024(self) -> None: + cutout = Image.new("RGBA", (64, 64), (0, 0, 0, 0)) + cutout.paste((255, 0, 0, 255), (0, 0, 8, 8)) + result = image_ops_registry.run("auto_frame", self._save(cutout, "dot.png"), {}, self.context) + self.assertEqual(_open(result.path).size, (1024, 1024)) + self.assertEqual(_visible_box(_open(result.path))[0], 102) + + def test_auto_frame_rejects_an_unknown_resolution(self) -> None: + with self.assertRaises(ValueError): + image_ops_registry.resolve_params("auto_frame", {"size": 999}) + + def test_auto_frame_needs_transparency(self) -> None: + opaque = self._save(Image.new("RGB", (16, 16), (9, 9, 9)), "photo.jpg") + with self.assertRaisesRegex(ImageOpInputError, "remove the background"): + image_ops_registry.run("auto_frame", opaque, {}, self.context) + clear = self._save(Image.new("RGBA", (16, 16), (9, 9, 9, 0)), "clear.png") + with self.assertRaisesRegex(ImageOpInputError, "no subject"): + image_ops_registry.run("auto_frame", clear, {}, self.context) + self.assertFalse((self.workspace / "Images").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_image_ops_registry.py b/api/tests/test_image_ops_registry.py index d7874f4b9..d9ef80012 100644 --- a/api/tests/test_image_ops_registry.py +++ b/api/tests/test_image_ops_registry.py @@ -56,8 +56,8 @@ def make_op(requirements=(), fn=None) -> ImageOp: class ImageOpsRegistryTests(unittest.TestCase): def test_builtin_metadata_is_serializable(self) -> None: described = image_ops_registry.describe() - self.assertEqual([d["id"] for d in described], ["remove_background"]) - op = described[0] + self.assertEqual([d["id"] for d in described], ["crop", "remove_background", "auto_frame"]) + op = described[1] self.assertEqual(op["label"], "Remove background") self.assertEqual( [(r["kind"], r["id"]) for r in op["requirements"]], From 47f88a9164c4bf941d61c7ae95338d10a4c86f25 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Tue, 17 Nov 2026 17:53:32 +0100 Subject: [PATCH 076/108] feat(print): add a Print page with a build-plate view A new Print page (Create, after Workflows) shows a model on a printer's build plate at its real size: - the model is Generate's current mesh or any mesh from the Library - printer presets (Bambu Lab, Prusa, Creality, Voron) or a custom volume, remembered between sessions; the default is 256 x 256 x 256 mm - the model is read in metres (glTF), drawn in millimetres, centred on the plate and resting on it; its size is shown in mm - a model larger than the build volume is flagged, side by side, and the volume outline turns red - a model without a declared real size (set_real_scale) says so and points to Set Real Size Closes #425 --- package.json | 2 +- src/areas/print/PrintPage.tsx | 379 ++++++++++++++++++ src/areas/print/printBed.test.ts | 79 ++++ src/areas/print/printBed.ts | 135 +++++++ src/areas/print/printStore.ts | 17 + .../components/agent/clientTools.test.ts | 2 +- src/shared/pages/pageRegistry.test.ts | 2 +- src/shared/pages/pageRegistry.ts | 12 + 8 files changed, 625 insertions(+), 3 deletions(-) create mode 100644 src/areas/print/PrintPage.tsx create mode 100644 src/areas/print/printBed.test.ts create mode 100644 src/areas/print/printBed.ts create mode 100644 src/areas/print/printStore.ts diff --git a/package.json b/package.json index 239b30074..00096849c 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/main/gpu-memory.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/shared/pages/startPage.test.ts src/areas/home/recents.test.ts src/areas/workflows/workflowRunRecord.test.ts src/areas/queue/queueModel.test.ts electron/main/workspace-images.test.ts src/areas/image-studio/imageStudioStore.test.ts src/areas/image-studio/imageOps.test.ts src/shared/utils/imageInput.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/main/gpu-memory.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/shared/pages/startPage.test.ts src/areas/home/recents.test.ts src/areas/workflows/workflowRunRecord.test.ts src/areas/queue/queueModel.test.ts electron/main/workspace-images.test.ts src/areas/image-studio/imageStudioStore.test.ts src/areas/image-studio/imageOps.test.ts src/shared/utils/imageInput.test.ts src/areas/print/printBed.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/print/PrintPage.tsx b/src/areas/print/PrintPage.tsx new file mode 100644 index 000000000..ecbd4c865 --- /dev/null +++ b/src/areas/print/PrintPage.tsx @@ -0,0 +1,379 @@ +import { Component, Suspense, useEffect, useMemo, useState } from 'react' +import type { ReactNode } from 'react' +import { Canvas } from '@react-three/fiber' +import { Environment, Lightformer, OrbitControls, useGLTF } from '@react-three/drei' +import * as THREE from 'three' +import { useAppStore } from '@shared/stores/appStore' +import { useNavStore } from '@shared/stores/navStore' +import type { ProjectedAssetLibraryEntry } from '@areas/generate/assetLibraryProjection' +import { getDefaultAssetLibraryService } from '@areas/generate/assetLibraryService' +import { + filterVisibleAssetLibraryEntries, + isAssetLibraryEntryOpenable, + openAssetLibraryEntry, + sortAssetLibraryEntries, +} from '@areas/generate/assetLibraryUi' +import { + CUSTOM_PRINTER_ID, + MAX_VOLUME_MM, + MIN_VOLUME_MM, + PRINTER_PRESETS, + checkFit, + describeOverflow, + isGlbUrl, + formatVolume, + isCalibrated, + modelSizeMm, + parsePrinterChoice, + resolveBuildVolume, + type BuildVolume, + type PrinterChoice, +} from './printBed' +import { usePrintStore } from './printStore' + +const PRINTER_STORAGE_KEY = 'modly.print.printer' +const MM_PER_M = 1000 + +function readStoredPrinter(): PrinterChoice { + try { return parsePrinterChoice(localStorage.getItem(PRINTER_STORAGE_KEY)) } catch { return parsePrinterChoice(null) } +} + +interface Measured { + /** Size in millimetres, glTF units read as metres. */ + size: BuildVolume + calibrated: boolean +} + +// ─── 3D scene (millimetres) ────────────────────────────────────────────────── + +/** Plate grid: a line every 10 mm, a brighter one every 50 mm. */ +function PlateGrid({ volume }: { volume: BuildVolume }): JSX.Element { + const [minor, major] = useMemo(() => { + const minorPoints: number[] = [] + const majorPoints: number[] = [] + const w = volume.width / 2 + const d = volume.depth / 2 + for (let x = 0; x <= volume.width + 1e-6; x += 10) { + const target = Math.round(x) % 50 === 0 ? majorPoints : minorPoints + target.push(x - w, 0, -d, x - w, 0, d) + } + for (let z = 0; z <= volume.depth + 1e-6; z += 10) { + const target = Math.round(z) % 50 === 0 ? majorPoints : minorPoints + target.push(-w, 0, z - d, w, 0, z - d) + } + const geometry = (points: number[]) => { + const g = new THREE.BufferGeometry() + g.setAttribute('position', new THREE.Float32BufferAttribute(points, 3)) + return g + } + return [geometry(minorPoints), geometry(majorPoints)] + }, [volume.width, volume.depth]) + + useEffect(() => () => { minor.dispose(); major.dispose() }, [minor, major]) + + return ( + + + + + ) +} + +function BuildPlate({ volume, fits }: { volume: BuildVolume, fits: boolean }): JSX.Element { + const edges = useMemo( + () => new THREE.EdgesGeometry(new THREE.BoxGeometry(volume.width, volume.height, volume.depth)), + [volume.width, volume.height, volume.depth], + ) + useEffect(() => () => edges.dispose(), [edges]) + + return ( + + + + + + + + + + + ) +} + +/** The model at its real size, centred on the plate and resting on it. */ +function PlacedModel({ url, onMeasured }: { url: string, onMeasured: (measured: Measured) => void }): JSX.Element { + const { scene } = useGLTF(url) + + const offset = useMemo(() => { + scene.position.set(0, 0, 0) + scene.rotation.set(0, 0, 0) + scene.scale.set(1, 1, 1) + scene.updateMatrixWorld(true) + const box = new THREE.Box3().setFromObject(scene) + const center = box.getCenter(new THREE.Vector3()) + const size = box.getSize(new THREE.Vector3()) + return { center, minY: box.min.y, size } + }, [scene]) + + useEffect(() => { + onMeasured({ size: modelSizeMm(offset.size), calibrated: isCalibrated(scene.userData) }) + }, [offset, scene, onMeasured]) + + // Free the loader cache and GPU memory when the model changes, as Generate does. + useEffect(() => () => { + useGLTF.clear(url) + scene.traverse((child) => { + if (child instanceof THREE.Mesh) { + child.geometry.dispose() + const materials = Array.isArray(child.material) ? child.material : [child.material] + materials.forEach((m: THREE.Material) => m.dispose()) + } + }) + }, [url, scene]) + + return ( + + + + + + ) +} + +class ModelBoundary extends Component<{ children: ReactNode, resetKey: string, onError: () => void }, { failed: boolean }> { + state = { failed: false } + static getDerivedStateFromError(): { failed: boolean } { return { failed: true } } + componentDidCatch(error: Error): void { + console.warn('[PrintPage] Failed to load model:', error.message) + this.props.onError() + } + componentDidUpdate(prev: { resetKey: string }): void { + if (prev.resetKey !== this.props.resetKey && this.state.failed) this.setState({ failed: false }) + } + render(): ReactNode { return this.state.failed ? null : this.props.children } +} + +function PlateScene({ volume, modelUrl, fits, onMeasured, onError }: { + volume: BuildVolume + modelUrl: string | null + fits: boolean + onMeasured: (measured: Measured) => void + onError: () => void +}): JSX.Element { + const span = Math.max(volume.width, volume.depth, volume.height) + return ( + + + + + + + + + + + + {modelUrl && ( + + + + + + )} + + + + ) +} + +// ─── Side panel ────────────────────────────────────────────────────────────── + +const FIELD = 'h-8 w-full rounded-lg border border-zinc-800 bg-zinc-900/70 px-2.5 text-xs text-zinc-200 focus:outline-none focus-visible:border-accent/50' + +function Section({ title, children }: { title: string, children: ReactNode }): JSX.Element { + return ( +
    +

    {title}

    + {children} +
    + ) +} + +function VolumeInput({ label, value, onChange }: { label: string, value: number, onChange: (value: number) => void }): JSX.Element { + return ( + + ) +} + +export default function PrintPage(): JSX.Element { + const apiUrl = useAppStore((s) => s.apiUrl) + const currentJob = useAppStore((s) => s.currentJob) + const navigate = useNavStore((s) => s.navigate) + const source = usePrintStore((s) => s.source) + const setSource = usePrintStore((s) => s.setSource) + const service = useMemo(() => getDefaultAssetLibraryService(), []) + + const [printer, setPrinter] = useState(readStoredPrinter) + const [assets, setAssets] = useState([]) + const [measured, setMeasured] = useState(null) + const [loadError, setLoadError] = useState(null) + + useEffect(() => { + let cancelled = false + service.list() + .then((result) => { + if (cancelled || !result.success) return + const meshes = filterVisibleAssetLibraryEntries(result.entries).filter(isAssetLibraryEntryOpenable) + setAssets(sortAssetLibraryEntries(meshes, 'date')) + }) + .catch(() => { /* the picker just lists fewer assets */ }) + return () => { cancelled = true } + }, [service]) + + function changePrinter(next: PrinterChoice) { + setPrinter(next) + try { localStorage.setItem(PRINTER_STORAGE_KEY, JSON.stringify(next)) } catch { /* private mode / quota */ } + } + + async function chooseSource(value: string) { + setLoadError(null) + if (value === 'current') { setSource({ kind: 'current' }); return } + const entry = assets.find((a) => a.id === value) + if (!entry) return + const result = await openAssetLibraryEntry(service, entry) + if (!result.success) { setLoadError(result.message); return } + setSource({ kind: 'library', id: entry.id, name: entry.displayName, url: result.selection.job.outputUrl ?? '' }) + } + + // GLB only: splats have no plate size, and imports are converted to GLB. + const currentUrl = currentJob?.status === 'done' && currentJob.outputUrl && isGlbUrl(currentJob.outputUrl) + ? currentJob.outputUrl + : null + const sourceUrl = source.kind === 'library' ? source.url : currentUrl + const modelUrl = sourceUrl ? `${apiUrl}${sourceUrl}` : null + + useEffect(() => { setMeasured(null) }, [modelUrl]) + + const volume = resolveBuildVolume(printer) + const fit = measured ? checkFit(measured.size, volume) : null + + return ( +
    + + +
    + setLoadError('The model could not be loaded.')} + /> + {!modelUrl && ( +
    +

    No model to print

    +

    Generate or open a mesh, or pick one from the Library on the left.

    + +
    + )} + {modelUrl && ( +

    Drag to rotate • Scroll to zoom

    + )} +
    +
    + ) +} diff --git a/src/areas/print/printBed.test.ts b/src/areas/print/printBed.test.ts new file mode 100644 index 000000000..ec9900a8e --- /dev/null +++ b/src/areas/print/printBed.test.ts @@ -0,0 +1,79 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { + CUSTOM_PRINTER_ID, + DEFAULT_PRINTER_CHOICE, + MAX_VOLUME_MM, + MIN_VOLUME_MM, + checkFit, + describeOverflow, + formatVolume, + isCalibrated, + isGlbUrl, + modelSizeMm, + parsePrinterChoice, + resolveBuildVolume, +} from './printBed' + +const PLATE_256 = { width: 256, depth: 256, height: 256 } + +test('the default printer is a 256 mm cube', () => { + assert.deepEqual(resolveBuildVolume(DEFAULT_PRINTER_CHOICE), PLATE_256) + assert.deepEqual(resolveBuildVolume(parsePrinterChoice(null)), PLATE_256) +}) + +test('a calibrated asset is shown at its real size and fits the plate', () => { + // A 30 cm × 15 cm × 7.5 cm GLB (metres, Y up): 300 wide, 75 deep, 150 tall. + const size = modelSizeMm({ x: 0.3, y: 0.15, z: 0.075 }) + assert.deepEqual(size, { width: 300, depth: 75, height: 150 }) + + const small = modelSizeMm({ x: 0.2, y: 0.25, z: 0.1 }) + assert.deepEqual(checkFit(small, PLATE_256), { fits: true, over: [] }) +}) + +test('a model larger than the build volume is flagged, side by side', () => { + const size = modelSizeMm({ x: 0.3, y: 0.3, z: 0.1 }) + const fit = checkFit(size, PLATE_256) + assert.deepEqual(fit, { fits: false, over: ['width', 'height'] }) + assert.equal( + describeOverflow(size, PLATE_256, fit.over), + 'Too large for this printer: 300 mm wide (max 256), 300 mm tall (max 256).', + ) +}) + +test('a model sized exactly to the plate fits despite float noise', () => { + const size = modelSizeMm({ x: 0.256, y: 0.256, z: 0.256 }) + assert.ok(checkFit(size, PLATE_256).fits) +}) + +test('the calibration marker is read from the scene userData', () => { + assert.ok(isCalibrated({ modly: { real_scale: { unit: 'm' } } })) + assert.ok(!isCalibrated({})) + assert.ok(!isCalibrated(null)) + assert.ok(!isCalibrated({ modly: { real_scale: { unit: 'cm' } } })) +}) + +test('a stored custom printer is restored and clamped', () => { + const choice = parsePrinterChoice(JSON.stringify({ + presetId: CUSTOM_PRINTER_ID, + custom: { width: 5, depth: 300, height: 99999 }, + })) + assert.deepEqual(resolveBuildVolume(choice), { width: MIN_VOLUME_MM, depth: 300, height: MAX_VOLUME_MM }) +}) + +test('an unknown or unreadable stored printer falls back to the default', () => { + assert.deepEqual(resolveBuildVolume(parsePrinterChoice('{"presetId":"gone"}')), PLATE_256) + assert.deepEqual(resolveBuildVolume(parsePrinterChoice('not json')), PLATE_256) +}) + +test('presets and formatting', () => { + assert.deepEqual(resolveBuildVolume({ ...DEFAULT_PRINTER_CHOICE, presetId: 'prusa-mk4' }), { width: 250, depth: 210, height: 220 }) + assert.equal(formatVolume({ width: 256, depth: 42.25, height: 7.5 }), '256 × 42.3 × 7.50 mm') +}) + +test('only GLB meshes are shown on the plate', () => { + assert.ok(isGlbUrl('/workspace/Workflows/a.glb')) + assert.ok(isGlbUrl('/optimize/serve-file?path=C%3A%5Cmodels%5Cpart.glb')) + assert.ok(!isGlbUrl('/workspace/Workflows/splat.ply')) + assert.ok(!isGlbUrl('/optimize/serve-file?path=%2Ftmp%2Fsplat.splat')) +}) diff --git a/src/areas/print/printBed.ts b/src/areas/print/printBed.ts new file mode 100644 index 000000000..be4602fd9 --- /dev/null +++ b/src/areas/print/printBed.ts @@ -0,0 +1,135 @@ +// Pure logic behind the Print page: printer volumes, the model's real size and +// whether it fits. Kept free of React and three.js so node --test can load it. + +/** A printer's build volume, in millimetres. Height is the printer's Z (up). */ +export interface BuildVolume { + width: number + depth: number + height: number +} + +export interface PrinterPreset { + id: string + label: string + volume: BuildVolume +} + +export const PRINTER_PRESETS: readonly PrinterPreset[] = [ + { id: 'bambu-x1', label: 'Bambu Lab X1 / P1 / A1', volume: { width: 256, depth: 256, height: 256 } }, + { id: 'bambu-a1-mini', label: 'Bambu Lab A1 mini', volume: { width: 180, depth: 180, height: 180 } }, + { id: 'prusa-mk4', label: 'Prusa MK4', volume: { width: 250, depth: 210, height: 220 } }, + { id: 'prusa-mini', label: 'Prusa MINI+', volume: { width: 180, depth: 180, height: 180 } }, + { id: 'ender-3', label: 'Creality Ender-3', volume: { width: 220, depth: 220, height: 250 } }, + { id: 'voron-350', label: 'Voron 2.4 (350)', volume: { width: 350, depth: 350, height: 340 } }, +] + +export const CUSTOM_PRINTER_ID = 'custom' +export const DEFAULT_PRINTER_ID = 'bambu-x1' + +/** Bounds of a custom build volume side, in millimetres. */ +export const MIN_VOLUME_MM = 10 +export const MAX_VOLUME_MM = 2000 + +export interface PrinterChoice { + /** A PRINTER_PRESETS id, or CUSTOM_PRINTER_ID. */ + presetId: string + /** The custom volume, kept while a preset is selected. */ + custom: BuildVolume +} + +export const DEFAULT_PRINTER_CHOICE: PrinterChoice = { + presetId: DEFAULT_PRINTER_ID, + custom: { width: 200, depth: 200, height: 200 }, +} + +function clampSide(value: unknown, fallback: number): number { + if (typeof value !== 'number' || !Number.isFinite(value)) return fallback + return Math.min(MAX_VOLUME_MM, Math.max(MIN_VOLUME_MM, value)) +} + +export function sanitizeVolume(value: unknown, fallback: BuildVolume = DEFAULT_PRINTER_CHOICE.custom): BuildVolume { + const v = (value && typeof value === 'object' ? value : {}) as Partial> + return { + width: clampSide(v.width, fallback.width), + depth: clampSide(v.depth, fallback.depth), + height: clampSide(v.height, fallback.height), + } +} + +/** A stored choice (JSON), falling back to the default for anything unreadable. */ +export function parsePrinterChoice(raw: string | null): PrinterChoice { + if (!raw) return DEFAULT_PRINTER_CHOICE + try { + const parsed = JSON.parse(raw) as { presetId?: unknown, custom?: unknown } + const known = parsed.presetId === CUSTOM_PRINTER_ID || PRINTER_PRESETS.some((p) => p.id === parsed.presetId) + return { + presetId: known ? String(parsed.presetId) : DEFAULT_PRINTER_ID, + custom: sanitizeVolume(parsed.custom), + } + } catch { + return DEFAULT_PRINTER_CHOICE + } +} + +export function resolveBuildVolume(choice: PrinterChoice): BuildVolume { + if (choice.presetId === CUSTOM_PRINTER_ID) return sanitizeVolume(choice.custom) + return (PRINTER_PRESETS.find((p) => p.id === choice.presetId) ?? PRINTER_PRESETS[0]).volume +} + +/** + * Whether a loaded GLB declares its real size (T-H1): the backend's + * set_real_scale op writes `scenes[n].extras.modly.real_scale = { unit: 'm' }`, + * which three's GLTFLoader puts on the scene's userData. + */ +export function isCalibrated(userData: unknown): boolean { + const modly = (userData as { modly?: { real_scale?: { unit?: unknown } } } | null)?.modly + return modly?.real_scale?.unit === 'm' +} + +/** A viewer mesh URL (/workspace/… or /optimize/serve-file?path=…) that points at a GLB. */ +export function isGlbUrl(url: string): boolean { + let path = url + try { path = decodeURIComponent(url) } catch { /* keep it as is */ } + return /\.(glb|gltf)$/i.test(path) +} + +const MM_PER_M = 1000 + +/** A model's size in millimetres, from its glTF bounding box (metres, Y up). */ +export function modelSizeMm(size: { x: number, y: number, z: number }): BuildVolume { + return { width: size.x * MM_PER_M, depth: size.z * MM_PER_M, height: size.y * MM_PER_M } +} + +export type VolumeSide = keyof BuildVolume + +export interface FitResult { + fits: boolean + /** The sides the model is larger than the build volume on. */ + over: VolumeSide[] +} + +/** Sub-micron slack, so a model sized exactly to the plate still fits. */ +const FIT_EPSILON_MM = 1e-3 + +export function checkFit(model: BuildVolume, volume: BuildVolume): FitResult { + const sides: VolumeSide[] = ['width', 'depth', 'height'] + const over = sides.filter((side) => model[side] > volume[side] + FIT_EPSILON_MM) + return { fits: over.length === 0, over } +} + +export function formatMm(value: number): string { + if (value >= 100) return value.toFixed(0) + if (value >= 10) return value.toFixed(1) + return value.toFixed(2) +} + +export function formatVolume(volume: BuildVolume): string { + return `${formatMm(volume.width)} × ${formatMm(volume.depth)} × ${formatMm(volume.height)} mm` +} + +/** "Too large: 312 mm wide for a 256 mm plate." */ +export function describeOverflow(model: BuildVolume, volume: BuildVolume, over: VolumeSide[]): string { + const words: Record = { width: 'wide', depth: 'deep', height: 'tall' } + const parts = over.map((side) => `${formatMm(model[side])} mm ${words[side]} (max ${formatMm(volume[side])})`) + return `Too large for this printer: ${parts.join(', ')}.` +} diff --git a/src/areas/print/printStore.ts b/src/areas/print/printStore.ts new file mode 100644 index 000000000..249203d1b --- /dev/null +++ b/src/areas/print/printStore.ts @@ -0,0 +1,17 @@ +import { create } from 'zustand' + +/** What the Print page shows: Generate's current mesh, or a Library asset. */ +export type PrintSource = + | { kind: 'current' } + | { kind: 'library', id: string, name: string, url: string } + +interface PrintState { + source: PrintSource + setSource: (source: PrintSource) => void +} + +// Kept across page switches, like Image Studio's history. +export const usePrintStore = create((set) => ({ + source: { kind: 'current' }, + setSource: (source) => set({ source }), +})) diff --git a/src/shared/components/agent/clientTools.test.ts b/src/shared/components/agent/clientTools.test.ts index ef6754bb9..53384f13f 100644 --- a/src/shared/components/agent/clientTools.test.ts +++ b/src/shared/components/agent/clientTools.test.ts @@ -339,7 +339,7 @@ test('navigate changes page and validates it', async () => { const f = fake() assert.equal(await run('navigate', { page: 'models' }, f), 'Opened the Extensions page.') assert.deepEqual(f.calls, [['navigate', 'models']]) - assert.match(await run('navigate', { page: 'nope' }, f), /^Error: page must be one of: home, generate, image-studio, workflows, library, queue, models, settings/) + assert.match(await run('navigate', { page: 'nope' }, f), /^Error: page must be one of: home, generate, image-studio, workflows, print, library, queue, models, settings/) }) test('a throwing dependency comes back as an error text', async () => { diff --git a/src/shared/pages/pageRegistry.test.ts b/src/shared/pages/pageRegistry.test.ts index 5b970af37..66eda1b55 100644 --- a/src/shared/pages/pageRegistry.test.ts +++ b/src/shared/pages/pageRegistry.test.ts @@ -22,7 +22,7 @@ test('the pages keep their ids, labels and sidebar order', () => { pagesByGroup().map((s) => [s.group, s.pages.map((p) => `${p.id}:${p.label}`)]), [ ['home', ['home:Home']], - ['create', ['generate:Generate', 'image-studio:Image Studio', 'workflows:Workflows']], + ['create', ['generate:Generate', 'image-studio:Image Studio', 'workflows:Workflows', 'print:Print']], ['manage', ['library:Library', 'queue:Queue']], ['system', ['models:Extensions', 'settings:Settings']], ], diff --git a/src/shared/pages/pageRegistry.ts b/src/shared/pages/pageRegistry.ts index 726a6f998..b8154240a 100644 --- a/src/shared/pages/pageRegistry.ts +++ b/src/shared/pages/pageRegistry.ts @@ -75,6 +75,18 @@ export const PAGES = [ load: () => import('@areas/workflows/WorkflowsPage'), wrapperClass: 'flex flex-1 overflow-hidden', }, + { + id: 'print', + label: 'Print', + group: 'create', + icon: [ + ['path', { d: 'M6 9V3h12v6' }], + ['rect', { x: 3, y: 9, width: 18, height: 8, rx: 1.5 }], + ['rect', { x: 7, y: 14, width: 10, height: 7, rx: 1 }], + ], + load: () => import('@areas/print/PrintPage'), + wrapperClass: 'flex flex-1 overflow-hidden', + }, { id: 'library', label: 'Library', From 4a4f45be4109ec0837b49704e4a9a5cf6caf4a8d Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Tue, 17 Nov 2026 17:54:09 +0100 Subject: [PATCH 077/108] feat(image-studio): show what the model sees A canvas toggle shows the selected file exactly as Generate uploads it, at its own resolution with its edges outlined, and says when it is not square. Crop's four edges sit in two columns in the toolbar. --- src/areas/image-studio/ImageOpsToolbar.tsx | 3 ++- src/areas/image-studio/ImageStudioPage.tsx | 30 ++++++++++++++++++++++ src/areas/image-studio/imageOps.test.ts | 23 +++++++++++++++++ src/areas/image-studio/imageOps.ts | 15 +++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/areas/image-studio/ImageOpsToolbar.tsx b/src/areas/image-studio/ImageOpsToolbar.tsx index 8f7f272c4..ba5b8e776 100644 --- a/src/areas/image-studio/ImageOpsToolbar.tsx +++ b/src/areas/image-studio/ImageOpsToolbar.tsx @@ -178,7 +178,8 @@ function ImageOpControl({ op, entries, selected, onResult, onActivity }: Toolbar values={params} onChange={(id, value) => setParams((current) => ({ ...current, [id]: value }))} disabled={running} - className="flex w-52 flex-col gap-1" + // Crop's four edges sit in two columns to keep the header short. + className={op.params_schema.length > 2 ? 'grid w-80 grid-cols-2 gap-x-3 gap-y-1' : 'flex w-52 flex-col gap-1'} /> {image.maskPath && (
    ) @@ -262,12 +274,12 @@ function stepSummary(row: QueueRow): string { } function StepList({ row, now }: { row: QueueRow, now: number }): JSX.Element { - if (row.steps.length === 0) return

    No steps recorded.

    + if (row.steps.length === 0) return

    No steps recorded.

    return (
      {row.steps.map((step, index) => ( -
    1. - {index + 1} +
    2. + {index + 1} {step.name} {formatDuration(runElapsedSeconds({ startedAt: step.started_at, endedAt: step.ended_at ?? null }, now))} @@ -281,19 +293,19 @@ function StepList({ row, now }: { row: QueueRow, now: number }): JSX.Element { function ChildRuns({ row, apiUrl, resolveModel, now }: { row: QueueRow, apiUrl: string, resolveModel: ModelNameResolver, now: number }): JSX.Element { const { children, error } = useChildRuns(apiUrl, row.id, !isFinishedStatus(row.status)) - if (error && !children) return

      Could not load the generations: {error}

      - if (!children) return

      Loading generations…

      - if (children.length === 0) return

      No model generations in this run.

      + if (error && !children) return

      Could not load the generations: {error}

      + if (!children) return

      Loading generations…

      + if (children.length === 0) return

      No model generations in this run.

      return (
        {children.map((record) => { const child = rowFromRecord(record, resolveModel) return ( -
      1. +
      2. {child.name} {formatDuration(runElapsedSeconds(child, now))} - +
      3. ) })} @@ -306,35 +318,35 @@ function RunCard({ row, now, apiUrl, resolveModel }: { row: QueueRow, now: numbe const expandable = row.kind === 'workflow' const summary = stepSummary(row) const failed = row.status === 'error' && !row.current + const finished = isFinishedStatus(row.status) return ( -
      4. +
      5. setExpanded((v) => !v) : undefined} > - {expandable ? ( - - ) :