From 1a93db28cfe55bc76c767124f29133f3a13efcac Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 16 Nov 2026 14:07:34 +0100 Subject: [PATCH 1/3] feat(image-studio): remove an image's background on the CPU Image Studio gets a "Remove background" tool. The cut-out lands in the history with the original as its source, so the before/after slider compares them; a Mask toggle shows the matte and an edge threshold hardens soft edges. It needs no extension and no GPU: BiRefNet-lite (MIT, ONNX, pinned revision, checked by size and SHA-256) runs with onnxruntime on the CPU. Both are fetched on first use behind a one-time setup card with progress (~249 MB): onnxruntime 1.23.2 is pip-installed into the backend's own environment on demand rather than added to requirements.txt, which would force every user through a full Python reinstall; 1.23.2 is the last release with Intel Mac wheels. Backend: an image_ops registry modelled on mesh_ops, under /image: GET /image/ops, GET /image/op/{op}/status, POST /image/op/{op}/prepare (SSE, LLM download format) and POST /image/op/{op} (409 until prepared). Outputs go to /Images/ and never overwrite. EditPanel's param fields move to a shared ParamFields component, used by both panels. Closes #421 --- api/main.py | 3 +- api/routers/image.py | 237 ++++++++++++++++ api/services/image_ops/__init__.py | 30 +++ api/services/image_ops/birefnet.py | 90 +++++++ api/services/image_ops/builtin.py | 88 ++++++ api/services/image_ops/outputs.py | 72 +++++ api/services/image_ops/registry.py | 125 +++++++++ api/services/image_ops/requirements.py | 248 +++++++++++++++++ api/services/image_ops/types.py | 101 +++++++ api/tests/test_image_ops_registry.py | 137 ++++++++++ api/tests/test_image_ops_remove_background.py | 253 ++++++++++++++++++ api/tests/test_image_ops_requirements.py | 177 ++++++++++++ api/tests/test_image_ops_router.py | 235 ++++++++++++++++ package.json | 2 +- src/areas/generate/components/EditPanel.tsx | 184 +------------ src/areas/image-studio/ImageOpsToolbar.tsx | 247 +++++++++++++++++ src/areas/image-studio/ImageStudioPage.tsx | 125 +++++---- src/areas/image-studio/imageOps.test.ts | 134 ++++++++++ src/areas/image-studio/imageOps.ts | 150 +++++++++++ src/areas/image-studio/imageOpsApi.ts | 50 ++++ src/areas/image-studio/imageStudioStore.ts | 4 + src/shared/components/ui/ParamFields.tsx | 186 +++++++++++++ src/shared/services/llmDownloads.ts | 9 +- 23 files changed, 2647 insertions(+), 240 deletions(-) create mode 100644 api/routers/image.py create mode 100644 api/services/image_ops/__init__.py create mode 100644 api/services/image_ops/birefnet.py create mode 100644 api/services/image_ops/builtin.py create mode 100644 api/services/image_ops/outputs.py create mode 100644 api/services/image_ops/registry.py create mode 100644 api/services/image_ops/requirements.py create mode 100644 api/services/image_ops/types.py create mode 100644 api/tests/test_image_ops_registry.py create mode 100644 api/tests/test_image_ops_remove_background.py create mode 100644 api/tests/test_image_ops_requirements.py create mode 100644 api/tests/test_image_ops_router.py create mode 100644 src/areas/image-studio/ImageOpsToolbar.tsx create mode 100644 src/areas/image-studio/imageOps.test.ts create mode 100644 src/areas/image-studio/imageOps.ts create mode 100644 src/areas/image-studio/imageOpsApi.ts create mode 100644 src/shared/components/ui/ParamFields.tsx diff --git a/api/main.py b/api/main.py index 9bf352304..0526d2e37 100644 --- a/api/main.py +++ b/api/main.py @@ -12,7 +12,7 @@ from services.stdio_utf8 import ensure_utf8_stdio ensure_utf8_stdio() # must run before any print/logging hits the pipe -from routers import generation, model, optimize, status, settings, extensions, export, workflow_runs, agent, llm +from routers import generation, model, optimize, status, settings, extensions, export, workflow_runs, agent, llm, image @asynccontextmanager @@ -68,6 +68,7 @@ def filter(self, record): app.include_router(workflow_runs.router, prefix="/workflow-runs") app.include_router(agent.router) app.include_router(llm.router, prefix="/llm") +app.include_router(image.router, prefix="/image") # Serve generated files from workspace — dynamic so path changes take effect immediately @app.get("/workspace/{full_path:path}") diff --git a/api/routers/image.py b/api/routers/image.py new file mode 100644 index 000000000..f440032ba --- /dev/null +++ b/api/routers/image.py @@ -0,0 +1,237 @@ +""" +Image operations (background removal, ...) on local files. + +Ops may need things installed on first use (a runtime, model weights): +`GET /image/op/{op}/status` says what is missing, `POST /image/op/{op}/prepare` +installs it while streaming progress in the same SSE format as the LLM +downloads (src/shared/services/llmDownloads.ts), and `POST /image/op/{op}` +runs the op once everything is there. +""" +import asyncio +import json +import threading +from pathlib import Path +from typing import Any, Optional + +from fastapi import APIRouter, HTTPException +from fastapi.responses import StreamingResponse +from pydantic import BaseModel, Field +from starlette.concurrency import run_in_threadpool + +import services.generator_registry as registry +from services.image_ops import ( + ImageOp, + ImageOpContext, + ImageOpExecutionError, + ImageOpInputError, + ImageOpNotFoundError, + ImageOpNotReadyError, + image_ops_registry, +) + +router = APIRouter(tags=["image"]) + +IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"} + + +class ImageOpRequest(BaseModel): + path: str + params: dict[str, Any] = Field(default_factory=dict) + + +def _fmt(data: dict) -> str: + return f"data: {json.dumps(data)}\n\n" + + +def _get_op(op_name: str) -> ImageOp: + try: + return image_ops_registry.get(op_name) + except ImageOpNotFoundError as exc: + raise HTTPException(404, f"Unknown image operation: {op_name}") from exc + + +def _missing_bytes(op: ImageOp) -> Optional[int]: + sizes = [req.download_bytes() for req in op.requirements if not req.is_ready()] + known = [size for size in sizes if size is not None] + return sum(known) if known else None + + +# ─── Catalog / status ───────────────────────────────────────────────────────── + +@router.get("/ops") +def list_image_operations(): + return {"ops": image_ops_registry.describe()} + + +@router.get("/op/{op_name}/status") +def image_operation_status(op_name: str): + op = _get_op(op_name) + missing = op.missing() + state = _prepares.get(op_name) + return { + "ready": not missing, + "missing": missing, + "download_bytes": _missing_bytes(op) if missing else None, + "preparing": bool(state and state.task and not state.task.done()), + } + + +# ─── Prepare (SSE) ──────────────────────────────────────────────────────────── + +class _PrepareState: + """One op's in-flight prepare, independent of any SSE connection, so a + second caller (or a reopened panel) attaches instead of starting a second + pip install / download racing the first.""" + + def __init__(self) -> None: + self.last_msg: dict = {} + self.subscribers: list[asyncio.Queue] = [] + self.task: Optional[asyncio.Task] = None + + +_prepares: dict[str, _PrepareState] = {} +# Two ops sharing a requirement (same runtime) must not install it twice at once. +_install_locks: dict[str, threading.Lock] = {} + + +def _is_terminal(msg: dict) -> bool: + return msg.get("status") == "done" or "error" in msg + + +async def _run_prepare(op: ImageOp, state: _PrepareState) -> None: + loop = asyncio.get_running_loop() + + def _broadcast(msg: dict) -> None: + def _do() -> None: + state.last_msg = msg + for queue in list(state.subscribers): + queue.put_nowait(msg) + loop.call_soon_threadsafe(_do) + + try: + pending = [req for req in op.requirements if not req.is_ready()] + # Weight each step by its download size so the bar is roughly linear in time. + weights = [max(1, req.download_bytes() or 1) for req in pending] + total = sum(weights) or 1 + done_weight = 0 + _broadcast({"percent": 0, "status": "Starting…"}) + + for req, weight in zip(pending, weights): + def _progress(msg: dict, _base: int = done_weight, _weight: int = weight, _kind: str = req.kind) -> None: + step_pct = max(0, min(100, msg.get("percent") or 0)) + overall = min(99, round((_base + _weight * step_pct / 100) / total * 100)) + _broadcast({**msg, "percent": overall, "step": _kind}) + + lock = _install_locks.setdefault(req.id, threading.Lock()) + + def _install(req=req, lock=lock, cb=_progress) -> None: + with lock: + req.install(cb) + + await loop.run_in_executor(None, _install) + done_weight += weight + + _broadcast({"percent": 100, "status": "done"}) + except Exception as exc: + _broadcast({"error": str(exc)}) + + +@router.api_route("/op/{op_name}/prepare", methods=["POST", "GET"]) +async def prepare_image_operation(op_name: str): + """Install what the op is missing, streaming progress. Idempotent: an op + that is already ready gets a single `done` frame; a call while a prepare is + running joins it. GET is accepted too so the renderer's existing + `consumeSse` (a plain fetch) can be reused unchanged.""" + op = _get_op(op_name) + state = _prepares.get(op_name) + running = state is not None and state.task is not None and not state.task.done() + + if not running and not op.missing(): + async def already(): + yield _fmt({"percent": 100, "status": "done"}) + return StreamingResponse(already(), media_type="text/event-stream") + + if not running: + state = _PrepareState() + _prepares[op_name] = state + state.task = asyncio.create_task(_run_prepare(op, state)) + + queue: asyncio.Queue[dict] = asyncio.Queue() + state.subscribers.append(queue) + if state.last_msg: + queue.put_nowait(state.last_msg) # replay current progress on (re)connect + + async def stream(): + try: + while True: + msg = await queue.get() + yield _fmt(msg) + if _is_terminal(msg): + break + finally: + if queue in state.subscribers: + state.subscribers.remove(queue) + + return StreamingResponse(stream(), media_type="text/event-stream") + + +# ─── Run ────────────────────────────────────────────────────────────────────── + +def _resolve_image_path(raw_path: str) -> Path: + if not raw_path or not raw_path.strip(): + raise HTTPException(400, "path is required") + candidate = Path(raw_path) + if candidate.is_absolute(): + resolved = candidate.resolve() + else: + resolved = (registry.WORKSPACE_DIR / raw_path).resolve() + if not registry.is_within_workspace(resolved): + raise HTTPException(400, "Invalid path") + if resolved.suffix.lower() not in IMAGE_EXTENSIONS: + raise HTTPException(400, f"Unsupported image type {resolved.suffix or '(none)'}: use png, jpg, jpeg or webp") + if not resolved.is_file(): + raise HTTPException(404, f"File not found: {raw_path}") + return resolved + + +def _workspace_url(path: Path) -> Optional[str]: + try: + relative = path.resolve().relative_to(registry.WORKSPACE_DIR.resolve()).as_posix() + except ValueError: + return None + return f"/workspace/{relative}" + + +@router.post("/op/{op_name}") +async def run_image_operation(op_name: str, body: ImageOpRequest): + op = _get_op(op_name) + input_path = _resolve_image_path(body.path) + missing = op.missing() + if missing: + raise HTTPException(409, {"code": "not_ready", "missing": missing}) + + context = ImageOpContext(workspace_dir=registry.WORKSPACE_DIR) + try: + # Inference takes seconds on CPU: keep the event loop free. + result = await run_in_threadpool(image_ops_registry.run, op_name, input_path, body.params, context) + except ImageOpNotReadyError as exc: + raise HTTPException(409, {"code": "not_ready", "missing": exc.missing}) from exc + except FileNotFoundError as exc: + raise HTTPException(404, str(exc)) from exc + except (ImageOpInputError, TypeError, ValueError) as exc: + raise HTTPException(400, str(exc)) from exc + except ImageOpExecutionError as exc: + raise HTTPException(500, str(exc)) from exc + + response: dict[str, Any] = { + "path": str(result.path.resolve()), + "mask_path": str(result.mask_path.resolve()) if result.mask_path else None, + } + url = _workspace_url(result.path) + if url: + response["url"] = url + if result.mask_path is not None: + mask_url = _workspace_url(result.mask_path) + if mask_url: + response["mask_url"] = mask_url + return response diff --git a/api/services/image_ops/__init__.py b/api/services/image_ops/__init__.py new file mode 100644 index 000000000..0be994d43 --- /dev/null +++ b/api/services/image_ops/__init__.py @@ -0,0 +1,30 @@ +"""Image operation registry (background removal, ...) used by the /image API.""" + +from .builtin import BUILTIN_IMAGE_OPS +from .registry import ImageOpsRegistry +from .types import ( + ImageOp, + ImageOpContext, + ImageOpExecutionError, + ImageOpInputError, + ImageOpNotFoundError, + ImageOpNotReadyError, + ImageOpResult, + Requirement, +) + + +image_ops_registry = ImageOpsRegistry(BUILTIN_IMAGE_OPS) + +__all__ = [ + "ImageOp", + "ImageOpContext", + "ImageOpExecutionError", + "ImageOpInputError", + "ImageOpNotFoundError", + "ImageOpNotReadyError", + "ImageOpResult", + "ImageOpsRegistry", + "Requirement", + "image_ops_registry", +] diff --git a/api/services/image_ops/birefnet.py b/api/services/image_ops/birefnet.py new file mode 100644 index 000000000..da3ca87b1 --- /dev/null +++ b/api/services/image_ops/birefnet.py @@ -0,0 +1,90 @@ +"""BiRefNet-lite background matting on CPU through onnxruntime. + +onnxruntime is installed on first use (see requirements.py), so it is only +imported inside `_create_onnx_session`, never at module import. +Pre/post-processing follows the model's preprocessor_config.json: +1024x1024 bilinear resize, x/255, ImageNet mean/std; the output is logits. +""" + +import threading +from pathlib import Path +from typing import Any, Callable, Optional + +import numpy as np +from PIL import Image + +INPUT_SIZE = 1024 +_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32) +_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32) + +SessionFactory = Callable[[Path], Any] + + +def _create_onnx_session(model_path: Path) -> Any: + import onnxruntime as ort # lazy: installed on demand + + options = ort.SessionOptions() + options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + return ort.InferenceSession(str(model_path), sess_options=options, providers=["CPUExecutionProvider"]) + + +class SessionCache: + """One InferenceSession per model file, built on first use and reused. + Tests inject a fake factory so they never need onnxruntime or the model.""" + + def __init__(self, factory: SessionFactory = _create_onnx_session) -> None: + self._factory = factory + self._lock = threading.Lock() + self._key: Optional[tuple[str, int]] = None + self._session: Any = None + + def get(self, model_path: Path) -> Any: + stat = model_path.stat() + key = (str(model_path.resolve()), stat.st_mtime_ns) + with self._lock: + if self._session is None or self._key != key: + self._session = self._factory(model_path) + self._key = key + return self._session + + def clear(self) -> None: + with self._lock: + self._session = None + self._key = None + + +session_cache = SessionCache() + + +def preprocess(rgb: Image.Image) -> np.ndarray: + """RGB PIL image -> float32 NCHW tensor (1, 3, 1024, 1024).""" + resized = rgb.resize((INPUT_SIZE, INPUT_SIZE), Image.Resampling.BILINEAR) + array = np.asarray(resized, dtype=np.float32) / 255.0 + array = (array - _MEAN) / _STD + return np.ascontiguousarray(array.transpose(2, 0, 1)[None, ...], dtype=np.float32) + + +def _sigmoid(x: np.ndarray) -> np.ndarray: + return 1.0 / (1.0 + np.exp(-np.clip(x, -60.0, 60.0))) + + +def predict_alpha(session: Any, rgb: Image.Image) -> np.ndarray: + """Soft foreground alpha in [0, 1], float32, at the image's own size (H, W).""" + input_name = session.get_inputs()[0].name + outputs = session.run(None, {input_name: preprocess(rgb)}) + logits = np.asarray(outputs[0], dtype=np.float32) + matte = _sigmoid(logits.reshape(logits.shape[-2], logits.shape[-1])) + resized = Image.fromarray(matte.astype(np.float32)).resize(rgb.size, Image.Resampling.BILINEAR) + return np.clip(np.asarray(resized, dtype=np.float32), 0.0, 1.0) + + +def apply_edge_threshold(alpha: np.ndarray, threshold: float) -> np.ndarray: + """0 keeps the soft alpha; t > 0 hardens: clamp((a - t) / (1 - t), 0, 1).""" + if threshold <= 0: + return np.clip(alpha, 0.0, 1.0) + t = min(float(threshold), 0.999) + return np.clip((alpha - t) / (1.0 - t), 0.0, 1.0) + + +def to_uint8(alpha: np.ndarray) -> np.ndarray: + return np.clip(np.rint(alpha * 255.0), 0, 255).astype(np.uint8) diff --git a/api/services/image_ops/builtin.py b/api/services/image_ops/builtin.py new file mode 100644 index 000000000..d603e6011 --- /dev/null +++ b/api/services/image_ops/builtin.py @@ -0,0 +1,88 @@ +"""Built-in image operations.""" + +from pathlib import Path +from typing import Any, Mapping + +import numpy as np +from PIL import Image, ImageOps, UnidentifiedImageError + +from . import birefnet +from .outputs import png_bytes, sanitize_stem, write_new_files +from .requirements import BIREFNET_LITE, ONNXRUNTIME +from .types import ImageOp, ImageOpContext, ImageOpExecutionError, ImageOpInputError, ImageOpResult + + +def load_image(path: Path) -> Image.Image: + """Open an image upright (EXIF orientation applied), fully decoded.""" + try: + with Image.open(path) as opened: + image = ImageOps.exif_transpose(opened) + image.load() + return image + except (UnidentifiedImageError, OSError, Image.DecompressionBombError) as exc: + raise ImageOpInputError(f"Cannot read image {path.name}: {exc}") from exc + + +def _source_alpha(image: Image.Image) -> np.ndarray | None: + """The input's own transparency in [0, 1], if it has any.""" + if image.mode in ("RGBA", "LA", "PA") or (image.mode == "P" and "transparency" in image.info): + return np.asarray(image.convert("RGBA").getchannel("A"), dtype=np.float32) / 255.0 + return None + + +def remove_background(input_path: Path, params: Mapping[str, Any], context: ImageOpContext) -> ImageOpResult: + image = load_image(input_path) + rgb = image.convert("RGB") + + try: + session = birefnet.session_cache.get(BIREFNET_LITE.path) + alpha = birefnet.predict_alpha(session, rgb) + except Exception as exc: + raise ImageOpExecutionError(f"Background removal failed: {exc}") from exc + + alpha = birefnet.apply_edge_threshold(alpha, float(params.get("edge_threshold", 0.0))) + source_alpha = _source_alpha(image) + if source_alpha is not None: + # Already-transparent pixels stay transparent. + alpha = alpha * source_alpha + mask = Image.fromarray(birefnet.to_uint8(alpha)) # 2D uint8 -> mode "L" + + cutout = rgb.copy() + cutout.putalpha(mask) + + out_path, mask_path = write_new_files( + context.images_dir, + sanitize_stem(input_path.name), + [("nobg", png_bytes(cutout)), ("mask", png_bytes(mask))], + ) + return ImageOpResult( + path=out_path, + mask_path=mask_path, + details={"width": rgb.width, "height": rgb.height}, + ) + + +REMOVE_BACKGROUND = ImageOp( + id="remove_background", + label="Remove background", + params_schema=( + { + "id": "edge_threshold", + "label": "Edge hardness", + "type": "float", + "default": 0.0, + "min": 0.0, + "max": 0.95, + "step": 0.05, + "tooltip": ( + "0 keeps soft, natural edges (hair, fur, blur). " + "Raise it to get crisper edges and drop faint leftovers of the background; " + "too high can eat into thin details of the subject." + ), + }, + ), + fn=remove_background, + requirements=(ONNXRUNTIME, BIREFNET_LITE), +) + +BUILTIN_IMAGE_OPS = (REMOVE_BACKGROUND,) diff --git a/api/services/image_ops/outputs.py b/api/services/image_ops/outputs.py new file mode 100644 index 000000000..870f7057c --- /dev/null +++ b/api/services/image_ops/outputs.py @@ -0,0 +1,72 @@ +"""Output file naming for image ops, matching electron/main/workspace-images.ts: +`--YYYYMMDD-HHMMSS-.png`, sortable by time, never overwriting.""" + +import io +import re +import secrets +import unicodedata +from datetime import datetime +from pathlib import Path +from typing import Callable, Optional, Sequence + +_WINDOWS_RESERVED = re.compile(r"^(con|prn|aux|nul|com[0-9]|lpt[0-9])$", re.IGNORECASE) +_MAX_STEM = 60 + + +def sanitize_stem(name: str) -> str: + """Filesystem-safe stem: only [A-Za-z0-9_-], no reserved Windows name, else "image".""" + leaf = re.split(r"[\\/]", name or "")[-1] + stem = Path(leaf).stem or leaf + ascii_only = unicodedata.normalize("NFKD", stem).encode("ascii", "ignore").decode("ascii") + safe = re.sub(r"[^A-Za-z0-9_-]+", "-", ascii_only) + safe = re.sub(r"-{2,}", "-", safe).strip("-_")[:_MAX_STEM].rstrip("-_") + if not safe or _WINDOWS_RESERVED.match(safe): + return "image" + return safe + + +def output_name(stem: str, tag: str, now: datetime, suffix: str) -> str: + return f"{stem}-{tag}-{now:%Y%m%d-%H%M%S}-{suffix}.png" + + +def write_new_files( + directory: Path, + stem: str, + payloads: Sequence[tuple[str, bytes]], + now: Optional[datetime] = None, + token: Callable[[], str] = lambda: secrets.token_hex(3), + attempts: int = 20, +) -> list[Path]: + """Write each (tag, bytes) as a new file sharing one timestamp and suffix. + + Files are created exclusively ("xb"), so an existing file is never + overwritten: on a name clash every file of the set is removed and the set + is retried with a fresh suffix. + """ + directory.mkdir(parents=True, exist_ok=True) + stamp = now or datetime.now() + for _ in range(attempts): + suffix = token() + written: list[Path] = [] + try: + for tag, data in payloads: + path = directory / output_name(stem, tag, stamp, suffix) + with open(path, "xb") as handle: + written.append(path) + handle.write(data) + return written + except FileExistsError: + for path in written: + path.unlink(missing_ok=True) + continue + except BaseException: + for path in written: + path.unlink(missing_ok=True) + raise + raise FileExistsError(f"Could not find a free output name in {directory}") + + +def png_bytes(image) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() diff --git a/api/services/image_ops/registry.py b/api/services/image_ops/registry.py new file mode 100644 index 000000000..b5bf873e5 --- /dev/null +++ b/api/services/image_ops/registry.py @@ -0,0 +1,125 @@ +"""Registry and dispatcher for image operations. + +Param handling follows services.mesh_ops.registry (same schema format: defaults, +then type checks and clamping to min/max). It is not imported from there: that +package pulls in trimesh and the whole mesh stack at import. +""" + +import re +from copy import deepcopy +from math import isfinite +from pathlib import Path +from typing import Any, Iterable, Iterator, Mapping, Optional + +from .types import ( + ImageOp, + ImageOpContext, + ImageOpNotFoundError, + ImageOpNotReadyError, + ImageOpResult, +) + + +_OP_ID = re.compile(r"^[a-z][a-z0-9_-]*$") + + +def _validate_param(parameter_id: str, value: Any, schema: Mapping[str, Any]) -> Any: + """Validate a supplied param against its schema; clamp numbers to min/max.""" + param_type = schema.get("type") + + if param_type == "boolean": + if not isinstance(value, bool): + raise TypeError(f"Parameter {parameter_id!r} must be a boolean") + return value + + if param_type == "select": + allowed = {option["value"] for option in schema.get("options", [])} + if allowed and value not in allowed: + raise ValueError(f"Parameter {parameter_id!r} must be one of {sorted(allowed)}") + return value + + if param_type in ("int", "float"): + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"Parameter {parameter_id!r} must be numeric") + if isinstance(value, float) and not isfinite(value): + raise ValueError(f"Parameter {parameter_id!r} must be finite") + if param_type == "int": + if isinstance(value, float) and not value.is_integer(): + raise TypeError(f"Parameter {parameter_id!r} must be an integer") + value = int(value) + else: + value = float(value) + if schema.get("min") is not None: + value = max(schema["min"], value) + if schema.get("max") is not None: + value = min(schema["max"], value) + return value + + return value + + +class ImageOpsRegistry: + def __init__(self, operations: Iterable[ImageOp] = ()) -> None: + self._operations: dict[str, ImageOp] = {} + for operation in operations: + self.register(operation) + + def register(self, operation: ImageOp) -> None: + if not _OP_ID.fullmatch(operation.id): + raise ValueError(f"Invalid image operation id: {operation.id!r}") + if operation.id in self._operations: + raise ValueError(f"Duplicate image operation id: {operation.id!r}") + self._operations[operation.id] = operation + + def get(self, operation_id: str) -> ImageOp: + try: + return self._operations[operation_id] + except KeyError as exc: + raise ImageOpNotFoundError(operation_id) from exc + + def __iter__(self) -> Iterator[ImageOp]: + return iter(self._operations.values()) + + def describe(self) -> list[dict[str, Any]]: + return [operation.describe() for operation in self._operations.values()] + + def resolve_params( + self, + operation_id: str, + params: Optional[Mapping[str, Any]], + ) -> dict[str, Any]: + """Apply the schema defaults, then validate and clamp. Unknown keys are rejected.""" + operation = self.get(operation_id) + known = {schema["id"] for schema in operation.params_schema if "id" in schema} + unknown = sorted(set(params or {}) - known) + if unknown: + raise ValueError(f"Unknown parameter(s) for {operation_id}: {', '.join(unknown)}") + + resolved = { + schema["id"]: deepcopy(schema["default"]) + for schema in operation.params_schema + if "id" in schema and "default" in schema + } + resolved.update(params or {}) + for schema in operation.params_schema: + parameter_id = schema.get("id") + if parameter_id in resolved: + resolved[parameter_id] = _validate_param(parameter_id, resolved[parameter_id], schema) + return resolved + + def run( + self, + operation_id: str, + input_path: Path, + params: Optional[Mapping[str, Any]], + context: ImageOpContext, + ) -> ImageOpResult: + operation = self.get(operation_id) + resolved = self.resolve_params(operation_id, params) + missing = operation.missing() + if missing: + raise ImageOpNotReadyError(operation_id, missing) + path = Path(input_path) + if not path.is_file(): + raise FileNotFoundError(f"Input image not found: {path}") + return operation.fn(path, resolved, context) diff --git a/api/services/image_ops/requirements.py b/api/services/image_ops/requirements.py new file mode 100644 index 000000000..5a285f4d1 --- /dev/null +++ b/api/services/image_ops/requirements.py @@ -0,0 +1,248 @@ +"""Things image ops install on first use: a pip package, a model file. + +Neither is in api/requirements.txt on purpose: changing that file sends every +user through a full venv reinstall (electron/main/python-setup.ts), for a +feature many will never open. They are fetched the first time the op is +prepared, into the backend's own environment and MODELS_DIR. +""" + +import hashlib +import importlib +import importlib.util +import os +import subprocess +import sys +import threading +from collections import deque +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Optional + +from .types import ImageOpExecutionError, ProgressCallback + + +def _models_dir() -> Path: + # Read at call time: POST /settings/paths rebinds it when the user moves it. + import services.generator_registry as registry + return registry.MODELS_DIR + + +def _hf_offline() -> bool: + return os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in ("1", "true", "yes", "on") + + +# ─── pip package ────────────────────────────────────────────────────────────── + +@dataclass(frozen=True) +class PipPackageRequirement: + """A Python package pip-installed into the running interpreter's environment.""" + + id: str # pip / import name, e.g. "onnxruntime" + label: str + pip_spec: str # e.g. "onnxruntime==1.23.2" + approx_bytes: int # wheel + dependencies, for the status estimate + kind: str = "runtime" + module: Optional[str] = None # import name when it differs from `id` + + @property + def module_name(self) -> str: + return self.module or self.id + + def is_ready(self) -> bool: + return importlib.util.find_spec(self.module_name) is not None + + def download_bytes(self) -> Optional[int]: + return None if self.is_ready() else self.approx_bytes + + def install(self, progress: ProgressCallback) -> None: + if self.is_ready(): + progress({"percent": 100, "status": f"{self.label} already installed"}) + return + + progress({"percent": 0, "status": f"Installing {self.label}…"}) + command = [ + sys.executable, "-m", "pip", "install", self.pip_spec, + "--disable-pip-version-check", "--no-input", + "--no-warn-script-location", "--progress-bar", "off", + ] + creationflags = getattr(subprocess, "CREATE_NO_WINDOW", 0) if sys.platform == "win32" else 0 + try: + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, + text=True, + encoding="utf-8", + errors="replace", + creationflags=creationflags, + ) + except OSError as exc: + raise ImageOpExecutionError(f"Could not start pip to install {self.label}: {exc}") from exc + + # pip prints no byte counts with the progress bar off: advance on its + # phase lines so the bar moves, and keep the tail for the error message. + tail: deque[str] = deque(maxlen=12) + percent = 0 + assert process.stdout is not None + for raw in process.stdout: + line = raw.strip() + if not line: + continue + tail.append(line) + if line.startswith("Collecting "): + percent = min(60, percent + 8) + elif line.startswith("Downloading "): + percent = min(70, percent + 4) + elif line.startswith("Installing collected packages"): + percent = 80 + else: + continue + progress({"percent": percent, "status": f"Installing {self.label}… {line[:80]}"}) + code = process.wait() + + if code != 0: + detail = " | ".join(line for line in tail if "error" in line.lower()) or " | ".join(list(tail)[-3:]) + hint = "" + if any(token in detail.lower() for token in ("connection", "network", "resolve", "proxy", "timed out", "no matching distribution")): + hint = " Check your internet connection and try again." + raise ImageOpExecutionError( + f"Installing {self.label} failed (pip exited with code {code}).{hint} {detail}".strip() + ) + + importlib.invalidate_caches() + if not self.is_ready(): + raise ImageOpExecutionError(f"{self.label} was installed but cannot be found by Python") + try: + importlib.import_module(self.module_name) + except Exception as exc: # broken wheel, missing system library… + raise ImageOpExecutionError(f"{self.label} was installed but fails to load: {exc}") from exc + progress({"percent": 100, "status": f"{self.label} installed"}) + + +# ─── Hugging Face model file ────────────────────────────────────────────────── + +@dataclass(frozen=True) +class HFFileRequirement: + """One model file downloaded from a pinned Hugging Face revision into + //, verified by size and SHA-256.""" + + id: str # also the folder name under MODELS_DIR + label: str + repo_id: str + filename: str # path inside the repo + revision: str # pinned commit + size: int + sha256: str + local_name: str = "model.onnx" + kind: str = "model" + + @property + def path(self) -> Path: + return _models_dir() / self.id / self.local_name + + def is_ready(self) -> bool: + try: + return self.path.is_file() and self.path.stat().st_size == self.size + except OSError: + return False + + def download_bytes(self) -> Optional[int]: + """Bytes left to fetch; a resumable .part file counts as already fetched.""" + if self.is_ready(): + return None + part = self.path.with_suffix(self.path.suffix + ".part") + try: + done = part.stat().st_size if part.is_file() else 0 + except OSError: + done = 0 + return max(0, self.size - done) + + def install(self, progress: ProgressCallback) -> None: + if self.is_ready(): + progress({"percent": 100, "status": f"{self.label} already downloaded"}) + return + if _hf_offline(): + raise ImageOpExecutionError( + f"{self.label} is not downloaded and HF_HUB_OFFLINE is set; go online once to fetch it" + ) + + from huggingface_hub import hf_hub_url + from routers.model import _download_file_streamed + + target = self.path + # A complete-looking file of the wrong size (interrupted copy, older + # revision) would be returned as-is by the downloader: start clean. + if target.exists(): + target.unlink() + + def _on_progress(msg: dict) -> None: + done = msg.get("bytesDownloaded") or 0 + total = msg.get("totalBytes") or self.size + pct = min(99, round(done / total * 100)) if total else 0 + progress({ + "percent": pct, + "status": f"Downloading {self.label}… {pct}%" + if str(msg.get("status", "")).startswith(("Downloading", "Resuming")) + else str(msg.get("status") or f"Downloading {self.label}…"), + "bytesDownloaded": done, + "totalBytes": total, + }) + + control = {"pause": threading.Event(), "cancel": threading.Event()} + try: + _download_file_streamed( + url=hf_hub_url(repo_id=self.repo_id, filename=self.filename, revision=self.revision), + filename=self.local_name, + dest_dir=str(target.parent), + file_index=1, + total_files=1, + base_percent=0, + progress_cb=_on_progress, + control=control, + ) + except Exception as exc: + # The downloader wraps network failures (after its retries) in + # RuntimeError; a bare OSError here is local (disk full, permissions). + hint = "" if isinstance(exc, OSError) else " Check your internet connection and try again." + raise ImageOpExecutionError(f"Downloading {self.label} failed: {exc}.{hint}") from exc + + progress({"percent": 99, "status": f"Verifying {self.label}…"}) + digest = _sha256(target) + if target.stat().st_size != self.size or digest != self.sha256: + target.unlink(missing_ok=True) + raise ImageOpExecutionError(f"{self.label} download is corrupted (checksum mismatch); try again") + progress({"percent": 100, "status": f"{self.label} ready", "bytesDownloaded": self.size, "totalBytes": self.size}) + + +def _sha256(path: Path, chunk: int = 4 * 1024 * 1024) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(chunk), b""): + h.update(block) + return h.hexdigest() + + +# ─── The concrete requirements ──────────────────────────────────────────────── + +# 1.23.2 is the newest onnxruntime with CPU wheels for every platform Modly runs +# on: win_amd64, macOS arm64 *and* x86_64 (1.24 dropped Intel Macs), manylinux +# x86_64/aarch64, for CPython 3.10–3.13 including the bundled 3.11. +ONNXRUNTIME = PipPackageRequirement( + id="onnxruntime", + label="ONNX Runtime", + pip_spec="onnxruntime==1.23.2", + approx_bytes=25_000_000, +) + +# BiRefNet-lite (MIT), fp32 ONNX export. fp16 halves the download but CPU +# execution of fp16 graphs in onnxruntime is slower and less accurate. +BIREFNET_LITE = HFFileRequirement( + id="birefnet-lite", + label="BiRefNet-lite model", + repo_id="onnx-community/BiRefNet_lite-ONNX", + filename="onnx/model.onnx", + revision="de15b22ba131738a16dff04aab8bdf8dc32e3ac1", + size=224_005_088, + sha256="5600024376f572a557870a5eb0afb1e5961636bef4e1e22132025467d0f03333", +) diff --git a/api/services/image_ops/types.py b/api/services/image_ops/types.py new file mode 100644 index 000000000..026ff03e3 --- /dev/null +++ b/api/services/image_ops/types.py @@ -0,0 +1,101 @@ +"""Shared types for Modly image operations. + +Mirrors services.mesh_ops.types, plus what mesh ops never needed: an image op +can depend on things that are fetched on first use (a Python runtime, model +weights). Those are `Requirement`s; the router reports which are missing and +installs them on demand. +""" + +from copy import deepcopy +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Mapping, Optional, Protocol + + +# One progress frame, in the SSE format the renderer already consumes for LLM +# downloads (src/shared/services/llmDownloads.ts: percent, status, +# bytesDownloaded, totalBytes). Percent is 0..100 within the requirement. +ProgressCallback = Callable[[dict], None] + + +class ImageOpNotFoundError(LookupError): + """Raised when a caller requests an operation that is not registered.""" + + +class ImageOpNotReadyError(RuntimeError): + """Raised when an operation runs before its requirements are installed.""" + + def __init__(self, operation_id: str, missing: list[str]) -> None: + super().__init__(f"{operation_id} is not ready: missing {', '.join(missing)}") + self.operation_id = operation_id + self.missing = missing + + +class ImageOpInputError(ValueError): + """Raised when the input file is not an image the operation can read.""" + + +class ImageOpExecutionError(RuntimeError): + """Raised when an operation, or installing one of its requirements, fails.""" + + +class Requirement(Protocol): + """Something an op needs that is installed on first use.""" + + kind: str # "runtime" | "model" + id: str # e.g. "onnxruntime", "birefnet-lite" + label: str # shown in progress messages + + def is_ready(self) -> bool: ... + + def download_bytes(self) -> Optional[int]: + """Approximate bytes still to fetch (best effort), None when unknown.""" + + def install(self, progress: ProgressCallback) -> None: + """Fetch and install; idempotent. Raises ImageOpExecutionError on failure.""" + + +@dataclass(frozen=True) +class ImageOpContext: + """Where an operation writes its outputs (read from the registry at call time).""" + + workspace_dir: Path + + @property + def images_dir(self) -> Path: + # Same folder Electron saves pasted images to (electron/main/workspace-images.ts). + return self.workspace_dir / "Images" + + +@dataclass(frozen=True) +class ImageOpResult: + path: Path + mask_path: Optional[Path] = None + details: Mapping[str, Any] = field(default_factory=dict) + + +ImageOpFn = Callable[[Path, Mapping[str, Any], ImageOpContext], ImageOpResult] + + +@dataclass(frozen=True) +class ImageOp: + id: str + label: str + params_schema: tuple[Mapping[str, Any], ...] + fn: ImageOpFn + requirements: tuple[Requirement, ...] = () + + def missing(self) -> list[str]: + """Kinds of the requirements not installed yet, in install order.""" + return [req.kind for req in self.requirements if not req.is_ready()] + + def describe(self) -> dict[str, Any]: + return { + "id": self.id, + "label": self.label, + "params_schema": deepcopy(list(self.params_schema)), + "requirements": [ + {"kind": req.kind, "id": req.id, "label": req.label} + for req in self.requirements + ], + } diff --git a/api/tests/test_image_ops_registry.py b/api/tests/test_image_ops_registry.py new file mode 100644 index 000000000..d7874f4b9 --- /dev/null +++ b/api/tests/test_image_ops_registry.py @@ -0,0 +1,137 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from services.image_ops import ( + ImageOp, + ImageOpContext, + ImageOpNotFoundError, + ImageOpNotReadyError, + ImageOpResult, + ImageOpsRegistry, + image_ops_registry, +) + + +class FakeRequirement: + def __init__(self, kind: str, ready: bool, size: int | None = 10, id: str | None = None) -> None: + self.kind = kind + self.id = id or f"fake-{kind}" + self.label = f"Fake {kind}" + self.ready = ready + self.size = size + self.installs = 0 + + def is_ready(self) -> bool: + return self.ready + + def download_bytes(self): + return None if self.ready else self.size + + def install(self, progress) -> None: + self.installs += 1 + progress({"percent": 50, "status": "half"}) + self.ready = True + progress({"percent": 100, "status": f"{self.label} ready"}) + + +SCHEMA = ( + {"id": "strength", "label": "Strength", "type": "float", "default": 0.0, "min": 0.0, "max": 0.95}, + {"id": "count", "label": "Count", "type": "int", "default": 2, "min": 1, "max": 5}, + {"id": "flag", "label": "Flag", "type": "boolean", "default": False}, +) + + +def make_op(requirements=(), fn=None) -> ImageOp: + return ImageOp( + id="example", + label="Example", + params_schema=SCHEMA, + fn=fn or (lambda path, params, ctx: ImageOpResult(path=path, details={"params": params})), + requirements=tuple(requirements), + ) + + +class ImageOpsRegistryTests(unittest.TestCase): + def test_builtin_metadata_is_serializable(self) -> None: + described = image_ops_registry.describe() + self.assertEqual([d["id"] for d in described], ["remove_background"]) + op = described[0] + self.assertEqual(op["label"], "Remove background") + self.assertEqual( + [(r["kind"], r["id"]) for r in op["requirements"]], + [("runtime", "onnxruntime"), ("model", "birefnet-lite")], + ) + (param,) = op["params_schema"] + self.assertEqual( + {k: param[k] for k in ("id", "type", "default", "min", "max")}, + {"id": "edge_threshold", "type": "float", "default": 0.0, "min": 0.0, "max": 0.95}, + ) + self.assertTrue(param["tooltip"]) + json.dumps(described) + + def test_defaults_are_applied_without_mutating_schema(self) -> None: + registry = ImageOpsRegistry([make_op()]) + self.assertEqual(registry.resolve_params("example", None), {"strength": 0.0, "count": 2, "flag": False}) + resolved = registry.resolve_params("example", {"strength": 0.3}) + self.assertEqual(resolved["strength"], 0.3) + self.assertEqual(SCHEMA[0]["default"], 0.0) + + def test_numbers_are_clamped_and_coerced(self) -> None: + registry = ImageOpsRegistry([make_op()]) + self.assertEqual(registry.resolve_params("example", {"strength": 2})["strength"], 0.95) + self.assertIsInstance(registry.resolve_params("example", {"strength": 1})["strength"], float) + self.assertEqual(registry.resolve_params("example", {"strength": -1.0})["strength"], 0.0) + self.assertEqual(registry.resolve_params("example", {"count": 9.0})["count"], 5) + + def test_invalid_params_are_rejected(self) -> None: + registry = ImageOpsRegistry([make_op()]) + for bad in ({"strength": "high"}, {"strength": True}, {"count": 1.5}, {"flag": 1}): + with self.subTest(bad=bad), self.assertRaises(TypeError): + registry.resolve_params("example", bad) + with self.assertRaises(ValueError): + registry.resolve_params("example", {"strength": float("nan")}) + with self.assertRaisesRegex(ValueError, "Unknown parameter"): + registry.resolve_params("example", {"nope": 1}) + + def test_ids_are_validated_and_unique(self) -> None: + registry = ImageOpsRegistry([make_op()]) + with self.assertRaises(ValueError): + registry.register(make_op()) + with self.assertRaises(ValueError): + ImageOpsRegistry([ImageOp(id="Bad Id", label="x", params_schema=(), fn=lambda *a: None)]) + with self.assertRaises(ImageOpNotFoundError): + registry.get("missing") + + def test_missing_lists_unready_requirement_kinds_in_order(self) -> None: + op = make_op([FakeRequirement("runtime", False), FakeRequirement("model", True)]) + self.assertEqual(op.missing(), ["runtime"]) + op = make_op([FakeRequirement("runtime", False), FakeRequirement("model", False)]) + self.assertEqual(op.missing(), ["runtime", "model"]) + + def test_run_refuses_when_not_ready(self) -> None: + calls = [] + op = make_op([FakeRequirement("model", False)], fn=lambda *a: calls.append(a)) + registry = ImageOpsRegistry([op]) + with tempfile.TemporaryDirectory() as tmp: + image = Path(tmp) / "a.png" + image.write_bytes(b"x") + with self.assertRaises(ImageOpNotReadyError) as caught: + registry.run("example", image, {}, ImageOpContext(Path(tmp))) + self.assertEqual(caught.exception.missing, ["model"]) + self.assertEqual(calls, []) + + def test_run_passes_resolved_params(self) -> None: + registry = ImageOpsRegistry([make_op([FakeRequirement("model", True)])]) + with tempfile.TemporaryDirectory() as tmp: + image = Path(tmp) / "a.png" + image.write_bytes(b"x") + result = registry.run("example", image, {"count": 3}, ImageOpContext(Path(tmp))) + self.assertEqual(result.details["params"], {"strength": 0.0, "count": 3, "flag": False}) + with self.assertRaises(FileNotFoundError): + registry.run("example", Path(tmp) / "missing.png", {}, ImageOpContext(Path(tmp))) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_image_ops_remove_background.py b/api/tests/test_image_ops_remove_background.py new file mode 100644 index 000000000..d709842f1 --- /dev/null +++ b/api/tests/test_image_ops_remove_background.py @@ -0,0 +1,253 @@ +import importlib.util +import os +import tempfile +import time +import unittest +from datetime import datetime +from pathlib import Path +from unittest.mock import patch + +import numpy as np +from PIL import Image + +import services.generator_registry as registry +from services.image_ops import ImageOpContext, ImageOpInputError, image_ops_registry +from services.image_ops import birefnet, builtin, outputs +from services.image_ops.requirements import BIREFNET_LITE + +HAS_ONNXRUNTIME = importlib.util.find_spec("onnxruntime") is not None + + +def _open(path) -> Image.Image: + with Image.open(path) as image: + image.load() + return image + + +class _Input: + def __init__(self, name: str) -> None: + self.name = name + + +class FakeSession: + """Stands in for an onnxruntime InferenceSession: the left half of the + frame is foreground (logit +8), the right half background (logit -8).""" + + def __init__(self) -> None: + self.feeds = [] + + def get_inputs(self): + return [_Input("input_image")] + + def run(self, output_names, feeds): + self.feeds.append(feeds) + logits = np.full((1, 1, birefnet.INPUT_SIZE, birefnet.INPUT_SIZE), -8.0, dtype=np.float32) + logits[..., : birefnet.INPUT_SIZE // 2] = 8.0 + return [logits] + + +class AlphaMathTests(unittest.TestCase): + def test_zero_threshold_keeps_soft_alpha(self) -> None: + alpha = np.array([[0.0, 0.25, 0.5, 1.0]], dtype=np.float32) + np.testing.assert_allclose(birefnet.apply_edge_threshold(alpha, 0.0), alpha) + + def test_threshold_hardens(self) -> None: + alpha = np.array([[0.0, 0.25, 0.5, 0.75, 1.0]], dtype=np.float32) + np.testing.assert_allclose(birefnet.apply_edge_threshold(alpha, 0.5), [[0.0, 0.0, 0.0, 0.5, 1.0]]) + np.testing.assert_allclose( + birefnet.apply_edge_threshold(alpha, 0.2), np.clip((alpha - 0.2) / 0.8, 0, 1), rtol=1e-6 + ) + + def test_to_uint8_rounds_and_clamps(self) -> None: + self.assertEqual(birefnet.to_uint8(np.array([-1.0, 0.0, 0.5, 1.0, 2.0])).tolist(), [0, 0, 128, 255, 255]) + + def test_preprocess_normalizes_to_nchw(self) -> None: + tensor = birefnet.preprocess(Image.new("RGB", (8, 6), (255, 0, 128))) + self.assertEqual(tensor.shape, (1, 3, 1024, 1024)) + self.assertEqual(tensor.dtype, np.float32) + np.testing.assert_allclose(tensor[0, :, 0, 0], [(1 - 0.485) / 0.229, (0 - 0.456) / 0.224, (128 / 255 - 0.406) / 0.225], rtol=1e-5) + + def test_predict_alpha_applies_sigmoid_and_resizes_back(self) -> None: + alpha = birefnet.predict_alpha(FakeSession(), Image.new("RGB", (40, 10))) + self.assertEqual(alpha.shape, (10, 40)) + self.assertGreater(alpha[:, :19].min(), 0.99) + self.assertLess(alpha[:, 21:].max(), 0.01) + + +class OutputNamingTests(unittest.TestCase): + def test_sanitize_stem(self) -> None: + self.assertEqual(outputs.sanitize_stem("My Photo (1).JPG"), "My-Photo-1") + self.assertEqual(outputs.sanitize_stem("Éclair.png"), "Eclair") + self.assertEqual(outputs.sanitize_stem("con.png"), "image") + self.assertEqual(outputs.sanitize_stem("!!!.png"), "image") + self.assertEqual(len(outputs.sanitize_stem("x" * 200 + ".png")), 60) + + def test_names_share_stamp_and_suffix(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + now = datetime(2026, 1, 2, 3, 4, 5) + paths = outputs.write_new_files(Path(tmp) / "Images", "cat", [("nobg", b"a"), ("mask", b"b")], now=now, token=lambda: "abcdef") + self.assertEqual([p.name for p in paths], ["cat-nobg-20260102-030405-abcdef.png", "cat-mask-20260102-030405-abcdef.png"]) + self.assertEqual(paths[0].read_bytes(), b"a") + + def test_never_overwrites(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + images = Path(tmp) + now = datetime(2026, 1, 2, 3, 4, 5) + # Only the mask name of the first suffix is taken: the set moves on as a pair. + taken = images / "cat-mask-20260102-030405-aaaaaa.png" + taken.write_bytes(b"keep") + tokens = iter(["aaaaaa", "bbbbbb"]) + paths = outputs.write_new_files(images, "cat", [("nobg", b"a"), ("mask", b"b")], now=now, token=lambda: next(tokens)) + self.assertEqual([p.name for p in paths], ["cat-nobg-20260102-030405-bbbbbb.png", "cat-mask-20260102-030405-bbbbbb.png"]) + self.assertEqual(taken.read_bytes(), b"keep") + self.assertFalse((images / "cat-nobg-20260102-030405-aaaaaa.png").exists()) + + def test_gives_up_after_too_many_clashes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + now = datetime(2026, 1, 2, 3, 4, 5) + (Path(tmp) / "cat-nobg-20260102-030405-aaaaaa.png").write_bytes(b"x") + with self.assertRaises(FileExistsError): + outputs.write_new_files(Path(tmp), "cat", [("nobg", b"a")], now=now, token=lambda: "aaaaaa", attempts=3) + + +class SessionCacheTests(unittest.TestCase): + def test_session_is_built_once_and_rebuilt_when_the_file_changes(self) -> None: + built = [] + cache = birefnet.SessionCache(lambda path: built.append(path) or object()) + with tempfile.TemporaryDirectory() as tmp: + model = Path(tmp) / "model.onnx" + model.write_bytes(b"a") + first = cache.get(model) + self.assertIs(cache.get(model), first) + time.sleep(0.01) + model.write_bytes(b"bb") + os.utime(model, ns=(time.time_ns(), time.time_ns() + 10_000_000)) + self.assertIsNot(cache.get(model), first) + self.assertEqual(len(built), 2) + + +class RemoveBackgroundTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + root = Path(self._tmp.name) + self.workspace = root / "workspace" + self.models = root / "models" + self.session = FakeSession() + self._patches = [ + patch.object(registry, "WORKSPACE_DIR", self.workspace), + patch.object(registry, "MODELS_DIR", self.models), + patch.object(birefnet, "session_cache", birefnet.SessionCache(lambda path: self.session)), + ] + for p in self._patches: + p.start() + # The model "is downloaded" (right size), the runtime is reported present. + BIREFNET_LITE.path.parent.mkdir(parents=True) + with open(BIREFNET_LITE.path, "wb") as f: + f.truncate(BIREFNET_LITE.size) + self._patches.append(patch("importlib.util.find_spec", side_effect=lambda name, *a: object() if name == "onnxruntime" else _real_find_spec(name, *a))) + self._patches[-1].start() + self.context = ImageOpContext(workspace_dir=self.workspace) + + def tearDown(self) -> None: + for p in reversed(self._patches): + p.stop() + self._tmp.cleanup() + + def _photo(self, name: str = "photo.jpg", size=(64, 32), mode="RGB", **save) -> Path: + rng = np.random.default_rng(0) + pixels = rng.integers(0, 255, (size[1], size[0], 3), dtype=np.uint8) + image = Image.fromarray(pixels).convert(mode) + path = self.workspace / name + path.parent.mkdir(parents=True, exist_ok=True) + image.save(path, **save) + return path + + def test_end_to_end_with_fake_session(self) -> None: + photo = self._photo("photo.png") + result = image_ops_registry.run("remove_background", photo, {}, self.context) + + self.assertEqual(result.path.parent, self.workspace / "Images") + self.assertRegex(result.path.name, r"^photo-nobg-\d{8}-\d{6}-[0-9a-f]{6}\.png$") + self.assertRegex(result.mask_path.name, r"^photo-mask-\d{8}-\d{6}-[0-9a-f]{6}\.png$") + out = _open(result.path) + mask = _open(result.mask_path) + self.assertEqual((out.mode, out.size), ("RGBA", (64, 32))) + self.assertEqual((mask.mode, mask.size), ("L", (64, 32))) + alpha = np.asarray(out.getchannel("A")) + self.assertTrue((alpha[:, :30] == 255).all()) + self.assertTrue((alpha[:, 34:] == 0).all()) + np.testing.assert_array_equal(np.asarray(mask), alpha) + # Original pixels are kept untouched under the alpha. + np.testing.assert_array_equal(np.asarray(out.convert("RGB")), np.asarray(_open(photo).convert("RGB"))) + self.assertEqual(self.session.feeds[0]["input_image"].shape, (1, 3, 1024, 1024)) + + def test_two_runs_never_overwrite(self) -> None: + photo = self._photo("photo.png") + first = image_ops_registry.run("remove_background", photo, {}, self.context) + second = image_ops_registry.run("remove_background", photo, {}, self.context) + self.assertNotEqual(first.path, second.path) + self.assertEqual(len(list((self.workspace / "Images").iterdir())), 4) + + def test_edge_threshold_is_applied(self) -> None: + class HalfSession(FakeSession): + def run(self, output_names, feeds): + return [np.zeros((1, 1, 1024, 1024), dtype=np.float32)] # sigmoid(0) = 0.5 everywhere + + birefnet.session_cache = birefnet.SessionCache(lambda path: HalfSession()) + photo = self._photo("photo.png") + soft = image_ops_registry.run("remove_background", photo, {"edge_threshold": 0}, self.context) + hard = image_ops_registry.run("remove_background", photo, {"edge_threshold": 0.6}, self.context) + self.assertTrue((np.asarray(_open(soft.mask_path)) == 128).all()) + self.assertTrue((np.asarray(_open(hard.mask_path)) == 0).all()) + + def test_exif_orientation_is_applied(self) -> None: + exif = Image.Exif() + exif[0x0112] = 6 # rotate 90° CW on display: 64x32 stored -> 32x64 shown + photo = self._photo("phone.jpg", exif=exif.tobytes()) + result = image_ops_registry.run("remove_background", photo, {}, self.context) + self.assertEqual(_open(result.path).size, (32, 64)) + + def test_any_mode_is_accepted_and_existing_transparency_is_kept(self) -> None: + for mode in ("L", "P", "RGBA"): + with self.subTest(mode=mode): + photo = self._photo(f"in-{mode}.png", mode=mode) + result = image_ops_registry.run("remove_background", photo, {}, self.context) + self.assertEqual(_open(result.path).mode, "RGBA") + + rgba = Image.new("RGBA", (64, 32), (200, 10, 10, 0)) # fully transparent input + path = self.workspace / "clear.png" + rgba.save(path) + result = image_ops_registry.run("remove_background", path, {}, self.context) + self.assertTrue((np.asarray(_open(result.path).getchannel("A")) == 0).all()) + + def test_unreadable_image_is_an_input_error(self) -> None: + bogus = self.workspace / "bogus.png" + bogus.parent.mkdir(parents=True, exist_ok=True) + bogus.write_bytes(b"not an image") + with self.assertRaises(ImageOpInputError): + image_ops_registry.run("remove_background", bogus, {}, self.context) + self.assertFalse((self.workspace / "Images").exists()) + + +_real_find_spec = importlib.util.find_spec + + +@unittest.skipUnless(HAS_ONNXRUNTIME, "onnxruntime is not installed") +class RealModelTests(unittest.TestCase): + """Runs the real model when this environment has onnxruntime and the + weights are already downloaded; never downloads anything itself.""" + + def test_real_inference_produces_a_mask(self) -> None: + if not BIREFNET_LITE.is_ready(): + self.skipTest("BiRefNet-lite weights are not downloaded") + session = birefnet.SessionCache().get(BIREFNET_LITE.path) + image = Image.new("RGB", (128, 96), (240, 240, 240)) + image.paste((200, 30, 30), (40, 24, 88, 72)) + alpha = birefnet.predict_alpha(session, image) + self.assertEqual(alpha.shape, (96, 128)) + self.assertGreaterEqual(alpha.min(), 0.0) + self.assertLessEqual(alpha.max(), 1.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_image_ops_requirements.py b/api/tests/test_image_ops_requirements.py new file mode 100644 index 000000000..d12cae022 --- /dev/null +++ b/api/tests/test_image_ops_requirements.py @@ -0,0 +1,177 @@ +import hashlib +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import services.generator_registry as registry +from services.image_ops import ImageOpExecutionError +from services.image_ops import requirements as reqs + + +def _model_req(content: bytes) -> reqs.HFFileRequirement: + return reqs.HFFileRequirement( + id="tiny-model", + label="Tiny model", + repo_id="org/repo", + filename="onnx/model.onnx", + revision="abc", + size=len(content), + sha256=hashlib.sha256(content).hexdigest(), + ) + + +class HFFileRequirementTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.models = Path(self._tmp.name) / "models" + self._patch = patch.object(registry, "MODELS_DIR", self.models) + self._patch.start() + + def tearDown(self) -> None: + self._patch.stop() + self._tmp.cleanup() + + def test_path_follows_models_dir_at_call_time(self) -> None: + req = _model_req(b"weights") + self.assertEqual(req.path, self.models / "tiny-model" / "model.onnx") + moved = Path(self._tmp.name) / "moved" + with patch.object(registry, "MODELS_DIR", moved): + self.assertEqual(req.path, moved / "tiny-model" / "model.onnx") + + def test_ready_only_when_file_has_expected_size(self) -> None: + req = _model_req(b"weights") + self.assertFalse(req.is_ready()) + self.assertEqual(req.download_bytes(), 7) + req.path.parent.mkdir(parents=True) + req.path.write_bytes(b"wei") # truncated + self.assertFalse(req.is_ready()) + req.path.write_bytes(b"weights") + self.assertTrue(req.is_ready()) + self.assertIsNone(req.download_bytes()) + + def test_download_bytes_discounts_a_resumable_part_file(self) -> None: + req = _model_req(b"0123456789") + req.path.parent.mkdir(parents=True) + req.path.with_suffix(".onnx.part").write_bytes(b"0123") + self.assertEqual(req.download_bytes(), 6) + + def _fake_download(self, content: bytes): + calls = [] + + def download(*, url, filename, dest_dir, progress_cb, **_): + calls.append(url) + progress_cb({"status": "Downloading… 50%", "bytesDownloaded": len(content) // 2, "totalBytes": len(content)}) + Path(dest_dir).mkdir(parents=True, exist_ok=True) # as the real downloader does + (Path(dest_dir) / filename).write_bytes(content) + return len(content) + + return download, calls + + def test_install_downloads_pinned_revision_and_verifies(self) -> None: + req = _model_req(b"weights!") + download, calls = self._fake_download(b"weights!") + frames = [] + with patch("routers.model._download_file_streamed", download): + req.install(frames.append) + self.assertTrue(req.is_ready()) + self.assertIn("org/repo", calls[0]) + self.assertIn("/abc/", calls[0]) # pinned revision + self.assertEqual(frames[-1]["percent"], 100) + self.assertEqual(frames[0]["percent"], 50) + self.assertEqual(frames[0]["status"], "Downloading Tiny model… 50%") + + def test_install_rejects_a_corrupted_download(self) -> None: + req = _model_req(b"weights") + download, _ = self._fake_download(b"WEIGHTS") # right size, wrong bytes + with patch("routers.model._download_file_streamed", download): + with self.assertRaisesRegex(ImageOpExecutionError, "checksum"): + req.install(lambda msg: None) + self.assertFalse(req.path.exists()) + + def test_install_reports_network_failure_clearly(self) -> None: + req = _model_req(b"weights") + + def download(**_): + raise RuntimeError("Failed to download model.onnx: ") + + with patch("routers.model._download_file_streamed", download): + with self.assertRaisesRegex(ImageOpExecutionError, "internet connection"): + req.install(lambda msg: None) + + def test_install_is_a_noop_when_ready(self) -> None: + req = _model_req(b"weights") + req.path.parent.mkdir(parents=True) + req.path.write_bytes(b"weights") + with patch("routers.model._download_file_streamed", side_effect=AssertionError("no download")): + req.install(lambda msg: None) + + def test_offline_mode_fails_without_network(self) -> None: + req = _model_req(b"weights") + with patch.dict("os.environ", {"HF_HUB_OFFLINE": "1"}), \ + patch("routers.model._download_file_streamed", side_effect=AssertionError("no download")): + with self.assertRaisesRegex(ImageOpExecutionError, "HF_HUB_OFFLINE"): + req.install(lambda msg: None) + + +class FakePopen: + def __init__(self, lines, code): + self.stdout = iter(line + "\n" for line in lines) + self._code = code + + def wait(self): + return self._code + + +class PipPackageRequirementTests(unittest.TestCase): + def setUp(self) -> None: + self.req = reqs.PipPackageRequirement( + id="definitely_not_installed_pkg", label="Fake runtime", + pip_spec="definitely_not_installed_pkg==1.0", approx_bytes=123, + ) + + def test_not_ready_reports_approximate_size(self) -> None: + self.assertFalse(self.req.is_ready()) + self.assertEqual(self.req.download_bytes(), 123) + + def test_already_installed_skips_pip(self) -> None: + req = reqs.PipPackageRequirement(id="json", label="json", pip_spec="json", approx_bytes=1) + self.assertTrue(req.is_ready()) + self.assertIsNone(req.download_bytes()) + with patch.object(subprocess, "Popen", side_effect=AssertionError("pip must not run")): + req.install(lambda msg: None) + + def test_pip_failure_raises_a_clear_error(self) -> None: + lines = [ + "Collecting definitely_not_installed_pkg==1.0", + "WARNING: Retrying ... NewConnectionError: Failed to establish a new connection", + "ERROR: Could not find a version that satisfies the requirement definitely_not_installed_pkg==1.0", + "ERROR: No matching distribution found for definitely_not_installed_pkg==1.0", + ] + frames = [] + with patch.object(subprocess, "Popen", return_value=FakePopen(lines, 1)) as popen: + with self.assertRaises(ImageOpExecutionError) as caught: + self.req.install(frames.append) + message = str(caught.exception) + self.assertIn("pip exited with code 1", message) + self.assertIn("internet connection", message) + self.assertIn("No matching distribution", message) + command = popen.call_args.args[0] + self.assertEqual(command[1:4], ["-m", "pip", "install"]) + self.assertIn("definitely_not_installed_pkg==1.0", command) + self.assertTrue(any(f["percent"] > 0 for f in frames)) + + def test_success_without_importable_module_is_an_error(self) -> None: + with patch.object(subprocess, "Popen", return_value=FakePopen(["Successfully installed x"], 0)): + with self.assertRaisesRegex(ImageOpExecutionError, "cannot be found"): + self.req.install(lambda msg: None) + + def test_onnxruntime_is_pinned_and_not_in_requirements_txt(self) -> None: + self.assertRegex(reqs.ONNXRUNTIME.pip_spec, r"^onnxruntime==\d+\.\d+\.\d+$") + requirements_txt = (Path(__file__).resolve().parents[1] / "requirements.txt").read_text() + self.assertNotIn("onnxruntime", requirements_txt) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_image_ops_router.py b/api/tests/test_image_ops_router.py new file mode 100644 index 000000000..561a6cbb9 --- /dev/null +++ b/api/tests/test_image_ops_router.py @@ -0,0 +1,235 @@ +import asyncio +import json +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import patch + +import numpy as np +from fastapi import HTTPException +from PIL import Image + +import routers.image as image_router +import services.generator_registry as registry +from services.image_ops import ImageOpExecutionError, ImageOpResult, ImageOpsRegistry +from test_image_ops_registry import FakeRequirement, make_op + + +async def _collect(response) -> list[dict]: + frames = [] + async for chunk in response.body_iterator: + text = chunk.decode() if isinstance(chunk, bytes) else chunk + for part in text.split("\n\n"): + if part.startswith("data: "): + frames.append(json.loads(part[6:])) + return frames + + +class ImageRouterTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.workspace = Path(self._tmp.name) / "workspace" + self.workspace.mkdir() + self.runtime = FakeRequirement("runtime", False, size=100) + self.model = FakeRequirement("model", False, size=300) + self.calls = [] + + def fn(path, params, ctx): + self.calls.append((path, params, ctx)) + out = ctx.images_dir / "out-nobg.png" + mask = ctx.images_dir / "out-mask.png" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(b"png") + mask.write_bytes(b"png") + return ImageOpResult(path=out, mask_path=mask) + + op = make_op([self.runtime, self.model], fn=fn) + self.registry = ImageOpsRegistry([op]) + self._patches = [ + patch.object(registry, "WORKSPACE_DIR", self.workspace), + patch.object(image_router, "image_ops_registry", self.registry), + patch.dict(image_router._prepares, clear=True), + ] + for p in self._patches: + p.start() + self.image = self.workspace / "photo.jpg" + Image.fromarray(np.zeros((4, 4, 3), dtype=np.uint8)).save(self.image) + + def tearDown(self) -> None: + for p in reversed(self._patches): + p.stop() + self._tmp.cleanup() + + def _run(self, path: str, params=None): + return asyncio.run(image_router.run_image_operation("example", image_router.ImageOpRequest(path=path, params=params or {}))) + + def test_ops_lists_the_registry(self) -> None: + body = image_router.list_image_operations() + self.assertEqual([op["id"] for op in body["ops"]], ["example"]) + self.assertEqual([r["kind"] for r in body["ops"][0]["requirements"]], ["runtime", "model"]) + + def test_builtin_op_is_exposed(self) -> None: + from services.image_ops import image_ops_registry + self.assertIn("remove_background", [op["id"] for op in image_ops_registry.describe()]) + + def test_status_reports_missing_and_bytes(self) -> None: + self.assertEqual( + image_router.image_operation_status("example"), + {"ready": False, "missing": ["runtime", "model"], "download_bytes": 400, "preparing": False}, + ) + self.runtime.ready = True + self.assertEqual(image_router.image_operation_status("example")["download_bytes"], 300) + self.model.ready = True + self.assertEqual( + image_router.image_operation_status("example"), + {"ready": True, "missing": [], "download_bytes": None, "preparing": False}, + ) + + def test_unknown_op_is_404(self) -> None: + for call in ( + lambda: image_router.image_operation_status("nope"), + lambda: asyncio.run(image_router.run_image_operation("nope", image_router.ImageOpRequest(path=str(self.image)))), + lambda: asyncio.run(image_router.prepare_image_operation("nope")), + ): + with self.assertRaises(HTTPException) as caught: + call() + self.assertEqual(caught.exception.status_code, 404) + + def test_run_when_not_ready_is_409_with_missing(self) -> None: + self.runtime.ready = True + with self.assertRaises(HTTPException) as caught: + self._run(str(self.image)) + self.assertEqual(caught.exception.status_code, 409) + self.assertEqual(caught.exception.detail, {"code": "not_ready", "missing": ["model"]}) + self.assertEqual(self.calls, []) + + def test_run_rejects_non_images_and_missing_files(self) -> None: + self.runtime.ready = self.model.ready = True + other = self.workspace / "model.glb" + other.write_bytes(b"glb") + cases = [(str(other), 400), ("../outside.png", 400), ("", 400), (str(self.workspace / "gone.png"), 404)] + for path, status in cases: + with self.subTest(path=path), self.assertRaises(HTTPException) as caught: + self._run(path) + self.assertEqual(caught.exception.status_code, status) + self.assertEqual(self.calls, []) + + def test_run_accepts_absolute_and_workspace_relative_paths(self) -> None: + self.runtime.ready = self.model.ready = True + for path in (str(self.image), "photo.jpg"): + with self.subTest(path=path): + body = self._run(path, {"strength": 0.5}) + self.assertEqual(Path(body["path"]), (self.workspace / "Images" / "out-nobg.png").resolve()) + self.assertEqual(Path(body["mask_path"]), (self.workspace / "Images" / "out-mask.png").resolve()) + self.assertEqual(body["url"], "/workspace/Images/out-nobg.png") + path, params, ctx = self.calls[-1] + self.assertEqual(path, self.image.resolve()) + self.assertEqual(params["strength"], 0.5) + self.assertEqual(ctx.workspace_dir, self.workspace) + + def test_run_maps_bad_params_and_failures(self) -> None: + self.runtime.ready = self.model.ready = True + with self.assertRaises(HTTPException) as caught: + self._run(str(self.image), {"strength": "x"}) + self.assertEqual(caught.exception.status_code, 400) + + def boom(*_): + raise ImageOpExecutionError("inference failed") + + self.registry._operations["example"] = make_op([self.runtime, self.model], fn=boom) + with self.assertRaises(HTTPException) as caught: + self._run(str(self.image)) + self.assertEqual(caught.exception.status_code, 500) + + def test_run_happens_off_the_event_loop(self) -> None: + self.runtime.ready = self.model.ready = True + threads = [] + self.registry._operations["example"] = make_op( + [self.runtime, self.model], + fn=lambda p, params, ctx: threads.append(threading.current_thread()) or ImageOpResult(path=p), + ) + self._run(str(self.image)) + self.assertIsNot(threads[0], threading.main_thread()) + + def test_prepare_when_ready_is_a_single_done_frame(self) -> None: + self.runtime.ready = self.model.ready = True + + async def go(): + return await _collect(await image_router.prepare_image_operation("example")) + + self.assertEqual(asyncio.run(go()), [{"percent": 100, "status": "done"}]) + self.assertEqual(self.runtime.installs + self.model.installs, 0) + + def test_prepare_installs_runtime_then_model_with_weighted_progress(self) -> None: + order = [] + for req in (self.runtime, self.model): + original = req.install + req.install = lambda progress, _o=original, _r=req: (order.append(_r.kind), _o(progress)) + + async def go(): + return await _collect(await image_router.prepare_image_operation("example")) + + frames = asyncio.run(go()) + self.assertEqual(order, ["runtime", "model"]) + self.assertEqual(frames[-1], {"percent": 100, "status": "done"}) + percents = [f["percent"] for f in frames] + self.assertEqual(percents, sorted(percents)) + # runtime is 100 of 400 bytes: its half-way frame is 12-13 %. + half_runtime = next(f for f in frames if f.get("step") == "runtime" and f["status"] == "half") + self.assertEqual(half_runtime["percent"], 12) + half_model = next(f for f in frames if f.get("step") == "model" and f["status"] == "half") + self.assertEqual(half_model["percent"], 62) + self.assertEqual(self.registry.get("example").missing(), []) + + def test_prepare_skips_requirements_already_installed(self) -> None: + self.runtime.ready = True + + async def go(): + return await _collect(await image_router.prepare_image_operation("example")) + + asyncio.run(go()) + self.assertEqual((self.runtime.installs, self.model.installs), (0, 1)) + + def test_prepare_failure_is_an_error_frame(self) -> None: + def fail(progress): + raise ImageOpExecutionError("Installing ONNX Runtime failed (pip exited with code 1).") + + self.runtime.install = fail + + async def go(): + return await _collect(await image_router.prepare_image_operation("example")) + + frames = asyncio.run(go()) + self.assertEqual(frames[-1], {"error": "Installing ONNX Runtime failed (pip exited with code 1)."}) + self.assertEqual(self.model.installs, 0) + + def test_a_second_prepare_joins_the_running_one(self) -> None: + gate = threading.Event() + original = self.model.install + + def slow_install(progress): + gate.wait(5) + original(progress) + + self.model.install = slow_install + + async def go(): + first = await image_router.prepare_image_operation("example") + first_frames = asyncio.ensure_future(_collect(first)) + await asyncio.sleep(0.05) + self.assertTrue(image_router.image_operation_status("example")["preparing"]) + second = await image_router.prepare_image_operation("example") + second_frames = asyncio.ensure_future(_collect(second)) + await asyncio.sleep(0.05) + gate.set() + return await first_frames, await second_frames + + first, second = asyncio.run(go()) + self.assertEqual(first[-1]["status"], "done") + self.assertEqual(second[-1]["status"], "done") + self.assertEqual((self.runtime.installs, self.model.installs), (1, 1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/package.json b/package.json index 063a2f9fd..239b30074 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/main/gpu-memory.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/shared/pages/startPage.test.ts src/areas/home/recents.test.ts src/areas/workflows/workflowRunRecord.test.ts src/areas/queue/queueModel.test.ts electron/main/workspace-images.test.ts src/areas/image-studio/imageStudioStore.test.ts src/shared/utils/imageInput.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/library/libraryView.test.ts src/areas/library/gridWindow.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts src/shared/thumbnails/assetThumbnailQueue.test.ts src/shared/thumbnails/assetThumbnailStore.test.ts src/shared/components/agent/clientTools.test.ts src/shared/components/agent/agentContext.test.ts src/shared/utils/ollamaHealth.test.ts src/shared/utils/notification.test.ts src/shared/utils/agentConversations.test.ts electron/main/artifact-registry-service.test.ts electron/main/asset-thumbnail-cache.test.ts electron/main/extension-path-guard.test.ts electron/main/storage-directory-guard.test.ts electron/main/storage-ownership.test.ts electron/main/startup-error.test.ts electron/main/api-port.test.ts electron/main/agent-conversations.test.ts electron/main/gpu-memory.test.ts electron/preload/artifact-registry-preload.test.ts src/shared/stores/meshHistory.test.ts src/areas/generate/meshEditPanel.test.ts src/areas/generate/sculpt/SculptMesh.test.ts src/areas/generate/sculpt/SculptMesh.perf.test.ts src/areas/generate/sculpt/sculptLogic.test.ts src/shared/pages/pageRegistry.test.ts src/shared/pages/startPage.test.ts src/areas/home/recents.test.ts src/areas/workflows/workflowRunRecord.test.ts src/areas/queue/queueModel.test.ts electron/main/workspace-images.test.ts src/areas/image-studio/imageStudioStore.test.ts src/areas/image-studio/imageOps.test.ts src/shared/utils/imageInput.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/components/EditPanel.tsx b/src/areas/generate/components/EditPanel.tsx index ed44b8128..2e22c45df 100644 --- a/src/areas/generate/components/EditPanel.tsx +++ b/src/areas/generate/components/EditPanel.tsx @@ -1,27 +1,21 @@ import { useCallback, useEffect, useState } from 'react' import { useAppStore } from '@shared/stores/appStore' import { useApi } from '@shared/hooks/useApi' -import { Tooltip } from '@shared/components/ui' +import { ParamFields } from '@shared/components/ui/ParamFields' import { INSPECT_OP_ID, - LOG_SLIDER_STEPS, apiErrorMessage, - coerceNumber, defaultParamValues, - fromSliderPosition, groupMeshOps, inspectionBadges, isMeshUpdate, meshPathFromUrl, - paramControl, requiresConfirmation, stepSummaries, summarizeDetails, - toSliderPosition, type BadgeState, type MeshOpDescriptor, type MeshOpResponse, - type ParamControl, } from '../meshEditPanel' import { commitSculpt, useSculptStore } from '../sculpt/sculptStore' import SculptSection from './SculptSection' @@ -63,178 +57,6 @@ function Chevron({ open }: { open: boolean }): JSX.Element { ) } -// ─── Param controls ─────────────────────────────────────────────────────────── - -const INPUT_CLASS = - 'bg-zinc-800 border border-zinc-700 rounded-lg px-2 py-1 text-xs text-zinc-200 focus:outline-none focus:border-violet-500 transition-colors disabled:opacity-50' - -/** Number text field that commits (clamped) on blur or Enter. */ -function NumberField({ control, value, onChange, disabled }: { - control: Extract - value: number - onChange: (v: number) => void - disabled: boolean -}): JSX.Element { - const [text, setText] = useState(String(value)) - useEffect(() => { setText(String(value)) }, [value]) - - function commit() { - const parsed = Number(text.replace(',', '.')) - if (text.trim() === '' || !Number.isFinite(parsed)) { setText(String(value)); return } - const next = coerceNumber(control, parsed) - setText(String(next)) - if (next !== value) onChange(next) - } - - return ( - setText(e.target.value)} - onBlur={commit} - onKeyDown={(e) => { if (e.key === 'Enter') commit() }} - className={`${INPUT_CLASS} w-20 shrink-0 text-right tabular-nums`} - /> - ) -} - -function ParamField({ control, value, onChange, disabled }: { - control: ParamControl - value: unknown - onChange: (v: unknown) => void - disabled: boolean -}): JSX.Element { - const label = ( -
- {control.label} - {control.tooltip && ( - - ⓘ - - )} -
- ) - - switch (control.kind) { - case 'toggle': { - const on = value === true - return ( -
- {label} - -
- ) - } - case 'select': - return ( -
- {label} - -
- ) - case 'slider': { - const numeric = typeof value === 'number' ? value : control.defaultValue - const log = control.scale === 'log' - return ( -
- {label} -
- onChange(fromSliderPosition(control, e.currentTarget.valueAsNumber))} - className="min-w-0 flex-1 accent-violet-500 cursor-pointer disabled:cursor-default" - /> - -
-
- ) - } - case 'number': - return ( -
- {label} - -
- ) - case 'text': - return ( -
- {label} - onChange(e.target.value)} - className={`${INPUT_CLASS} w-full`} - /> -
- ) - } -} - -function ParamFields({ op, values, onChange, disabled }: { - op: MeshOpDescriptor - values: Params - onChange: (id: string, value: unknown) => void - disabled: boolean -}): JSX.Element | null { - if (op.params_schema.length === 0) return null - return ( -
- {op.params_schema.map((schema) => { - const control = paramControl(schema) - return ( - onChange(schema.id, v)} - disabled={disabled} - /> - ) - })} -
- ) -} - // ─── Reports ────────────────────────────────────────────────────────────────── const BADGE_CLASS: Record = { @@ -470,7 +292,7 @@ export default function EditPanel({ onClose }: { onClose: () => void }): JSX.Ele setParam(inspectOp, id, v)} disabled={running !== null} @@ -526,7 +348,7 @@ export default function EditPanel({ onClose }: { onClose: () => void }): JSX.Ele {open && hasParams && (
- setParam(op, id, v)} disabled={running !== null} /> + setParam(op, id, v)} disabled={running !== null} /> {params[op.id] && ( + + {card.phase !== 'hidden' && card.phase !== 'checking' && ( + { void download() }} + onClose={() => { pending.current = null; setCard(HIDDEN_CARD) }} + /> + )} +
+ ) +} + +function SetupCardView({ card, message, onDownload, onClose }: { + card: Exclude + message: string + onDownload: () => void + onClose: () => void +}): JSX.Element { + return ( +
+ {card.phase === 'confirm' && ( + <> +

{message}

+
+ + +
+ + )} + {card.phase === 'installing' && ( + <> +

{stepLabel(card) ?? 'Setting up…'}

+ +

This happens once. It will run as soon as it is done.

+ + )} + {card.phase === 'error' && ( + <> +

{card.message}

+
+ + +
+ + )} +
+ ) +} diff --git a/src/areas/image-studio/ImageStudioPage.tsx b/src/areas/image-studio/ImageStudioPage.tsx index 59cdd5eec..7f0d6eac9 100644 --- a/src/areas/image-studio/ImageStudioPage.tsx +++ b/src/areas/image-studio/ImageStudioPage.tsx @@ -4,6 +4,7 @@ import { useNavStore } from '@shared/stores/navStore' import { SUPPORTED_IMAGE_TYPES, saveImageToWorkspace, takePastedImage, useImageHandoff } from '@shared/utils/imageInput' import { mimeFromPath } from '@areas/workflows/nodes/imageUtils' import { fileNameOf, findByPath, sourceOf, useImageStudioStore, type StudioImage } from './imageStudioStore' +import ImageOpsToolbar, { type ImageOpActivity, type ImageOpResult } from './ImageOpsToolbar' // ─── Icons ──────────────────────────────────────────────────────────────────── @@ -40,51 +41,6 @@ async function previewUrlForPath(path: string): Promise { return URL.createObjectURL(new Blob([bytes], { type: mimeFromPath(path) })) } -// ─── Image operations (slot) ────────────────────────────────────────────────── -// Image operations (background removal, … — ticket #421) plug in here. An -// operation reads the selected image and writes a NEW file; the page adds it -// to the history with the selected image as its source, so the canvas shows -// the result before/after against it. The originals are never modified. - -interface ImageOp { - id: string - label: string - icon: string - /** Produces a new image file from `image` and returns its absolute path. */ - run: (image: StudioImage) => Promise -} - -const IMAGE_OPS: readonly ImageOp[] = [] - -function ImageOpsToolbar({ image, onResult }: { image: StudioImage, onResult: (path: string, source: StudioImage) => Promise }): JSX.Element | null { - const showError = useAppStore((s) => s.showError) - const [running, setRunning] = useState(null) - if (IMAGE_OPS.length === 0) return null - - async function run(op: ImageOp) { - setRunning(op.id) - try { - await onResult(await op.run(image), image) - } catch (err) { - showError(err instanceof Error ? err.message : String(err)) - } finally { - setRunning(null) - } - } - - return ( -
- {IMAGE_OPS.map((op) => ( - - ))} -
- ) -} - // ─── Canvas ─────────────────────────────────────────────────────────────────── function CompareView({ before, after }: { before: StudioImage, after: StudioImage }): JSX.Element { @@ -130,12 +86,64 @@ function CompareView({ before, after }: { before: StudioImage, after: StudioImag ) } -function Canvas({ image, source }: { image: StudioImage, source?: StudioImage }): JSX.Element { +/** Object URL of the mask file while `path` is set; revoked when it changes. */ +function useMaskPreview(path: string | undefined): { url: string | null, failed: boolean } { + const [state, setState] = useState<{ path: string, url: string | null, failed: boolean } | null>(null) + useEffect(() => { + if (!path) return + let cancelled = false + let created: string | null = null + previewUrlForPath(path) + .then((url) => { + if (cancelled) { URL.revokeObjectURL(url); return } + created = url + setState({ path, url, failed: false }) + }) + .catch(() => { if (!cancelled) setState({ path, url: null, failed: true }) }) + return () => { + cancelled = true + if (created) URL.revokeObjectURL(created) + } + }, [path]) + return state && state.path === path ? state : { url: null, failed: false } +} + +function Canvas({ image, source, activity }: { image: StudioImage, source?: StudioImage, activity: string | null }): JSX.Element { + const [maskFor, setMaskFor] = useState(null) + const showMask = Boolean(image.maskPath) && maskFor === image.id + const mask = useMaskPreview(showMask ? image.maskPath : undefined) + + let content: JSX.Element + if (showMask) { + content = mask.url + ? {`Mask + : {mask.failed ? 'Could not load the mask' : 'Loading mask…'} + } else if (source) { + content = + } else { + content = {image.name} + } + return (
- {source - ? - : {image.name}} + {content} + {image.maskPath && ( + + )} + {activity && ( +
+ + + {activity} + +
+ )}
) } @@ -205,21 +213,28 @@ export default function ImageStudioPage(): JSX.Element { const { add, select, remove } = useImageStudioStore.getState() const [busy, setBusy] = useState(false) const [dragging, setDragging] = useState(false) + const [activity, setActivity] = useState(null) const selected = entries.find((e) => e.id === selectedId) const source = sourceOf(entries, selected) /** An image already on disk: used where it is, not copied. */ - const addPath = useCallback(async (path: string, sourceImage?: StudioImage) => { - const existing = sourceImage ? undefined : findByPath(useImageStudioStore.getState().entries, path) + const addPath = useCallback(async (path: string) => { + const existing = findByPath(useImageStudioStore.getState().entries, path) if (existing) { select(existing.id) return } const previewUrl = await previewUrlForPath(path) - add({ path, previewUrl, name: fileNameOf(path), sourceId: sourceImage?.id }) + add({ path, previewUrl, name: fileNameOf(path) }) }, [add, select]) + /** An image operation's output: a new entry made from `sourceImage`, selected. */ + const addResult = useCallback(async (result: ImageOpResult, sourceImage: StudioImage) => { + const previewUrl = await previewUrlForPath(result.path) + add({ path: result.path, previewUrl, name: fileNameOf(result.path), sourceId: sourceImage.id, op: result.op, maskPath: result.maskPath }) + }, [add]) + /** Image bytes with no file behind them (clipboard, browser drag): saved to the workspace first. */ const addBlob = useCallback(async (file: File, name: string) => { const path = await saveImageToWorkspace(file, name) @@ -293,7 +308,7 @@ export default function ImageStudioPage(): JSX.Element {

- {selected && } + {selected && } - {selected ? : } + {selected + ? +: } {entries.length > 0 && (
diff --git a/src/areas/image-studio/imageOps.test.ts b/src/areas/image-studio/imageOps.test.ts new file mode 100644 index 000000000..4d4ff6bd0 --- /dev/null +++ b/src/areas/image-studio/imageOps.test.ts @@ -0,0 +1,134 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { + HIDDEN_CARD, + applyPrepareEvent, + approxSize, + cardForStatus, + failInstall, + notReadyMissing, + runInputFor, + setupMessage, + startInstall, + stepLabel, + type SetupCard, +} from './imageOps' +import type { StudioImage } from './imageStudioStore' + +const image = (id: string, extra: Partial = {}): StudioImage => ({ + id, path: `C:\\work\\${id}.png`, previewUrl: `blob:${id}`, name: `${id}.png`, createdAt: 0, ...extra, +}) + +const OP = 'remove_background' + +// ─── runInputFor ────────────────────────────────────────────────────────────── + +test('runInputFor runs on the selected image when it is an original', () => { + const photo = image('photo') + assert.equal(runInputFor([photo], photo, OP), photo) +}) + +test('runInputFor re-runs on the source when the selected image is a result of the same op', () => { + const photo = image('photo') + const cut = image('cut', { sourceId: 'photo', op: OP }) + assert.equal(runInputFor([cut, photo], cut, OP), photo) +}) + +test('runInputFor keeps a result of another op as the input', () => { + const photo = image('photo') + const upscaled = image('up', { sourceId: 'photo', op: 'upscale' }) + assert.equal(runInputFor([upscaled, photo], upscaled, OP), upscaled) +}) + +test('runInputFor falls back to the selected image when its source left the history', () => { + const cut = image('cut', { sourceId: 'gone', op: OP }) + assert.equal(runInputFor([cut], cut, OP), cut) +}) + +// ─── Status → card ──────────────────────────────────────────────────────────── + +test('cardForStatus hides the card when the op is ready', () => { + assert.deepEqual(cardForStatus({ ready: true, missing: [], download_bytes: null }), HIDDEN_CARD) +}) + +test('cardForStatus offers the download with the missing steps in install order', () => { + assert.deepEqual( + cardForStatus({ ready: false, missing: ['model', 'runtime'], download_bytes: 176_000_000 }), + { phase: 'confirm', missing: ['runtime', 'model'], downloadBytes: 176_000_000 }, + ) +}) + +test('cardForStatus ignores unknown steps and a missing size', () => { + assert.deepEqual( + cardForStatus({ ready: false, missing: ['model', 'gpu' as never], download_bytes: null }), + { phase: 'confirm', missing: ['model'], downloadBytes: null }, + ) +}) + +test('setupMessage states the size and the CPU note for background removal', () => { + assert.equal( + setupMessage({ id: OP, label: 'Remove background' }, 176_000_000), + 'Background removal needs a one-time download (~180 MB). It runs on your CPU, no GPU needed.', + ) + assert.equal(setupMessage({ id: 'other', label: 'Upscale' }, null), 'Upscale needs a one-time download.') +}) + +test('approxSize rounds to what a person reads', () => { + assert.equal(approxSize(null), null) + assert.equal(approxSize(0), null) + assert.equal(approxSize(3_200_000), '~3 MB') + assert.equal(approxSize(204_000_000), '~200 MB') + assert.equal(approxSize(1_430_000_000), '~1.4 GB') +}) + +// ─── Prepare stream → card ──────────────────────────────────────────────────── + +const confirm: SetupCard = { phase: 'confirm', missing: ['runtime', 'model'], downloadBytes: 200_000_000 } + +test('startInstall begins on the first missing step at 0%', () => { + const card = startInstall(confirm) + assert.equal(card.phase, 'installing') + assert.equal(stepLabel(card), 'Step 1 of 2 · Installing the runtime') + assert.equal(card.phase === 'installing' && card.event.percent, 0) +}) + +test('applyPrepareEvent follows the step the backend reports, and keeps it until the next one', () => { + let card = startInstall(confirm) + card = applyPrepareEvent(card, { step: 'model', percent: 12, status: 'Downloading model', bytesDownloaded: 24, totalBytes: 200 }) + assert.equal(stepLabel(card), 'Step 2 of 2 · Downloading the model') + card = applyPrepareEvent(card, { percent: 50 }) + assert.equal(stepLabel(card), 'Step 2 of 2 · Downloading the model') + assert.equal(card.phase === 'installing' && card.event.percent, 50) +}) + +test('a single missing step is labelled without a counter', () => { + const card = startInstall({ phase: 'confirm', missing: ['model'], downloadBytes: null }) + assert.equal(stepLabel(card), 'Downloading the model') +}) + +test('an error frame turns the card into an error that keeps the setup for Retry', () => { + const card = applyPrepareEvent(startInstall(confirm), { error: 'pip failed' }) + assert.deepEqual(card, { phase: 'error', missing: ['runtime', 'model'], downloadBytes: 200_000_000, message: 'pip failed' }) + const retried = startInstall(card) + assert.equal(retried.phase, 'installing') + assert.equal(stepLabel(retried), 'Step 1 of 2 · Installing the runtime') +}) + +test('a cancelled frame and a broken stream are errors too', () => { + assert.equal(applyPrepareEvent(startInstall(confirm), { cancelled: true }).phase, 'error') + const failed = failInstall(startInstall(confirm), 'HTTP 500') + assert.equal(failed.phase === 'error' && failed.message, 'HTTP 500') +}) + +// ─── 409 not_ready ──────────────────────────────────────────────────────────── + +test('notReadyMissing reads the missing steps of a 409 not_ready', () => { + const err = { response: { status: 409, data: { detail: { code: 'not_ready', missing: ['model'] } } } } + assert.deepEqual(notReadyMissing(err), ['model']) +}) + +test('notReadyMissing ignores other errors', () => { + assert.equal(notReadyMissing(new Error('boom')), null) + assert.equal(notReadyMissing({ response: { status: 409, data: { detail: 'busy' } } }), null) + assert.equal(notReadyMissing({ response: { status: 500, data: { detail: { code: 'not_ready' } } } }), null) +}) diff --git a/src/areas/image-studio/imageOps.ts b/src/areas/image-studio/imageOps.ts new file mode 100644 index 000000000..0dde671f6 --- /dev/null +++ b/src/areas/image-studio/imageOps.ts @@ -0,0 +1,150 @@ +// Pure logic behind the Image Studio's operations (GET /image/ops): which +// image a run reads, the one-time setup card and its download progress. +// Kept free of React so it can be unit-tested with node --test. + +import type { MeshOpParamSchema } from '../generate/meshEditPanel' +import type { SseEvent } from '../../shared/services/llmDownloads' +import { sourceOf, type StudioImage } from './imageStudioStore' + +/** What an op may need installed before its first run. */ +export type SetupStep = 'runtime' | 'model' + +/** One entry of GET /image/ops. */ +export interface ImageOpDescriptor { + id: string + label: string + params_schema: MeshOpParamSchema[] + requirements: string[] +} + +/** GET /image/op/{op}/status */ +export interface ImageOpStatus { + ready: boolean + missing: SetupStep[] + download_bytes: number | null +} + +/** A frame of POST /image/op/{op}/prepare: the LLM download format, plus the step it belongs to. */ +export type PrepareEvent = SseEvent & { step?: SetupStep } + +// ─── Run input ──────────────────────────────────────────────────────────────── + +/** + * The image a run of `opId` reads. Re-running an op on its own result (e.g. + * with another edge threshold) starts again from the original, not from the + * already-processed image. + */ +export function runInputFor(entries: readonly StudioImage[], selected: StudioImage, opId: string): StudioImage { + if (selected.op === opId) return sourceOf(entries, selected) ?? selected + return selected +} + +// ─── Setup card ─────────────────────────────────────────────────────────────── + +export type SetupCard = + | { phase: 'hidden' } + | { phase: 'checking' } + | { phase: 'confirm', missing: SetupStep[], downloadBytes: number | null } + | { phase: 'installing', missing: SetupStep[], downloadBytes: number | null, step: SetupStep | null, event: SseEvent } + | { phase: 'error', missing: SetupStep[], downloadBytes: number | null, message: string } + +export const HIDDEN_CARD: SetupCard = { phase: 'hidden' } + +const STEPS: readonly SetupStep[] = ['runtime', 'model'] + +function knownSteps(missing: unknown): SetupStep[] { + return Array.isArray(missing) ? STEPS.filter((s) => missing.includes(s)) : [] +} + +/** What the card shows for a status: nothing when the op is ready, else the download offer. */ +export function cardForStatus(status: ImageOpStatus): SetupCard { + if (status.ready) return HIDDEN_CARD + const bytes = typeof status.download_bytes === 'number' && status.download_bytes > 0 ? status.download_bytes : null + return { phase: 'confirm', missing: knownSteps(status.missing), downloadBytes: bytes } +} + +function setupOf(card: SetupCard): { missing: SetupStep[], downloadBytes: number | null } { + return 'missing' in card ? { missing: card.missing, downloadBytes: card.downloadBytes } : { missing: [], downloadBytes: null } +} + +/** The card once the download starts (before the first frame arrives). */ +export function startInstall(card: SetupCard): SetupCard { + const setup = setupOf(card) + return { phase: 'installing', ...setup, step: setup.missing[0] ?? null, event: { percent: 0, status: 'Starting…' } } +} + +/** Folds one SSE frame of the prepare stream into the card. */ +export function applyPrepareEvent(card: SetupCard, event: PrepareEvent): SetupCard { + const setup = setupOf(card) + if (event.error) return { phase: 'error', ...setup, message: event.error } + if (event.cancelled) return { phase: 'error', ...setup, message: 'The download was cancelled.' } + const previous = card.phase === 'installing' ? card.step : null + const step = event.step && STEPS.includes(event.step) ? event.step : previous ?? setup.missing[0] ?? null + return { phase: 'installing', ...setup, step, event } +} + +/** The card when the stream failed outright (backend gone, HTTP error). */ +export function failInstall(card: SetupCard, message: string): SetupCard { + return { phase: 'error', ...setupOf(card), message } +} + +const STEP_LABELS: Record = { + runtime: 'Installing the runtime', + model: 'Downloading the model', +} + +/** "Step 1 of 2 · Installing the runtime", or null when the step is unknown. */ +export function stepLabel(card: SetupCard): string | null { + if (card.phase !== 'installing' || !card.step) return null + const label = STEP_LABELS[card.step] + const total = card.missing.length + const index = card.missing.indexOf(card.step) + return total > 1 && index >= 0 ? `Step ${index + 1} of ${total} · ${label}` : label +} + +/** "~200 MB" (rounded to what a person reads), or null when unknown. */ +export function approxSize(bytes: number | null): string | null { + if (bytes == null || bytes <= 0) return null + const mb = bytes / 1e6 + if (mb >= 1000) return `~${(mb / 1000).toFixed(1)} GB` + return `~${Math.max(1, Math.round(mb / 10) * 10 || Math.round(mb))} MB` +} + +// ─── Copy ───────────────────────────────────────────────────────────────────── + +interface OpCopy { + /** Subject of the setup sentence ("Background removal needs…"). */ + noun: string + /** Shown on the button while the op runs. */ + running: string + /** Extra line under the download offer. */ + note?: string +} + +const OP_COPY: Record = { + remove_background: { noun: 'Background removal', running: 'Removing background…', note: 'It runs on your CPU, no GPU needed.' }, +} + +export function opCopy(op: Pick): OpCopy { + return OP_COPY[op.id] ?? { noun: op.label, running: `${op.label}…` } +} + +/** The download offer's sentence. */ +export function setupMessage(op: Pick, downloadBytes: number | null): string { + const copy = opCopy(op) + const size = approxSize(downloadBytes) + const first = `${copy.noun} needs a one-time download${size ? ` (${size})` : ''}.` + return copy.note ? `${first} ${copy.note}` : first +} + +// ─── API errors ─────────────────────────────────────────────────────────────── + +/** The missing steps of a 409 `{detail:{code:'not_ready', missing}}`, or null for any other error. */ +export function notReadyMissing(err: unknown): SetupStep[] | null { + if (!err || typeof err !== 'object') return null + const response = (err as { response?: { status?: number, data?: { detail?: unknown } } }).response + if (response?.status !== 409) return null + const detail = response.data?.detail + if (!detail || typeof detail !== 'object' || (detail as { code?: unknown }).code !== 'not_ready') return null + return knownSteps((detail as { missing?: unknown }).missing) +} diff --git a/src/areas/image-studio/imageOpsApi.ts b/src/areas/image-studio/imageOpsApi.ts new file mode 100644 index 000000000..16a133ba1 --- /dev/null +++ b/src/areas/image-studio/imageOpsApi.ts @@ -0,0 +1,50 @@ +import axios from 'axios' +import { useAppStore } from '@shared/stores/appStore' +import { consumeSse } from '@shared/services/llmDownloads' +import { apiErrorMessage } from '@areas/generate/meshEditPanel' +import { notReadyMissing, type ImageOpDescriptor, type ImageOpStatus, type PrepareEvent, type SetupStep } from './imageOps' + +// Client of the image-ops routes (api/routers/image*.py): list, status, +// one-time setup (SSE) and run. Paths are absolute files in /Images. + +/** POST /image/op/{op} answered 409: the op needs its one-time setup first. */ +export class ImageOpNotReadyError extends Error { + constructor(readonly missing: SetupStep[]) { + super('This operation needs a one-time download first.') + this.name = 'ImageOpNotReadyError' + } +} + +function opUrl(op: string, suffix = ''): string { + return `${useAppStore.getState().apiUrl}/image/op/${encodeURIComponent(op)}${suffix}` +} + +export async function listImageOps(): Promise { + const { data } = await axios.get<{ ops?: ImageOpDescriptor[] }>(`${useAppStore.getState().apiUrl}/image/ops`) + return Array.isArray(data?.ops) ? data.ops : [] +} + +export async function getImageOpStatus(op: string): Promise { + try { + const { data } = await axios.get(opUrl(op, '/status')) + return data + } catch (err) { + throw new Error(apiErrorMessage(err)) + } +} + +/** Streams the install of what `op` is missing; resolves when the stream ends. */ +export function prepareImageOp(op: string, onEvent: (event: PrepareEvent) => void, signal?: AbortSignal): Promise { + return consumeSse(opUrl(op, '/prepare'), onEvent, signal, { method: 'POST' }) +} + +export async function runImageOp(op: string, path: string, params: Record): Promise<{ path: string, maskPath?: string }> { + try { + const { data } = await axios.post<{ path: string, mask_path?: string | null }>(opUrl(op), { path, params }) + return { path: data.path, maskPath: data.mask_path ?? undefined } + } catch (err) { + const missing = notReadyMissing(err) + if (missing) throw new ImageOpNotReadyError(missing) + throw new Error(apiErrorMessage(err)) + } +} diff --git a/src/areas/image-studio/imageStudioStore.ts b/src/areas/image-studio/imageStudioStore.ts index 47c1a4e39..a04b55e55 100644 --- a/src/areas/image-studio/imageStudioStore.ts +++ b/src/areas/image-studio/imageStudioStore.ts @@ -16,6 +16,10 @@ export interface StudioImage { name: string /** The image this one was produced from, when it came from an operation. */ sourceId?: string + /** Id of the image operation that produced it (e.g. background removal). */ + op?: string + /** Absolute path of the mask the operation wrote alongside, if any. */ + maskPath?: string createdAt: number } diff --git a/src/shared/components/ui/ParamFields.tsx b/src/shared/components/ui/ParamFields.tsx new file mode 100644 index 000000000..dbe0b8dfd --- /dev/null +++ b/src/shared/components/ui/ParamFields.tsx @@ -0,0 +1,186 @@ +import { useEffect, useState } from 'react' +import { Tooltip } from './Tooltip' +import { + LOG_SLIDER_STEPS, + coerceNumber, + fromSliderPosition, + paramControl, + toSliderPosition, + type MeshOpParamSchema, + type ParamControl, +} from '@areas/generate/meshEditPanel' + +// Controls generated from an op's params_schema (mesh ops, image ops): the +// same param dict format everywhere, so one renderer. + +const INPUT_CLASS = + 'bg-zinc-800 border border-zinc-700 rounded-lg px-2 py-1 text-xs text-zinc-200 focus:outline-none focus:border-violet-500 transition-colors disabled:opacity-50' + +/** Number text field that commits (clamped) on blur or Enter. */ +function NumberField({ control, value, onChange, disabled }: { + control: Extract + value: number + onChange: (v: number) => void + disabled: boolean +}): JSX.Element { + const [text, setText] = useState(String(value)) + useEffect(() => { setText(String(value)) }, [value]) + + function commit() { + const parsed = Number(text.replace(',', '.')) + if (text.trim() === '' || !Number.isFinite(parsed)) { setText(String(value)); return } + const next = coerceNumber(control, parsed) + setText(String(next)) + if (next !== value) onChange(next) + } + + return ( + setText(e.target.value)} + onBlur={commit} + onKeyDown={(e) => { if (e.key === 'Enter') commit() }} + className={`${INPUT_CLASS} w-20 shrink-0 text-right tabular-nums`} + /> + ) +} + +export function ParamField({ control, value, onChange, disabled }: { + control: ParamControl + value: unknown + onChange: (v: unknown) => void + disabled: boolean +}): JSX.Element { + const label = ( +
+ {control.label} + {control.tooltip && ( + + ⓘ + + )} +
+ ) + + switch (control.kind) { + case 'toggle': { + const on = value === true + return ( +
+ {label} + +
+ ) + } + case 'select': + return ( +
+ {label} + +
+ ) + case 'slider': { + const numeric = typeof value === 'number' ? value : control.defaultValue + const log = control.scale === 'log' + return ( +
+ {label} +
+ onChange(fromSliderPosition(control, e.currentTarget.valueAsNumber))} + className="min-w-0 flex-1 accent-violet-500 cursor-pointer disabled:cursor-default" + /> + +
+
+ ) + } + case 'number': + return ( +
+ {label} + +
+ ) + case 'text': + return ( +
+ {label} + onChange(e.target.value)} + className={`${INPUT_CLASS} w-full`} + /> +
+ ) + } +} + +/** One control per entry of `schema`; nothing when the op takes no params. */ +export function ParamFields({ schema, values, onChange, disabled, className = 'flex flex-col gap-2.5' }: { + schema: MeshOpParamSchema[] + values: Record + onChange: (id: string, value: unknown) => void + disabled: boolean + className?: string +}): JSX.Element | null { + if (schema.length === 0) return null + return ( +
+ {schema.map((param) => { + const control = paramControl(param) + return ( + onChange(param.id, v)} + disabled={disabled} + /> + ) + })} +
+ ) +} diff --git a/src/shared/services/llmDownloads.ts b/src/shared/services/llmDownloads.ts index 18de4412f..46d58235b 100644 --- a/src/shared/services/llmDownloads.ts +++ b/src/shared/services/llmDownloads.ts @@ -17,9 +17,12 @@ export interface SseEvent { paused?: boolean } -/** Reads one SSE `data: {...}` frame at a time and calls `onEvent` for each. */ -export async function consumeSse(url: string, onEvent: (data: SseEvent) => void, signal?: AbortSignal): Promise { - const res = await fetch(url, { signal }) +/** + * Reads one SSE `data: {...}` frame at a time and calls `onEvent` for each. + * `init` sets the method/body for streams opened with a POST. + */ +export async function consumeSse(url: string, onEvent: (data: SseEvent) => void, signal?: AbortSignal, init?: RequestInit): Promise { + const res = await fetch(url, { ...init, signal }) if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`) const reader = res.body.getReader() const decoder = new TextDecoder() From 771ddd0d0c6c57b812e23bdd6063d6be14df95af Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 16 Nov 2026 14:12:57 +0100 Subject: [PATCH 2/3] fix(image-studio): accept the background-removal setup only as POST A GET could start the one-time install from any page (an image tag is enough); the renderer already uses POST. --- api/routers/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/routers/image.py b/api/routers/image.py index f440032ba..08f4bfd44 100644 --- a/api/routers/image.py +++ b/api/routers/image.py @@ -136,7 +136,7 @@ def _install(req=req, lock=lock, cb=_progress) -> None: _broadcast({"error": str(exc)}) -@router.api_route("/op/{op_name}/prepare", methods=["POST", "GET"]) +@router.post("/op/{op_name}/prepare") async def prepare_image_operation(op_name: str): """Install what the op is missing, streaming progress. Idempotent: an op that is already ready gets a single `done` frame; a call while a prepare is From fd09eabe4cff45e959182ba232410fbd292e83bb Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Mon, 16 Nov 2026 14:58:19 +0100 Subject: [PATCH 3/3] refactor(image-studio): show the selected image only, without a before/after slider The canvas shows the cut-out itself; the original stays one click away in the history strip. --- src/areas/image-studio/ImageStudioPage.tsx | 54 ++-------------------- src/areas/image-studio/imageStudioStore.ts | 4 +- 2 files changed, 6 insertions(+), 52 deletions(-) diff --git a/src/areas/image-studio/ImageStudioPage.tsx b/src/areas/image-studio/ImageStudioPage.tsx index 7f0d6eac9..e61721e64 100644 --- a/src/areas/image-studio/ImageStudioPage.tsx +++ b/src/areas/image-studio/ImageStudioPage.tsx @@ -1,9 +1,9 @@ -import { useCallback, useEffect, useRef, useState, type CSSProperties } from 'react' +import { useCallback, useEffect, useState, type CSSProperties } from 'react' import { useAppStore } from '@shared/stores/appStore' import { useNavStore } from '@shared/stores/navStore' import { SUPPORTED_IMAGE_TYPES, saveImageToWorkspace, takePastedImage, useImageHandoff } from '@shared/utils/imageInput' import { mimeFromPath } from '@areas/workflows/nodes/imageUtils' -import { fileNameOf, findByPath, sourceOf, useImageStudioStore, type StudioImage } from './imageStudioStore' +import { fileNameOf, findByPath, useImageStudioStore, type StudioImage } from './imageStudioStore' import ImageOpsToolbar, { type ImageOpActivity, type ImageOpResult } from './ImageOpsToolbar' // ─── Icons ──────────────────────────────────────────────────────────────────── @@ -43,49 +43,6 @@ async function previewUrlForPath(path: string): Promise { // ─── Canvas ─────────────────────────────────────────────────────────────────── -function CompareView({ before, after }: { before: StudioImage, after: StudioImage }): JSX.Element { - const [position, setPosition] = useState(50) - const box = useRef(null) - - const moveTo = (clientX: number) => { - const rect = box.current?.getBoundingClientRect() - if (!rect || rect.width === 0) return - setPosition(Math.min(100, Math.max(0, ((clientX - rect.left) / rect.width) * 100))) - } - - return ( -
{ event.currentTarget.setPointerCapture(event.pointerId); moveTo(event.clientX) }} - onPointerMove={(event) => { if (event.currentTarget.hasPointerCapture(event.pointerId)) moveTo(event.clientX) }} - role="slider" - aria-label="Before / after" - aria-valuemin={0} - aria-valuemax={100} - aria-valuenow={Math.round(position)} - tabIndex={0} - onKeyDown={(event) => { - if (event.key === 'ArrowLeft') setPosition((p) => Math.max(0, p - 5)) - if (event.key === 'ArrowRight') setPosition((p) => Math.min(100, p + 5)) - }} - > - {after.name} - {before.name} -
- - - -
- Before - After -
- ) -} - /** Object URL of the mask file while `path` is set; revoked when it changes. */ function useMaskPreview(path: string | undefined): { url: string | null, failed: boolean } { const [state, setState] = useState<{ path: string, url: string | null, failed: boolean } | null>(null) @@ -108,7 +65,7 @@ function useMaskPreview(path: string | undefined): { url: string | null, failed: return state && state.path === path ? state : { url: null, failed: false } } -function Canvas({ image, source, activity }: { image: StudioImage, source?: StudioImage, activity: string | null }): JSX.Element { +function Canvas({ image, activity }: { image: StudioImage, activity: string | null }): JSX.Element { const [maskFor, setMaskFor] = useState(null) const showMask = Boolean(image.maskPath) && maskFor === image.id const mask = useMaskPreview(showMask ? image.maskPath : undefined) @@ -118,8 +75,6 @@ function Canvas({ image, source, activity }: { image: StudioImage, source?: Stud content = mask.url ? {`Mask : {mask.failed ? 'Could not load the mask' : 'Loading mask…'} - } else if (source) { - content = } else { content = {image.name} } @@ -216,7 +171,6 @@ export default function ImageStudioPage(): JSX.Element { const [activity, setActivity] = useState(null) const selected = entries.find((e) => e.id === selectedId) - const source = sourceOf(entries, selected) /** An image already on disk: used where it is, not copied. */ const addPath = useCallback(async (path: string) => { @@ -330,7 +284,7 @@ export default function ImageStudioPage(): JSX.Element { {selected - ? + ? : } {entries.length > 0 && ( diff --git a/src/areas/image-studio/imageStudioStore.ts b/src/areas/image-studio/imageStudioStore.ts index a04b55e55..9d02c9da2 100644 --- a/src/areas/image-studio/imageStudioStore.ts +++ b/src/areas/image-studio/imageStudioStore.ts @@ -5,7 +5,7 @@ import { create } from 'zustand' // level, and survives navigation (not app restarts). // // An image made from another one (an image operation, e.g. background -// removal) records it as `sourceId`; the studio compares the two before/after. +// removal) records it as `sourceId`, so re-running the op starts from the original. export interface StudioImage { id: string @@ -41,7 +41,7 @@ export function addToHistory(history: StudioHistory, entry: StudioImage, max = M /** * Drops entries from the history. Images made from a dropped one lose their - * source (nothing to compare with any more); the selection moves to the + * source (a re-run then starts from them); the selection moves to the * neighbour of the removed image. */ export function withoutEntries(history: StudioHistory, ids: readonly string[]): { history: StudioHistory, dropped: StudioImage[] } {