Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion api/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from services.stdio_utf8 import ensure_utf8_stdio
ensure_utf8_stdio() # must run before any print/logging hits the pipe

from routers import generation, model, optimize, status, settings, extensions, export, workflow_runs, agent, llm
from routers import generation, model, optimize, status, settings, extensions, export, workflow_runs, agent, llm, image


@asynccontextmanager
Expand Down Expand Up @@ -68,6 +68,7 @@ def filter(self, record):
app.include_router(workflow_runs.router, prefix="/workflow-runs")
app.include_router(agent.router)
app.include_router(llm.router, prefix="/llm")
app.include_router(image.router, prefix="/image")

# Serve generated files from workspace — dynamic so path changes take effect immediately
@app.get("/workspace/{full_path:path}")
Expand Down
237 changes: 237 additions & 0 deletions api/routers/image.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
"""
Image operations (background removal, ...) on local files.

Ops may need things installed on first use (a runtime, model weights):
`GET /image/op/{op}/status` says what is missing, `POST /image/op/{op}/prepare`
installs it while streaming progress in the same SSE format as the LLM
downloads (src/shared/services/llmDownloads.ts), and `POST /image/op/{op}`
runs the op once everything is there.
"""
import asyncio
import json
import threading
from pathlib import Path
from typing import Any, Optional

from fastapi import APIRouter, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from starlette.concurrency import run_in_threadpool

import services.generator_registry as registry
from services.image_ops import (
ImageOp,
ImageOpContext,
ImageOpExecutionError,
ImageOpInputError,
ImageOpNotFoundError,
ImageOpNotReadyError,
image_ops_registry,
)

router = APIRouter(tags=["image"])

IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}


class ImageOpRequest(BaseModel):
path: str
params: dict[str, Any] = Field(default_factory=dict)


def _fmt(data: dict) -> str:
return f"data: {json.dumps(data)}\n\n"


def _get_op(op_name: str) -> ImageOp:
try:
return image_ops_registry.get(op_name)
except ImageOpNotFoundError as exc:
raise HTTPException(404, f"Unknown image operation: {op_name}") from exc


def _missing_bytes(op: ImageOp) -> Optional[int]:
sizes = [req.download_bytes() for req in op.requirements if not req.is_ready()]
known = [size for size in sizes if size is not None]
return sum(known) if known else None


# ─── Catalog / status ─────────────────────────────────────────────────────────

@router.get("/ops")
def list_image_operations():
return {"ops": image_ops_registry.describe()}


@router.get("/op/{op_name}/status")
def image_operation_status(op_name: str):
op = _get_op(op_name)
missing = op.missing()
state = _prepares.get(op_name)
return {
"ready": not missing,
"missing": missing,
"download_bytes": _missing_bytes(op) if missing else None,
"preparing": bool(state and state.task and not state.task.done()),
}


# ─── Prepare (SSE) ────────────────────────────────────────────────────────────

class _PrepareState:
"""One op's in-flight prepare, independent of any SSE connection, so a
second caller (or a reopened panel) attaches instead of starting a second
pip install / download racing the first."""

def __init__(self) -> None:
self.last_msg: dict = {}
self.subscribers: list[asyncio.Queue] = []
self.task: Optional[asyncio.Task] = None


_prepares: dict[str, _PrepareState] = {}
# Two ops sharing a requirement (same runtime) must not install it twice at once.
_install_locks: dict[str, threading.Lock] = {}


def _is_terminal(msg: dict) -> bool:
return msg.get("status") == "done" or "error" in msg


async def _run_prepare(op: ImageOp, state: _PrepareState) -> None:
loop = asyncio.get_running_loop()

def _broadcast(msg: dict) -> None:
def _do() -> None:
state.last_msg = msg
for queue in list(state.subscribers):
queue.put_nowait(msg)
loop.call_soon_threadsafe(_do)

try:
pending = [req for req in op.requirements if not req.is_ready()]
# Weight each step by its download size so the bar is roughly linear in time.
weights = [max(1, req.download_bytes() or 1) for req in pending]
total = sum(weights) or 1
done_weight = 0
_broadcast({"percent": 0, "status": "Starting…"})

for req, weight in zip(pending, weights):
def _progress(msg: dict, _base: int = done_weight, _weight: int = weight, _kind: str = req.kind) -> None:
step_pct = max(0, min(100, msg.get("percent") or 0))
overall = min(99, round((_base + _weight * step_pct / 100) / total * 100))
_broadcast({**msg, "percent": overall, "step": _kind})

lock = _install_locks.setdefault(req.id, threading.Lock())

def _install(req=req, lock=lock, cb=_progress) -> None:
with lock:
req.install(cb)

await loop.run_in_executor(None, _install)
done_weight += weight

_broadcast({"percent": 100, "status": "done"})
except Exception as exc:
_broadcast({"error": str(exc)})


@router.post("/op/{op_name}/prepare")
async def prepare_image_operation(op_name: str):
"""Install what the op is missing, streaming progress. Idempotent: an op
that is already ready gets a single `done` frame; a call while a prepare is
running joins it. GET is accepted too so the renderer's existing
`consumeSse` (a plain fetch) can be reused unchanged."""
op = _get_op(op_name)
state = _prepares.get(op_name)
running = state is not None and state.task is not None and not state.task.done()

if not running and not op.missing():
async def already():
yield _fmt({"percent": 100, "status": "done"})
return StreamingResponse(already(), media_type="text/event-stream")

if not running:
state = _PrepareState()
_prepares[op_name] = state
state.task = asyncio.create_task(_run_prepare(op, state))

queue: asyncio.Queue[dict] = asyncio.Queue()
state.subscribers.append(queue)
if state.last_msg:
queue.put_nowait(state.last_msg) # replay current progress on (re)connect

async def stream():
try:
while True:
msg = await queue.get()
yield _fmt(msg)
if _is_terminal(msg):
break
finally:
if queue in state.subscribers:
state.subscribers.remove(queue)

return StreamingResponse(stream(), media_type="text/event-stream")


# ─── Run ──────────────────────────────────────────────────────────────────────

def _resolve_image_path(raw_path: str) -> Path:
if not raw_path or not raw_path.strip():
raise HTTPException(400, "path is required")
candidate = Path(raw_path)
if candidate.is_absolute():
resolved = candidate.resolve()
else:
resolved = (registry.WORKSPACE_DIR / raw_path).resolve()
if not registry.is_within_workspace(resolved):
raise HTTPException(400, "Invalid path")
if resolved.suffix.lower() not in IMAGE_EXTENSIONS:
raise HTTPException(400, f"Unsupported image type {resolved.suffix or '(none)'}: use png, jpg, jpeg or webp")
if not resolved.is_file():
raise HTTPException(404, f"File not found: {raw_path}")
return resolved


def _workspace_url(path: Path) -> Optional[str]:
try:
relative = path.resolve().relative_to(registry.WORKSPACE_DIR.resolve()).as_posix()
except ValueError:
return None
return f"/workspace/{relative}"


@router.post("/op/{op_name}")
async def run_image_operation(op_name: str, body: ImageOpRequest):
op = _get_op(op_name)
input_path = _resolve_image_path(body.path)
missing = op.missing()
if missing:
raise HTTPException(409, {"code": "not_ready", "missing": missing})

context = ImageOpContext(workspace_dir=registry.WORKSPACE_DIR)
try:
# Inference takes seconds on CPU: keep the event loop free.
result = await run_in_threadpool(image_ops_registry.run, op_name, input_path, body.params, context)
except ImageOpNotReadyError as exc:
raise HTTPException(409, {"code": "not_ready", "missing": exc.missing}) from exc
except FileNotFoundError as exc:
raise HTTPException(404, str(exc)) from exc
except (ImageOpInputError, TypeError, ValueError) as exc:
raise HTTPException(400, str(exc)) from exc
except ImageOpExecutionError as exc:
raise HTTPException(500, str(exc)) from exc

response: dict[str, Any] = {
"path": str(result.path.resolve()),
"mask_path": str(result.mask_path.resolve()) if result.mask_path else None,
}
url = _workspace_url(result.path)
if url:
response["url"] = url
if result.mask_path is not None:
mask_url = _workspace_url(result.mask_path)
if mask_url:
response["mask_url"] = mask_url
return response
30 changes: 30 additions & 0 deletions api/services/image_ops/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
"""Image operation registry (background removal, ...) used by the /image API."""

from .builtin import BUILTIN_IMAGE_OPS
from .registry import ImageOpsRegistry
from .types import (
ImageOp,
ImageOpContext,
ImageOpExecutionError,
ImageOpInputError,
ImageOpNotFoundError,
ImageOpNotReadyError,
ImageOpResult,
Requirement,
)


image_ops_registry = ImageOpsRegistry(BUILTIN_IMAGE_OPS)

__all__ = [
"ImageOp",
"ImageOpContext",
"ImageOpExecutionError",
"ImageOpInputError",
"ImageOpNotFoundError",
"ImageOpNotReadyError",
"ImageOpResult",
"ImageOpsRegistry",
"Requirement",
"image_ops_registry",
]
90 changes: 90 additions & 0 deletions api/services/image_ops/birefnet.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
"""BiRefNet-lite background matting on CPU through onnxruntime.
onnxruntime is installed on first use (see requirements.py), so it is only
imported inside `_create_onnx_session`, never at module import.
Pre/post-processing follows the model's preprocessor_config.json:
1024x1024 bilinear resize, x/255, ImageNet mean/std; the output is logits.
"""

import threading
from pathlib import Path
from typing import Any, Callable, Optional

import numpy as np
from PIL import Image

INPUT_SIZE = 1024
_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)

SessionFactory = Callable[[Path], Any]


def _create_onnx_session(model_path: Path) -> Any:
import onnxruntime as ort # lazy: installed on demand

options = ort.SessionOptions()
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
return ort.InferenceSession(str(model_path), sess_options=options, providers=["CPUExecutionProvider"])


class SessionCache:
"""One InferenceSession per model file, built on first use and reused.
Tests inject a fake factory so they never need onnxruntime or the model."""

def __init__(self, factory: SessionFactory = _create_onnx_session) -> None:
self._factory = factory
self._lock = threading.Lock()
self._key: Optional[tuple[str, int]] = None
self._session: Any = None

def get(self, model_path: Path) -> Any:
stat = model_path.stat()
key = (str(model_path.resolve()), stat.st_mtime_ns)
with self._lock:
if self._session is None or self._key != key:
self._session = self._factory(model_path)
self._key = key
return self._session

def clear(self) -> None:
with self._lock:
self._session = None
self._key = None


session_cache = SessionCache()


def preprocess(rgb: Image.Image) -> np.ndarray:
"""RGB PIL image -> float32 NCHW tensor (1, 3, 1024, 1024)."""
resized = rgb.resize((INPUT_SIZE, INPUT_SIZE), Image.Resampling.BILINEAR)
array = np.asarray(resized, dtype=np.float32) / 255.0
array = (array - _MEAN) / _STD
return np.ascontiguousarray(array.transpose(2, 0, 1)[None, ...], dtype=np.float32)


def _sigmoid(x: np.ndarray) -> np.ndarray:
return 1.0 / (1.0 + np.exp(-np.clip(x, -60.0, 60.0)))


def predict_alpha(session: Any, rgb: Image.Image) -> np.ndarray:
"""Soft foreground alpha in [0, 1], float32, at the image's own size (H, W)."""
input_name = session.get_inputs()[0].name
outputs = session.run(None, {input_name: preprocess(rgb)})
logits = np.asarray(outputs[0], dtype=np.float32)
matte = _sigmoid(logits.reshape(logits.shape[-2], logits.shape[-1]))
resized = Image.fromarray(matte.astype(np.float32)).resize(rgb.size, Image.Resampling.BILINEAR)
return np.clip(np.asarray(resized, dtype=np.float32), 0.0, 1.0)


def apply_edge_threshold(alpha: np.ndarray, threshold: float) -> np.ndarray:
"""0 keeps the soft alpha; t > 0 hardens: clamp((a - t) / (1 - t), 0, 1)."""
if threshold <= 0:
return np.clip(alpha, 0.0, 1.0)
t = min(float(threshold), 0.999)
return np.clip((alpha - t) / (1.0 - t), 0.0, 1.0)


def to_uint8(alpha: np.ndarray) -> np.ndarray:
return np.clip(np.rint(alpha * 255.0), 0, 255).astype(np.uint8)
Loading
Loading