Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions api/routers/image.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,8 @@ def _install(req=req, lock=lock, cb=_progress) -> None:
async def prepare_image_operation(op_name: str):
"""Install what the op is missing, streaming progress. Idempotent: an op
that is already ready gets a single `done` frame; a call while a prepare is
running joins it. GET is accepted too so the renderer's existing
`consumeSse` (a plain fetch) can be reused unchanged."""
running joins it. POST only: it downloads and installs, so a plain GET (a
link, a prefetch) must never start it."""
op = _get_op(op_name)
state = _prepares.get(op_name)
running = state is not None and state.task is not None and not state.task.done()
Expand Down Expand Up @@ -178,15 +178,21 @@ async def stream():
# ─── Run ──────────────────────────────────────────────────────────────────────

def _resolve_image_path(raw_path: str) -> Path:
"""The input file, which must be inside the workspace.

Absolute or workspace-relative; checked once fully resolved, so `..` and
symlinks pointing out of the workspace are refused too. The Image Studio
copies an image from elsewhere into <workspace>/Images before running an
op on it (it reads the 403 code `outside_workspace`).
"""
if not raw_path or not raw_path.strip():
raise HTTPException(400, "path is required")
candidate = Path(raw_path)
if candidate.is_absolute():
resolved = candidate.resolve()
else:
resolved = (registry.WORKSPACE_DIR / raw_path).resolve()
if not registry.is_within_workspace(resolved):
raise HTTPException(400, "Invalid path")
if not candidate.is_absolute():
candidate = registry.WORKSPACE_DIR / candidate
resolved = candidate.resolve()
if not registry.is_within_workspace(resolved):
raise HTTPException(403, {"code": "outside_workspace", "message": "The image must be inside the workspace folder."})
if resolved.suffix.lower() not in IMAGE_EXTENSIONS:
raise HTTPException(400, f"Unsupported image type {resolved.suffix or '(none)'}: use png, jpg, jpeg or webp")
if not resolved.is_file():
Expand Down
108 changes: 103 additions & 5 deletions api/services/image_ops/builtin.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import numpy as np
from PIL import Image, ImageOps, UnidentifiedImageError

from . import birefnet
from . import birefnet, framing
from .outputs import png_bytes, sanitize_stem, write_new_files
from .requirements import BIREFNET_LITE, ONNXRUNTIME
from .types import ImageOp, ImageOpContext, ImageOpExecutionError, ImageOpInputError, ImageOpResult
Expand All @@ -23,11 +23,23 @@ def load_image(path: Path) -> Image.Image:
raise ImageOpInputError(f"Cannot read image {path.name}: {exc}") from exc


def _has_alpha(image: Image.Image) -> bool:
"""Whether the image can hold transparency: an alpha channel, or a palette /
colour-key entry (PNG tRNS on P, RGB or L images). Reads no pixels."""
return image.mode in ("RGBA", "LA", "PA", "RGBa", "La") or "transparency" in image.info


def _alpha_u8(image: Image.Image) -> np.ndarray | None:
"""The input's own alpha channel (uint8), None when it has no transparency."""
if not _has_alpha(image):
return None
return np.asarray(image.convert("RGBA").getchannel("A"))


def _source_alpha(image: Image.Image) -> np.ndarray | None:
"""The input's own transparency in [0, 1], if it has any."""
if image.mode in ("RGBA", "LA", "PA") or (image.mode == "P" and "transparency" in image.info):
return np.asarray(image.convert("RGBA").getchannel("A"), dtype=np.float32) / 255.0
return None
alpha = _alpha_u8(image)
return None if alpha is None else alpha.astype(np.float32) / 255.0


def remove_background(input_path: Path, params: Mapping[str, Any], context: ImageOpContext) -> ImageOpResult:
Expand Down Expand Up @@ -85,4 +97,90 @@ def remove_background(input_path: Path, params: Mapping[str, Any], context: Imag
requirements=(ONNXRUNTIME, BIREFNET_LITE),
)

BUILTIN_IMAGE_OPS = (REMOVE_BACKGROUND,)

def _edge_param(param_id: str, label: str) -> dict[str, Any]:
return {
"id": param_id,
"label": label,
"type": "int",
"default": 0,
"min": 0,
"max": 45,
"step": 1,
"tooltip": f"Percentage of the image cut off the {param_id} edge.",
}


def crop(input_path: Path, params: Mapping[str, Any], context: ImageOpContext) -> ImageOpResult:
image = load_image(input_path)
image = image.convert("RGBA" if _has_alpha(image) else "RGB")
box = framing.crop_box(
image.width, image.height,
params.get("left", 0), params.get("top", 0), params.get("right", 0), params.get("bottom", 0),
)
cropped = image.crop(box)
(out_path,) = write_new_files(context.images_dir, sanitize_stem(input_path.name), [("crop", png_bytes(cropped))])
return ImageOpResult(path=out_path, details={"width": cropped.width, "height": cropped.height, "box": list(box)})


CROP = ImageOp(
id="crop",
label="Crop",
params_schema=(
_edge_param("left", "Left %"),
_edge_param("top", "Top %"),
_edge_param("right", "Right %"),
_edge_param("bottom", "Bottom %"),
),
fn=crop,
)


def auto_frame(input_path: Path, params: Mapping[str, Any], context: ImageOpContext) -> ImageOpResult:
image = load_image(input_path)
alpha = _alpha_u8(image)
# An alpha channel with no transparent pixel (clipboard pastes, screenshots)
# is an opaque photo: framing it would only letterbox the whole picture.
if alpha is None or not (alpha <= framing.SUBJECT_ALPHA_THRESHOLD).any():
raise ImageOpInputError("Auto-frame needs a transparent background: remove the background first.")
bbox = framing.subject_bbox(alpha)
if bbox is None:
raise ImageOpInputError("Auto-frame found no subject: the image is fully transparent.")
size = int(params.get("size", 1024))
framed = framing.frame_square(image, bbox, size, float(params.get("margin", 10)))
(out_path,) = write_new_files(context.images_dir, sanitize_stem(input_path.name), [("framed", png_bytes(framed))])
return ImageOpResult(path=out_path, details={"width": size, "height": size, "subject_box": list(bbox)})


AUTO_FRAME = ImageOp(
id="auto_frame",
label="Auto-frame",
params_schema=(
{
"id": "margin",
"label": "Margin %",
"type": "int",
"default": 10,
"min": 0,
"max": 40,
"step": 1,
"tooltip": "Empty space kept around the subject on every side, as a percentage of the square.",
},
{
"id": "size",
"label": "Resolution",
"type": "select",
"default": 1024,
"options": [
{"value": 512, "label": "512 × 512"},
{"value": 768, "label": "768 × 768"},
{"value": 1024, "label": "1024 × 1024"},
{"value": 2048, "label": "2048 × 2048"},
],
"tooltip": "Size of the square image written, and sent to Generate as is.",
},
),
fn=auto_frame,
)

BUILTIN_IMAGE_OPS = (CROP, REMOVE_BACKGROUND, AUTO_FRAME)
57 changes: 57 additions & 0 deletions api/services/image_ops/framing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
"""Crop and auto-frame geometry, kept apart from file I/O so it can be tested on its own.

Generators resize and crop their input each in their own way; framing the
subject ourselves (centered, square, known margin, known resolution) means the
file sent to Generate is already what the model should see.
"""

from typing import Optional

import numpy as np
from PIL import Image

# Alpha (0..255) at or below this is background: background removal leaves a
# faint haze of near-zero alpha that must not stretch the subject's box.
SUBJECT_ALPHA_THRESHOLD = 12

Box = tuple[int, int, int, int] # left, top, right, bottom (right/bottom exclusive)


def crop_box(width: int, height: int, left: float, top: float, right: float, bottom: float) -> Box:
"""The box left after trimming each edge by a percentage of the image side.

Always at least 1 px wide and high.
"""
x0 = min(width - 1, round(width * left / 100))
y0 = min(height - 1, round(height * top / 100))
x1 = max(x0 + 1, width - round(width * right / 100))
y1 = max(y0 + 1, height - round(height * bottom / 100))
return x0, y0, x1, y1


def subject_bbox(alpha: np.ndarray, threshold: int = SUBJECT_ALPHA_THRESHOLD) -> Optional[Box]:
"""Bounding box of the pixels whose alpha is above `threshold`, None if there are none."""
visible = alpha > threshold
rows = np.flatnonzero(visible.any(axis=1))
cols = np.flatnonzero(visible.any(axis=0))
if rows.size == 0:
return None
return int(cols[0]), int(rows[0]), int(cols[-1]) + 1, int(rows[-1]) + 1


def frame_square(image: Image.Image, bbox: Box, size: int, margin_percent: float) -> Image.Image:
"""The subject in `bbox`, scaled so its longer side spans `size` minus a
margin on each side, centered on a transparent `size` x `size` canvas."""
margin = round(size * margin_percent / 100)
inner = max(1, size - 2 * margin)
subject = image.convert("RGBA").crop(bbox)
scale = inner / max(subject.width, subject.height)
scaled_size = (
max(1, min(inner, round(subject.width * scale))),
max(1, min(inner, round(subject.height * scale))),
)
# Resample premultiplied so transparent pixels don't bleed their color into the edges.
scaled = subject.convert("RGBa").resize(scaled_size, Image.LANCZOS).convert("RGBA")
canvas = Image.new("RGBA", (size, size), (0, 0, 0, 0))
canvas.paste(scaled, ((size - scaled.width) // 2, (size - scaled.height) // 2))
return canvas
170 changes: 170 additions & 0 deletions api/tests/test_image_ops_framing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
import tempfile
import unittest
from pathlib import Path

import numpy as np
from PIL import Image

from services.image_ops import ImageOpContext, ImageOpInputError, image_ops_registry
from services.image_ops import framing


def _open(path) -> Image.Image:
with Image.open(path) as image:
image.load()
return image


def _visible_box(image: Image.Image) -> tuple[int, int, int, int]:
return framing.subject_bbox(np.asarray(image.getchannel("A")))


class CropBoxTests(unittest.TestCase):
def test_trims_each_edge_by_its_percentage(self) -> None:
self.assertEqual(framing.crop_box(200, 100, 10, 20, 25, 0), (20, 20, 150, 100))

def test_no_trim_keeps_the_whole_image(self) -> None:
self.assertEqual(framing.crop_box(64, 32, 0, 0, 0, 0), (0, 0, 64, 32))

def test_never_empty(self) -> None:
self.assertEqual(framing.crop_box(1, 1, 45, 45, 45, 45), (0, 0, 1, 1))
x0, y0, x1, y1 = framing.crop_box(3, 3, 60, 60, 60, 60)
self.assertGreater(x1, x0)
self.assertGreater(y1, y0)


class SubjectBoxTests(unittest.TestCase):
def test_box_of_the_visible_pixels(self) -> None:
alpha = np.zeros((10, 20), dtype=np.uint8)
alpha[2:5, 3:9] = 255
self.assertEqual(framing.subject_bbox(alpha), (3, 2, 9, 5))

def test_faint_haze_is_ignored(self) -> None:
alpha = np.full((10, 20), framing.SUBJECT_ALPHA_THRESHOLD, dtype=np.uint8)
alpha[4, 4] = 200
self.assertEqual(framing.subject_bbox(alpha), (4, 4, 5, 5))

def test_none_when_fully_transparent(self) -> None:
self.assertIsNone(framing.subject_bbox(np.zeros((4, 4), dtype=np.uint8)))


class FrameSquareTests(unittest.TestCase):
def test_off_center_subject_is_centered_with_a_uniform_margin(self) -> None:
image = Image.new("RGBA", (300, 200), (0, 0, 0, 0))
image.paste((200, 30, 30, 255), (200, 20, 280, 60)) # 80x40, near the top-right corner
framed = framing.frame_square(image, (200, 20, 280, 60), 100, 10)
self.assertEqual((framed.mode, framed.size), ("RGBA", (100, 100)))
# The longer side spans the square minus 10 px on each side; the other is centered.
self.assertEqual(_visible_box(framed), (10, 30, 90, 70))

def test_tall_subject_keeps_its_aspect_ratio(self) -> None:
image = Image.new("RGBA", (50, 100), (0, 0, 0, 0))
image.paste((0, 0, 255, 255), (0, 0, 25, 100))
framed = framing.frame_square(image, (0, 0, 25, 100), 200, 0)
self.assertEqual(_visible_box(framed), (75, 0, 125, 200))

def test_transparent_edges_do_not_darken(self) -> None:
image = Image.new("RGBA", (40, 40), (0, 0, 0, 0))
image.paste((255, 255, 255, 255), (10, 10, 30, 30))
framed = framing.frame_square(image, (0, 0, 40, 40), 97, 0)
rgba = np.asarray(framed)
partly = (rgba[..., 3] >= 32) & (rgba[..., 3] < 255)
self.assertTrue(partly.any())
self.assertGreater(rgba[partly][:, :3].min(), 240)


class FramingOpsTests(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.workspace = Path(self._tmp.name) / "workspace"
self.workspace.mkdir()
self.context = ImageOpContext(workspace_dir=self.workspace)

def tearDown(self) -> None:
self._tmp.cleanup()

def _save(self, image: Image.Image, name: str) -> Path:
path = self.workspace / name
image.save(path)
return path

def test_ops_need_no_download(self) -> None:
for op_id in ("crop", "auto_frame"):
with self.subTest(op=op_id):
self.assertEqual(image_ops_registry.get(op_id).missing(), [])

def test_crop_writes_a_new_image(self) -> None:
pixels = np.arange(40 * 20 * 3, dtype=np.uint32).reshape(20, 40, 3).astype(np.uint8)
photo = self._save(Image.fromarray(pixels), "photo.png")
result = image_ops_registry.run("crop", photo, {"left": 25, "bottom": 40}, self.context)
self.assertRegex(result.path.name, r"^photo-crop-\d{8}-\d{6}-[0-9a-f]{6}\.png$")
self.assertIsNone(result.mask_path)
out = _open(result.path)
self.assertEqual((out.mode, out.size), ("RGB", (30, 12)))
np.testing.assert_array_equal(np.asarray(out), pixels[:12, 10:])
self.assertEqual(_open(photo).size, (40, 20))

def test_crop_keeps_transparency(self) -> None:
photo = self._save(Image.new("RGBA", (10, 10), (1, 2, 3, 0)), "cut.png")
result = image_ops_registry.run("crop", photo, {"top": 20}, self.context)
self.assertEqual(_open(result.path).mode, "RGBA")

def test_colour_key_transparency_is_honoured(self) -> None:
# RGB and L PNGs can carry their transparency as a tRNS colour key.
for mode, key in (("RGB", (0, 0, 0)), ("L", 0)):
with self.subTest(mode=mode):
keyed = Image.new(mode, (40, 20), key)
keyed.paste((200, 30, 30) if mode == "RGB" else 200, (25, 2, 35, 12))
path = self.workspace / f"keyed-{mode}.png"
keyed.save(path, transparency=key)
self.assertIn("transparency", _open(path).info)

cropped = _open(image_ops_registry.run("crop", path, {"left": 10}, self.context).path)
self.assertEqual(cropped.mode, "RGBA")
self.assertEqual(cropped.getpixel((0, 0))[3], 0)

framed = _open(image_ops_registry.run("auto_frame", path, {"margin": 0, "size": 512}, self.context).path)
self.assertEqual(_visible_box(framed), (0, 0, 512, 512))

def test_crop_edges_are_clamped(self) -> None:
self.assertEqual(image_ops_registry.resolve_params("crop", {"left": 90})["left"], 45)

def test_auto_frame_centers_an_off_center_cutout(self) -> None:
cutout = Image.new("RGBA", (400, 300), (0, 0, 0, 0))
cutout.paste((30, 200, 30, 255), (10, 200, 110, 290)) # 100x90, bottom-left
path = self._save(cutout, "cat-nobg.png")
result = image_ops_registry.run("auto_frame", path, {"margin": 10, "size": 512}, self.context)
self.assertRegex(result.path.name, r"^cat-nobg-framed-\d{8}-\d{6}-[0-9a-f]{6}\.png$")
framed = _open(result.path)
self.assertEqual((framed.mode, framed.size), ("RGBA", (512, 512)))
left, top, right, bottom = _visible_box(framed)
margin = round(512 * 0.10)
self.assertEqual((left, 512 - right), (margin, margin))
self.assertLessEqual(abs(top - (512 - bottom)), 1)

def test_auto_frame_defaults_to_1024(self) -> None:
cutout = Image.new("RGBA", (64, 64), (0, 0, 0, 0))
cutout.paste((255, 0, 0, 255), (0, 0, 8, 8))
result = image_ops_registry.run("auto_frame", self._save(cutout, "dot.png"), {}, self.context)
self.assertEqual(_open(result.path).size, (1024, 1024))
self.assertEqual(_visible_box(_open(result.path))[0], 102)

def test_auto_frame_rejects_an_unknown_resolution(self) -> None:
with self.assertRaises(ValueError):
image_ops_registry.resolve_params("auto_frame", {"size": 999})

def test_auto_frame_needs_transparency(self) -> None:
opaque = self._save(Image.new("RGB", (16, 16), (9, 9, 9)), "photo.jpg")
with self.assertRaisesRegex(ImageOpInputError, "remove the background"):
image_ops_registry.run("auto_frame", opaque, {}, self.context)
pasted = self._save(Image.new("RGBA", (16, 16), (9, 9, 9, 255)), "pasted.png")
with self.assertRaisesRegex(ImageOpInputError, "remove the background"):
image_ops_registry.run("auto_frame", pasted, {}, self.context)
clear = self._save(Image.new("RGBA", (16, 16), (9, 9, 9, 0)), "clear.png")
with self.assertRaisesRegex(ImageOpInputError, "no subject"):
image_ops_registry.run("auto_frame", clear, {}, self.context)
self.assertFalse((self.workspace / "Images").exists())


if __name__ == "__main__":
unittest.main()
Loading
Loading