From c07c2a3d860cc3a53d8121a201c7a196999b2011 Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Tue, 17 Nov 2026 17:53:12 +0100 Subject: [PATCH 1/9] feat(image-studio): add crop and auto-frame image ops Crop trims a percentage off each edge. Auto-frame recenters the subject on its alpha bounding box and pads it to a transparent square at a chosen resolution with a uniform margin, so generators get a framed input instead of cropping it each their own way. --- api/services/image_ops/builtin.py | 88 +++++++++++++++- api/services/image_ops/framing.py | 57 ++++++++++ api/tests/test_image_ops_framing.py | 150 +++++++++++++++++++++++++++ api/tests/test_image_ops_registry.py | 4 +- 4 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 api/services/image_ops/framing.py create mode 100644 api/tests/test_image_ops_framing.py diff --git a/api/services/image_ops/builtin.py b/api/services/image_ops/builtin.py index d603e601..c96d38fc 100644 --- a/api/services/image_ops/builtin.py +++ b/api/services/image_ops/builtin.py @@ -6,7 +6,7 @@ import numpy as np from PIL import Image, ImageOps, UnidentifiedImageError -from . import birefnet +from . import birefnet, framing from .outputs import png_bytes, sanitize_stem, write_new_files from .requirements import BIREFNET_LITE, ONNXRUNTIME from .types import ImageOp, ImageOpContext, ImageOpExecutionError, ImageOpInputError, ImageOpResult @@ -85,4 +85,88 @@ def remove_background(input_path: Path, params: Mapping[str, Any], context: Imag requirements=(ONNXRUNTIME, BIREFNET_LITE), ) -BUILTIN_IMAGE_OPS = (REMOVE_BACKGROUND,) + +def _edge_param(param_id: str, label: str) -> dict[str, Any]: + return { + "id": param_id, + "label": label, + "type": "int", + "default": 0, + "min": 0, + "max": 45, + "step": 1, + "tooltip": f"Percentage of the image cut off the {param_id} edge.", + } + + +def crop(input_path: Path, params: Mapping[str, Any], context: ImageOpContext) -> ImageOpResult: + image = load_image(input_path) + image = image.convert("RGBA" if _source_alpha(image) is not None else "RGB") + box = framing.crop_box( + image.width, image.height, + params.get("left", 0), params.get("top", 0), params.get("right", 0), params.get("bottom", 0), + ) + cropped = image.crop(box) + (out_path,) = write_new_files(context.images_dir, sanitize_stem(input_path.name), [("crop", png_bytes(cropped))]) + return ImageOpResult(path=out_path, details={"width": cropped.width, "height": cropped.height, "box": list(box)}) + + +CROP = ImageOp( + id="crop", + label="Crop", + params_schema=( + _edge_param("left", "Left %"), + _edge_param("top", "Top %"), + _edge_param("right", "Right %"), + _edge_param("bottom", "Bottom %"), + ), + fn=crop, +) + + +def auto_frame(input_path: Path, params: Mapping[str, Any], context: ImageOpContext) -> ImageOpResult: + image = load_image(input_path) + alpha = _source_alpha(image) + if alpha is None: + raise ImageOpInputError("Auto-frame needs a transparent background: remove the background first.") + bbox = framing.subject_bbox(birefnet.to_uint8(alpha)) + if bbox is None: + raise ImageOpInputError("Auto-frame found no subject: the image is fully transparent.") + size = int(params.get("size", 1024)) + framed = framing.frame_square(image, bbox, size, float(params.get("margin", 10))) + (out_path,) = write_new_files(context.images_dir, sanitize_stem(input_path.name), [("framed", png_bytes(framed))]) + return ImageOpResult(path=out_path, details={"width": size, "height": size, "subject_box": list(bbox)}) + + +AUTO_FRAME = ImageOp( + id="auto_frame", + label="Auto-frame", + params_schema=( + { + "id": "margin", + "label": "Margin %", + "type": "int", + "default": 10, + "min": 0, + "max": 40, + "step": 1, + "tooltip": "Empty space kept around the subject on every side, as a percentage of the square.", + }, + { + "id": "size", + "label": "Resolution", + "type": "select", + "default": 1024, + "options": [ + {"value": 512, "label": "512 × 512"}, + {"value": 768, "label": "768 × 768"}, + {"value": 1024, "label": "1024 × 1024"}, + {"value": 2048, "label": "2048 × 2048"}, + ], + "tooltip": "Size of the square image written, and sent to Generate as is.", + }, + ), + fn=auto_frame, +) + +BUILTIN_IMAGE_OPS = (CROP, REMOVE_BACKGROUND, AUTO_FRAME) diff --git a/api/services/image_ops/framing.py b/api/services/image_ops/framing.py new file mode 100644 index 00000000..b37a7d6b --- /dev/null +++ b/api/services/image_ops/framing.py @@ -0,0 +1,57 @@ +"""Crop and auto-frame geometry, kept apart from file I/O so it can be tested on its own. + +Generators resize and crop their input each in their own way; framing the +subject ourselves (centered, square, known margin, known resolution) means the +file sent to Generate is already what the model should see. +""" + +from typing import Optional + +import numpy as np +from PIL import Image + +# Alpha (0..255) at or below this is background: background removal leaves a +# faint haze of near-zero alpha that must not stretch the subject's box. +SUBJECT_ALPHA_THRESHOLD = 12 + +Box = tuple[int, int, int, int] # left, top, right, bottom (right/bottom exclusive) + + +def crop_box(width: int, height: int, left: float, top: float, right: float, bottom: float) -> Box: + """The box left after trimming each edge by a percentage of the image side. + + Always at least 1 px wide and high. + """ + x0 = min(width - 1, round(width * left / 100)) + y0 = min(height - 1, round(height * top / 100)) + x1 = max(x0 + 1, width - round(width * right / 100)) + y1 = max(y0 + 1, height - round(height * bottom / 100)) + return x0, y0, x1, y1 + + +def subject_bbox(alpha: np.ndarray, threshold: int = SUBJECT_ALPHA_THRESHOLD) -> Optional[Box]: + """Bounding box of the pixels whose alpha is above `threshold`, None if there are none.""" + visible = alpha > threshold + rows = np.flatnonzero(visible.any(axis=1)) + cols = np.flatnonzero(visible.any(axis=0)) + if rows.size == 0: + return None + return int(cols[0]), int(rows[0]), int(cols[-1]) + 1, int(rows[-1]) + 1 + + +def frame_square(image: Image.Image, bbox: Box, size: int, margin_percent: float) -> Image.Image: + """The subject in `bbox`, scaled so its longer side spans `size` minus a + margin on each side, centered on a transparent `size` x `size` canvas.""" + margin = round(size * margin_percent / 100) + inner = max(1, size - 2 * margin) + subject = image.convert("RGBA").crop(bbox) + scale = inner / max(subject.width, subject.height) + scaled_size = ( + max(1, min(inner, round(subject.width * scale))), + max(1, min(inner, round(subject.height * scale))), + ) + # Resample premultiplied so transparent pixels don't bleed their color into the edges. + scaled = subject.convert("RGBa").resize(scaled_size, Image.LANCZOS).convert("RGBA") + canvas = Image.new("RGBA", (size, size), (0, 0, 0, 0)) + canvas.paste(scaled, ((size - scaled.width) // 2, (size - scaled.height) // 2)) + return canvas diff --git a/api/tests/test_image_ops_framing.py b/api/tests/test_image_ops_framing.py new file mode 100644 index 00000000..b99fcd0d --- /dev/null +++ b/api/tests/test_image_ops_framing.py @@ -0,0 +1,150 @@ +import tempfile +import unittest +from pathlib import Path + +import numpy as np +from PIL import Image + +from services.image_ops import ImageOpContext, ImageOpInputError, image_ops_registry +from services.image_ops import framing + + +def _open(path) -> Image.Image: + with Image.open(path) as image: + image.load() + return image + + +def _visible_box(image: Image.Image) -> tuple[int, int, int, int]: + return framing.subject_bbox(np.asarray(image.getchannel("A"))) + + +class CropBoxTests(unittest.TestCase): + def test_trims_each_edge_by_its_percentage(self) -> None: + self.assertEqual(framing.crop_box(200, 100, 10, 20, 25, 0), (20, 20, 150, 100)) + + def test_no_trim_keeps_the_whole_image(self) -> None: + self.assertEqual(framing.crop_box(64, 32, 0, 0, 0, 0), (0, 0, 64, 32)) + + def test_never_empty(self) -> None: + self.assertEqual(framing.crop_box(1, 1, 45, 45, 45, 45), (0, 0, 1, 1)) + x0, y0, x1, y1 = framing.crop_box(3, 3, 60, 60, 60, 60) + self.assertGreater(x1, x0) + self.assertGreater(y1, y0) + + +class SubjectBoxTests(unittest.TestCase): + def test_box_of_the_visible_pixels(self) -> None: + alpha = np.zeros((10, 20), dtype=np.uint8) + alpha[2:5, 3:9] = 255 + self.assertEqual(framing.subject_bbox(alpha), (3, 2, 9, 5)) + + def test_faint_haze_is_ignored(self) -> None: + alpha = np.full((10, 20), framing.SUBJECT_ALPHA_THRESHOLD, dtype=np.uint8) + alpha[4, 4] = 200 + self.assertEqual(framing.subject_bbox(alpha), (4, 4, 5, 5)) + + def test_none_when_fully_transparent(self) -> None: + self.assertIsNone(framing.subject_bbox(np.zeros((4, 4), dtype=np.uint8))) + + +class FrameSquareTests(unittest.TestCase): + def test_off_center_subject_is_centered_with_a_uniform_margin(self) -> None: + image = Image.new("RGBA", (300, 200), (0, 0, 0, 0)) + image.paste((200, 30, 30, 255), (200, 20, 280, 60)) # 80x40, near the top-right corner + framed = framing.frame_square(image, (200, 20, 280, 60), 100, 10) + self.assertEqual((framed.mode, framed.size), ("RGBA", (100, 100))) + # The longer side spans the square minus 10 px on each side; the other is centered. + self.assertEqual(_visible_box(framed), (10, 30, 90, 70)) + + def test_tall_subject_keeps_its_aspect_ratio(self) -> None: + image = Image.new("RGBA", (50, 100), (0, 0, 0, 0)) + image.paste((0, 0, 255, 255), (0, 0, 25, 100)) + framed = framing.frame_square(image, (0, 0, 25, 100), 200, 0) + self.assertEqual(_visible_box(framed), (75, 0, 125, 200)) + + def test_transparent_edges_do_not_darken(self) -> None: + image = Image.new("RGBA", (40, 40), (0, 0, 0, 0)) + image.paste((255, 255, 255, 255), (10, 10, 30, 30)) + framed = framing.frame_square(image, (0, 0, 40, 40), 97, 0) + rgba = np.asarray(framed) + partly = (rgba[..., 3] >= 32) & (rgba[..., 3] < 255) + self.assertTrue(partly.any()) + self.assertGreater(rgba[partly][:, :3].min(), 240) + + +class FramingOpsTests(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.workspace = Path(self._tmp.name) / "workspace" + self.workspace.mkdir() + self.context = ImageOpContext(workspace_dir=self.workspace) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def _save(self, image: Image.Image, name: str) -> Path: + path = self.workspace / name + image.save(path) + return path + + def test_ops_need_no_download(self) -> None: + for op_id in ("crop", "auto_frame"): + with self.subTest(op=op_id): + self.assertEqual(image_ops_registry.get(op_id).missing(), []) + + def test_crop_writes_a_new_image(self) -> None: + pixels = np.arange(40 * 20 * 3, dtype=np.uint32).reshape(20, 40, 3).astype(np.uint8) + photo = self._save(Image.fromarray(pixels), "photo.png") + result = image_ops_registry.run("crop", photo, {"left": 25, "bottom": 40}, self.context) + self.assertRegex(result.path.name, r"^photo-crop-\d{8}-\d{6}-[0-9a-f]{6}\.png$") + self.assertIsNone(result.mask_path) + out = _open(result.path) + self.assertEqual((out.mode, out.size), ("RGB", (30, 12))) + np.testing.assert_array_equal(np.asarray(out), pixels[:12, 10:]) + self.assertEqual(_open(photo).size, (40, 20)) + + def test_crop_keeps_transparency(self) -> None: + photo = self._save(Image.new("RGBA", (10, 10), (1, 2, 3, 0)), "cut.png") + result = image_ops_registry.run("crop", photo, {"top": 20}, self.context) + self.assertEqual(_open(result.path).mode, "RGBA") + + def test_crop_edges_are_clamped(self) -> None: + self.assertEqual(image_ops_registry.resolve_params("crop", {"left": 90})["left"], 45) + + def test_auto_frame_centers_an_off_center_cutout(self) -> None: + cutout = Image.new("RGBA", (400, 300), (0, 0, 0, 0)) + cutout.paste((30, 200, 30, 255), (10, 200, 110, 290)) # 100x90, bottom-left + path = self._save(cutout, "cat-nobg.png") + result = image_ops_registry.run("auto_frame", path, {"margin": 10, "size": 512}, self.context) + self.assertRegex(result.path.name, r"^cat-nobg-framed-\d{8}-\d{6}-[0-9a-f]{6}\.png$") + framed = _open(result.path) + self.assertEqual((framed.mode, framed.size), ("RGBA", (512, 512))) + left, top, right, bottom = _visible_box(framed) + margin = round(512 * 0.10) + self.assertEqual((left, 512 - right), (margin, margin)) + self.assertLessEqual(abs(top - (512 - bottom)), 1) + + def test_auto_frame_defaults_to_1024(self) -> None: + cutout = Image.new("RGBA", (64, 64), (0, 0, 0, 0)) + cutout.paste((255, 0, 0, 255), (0, 0, 8, 8)) + result = image_ops_registry.run("auto_frame", self._save(cutout, "dot.png"), {}, self.context) + self.assertEqual(_open(result.path).size, (1024, 1024)) + self.assertEqual(_visible_box(_open(result.path))[0], 102) + + def test_auto_frame_rejects_an_unknown_resolution(self) -> None: + with self.assertRaises(ValueError): + image_ops_registry.resolve_params("auto_frame", {"size": 999}) + + def test_auto_frame_needs_transparency(self) -> None: + opaque = self._save(Image.new("RGB", (16, 16), (9, 9, 9)), "photo.jpg") + with self.assertRaisesRegex(ImageOpInputError, "remove the background"): + image_ops_registry.run("auto_frame", opaque, {}, self.context) + clear = self._save(Image.new("RGBA", (16, 16), (9, 9, 9, 0)), "clear.png") + with self.assertRaisesRegex(ImageOpInputError, "no subject"): + image_ops_registry.run("auto_frame", clear, {}, self.context) + self.assertFalse((self.workspace / "Images").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_image_ops_registry.py b/api/tests/test_image_ops_registry.py index d7874f4b..d9ef8001 100644 --- a/api/tests/test_image_ops_registry.py +++ b/api/tests/test_image_ops_registry.py @@ -56,8 +56,8 @@ def make_op(requirements=(), fn=None) -> ImageOp: class ImageOpsRegistryTests(unittest.TestCase): def test_builtin_metadata_is_serializable(self) -> None: described = image_ops_registry.describe() - self.assertEqual([d["id"] for d in described], ["remove_background"]) - op = described[0] + self.assertEqual([d["id"] for d in described], ["crop", "remove_background", "auto_frame"]) + op = described[1] self.assertEqual(op["label"], "Remove background") self.assertEqual( [(r["kind"], r["id"]) for r in op["requirements"]], From 4a4f45be4109ec0837b49704e4a9a5cf6caf4a8d Mon Sep 17 00:00:00 2001 From: Lightning Pixel Date: Tue, 17 Nov 2026 17:54:09 +0100 Subject: [PATCH 2/9] feat(image-studio): show what the model sees A canvas toggle shows the selected file exactly as Generate uploads it, at its own resolution with its edges outlined, and says when it is not square. Crop's four edges sit in two columns in the toolbar. --- src/areas/image-studio/ImageOpsToolbar.tsx | 3 ++- src/areas/image-studio/ImageStudioPage.tsx | 30 ++++++++++++++++++++++ src/areas/image-studio/imageOps.test.ts | 23 +++++++++++++++++ src/areas/image-studio/imageOps.ts | 15 +++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/areas/image-studio/ImageOpsToolbar.tsx b/src/areas/image-studio/ImageOpsToolbar.tsx index 8f7f272c..ba5b8e77 100644 --- a/src/areas/image-studio/ImageOpsToolbar.tsx +++ b/src/areas/image-studio/ImageOpsToolbar.tsx @@ -178,7 +178,8 @@ function ImageOpControl({ op, entries, selected, onResult, onActivity }: Toolbar values={params} onChange={(id, value) => setParams((current) => ({ ...current, [id]: value }))} disabled={running} - className="flex w-52 flex-col gap-1" + // Crop's four edges sit in two columns to keep the header short. + className={op.params_schema.length > 2 ? 'grid w-80 grid-cols-2 gap-x-3 gap-y-1' : 'flex w-52 flex-col gap-1'} /> {image.maskPath && (