diff --git a/backend/scene_physics.py b/backend/scene_physics.py new file mode 100644 index 0000000..2884ec1 --- /dev/null +++ b/backend/scene_physics.py @@ -0,0 +1,107 @@ +"""Native geometric placement constraints for static scene assets.""" + +from __future__ import annotations + +import numpy as np + + +RELATIONS = ("floor", "on", "left_of", "right_of", "front_of", "behind", + "wall_left", "wall_right", "wall_front", "wall_back", "ceiling") + + +def transformed_bounds(bounds: np.ndarray, transform: np.ndarray) -> np.ndarray: + """World AABB of a local AABB after rotation, scale and translation.""" + lo, hi = np.asarray(bounds, dtype=float) + corners = np.array([[x, y, z, 1.0] for x in (lo[0], hi[0]) + for y in (lo[1], hi[1]) for z in (lo[2], hi[2])]) + points = (transform @ corners.T).T[:, :3] + return np.stack((points.min(axis=0), points.max(axis=0))) + + +def resolve_translation(local_bounds: np.ndarray, relation: str, + target_bounds: np.ndarray | None = None, + clearance: float = 0.05) -> np.ndarray: + """Place an unrotated/rotated asset relative to target bounds or the floor.""" + if relation not in RELATIONS: + raise ValueError(f"Unsupported placement relation: {relation}") + if not np.isfinite(clearance) or clearance < 0 or clearance > 10: + raise ValueError("clearance must be finite and between 0 and 10") + lo, hi = local_bounds + if relation == "floor": + return np.array([0.0, 0.0, -lo[2]]) + if target_bounds is None: + raise ValueError(f"{relation} requires a previously placed target") + target_lo, target_hi = target_bounds + center = (target_lo + target_hi) / 2 + own_center = (lo + hi) / 2 + position = np.array([center[0] - own_center[0], center[1] - own_center[1], + target_lo[2] - lo[2]]) + if relation == "on": + position[2] = target_hi[2] + clearance - lo[2] + elif relation == "ceiling": + position[2] = target_hi[2] - clearance - hi[2] + elif relation == "wall_left": + position[0] = target_lo[0] + clearance - lo[0] + position[2] = center[2] - own_center[2] + elif relation == "wall_right": + position[0] = target_hi[0] - clearance - hi[0] + position[2] = center[2] - own_center[2] + elif relation == "wall_front": + position[1] = target_lo[1] + clearance - lo[1] + position[2] = center[2] - own_center[2] + elif relation == "wall_back": + position[1] = target_hi[1] - clearance - hi[1] + position[2] = center[2] - own_center[2] + elif relation == "left_of": + position[0] = target_lo[0] - clearance - hi[0] + elif relation == "right_of": + position[0] = target_hi[0] + clearance - lo[0] + elif relation == "front_of": + position[1] = target_lo[1] - clearance - hi[1] + else: + position[1] = target_hi[1] + clearance - lo[1] + return position + + +def penetration_depth(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """Positive on all axes only when AABBs overlap volumetrically.""" + return np.minimum(a[1], b[1]) - np.maximum(a[0], b[0]) + + +def separate_static_aabb(candidate: np.ndarray, others: list[np.ndarray], + clearance: float = 0.05, *, + containment: np.ndarray | None = None, + support: np.ndarray | None = None) -> np.ndarray: + """Remove XY overlap while retaining room and horizontal support bounds.""" + shifted = np.asarray(candidate, dtype=float).copy() + delta = np.zeros(3) + def permitted(bounds: np.ndarray) -> bool: + if containment is not None and (np.any(bounds[0] < containment[0] - 1e-6) + or np.any(bounds[1] > containment[1] + 1e-6)): + return False + if support is not None and (np.any(bounds[0, :2] < support[0, :2] - 1e-6) + or np.any(bounds[1, :2] > support[1, :2] + 1e-6)): + return False + return True + if not permitted(shifted): + raise ValueError("Object exceeds room or support bounds") + for _ in range(max(1, len(others) * 4)): + collision = next((other for other in others + if np.all(penetration_depth(shifted, other) > 1e-6)), None) + if collision is None: + return delta + options = [] + for axis in (0, 1): + for amount in (collision[0, axis] - shifted[1, axis] - clearance, + collision[1, axis] - shifted[0, axis] + clearance): + proposed = shifted.copy() + proposed[:, axis] += amount + if permitted(proposed): + remaining = sum(np.all(penetration_depth(proposed, other) > 1e-6) + for other in others) + options.append((remaining, abs(amount), axis, amount, proposed)) + if not options: + raise ValueError("Unable to resolve collisions within room or support bounds") + _, _, axis, amount, shifted = min(options, key=lambda option: option[:3]) + delta[axis] += amount + raise ValueError("Unable to resolve static scene collisions") diff --git a/backend/scenesmith.py b/backend/scenesmith.py new file mode 100644 index 0000000..18bf609 --- /dev/null +++ b/backend/scenesmith.py @@ -0,0 +1,420 @@ +"""Native asset layout backend inspired by SceneSmith's scene-spec boundary. + +The upstream agent and Drake physics solver are not included here. +""" + +from __future__ import annotations + +import json +import hashlib +import os +from pathlib import Path +import re +import shutil +import subprocess +from typing import Any + +import numpy as np +import trimesh + +from backend.scene_physics import RELATIONS, resolve_translation, separate_static_aabb, transformed_bounds +from backend.scenesmith_preview import render_native_preview + +STAGES = ("floor_plan", "furniture", "wall_mounted", "ceiling_mounted", "manipuland") + + +def _object_stage(item: dict[str, Any]) -> str: + relation = item.get("relation", "floor") + inferred = ("wall_mounted" if isinstance(relation, str) and relation.startswith("wall_") else + "ceiling_mounted" if relation == "ceiling" else + "manipuland" if relation == "on" else "furniture") + stage = item.get("stage", inferred) + if stage not in STAGES[1:]: + raise ValueError(f"Invalid object stage: {stage}") + return stage + + +def asset_catalog(directory: str | Path) -> dict[str, Path]: + root = Path(directory).expanduser().resolve() + if not root.is_dir(): + raise ValueError(f"3D asset directory does not exist: {root}") + catalog = {path.relative_to(root).with_suffix("").as_posix(): path + for path in sorted(root.rglob("*.glb")) if path.is_file()} + if not catalog: + raise ValueError(f"No GLB assets found in {root}") + return catalog + + +def asset_descriptors(catalog: dict[str, Path]) -> list[dict[str, Any]]: + """Read optional adjacent JSON metadata without importing model runtimes.""" + descriptors = [] + for key, path in catalog.items(): + descriptor = {"id": key, "name": path.stem} + sidecar = path.with_suffix(".json") + if sidecar.is_file(): + if sidecar.stat().st_size > 65536: + raise ValueError(f"Asset metadata is too large: {sidecar}") + metadata = json.loads(sidecar.read_text(encoding="utf-8")) + if not isinstance(metadata, dict): + raise ValueError(f"Asset metadata must be an object: {sidecar}") + for field in ("description", "category"): + if field in metadata: + value = metadata[field] + if not isinstance(value, str) or len(value) > 500: + raise ValueError(f"Invalid {field} in {sidecar}") + descriptor[field] = value + if "tags" in metadata: + tags = metadata["tags"] + if not isinstance(tags, list) or len(tags) > 32 or any( + not isinstance(tag, str) or len(tag) > 80 for tag in tags + ): + raise ValueError(f"Invalid tags in {sidecar}") + descriptor["tags"] = tags + descriptors.append(descriptor) + return descriptors + + +def offline_plan(prompt: str, catalog: dict[str, Path]) -> dict[str, Any]: + """Baseline: named assets in a deterministic row, pending a planner model.""" + text = prompt.lower() + selected = [] + for item in asset_descriptors(catalog): + aliases = [item["id"], item["name"], item.get("category", ""), *item.get("tags", [])] + if any(alias and alias.lower() in text for alias in aliases): + selected.append(item["id"]) + if not selected: + raise ValueError("Prompt did not name an asset; use an asset ID or provide a scene spec") + return {"schema": "ssui.scene-layout/1", "objects": [ + {"id": f"object-{index:03d}", "asset": key, + "position": [float(index * 3), 0.0, 0.0], "rotation_deg": 0.0, "scale": 1.0} + for index, key in enumerate(selected)]} + + +def enforce_text_relations(prompt: str, spec: dict[str, Any], + catalog: dict[str, Path]) -> dict[str, Any]: + """Turn explicit pairwise language constraints into ordered scene relations.""" + objects = spec.get("objects") + if not isinstance(objects, list): + return spec + aliases = {} + for descriptor in asset_descriptors(catalog): + for alias in [descriptor["id"], descriptor["name"], *descriptor.get("tags", [])]: + if alias and len(alias) >= 2: + aliases[alias.lower()] = descriptor["id"] + choices = sorted(aliases, key=len, reverse=True) + text = prompt.lower() + constraints = [] + for subject in choices: + for target in choices: + if aliases[subject] == aliases[target]: + continue + pattern = (re.escape(subject) + r"\s*(?:放在|摆在|位于|在)?\s*" + + re.escape(target) + r"\s*(?:的)?\s*(左边|右边|前面|后面|上面)") + match = re.search(pattern, text) + english = None + if not match: + english = re.search( + re.escape(subject) + r"\s+(?:is\s+)?(?:to\s+the\s+)?" + r"(left of|right of|in front of|behind|on top of)\s+" + re.escape(target), text) + if not match and not english: + continue + subject_item = next((item for item in objects if isinstance(item, dict) and item.get("asset") == aliases[subject]), None) + target_item = next((item for item in objects if isinstance(item, dict) and item.get("asset") == aliases[target]), None) + if subject_item is None or target_item is None or subject_item is target_item: + continue + relation = ({"左边": "left_of", "右边": "right_of", "前面": "front_of", + "后面": "behind", "上面": "on"}[match.group(1)] if match else + {"left of": "left_of", "right of": "right_of", "in front of": "front_of", + "behind": "behind", "on top of": "on"}[english.group(1)]) + if not target_item.get("id"): + target_item["id"] = f"object-{objects.index(target_item):03d}" + existing = next((entry for entry in constraints if entry[0] is subject_item), None) + if existing is not None: + if existing[1] is target_item and subject_item["relation"] == relation: + break + raise ValueError(f"Conflicting placement relations for {subject_item['asset']}") + subject_item["relation"] = relation + subject_item["relative_to"] = target_item.get("id") + subject_item.pop("position", None) + constraints.append((subject_item, target_item)) + break + if constraints: + ordered = [] + pending = list(objects) + while pending: + ready = next((item for item in pending if all( + subject is not item or target in ordered + for subject, target in constraints)), None) + if ready is None: + raise ValueError("Cyclic placement relations in prompt") + ordered.append(ready) + pending.remove(ready) + objects[:] = ordered + return spec + + +def plan_with_local_llm(prompt: str, catalog: dict[str, Path], output_dir: str | Path, + python: str | Path, model_path: str | Path) -> dict[str, Any]: + """Native subprocess keeps GPU model dependencies out of the CPU backend.""" + output = Path(output_dir) + output.mkdir(parents=True, exist_ok=True) + request_path, response_path, log_path = (output / "planner-request.json", + output / "planner-response.txt", output / "planner.log") + if not Path(python).is_file() or not Path(model_path).is_dir(): + raise RuntimeError("Local planner Python or model directory is missing") + request_path.write_text(json.dumps({"prompt": prompt, "assets": asset_descriptors(catalog)}, ensure_ascii=False), encoding="utf-8") + runner = Path(__file__).with_name("scenesmith_llm_runner.py") + environment = os.environ.copy() + temp = output / "tmp" + temp.mkdir(exist_ok=True) + environment.update(TEMP=str(temp), TMP=str(temp), TRITON_CACHE_DIR=str(output / "triton"), + HF_HOME=str(output / "huggingface"), HF_MODULES_CACHE=str(output / "huggingface/modules"), + PYTHONUTF8="1", PYTHONIOENCODING="utf-8:replace") + command = [str(python), str(runner), "--model", str(model_path), "--request", str(request_path), + "--output", str(response_path)] + with log_path.open("w", encoding="utf-8") as stream: + result = subprocess.run(command, cwd=output, env=environment, stdout=stream, + stderr=subprocess.STDOUT, timeout=900, + creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0) + if result.returncode or not response_path.is_file(): + raise RuntimeError(f"Local scene planner failed; log: {log_path}") + response = response_path.read_text(encoding="utf-8") + try: + first, last = response.index("{"), response.rindex("}") + 1 + spec = json.loads(response[first:last]) + except (ValueError, json.JSONDecodeError) as exc: + raise ValueError(f"Local scene planner did not return valid JSON; response: {response_path}") from exc + if not isinstance(spec, dict): + raise ValueError("Local scene planner must return a JSON object") + return enforce_text_relations(prompt, spec, catalog) + + +def _vector(value: Any, name: str) -> np.ndarray: + if not isinstance(value, (list, tuple)) or len(value) != 3: + raise ValueError(f"{name} must have three numbers") + result = np.asarray(value, dtype=float) + if not np.isfinite(result).all(): + raise ValueError(f"{name} must be finite") + return result + + +def _add_room(scene: trimesh.Scene, room: dict[str, Any]) -> dict[str, Any]: + """Create an editable floor and walls with optional rectangular openings.""" + size = _vector(room.get("size", [6, 6, 3]), "room.size") + if np.any(size < 1) or np.any(size > 100): + raise ValueError("room.size must be between 1 and 100 meters per axis") + width, depth, height = size + thickness = 0.1 + openings = room.get("openings", []) + if not isinstance(openings, list) or len(openings) > 4: + raise ValueError("room.openings must be a list of at most four openings") + by_wall = {} + for opening in openings: + if not isinstance(opening, dict) or opening.get("wall") not in ("left", "right", "front", "back"): + raise ValueError("Opening needs a valid wall") + wall = opening["wall"] + if wall in by_wall: + raise ValueError(f"Only one opening per wall is supported: {wall}") + span = depth if wall in ("left", "right") else width + center = float(opening.get("center", 0.0)) + opening_width = float(opening.get("width", 1.0)) + bottom = float(opening.get("bottom", 0.0)) + opening_height = float(opening.get("height", 2.0)) + if (not np.isfinite([center, opening_width, bottom, opening_height]).all() + or opening_width <= 0 or opening_height <= 0 or bottom < 0 + or abs(center) + opening_width / 2 >= span / 2 + or bottom + opening_height > height): + raise ValueError(f"Opening exceeds wall bounds: {wall}") + by_wall[wall] = {"wall": wall, "center": center, "width": opening_width, + "bottom": bottom, "height": opening_height, + "kind": str(opening.get("kind", "door"))} + parts = {"room.floor": ((width, depth, thickness), (0, 0, -thickness / 2))} + for wall in ("left", "right", "front", "back"): + horizontal_x = wall in ("front", "back") + span = width if horizontal_x else depth + fixed = (-depth / 2 if wall == "front" else depth / 2) if horizontal_x else (-width / 2 if wall == "left" else width / 2) + + def add_piece(label: str, low: float, high: float, z_low: float, z_high: float) -> None: + if high - low < 1e-6 or z_high - z_low < 1e-6: + return + center = (low + high) / 2 + z_center = (z_low + z_high) / 2 + extents = ((high - low, thickness, z_high - z_low) if horizontal_x + else (thickness, high - low, z_high - z_low)) + location = ((center, fixed, z_center) if horizontal_x else (fixed, center, z_center)) + parts[f"room.wall.{wall}.{label}"] = (extents, location) + + opening = by_wall.get(wall) + if opening is None: + add_piece("full", -span / 2, span / 2, 0, height) + else: + left = opening["center"] - opening["width"] / 2 + right = opening["center"] + opening["width"] / 2 + add_piece("left", -span / 2, left, 0, height) + add_piece("right", right, span / 2, 0, height) + add_piece("bottom", left, right, 0, opening["bottom"]) + add_piece("top", left, right, opening["bottom"] + opening["height"], height) + for name, (extents, center) in parts.items(): + scene.add_geometry(trimesh.creation.box(extents=extents), node_name=name, + geom_name=name, transform=trimesh.transformations.translation_matrix(center)) + return {"size": size.tolist(), "openings": list(by_wall.values())} + + +def build_scene(spec: dict[str, Any], catalog: dict[str, Path]) -> tuple[trimesh.Scene, dict[str, Any]]: + """Compose exact transforms, preserving separate object nodes in the GLB.""" + if spec.get("schema") != "ssui.scene-layout/1" or not isinstance(spec.get("objects"), list): + raise ValueError("Expected ssui.scene-layout/1 with an objects array") + if len(spec["objects"]) > 256 or (not spec["objects"] and "room" not in spec): + raise ValueError("Scene must contain a room or 1 to 256 objects") + scene = trimesh.Scene() + manifest = {"schema": "ssui.scene-layout/1", "objects": []} + if "room" in spec: + if not isinstance(spec["room"], dict): + raise ValueError("room must be an object") + manifest["room"] = _add_room(scene, spec["room"]) + room_bounds = None + if "room" in manifest: + width, depth, height = manifest["room"]["size"] + room_bounds = np.array([[-width / 2, -depth / 2, 0], + [width / 2, depth / 2, height]]) + seen = set() + placed_bounds = {} + asset_hashes = {} + for index, item in enumerate(spec["objects"]): + if not isinstance(item, dict): + raise ValueError(f"objects[{index}] must be an object") + ident = str(item.get("id") or f"object-{index:03d}") + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,80}", ident) or ident in seen: + raise ValueError(f"Invalid or duplicate object ID: {ident}") + seen.add(ident) + key = item.get("asset") + if key not in catalog: + raise ValueError(f"Unknown asset ID: {key}") + if key not in asset_hashes: + with catalog[key].open("rb") as stream: + asset_hashes[key] = hashlib.file_digest(stream, "sha256").hexdigest() + if item.get("source_sha256") and item["source_sha256"] != asset_hashes[key]: + raise ValueError(f"Asset changed since saved scene state: {key}") + rotation = float(item.get("rotation_deg", 0.0)) + scale = float(item.get("scale", 1.0)) + if not np.isfinite(rotation) or not np.isfinite(scale) or not 0.01 <= scale <= 100: + raise ValueError(f"Invalid rotation or scale for {ident}") + asset = trimesh.load(str(catalog[key]), force="scene") + if not isinstance(asset, trimesh.Scene) or not asset.geometry: + raise ValueError(f"Asset has no geometry: {key}") + base_transform = trimesh.transformations.rotation_matrix(np.radians(rotation), [0, 0, 1]) + base_transform = base_transform @ trimesh.transformations.scale_matrix(scale) + bounds = transformed_bounds(asset.bounds, base_transform) + relation = item.get("relation", "floor") + target = item.get("relative_to") + if relation not in RELATIONS: + raise ValueError(f"Unsupported placement relation: {relation}") + if "position" in item: + position = _vector(item["position"], "position") + else: + wall_or_ceiling = relation.startswith("wall_") or relation == "ceiling" + if wall_or_ceiling and room_bounds is None: + raise ValueError(f"{relation} requires a room") + if target is not None and target not in placed_bounds and target != "room": + raise ValueError(f"Unknown or not yet placed target: {target}") + target_bounds = room_bounds if wall_or_ceiling else placed_bounds.get(target) + position = resolve_translation(bounds, relation, target_bounds, + float(item.get("clearance", 0.05))) + if relation == "floor" and position[2] + bounds[0, 2] < 0: + position[2] = -bounds[0, 2] + transform = trimesh.transformations.translation_matrix(position) @ base_transform + world_bounds = transformed_bounds(asset.bounds, transform) + correction = np.zeros(3) + support_bounds = placed_bounds.get(target) if relation == "on" else None + if room_bounds is not None or support_bounds is not None or not item.get("allow_overlap", False): + correction = separate_static_aabb( + world_bounds, [] if item.get("allow_overlap", False) else list(placed_bounds.values()), + containment=room_bounds, support=support_bounds) + position += correction + transform = trimesh.transformations.translation_matrix(position) @ base_transform + world_bounds = transformed_bounds(asset.bounds, transform) + placed_bounds[ident] = world_bounds + for part, geometry in enumerate(asset.dump()): + scene.add_geometry(geometry, node_name=f"{ident}.part-{part:03d}", + geom_name=f"{ident}.mesh-{part:03d}", transform=transform) + manifest["objects"].append({"id": ident, "asset": key, "position": position.tolist(), + "source_sha256": asset_hashes[key], + "rotation_deg": rotation, "scale": scale, + "relation": relation, "relative_to": target, + "collision_correction": correction.tolist(), + "stage": _object_stage(item)}) + return scene, manifest + + +def _stage_spec(spec: dict[str, Any], stage: str) -> dict[str, Any]: + selected = {"schema": "ssui.scene-layout/1"} + if "room" in spec: + selected["room"] = spec["room"] + cutoff = STAGES.index(stage) + selected["objects"] = [item for item in spec["objects"] + if STAGES.index(_object_stage(item)) <= cutoff] + return selected + + +def _resume_spec(spec: dict[str, Any], resume_from: str | Path) -> dict[str, Any]: + source = Path(resume_from) + state_path = source / "scene.json" if source.is_dir() else source + previous = json.loads(state_path.read_text(encoding="utf-8")) + if previous.get("schema") != "ssui.scene-layout/1" or not isinstance(previous.get("objects"), list): + raise ValueError("Resume state is not a valid SSUI scene layout") + ids = {item.get("id") for item in previous["objects"]} + additions = spec.get("objects", []) + if any(item.get("id") in ids for item in additions): + raise ValueError("New scene objects duplicate IDs in resume state") + merged = {"schema": "ssui.scene-layout/1", + "objects": [*previous["objects"], *additions]} + if "room" in spec or "room" in previous: + merged["room"] = spec.get("room", previous.get("room")) + return merged + + +def generate_scene(prompt: str, asset_dir: str | Path, output_dir: str | Path, + spec: dict[str, Any] | None = None, *, stop_stage: str = "manipuland", + resume_from: str | Path | None = None) -> Path: + if stop_stage not in STAGES: + raise ValueError(f"Invalid stop stage: {stop_stage}") + catalog = asset_catalog(asset_dir) + layout = spec if spec is not None else offline_plan(prompt, catalog) + if not isinstance(layout, dict) or layout.get("schema") != "ssui.scene-layout/1" or not isinstance(layout.get("objects"), list): + raise ValueError("Expected ssui.scene-layout/1 with an objects array") + if any(not isinstance(item, dict) for item in layout["objects"]): + raise ValueError("Scene objects must be JSON objects") + if resume_from is not None: + layout = _resume_spec(layout, resume_from) + output = Path(output_dir) + output.mkdir(parents=True, exist_ok=True) + scene = None + manifest = None + previous_ids = None + previous_stage_dir = None + for stage in STAGES[:STAGES.index(stop_stage) + 1]: + stage_spec = _stage_spec(layout, stage) + if not stage_spec["objects"] and "room" not in stage_spec: + continue + stage_dir = output / "stages" / stage + stage_dir.mkdir(parents=True, exist_ok=True) + current_ids = tuple(item.get("id", index) for index, item in enumerate(stage_spec["objects"])) + if previous_ids == current_ids and previous_stage_dir is not None: + shutil.copyfile(previous_stage_dir / "scene.glb", stage_dir / "scene.glb") + shutil.copyfile(previous_stage_dir / "scene.json", stage_dir / "scene.json") + shutil.copyfile(previous_stage_dir / "preview.png", stage_dir / "preview.png") + else: + scene, manifest = build_scene(stage_spec, catalog) + scene.export(str(stage_dir / "scene.glb")) + (stage_dir / "scene.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + render_native_preview(stage_dir / "scene.glb", stage_dir / "preview.png") + previous_ids = current_ids + previous_stage_dir = stage_dir + if scene is None or manifest is None: + raise ValueError(f"No scene content is available through stage {stop_stage}") + glb = output / "scene.glb" + shutil.copyfile(previous_stage_dir / "scene.glb", glb) + shutil.copyfile(previous_stage_dir / "scene.json", output / "scene.json") + shutil.copyfile(previous_stage_dir / "preview.png", output / "preview.png") + return glb diff --git a/backend/scenesmith_blender.py b/backend/scenesmith_blender.py new file mode 100644 index 0000000..1a84fe1 --- /dev/null +++ b/backend/scenesmith_blender.py @@ -0,0 +1,45 @@ +"""Optional native Blender preview for SceneSmith GLB scenes.""" + +from __future__ import annotations + +import os +from pathlib import Path +import shutil +import subprocess + + +def render_scene_preview(glb: str | Path, output_dir: str | Path, + blender_executable: str | None = None) -> tuple[Path, Path]: + source = Path(glb).expanduser().resolve() + if not source.is_file(): + raise FileNotFoundError(source) + output = Path(output_dir).expanduser().resolve() + output.mkdir(parents=True, exist_ok=True) + preview = output / "preview.png" + blend = output / "scene.blend" + script = Path(__file__).with_name("scenesmith_blender_render.py") + arguments = ["--background", "--factory-startup", "--python", str(script), "--", + str(source), str(preview), str(blend)] + executable = blender_executable or os.environ.get("BLENDER_EXECUTABLE") or shutil.which("blender") or shutil.which("blender.exe") + if executable: + if not Path(executable).is_file(): + raise FileNotFoundError(executable) + command = [executable, *arguments] + elif os.name == "nt": + helper = Path(__file__).resolve().parents[1] / "extensions/Motion/blender/run_msix_blender.ps1" + powershell = shutil.which("pwsh") or shutil.which("powershell") + if not helper.is_file() or not powershell: + raise RuntimeError("Native Blender launcher is unavailable") + helper_quoted = str(helper).replace("'", "''") + arguments_quoted = subprocess.list2cmdline(arguments).replace("'", "''") + script_call = f"& '{helper_quoted}' -Arguments '{arguments_quoted}' -Wait" + command = [powershell, "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", script_call] + else: + raise RuntimeError("Blender not found; set BLENDER_EXECUTABLE") + log = output / "blender.log" + with log.open("w", encoding="utf-8") as stream: + result = subprocess.run(command, stdout=stream, stderr=subprocess.STDOUT, + timeout=600, creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0) + if result.returncode or not preview.is_file() or not blend.is_file(): + raise RuntimeError(f"Blender preview failed; see {log}") + return preview, blend diff --git a/backend/scenesmith_blender_render.py b/backend/scenesmith_blender_render.py new file mode 100644 index 0000000..72766ae --- /dev/null +++ b/backend/scenesmith_blender_render.py @@ -0,0 +1,54 @@ +"""Executed inside Blender's bundled Python; no bpy dependency in SSUI backend.""" + +from pathlib import Path +import sys + +import bpy +from mathutils import Vector + + +def main(): + source, preview, blend = map(Path, sys.argv[sys.argv.index("--") + 1:]) + bpy.ops.object.select_all(action="SELECT") + bpy.ops.object.delete(use_global=False) + bpy.ops.import_scene.gltf(filepath=str(source)) + meshes = [obj for obj in bpy.context.scene.objects if obj.type == "MESH"] + if not meshes: + raise RuntimeError("Imported scene has no mesh") + points = [obj.matrix_world @ Vector(corner) for obj in meshes for corner in obj.bound_box] + low = Vector([min(point[axis] for point in points) for axis in range(3)]) + high = Vector([max(point[axis] for point in points) for axis in range(3)]) + center = (low + high) / 2 + extent = max(high.x - low.x, high.y - low.y, high.z - low.z, 1.0) + bpy.ops.object.camera_add(location=(center.x + extent * 1.6, + center.y - extent * 2.2, + center.z + extent * 1.5)) + camera = bpy.context.object + direction = center - camera.location + camera.rotation_euler = direction.to_track_quat("-Z", "Y").to_euler() + camera.data.type = "ORTHO" + camera.data.ortho_scale = extent * 2.4 + bpy.context.scene.camera = camera + for location, power in (((center.x, center.y, center.z + extent * 2), 1500), + ((center.x - extent, center.y, center.z + extent), 700)): + bpy.ops.object.light_add(type="AREA", location=location) + light = bpy.context.object + light.data.energy = power + light.data.shape = "DISK" + light.data.size = extent * 2 + light.rotation_euler = (center - light.location).to_track_quat("-Z", "Y").to_euler() + scene = bpy.context.scene + scene.render.engine = "BLENDER_EEVEE_NEXT" + scene.render.resolution_x = 640 + scene.render.resolution_y = 480 + scene.render.resolution_percentage = 100 + scene.render.image_settings.file_format = "PNG" + scene.render.filepath = str(preview) + scene.world.color = (0.12, 0.12, 0.12) + bpy.ops.wm.save_as_mainfile(filepath=str(blend)) + bpy.ops.render.render(write_still=True) + print(f"SCENESMITH_RENDERED {preview} {blend}") + + +if __name__ == "__main__": + main() diff --git a/backend/scenesmith_llm_runner.py b/backend/scenesmith_llm_runner.py new file mode 100644 index 0000000..4aa8df3 --- /dev/null +++ b/backend/scenesmith_llm_runner.py @@ -0,0 +1,50 @@ +"""Run a local Qwen scene planner in the native ARM64 CUDA model environment.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--request", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + request = json.loads(Path(args.request).read_text(encoding="utf-8")) + catalog = request["assets"] + prompt = ( + "Return JSON only. Use exactly the keys schema and objects. Each object has id, asset, " + "position, rotation_deg, scale. Positions are [x,y,z] in meters with Z up. " + "Example: assets=[\"sofa\",\"plant\"], request='plant to the left of sofa' => " + "{\"schema\":\"ssui.scene-layout/1\",\"objects\":[" + "{\"id\":\"sofa-1\",\"asset\":\"sofa\",\"position\":[0,0,0],\"rotation_deg\":0,\"scale\":1}," + "{\"id\":\"plant-1\",\"asset\":\"plant\",\"position\":[-2,0,0],\"rotation_deg\":0,\"scale\":1}]}. " + "Use only IDs from available assets.\n" + f"Available assets (use their id fields): {json.dumps(catalog, ensure_ascii=False)}\n" + f"Request: {request['prompt']}\nJSON:" + ) + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=False, local_files_only=True) + model = AutoModelForCausalLM.from_pretrained( + args.model, device_map="auto", trust_remote_code=False, local_files_only=True + ).eval() + if callable(getattr(model, "chat", None)): + with torch.inference_mode(): + answer, _ = model.chat(tokenizer, prompt, history=None) + else: + messages = [{"role": "user", "content": prompt}] + text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + inputs = tokenizer(text, return_tensors="pt").to(model.device) + with torch.inference_mode(): + output = model.generate(**inputs, do_sample=False, max_new_tokens=1024) + answer = tokenizer.decode(output[0, inputs.input_ids.shape[1]:], skip_special_tokens=True) + Path(args.output).write_text(answer, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/backend/scenesmith_preview.py b/backend/scenesmith_preview.py new file mode 100644 index 0000000..2092ff1 --- /dev/null +++ b/backend/scenesmith_preview.py @@ -0,0 +1,72 @@ +"""Small native CPU preview renderer for static SceneSmith GLB geometry.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +from PIL import Image, ImageDraw +import trimesh + + +def render_native_preview(glb: str | Path, output: str | Path, + size: tuple[int, int] = (640, 480)) -> Path: + """Render shaded geometry without a GPU, display server, or Blender process.""" + source = Path(glb).expanduser().resolve() + scene = trimesh.load(source, force="scene") + if not isinstance(scene, trimesh.Scene) or not scene.geometry: + raise ValueError("GLB scene has no geometry") + if not 64 <= size[0] <= 4096 or not 64 <= size[1] <= 4096: + raise ValueError("Preview dimensions must be 64 to 4096 pixels") + meshes = [] + for node in scene.graph.nodes_geometry: + if node.startswith(("room.wall.front", "room.wall.right")): + continue # Cut away camera-facing walls so room contents remain visible. + matrix, geometry_name = scene.graph.get(node) + geometry = scene.geometry[geometry_name] + if not isinstance(geometry, trimesh.Trimesh) or not len(geometry.faces): + continue + vertices = trimesh.transform_points(geometry.vertices, matrix) + meshes.append((vertices, geometry)) + if not meshes: + raise ValueError("GLB scene has no visible triangle meshes") + all_vertices = np.concatenate([vertices for vertices, _ in meshes]) + center = (all_vertices.min(axis=0) + all_vertices.max(axis=0)) / 2 + forward = np.array([-1.0, 1.2, -0.85], dtype=float) + forward /= np.linalg.norm(forward) + right = np.cross(forward, [0.0, 0.0, 1.0]) + right /= np.linalg.norm(right) + up = np.cross(right, forward) + projection = np.stack([right, up, forward], axis=1) + projected_all = (all_vertices - center) @ projection + horizontal = max(float(np.ptp(projected_all[:, 0])), 0.1) + vertical = max(float(np.ptp(projected_all[:, 1])), 0.1) + pixels_per_unit = min(size[0] * 0.82 / horizontal, size[1] * 0.82 / vertical) + image = Image.new("RGB", size, (35, 39, 47)) + draw = ImageDraw.Draw(image) + triangles = [] + light = np.array([0.3, -0.5, 0.82]) + light /= np.linalg.norm(light) + for vertices, mesh in meshes: + projected = (vertices - center) @ projection + points = np.column_stack((size[0] / 2 + projected[:, 0] * pixels_per_unit, + size[1] / 2 - projected[:, 1] * pixels_per_unit)) + visual = mesh.visual + colors = (visual.face_colors if hasattr(visual, "face_colors") else + visual.to_color().face_colors if hasattr(visual, "to_color") else + np.tile(np.array([[180, 185, 192, 255]], dtype=np.uint8), (len(mesh.faces), 1))) + for index, face in enumerate(mesh.faces): + corners = vertices[face] + normal = np.cross(corners[1] - corners[0], corners[2] - corners[0]) + length = np.linalg.norm(normal) + if length < 1e-12: + continue + brightness = 0.4 + 0.6 * abs(float(np.dot(normal / length, light))) + rgb = tuple(int(channel * brightness) for channel in colors[index, :3]) + triangles.append((float(projected[face, 2].mean()), points[face], rgb)) + for _, points, color in sorted(triangles, key=lambda entry: entry[0], reverse=True): + draw.polygon([tuple(point) for point in points], fill=color) + destination = Path(output).expanduser().resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + image.save(destination) + return destination diff --git a/doc/SceneSmithNativePort.md b/doc/SceneSmithNativePort.md new file mode 100644 index 0000000..989d9b5 --- /dev/null +++ b/doc/SceneSmithNativePort.md @@ -0,0 +1,27 @@ +# SceneSmith 完整原生移植检查表 + +上游固定版本:`nepfaff/scenesmith@67cc408fd38334b4a926efef45e284302ed5055b`。本文件记录完整移植所需的工作,不把当前 `backend/scenesmith.py` 的静态 GLB 摆放实现视为完成。 + +## 上游功能与 SSUI 对应项 + +| 上游阶段/子系统 | 完整移植的验收产物 | 当前状态 | +|---|---|---| +| `floor_plan` | 文字生成多房间墙、门、窗、地面及可恢复状态 | 单房间矩形地面/四墙和每墙一个门窗开口可由 JSON 创建;文本生成多房间、状态恢复未完成 | +| `furniture` | 语义检索/生成资产、约束摆放、尺度与碰撞校正 | 结构化 GLB 摆放、Qwen2 单提示词规划(含 SSUI 工作流执行)、多条明确文字关系的依赖排序与循环检测、接地和受房间边界约束的 AABB 碰撞修正已验证;语义检索/生成、复杂碰撞未完成 | +| `wall_mounted` | 墙面支撑和挂装校正 | 静态房间边界定位已测试;真实墙洞/挂装碰撞未完成 | +| `ceiling_mounted` | 天花支撑和悬挂校正 | 静态房间顶面定位已测试;悬挂稳定性未完成 | +| `manipuland` | 桌面小物、堆叠、装填和物理稳定性 | 单物体 `on` 支撑及支撑面边界检查已测试;堆叠、装填、仿真未完成 | +| 资产服务 | HSSD、Objaverse、生成模型与材质索引 | 用户 GLB 目录及名称/类别/标签描述索引;HSSD、Objaverse、资产生成和材质索引未完成 | +| 渲染/物理 | Blender 观察反馈、Drake 投影/仿真 | backend 已为每阶段生成 CPU 静态 PNG 预览,真实 Qwen2 场景画面已检查;原生 Blender 导入/渲染代码已接入,但商店版后台启动未产出文件,Drake 等价物理仿真未完成 | +| 状态/导出 | 五阶段保存/恢复、Blender、Drake、可编辑 GLB | 五阶段静态 GLB/JSON 快照和续接已测试;代理对话、物理状态、Blender/Drake 输出未完成 | + +## 已确认的平台阻碍 + +- 当前 SSUI 共享 backend:Windows ARM64 / CPython 3.12;上游 `pyproject.toml` 要求 Python `>=3.11,<3.12` 和 Torch `>=2.5,<2.6`。 +- 当前共享 backend 中没有 `pydrake`、`bpy`、Torch、Transformers 或 OpenAI Agents;上游至少 34 个 Python 文件直接导入 `pydrake`。 +- Drake 官方支持表不包含 Windows;1.47.0 的 PyPI 文件没有 Windows ARM64 wheel。完整上游算法若坚持在 Windows ARM64 原生运行,需要移植/替换 Drake 几何、碰撞、优化与仿真接口,并验证行为等价。 +- 当前环境没有 `OPENAI_API_KEY` 或 `SSUI_SCENE_ASSETS`;上游完整流程还需要模型权重、资产和相应许可。 + +## 完成判据 + +使用官方或明确等价的依赖,依次跑通五阶段的真实文字输入;保留资产来源和阶段状态;导出可编辑场景;检查物体语义、位置、墙/顶/支撑面约束、碰撞与物理稳定性;通过 SSUI 工作流和桌面实际运行。仅代码导入、模拟测试或静态 GLB 合成不能作为完整移植验收。 diff --git a/examples/advance/workflow-scenesmith.py b/examples/advance/workflow-scenesmith.py new file mode 100644 index 0000000..8f40a47 --- /dev/null +++ b/examples/advance/workflow-scenesmith.py @@ -0,0 +1,11 @@ +"""Text to editable indoor scene through an isolated SceneSmith runtime.""" +from ssui import Mesh, Prompt, workflow +from ssui.config import SSUIConfig +from ssui_3dmodel.SceneSmith import GenSceneSmithScene, SceneSmithModel + +config = SSUIConfig() + + +@workflow +def scenesmith_scene(prompt: Prompt) -> Mesh: + return GenSceneSmithScene(config("SceneSmith Scene"), SceneSmithModel.load_local(), prompt) diff --git a/extensions/3D-Model/README.md b/extensions/3D-Model/README.md index 6b93144..49c7194 100644 --- a/extensions/3D-Model/README.md +++ b/extensions/3D-Model/README.md @@ -1,5 +1,9 @@ # SSUI 3D Model extension +## SceneSmith 室内场景 + +`GenSceneSmithScene` 使用仓库内的 Python backend 摆放已有 GLB 资产,导出带独立网格对象的 GLB、阶段快照和预览图。当前移植的是静态场景布局,上游的多代理与物理求解尚未完成。接口、示例和验证状态见 [SCENESMITH.md](SCENESMITH.md)。 + ## 自动骨骼绑定 新增 UniRig、Make-It-Animatable(MIA)和 RigAnything 隔离运行适配器, diff --git a/extensions/3D-Model/SCENESMITH.md b/extensions/3D-Model/SCENESMITH.md new file mode 100644 index 0000000..0605267 --- /dev/null +++ b/extensions/3D-Model/SCENESMITH.md @@ -0,0 +1,32 @@ +# 原生场景布局 backend + +`backend/scenesmith.py` 在 SSUI 共享 Python 环境中运行,不使用 Docker、WSL 或 SceneSmith 服务。可选的本机 Qwen 规划器通过 Windows ARM64 CUDA Python 子进程运行模型;几何生成和场景导出仍在共享 backend 中完成。工作流入口是 `GenSceneSmithScene`,示例为 `examples/advance/workflow-scenesmith.py`。 + +## 使用 + +准备一个包含静态 GLB 的目录,并将节点的 `asset_dir` 指向它,或设置 `SSUI_SCENE_ASSETS`。相对目录层级去掉 `.glb` 后成为资产 ID,例如 `furniture/chair.glb` → `furniture/chair`。 + +默认文字模式按资产 ID、文件名或资产旁 `同名.json` 的 `category`/`tags` 匹配,再把匹配到的资产按固定间距摆成一行。资产元数据可含 `description`、`category`、`tags`,供本地模型选择。可设置 `planner_model` 为本地 Qwen2 模型目录,并设置 `SSUI_SCENE_PLANNER_PYTHON` 为 Windows ARM64 CUDA 虚拟环境的 Python;此时 backend 调用本机模型生成布局 JSON,再进行格式与资产校验、接地和静态 AABB 碰撞修正。明确的中英文“左/右/前/后/上”关系会作为硬约束校正模型坐标。这个规划器是 Windows ARM64 替代实现,**尚未达到 SceneSmith 上游的多代理空间推理与物理仿真**。 + +精确摆放时,将节点的 `spec_path` 指向以下 JSON(或设置 `SSUI_SCENE_SPEC`): + +```json +{ + "schema": "ssui.scene-layout/1", + "objects": [ + {"id": "chair-1", "asset": "furniture/chair", "position": [1, 2, 0], "rotation_deg": 90, "scale": 1} + ] +} +``` + +位置单位与资产的 GLB 单位相同;Z 为上,旋转绕 Z 轴。可选的 `room: {"size": [宽, 深, 高], "openings": [{"wall": "front", "kind": "door", "width": 1, "height": 2}]}` 会生成独立的地板与墙体,并在指定墙上留矩形门窗洞。对象可指定 `relation` 为 `floor`、`on`、`left_of`、`right_of`、`front_of`、`behind`、`wall_left/right/front/back` 或 `ceiling`;对象间关系用 `relative_to` 指向前面已放置的对象。后端验证资产 ID、物体 ID、有限坐标和缩放范围,对 `floor` 避免穿地,在房间及支撑面边界内消除静态包围盒重叠;无法满足边界时明确报错。输出是 `output/scenesmith/scene_*/scene.glb` 与 `scene.json`。GLB 保留独立节点,SSUI 通过原有 path-backed `Mesh` 返回。当前实现针对静态几何;蒙皮、动画和复杂层级保真仍需单独验证。 + +验证命令:`yarn test:scenesmith` 和 `yarn test:scenesmith:planner`。前者在当前 Windows ARM64 共享 Python 环境创建真实 GLB 资产、生成场景并重新载入;后者使用本地 Qwen2 权重完成真实文字推理、GLB 导出和 SSUI 沙盒工作流执行,仅验证这一个固定提示词,不代表泛化质量或完整上游效果。 + +节点的 `stop_stage` 可设为 `floor_plan`、`furniture`、`wall_mounted`、`ceiling_mounted` 或 `manipuland`。每阶段输出保存在 `output/scenesmith/scene_*/stages//`,包括 `scene.glb`、`scene.json` 与 backend CPU 生成的 `preview.png`;最终目录也有预览图。预览采用简单三角形投影和着色,用于检查位置,不代表材质真实渲染。将旧阶段目录或 `scene.json` 填入 `resume_from`,再提供新增对象规格即可继续;重复物体 ID 会报错。快照记录资产 SHA-256,续接时若资产内容已变会报错。阶段快照目前保存静态几何和位置,不包含上游代理对话或物理状态。可选的 `backend/scenesmith_blender.py` 能调用原生 Blender 导出 `.blend` 和 PNG,但当前机器的商店版后台启动没有产出文件,尚未通过真实渲染验收。 + +## 与上游 SceneSmith 的关系 + +此模块只移植了“资产目录 → 场景规格 → 独立实例”的集成边界。上游 SceneSmith 提交 `67cc408fd38334b4a926efef45e284302ed5055b` 的代理规划、检索服务、Blender 渲染和 Drake 物理求解尚未进入 SSUI backend。上游限制 Python 3.11/Torch 2.5,Drake 1.47.0 没有 Windows ARM64 wheel;SSUI 当前共享环境是 Windows ARM64/Python 3.12。要得到完整上游效果,需要把这些依赖的职责逐一替换为本机兼容实现,不能通过复制上游包直接完成。 + +后续顺序:原生语言规划器输出 `ssui.scene-layout/1`;资产语义索引;碰撞、支撑面及落地校正;桌面 UI 的逐物体编辑;最后做真实提示词质量和速度验收。当前这些项目尚未验收。 diff --git a/extensions/3D-Model/ssui_3dmodel/SceneSmith.py b/extensions/3D-Model/ssui_3dmodel/SceneSmith.py new file mode 100644 index 0000000..ef5e285 --- /dev/null +++ b/extensions/3D-Model/ssui_3dmodel/SceneSmith.py @@ -0,0 +1,68 @@ +"""SSUI workflow entry for the native 3D scene layout backend.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import tempfile + +from backend.scenesmith import STAGES, asset_catalog, generate_scene, plan_with_local_llm +from ssui.annotation import param +from ssui.base import Mesh, Prompt +from ssui.config import SSUIConfig +from ssui.controller import Input, Select + + +ROOT = Path(__file__).resolve().parents[3] + + +class SceneSmithModel: + def __init__(self, asset_dir=None, spec_path=None, planner_model=None, planner_python=None, + stop_stage="manipuland", resume_from=None): + self.asset_dir = asset_dir or os.environ.get("SSUI_SCENE_ASSETS") + self.spec_path = spec_path or os.environ.get("SSUI_SCENE_SPEC") + self.planner_model = planner_model or os.environ.get("SSUI_SCENE_PLANNER_MODEL") + self.planner_python = planner_python or os.environ.get("SSUI_SCENE_PLANNER_PYTHON") + self.stop_stage = stop_stage + self.resume_from = resume_from + + @classmethod + def load_local(cls, asset_dir=None, spec_path=None, planner_model=None, planner_python=None, + stop_stage="manipuland", resume_from=None): + return cls(asset_dir, spec_path, planner_model, planner_python, stop_stage, resume_from) + + def generate(self, prompt: str) -> Mesh: + if not prompt.strip(): + raise ValueError("Scene prompt must be nonempty") + if not self.asset_dir: + raise ValueError("Set SSUI_SCENE_ASSETS to a GLB asset directory") + spec = json.loads(Path(self.spec_path).read_text(encoding="utf-8")) if self.spec_path else None + output = ROOT / "output/scenesmith" + output.mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix="scene_", dir=output)) + if spec is None and self.planner_model: + if not self.planner_python: + raise ValueError("Set SSUI_SCENE_PLANNER_PYTHON for the local Qwen runtime") + spec = plan_with_local_llm(prompt, asset_catalog(self.asset_dir), work, + self.planner_python, self.planner_model) + return Mesh(path=str(generate_scene(prompt, self.asset_dir, work, spec, + stop_stage=self.stop_stage, resume_from=self.resume_from))) + + +@param("asset_dir", Input("GLB asset directory"), default="") +@param("spec_path", Input("Optional scene JSON path"), default="") +@param("planner_model", Input("Optional local Qwen model directory"), default="") +@param("stop_stage", Select(*STAGES), default="manipuland") +@param("resume_from", Input("Optional prior stage scene.json"), default="") +def GenSceneSmithScene(config: SSUIConfig, model: SceneSmithModel, prompt: Prompt) -> Mesh: + if config.is_prepare(): + return Mesh() + text = prompt.text if isinstance(prompt, Prompt) else str(prompt) + asset_dir = config["asset_dir"] or model.asset_dir + spec_path = config["spec_path"] or model.spec_path + planner_model = config["planner_model"] or model.planner_model + stop_stage = config["stop_stage"] or model.stop_stage + resume_from = config["resume_from"] or model.resume_from + return SceneSmithModel.load_local(asset_dir, spec_path, planner_model, model.planner_python, + stop_stage, resume_from).generate(text) diff --git a/extensions/3D-Model/ssui_3dmodel/__init__.py b/extensions/3D-Model/ssui_3dmodel/__init__.py index f7d5e33..e2398e0 100644 --- a/extensions/3D-Model/ssui_3dmodel/__init__.py +++ b/extensions/3D-Model/ssui_3dmodel/__init__.py @@ -1,4 +1,5 @@ from .WorldClaw import GenWorldClawScene, WorldClawModel, build_worldclaw_scene from .AutoRig import AutoRigMesh, AutoRigModel +from .SceneSmith import GenSceneSmithScene, SceneSmithModel -__all__ = ["GenWorldClawScene", "WorldClawModel", "build_worldclaw_scene", "AutoRigMesh", "AutoRigModel"] +__all__ = ["GenWorldClawScene", "WorldClawModel", "build_worldclaw_scene", "AutoRigMesh", "AutoRigModel", "GenSceneSmithScene", "SceneSmithModel"] diff --git a/package.json b/package.json index e9db213..df54975 100644 --- a/package.json +++ b/package.json @@ -156,6 +156,9 @@ "test:scene-composite": "node dependencies/venv.cjs python -m unittest tests.scene_composite_test -v", "test:blender-scene": "node dependencies/venv.cjs python -m unittest tests.blender_scene_test -v", "test:rigging": "node dependencies/venv.cjs python -m unittest tests.auto_rig_test -v", + "test:scenesmith": "node dependencies/venv.cjs python -m unittest tests.scenesmith_test -v", + "test:scenesmith:planner": "node dependencies/venv.cjs python scripts/scenesmith_planner_smoke.py", + "test:scenesmith:blender": "node dependencies/venv.cjs python scripts/scenesmith_blender_smoke.py", "test:rigging:smoke": "node dependencies/venv.cjs python scripts/rigging_smoke.py", "build:rigging:cuda": "node dependencies/venv.cjs python scripts/build_rigging_cuda.py", "test:rigging:cuda": "node dependencies/venv.cjs python scripts/rigging_cuda_check.py", diff --git a/scripts/scenesmith_blender_smoke.py b/scripts/scenesmith_blender_smoke.py new file mode 100644 index 0000000..171e9f9 --- /dev/null +++ b/scripts/scenesmith_blender_smoke.py @@ -0,0 +1,26 @@ +"""Render the local SceneSmith planner scene in native Blender.""" + +import json +from pathlib import Path + +from PIL import Image + +from backend.scenesmith_blender import render_scene_preview + + +root = Path(__file__).resolve().parents[1] +scene = root / ".build/scenesmith-planner-smoke/scene.glb" +if not scene.is_file(): + raise FileNotFoundError(f"Run yarn test:scenesmith:planner first: {scene}") +output = root / ".build/scenesmith-blender-smoke" +preview, blend = render_scene_preview(scene, output) +with Image.open(preview) as image: + image.verify() +with Image.open(preview) as image: + if image.size != (640, 480): + raise RuntimeError(f"Unexpected preview dimensions: {image.size}") +if blend.stat().st_size < 1024: + raise RuntimeError("Blender scene output is incomplete") +print(json.dumps({"preview": str(preview), "blend": str(blend), + "preview_bytes": preview.stat().st_size, + "blend_bytes": blend.stat().st_size})) diff --git a/scripts/scenesmith_planner_smoke.py b/scripts/scenesmith_planner_smoke.py new file mode 100644 index 0000000..8c34648 --- /dev/null +++ b/scripts/scenesmith_planner_smoke.py @@ -0,0 +1,50 @@ +"""Real local Qwen planner acceptance for native scene backend.""" + +import json +import os +from pathlib import Path +import sys + +import trimesh + +from backend.scenesmith import generate_scene, asset_catalog, plan_with_local_llm +from ssui.base import Prompt +from ss_executor.loader import SSLoader + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "extensions/3D-Model")) +output = ROOT / ".build/scenesmith-planner-smoke" +assets = output / "assets" +assets.mkdir(parents=True, exist_ok=True) +trimesh.creation.box(extents=(2, 1, 1)).export(assets / "table.glb") +trimesh.creation.box(extents=(0.5, 0.5, 1)).export(assets / "chair.glb") +(assets / "table.json").write_text(json.dumps({"tags": ["桌子"]}, ensure_ascii=False), encoding="utf-8") +(assets / "chair.json").write_text(json.dumps({"tags": ["椅子"]}, ensure_ascii=False), encoding="utf-8") +python = Path(os.environ.get("SSUI_SCENESMITH_TEST_PYTHON", ROOT / ".build/cuda-arm64-venv/Scripts/python.exe")) +model = Path(os.environ.get("SSUI_SCENESMITH_TEST_MODEL", ROOT / ".build/arm64-models/qwen")) +prompt = "用 table 和 chair 摆一个小场景,椅子放在桌子的左边。" +spec = plan_with_local_llm(prompt, asset_catalog(assets), output, python, model) +glb = generate_scene(prompt, assets, output, spec) +loaded = trimesh.load(glb, force="scene") +spec_path = output / "planner-spec.json" +spec_path.write_text(json.dumps(spec, ensure_ascii=False), encoding="utf-8") +os.environ["SSUI_SCENE_ASSETS"] = str(assets) +os.environ["SSUI_SCENE_SPEC"] = str(spec_path) +loader = SSLoader() +loader.load(str(ROOT / "examples/advance/workflow-scenesmith.py")) +loader.Execute() +loader.GetConfig("scenesmith_scene") +loader.config.set_prepared(False) +workflow = next(fn for fn, _, _ in loader.callables if fn.__name__ == "scenesmith_scene") +workflow_result = workflow(Prompt(prompt)) +workflow_scene = trimesh.load(workflow_result.path, force="scene") +report = {"spec": spec, "mesh_count": len(loaded.geometry), "glb": str(glb), + "workflow_glb": workflow_result.path, "workflow_mesh_count": len(workflow_scene.geometry)} +(output / "report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") +if len(loaded.geometry) < 2 or len(workflow_scene.geometry) != len(loaded.geometry): + raise RuntimeError("Planner did not create both table and chair") +positions = {item["asset"]: item["position"] for item in json.loads((output / "scene.json").read_text(encoding="utf-8"))["objects"]} +if not positions["chair"][0] < positions["table"][0]: + raise RuntimeError("Planner violated left-of relation") +print(json.dumps({"mesh_count": len(loaded.geometry), "report": str(output / "report.json")}, ensure_ascii=False)) diff --git a/tests/scenesmith_test.py b/tests/scenesmith_test.py new file mode 100644 index 0000000..6a1d1be --- /dev/null +++ b/tests/scenesmith_test.py @@ -0,0 +1,278 @@ +"""Native scene layout backend with real GLB round trips.""" +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import trimesh +from PIL import Image + +from backend.scenesmith import asset_catalog, asset_descriptors, build_scene, enforce_text_relations, generate_scene, offline_plan +from backend.scene_physics import penetration_depth, transformed_bounds +from backend.scenesmith_preview import render_native_preview +from ssui.base import Mesh, Prompt +from ssui.config import SSUIConfig +from tests.utils import add_extension_paths + +add_extension_paths() +from ssui_3dmodel.SceneSmith import GenSceneSmithScene, SceneSmithModel + + +class SceneSmithTest(unittest.TestCase): + def test_prepare_and_loader(self): + config = SSUIConfig()("SceneSmith") + config.set_prepared() + self.assertIsInstance(GenSceneSmithScene(config, SceneSmithModel.load_local(), Prompt("room")), Mesh) + from ss_executor.loader import SSLoader + loader = SSLoader() + loader.load(str(Path(__file__).resolve().parents[1] / "examples/advance/workflow-scenesmith.py")) + loader.Execute() + loader.GetConfig("scenesmith_scene") + + def test_native_scene_with_existing_assets(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + base = Path(folder) + assets = base / "assets" + assets.mkdir() + trimesh.creation.box(extents=(2, 2, 2)).export(assets / "chair.glb") + trimesh.creation.icosphere(radius=0.5).export(assets / "lamp.glb") + spec = {"schema": "ssui.scene-layout/1", "objects": [ + {"id": "chair-a", "asset": "chair", "position": [1, 2, 0], "rotation_deg": 90}, + {"id": "lamp-a", "asset": "lamp", "position": [3, 2, 1]}]} + output = generate_scene("chair and lamp", assets, base / "out", spec) + loaded = trimesh.load(output, force="scene") + self.assertEqual(len(loaded.geometry), 2) + self.assertTrue(any("chair-a" in name for name in loaded.graph.nodes_geometry)) + manifest = json.loads((base / "out/scene.json").read_text(encoding="utf-8")) + self.assertEqual([item["id"] for item in manifest["objects"]], ["chair-a", "lamp-a"]) + preview = render_native_preview(output, base / "out/preview.png") + with Image.open(preview) as image: + self.assertEqual(image.size, (640, 480)) + self.assertGreater(len(image.getcolors(640 * 480)), 1) + with patch("ssui_3dmodel.SceneSmith.ROOT", base): + result = SceneSmithModel.load_local(assets).generate("place chair and lamp") + self.assertTrue(Path(result.path).is_file()) + + def test_invalid_spec_and_assets(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + assets = Path(folder) + trimesh.creation.box().export(assets / "box.glb") + catalog = asset_catalog(assets) + with self.assertRaisesRegex(ValueError, "Unknown asset"): + build_scene({"schema": "ssui.scene-layout/1", "objects": [{"asset": "missing"}]}, catalog) + with self.assertRaisesRegex(ValueError, "duplicate"): + build_scene({"schema": "ssui.scene-layout/1", "objects": [ + {"id": "same", "asset": "box"}, {"id": "same", "asset": "box"}]}, catalog) + with self.assertRaisesRegex(ValueError, "finite"): + build_scene({"schema": "ssui.scene-layout/1", "objects": [ + {"asset": "box", "position": [float("nan"), 0, 0]}]}, catalog) + + def test_asset_metadata_for_retrieval(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + assets = Path(folder) + trimesh.creation.box().export(assets / "seat.glb") + (assets / "seat.json").write_text(json.dumps({"description": "Wooden dining chair", + "category": "furniture", "tags": ["椅子", "chair"]}), + encoding="utf-8") + catalog = asset_catalog(assets) + self.assertEqual(asset_descriptors(catalog)[0]["tags"], ["椅子", "chair"]) + self.assertEqual(offline_plan("摆一把椅子", catalog)["objects"][0]["asset"], "seat") + + def test_explicit_text_relation_overrides_wrong_model_coordinates(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + assets = Path(folder) + for name, tag in (("table", "桌子"), ("chair", "椅子")): + trimesh.creation.box().export(assets / f"{name}.glb") + (assets / f"{name}.json").write_text(json.dumps({"tags": [tag]}, ensure_ascii=False), encoding="utf-8") + spec = {"schema": "ssui.scene-layout/1", "objects": [ + {"id": "chair-1", "asset": "chair", "position": [2, 0, 0]}, + {"id": "table-1", "asset": "table", "position": [0, 0, 0]}]} + corrected = enforce_text_relations("椅子放在桌子的左边", spec, asset_catalog(assets)) + self.assertEqual([item["id"] for item in corrected["objects"]], ["table-1", "chair-1"]) + _, manifest = build_scene(corrected, asset_catalog(assets)) + positions = {item["asset"]: item["position"] for item in manifest["objects"]} + self.assertLess(positions["chair"][0], positions["table"][0]) + english = {"schema": "ssui.scene-layout/1", "objects": [ + {"id": "chair-1", "asset": "chair", "position": [2, 0, 0]}, + {"id": "table-1", "asset": "table", "position": [0, 0, 0]}]} + enforce_text_relations("chair to the left of table", english, asset_catalog(assets)) + self.assertEqual(english["objects"][1]["relation"], "left_of") + + def test_multiple_text_relations_and_cycle(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + assets = Path(folder) + for name in ("table", "chair", "cup"): + trimesh.creation.box().export(assets / f"{name}.glb") + catalog = asset_catalog(assets) + spec = {"schema": "ssui.scene-layout/1", "objects": [ + {"id": name, "asset": name, "position": [0, 0, 0]} + for name in ("cup", "chair", "table")]} + enforce_text_relations("cup on top of table; chair to the left of table", spec, catalog) + self.assertEqual(spec["objects"][0]["id"], "table") + self.assertEqual({item["id"]: item.get("relation") for item in spec["objects"]}, + {"table": None, "cup": "on", "chair": "left_of"}) + _, manifest = build_scene(spec, catalog) + placed = {item["id"]: item["position"] for item in manifest["objects"]} + self.assertGreater(placed["cup"][2], placed["table"][2]) + self.assertLess(placed["chair"][0], placed["table"][0]) + cycle = {"schema": "ssui.scene-layout/1", "objects": [ + {"id": name, "asset": name} for name in ("table", "chair")]} + with self.assertRaisesRegex(ValueError, "Cyclic"): + enforce_text_relations("table to the left of chair; chair to the left of table", + cycle, catalog) + + def test_native_support_and_collision_geometry(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + assets = Path(folder) + trimesh.creation.box(extents=(2, 2, 2)).export(assets / "table.glb") + trimesh.creation.box(extents=(0.4, 0.4, 0.4)).export(assets / "cup.glb") + catalog = asset_catalog(assets) + scene, manifest = build_scene({"schema": "ssui.scene-layout/1", "objects": [ + {"id": "table", "asset": "table", "relation": "floor"}, + {"id": "cup", "asset": "cup", "relation": "on", "relative_to": "table"}]}, catalog) + self.assertEqual(len(scene.geometry), 2) + self.assertAlmostEqual(manifest["objects"][0]["position"][2], 1.0) + self.assertAlmostEqual(manifest["objects"][1]["position"][2], 2.25) + bounds = {} + for node in scene.graph.nodes_geometry: + matrix, geometry = scene.graph.get(node) + bounds[node.split(".")[0]] = transformed_bounds(scene.geometry[geometry].bounds, matrix) + self.assertLess(penetration_depth(bounds["table"], bounds["cup"])[2], 0) + + def test_overlapping_furniture_is_separated(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + assets = Path(folder) + trimesh.creation.box(extents=(2, 2, 2)).export(assets / "box.glb") + scene, manifest = build_scene({"schema": "ssui.scene-layout/1", "objects": [ + {"id": "a", "asset": "box", "position": [0, 0, 1]}, + {"id": "b", "asset": "box", "position": [0.5, 0, 1]}]}, asset_catalog(assets)) + self.assertEqual(len(scene.geometry), 2) + self.assertNotEqual(manifest["objects"][1]["collision_correction"], [0, 0, 0]) + + def test_collision_correction_stays_inside_room_and_support(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + assets = Path(folder) + trimesh.creation.box(extents=(1, 1, 1)).export(assets / "box.glb") + trimesh.creation.box(extents=(2, 2, 1)).export(assets / "table.glb") + catalog = asset_catalog(assets) + _, manifest = build_scene({"schema": "ssui.scene-layout/1", + "room": {"size": [3, 3, 3]}, "objects": [ + {"id": "a", "asset": "box", "position": [-0.8, 0, 0.5]}, + {"id": "b", "asset": "box", "position": [-0.8, 0, 0.5]}]}, catalog) + self.assertLessEqual(abs(manifest["objects"][1]["position"][0]), 1.0) + with self.assertRaisesRegex(ValueError, "room or support bounds"): + build_scene({"schema": "ssui.scene-layout/1", "room": {"size": [3, 3, 3]}, + "objects": [{"asset": "box", "position": [2, 0, 0.5]}]}, catalog) + with self.assertRaisesRegex(ValueError, "room or support bounds"): + build_scene({"schema": "ssui.scene-layout/1", "objects": [ + {"id": "table", "asset": "box"}, + {"id": "large", "asset": "table", "relation": "on", "relative_to": "table"}]}, catalog) + + def test_room_floor_and_walls_are_separate_nodes(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + assets = Path(folder) + trimesh.creation.box().export(assets / "chair.glb") + scene, manifest = build_scene({"schema": "ssui.scene-layout/1", + "room": {"size": [8, 6, 3]}, + "objects": [{"asset": "chair"}]}, asset_catalog(assets)) + self.assertEqual(manifest["room"]["size"], [8.0, 6.0, 3.0]) + self.assertEqual(len(scene.geometry), 6) + self.assertIn("room.floor", scene.graph.nodes_geometry) + floor_plan, _ = build_scene({"schema": "ssui.scene-layout/1", + "room": {"size": [8, 6, 3]}, "objects": []}, asset_catalog(assets)) + self.assertEqual(len(floor_plan.geometry), 5) + + def test_wall_and_ceiling_constraints(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + assets = Path(folder) + trimesh.creation.box(extents=(0.5, 0.2, 0.5)).export(assets / "lamp.glb") + scene, manifest = build_scene({"schema": "ssui.scene-layout/1", + "room": {"size": [8, 6, 3]}, + "objects": [ + {"id": "sconce", "asset": "lamp", "relation": "wall_left"}, + {"id": "light", "asset": "lamp", "relation": "ceiling"} + ]}, asset_catalog(assets)) + self.assertEqual(len(scene.geometry), 7) + self.assertEqual([x["stage"] for x in manifest["objects"]], + ["wall_mounted", "ceiling_mounted"]) + self.assertLess(manifest["objects"][0]["position"][0], -3) + self.assertGreater(manifest["objects"][1]["position"][2], 2) + + def test_floor_plan_door_and_window_openings(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + assets = Path(folder) + trimesh.creation.box().export(assets / "box.glb") + scene, manifest = build_scene({"schema": "ssui.scene-layout/1", + "room": {"size": [8, 6, 3], "openings": [ + {"wall": "front", "kind": "door", "width": 1, "height": 2}, + {"wall": "left", "kind": "window", "width": 1, + "bottom": 1, "height": 1}]}, + "objects": []}, asset_catalog(assets)) + self.assertEqual(len(manifest["room"]["openings"]), 2) + self.assertIn("room.wall.front.top", scene.graph.nodes_geometry) + self.assertIn("room.wall.left.bottom", scene.graph.nodes_geometry) + self.assertNotIn("room.wall.front.full", scene.graph.nodes_geometry) + + def test_stage_snapshots_and_resume(self): + root = Path(__file__).resolve().parents[1] / ".build" + root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=root) as folder: + base = Path(folder) + assets = base / "assets" + assets.mkdir() + trimesh.creation.box(extents=(1, 1, 1)).export(assets / "chair.glb") + trimesh.creation.box(extents=(0.3, 0.3, 0.3)).export(assets / "picture.glb") + first = {"schema": "ssui.scene-layout/1", "room": {"size": [6, 6, 3]}, + "objects": [{"id": "chair-1", "asset": "chair", "relation": "floor"}]} + generate_scene("chair", assets, base / "first", first, stop_stage="furniture") + floor_state = base / "first/stages/floor_plan/scene.json" + furniture_state = base / "first/stages/furniture/scene.json" + self.assertEqual(len(json.loads(floor_state.read_text(encoding="utf-8"))["objects"]), 0) + self.assertEqual(len(json.loads(furniture_state.read_text(encoding="utf-8"))["objects"]), 1) + self.assertTrue((base / "first/stages/floor_plan/preview.png").is_file()) + self.assertTrue((base / "first/stages/furniture/preview.png").is_file()) + self.assertFalse((base / "first/stages/wall_mounted").exists()) + addition = {"schema": "ssui.scene-layout/1", "objects": [ + {"id": "picture-1", "asset": "picture", "relation": "wall_back"}]} + generate_scene("picture on back wall", assets, base / "second", addition, + stop_stage="wall_mounted", resume_from=furniture_state) + result = json.loads((base / "second/scene.json").read_text(encoding="utf-8")) + self.assertEqual([item["id"] for item in result["objects"]], ["chair-1", "picture-1"]) + self.assertEqual(result["objects"][1]["stage"], "wall_mounted") + self.assertTrue((base / "second/stages/wall_mounted/scene.glb").is_file()) + with self.assertRaisesRegex(ValueError, "duplicate IDs"): + generate_scene("chair again", assets, base / "duplicate", first, + resume_from=furniture_state) + with self.assertRaisesRegex(ValueError, "Invalid stop stage"): + generate_scene("chair", assets, base / "invalid", first, stop_stage="unknown") + trimesh.creation.icosphere().export(assets / "chair.glb") + with self.assertRaisesRegex(ValueError, "Asset changed"): + generate_scene("resume", assets, base / "changed", addition, + resume_from=furniture_state) + + +if __name__ == "__main__": + unittest.main()