+ This demo needs a Gemini API key with image generation access. Get one
+ from
+ aistudio.google.com. The key is saved in your browser's localStorage and only sent to
+ Google.
+
+
+
+
+
+
+
+
+
+ starting...
+
+
+
+
diff --git a/demos/generative_object/src/GenerativeObject.ts b/demos/generative_object/src/GenerativeObject.ts
new file mode 100644
index 000000000..b99bc94ae
--- /dev/null
+++ b/demos/generative_object/src/GenerativeObject.ts
@@ -0,0 +1,132 @@
+import * as THREE from 'three';
+import {
+ computeBillboardScale,
+ DragMode,
+ type Draggable,
+ type HasDraggingMode,
+ OCCLUDABLE_ITEMS_LAYER,
+ Script,
+} from 'xrblocks';
+
+import type {LoadedTexture} from './TextureSource.js';
+
+/** How to build a {@link GenerativeObject}'s mesh. */
+export interface GenerativeObjectStyle {
+ /** Largest dimension of the object, in meters. */
+ maxSize: number;
+ /** Build a displaced 2.5D relief instead of a flat cutout. */
+ relief?: boolean;
+ /** Relief displacement depth in meters. */
+ reliefStrength?: number;
+ /** Plane subdivisions per side used for the relief mesh. */
+ reliefSegments?: number;
+}
+
+/**
+ * A generated image placed in the scene as a draggable object: a flat textured
+ * cutout by default, or a displaced relief mesh when
+ * {@link GenerativeObjectStyle.relief} is set. Opts into
+ * `OCCLUDABLE_ITEMS_LAYER` so depth occlusion can hide it behind real geometry.
+ */
+export class GenerativeObject
+ extends Script
+ implements Draggable, HasDraggingMode
+{
+ draggable = true;
+ // Lets the global DragManager pick the object up and move it; without a
+ // draggingMode the manager bails out of beginDragging.
+ draggingMode = DragMode.TRANSLATING;
+
+ /** The prompt that produced this object. */
+ readonly prompt: string;
+
+ /** The mesh that renders the generated image. */
+ readonly mesh: THREE.Mesh;
+
+ /**
+ * @param prompt - The prompt that produced the image.
+ * @param loaded - The decoded texture and its pixel dimensions.
+ * @param style - How to size and build the mesh.
+ */
+ constructor(
+ prompt: string,
+ loaded: LoadedTexture,
+ style: GenerativeObjectStyle
+ ) {
+ super();
+ this.prompt = prompt;
+
+ this.mesh = style.relief
+ ? buildReliefMesh(loaded, style)
+ : buildFlatMesh(loaded.texture);
+
+ const size = computeBillboardScale(
+ loaded.width,
+ loaded.height,
+ style.maxSize
+ );
+ this.mesh.scale.set(size.x, size.y, 1);
+ this.add(this.mesh);
+
+ // Allow real-world depth to occlude the generated object.
+ this.mesh.layers.enable(OCCLUDABLE_ITEMS_LAYER);
+ }
+
+ /** Releases GPU resources held by this object. */
+ dispose() {
+ this.mesh.geometry.dispose();
+ const material = this.mesh.material as THREE.Material & {
+ map?: THREE.Texture | null;
+ displacementMap?: THREE.Texture | null;
+ bumpMap?: THREE.Texture | null;
+ };
+ // Dispose every distinct texture the material references (the relief mesh
+ // reuses one map across displacement + bump, so guard against double free).
+ const textures = new Set();
+ for (const tex of [
+ material.map,
+ material.displacementMap,
+ material.bumpMap,
+ ]) {
+ if (tex) textures.add(tex);
+ }
+ for (const tex of textures) tex.dispose();
+ material.dispose();
+ }
+}
+
+function buildFlatMesh(texture: THREE.Texture) {
+ const material = new THREE.MeshBasicMaterial({
+ map: texture,
+ transparent: true,
+ // Discard the keyed-out (transparent) pixels so edge filtering doesn't blend
+ // the chroma-key background color into a halo around the cutout.
+ alphaTest: 0.5,
+ side: THREE.DoubleSide,
+ });
+ return new THREE.Mesh(new THREE.PlaneGeometry(1, 1), material);
+}
+
+function buildReliefMesh(loaded: LoadedTexture, style: GenerativeObjectStyle) {
+ const segments = style.reliefSegments ?? 96;
+ const strength = style.reliefStrength ?? 0.04;
+ // Displace/bump from an alpha-masked grayscale map (background stays flat) so
+ // brighter subject regions stand out and pick up shading. Falls back to the
+ // color texture when no masked map is available.
+ const displacementMap = loaded.displacementTexture ?? loaded.texture;
+ const material = new THREE.MeshStandardMaterial({
+ map: loaded.texture,
+ displacementMap,
+ displacementScale: strength,
+ bumpMap: displacementMap,
+ roughness: 0.9,
+ metalness: 0,
+ transparent: true,
+ alphaTest: 0.5,
+ side: THREE.DoubleSide,
+ });
+ return new THREE.Mesh(
+ new THREE.PlaneGeometry(1, 1, segments, segments),
+ material
+ );
+}
diff --git a/demos/generative_object/src/GenerativeObjects.ts b/demos/generative_object/src/GenerativeObjects.ts
new file mode 100644
index 000000000..153e588a7
--- /dev/null
+++ b/demos/generative_object/src/GenerativeObjects.ts
@@ -0,0 +1,321 @@
+import * as THREE from 'three';
+import {
+ AI,
+ Depth,
+ OcclusionUtils,
+ poseInFrontOfCamera,
+ quaternionFacingCamera,
+ Script,
+} from 'xrblocks';
+
+import {GenerativeObject} from './GenerativeObject.js';
+import {GenerativeOptions} from './GenerativeOptions.js';
+import {
+ CanvasBackgroundTextureSource,
+ DataUrlTextureSource,
+ type TextureSource,
+} from './TextureSource.js';
+
+const scratchCameraPosition = new THREE.Vector3();
+const scratchOrigin = new THREE.Vector3();
+const scratchDirection = new THREE.Vector3();
+const WORLD_UP = new THREE.Vector3(0, 1, 0);
+/** Clearance in meters to float an object off a vertical surface. */
+const SURFACE_CLEARANCE = 0.08;
+
+// Saved original raycast of each depth mesh we no-op so it stays out of the
+// reticle AND the spatial-UI hover raycast; raycastSurface_ restores it for
+// placement. (ignoreReticleRaycast covers only the reticle, not UI hover.)
+type DepthMeshRaycast = (
+ raycaster: THREE.Raycaster,
+ intersects: THREE.Intersection[]
+) => boolean;
+const originalDepthRaycast = new WeakMap();
+
+/** Per-call overrides for {@link GenerativeObjects.imagine}. */
+export interface ImagineOptions {
+ /** Distance in meters in front of the user. Defaults to the options value. */
+ distance?: number;
+ /** Largest dimension of the object in meters. Defaults to the options value. */
+ maxSize?: number;
+}
+
+/**
+ * Demo helper that turns a text prompt into a placed, draggable
+ * {@link GenerativeObject}: it asks the AI model to generate an image, decodes
+ * it into a texture, and drops the result into the scene in front of the user.
+ *
+ * Lives in the demo (rather than the SDK) so the high-level shape can keep
+ * evolving. The generation step is split into {@link generateBillboard}, an
+ * image-to-placed-object primitive, to mirror where an SDK
+ * `ai.generateBillboard(image)` could eventually sit.
+ *
+ * If AI is unavailable or generation yields no image, {@link imagine} resolves
+ * to `null` instead of throwing.
+ */
+export class GenerativeObjects extends Script {
+ static dependencies = {
+ ai: AI,
+ camera: THREE.Camera,
+ scene: THREE.Scene,
+ depth: Depth,
+ };
+
+ options = new GenerativeOptions();
+
+ /** Decodes generated image data into a texture. Built from options on init. */
+ textureSource: TextureSource = new DataUrlTextureSource();
+
+ /** All objects created this session, in creation order. */
+ readonly objects: GenerativeObject[] = [];
+
+ private ai!: AI;
+ private camera!: THREE.Camera;
+ private scene!: THREE.Scene;
+ private depth?: Depth;
+ private raycaster = new THREE.Raycaster();
+ // Per-object teardown that removes the object's occlusion shader from the
+ // engine-wide depth.occludableShaders set, so cleared objects don't leak.
+ private readonly shaderCleanups = new Map void>();
+ // Bumped whenever objects are cleared, so an in-flight imagine() that resolves
+ // after a clear/teardown does not add a stale object to the scene.
+ private generation = 0;
+
+ init({
+ ai,
+ camera,
+ scene,
+ depth,
+ }: {
+ ai: AI;
+ camera: THREE.Camera;
+ scene: THREE.Scene;
+ depth?: Depth;
+ }) {
+ this.ai = ai;
+ this.camera = camera;
+ this.scene = scene;
+ this.depth = depth;
+ this.textureSource = this.options.removeBackground
+ ? new CanvasBackgroundTextureSource({
+ buildDisplacement: this.options.relief,
+ })
+ : new DataUrlTextureSource();
+ }
+
+ /** Whether image generation can run in the current session. */
+ get isSupported(): boolean {
+ return !!this.ai?.isAvailable?.();
+ }
+
+ /** Billboards tracked objects toward the user each frame, when enabled. */
+ override update() {
+ this.ensureDepthMeshNonInteractive_();
+ if (!this.options.billboard || this.objects.length === 0) {
+ return;
+ }
+ const cameraPosition = this.camera.getWorldPosition(scratchCameraPosition);
+ for (const object of this.objects) {
+ quaternionFacingCamera(
+ object.position,
+ cameraPosition,
+ object.quaternion
+ );
+ }
+ }
+
+ // The depth mesh is in the scene for occlusion + placement, so both the
+ // reticle and the spatial-UI button hover (its own scene raycast) hit it;
+ // close to a wall it steals hover from the panel. No-op its raycast so every
+ // raycaster skips it; raycastSurface_ restores it briefly for placement.
+ private ensureDepthMeshNonInteractive_() {
+ const mesh = this.depth?.depthMesh;
+ if (!mesh || originalDepthRaycast.has(mesh)) {
+ return;
+ }
+ originalDepthRaycast.set(mesh, mesh.raycast);
+ mesh.raycast = () => false;
+ }
+
+ /**
+ * Generates an image for `prompt` and places it as a draggable object in
+ * front of the user.
+ * @param prompt - What to generate, e.g. "a small red dragon".
+ * @param options - Optional per-call placement overrides.
+ * @returns The placed object, or `null` if generation was unavailable or
+ * produced no image.
+ */
+ async imagine(
+ prompt: string,
+ options: ImagineOptions = {}
+ ): Promise {
+ if (!this.isSupported) {
+ return null;
+ }
+
+ const result = await this.ai.generate(
+ prompt,
+ 'image',
+ this.options.systemInstruction
+ );
+ if (typeof result !== 'string' || result.length === 0) {
+ return null;
+ }
+
+ return this.generateBillboard(result, prompt, options);
+ }
+
+ /**
+ * Builds and places a draggable billboard from an already-generated image.
+ * This is the image-to-object half of {@link imagine}, kept separate to model
+ * the shape of a future `ai.generateBillboard(image)` primitive.
+ * @param image - Image data (typically a `data:` URL).
+ * @param prompt - Label describing the image, stored on the object.
+ * @param options - Optional per-call placement overrides.
+ * @returns The placed object, or `null` if it was cleared mid-load.
+ */
+ async generateBillboard(
+ image: string,
+ prompt = '',
+ options: ImagineOptions = {}
+ ): Promise {
+ const generation = this.generation;
+ const loaded = await this.textureSource.load(image);
+ // Dropped/cleared while the texture was decoding: discard so we never add a
+ // stale object after a clearObjects()/teardown.
+ if (generation !== this.generation) {
+ loaded.texture.dispose();
+ loaded.displacementTexture?.dispose();
+ return null;
+ }
+
+ const maxSize = options.maxSize ?? this.options.maxSize;
+ const distance = options.distance ?? this.options.distance;
+
+ const object = new GenerativeObject(prompt, loaded, {
+ maxSize,
+ relief: this.options.relief,
+ reliefStrength: this.options.reliefStrength,
+ reliefSegments: this.options.reliefSegments,
+ });
+ this.setupOcclusion_(object);
+ this.placeObject_(object, distance);
+
+ this.scene.add(object);
+ this.objects.push(object);
+ return object;
+ }
+
+ /**
+ * Makes the object's material occluded by the real-world depth mesh: enabling
+ * the occludable layer alone only builds the occlusion mask, so the material's
+ * shader must also sample it (mirrors `ModelViewer`). No-op when depth is not
+ * enabled, so the object stays plainly visible instead of sampling an empty
+ * occlusion map and rendering transparent.
+ */
+ private setupOcclusion_(object: GenerativeObject) {
+ const depth = this.depth;
+ if (!depth?.occludableShaders) {
+ return;
+ }
+ const material = object.mesh.material;
+ material.onBeforeCompile = (shader) => {
+ OcclusionUtils.addOcclusionToShader(shader);
+ depth.occludableShaders.add(shader);
+ // Remember how to remove this shader so clearObjects() doesn't leak it.
+ this.shaderCleanups.set(object, () =>
+ depth.occludableShaders.delete(shader)
+ );
+ };
+ material.needsUpdate = true;
+ }
+
+ /**
+ * Positions a freshly built object: on the real-world surface the user is
+ * looking at when grounding is enabled and a hit is found, otherwise in front
+ * of the camera. Stands on horizontal surfaces and floats a little off
+ * vertical ones so it never blends into a wall. Always upright toward the user.
+ */
+ private placeObject_(object: GenerativeObject, distance: number) {
+ const hit = this.options.groundOnSurface ? this.raycastSurface_() : null;
+ if (hit) {
+ const isHorizontal = Math.abs(hit.normal.dot(WORLD_UP)) > 0.7;
+ if (isHorizontal) {
+ // Stand the cutout on the surface by lifting it half its height.
+ const halfHeight = object.mesh.scale.y / 2;
+ object.position.copy(hit.point).addScaledVector(WORLD_UP, halfHeight);
+ } else {
+ // Float it off the vertical surface so it doesn't z-fight / blend in.
+ object.position
+ .copy(hit.point)
+ .addScaledVector(hit.normal, SURFACE_CLEARANCE);
+ }
+ } else {
+ poseInFrontOfCamera(this.camera, distance, object.position);
+ }
+ const cameraPosition = this.camera.getWorldPosition(scratchCameraPosition);
+ quaternionFacingCamera(object.position, cameraPosition, object.quaternion);
+ }
+
+ /**
+ * Raycasts from the camera forward against the depth mesh.
+ * @returns The world-space hit point and surface normal, or `null` if there is
+ * no depth mesh or no intersection.
+ */
+ protected raycastSurface_(): {
+ point: THREE.Vector3;
+ normal: THREE.Vector3;
+ } | null {
+ const depthMesh = this.depth?.depthMesh;
+ if (!depthMesh) {
+ return null;
+ }
+ const origin = this.camera.getWorldPosition(scratchOrigin);
+ const direction = this.camera.getWorldDirection(scratchDirection);
+ this.raycaster.set(origin, direction);
+ // depthMesh.raycast is no-op'd so walls don't steal hover; restore it just
+ // for this placement query, in a finally so a throw can't leave it active.
+ const original = originalDepthRaycast.get(depthMesh);
+ const nooped = depthMesh.raycast;
+ if (original) depthMesh.raycast = original;
+ let intersections;
+ try {
+ intersections = this.raycaster.intersectObject(depthMesh, false);
+ } finally {
+ depthMesh.raycast = nooped;
+ }
+ if (intersections.length === 0) {
+ return null;
+ }
+ const hit = intersections[0];
+ // Ignore far surfaces (e.g. a wall across the room): placing a small object
+ // metres away makes it tiny and easy to miss. Fall back to in-front
+ // placement by returning null when the hit is beyond the comfortable reach.
+ if (hit.distance > this.options.maxGroundDistance) {
+ return null;
+ }
+ const point = hit.point.clone();
+ // Prefer the triangle's geometric face normal: the depth mesh does not keep
+ // per-vertex normals fresh, so the interpolated hit.normal can be stale,
+ // whereas the face normal is derived from the current vertex positions.
+ const local = hit.face?.normal ?? hit.normal ?? WORLD_UP;
+ const normal = local
+ .clone()
+ .transformDirection(depthMesh.matrixWorld)
+ .normalize();
+ return {point, normal};
+ }
+
+ /** Removes all generated objects from the scene and frees their resources. */
+ clearObjects() {
+ // Invalidate any in-flight generateBillboard() so its result is discarded.
+ this.generation++;
+ for (const object of this.objects) {
+ this.scene.remove(object);
+ this.shaderCleanups.get(object)?.();
+ this.shaderCleanups.delete(object);
+ object.dispose();
+ }
+ this.objects.length = 0;
+ }
+}
diff --git a/demos/generative_object/src/GenerativeOptions.ts b/demos/generative_object/src/GenerativeOptions.ts
new file mode 100644
index 000000000..22464bf4a
--- /dev/null
+++ b/demos/generative_object/src/GenerativeOptions.ts
@@ -0,0 +1,65 @@
+/**
+ * Configuration for the `GenerativeObjects` demo helper, which turns a text
+ * prompt into a placed, draggable object in the scene.
+ */
+export class GenerativeOptions {
+ /**
+ * System instruction passed to the image model. Asks for a single subject on
+ * a saturated, uniform background that contrasts with most subjects, so the
+ * background keyer can cut it out cleanly (a plain white background fails for
+ * pale subjects like a paper airplane, which get keyed out with it).
+ */
+ systemInstruction =
+ 'Generate a single, centered subject that fills most of the frame on a ' +
+ 'plain, solid chroma-green (#00b140) background. The subject itself must ' +
+ 'not be green. No text, no watermark, no border, no shadows on the ' +
+ 'background.';
+
+ /** Distance in meters in front of the user to place a new object. */
+ distance = 1.0;
+
+ /**
+ * Place new objects where the user is looking hits the real-world depth mesh
+ * (so they sit on your table/floor), falling back to {@link distance} in front
+ * of the camera when there is no surface hit. Requires depth to be enabled.
+ */
+ groundOnSurface = true;
+
+ /**
+ * Farthest a grounded object may be placed, in meters. Surface hits beyond
+ * this (e.g. a wall across the room) are ignored so the object appears at a
+ * comfortable, visible reach in front of you rather than tiny and far away.
+ */
+ maxGroundDistance = 2.0;
+
+ /** Largest dimension (meters) of a placed object; aspect ratio is preserved. */
+ maxSize = 0.6;
+
+ /**
+ * Whether generated objects continuously turn to face the user (billboard).
+ * Keeps the flat cutout from ever looking paper-thin from the side.
+ */
+ billboard = true;
+
+ /**
+ * Experimental: build the object as a 2.5D relief instead of a flat cutout.
+ * A densely subdivided plane is displaced by the generated image's
+ * brightness (via a three.js displacement + bump map), giving the subject
+ * real, shaded surface relief. Approximate (brightness is not true depth) and
+ * requires a light in the scene. Best viewed with {@link billboard} off.
+ */
+ relief = false;
+
+ /** Relief displacement depth in meters (when {@link relief} is on). */
+ reliefStrength = 0.04;
+
+ /** Plane subdivisions per side used to build the relief mesh. */
+ reliefSegments = 96;
+
+ /**
+ * Whether to key out the (plain) background of the generated image so the
+ * subject reads as a clean cutout instead of a flat card. Requires a browser
+ * 2D canvas; ignored in non-browser environments.
+ */
+ removeBackground = true;
+}
diff --git a/demos/generative_object/src/TextureSource.ts b/demos/generative_object/src/TextureSource.ts
new file mode 100644
index 000000000..38c89367c
--- /dev/null
+++ b/demos/generative_object/src/TextureSource.ts
@@ -0,0 +1,139 @@
+import * as THREE from 'three';
+import {buildDisplacementMap, keyOutBackground} from 'xrblocks';
+
+/** A loaded texture together with its source pixel dimensions. */
+export interface LoadedTexture {
+ texture: THREE.Texture;
+ width: number;
+ height: number;
+ /**
+ * Optional alpha-masked grayscale map for relief displacement/bump, where the
+ * (transparent) background is black so it does not displace.
+ */
+ displacementTexture?: THREE.Texture;
+}
+
+/**
+ * Loads a texture from an image source (typically a `data:` URL produced by
+ * image generation). Abstracted behind an interface so the orchestration in
+ * `GenerativeObjects` can be swapped without decoding real images.
+ */
+export interface TextureSource {
+ load(dataUrl: string): Promise;
+}
+
+/**
+ * Default {@link TextureSource} backed by `THREE.TextureLoader`. Resolves once
+ * the browser has decoded the image, reporting its natural pixel dimensions.
+ */
+export class DataUrlTextureSource implements TextureSource {
+ private loader = new THREE.TextureLoader();
+
+ load(dataUrl: string): Promise {
+ return new Promise((resolve, reject) => {
+ this.loader.load(
+ dataUrl,
+ (texture) => {
+ texture.colorSpace = THREE.SRGBColorSpace;
+ const image = texture.image as
+ | {width?: number; height?: number}
+ | undefined;
+ resolve({
+ texture,
+ width: image?.width ?? 0,
+ height: image?.height ?? 0,
+ });
+ },
+ undefined,
+ (error) => reject(error)
+ );
+ });
+ }
+}
+
+/**
+ * A {@link TextureSource} that removes the (plain) background of the generated
+ * image so the subject reads as a clean cutout. Decodes the image to a 2D
+ * canvas, keys out background pixels via `keyOutBackground`, and returns a
+ * `CanvasTexture`. Browser-only (requires `Image` and a 2D canvas context).
+ *
+ * The relief displacement map is built lazily, only when `buildDisplacement` is
+ * set, so flat cutouts do not allocate a texture they never use.
+ */
+export class CanvasBackgroundTextureSource implements TextureSource {
+ /** Maximum RGB distance from the sampled background color to key out. */
+ tolerance?: number;
+ /** Whether to also build the relief displacement map. */
+ buildDisplacement: boolean;
+
+ constructor(options: {tolerance?: number; buildDisplacement?: boolean} = {}) {
+ this.tolerance = options.tolerance;
+ this.buildDisplacement = options.buildDisplacement ?? false;
+ }
+
+ load(dataUrl: string): Promise {
+ return new Promise((resolve, reject) => {
+ const image = new Image();
+ image.crossOrigin = 'anonymous';
+ image.onload = () => {
+ try {
+ resolve(this.process(image));
+ } catch (error) {
+ reject(error);
+ }
+ };
+ image.onerror = () =>
+ reject(new Error('Failed to decode generated image'));
+ image.src = dataUrl;
+ });
+ }
+
+ private process(image: HTMLImageElement): LoadedTexture {
+ const width = image.naturalWidth || image.width;
+ const height = image.naturalHeight || image.height;
+ const canvas = document.createElement('canvas');
+ canvas.width = width;
+ canvas.height = height;
+ const context = canvas.getContext('2d');
+ if (!context) {
+ throw new Error('2D canvas context unavailable for background removal');
+ }
+ context.drawImage(image, 0, 0, width, height);
+ const imageData = context.getImageData(0, 0, width, height);
+ const keyed = keyOutBackground(
+ {data: imageData.data, width, height},
+ {tolerance: this.tolerance}
+ );
+ imageData.data.set(keyed.data);
+ context.putImageData(imageData, 0, 0);
+
+ const texture = new THREE.CanvasTexture(canvas);
+ texture.colorSpace = THREE.SRGBColorSpace;
+
+ let displacementTexture: THREE.Texture | undefined;
+ if (this.buildDisplacement) {
+ // An alpha-masked grayscale map for relief: the transparent background is
+ // black so it stays flat instead of displacing into stray geometry.
+ const displacement = buildDisplacementMap({
+ data: keyed.data,
+ width,
+ height,
+ });
+ const displacementCanvas = document.createElement('canvas');
+ displacementCanvas.width = width;
+ displacementCanvas.height = height;
+ const displacementContext = displacementCanvas.getContext('2d');
+ if (displacementContext) {
+ const displacementImageData = displacementContext.createImageData(
+ width,
+ height
+ );
+ displacementImageData.data.set(displacement.data);
+ displacementContext.putImageData(displacementImageData, 0, 0);
+ displacementTexture = new THREE.CanvasTexture(displacementCanvas);
+ }
+ }
+
+ return {texture, width, height, displacementTexture};
+ }
+}
diff --git a/demos/generative_object/src/main.ts b/demos/generative_object/src/main.ts
new file mode 100644
index 000000000..24ed4741f
--- /dev/null
+++ b/demos/generative_object/src/main.ts
@@ -0,0 +1,375 @@
+import 'xrblocks/addons/simulator/SimulatorAddons.js';
+
+import * as THREE from 'three';
+import {
+ HeadLeashBehavior,
+ ManipulationBehavior,
+ UICore,
+ UIIcon,
+ UIPanel,
+ UIText,
+} from 'xrblocks/addons/uiblocks/src/index.js';
+import * as xb from 'xrblocks';
+
+import {GenerativeObjects} from './GenerativeObjects.js';
+
+// Demo for a prompt-to-object generative helper. Buttons or voice summon an
+// AI-generated cutout onto the surface you're looking at; grab to move it.
+//
+// The generative helper lives in this demo (demos/generative_object/src/), not
+// the SDK: a Gemini API key is required, pass it in the URL as ?key=... or place
+// a keys.json next to this file (or at the served repo root).
+
+const PRESET_PROMPTS = [
+ 'a small friendly red dragon',
+ 'a potted succulent plant',
+ 'a vintage robot toy',
+ 'a slice of watermelon',
+ 'a rubber duck wearing sunglasses',
+ 'a paper airplane',
+];
+
+class GenerativeObjectDemo extends xb.Script {
+ private presetIndex = 0;
+ private busy = false;
+ private listening = false;
+ private recognizer: xb.SpeechRecognizer | null = null;
+ private domSpeakButton: HTMLButtonElement | null = null;
+ private xrStatusText: UIText | null = null;
+ private uiCore?: UICore;
+
+ /**
+ * @param generative - The demo-owned generative helper, added to the engine
+ * separately so its dependencies (AI, camera, scene, depth) are injected.
+ */
+ constructor(private generative: GenerativeObjects) {
+ super();
+ }
+
+ override init() {
+ // Lights so the relief (lit standard material) shows surface shading.
+ const ambient = new THREE.AmbientLight(0xffffff, 1.2);
+ const key = new THREE.DirectionalLight(0xffffff, 1.5);
+ key.position.set(0.5, 1, 1);
+ xb.core.scene.add(ambient, key);
+
+ // Voice trigger: imagine whatever you say.
+ this.recognizer = xb.core.sound?.speechRecognizer ?? null;
+ if (this.recognizer) {
+ this.recognizer.addEventListener('result', (event) => {
+ if (event.isFinal && event.transcript.trim()) {
+ this.imagine(event.transcript.trim());
+ this.setListening_(false);
+ }
+ });
+ this.recognizer.addEventListener('end', () => this.setListening_(false));
+ this.recognizer.addEventListener('error', () =>
+ this.setListening_(false)
+ );
+ }
+
+ this.buildDomControls_();
+ this.buildSpatialPanel_();
+ this.setStatus_('summon an object with the buttons or your voice.');
+ }
+
+ // ---- actions (shared by DOM buttons, spatial buttons, and keys) ----
+
+ private summonPreset_() {
+ const prompt = PRESET_PROMPTS[this.presetIndex % PRESET_PROMPTS.length];
+ this.presetIndex++;
+ this.imagine(prompt);
+ }
+
+ private toggleSpeak_() {
+ if (!this.recognizer) return;
+ if (this.listening) {
+ this.recognizer.stop();
+ this.setListening_(false);
+ } else {
+ this.recognizer.start();
+ this.setListening_(true);
+ this.setStatus_('listening... say what to summon.');
+ }
+ }
+
+ private toggleRelief_() {
+ const opts = this.generative.options;
+ opts.relief = !opts.relief;
+ // Relief reads best when you can move around it, so pause billboarding.
+ opts.billboard = !opts.relief;
+ this.setStatus_(
+ opts.relief
+ ? 'relief ON (2.5D). summon something; billboarding paused to orbit it.'
+ : 'relief OFF (flat cutout). billboarding back on.'
+ );
+ }
+
+ private clearObjects_() {
+ this.generative.clearObjects();
+ this.setStatus_('cleared. summon something new.');
+ }
+
+ private async imagine(prompt: string) {
+ if (this.busy) return;
+ if (!this.generative.isSupported) {
+ this.setStatus_('generation unavailable. check your Gemini key.');
+ return;
+ }
+ this.busy = true;
+ this.setStatus_(`summoning "${prompt}"...`);
+ try {
+ const object = await this.generative.imagine(prompt);
+ this.setStatus_(
+ object
+ ? `summoned "${prompt}". grab to move it. summon more anytime.`
+ : `couldn't generate "${prompt}". try again.`
+ );
+ } catch (error) {
+ console.error('[generative_object]', error);
+ this.setStatus_(`error generating "${prompt}".`);
+ } finally {
+ this.busy = false;
+ }
+ }
+
+ // ---- input: keyboard shortcuts (summoning is via the buttons / voice) ----
+
+ override onKeyDown(event: KeyboardEvent) {
+ if (event.code === 'KeyG') {
+ this.summonPreset_();
+ } else if (event.code === 'KeyR') {
+ this.toggleRelief_();
+ }
+ }
+
+ // ---- DOM controls (desktop) ----
+
+ private buildDomControls_() {
+ const bar = document.createElement('div');
+ Object.assign(bar.style, {
+ position: 'fixed',
+ top: '12px',
+ right: '12px',
+ display: 'flex',
+ flexDirection: 'column',
+ gap: '10px',
+ zIndex: '999',
+ });
+ bar.appendChild(
+ this.makeDomButton_('✨ Summon', () => this.summonPreset_())
+ );
+ this.domSpeakButton = this.makeDomButton_('🎙️ Speak', () =>
+ this.toggleSpeak_()
+ );
+ bar.appendChild(this.domSpeakButton);
+ bar.appendChild(
+ this.makeDomButton_('🌀 Relief', () => this.toggleRelief_())
+ );
+ bar.appendChild(
+ this.makeDomButton_('🗑️ Clear', () => this.clearObjects_())
+ );
+ document.body.appendChild(bar);
+ }
+
+ private makeDomButton_(
+ label: string,
+ onClick: () => void
+ ): HTMLButtonElement {
+ const button = document.createElement('button');
+ button.textContent = label;
+ Object.assign(button.style, {
+ padding: '10px 18px',
+ background: '#9177c7',
+ color: '#fff',
+ border: 'none',
+ borderRadius: '24px',
+ fontSize: '14px',
+ cursor: 'pointer',
+ });
+ button.addEventListener('click', onClick);
+ return button;
+ }
+
+ private setListening_(listening: boolean) {
+ this.listening = listening;
+ if (this.domSpeakButton) {
+ this.domSpeakButton.textContent = listening
+ ? '🔴 Listening...'
+ : '🎙️ Speak';
+ }
+ }
+
+ // ---- spatial control panel (XR) ----
+
+ private buildSpatialPanel_() {
+ this.uiCore = new UICore(this);
+ const card = this.uiCore.createCard({
+ name: 'GenerativeObjectControlCard',
+ position: new THREE.Vector3(0, 1.3, -0.8),
+ sizeX: 0.62,
+ sizeY: 0.24,
+ });
+ const panel = new UIPanel({
+ width: '100%',
+ height: '100%',
+ fillColor: 'rgba(16, 14, 26, 0.94)',
+ strokeWidth: 2,
+ strokeColor: 'rgba(145, 119, 199, 0.55)',
+ cornerRadius: 18,
+ padding: 14,
+ flexDirection: 'column',
+ gap: 8,
+ alignItems: 'stretch',
+ justifyContent: 'center',
+ });
+ panel.add(
+ new UIText('GENERATIVE OBJECTS', {
+ fontSize: 18,
+ fontWeight: 'bold',
+ color: '#c4b5ff',
+ textAlign: 'center',
+ width: '100%',
+ })
+ );
+ this.xrStatusText = new UIText('idle', {
+ fontSize: 12,
+ color: '#8b97a7',
+ textAlign: 'center',
+ width: '100%',
+ });
+ panel.add(this.xrStatusText);
+ panel.add(
+ new UIPanel({
+ width: '100%',
+ height: 1,
+ fillColor: 'rgba(255, 255, 255, 0.10)',
+ })
+ );
+ const row = new UIPanel({
+ width: '100%',
+ flexDirection: 'row',
+ gap: 10,
+ justifyContent: 'center',
+ alignItems: 'center',
+ });
+ row.add(this.makeXrButton_('flare', 'summon', () => this.summonPreset_()));
+ row.add(this.makeXrButton_('mic', 'speak', () => this.toggleSpeak_()));
+ row.add(
+ this.makeXrButton_('deployed_code', 'relief', () => this.toggleRelief_())
+ );
+ row.add(
+ this.makeXrButton_('delete_sweep', 'clear', () => this.clearObjects_())
+ );
+ panel.add(row);
+ card.add(panel);
+ card.addBehavior(
+ new ManipulationBehavior({draggable: true, faceCamera: false})
+ );
+ // Gently follow the user so the controls stay in reach as they move.
+ card.addBehavior(
+ new HeadLeashBehavior({
+ offset: new THREE.Vector3(0, 0.3, -1.0),
+ posLerp: 0.08,
+ rotLerp: 0.1,
+ })
+ );
+ }
+
+ // Icon + caption button mirroring a DOM control (matches world_companion).
+ private makeXrButton_(
+ iconName: string,
+ label: string,
+ onClick: () => void
+ ): UIPanel {
+ // Idle is a dark chip; hover is a clear purple so the highlight is
+ // unmistakable (the old near-black hover was invisible against idle).
+ const idle = '#3a3550';
+ const hover = '#7a5fc7';
+ const btn = new UIPanel({
+ paddingTop: 8,
+ paddingBottom: 8,
+ paddingLeft: 16,
+ paddingRight: 16,
+ cornerRadius: 12,
+ fillColor: idle,
+ strokeWidth: 1,
+ strokeColor: '#6b5fa0',
+ flexDirection: 'row',
+ alignItems: 'center',
+ justifyContent: 'center',
+ gap: 8,
+ renderOrder: 10,
+ onHoverEnter: () => btn.setFillColor(hover),
+ onHoverExit: () => btn.setFillColor(idle),
+ onClick: () => {
+ btn.setFillColor('#b49aff');
+ setTimeout(() => btn.setFillColor(idle), 180);
+ onClick();
+ },
+ });
+ btn.add(
+ new UIIcon(iconName, {
+ color: 'white',
+ width: 22,
+ height: 22,
+ renderOrder: 12,
+ })
+ );
+ btn.add(
+ new UIText(label, {
+ fontSize: 14,
+ color: '#ffffff',
+ fontWeight: 'bold',
+ depthTest: false,
+ renderOrder: 100,
+ })
+ );
+ return btn;
+ }
+
+ private setStatus_(text: string) {
+ console.log('[generative_object]', text);
+ const el = document.getElementById('status');
+ if (el) el.textContent = text;
+ // The spatial font lacks some glyphs (e.g. the ellipsis), so normalize.
+ if (this.xrStatusText) this.xrStatusText.setText(text.replace(/…/g, '...'));
+ }
+}
+
+function start() {
+ const options = new xb.Options();
+ // AI for image generation (the generative helper lives in the demo now).
+ options.enableAI();
+
+ // Spatial UI (the control panel) + reticle for pointing at it.
+ options.enableUI();
+ options.reticles.enabled = true;
+
+ // Real-world depth so generated objects are occluded by your environment, and
+ // so placement raycasts hit the current full-resolution surface.
+ options.depth.enabled = true;
+ options.depth.depthMesh.enabled = true;
+ options.depth.depthMesh.updateFullResolutionGeometry = true;
+ options.depth.depthTexture.enabled = true;
+ options.depth.occlusion.enabled = true;
+
+ // Voice input to describe objects.
+ options.sound.speechRecognizer.enabled = true;
+
+ options.setAppTitle('Generative Object');
+ options.setAppDescription(
+ 'Summon AI-generated objects onto the surfaces around you with buttons or ' +
+ 'voice, then grab them. Provide a Gemini key via ?key=...'
+ );
+ options.xrButton.showEnterSimulatorButton = true;
+
+ // The generative helper is its own Script so dependency injection resolves
+ // AI/camera/scene/depth for it, just like a core subsystem would.
+ const generative = new GenerativeObjects();
+ xb.add(generative);
+ xb.add(new GenerativeObjectDemo(generative));
+ xb.init(options);
+}
+
+document.addEventListener('DOMContentLoaded', start);
diff --git a/src/generative/BackgroundKeyer.test.ts b/src/generative/BackgroundKeyer.test.ts
new file mode 100644
index 000000000..df0f6b8e6
--- /dev/null
+++ b/src/generative/BackgroundKeyer.test.ts
@@ -0,0 +1,95 @@
+import {describe, it, expect} from 'vitest';
+
+import {
+ buildDisplacementMap,
+ estimateBackgroundColor,
+ keyOutBackground,
+ RgbaImage,
+} from './BackgroundKeyer';
+
+/**
+ * Builds a `size` x `size` RGBA image with a uniform `bg` border color and a
+ * single `fg` pixel at the center.
+ */
+function imageWithCenter(
+ size: number,
+ bg: [number, number, number],
+ fg: [number, number, number]
+): RgbaImage {
+ const data = new Uint8ClampedArray(size * size * 4);
+ for (let i = 0; i < size * size; i++) {
+ data[i * 4] = bg[0];
+ data[i * 4 + 1] = bg[1];
+ data[i * 4 + 2] = bg[2];
+ data[i * 4 + 3] = 255;
+ }
+ const center = (Math.floor(size / 2) * size + Math.floor(size / 2)) * 4;
+ data[center] = fg[0];
+ data[center + 1] = fg[1];
+ data[center + 2] = fg[2];
+ data[center + 3] = 255;
+ return {data, width: size, height: size};
+}
+
+describe('estimateBackgroundColor', () => {
+ it('averages the four corner pixels', () => {
+ const image = imageWithCenter(4, [255, 255, 255], [200, 0, 0]);
+ expect(estimateBackgroundColor(image)).toEqual([255, 255, 255]);
+ });
+});
+
+describe('keyOutBackground', () => {
+ it('makes background pixels transparent and keeps the subject opaque', () => {
+ const image = imageWithCenter(4, [255, 255, 255], [200, 0, 0]);
+ const result = keyOutBackground(image);
+
+ // A corner (background) is now transparent.
+ expect(result.data[3]).toBe(0);
+ // The center (subject) stays opaque.
+ const center = (2 * 4 + 2) * 4;
+ expect(result.data[center + 3]).toBe(255);
+ });
+
+ it('does not mutate the input image', () => {
+ const image = imageWithCenter(4, [255, 255, 255], [200, 0, 0]);
+ keyOutBackground(image);
+ // Original corner alpha is unchanged.
+ expect(image.data[3]).toBe(255);
+ });
+
+ it('keeps near-background colors within tolerance transparent', () => {
+ // Subject color is close to white; a wide tolerance keys it out too.
+ const image = imageWithCenter(4, [255, 255, 255], [250, 250, 250]);
+ const result = keyOutBackground(image, {tolerance: 100});
+ const center = (2 * 4 + 2) * 4;
+ expect(result.data[center + 3]).toBe(0);
+ });
+
+ it('preserves distinct subjects under a tight tolerance', () => {
+ const image = imageWithCenter(4, [255, 255, 255], [10, 10, 10]);
+ const result = keyOutBackground(image, {tolerance: 10});
+ const center = (2 * 4 + 2) * 4;
+ expect(result.data[center + 3]).toBe(255);
+ });
+});
+
+describe('buildDisplacementMap', () => {
+ it('maps transparent background to black (no displacement)', () => {
+ const image = imageWithCenter(4, [255, 255, 255], [200, 0, 0]);
+ const keyed = keyOutBackground(image);
+ const disp = buildDisplacementMap(keyed);
+ // A corner was keyed transparent -> displacement 0, opaque.
+ expect(disp.data[0]).toBe(0);
+ expect(disp.data[3]).toBe(255);
+ });
+
+ it('maps the subject to its luminance', () => {
+ const image = imageWithCenter(4, [255, 255, 255], [200, 0, 0]);
+ const keyed = keyOutBackground(image);
+ const disp = buildDisplacementMap(keyed);
+ const center = (2 * 4 + 2) * 4;
+ const expected = Math.round(0.2126 * 200);
+ expect(disp.data[center]).toBe(expected);
+ expect(disp.data[center + 3]).toBe(255);
+ });
+});
diff --git a/src/generative/BackgroundKeyer.ts b/src/generative/BackgroundKeyer.ts
new file mode 100644
index 000000000..5b83c7560
--- /dev/null
+++ b/src/generative/BackgroundKeyer.ts
@@ -0,0 +1,100 @@
+/** A raw RGBA image: `data` is width*height*4 bytes, row-major. */
+export interface RgbaImage {
+ data: Uint8ClampedArray;
+ width: number;
+ height: number;
+}
+
+/** Options for {@link keyOutBackground}. */
+export interface BackgroundKeyOptions {
+ /**
+ * Maximum Euclidean RGB distance (0-441) from the sampled background color
+ * for a pixel to be treated as background and made transparent.
+ */
+ tolerance?: number;
+}
+
+const DEFAULT_TOLERANCE = 48;
+
+/**
+ * Estimates the background color of an image by averaging its four corner
+ * pixels. Generated images that place the subject on a plain, uniform
+ * background (the generative_object demo instructs the model to do so) have
+ * corners that are a reliable sample.
+ * @param image - The source RGBA image.
+ * @returns The estimated `[r, g, b]` background color (0-255).
+ */
+export function estimateBackgroundColor(
+ image: RgbaImage
+): [number, number, number] {
+ const {data, width, height} = image;
+ const corners = [
+ 0,
+ (width - 1) * 4,
+ (height - 1) * width * 4,
+ ((height - 1) * width + (width - 1)) * 4,
+ ];
+ let r = 0;
+ let g = 0;
+ let b = 0;
+ for (const offset of corners) {
+ r += data[offset];
+ g += data[offset + 1];
+ b += data[offset + 2];
+ }
+ return [r / corners.length, g / corners.length, b / corners.length];
+}
+
+/**
+ * Makes background-colored pixels transparent, turning a subject-on-a-plain-
+ * background image into a clean cutout. Operates on a copy; the input is not
+ * mutated.
+ * @param image - The source RGBA image.
+ * @param options - Keying options.
+ * @returns A new {@link RgbaImage} with background pixels set to alpha 0.
+ */
+export function keyOutBackground(
+ image: RgbaImage,
+ options: BackgroundKeyOptions = {}
+): RgbaImage {
+ const tolerance = options.tolerance ?? DEFAULT_TOLERANCE;
+ const [bgR, bgG, bgB] = estimateBackgroundColor(image);
+ const toleranceSquared = tolerance * tolerance;
+
+ const out = new Uint8ClampedArray(image.data);
+ for (let i = 0; i < out.length; i += 4) {
+ const dr = out[i] - bgR;
+ const dg = out[i + 1] - bgG;
+ const db = out[i + 2] - bgB;
+ if (dr * dr + dg * dg + db * db <= toleranceSquared) {
+ out[i + 3] = 0;
+ }
+ }
+ return {data: out, width: image.width, height: image.height};
+}
+
+/**
+ * Builds a grayscale displacement map from a keyed image: background pixels
+ * (alpha 0) become black (no displacement) and subject pixels become their
+ * luminance. Masking by alpha keeps the transparent background from displacing
+ * into stray geometry. The result is opaque RGBA.
+ * @param image - A keyed RGBA image (background already at alpha 0).
+ * @returns A new opaque {@link RgbaImage} usable as a displacement/bump map.
+ */
+export function buildDisplacementMap(image: RgbaImage): RgbaImage {
+ const {data, width, height} = image;
+ const out = new Uint8ClampedArray(data.length);
+ for (let i = 0; i < data.length; i += 4) {
+ const luminance =
+ data[i + 3] === 0
+ ? 0
+ : Math.round(
+ 0.2126 * data[i] + 0.7152 * data[i + 1] + 0.0722 * data[i + 2]
+ );
+ out[i] = luminance;
+ out[i + 1] = luminance;
+ out[i + 2] = luminance;
+ out[i + 3] = 255;
+ }
+ return {data: out, width, height};
+}
diff --git a/src/generative/GenerativeObjectUtils.test.ts b/src/generative/GenerativeObjectUtils.test.ts
new file mode 100644
index 000000000..7f6606ddb
--- /dev/null
+++ b/src/generative/GenerativeObjectUtils.test.ts
@@ -0,0 +1,115 @@
+import * as THREE from 'three';
+import {describe, it, expect} from 'vitest';
+
+import {
+ computeBillboardScale,
+ poseInFrontOfCamera,
+ quaternionFacingCamera,
+} from './GenerativeObjectUtils';
+
+describe('computeBillboardScale', () => {
+ it('keeps a landscape image within maxSize on its longest side', () => {
+ const size = computeBillboardScale(200, 100, 0.6);
+ expect(size.x).toBeCloseTo(0.6);
+ expect(size.y).toBeCloseTo(0.3);
+ });
+
+ it('keeps a portrait image within maxSize on its longest side', () => {
+ const size = computeBillboardScale(100, 200, 0.6);
+ expect(size.x).toBeCloseTo(0.3);
+ expect(size.y).toBeCloseTo(0.6);
+ });
+
+ it('returns a square for a square image', () => {
+ const size = computeBillboardScale(512, 512, 0.6);
+ expect(size.x).toBeCloseTo(0.6);
+ expect(size.y).toBeCloseTo(0.6);
+ });
+
+ it('falls back to a square for degenerate dimensions', () => {
+ const size = computeBillboardScale(0, 0, 0.6);
+ expect(size.x).toBeCloseTo(0.6);
+ expect(size.y).toBeCloseTo(0.6);
+ });
+
+ it('writes into the provided target vector', () => {
+ const target = new THREE.Vector2();
+ const result = computeBillboardScale(200, 100, 0.6, target);
+ expect(result).toBe(target);
+ });
+});
+
+describe('poseInFrontOfCamera', () => {
+ it('places the object distance meters along the camera forward axis', () => {
+ const camera = new THREE.PerspectiveCamera();
+ camera.position.set(0, 0, 0);
+ camera.updateMatrixWorld(true);
+ const {position} = poseInFrontOfCamera(camera, 1.0);
+ // Default camera looks down -Z.
+ expect(position.x).toBeCloseTo(0);
+ expect(position.y).toBeCloseTo(0);
+ expect(position.z).toBeCloseTo(-1);
+ });
+
+ it('offsets from the camera world position', () => {
+ const camera = new THREE.PerspectiveCamera();
+ camera.position.set(2, 1, 3);
+ camera.updateMatrixWorld(true);
+ const {position} = poseInFrontOfCamera(camera, 2.0);
+ expect(position.x).toBeCloseTo(2);
+ expect(position.y).toBeCloseTo(1);
+ expect(position.z).toBeCloseTo(1); // 3 + (-1 * 2)
+ });
+
+ it('faces the user (+Z toward the camera)', () => {
+ const camera = new THREE.PerspectiveCamera();
+ camera.position.set(0, 0, 0);
+ camera.updateMatrixWorld(true);
+ const {position, quaternion} = poseInFrontOfCamera(camera, 1.0);
+ const normal = new THREE.Vector3(0, 0, 1).applyQuaternion(quaternion);
+ // The plane normal should point from the object back toward the camera.
+ const toCamera = camera.position.clone().sub(position).normalize();
+ expect(normal.dot(toCamera)).toBeGreaterThan(0.99);
+ });
+});
+
+describe('quaternionFacingCamera', () => {
+ it('orients the front face (+Z) toward the camera', () => {
+ const objectPosition = new THREE.Vector3(0, 0, -2);
+ const cameraPosition = new THREE.Vector3(0, 0, 0);
+ const q = quaternionFacingCamera(objectPosition, cameraPosition);
+ const normal = new THREE.Vector3(0, 0, 1).applyQuaternion(q);
+ const toCamera = cameraPosition.clone().sub(objectPosition).normalize();
+ expect(normal.dot(toCamera)).toBeGreaterThan(0.99);
+ });
+
+ it('faces the camera from an off-axis position', () => {
+ const objectPosition = new THREE.Vector3(3, 0, 1);
+ const cameraPosition = new THREE.Vector3(0, 0, 0);
+ const q = quaternionFacingCamera(objectPosition, cameraPosition);
+ const normal = new THREE.Vector3(0, 0, 1).applyQuaternion(q);
+ const toCamera = cameraPosition.clone().sub(objectPosition).normalize();
+ expect(normal.dot(toCamera)).toBeGreaterThan(0.99);
+ });
+
+ it('returns identity when object and camera coincide', () => {
+ const q = quaternionFacingCamera(
+ new THREE.Vector3(1, 1, 1),
+ new THREE.Vector3(1, 1, 1)
+ );
+ expect(q.x).toBe(0);
+ expect(q.y).toBe(0);
+ expect(q.z).toBe(0);
+ expect(q.w).toBe(1);
+ });
+});
+
+describe('quaternionFacingCamera uprightness', () => {
+ it('stays upright (world up preserved) when the camera is above', () => {
+ const objectPosition = new THREE.Vector3(0, 0, -2);
+ const cameraPosition = new THREE.Vector3(0, 3, 0);
+ const q = quaternionFacingCamera(objectPosition, cameraPosition);
+ const up = new THREE.Vector3(0, 1, 0).applyQuaternion(q);
+ expect(up.y).toBeGreaterThan(0.99);
+ });
+});
diff --git a/src/generative/GenerativeObjectUtils.ts b/src/generative/GenerativeObjectUtils.ts
new file mode 100644
index 000000000..7f8a29409
--- /dev/null
+++ b/src/generative/GenerativeObjectUtils.ts
@@ -0,0 +1,82 @@
+import * as THREE from 'three';
+
+import {lookAtRotation} from '../utils/RotationUtils';
+
+/**
+ * Computes an aspect-ratio-preserving plane size whose largest side equals
+ * `maxSize`. Used to scale a generated image so it reads at a comfortable size
+ * regardless of the model's output resolution.
+ * @param imageWidth - Source image width in pixels.
+ * @param imageHeight - Source image height in pixels.
+ * @param maxSize - Largest dimension of the resulting plane, in meters.
+ * @param target - Optional output vector to write into.
+ * @returns `target` set to the plane's [width, height] in meters.
+ */
+export function computeBillboardScale(
+ imageWidth: number,
+ imageHeight: number,
+ maxSize: number,
+ target = new THREE.Vector2()
+): THREE.Vector2 {
+ if (imageWidth <= 0 || imageHeight <= 0 || maxSize <= 0) {
+ // Degenerate input: fall back to a square so the object is still visible.
+ return target.set(maxSize, maxSize);
+ }
+ const aspect = imageWidth / imageHeight;
+ if (aspect >= 1) {
+ return target.set(maxSize, maxSize / aspect);
+ }
+ return target.set(maxSize * aspect, maxSize);
+}
+
+/**
+ * Computes a pose `distance` meters in front of the camera, oriented so its
+ * front face (+Z) points back toward the user.
+ * @param camera - The user's camera.
+ * @param distance - Distance in front of the camera, in meters.
+ * @param position - Optional output position.
+ * @param quaternion - Optional output orientation.
+ * @returns The position and orientation.
+ */
+export function poseInFrontOfCamera(
+ camera: THREE.Camera,
+ distance: number,
+ position = new THREE.Vector3(),
+ quaternion = new THREE.Quaternion()
+): {position: THREE.Vector3; quaternion: THREE.Quaternion} {
+ const forward = new THREE.Vector3();
+ camera.getWorldDirection(forward);
+ camera.getWorldPosition(position);
+ position.addScaledVector(forward, distance);
+ // lookAtRotation orients local -Z along `forward` (into the scene), so the
+ // plane's +Z normal faces back toward the user.
+ lookAtRotation(forward, undefined, quaternion);
+ return {position, quaternion};
+}
+
+/**
+ * Computes an orientation that turns a plane's front face (+Z) toward the
+ * camera while keeping the object upright (yaw only). Used to billboard a
+ * generated cutout so it faces the user like a standee, without tilting.
+ * @param objectPosition - World position of the object.
+ * @param cameraPosition - World position of the camera.
+ * @param target - Optional output orientation.
+ * @returns `target` oriented so +Z points toward the camera, staying upright.
+ */
+export function quaternionFacingCamera(
+ objectPosition: THREE.Vector3,
+ cameraPosition: THREE.Vector3,
+ target = new THREE.Quaternion()
+): THREE.Quaternion {
+ const awayFromCamera = new THREE.Vector3().subVectors(
+ objectPosition,
+ cameraPosition
+ );
+ // Keep it upright: only yaw toward the camera, never pitch/roll.
+ awayFromCamera.y = 0;
+ if (awayFromCamera.lengthSq() === 0) {
+ return target.identity();
+ }
+ // lookAtRotation orients -Z along `awayFromCamera`, so +Z faces the camera.
+ return lookAtRotation(awayFromCamera, undefined, target);
+}
diff --git a/src/generative/index.ts b/src/generative/index.ts
new file mode 100644
index 000000000..6348269bc
--- /dev/null
+++ b/src/generative/index.ts
@@ -0,0 +1,2 @@
+export * from './BackgroundKeyer';
+export * from './GenerativeObjectUtils';
diff --git a/src/xrblocks.ts b/src/xrblocks.ts
index 9a8084224..1facc19e3 100644
--- a/src/xrblocks.ts
+++ b/src/xrblocks.ts
@@ -38,6 +38,8 @@ export * from './depth/DepthOptions';
export * from './depth/DepthTextures';
export * from './depth/occlusion/OcclusionPass';
export * from './depth/occlusion/OcclusionUtils';
+export * from './generative/BackgroundKeyer';
+export * from './generative/GenerativeObjectUtils';
export * from './input/components/HandJointNames';
export * from './input/GamepadController';
export * from './input/GamepadBindings';