feat(react-native): createCameraEffectMiddleware and release-safe model loading

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+- `createCameraEffectMiddleware(effect)` for React Native: a camera-track middleware for `useCamera().setCameraTrackMiddleware`, so an effect can be switched on from any screen without a provider or a hook.
+- The TypeGPU segmentation model is fetched and parsed once per URL and shared by later sessions, so switching an effect on again or restarting the camera does not reload it.
+- `typeGpuPersonSegmentation({ loadModel })` supplies the model bytes for platforms that cannot `fetch` the model URL, such as an asset embedded in an Android release build.
+
 ## 0.1.2
 
 - Background blur: the person no longer bleeds into the blurred background (no halo), the outline is a smooth ramp that hugs the body, and similar-coloured background next to the person is no longer left sharp.

README.md

@@ -38,28 +38,33 @@ const effect = useBackgroundImage({
 
 `@fishjam-cloud/video-effects/fishjam-react-native` runs an effect on the camera track Fishjam already publishes, through `useCamera`'s camera-track middleware. No separate camera and no custom track: the app keeps using `useCamera`, and remote peers keep seeing `peer.cameraTrack`.
 
-```tsx
-import { useBackgroundBlur } from "@fishjam-cloud/video-effects/background-blur";
-import { useFishjamCameraEffect } from "@fishjam-cloud/video-effects/fishjam-react-native";
+```ts
+import { createBackgroundBlurEffect } from "@fishjam-cloud/video-effects/background-blur";
+import { createCameraEffectMiddleware } from "@fishjam-cloud/video-effects/fishjam-react-native";
 import { typeGpuPersonSegmentation } from "@fishjam-cloud/video-effects/segmentation/typegpu";
 
 const segmentation = typeGpuPersonSegmentation({ modelUrl });
+export const backgroundBlur = createCameraEffectMiddleware(
+  createBackgroundBlurEffect(() => ({ segmentation, radius: 24 })),
+);
+```
 
-function BackgroundBlur({ enabled }: { enabled: boolean }) {
-  const blur = useBackgroundBlur({ segmentation, radius: 24 });
-  const { status, error } = useFishjamCameraEffect(enabled ? blur : null);
-  return null;
-}
+```tsx
+const { currentCameraMiddleware, setCameraTrackMiddleware } = useCamera();
+const isBlurOn = currentCameraMiddleware === backgroundBlur;
+<Button
+  onPress={() => setCameraTrackMiddleware(isBlurOn ? null : backgroundBlur)}
+/>;
 ```
 
-Mount it anywhere inside `FishjamProvider`. While the effect loads, the plain camera is published, so the track is never black. `status` goes `loading` → `ready`, or `unsupported` / `error` with `error` set; `retry()` builds the effect again.
+The middleware lives in Fishjam's camera state, so it stays applied across screens until it is cleared with `null`. Switch it on once the camera is on. Pass `onStatus` in the options to follow loading and errors.
+
+For a component-scoped version there is `useFishjamCameraEffect(effect | null)`, the twin of the web hook; it clears the effect when the component unmounts.
 
 The model file ships in the package: `require("@fishjam-cloud/video-effects/assets/selfie_segmenter.ssgbin")` through `expo-asset` gives the `modelUrl`. Add `ssgbin` to Metro's `resolver.assetExts`.
 
 The app must have these installed and linked: `@fishjam-cloud/react-native-client`, `@fishjam-cloud/react-native-webrtc` (0.30.2 or newer), `@fishjam-cloud/react-native-worklets`, `react-native-worklets` (0.12 or newer, with its Babel plugin) and `react-native-webgpu`, with the New Architecture on. Android needs API 26.
 
-For custom rendering, `createCameraFrameProcessorSession` and the WebGPU helpers are exported from the same entry; the hook is built on them.
-
 ## Entry points
 
 - `@fishjam-cloud/video-effects` — provider and effect contracts

src/fishjam-react-native.ts

@@ -10,6 +10,10 @@
  * @packageDocumentation
  */
 
+export {
+  type CameraEffectMiddlewareOptions,
+  createCameraEffectMiddleware,
+} from "./react-native/createCameraEffectMiddleware";
 export {
   type FishjamCameraEffectOptions,
   type FishjamCameraEffectResult,
@@ -58,3 +62,4 @@ export {
   useCameraWebGpuDevice,
   type UseCameraWebGpuDeviceResult,
 } from "./react-native/webgpu/useCameraWebGpuDevice";
+export { getCameraWebGpuDevice } from "./react-native/webgpu/useCameraWebGpuDevice";

src/react-native/createCameraEffectMiddleware.ts

@@ -0,0 +1,116 @@
+/// <reference types="@webgpu/types" />
+import type { TrackMiddleware } from "@fishjam-cloud/react-native-client";
+
+import type { VideoEffect, VideoEffectStatus } from "../core/types";
+import {
+  type CameraFrameInfo,
+  createCameraFrameProcessorSession,
+} from "./webgpu/cameraFrameProcessorSession";
+import {
+  createCameraTextureResolver,
+  resolveCameraTexture,
+} from "./webgpu/cameraTextureResolver";
+import type { WebGpuFrameRenderFunction } from "./webgpu/frameRenderContext";
+import { getOutputSurfaceFormat } from "./webgpu/requiredFeatures";
+import { getCameraWebGpuDevice } from "./webgpu/useCameraWebGpuDevice";
+
+const DEFAULT_WIDTH = 720;
+const DEFAULT_HEIGHT = 1280;
+
+export interface CameraEffectMiddlewareOptions {
+  /** Width of the published video, in pixels. Defaults to 720. */
+  readonly width?: number;
+  /** Height of the published video, in pixels. Defaults to 1280. */
+  readonly height?: number;
+  /** Reports the effect's loading progress and failures. */
+  readonly onStatus?: (status: VideoEffectStatus, error?: Error) => void;
+}
+
+/**
+ * A camera-track middleware that applies an effect to the camera track Fishjam publishes. Hand it
+ * to `useCamera().setCameraTrackMiddleware`; it stays in place across screens until it is
+ * replaced or cleared with `null`, and `currentCameraMiddleware` tells whether it is active.
+ *
+ * ```ts
+ * const blur = createCameraEffectMiddleware(createBackgroundBlurEffect(() => ({ segmentation })));
+ * setCameraTrackMiddleware(enabled ? blur : null);
+ * ```
+ */
+export function createCameraEffectMiddleware(
+  effect: VideoEffect,
+  options: CameraEffectMiddlewareOptions = {},
+): TrackMiddleware {
+  const width = options.width ?? DEFAULT_WIDTH;
+  const height = options.height ?? DEFAULT_HEIGHT;
+
+  return async (rawTrack) => {
+    const device = await getCameraWebGpuDevice();
+    const session = await effect.create({
+      device,
+      width,
+      height,
+      outputFormat: getOutputSurfaceFormat(),
+      onStatus: options.onStatus,
+    });
+    const resolvedCamera = createCameraTextureResolver(device, {
+      width,
+      height,
+      cameraPixelLayout: "rgb",
+    });
+    const kernel = session.frameKernel;
+    const frameOptions = effect.frameOptions?.();
+
+    const frameKernel = (
+      frame: CameraFrameInfo,
+      render: WebGpuFrameRenderFunction,
+    ) => {
+      "worklet";
+      render((context) => {
+        "worklet";
+        const timestampUs = Math.floor(frame.timestampNanoseconds / 1_000);
+        resolveCameraTexture(
+          context.device,
+          resolvedCamera,
+          context.cameraTexture,
+          context.cameraWidth,
+          context.cameraHeight,
+          context.commandEncoder,
+        );
+        kernel.offer(kernel.state, {
+          kind: "gpu-texture",
+          timestampUs,
+          width: resolvedCamera.width,
+          height: resolvedCamera.height,
+          texture: resolvedCamera.view,
+          commandEncoder: context.commandEncoder,
+        });
+        kernel.encode(
+          kernel.state,
+          {
+            timestampUs,
+            source: resolvedCamera.view,
+            output: context.outputView,
+            commandEncoder: context.commandEncoder,
+          },
+          frameOptions,
+        );
+      });
+    };
+
+    const processor = await createCameraFrameProcessorSession({
+      track: rawTrack,
+      device,
+      width,
+      height,
+      frameKernel,
+    });
+    return {
+      track: processor.track,
+      onClear: () => {
+        void processor.dispose();
+        session.dispose();
+        resolvedCamera.texture.destroy();
+      },
+    };
+  };
+}

src/react-native/webgpu/useCameraWebGpuDevice.ts

@@ -25,7 +25,8 @@ async function acquireSharedCameraWebGpuDevice(): Promise<GPUDevice> {
   });
 }
 
-function getSharedCameraWebGpuDevice(): Promise<GPUDevice> {
+/** The app-wide GPUDevice for camera work, acquired on first use and shared afterwards. */
+export function getCameraWebGpuDevice(): Promise<GPUDevice> {
   if (sharedDevicePromise == null) {
     // Only this promise may clear the slot: by the time a loss (or failure) callback fires, a
     // replacement device may already occupy it, and unconditionally nulling would discard that
@@ -80,7 +81,7 @@ export function useCameraWebGpuDevice(): UseCameraWebGpuDeviceResult {
 
   useEffect(() => {
     let cancelled = false;
-    getSharedCameraWebGpuDevice()
+    getCameraWebGpuDevice()
       .then((device) => {
         if (!cancelled) {
           setResult({ device, error: null });

src/segmentation/typegpu/index.ts

@@ -17,7 +17,10 @@ import {
   packFrameCropParams,
   packUpsampleParams,
 } from "./internal/frameParams";
-import { parseSegmenterPlan } from "./internal/inference/bundle";
+import {
+  parseSegmenterPlan,
+  type SegmenterPlan,
+} from "./internal/inference/bundle";
 import {
   buildSegmentationBundle,
   type SegmentationBundle,
@@ -33,6 +36,12 @@ const PROVIDER_ID = "typegpu-selfie-segmentation-experimental";
 export interface TypeGpuPersonSegmentationOptions {
   /** CDN or application asset URL. The default points at this package's bundled model. */
   readonly modelUrl?: string;
+  /**
+   * Supplies the model bytes instead of fetching `modelUrl`, for a model the platform cannot serve
+   * over `fetch`, such as an asset embedded in a React Native release build. Keep the function's
+   * identity stable (create it once), so the parsed model is shared between sessions.
+   */
+  readonly loadModel?: () => Promise<ArrayBuffer>;
 }
 
 // Plain data only (numbers plus GPU objects): the state is copied onto the camera thread's
@@ -65,8 +74,7 @@ async function createTypeGpuSession(
   context: SegmentationContext,
   options: TypeGpuPersonSegmentationOptions,
 ): Promise<PersonSegmentationSession> {
-  const buffer = await loadModel(options.modelUrl ?? DEFAULT_MODEL_URL);
-  const plan = parseSegmenterPlan(buffer);
+  const plan = await loadSegmenterPlan(options);
   const root = await tgpu.initFromDevice({ device: context.device });
   const bundle = buildSegmentationBundle(
     root,
@@ -213,7 +221,43 @@ function resetTimeline(kernelState: PersonSegmentationKernelState): void {
   state.timestampUs = Number.NEGATIVE_INFINITY;
 }
 
-async function loadModel(url: string): Promise<ArrayBuffer> {
+// The parsed plan is immutable and shared by every session made from the same URL or loader, so
+// a camera restart or an effect toggle skips the fetch and the parse. A failed load is forgotten,
+// so the next attempt loads again.
+type ModelLoader = () => Promise<ArrayBuffer>;
+const plansByUrl = new Map<string, Promise<SegmenterPlan>>();
+const plansByLoader = new WeakMap<ModelLoader, Promise<SegmenterPlan>>();
+
+function loadSegmenterPlan(
+  options: TypeGpuPersonSegmentationOptions,
+): Promise<SegmenterPlan> {
+  if (options.loadModel) {
+    return cachedPlan(plansByLoader, options.loadModel, options.loadModel);
+  }
+  const url = options.modelUrl ?? DEFAULT_MODEL_URL;
+  return cachedPlan(plansByUrl, url, () => fetchModel(url));
+}
+
+interface PlanCache<Key> {
+  get(key: Key): Promise<SegmenterPlan> | undefined;
+  set(key: Key, plan: Promise<SegmenterPlan>): unknown;
+  delete(key: Key): unknown;
+}
+
+function cachedPlan<Key>(
+  cache: PlanCache<Key>,
+  key: Key,
+  load: ModelLoader,
+): Promise<SegmenterPlan> {
+  const cached = cache.get(key);
+  if (cached) return cached;
+  const loading = load().then(parseSegmenterPlan);
+  cache.set(key, loading);
+  loading.catch(() => cache.delete(key));
+  return loading;
+}
+
+async function fetchModel(url: string): Promise<ArrayBuffer> {
   const response = await fetch(url);
   if (!response.ok)
     throw new Error(