From 8f31930b4f5c1d7782231c3706dfd79e30b08248 Mon Sep 17 00:00:00 2001
From: Milosz Filimowski <milosz.filimowski@swmansion.com>
Date: Thu, 10 Sep 2026 15:22:38 +0200
Subject: [PATCH 1/3] feat(react-native): createCameraEffectMiddleware

A camera-track middleware for useCamera().setCameraTrackMiddleware. It
acquires the GPU device, builds the effect session and the frame kernel,
and disposes them when cleared, so an app can switch an effect on from any
screen with no provider and no hook.
---
 CHANGELOG.md                                  |   4 +
 README.md                                     |  27 ++--
 src/fishjam-react-native.ts                   |   5 +
 .../createCameraEffectMiddleware.ts           | 116 ++++++++++++++++++
 .../webgpu/useCameraWebGpuDevice.ts           |   5 +-
 5 files changed, 144 insertions(+), 13 deletions(-)
 create mode 100644 src/react-native/createCameraEffectMiddleware.ts

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 50bb809..2a2c630 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## Unreleased
+
+- `createCameraEffectMiddleware(effect)` for React Native: a camera-track middleware for `useCamera().setCameraTrackMiddleware`, so an effect can be switched on from any screen without a provider or a hook.
+
 ## 0.1.2
 
 - Background blur: the person no longer bleeds into the blurred background (no halo), the outline is a smooth ramp that hugs the body, and similar-coloured background next to the person is no longer left sharp.
diff --git a/README.md b/README.md
index 8420830..77cb1c4 100644
--- a/README.md
+++ b/README.md
@@ -38,28 +38,33 @@ const effect = useBackgroundImage({
 
 `@fishjam-cloud/video-effects/fishjam-react-native` runs an effect on the camera track Fishjam already publishes, through `useCamera`'s camera-track middleware. No separate camera and no custom track: the app keeps using `useCamera`, and remote peers keep seeing `peer.cameraTrack`.
 
-```tsx
-import { useBackgroundBlur } from "@fishjam-cloud/video-effects/background-blur";
-import { useFishjamCameraEffect } from "@fishjam-cloud/video-effects/fishjam-react-native";
+```ts
+import { createBackgroundBlurEffect } from "@fishjam-cloud/video-effects/background-blur";
+import { createCameraEffectMiddleware } from "@fishjam-cloud/video-effects/fishjam-react-native";
 import { typeGpuPersonSegmentation } from "@fishjam-cloud/video-effects/segmentation/typegpu";
 
 const segmentation = typeGpuPersonSegmentation({ modelUrl });
+export const backgroundBlur = createCameraEffectMiddleware(
+  createBackgroundBlurEffect(() => ({ segmentation, radius: 24 })),
+);
+```
 
-function BackgroundBlur({ enabled }: { enabled: boolean }) {
-  const blur = useBackgroundBlur({ segmentation, radius: 24 });
-  const { status, error } = useFishjamCameraEffect(enabled ? blur : null);
-  return null;
-}
+```tsx
+const { currentCameraMiddleware, setCameraTrackMiddleware } = useCamera();
+const isBlurOn = currentCameraMiddleware === backgroundBlur;
+<Button
+  onPress={() => setCameraTrackMiddleware(isBlurOn ? null : backgroundBlur)}
+/>;
 ```
 
-Mount it anywhere inside `FishjamProvider`. While the effect loads, the plain camera is published, so the track is never black. `status` goes `loading` → `ready`, or `unsupported` / `error` with `error` set; `retry()` builds the effect again.
+The middleware lives in Fishjam's camera state, so it stays applied across screens until it is cleared with `null`. Switch it on once the camera is on. Pass `onStatus` in the options to follow loading and errors.
+
+For a component-scoped version there is `useFishjamCameraEffect(effect | null)`, the twin of the web hook; it clears the effect when the component unmounts.
 
 The model file ships in the package: `require("@fishjam-cloud/video-effects/assets/selfie_segmenter.ssgbin")` through `expo-asset` gives the `modelUrl`. Add `ssgbin` to Metro's `resolver.assetExts`.
 
 The app must have these installed and linked: `@fishjam-cloud/react-native-client`, `@fishjam-cloud/react-native-webrtc` (0.30.2 or newer), `@fishjam-cloud/react-native-worklets`, `react-native-worklets` (0.12 or newer, with its Babel plugin) and `react-native-webgpu`, with the New Architecture on. Android needs API 26.
 
-For custom rendering, `createCameraFrameProcessorSession` and the WebGPU helpers are exported from the same entry; the hook is built on them.
-
 ## Entry points
 
 - `@fishjam-cloud/video-effects` — provider and effect contracts
diff --git a/src/fishjam-react-native.ts b/src/fishjam-react-native.ts
index 6c6d47e..b93310e 100644
--- a/src/fishjam-react-native.ts
+++ b/src/fishjam-react-native.ts
@@ -10,6 +10,10 @@
  * @packageDocumentation
  */
 
+export {
+  type CameraEffectMiddlewareOptions,
+  createCameraEffectMiddleware,
+} from "./react-native/createCameraEffectMiddleware";
 export {
   type FishjamCameraEffectOptions,
   type FishjamCameraEffectResult,
@@ -58,3 +62,4 @@ export {
   useCameraWebGpuDevice,
   type UseCameraWebGpuDeviceResult,
 } from "./react-native/webgpu/useCameraWebGpuDevice";
+export { getCameraWebGpuDevice } from "./react-native/webgpu/useCameraWebGpuDevice";
diff --git a/src/react-native/createCameraEffectMiddleware.ts b/src/react-native/createCameraEffectMiddleware.ts
new file mode 100644
index 0000000..6d852a7
--- /dev/null
+++ b/src/react-native/createCameraEffectMiddleware.ts
@@ -0,0 +1,116 @@
+/// <reference types="@webgpu/types" />
+import type { TrackMiddleware } from "@fishjam-cloud/react-native-client";
+
+import type { VideoEffect, VideoEffectStatus } from "../core/types";
+import {
+  type CameraFrameInfo,
+  createCameraFrameProcessorSession,
+} from "./webgpu/cameraFrameProcessorSession";
+import {
+  createCameraTextureResolver,
+  resolveCameraTexture,
+} from "./webgpu/cameraTextureResolver";
+import type { WebGpuFrameRenderFunction } from "./webgpu/frameRenderContext";
+import { getOutputSurfaceFormat } from "./webgpu/requiredFeatures";
+import { getCameraWebGpuDevice } from "./webgpu/useCameraWebGpuDevice";
+
+const DEFAULT_WIDTH = 720;
+const DEFAULT_HEIGHT = 1280;
+
+export interface CameraEffectMiddlewareOptions {
+  /** Width of the published video, in pixels. Defaults to 720. */
+  readonly width?: number;
+  /** Height of the published video, in pixels. Defaults to 1280. */
+  readonly height?: number;
+  /** Reports the effect's loading progress and failures. */
+  readonly onStatus?: (status: VideoEffectStatus, error?: Error) => void;
+}
+
+/**
+ * A camera-track middleware that applies an effect to the camera track Fishjam publishes. Hand it
+ * to `useCamera().setCameraTrackMiddleware`; it stays in place across screens until it is
+ * replaced or cleared with `null`, and `currentCameraMiddleware` tells whether it is active.
+ *
+ * ```ts
+ * const blur = createCameraEffectMiddleware(createBackgroundBlurEffect(() => ({ segmentation })));
+ * setCameraTrackMiddleware(enabled ? blur : null);
+ * ```
+ */
+export function createCameraEffectMiddleware(
+  effect: VideoEffect,
+  options: CameraEffectMiddlewareOptions = {},
+): TrackMiddleware {
+  const width = options.width ?? DEFAULT_WIDTH;
+  const height = options.height ?? DEFAULT_HEIGHT;
+
+  return async (rawTrack) => {
+    const device = await getCameraWebGpuDevice();
+    const session = await effect.create({
+      device,
+      width,
+      height,
+      outputFormat: getOutputSurfaceFormat(),
+      onStatus: options.onStatus,
+    });
+    const resolvedCamera = createCameraTextureResolver(device, {
+      width,
+      height,
+      cameraPixelLayout: "rgb",
+    });
+    const kernel = session.frameKernel;
+    const frameOptions = effect.frameOptions?.();
+
+    const frameKernel = (
+      frame: CameraFrameInfo,
+      render: WebGpuFrameRenderFunction,
+    ) => {
+      "worklet";
+      render((context) => {
+        "worklet";
+        const timestampUs = Math.floor(frame.timestampNanoseconds / 1_000);
+        resolveCameraTexture(
+          context.device,
+          resolvedCamera,
+          context.cameraTexture,
+          context.cameraWidth,
+          context.cameraHeight,
+          context.commandEncoder,
+        );
+        kernel.offer(kernel.state, {
+          kind: "gpu-texture",
+          timestampUs,
+          width: resolvedCamera.width,
+          height: resolvedCamera.height,
+          texture: resolvedCamera.view,
+          commandEncoder: context.commandEncoder,
+        });
+        kernel.encode(
+          kernel.state,
+          {
+            timestampUs,
+            source: resolvedCamera.view,
+            output: context.outputView,
+            commandEncoder: context.commandEncoder,
+          },
+          frameOptions,
+        );
+      });
+    };
+
+    const processor = await createCameraFrameProcessorSession({
+      track: rawTrack,
+      device,
+      width,
+      height,
+      frameKernel,
+    });
+    return {
+      track: processor.track,
+      onClear: () => {
+        void processor.dispose();
+        session.dispose();
+        resolvedCamera.texture.destroy();
+      },
+    };
+  };
+}
diff --git a/src/react-native/webgpu/useCameraWebGpuDevice.ts b/src/react-native/webgpu/useCameraWebGpuDevice.ts
index 8518d29..ea67dd6 100644
--- a/src/react-native/webgpu/useCameraWebGpuDevice.ts
+++ b/src/react-native/webgpu/useCameraWebGpuDevice.ts
@@ -25,7 +25,8 @@ async function acquireSharedCameraWebGpuDevice(): Promise<GPUDevice> {
   });
 }
 
-function getSharedCameraWebGpuDevice(): Promise<GPUDevice> {
+/** The app-wide GPUDevice for camera work, acquired on first use and shared afterwards. */
+export function getCameraWebGpuDevice(): Promise<GPUDevice> {
   if (sharedDevicePromise == null) {
     // Only this promise may clear the slot: by the time a loss (or failure) callback fires, a
     // replacement device may already occupy it, and unconditionally nulling would discard that
@@ -80,7 +81,7 @@ export function useCameraWebGpuDevice(): UseCameraWebGpuDeviceResult {
 
   useEffect(() => {
     let cancelled = false;
-    getSharedCameraWebGpuDevice()
+    getCameraWebGpuDevice()
       .then((device) => {
         if (!cancelled) {
           setResult({ device, error: null });

From f5fbaecd440fa7071524cfbbe5c9e1de0bd06437 Mon Sep 17 00:00:00 2001
From: Milosz Filimowski <milosz.filimowski@swmansion.com>
Date: Fri, 11 Sep 2026 11:58:49 +0200
Subject: [PATCH 2/3] perf(segmentation): share the parsed TypeGPU model
 between sessions

The plan is fetched and parsed once per model URL and reused by every
later session, so switching an effect on again or restarting the camera
skips the download and the parse. A failed load is not cached.
---
 CHANGELOG.md                      |  1 +
 src/segmentation/typegpu/index.ts | 22 +++++++++++++++++++---
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2a2c630..eccc4fe 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## Unreleased
 
 - `createCameraEffectMiddleware(effect)` for React Native: a camera-track middleware for `useCamera().setCameraTrackMiddleware`, so an effect can be switched on from any screen without a provider or a hook.
+- The TypeGPU segmentation model is fetched and parsed once per URL and shared by later sessions, so switching an effect on again or restarting the camera does not reload it.
 
 ## 0.1.2
 
diff --git a/src/segmentation/typegpu/index.ts b/src/segmentation/typegpu/index.ts
index 01d612a..4025103 100644
--- a/src/segmentation/typegpu/index.ts
+++ b/src/segmentation/typegpu/index.ts
@@ -17,7 +17,10 @@ import {
   packFrameCropParams,
   packUpsampleParams,
 } from "./internal/frameParams";
-import { parseSegmenterPlan } from "./internal/inference/bundle";
+import {
+  parseSegmenterPlan,
+  type SegmenterPlan,
+} from "./internal/inference/bundle";
 import {
   buildSegmentationBundle,
   type SegmentationBundle,
@@ -65,8 +68,7 @@ async function createTypeGpuSession(
   context: SegmentationContext,
   options: TypeGpuPersonSegmentationOptions,
 ): Promise<PersonSegmentationSession> {
-  const buffer = await loadModel(options.modelUrl ?? DEFAULT_MODEL_URL);
-  const plan = parseSegmenterPlan(buffer);
+  const plan = await loadSegmenterPlan(options.modelUrl ?? DEFAULT_MODEL_URL);
   const root = await tgpu.initFromDevice({ device: context.device });
   const bundle = buildSegmentationBundle(
     root,
@@ -213,6 +215,20 @@ function resetTimeline(kernelState: PersonSegmentationKernelState): void {
   state.timestampUs = Number.NEGATIVE_INFINITY;
 }
 
+// The parsed plan is immutable and shared by every session made from the same URL, so a camera
+// restart or an effect toggle skips the fetch and the parse. A failed load is forgotten, so the
+// next attempt fetches again.
+const segmenterPlanCache = new Map<string, Promise<SegmenterPlan>>();
+
+function loadSegmenterPlan(url: string): Promise<SegmenterPlan> {
+  const cached = segmenterPlanCache.get(url);
+  if (cached) return cached;
+  const loading = loadModel(url).then(parseSegmenterPlan);
+  segmenterPlanCache.set(url, loading);
+  loading.catch(() => segmenterPlanCache.delete(url));
+  return loading;
+}
+
 async function loadModel(url: string): Promise<ArrayBuffer> {
   const response = await fetch(url);
   if (!response.ok)

From 836bfff61fd4dcf8af0d094c5f7b0d3019944467 Mon Sep 17 00:00:00 2001
From: Milosz Filimowski <milosz.filimowski@swmansion.com>
Date: Fri, 11 Sep 2026 12:11:03 +0200
Subject: [PATCH 3/3] feat(segmentation): loadModel option for models that
 cannot be fetched by URL

An asset embedded in an Android release build is exposed as a
file:///android_res URL that fetch cannot read. The option supplies the
bytes instead, and the parsed model is cached per loader like it is per
URL.
---
 CHANGELOG.md                      |  1 +
 src/segmentation/typegpu/index.ts | 50 ++++++++++++++++++++++++-------
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index eccc4fe..ee5b176 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 - `createCameraEffectMiddleware(effect)` for React Native: a camera-track middleware for `useCamera().setCameraTrackMiddleware`, so an effect can be switched on from any screen without a provider or a hook.
 - The TypeGPU segmentation model is fetched and parsed once per URL and shared by later sessions, so switching an effect on again or restarting the camera does not reload it.
+- `typeGpuPersonSegmentation({ loadModel })` supplies the model bytes for platforms that cannot `fetch` the model URL, such as an asset embedded in an Android release build.
 
 ## 0.1.2
 
diff --git a/src/segmentation/typegpu/index.ts b/src/segmentation/typegpu/index.ts
index 4025103..afd96f6 100644
--- a/src/segmentation/typegpu/index.ts
+++ b/src/segmentation/typegpu/index.ts
@@ -36,6 +36,12 @@ const PROVIDER_ID = "typegpu-selfie-segmentation-experimental";
 export interface TypeGpuPersonSegmentationOptions {
   /** CDN or application asset URL. The default points at this package's bundled model. */
   readonly modelUrl?: string;
+  /**
+   * Supplies the model bytes instead of fetching `modelUrl`, for a model the platform cannot serve
+   * over `fetch`, such as an asset embedded in a React Native release build. Keep the function's
+   * identity stable (create it once), so the parsed model is shared between sessions.
+   */
+  readonly loadModel?: () => Promise<ArrayBuffer>;
 }
 
 // Plain data only (numbers plus GPU objects): the state is copied onto the camera thread's
@@ -68,7 +74,7 @@ async function createTypeGpuSession(
   context: SegmentationContext,
   options: TypeGpuPersonSegmentationOptions,
 ): Promise<PersonSegmentationSession> {
-  const plan = await loadSegmenterPlan(options.modelUrl ?? DEFAULT_MODEL_URL);
+  const plan = await loadSegmenterPlan(options);
   const root = await tgpu.initFromDevice({ device: context.device });
   const bundle = buildSegmentationBundle(
     root,
@@ -215,21 +221,43 @@ function resetTimeline(kernelState: PersonSegmentationKernelState): void {
   state.timestampUs = Number.NEGATIVE_INFINITY;
 }
 
-// The parsed plan is immutable and shared by every session made from the same URL, so a camera
-// restart or an effect toggle skips the fetch and the parse. A failed load is forgotten, so the
-// next attempt fetches again.
-const segmenterPlanCache = new Map<string, Promise<SegmenterPlan>>();
+// The parsed plan is immutable and shared by every session made from the same URL or loader, so
+// a camera restart or an effect toggle skips the fetch and the parse. A failed load is forgotten,
+// so the next attempt loads again.
+type ModelLoader = () => Promise<ArrayBuffer>;
+const plansByUrl = new Map<string, Promise<SegmenterPlan>>();
+const plansByLoader = new WeakMap<ModelLoader, Promise<SegmenterPlan>>();
 
-function loadSegmenterPlan(url: string): Promise<SegmenterPlan> {
-  const cached = segmenterPlanCache.get(url);
+function loadSegmenterPlan(
+  options: TypeGpuPersonSegmentationOptions,
+): Promise<SegmenterPlan> {
+  if (options.loadModel) {
+    return cachedPlan(plansByLoader, options.loadModel, options.loadModel);
+  }
+  const url = options.modelUrl ?? DEFAULT_MODEL_URL;
+  return cachedPlan(plansByUrl, url, () => fetchModel(url));
+}
+
+interface PlanCache<Key> {
+  get(key: Key): Promise<SegmenterPlan> | undefined;
+  set(key: Key, plan: Promise<SegmenterPlan>): unknown;
+  delete(key: Key): unknown;
+}
+
+function cachedPlan<Key>(
+  cache: PlanCache<Key>,
+  key: Key,
+  load: ModelLoader,
+): Promise<SegmenterPlan> {
+  const cached = cache.get(key);
   if (cached) return cached;
-  const loading = loadModel(url).then(parseSegmenterPlan);
-  segmenterPlanCache.set(url, loading);
-  loading.catch(() => segmenterPlanCache.delete(url));
+  const loading = load().then(parseSegmenterPlan);
+  cache.set(key, loading);
+  loading.catch(() => cache.delete(key));
   return loading;
 }
 
-async function loadModel(url: string): Promise<ArrayBuffer> {
+async function fetchModel(url: string): Promise<ArrayBuffer> {
   const response = await fetch(url);
   if (!response.ok)
     throw new Error(