fix(capture): zero-copy dmabuf→VAAPI capture — fix whole-screen recording freeze on GNOME/AMD (#507)

electron/native/pipewire-capture/build.rs

@@ -24,7 +24,7 @@ fn main() {
 
 fn build_pipewire_shim(root: &Path) {
     let vendor = root.join("vendor/pipewire-1.0.5/include");
-    let sources = ["csrc/pw_shim.c", "csrc/pw_audio.c"];
+    let sources = ["csrc/pw_shim.c", "csrc/pw_audio.c", "csrc/dmabuf_modifiers.c"];
 
     assert!(
         vendor.join("pipewire/pipewire.h").is_file(),
@@ -127,7 +127,9 @@ fn link_ffmpeg(root: &Path) {
     );
 
     println!("cargo:rustc-link-search=native={}", lib.display());
-    for name in ["avcodec", "avformat", "avutil", "swscale", "swresample"] {
+    // avfilter is for the VAAPI VPP (scale_vaapi) that converts an imported
+    // dmabuf surface to NV12 for the encoder — see the dmabuf import path.
+    for name in ["avcodec", "avformat", "avutil", "avfilter", "swscale", "swresample"] {
         println!("cargo:rustc-link-lib={name}");
     }
     // A SUBDIRECTORY, NOT `$ORIGIN`. The helper is staged into
@@ -167,8 +169,12 @@ fn link_ffmpeg(root: &Path) {
             #include <libavformat/avformat.h>
             #include <libavutil/avutil.h>
             #include <libavutil/hwcontext.h>
+            #include <libavutil/hwcontext_drm.h>
             #include <libavutil/imgutils.h>
             #include <libavutil/opt.h>
+            #include <libavfilter/avfilter.h>
+            #include <libavfilter/buffersrc.h>
+            #include <libavfilter/buffersink.h>
             #include <libswresample/swresample.h>
             #include <libswscale/swscale.h>
             "#,
@@ -181,6 +187,9 @@ fn link_ffmpeg(root: &Path) {
         .allowlist_function("avcodec_.*")
         .allowlist_function("avformat_.*")
         .allowlist_function("avio_.*")
+        .allowlist_function("avfilter_.*")
+        .allowlist_function("av_buffersrc_.*")
+        .allowlist_function("av_buffersink_.*")
         .allowlist_function("sws_.*")
         .allowlist_function("swr_.*")
         .allowlist_type("AV.*")

electron/native/pipewire-capture/csrc/dmabuf_modifiers.c

@@ -0,0 +1,94 @@
+#include "dmabuf_modifiers.h"
+
+#include <dlfcn.h>
+#include <stddef.h>
+
+/*
+ * Minimal EGL surface, spelled out rather than pulled from <EGL/egl.h> so the
+ * build needs no EGL dev package (same reasoning as the DRM modifier constants
+ * in pw_shim.c). libEGL itself is dlopen'd at runtime; if it is absent the
+ * caller degrades to the LINEAR/INVALID offer.
+ */
+typedef void *EGLDisplay;
+typedef unsigned int EGLBoolean;
+typedef int EGLint;
+typedef intptr_t EGLAttrib;
+typedef uint64_t EGLuint64KHR;
+
+#define OSC_EGL_TRUE 1
+#define OSC_EGL_NO_DISPLAY ((EGLDisplay)0)
+#define OSC_EGL_DEFAULT_DISPLAY ((void *)0)
+/* EGL_MESA_platform_surfaceless — a display with no window system, exactly what
+ * a one-shot capability query wants. */
+#define OSC_EGL_PLATFORM_SURFACELESS_MESA 0x31DD
+
+typedef void *(*osc_eglGetProcAddress)(const char *);
+typedef EGLDisplay (*osc_eglGetPlatformDisplay)(EGLint platform, void *native,
+                                                const EGLAttrib *attrib_list);
+typedef EGLBoolean (*osc_eglInitialize)(EGLDisplay, EGLint *major, EGLint *minor);
+typedef EGLBoolean (*osc_eglTerminate)(EGLDisplay);
+typedef EGLBoolean (*osc_eglQueryDmaBufModifiersEXT)(EGLDisplay, EGLint format,
+                                                     EGLint max_modifiers,
+                                                     EGLuint64KHR *modifiers,
+                                                     EGLBoolean *external_only,
+                                                     EGLint *num_modifiers);
+
+int osc_query_dmabuf_modifiers(uint32_t fourcc, uint64_t *out, int max_out)
+{
+    if (out == NULL || max_out <= 0) {
+        return 0;
+    }
+
+    /* RTLD_NODELETE: EGL keeps process-global state, so never let dlclose run
+     * its destructors — we deliberately do not dlclose at all. */
+    void *egl = dlopen("libEGL.so.1", RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE);
+    if (egl == NULL) {
+        return 0;
+    }
+
+    osc_eglGetProcAddress get_proc =
+        (osc_eglGetProcAddress)dlsym(egl, "eglGetProcAddress");
+    osc_eglInitialize egl_init = (osc_eglInitialize)dlsym(egl, "eglInitialize");
+    osc_eglTerminate egl_terminate = (osc_eglTerminate)dlsym(egl, "eglTerminate");
+    if (get_proc == NULL || egl_init == NULL || egl_terminate == NULL) {
+        return 0;
+    }
+
+    osc_eglGetPlatformDisplay get_display =
+        (osc_eglGetPlatformDisplay)get_proc("eglGetPlatformDisplayEXT");
+    osc_eglQueryDmaBufModifiersEXT query_mods =
+        (osc_eglQueryDmaBufModifiersEXT)get_proc("eglQueryDmaBufModifiersEXT");
+    if (get_display == NULL || query_mods == NULL) {
+        return 0;
+    }
+
+    EGLDisplay dpy = get_display(OSC_EGL_PLATFORM_SURFACELESS_MESA,
+                                 OSC_EGL_DEFAULT_DISPLAY, NULL);
+    if (dpy == OSC_EGL_NO_DISPLAY) {
+        return 0;
+    }
+    if (egl_init(dpy, NULL, NULL) != OSC_EGL_TRUE) {
+        return 0;
+    }
+
+    int written = 0;
+    EGLint count = 0;
+    if (query_mods(dpy, (EGLint)fourcc, 0, NULL, NULL, &count) == OSC_EGL_TRUE &&
+        count > 0) {
+        EGLuint64KHR mods[128];
+        EGLBoolean external[128];
+        EGLint cap = (EGLint)(sizeof(mods) / sizeof(mods[0]));
+        if (count > cap) {
+            count = cap;
+        }
+        if (query_mods(dpy, (EGLint)fourcc, count, mods, external, &count) ==
+            OSC_EGL_TRUE) {
+            for (EGLint i = 0; i < count && written < max_out; i++) {
+                out[written++] = (uint64_t)mods[i];
+            }
+        }
+    }
+
+    egl_terminate(dpy);
+    return written;
+}

electron/native/pipewire-capture/csrc/dmabuf_modifiers.h

@@ -0,0 +1,20 @@
+#ifndef OSC_DMABUF_MODIFIERS_H
+#define OSC_DMABUF_MODIFIERS_H
+
+#include <stdint.h>
+
+/*
+ * Query the DRM format modifiers the local GPU's EGL stack can import for a
+ * given DRM fourcc (e.g. XRGB8888). These are the modifiers we can legitimately
+ * advertise to the compositor in the dmabuf EnumFormat: a compositor buffer
+ * whose modifier is in this set is one we can hand to VAAPI. Fills `out` with up
+ * to `max_out` modifiers and returns the count, or 0 when enumeration is
+ * unavailable (no libEGL, no surfaceless platform, driver refuses) — in which
+ * case the caller falls back to LINEAR/INVALID only.
+ *
+ * libEGL is loaded with dlopen, matching how this crate treats libpipewire: the
+ * helper stays buildable and runnable on a box without EGL dev packages.
+ */
+int osc_query_dmabuf_modifiers(uint32_t fourcc, uint64_t *out, int max_out);
+
+#endif