From 2896468fc4985b3dc4340de004a90818e530dc63 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 5 Sep 2026 05:56:09 +0000 Subject: [PATCH 01/45] feat(linux): add Wayland CLI recording with VA-API --- .github/workflows/ci.yml | 37 + Cargo.lock | 1753 +++++++++++++++++++++++++++-- Cargo.toml | 1 + README.md | 4 + crates/control/src/client.rs | 31 +- crates/daemon/Cargo.toml | 7 +- crates/daemon/src/coordinator.rs | 126 ++- crates/daemon/src/runtime.rs | 63 ++ crates/daemon/src/server.rs | 17 +- crates/daemon/src/test_support.rs | 26 +- crates/linux/Cargo.toml | 18 + crates/linux/src/lib.rs | 201 ++++ crates/linux/src/pipeline.rs | 706 ++++++++++++ crates/linux/src/portal.rs | 175 +++ crates/linux/src/portal_tests.rs | 219 ++++ crates/macos/src/lib.rs | 6 +- packaging/linux/README.md | 112 ++ scripts/package-cli-linux.sh | 21 + scripts/test-cli-linux.py | 82 ++ 19 files changed, 3500 insertions(+), 105 deletions(-) create mode 100644 crates/linux/Cargo.toml create mode 100644 crates/linux/src/lib.rs create mode 100644 crates/linux/src/pipeline.rs create mode 100644 crates/linux/src/portal.rs create mode 100644 crates/linux/src/portal_tests.rs create mode 100644 packaging/linux/README.md create mode 100755 scripts/package-cli-linux.sh create mode 100644 scripts/test-cli-linux.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5321f40..fd2c433 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,43 @@ concurrency: cancel-in-progress: true jobs: + linux: + name: Linux CLI and capture lifecycle + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - name: Install native dependencies + run: | + sudo apt-get update + sudo apt-get install -y pkg-config libgstreamer1.0-dev \ + libgstreamer-plugins-base1.0-dev gstreamer1.0-pipewire \ + gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ + gstreamer1.0-plugins-bad gstreamer1.0-libav ffmpeg dbus-daemon + - name: Build and test + run: | + cargo fmt --check + cargo check --workspace --locked + cargo test --workspace --locked + dbus-run-session -- cargo test -p linux portal_roundtrip --locked -- --ignored + ./scripts/package-cli-linux.sh + - name: Test packaged CLI + run: | + stage="$(mktemp -d)" + trap 'rm -rf "$stage"' EXIT + tar -xzf dist/cli/linux/wrec-cli-x86_64-unknown-linux-gnu.tar.gz -C "$stage" + python3 scripts/test-cli-linux.py "$stage" + - name: Upload Linux CLI package + uses: actions/upload-artifact@v4 + with: + name: wrec-cli-linux + path: dist/cli/linux/*.tar.gz + rust: name: Rust checks runs-on: blacksmith-6vcpu-macos-latest diff --git a/Cargo.lock b/Cargo.lock index 58b0e25..1722912 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,12 +17,167 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "ashpd" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f8bd58b44ea371b48d21cdc217380cfcafd4b2bb1ad50d27514ec5beca71a2d" +dependencies = [ + "enumflags2", + "futures-channel", + "futures-util", + "rand", + "serde", + "serde_repr", + "tokio", + "url", + "zbus", +] + +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix", +] + +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-signal" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43c070bbf59cd3570b6b2dd54cd772527c7c3620fce8be898406dd3ed6adc64c" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "bitflags" version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -32,12 +187,31 @@ dependencies = [ "objc2", ] +[[package]] +name = "blocking" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "bumpalo" version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.2.62" @@ -48,6 +222,16 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfg-expr" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2b34126159980f92da2a08bdec0694fd80fb5eb9e48aff25d20a0d8dfa710d" +dependencies = [ + "smallvec", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -71,6 +255,15 @@ dependencies = [ "wrec-channel", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "config" version = "0.3.0" @@ -93,6 +286,31 @@ dependencies = [ "wrec-channel", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "ctrlc" version = "3.5.2" @@ -100,8 +318,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" dependencies = [ "dispatch2", - "nix", - "windows-sys", + "nix 0.31.3", + "windows-sys 0.61.2", ] [[package]] @@ -112,6 +330,7 @@ dependencies = [ "control", "domain", "libc", + "linux", "macos", "rusqlite", "serde", @@ -121,6 +340,16 @@ dependencies = [ "wrec-channel", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dispatch2" version = "0.3.1" @@ -133,6 +362,17 @@ dependencies = [ "objc2", ] +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "domain" version = "0.3.0" @@ -144,6 +384,75 @@ dependencies = [ "wrec-channel", ] +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "endi" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", + "serde", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -156,6 +465,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -169,145 +484,669 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] -name = "hashbrown" -version = "0.16.1" +name = "form_urlencoded" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ - "foldhash", + "percent-encoding", ] [[package]] -name = "hashlink" -version = "0.11.0" +name = "futures-channel" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0b22561a9c04a7cb1a302c013e0259cd3b4bb619f145b32f72b8b4bcbed230" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ - "hashbrown", + "futures-core", ] [[package]] -name = "itoa" -version = "1.0.18" +name = "futures-core" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] -name = "js-sys" -version = "0.3.98" +name = "futures-executor" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ - "cfg-if", - "once_cell", - "wasm-bindgen", + "futures-core", + "futures-task", + "futures-util", ] [[package]] -name = "lazy_static" -version = "1.5.0" +name = "futures-io" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] -name = "libc" -version = "0.2.186" +name = "futures-lite" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] [[package]] -name = "libsqlite3-sys" -version = "0.37.0" +name = "futures-macro" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ - "cc", - "pkg-config", - "vcpkg", + "proc-macro2", + "quote", + "syn 3.0.5", ] [[package]] -name = "log" -version = "0.4.29" +name = "futures-sink" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] -name = "macos" -version = "0.3.0" +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ - "anyhow", - "domain", - "objc2", - "objc2-av-foundation", - "objc2-core-foundation", - "objc2-core-graphics", - "objc2-core-media", - "objc2-core-video", - "objc2-foundation", - "objc2-metal", - "objc2-screen-capture-kit", - "objc2-video-toolbox", - "serde", - "serde_json", - "tracing", + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", ] [[package]] -name = "matchers" -version = "0.2.0" +name = "generic-array" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ - "regex-automata", + "typenum", + "version_check", ] [[package]] -name = "memchr" -version = "2.8.0" +name = "getrandom" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] [[package]] -name = "nix" -version = "0.31.3" +name = "getrandom" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "bitflags", "cfg-if", - "cfg_aliases", "libc", + "r-efi", + "wasip2", ] [[package]] -name = "nu-ansi-term" -version = "0.50.3" +name = "gio-sys" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +checksum = "521e93a7e56fc89e84aea9a52cfc9436816a4b363b030260b699950ff1336c83" dependencies = [ - "windows-sys", + "glib-sys", + "gobject-sys", + "libc", + "system-deps", + "windows-sys 0.59.0", ] [[package]] -name = "objc2" -version = "0.6.4" +name = "glib" +version = "0.20.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +checksum = "ffc4b6e352d4716d84d7dde562dd9aee2a7d48beb872dd9ece7f2d1515b2d683" dependencies = [ - "objc2-encode", + "bitflags", + "futures-channel", + "futures-core", + "futures-executor", + "futures-task", + "futures-util", + "gio-sys", + "glib-macros", + "glib-sys", + "gobject-sys", + "libc", + "memchr", + "smallvec", ] [[package]] -name = "objc2-av-foundation" -version = "0.3.2" +name = "glib-macros" +version = "0.20.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +checksum = "e8084af62f09475a3f529b1629c10c429d7600ee1398ae12dd3bf175d74e7145" dependencies = [ - "bitflags", - "block2", - "dispatch2", - "objc2", + "heck", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "glib-sys" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ab79e1ed126803a8fb827e3de0e2ff95191912b8db65cee467edb56fc4cc215" +dependencies = [ + "libc", + "system-deps", +] + +[[package]] +name = "gobject-sys" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9aca94bb73989e3cfdbf8f2e0f1f6da04db4d291c431f444838925c4c63eda" +dependencies = [ + "glib-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer" +version = "0.23.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8757a87f3706560037a01a9f06a59fcc7bdb0864744dcf73546606e60c4316e1" +dependencies = [ + "cfg-if", + "futures-channel", + "futures-core", + "futures-util", + "glib", + "gstreamer-sys", + "itertools", + "libc", + "muldiv", + "num-integer", + "num-rational", + "once_cell", + "option-operations", + "paste", + "pin-project-lite", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gstreamer-allocators" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45f3c36aa5bc98e5dbfd19a27536373167ab1ad734ffafb84d5d65203fe75208" +dependencies = [ + "glib", + "gstreamer", + "gstreamer-allocators-sys", + "libc", + "once_cell", +] + +[[package]] +name = "gstreamer-allocators-sys" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c005b6cfae0c7e97b0ea95921f561fc8a8786b734cec25c29363a52305074a10" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-sys" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feea73b4d92dbf9c24a203c9cd0bcc740d584f6b5960d5faf359febf288919b2" +dependencies = [ + "glib-sys", + "gobject-sys", + "libc", + "system-deps", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0b22561a9c04a7cb1a302c013e0259cd3b4bb619f145b32f72b8b4bcbed230" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libsqlite3-sys" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux" +version = "0.3.0" +dependencies = [ + "ashpd", + "domain", + "futures-util", + "gstreamer", + "gstreamer-allocators", + "serde_json", + "tokio", + "zbus", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "macos" +version = "0.3.0" +dependencies = [ + "anyhow", + "domain", + "objc2", + "objc2-av-foundation", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation", + "objc2-metal", + "objc2-screen-capture-kit", + "objc2-video-toolbox", + "serde", + "serde_json", + "tracing", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "muldiv" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "956787520e75e9bd233246045d19f42fb73242759cc57fba9611d940ae96d4b0" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags", + "block2", + "dispatch2", + "objc2", "objc2-avf-audio", "objc2-core-audio-types", "objc2-core-foundation", @@ -552,18 +1391,98 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "option-operations" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c26d27bb1aeab65138e4bf7666045169d1717febcc9ff870166be8348b223d0" +dependencies = [ + "paste", +] + +[[package]] +name = "ordered-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkg-config" version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-crate" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -582,6 +1501,42 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -624,6 +1579,19 @@ dependencies = [ "sqlite-wasm-rs", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -657,7 +1625,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -673,6 +1641,37 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "serde_spanned" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +dependencies = [ + "serde_core", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -688,12 +1687,38 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "sqlite-wasm-rs" version = "0.5.4" @@ -706,6 +1731,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "syn" version = "2.0.117" @@ -717,6 +1754,60 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "system-deps" +version = "7.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396a35feb67335377e0251fcbc1092fc85c484bd4e3a7a54319399da127796e7" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "target-lexicon" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -743,7 +1834,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -754,7 +1845,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -766,6 +1857,104 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "toml" +version = "1.0.7+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd28d57d8a6f6e458bc0b8784f8fdcc4b99a437936056fa122cb234f18656a96" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime 1.0.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.4", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_datetime" +version = "1.0.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b320e741db58cac564e26c607d3cc1fdc4a88fd36c879568c07856ed83ff3e9" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.23.10+spec-1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" +dependencies = [ + "indexmap", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "winnow 0.7.15", +] + +[[package]] +name = "toml_parser" +version = "1.0.10+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_writer" +version = "1.0.7+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17aaa1c6e3dc22b1da4b6bba97d066e354c7945cac2f7852d4e4e7ca7a6b56d" + [[package]] name = "tracing" version = "0.1.44" @@ -785,7 +1974,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -827,12 +2016,54 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uds_windows" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89daebc3e6fd160ac4aa9fc8b3bf71e1f74fbf92367ae71fb83a037e8bf164b9" +dependencies = [ + "memoffset", + "tempfile", + "winapi", +] + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "valuable" version = "0.1.1" @@ -845,6 +2076,33 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version-compare" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.121" @@ -877,7 +2135,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -890,12 +2148,52 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -905,6 +2203,91 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wrec-channel" version = "0.3.0" @@ -912,12 +2295,216 @@ dependencies = [ "serde", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "xdg-home" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec1cdab258fb55c0da61328dc52c8764709b249011b2cad0454c72f0bf10a1f6" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "xtask" version = "0.3.0" +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zbus" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb97012beadd29e654708a0fdb4c84bc046f537aecfde2c3ee0a9e4b4d48c725" +dependencies = [ + "async-broadcast", + "async-process", + "async-recursion", + "async-trait", + "enumflags2", + "event-listener", + "futures-core", + "futures-sink", + "futures-util", + "hex", + "nix 0.29.0", + "ordered-stream", + "rand", + "serde", + "serde_repr", + "sha1", + "static_assertions", + "tokio", + "tracing", + "uds_windows", + "windows-sys 0.52.0", + "xdg-home", + "zbus_macros", + "zbus_names", + "zvariant", +] + +[[package]] +name = "zbus_macros" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267db9407081e90bbfa46d841d3cbc60f59c0351838c4bc65199ecd79ab1983e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", + "zvariant_utils", +] + +[[package]] +name = "zbus_names" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9b1fef7d021261cc16cba64c351d291b715febe0fa10dc3a443ac5a5022e6c" +dependencies = [ + "serde", + "static_assertions", + "zvariant", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e3c6377872d72510393f688a555d7097b0f741995c7a00f0407f786dd486b2d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zvariant" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2084290ab9a1c471c38fc524945837734fbf124487e105daec2bb57fd48c81fe" +dependencies = [ + "endi", + "enumflags2", + "serde", + "static_assertions", + "url", + "zvariant_derive", +] + +[[package]] +name = "zvariant_derive" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73e2ba546bda683a90652bac4a279bc146adad1386f25379cf73200d2002c449" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", + "zvariant_utils", +] + +[[package]] +name = "zvariant_utils" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] diff --git a/Cargo.toml b/Cargo.toml index d44334a..2fb8035 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/domain", "crates/daemon", "crates/macos", + "crates/linux", "crates/xtask", ] resolver = "2" diff --git a/README.md b/README.md index 9b86902..f3b91f6 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,10 @@ Wrec records displays or windows with a native ScreenCaptureKit pipeline, writes hardware-encoded `.mov` files, and gives you both a native SwiftUI menu-bar app and a JSON-friendly CLI for scripts and agents. +An initial Linux CLI backend uses the Wayland ScreenCast portal, PipeWire +DMA-BUFs, and VA-API hardware encoding. It requires a compatible Intel/AMD +GPU and desktop. See [Linux setup and limitations](packaging/linux/README.md). + > [!NOTE] > Wrec is still early public software. Release builds are not notarized, so > macOS blocks the first launch of an app downloaded with a browser. The diff --git a/crates/control/src/client.rs b/crates/control/src/client.rs index baa7326..f42f29d 100644 --- a/crates/control/src/client.rs +++ b/crates/control/src/client.rs @@ -255,6 +255,7 @@ pub fn ensure_daemon() -> Result<(), AgentError> { next: "Create the directory manually or set WREC_HOME to a writable path.".into(), })?; + #[cfg(target_os = "macos")] if std::env::var_os("WREC_DAEMON_BIN").is_none() && std::env::var_os("WREC_HEADLESS").as_deref() != Some(std::ffi::OsStr::new("1")) { @@ -338,6 +339,7 @@ pub fn ensure_daemon() -> Result<(), AgentError> { }) } +#[cfg(target_os = "macos")] fn launch_app_daemon(client: &DaemonClient) -> Result<(), AgentError> { let channel = Channel::current(); let status = Command::new("/usr/bin/open") @@ -605,6 +607,13 @@ fn daemon_candidates(current: &Path) -> Vec { [ Some(current_dir.join("daemon")), profile_dir.map(|dir| dir.join("daemon")), + #[cfg(target_os = "linux")] + current_dir.parent().map(|prefix| { + prefix + .join("lib") + .join(Channel::current().runtime_dir_name()) + .join("daemon") + }), Some(installed_daemon_path()), ] .into_iter() @@ -623,7 +632,7 @@ fn daemon_executable_launch(path: PathBuf) -> Option { return None; } - if cargo_profile_dir(&path).is_some() { + if cfg!(target_os = "linux") || cargo_profile_dir(&path).is_some() { return Some(DaemonLaunch::executable(path)); } @@ -756,7 +765,27 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + #[cfg(target_os = "linux")] #[test] + fn linux_package_finds_runtime_relative_to_install_prefix() { + let candidates = daemon_candidates(std::path::Path::new("/opt/wrec/bin/wrec")); + assert!(candidates.contains( + &PathBuf::from("/opt/wrec/lib") + .join(wrec_channel::Channel::current().runtime_dir_name()) + .join("daemon") + )); + let dir = std::env::temp_dir().join(format!("wrec-linux-daemon-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let daemon = dir.join("daemon"); + std::fs::write(&daemon, "").unwrap(); + let launch = daemon_executable_launch(daemon.clone()).unwrap(); + assert_eq!(launch.program, daemon); + assert!(launch.envs.is_empty()); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + #[cfg(target_os = "macos")] fn packaged_daemon_launch_requires_sibling_capture_engine() { let dir = std::env::temp_dir().join(format!( "wrec-control-packaged-daemon-{}", diff --git a/crates/daemon/Cargo.toml b/crates/daemon/Cargo.toml index 8293a7e..c38c66f 100644 --- a/crates/daemon/Cargo.toml +++ b/crates/daemon/Cargo.toml @@ -10,7 +10,6 @@ control = { path = "../control" } config = { path = "../config" } domain = { path = "../domain" } libc = "0.2" -macos = { path = "../macos" } rusqlite = { version = "0.39.0", features = ["bundled"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" @@ -18,6 +17,12 @@ thiserror.workspace = true tracing.workspace = true wrec-channel.workspace = true +[target.'cfg(target_os = "macos")'.dependencies] +macos = { path = "../macos" } + +[target.'cfg(target_os = "linux")'.dependencies] +linux = { path = "../linux" } + [[bin]] name = "daemon" path = "src/main.rs" diff --git a/crates/daemon/src/coordinator.rs b/crates/daemon/src/coordinator.rs index 5c4349d..fc36109 100644 --- a/crates/daemon/src/coordinator.rs +++ b/crates/daemon/src/coordinator.rs @@ -55,6 +55,40 @@ impl Coordinator { .unwrap_or(true) } + #[cfg(target_os = "linux")] + pub(crate) fn stop_for_shutdown(state: &SharedCoordinator) { + let control = { + let Ok(mut state) = lock_state(state) else { + return; + }; + state.shutdown_requested = true; + while let Some(id) = state.queue.pop_front() { + if let Some(job) = state.jobs.get_mut(&id) { + job.mark_cancelled(); + } + } + state + .active_job_id + .and_then(|id| state.jobs.get_mut(&id)) + .and_then(|job| { + job.mark_finishing(); + job.control.clone() + }) + }; + if let Some(control) = control { + if let Ok(mut engine) = control.lock() { + let _ = engine.stop(); + } + } + } + + #[cfg(target_os = "linux")] + pub(crate) fn capture_active(state: &SharedCoordinator) -> bool { + lock_state(state) + .map(|state| state.active_job_id.is_some()) + .unwrap_or(false) + } + pub(crate) fn status(&self) -> Value { json!({ "daemon_version": env!("CARGO_PKG_VERSION"), @@ -535,18 +569,16 @@ fn run_job( return; } - if let Ok(mut state) = lock_state(&state) { - if let Some(job) = state.jobs.get_mut(&job_id) { - job.mark_recording(); - } - } - notify_job_changed(); - - let started = Instant::now(); + // A portal can wait for user selection after start() returns. Only the + // engine's Started event means frames are flowing and duration can begin. + let mut started = None; let mut duration_stop_requested = false; loop { match rx.recv_timeout(Duration::from_millis(200)) { Ok(event) => { + if matches!(event, RecorderEvent::Started { .. }) && started.is_none() { + started = Some(Instant::now()); + } let done = handle_recorder_event(&state, job_id, event); if done { break; @@ -559,7 +591,7 @@ fn run_job( } } - if let Some(duration_ms) = duration_ms { + if let (Some(duration_ms), Some(started)) = (duration_ms, started) { if !duration_stop_requested && started.elapsed() >= Duration::from_millis(duration_ms) { duration_stop_requested = true; append_job_message( @@ -623,6 +655,9 @@ fn handle_recorder_event( false } BackendEvent::Started => { + if job.status == JobStatus::Starting { + job.mark_recording(); + } job.push_event(EventLevel::Info, "recording started".to_string()); false } @@ -659,7 +694,7 @@ fn handle_recorder_event( true } }; - if done { + if done || matches!(event, RecorderEvent::Started { .. }) { drop(state); notify_job_changed(); } @@ -818,6 +853,7 @@ fn target_list_lock() -> &'static Mutex<()> { /// payload: observers just re-query `wrec jobs`/`job show`, so a stale or /// coalesced delivery is harmless. fn notify_job_changed() { + #[cfg(target_os = "macos")] macos::post_distributed_notification(&Channel::current().notification_name("job-changed")); } @@ -884,6 +920,76 @@ mod tests { use control::{JobSnapshot, RecordingOptions, TargetSelector}; use domain::{CaptureSourceKind, PermissionStatus}; + #[test] + fn duration_waits_for_capture_readiness() { + let _guard = env_lock(); + isolate_env(); + let runtime = FakeRuntime::new(); + runtime + .start_gate + .store(false, std::sync::atomic::Ordering::Relaxed); + let state = Arc::new(Mutex::new(Coordinator::new(runtime.clone()))); + let value = Coordinator::record_start( + state.clone(), + StartRecordingParams { + selector: Some(TargetSelector::Id { + kind: CaptureSourceKind::Display, + id: 1, + }), + options: RecordingOptions::default(), + duration_ms: Some(100), + queue: true, + }, + ) + .unwrap(); + let id = value["job"]["id"].as_u64().unwrap(); + wait_for_status(&state, id, JobStatus::Starting); + std::thread::sleep(Duration::from_millis(350)); + assert_eq!(job_status(&state, id), JobStatus::Starting); + runtime + .start_gate + .store(true, std::sync::atomic::Ordering::Relaxed); + wait_for_status(&state, id, JobStatus::Recording); + wait_for_status(&state, id, JobStatus::Completed); + } + + #[test] + fn pending_source_selection_can_be_stopped() { + let _guard = env_lock(); + isolate_env(); + let runtime = FakeRuntime::new(); + runtime + .start_gate + .store(false, std::sync::atomic::Ordering::Relaxed); + let state = Arc::new(Mutex::new(Coordinator::new(runtime))); + let id = start_job(state.clone()).id; + wait_for_status(&state, id, JobStatus::Starting); + // Wait for the asynchronous Starting event, so the fake owns its session. + for _ in 0..50 { + if lock_state(&state).unwrap().jobs[&id].output_path.is_some() { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + Coordinator::job_stop(state.clone(), id).unwrap(); + wait_for_status(&state, id, JobStatus::Completed); + } + + #[cfg(target_os = "linux")] + #[test] + fn shutdown_finishes_capture_and_cancels_queued_jobs() { + let _guard = env_lock(); + isolate_env(); + let state = Arc::new(Mutex::new(Coordinator::new(FakeRuntime::new()))); + let active = start_job(state.clone()).id; + wait_for_status(&state, active, JobStatus::Recording); + let queued = start_job(state.clone()).id; + Coordinator::stop_for_shutdown(&state); + wait_for_status(&state, active, JobStatus::Completed); + assert_eq!(job_status(&state, queued), JobStatus::Cancelled); + assert!(Coordinator::shutdown_requested(&state)); + } + #[test] fn queued_job_launches_after_active_job_stops() { let _guard = env_lock(); diff --git a/crates/daemon/src/runtime.rs b/crates/daemon/src/runtime.rs index d547a89..57ab831 100644 --- a/crates/daemon/src/runtime.rs +++ b/crates/daemon/src/runtime.rs @@ -1,5 +1,6 @@ use control::AgentError; use domain::{CaptureTarget, PermissionStatus, RecorderEngine, RecorderError, RecorderEvent}; +#[cfg(target_os = "macos")] use macos::MacosRecorder; use std::sync::mpsc; @@ -18,9 +19,11 @@ pub(crate) trait RecordingRuntime: Clone + Send + Sync + 'static { fn new_engine(&self, events: mpsc::Sender) -> Self::Engine; } +#[cfg(target_os = "macos")] #[derive(Clone, Default)] pub(crate) struct MacosRuntime; +#[cfg(target_os = "macos")] impl RecordingRuntime for MacosRuntime { type Engine = MacosRecorder; @@ -70,6 +73,7 @@ impl RecordingRuntime for MacosRuntime { } } +#[cfg(any(target_os = "macos", test))] fn permission_error(error: RecorderError) -> AgentError { match error { RecorderError::MissingScreenRecordingPermission => { @@ -100,6 +104,7 @@ fn permission_error(error: RecorderError) -> AgentError { /// (Wrec.app or the terminal running `wrec daemon start`), not to the daemon /// binary itself, so a denial here otherwise leaves no trail in daemon.log /// pointing anyone at the actual app to go re-approve. +#[cfg(any(target_os = "macos", test))] fn warn_screen_recording_permission_missing() { tracing::warn!( "screen recording permission is not granted; grant it to the app that launched this \ @@ -107,6 +112,64 @@ fn warn_screen_recording_permission_missing() { ); } +#[cfg(target_os = "linux")] +pub(crate) use LinuxRuntime as PlatformRuntime; +#[cfg(target_os = "macos")] +pub(crate) use MacosRuntime as PlatformRuntime; + +#[cfg(target_os = "linux")] +#[derive(Clone, Default)] +pub(crate) struct LinuxRuntime; + +#[cfg(target_os = "linux")] +impl RecordingRuntime for LinuxRuntime { + type Engine = linux::LinuxRecorder; + + fn list_targets(&self) -> Result, AgentError> { + linux::list_targets().map_err(linux_error) + } + + fn screen_recording_permission_status(&self) -> Result { + linux::check_desktop().map_err(linux_error)?; + // The portal grants access per session when recording starts. + Ok(PermissionStatus::Unknown) + } + + fn request_screen_recording_permission(&self) -> Result { + self.screen_recording_permission_status() + } + + fn microphone_permission_status(&self) -> Result { + // Native PulseAudio clients have no separate TCC permission prompt. + // The audio server checks access when pulsesrc connects. + Ok(PermissionStatus::Granted) + } + + fn request_microphone_permission(&self) -> Result { + self.microphone_permission_status() + } + + fn open_microphone_settings(&self) -> Result<(), AgentError> { + Err(linux_error(RecorderError::Backend( + "Select the microphone in your desktop's sound settings.".into(), + ))) + } + + fn new_engine(&self, events: mpsc::Sender) -> Self::Engine { + linux::LinuxRecorder::new(events) + } +} + +#[cfg(target_os = "linux")] +fn linux_error(error: RecorderError) -> AgentError { + AgentError { + code: "linux_capture_unavailable".into(), + message: error.to_string(), + recoverable: true, + next: "Run wrec inside a Wayland desktop with PipeWire and the desktop's ScreenCast portal installed. See packaging/linux/README.md.".into(), + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/daemon/src/server.rs b/crates/daemon/src/server.rs index d0cc226..10b880c 100644 --- a/crates/daemon/src/server.rs +++ b/crates/daemon/src/server.rs @@ -1,6 +1,6 @@ use crate::{ coordinator::{lock_state, Coordinator, SharedCoordinator}, - runtime::{MacosRuntime, RecordingRuntime}, + runtime::{PlatformRuntime, RecordingRuntime}, }; use control::{ daemon_log_path, response_error, socket_path, wrec_home, AgentError, IpcRequest, IpcResponse, @@ -73,7 +73,7 @@ pub fn serve_forever() -> Result<(), String> { listener .set_nonblocking(true) .map_err(|err| format!("failed to configure {}: {err}", socket.display()))?; - let state = Arc::new(Mutex::new(Coordinator::new(MacosRuntime))); + let state = Arc::new(Mutex::new(Coordinator::new(PlatformRuntime))); while !TERMINATED.load(Ordering::SeqCst) && !Coordinator::shutdown_requested(&state) { match listener.accept() { @@ -88,12 +88,23 @@ pub fn serve_forever() -> Result<(), String> { } } + #[cfg(target_os = "linux")] + { + Coordinator::stop_for_shutdown(&state); + let deadline = std::time::Instant::now() + Duration::from_secs(15); + while Coordinator::capture_active(&state) && std::time::Instant::now() < deadline { + thread::sleep(POLL_INTERVAL); + } + if Coordinator::capture_active(&state) { + tracing::error!("capture did not finalize within 15s during daemon shutdown"); + } + } tracing::info!("daemon stopped"); let _ = std::fs::remove_file(&socket); Ok(()) } -fn handle_client(stream: UnixStream, state: SharedCoordinator) { +fn handle_client(stream: UnixStream, state: SharedCoordinator) { if let Err(err) = stream.set_nonblocking(false) { tracing::warn!("client blocking mode failed: {err}"); } diff --git a/crates/daemon/src/test_support.rs b/crates/daemon/src/test_support.rs index ccb766b..50246c9 100644 --- a/crates/daemon/src/test_support.rs +++ b/crates/daemon/src/test_support.rs @@ -7,7 +7,7 @@ use domain::{ use std::{ path::PathBuf, sync::{ - atomic::{AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, mpsc, Arc, Mutex, MutexGuard, PoisonError, }, }; @@ -41,12 +41,15 @@ pub(crate) struct FakeRuntime { mic_request_result: Arc>, mic_requests: Arc, mic_settings_opens: Arc, + pub(crate) start_gate: Arc, } pub(crate) struct FakeEngine { events: mpsc::Sender, next_session_id: Arc, active: Option, + start_gate: Arc, + stopped: Arc, } impl FakeRuntime { @@ -63,6 +66,7 @@ impl FakeRuntime { mic_request_result: Arc::new(Mutex::new(PermissionStatus::Granted)), mic_requests: Arc::new(AtomicU64::new(0)), mic_settings_opens: Arc::new(AtomicU64::new(0)), + start_gate: Arc::new(AtomicBool::new(true)), } } @@ -125,6 +129,8 @@ impl RecordingRuntime for FakeRuntime { events, next_session_id: self.next_session_id.clone(), active: None, + start_gate: self.start_gate.clone(), + stopped: Arc::new(AtomicBool::new(false)), } } } @@ -155,12 +161,21 @@ impl RecorderEngine for FakeEngine { output_path: session.output_path.clone(), }) .unwrap(); - self.events - .send(RecorderEvent::Started { + let events = self.events.clone(); + let gate = self.start_gate.clone(); + let stopped = self.stopped.clone(); + std::thread::spawn(move || { + while !gate.load(Ordering::Relaxed) { + if stopped.load(Ordering::Relaxed) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + let _ = events.send(RecorderEvent::Started { session_id: id, dimensions: None, - }) - .unwrap(); + }); + }); Ok(session) } @@ -173,6 +188,7 @@ impl RecorderEngine for FakeEngine { } fn stop(&mut self) -> RecorderResult<()> { + self.stopped.store(true, Ordering::Relaxed); let session = self .active .take() diff --git a/crates/linux/Cargo.toml b/crates/linux/Cargo.toml new file mode 100644 index 0000000..16b35f4 --- /dev/null +++ b/crates/linux/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "linux" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[target.'cfg(target_os = "linux")'.dependencies] +domain = { path = "../domain" } +ashpd = { version = "0.9", default-features = false, features = ["tokio"] } +tokio = { version = "1", features = ["rt", "time", "sync", "macros"] } +gstreamer = { version = "0.23", features = ["v1_22"] } +gstreamer-allocators = { version = "0.23", features = ["v1_22"] } +futures-util = { version = "0.3", default-features = false, features = ["std", "async-await"] } + +[target.'cfg(target_os = "linux")'.dev-dependencies] +serde_json = "1" +zbus = { version = "4.4", default-features = false, features = ["tokio"] } diff --git a/crates/linux/src/lib.rs b/crates/linux/src/lib.rs new file mode 100644 index 0000000..59d9b85 --- /dev/null +++ b/crates/linux/src/lib.rs @@ -0,0 +1,201 @@ +#![cfg(target_os = "linux")] + +mod pipeline; +mod portal; + +use domain::{ + CaptureTarget, RecorderEngine, RecorderError, RecorderEvent, RecorderSettings, + RecordingSession, Result, +}; +use futures_util::StreamExt; +use std::{ + sync::{ + atomic::{AtomicU64, Ordering}, + mpsc, + }, + thread, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tokio::sync::watch; + +pub use portal::{check_desktop, list_targets}; + +pub(crate) fn backend(error: impl std::fmt::Display) -> RecorderError { + RecorderError::Backend(error.to_string()) +} + +pub(crate) enum Command { + Pause(mpsc::SyncSender>), + Resume(mpsc::SyncSender>), +} + +struct Active { + commands: mpsc::SyncSender, + stop: watch::Sender, + worker: thread::JoinHandle<()>, +} + +/// Rust owns lifecycle and buffer handles. GStreamer owns the native pixel path. +pub struct LinuxRecorder { + events: mpsc::Sender, + active: Option, + stop_requested: bool, +} + +impl LinuxRecorder { + pub fn new(events: mpsc::Sender) -> Self { + Self { + events, + active: None, + stop_requested: false, + } + } + + fn control(&self, command: impl FnOnce(mpsc::SyncSender>) -> Command) -> Result<()> { + let active = self + .active + .as_ref() + .ok_or_else(|| backend("no active recording"))?; + let (tx, rx) = mpsc::sync_channel(1); + active.commands.try_send(command(tx)).map_err(backend)?; + rx.recv_timeout(Duration::from_secs(5)).map_err(backend)? + } +} + +impl RecorderEngine for LinuxRecorder { + fn list_targets(&self) -> Result> { + list_targets() + } + + fn start( + &mut self, + target: CaptureTarget, + settings: RecorderSettings, + ) -> Result { + if self.active.is_none() && self.stop_requested { + return Err(backend("stopped before recording startup")); + } + if let Some(active) = &self.active { + if !active.worker.is_finished() { + return Err(backend("recording is already active")); + } + } + if let Some(active) = self.active.take() { + let _ = active.worker.join(); + } + self.stop_requested = false; + check_desktop()?; + portal::validate_target(&target)?; + pipeline::check_plugins(&settings)?; + std::fs::create_dir_all(&settings.output_dir).map_err(backend)?; + static LAST_ID: AtomicU64 = AtomicU64::new(0); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(backend)? + .as_micros() as u64; + let previous = LAST_ID + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |last| { + Some(now.max(last.saturating_add(1))) + }) + .unwrap(); + let id = now.max(previous.saturating_add(1)); + let session = RecordingSession { + id, + output_path: settings.output_dir.join(format!("wrec-{id}.mov")), + }; + let events = self.events.clone(); + let completion_events = events.clone(); + let worker_session = session.clone(); + let (commands, receiver) = mpsc::sync_channel(1); + let (stop, mut stopped) = watch::channel(false); + let worker_stop = stop.clone(); + let worker = thread::Builder::new() + .name("wrec-linux-capture".into()) + .spawn(move || { + let _ = events.send(RecorderEvent::Starting { + session_id: id, + target: target.clone(), + settings: settings.clone(), + output_path: worker_session.output_path.clone(), + }); + let result = (|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(backend)?; + runtime.block_on(async { + let capture = + portal::open(&target, settings.include_cursor, &mut stopped).await?; + let result = async { + let mut closed = + capture.session.receive_closed().await.map_err(backend)?; + let stream = portal::PipeWireStream { + fd: capture.stream.fd.try_clone().map_err(backend)?, + node: capture.stream.node, + }; + let mut recording = tokio::task::spawn_blocking(move || { + pipeline::record( + &stream, + &worker_session, + &settings, + &events, + receiver, + &stopped, + ) + }); + tokio::select! { + result = &mut recording => result.map_err(backend)?, + _ = closed.next() => { + let _ = worker_stop.send(true); + recording.await.map_err(backend)? + } + } + } + .await; + // Release the remote only after all native elements have stopped using its fd. + capture.close().await; + result + }) + })(); + let (success, status) = match result { + Ok(()) => (true, "recording finalized".to_string()), + Err(error) => (false, error.to_string()), + }; + let _ = completion_events.send(RecorderEvent::Exited { + session_id: id, + success, + status, + }); + }) + .map_err(backend)?; + self.active = Some(Active { + commands, + stop, + worker, + }); + Ok(session) + } + + fn pause(&mut self) -> Result<()> { + self.control(Command::Pause) + } + fn resume(&mut self) -> Result<()> { + self.control(Command::Resume) + } + fn stop(&mut self) -> Result<()> { + self.stop_requested = true; + if let Some(active) = &self.active { + let _ = active.stop.send(true); + } + Ok(()) + } +} + +impl Drop for LinuxRecorder { + fn drop(&mut self) { + if let Some(active) = self.active.take() { + let _ = active.stop.send(true); + let _ = active.worker.join(); + } + } +} diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs new file mode 100644 index 0000000..fdd55dc --- /dev/null +++ b/crates/linux/src/pipeline.rs @@ -0,0 +1,706 @@ +use crate::{backend, portal::PipeWireStream, Command}; +use domain::{ + CaptureDimensions, Codec, Quality, RecorderEvent, RecorderMetrics, RecorderSettings, + RecordingSession, Resolution, Result, +}; +use gstreamer::{self as gst, prelude::*}; +use std::{ + os::fd::AsRawFd, + path::Path, + sync::{ + atomic::{AtomicU64, Ordering}, + mpsc, Arc, Mutex, + }, + time::{Duration, Instant}, +}; +use tokio::sync::watch; + +fn element(factory: &str) -> Result { + gst::ElementFactory::make(factory).build().map_err(|error| backend(format!("GStreamer element {factory} is unavailable: {error}. Install the Linux runtime dependencies in packaging/linux/README.md."))) +} + +fn encoder_name(codec: Codec) -> Result<&'static str> { + let names = match codec { + Codec::H264 => ["vah264lpenc", "vah264enc"], + Codec::Hevc => ["vah265lpenc", "vah265enc"], + }; + names.into_iter().find(|name| gst::ElementFactory::find(name).is_some()) + .ok_or_else(|| backend(format!("No VA-API {} hardware encoder is available. Install GStreamer's va plugin and an Intel/AMD encoding driver, and check access to /dev/dri/renderD*. NVIDIA/NVENC and software video encoding are not supported by this backend.", codec.as_arg()))) +} + +pub(crate) fn check_plugins(settings: &RecorderSettings) -> Result<()> { + gst::init().map_err(backend)?; + encoder_name(settings.codec)?; + for factory in [ + "pipewiresrc", + "vapostproc", + "capsfilter", + "queue", + parser_name(settings.codec), + "qtmux", + "filesink", + ] { + element(factory)?; + } + if settings.include_system_audio || settings.include_microphone { + for factory in [ + "pulsesrc", + "audioconvert", + "audioresample", + "avenc_aac", + "aacparse", + ] { + element(factory)?; + } + } + Ok(()) +} + +fn parser_name(codec: Codec) -> &'static str { + match codec { + Codec::H264 => "h264parse", + Codec::Hevc => "h265parse", + } +} + +/// Fit inside the requested bounds without upscaling or odd chroma dimensions. +fn output_size(width: i32, height: i32, resolution: Resolution) -> (i32, i32) { + let (max_width, max_height) = match resolution { + Resolution::Native => (width, height), + Resolution::R720p => (1280, 720), + Resolution::R1080p => (1920, 1080), + Resolution::R2k => (2560, 1440), + Resolution::R4k => (3840, 2160), + }; + let scale = (max_width as f64 / width as f64) + .min(max_height as f64 / height as f64) + .min(1.0); + let even = |size: i32| (((size as f64 * scale) as i32) & !1).max(2); + (even(width), even(height)) +} + +fn video_caps(size: Option<(i32, i32)>) -> gst::Caps { + let builder = gst::Caps::builder("video/x-raw") + .features(["memory:VAMemory"]) + .field("format", "NV12") + .field("pixel-aspect-ratio", gst::Fraction::new(1, 1)); + match size { + Some((w, h)) => builder.field("width", w).field("height", h).build(), + None => builder.build(), + } +} + +fn video_queue() -> Result { + let queue = element("queue")?; + queue.set_property("max-size-buffers", 2u32); + queue.set_property("max-size-bytes", 0u32); + queue.set_property("max-size-time", 0u64); + queue.set_property_from_str("leaky", "downstream"); + Ok(queue) +} + +struct PipelineGuard(gst::Pipeline); +impl Drop for PipelineGuard { + fn drop(&mut self) { + let _ = self.0.set_state(gst::State::Null); + } +} + +#[derive(Default)] +struct Counters { + frames: AtomicU64, + dropped: AtomicU64, + dimensions: Mutex>, +} + +pub(crate) fn record( + capture: &PipeWireStream, + session: &RecordingSession, + settings: &RecorderSettings, + events: &mpsc::Sender, + commands: mpsc::Receiver, + stop: &watch::Receiver, +) -> Result<()> { + if *stop.borrow() { + return Err(backend("stopped before recording started")); + } + let settings = settings.clone().with_preset_limits(); + let pipeline = PipelineGuard(gst::Pipeline::new()); + pipeline.0.use_clock(Some(&gst::SystemClock::obtain())); + let source = element("pipewiresrc")?; + source.set_property("fd", capture.fd.as_raw_fd()); + // The portal returns a node ID, whereas target-object expects an object serial. + source.set_property("path", capture.node.to_string()); + source.set_property("always-copy", false); + source.set_property("min-buffers", 4i32); + source.set_property("max-buffers", 8i32); + source.set_property("keepalive-time", 1000i32); + source.set_property("resend-last", true); + source.set_property("do-timestamp", true); + let input = element("capsfilter")?; + input.set_property( + "caps", + gst::Caps::builder("video/x-raw") + .features(["memory:DMABuf"]) + .field( + "framerate", + gst::Fraction::new(settings.fps.as_u32() as i32, 1), + ) + .build(), + ); + let queue = video_queue()?; + let converter = element("vapostproc")?; + converter.set_property("add-borders", true); + let output = element("capsfilter")?; + output.set_property("caps", video_caps(None)); + let encoder_factory = encoder_name(settings.codec)?; + let encoder = element(encoder_factory)?; + encoder.set_property_from_str("rate-control", "cqp"); + let qp = match settings.quality { + Quality::Efficient => 30u32, + Quality::Balanced => 25u32, + Quality::High => 20u32, + }; + for property in ["qpi", "qpp", "qpb"] { + encoder.set_property(property, qp); + } + encoder.set_property("key-int-max", settings.fps.as_u32() * 2); + encoder.set_property("b-frames", 0u32); + let parser = element(parser_name(settings.codec))?; + let mux = element("qtmux")?; + mux.set_property("fragment-duration", 10000u32); + let sink = element("filesink")?; + sink.set_property( + "location", + session + .output_path + .to_str() + .ok_or_else(|| backend("recording output path must be UTF-8"))?, + ); + sink.set_property("sync", false); + let chain = [ + &source, &input, &queue, &converter, &output, &encoder, &parser, &mux, &sink, + ]; + pipeline.0.add_many(chain).map_err(backend)?; + gst::Element::link_many(chain).map_err(|error| backend(format!("Cannot link the DMA-BUF/VA-API pipeline: {error}. Use GStreamer 1.24 or newer with DMA-BUF support in vapostproc.")))?; + if settings.include_system_audio { + add_audio(&pipeline.0, &mux, Some("@DEFAULT_MONITOR@"))?; + } + if settings.include_microphone { + add_audio(&pipeline.0, &mux, None)?; + } + let counters = Arc::new(Counters::default()); + attach_capture_probe(&source, &output, settings.resolution, counters.clone()); + attach_encoded_probe(&parser, counters.clone()); + // One overrun accompanies each incoming frame that replaces the oldest frame. + let dropped = counters.clone(); + queue.connect("overrun", false, move |_| { + dropped.dropped.fetch_add(1, Ordering::Relaxed); + None + }); + let _ = events.send(RecorderEvent::Log { session_id: Some(session.id), message: format!("capture-engine: PipeWire DMA-BUF → vapostproc → {encoder_factory}; 8 capture buffers, 2 queued frames, no CPU video fallback") }); + // Reserve a unique file so a collision never truncates an existing recording. + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&session.output_path) + .map_err(backend)?; + let result = run(&pipeline.0, session, events, commands, stop, &counters); + let _ = pipeline.0.set_state(gst::State::Null); + if counters.frames.load(Ordering::Relaxed) == 0 { + let _ = std::fs::remove_file(&session.output_path); + } + result +} + +fn attach_encoded_probe(parser: &gst::Element, counters: Arc) { + parser + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, _| { + counters.frames.fetch_add(1, Ordering::Relaxed); + gst::PadProbeReturn::Ok + }); +} + +fn add_audio(pipeline: &gst::Pipeline, mux: &gst::Element, device: Option<&str>) -> Result<()> { + let source = element("pulsesrc")?; + source.set_property("provide-clock", false); + if let Some(device) = device { + source.set_property("device", device); + } + add_audio_source(pipeline, mux, &source) +} + +fn add_audio_source( + pipeline: &gst::Pipeline, + mux: &gst::Element, + source: &gst::Element, +) -> Result<()> { + let convert = element("audioconvert")?; + let resample = element("audioresample")?; + let caps = element("capsfilter")?; + caps.set_property( + "caps", + gst::Caps::builder("audio/x-raw") + .field("rate", 48000i32) + .field("channels", 2i32) + .build(), + ); + let encoder = element("avenc_aac")?; + encoder.set_property("bitrate", 128000i32); + let parser = element("aacparse")?; + let queue = element("queue")?; + queue.set_property("max-size-time", 2_000_000_000u64); + queue.set_property("max-size-buffers", 0u32); + queue.set_property("max-size-bytes", 0u32); + let chain = [ + source, &convert, &resample, &caps, &encoder, &parser, &queue, + ]; + pipeline.add_many(chain).map_err(backend)?; + gst::Element::link_many(chain).map_err(backend)?; + queue.link(mux).map_err(backend)?; + Ok(()) +} + +fn attach_capture_probe( + source: &gst::Element, + output: &gst::Element, + resolution: Resolution, + counters: Arc, +) { + let output = output.clone(); + source.static_pad("src").unwrap().add_probe(gst::PadProbeType::EVENT_DOWNSTREAM | gst::PadProbeType::BUFFER, move |pad, info| { + if let Some(gst::PadProbeData::Event(event)) = &info.data { + if let gst::EventView::Caps(caps) = event.view() { + if let Some(structure) = caps.caps().structure(0) { + if let (Ok(width), Ok(height)) = (structure.get::("width"), structure.get::("height")) { + if width >= 2 && height >= 2 { + let mut dimensions = counters.dimensions.lock().unwrap(); + // A MOV track has a fixed canvas. Window resizes are + // scaled/letterboxed by VA into the initial output size. + let (w, h) = dimensions.map(|d| (d.output_width as i32, d.output_height as i32)) + .unwrap_or_else(|| output_size(width, height, resolution)); + dimensions.get_or_insert(CaptureDimensions { native_width: width.into(), native_height: height.into(), output_width: w.into(), output_height: h.into() }); + drop(dimensions); + output.set_property("caps", video_caps(Some((w, h)))); + } + } + } + } + } + if let Some(gst::PadProbeData::Buffer(buffer)) = &info.data { + if buffer.n_memory() == 0 || buffer.iter_memories().any(|memory| !memory.is_memory_type::()) { + if let Some(element) = pad.parent_element() { + gst::element_error!(element, gst::StreamError::Format, ("PipeWire delivered a non-DMA-BUF frame. This backend requires GPU buffer sharing; check the compositor and graphics driver.")); + } + return gst::PadProbeReturn::Drop; + } + } + gst::PadProbeReturn::Ok + }); +} + +fn set_state(pipeline: &gst::Pipeline, state: gst::State) -> Result<()> { + pipeline.set_state(state).map_err(backend)?; + let (result, current, _) = pipeline.state(gst::ClockTime::from_seconds(3)); + result.map_err(backend)?; + if current != state { + return Err(backend(format!( + "recording did not enter {state:?} within 3s" + ))); + } + Ok(()) +} + +fn run( + pipeline: &gst::Pipeline, + session: &RecordingSession, + events: &mpsc::Sender, + commands: mpsc::Receiver, + stop: &watch::Receiver, + counters: &Counters, +) -> Result<()> { + pipeline.set_state(gst::State::Playing).map_err(backend)?; + let bus = pipeline + .bus() + .ok_or_else(|| backend("recording pipeline has no bus"))?; + let launched = Instant::now(); + let mut started = false; + let mut stopping = None; + let mut last_metrics = Instant::now(); + loop { + if *stop.borrow() && stopping.is_none() { + if !started { + return Err(backend("stopped before the first encoded frame")); + } + // Live sources must run to drain their pending buffers and finalize a paused movie. + pipeline.set_state(gst::State::Playing).map_err(backend)?; + if !pipeline.send_event(gst::event::Eos::new()) { + return Err(backend("recording pipeline rejected finalization")); + } + stopping = Some(Instant::now()); + } + if let Ok(command) = commands.try_recv() { + let (state, reply) = match command { + Command::Pause(reply) => (gst::State::Paused, reply), + Command::Resume(reply) => (gst::State::Playing, reply), + }; + let result = if stopping.is_some() || !started { + Err(backend("recording is not ready for pause/resume")) + } else { + set_state(pipeline, state) + }; + let _ = reply.send(result); + } + if let Some(message) = bus.timed_pop(gst::ClockTime::from_mseconds(100)) { + match message.view() { + gst::MessageView::Error(error) => { + return Err(backend(format!( + "{}: {} ({})", + error + .src() + .map(|src| src.name().to_string()) + .unwrap_or_default(), + error.error(), + error.debug().unwrap_or_default() + ))) + } + gst::MessageView::Eos(_) => { + if counters.frames.load(Ordering::Relaxed) == 0 { + return Err(backend("capture ended without an encoded frame")); + } + emit_metrics(pipeline, session, events, counters); + return Ok(()); + } + _ => {} + } + } + if !started && counters.frames.load(Ordering::Relaxed) > 0 { + started = true; + let _ = events.send(RecorderEvent::Started { + session_id: session.id, + dimensions: *counters.dimensions.lock().unwrap(), + }); + } + if !started && launched.elapsed() > Duration::from_secs(15) { + return Err(backend("No encoded frame arrived within 15s. Check DMA-BUF support, VA-API driver access, and audio devices.")); + } + if stopping.is_some_and(|at| at.elapsed() > Duration::from_secs(10)) { + return Err(backend( + "Movie finalization timed out after 10s; only completed fragments may be playable.", + )); + } + if started && last_metrics.elapsed() >= Duration::from_secs(1) { + emit_metrics(pipeline, session, events, counters); + last_metrics = Instant::now(); + } + } +} + +fn emit_metrics( + pipeline: &gst::Pipeline, + session: &RecordingSession, + events: &mpsc::Sender, + counters: &Counters, +) { + let elapsed_secs = pipeline + .query_position::() + .map(|time| time.seconds()) + .unwrap_or(0); + let output_bytes = file_size(&session.output_path); + let _ = events.send(RecorderEvent::Metrics { + session_id: session.id, + metrics: RecorderMetrics { + elapsed_secs, + output_bytes, + estimated_bitrate_mbps: if elapsed_secs > 0 { + output_bytes as f32 * 8.0 / elapsed_secs as f32 / 1_000_000.0 + } else { + 0.0 + }, + frames: Some(counters.frames.load(Ordering::Relaxed)), + dropped_frames: Some(counters.dropped.load(Ordering::Relaxed)), + }, + }); +} + +fn file_size(path: &Path) -> u64 { + std::fs::metadata(path) + .map(|metadata| metadata.len()) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dimensions_preserve_aspect_and_do_not_upscale() { + assert_eq!(output_size(3840, 2160, Resolution::R1080p), (1920, 1080)); + assert_eq!(output_size(1080, 1920, Resolution::R1080p), (606, 1080)); + assert_eq!(output_size(800, 600, Resolution::R1080p), (800, 600)); + assert_eq!(output_size(1919, 1079, Resolution::Native), (1918, 1078)); + } + + struct TestRecording { + pipeline: gst::Pipeline, + session: RecordingSession, + events: mpsc::Receiver, + commands: mpsc::SyncSender, + stop: watch::Sender, + worker: Option>>, + } + + impl TestRecording { + fn start(audio_tracks: usize) -> Self { + gst::init().unwrap(); + static ID: AtomicU64 = AtomicU64::new(0); + let id = ID.fetch_add(1, Ordering::Relaxed); + let session = RecordingSession { + id, + output_path: std::env::temp_dir() + .join(format!("wrec-linux-test-{}-{id}.mov", std::process::id())), + }; + // Synthetic software encoding is confined to tests. Exercise the same + // bus/control/mux code without pretending this is a hardware benchmark. + let pipeline = gst::parse::launch("videotestsrc is-live=true pattern=ball ! video/x-raw,width=320,height=180,framerate=30/1 ! openh264enc ! h264parse name=parser ! qtmux name=mux fragment-duration=10000 ! filesink name=output sync=false") + .unwrap().downcast::().unwrap(); + pipeline + .by_name("output") + .unwrap() + .set_property("location", session.output_path.to_str().unwrap()); + for _ in 0..audio_tracks { + let source = element("audiotestsrc").unwrap(); + source.set_property("is-live", true); + add_audio_source(&pipeline, &pipeline.by_name("mux").unwrap(), &source).unwrap(); + } + let counters = Arc::new(Counters::default()); + attach_encoded_probe(&pipeline.by_name("parser").unwrap(), counters.clone()); + let (events_tx, events) = mpsc::channel(); + let (commands, commands_rx) = mpsc::sync_channel(1); + let (stop, stopped) = watch::channel(false); + let worker_pipeline = pipeline.clone(); + let worker_session = session.clone(); + let worker = std::thread::spawn(move || { + let guard = PipelineGuard(worker_pipeline); + run( + &guard.0, + &worker_session, + &events_tx, + commands_rx, + &stopped, + &counters, + ) + }); + let recording = Self { + pipeline, + session, + events, + commands, + stop, + worker: Some(worker), + }; + loop { + if matches!( + recording + .events + .recv_timeout(Duration::from_secs(10)) + .unwrap(), + RecorderEvent::Started { .. } + ) { + break; + } + } + recording + } + + fn control(&self, pause: bool) { + let (tx, rx) = mpsc::sync_channel(1); + self.commands + .send(if pause { + Command::Pause(tx) + } else { + Command::Resume(tx) + }) + .unwrap(); + rx.recv_timeout(Duration::from_secs(5)).unwrap().unwrap(); + } + + fn finish(&mut self) -> Result<()> { + self.stop.send(true).unwrap(); + self.worker.take().unwrap().join().unwrap() + } + + fn probe(&self) -> serde_json::Value { + let output = std::process::Command::new("ffprobe") + .args([ + "-v", + "error", + "-show_streams", + "-show_format", + "-show_packets", + "-of", + "json", + ]) + .arg(&self.session.output_path) + .output() + .expect("install ffmpeg to run Linux recording tests"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let probe: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let mut previous = std::collections::HashMap::new(); + for packet in probe["packets"].as_array().unwrap() { + let stream = packet["stream_index"].as_u64().unwrap(); + let dts = packet["dts"].as_i64().unwrap(); + if let Some(last) = previous.insert(stream, dts) { + assert!( + dts > last, + "stream {stream} has non-increasing DTS: {last} -> {dts}" + ); + } + } + let decoded = std::process::Command::new("ffmpeg") + .args(["-v", "error", "-i"]) + .arg(&self.session.output_path) + // Preserve the movie's timestamp precision when decoding VFR + // into null output; rounding to 1/30 can create duplicate DTS. + .args([ + "-map", + "0", + "-vsync", + "0", + "-enc_time_base", + "-1", + "-f", + "null", + "-", + ]) + .output() + .unwrap(); + assert!( + decoded.status.success() && decoded.stderr.is_empty(), + "{}", + String::from_utf8_lossy(&decoded.stderr) + ); + probe + } + } + + impl Drop for TestRecording { + fn drop(&mut self) { + let _ = self.stop.send(true); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + let _ = std::fs::remove_file(&self.session.output_path); + } + } + + #[test] + fn finalizes_playable_video_and_two_audio_tracks() { + let mut recording = TestRecording::start(2); + std::thread::sleep(Duration::from_millis(600)); + recording.finish().unwrap(); + let probe = recording.probe(); + let streams = probe["streams"].as_array().unwrap(); + assert_eq!( + streams.iter().filter(|s| s["codec_name"] == "h264").count(), + 1 + ); + assert_eq!( + streams.iter().filter(|s| s["codec_name"] == "aac").count(), + 2 + ); + assert!( + probe["format"]["duration"] + .as_str() + .unwrap() + .parse::() + .unwrap() + > 0.4 + ); + } + + #[test] + fn pause_resume_omits_paused_time_and_stop_while_paused_finalizes() { + let mut recording = TestRecording::start(0); + std::thread::sleep(Duration::from_millis(400)); + recording.control(true); + let before = recording + .pipeline + .query_position::() + .unwrap(); + std::thread::sleep(Duration::from_millis(600)); + let after = recording + .pipeline + .query_position::() + .unwrap(); + assert!(after.saturating_sub(before) < gst::ClockTime::from_mseconds(100)); + recording.control(false); + std::thread::sleep(Duration::from_millis(400)); + recording.control(true); + let active_duration = recording + .pipeline + .query_position::() + .unwrap() + .nseconds() as f64 + / 1_000_000_000.0; + recording.finish().unwrap(); + let probe = recording.probe(); + let duration = probe["format"]["duration"] + .as_str() + .unwrap() + .parse::() + .unwrap(); + assert!( + (duration - active_duration).abs() < 0.2, + "movie duration {duration}, active timeline {active_duration}" + ); + } + + #[test] + fn pipeline_errors_fail_the_recording() { + let mut recording = TestRecording::start(0); + gst::element_error!( + recording.pipeline, + gst::StreamError::Failed, + ("injected capture failure") + ); + let error = recording + .worker + .take() + .unwrap() + .join() + .unwrap() + .unwrap_err(); + assert!(error.to_string().contains("injected capture failure")); + } + + #[test] + fn rejects_cpu_buffers_before_pixel_processing() { + gst::init().unwrap(); + let pipeline = PipelineGuard(gst::parse::launch("videotestsrc num-buffers=1 name=source ! video/x-raw,width=320,height=180 ! fakesink").unwrap().downcast::().unwrap()); + let output = element("capsfilter").unwrap(); + attach_capture_probe( + &pipeline.0.by_name("source").unwrap(), + &output, + Resolution::Native, + Arc::new(Counters::default()), + ); + pipeline.0.set_state(gst::State::Playing).unwrap(); + let message = pipeline + .0 + .bus() + .unwrap() + .timed_pop_filtered(gst::ClockTime::from_seconds(5), &[gst::MessageType::Error]) + .unwrap(); + let gst::MessageView::Error(error) = message.view() else { + panic!("expected error") + }; + assert!(error.error().to_string().contains("non-DMA-BUF")); + } +} diff --git a/crates/linux/src/portal.rs b/crates/linux/src/portal.rs new file mode 100644 index 0000000..3f086a3 --- /dev/null +++ b/crates/linux/src/portal.rs @@ -0,0 +1,175 @@ +use crate::backend; +use ashpd::{ + desktop::{ + screencast::{CursorMode, Screencast, SourceType}, + PersistMode, Session, + }, + WindowIdentifier, +}; +use domain::{CaptureSourceKind, CaptureTarget, Result}; +use std::{os::fd::OwnedFd, time::Duration}; +use tokio::sync::watch; + +#[cfg(test)] +#[path = "portal_tests.rs"] +mod tests; + +pub fn check_desktop() -> Result<()> { + if std::env::var_os("WAYLAND_DISPLAY").is_none() + || std::env::var_os("XDG_RUNTIME_DIR").is_none() + { + return Err(backend("Linux recording requires a Wayland desktop session. X11 and headless capture are not supported.")); + } + Ok(()) +} + +fn runtime() -> Result { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(backend) +} + +pub fn list_targets() -> Result> { + check_desktop()?; + runtime()?.block_on(async { + tokio::time::timeout(Duration::from_secs(5), async { + let portal = Screencast::new().await.map_err(backend)?; + let types = portal.available_source_types().await.map_err(backend)?; + let mut targets = Vec::new(); + for (source, kind, name) in [ + ( + SourceType::Monitor, + CaptureSourceKind::Display, + "Choose a display in the desktop picker", + ), + ( + SourceType::Window, + CaptureSourceKind::Window, + "Choose a window in the desktop picker", + ), + ] { + if types.contains(source) { + targets.push(CaptureTarget { + id: 0, + kind, + name: name.into(), + }); + } + } + Ok(targets) + }) + .await + .map_err(|_| backend("ScreenCast portal did not respond within 5s"))? + }) +} + +pub(crate) fn validate_target(target: &CaptureTarget) -> Result<()> { + if target.id != 0 { + return Err(backend("Wayland targets use display:0 or window:0 to open the desktop picker; application names and native window IDs are unavailable.")); + } + Ok(()) +} + +pub(crate) struct Capture { + pub stream: PipeWireStream, + pub session: Session<'static, Screencast<'static>>, + // Keep the portal connection alive until the pipeline and session close. + _portal: Screencast<'static>, +} + +pub(crate) struct PipeWireStream { + pub fd: OwnedFd, + pub node: u32, +} + +impl Capture { + pub async fn close(self) { + let _ = tokio::time::timeout(Duration::from_secs(3), self.session.close()).await; + } +} + +pub(crate) async fn open( + target: &CaptureTarget, + cursor: bool, + stop: &mut watch::Receiver, +) -> Result { + if *stop.borrow() { + return Err(backend("stopped before source selection")); + } + let portal = tokio::time::timeout(Duration::from_secs(5), Screencast::new()) + .await + .map_err(|_| backend("ScreenCast portal connection timed out"))? + .map_err(backend)?; + let session = tokio::time::timeout(Duration::from_secs(5), portal.create_session()) + .await + .map_err(|_| backend("ScreenCast session creation timed out"))? + .map_err(backend)?; + let selected = async { + let source = match target.kind { + CaptureSourceKind::Display => SourceType::Monitor, + CaptureSourceKind::Window => SourceType::Window, + }; + let cursor = if cursor { + CursorMode::Embedded + } else { + CursorMode::Hidden + }; + if !portal + .available_cursor_modes() + .await + .map_err(backend)? + .contains(cursor) + { + return Err(backend( + "The desktop portal cannot provide the requested cursor mode.", + )); + } + portal + .select_sources( + &session, + cursor, + source.into(), + false, + None, + PersistMode::DoNot, + ) + .await + .map_err(backend)? + .response() + .map_err(backend)?; + let response = portal + .start(&session, &WindowIdentifier::default()) + .await + .map_err(backend)? + .response() + .map_err(backend)?; + if response.streams().len() != 1 { + return Err(backend( + "The desktop portal must return exactly one capture stream.", + )); + } + let node = response.streams()[0].pipe_wire_node_id(); + let fd = portal + .open_pipe_wire_remote(&session) + .await + .map_err(backend)?; + Ok((fd, node)) + }; + let result = tokio::select! { + result = selected => result, + _ = stop.changed() => Err(backend("stopped during source selection")), + _ = tokio::time::sleep(Duration::from_secs(120)) => Err(backend("Desktop source selection timed out after 120s; start again and choose a source.")), + }; + match result { + Ok((fd, node)) => Ok(Capture { + stream: PipeWireStream { fd, node }, + session, + _portal: portal, + }), + Err(error) => { + let _ = tokio::time::timeout(Duration::from_secs(3), session.close()).await; + Err(error) + } + } +} diff --git a/crates/linux/src/portal_tests.rs b/crates/linux/src/portal_tests.rs new file mode 100644 index 0000000..bb554f6 --- /dev/null +++ b/crates/linux/src/portal_tests.rs @@ -0,0 +1,219 @@ +use super::*; +use std::{ + collections::HashMap, + os::unix::net::UnixStream, + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, +}; +use zbus::{ + message::Header, + zvariant::{OwnedObjectPath, OwnedValue, Value}, + Connection, +}; + +#[derive(Clone, Default)] +struct MockPortal { + mode: Arc, + closes: Arc, + selections: Arc>>, +} + +struct MockSession(Arc); + +#[zbus::interface(name = "org.freedesktop.portal.Session")] +impl MockSession { + fn close(&self) { + self.0.fetch_add(1, Ordering::Relaxed); + } +} + +fn path( + header: &Header<'_>, + options: &HashMap, + session: bool, +) -> OwnedObjectPath { + let sender = header + .sender() + .unwrap() + .as_str() + .trim_start_matches(':') + .replace('.', "_"); + let (kind, key) = if session { + ("session", "session_handle_token") + } else { + ("request", "handle_token") + }; + let token = <&str>::try_from(options.get(key).unwrap()).unwrap(); + format!("/org/freedesktop/portal/desktop/{kind}/{sender}/{token}") + .try_into() + .unwrap() +} + +async fn response( + connection: &Connection, + request: &OwnedObjectPath, + code: u32, + data: HashMap<&str, Value<'_>>, +) { + connection + .emit_signal( + None::<&str>, + request, + "org.freedesktop.portal.Request", + "Response", + &(code, data), + ) + .await + .unwrap(); +} + +#[zbus::interface(name = "org.freedesktop.portal.ScreenCast")] +impl MockPortal { + #[zbus(property, name = "version")] + fn version(&self) -> u32 { + 5 + } + #[zbus(property)] + fn available_source_types(&self) -> u32 { + 3 + } + #[zbus(property)] + fn available_cursor_modes(&self) -> u32 { + 3 + } + + async fn create_session( + &self, + options: HashMap, + #[zbus(header)] header: Header<'_>, + #[zbus(connection)] connection: &Connection, + ) -> OwnedObjectPath { + let session = path(&header, &options, true); + let request = path(&header, &options, false); + connection + .object_server() + .at(session.clone(), MockSession(self.closes.clone())) + .await + .unwrap(); + response( + connection, + &request, + 0, + HashMap::from([("session_handle", Value::from(session.as_str()))]), + ) + .await; + request + } + + async fn select_sources( + &self, + _session: OwnedObjectPath, + options: HashMap, + #[zbus(header)] header: Header<'_>, + #[zbus(connection)] connection: &Connection, + ) -> OwnedObjectPath { + self.selections.lock().unwrap().push(( + u32::try_from(&options["types"]).unwrap(), + u32::try_from(&options["cursor_mode"]).unwrap(), + bool::try_from(&options["multiple"]).unwrap(), + )); + let request = path(&header, &options, false); + response(connection, &request, 0, HashMap::new()).await; + request + } + + async fn start( + &self, + _session: OwnedObjectPath, + _parent: String, + options: HashMap, + #[zbus(header)] header: Header<'_>, + #[zbus(connection)] connection: &Connection, + ) -> OwnedObjectPath { + let request = path(&header, &options, false); + match self.mode.load(Ordering::Relaxed) { + 0 => { + let streams = vec![(42u32, HashMap::<&str, Value<'_>>::new())]; + response( + connection, + &request, + 0, + HashMap::from([("streams", Value::from(streams))]), + ) + .await; + } + 1 => response(connection, &request, 1, HashMap::new()).await, + _ => {} // Leave the picker pending until wrec cancels it. + } + request + } + + fn open_pipe_wire_remote( + &self, + _session: OwnedObjectPath, + _options: HashMap, + ) -> zbus::zvariant::OwnedFd { + let (socket, _peer) = UnixStream::pair().unwrap(); + let fd: OwnedFd = socket.into(); + fd.into() + } +} + +#[test] +#[ignore = "requires an isolated D-Bus: dbus-run-session -- cargo test -p linux portal_roundtrip -- --ignored"] +fn portal_roundtrip_and_cancellation_close_sessions() { + std::env::set_var("WAYLAND_DISPLAY", "wayland-test"); + std::env::set_var("XDG_RUNTIME_DIR", std::env::temp_dir()); + let mock = MockPortal::default(); + runtime().unwrap().block_on(async { + let _server = zbus::connection::Builder::session() + .unwrap() + .name("org.freedesktop.portal.Desktop") + .unwrap() + .serve_at("/org/freedesktop/portal/desktop", mock.clone()) + .unwrap() + .build() + .await + .unwrap(); + let target = CaptureTarget { + id: 0, + name: "picker".into(), + kind: CaptureSourceKind::Display, + }; + let (_stop, mut stopped) = watch::channel(false); + let capture = + tokio::time::timeout(Duration::from_secs(5), open(&target, true, &mut stopped)) + .await + .unwrap() + .unwrap(); + assert_eq!(capture.stream.node, 42); + capture.close().await; + assert_eq!(mock.closes.load(Ordering::Relaxed), 1); + assert_eq!(*mock.selections.lock().unwrap(), vec![(1, 2, false)]); + + mock.mode.store(1, Ordering::Relaxed); + let denied = + tokio::time::timeout(Duration::from_secs(5), open(&target, false, &mut stopped)) + .await + .unwrap(); + assert!(denied.is_err()); + assert_eq!(mock.closes.load(Ordering::Relaxed), 2); + + mock.mode.store(2, Ordering::Relaxed); + let (stop, mut stopped) = watch::channel(false); + let cancelled = open(&target, true, &mut stopped); + let cancel = async { + tokio::time::sleep(Duration::from_millis(200)).await; + stop.send(true).unwrap(); + }; + let (result, ()) = tokio::join!(cancelled, cancel); + assert!(result + .err() + .unwrap() + .to_string() + .contains("stopped during source selection")); + assert_eq!(mock.closes.load(Ordering::Relaxed), 3); + }); +} diff --git a/crates/macos/src/lib.rs b/crates/macos/src/lib.rs index d52bc8e..38f9b54 100644 --- a/crates/macos/src/lib.rs +++ b/crates/macos/src/lib.rs @@ -1,6 +1,8 @@ +#[cfg(target_os = "macos")] +use domain::{CaptureDimensions, CaptureSourceKind, RecorderMetrics}; use domain::{ - CaptureDimensions, CaptureSourceKind, CaptureTarget, PermissionStatus, RecorderEngine, - RecorderError, RecorderEvent, RecorderMetrics, RecorderSettings, RecordingSession, Result, + CaptureTarget, PermissionStatus, RecorderEngine, RecorderError, RecorderEvent, + RecorderSettings, RecordingSession, Result, }; static LAST_SESSION_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); diff --git a/packaging/linux/README.md b/packaging/linux/README.md new file mode 100644 index 0000000..7aa3779 --- /dev/null +++ b/packaging/linux/README.md @@ -0,0 +1,112 @@ +# Linux CLI and daemon + +The Linux backend records a Wayland display or window selected through the +desktop's ScreenCast portal. It requires PipeWire DMA-BUF capture, GStreamer +1.24 or newer, and an Intel or AMD GPU with VA-API encoding support. This is +an initial backend; hardware compatibility and efficiency still need measurements +on real desktops. There is no Linux GUI in this release. + +The pipeline is `pipewiresrc → DMA-BUF → vapostproc → VA encoder → qtmux`. +Rust manages sessions and buffer metadata and never maps video pixels. The +source negotiates four to eight buffers, the video queue holds at most two +frames, and the encoder uses no B-frames. This bounds application queues; +the compositor and driver have their own allocations. Colour conversion and +scaling can still allocate GPU surfaces, so this is not a claim of zero total +copies. The backend rejects CPU frame buffers and has no software video fallback. + +## Install dependencies and build + +Ubuntu 24.04 or newer is a starting point. The desktop must already have a +working PipeWire service and its own `xdg-desktop-portal` backend, such as +`xdg-desktop-portal-gnome` or `xdg-desktop-portal-kde`. Use the backend for your +desktop; do not install an arbitrary mix of portal backends. + +Use the current stable Rust toolchain. + +```bash +sudo apt install build-essential pkg-config libgstreamer1.0-dev \ + libgstreamer-plugins-base1.0-dev gstreamer1.0-tools \ + gstreamer1.0-pipewire gstreamer1.0-plugins-base \ + gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav \ + pipewire-pulse vainfo +cargo build --release -p cli -p daemon +``` + +Install the VA-API driver for your GPU. Distribution codec packaging and GPU +capabilities vary. Inspect `vainfo`, `gst-inspect-1.0 vapostproc`, and +`gst-inspect-1.0 vah264enc` or `vah264lpenc`. HEVC needs `vah265enc` or +`vah265lpenc`. If the `va` plugin lists zero elements, check the driver and +access to `/dev/dri/renderD*` from the same user running wrec. + +Build a relocatable CLI archive with: + +```bash +./scripts/package-cli-linux.sh +``` + +Extract the archive into a prefix such as `~/.local`. It contains `bin/wrec` +and `lib/wrec/daemon`. Put the prefix's `bin` directory on PATH. Native +GStreamer and driver libraries remain system dependencies. Run the CLI in +the logged-in desktop session; it starts the daemon automatically and +inherits the session's Wayland, D-Bus, PipeWire, and audio environment. +After changing sessions, stop and restart the daemon. + +## Record + +```bash +wrec targets --json +wrec record --target display:0 --codec h264 --duration 10s +wrec record --target window:0 --codec h264 --no-system-audio +``` + +`display:0` and `window:0` open the desktop's source picker. They are not +native display or window IDs. Only source types advertised by the portal +appear in `targets`. The duration starts after the first encoded frame, +so time spent choosing a source does not shorten the recording. The picker +times out after two minutes and `wrec job stop ` cancels it. + +Permission status is `unknown` outside a recording because portal grants +belong to individual sessions. `permission.request` does not open a second +picker. Every recording asks for a source; restore tokens, app-name selection, +unattended capture, and arbitrary window enumeration are not implemented. + +System audio records the default output monitor through `pipewire-pulse`. +`--mic` adds the default microphone as a separate AAC track. Audio encoding +runs on the CPU. Use `--no-system-audio` to omit system audio; microphone capture +remains opt-in. Configure devices in the desktop's sound settings. A Linux +wrec window-hiding filter and custom microphone indicator are not provided; +the desktop controls its own screen-sharing indication. + +Pause and resume use the pipeline clock so paused time is omitted. Stop +drains the encoders and finalizes the movie, including when paused. The +writer emits ten-second movie fragments. An interrupted recording may only +be playable through its last completed fragment. A static desktop requests +one keepalive frame per second to keep the movie timeline advancing. + +X11, NVIDIA/NVENC, HDR capture, and desktop-specific capture shortcuts are +outside this first backend. A compositor/driver combination that cannot +share importable DMA-BUFs fails with an error. There is no silent fallback. + +## Validate + +```bash +sudo apt install ffmpeg dbus-daemon +cargo fmt --check +cargo check --workspace --locked +cargo test --workspace --locked +dbus-run-session -- cargo test -p linux portal_roundtrip -- --ignored +``` + +The movie tests use synthetic video and a software encoder only inside the +test module. They check pause timing, stop while paused, real AAC tracks, +file readability, capture errors, and rejection of CPU buffers. The portal +test uses a mock ScreenCast service on an isolated D-Bus and checks source +options, denial, cancellation, and session cleanup. These tests do not prove +that a desktop exports DMA-BUFs or that a driver can import them. + +Before calling a GPU/desktop combination supported, record actual displays +and windows, include a static screen and motion, check cursor on/off, audio +and microphone sync, pause/resume, stop from the desktop sharing control, +and inspect files with `ffprobe`. Measure CPU, RSS, GPU memory, power, and +compositor overhead at 1080p30 and 4K60. Compare against an idle baseline on +the same machine and report the GPU, driver, desktop, and library versions. diff --git a/scripts/package-cli-linux.sh b/scripts/package-cli-linux.sh new file mode 100755 index 0000000..0d0ddca --- /dev/null +++ b/scripts/package-cli-linux.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$(uname -s)" != Linux ]]; then + echo 'Build the Linux CLI package on Linux.' >&2 + exit 1 +fi +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" +cargo build --release --locked -p cli -p daemon +task_target="${CARGO_TARGET_DIR:-$repo_root/target}" +stage="$(mktemp -d)" +trap 'rm -rf "$stage"' EXIT +install -Dm755 "$task_target/release/wrec" "$stage/bin/wrec" +install -Dm755 "$task_target/release/daemon" "$stage/lib/wrec/daemon" +install -Dm644 packaging/linux/README.md "$stage/share/doc/wrec/README.md" +install -Dm644 LICENSE "$stage/share/doc/wrec/LICENSE" +mkdir -p dist/cli/linux +archive="$repo_root/dist/cli/linux/wrec-cli-$(uname -m)-unknown-linux-gnu.tar.gz" +tar -czf "$archive" -C "$stage" bin lib share +echo "$archive" diff --git a/scripts/test-cli-linux.py b/scripts/test-cli-linux.py new file mode 100644 index 0000000..b59c88f --- /dev/null +++ b/scripts/test-cli-linux.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Exercise an extracted Linux package without a desktop, GPU, or user config.""" + +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time + + +def main(): + prefix = Path(sys.argv[1]).resolve() + binary = prefix / "bin" / "wrec" + with tempfile.TemporaryDirectory(prefix="wrec-linux-smoke-") as directory: + env = os.environ.copy() + for key in ("WREC_DAEMON_BIN", "WREC_CAPTURE_ENGINE_PATH", "WREC_HEADLESS", + "WREC_CHANNEL", "WAYLAND_DISPLAY", "XDG_RUNTIME_DIR"): + env.pop(key, None) + env["WREC_HOME"] = str(Path(directory) / "home") + env["WREC_DATA_DIR"] = str(Path(directory) / "data") + + def run(*args, success=True): + result = subprocess.run([str(binary), *args], env=env, text=True, + capture_output=True, timeout=20) + assert (result.returncode == 0) == success, (args, result.stdout, result.stderr) + return result + + pid = None + try: + status = json.loads(run("daemon", "start", "--json").stdout) + assert status["channel"] == "release", status + pid = status["pid"] + executable = Path(f"/proc/{pid}/exe").resolve() + assert executable == prefix / "lib" / "wrec" / "daemon", executable + print("PASS: packaged CLI automatically launches its relative Linux daemon") + + result = run("targets", "--json", success=False) + assert "Wayland desktop session" in result.stdout + result.stderr + print("PASS: headless target discovery returns a useful error") + + result = run("record", "--target", "display:0", "--duration", "1s", "--json", success=False) + assert "Wayland desktop session" in result.stdout + result.stderr + assert not list(Path(directory).rglob("*.mov")) + print("PASS: unsupported capture fails without a recording artifact") + + jobs = json.loads(run("jobs", "--json").stdout) + assert jobs["jobs"] == [], jobs + print("PASS: rejected requests leave no active or queued jobs") + + run("daemon", "stop", "--json") + socket = Path(env["WREC_HOME"]) / "wrec.sock" + for _ in range(100): + if not socket.exists(): + break + time.sleep(0.05) + assert not socket.exists(), "daemon socket survived shutdown" + pid = None + print("PASS: clean daemon shutdown removes the socket") + + status = json.loads(run("daemon", "start", "--json").stdout) + pid = status["pid"] + os.kill(pid, signal.SIGTERM) + for _ in range(100): + if not socket.exists(): + break + time.sleep(0.05) + assert not socket.exists(), "daemon socket survived SIGTERM" + pid = None + print("PASS: restarted daemon handles SIGTERM and removes the socket") + finally: + if pid is not None: + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + + +if __name__ == "__main__": + main() From afbd7153dbb27e9d98093daadbc22dadbe5c4dd5 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 5 Sep 2026 06:02:19 +0000 Subject: [PATCH 02/45] fix(linux): preserve cancellation and bound shutdown waits --- crates/daemon/src/backend.rs | 9 +++++ crates/daemon/src/coordinator.rs | 24 +++++++++++- crates/daemon/src/jobs.rs | 2 +- crates/daemon/src/store.rs | 31 ++++++++++++++++ crates/daemon/src/test_support.rs | 13 +++++-- crates/domain/src/lib.rs | 6 +++ crates/linux/src/lib.rs | 62 ++++++++++++++++++++++++++++++- crates/linux/src/pipeline.rs | 14 ++++--- crates/linux/src/portal.rs | 6 +-- crates/linux/src/portal_tests.rs | 6 +-- packaging/linux/README.md | 3 ++ 11 files changed, 153 insertions(+), 23 deletions(-) diff --git a/crates/daemon/src/backend.rs b/crates/daemon/src/backend.rs index 2be6e20..ef5f5ec 100644 --- a/crates/daemon/src/backend.rs +++ b/crates/daemon/src/backend.rs @@ -39,6 +39,7 @@ pub struct RecordingOverrides { #[derive(Debug, Clone)] pub enum BackendEvent { + Cancelled, Starting { output_path: PathBuf, }, @@ -86,6 +87,14 @@ impl WrecBackend { pub fn handle_recorder_event(&mut self, event: &RecorderEvent) -> BackendEvent { match event { + RecorderEvent::Cancelled { session_id } => { + if let Some(store) = &self.store { + store.mark_recording_cancelled(*session_id, now_ms()); + } + self.active_session_id = None; + self.active_output_path = None; + BackendEvent::Cancelled + } RecorderEvent::Starting { session_id, target, diff --git a/crates/daemon/src/coordinator.rs b/crates/daemon/src/coordinator.rs index fc36109..4282034 100644 --- a/crates/daemon/src/coordinator.rs +++ b/crates/daemon/src/coordinator.rs @@ -10,7 +10,7 @@ use control::{ daemon_log_path, now_ms, socket_path, wrec_home, AgentError, AgentWarning, EventLevel, JobStatus, RecordingOptions, StartRecordingParams, PROTOCOL_VERSION, }; -use domain::{CaptureTarget, RecorderEngine, RecorderEvent}; +use domain::{CaptureTarget, RecorderEngine, RecorderError, RecorderEvent}; use serde_json::{json, Value}; use std::{ collections::{BTreeMap, VecDeque}, @@ -565,6 +565,18 @@ fn run_job( }; if let Err(err) = start_result { drain_recorder_events(&state, job_id, &rx); + if matches!(err, RecorderError::Cancelled) { + if let Ok(mut state) = lock_state(&state) { + if let Some(job) = state.jobs.get_mut(&job_id) { + job.mark_cancelled(); + } + if state.active_job_id == Some(job_id) { + state.active_job_id = None; + } + } + notify_job_changed(); + return; + } finish_job_failed(&state, job_id, format!("recording failed to start: {err}")); return; } @@ -646,6 +658,14 @@ fn handle_recorder_event( } let done = match backend_event { + BackendEvent::Cancelled => { + job.output_path = None; + job.mark_cancelled(); + if active_matches { + state.active_job_id = None; + } + true + } BackendEvent::Starting { output_path, .. } => { job.output_path = Some(output_path.clone()); job.push_event( @@ -972,7 +992,7 @@ mod tests { std::thread::sleep(Duration::from_millis(10)); } Coordinator::job_stop(state.clone(), id).unwrap(); - wait_for_status(&state, id, JobStatus::Completed); + wait_for_status(&state, id, JobStatus::Cancelled); } #[cfg(target_os = "linux")] diff --git a/crates/daemon/src/jobs.rs b/crates/daemon/src/jobs.rs index 660f24b..824e361 100644 --- a/crates/daemon/src/jobs.rs +++ b/crates/daemon/src/jobs.rs @@ -95,7 +95,7 @@ impl JobRecord { self.status = JobStatus::Cancelled; self.finished_at_ms = Some(now_ms()); self.control = None; - self.push_event(EventLevel::Warning, "queued job cancelled"); + self.push_event(EventLevel::Warning, "job cancelled"); } } diff --git a/crates/daemon/src/store.rs b/crates/daemon/src/store.rs index 01f0cc8..1ce63cb 100644 --- a/crates/daemon/src/store.rs +++ b/crates/daemon/src/store.rs @@ -21,6 +21,7 @@ pub type Result = std::result::Result; #[derive(Debug, Clone, Copy)] pub enum RecordingStatus { + Cancelled, Starting, Recording, Completed, @@ -30,6 +31,7 @@ pub enum RecordingStatus { impl RecordingStatus { const fn as_str(self) -> &'static str { match self { + Self::Cancelled => "cancelled", Self::Starting => "starting", Self::Recording => "recording", Self::Completed => "completed", @@ -201,6 +203,16 @@ impl Store { }); } + pub fn mark_recording_cancelled(&self, id: u64, stopped_at_ms: i64) { + self.send(StoreCommand::MarkRecordingFinished { + id, + status: RecordingStatus::Cancelled, + stopped_at_ms, + file_size_bytes: None, + error_message: None, + }); + } + pub fn update_dimensions(&self, id: u64, dimensions: CaptureDimensions) { self.send(StoreCommand::UpdateDimensions { id, dimensions }); } @@ -1160,6 +1172,25 @@ mod tests { assert_eq!(row.2, None); } + #[test] + fn cancelled_selection_is_persisted_without_a_capture_error() { + let db = TempDb::new("cancelled"); + { + let store = Store::open(db.path()).unwrap(); + store.upsert_recording(sample_recording(1)); + store.mark_recording_cancelled(1, 4_000); + } + let (status, error): (String, Option) = read_connection(&db) + .query_row( + "SELECT status, error_message FROM recordings WHERE id = 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(status, "cancelled"); + assert_eq!(error, None); + } + #[test] fn updates_for_unknown_recording_change_nothing() { let db = TempDb::new("missing-row"); diff --git a/crates/daemon/src/test_support.rs b/crates/daemon/src/test_support.rs index 50246c9..6b3f22d 100644 --- a/crates/daemon/src/test_support.rs +++ b/crates/daemon/src/test_support.rs @@ -199,13 +199,18 @@ impl RecorderEngine for FakeEngine { message: "stopping recording".into(), }) .unwrap(); - self.events - .send(RecorderEvent::Exited { + let event = if !self.start_gate.load(Ordering::Relaxed) { + RecorderEvent::Cancelled { + session_id: session.id, + } + } else { + RecorderEvent::Exited { session_id: session.id, success: true, status: "exit status: 0".into(), - }) - .unwrap(); + } + }; + self.events.send(event).unwrap(); Ok(()) } } diff --git a/crates/domain/src/lib.rs b/crates/domain/src/lib.rs index 8513554..877805b 100644 --- a/crates/domain/src/lib.rs +++ b/crates/domain/src/lib.rs @@ -286,10 +286,16 @@ pub enum RecorderEvent { success: bool, status: String, }, + Cancelled { + session_id: u64, + }, } #[derive(Debug, thiserror::Error)] pub enum RecorderError { + #[error("recording cancelled before capture started")] + Cancelled, + #[error("screen recording permission is not granted")] MissingScreenRecordingPermission, diff --git a/crates/linux/src/lib.rs b/crates/linux/src/lib.rs index 59d9b85..cdbcc57 100644 --- a/crates/linux/src/lib.rs +++ b/crates/linux/src/lib.rs @@ -73,7 +73,7 @@ impl RecorderEngine for LinuxRecorder { settings: RecorderSettings, ) -> Result { if self.active.is_none() && self.stop_requested { - return Err(backend("stopped before recording startup")); + return Err(RecorderError::Cancelled); } if let Some(active) = &self.active { if !active.worker.is_finished() { @@ -159,6 +159,10 @@ impl RecorderEngine for LinuxRecorder { })(); let (success, status) = match result { Ok(()) => (true, "recording finalized".to_string()), + Err(RecorderError::Cancelled) => { + let _ = completion_events.send(RecorderEvent::Cancelled { session_id: id }); + return; + } Err(error) => (false, error.to_string()), }; let _ = completion_events.send(RecorderEvent::Exited { @@ -195,7 +199,61 @@ impl Drop for LinuxRecorder { fn drop(&mut self) { if let Some(active) = self.active.take() { let _ = active.stop.send(true); - let _ = active.worker.join(); + // The coordinator waits for finalization before normal shutdown. + // A stuck native call must not turn destruction into an unbounded join. + if active.worker.is_finished() { + let _ = active.worker.join(); + } } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn destruction_requests_stop_without_waiting_for_a_stuck_worker() { + let (events, _) = mpsc::channel(); + let (commands, _) = mpsc::sync_channel(1); + let (stop, stopped) = watch::channel(false); + let (release, waiting) = mpsc::sync_channel::<()>(1); + let worker = thread::spawn(move || { + let _ = waiting.recv_timeout(Duration::from_secs(1)); + }); + let recorder = LinuxRecorder { + events, + active: Some(Active { + commands, + stop, + worker, + }), + stop_requested: false, + }; + let at = std::time::Instant::now(); + drop(recorder); + let elapsed = at.elapsed(); + let _ = release.send(()); + assert!( + elapsed < Duration::from_millis(200), + "destructor waited {elapsed:?}" + ); + assert!(*stopped.borrow()); + } + + #[test] + fn stop_before_start_preserves_cancellation() { + let (events, _) = mpsc::channel(); + let mut recorder = LinuxRecorder::new(events); + recorder.stop().unwrap(); + let target = CaptureTarget { + id: 0, + kind: domain::CaptureSourceKind::Display, + name: "picker".into(), + }; + assert!(matches!( + recorder.start(target, RecorderSettings::default()), + Err(RecorderError::Cancelled) + )); + } +} diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index fdd55dc..e72ddb4 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -1,7 +1,7 @@ use crate::{backend, portal::PipeWireStream, Command}; use domain::{ - CaptureDimensions, Codec, Quality, RecorderEvent, RecorderMetrics, RecorderSettings, - RecordingSession, Resolution, Result, + CaptureDimensions, Codec, Quality, RecorderError, RecorderEvent, RecorderMetrics, + RecorderSettings, RecordingSession, Resolution, Result, }; use gstreamer::{self as gst, prelude::*}; use std::{ @@ -122,7 +122,7 @@ pub(crate) fn record( stop: &watch::Receiver, ) -> Result<()> { if *stop.borrow() { - return Err(backend("stopped before recording started")); + return Err(RecorderError::Cancelled); } let settings = settings.clone().with_preset_limits(); let pipeline = PipelineGuard(gst::Pipeline::new()); @@ -207,7 +207,9 @@ pub(crate) fn record( .map_err(backend)?; let result = run(&pipeline.0, session, events, commands, stop, &counters); let _ = pipeline.0.set_state(gst::State::Null); - if counters.frames.load(Ordering::Relaxed) == 0 { + if matches!(result, Err(RecorderError::Cancelled)) + || counters.frames.load(Ordering::Relaxed) == 0 + { let _ = std::fs::remove_file(&session.output_path); } result @@ -331,8 +333,8 @@ fn run( let mut last_metrics = Instant::now(); loop { if *stop.borrow() && stopping.is_none() { - if !started { - return Err(backend("stopped before the first encoded frame")); + if !started && counters.frames.load(Ordering::Relaxed) == 0 { + return Err(RecorderError::Cancelled); } // Live sources must run to drain their pending buffers and finalize a paused movie. pipeline.set_state(gst::State::Playing).map_err(backend)?; diff --git a/crates/linux/src/portal.rs b/crates/linux/src/portal.rs index 3f086a3..c8ac5a1 100644 --- a/crates/linux/src/portal.rs +++ b/crates/linux/src/portal.rs @@ -6,7 +6,7 @@ use ashpd::{ }, WindowIdentifier, }; -use domain::{CaptureSourceKind, CaptureTarget, Result}; +use domain::{CaptureSourceKind, CaptureTarget, RecorderError, Result}; use std::{os::fd::OwnedFd, time::Duration}; use tokio::sync::watch; @@ -95,7 +95,7 @@ pub(crate) async fn open( stop: &mut watch::Receiver, ) -> Result { if *stop.borrow() { - return Err(backend("stopped before source selection")); + return Err(RecorderError::Cancelled); } let portal = tokio::time::timeout(Duration::from_secs(5), Screencast::new()) .await @@ -158,7 +158,7 @@ pub(crate) async fn open( }; let result = tokio::select! { result = selected => result, - _ = stop.changed() => Err(backend("stopped during source selection")), + _ = stop.changed() => Err(RecorderError::Cancelled), _ = tokio::time::sleep(Duration::from_secs(120)) => Err(backend("Desktop source selection timed out after 120s; start again and choose a source.")), }; match result { diff --git a/crates/linux/src/portal_tests.rs b/crates/linux/src/portal_tests.rs index bb554f6..4517b4e 100644 --- a/crates/linux/src/portal_tests.rs +++ b/crates/linux/src/portal_tests.rs @@ -209,11 +209,7 @@ fn portal_roundtrip_and_cancellation_close_sessions() { stop.send(true).unwrap(); }; let (result, ()) = tokio::join!(cancelled, cancel); - assert!(result - .err() - .unwrap() - .to_string() - .contains("stopped during source selection")); + assert!(matches!(result, Err(RecorderError::Cancelled))); assert_eq!(mock.closes.load(Ordering::Relaxed), 3); }); } diff --git a/packaging/linux/README.md b/packaging/linux/README.md index 7aa3779..b380c4a 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -64,6 +64,9 @@ native display or window IDs. Only source types advertised by the portal appear in `targets`. The duration starts after the first encoded frame, so time spent choosing a source does not shorten the recording. The picker times out after two minutes and `wrec job stop ` cancels it. +Stopping before capture begins produces a cancelled job, without a movie. +Once capture starts, `--duration` counts wall time, including pauses; the +movie itself omits paused time. Permission status is `unknown` outside a recording because portal grants belong to individual sessions. `permission.request` does not open a second From b4b8ae77d4d3e169106b77e74b06d7de63ada63f Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 5 Sep 2026 06:03:03 +0000 Subject: [PATCH 03/45] ci(linux): pin actions used to build CLI packages --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd2c433..352d4d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,10 +19,10 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 20 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable with: components: rustfmt - name: Install native dependencies @@ -46,7 +46,7 @@ jobs: tar -xzf dist/cli/linux/wrec-cli-x86_64-unknown-linux-gnu.tar.gz -C "$stage" python3 scripts/test-cli-linux.py "$stage" - name: Upload Linux CLI package - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: wrec-cli-linux path: dist/cli/linux/*.tar.gz From a408279d846d2fc0a02ee5f0aae1ae9b79f5e760 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 5 Sep 2026 06:25:54 +0000 Subject: [PATCH 04/45] feat(linux): support X11 and automatic encoder fallback --- .github/workflows/ci.yml | 4 +- Cargo.lock | 40 ++++++ README.md | 5 +- crates/control/src/client.rs | 49 +++++-- crates/daemon/src/backend.rs | 37 ++++- crates/daemon/src/coordinator.rs | 13 +- crates/daemon/src/runtime.rs | 25 +++- crates/linux/Cargo.toml | 2 + crates/linux/src/desktop.rs | 47 +++++++ crates/linux/src/encoding.rs | 209 ++++++++++++++++++++++++++++ crates/linux/src/lib.rs | 23 +++- crates/linux/src/pipeline.rs | 226 ++++++++++++++++++++----------- crates/linux/src/x11.rs | 109 +++++++++++++++ packaging/linux/README.md | 164 +++++++++++----------- scripts/test-capture-linux.py | 110 +++++++++++++++ scripts/test-cli-linux.py | 12 +- 16 files changed, 898 insertions(+), 177 deletions(-) create mode 100644 crates/linux/src/desktop.rs create mode 100644 crates/linux/src/encoding.rs create mode 100644 crates/linux/src/x11.rs create mode 100644 scripts/test-capture-linux.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 352d4d0..e681d75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,8 @@ jobs: sudo apt-get install -y pkg-config libgstreamer1.0-dev \ libgstreamer-plugins-base1.0-dev gstreamer1.0-pipewire \ gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ - gstreamer1.0-plugins-bad gstreamer1.0-libav ffmpeg dbus-daemon + gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly \ + gstreamer1.0-libav ffmpeg dbus-daemon xvfb x11-apps - name: Build and test run: | cargo fmt --check @@ -45,6 +46,7 @@ jobs: trap 'rm -rf "$stage"' EXIT tar -xzf dist/cli/linux/wrec-cli-x86_64-unknown-linux-gnu.tar.gz -C "$stage" python3 scripts/test-cli-linux.py "$stage" + python3 scripts/test-capture-linux.py "$stage/bin/wrec" - name: Upload Linux CLI package uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: diff --git a/Cargo.lock b/Cargo.lock index 1722912..385e4b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -584,6 +584,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix", + "windows-link", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -983,6 +993,8 @@ dependencies = [ "gstreamer-allocators", "serde_json", "tokio", + "x11-dl", + "x11rb", "zbus", ] @@ -2307,6 +2319,34 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "x11-dl" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38735924fedd5314a6e548792904ed8c6de6636285cb9fec04d5b1db85c1516f" +dependencies = [ + "libc", + "once_cell", + "pkg-config", +] + +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + [[package]] name = "xdg-home" version = "1.3.0" diff --git a/README.md b/README.md index f3b91f6..8e69da8 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,8 @@ Wrec records displays or windows with a native ScreenCaptureKit pipeline, writes hardware-encoded `.mov` files, and gives you both a native SwiftUI menu-bar app and a JSON-friendly CLI for scripts and agents. -An initial Linux CLI backend uses the Wayland ScreenCast portal, PipeWire -DMA-BUFs, and VA-API hardware encoding. It requires a compatible Intel/AMD -GPU and desktop. See [Linux setup and limitations](packaging/linux/README.md). +The Linux CLI records Wayland and X11 desktops. It prefers VA-API or NVIDIA +hardware encoding and falls back to software encoding when needed. See [Linux setup and limitations](packaging/linux/README.md). > [!NOTE] > Wrec is still early public software. Release builds are not notarized, so diff --git a/crates/control/src/client.rs b/crates/control/src/client.rs index f42f29d..880a672 100644 --- a/crates/control/src/client.rs +++ b/crates/control/src/client.rs @@ -605,15 +605,18 @@ fn daemon_candidates(current: &Path) -> Vec { .filter(|_| current_dir.file_name().is_some_and(|name| name == "deps")); [ + #[cfg(target_os = "linux")] + current_dir + .parent() + .filter(|_| current_dir.file_name().is_some_and(|name| name == "bin")) + .map(|prefix| { + prefix + .join("lib") + .join(Channel::current().runtime_dir_name()) + .join("daemon") + }), Some(current_dir.join("daemon")), profile_dir.map(|dir| dir.join("daemon")), - #[cfg(target_os = "linux")] - current_dir.parent().map(|prefix| { - prefix - .join("lib") - .join(Channel::current().runtime_dir_name()) - .join("daemon") - }), Some(installed_daemon_path()), ] .into_iter() @@ -632,6 +635,13 @@ fn daemon_executable_launch(path: PathBuf) -> Option { return None; } + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::PermissionsExt; + if std::fs::metadata(&path).ok()?.permissions().mode() & 0o111 == 0 { + return None; + } + } if cfg!(target_os = "linux") || cargo_profile_dir(&path).is_some() { return Some(DaemonLaunch::executable(path)); } @@ -727,10 +737,18 @@ mod tests { } #[test] - fn daemon_candidates_prefer_sibling_daemon() { + fn daemon_candidates_prefer_the_platform_package_layout() { let candidates = daemon_candidates(&PathBuf::from("/tmp/wrec/bin/wrec")); + #[cfg(target_os = "macos")] assert_eq!(candidates[0], PathBuf::from("/tmp/wrec/bin/daemon")); + #[cfg(target_os = "linux")] + assert_eq!( + candidates[0], + PathBuf::from("/tmp/wrec/lib") + .join(wrec_channel::Channel::current().runtime_dir_name()) + .join("daemon") + ); } #[test] @@ -755,6 +773,11 @@ mod tests { let daemon = debug.join("daemon"); std::fs::create_dir_all(&debug).unwrap(); std::fs::write(&daemon, "").unwrap(); + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&daemon, std::fs::Permissions::from_mode(0o755)).unwrap(); + } let launch = daemon_executable_launch(daemon.clone()).unwrap(); @@ -778,6 +801,11 @@ mod tests { std::fs::create_dir_all(&dir).unwrap(); let daemon = dir.join("daemon"); std::fs::write(&daemon, "").unwrap(); + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&daemon, std::fs::Permissions::from_mode(0o755)).unwrap(); + } let launch = daemon_executable_launch(daemon.clone()).unwrap(); assert_eq!(launch.program, daemon); assert!(launch.envs.is_empty()); @@ -795,6 +823,11 @@ mod tests { let capture_engine = dir.join("capture-engine"); std::fs::create_dir_all(&dir).unwrap(); std::fs::write(&daemon, "").unwrap(); + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&daemon, std::fs::Permissions::from_mode(0o755)).unwrap(); + } assert!(daemon_executable_launch(daemon.clone()).is_none()); diff --git a/crates/daemon/src/backend.rs b/crates/daemon/src/backend.rs index ef5f5ec..bdc84f3 100644 --- a/crates/daemon/src/backend.rs +++ b/crates/daemon/src/backend.rs @@ -184,9 +184,12 @@ impl WrecBackend { success, status, } => { - let output_path = self.active_output_path.clone(); let failed_before_exit = self.failed_session_ids.remove(session_id); let success = *success && !failed_before_exit; + let output_path = self + .active_output_path + .clone() + .filter(|path| success || path.is_file()); if success { self.mark_recording_completed(*session_id, output_path.as_deref()); @@ -615,4 +618,36 @@ mod tests { assert!(matches!(event, BackendEvent::Exited { success: false, .. })); } + + #[test] + fn failed_start_omits_missing_output_and_failed_capture_keeps_existing_output() { + let path = + std::env::temp_dir().join(format!("wrec-failure-artifact-{}", std::process::id())); + for (stage, artifact) in [ + ("portal rejected", false), + ("setup failed", false), + ("zero-frame timeout", false), + ("capture failed after frames", true), + ] { + if artifact { + std::fs::write(&path, b"partial movie fragment").unwrap(); + } + let mut backend = WrecBackend { + store: None, + active_session_id: Some(1), + active_output_path: Some(path.clone()), + failed_session_ids: HashSet::new(), + }; + let event = backend.handle_recorder_event(&RecorderEvent::Exited { + session_id: 1, + success: false, + status: stage.into(), + }); + let BackendEvent::Exited { output_path, .. } = event else { + panic!("expected exit") + }; + assert_eq!(output_path.is_some(), artifact, "{stage}"); + } + std::fs::remove_file(path).unwrap(); + } } diff --git a/crates/daemon/src/coordinator.rs b/crates/daemon/src/coordinator.rs index 4282034..9929e5e 100644 --- a/crates/daemon/src/coordinator.rs +++ b/crates/daemon/src/coordinator.rs @@ -176,8 +176,8 @@ impl Coordinator { let (job, should_launch) = { let config = load_config(); let overrides = recording_overrides(¶ms.options); - let (settings, warning) = build_settings_report(&config.settings, &overrides); - let warnings = warning + let (mut settings, warning) = build_settings_report(&config.settings, &overrides); + let mut warnings = warning .map(|message| AgentWarning { code: "preset_limited".into(), message, @@ -185,6 +185,9 @@ impl Coordinator { }) .into_iter() .collect::>(); + lock_state(&state)? + .runtime + .prepare_settings(&mut settings, &mut warnings); let targets = list_targets_with_cache(&state, true)?; let target = resolve_record_target( &targets, @@ -702,7 +705,11 @@ fn handle_recorder_event( output_path, .. } => { - job.output_path = output_path.or_else(|| job.output_path.clone()); + job.output_path = output_path.or_else(|| { + job.output_path + .clone() + .filter(|path| success || path.is_file()) + }); if success { job.mark_completed(format!("capture engine exited: {status}")); } else { diff --git a/crates/daemon/src/runtime.rs b/crates/daemon/src/runtime.rs index 57ab831..16bfefd 100644 --- a/crates/daemon/src/runtime.rs +++ b/crates/daemon/src/runtime.rs @@ -7,6 +7,13 @@ use std::sync::mpsc; pub(crate) trait RecordingRuntime: Clone + Send + Sync + 'static { type Engine: RecorderEngine + Send + 'static; + fn prepare_settings( + &self, + _settings: &mut domain::RecorderSettings, + _warnings: &mut Vec, + ) { + } + fn list_targets(&self) -> Result, AgentError>; fn screen_recording_permission_status(&self) -> Result; fn request_screen_recording_permission(&self) -> Result; @@ -125,6 +132,22 @@ pub(crate) struct LinuxRuntime; impl RecordingRuntime for LinuxRuntime { type Engine = linux::LinuxRecorder; + fn prepare_settings( + &self, + settings: &mut domain::RecorderSettings, + warnings: &mut Vec, + ) { + if settings.hide_wrec || (settings.include_microphone && settings.show_mic_indicator) { + warnings.push(control::AgentWarning { + code: "linux_settings_unavailable".into(), + message: "Linux does not provide Wrec-window exclusion or a custom microphone indicator; these settings are disabled for this job.".into(), + next: "Use the desktop's sharing controls and sound settings.".into(), + }); + } + settings.hide_wrec = false; + settings.show_mic_indicator = false; + } + fn list_targets(&self) -> Result, AgentError> { linux::list_targets().map_err(linux_error) } @@ -166,7 +189,7 @@ fn linux_error(error: RecorderError) -> AgentError { code: "linux_capture_unavailable".into(), message: error.to_string(), recoverable: true, - next: "Run wrec inside a Wayland desktop with PipeWire and the desktop's ScreenCast portal installed. See packaging/linux/README.md.".into(), + next: "Run wrec inside a Wayland desktop with its ScreenCast portal, or an X11 desktop with DISPLAY set. See packaging/linux/README.md.".into(), } } diff --git a/crates/linux/Cargo.toml b/crates/linux/Cargo.toml index 16b35f4..f14be8d 100644 --- a/crates/linux/Cargo.toml +++ b/crates/linux/Cargo.toml @@ -12,6 +12,8 @@ tokio = { version = "1", features = ["rt", "time", "sync", "macros"] } gstreamer = { version = "0.23", features = ["v1_22"] } gstreamer-allocators = { version = "0.23", features = ["v1_22"] } futures-util = { version = "0.3", default-features = false, features = ["std", "async-await"] } +x11rb = "0.13" +x11-dl = "2.21" [target.'cfg(target_os = "linux")'.dev-dependencies] serde_json = "1" diff --git a/crates/linux/src/desktop.rs b/crates/linux/src/desktop.rs new file mode 100644 index 0000000..cfaed37 --- /dev/null +++ b/crates/linux/src/desktop.rs @@ -0,0 +1,47 @@ +use crate::{backend, portal, x11}; +use domain::{CaptureTarget, Result}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Desktop { + Wayland, + X11, +} + +pub(crate) fn detect() -> Result { + detect_values( + std::env::var_os("WAYLAND_DISPLAY").is_some_and(|s| !s.is_empty()), + std::env::var_os("DISPLAY").is_some_and(|s| !s.is_empty()), + ) +} + +fn detect_values(wayland: bool, x11: bool) -> Result { + // DISPLAY in a Wayland session is usually XWayland, which cannot capture + // the full Wayland desktop. Use the portal whenever Wayland is active. + match (wayland, x11) { + (true, _) => Ok(Desktop::Wayland), + (false, true) => Ok(Desktop::X11), + _ => Err(backend("Linux recording requires a Wayland or X11 desktop session. This process has neither WAYLAND_DISPLAY nor DISPLAY.")), + } +} + +pub fn check_desktop() -> Result<()> { + detect().map(|_| ()) +} + +pub fn list_targets() -> Result> { + match detect()? { + Desktop::Wayland => portal::list_targets(), + Desktop::X11 => x11::list_targets(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn chooses_portal_over_xwayland_and_supports_x11_only() { + assert_eq!(detect_values(true, true).unwrap(), Desktop::Wayland); + assert_eq!(detect_values(false, true).unwrap(), Desktop::X11); + assert!(detect_values(false, false).is_err()); + } +} diff --git a/crates/linux/src/encoding.rs b/crates/linux/src/encoding.rs new file mode 100644 index 0000000..270a213 --- /dev/null +++ b/crates/linux/src/encoding.rs @@ -0,0 +1,209 @@ +use crate::{backend, pipeline::element}; +use domain::{Codec, Quality, RecorderSettings, Result}; +use gstreamer::{self as gst, prelude::*}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kind { + Va, + Cuda, + Nvidia, + Software, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Mode { + pub kind: Kind, + pub factory: &'static str, + pub dmabuf: bool, +} + +pub(crate) fn modes(codec: Codec, pipewire: bool) -> Vec { + modes_with(codec, pipewire, |name| { + gst::ElementFactory::find(name).is_some() + }) +} + +fn modes_with(codec: Codec, pipewire: bool, has: impl Fn(&str) -> bool) -> Vec { + let (va, nv, software): (&[&str], &str, &[&str]) = match codec { + Codec::H264 => ( + &["vah264lpenc", "vah264enc"], + "nvh264enc", + &["x264enc", "openh264enc"], + ), + Codec::Hevc => (&["vah265lpenc", "vah265enc"], "nvh265enc", &["x265enc"]), + }; + let mut modes = Vec::new(); + for factory in va + .iter() + .copied() + .filter(|name| has(name) && has("vapostproc")) + { + if pipewire { + modes.push(Mode { + kind: Kind::Va, + factory, + dmabuf: true, + }); + } + modes.push(Mode { + kind: Kind::Va, + factory, + dmabuf: false, + }); + } + if has(nv) { + if has("cudaupload") && has("cudaconvertscale") { + modes.push(Mode { + kind: Kind::Cuda, + factory: nv, + dmabuf: false, + }); + } + modes.push(Mode { + kind: Kind::Nvidia, + factory: nv, + dmabuf: false, + }); + } + for factory in software.iter().copied().filter(|name| has(name)) { + modes.push(Mode { + kind: Kind::Software, + factory, + dmabuf: false, + }); + } + modes +} + +impl Mode { + pub fn description(self) -> String { + let input = if self.dmabuf { + "shared GPU buffers" + } else { + "system-memory capture" + }; + let processing = match self.kind { + Kind::Va | Kind::Cuda => "GPU conversion and encoding", + Kind::Nvidia => "CPU conversion, NVIDIA hardware encoding", + Kind::Software => "CPU conversion and software encoding", + }; + format!("{input}; {processing}; {}", self.factory) + } + + pub fn caps(self, size: Option<(i32, i32)>) -> gst::Caps { + let builder = + gst::Caps::builder("video/x-raw").field("pixel-aspect-ratio", gst::Fraction::new(1, 1)); + let builder = match self.kind { + Kind::Va => builder + .features(["memory:VAMemory"]) + .field("format", "NV12"), + Kind::Cuda => builder + .features(["memory:CUDAMemory"]) + .field("format", "NV12"), + Kind::Nvidia => builder + .features(["memory:SystemMemory"]) + .field("format", "NV12"), + Kind::Software => builder + .features(["memory:SystemMemory"]) + .field("format", "I420"), + }; + match size { + Some((w, h)) => builder.field("width", w).field("height", h).build(), + None => builder.build(), + } + } + + pub fn converters(self) -> Result> { + match self.kind { + Kind::Va => { + let converter = element("vapostproc")?; + converter.set_property("add-borders", true); + Ok(vec![converter]) + } + Kind::Cuda => { + let converter = element("cudaconvertscale")?; + if converter.find_property("add-borders").is_some() { + converter.set_property("add-borders", true); + } + Ok(vec![element("cudaupload")?, converter]) + } + _ => { + let scaler = element("videoscale")?; + scaler.set_property("add-borders", true); + Ok(vec![element("videoconvert")?, scaler]) + } + } + } + + pub fn encoder(self, settings: &RecorderSettings) -> Result { + let encoder = element(self.factory)?; + let qp = match settings.quality { + Quality::Efficient => 30u32, + Quality::Balanced => 25u32, + Quality::High => 20u32, + }; + let bitrate = match settings.quality { + Quality::Efficient => 2500u32, + Quality::Balanced => 6000u32, + Quality::High => 16000u32, + }; + match self.kind { + Kind::Va => { + encoder.set_property_from_str("rate-control", "cqp"); + for name in ["qpi", "qpp", "qpb"] { + encoder.set_property(name, qp); + } + encoder.set_property("key-int-max", settings.fps.as_u32() * 2); + encoder.set_property("b-frames", 0u32); + } + Kind::Cuda | Kind::Nvidia => { + encoder.set_property("bitrate", bitrate); + encoder.set_property("gop-size", settings.fps.as_u32() as i32 * 2); + encoder.set_property("bframes", 0u32); + } + Kind::Software => match self.factory { + "x264enc" => { + encoder.set_property_from_str("speed-preset", "ultrafast"); + encoder.set_property_from_str("tune", "zerolatency"); + encoder.set_property_from_str("pass", "qual"); + encoder.set_property("quantizer", qp); + encoder.set_property("threads", 2u32); + encoder.set_property("key-int-max", settings.fps.as_u32() * 2); + } + "x265enc" => { + encoder.set_property_from_str("speed-preset", "ultrafast"); + encoder.set_property_from_str("tune", "zerolatency"); + encoder.set_property("qp", qp as i32); + encoder.set_property("option-string", "pools=2:frame-threads=2"); + encoder.set_property("key-int-max", settings.fps.as_u32() as i32 * 2); + } + "openh264enc" => { + encoder.set_property("bitrate", bitrate * 1000); + encoder.set_property("multi-thread", 2u32); + encoder.set_property("gop-size", settings.fps.as_u32() * 2); + } + _ => return Err(backend("unsupported software encoder")), + }, + } + Ok(encoder) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn prefers_hardware_and_retains_software_compatibility() { + let modes = modes_with(Codec::H264, true, |_| true); + assert!(modes[0].dmabuf); + assert_eq!(modes[0].kind, Kind::Va); + assert!(modes.iter().any(|m| m.kind == Kind::Cuda)); + assert_eq!(modes.last().unwrap().kind, Kind::Software); + assert!(modes_with(Codec::Hevc, false, |_| true) + .iter() + .all(|m| !m.dmabuf)); + let modes = modes_with(Codec::H264, false, |name| name == "openh264enc"); + assert_eq!(modes.len(), 1); + assert_eq!(modes[0].kind, Kind::Software); + } +} diff --git a/crates/linux/src/lib.rs b/crates/linux/src/lib.rs index cdbcc57..5e9db61 100644 --- a/crates/linux/src/lib.rs +++ b/crates/linux/src/lib.rs @@ -1,7 +1,10 @@ #![cfg(target_os = "linux")] +mod desktop; +mod encoding; mod pipeline; mod portal; +mod x11; use domain::{ CaptureTarget, RecorderEngine, RecorderError, RecorderEvent, RecorderSettings, @@ -18,7 +21,7 @@ use std::{ }; use tokio::sync::watch; -pub use portal::{check_desktop, list_targets}; +pub use desktop::{check_desktop, list_targets}; pub(crate) fn backend(error: impl std::fmt::Display) -> RecorderError { RecorderError::Backend(error.to_string()) @@ -85,8 +88,9 @@ impl RecorderEngine for LinuxRecorder { } self.stop_requested = false; check_desktop()?; - portal::validate_target(&target)?; - pipeline::check_plugins(&settings)?; + if desktop::detect()? == desktop::Desktop::Wayland { + portal::validate_target(&target)?; + } std::fs::create_dir_all(&settings.output_dir).map_err(backend)?; static LAST_ID: AtomicU64 = AtomicU64::new(0); let now = SystemTime::now() @@ -119,6 +123,17 @@ impl RecorderEngine for LinuxRecorder { output_path: worker_session.output_path.clone(), }); let result = (|| { + if desktop::detect()? == desktop::Desktop::X11 { + let (display, xid) = x11::source(&target)?; + return pipeline::record( + &pipeline::CaptureInput::X11 { display, xid }, + &worker_session, + &settings, + &events, + receiver, + &stopped, + ); + } let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -135,7 +150,7 @@ impl RecorderEngine for LinuxRecorder { }; let mut recording = tokio::task::spawn_blocking(move || { pipeline::record( - &stream, + &pipeline::CaptureInput::PipeWire(stream), &worker_session, &settings, &events, diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index e72ddb4..815575f 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -1,7 +1,12 @@ -use crate::{backend, portal::PipeWireStream, Command}; +use crate::{ + backend, + encoding::{self, Mode}, + portal::PipeWireStream, + Command, +}; use domain::{ - CaptureDimensions, Codec, Quality, RecorderError, RecorderEvent, RecorderMetrics, - RecorderSettings, RecordingSession, Resolution, Result, + CaptureDimensions, Codec, RecorderError, RecorderEvent, RecorderMetrics, RecorderSettings, + RecordingSession, Resolution, Result, }; use gstreamer::{self as gst, prelude::*}; use std::{ @@ -15,25 +20,13 @@ use std::{ }; use tokio::sync::watch; -fn element(factory: &str) -> Result { +pub(crate) fn element(factory: &str) -> Result { gst::ElementFactory::make(factory).build().map_err(|error| backend(format!("GStreamer element {factory} is unavailable: {error}. Install the Linux runtime dependencies in packaging/linux/README.md."))) } -fn encoder_name(codec: Codec) -> Result<&'static str> { - let names = match codec { - Codec::H264 => ["vah264lpenc", "vah264enc"], - Codec::Hevc => ["vah265lpenc", "vah265enc"], - }; - names.into_iter().find(|name| gst::ElementFactory::find(name).is_some()) - .ok_or_else(|| backend(format!("No VA-API {} hardware encoder is available. Install GStreamer's va plugin and an Intel/AMD encoding driver, and check access to /dev/dri/renderD*. NVIDIA/NVENC and software video encoding are not supported by this backend.", codec.as_arg()))) -} - pub(crate) fn check_plugins(settings: &RecorderSettings) -> Result<()> { gst::init().map_err(backend)?; - encoder_name(settings.codec)?; for factory in [ - "pipewiresrc", - "vapostproc", "capsfilter", "queue", parser_name(settings.codec), @@ -79,17 +72,6 @@ fn output_size(width: i32, height: i32, resolution: Resolution) -> (i32, i32) { (even(width), even(height)) } -fn video_caps(size: Option<(i32, i32)>) -> gst::Caps { - let builder = gst::Caps::builder("video/x-raw") - .features(["memory:VAMemory"]) - .field("format", "NV12") - .field("pixel-aspect-ratio", gst::Fraction::new(1, 1)); - match size { - Some((w, h)) => builder.field("width", w).field("height", h).build(), - None => builder.build(), - } -} - fn video_queue() -> Result { let queue = element("queue")?; queue.set_property("max-size-buffers", 2u32); @@ -111,15 +93,75 @@ struct Counters { frames: AtomicU64, dropped: AtomicU64, dimensions: Mutex>, + last_pts: Mutex>, +} + +pub(crate) enum CaptureInput { + PipeWire(PipeWireStream), + X11 { display: String, xid: u64 }, +} + +struct Attempt { + mode: Mode, + counters: Arc, } pub(crate) fn record( - capture: &PipeWireStream, + capture: &CaptureInput, session: &RecordingSession, settings: &RecorderSettings, events: &mpsc::Sender, commands: mpsc::Receiver, stop: &watch::Receiver, +) -> Result<()> { + if matches!(capture, CaptureInput::X11 { .. }) { + crate::x11::initialize()?; + } + check_plugins(settings)?; + let modes = encoding::modes(settings.codec, matches!(capture, CaptureInput::PipeWire(_))); + if modes.is_empty() { + return Err(backend(format!("No {} encoder is installed. Install GStreamer VA/NVENC plugins or x264/x265/openh264 for software compatibility; see packaging/linux/README.md.", settings.codec.as_arg()))); + } + let mut last_error = None; + for mode in modes { + if *stop.borrow() { + return Err(RecorderError::Cancelled); + } + let attempt = Attempt { + mode, + counters: Arc::new(Counters::default()), + }; + let _ = events.send(RecorderEvent::Log { + session_id: Some(session.id), + message: format!("capture-engine: trying {}", mode.description()), + }); + let result = record_attempt( + capture, session, settings, events, &commands, stop, &attempt, + ); + if result.is_ok() + || matches!(result, Err(RecorderError::Cancelled)) + || attempt.counters.frames.load(Ordering::Relaxed) > 0 + { + return result; + } + let error = result.unwrap_err(); + let _ = events.send(RecorderEvent::Log { + session_id: Some(session.id), + message: format!("capture-engine: {} could not start: {error}", mode.factory), + }); + last_error = Some(error); + } + Err(last_error.unwrap_or_else(|| backend("No recording pipeline could start"))) +} + +fn record_attempt( + capture: &CaptureInput, + session: &RecordingSession, + settings: &RecorderSettings, + events: &mpsc::Sender, + commands: &mpsc::Receiver, + stop: &watch::Receiver, + attempt: &Attempt, ) -> Result<()> { if *stop.borrow() { return Err(RecorderError::Cancelled); @@ -127,48 +169,51 @@ pub(crate) fn record( let settings = settings.clone().with_preset_limits(); let pipeline = PipelineGuard(gst::Pipeline::new()); pipeline.0.use_clock(Some(&gst::SystemClock::obtain())); - let source = element("pipewiresrc")?; - source.set_property("fd", capture.fd.as_raw_fd()); - // The portal returns a node ID, whereas target-object expects an object serial. - source.set_property("path", capture.node.to_string()); - source.set_property("always-copy", false); - source.set_property("min-buffers", 4i32); - source.set_property("max-buffers", 8i32); - source.set_property("keepalive-time", 1000i32); - source.set_property("resend-last", true); + let mode = attempt.mode; + let source = match capture { + CaptureInput::PipeWire(capture) => { + let source = element("pipewiresrc")?; + source.set_property("fd", capture.fd.as_raw_fd()); + source.set_property("path", capture.node.to_string()); + source.set_property("always-copy", false); + source.set_property("min-buffers", 4i32); + source.set_property("max-buffers", 8i32); + source.set_property("keepalive-time", 1000i32); + source.set_property("resend-last", true); + source + } + CaptureInput::X11 { display, xid } => { + let source = element("ximagesrc")?; + source.set_property("display-name", display); + source.set_property("xid", *xid); + source.set_property("show-pointer", settings.include_cursor); + source.set_property("use-damage", true); + source + } + }; source.set_property("do-timestamp", true); let input = element("capsfilter")?; - input.set_property( - "caps", - gst::Caps::builder("video/x-raw") - .features(["memory:DMABuf"]) - .field( - "framerate", - gst::Fraction::new(settings.fps.as_u32() as i32, 1), - ) - .build(), + let caps = gst::Caps::builder("video/x-raw").field( + "framerate", + gst::Fraction::new(settings.fps.as_u32() as i32, 1), ); + let caps = if mode.dmabuf { + caps.features(["memory:DMABuf"]).build() + } else { + caps.build() + }; + input.set_property("caps", caps); let queue = video_queue()?; - let converter = element("vapostproc")?; - converter.set_property("add-borders", true); + let converters = mode.converters()?; let output = element("capsfilter")?; - output.set_property("caps", video_caps(None)); - let encoder_factory = encoder_name(settings.codec)?; - let encoder = element(encoder_factory)?; - encoder.set_property_from_str("rate-control", "cqp"); - let qp = match settings.quality { - Quality::Efficient => 30u32, - Quality::Balanced => 25u32, - Quality::High => 20u32, - }; - for property in ["qpi", "qpp", "qpb"] { - encoder.set_property(property, qp); - } - encoder.set_property("key-int-max", settings.fps.as_u32() * 2); - encoder.set_property("b-frames", 0u32); + output.set_property("caps", mode.caps(None)); + let encoder = mode.encoder(&settings)?; let parser = element(parser_name(settings.codec))?; let mux = element("qtmux")?; mux.set_property("fragment-duration", 10000u32); + // Preserve sub-frame timestamps around pause/resume instead of rounding + // them to the default frame-rate-derived track timescale. + mux.set_property("trak-timescale", 1_000_000u32); let sink = element("filesink")?; sink.set_property( "location", @@ -178,19 +223,28 @@ pub(crate) fn record( .ok_or_else(|| backend("recording output path must be UTF-8"))?, ); sink.set_property("sync", false); - let chain = [ - &source, &input, &queue, &converter, &output, &encoder, &parser, &mux, &sink, - ]; - pipeline.0.add_many(chain).map_err(backend)?; - gst::Element::link_many(chain).map_err(|error| backend(format!("Cannot link the DMA-BUF/VA-API pipeline: {error}. Use GStreamer 1.24 or newer with DMA-BUF support in vapostproc.")))?; + let mut chain = vec![&source, &input, &queue]; + chain.extend(converters.iter()); + chain.extend([&output, &encoder, &parser, &mux, &sink]); + pipeline + .0 + .add_many(chain.iter().copied()) + .map_err(backend)?; + gst::Element::link_many(chain.iter().copied()).map_err(backend)?; if settings.include_system_audio { add_audio(&pipeline.0, &mux, Some("@DEFAULT_MONITOR@"))?; } if settings.include_microphone { add_audio(&pipeline.0, &mux, None)?; } - let counters = Arc::new(Counters::default()); - attach_capture_probe(&source, &output, settings.resolution, counters.clone()); + let counters = attempt.counters.clone(); + attach_capture_probe( + &source, + &output, + settings.resolution, + mode, + counters.clone(), + ); attach_encoded_probe(&parser, counters.clone()); // One overrun accompanies each incoming frame that replaces the oldest frame. let dropped = counters.clone(); @@ -198,7 +252,13 @@ pub(crate) fn record( dropped.dropped.fetch_add(1, Ordering::Relaxed); None }); - let _ = events.send(RecorderEvent::Log { session_id: Some(session.id), message: format!("capture-engine: PipeWire DMA-BUF → vapostproc → {encoder_factory}; 8 capture buffers, 2 queued frames, no CPU video fallback") }); + let _ = events.send(RecorderEvent::Log { + session_id: Some(session.id), + message: format!( + "capture-engine: selected {}; video queue limited to 2 frames", + mode.description() + ), + }); // Reserve a unique file so a collision never truncates an existing recording. std::fs::OpenOptions::new() .write(true) @@ -269,6 +329,7 @@ fn attach_capture_probe( source: &gst::Element, output: &gst::Element, resolution: Resolution, + mode: Mode, counters: Arc, ) { let output = output.clone(); @@ -280,21 +341,29 @@ fn attach_capture_probe( if width >= 2 && height >= 2 { let mut dimensions = counters.dimensions.lock().unwrap(); // A MOV track has a fixed canvas. Window resizes are - // scaled/letterboxed by VA into the initial output size. + // scaled/letterboxed into the initial output size. let (w, h) = dimensions.map(|d| (d.output_width as i32, d.output_height as i32)) .unwrap_or_else(|| output_size(width, height, resolution)); dimensions.get_or_insert(CaptureDimensions { native_width: width.into(), native_height: height.into(), output_width: w.into(), output_height: h.into() }); drop(dimensions); - output.set_property("caps", video_caps(Some((w, h)))); + output.set_property("caps", mode.caps(Some((w, h)))); } } } } } if let Some(gst::PadProbeData::Buffer(buffer)) = &info.data { - if buffer.n_memory() == 0 || buffer.iter_memories().any(|memory| !memory.is_memory_type::()) { + if let Some(pts) = buffer.pts() { + let mut previous = counters.last_pts.lock().unwrap(); + if previous.is_some_and(|last| pts <= last || pts.saturating_sub(last) < gst::ClockTime::from_useconds(1)) { + counters.dropped.fetch_add(1, Ordering::Relaxed); + return gst::PadProbeReturn::Drop; + } + *previous = Some(pts); + } + if mode.dmabuf && (buffer.n_memory() == 0 || buffer.iter_memories().any(|memory| !memory.is_memory_type::())) { if let Some(element) = pad.parent_element() { - gst::element_error!(element, gst::StreamError::Format, ("PipeWire delivered a non-DMA-BUF frame. This backend requires GPU buffer sharing; check the compositor and graphics driver.")); + gst::element_error!(element, gst::StreamError::Format, ("PipeWire delivered a non-DMA-BUF frame. This attempt requires GPU buffer sharing; another available mode will be tried before capture starts.")); } return gst::PadProbeReturn::Drop; } @@ -319,7 +388,7 @@ fn run( pipeline: &gst::Pipeline, session: &RecordingSession, events: &mpsc::Sender, - commands: mpsc::Receiver, + commands: &mpsc::Receiver, stop: &watch::Receiver, counters: &Counters, ) -> Result<()> { @@ -490,7 +559,7 @@ mod tests { &guard.0, &worker_session, &events_tx, - commands_rx, + &commands_rx, &stopped, &counters, ) @@ -691,6 +760,11 @@ mod tests { &pipeline.0.by_name("source").unwrap(), &output, Resolution::Native, + Mode { + kind: encoding::Kind::Va, + factory: "vah264enc", + dmabuf: true, + }, Arc::new(Counters::default()), ); pipeline.0.set_state(gst::State::Playing).unwrap(); diff --git a/crates/linux/src/x11.rs b/crates/linux/src/x11.rs new file mode 100644 index 0000000..53ed442 --- /dev/null +++ b/crates/linux/src/x11.rs @@ -0,0 +1,109 @@ +use crate::backend; +use domain::{CaptureSourceKind, CaptureTarget, Result}; +use x11rb::{ + connection::Connection, + protocol::xproto::{AtomEnum, ConnectionExt, MapState}, +}; + +pub(crate) fn initialize() -> Result<()> { + static XLIB: std::sync::OnceLock> = + std::sync::OnceLock::new(); + static INITIALIZED: std::sync::OnceLock = std::sync::OnceLock::new(); + let xlib = XLIB + .get_or_init(|| x11_dl::xlib::Xlib::open().map_err(|e| e.to_string())) + .as_ref() + .map_err(backend)?; + // Must run before GStreamer opens any Xlib display. Discovery uses x11rb, + // which talks to the server directly and does not initialize Xlib. + if !INITIALIZED.get_or_init(|| unsafe { (xlib.XInitThreads)() } != 0) { + return Err(backend("XInitThreads failed")); + } + Ok(()) +} + +pub(crate) fn list_targets() -> Result> { + let (connection, default_screen) = x11rb::connect(None).map_err(backend)?; + let atoms = |name: &[u8]| -> Result { + Ok(connection + .intern_atom(false, name) + .map_err(backend)? + .reply() + .map_err(backend)? + .atom) + }; + let clients = atoms(b"_NET_CLIENT_LIST_STACKING")?; + let utf8_name = atoms(b"_NET_WM_NAME")?; + let mut targets = Vec::new(); + for (index, screen) in connection.setup().roots.iter().enumerate() { + targets.push(CaptureTarget { + id: if index == default_screen { + 0 + } else { + screen.root.into() + }, + kind: CaptureSourceKind::Display, + name: format!( + "X11 display {index} ({}×{})", + screen.width_in_pixels, screen.height_in_pixels + ), + }); + let reply = connection + .get_property(false, screen.root, clients, AtomEnum::WINDOW, 0, 4096) + .map_err(backend)? + .reply() + .map_err(backend)?; + let windows: Vec = reply.value32().map(|v| v.collect()).unwrap_or_else(|| { + connection + .query_tree(screen.root) + .ok() + .and_then(|c| c.reply().ok()) + .map(|r| r.children) + .unwrap_or_default() + }); + for window in windows { + let visible = connection + .get_window_attributes(window) + .ok() + .and_then(|c| c.reply().ok()) + .is_some_and(|a| a.map_state == MapState::VIEWABLE); + if !visible { + continue; + } + let property = |atom: u32| { + connection + .get_property(false, window, atom, AtomEnum::ANY, 0, 1024) + .ok() + .and_then(|c| c.reply().ok()) + .map(|r| { + String::from_utf8_lossy(&r.value) + .trim_end_matches('\0') + .to_string() + }) + }; + let name = property(utf8_name) + .filter(|s| !s.is_empty()) + .or_else(|| property(AtomEnum::WM_NAME.into())) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| format!("X11 window {window}")); + targets.push(CaptureTarget { + id: window.into(), + name, + kind: CaptureSourceKind::Window, + }); + } + } + Ok(targets) +} + +pub(crate) fn source(target: &CaptureTarget) -> Result<(String, u64)> { + let display = std::env::var("DISPLAY").map_err(backend)?; + if !list_targets()? + .iter() + .any(|t| t.id == target.id && t.kind == target.kind) + { + return Err(backend( + "The selected X11 display or window is no longer available.", + )); + } + Ok((display, target.id)) +} diff --git a/packaging/linux/README.md b/packaging/linux/README.md index b380c4a..ec486dc 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -1,55 +1,66 @@ # Linux CLI and daemon -The Linux backend records a Wayland display or window selected through the -desktop's ScreenCast portal. It requires PipeWire DMA-BUF capture, GStreamer -1.24 or newer, and an Intel or AMD GPU with VA-API encoding support. This is -an initial backend; hardware compatibility and efficiency still need measurements -on real desktops. There is no Linux GUI in this release. - -The pipeline is `pipewiresrc → DMA-BUF → vapostproc → VA encoder → qtmux`. -Rust manages sessions and buffer metadata and never maps video pixels. The -source negotiates four to eight buffers, the video queue holds at most two -frames, and the encoder uses no B-frames. This bounds application queues; -the compositor and driver have their own allocations. Colour conversion and -scaling can still allocate GPU surfaces, so this is not a claim of zero total -copies. The backend rejects CPU frame buffers and has no software video fallback. +The Linux backend records Wayland desktops through the ScreenCast portal and +PipeWire, or X11 displays and windows through XShm/XImage. It selects an installed +encoder automatically: Intel/AMD VA-API, NVIDIA NVENC, then software encoding. +A working desktop and the required native plugins are needed. There is no Linux +GUI yet. Hardware compatibility and efficiency still need measurements on real +desktops; software fallback makes recording possible without a supported GPU, +but uses more CPU. + +On compatible Wayland/VA-API systems, the preferred pipeline is +`pipewiresrc → DMA-BUF → vapostproc → VA encoder → qtmux`. Rust manages sessions +and buffer metadata without mapping video pixels. The source negotiates four to +eight buffers and the video queue holds at most two frames. Conversion and scaling +can allocate GPU surfaces; this does not imply zero total copies. + +When shared GPU buffers cannot be imported, wrec retries with system-memory +capture and hardware encoding. NVIDIA uses CUDA conversion when available, +otherwise CPU conversion before NVENC. Software encoding uses x264/OpenH264 for +H.264 and x265 for HEVC. Retries happen only before the first encoded frame and +preserve the requested codec. Job events identify the attempted and selected +paths. X11 capture uses system memory even when encoding runs on the GPU. ## Install dependencies and build -Ubuntu 24.04 or newer is a starting point. The desktop must already have a -working PipeWire service and its own `xdg-desktop-portal` backend, such as -`xdg-desktop-portal-gnome` or `xdg-desktop-portal-kde`. Use the backend for your -desktop; do not install an arbitrary mix of portal backends. - -Use the current stable Rust toolchain. +Use GStreamer 1.22 or newer and current stable Rust. GStreamer 1.24 or newer is +recommended for the DMA-BUF/VA path. Ubuntu 24.04 is a starting point; package +names differ across distributions. ```bash sudo apt install build-essential pkg-config libgstreamer1.0-dev \ libgstreamer-plugins-base1.0-dev gstreamer1.0-tools \ gstreamer1.0-pipewire gstreamer1.0-plugins-base \ - gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav \ - pipewire-pulse vainfo + gstreamer1.0-plugins-good gstreamer1.0-plugins-bad \ + gstreamer1.0-plugins-ugly gstreamer1.0-libav cargo build --release -p cli -p daemon ``` -Install the VA-API driver for your GPU. Distribution codec packaging and GPU -capabilities vary. Inspect `vainfo`, `gst-inspect-1.0 vapostproc`, and -`gst-inspect-1.0 vah264enc` or `vah264lpenc`. HEVC needs `vah265enc` or -`vah265lpenc`. If the `va` plugin lists zero elements, check the driver and -access to `/dev/dri/renderD*` from the same user running wrec. +Wayland needs PipeWire and the ScreenCast portal backend for the desktop, such as +`xdg-desktop-portal-gnome` or `xdg-desktop-portal-kde`. Use the backend for your +desktop. X11 needs an accessible X server and the `ximagesrc` plugin. -Build a relocatable CLI archive with: +Install the driver for your GPU. Check `vainfo` and +`gst-inspect-1.0 vah264enc` / `vah264lpenc` for Intel/AMD, or +`gst-inspect-1.0 nvh264enc` for NVIDIA. HEVC uses the corresponding H.265 encoder. +If hardware elements are absent, inspect `x264enc`, `openh264enc`, or `x265enc`. +Distribution codec packaging and GPU capabilities vary. VA-API also needs access +to `/dev/dri/renderD*`; NVIDIA needs its driver and encode libraries. Missing +hardware plugins do not prevent software recording. ```bash ./scripts/package-cli-linux.sh ``` Extract the archive into a prefix such as `~/.local`. It contains `bin/wrec` -and `lib/wrec/daemon`. Put the prefix's `bin` directory on PATH. Native -GStreamer and driver libraries remain system dependencies. Run the CLI in -the logged-in desktop session; it starts the daemon automatically and -inherits the session's Wayland, D-Bus, PipeWire, and audio environment. -After changing sessions, stop and restart the daemon. +and `lib/wrec/daemon`. Put the prefix's `bin` directory on PATH. Native libraries +remain system dependencies. The archive targets the build machine's architecture +and libc; it is not a universal binary for every Linux distribution. + +Run the CLI in the logged-in desktop session. It starts the daemon automatically +and inherits the display, D-Bus, PipeWire, and audio environment. After changing +sessions, stop and restart the daemon. A container without the desktop sockets or +GPU device access cannot record the host desktop merely because the host has a GPU. ## Record @@ -59,57 +70,56 @@ wrec record --target display:0 --codec h264 --duration 10s wrec record --target window:0 --codec h264 --no-system-audio ``` -`display:0` and `window:0` open the desktop's source picker. They are not -native display or window IDs. Only source types advertised by the portal -appear in `targets`. The duration starts after the first encoded frame, -so time spent choosing a source does not shorten the recording. The picker -times out after two minutes and `wrec job stop ` cancels it. -Stopping before capture begins produces a cancelled job, without a movie. -Once capture starts, `--duration` counts wall time, including pauses; the -movie itself omits paused time. - -Permission status is `unknown` outside a recording because portal grants -belong to individual sessions. `permission.request` does not open a second -picker. Every recording asks for a source; restore tokens, app-name selection, -unattended capture, and arbitrary window enumeration are not implemented. - -System audio records the default output monitor through `pipewire-pulse`. -`--mic` adds the default microphone as a separate AAC track. Audio encoding -runs on the CPU. Use `--no-system-audio` to omit system audio; microphone capture -remains opt-in. Configure devices in the desktop's sound settings. A Linux -wrec window-hiding filter and custom microphone indicator are not provided; -the desktop controls its own screen-sharing indication. - -Pause and resume use the pipeline clock so paused time is omitted. Stop -drains the encoders and finalizes the movie, including when paused. The -writer emits ten-second movie fragments. An interrupted recording may only -be playable through its last completed fragment. A static desktop requests -one keepalive frame per second to keep the movie timeline advancing. - -X11, NVIDIA/NVENC, HDR capture, and desktop-specific capture shortcuts are -outside this first backend. A compositor/driver combination that cannot -share importable DMA-BUFs fails with an error. There is no silent fallback. +On Wayland, `display:0` and `window:0` open the desktop's source picker; only +advertised source types appear. On X11, use the window ID returned by `targets` +instead of `window:0`; named windows and X screen roots are enumerated directly. +A multi-monitor X screen is captured as one desktop. Wayland is preferred when +both session variables exist, because XWayland cannot capture the whole desktop. + +The duration starts after the first encoded frame, so time spent choosing a +source does not shorten recording. The portal picker times out after two minutes; +`wrec job stop ` cancels it. Stopping before capture begins produces a cancelled +job without a movie. After capture starts, `--duration` counts wall time including +pauses; the movie omits paused time. Failed starts do not report nonexistent files. + +Wayland permission status is `unknown` outside a recording because grants belong +to individual sessions. Every recording asks for a source. Restore tokens, +unattended Wayland capture, and Wayland app-name selection are not implemented. + +System audio uses the default PulseAudio output monitor, including through +`pipewire-pulse`. `--mic` adds the default microphone as a separate AAC track. +Audio encoding runs on the CPU. Use `--no-system-audio` when no audio service is +available. Configure devices in desktop sound settings. Linux cannot apply the +shared wrec window-hiding or custom microphone-indicator options; job settings +report them disabled with a warning. The desktop controls its sharing indicator. + +Stop drains the encoders and finalizes the movie, including when paused. The +writer emits ten-second movie fragments; an interrupted recording may only be +playable through its last completed fragment. PipeWire requests one keepalive +frame per second on a static desktop. HDR is not supported. ## Validate ```bash -sudo apt install ffmpeg dbus-daemon +sudo apt install ffmpeg dbus-daemon xvfb x11-apps cargo fmt --check cargo check --workspace --locked cargo test --workspace --locked -dbus-run-session -- cargo test -p linux portal_roundtrip -- --ignored +dbus-run-session -- cargo test -p linux portal_roundtrip --locked -- --ignored +cargo build -p cli -p daemon +python3 scripts/test-capture-linux.py target/debug/wrec ``` -The movie tests use synthetic video and a software encoder only inside the -test module. They check pause timing, stop while paused, real AAC tracks, -file readability, capture errors, and rejection of CPU buffers. The portal -test uses a mock ScreenCast service on an isolated D-Bus and checks source -options, denial, cancellation, and session cleanup. These tests do not prove -that a desktop exports DMA-BUFs or that a driver can import them. - -Before calling a GPU/desktop combination supported, record actual displays -and windows, include a static screen and motion, check cursor on/off, audio -and microphone sync, pause/resume, stop from the desktop sharing control, -and inspect files with `ffprobe`. Measure CPU, RSS, GPU memory, power, and -compositor overhead at 1080p30 and 4K60. Compare against an idle baseline on -the same machine and report the GPU, driver, desktop, and library versions. +The isolated Xvfb test records actual X11 display/window pixels through the CLI +and daemon, checks H.264/HEVC decoding and timestamps, and exercises pause/resume +and stopping while paused. It expects a machine without a hardware encoder so +software fallback is exercised. CI also tests the extracted package and headless +errors. Native pipeline tests cover AAC tracks, timing, capture errors, and strict +DMA-BUF rejection; an isolated mock portal covers options and session cleanup. + +These checks do not validate GPU buffer import or a real desktop portal. Before +calling a GPU/desktop combination validated, record displays and windows, static +screens and motion, cursor on/off, audio/microphone sync, and stopping from the +desktop sharing control. Measure CPU, RSS, GPU memory, power, and compositor cost +at 1080p30 and 4K60 against an idle baseline. Report GPU, driver, desktop, encoder, +and library versions. VA-API and NVIDIA performance remain unverified here. diff --git a/scripts/test-capture-linux.py b/scripts/test-capture-linux.py new file mode 100644 index 0000000..991bd2b --- /dev/null +++ b/scripts/test-capture-linux.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Record a real, isolated X11 desktop through the CLI, daemon, and native engine.""" +import json +import os +from pathlib import Path +import selectors +import subprocess +import sys +import tempfile +import time + + +def main(): + binary = Path(sys.argv[1]).resolve() + with tempfile.TemporaryDirectory(prefix="wrec-x11-capture-") as directory: + root = Path(directory) + env = os.environ.copy() + for key in ("WAYLAND_DISPLAY", "WREC_DAEMON_BIN", "WREC_CHANNEL", "WREC_HEADLESS"): + env.pop(key, None) + env.update(WREC_HOME=str(root / "home"), WREC_DATA_DIR=str(root / "data")) + with open(root / "xvfb.log", "w") as log: + desktop = subprocess.Popen(["Xvfb", "-displayfd", "1", "-screen", "0", "800x600x24", "-nolisten", "tcp"], stdout=subprocess.PIPE, stderr=log, text=True) + window = None + try: + with selectors.DefaultSelector() as selector: + selector.register(desktop.stdout, selectors.EVENT_READ) + assert selector.select(10), "Xvfb did not announce a display" + env["DISPLAY"] = ":" + desktop.stdout.readline().strip() + window = subprocess.Popen(["xmessage", "-title", "Wrec capture test", "-geometry", "500x250+50+50", "Wrec Linux capture: this is a real X11 test window."], env=env, stdout=log, stderr=log) + + def run(*args): + result = subprocess.run([str(binary), *args], env=env, capture_output=True, text=True, timeout=25) + assert result.returncode == 0, (args, result.stdout, result.stderr) + return json.loads(result.stdout) + + def job(job_id): + return run("job", "show", str(job_id), "--json")["job"] + + def wait(job_id, status): + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + value = job(job_id) + if value["status"] == status: + return value + assert value["status"] not in ("failed", "cancelled"), value + time.sleep(0.1) + raise AssertionError(value) + + def submit(target, codec, duration=None): + args = ["record", "--target", target, "--codec", codec, "--no-system-audio", "--out", str(root / "movies"), "--detach", "--json"] + if duration: + args.extend(["--duration", duration]) + return run(*args)["job"]["id"] + + def verify(value, codec): + path = value["output_path"] + assert Path(path).is_file(), value + probe = subprocess.run(["ffprobe", "-v", "error", "-show_streams", "-show_packets", "-of", "json", path], capture_output=True, text=True, check=True) + data = json.loads(probe.stdout) + assert data["streams"][0]["codec_name"] == codec, data["streams"] + previous = -float("inf") + for packet in data["packets"]: + dts = int(packet["dts"]) + assert dts > previous, (previous, dts) + previous = dts + decoded = subprocess.run(["ffmpeg", "-v", "error", "-i", path, "-vsync", "0", "-enc_time_base", "-1", "-f", "null", "-"], capture_output=True, text=True) + assert decoded.returncode == 0 and not decoded.stderr, decoded.stderr + assert value["settings"]["hide_wrec"] is False + assert value["settings"]["show_mic_indicator"] is False + assert any(w["code"] == "linux_settings_unavailable" for w in value["warnings"]), value + + for _ in range(50): + targets = run("targets", "--json") + windows = [t for t in targets if t["kind"] == "window" and "Wrec capture test" in t["name"]] + if windows: + break + time.sleep(0.1) + assert windows, targets + print("PASS: X11 display and named-window discovery") + + display_id = submit("display:0", "h264", "2s") + display = wait(display_id, "completed") + verify(display, "h264") + assert any("software encoding" in e["message"] for e in display["events"]), display + print("PASS: real X11 display capture, automatic software fallback, H.264 decode, increasing timestamps") + + window_id = submit(f'window:{windows[0]["id"]}', "hevc") + wait(window_id, "recording") + time.sleep(0.5) + run("job", "pause", str(window_id), "--json") + wait(window_id, "paused") + time.sleep(0.3) + run("job", "resume", str(window_id), "--json") + time.sleep(0.5) + run("job", "pause", str(window_id), "--json") + run("job", "stop", str(window_id), "--json") + verify(wait(window_id, "completed"), "hevc") + print("PASS: real X11 window capture, HEVC decode, pause/resume, and finalization while paused") + run("daemon", "stop", "--json") + finally: + subprocess.run([str(binary), "daemon", "stop", "--json"], env=env, capture_output=True, timeout=20) + if window: + window.terminate() + window.wait(timeout=5) + desktop.terminate() + desktop.wait(timeout=5) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-cli-linux.py b/scripts/test-cli-linux.py index b59c88f..570f59a 100644 --- a/scripts/test-cli-linux.py +++ b/scripts/test-cli-linux.py @@ -17,7 +17,7 @@ def main(): with tempfile.TemporaryDirectory(prefix="wrec-linux-smoke-") as directory: env = os.environ.copy() for key in ("WREC_DAEMON_BIN", "WREC_CAPTURE_ENGINE_PATH", "WREC_HEADLESS", - "WREC_CHANNEL", "WAYLAND_DISPLAY", "XDG_RUNTIME_DIR"): + "WREC_CHANNEL", "WAYLAND_DISPLAY", "DISPLAY", "XDG_RUNTIME_DIR"): env.pop(key, None) env["WREC_HOME"] = str(Path(directory) / "home") env["WREC_DATA_DIR"] = str(Path(directory) / "data") @@ -28,6 +28,11 @@ def run(*args, success=True): assert (result.returncode == 0) == success, (args, result.stdout, result.stderr) return result + # A sibling executable must not shadow the archive's own runtime. + decoy = prefix / "bin" / "daemon" + assert not decoy.exists(), decoy + decoy.write_text("#!/bin/sh\nexit 99\n") + decoy.chmod(0o755) pid = None try: status = json.loads(run("daemon", "start", "--json").stdout) @@ -38,11 +43,11 @@ def run(*args, success=True): print("PASS: packaged CLI automatically launches its relative Linux daemon") result = run("targets", "--json", success=False) - assert "Wayland desktop session" in result.stdout + result.stderr + assert "Wayland or X11 desktop session" in result.stdout + result.stderr print("PASS: headless target discovery returns a useful error") result = run("record", "--target", "display:0", "--duration", "1s", "--json", success=False) - assert "Wayland desktop session" in result.stdout + result.stderr + assert "Wayland or X11 desktop session" in result.stdout + result.stderr assert not list(Path(directory).rglob("*.mov")) print("PASS: unsupported capture fails without a recording artifact") @@ -71,6 +76,7 @@ def run(*args, success=True): pid = None print("PASS: restarted daemon handles SIGTERM and removes the socket") finally: + decoy.unlink() if pid is not None: try: os.kill(pid, signal.SIGTERM) From 537ceeecf999bc20ecbb1c7b1dc5af0dab33a78e Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 5 Sep 2026 06:31:43 +0000 Subject: [PATCH 05/45] fix: exclude source selection from recording history duration --- crates/daemon/src/backend.rs | 2 +- crates/daemon/src/store.rs | 59 +++++++++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/crates/daemon/src/backend.rs b/crates/daemon/src/backend.rs index bdc84f3..44ab7ec 100644 --- a/crates/daemon/src/backend.rs +++ b/crates/daemon/src/backend.rs @@ -241,7 +241,7 @@ impl WrecBackend { fn mark_recording_started(&self, session_id: u64) { if let Some(store) = &self.store { - store.mark_recording_started(session_id); + store.mark_recording_started(session_id, now_ms()); } } diff --git a/crates/daemon/src/store.rs b/crates/daemon/src/store.rs index 1ce63cb..adae4a0 100644 --- a/crates/daemon/src/store.rs +++ b/crates/daemon/src/store.rs @@ -121,6 +121,7 @@ enum StoreCommand { UpsertRecording(RecordingRecord), MarkRecordingStarted { id: u64, + started_at_ms: i64, }, MarkRecordingFinished { id: u64, @@ -174,8 +175,8 @@ impl Store { self.send(StoreCommand::UpsertRecording(recording)); } - pub fn mark_recording_started(&self, id: u64) { - self.send(StoreCommand::MarkRecordingStarted { id }); + pub fn mark_recording_started(&self, id: u64, started_at_ms: i64) { + self.send(StoreCommand::MarkRecordingStarted { id, started_at_ms }); } pub fn mark_recording_completed( @@ -403,7 +404,9 @@ fn writer_loop(conn: Connection, receiver: mpsc::Receiver) { let kind = command.kind(); let result = match command { StoreCommand::UpsertRecording(recording) => upsert_recording(&conn, &recording), - StoreCommand::MarkRecordingStarted { id } => mark_recording_started(&conn, id), + StoreCommand::MarkRecordingStarted { id, started_at_ms } => { + mark_recording_started(&conn, id, started_at_ms) + } StoreCommand::MarkRecordingFinished { id, status, @@ -496,10 +499,14 @@ fn upsert_recording(conn: &Connection, recording: &RecordingRecord) -> rusqlite: Ok(()) } -fn mark_recording_started(conn: &Connection, id: u64) -> rusqlite::Result<()> { +fn mark_recording_started(conn: &Connection, id: u64, started_at_ms: i64) -> rusqlite::Result<()> { conn.execute( - "UPDATE recordings SET status = ?1, error_message = NULL WHERE id = ?2", - params![RecordingStatus::Recording.as_str(), u64_to_i64(id)], + "UPDATE recordings SET status = ?1, started_at_ms = ?3, error_message = NULL WHERE id = ?2", + params![ + RecordingStatus::Recording.as_str(), + u64_to_i64(id), + started_at_ms + ], )?; Ok(()) } @@ -518,7 +525,7 @@ fn mark_recording_finished( SET status = ?1, stopped_at_ms = ?2, - duration_ms = MAX(0, ?2 - started_at_ms), + duration_ms = CASE WHEN status = 'starting' AND ?1 != 'completed' THEN 0 ELSE MAX(0, ?2 - started_at_ms) END, file_size_bytes = COALESCE(?3, file_size_bytes), error_message = ?4 WHERE id = ?5 @@ -937,9 +944,9 @@ mod tests { { let store = Store::open(db.path()).expect("first open"); store.upsert_recording(sample_recording(1)); - store.mark_recording_started(1); + store.mark_recording_started(1, 1_000); store.upsert_recording(sample_recording(2)); - store.mark_recording_started(2); + store.mark_recording_started(2, 1_000); store.mark_recording_completed(2, 2000, Some(64)); } @@ -1050,7 +1057,7 @@ mod tests { let store = Store::open(db.path()).expect("open store"); store.upsert_recording(sample_recording(1)); store.mark_recording_failed(1, 2_000, "engine crashed".to_string()); - store.mark_recording_started(1); + store.mark_recording_started(1, 1_000); } let conn = read_connection(&db); @@ -1066,6 +1073,36 @@ mod tests { assert_eq!(error, None); } + #[test] + fn history_duration_excludes_picker_delay_and_cancelled_attempts() { + let db = TempDb::new("picker-duration"); + { + let store = Store::open(db.path()).unwrap(); + store.upsert_recording(sample_recording(1)); // Attempt at 1s. + store.mark_recording_started(1, 61_000); // Picker took one minute. + store.mark_recording_completed(1, 66_000, Some(1)); + store.upsert_recording(sample_recording(2)); + store.mark_recording_cancelled(2, 61_000); + } + let conn = read_connection(&db); + let recorded: (i64, i64) = conn + .query_row( + "SELECT started_at_ms, duration_ms FROM recordings WHERE id = 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(recorded, (61_000, 5_000)); + let cancelled: (String, i64) = conn + .query_row( + "SELECT status, duration_ms FROM recordings WHERE id = 2", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(cancelled, ("cancelled".into(), 0)); + } + #[test] fn mark_recording_completed_sets_duration_and_file_size() { let db = TempDb::new("completed"); @@ -1196,7 +1233,7 @@ mod tests { let db = TempDb::new("missing-row"); { let store = Store::open(db.path()).expect("open store"); - store.mark_recording_started(99); + store.mark_recording_started(99, 1_000); store.mark_recording_completed(99, 5_000, Some(1)); store.update_dimensions( 99, From c278524b2fc39345f48a396ceb1679ca5940ac8b Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 5 Sep 2026 06:36:33 +0000 Subject: [PATCH 06/45] fix(linux): open a fresh PipeWire remote for encoder retries --- crates/linux/src/lib.rs | 19 ++++++++++-------- crates/linux/src/pipeline.rs | 9 +++++++-- crates/linux/src/portal.rs | 34 +++++++++++++++++++++----------- crates/linux/src/portal_tests.rs | 8 +++++++- 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/crates/linux/src/lib.rs b/crates/linux/src/lib.rs index 5e9db61..af07370 100644 --- a/crates/linux/src/lib.rs +++ b/crates/linux/src/lib.rs @@ -14,7 +14,7 @@ use futures_util::StreamExt; use std::{ sync::{ atomic::{AtomicU64, Ordering}, - mpsc, + mpsc, Arc, }, thread, time::{Duration, SystemTime, UNIX_EPOCH}, @@ -139,18 +139,21 @@ impl RecorderEngine for LinuxRecorder { .build() .map_err(backend)?; runtime.block_on(async { - let capture = - portal::open(&target, settings.include_cursor, &mut stopped).await?; + let capture = Arc::new( + portal::open(&target, settings.include_cursor, &mut stopped).await?, + ); let result = async { let mut closed = capture.session.receive_closed().await.map_err(backend)?; - let stream = portal::PipeWireStream { - fd: capture.stream.fd.try_clone().map_err(backend)?, - node: capture.stream.node, - }; + let worker_capture = capture.clone(); + let runtime = tokio::runtime::Handle::current(); let mut recording = tokio::task::spawn_blocking(move || { pipeline::record( - &pipeline::CaptureInput::PipeWire(stream), + &pipeline::CaptureInput::PipeWire(Box::new(move || { + // Each retry needs a new protocol connection. Duplicating + // an already-used socket does not reset its remote state. + runtime.block_on(worker_capture.connect()) + })), &worker_session, &settings, &events, diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 815575f..88d276b 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -97,7 +97,7 @@ struct Counters { } pub(crate) enum CaptureInput { - PipeWire(PipeWireStream), + PipeWire(Box Result + Send>), X11 { display: String, xid: u64 }, } @@ -167,11 +167,16 @@ fn record_attempt( return Err(RecorderError::Cancelled); } let settings = settings.clone().with_preset_limits(); + let stream = match capture { + CaptureInput::PipeWire(connect) => Some(connect()?), + CaptureInput::X11 { .. } => None, + }; let pipeline = PipelineGuard(gst::Pipeline::new()); pipeline.0.use_clock(Some(&gst::SystemClock::obtain())); let mode = attempt.mode; let source = match capture { - CaptureInput::PipeWire(capture) => { + CaptureInput::PipeWire(_) => { + let capture = stream.as_ref().unwrap(); let source = element("pipewiresrc")?; source.set_property("fd", capture.fd.as_raw_fd()); source.set_property("path", capture.node.to_string()); diff --git a/crates/linux/src/portal.rs b/crates/linux/src/portal.rs index c8ac5a1..a0043f0 100644 --- a/crates/linux/src/portal.rs +++ b/crates/linux/src/portal.rs @@ -18,7 +18,7 @@ pub fn check_desktop() -> Result<()> { if std::env::var_os("WAYLAND_DISPLAY").is_none() || std::env::var_os("XDG_RUNTIME_DIR").is_none() { - return Err(backend("Linux recording requires a Wayland desktop session. X11 and headless capture are not supported.")); + return Err(backend("Wayland capture requires WAYLAND_DISPLAY and XDG_RUNTIME_DIR from the logged-in desktop session.")); } Ok(()) } @@ -72,10 +72,10 @@ pub(crate) fn validate_target(target: &CaptureTarget) -> Result<()> { } pub(crate) struct Capture { - pub stream: PipeWireStream, + node: u32, pub session: Session<'static, Screencast<'static>>, // Keep the portal connection alive until the pipeline and session close. - _portal: Screencast<'static>, + portal: Screencast<'static>, } pub(crate) struct PipeWireStream { @@ -84,7 +84,21 @@ pub(crate) struct PipeWireStream { } impl Capture { - pub async fn close(self) { + pub async fn connect(&self) -> Result { + let fd = tokio::time::timeout( + Duration::from_secs(5), + self.portal.open_pipe_wire_remote(&self.session), + ) + .await + .map_err(|_| backend("Opening the PipeWire remote timed out"))? + .map_err(backend)?; + Ok(PipeWireStream { + fd, + node: self.node, + }) + } + + pub async fn close(&self) { let _ = tokio::time::timeout(Duration::from_secs(3), self.session.close()).await; } } @@ -150,11 +164,7 @@ pub(crate) async fn open( )); } let node = response.streams()[0].pipe_wire_node_id(); - let fd = portal - .open_pipe_wire_remote(&session) - .await - .map_err(backend)?; - Ok((fd, node)) + Ok(node) }; let result = tokio::select! { result = selected => result, @@ -162,10 +172,10 @@ pub(crate) async fn open( _ = tokio::time::sleep(Duration::from_secs(120)) => Err(backend("Desktop source selection timed out after 120s; start again and choose a source.")), }; match result { - Ok((fd, node)) => Ok(Capture { - stream: PipeWireStream { fd, node }, + Ok(node) => Ok(Capture { + node, session, - _portal: portal, + portal, }), Err(error) => { let _ = tokio::time::timeout(Duration::from_secs(3), session.close()).await; diff --git a/crates/linux/src/portal_tests.rs b/crates/linux/src/portal_tests.rs index 4517b4e..a577f96 100644 --- a/crates/linux/src/portal_tests.rs +++ b/crates/linux/src/portal_tests.rs @@ -17,6 +17,7 @@ use zbus::{ struct MockPortal { mode: Arc, closes: Arc, + remotes: Arc, selections: Arc>>, } @@ -155,6 +156,7 @@ impl MockPortal { _session: OwnedObjectPath, _options: HashMap, ) -> zbus::zvariant::OwnedFd { + self.remotes.fetch_add(1, Ordering::Relaxed); let (socket, _peer) = UnixStream::pair().unwrap(); let fd: OwnedFd = socket.into(); fd.into() @@ -188,7 +190,11 @@ fn portal_roundtrip_and_cancellation_close_sessions() { .await .unwrap() .unwrap(); - assert_eq!(capture.stream.node, 42); + assert_eq!(capture.connect().await.unwrap().node, 42); + assert_eq!(capture.connect().await.unwrap().node, 42); + assert_eq!(mock.remotes.load(Ordering::Relaxed), 2); + // Retrying the encoder opens a new remote, not a second source picker. + assert_eq!(mock.selections.lock().unwrap().len(), 1); capture.close().await; assert_eq!(mock.closes.load(Ordering::Relaxed), 1); assert_eq!(*mock.selections.lock().unwrap(), vec![(1, 2, false)]); From 5afccdbb1d3d379a3de403a57130d7c2298e1650 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 5 Sep 2026 09:31:46 +0000 Subject: [PATCH 07/45] fix(linux): validate and repair native Wayland capture on Radeon --- crates/linux/src/lib.rs | 5 +- crates/linux/src/pipeline.rs | 236 ++++++++++++++++++++++++------- crates/linux/src/portal.rs | 17 ++- crates/linux/src/portal_tests.rs | 6 + packaging/linux/README.md | 28 ++-- 5 files changed, 225 insertions(+), 67 deletions(-) diff --git a/crates/linux/src/lib.rs b/crates/linux/src/lib.rs index af07370..d7a3061 100644 --- a/crates/linux/src/lib.rs +++ b/crates/linux/src/lib.rs @@ -134,10 +134,7 @@ impl RecorderEngine for LinuxRecorder { &stopped, ); } - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(backend)?; + let runtime = portal::runtime()?; runtime.block_on(async { let capture = Arc::new( portal::open(&target, settings.include_cursor, &mut stopped).await?, diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 88d276b..96c59db 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -72,6 +72,24 @@ fn output_size(width: i32, height: i32, resolution: Resolution) -> (i32, i32) { (even(width), even(height)) } +fn capture_caps(dmabuf: bool, pipewire: bool, fps: u32) -> gst::Caps { + let caps = gst::Caps::builder("video/x-raw").field( + // Wayland compositors advertise variable-rate frames (framerate=0/1). + // Bound their maximum rate without requiring a fixed-rate source. + if pipewire { + "max-framerate" + } else { + "framerate" + }, + gst::Fraction::new(fps as i32, 1), + ); + if dmabuf { + caps.features(["memory:DMABuf"]).build() + } else { + caps.build() + } +} + fn video_queue() -> Result { let queue = element("queue")?; queue.set_property("max-size-buffers", 2u32); @@ -94,6 +112,45 @@ struct Counters { dropped: AtomicU64, dimensions: Mutex>, last_pts: Mutex>, + timeline: Mutex, +} + +#[derive(Default)] +struct Timeline { + paused_at: Option, + offset: gst::ClockTime, + accept_from: gst::ClockTime, +} + +impl Timeline { + fn pause(&mut self, now: gst::ClockTime) { + self.paused_at.get_or_insert(now); + } + + fn resume(&mut self, now: gst::ClockTime) { + if let Some(paused_at) = self.paused_at.take() { + self.offset += now.saturating_sub(paused_at); + self.accept_from = now; + } + } + + fn position(&self, now: gst::ClockTime) -> gst::ClockTime { + self.paused_at.unwrap_or(now).saturating_sub(self.offset) + } + + fn retime(&self, buffer: &mut gst::Buffer) -> bool { + if self.paused_at.is_some() || buffer.pts().is_some_and(|pts| pts < self.accept_from) { + return false; + } + if self.offset > gst::ClockTime::ZERO { + // make_mut copies a shared buffer header, retaining references to + // its memory. DMA-BUF pixels remain on the GPU. + let buffer = buffer.make_mut(); + buffer.set_pts(buffer.pts().map(|pts| pts.saturating_sub(self.offset))); + buffer.set_dts(buffer.dts().map(|dts| dts.saturating_sub(self.offset))); + } + true + } } pub(crate) enum CaptureInput { @@ -198,16 +255,14 @@ fn record_attempt( }; source.set_property("do-timestamp", true); let input = element("capsfilter")?; - let caps = gst::Caps::builder("video/x-raw").field( - "framerate", - gst::Fraction::new(settings.fps.as_u32() as i32, 1), + input.set_property( + "caps", + capture_caps( + mode.dmabuf, + matches!(capture, CaptureInput::PipeWire(_)), + settings.fps.as_u32(), + ), ); - let caps = if mode.dmabuf { - caps.features(["memory:DMABuf"]).build() - } else { - caps.build() - }; - input.set_property("caps", caps); let queue = video_queue()?; let converters = mode.converters()?; let output = element("capsfilter")?; @@ -236,13 +291,13 @@ fn record_attempt( .add_many(chain.iter().copied()) .map_err(backend)?; gst::Element::link_many(chain.iter().copied()).map_err(backend)?; + let counters = attempt.counters.clone(); if settings.include_system_audio { - add_audio(&pipeline.0, &mux, Some("@DEFAULT_MONITOR@"))?; + add_audio(&pipeline.0, &mux, Some("@DEFAULT_MONITOR@"), &counters)?; } if settings.include_microphone { - add_audio(&pipeline.0, &mux, None)?; + add_audio(&pipeline.0, &mux, None, &counters)?; } - let counters = attempt.counters.clone(); attach_capture_probe( &source, &output, @@ -290,20 +345,27 @@ fn attach_encoded_probe(parser: &gst::Element, counters: Arc) { }); } -fn add_audio(pipeline: &gst::Pipeline, mux: &gst::Element, device: Option<&str>) -> Result<()> { +fn add_audio( + pipeline: &gst::Pipeline, + mux: &gst::Element, + device: Option<&str>, + counters: &Arc, +) -> Result<()> { let source = element("pulsesrc")?; source.set_property("provide-clock", false); if let Some(device) = device { source.set_property("device", device); } - add_audio_source(pipeline, mux, &source) + add_audio_source(pipeline, mux, &source, counters) } fn add_audio_source( pipeline: &gst::Pipeline, mux: &gst::Element, source: &gst::Element, + counters: &Arc, ) -> Result<()> { + attach_timing_probe(source, counters.clone()); let convert = element("audioconvert")?; let resample = element("audioresample")?; let caps = element("capsfilter")?; @@ -330,6 +392,26 @@ fn add_audio_source( Ok(()) } +fn retime(info: &mut gst::PadProbeInfo<'_>, counters: &Counters) -> bool { + if let Some(gst::PadProbeData::Buffer(buffer)) = info.data.as_mut() { + return counters.timeline.lock().unwrap().retime(buffer); + } + true +} + +fn attach_timing_probe(source: &gst::Element, counters: Arc) { + source + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + if retime(info, &counters) { + gst::PadProbeReturn::Ok + } else { + gst::PadProbeReturn::Drop + } + }); +} + fn attach_capture_probe( source: &gst::Element, output: &gst::Element, @@ -339,6 +421,7 @@ fn attach_capture_probe( ) { let output = output.clone(); source.static_pad("src").unwrap().add_probe(gst::PadProbeType::EVENT_DOWNSTREAM | gst::PadProbeType::BUFFER, move |pad, info| { + if !retime(info, &counters) { return gst::PadProbeReturn::Drop; } if let Some(gst::PadProbeData::Event(event)) = &info.data { if let gst::EventView::Caps(caps) = event.view() { if let Some(structure) = caps.caps().structure(0) { @@ -377,16 +460,26 @@ fn attach_capture_probe( }); } -fn set_state(pipeline: &gst::Pipeline, state: gst::State) -> Result<()> { - pipeline.set_state(state).map_err(backend)?; - let (result, current, _) = pipeline.state(gst::ClockTime::from_seconds(3)); - result.map_err(backend)?; - if current != state { - return Err(backend(format!( - "recording did not enter {state:?} within 3s" - ))); +fn state_error(pipeline: &gst::Pipeline, fallback: impl std::fmt::Display) -> RecorderError { + if let Some(message) = pipeline.bus().and_then(|bus| { + bus.timed_pop_filtered( + gst::ClockTime::from_mseconds(100), + &[gst::MessageType::Error], + ) + }) { + if let gst::MessageView::Error(error) = message.view() { + return backend(format!( + "{}: {} ({})", + error + .src() + .map(|src| src.name().to_string()) + .unwrap_or_default(), + error.error(), + error.debug().unwrap_or_default() + )); + } } - Ok(()) + backend(fallback) } fn run( @@ -397,7 +490,9 @@ fn run( stop: &watch::Receiver, counters: &Counters, ) -> Result<()> { - pipeline.set_state(gst::State::Playing).map_err(backend)?; + pipeline + .set_state(gst::State::Playing) + .map_err(|error| state_error(pipeline, error))?; let bus = pipeline .bus() .ok_or_else(|| backend("recording pipeline has no bus"))?; @@ -410,22 +505,34 @@ fn run( if !started && counters.frames.load(Ordering::Relaxed) == 0 { return Err(RecorderError::Cancelled); } - // Live sources must run to drain their pending buffers and finalize a paused movie. - pipeline.set_state(gst::State::Playing).map_err(backend)?; + // Reopen the input gate so EOS can carry the final frame. Keeping + // native sources PLAYING avoids PipeWire pause/resume renegotiation. + counters + .timeline + .lock() + .unwrap() + .resume(pipeline.current_running_time().unwrap_or_default()); if !pipeline.send_event(gst::event::Eos::new()) { return Err(backend("recording pipeline rejected finalization")); } stopping = Some(Instant::now()); } if let Ok(command) = commands.try_recv() { - let (state, reply) = match command { - Command::Pause(reply) => (gst::State::Paused, reply), - Command::Resume(reply) => (gst::State::Playing, reply), + let (pause, reply) = match command { + Command::Pause(reply) => (true, reply), + Command::Resume(reply) => (false, reply), }; let result = if stopping.is_some() || !started { Err(backend("recording is not ready for pause/resume")) } else { - set_state(pipeline, state) + let now = pipeline.current_running_time().unwrap_or_default(); + let mut timeline = counters.timeline.lock().unwrap(); + if pause { + timeline.pause(now); + } else { + timeline.resume(now); + } + Ok(()) }; let _ = reply.send(result); } @@ -480,10 +587,12 @@ fn emit_metrics( events: &mpsc::Sender, counters: &Counters, ) { - let elapsed_secs = pipeline - .query_position::() - .map(|time| time.seconds()) - .unwrap_or(0); + let elapsed_secs = counters + .timeline + .lock() + .unwrap() + .position(pipeline.current_running_time().unwrap_or_default()) + .seconds(); let output_bytes = file_size(&session.output_path); let _ = events.send(RecorderEvent::Metrics { session_id: session.id, @@ -511,6 +620,26 @@ fn file_size(path: &Path) -> u64 { mod tests { use super::*; + #[test] + fn accepts_variable_rate_portal_frames_and_caps_their_maximum() { + gst::init().unwrap(); + let portal = gst::Caps::builder("video/x-raw") + .features(["memory:DMABuf"]) + .field("framerate", gst::Fraction::new(0, 1)) + .field("max-framerate", gst::Fraction::new(30, 1)) + .build(); + assert!(capture_caps(true, true, 30).can_intersect(&portal)); + assert!(!capture_caps(true, false, 30).can_intersect(&portal)); + assert_eq!( + capture_caps(true, true, 30) + .structure(0) + .unwrap() + .get::("max-framerate") + .unwrap(), + gst::Fraction::new(30, 1) + ); + } + #[test] fn dimensions_preserve_aspect_and_do_not_upscale() { assert_eq!(output_size(3840, 2160, Resolution::R1080p), (1920, 1080)); @@ -526,6 +655,7 @@ mod tests { commands: mpsc::SyncSender, stop: watch::Sender, worker: Option>>, + counters: Arc, } impl TestRecording { @@ -540,24 +670,32 @@ mod tests { }; // Synthetic software encoding is confined to tests. Exercise the same // bus/control/mux code without pretending this is a hardware benchmark. - let pipeline = gst::parse::launch("videotestsrc is-live=true pattern=ball ! video/x-raw,width=320,height=180,framerate=30/1 ! openh264enc ! h264parse name=parser ! qtmux name=mux fragment-duration=10000 ! filesink name=output sync=false") + let pipeline = gst::parse::launch("videotestsrc name=video is-live=true pattern=ball ! video/x-raw,width=320,height=180,framerate=30/1 ! openh264enc ! h264parse name=parser ! qtmux name=mux fragment-duration=10000 trak-timescale=1000000 ! filesink name=output sync=false") .unwrap().downcast::().unwrap(); pipeline .by_name("output") .unwrap() .set_property("location", session.output_path.to_str().unwrap()); + let counters = Arc::new(Counters::default()); + attach_timing_probe(&pipeline.by_name("video").unwrap(), counters.clone()); for _ in 0..audio_tracks { let source = element("audiotestsrc").unwrap(); source.set_property("is-live", true); - add_audio_source(&pipeline, &pipeline.by_name("mux").unwrap(), &source).unwrap(); + add_audio_source( + &pipeline, + &pipeline.by_name("mux").unwrap(), + &source, + &counters, + ) + .unwrap(); } - let counters = Arc::new(Counters::default()); attach_encoded_probe(&pipeline.by_name("parser").unwrap(), counters.clone()); let (events_tx, events) = mpsc::channel(); let (commands, commands_rx) = mpsc::sync_channel(1); let (stop, stopped) = watch::channel(false); let worker_pipeline = pipeline.clone(); let worker_session = session.clone(); + let worker_counters = counters.clone(); let worker = std::thread::spawn(move || { let guard = PipelineGuard(worker_pipeline); run( @@ -566,7 +704,7 @@ mod tests { &events_tx, &commands_rx, &stopped, - &counters, + &worker_counters, ) }); let recording = Self { @@ -576,6 +714,7 @@ mod tests { commands, stop, worker: Some(worker), + counters, }; loop { if matches!( @@ -703,26 +842,23 @@ mod tests { #[test] fn pause_resume_omits_paused_time_and_stop_while_paused_finalizes() { - let mut recording = TestRecording::start(0); + let mut recording = TestRecording::start(2); std::thread::sleep(Duration::from_millis(400)); recording.control(true); - let before = recording - .pipeline - .query_position::() - .unwrap(); + std::thread::sleep(Duration::from_millis(100)); // Drain frames already accepted. + let before = recording.counters.frames.load(Ordering::Relaxed); std::thread::sleep(Duration::from_millis(600)); - let after = recording - .pipeline - .query_position::() - .unwrap(); - assert!(after.saturating_sub(before) < gst::ClockTime::from_mseconds(100)); + let after = recording.counters.frames.load(Ordering::Relaxed); + assert_eq!(after, before, "video frames must stop while paused"); recording.control(false); std::thread::sleep(Duration::from_millis(400)); recording.control(true); let active_duration = recording - .pipeline - .query_position::() + .counters + .timeline + .lock() .unwrap() + .position(recording.pipeline.current_running_time().unwrap()) .nseconds() as f64 / 1_000_000_000.0; recording.finish().unwrap(); diff --git a/crates/linux/src/portal.rs b/crates/linux/src/portal.rs index a0043f0..3324446 100644 --- a/crates/linux/src/portal.rs +++ b/crates/linux/src/portal.rs @@ -23,10 +23,19 @@ pub fn check_desktop() -> Result<()> { Ok(()) } -fn runtime() -> Result { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() +pub(crate) fn runtime() -> Result<&'static tokio::runtime::Runtime> { + // ashpd caches its D-Bus connection globally. Its I/O tasks must outlive + // individual target queries and recordings, even when called on new threads. + static RUNTIME: std::sync::OnceLock> = + std::sync::OnceLock::new(); + RUNTIME + .get_or_init(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| error.to_string()) + }) + .as_ref() .map_err(backend) } diff --git a/crates/linux/src/portal_tests.rs b/crates/linux/src/portal_tests.rs index a577f96..b8c558f 100644 --- a/crates/linux/src/portal_tests.rs +++ b/crates/linux/src/portal_tests.rs @@ -179,6 +179,12 @@ fn portal_roundtrip_and_cancellation_close_sessions() { .build() .await .unwrap(); + tokio::task::spawn_blocking(|| { + assert!(!list_targets().unwrap().is_empty()); + assert!(!list_targets().unwrap().is_empty()); + }) + .await + .unwrap(); let target = CaptureTarget { id: 0, name: "picker".into(), diff --git a/packaging/linux/README.md b/packaging/linux/README.md index ec486dc..aa953fd 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -4,8 +4,8 @@ The Linux backend records Wayland desktops through the ScreenCast portal and PipeWire, or X11 displays and windows through XShm/XImage. It selects an installed encoder automatically: Intel/AMD VA-API, NVIDIA NVENC, then software encoding. A working desktop and the required native plugins are needed. There is no Linux -GUI yet. Hardware compatibility and efficiency still need measurements on real -desktops; software fallback makes recording possible without a supported GPU, +GUI yet. AMD Radeon capture has been tested on GNOME Wayland; Intel and NVIDIA hardware +remain unverified. Software fallback makes recording possible without a supported GPU, but uses more CPU. On compatible Wayland/VA-API systems, the preferred pipeline is @@ -80,7 +80,10 @@ The duration starts after the first encoded frame, so time spent choosing a source does not shorten recording. The portal picker times out after two minutes; `wrec job stop ` cancels it. Stopping before capture begins produces a cancelled job without a movie. After capture starts, `--duration` counts wall time including -pauses; the movie omits paused time. Failed starts do not report nonexistent files. +pauses; the movie omits paused time. Pause keeps the approved source streams alive but drops +video and audio before conversion/encoding; timestamps remove the pause on resume. +Rust only changes buffer metadata, preserving GPU memory. This also avoids +PipeWire source renegotiation on resume. Failed starts do not report nonexistent files. Wayland permission status is `unknown` outside a recording because grants belong to individual sessions. Every recording asks for a source. Restore tokens, @@ -117,9 +120,16 @@ software fallback is exercised. CI also tests the extracted package and headless errors. Native pipeline tests cover AAC tracks, timing, capture errors, and strict DMA-BUF rejection; an isolated mock portal covers options and session cleanup. -These checks do not validate GPU buffer import or a real desktop portal. Before -calling a GPU/desktop combination validated, record displays and windows, static -screens and motion, cursor on/off, audio/microphone sync, and stopping from the -desktop sharing control. Measure CPU, RSS, GPU memory, power, and compositor cost -at 1080p30 and 4K60 against an idle baseline. Report GPU, driver, desktop, encoder, -and library versions. VA-API and NVIDIA performance remain unverified here. +Real hardware validation on Corex (Ryzen 9 5900H / Radeon Cezanne, Ubuntu 26.04, +GNOME Wayland, GStreamer 1.28.2) recorded H.264 displays and an animated HEVC +window through DMA-BUF and VA-API with no fallback. Video/audio decode and +strictly increasing timestamps passed, including live system audio, pause/resume, +and stopping while paused. The 1080p animated-window sample used about 4% of one +CPU core and 129 MiB peak daemon RSS. It recorded about 31 fps with a requested +60 fps ceiling; this is not sustained-60-fps validation or a general benchmark. + +Intel/NVIDIA, other compositors, live microphone sync, desktop sharing-control +stops, HDR, sustained 60 fps, 4K, GPU memory and power need separate validation. +For each configuration, compare CPU, RSS, GPU memory, power and compositor cost +against an idle baseline, and report GPU, driver, desktop, encoder and library +versions. Software/Xvfb tests alone cannot prove DMA-BUF import or GPU efficiency. From 0624a9fa1a04e617630f0596b08f8a96e1bbd6bd Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Wed, 30 Sep 2026 08:06:33 +0000 Subject: [PATCH 08/45] fix(linux): survive X11 source loss and reclaim idle capture memory --- crates/linux/src/pipeline.rs | 37 +++++++- crates/linux/src/x11.rs | 174 +++++++++++++++++++++++++++++++++- packaging/linux/README.md | 4 + scripts/test-capture-linux.py | 35 ++++++- 4 files changed, 244 insertions(+), 6 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 96c59db..7626b4a 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -195,6 +195,7 @@ pub(crate) fn record( let result = record_attempt( capture, session, settings, events, &commands, stop, &attempt, ); + release_freed_memory(); if result.is_ok() || matches!(result, Err(RecorderError::Cancelled)) || attempt.counters.frames.load(Ordering::Relaxed) > 0 @@ -211,6 +212,19 @@ pub(crate) fn record( Err(last_error.unwrap_or_else(|| backend("No recording pipeline could start"))) } +#[cfg(target_env = "gnu")] +fn release_freed_memory() { + extern "C" { + fn malloc_trim(pad: usize) -> std::os::raw::c_int; + } + unsafe { + malloc_trim(0); + } +} + +#[cfg(not(target_env = "gnu"))] +fn release_freed_memory() {} + fn record_attempt( capture: &CaptureInput, session: &RecordingSession, @@ -228,6 +242,10 @@ fn record_attempt( CaptureInput::PipeWire(connect) => Some(connect()?), CaptureInput::X11 { .. } => None, }; + let source_watch = match capture { + CaptureInput::X11 { xid, .. } if *xid != 0 => Some(crate::x11::SourceWatch::new(*xid)?), + _ => None, + }; let pipeline = PipelineGuard(gst::Pipeline::new()); pipeline.0.use_clock(Some(&gst::SystemClock::obtain())); let mode = attempt.mode; @@ -325,7 +343,15 @@ fn record_attempt( .create_new(true) .open(&session.output_path) .map_err(backend)?; - let result = run(&pipeline.0, session, events, commands, stop, &counters); + let result = run( + &pipeline.0, + session, + events, + commands, + stop, + &counters, + &|| source_watch.as_ref().is_some_and(|watch| watch.lost()), + ); let _ = pipeline.0.set_state(gst::State::Null); if matches!(result, Err(RecorderError::Cancelled)) || counters.frames.load(Ordering::Relaxed) == 0 @@ -482,6 +508,9 @@ fn state_error(pipeline: &gst::Pipeline, fallback: impl std::fmt::Display) -> Re backend(fallback) } +const SOURCE_LOST: &str = + "The captured X11 window closed or became unavailable; the recording stopped."; + fn run( pipeline: &gst::Pipeline, session: &RecordingSession, @@ -489,6 +518,7 @@ fn run( commands: &mpsc::Receiver, stop: &watch::Receiver, counters: &Counters, + source_lost: &dyn Fn() -> bool, ) -> Result<()> { pipeline .set_state(gst::State::Playing) @@ -501,6 +531,9 @@ fn run( let mut stopping = None; let mut last_metrics = Instant::now(); loop { + if stopping.is_none() && source_lost() { + return Err(backend(SOURCE_LOST)); + } if *stop.borrow() && stopping.is_none() { if !started && counters.frames.load(Ordering::Relaxed) == 0 { return Err(RecorderError::Cancelled); @@ -538,6 +571,7 @@ fn run( } if let Some(message) = bus.timed_pop(gst::ClockTime::from_mseconds(100)) { match message.view() { + gst::MessageView::Error(_) if source_lost() => return Err(backend(SOURCE_LOST)), gst::MessageView::Error(error) => { return Err(backend(format!( "{}: {} ({})", @@ -705,6 +739,7 @@ mod tests { &commands_rx, &stopped, &worker_counters, + &|| false, ) }); let recording = Self { diff --git a/crates/linux/src/x11.rs b/crates/linux/src/x11.rs index 53ed442..65d3165 100644 --- a/crates/linux/src/x11.rs +++ b/crates/linux/src/x11.rs @@ -1,16 +1,136 @@ use crate::backend; use domain::{CaptureSourceKind, CaptureTarget, Result}; +use std::{ + cell::Cell, + os::raw::{c_int, c_ulong}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, OnceLock, + }, + time::{Duration, Instant}, +}; +use x11_dl::xlib; use x11rb::{ connection::Connection, protocol::xproto::{AtomEnum, ConnectionExt, MapState}, + rust_connection::RustConnection, }; +type ErrorHandler = unsafe extern "C" fn(*mut xlib::Display, *mut xlib::XErrorEvent) -> c_int; + +static PREVIOUS_HANDLER: OnceLock> = OnceLock::new(); +static SOURCES: Mutex> = Mutex::new(Vec::new()); + +struct Source { + xid: u32, + damage: Option<(u8, u8)>, + lost: Arc, +} + +impl Source { + fn claims(&self, error: &xlib::XErrorEvent) -> bool { + if matches!(error.error_code, xlib::BadWindow | xlib::BadDrawable) + && error.resourceid == c_ulong::from(self.xid) + { + self.lost.store(true, Ordering::SeqCst); + return true; + } + self.lost.load(Ordering::SeqCst) + && self.damage == Some((error.request_code, error.error_code)) + } +} + +unsafe extern "C" fn handle_error( + display: *mut xlib::Display, + event: *mut xlib::XErrorEvent, +) -> c_int { + if let Some(error) = event.as_ref() { + if SOURCES + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .iter() + .any(|source| source.claims(error)) + { + return 0; + } + } + match PREVIOUS_HANDLER.get().copied().flatten() { + Some(previous) => previous(display, event), + None => 0, + } +} + +pub(crate) struct SourceWatch { + xid: u32, + lost: Arc, + connection: RustConnection, + checked: Cell>, +} + +impl SourceWatch { + pub(crate) fn new(xid: u64) -> Result { + let xid = u32::try_from(xid).map_err(backend)?; + let (connection, _) = x11rb::connect(None).map_err(backend)?; + let damage = connection + .query_extension(b"DAMAGE") + .map_err(backend)? + .reply() + .map_err(backend)?; + let lost = Arc::new(AtomicBool::new(false)); + SOURCES + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(Source { + xid, + damage: damage + .present + .then_some((damage.major_opcode, damage.first_error)), + lost: lost.clone(), + }); + Ok(Self { + xid, + lost, + connection, + checked: Cell::new(None), + }) + } + + pub(crate) fn lost(&self) -> bool { + if !self.lost.load(Ordering::SeqCst) + && self + .checked + .get() + .map_or(true, |at| at.elapsed() >= Duration::from_millis(500)) + { + self.checked.set(Some(Instant::now())); + let exists = self + .connection + .get_window_attributes(self.xid) + .ok() + .and_then(|cookie| cookie.reply().ok()) + .is_some(); + if !exists { + self.lost.store(true, Ordering::SeqCst); + } + } + self.lost.load(Ordering::SeqCst) + } +} + +impl Drop for SourceWatch { + fn drop(&mut self) { + SOURCES + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .retain(|source| !Arc::ptr_eq(&source.lost, &self.lost)); + } +} + pub(crate) fn initialize() -> Result<()> { - static XLIB: std::sync::OnceLock> = - std::sync::OnceLock::new(); - static INITIALIZED: std::sync::OnceLock = std::sync::OnceLock::new(); + static XLIB: OnceLock> = OnceLock::new(); + static INITIALIZED: OnceLock = OnceLock::new(); let xlib = XLIB - .get_or_init(|| x11_dl::xlib::Xlib::open().map_err(|e| e.to_string())) + .get_or_init(|| xlib::Xlib::open().map_err(|e| e.to_string())) .as_ref() .map_err(backend)?; // Must run before GStreamer opens any Xlib display. Discovery uses x11rb, @@ -18,6 +138,7 @@ pub(crate) fn initialize() -> Result<()> { if !INITIALIZED.get_or_init(|| unsafe { (xlib.XInitThreads)() } != 0) { return Err(backend("XInitThreads failed")); } + PREVIOUS_HANDLER.get_or_init(|| unsafe { (xlib.XSetErrorHandler)(Some(handle_error)) }); Ok(()) } @@ -107,3 +228,48 @@ pub(crate) fn source(target: &CaptureTarget) -> Result<(String, u64)> { } Ok((display, target.id)) } + +#[cfg(test)] +mod tests { + use super::*; + + fn error(resourceid: c_ulong, error_code: u8, request_code: u8) -> xlib::XErrorEvent { + xlib::XErrorEvent { + type_: 0, + display: std::ptr::null_mut(), + resourceid, + serial: 0, + error_code, + request_code, + minor_code: 0, + } + } + + #[test] + fn source_errors_only_claim_the_recorded_window() { + let source = Source { + xid: 42, + damage: None, + lost: Arc::new(AtomicBool::new(false)), + }; + assert!(!source.claims(&error(43, xlib::BadWindow, 3))); + assert!(!source.lost.load(Ordering::SeqCst)); + assert!(source.claims(&error(42, xlib::BadWindow, 3))); + assert!(source.lost.load(Ordering::SeqCst)); + assert!(!source.claims(&error(43, xlib::BadDrawable, 73))); + } + + #[test] + fn damage_cleanup_errors_require_a_lost_source_and_matching_extension() { + let source = Source { + xid: 42, + damage: Some((143, 152)), + lost: Arc::new(AtomicBool::new(false)), + }; + assert!(!source.claims(&error(99, 152, 143))); + assert!(source.claims(&error(42, xlib::BadDrawable, 73))); + assert!(source.claims(&error(99, 152, 143))); + assert!(!source.claims(&error(99, 151, 143))); + assert!(!source.claims(&error(99, 152, 144))); + } +} diff --git a/packaging/linux/README.md b/packaging/linux/README.md index aa953fd..649874f 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -7,6 +7,10 @@ A working desktop and the required native plugins are needed. There is no Linux GUI yet. AMD Radeon capture has been tested on GNOME Wayland; Intel and NVIDIA hardware remain unverified. Software fallback makes recording possible without a supported GPU, but uses more CPU. +Software 4K HEVC can also need hundreds of MiB while encoding. On glibc builds, +wrec asks the allocator to return freed heap pages after each recording pipeline +has been fully torn down. This reduces idle memory retention, not the live +encoder working set; other allocators retain their own reclamation policy. On compatible Wayland/VA-API systems, the preferred pipeline is `pipewiresrc → DMA-BUF → vapostproc → VA encoder → qtmux`. Rust manages sessions diff --git a/scripts/test-capture-linux.py b/scripts/test-capture-linux.py index 991bd2b..9de90a2 100644 --- a/scripts/test-capture-linux.py +++ b/scripts/test-capture-linux.py @@ -19,7 +19,7 @@ def main(): env.pop(key, None) env.update(WREC_HOME=str(root / "home"), WREC_DATA_DIR=str(root / "data")) with open(root / "xvfb.log", "w") as log: - desktop = subprocess.Popen(["Xvfb", "-displayfd", "1", "-screen", "0", "800x600x24", "-nolisten", "tcp"], stdout=subprocess.PIPE, stderr=log, text=True) + desktop = subprocess.Popen(["Xvfb", "-displayfd", "1", "-screen", "0", "800x600x24", "-screen", "1", "3840x2160x24", "-nolisten", "tcp"], stdout=subprocess.PIPE, stderr=log, text=True) window = None try: with selectors.DefaultSelector() as selector: @@ -96,6 +96,39 @@ def verify(value, codec): run("job", "stop", str(window_id), "--json") verify(wait(window_id, "completed"), "hevc") print("PASS: real X11 window capture, HEVC decode, pause/resume, and finalization while paused") + + daemon_pid = run("daemon", "status", "--json")["pid"] + closing = subprocess.Popen(["xmessage", "-title", "Wrec closing window", "-geometry", "300x150+100+100", "Wrec Linux capture: this window closes mid-recording."], env=env, stdout=log, stderr=log) + try: + for _ in range(50): + closing_windows = [target for target in run("targets", "--json") if target["kind"] == "window" and "Wrec closing window" in target["name"]] + if closing_windows: + break + time.sleep(0.1) + assert closing_windows, "closing window was not discovered" + closing_id = submit(f'window:{closing_windows[0]["id"]}', "h264") + wait(closing_id, "recording") + time.sleep(1) + finally: + closing.terminate() + closing.wait(timeout=5) + lost = wait(closing_id, "failed") + assert any("X11 window closed" in event["message"] for event in lost["events"]), lost + assert run("daemon", "status", "--json")["pid"] == daemon_pid, "daemon restarted after the captured window closed" + after_id = submit("display:0", "h264", "2s") + verify(wait(after_id, "completed"), "h264") + assert run("daemon", "status", "--json")["pid"] == daemon_pid, "daemon restarted during capture after source loss" + print("PASS: closing a captured X11 window fails only that recording; the same daemon records again") + + large_display = next(target for target in targets if target["kind"] == "display" and "3840×2160" in target["name"]) + for codec in ("hevc", "h264", "hevc"): + memory_id = run("record", "--target", f'display:{large_display["id"]}', "--codec", codec, "--quality", "high", "--resolution", "native", "--no-system-audio", "--no-mic", "--duration", "2s", "--out", str(root / "movies"), "--detach", "--json")["job"]["id"] + verify(wait(memory_id, "completed"), codec) + time.sleep(0.3) + status = Path(f"/proc/{daemon_pid}/status").read_text() + idle_rss_mib = int(next(line.split()[1] for line in status.splitlines() if line.startswith("VmRSS:"))) / 1024 + assert idle_rss_mib < 256, f"4K encoder retained {idle_rss_mib:.1f} MiB after teardown" + print(f"PASS: repeated 4K capture releases idle heap pages ({idle_rss_mib:.1f} MiB daemon RSS)") run("daemon", "stop", "--json") finally: subprocess.run([str(binary), "daemon", "stop", "--json"], env=env, capture_output=True, timeout=20) From 79256b934a61ad1536329ffbba98f5188be7585b Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:56:35 +0000 Subject: [PATCH 09/45] fix(linux): isolate capture faults and preserve audio timing --- .github/workflows/ci.yml | 3 +- Cargo.lock | 2 + crates/daemon/src/main.rs | 7 + crates/domain/src/lib.rs | 4 +- crates/linux/Cargo.toml | 4 +- crates/linux/src/lib.rs | 280 +++++------ crates/linux/src/pipeline.rs | 309 ++++++++---- crates/linux/src/portal.rs | 168 ++++++- crates/linux/src/portal_tests.rs | 221 ++++++++- crates/linux/src/worker.rs | 803 +++++++++++++++++++++++++++++++ crates/linux/src/x11.rs | 6 +- packaging/linux/README.md | 128 ++++- scripts/test-capture-linux.py | 149 +++++- 13 files changed, 1817 insertions(+), 267 deletions(-) create mode 100644 crates/linux/src/worker.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e681d75..07214af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,13 +32,14 @@ jobs: libgstreamer-plugins-base1.0-dev gstreamer1.0-pipewire \ gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly \ - gstreamer1.0-libav ffmpeg dbus-daemon xvfb x11-apps + gstreamer1.0-libav ffmpeg dbus-daemon xvfb x11-apps pipewire - name: Build and test run: | cargo fmt --check cargo check --workspace --locked cargo test --workspace --locked dbus-run-session -- cargo test -p linux portal_roundtrip --locked -- --ignored + cargo test -p linux node_watch --locked -- --ignored ./scripts/package-cli-linux.sh - name: Test packaged CLI run: | diff --git a/Cargo.lock b/Cargo.lock index 385e4b5..0bdc87b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -991,6 +991,8 @@ dependencies = [ "futures-util", "gstreamer", "gstreamer-allocators", + "libc", + "serde", "serde_json", "tokio", "x11-dl", diff --git a/crates/daemon/src/main.rs b/crates/daemon/src/main.rs index aada87f..5dfa44d 100644 --- a/crates/daemon/src/main.rs +++ b/crates/daemon/src/main.rs @@ -1,4 +1,11 @@ fn main() { + #[cfg(target_os = "linux")] + if std::env::args_os() + .nth(1) + .is_some_and(|argument| argument == linux::CAPTURE_WORKER_ARGUMENT) + { + linux::run_capture_worker(); + } if let Err(message) = daemon::serve_forever() { eprintln!("error: {message}"); std::process::exit(1); diff --git a/crates/domain/src/lib.rs b/crates/domain/src/lib.rs index 877805b..30159c4 100644 --- a/crates/domain/src/lib.rs +++ b/crates/domain/src/lib.rs @@ -257,7 +257,7 @@ pub struct CaptureDimensions { pub output_height: i64, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum RecorderEvent { Starting { session_id: u64, @@ -291,7 +291,7 @@ pub enum RecorderEvent { }, } -#[derive(Debug, thiserror::Error)] +#[derive(Debug, thiserror::Error, Serialize, Deserialize)] pub enum RecorderError { #[error("recording cancelled before capture started")] Cancelled, diff --git a/crates/linux/Cargo.toml b/crates/linux/Cargo.toml index f14be8d..b703f55 100644 --- a/crates/linux/Cargo.toml +++ b/crates/linux/Cargo.toml @@ -14,7 +14,9 @@ gstreamer-allocators = { version = "0.23", features = ["v1_22"] } futures-util = { version = "0.3", default-features = false, features = ["std", "async-await"] } x11rb = "0.13" x11-dl = "2.21" +libc = "0.2" +serde = { version = "1", features = ["derive"] } +serde_json = "1" [target.'cfg(target_os = "linux")'.dev-dependencies] -serde_json = "1" zbus = { version = "4.4", default-features = false, features = ["tokio"] } diff --git a/crates/linux/src/lib.rs b/crates/linux/src/lib.rs index d7a3061..d057fa9 100644 --- a/crates/linux/src/lib.rs +++ b/crates/linux/src/lib.rs @@ -4,24 +4,25 @@ mod desktop; mod encoding; mod pipeline; mod portal; +mod worker; mod x11; use domain::{ CaptureTarget, RecorderEngine, RecorderError, RecorderEvent, RecorderSettings, RecordingSession, Result, }; -use futures_util::StreamExt; use std::{ + path::Path, sync::{ atomic::{AtomicU64, Ordering}, mpsc, Arc, }, - thread, - time::{Duration, SystemTime, UNIX_EPOCH}, + time::{SystemTime, UNIX_EPOCH}, }; use tokio::sync::watch; pub use desktop::{check_desktop, list_targets}; +pub use worker::{run as run_capture_worker, ARGUMENT as CAPTURE_WORKER_ARGUMENT}; pub(crate) fn backend(error: impl std::fmt::Display) -> RecorderError { RecorderError::Backend(error.to_string()) @@ -32,16 +33,10 @@ pub(crate) enum Command { Resume(mpsc::SyncSender>), } -struct Active { - commands: mpsc::SyncSender, - stop: watch::Sender, - worker: thread::JoinHandle<()>, -} - /// Rust owns lifecycle and buffer handles. GStreamer owns the native pixel path. pub struct LinuxRecorder { events: mpsc::Sender, - active: Option, + active: Option, stop_requested: bool, } @@ -54,14 +49,11 @@ impl LinuxRecorder { } } - fn control(&self, command: impl FnOnce(mpsc::SyncSender>) -> Command) -> Result<()> { - let active = self - .active - .as_ref() - .ok_or_else(|| backend("no active recording"))?; - let (tx, rx) = mpsc::sync_channel(1); - active.commands.try_send(command(tx)).map_err(backend)?; - rx.recv_timeout(Duration::from_secs(5)).map_err(backend)? + fn control(&mut self, request: worker::Request) -> Result<()> { + self.active + .as_mut() + .ok_or_else(|| backend("no active recording"))? + .control(request) } } @@ -78,14 +70,14 @@ impl RecorderEngine for LinuxRecorder { if self.active.is_none() && self.stop_requested { return Err(RecorderError::Cancelled); } - if let Some(active) = &self.active { - if !active.worker.is_finished() { - return Err(backend("recording is already active")); - } - } - if let Some(active) = self.active.take() { - let _ = active.worker.join(); + if self + .active + .as_ref() + .is_some_and(|active| !active.finished()) + { + return Err(backend("recording is already active")); } + self.active = None; self.stop_requested = false; check_desktop()?; if desktop::detect()? == desktop::Desktop::Wayland { @@ -107,155 +99,139 @@ impl RecorderEngine for LinuxRecorder { id, output_path: settings.output_dir.join(format!("wrec-{id}.mov")), }; - let events = self.events.clone(); - let completion_events = events.clone(); - let worker_session = session.clone(); - let (commands, receiver) = mpsc::sync_channel(1); - let (stop, mut stopped) = watch::channel(false); - let worker_stop = stop.clone(); - let worker = thread::Builder::new() - .name("wrec-linux-capture".into()) - .spawn(move || { - let _ = events.send(RecorderEvent::Starting { - session_id: id, - target: target.clone(), - settings: settings.clone(), - output_path: worker_session.output_path.clone(), - }); - let result = (|| { - if desktop::detect()? == desktop::Desktop::X11 { - let (display, xid) = x11::source(&target)?; - return pipeline::record( - &pipeline::CaptureInput::X11 { display, xid }, - &worker_session, - &settings, - &events, - receiver, - &stopped, - ); - } - let runtime = portal::runtime()?; - runtime.block_on(async { - let capture = Arc::new( - portal::open(&target, settings.include_cursor, &mut stopped).await?, - ); - let result = async { - let mut closed = - capture.session.receive_closed().await.map_err(backend)?; - let worker_capture = capture.clone(); - let runtime = tokio::runtime::Handle::current(); - let mut recording = tokio::task::spawn_blocking(move || { - pipeline::record( - &pipeline::CaptureInput::PipeWire(Box::new(move || { - // Each retry needs a new protocol connection. Duplicating - // an already-used socket does not reset its remote state. - runtime.block_on(worker_capture.connect()) - })), - &worker_session, - &settings, - &events, - receiver, - &stopped, - ) - }); - tokio::select! { - result = &mut recording => result.map_err(backend)?, - _ = closed.next() => { - let _ = worker_stop.send(true); - recording.await.map_err(backend)? - } - } - } - .await; - // Release the remote only after all native elements have stopped using its fd. - capture.close().await; - result - }) - })(); - let (success, status) = match result { - Ok(()) => (true, "recording finalized".to_string()), - Err(RecorderError::Cancelled) => { - let _ = completion_events.send(RecorderEvent::Cancelled { session_id: id }); - return; - } - Err(error) => (false, error.to_string()), - }; - let _ = completion_events.send(RecorderEvent::Exited { - session_id: id, - success, - status, - }); - }) - .map_err(backend)?; - self.active = Some(Active { - commands, - stop, - worker, - }); + self.active = Some(worker::Worker::spawn( + Path::new(worker::PROGRAM), + &[worker::ARGUMENT], + session.clone(), + target, + settings, + self.events.clone(), + worker::EXIT_DEADLINE, + )?); Ok(session) } fn pause(&mut self) -> Result<()> { - self.control(Command::Pause) + self.control(worker::Request::Pause) } fn resume(&mut self) -> Result<()> { - self.control(Command::Resume) + self.control(worker::Request::Resume) } fn stop(&mut self) -> Result<()> { self.stop_requested = true; - if let Some(active) = &self.active { - let _ = active.stop.send(true); + if let Some(active) = &mut self.active { + active.stop(); } Ok(()) } } -impl Drop for LinuxRecorder { - fn drop(&mut self) { - if let Some(active) = self.active.take() { - let _ = active.stop.send(true); - // The coordinator waits for finalization before normal shutdown. - // A stuck native call must not turn destruction into an unbounded join. - if active.worker.is_finished() { - let _ = active.worker.join(); +pub(crate) fn capture( + target: CaptureTarget, + settings: RecorderSettings, + session: RecordingSession, + events: mpsc::Sender, + receiver: mpsc::Receiver, + worker_stop: watch::Sender, + stopping: Arc, +) { + let mut stopped = worker_stop.subscribe(); + let id = session.id; + let completion_events = events.clone(); + let worker_session = session; + let _ = events.send(RecorderEvent::Starting { + session_id: id, + target: target.clone(), + settings: settings.clone(), + output_path: worker_session.output_path.clone(), + }); + let result = (|| { + if desktop::detect()? == desktop::Desktop::X11 { + let (display, xid) = x11::source(&target)?; + return pipeline::record( + &pipeline::CaptureInput::X11 { display, xid }, + &worker_session, + &settings, + &events, + receiver, + &stopped, + &*stopping, + ); + } + let runtime = portal::runtime()?; + runtime.block_on(async { + let capture = + Arc::new(portal::open(&target, settings.include_cursor, &mut stopped).await?); + let result = async { + let mut endings = capture.endings().await?; + let node_watch = portal::NodeWatch::start(capture.connect().await?).await; + if let portal::NodeWatch::Unavailable(reason) = &node_watch { + let _ = completion_events.send(RecorderEvent::Log { + session_id: Some(id), + message: format!("capture-engine: removal of the screen-cast stream will not be detected: {reason}"), + }); + } + let ending_events = completion_events.clone(); + let ending_stopping = stopping.clone(); + let worker_capture = capture.clone(); + let runtime = tokio::runtime::Handle::current(); + let mut recording = tokio::task::spawn_blocking(move || { + pipeline::record( + &pipeline::CaptureInput::PipeWire(Box::new(move || { + // Each retry needs a new protocol connection. Duplicating + // an already-used socket does not reset its remote state. + runtime.block_on(worker_capture.connect()) + })), + &worker_session, + &settings, + &events, + receiver, + &stopped, + &*stopping, + ) + }); + let result = tokio::select! { + result = &mut recording => result.map_err(backend)?, + ending = endings.next() => { + if matches!(ending, portal::Ending::Closed) { + let _ = ending_events.send(RecorderEvent::Log { + session_id: Some(id), + message: "capture-engine: the desktop closed the screen-cast session; finalizing the recording".into(), + }); + ending_stopping(); + let _ = worker_stop.send(true); + } + recording.await.map_err(backend)? + } + }; + drop(node_watch); + result } + .await; + // Release the remote only after all native elements have stopped using its fd. + capture.close().await; + result + }) + })(); + let (success, status) = match result { + Ok(()) => (true, "recording finalized".to_string()), + Err(RecorderError::Cancelled) => { + let _ = completion_events.send(RecorderEvent::Cancelled { session_id: id }); + return; } - } + Err(error) => (false, error.to_string()), + }; + let _ = completion_events.send(RecorderEvent::Exited { + session_id: id, + success, + status, + }); } #[cfg(test)] mod tests { use super::*; - #[test] - fn destruction_requests_stop_without_waiting_for_a_stuck_worker() { - let (events, _) = mpsc::channel(); - let (commands, _) = mpsc::sync_channel(1); - let (stop, stopped) = watch::channel(false); - let (release, waiting) = mpsc::sync_channel::<()>(1); - let worker = thread::spawn(move || { - let _ = waiting.recv_timeout(Duration::from_secs(1)); - }); - let recorder = LinuxRecorder { - events, - active: Some(Active { - commands, - stop, - worker, - }), - stop_requested: false, - }; - let at = std::time::Instant::now(); - drop(recorder); - let elapsed = at.elapsed(); - let _ = release.send(()); - assert!( - elapsed < Duration::from_millis(200), - "destructor waited {elapsed:?}" - ); - assert!(*stopped.borrow()); - } - #[test] fn stop_before_start_preserves_cancellation() { let (events, _) = mpsc::channel(); diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 7626b4a..40d7c85 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -90,6 +90,16 @@ fn capture_caps(dmabuf: bool, pipewire: bool, fps: u32) -> gst::Caps { } } +fn movie_mux() -> Result { + let mux = element("qtmux")?; + mux.set_property("fragment-duration", 10000u32); + mux.set_property_from_str("fragment-mode", "first-moov-then-finalise"); + // Preserve sub-frame timestamps around pause/resume instead of rounding + // them to the default frame-rate-derived track timescale. + mux.set_property("trak-timescale", 1_000_000u32); + Ok(mux) +} + fn video_queue() -> Result { let queue = element("queue")?; queue.set_property("max-size-buffers", 2u32); @@ -158,9 +168,20 @@ pub(crate) enum CaptureInput { X11 { display: String, xid: u64 }, } -struct Attempt { +struct Attempt<'a> { mode: Mode, counters: Arc, + last: bool, + stopping: &'a dyn Fn(), +} + +fn finished(result: &Result<()>, counters: &Counters) -> bool { + matches!(result, Ok(()) | Err(RecorderError::Cancelled)) + || counters.frames.load(Ordering::Relaxed) > 0 +} + +fn ends_recording(result: &Result<()>, counters: &Counters, last: bool) -> bool { + last || finished(result, counters) } pub(crate) fn record( @@ -170,6 +191,7 @@ pub(crate) fn record( events: &mpsc::Sender, commands: mpsc::Receiver, stop: &watch::Receiver, + stopping: &dyn Fn(), ) -> Result<()> { if matches!(capture, CaptureInput::X11 { .. }) { crate::x11::initialize()?; @@ -180,13 +202,16 @@ pub(crate) fn record( return Err(backend(format!("No {} encoder is installed. Install GStreamer VA/NVENC plugins or x264/x265/openh264 for software compatibility; see packaging/linux/README.md.", settings.codec.as_arg()))); } let mut last_error = None; - for mode in modes { + let count = modes.len(); + for (index, mode) in modes.into_iter().enumerate() { if *stop.borrow() { return Err(RecorderError::Cancelled); } let attempt = Attempt { mode, counters: Arc::new(Counters::default()), + last: index + 1 == count, + stopping, }; let _ = events.send(RecorderEvent::Log { session_id: Some(session.id), @@ -196,10 +221,7 @@ pub(crate) fn record( capture, session, settings, events, &commands, stop, &attempt, ); release_freed_memory(); - if result.is_ok() - || matches!(result, Err(RecorderError::Cancelled)) - || attempt.counters.frames.load(Ordering::Relaxed) > 0 - { + if finished(&result, &attempt.counters) { return result; } let error = result.unwrap_err(); @@ -232,7 +254,7 @@ fn record_attempt( events: &mpsc::Sender, commands: &mpsc::Receiver, stop: &watch::Receiver, - attempt: &Attempt, + attempt: &Attempt<'_>, ) -> Result<()> { if *stop.borrow() { return Err(RecorderError::Cancelled); @@ -287,11 +309,7 @@ fn record_attempt( output.set_property("caps", mode.caps(None)); let encoder = mode.encoder(&settings)?; let parser = element(parser_name(settings.codec))?; - let mux = element("qtmux")?; - mux.set_property("fragment-duration", 10000u32); - // Preserve sub-frame timestamps around pause/resume instead of rounding - // them to the default frame-rate-derived track timescale. - mux.set_property("trak-timescale", 1_000_000u32); + let mux = movie_mux()?; let sink = element("filesink")?; sink.set_property( "location", @@ -350,8 +368,15 @@ fn record_attempt( commands, stop, &counters, - &|| source_watch.as_ref().is_some_and(|watch| watch.lost()), + &|| match (&stream, &source_watch) { + (Some(stream), _) => stream.lost(), + (None, Some(watch)) => watch.lost().then_some(SOURCE_LOST), + (None, None) => None, + }, ); + if ends_recording(&result, &counters, attempt.last) { + (attempt.stopping)(); + } let _ = pipeline.0.set_state(gst::State::Null); if matches!(result, Err(RecorderError::Cancelled)) || counters.frames.load(Ordering::Relaxed) == 0 @@ -509,7 +534,7 @@ fn state_error(pipeline: &gst::Pipeline, fallback: impl std::fmt::Display) -> Re } const SOURCE_LOST: &str = - "The captured X11 window closed or became unavailable; the recording stopped."; + "The captured X11 window closed, was minimized, or became unavailable; the recording stopped."; fn run( pipeline: &gst::Pipeline, @@ -518,7 +543,7 @@ fn run( commands: &mpsc::Receiver, stop: &watch::Receiver, counters: &Counters, - source_lost: &dyn Fn() -> bool, + source_lost: &dyn Fn() -> Option<&'static str>, ) -> Result<()> { pipeline .set_state(gst::State::Playing) @@ -531,8 +556,8 @@ fn run( let mut stopping = None; let mut last_metrics = Instant::now(); loop { - if stopping.is_none() && source_lost() { - return Err(backend(SOURCE_LOST)); + if let Some(message) = stopping.is_none().then(source_lost).flatten() { + return Err(backend(message)); } if *stop.borrow() && stopping.is_none() { if !started && counters.frames.load(Ordering::Relaxed) == 0 { @@ -571,8 +596,10 @@ fn run( } if let Some(message) = bus.timed_pop(gst::ClockTime::from_mseconds(100)) { match message.view() { - gst::MessageView::Error(_) if source_lost() => return Err(backend(SOURCE_LOST)), gst::MessageView::Error(error) => { + if let Some(lost) = source_lost() { + return Err(backend(lost)); + } return Err(backend(format!( "{}: {} ({})", error @@ -581,7 +608,7 @@ fn run( .unwrap_or_default(), error.error(), error.debug().unwrap_or_default() - ))) + ))); } gst::MessageView::Eos(_) => { if counters.frames.load(Ordering::Relaxed) == 0 { @@ -690,10 +717,15 @@ mod tests { stop: watch::Sender, worker: Option>>, counters: Arc, + lost: Arc, } impl TestRecording { fn start(audio_tracks: usize) -> Self { + Self::start_with_audio_delays(&vec![gst::ClockTime::ZERO; audio_tracks]) + } + + fn start_with_audio_delays(audio_delays: &[gst::ClockTime]) -> Self { gst::init().unwrap(); static ID: AtomicU64 = AtomicU64::new(0); let id = ID.fetch_add(1, Ordering::Relaxed); @@ -704,24 +736,27 @@ mod tests { }; // Synthetic software encoding is confined to tests. Exercise the same // bus/control/mux code without pretending this is a hardware benchmark. - let pipeline = gst::parse::launch("videotestsrc name=video is-live=true pattern=ball ! video/x-raw,width=320,height=180,framerate=30/1 ! openh264enc ! h264parse name=parser ! qtmux name=mux fragment-duration=10000 trak-timescale=1000000 ! filesink name=output sync=false") + let pipeline = gst::parse::launch("videotestsrc name=video is-live=true pattern=ball ! video/x-raw,width=320,height=180,framerate=30/1 ! openh264enc ! h264parse name=parser") .unwrap().downcast::().unwrap(); - pipeline - .by_name("output") - .unwrap() - .set_property("location", session.output_path.to_str().unwrap()); + let mux = movie_mux().unwrap(); + let sink = element("filesink").unwrap(); + sink.set_property("sync", false); + sink.set_property("location", session.output_path.to_str().unwrap()); + pipeline.add_many([&mux, &sink]).unwrap(); + gst::Element::link_many([&pipeline.by_name("parser").unwrap(), &mux, &sink]).unwrap(); let counters = Arc::new(Counters::default()); attach_timing_probe(&pipeline.by_name("video").unwrap(), counters.clone()); - for _ in 0..audio_tracks { + for &delay in audio_delays { let source = element("audiotestsrc").unwrap(); source.set_property("is-live", true); - add_audio_source( - &pipeline, - &pipeline.by_name("mux").unwrap(), - &source, - &counters, - ) - .unwrap(); + source.static_pad("src").unwrap().add_probe( + gst::PadProbeType::BUFFER, + move |_, info| match info.buffer().and_then(|buffer| buffer.pts()) { + Some(pts) if pts < delay => gst::PadProbeReturn::Drop, + _ => gst::PadProbeReturn::Ok, + }, + ); + add_audio_source(&pipeline, &mux, &source, &counters).unwrap(); } attach_encoded_probe(&pipeline.by_name("parser").unwrap(), counters.clone()); let (events_tx, events) = mpsc::channel(); @@ -730,6 +765,8 @@ mod tests { let worker_pipeline = pipeline.clone(); let worker_session = session.clone(); let worker_counters = counters.clone(); + let lost = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let worker_lost = lost.clone(); let worker = std::thread::spawn(move || { let guard = PipelineGuard(worker_pipeline); run( @@ -739,7 +776,11 @@ mod tests { &commands_rx, &stopped, &worker_counters, - &|| false, + &|| { + worker_lost + .load(Ordering::SeqCst) + .then_some(crate::portal::PIPEWIRE_LOST) + }, ) }); let recording = Self { @@ -750,6 +791,7 @@ mod tests { stop, worker: Some(worker), counters, + lost, }; loop { if matches!( @@ -783,61 +825,69 @@ mod tests { } fn probe(&self) -> serde_json::Value { - let output = std::process::Command::new("ffprobe") - .args([ - "-v", - "error", - "-show_streams", - "-show_format", - "-show_packets", - "-of", - "json", - ]) - .arg(&self.session.output_path) - .output() - .expect("install ffmpeg to run Linux recording tests"); - assert!( - output.status.success(), - "{}", - String::from_utf8_lossy(&output.stderr) - ); - let probe: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); - let mut previous = std::collections::HashMap::new(); - for packet in probe["packets"].as_array().unwrap() { - let stream = packet["stream_index"].as_u64().unwrap(); - let dts = packet["dts"].as_i64().unwrap(); - if let Some(last) = previous.insert(stream, dts) { - assert!( - dts > last, - "stream {stream} has non-increasing DTS: {last} -> {dts}" - ); - } + probe_movie(&self.session.output_path) + } + } + + fn probe_movie(movie: &Path) -> serde_json::Value { + let probe = decode_movie(movie); + let mut previous = std::collections::HashMap::new(); + for packet in probe["packets"].as_array().unwrap() { + let stream = packet["stream_index"].as_u64().unwrap(); + let dts = packet["dts"].as_i64().unwrap(); + if let Some(last) = previous.insert(stream, dts) { + assert!( + dts > last, + "stream {stream} has non-increasing DTS: {last} -> {dts}" + ); } - let decoded = std::process::Command::new("ffmpeg") - .args(["-v", "error", "-i"]) - .arg(&self.session.output_path) - // Preserve the movie's timestamp precision when decoding VFR - // into null output; rounding to 1/30 can create duplicate DTS. - .args([ - "-map", - "0", - "-vsync", - "0", - "-enc_time_base", - "-1", - "-f", - "null", - "-", - ]) - .output() - .unwrap(); - assert!( - decoded.status.success() && decoded.stderr.is_empty(), - "{}", - String::from_utf8_lossy(&decoded.stderr) - ); - probe } + probe + } + + fn decode_movie(movie: &Path) -> serde_json::Value { + let output = std::process::Command::new("ffprobe") + .args([ + "-v", + "error", + "-show_streams", + "-show_format", + "-show_packets", + "-of", + "json", + ]) + .arg(movie) + .output() + .expect("install ffmpeg to run Linux recording tests"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let decoded = std::process::Command::new("ffmpeg") + .args(["-v", "error", "-i"]) + .arg(movie) + // Preserve the movie's timestamp precision when decoding VFR + // into null output; rounding to 1/30 can create duplicate DTS. + .args([ + "-map", + "0", + "-vsync", + "0", + "-enc_time_base", + "-1", + "-f", + "null", + "-", + ]) + .output() + .unwrap(); + assert!( + decoded.status.success() && decoded.stderr.is_empty(), + "{}", + String::from_utf8_lossy(&decoded.stderr) + ); + serde_json::from_slice(&output.stdout).unwrap() } impl Drop for TestRecording { @@ -850,6 +900,54 @@ mod tests { } } + fn start_seconds(probe: &serde_json::Value, codec: &str) -> Vec { + probe["streams"] + .as_array() + .unwrap() + .iter() + .filter(|stream| stream["codec_name"] == codec) + .map(|stream| stream["start_time"].as_str().unwrap().parse().unwrap()) + .collect() + } + + fn assert_late_audio_keeps_its_offset(movie: &Path, probe: &serde_json::Value, delay: f64) { + let video = start_seconds(probe, "h264"); + let audio = start_seconds(probe, "aac"); + assert_eq!((video.len(), audio.len()), (1, 2), "{}", movie.display()); + let (prompt, late) = (audio[0], audio[1]); + assert!( + (late - prompt - delay).abs() < 0.05, + "{}: audio starts {prompt}s and {late}s, expected {delay}s apart", + movie.display() + ); + assert!( + (late - video[0] - delay).abs() < 0.1, + "{}: late audio starts {late}s, video {}s, expected {delay}s apart", + movie.display(), + video[0] + ); + } + + #[test] + fn late_audio_keeps_its_capture_timestamp_in_partial_and_finalized_movies() { + let delay = gst::ClockTime::from_seconds(1); + let mut recording = TestRecording::start_with_audio_delays(&[gst::ClockTime::ZERO, delay]); + std::thread::sleep(Duration::from_secs(13)); + let partial = recording.session.output_path.with_extension("partial.mov"); + std::fs::copy(&recording.session.output_path, &partial).unwrap(); + recording.finish().unwrap(); + let seconds = delay.nseconds() as f64 / 1_000_000_000.0; + let checks = std::panic::catch_unwind(|| { + assert_late_audio_keeps_its_offset(&partial, &decode_movie(&partial), seconds); + let movie = &recording.session.output_path; + assert_late_audio_keeps_its_offset(movie, &probe_movie(movie), seconds); + }); + let _ = std::fs::remove_file(&partial); + if let Err(panic) = checks { + std::panic::resume_unwind(panic); + } + } + #[test] fn finalizes_playable_video_and_two_audio_tracks() { let mut recording = TestRecording::start(2); @@ -909,6 +1007,45 @@ mod tests { ); } + #[test] + fn every_attempt_that_ends_the_recording_signals_teardown() { + let idle = Counters::default(); + let encoded = Counters::default(); + encoded.frames.store(1, Ordering::Relaxed); + let failed = Err(backend("lost")); + assert!(ends_recording(&Ok(()), &encoded, false)); + assert!(ends_recording(&Ok(()), &encoded, true)); + assert!(ends_recording(&Err(RecorderError::Cancelled), &idle, false)); + assert!(ends_recording(&failed, &encoded, false)); + assert!(ends_recording(&failed, &idle, true)); + assert!(!ends_recording(&failed, &idle, false)); + } + + #[test] + fn source_loss_fails_active_and_paused_recordings_promptly() { + for paused in [false, true] { + let mut recording = TestRecording::start(1); + std::thread::sleep(Duration::from_millis(300)); + if paused { + recording.control(true); + } + let at = Instant::now(); + recording.lost.store(true, Ordering::SeqCst); + let error = recording + .worker + .take() + .unwrap() + .join() + .unwrap() + .unwrap_err(); + assert!(at.elapsed() < Duration::from_secs(1), "{:?}", at.elapsed()); + assert_eq!( + error.to_string(), + format!("backend error: {}", crate::portal::PIPEWIRE_LOST) + ); + } + } + #[test] fn pipeline_errors_fail_the_recording() { let mut recording = TestRecording::start(0); diff --git a/crates/linux/src/portal.rs b/crates/linux/src/portal.rs index 3324446..8aa1515 100644 --- a/crates/linux/src/portal.rs +++ b/crates/linux/src/portal.rs @@ -7,8 +7,21 @@ use ashpd::{ WindowIdentifier, }; use domain::{CaptureSourceKind, CaptureTarget, RecorderError, Result}; -use std::{os::fd::OwnedFd, time::Duration}; -use tokio::sync::watch; +use futures_util::{Stream, StreamExt}; +use gstreamer::{self as gst, prelude::*}; +use std::{ + os::fd::{AsRawFd, OwnedFd}, + pin::Pin, + sync::{mpsc, Arc, OnceLock}, + thread, + time::Duration, +}; +use tokio::sync::{oneshot, watch}; + +const PORTAL: &str = "org.freedesktop.portal.Desktop"; +pub(crate) const PIPEWIRE_LOST: &str = "The PipeWire connection closed, so the screen recording stopped. Restart PipeWire and WirePlumber (or the desktop session), then record again."; +pub(crate) const NODE_LOST: &str = "The desktop's screen-cast stream ended because its PipeWire node was removed, so the screen recording stopped. If the portal backend (such as xdg-desktop-portal-wlr) or the compositor exited, restart it, then record again."; +pub(crate) const PORTAL_LOST: &str = "The desktop portal (org.freedesktop.portal.Desktop) exited, so the screen recording stopped. Restart xdg-desktop-portal and its desktop backend, then record again."; #[cfg(test)] #[path = "portal_tests.rs"] @@ -85,14 +98,163 @@ pub(crate) struct Capture { pub session: Session<'static, Screencast<'static>>, // Keep the portal connection alive until the pipeline and session close. portal: Screencast<'static>, + lost: Arc>, } pub(crate) struct PipeWireStream { pub fd: OwnedFd, pub node: u32, + lost: Arc>, +} + +impl PipeWireStream { + fn lose(&self, message: &'static str) { + let _ = self.lost.set(message); + } + + pub(crate) fn lost(&self) -> Option<&'static str> { + if let Some(message) = self.lost.get() { + return Some(message); + } + let mut remote = libc::pollfd { + fd: self.fd.as_raw_fd(), + events: libc::POLLRDHUP, + revents: 0, + }; + let ready = unsafe { libc::poll(&mut remote, 1, 0) }; + let closed = libc::POLLHUP | libc::POLLRDHUP | libc::POLLERR | libc::POLLNVAL; + (ready > 0 && remote.revents & closed != 0).then_some(PIPEWIRE_LOST) + } +} + +pub(crate) enum NodeWatch { + Watching { + stop: mpsc::Sender<()>, + done: mpsc::Receiver<()>, + }, + Unavailable(String), +} + +impl NodeWatch { + pub(crate) async fn start(remote: PipeWireStream) -> Self { + match Self::watch(remote).await { + Ok(watch) => watch, + Err(error) => Self::Unavailable(error.to_string()), + } + } + + async fn watch(remote: PipeWireStream) -> Result { + gst::init().map_err(backend)?; + let provider = gst::DeviceProviderFactory::by_name("pipewiredeviceprovider") + .ok_or_else(|| backend("GStreamer's PipeWire device provider is not installed"))?; + provider.set_property("fd", remote.fd.as_raw_fd()); + let bus = provider.bus(); + let (stop, stops) = mpsc::channel(); + let (finished, done) = mpsc::channel(); + let (ready, synced) = oneshot::channel(); + thread::Builder::new() + .name("wrec-pipewire-node".into()) + .spawn(move || { + let present = provider.start().is_ok() + && provider + .devices() + .iter() + .any(|device| node_id(device) == Some(remote.node)); + let _ = ready.send(present); + while present && matches!(stops.try_recv(), Err(mpsc::TryRecvError::Empty)) { + let removed = bus + .timed_pop_filtered( + gst::ClockTime::from_mseconds(100), + &[gst::MessageType::DeviceRemoved], + ) + .is_some_and(|message| match message.view() { + gst::MessageView::DeviceRemoved(removed) => { + node_id(&removed.device()) == Some(remote.node) + } + _ => false, + }); + if removed { + remote.lose(NODE_LOST); + break; + } + } + provider.stop(); + drop(remote); + let _ = finished.send(()); + }) + .map_err(backend)?; + let watch = Self::Watching { stop, done }; + match tokio::time::timeout(Duration::from_secs(5), synced).await { + Ok(Ok(true)) => Ok(watch), + Ok(_) => Err(backend( + "the selected node was not visible on a second PipeWire connection", + )), + Err(_) => Err(backend("PipeWire did not list its nodes within 5s")), + } + } +} + +impl Drop for NodeWatch { + fn drop(&mut self) { + if let Self::Watching { stop, done } = self { + let _ = stop.send(()); + let _ = done.recv_timeout(Duration::from_secs(3)); + } + } +} + +fn node_id(device: &gst::Device) -> Option { + device + .has_property("id", Some(gst::glib::Type::U32)) + .then(|| device.property::("id")) +} + +pub(crate) enum Ending { + Closed, + PortalLost, +} + +pub(crate) struct Endings<'a> { + closed: Pin + 'a>>, + owners: ashpd::zbus::proxy::OwnerChangedStream<'a>, + lost: Arc>, +} + +impl Endings<'_> { + pub(crate) async fn next(&mut self) -> Ending { + if self.lost.get().is_some() { + return Ending::PortalLost; + } + tokio::select! { + _ = self.closed.next() => Ending::Closed, + _ = self.owners.next() => { + let _ = self.lost.set(PORTAL_LOST); + Ending::PortalLost + } + } + } } impl Capture { + pub async fn endings(&self) -> Result> { + let closed = self.session.receive_closed().await.map_err(backend)?; + let owners = self.portal.receive_owner_changed().await.map_err(backend)?; + let present = ashpd::zbus::fdo::DBusProxy::new(self.portal.connection()) + .await + .map_err(backend)? + .name_has_owner(PORTAL.try_into().map_err(backend)?) + .await + .map_err(backend)?; + if !present { + let _ = self.lost.set(PORTAL_LOST); + } + Ok(Endings { + closed: Box::pin(closed.map(|_| ())), + owners, + lost: self.lost.clone(), + }) + } + pub async fn connect(&self) -> Result { let fd = tokio::time::timeout( Duration::from_secs(5), @@ -104,6 +266,7 @@ impl Capture { Ok(PipeWireStream { fd, node: self.node, + lost: self.lost.clone(), }) } @@ -185,6 +348,7 @@ pub(crate) async fn open( node, session, portal, + lost: Arc::new(OnceLock::new()), }), Err(error) => { let _ = tokio::time::timeout(Duration::from_secs(3), session.close()).await; diff --git a/crates/linux/src/portal_tests.rs b/crates/linux/src/portal_tests.rs index b8c558f..f58a350 100644 --- a/crates/linux/src/portal_tests.rs +++ b/crates/linux/src/portal_tests.rs @@ -1,7 +1,8 @@ use super::*; use std::{ collections::HashMap, - os::unix::net::UnixStream, + io::{Read, Write}, + os::{fd::AsRawFd, unix::net::UnixStream}, sync::{ atomic::{AtomicU32, Ordering}, Arc, @@ -18,6 +19,8 @@ struct MockPortal { mode: Arc, closes: Arc, remotes: Arc, + peers: Arc>>, + sessions: Arc>>, selections: Arc>>, } @@ -93,6 +96,7 @@ impl MockPortal { ) -> OwnedObjectPath { let session = path(&header, &options, true); let request = path(&header, &options, false); + self.sessions.lock().unwrap().push(session.clone()); connection .object_server() .at(session.clone(), MockSession(self.closes.clone())) @@ -157,7 +161,8 @@ impl MockPortal { _options: HashMap, ) -> zbus::zvariant::OwnedFd { self.remotes.fetch_add(1, Ordering::Relaxed); - let (socket, _peer) = UnixStream::pair().unwrap(); + let (socket, peer) = UnixStream::pair().unwrap(); + self.peers.lock().unwrap().push(peer); let fd: OwnedFd = socket.into(); fd.into() } @@ -170,7 +175,7 @@ fn portal_roundtrip_and_cancellation_close_sessions() { std::env::set_var("XDG_RUNTIME_DIR", std::env::temp_dir()); let mock = MockPortal::default(); runtime().unwrap().block_on(async { - let _server = zbus::connection::Builder::session() + let server = zbus::connection::Builder::session() .unwrap() .name("org.freedesktop.portal.Desktop") .unwrap() @@ -223,5 +228,215 @@ fn portal_roundtrip_and_cancellation_close_sessions() { let (result, ()) = tokio::join!(cancelled, cancel); assert!(matches!(result, Err(RecorderError::Cancelled))); assert_eq!(mock.closes.load(Ordering::Relaxed), 3); + + mock.mode.store(0, Ordering::Relaxed); + let (_stop, mut stopped) = watch::channel(false); + let capture = open(&target, true, &mut stopped).await.unwrap(); + let mut endings = capture.endings().await.unwrap(); + let stream = capture.connect().await.unwrap(); + assert_eq!(stream.lost(), None); + let peer = mock.peers.lock().unwrap().pop().unwrap(); + (&peer).write_all(b"pipewire").unwrap(); + drop(peer); + assert_eq!(stream.lost(), Some(PIPEWIRE_LOST)); + let mut unread = Vec::new(); + UnixStream::from(stream.fd.try_clone().unwrap()) + .read_to_end(&mut unread) + .unwrap(); + assert_eq!( + unread, b"pipewire", + "loss detection must not consume PipeWire data" + ); + + let session = mock.sessions.lock().unwrap().last().unwrap().clone(); + server + .emit_signal( + None::<&str>, + &session, + "org.freedesktop.portal.Session", + "Closed", + &(HashMap::<&str, Value<'_>>::new(),), + ) + .await + .unwrap(); + assert!(matches!( + tokio::time::timeout(Duration::from_secs(5), endings.next()) + .await + .unwrap(), + Ending::Closed + )); + capture.close().await; + + let capture = open(&target, true, &mut stopped).await.unwrap(); + let mut endings = capture.endings().await.unwrap(); + let stream = capture.connect().await.unwrap(); + assert_eq!(stream.lost(), None); + server.release_name(PORTAL).await.unwrap(); + assert!(matches!( + tokio::time::timeout(Duration::from_secs(5), endings.next()) + .await + .unwrap(), + Ending::PortalLost + )); + assert_eq!(stream.lost(), Some(PORTAL_LOST)); + assert!(matches!(endings.next().await, Ending::PortalLost)); + drop(endings); + capture.close().await; + + server.request_name(PORTAL).await.unwrap(); + let capture = open(&target, true, &mut stopped).await.unwrap(); + let stream = capture.connect().await.unwrap(); + assert_eq!(stream.lost(), None); + assert!(capture.endings().await.is_ok()); + capture.close().await; }); } + +struct PrivatePipeWire { + daemon: std::process::Child, + directory: std::path::PathBuf, +} + +impl PrivatePipeWire { + fn start() -> Self { + let directory = std::env::temp_dir().join(format!("wrec-pipewire-{}", std::process::id())); + std::fs::create_dir_all(&directory).unwrap(); + let daemon = std::process::Command::new("pipewire") + .env_remove("DBUS_SESSION_BUS_ADDRESS") + .env("PIPEWIRE_RUNTIME_DIR", &directory) + .env("XDG_RUNTIME_DIR", &directory) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("install pipewire to run this test"); + let pipewire = Self { daemon, directory }; + for _ in 0..50 { + if pipewire.directory.join("pipewire-0").exists() { + return pipewire; + } + std::thread::sleep(Duration::from_millis(100)); + } + panic!("private PipeWire did not create its socket") + } + + fn connect(&self) -> OwnedFd { + UnixStream::connect(self.directory.join("pipewire-0")) + .unwrap() + .into() + } + + fn stream(&self, node: u32, lost: &Arc>) -> PipeWireStream { + PipeWireStream { + fd: self.connect(), + node, + lost: lost.clone(), + } + } + + fn node(&self, name: &str) -> u32 { + let provider = gst::DeviceProviderFactory::by_name("pipewiredeviceprovider").unwrap(); + for _ in 0..50 { + let fd = self.connect(); + provider.set_property("fd", fd.as_raw_fd()); + provider.start().unwrap(); + let found = provider.devices().iter().find_map(|device| { + let properties = device.properties()?; + (properties.get::("node.name").ok()? == name) + .then(|| node_id(device)) + .flatten() + }); + provider.stop(); + if let Some(node) = found { + return node; + } + std::thread::sleep(Duration::from_millis(100)); + } + panic!("PipeWire node {name} did not appear") + } +} + +impl Drop for PrivatePipeWire { + fn drop(&mut self) { + let _ = self.daemon.kill(); + let _ = self.daemon.wait(); + let _ = std::fs::remove_dir_all(&self.directory); + } +} + +fn wait_for(lost: &Arc>) -> Option<&'static str> { + for _ in 0..50 { + if let Some(message) = lost.get() { + return Some(message); + } + std::thread::sleep(Duration::from_millis(100)); + } + None +} + +#[test] +#[ignore = "requires the pipewire daemon and GStreamer PipeWire plugin: cargo test -p linux node_watch -- --ignored"] +fn node_watch_reports_removal_absence_and_stops_after_daemon_loss() { + gst::init().unwrap(); + let mut pipewire = PrivatePipeWire::start(); + let source = gst::parse::launch( + "videotestsrc is-live=true ! pipewiresink name=sink client-name=wrec-node-watch", + ) + .unwrap() + .downcast::() + .unwrap(); + let sink_fd = pipewire.connect(); + source + .by_name("sink") + .unwrap() + .set_property("fd", sink_fd.as_raw_fd()); + source.set_state(gst::State::Playing).unwrap(); + let node = pipewire.node("wrec-node-watch"); + + let lost = Arc::new(OnceLock::new()); + let watch = runtime() + .unwrap() + .block_on(NodeWatch::start(pipewire.stream(node, &lost))); + assert!(matches!(watch, NodeWatch::Watching { .. })); + std::thread::sleep(Duration::from_millis(500)); + assert_eq!( + lost.get(), + None, + "a live, idle node must not be reported lost" + ); + source.set_state(gst::State::Null).unwrap(); + assert_eq!(wait_for(&lost), Some(NODE_LOST)); + let at = std::time::Instant::now(); + drop(watch); + assert!(at.elapsed() < Duration::from_secs(1), "{:?}", at.elapsed()); + + let missing = Arc::new(OnceLock::new()); + let watch = runtime() + .unwrap() + .block_on(NodeWatch::start(pipewire.stream(node, &missing))); + assert!(matches!(watch, NodeWatch::Unavailable(_))); + assert_eq!(missing.get(), None); + drop(watch); + + let restarted_fd = pipewire.connect(); + source + .by_name("sink") + .unwrap() + .set_property("fd", restarted_fd.as_raw_fd()); + source.set_state(gst::State::Playing).unwrap(); + let node = pipewire.node("wrec-node-watch"); + let remote = Arc::new(OnceLock::new()); + let watch = runtime() + .unwrap() + .block_on(NodeWatch::start(pipewire.stream(node, &remote))); + assert!(matches!(watch, NodeWatch::Watching { .. })); + let stream = pipewire.stream(node, &remote); + assert_eq!(stream.lost(), None); + let _ = pipewire.daemon.kill(); + let _ = pipewire.daemon.wait(); + std::thread::sleep(Duration::from_millis(200)); + assert_eq!(stream.lost(), Some(PIPEWIRE_LOST)); + let at = std::time::Instant::now(); + drop(watch); + assert!(at.elapsed() < Duration::from_secs(4), "{:?}", at.elapsed()); + let _ = source.set_state(gst::State::Null); +} diff --git a/crates/linux/src/worker.rs b/crates/linux/src/worker.rs new file mode 100644 index 0000000..fabdf6c --- /dev/null +++ b/crates/linux/src/worker.rs @@ -0,0 +1,803 @@ +use crate::{backend, capture, Command}; +use domain::{CaptureTarget, RecorderEvent, RecorderSettings, RecordingSession, Result}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::File, + io::{BufRead, BufReader, Read, Write}, + os::{ + fd::{AsFd, AsRawFd}, + unix::process::CommandExt, + }, + path::Path, + process::{ChildStderr, ChildStdin, ChildStdout, ExitStatus, Stdio}, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, Mutex, + }, + thread, + time::{Duration, Instant}, +}; +use tokio::sync::watch; + +pub const ARGUMENT: &str = "--wrec-capture-worker"; +pub(crate) const PROGRAM: &str = "/proc/self/exe"; +pub(crate) const EXIT_DEADLINE: Duration = Duration::from_secs(20); +const REPLY_TIMEOUT: Duration = Duration::from_secs(5); +const DRAIN_WAIT: Duration = Duration::from_secs(1); +const POLL: Duration = Duration::from_millis(100); +const MESSAGE_LIMIT: u64 = 1 << 20; +const STDERR_LINE_LIMIT: u64 = 4096; + +#[derive(Serialize, Deserialize)] +pub(crate) enum Request { + Start { + session: RecordingSession, + target: CaptureTarget, + settings: RecorderSettings, + }, + Pause, + Resume, + Stop, +} + +#[derive(Serialize, Deserialize)] +enum Message { + Event(RecorderEvent), + Reply(Result<()>), + Stopping, +} + +type Received = std::result::Result; + +pub(crate) struct Worker { + stdin: ChildStdin, + replies: mpsc::Receiver>, + sent: u64, + answered: u64, + stopping: mpsc::Sender<()>, + finished: Arc, +} + +impl Worker { + pub(crate) fn spawn( + program: &Path, + arguments: &[&str], + session: RecordingSession, + target: CaptureTarget, + settings: RecorderSettings, + events: mpsc::Sender, + deadline: Duration, + ) -> Result { + let mut command = std::process::Command::new(program); + command + .arg0("wrec-capture") + .args(arguments) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0); + let parent = std::process::id(); + unsafe { + command.pre_exec(move || die_with_parent(parent)); + } + let (ready, launched) = mpsc::sync_channel(1); + let (replies_sender, replies) = mpsc::channel(); + let (stopping, stop_requests) = mpsc::channel(); + let finished = Arc::new(AtomicBool::new(false)); + let supervisor = Supervisor { + start: Request::Start { + session: session.clone(), + target, + settings, + }, + session, + events, + replies: replies_sender, + stop_requests, + finished: finished.clone(), + deadline, + }; + thread::Builder::new() + .name("wrec-capture-supervisor".into()) + .spawn(move || supervisor.run(command, ready)) + .map_err(|error| backend(format!("could not start the capture worker: {error}")))?; + let stdin = launched + .recv() + .map_err(|_| backend("could not start the capture worker"))??; + Ok(Self { + stdin, + replies, + sent: 0, + answered: 0, + stopping, + finished, + }) + } + + pub(crate) fn control(&mut self, request: Request) -> Result<()> { + write_line(&mut self.stdin, &request).map_err(|error| { + backend(format!( + "the capture worker is not accepting commands: {error}" + )) + })?; + self.sent += 1; + let deadline = Instant::now() + REPLY_TIMEOUT; + loop { + let reply = match self + .replies + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + { + Ok(reply) => reply, + Err(mpsc::RecvTimeoutError::Timeout) => { + return Err(backend("the capture worker did not reply within 5s")) + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + return Err(backend("the capture worker exited before replying")) + } + }; + self.answered += 1; + if self.answered == self.sent { + return reply; + } + } + } + + pub(crate) fn stop(&mut self) { + let _ = write_line(&mut self.stdin, &Request::Stop); + let _ = self.stopping.send(()); + } + + pub(crate) fn finished(&self) -> bool { + self.finished.load(Ordering::SeqCst) + } +} + +fn die_with_parent(parent: u32) -> std::io::Result<()> { + if unsafe { libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL as libc::c_ulong) } != 0 { + return Err(std::io::Error::last_os_error()); + } + if unsafe { libc::getppid() } as u32 != parent { + return Err(std::io::Error::from_raw_os_error(libc::ESRCH)); + } + Ok(()) +} + +fn write_line(writer: &mut impl Write, value: &impl Serialize) -> std::io::Result<()> { + let mut line = serde_json::to_vec(value)?; + line.push(b'\n'); + writer.write_all(&line)?; + writer.flush() +} + +fn set_nonblocking(stdin: &ChildStdin) -> std::io::Result<()> { + let descriptor = stdin.as_raw_fd(); + let flags = unsafe { libc::fcntl(descriptor, libc::F_GETFL) }; + if flags < 0 || unsafe { libc::fcntl(descriptor, libc::F_SETFL, flags | libc::O_NONBLOCK) } < 0 + { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +struct Supervisor { + start: Request, + session: RecordingSession, + events: mpsc::Sender, + replies: mpsc::Sender>, + stop_requests: mpsc::Receiver<()>, + finished: Arc, + deadline: Duration, +} + +impl Supervisor { + fn run(self, mut command: std::process::Command, ready: mpsc::SyncSender>) { + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => { + let _ = ready.send(Err(backend(format!( + "could not start the capture worker: {error}" + )))); + return; + } + }; + let mut stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + let launched = write_line(&mut stdin, &self.start) + .and_then(|()| set_nonblocking(&stdin)) + .and_then(|()| Ok((read_messages(stdout)?, last_lines(stderr)?))); + let (messages, last_error) = match launched { + Ok(readers) => readers, + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + let _ = ready.send(Err(backend(format!( + "could not start the capture worker: {error}" + )))); + return; + } + }; + if ready.send(Ok(stdin)).is_err() { + let _ = child.kill(); + let _ = child.wait(); + return; + } + let mut terminal = None; + let mut started = false; + let mut problem = None; + let mut output_open = true; + let mut exit_by: Option = None; + let mut drained_by: Option = None; + let mut status: Option = None; + loop { + let received = if output_open { + messages.recv_timeout(POLL) + } else { + thread::sleep(POLL); + Err(mpsc::RecvTimeoutError::Timeout) + }; + match received { + Ok(Ok(Message::Event(event))) => { + if matches!( + event, + RecorderEvent::Exited { .. } | RecorderEvent::Cancelled { .. } + ) { + terminal = Some(event); + exit_by.get_or_insert(Instant::now() + self.deadline); + } else { + started |= matches!(event, RecorderEvent::Started { .. }); + let _ = self.events.send(event); + } + } + Ok(Ok(Message::Reply(reply))) => { + let _ = self.replies.send(reply); + } + Ok(Ok(Message::Stopping)) => { + exit_by.get_or_insert(Instant::now() + self.deadline); + } + Ok(Err(error)) => { + problem = Some(format!("capture worker sent an invalid message ({error})")); + let _ = child.kill(); + status = child.wait().ok(); + break; + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + output_open = false; + exit_by.get_or_insert(Instant::now() + self.deadline); + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + if !matches!( + self.stop_requests.try_recv(), + Err(mpsc::TryRecvError::Empty) + ) { + exit_by.get_or_insert(Instant::now() + self.deadline); + } + if status.is_none() { + status = child.try_wait().ok().flatten(); + } + if status.is_some() { + let drained = *drained_by.get_or_insert(Instant::now() + DRAIN_WAIT); + if !output_open || Instant::now() >= drained { + break; + } + } else if exit_by.is_some_and(|by| Instant::now() >= by) { + let _ = child.kill(); + status = child.wait().ok(); + if terminal.is_none() { + problem = Some(format!( + "capture worker did not exit within {}s", + self.deadline.as_secs() + )); + } + break; + } + } + drop(self.replies); + let event = match (terminal, problem) { + (Some(event), None) => event, + (_, problem) => { + let empty = + std::fs::metadata(&self.session.output_path).is_ok_and(|file| file.len() == 0); + if !started || empty { + let _ = std::fs::remove_file(&self.session.output_path); + } + let problem = + problem.unwrap_or_else(|| "capture worker exited unexpectedly".to_string()); + let status = status + .map_or_else(|| "unknown status".to_string(), |status| status.to_string()); + let detail = last_error.recv_timeout(DRAIN_WAIT).unwrap_or_default(); + RecorderEvent::Exited { + session_id: self.session.id, + success: false, + status: if detail.is_empty() { + format!("{problem} ({status})") + } else { + format!("{problem} ({status}): {detail}") + }, + } + } + }; + self.finished.store(true, Ordering::SeqCst); + let _ = self.events.send(event); + } +} + +fn read_messages(stdout: ChildStdout) -> std::io::Result> { + let (sender, receiver) = mpsc::channel(); + thread::Builder::new() + .name("wrec-capture-output".into()) + .spawn(move || { + let mut reader = BufReader::new(stdout); + let mut line = Vec::new(); + loop { + line.clear(); + match reader + .by_ref() + .take(MESSAGE_LIMIT) + .read_until(b'\n', &mut line) + { + Ok(0) => return, + Ok(_) => {} + Err(error) => { + let _ = sender.send(Err(error.to_string())); + return; + } + } + if !line.ends_with(b"\n") && (line.len() as u64) < MESSAGE_LIMIT { + return; + } + let message = serde_json::from_slice(&line).map_err(|error| error.to_string()); + let invalid = message.is_err(); + if sender.send(message).is_err() || invalid { + return; + } + } + })?; + Ok(receiver) +} + +fn last_lines(stderr: ChildStderr) -> std::io::Result> { + let (sender, receiver) = mpsc::channel(); + thread::Builder::new() + .name("wrec-capture-stderr".into()) + .spawn(move || { + let mut reader = BufReader::new(stderr); + let mut line = Vec::new(); + let mut previous = String::new(); + let mut last = String::new(); + loop { + line.clear(); + match reader + .by_ref() + .take(STDERR_LINE_LIMIT) + .read_until(b'\n', &mut line) + { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + let text = String::from_utf8_lossy(&line).trim().to_string(); + let _ = writeln!( + std::io::stderr(), + "{}", + String::from_utf8_lossy(&line).trim_end() + ); + if !text.is_empty() { + previous = std::mem::replace(&mut last, text); + } + } + let _ = sender.send(format!("{previous} {last}").trim().to_string()); + })?; + Ok(receiver) +} + +pub fn run() -> ! { + unsafe { + libc::prctl(libc::PR_SET_NAME, c"wrec-capture".as_ptr() as libc::c_ulong); + } + let protocol = match std::io::stdout().as_fd().try_clone_to_owned() { + Ok(protocol) => File::from(protocol), + Err(error) => fail(format!("could not open the protocol stream: {error}")), + }; + if unsafe { libc::dup2(libc::STDERR_FILENO, libc::STDOUT_FILENO) } < 0 { + fail(format!( + "could not redirect stdout: {}", + std::io::Error::last_os_error() + )); + } + let output = Arc::new(Mutex::new(protocol)); + let mut line = String::new(); + let _ = std::io::stdin().read_line(&mut line); + let Ok(Request::Start { + session, + target, + settings, + }) = serde_json::from_str(&line) + else { + fail(format!("expected a start request, received {line:?}")); + }; + let (events, outgoing) = mpsc::channel(); + let (commands, receiver) = mpsc::sync_channel(1); + let (stop, _) = watch::channel(false); + let (pending, answers) = mpsc::channel::>>(); + let reply_output = output.clone(); + let control_stop = stop.clone(); + spawn("wrec-capture-replies", move || { + for answer in answers { + let reply = answer + .recv() + .unwrap_or_else(|_| Err(backend("recording stopped before replying"))); + send(&reply_output, &Message::Reply(reply)); + } + }); + spawn("wrec-capture-control", move || { + for line in std::io::stdin().lines() { + let Ok(line) = line else { break }; + match serde_json::from_str(&line) { + Ok(Request::Pause) => queue(&commands, &pending, Command::Pause), + Ok(Request::Resume) => queue(&commands, &pending, Command::Resume), + Ok(Request::Stop) => { + control_stop.send_replace(true); + } + _ => report(&format!("ignored an invalid request: {line:?}")), + } + } + control_stop.send_replace(true); + }); + let stopping_output = output.clone(); + let stopping = Arc::new(move || send(&stopping_output, &Message::Stopping)); + let forwarder = spawn("wrec-capture-events", move || { + for event in outgoing { + let finished = matches!( + event, + RecorderEvent::Exited { .. } | RecorderEvent::Cancelled { .. } + ); + send(&output, &Message::Event(event)); + if finished { + break; + } + } + }); + capture(target, settings, session, events, receiver, stop, stopping); + let _ = forwarder.join(); + std::process::exit(0) +} + +fn spawn(name: &str, body: impl FnOnce() + Send + 'static) -> thread::JoinHandle<()> { + thread::Builder::new() + .name(name.into()) + .spawn(body) + .unwrap_or_else(|error| fail(format!("could not start {name}: {error}"))) +} + +fn queue( + commands: &mpsc::SyncSender, + pending: &mpsc::Sender>>, + command: fn(mpsc::SyncSender>) -> Command, +) { + let (reply, answer) = mpsc::sync_channel(1); + if let Err(error) = commands.try_send(command(reply.clone())) { + let _ = reply.send(Err(backend(error))); + } + let _ = pending.send(answer); +} + +fn send(output: &Mutex, message: &Message) { + let mut output = output + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let _ = write_line(&mut *output, message); +} + +fn report(message: &str) { + let _ = writeln!(std::io::stderr(), "capture worker: {message}"); +} + +fn fail(message: String) -> ! { + report(&message); + std::process::exit(2) +} + +#[cfg(test)] +mod tests { + use super::*; + use domain::{CaptureSourceKind, RecorderError}; + use std::sync::atomic::AtomicU64; + + const TEST_DEADLINE: Duration = Duration::from_secs(1); + + struct Fake { + worker: Option, + events: mpsc::Receiver, + session: RecordingSession, + } + + impl Fake { + fn start(script: &str) -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + let id = NEXT.fetch_add(1, Ordering::Relaxed); + let session = RecordingSession { + id, + output_path: std::env::temp_dir() + .join(format!("wrec-worker-test-{}-{id}.mov", std::process::id())), + }; + let (events, received) = mpsc::channel(); + let worker = Worker::spawn( + Path::new("/bin/sh"), + &[ + "-c", + &format!(r#"echo $$ > "$0.pid"; {script}"#), + session.output_path.to_str().unwrap(), + ], + session.clone(), + CaptureTarget { + id: 0, + name: "display".into(), + kind: CaptureSourceKind::Display, + }, + RecorderSettings::default(), + events, + TEST_DEADLINE, + ) + .unwrap(); + Self { + worker: Some(worker), + events: received, + session, + } + } + + fn worker(&mut self) -> &mut Worker { + self.worker.as_mut().unwrap() + } + + fn terminal(&self) -> RecorderEvent { + loop { + let event = self.events.recv_timeout(Duration::from_secs(10)).unwrap(); + if matches!( + event, + RecorderEvent::Exited { .. } | RecorderEvent::Cancelled { .. } + ) { + if let Some(worker) = &self.worker { + assert!(worker.finished(), "terminal event published before reaping"); + } + assert!(self.reaped(), "terminal event published before reaping"); + return event; + } + } + } + + fn failure(&self) -> String { + match self.terminal() { + RecorderEvent::Exited { + success: false, + status, + .. + } => status, + event => panic!("expected a failed exit, received {event:?}"), + } + } + + fn reaped(&self) -> bool { + let pid = std::fs::read_to_string(self.pid_file()).unwrap(); + !Path::new(&format!("/proc/{}", pid.trim())).exists() + } + + fn pid_file(&self) -> std::path::PathBuf { + self.session.output_path.with_extension("mov.pid") + } + } + + impl Drop for Fake { + fn drop(&mut self) { + let _ = std::fs::remove_file(self.pid_file()); + let _ = std::fs::remove_file(&self.session.output_path); + } + } + + #[test] + fn protocol_round_trips_replies_and_events() { + let line = serde_json::to_string(&Message::Reply(Err(RecorderError::Backend( + "not ready".into(), + )))) + .unwrap(); + let Message::Reply(Err(RecorderError::Backend(message))) = + serde_json::from_str(&line).unwrap() + else { + panic!("reply changed shape: {line}") + }; + assert_eq!(message, "not ready"); + let line = serde_json::to_string(&Message::Event(RecorderEvent::Exited { + session_id: 7, + success: true, + status: "recording finalized".into(), + })) + .unwrap(); + assert!(matches!( + serde_json::from_str(&line).unwrap(), + Message::Event(RecorderEvent::Exited { + session_id: 7, + success: true, + .. + }) + )); + } + + #[test] + fn worker_crash_fails_the_recording_with_its_last_error() { + let fake = Fake::start( + r#"read start; : > "$0"; echo 'XIO: fatal IO error on X server' >&2; echo ' after 9 requests' >&2; exit 1"#, + ); + let status = fake.failure(); + assert!(status.contains("exited unexpectedly"), "{status}"); + assert!(status.contains("exit status: 1"), "{status}"); + assert!( + status.ends_with("XIO: fatal IO error on X server after 9 requests"), + "{status}" + ); + assert!(!fake.session.output_path.exists()); + } + + #[test] + fn worker_signal_after_frames_keeps_the_partial_movie() { + let fake = Fake::start( + r#"read start; echo partial > "$0"; echo '{"Event":{"Started":{"session_id":0,"dimensions":null}}}'; kill -9 $$"#, + ); + let status = fake.failure(); + assert!(status.contains("SIGKILL"), "{status}"); + assert!(fake.session.output_path.exists()); + } + + #[test] + fn worker_crash_before_a_fragment_is_written_removes_the_empty_movie() { + let fake = Fake::start( + r#"read start; : > "$0"; echo '{"Event":{"Started":{"session_id":0,"dimensions":null}}}'; exit 1"#, + ); + fake.failure(); + assert!(!fake.session.output_path.exists()); + } + + #[test] + fn replies_reach_the_waiting_command_and_finish_follows_reaping() { + let mut fake = Fake::start( + r#"read start; read pause; echo '{"Reply":{"Err":{"Backend":"not ready"}}}'; read resume; echo '{"Reply":{"Ok":null}}'; read stop; echo '{"Event":{"Cancelled":{"session_id":0}}}'"#, + ); + assert!( + matches!(fake.worker().control(Request::Pause), Err(RecorderError::Backend(message)) if message == "not ready") + ); + fake.worker().control(Request::Resume).unwrap(); + fake.worker().stop(); + assert!(matches!(fake.terminal(), RecorderEvent::Cancelled { .. })); + assert!(matches!( + fake.events.recv_timeout(Duration::from_secs(5)), + Err(mpsc::RecvTimeoutError::Disconnected) + )); + } + + #[test] + fn late_replies_are_not_given_to_the_next_command() { + let mut fake = Fake::start( + r#"read start; read pause; sleep 6; echo '{"Reply":{"Err":{"Backend":"late pause"}}}'; read resume; echo '{"Reply":{"Ok":null}}'; read stop; exit 0"#, + ); + let error = fake.worker().control(Request::Pause).unwrap_err(); + assert!( + error.to_string().contains("did not reply within 5s"), + "{error}" + ); + fake.worker().control(Request::Resume).unwrap(); + } + + #[test] + fn worker_death_while_a_command_waits_returns_promptly() { + let mut fake = Fake::start("read start; read pause; exit 3"); + let at = Instant::now(); + let error = fake.worker().control(Request::Pause).unwrap_err(); + assert!(at.elapsed() < Duration::from_secs(2), "{:?}", at.elapsed()); + assert!( + error.to_string().contains("exited before replying"), + "{error}" + ); + assert!(fake.failure().contains("exit status: 3")); + assert!(fake.worker().control(Request::Resume).is_err()); + } + + #[test] + fn invalid_worker_output_stops_the_worker() { + let fake = Fake::start("read start; echo not-json; exec sleep 30"); + let at = Instant::now(); + let status = fake.failure(); + assert!(status.contains("invalid message"), "{status}"); + assert!(at.elapsed() < Duration::from_secs(5), "{:?}", at.elapsed()); + } + + #[test] + fn frozen_worker_is_killed_after_the_stop_deadline() { + let mut fake = Fake::start( + r#"read start; echo '{"Event":{"Started":{"session_id":0,"dimensions":null}}}'; kill -STOP $$"#, + ); + assert!(matches!( + fake.events.recv_timeout(Duration::from_secs(5)).unwrap(), + RecorderEvent::Started { .. } + )); + let error = fake.worker().control(Request::Pause).unwrap_err(); + assert!( + error.to_string().contains("did not reply within 5s"), + "{error}" + ); + assert!(!fake.worker().finished()); + let at = Instant::now(); + fake.worker().stop(); + let status = fake.failure(); + assert!(at.elapsed() >= TEST_DEADLINE, "{:?}", at.elapsed()); + assert!(status.contains("did not exit within 1s"), "{status}"); + assert!(status.contains("SIGKILL"), "{status}"); + } + + #[test] + fn final_event_waits_until_a_hung_worker_is_killed_and_reaped() { + let mut fake = Fake::start( + r#"read start; echo '{"Event":{"Exited":{"session_id":0,"success":true,"status":"recording finalized"}}}'; exec sleep 30"#, + ); + assert!(matches!( + fake.events.recv_timeout(TEST_DEADLINE / 2), + Err(mpsc::RecvTimeoutError::Timeout) + )); + assert!(!fake.worker().finished()); + assert!(matches!( + fake.terminal(), + RecorderEvent::Exited { success: true, .. } + )); + } + + #[test] + fn teardown_that_hangs_after_a_failure_is_killed_without_a_stop_request() { + let fake = Fake::start( + r#"read start; echo '{"Event":{"Started":{"session_id":0,"dimensions":null}}}'; echo '"Stopping"'; exec sleep 30"#, + ); + let at = Instant::now(); + let status = fake.failure(); + assert!(at.elapsed() >= TEST_DEADLINE, "{:?}", at.elapsed()); + assert!(at.elapsed() < Duration::from_secs(5), "{:?}", at.elapsed()); + assert!(status.contains("did not exit within 1s"), "{status}"); + } + + #[test] + fn result_reported_before_a_hung_teardown_is_kept() { + let fake = Fake::start( + r#"read start; echo '"Stopping"'; echo '{"Event":{"Exited":{"session_id":0,"success":false,"status":"source lost"}}}'; exec sleep 30"#, + ); + assert!(matches!( + fake.terminal(), + RecorderEvent::Exited { success: false, status, .. } if status == "source lost" + )); + } + + #[test] + fn exiting_worker_is_finished_when_its_final_event_arrives() { + let fake = Fake::start( + r#"read start; echo '{"Event":{"Exited":{"session_id":0,"success":true,"status":"recording finalized"}}}'"#, + ); + let at = Instant::now(); + assert!(matches!( + fake.terminal(), + RecorderEvent::Exited { success: true, .. } + )); + assert!(at.elapsed() < TEST_DEADLINE, "{:?}", at.elapsed()); + } + + #[test] + fn dropping_the_handle_of_a_hung_worker_kills_it_after_the_deadline() { + let mut fake = Fake::start("read start; exec sleep 30"); + let at = Instant::now(); + drop(fake.worker.take()); + assert!(at.elapsed() < Duration::from_millis(200)); + assert!(fake.failure().contains("did not exit within 1s")); + } + + #[test] + fn dropping_the_handle_lets_a_responsive_worker_stop() { + let mut fake = Fake::start("read start; read stop; exit 4"); + drop(fake.worker.take()); + assert!(fake.failure().contains("exit status: 4")); + } +} diff --git a/crates/linux/src/x11.rs b/crates/linux/src/x11.rs index 65d3165..6f171a7 100644 --- a/crates/linux/src/x11.rs +++ b/crates/linux/src/x11.rs @@ -103,13 +103,13 @@ impl SourceWatch { .map_or(true, |at| at.elapsed() >= Duration::from_millis(500)) { self.checked.set(Some(Instant::now())); - let exists = self + let viewable = self .connection .get_window_attributes(self.xid) .ok() .and_then(|cookie| cookie.reply().ok()) - .is_some(); - if !exists { + .is_some_and(|attributes| attributes.map_state == MapState::VIEWABLE); + if !viewable { self.lost.store(true, Ordering::SeqCst); } } diff --git a/packaging/linux/README.md b/packaging/linux/README.md index 649874f..8f60fef 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -7,10 +7,10 @@ A working desktop and the required native plugins are needed. There is no Linux GUI yet. AMD Radeon capture has been tested on GNOME Wayland; Intel and NVIDIA hardware remain unverified. Software fallback makes recording possible without a supported GPU, but uses more CPU. -Software 4K HEVC can also need hundreds of MiB while encoding. On glibc builds, -wrec asks the allocator to return freed heap pages after each recording pipeline -has been fully torn down. This reduces idle memory retention, not the live -encoder working set; other allocators retain their own reclamation policy. +Software 4K HEVC can also need hundreds of MiB while encoding. That memory +belongs to the capture worker described below, and the system reclaims it when +the worker exits at the end of each recording. On glibc builds, the worker also +asks the allocator to return freed heap pages between encoder attempts. On compatible Wayland/VA-API systems, the preferred pipeline is `pipewiresrc → DMA-BUF → vapostproc → VA encoder → qtmux`. Rust manages sessions @@ -25,6 +25,51 @@ H.264 and x265 for HEVC. Retries happen only before the first encoded frame and preserve the requested codec. Job events identify the attempted and selected paths. X11 capture uses system memory even when encoding runs on the GPU. +## Capture worker + +The daemon does not run native capture code. For each recording it starts a +capture worker, which is the same daemon executable started again with a private +argument and the daemon's environment. The worker opens the X11 display or the +ScreenCast portal session, runs the GStreamer pipeline, and writes the movie. +Target discovery stays in the daemon. The daemon sends the start request, pause, +resume and stop to the worker's stdin as JSON lines. The worker answers on stdout +with recorder events and pause/resume replies. Anything native libraries print +goes to the worker's stderr, which the daemon copies into its log. + +Xlib exits the whole process when its X server connection breaks, and a GPU +driver can crash its process. Either now ends only the worker. When the worker +exits before reporting a result, the daemon marks that job failed with the exit +status and the worker's last stderr lines, then starts the next queued job. The +daemon PID stays the same. The daemon keeps a partial movie only if the worker +reported encoded frames and the file is not empty. + +A worker can also freeze or hang inside a native call. The daemon gives every +worker 20 seconds to exit after a stop request, after its final event, after its +output closes, or after the worker reports that it is tearing down a failed, +lost or cancelled recording, then kills it with SIGKILL. The worker sends that +report before it stops the native pipeline, so a GPU or plugin cleanup that +hangs cannot hold the job open. Recording, idle and paused time never start the +deadline, and neither does tearing down an encoder attempt that will be retried. The job's final status is +published only after the worker has been reaped, so the next queued job never +starts while the previous worker still runs. A worker killed this way before it +reported a result fails its job. While a worker is frozen, pause and resume fail +after the usual 5 seconds and the recording keeps its current state. + +The worker runs in its own process group, so Ctrl-C in a terminal reaches the +daemon, which then stops the worker gracefully. `kill ` does the +same and waits up to 15 seconds for finalization before the daemon exits. The +kernel kills the worker with SIGKILL as soon as the daemon process exits for any +reason, so a worker never outlives its daemon. If the daemon is killed or crashes +during a recording, the movie is not finalized. At most, it is playable through +the last completed fragment that reached the disk. Fragments still buffered by +the file writer are lost. In one test, killing the daemon 12 seconds into a +static-desktop recording left an empty file. + +While recording, resource use is the daemon plus its worker. Measuring the +daemon PID alone misses the capture pipeline. Find the worker with +`pgrep -P -x wrec-capture`, where the daemon PID comes from +`wrec daemon status --json`, and add its CPU and RSS to the daemon's. + ## Install dependencies and build Use GStreamer 1.22 or newer and current stable Rust. GStreamer 1.24 or newer is @@ -77,7 +122,10 @@ wrec record --target window:0 --codec h264 --no-system-audio On Wayland, `display:0` and `window:0` open the desktop's source picker; only advertised source types appear. On X11, use the window ID returned by `targets` instead of `window:0`; named windows and X screen roots are enumerated directly. -A multi-monitor X screen is captured as one desktop. Wayland is preferred when +A multi-monitor X screen is captured as one desktop. The captured X11 window +must stay viewable: closing it, minimizing it, or any other unmap fails the +recording within about half a second, including window managers that unmap +windows on other workspaces. Wayland is preferred when both session variables exist, because XWayland cannot capture the whole desktop. The duration starts after the first encoded frame, so time spent choosing a @@ -100,29 +148,75 @@ available. Configure devices in desktop sound settings. Linux cannot apply the shared wrec window-hiding or custom microphone-indicator options; job settings report them disabled with a warning. The desktop controls its sharing indicator. +On Wayland, a recording watches four things besides its frames: the ScreenCast +session's `Closed` signal, the D-Bus owner of `org.freedesktop.portal.Desktop`, +the PipeWire remote socket the portal returned, and the selected PipeWire node. +Frames alone cannot show that a source ended, because a static desktop or a +paused recording may deliver none for any length of time. When the desktop +closes the session, for example from its sharing indicator, wrec finalizes the +movie as if `wrec job stop` had run. When the portal loses its owner, the +PipeWire socket hangs up, or the node is removed, the job fails within a +fraction of a second, active or paused, with an event that names the service to restart. Any +partial movie stays on disk. wrec checks the socket with `poll` and never reads +it, so PipeWire's own reader sees all of its data. + +The node watch opens a second PipeWire remote through the same portal session, +so it sees only the nodes that session may use. It runs GStreamer's PipeWire +device provider, from the `gstreamer1.0-pipewire` package that recording already +needs, and fails the recording when the device for the selected node is removed. +A portal backend such as xdg-desktop-portal-wlr owns that node, so the node +disappears when the backend exits. A compositor exit is detected only when it +causes one of the four signals above, which has not been verified. If the node is not +visible on the second remote, or PipeWire does not list its nodes within 5 +seconds, the job logs that stream removal will not be detected and records +without the watch. If a desktop removes the node before its `Closed` signal +arrives, a stop from its sharing indicator fails the job with the stream-ended +message instead of finalizing it. + Stop drains the encoders and finalizes the movie, including when paused. The -writer emits ten-second movie fragments; an interrupted recording may only be -playable through its last completed fragment. PipeWire requests one keepalive -frame per second on a static desktop. HDR is not supported. +writer requests ten-second movie fragments; encoder keyframes can close them +earlier. An interrupted recording may only be playable through its last completed +fragment. Audio sources can deliver their +first samples later than video. The writer keeps that offset with a per-track +edit list, written in the first fragment and again at finalization, instead of +moving each track to start at zero. To do this, it keeps every sample table in +memory until the movie is finalized and then writes a complete index at the end +of the file. Measure long recordings using the whole daemon process tree and +include the time needed to finalize the index. PipeWire requests one keepalive frame per second on a static +desktop. HDR is not supported. ## Validate ```bash -sudo apt install ffmpeg dbus-daemon xvfb x11-apps +sudo apt install ffmpeg dbus-daemon xvfb x11-apps pipewire cargo fmt --check cargo check --workspace --locked cargo test --workspace --locked dbus-run-session -- cargo test -p linux portal_roundtrip --locked -- --ignored +cargo test -p linux node_watch --locked -- --ignored cargo build -p cli -p daemon python3 scripts/test-capture-linux.py target/debug/wrec ``` The isolated Xvfb test records actual X11 display/window pixels through the CLI and daemon, checks H.264/HEVC decoding and timestamps, and exercises pause/resume -and stopping while paused. It expects a machine without a hardware encoder so +and stopping while paused. It also kills the X server during a recording and +during a pause, with a second job queued. Both jobs must fail, the daemon PID must +stay the same, the worker must be reaped, and the same daemon must record again +after a new X server starts on that display. A job queued behind a closed +window, behind an unmapped window, and behind a worker frozen with SIGSTOP must +start at once and complete. +The frozen worker must be killed 20 seconds after stop, and killing the daemon +must kill its worker. It expects a machine without a hardware encoder so software fallback is exercised. CI also tests the extracted package and headless errors. Native pipeline tests cover AAC tracks, timing, capture errors, and strict -DMA-BUF rejection; an isolated mock portal covers options and session cleanup. +DMA-BUF rejection. An isolated mock portal covers options, session cleanup, +session `Closed`, portal owner loss and recovery, and PipeWire socket hangup +without consuming socket data. The node watch test starts a private PipeWire +daemon in a temporary directory, without the session bus. It checks that an +idle node is not reported, that removing the node is reported, that a missing +node disables the watch instead of failing, and that the watch stops promptly +after the daemon dies. Real hardware validation on Corex (Ryzen 9 5900H / Radeon Cezanne, Ubuntu 26.04, GNOME Wayland, GStreamer 1.28.2) recorded H.264 displays and an animated HEVC @@ -131,6 +225,18 @@ strictly increasing timestamps passed, including live system audio, pause/resume and stopping while paused. The 1080p animated-window sample used about 4% of one CPU core and 129 MiB peak daemon RSS. It recorded about 31 fps with a requested 60 fps ceiling; this is not sustained-60-fps validation or a general benchmark. +That run predates the capture worker. The pipeline ran inside the daemon then, so +the 129 MiB covered both roles. The worker loads its own copy of the executable +and native libraries, so the sum of daemon and worker RSS is expected to be +higher. Corex has not been measured again since the split. + +A release build on x86_64 Ubuntu with GStreamer 1.28.2, recording an isolated +Xvfb with the hardware encoder plugins hidden, measured the split directly. The +idle daemon used 9 to 11 MiB RSS. While recording an 800×600 display, the daemon +stayed near 10 MiB and the worker used 32 MiB with software H.264 and 66 MiB +with software HEVC. Native 4K software recordings peaked near 600 MiB worker RSS +and left the daemon at 11 MiB. These numbers cover software encoding on a +virtual X server, not GPU capture. Intel/NVIDIA, other compositors, live microphone sync, desktop sharing-control stops, HDR, sustained 60 fps, 4K, GPU memory and power need separate validation. diff --git a/scripts/test-capture-linux.py b/scripts/test-capture-linux.py index 9de90a2..de1cfaf 100644 --- a/scripts/test-capture-linux.py +++ b/scripts/test-capture-linux.py @@ -1,15 +1,54 @@ #!/usr/bin/env python3 """Record a real, isolated X11 desktop through the CLI, daemon, and native engine.""" +import ctypes import json import os from pathlib import Path import selectors +import signal import subprocess import sys import tempfile import time +def memory_mib(pid, field): + try: + status = Path(f"/proc/{pid}/status").read_text() + except FileNotFoundError: + return None + values = [int(line.split()[1]) / 1024 for line in status.splitlines() if line.startswith(f"{field}:")] + return values[0] if values else None + + +def workers(daemon_pid): + capture = [] + for process in Path("/proc").glob("[0-9]*"): + try: + parent = int((process / "stat").read_text().rsplit(")", 1)[1].split()[1]) + if parent == daemon_pid and (process / "comm").read_text().strip() == "wrec-capture": + capture.append(int(process.name)) + except (FileNotFoundError, ProcessLookupError, IndexError, ValueError): + pass + return capture + + +def without_workers(daemon_pid): + deadline = time.monotonic() + 5 + while workers(daemon_pid) and time.monotonic() < deadline: + time.sleep(0.1) + return not workers(daemon_pid) + + +def unmap(display, window): + xlib = ctypes.CDLL("libX11.so.6") + xlib.XOpenDisplay.restype = ctypes.c_void_p + connection = xlib.XOpenDisplay(display.encode()) + assert connection, f"could not open {display}" + xlib.XUnmapWindow(ctypes.c_void_p(connection), ctypes.c_ulong(window)) + xlib.XCloseDisplay(ctypes.c_void_p(connection)) + + def main(): binary = Path(sys.argv[1]).resolve() with tempfile.TemporaryDirectory(prefix="wrec-x11-capture-") as directory: @@ -19,7 +58,8 @@ def main(): env.pop(key, None) env.update(WREC_HOME=str(root / "home"), WREC_DATA_DIR=str(root / "data")) with open(root / "xvfb.log", "w") as log: - desktop = subprocess.Popen(["Xvfb", "-displayfd", "1", "-screen", "0", "800x600x24", "-screen", "1", "3840x2160x24", "-nolisten", "tcp"], stdout=subprocess.PIPE, stderr=log, text=True) + screens = ["-screen", "0", "800x600x24", "-screen", "1", "3840x2160x24", "-nolisten", "tcp"] + desktop = subprocess.Popen(["Xvfb", "-displayfd", "1", *screens], stdout=subprocess.PIPE, stderr=log, text=True) window = None try: with selectors.DefaultSelector() as selector: @@ -108,6 +148,8 @@ def verify(value, codec): assert closing_windows, "closing window was not discovered" closing_id = submit(f'window:{closing_windows[0]["id"]}', "h264") wait(closing_id, "recording") + after_id = submit("display:0", "h264", "2s") + assert job(after_id)["status"] == "queued", job(after_id) time.sleep(1) finally: closing.terminate() @@ -115,20 +157,115 @@ def verify(value, codec): lost = wait(closing_id, "failed") assert any("X11 window closed" in event["message"] for event in lost["events"]), lost assert run("daemon", "status", "--json")["pid"] == daemon_pid, "daemon restarted after the captured window closed" - after_id = submit("display:0", "h264", "2s") verify(wait(after_id, "completed"), "h264") assert run("daemon", "status", "--json")["pid"] == daemon_pid, "daemon restarted during capture after source loss" - print("PASS: closing a captured X11 window fails only that recording; the same daemon records again") + print("PASS: closing a captured X11 window fails only that recording; the queued job records next in the same daemon") + + hidden = subprocess.Popen(["xmessage", "-title", "Wrec unmapped window", "-geometry", "300x150+120+120", "Wrec Linux capture: this window is unmapped mid-recording."], env=env, stdout=log, stderr=log) + try: + for _ in range(50): + hidden_windows = [target for target in run("targets", "--json") if target["kind"] == "window" and "Wrec unmapped window" in target["name"]] + if hidden_windows: + break + time.sleep(0.1) + assert hidden_windows, "unmapped window was not discovered" + hidden_id = submit(f'window:{hidden_windows[0]["id"]}', "h264") + wait(hidden_id, "recording") + after_id = submit("display:0", "h264", "2s") + time.sleep(1) + unmapped_at = time.monotonic() + unmap(env["DISPLAY"], hidden_windows[0]["id"]) + unmapped = wait(hidden_id, "failed") + assert time.monotonic() - unmapped_at < 5, "unmapping the captured window was not detected promptly" + assert any("was minimized" in event["message"] for event in unmapped["events"]), unmapped + verify(wait(after_id, "completed"), "h264") + finally: + hidden.terminate() + hidden.wait(timeout=5) + assert run("daemon", "status", "--json")["pid"] == daemon_pid, "daemon restarted after the captured window was unmapped" + print("PASS: unmapping a captured X11 window fails only that recording; the queued job records next in the same daemon") + + frozen_id = submit("display:0", "h264") + wait(frozen_id, "recording") + frozen_workers = workers(daemon_pid) + assert len(frozen_workers) == 1, frozen_workers + queued_id = submit("display:0", "h264", "2s") + assert job(queued_id)["status"] == "queued", job(queued_id) + os.kill(frozen_workers[0], signal.SIGSTOP) + run("job", "stop", str(frozen_id), "--json") + deadline = time.monotonic() + 40 + while job(frozen_id)["status"] == "finishing" and time.monotonic() < deadline: + time.sleep(0.2) + frozen = job(frozen_id) + assert frozen["status"] == "failed", frozen + assert any("did not exit within 20s" in event["message"] for event in frozen["events"]), frozen + assert not Path(f"/proc/{frozen_workers[0]}").exists(), "frozen capture worker was not reaped" + verify(wait(queued_id, "completed"), "h264") + assert run("daemon", "status", "--json")["pid"] == daemon_pid, "daemon restarted after a frozen worker" + print("PASS: a frozen capture worker is killed 20s after stop; the queued job records next in the same daemon") large_display = next(target for target in targets if target["kind"] == "display" and "3840×2160" in target["name"]) + worker_peak_mib = 0 for codec in ("hevc", "h264", "hevc"): memory_id = run("record", "--target", f'display:{large_display["id"]}', "--codec", codec, "--quality", "high", "--resolution", "native", "--no-system-audio", "--no-mic", "--duration", "2s", "--out", str(root / "movies"), "--detach", "--json")["job"]["id"] + deadline = time.monotonic() + 30 + while job(memory_id)["status"] in ("queued", "starting", "recording", "finishing") and time.monotonic() < deadline: + for worker in workers(daemon_pid): + worker_peak_mib = max(worker_peak_mib, memory_mib(worker, "VmHWM") or 0) + time.sleep(0.2) verify(wait(memory_id, "completed"), codec) time.sleep(0.3) - status = Path(f"/proc/{daemon_pid}/status").read_text() - idle_rss_mib = int(next(line.split()[1] for line in status.splitlines() if line.startswith("VmRSS:"))) / 1024 + idle_rss_mib = memory_mib(daemon_pid, "VmRSS") assert idle_rss_mib < 256, f"4K encoder retained {idle_rss_mib:.1f} MiB after teardown" - print(f"PASS: repeated 4K capture releases idle heap pages ({idle_rss_mib:.1f} MiB daemon RSS)") + assert without_workers(daemon_pid), "capture worker outlived its recording" + assert worker_peak_mib > 0, "4K capture did not run in a wrec-capture worker" + print(f"PASS: repeated 4K capture runs in a worker ({worker_peak_mib:.1f} MiB peak worker RSS) and leaves {idle_rss_mib:.1f} MiB daemon RSS") + + display_number = env["DISPLAY"][1:] + for interrupted in ("recording", "paused"): + lost_id = submit("display:0", "h264") + wait(lost_id, "recording") + capture_workers = workers(daemon_pid) + assert len(capture_workers) == 1, capture_workers + queued_id = submit("display:0", "h264", "2s") + assert job(queued_id)["status"] == "queued", job(queued_id) + time.sleep(0.5) + if interrupted == "paused": + run("job", "pause", str(lost_id), "--json") + wait(lost_id, "paused") + desktop.kill() + desktop.wait(timeout=5) + lost = wait(lost_id, "failed") + assert any("capture worker exited unexpectedly" in event["message"] for event in lost["events"]), lost + queued = job(queued_id) + deadline = time.monotonic() + 30 + while queued["status"] not in ("completed", "failed", "cancelled") and time.monotonic() < deadline: + time.sleep(0.1) + queued = job(queued_id) + assert queued["status"] == "failed", queued + assert run("daemon", "status", "--json")["pid"] == daemon_pid, "losing the X server restarted the daemon" + assert without_workers(daemon_pid), "crashed capture worker was not reaped" + desktop = subprocess.Popen(["Xvfb", f":{display_number}", *screens], stderr=log) + for _ in range(50): + if subprocess.run([str(binary), "targets", "--json"], env=env, capture_output=True).returncode == 0: + break + time.sleep(0.1) + after_id = submit("display:0", "h264", "2s") + verify(wait(after_id, "completed"), "h264") + assert run("daemon", "status", "--json")["pid"] == daemon_pid, "daemon restarted after X server loss" + print(f"PASS: losing the X server while {interrupted} fails that job and the queued job; the same daemon records after X returns") + + orphan_id = submit("display:0", "h264") + wait(orphan_id, "recording") + orphan_workers = workers(daemon_pid) + assert len(orphan_workers) == 1, orphan_workers + os.kill(daemon_pid, signal.SIGKILL) + deadline = time.monotonic() + 5 + while Path(f"/proc/{orphan_workers[0]}").exists() and time.monotonic() < deadline: + time.sleep(0.1) + assert not Path(f"/proc/{orphan_workers[0]}").exists(), "capture worker outlived a killed daemon" + assert run("daemon", "start", "--json")["pid"] != daemon_pid, "a new daemon did not start after the old one was killed" + print("PASS: killing the daemon kills its capture worker") run("daemon", "stop", "--json") finally: subprocess.run([str(binary), "daemon", "stop", "--json"], env=env, capture_output=True, timeout=20) From cfb77c40a0eaa9e9612a31504c5cb056b12a01eb Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:50:06 +0000 Subject: [PATCH 10/45] fix(linux): preserve capture time on older VA encoders --- crates/linux/src/pipeline.rs | 241 ++++++++++++++++++++++++++++++++++- packaging/linux/README.md | 10 ++ 2 files changed, 250 insertions(+), 1 deletion(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 40d7c85..1e5cf1d 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -341,6 +341,9 @@ fn record_attempt( mode, counters.clone(), ); + if rewrites_timestamps(mode, &encoder) { + keep_capture_timestamps(&encoder, &parser); + } attach_encoded_probe(&parser, counters.clone()); // One overrun accompanies each incoming frame that replaces the oldest frame. let dropped = counters.clone(); @@ -396,6 +399,91 @@ fn attach_encoded_probe(parser: &gst::Element, counters: Arc) { }); } +const CAPTURE_TIME: &str = "timestamp/x-wrec-capture"; + +fn rewrites_timestamps(mode: Mode, encoder: &gst::Element) -> bool { + mode.kind == encoding::Kind::Va + && encoder + .factory() + .and_then(|factory| factory.plugin()) + .is_some_and(|plugin| { + let version: Vec = plugin + .version() + .split('.') + .map_while(|part| part.parse().ok()) + .collect(); + version < vec![1, 24, 3] + }) +} + +fn segment(pad: &gst::Pad) -> Option> { + pad.sticky_event::(0) + .and_then(|event| event.segment().clone().downcast::().ok()) +} + +const LOST_CAPTURE_TIME: &str = "A frame lost its capture timestamp in a VA encoder that replaces timestamps (GStreamer VA before 1.24.3). The recording stopped instead of writing video with incorrect timing."; + +fn lose_capture_time(pad: &gst::Pad) -> gst::PadProbeReturn { + if let Some(element) = pad.parent_element() { + gst::element_error!(element, gst::StreamError::Format, ("{}", LOST_CAPTURE_TIME)); + } + gst::PadProbeReturn::Drop +} + +fn keep_capture_timestamps(encoder: &gst::Element, parser: &gst::Element) { + let reference = gst::Caps::new_empty_simple(CAPTURE_TIME); + encoder + .static_pad("sink") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |pad, info| { + let running_time = info + .buffer() + .and_then(|buffer| buffer.pts()) + .zip(segment(pad)) + .and_then(|(pts, segment)| segment.to_running_time(pts)); + match (running_time, info.data.as_mut()) { + (Some(running_time), Some(gst::PadProbeData::Buffer(buffer))) => { + gst::ReferenceTimestampMeta::add( + buffer.make_mut(), + &reference, + running_time, + gst::ClockTime::NONE, + ); + gst::PadProbeReturn::Ok + } + _ => lose_capture_time(pad), + } + }); + parser + .static_pad("sink") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |pad, info| { + let Some(gst::PadProbeData::Buffer(buffer)) = info.data.as_mut() else { + return lose_capture_time(pad); + }; + let running_time = buffer + .iter_meta::() + .find(|meta| { + meta.reference() + .structure(0) + .is_some_and(|structure| structure.name() == CAPTURE_TIME) + }) + .map(|meta| meta.timestamp()); + let Some(pts) = running_time + .zip(segment(pad)) + .and_then(|(running_time, segment)| { + segment.position_from_running_time(running_time) + }) + else { + return lose_capture_time(pad); + }; + let buffer = buffer.make_mut(); + buffer.set_pts(pts); + buffer.set_dts(pts); + gst::PadProbeReturn::Ok + }); +} + fn add_audio( pipeline: &gst::Pipeline, mux: &gst::Element, @@ -726,6 +814,10 @@ mod tests { } fn start_with_audio_delays(audio_delays: &[gst::ClockTime]) -> Self { + Self::start_with(audio_delays, false) + } + + fn start_with(audio_delays: &[gst::ClockTime], counting_encoder: bool) -> Self { gst::init().unwrap(); static ID: AtomicU64 = AtomicU64::new(0); let id = ID.fetch_add(1, Ordering::Relaxed); @@ -736,7 +828,7 @@ mod tests { }; // Synthetic software encoding is confined to tests. Exercise the same // bus/control/mux code without pretending this is a hardware benchmark. - let pipeline = gst::parse::launch("videotestsrc name=video is-live=true pattern=ball ! video/x-raw,width=320,height=180,framerate=30/1 ! openh264enc ! h264parse name=parser") + let pipeline = gst::parse::launch("videotestsrc name=video is-live=true pattern=ball ! video/x-raw,width=320,height=180,framerate=30/1 ! openh264enc name=encoder ! h264parse name=parser") .unwrap().downcast::().unwrap(); let mux = movie_mux().unwrap(); let sink = element("filesink").unwrap(); @@ -746,6 +838,11 @@ mod tests { gst::Element::link_many([&pipeline.by_name("parser").unwrap(), &mux, &sink]).unwrap(); let counters = Arc::new(Counters::default()); attach_timing_probe(&pipeline.by_name("video").unwrap(), counters.clone()); + let encoder = pipeline.by_name("encoder").unwrap(); + if counting_encoder { + count_frames_like_old_va_encoders(&pipeline.by_name("video").unwrap(), &encoder); + } + keep_capture_timestamps(&encoder, &pipeline.by_name("parser").unwrap()); for &delay in audio_delays { let source = element("audiotestsrc").unwrap(); source.set_property("is-live", true); @@ -829,6 +926,54 @@ mod tests { } } + fn count_frames_like_old_va_encoders(source: &gst::Element, encoder: &gst::Element) { + source.static_pad("src").unwrap().add_probe( + gst::PadProbeType::BUFFER, + |_, info| match info.buffer().map(|buffer| buffer.offset() % 3) { + Some(2) => gst::PadProbeReturn::Drop, + _ => gst::PadProbeReturn::Ok, + }, + ); + let start = gst::ClockTime::from_seconds(60 * 60 * 1000); + let frame_duration = gst::ClockTime::SECOND / 30; + let first_pts = Mutex::new(None); + let count = AtomicU64::new(0); + encoder.static_pad("src").unwrap().add_probe( + gst::PadProbeType::BUFFER | gst::PadProbeType::EVENT_DOWNSTREAM, + move |_, info| { + match info.data.as_mut() { + Some(gst::PadProbeData::Event(event)) => { + if let gst::EventView::Segment(segment) = event.view() { + let mut segment = segment + .segment() + .clone() + .downcast::() + .unwrap(); + segment.set_start(segment.start().map(|time| time + start)); + segment.set_position(segment.position().map(|time| time + start)); + segment.set_stop(segment.stop().map(|time| time + start)); + *event = gst::event::Segment::new(&segment); + } + } + Some(gst::PadProbeData::Buffer(buffer)) => { + let first = *first_pts + .lock() + .unwrap() + .get_or_insert(buffer.pts().unwrap_or_default()); + let pts = + start + first + frame_duration * count.fetch_add(1, Ordering::Relaxed); + let buffer = buffer.make_mut(); + buffer.set_pts(pts); + buffer.set_dts(pts); + buffer.set_duration(frame_duration); + } + _ => {} + } + gst::PadProbeReturn::Ok + }, + ); + } + fn probe_movie(movie: &Path) -> serde_json::Value { let probe = decode_movie(movie); let mut previous = std::collections::HashMap::new(); @@ -948,6 +1093,100 @@ mod tests { } } + fn end_seconds(probe: &serde_json::Value, codec: &str) -> f64 { + let stream = probe["streams"] + .as_array() + .unwrap() + .iter() + .find(|stream| stream["codec_name"] == codec) + .unwrap(); + let seconds = |key: &str| stream[key].as_str().unwrap().parse::().unwrap(); + seconds("start_time") + seconds("duration") + } + + #[test] + fn capture_timestamps_survive_encoders_that_count_frames() { + let mut recording = TestRecording::start_with(&[gst::ClockTime::ZERO], true); + std::thread::sleep(Duration::from_millis(1500)); + recording.control(true); + std::thread::sleep(Duration::from_millis(700)); + recording.control(false); + std::thread::sleep(Duration::from_millis(1500)); + let active = recording + .counters + .timeline + .lock() + .unwrap() + .position(recording.pipeline.current_running_time().unwrap()) + .nseconds() as f64 + / 1_000_000_000.0; + recording.finish().unwrap(); + let probe = recording.probe(); + let (video, audio) = (end_seconds(&probe, "h264"), end_seconds(&probe, "aac")); + assert!( + (video - active).abs() < 0.2, + "video ends at {video}s after {active}s of active recording" + ); + assert!( + (video - audio).abs() < 0.15, + "video ends at {video}s, audio at {audio}s" + ); + let index = probe["streams"] + .as_array() + .unwrap() + .iter() + .find(|stream| stream["codec_name"] == "h264") + .unwrap()["index"] + .clone(); + let mut pts: Vec = probe["packets"] + .as_array() + .unwrap() + .iter() + .filter(|packet| packet["stream_index"] == index) + .map(|packet| packet["pts"].as_i64().unwrap()) + .collect(); + pts.sort_unstable(); + let skipped = pts + .windows(2) + .filter(|pair| pair[1] - pair[0] > 50_000) + .count(); + assert!( + skipped > 10, + "{skipped} video gaps show dropped capture frames" + ); + } + + #[test] + fn lost_capture_timestamps_fail_instead_of_speeding_up_video() { + let mut recording = TestRecording::start_with(&[gst::ClockTime::ZERO], true); + std::thread::sleep(Duration::from_millis(500)); + recording + .pipeline + .by_name("encoder") + .unwrap() + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, |_, info| { + if let Some(gst::PadProbeData::Buffer(buffer)) = info.data.as_mut() { + if let Some(meta) = buffer.make_mut().meta_mut::() + { + meta.remove().unwrap(); + } + } + gst::PadProbeReturn::Ok + }); + std::thread::sleep(Duration::from_secs(1)); + let _ = recording.stop.send(true); + let error = recording + .worker + .take() + .unwrap() + .join() + .unwrap() + .unwrap_err(); + assert!(error.to_string().contains(LOST_CAPTURE_TIME), "{error}"); + } + #[test] fn finalizes_playable_video_and_two_audio_tracks() { let mut recording = TestRecording::start(2); diff --git a/packaging/linux/README.md b/packaging/linux/README.md index 8f60fef..878ca04 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -97,6 +97,16 @@ Distribution codec packaging and GPU capabilities vary. VA-API also needs access to `/dev/dri/renderD*`; NVIDIA needs its driver and encode libraries. Missing hardware plugins do not prevent software recording. +GStreamer VA encoders before 1.24.3 replace each frame's capture timestamp with +its frame number times the nominal frame duration. When the screen delivers +fewer frames than that rate, the movie plays faster than the recorded time. +With these versions, wrec attaches each frame's capture time to the frame as a +`GstReferenceTimestampMeta` and restores it after the encoder, so VA encoding +keeps the original timing. VA 1.24.3 and later keep timestamps themselves and +skip this step. If a frame loses its capture time inside an affected encoder, +the recording fails with an error instead of writing faster video. This is not +a minimum version, and it does not switch to software encoding. + ```bash ./scripts/package-cli-linux.sh ``` From 2680da7cc2960325229990e3d4ec574d649a54bb Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:02:02 +0000 Subject: [PATCH 11/45] fix(linux): fail timing guards without native assertions --- crates/linux/src/pipeline.rs | 108 ++++++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 32 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 1e5cf1d..d6f7b9e 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -423,11 +423,12 @@ fn segment(pad: &gst::Pad) -> Option> { const LOST_CAPTURE_TIME: &str = "A frame lost its capture timestamp in a VA encoder that replaces timestamps (GStreamer VA before 1.24.3). The recording stopped instead of writing video with incorrect timing."; -fn lose_capture_time(pad: &gst::Pad) -> gst::PadProbeReturn { +fn lose_capture_time(pad: &gst::Pad, info: &mut gst::PadProbeInfo<'_>) -> gst::PadProbeReturn { if let Some(element) = pad.parent_element() { gst::element_error!(element, gst::StreamError::Format, ("{}", LOST_CAPTURE_TIME)); } - gst::PadProbeReturn::Drop + info.flow_res = Err(gst::FlowError::Error); + gst::PadProbeReturn::Handled } fn keep_capture_timestamps(encoder: &gst::Element, parser: &gst::Element) { @@ -441,46 +442,47 @@ fn keep_capture_timestamps(encoder: &gst::Element, parser: &gst::Element) { .and_then(|buffer| buffer.pts()) .zip(segment(pad)) .and_then(|(pts, segment)| segment.to_running_time(pts)); - match (running_time, info.data.as_mut()) { - (Some(running_time), Some(gst::PadProbeData::Buffer(buffer))) => { - gst::ReferenceTimestampMeta::add( - buffer.make_mut(), - &reference, - running_time, - gst::ClockTime::NONE, - ); - gst::PadProbeReturn::Ok - } - _ => lose_capture_time(pad), + if let (Some(running_time), Some(gst::PadProbeData::Buffer(buffer))) = + (running_time, info.data.as_mut()) + { + gst::ReferenceTimestampMeta::add( + buffer.make_mut(), + &reference, + running_time, + gst::ClockTime::NONE, + ); + return gst::PadProbeReturn::Ok; } + lose_capture_time(pad, info) }); parser .static_pad("sink") .unwrap() .add_probe(gst::PadProbeType::BUFFER, move |pad, info| { - let Some(gst::PadProbeData::Buffer(buffer)) = info.data.as_mut() else { - return lose_capture_time(pad); - }; - let running_time = buffer - .iter_meta::() - .find(|meta| { - meta.reference() - .structure(0) - .is_some_and(|structure| structure.name() == CAPTURE_TIME) + let pts = info + .buffer() + .and_then(|buffer| { + buffer + .iter_meta::() + .find(|meta| { + meta.reference() + .structure(0) + .is_some_and(|structure| structure.name() == CAPTURE_TIME) + }) + .map(|meta| meta.timestamp()) }) - .map(|meta| meta.timestamp()); - let Some(pts) = running_time .zip(segment(pad)) .and_then(|(running_time, segment)| { segment.position_from_running_time(running_time) - }) - else { - return lose_capture_time(pad); - }; - let buffer = buffer.make_mut(); - buffer.set_pts(pts); - buffer.set_dts(pts); - gst::PadProbeReturn::Ok + }); + if let (Some(pts), Some(gst::PadProbeData::Buffer(buffer))) = (pts, info.data.as_mut()) + { + let buffer = buffer.make_mut(); + buffer.set_pts(pts); + buffer.set_dts(pts); + return gst::PadProbeReturn::Ok; + } + lose_capture_time(pad, info) }); } @@ -1187,6 +1189,48 @@ mod tests { assert!(error.to_string().contains(LOST_CAPTURE_TIME), "{error}"); } + #[test] + fn missing_capture_time_fails_the_push_with_a_stream_error() { + gst::init().unwrap(); + let pipeline = PipelineGuard(gst::Pipeline::new()); + let encoder = element("identity").unwrap(); + let parser = element("identity").unwrap(); + let sink = element("fakesink").unwrap(); + pipeline.0.add_many([&encoder, &parser, &sink]).unwrap(); + parser.link(&sink).unwrap(); + keep_capture_timestamps(&encoder, &parser); + let _ = pipeline.0.set_state(gst::State::Playing); + let source = gst::Pad::builder(gst::PadDirection::Src).build(); + source.set_active(true).unwrap(); + source.link(&parser.static_pad("sink").unwrap()).unwrap(); + assert!(source.push_event(gst::event::StreamStart::new("capture"))); + assert!( + source.push_event(gst::event::Segment::new(&gst::FormattedSegment::< + gst::ClockTime, + >::new())) + ); + let mut buffer = gst::Buffer::new(); + buffer + .get_mut() + .unwrap() + .set_pts(gst::ClockTime::from_seconds(1)); + assert_eq!(source.push(buffer), Err(gst::FlowError::Error)); + let message = pipeline + .0 + .bus() + .unwrap() + .timed_pop_filtered(gst::ClockTime::from_seconds(5), &[gst::MessageType::Error]) + .unwrap(); + let gst::MessageView::Error(error) = message.view() else { + panic!("expected error") + }; + assert!( + error.error().to_string().contains(LOST_CAPTURE_TIME), + "{}", + error.error() + ); + } + #[test] fn finalizes_playable_video_and_two_audio_tracks() { let mut recording = TestRecording::start(2); From c9ed97612cd47fcee14d751627eeb6c14520fb7d Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:02:46 +0000 Subject: [PATCH 12/45] fix(linux): preserve the initial static portal frame on older VA --- crates/linux/src/encoding.rs | 62 +++++++++++++++++++++++++++++++++--- crates/linux/src/pipeline.rs | 36 ++++++++++++++------- packaging/linux/README.md | 21 ++++++++++-- 3 files changed, 101 insertions(+), 18 deletions(-) diff --git a/crates/linux/src/encoding.rs b/crates/linux/src/encoding.rs index 270a213..512db27 100644 --- a/crates/linux/src/encoding.rs +++ b/crates/linux/src/encoding.rs @@ -17,13 +17,46 @@ pub(crate) struct Mode { pub dmabuf: bool, } -pub(crate) fn modes(codec: Codec, pipewire: bool) -> Vec { - modes_with(codec, pipewire, |name| { +pub(crate) fn modes(codec: Codec, dmabuf: bool) -> Vec { + modes_with(codec, dmabuf, |name| { gst::ElementFactory::find(name).is_some() }) } -fn modes_with(codec: Codec, pipewire: bool, has: impl Fn(&str) -> bool) -> Vec { +pub(crate) fn plugin_version(factory: &gst::ElementFactory) -> Option> { + factory.plugin().map(|plugin| { + plugin + .version() + .split('.') + .map_while(|part| part.parse().ok()) + .collect() + }) +} + +fn imports_dma_buf(sink: &gst::CapsRef, version: &[u32]) -> bool { + let dma_drm = gst::Structure::builder("video/x-raw") + .field("format", "DMA_DRM") + .build(); + version >= [1, 24, 6].as_slice() + && sink.iter_with_features().any(|(structure, features)| { + !features.is_any() + && features.contains("memory:DMABuf") + && structure.has_field("format") + && structure.can_intersect(&dma_drm) + }) +} + +pub(crate) fn va_imports_dma_buf() -> bool { + gst::ElementFactory::find("vapostproc").is_some_and(|factory| { + let version = plugin_version(&factory).unwrap_or_default(); + factory.static_pad_templates().iter().any(|template| { + template.direction() == gst::PadDirection::Sink + && imports_dma_buf(&template.caps(), &version) + }) + }) +} + +fn modes_with(codec: Codec, dmabuf: bool, has: impl Fn(&str) -> bool) -> Vec { let (va, nv, software): (&[&str], &str, &[&str]) = match codec { Codec::H264 => ( &["vah264lpenc", "vah264enc"], @@ -38,7 +71,7 @@ fn modes_with(codec: Codec, pipewire: bool, has: impl Fn(&str) -> bool) -> Vec().unwrap(); + let native = caps("video/x-raw(memory:VAMemory), format={ NV12, P010_10LE }; video/x-raw(memory:DMABuf), format={ BGRA, RGBA, BGRx, RGBx, NV12, P010_10LE }; video/x-raw(ANY); video/x-raw(ANY), format={ BGRx, NV12 }"); + let formatless = caps("video/x-raw(memory:DMABuf), width=[1, 16384]"); + let modern = caps("video/x-raw(memory:VAMemory), format={ NV12, P010_10LE }; video/x-raw(memory:DMABuf), format=DMA_DRM, drm-format={ NV12:0x0200000000000901, XR24:0x0200000000000901 }; video/x-raw, format={ BGRx, NV12 }"); + assert!(!imports_dma_buf(&native, &[1, 22, 0])); + assert!(!imports_dma_buf(&native, &[1, 28, 2])); + assert!(!imports_dma_buf(&formatless, &[1, 28, 2])); + assert!(!imports_dma_buf(&modern, &[1, 24, 0])); + assert!(!imports_dma_buf(&modern, &[1, 24, 5])); + assert!(imports_dma_buf(&modern, &[1, 24, 6])); + assert!(imports_dma_buf(&modern, &[1, 28, 2])); + if let Some(factory) = gst::ElementFactory::find("vapostproc") { + if plugin_version(&factory).is_some_and(|version| version >= vec![1, 24, 6]) { + assert!(va_imports_dma_buf()); + } + } + } + #[test] fn prefers_hardware_and_retains_software_compatibility() { let modes = modes_with(Codec::H264, true, |_| true); diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index d6f7b9e..b9e095e 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -84,7 +84,9 @@ fn capture_caps(dmabuf: bool, pipewire: bool, fps: u32) -> gst::Caps { gst::Fraction::new(fps as i32, 1), ); if dmabuf { - caps.features(["memory:DMABuf"]).build() + caps.features(["memory:DMABuf"]) + .field("format", "DMA_DRM") + .build() } else { caps.build() } @@ -197,7 +199,8 @@ pub(crate) fn record( crate::x11::initialize()?; } check_plugins(settings)?; - let modes = encoding::modes(settings.codec, matches!(capture, CaptureInput::PipeWire(_))); + let dmabuf = matches!(capture, CaptureInput::PipeWire(_)) && encoding::va_imports_dma_buf(); + let modes = encoding::modes(settings.codec, dmabuf); if modes.is_empty() { return Err(backend(format!("No {} encoder is installed. Install GStreamer VA/NVENC plugins or x264/x265/openh264 for software compatibility; see packaging/linux/README.md.", settings.codec.as_arg()))); } @@ -405,15 +408,8 @@ fn rewrites_timestamps(mode: Mode, encoder: &gst::Element) -> bool { mode.kind == encoding::Kind::Va && encoder .factory() - .and_then(|factory| factory.plugin()) - .is_some_and(|plugin| { - let version: Vec = plugin - .version() - .split('.') - .map_while(|part| part.parse().ok()) - .collect(); - version < vec![1, 24, 3] - }) + .and_then(|factory| encoding::plugin_version(&factory)) + .is_some_and(|version| version < vec![1, 24, 3]) } fn segment(pad: &gst::Pad) -> Option> { @@ -791,6 +787,24 @@ mod tests { ); } + #[test] + fn dma_buf_capture_requires_an_explicit_drm_format() { + gst::init().unwrap(); + let capture = capture_caps(true, true, 60); + let caps = |description: &str| description.parse::().unwrap(); + let legacy_source = caps("video/x-raw(memory:DMABuf), format=BGRx, width=1280, height=720, framerate=0/1, max-framerate=60/1"); + let legacy_converter = caps("video/x-raw(memory:DMABuf), width=[1, 16384], height=[1, 16384], format={ BGRA, RGBA, BGRx, RGBx, NV12, P010_10LE }"); + let modern_source = caps("video/x-raw(memory:DMABuf), format=DMA_DRM, drm-format=XR24:0x0200000000000901, width=1280, height=720, framerate=0/1, max-framerate=60/1"); + let modern_converter = caps("video/x-raw(memory:DMABuf), width=[1, 16384], height=[1, 16384], format=DMA_DRM, drm-format={ NV12:0x0200000000000901, XR24:0x0200000000000901 }"); + assert!(!capture.can_intersect(&legacy_source)); + assert!(!capture.can_intersect(&legacy_converter)); + assert!(capture.can_intersect(&modern_source)); + assert!(capture.can_intersect(&modern_converter)); + assert!(capture_caps(false, true, 60).can_intersect(&caps( + "video/x-raw, format=BGRx, width=1280, height=720, framerate=0/1, max-framerate=60/1" + ))); + } + #[test] fn dimensions_preserve_aspect_and_do_not_upscale() { assert_eq!(output_size(3840, 2160, Resolution::R1080p), (1920, 1080)); diff --git a/packaging/linux/README.md b/packaging/linux/README.md index 878ca04..c5ba80d 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -25,6 +25,21 @@ H.264 and x265 for HEVC. Retries happen only before the first encoded frame and preserve the requested codec. Job events identify the attempted and selected paths. X11 capture uses system memory even when encoding runs on the GPU. +wrec tries shared GPU buffers only when `vapostproc` lists DMA-BUF caps with +`format=DMA_DRM` and comes from GStreamer VA 1.24.6 or later. `DMA_DRM` caps +carry the pixel format and modifier with an explicit `drm-format`. GStreamer VA +1.22 describes DMA-BUF with a plain format such as `BGRx` and no modifier. +GStreamer VA before 1.24.6 imports a DMA-BUF only when each buffer's memory +covers a whole video plane. 1.24.6 needs one byte and reads strides from video +metadata. Producers decide the sizes that PipeWire reports for a DMA-BUF. +xdg-desktop-portal-wlr reports a memory size of 0, so its buffers fail that +older check, and the frame is lost before encoding starts. With older VA, wrec +skips the shared-buffer attempt and starts with system-memory capture, still +encoding on the GPU. The skipped attempt matters on a static screen. +xdg-desktop-portal-wlr sends a frame only when the screen changes, so an +attempt that receives the initial frame and fails leaves later attempts with +no frame until something on screen changes. + ## Capture worker The daemon does not run native capture code. For each recording it starts a @@ -72,9 +87,9 @@ daemon PID alone misses the capture pipeline. Find the worker with ## Install dependencies and build -Use GStreamer 1.22 or newer and current stable Rust. GStreamer 1.24 or newer is -recommended for the DMA-BUF/VA path. Ubuntu 24.04 is a starting point; package -names differ across distributions. +Use GStreamer 1.22 or newer and current stable Rust. The shared-buffer +DMA-BUF/VA path needs GStreamer VA 1.24.6 or newer. Ubuntu 24.04 is a starting +point; package names differ across distributions. ```bash sudo apt install build-essential pkg-config libgstreamer1.0-dev \ From 12e9b257840a2ad4ffcfc415d893082e55e6d7e6 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:21:04 +0000 Subject: [PATCH 13/45] fix(linux): keep portal startup clocks and streams stable --- crates/linux/src/pipeline.rs | 167 ++++++++++++++++++++++++++++++++++- packaging/linux/README.md | 23 ++++- 2 files changed, 187 insertions(+), 3 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index b9e095e..7f7f099 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -344,6 +344,7 @@ fn record_attempt( mode, counters.clone(), ); + keep_source_configuration(&queue); if rewrites_timestamps(mode, &encoder) { keep_capture_timestamps(&encoder, &parser); } @@ -549,6 +550,33 @@ fn attach_timing_probe(source: &gst::Element, counters: Arc) { }); } +thread_local! { + static SETTING_CANVAS: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +fn set_canvas(output: &gst::Element, caps: gst::Caps) { + SETTING_CANVAS.with(|setting| setting.set(true)); + output.set_property("caps", caps); + SETTING_CANVAS.with(|setting| setting.set(false)); +} + +fn keep_source_configuration(queue: &gst::Element) { + queue + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::EVENT_UPSTREAM, |_, info| { + let canvas = info + .event() + .is_some_and(|event| event.type_() == gst::EventType::Reconfigure) + && SETTING_CANVAS.with(|setting| setting.get()); + if canvas { + gst::PadProbeReturn::Handled + } else { + gst::PadProbeReturn::Ok + } + }); +} + fn attach_capture_probe( source: &gst::Element, output: &gst::Element, @@ -571,13 +599,19 @@ fn attach_capture_probe( .unwrap_or_else(|| output_size(width, height, resolution)); dimensions.get_or_insert(CaptureDimensions { native_width: width.into(), native_height: height.into(), output_width: w.into(), output_height: h.into() }); drop(dimensions); - output.set_property("caps", mode.caps(Some((w, h)))); + set_canvas(&output, mode.caps(Some((w, h)))); } } } } } if let Some(gst::PadProbeData::Buffer(buffer)) = &info.data { + if buffer.flags().contains(gst::BufferFlags::CORRUPTED) && counters.last_pts.lock().unwrap().is_none() { + if let Some(element) = pad.parent_element() { + gst::element_error!(element, gst::StreamError::Failed, ("{}", UNFINISHED_FIRST_FRAME)); + } + return gst::PadProbeReturn::Drop; + } if let Some(pts) = buffer.pts() { let mut previous = counters.last_pts.lock().unwrap(); if previous.is_some_and(|last| pts <= last || pts.saturating_sub(last) < gst::ClockTime::from_useconds(1)) { @@ -619,6 +653,8 @@ fn state_error(pipeline: &gst::Pipeline, fallback: impl std::fmt::Display) -> Re backend(fallback) } +const UNFINISHED_FIRST_FRAME: &str = "The screen-capture source started with an unfinished frame. Its timestamp would stall capture timing, so this attempt stopped before encoding and another available mode will be tried."; + const SOURCE_LOST: &str = "The captured X11 window closed, was minimized, or became unavailable; the recording stopped."; @@ -1361,6 +1397,135 @@ mod tests { assert!(error.to_string().contains("injected capture failure")); } + fn first_unfinished_frame(corrupted_offset: u64) -> Option { + gst::init().unwrap(); + let pipeline = PipelineGuard(gst::parse::launch("videotestsrc num-buffers=3 name=source ! video/x-raw,width=320,height=180 ! fakesink").unwrap().downcast::().unwrap()); + let source = pipeline.0.by_name("source").unwrap(); + source + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + if let Some(gst::PadProbeData::Buffer(buffer)) = info.data.as_mut() { + if buffer.offset() == corrupted_offset { + buffer.make_mut().set_flags(gst::BufferFlags::CORRUPTED); + } + } + gst::PadProbeReturn::Ok + }); + attach_capture_probe( + &source, + &element("capsfilter").unwrap(), + Resolution::Native, + Mode { + kind: encoding::Kind::Software, + factory: "x264enc", + dmabuf: false, + }, + Arc::new(Counters::default()), + ); + pipeline.0.set_state(gst::State::Playing).unwrap(); + let message = pipeline + .0 + .bus() + .unwrap() + .timed_pop_filtered( + gst::ClockTime::from_seconds(5), + &[gst::MessageType::Error, gst::MessageType::Eos], + ) + .unwrap(); + match message.view() { + gst::MessageView::Error(error) => Some(error.error().to_string()), + _ => None, + } + } + + #[test] + fn an_unfinished_first_frame_fails_the_attempt_before_encoding() { + let error = first_unfinished_frame(0).expect("an unfinished first frame must fail"); + assert!(error.contains(UNFINISHED_FIRST_FRAME), "{error}"); + assert_eq!(first_unfinished_frame(1), None); + } + + fn source_reconfigures_while_scaling_to_720p(keep_source: bool) -> (u64, (i32, i32)) { + gst::init().unwrap(); + let pipeline = PipelineGuard(gst::parse::launch("videotestsrc num-buffers=10 name=source ! video/x-raw,width=1920,height=1080,framerate=30/1 ! queue name=queue ! videoconvert ! videoscale add-borders=true ! capsfilter name=output ! fakesink name=sink").unwrap().downcast::().unwrap()); + let source = pipeline.0.by_name("source").unwrap(); + let output = pipeline.0.by_name("output").unwrap(); + let mode = Mode { + kind: encoding::Kind::Software, + factory: "x264enc", + dmabuf: false, + }; + output.set_property("caps", mode.caps(None)); + let configured = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let reconfigures = Arc::new(AtomicU64::new(0)); + let (seen, counted) = (configured.clone(), reconfigures.clone()); + source.static_pad("src").unwrap().add_probe( + gst::PadProbeType::EVENT_DOWNSTREAM | gst::PadProbeType::EVENT_UPSTREAM, + move |_, info| { + match info.event().map(|event| event.type_()) { + Some(gst::EventType::Caps) => seen.store(true, Ordering::SeqCst), + Some(gst::EventType::Reconfigure) if seen.load(Ordering::SeqCst) => { + counted.fetch_add(1, Ordering::SeqCst); + } + _ => {} + } + gst::PadProbeReturn::Ok + }, + ); + attach_capture_probe( + &source, + &output, + Resolution::R720p, + mode, + Arc::new(Counters::default()), + ); + if keep_source { + keep_source_configuration(&pipeline.0.by_name("queue").unwrap()); + } + pipeline.0.set_state(gst::State::Playing).unwrap(); + let message = pipeline + .0 + .bus() + .unwrap() + .timed_pop_filtered( + gst::ClockTime::from_seconds(5), + &[gst::MessageType::Error, gst::MessageType::Eos], + ) + .unwrap(); + assert!( + matches!(message.view(), gst::MessageView::Eos(_)), + "{message:?}" + ); + let caps = pipeline + .0 + .by_name("sink") + .unwrap() + .static_pad("sink") + .unwrap() + .current_caps() + .unwrap(); + let structure = caps.structure(0).unwrap(); + ( + reconfigures.load(Ordering::SeqCst), + ( + structure.get::("width").unwrap(), + structure.get::("height").unwrap(), + ), + ) + } + + #[test] + fn setting_the_output_canvas_does_not_renegotiate_the_source() { + assert_eq!( + source_reconfigures_while_scaling_to_720p(true), + (0, (1280, 720)) + ); + let (reconfigures, canvas) = source_reconfigures_while_scaling_to_720p(false); + assert!(reconfigures > 0); + assert_eq!(canvas, (1280, 720)); + } + #[test] fn rejects_cpu_buffers_before_pixel_processing() { gst::init().unwrap(); diff --git a/packaging/linux/README.md b/packaging/linux/README.md index c5ba80d..c7d5dfc 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -40,6 +40,25 @@ xdg-desktop-portal-wlr sends a frame only when the screen changes, so an attempt that receives the initial frame and fails leaves later attempts with no frame until something on screen changes. +`pipewiresrc` pauses its stream briefly while starting. When that pause +interrupts a capture, xdg-desktop-portal-wlr sends the unfinished buffer marked +corrupted, with timestamp 0. GStreamer aligns a live source's timestamps to +the first buffer it sees. When that buffer is the corrupted one, every real +frame looks due hours in the future, so the source waits instead of delivering +frames. This +was seen with shared GPU buffers on xdg-desktop-portal-wlr 0.8.1 and PipeWire +1.6.2. wrec stops an attempt whose first frame is marked corrupted before +anything is encoded and moves on to the next available mode. + +wrec sets the movie's output size when the first source caps arrive. Changing +a capsfilter normally asks every upstream element to renegotiate, and +`pipewiresrc` answers by disconnecting and reconnecting its PipeWire stream, +even with identical caps. xdg-desktop-portal-wlr 0.8.1 with +ext-image-copy-capture then requests a new frame while the first one is still +pending, breaks the Wayland protocol, and exits. wrec stops that request at +the video queue, so the converters still renegotiate their output and the +capture source keeps its stream. Other renegotiation requests pass through. + ## Capture worker The daemon does not run native capture code. For each recording it starts a @@ -88,8 +107,8 @@ daemon PID alone misses the capture pipeline. Find the worker with ## Install dependencies and build Use GStreamer 1.22 or newer and current stable Rust. The shared-buffer -DMA-BUF/VA path needs GStreamer VA 1.24.6 or newer. Ubuntu 24.04 is a starting -point; package names differ across distributions. +DMA-BUF/VA path needs GStreamer VA 1.24.6 or newer. Ubuntu 24.04 is a starting point; package +names differ across distributions. ```bash sudo apt install build-essential pkg-config libgstreamer1.0-dev \ From aa1cb1ee69c24d5109a40a2799ba7c54bdc6a8ad Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:47:02 +0000 Subject: [PATCH 14/45] fix(linux): stop rejected capture pushes and refresh resized video pools --- crates/linux/src/pipeline.rs | 308 ++++++++++++++++++++++++++++++----- packaging/linux/README.md | 20 ++- 2 files changed, 283 insertions(+), 45 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 7f7f099..e23bbd0 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -585,50 +585,79 @@ fn attach_capture_probe( counters: Arc, ) { let output = output.clone(); - source.static_pad("src").unwrap().add_probe(gst::PadProbeType::EVENT_DOWNSTREAM | gst::PadProbeType::BUFFER, move |pad, info| { - if !retime(info, &counters) { return gst::PadProbeReturn::Drop; } - if let Some(gst::PadProbeData::Event(event)) = &info.data { - if let gst::EventView::Caps(caps) = event.view() { - if let Some(structure) = caps.caps().structure(0) { - if let (Ok(width), Ok(height)) = (structure.get::("width"), structure.get::("height")) { - if width >= 2 && height >= 2 { - let mut dimensions = counters.dimensions.lock().unwrap(); - // A MOV track has a fixed canvas. Window resizes are - // scaled/letterboxed into the initial output size. - let (w, h) = dimensions.map(|d| (d.output_width as i32, d.output_height as i32)) - .unwrap_or_else(|| output_size(width, height, resolution)); - dimensions.get_or_insert(CaptureDimensions { native_width: width.into(), native_height: height.into(), output_width: w.into(), output_height: h.into() }); - drop(dimensions); - set_canvas(&output, mode.caps(Some((w, h)))); + let resized = std::sync::atomic::AtomicBool::new(false); + source.static_pad("src").unwrap().add_probe( + gst::PadProbeType::EVENT_DOWNSTREAM | gst::PadProbeType::BUFFER, + move |pad, info| { + if !retime(info, &counters) { + return gst::PadProbeReturn::Drop; + } + if let Some(gst::PadProbeData::Event(event)) = &info.data { + if let gst::EventView::Caps(caps) = event.view() { + if let Some(structure) = caps.caps().structure(0) { + if let (Ok(width), Ok(height)) = ( + structure.get::("width"), + structure.get::("height"), + ) { + if width >= 2 && height >= 2 { + let mut dimensions = counters.dimensions.lock().unwrap(); + resized.store(dimensions.is_some(), Ordering::Relaxed); + // A MOV track has a fixed canvas. Window resizes are + // scaled/letterboxed into the initial output size. + let (w, h) = dimensions + .map(|d| (d.output_width as i32, d.output_height as i32)) + .unwrap_or_else(|| output_size(width, height, resolution)); + dimensions.get_or_insert(CaptureDimensions { + native_width: width.into(), + native_height: height.into(), + output_width: w.into(), + output_height: h.into(), + }); + drop(dimensions); + set_canvas(&output, mode.caps(Some((w, h)))); + } } } } } - } - if let Some(gst::PadProbeData::Buffer(buffer)) = &info.data { - if buffer.flags().contains(gst::BufferFlags::CORRUPTED) && counters.last_pts.lock().unwrap().is_none() { - if let Some(element) = pad.parent_element() { - gst::element_error!(element, gst::StreamError::Failed, ("{}", UNFINISHED_FIRST_FRAME)); + if let Some(gst::PadProbeData::Buffer(buffer)) = &info.data { + if buffer.flags().contains(gst::BufferFlags::CORRUPTED) + && counters.last_pts.lock().unwrap().is_none() + { + return reject_capture( + pad, + info, + gst::StreamError::Failed, + UNFINISHED_FIRST_FRAME, + ); } - return gst::PadProbeReturn::Drop; - } - if let Some(pts) = buffer.pts() { - let mut previous = counters.last_pts.lock().unwrap(); - if previous.is_some_and(|last| pts <= last || pts.saturating_sub(last) < gst::ClockTime::from_useconds(1)) { - counters.dropped.fetch_add(1, Ordering::Relaxed); - return gst::PadProbeReturn::Drop; + if let Some(pts) = buffer.pts() { + let mut previous = counters.last_pts.lock().unwrap(); + if previous.is_some_and(|last| { + pts <= last || pts.saturating_sub(last) < gst::ClockTime::from_useconds(1) + }) { + counters.dropped.fetch_add(1, Ordering::Relaxed); + return gst::PadProbeReturn::Drop; + } + *previous = Some(pts); } - *previous = Some(pts); - } - if mode.dmabuf && (buffer.n_memory() == 0 || buffer.iter_memories().any(|memory| !memory.is_memory_type::())) { - if let Some(element) = pad.parent_element() { - gst::element_error!(element, gst::StreamError::Format, ("PipeWire delivered a non-DMA-BUF frame. This attempt requires GPU buffer sharing; another available mode will be tried before capture starts.")); + if mode.dmabuf + && (buffer.n_memory() == 0 + || buffer.iter_memories().any(|memory| { + !memory.is_memory_type::() + })) + { + return reject_capture(pad, info, gst::StreamError::Format, NON_DMA_BUF_FRAME); + } + if resized.swap(false, Ordering::Relaxed) { + let mut allocation = + gst::query::Allocation::new(pad.current_caps().as_ref(), true); + pad.peer_query(&mut allocation); } - return gst::PadProbeReturn::Drop; } - } - gst::PadProbeReturn::Ok - }); + gst::PadProbeReturn::Ok + }, + ); } fn state_error(pipeline: &gst::Pipeline, fallback: impl std::fmt::Display) -> RecorderError { @@ -653,6 +682,21 @@ fn state_error(pipeline: &gst::Pipeline, fallback: impl std::fmt::Display) -> Re backend(fallback) } +const NON_DMA_BUF_FRAME: &str = "PipeWire delivered a non-DMA-BUF frame. This attempt requires GPU buffer sharing; another available mode will be tried before capture starts."; + +fn reject_capture( + pad: &gst::Pad, + info: &mut gst::PadProbeInfo<'_>, + error: gst::StreamError, + message: &str, +) -> gst::PadProbeReturn { + if let Some(element) = pad.parent_element() { + gst::element_error!(element, error, ("{}", message)); + } + info.flow_res = Err(gst::FlowError::Error); + gst::PadProbeReturn::Handled +} + const UNFINISHED_FIRST_FRAME: &str = "The screen-capture source started with an unfinished frame. Its timestamp would stall capture timing, so this attempt stopped before encoding and another available mode will be tried."; const SOURCE_LOST: &str = @@ -1397,9 +1441,33 @@ mod tests { assert!(error.to_string().contains("injected capture failure")); } - fn first_unfinished_frame(corrupted_offset: u64) -> Option { + fn downstream_buffers(pipeline: &gst::Pipeline) -> Arc { + let count = Arc::new(AtomicU64::new(0)); + let counted = count.clone(); + pipeline + .by_name("sink") + .unwrap() + .static_pad("sink") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, _| { + counted.fetch_add(1, Ordering::SeqCst); + gst::PadProbeReturn::Ok + }); + count + } + + fn settle(pipeline: &gst::Pipeline) { + pipeline + .bus() + .unwrap() + .timed_pop_filtered(gst::ClockTime::from_seconds(1), &[gst::MessageType::Eos]); + let _ = pipeline.set_state(gst::State::Null); + } + + fn first_unfinished_frame(corrupted_offset: u64) -> (Option, u64) { gst::init().unwrap(); - let pipeline = PipelineGuard(gst::parse::launch("videotestsrc num-buffers=3 name=source ! video/x-raw,width=320,height=180 ! fakesink").unwrap().downcast::().unwrap()); + let pipeline = PipelineGuard(gst::parse::launch("videotestsrc num-buffers=3 name=source ! video/x-raw,width=320,height=180 ! fakesink name=sink").unwrap().downcast::().unwrap()); + let delivered = downstream_buffers(&pipeline.0); let source = pipeline.0.by_name("source").unwrap(); source .static_pad("src") @@ -1433,17 +1501,21 @@ mod tests { &[gst::MessageType::Error, gst::MessageType::Eos], ) .unwrap(); - match message.view() { + let error = match message.view() { gst::MessageView::Error(error) => Some(error.error().to_string()), _ => None, - } + }; + settle(&pipeline.0); + (error, delivered.load(Ordering::SeqCst)) } #[test] fn an_unfinished_first_frame_fails_the_attempt_before_encoding() { - let error = first_unfinished_frame(0).expect("an unfinished first frame must fail"); + let (error, delivered) = first_unfinished_frame(0); + let error = error.expect("an unfinished first frame must fail"); assert!(error.contains(UNFINISHED_FIRST_FRAME), "{error}"); - assert_eq!(first_unfinished_frame(1), None); + assert_eq!(delivered, 0, "no frame may follow a rejected first frame"); + assert_eq!(first_unfinished_frame(1), (None, 3)); } fn source_reconfigures_while_scaling_to_720p(keep_source: bool) -> (u64, (i32, i32)) { @@ -1526,10 +1598,158 @@ mod tests { assert_eq!(canvas, (1280, 720)); } + fn resize_without_source_allocation( + converter: &str, + ) -> ( + Vec>, + u64, + u64, + ) { + gst::init().unwrap(); + let pipeline = PipelineGuard(gst::parse::launch(&format!("queue name=source ! queue name=queue ! {converter} ! capsfilter name=output ! fakesink name=sink")).unwrap().downcast::().unwrap()); + let delivered = downstream_buffers(&pipeline.0); + let allocations = Arc::new(AtomicU64::new(0)); + let counted = allocations.clone(); + pipeline + .0 + .by_name("queue") + .unwrap() + .static_pad("sink") + .unwrap() + .add_probe( + gst::PadProbeType::QUERY_DOWNSTREAM | gst::PadProbeType::PUSH, + move |_, info| { + if info + .query() + .is_some_and(|query| matches!(query.view(), gst::QueryView::Allocation(_))) + { + counted.fetch_add(1, Ordering::SeqCst); + } + gst::PadProbeReturn::Ok + }, + ); + let mode = Mode { + kind: if converter == "vapostproc" { + encoding::Kind::Va + } else { + encoding::Kind::Software + }, + factory: "x264enc", + dmabuf: false, + }; + let output = pipeline.0.by_name("output").unwrap(); + output.set_property("caps", mode.caps(None)); + attach_capture_probe( + &pipeline.0.by_name("source").unwrap(), + &output, + Resolution::Native, + mode, + Arc::new(Counters::default()), + ); + pipeline.0.set_state(gst::State::Playing).unwrap(); + let source = gst::Pad::builder(gst::PadDirection::Src).build(); + source.set_active(true).unwrap(); + source + .link( + &pipeline + .0 + .by_name("source") + .unwrap() + .static_pad("sink") + .unwrap(), + ) + .unwrap(); + assert!(source.push_event(gst::event::StreamStart::new("capture"))); + let mut results = Vec::new(); + let mut pts = gst::ClockTime::ZERO; + for (index, (width, height)) in [(1920, 1080), (1920, 1080), (1280, 720), (1280, 720)] + .into_iter() + .enumerate() + { + if index == 0 || index == 2 { + let caps = + format!("video/x-raw,format=RGBx,width={width},height={height},framerate=0/1") + .parse::() + .unwrap(); + assert!(source.push_event(gst::event::Caps::new(&caps))); + if index == 0 { + assert!(source + .push_event(gst::event::Segment::new(&gst::FormattedSegment::< + gst::ClockTime, + >::new()))); + source.peer_query(&mut gst::query::Allocation::new(Some(&caps), true)); + } + } + if index == 2 { + allocations.store(0, Ordering::SeqCst); + } + let mut buffer = gst::Buffer::with_size((width * height * 4) as usize).unwrap(); + buffer.get_mut().unwrap().set_pts(pts); + pts += gst::ClockTime::from_mseconds(33); + results.push(source.push(buffer)); + } + assert!(source.push_event(gst::event::Eos::new())); + pipeline.0.bus().unwrap().timed_pop_filtered( + gst::ClockTime::from_seconds(5), + &[gst::MessageType::Eos, gst::MessageType::Error], + ); + ( + results, + delivered.load(Ordering::SeqCst), + allocations.load(Ordering::SeqCst), + ) + } + + #[test] + fn a_source_resize_renegotiates_converter_allocation() { + let (results, delivered, allocations) = + resize_without_source_allocation("videoconvert ! videoscale add-borders=true"); + assert!(results.iter().all(|result| result.is_ok()), "{results:?}"); + assert_eq!((delivered, allocations), (4, 1)); + if gst::ElementFactory::find("vapostproc").is_some() { + let (results, delivered, _) = resize_without_source_allocation("vapostproc"); + assert!(results.iter().all(|result| result.is_ok()), "{results:?}"); + assert_eq!(delivered, 4); + } + } + #[test] fn rejects_cpu_buffers_before_pixel_processing() { gst::init().unwrap(); - let pipeline = PipelineGuard(gst::parse::launch("videotestsrc num-buffers=1 name=source ! video/x-raw,width=320,height=180 ! fakesink").unwrap().downcast::().unwrap()); + let pipeline = PipelineGuard(gst::parse::launch("videotestsrc num-buffers=3 name=source ! video/x-raw,width=320,height=180 ! fakesink name=sink").unwrap().downcast::().unwrap()); + let delivered = downstream_buffers(&pipeline.0); + pipeline + .0 + .by_name("source") + .unwrap() + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, |_, info| { + if let Some(gst::PadProbeData::Buffer(buffer)) = info.data.as_mut() { + if buffer.offset() >= 1 { + let size = buffer.size(); + let path = std::env::temp_dir().join(format!( + "wrec-dmabuf-{}-{}", + std::process::id(), + buffer.offset() + )); + let file = std::fs::File::options() + .read(true) + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + std::fs::remove_file(&path).unwrap(); + file.set_len(size as u64).unwrap(); + let memory = unsafe { + gstreamer_allocators::DmaBufAllocator::new().alloc(file, size) + } + .unwrap(); + buffer.make_mut().replace_all_memory(memory); + } + } + gst::PadProbeReturn::Ok + }); let output = element("capsfilter").unwrap(); attach_capture_probe( &pipeline.0.by_name("source").unwrap(), @@ -1553,5 +1773,7 @@ mod tests { panic!("expected error") }; assert!(error.error().to_string().contains("non-DMA-BUF")); + settle(&pipeline.0); + assert_eq!(delivered.load(Ordering::SeqCst), 0); } } diff --git a/packaging/linux/README.md b/packaging/linux/README.md index c7d5dfc..8130955 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -59,8 +59,24 @@ pending, breaks the Wayland protocol, and exits. wrec stops that request at the video queue, so the converters still renegotiate their output and the capture source keeps its stream. Other renegotiation requests pass through. +When the captured output or window changes size, `pipewiresrc` sends the new +caps without asking downstream elements to set up their buffer pools again. +GStreamer VA `vapostproc` then copies system-memory frames into a pool sized +for the old caps and rejects the first resized frame. After a size change, wrec +sends the allocation query that normal GStreamer negotiation would send, before +the next frame, so the converters size their pools for the new frames. + ## Capture worker +xdg-desktop-portal-wlr 0.8.1 has an upstream startup race on its +ext-image-copy-capture path. A consumer pause can leave a capture frame +pending; resuming or starting the next consumer can then create a second +frame and make the backend exit with "session already has a frame object". +PipeWire's GStreamer source pauses and resumes during startup, so successful +recordings on this stack do not establish reliable startup. wrec detects the +backend loss and fails the job, but does not repair the portal. This remains +an unresolved compatibility issue, separate from the fixes above. + The daemon does not run native capture code. For each recording it starts a capture worker, which is the same daemon executable started again with a private argument and the daemon's environment. The worker opens the X11 display or the @@ -107,8 +123,8 @@ daemon PID alone misses the capture pipeline. Find the worker with ## Install dependencies and build Use GStreamer 1.22 or newer and current stable Rust. The shared-buffer -DMA-BUF/VA path needs GStreamer VA 1.24.6 or newer. Ubuntu 24.04 is a starting point; package -names differ across distributions. +DMA-BUF/VA path needs GStreamer VA 1.24.6 or newer. Ubuntu 24.04 is a starting +point; package names differ across distributions. ```bash sudo apt install build-essential pkg-config libgstreamer1.0-dev \ From 9cd2446b2fbd1dd44748c03b9f24c758fc60cbec Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:24:40 +0000 Subject: [PATCH 15/45] fix(linux): negotiate frame ceilings below the requested rate --- crates/linux/src/pipeline.rs | 83 +++++++++++++++++++++++++++++++----- packaging/linux/README.md | 9 ++++ 2 files changed, 81 insertions(+), 11 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index e23bbd0..a39d894 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -73,16 +73,21 @@ fn output_size(width: i32, height: i32, resolution: Resolution) -> (i32, i32) { } fn capture_caps(dmabuf: bool, pipewire: bool, fps: u32) -> gst::Caps { - let caps = gst::Caps::builder("video/x-raw").field( - // Wayland compositors advertise variable-rate frames (framerate=0/1). - // Bound their maximum rate without requiring a fixed-rate source. - if pipewire { - "max-framerate" - } else { - "framerate" - }, - gst::Fraction::new(fps as i32, 1), - ); + let caps = gst::Caps::builder("video/x-raw"); + // Wayland compositors advertise variable-rate frames (framerate=0/1). + // Bound their maximum rate without requiring a fixed-rate source. + let caps = if pipewire { + caps.field( + "max-framerate", + gst::List::new( + (1..=fps as i32) + .rev() + .map(|rate| gst::Fraction::new(rate, 1)), + ), + ) + } else { + caps.field("framerate", gst::Fraction::new(fps as i32, 1)) + }; if dmabuf { caps.features(["memory:DMABuf"]) .field("format", "DMA_DRM") @@ -861,12 +866,68 @@ mod tests { capture_caps(true, true, 30) .structure(0) .unwrap() - .get::("max-framerate") + .get::("max-framerate") + .unwrap() + .first() + .unwrap() + .get::() .unwrap(), gst::Fraction::new(30, 1) ); } + fn negotiated_ceiling(capture: &gst::Caps, portal: &str) -> Option { + let portal = portal.parse::().unwrap(); + let mut common = capture.intersect_with_mode(&portal, gst::CapsIntersectMode::First); + if common.is_empty() { + return None; + } + common.fixate(); + common + .structure(0) + .unwrap() + .get::("max-framerate") + .ok() + } + + #[test] + fn portal_maximum_rates_below_the_request_become_the_ceiling() { + gst::init().unwrap(); + for dmabuf in [false, true] { + let features = if dmabuf { + "(memory:DMABuf), format=DMA_DRM, drm-format=XR24:0x0200000000000901" + } else { + ", format=BGRx" + }; + let portal = |maximum: &str| { + format!("video/x-raw{features}, framerate=0/1, max-framerate={maximum}") + }; + for (fps, maximum, ceiling) in [ + (30, "[1/1, 5/1]", 5), + (30, "[1/1, 24/1]", 24), + (30, "[1/1, 60/1]", 30), + (30, "[1/1, 144/1]", 30), + (60, "[1/1, 5/1]", 5), + (60, "[1/1, 60000/1001]", 59), + (60, "[1/1, 60/1]", 60), + (60, "[1/1, 144/1]", 60), + ] { + assert_eq!( + negotiated_ceiling(&capture_caps(dmabuf, true, fps), &portal(maximum)), + Some(gst::Fraction::new(ceiling, 1)), + "dmabuf {dmabuf}, {fps} fps requested, portal maximum {maximum}" + ); + } + for (fps, maximum) in [(30, "60/1"), (30, "[40/1, 60/1]"), (60, "144/1")] { + assert_eq!( + negotiated_ceiling(&capture_caps(dmabuf, true, fps), &portal(maximum)), + None, + "dmabuf {dmabuf}, {fps} fps requested, portal maximum {maximum}" + ); + } + } + } + #[test] fn dma_buf_capture_requires_an_explicit_drm_format() { gst::init().unwrap(); diff --git a/packaging/linux/README.md b/packaging/linux/README.md index 8130955..486d18f 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -25,6 +25,15 @@ H.264 and x265 for HEVC. Retries happen only before the first encoded frame and preserve the requested codec. Job events identify the attempted and selected paths. X11 capture uses system memory even when encoding runs on the GPU. +On Wayland, the requested frame rate is a ceiling. wrec offers the portal every +whole-number maximum from the requested rate down to 1 fps, highest first, and +PipeWire picks the highest one the portal supports. A 60 Hz output recorded at +30 fps captures at up to 30 fps, and a 5 Hz output captures at up to 5 fps. A +fractional refresh rate below the request is rounded down. For example, a +59.94 Hz output recorded at 60 fps captures at up to 59 fps. A portal that +advertises a single fixed fractional maximum below the requested rate, +instead of a range, cannot be negotiated. + wrec tries shared GPU buffers only when `vapostproc` lists DMA-BUF caps with `format=DMA_DRM` and comes from GStreamer VA 1.24.6 or later. `DMA_DRM` caps carry the pixel format and modifier with an explicit `drm-format`. GStreamer VA From 11af9d74e2f1d35d5597cc781b66044bc0f1e683 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 2 Oct 2026 12:01:29 +0000 Subject: [PATCH 16/45] fix(linux): match VA keyframes to capture rate and bound fallback cleanup --- crates/linux/src/lib.rs | 4 +- crates/linux/src/pipeline.rs | 170 +++++++++++++++++++++++++++++++++-- crates/linux/src/worker.rs | 71 ++++++++++++++- packaging/linux/README.md | 17 ++++ 4 files changed, 254 insertions(+), 8 deletions(-) diff --git a/crates/linux/src/lib.rs b/crates/linux/src/lib.rs index d057fa9..fe1f789 100644 --- a/crates/linux/src/lib.rs +++ b/crates/linux/src/lib.rs @@ -133,7 +133,7 @@ pub(crate) fn capture( events: mpsc::Sender, receiver: mpsc::Receiver, worker_stop: watch::Sender, - stopping: Arc, + stopping: Arc, ) { let mut stopped = worker_stop.subscribe(); let id = session.id; @@ -198,7 +198,7 @@ pub(crate) fn capture( session_id: Some(id), message: "capture-engine: the desktop closed the screen-cast session; finalizing the recording".into(), }); - ending_stopping(); + ending_stopping(pipeline::Teardown::Recording); let _ = worker_stop.send(true); } recording.await.map_err(backend)? diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index a39d894..eaee6e2 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -175,11 +175,17 @@ pub(crate) enum CaptureInput { X11 { display: String, xid: u64 }, } +pub(crate) enum Teardown { + Recording, + AttemptStarted, + AttemptFinished, +} + struct Attempt<'a> { mode: Mode, counters: Arc, last: bool, - stopping: &'a dyn Fn(), + stopping: &'a dyn Fn(Teardown), } fn finished(result: &Result<()>, counters: &Counters) -> bool { @@ -198,7 +204,7 @@ pub(crate) fn record( events: &mpsc::Sender, commands: mpsc::Receiver, stop: &watch::Receiver, - stopping: &dyn Fn(), + stopping: &dyn Fn(Teardown), ) -> Result<()> { if matches!(capture, CaptureInput::X11 { .. }) { crate::x11::initialize()?; @@ -350,6 +356,9 @@ fn record_attempt( counters.clone(), ); keep_source_configuration(&queue); + if mode.kind == encoding::Kind::Va { + match_keyframes_to_capture(&source, &encoder, settings.fps.as_u32()); + } if rewrites_timestamps(mode, &encoder) { keep_capture_timestamps(&encoder, &parser); } @@ -386,10 +395,16 @@ fn record_attempt( (None, None) => None, }, ); - if ends_recording(&result, &counters, attempt.last) { - (attempt.stopping)(); - } + let ending = ends_recording(&result, &counters, attempt.last); + (attempt.stopping)(if ending { + Teardown::Recording + } else { + Teardown::AttemptStarted + }); let _ = pipeline.0.set_state(gst::State::Null); + if !ending { + (attempt.stopping)(Teardown::AttemptFinished); + } if matches!(result, Err(RecorderError::Cancelled)) || counters.frames.load(Ordering::Relaxed) == 0 { @@ -582,6 +597,35 @@ fn keep_source_configuration(queue: &gst::Element) { }); } +fn keyframe_interval(caps: &gst::CapsRef, fps: u32) -> Option { + let structure = caps.structure(0)?; + let rate = structure + .get::("max-framerate") + .or_else(|_| structure.get::("framerate")) + .ok()?; + let (numerator, denominator) = (rate.numer(), rate.denom()); + (numerator > 0 && denominator > 0).then(|| { + let frames = (2 * numerator as u64).div_ceil(denominator as u64); + frames.clamp(1, 2 * fps as u64) as u32 + }) +} + +fn match_keyframes_to_capture(source: &gst::Element, encoder: &gst::Element, fps: u32) { + let encoder = encoder.clone(); + source.static_pad("src").unwrap().add_probe( + gst::PadProbeType::EVENT_DOWNSTREAM, + move |_, info| match info.event().map(|event| event.view()) { + Some(gst::EventView::Caps(caps)) => { + if let Some(frames) = keyframe_interval(caps.caps(), fps) { + encoder.set_property("key-int-max", frames); + } + gst::PadProbeReturn::Remove + } + _ => gst::PadProbeReturn::Ok, + }, + ); +} + fn attach_capture_probe( source: &gst::Element, output: &gst::Element, @@ -946,6 +990,122 @@ mod tests { ))); } + #[test] + fn keyframes_follow_the_negotiated_capture_rate_within_the_request() { + gst::init().unwrap(); + let interval = + |caps: &str, fps| keyframe_interval(&caps.parse::().unwrap(), fps); + assert_eq!( + interval("video/x-raw, framerate=0/1, max-framerate=5/1", 60), + Some(10) + ); + assert_eq!( + interval("video/x-raw, framerate=0/1, max-framerate=5/1", 30), + Some(10) + ); + assert_eq!( + interval("video/x-raw, framerate=0/1, max-framerate=24/1", 30), + Some(48) + ); + assert_eq!( + interval("video/x-raw, framerate=0/1, max-framerate=30/1", 30), + Some(60) + ); + assert_eq!( + interval("video/x-raw, framerate=0/1, max-framerate=60/1", 60), + Some(120) + ); + assert_eq!( + interval("video/x-raw, framerate=0/1, max-framerate=60000/1001", 60), + Some(120) + ); + assert_eq!( + interval("video/x-raw, framerate=0/1, max-framerate=144/1", 60), + Some(120) + ); + assert_eq!(interval("video/x-raw, framerate=30/1", 30), Some(60)); + assert_eq!(interval("video/x-raw, framerate=60/1", 60), Some(120)); + assert_eq!(interval("video/x-raw, framerate=0/1", 60), None); + assert_eq!(interval("video/x-raw", 60), None); + } + + fn keyframes_from_va_encoder(factory: &'static str, match_capture: bool) -> (u32, usize) { + gst::init().unwrap(); + let pipeline = PipelineGuard(gst::parse::launch("videotestsrc name=source num-buffers=25 ! video/x-raw,width=320,height=180,framerate=5/1 ! vapostproc ! video/x-raw(memory:VAMemory),format=NV12 ! identity name=encoder-input ! fakesink name=sink").unwrap().downcast::().unwrap()); + let mode = Mode { + kind: encoding::Kind::Va, + factory, + dmabuf: false, + }; + let settings = RecorderSettings { + fps: domain::FrameRate::Fps60, + codec: if factory.contains("265") { + Codec::Hevc + } else { + Codec::H264 + }, + ..RecorderSettings::default() + }; + let encoder = mode.encoder(&settings).unwrap(); + let parser = element(parser_name(settings.codec)).unwrap(); + let input = pipeline.0.by_name("encoder-input").unwrap(); + let sink = pipeline.0.by_name("sink").unwrap(); + input.unlink(&sink); + pipeline.0.add_many([&encoder, &parser]).unwrap(); + gst::Element::link_many([&input, &encoder, &parser, &sink]).unwrap(); + if match_capture { + match_keyframes_to_capture(&pipeline.0.by_name("source").unwrap(), &encoder, 60); + } + let keyframes = Arc::new(AtomicU64::new(0)); + let counted = keyframes.clone(); + sink.static_pad("sink") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + if info + .buffer() + .is_some_and(|buffer| !buffer.flags().contains(gst::BufferFlags::DELTA_UNIT)) + { + counted.fetch_add(1, Ordering::SeqCst); + } + gst::PadProbeReturn::Ok + }); + pipeline.0.set_state(gst::State::Playing).unwrap(); + let message = pipeline + .0 + .bus() + .unwrap() + .timed_pop_filtered( + gst::ClockTime::from_seconds(20), + &[gst::MessageType::Eos, gst::MessageType::Error], + ) + .unwrap(); + assert!( + matches!(message.view(), gst::MessageView::Eos(_)), + "{message:?}" + ); + ( + encoder.property::("key-int-max"), + keyframes.load(Ordering::SeqCst) as usize, + ) + } + + #[test] + fn va_keyframes_are_two_seconds_apart_at_the_capture_rate() { + gst::init().unwrap(); + for factory in ["vah264enc", "vah265enc"] { + if gst::ElementFactory::find(factory).is_none() + || gst::ElementFactory::find("vapostproc").is_none() + { + continue; + } + assert_eq!( + keyframes_from_va_encoder(factory, true), + (10, 3), + "{factory}" + ); + } + } + #[test] fn dimensions_preserve_aspect_and_do_not_upscale() { assert_eq!(output_size(3840, 2160, Resolution::R1080p), (1920, 1080)); diff --git a/crates/linux/src/worker.rs b/crates/linux/src/worker.rs index fabdf6c..84291d5 100644 --- a/crates/linux/src/worker.rs +++ b/crates/linux/src/worker.rs @@ -45,6 +45,8 @@ enum Message { Event(RecorderEvent), Reply(Result<()>), Stopping, + CleaningAttempt, + CleanedAttempt, } type Received = std::result::Result; @@ -227,6 +229,7 @@ impl Supervisor { let mut problem = None; let mut output_open = true; let mut exit_by: Option = None; + let mut cleanup_by: Option = None; let mut drained_by: Option = None; let mut status: Option = None; loop { @@ -255,6 +258,12 @@ impl Supervisor { Ok(Ok(Message::Stopping)) => { exit_by.get_or_insert(Instant::now() + self.deadline); } + Ok(Ok(Message::CleaningAttempt)) => { + cleanup_by = Some(Instant::now() + self.deadline); + } + Ok(Ok(Message::CleanedAttempt)) => { + cleanup_by = None; + } Ok(Err(error)) => { problem = Some(format!("capture worker sent an invalid message ({error})")); let _ = child.kill(); @@ -291,6 +300,16 @@ impl Supervisor { )); } break; + } else if cleanup_by.is_some_and(|by| Instant::now() >= by) { + let _ = child.kill(); + status = child.wait().ok(); + if terminal.is_none() { + problem = Some(format!( + "capture worker did not finish stopping an encoder attempt within {}s", + self.deadline.as_secs() + )); + } + break; } } drop(self.replies); @@ -445,7 +464,16 @@ pub fn run() -> ! { control_stop.send_replace(true); }); let stopping_output = output.clone(); - let stopping = Arc::new(move || send(&stopping_output, &Message::Stopping)); + let stopping = Arc::new(move |teardown: crate::pipeline::Teardown| { + send( + &stopping_output, + &match teardown { + crate::pipeline::Teardown::Recording => Message::Stopping, + crate::pipeline::Teardown::AttemptStarted => Message::CleaningAttempt, + crate::pipeline::Teardown::AttemptFinished => Message::CleanedAttempt, + }, + ) + }); let forwarder = spawn("wrec-capture-events", move || { for event in outgoing { let finished = matches!( @@ -761,6 +789,47 @@ mod tests { assert!(status.contains("did not exit within 1s"), "{status}"); } + #[test] + fn a_hung_teardown_between_encoder_attempts_is_killed_without_a_stop_request() { + let fake = Fake::start(r#"read start; echo '"CleaningAttempt"'; exec sleep 30"#); + let at = Instant::now(); + let status = fake.failure(); + assert!(at.elapsed() >= TEST_DEADLINE, "{:?}", at.elapsed()); + assert!(at.elapsed() < Duration::from_secs(5), "{:?}", at.elapsed()); + assert!( + status.contains("did not finish stopping an encoder attempt within 1s"), + "{status}" + ); + assert!(status.contains("SIGKILL"), "{status}"); + } + + #[test] + fn a_finished_attempt_teardown_does_not_limit_the_next_attempt() { + let fake = Fake::start( + r#"read start; echo '"CleaningAttempt"'; echo '"CleanedAttempt"'; echo '{"Event":{"Started":{"session_id":0,"dimensions":null}}}'; sleep 3; echo '{"Event":{"Exited":{"session_id":0,"success":true,"status":"recording finalized"}}}'"#, + ); + let at = Instant::now(); + assert!(matches!( + fake.terminal(), + RecorderEvent::Exited { success: true, .. } + )); + assert!(at.elapsed() >= Duration::from_secs(3), "{:?}", at.elapsed()); + } + + #[test] + fn stopping_during_attempt_teardown_keeps_the_stop_deadline() { + let mut fake = Fake::start( + r#"read start; echo '"CleaningAttempt"'; read stop; sleep 0.5; echo '"CleanedAttempt"'; exec sleep 30"#, + ); + thread::sleep(Duration::from_millis(300)); + let at = Instant::now(); + fake.worker().stop(); + let status = fake.failure(); + assert!(at.elapsed() >= TEST_DEADLINE, "{:?}", at.elapsed()); + assert!(at.elapsed() < Duration::from_secs(5), "{:?}", at.elapsed()); + assert!(status.contains("did not exit within 1s"), "{status}"); + } + #[test] fn result_reported_before_a_hung_teardown_is_kept() { let fake = Fake::start( diff --git a/packaging/linux/README.md b/packaging/linux/README.md index 486d18f..f62ceac 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -34,6 +34,11 @@ fractional refresh rate below the request is rounded down. For example, a advertises a single fixed fractional maximum below the requested rate, instead of a range, cannot be negotiated. +VA encoders set their initial keyframe interval to about two seconds at the +negotiated capture rate, capped by the requested rate. A 5 fps capture gets +an interval of 10 frames even when 60 fps was requested. Using the requested +rate alone can stall older Radeon HEVC encoding at low capture rates. + wrec tries shared GPU buffers only when `vapostproc` lists DMA-BUF caps with `format=DMA_DRM` and comes from GStreamer VA 1.24.6 or later. `DMA_DRM` caps carry the pixel format and modifier with an explicit `drm-format`. GStreamer VA @@ -86,6 +91,14 @@ recordings on this stack do not establish reliable startup. wrec detects the backend loss and fails the job, but does not repair the portal. This remains an unresolved compatibility issue, separate from the fixes above. +The older wlr-screencopy path also has a static-start limitation. The portal +shares a screencopy-manager binding across sessions and asks for each new +session's first frame with damage tracking. The first session consumes the +binding's initial damage; later sessions on an unchanged output can wait +without receiving an initial image. This was reproduced with portal-wlr 0.7.0, +Sway 1.7 and wlroots 0.15.1. It remains unresolved; repainting or restarting +the portal is not treated as a repair. + The daemon does not run native capture code. For each recording it starts a capture worker, which is the same daemon executable started again with a private argument and the daemon's environment. The worker opens the X11 display or the @@ -129,6 +142,10 @@ daemon PID alone misses the capture pipeline. Find the worker with `pgrep -P -x wrec-capture`, where the daemon PID comes from `wrec daemon status --json`, and add its CPU and RSS to the daemon's. +Native cleanup between encoder attempts has its own 20-second kill/reap bound. +Completing that cleanup clears only its attempt deadline; it does not clear a +user-stop or final-exit deadline, or limit the next healthy recording. + ## Install dependencies and build Use GStreamer 1.22 or newer and current stable Rust. The shared-buffer From c10033311e2293817408481ab8ccaaf3b8b9ced1 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:09:47 +0000 Subject: [PATCH 17/45] fix(linux): keep rate changes playable and bound full attempt cleanup --- crates/linux/src/pipeline.rs | 515 ++++++++++++++++++++++++++++++++--- crates/linux/src/worker.rs | 23 +- packaging/linux/README.md | 21 +- 3 files changed, 514 insertions(+), 45 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index eaee6e2..e99363b 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -172,9 +172,15 @@ impl Timeline { pub(crate) enum CaptureInput { PipeWire(Box Result + Send>), - X11 { display: String, xid: u64 }, + X11 { + display: String, + xid: u64, + }, + #[cfg(test)] + Element(Box gst::Element + Send>), } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Teardown { Recording, AttemptStarted, @@ -235,6 +241,9 @@ pub(crate) fn record( capture, session, settings, events, &commands, stop, &attempt, ); release_freed_memory(); + if !ends_recording(&result, &attempt.counters, attempt.last) { + stopping(Teardown::AttemptFinished); + } if finished(&result, &attempt.counters) { return result; } @@ -276,7 +285,7 @@ fn record_attempt( let settings = settings.clone().with_preset_limits(); let stream = match capture { CaptureInput::PipeWire(connect) => Some(connect()?), - CaptureInput::X11 { .. } => None, + _ => None, }; let source_watch = match capture { CaptureInput::X11 { xid, .. } if *xid != 0 => Some(crate::x11::SourceWatch::new(*xid)?), @@ -306,6 +315,8 @@ fn record_attempt( source.set_property("use-damage", true); source } + #[cfg(test)] + CaptureInput::Element(source) => source(), }; source.set_property("do-timestamp", true); let input = element("capsfilter")?; @@ -313,7 +324,7 @@ fn record_attempt( "caps", capture_caps( mode.dmabuf, - matches!(capture, CaptureInput::PipeWire(_)), + !matches!(capture, CaptureInput::X11 { .. }), settings.fps.as_u32(), ), ); @@ -323,6 +334,8 @@ fn record_attempt( output.set_property("caps", mode.caps(None)); let encoder = mode.encoder(&settings)?; let parser = element(parser_name(settings.codec))?; + parser.set_property("config-interval", -1i32); + let format = movie_format(settings.codec)?; let mux = movie_mux()?; let sink = element("filesink")?; sink.set_property( @@ -335,7 +348,7 @@ fn record_attempt( sink.set_property("sync", false); let mut chain = vec![&source, &input, &queue]; chain.extend(converters.iter()); - chain.extend([&output, &encoder, &parser, &mux, &sink]); + chain.extend([&output, &encoder, &parser, &format, &mux, &sink]); pipeline .0 .add_many(chain.iter().copied()) @@ -357,7 +370,7 @@ fn record_attempt( ); keep_source_configuration(&queue); if mode.kind == encoding::Kind::Va { - match_keyframes_to_capture(&source, &encoder, settings.fps.as_u32()); + match_keyframes_to_capture(&encoder, settings.fps.as_u32()); } if rewrites_timestamps(mode, &encoder) { keep_capture_timestamps(&encoder, &parser); @@ -402,9 +415,6 @@ fn record_attempt( Teardown::AttemptStarted }); let _ = pipeline.0.set_state(gst::State::Null); - if !ending { - (attempt.stopping)(Teardown::AttemptFinished); - } if matches!(result, Err(RecorderError::Cancelled)) || counters.frames.load(Ordering::Relaxed) == 0 { @@ -597,35 +607,68 @@ fn keep_source_configuration(queue: &gst::Element) { }); } -fn keyframe_interval(caps: &gst::CapsRef, fps: u32) -> Option { - let structure = caps.structure(0)?; - let rate = structure - .get::("max-framerate") - .or_else(|_| structure.get::("framerate")) - .ok()?; - let (numerator, denominator) = (rate.numer(), rate.denom()); - (numerator > 0 && denominator > 0).then(|| { - let frames = (2 * numerator as u64).div_ceil(denominator as u64); - frames.clamp(1, 2 * fps as u64) as u32 - }) +fn keyframe_interval(caps: &gst::CapsRef, fps: u32) -> u32 { + let rate = caps.structure(0).and_then(|structure| { + ["framerate", "max-framerate"] + .into_iter() + .filter_map(|field| structure.get::(field).ok()) + .find(|rate| rate.numer() > 0 && rate.denom() > 0) + }); + let frames = rate.map_or(2 * fps as u64, |rate| { + (2 * rate.numer() as u64).div_ceil(rate.denom() as u64) + }); + frames.clamp(1, 2 * fps as u64) as u32 } -fn match_keyframes_to_capture(source: &gst::Element, encoder: &gst::Element, fps: u32) { - let encoder = encoder.clone(); - source.static_pad("src").unwrap().add_probe( +// VA encoders read key-int-max only when new caps reconfigure them, so set it +// as each CAPS event enters the encoder, after every frame of the old rate. +fn match_keyframes_to_capture(encoder: &gst::Element, fps: u32) { + encoder.static_pad("sink").unwrap().add_probe( gst::PadProbeType::EVENT_DOWNSTREAM, - move |_, info| match info.event().map(|event| event.view()) { - Some(gst::EventView::Caps(caps)) => { - if let Some(frames) = keyframe_interval(caps.caps(), fps) { + move |pad, info| { + if let (Some(gst::EventView::Caps(caps)), Some(encoder)) = + (info.event().map(|event| event.view()), pad.parent_element()) + { + let frames = keyframe_interval(caps.caps(), fps); + if encoder.property::("key-int-max") != frames { encoder.set_property("key-int-max", frames); } - gst::PadProbeReturn::Remove } - _ => gst::PadProbeReturn::Ok, + gst::PadProbeReturn::Ok }, ); } +// A capture rate change reconfigures the encoder, which writes new parameter +// sets. qtmux stores those as an extra sample description, which a fragmented +// movie cannot reference, so the movie keeps its first description and every +// keyframe carries its own parameter sets (avc3/hev1). +fn movie_format(codec: Codec) -> Result { + let format = element("capsfilter")?; + format.set_property( + "caps", + match codec { + Codec::H264 => gst::Caps::builder("video/x-h264").field("stream-format", "avc3"), + Codec::Hevc => gst::Caps::builder("video/x-h265").field("stream-format", "hev1"), + } + .field("alignment", "au") + .build(), + ); + let first = Mutex::new(None::); + format.static_pad("src").unwrap().add_probe( + gst::PadProbeType::EVENT_DOWNSTREAM, + move |_, info| { + if let Some(gst::PadProbeData::Event(event)) = &mut info.data { + if event.type_() == gst::EventType::Caps { + *event = first.lock().unwrap().get_or_insert(event.clone()).clone(); + } + } + gst::PadProbeReturn::Ok + }, + ); + Ok(format) +} + fn attach_capture_probe( source: &gst::Element, output: &gst::Element, @@ -997,36 +1040,40 @@ mod tests { |caps: &str, fps| keyframe_interval(&caps.parse::().unwrap(), fps); assert_eq!( interval("video/x-raw, framerate=0/1, max-framerate=5/1", 60), - Some(10) + 10 ); assert_eq!( interval("video/x-raw, framerate=0/1, max-framerate=5/1", 30), - Some(10) + 10 ); assert_eq!( interval("video/x-raw, framerate=0/1, max-framerate=24/1", 30), - Some(48) + 48 ); assert_eq!( interval("video/x-raw, framerate=0/1, max-framerate=30/1", 30), - Some(60) + 60 ); assert_eq!( interval("video/x-raw, framerate=0/1, max-framerate=60/1", 60), - Some(120) + 120 ); assert_eq!( interval("video/x-raw, framerate=0/1, max-framerate=60000/1001", 60), - Some(120) + 120 ); assert_eq!( interval("video/x-raw, framerate=0/1, max-framerate=144/1", 60), - Some(120) + 120 ); - assert_eq!(interval("video/x-raw, framerate=30/1", 30), Some(60)); - assert_eq!(interval("video/x-raw, framerate=60/1", 60), Some(120)); - assert_eq!(interval("video/x-raw, framerate=0/1", 60), None); - assert_eq!(interval("video/x-raw", 60), None); + assert_eq!(interval("video/x-raw, framerate=30/1", 30), 60); + assert_eq!(interval("video/x-raw, framerate=60/1", 60), 120); + assert_eq!( + interval("video/x-raw, framerate=24/1, max-framerate=60/1", 60), + 48 + ); + assert_eq!(interval("video/x-raw, framerate=0/1", 60), 120); + assert_eq!(interval("video/x-raw", 30), 60); } fn keyframes_from_va_encoder(factory: &'static str, match_capture: bool) -> (u32, usize) { @@ -1054,7 +1101,7 @@ mod tests { pipeline.0.add_many([&encoder, &parser]).unwrap(); gst::Element::link_many([&input, &encoder, &parser, &sink]).unwrap(); if match_capture { - match_keyframes_to_capture(&pipeline.0.by_name("source").unwrap(), &encoder, 60); + match_keyframes_to_capture(&encoder, 60); } let keyframes = Arc::new(AtomicU64::new(0)); let counted = keyframes.clone(); @@ -1605,6 +1652,396 @@ mod tests { ); } + fn test_session() -> RecordingSession { + static ID: AtomicU64 = AtomicU64::new(0); + let id = ID.fetch_add(1, Ordering::Relaxed); + RecordingSession { + id, + output_path: std::env::temp_dir().join(format!( + "wrec-linux-record-test-{}-{id}.mov", + std::process::id() + )), + } + } + + fn silent_settings(codec: Codec) -> RecorderSettings { + RecorderSettings { + fps: domain::FrameRate::Fps60, + codec, + quality: domain::Quality::High, + include_system_audio: false, + include_microphone: false, + ..RecorderSettings::default() + } + } + + #[test] + fn an_encoder_attempt_reports_its_cleanup_only_after_its_pipeline_is_released() { + gst::init().unwrap(); + let settings = silent_settings(Codec::H264); + let attempts = encoding::modes(settings.codec, false).len(); + assert!(attempts > 1, "this test needs two H.264 encoders"); + let sources = Arc::new(Mutex::new(Vec::>::new())); + let created = sources.clone(); + let capture = CaptureInput::Element(Box::new(move || { + let source = element("filesrc").unwrap(); + source.set_property("location", "/nonexistent/wrec-capture-source"); + created.lock().unwrap().push(source.downgrade()); + source + })); + let session = test_session(); + let (events, _received) = mpsc::channel(); + let (_commands, receiver) = mpsc::sync_channel(1); + let (_stop, stopped) = watch::channel(false); + let signals = Mutex::new(Vec::new()); + let result = record( + &capture, + &session, + &settings, + &events, + receiver, + &stopped, + &|teardown| { + let released = sources + .lock() + .unwrap() + .iter() + .all(|source| source.upgrade().is_none()); + signals.lock().unwrap().push((teardown, released)); + }, + ); + assert!(result.is_err()); + assert!(!session.output_path.exists()); + let mut expected = Vec::new(); + for _ in 1..attempts { + expected.push((Teardown::AttemptStarted, false)); + expected.push((Teardown::AttemptFinished, true)); + } + expected.push((Teardown::Recording, false)); + assert_eq!(signals.into_inner().unwrap(), expected); + } + + fn variable_rate(rate: i32) -> gst::Caps { + gst::Caps::builder("video/x-raw") + .field("format", "BGRx") + .field("width", 320i32) + .field("height", 180i32) + .field("framerate", gst::Fraction::new(0, 1)) + .field("max-framerate", gst::Fraction::new(rate, 1)) + .build() + } + + struct RateChangeRecording { + result: Result<()>, + factory: String, + session: RecordingSession, + partial: std::path::PathBuf, + scheduled: Vec, + source_rates: Vec, + metrics: RecorderMetrics, + } + + impl Drop for RateChangeRecording { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.session.output_path); + let _ = std::fs::remove_file(&self.partial); + } + } + + // Each recording paces real frames through a leaky queue; running two at + // once on a shared encoder would turn scheduling delays into frame drops. + static RATE_CHANGE_RECORDING: Mutex<()> = Mutex::new(()); + + fn record_rate_changes(codec: Codec, phases: &[(i32, u64)]) -> RateChangeRecording { + gst::init().unwrap(); + let _sequential = RATE_CHANGE_RECORDING + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let source = Arc::new(Mutex::new(None::)); + let source_rates = Arc::new(Mutex::new(Vec::new())); + let created = source.clone(); + let observed = source_rates.clone(); + let first_rate = phases[0].0; + let capture = CaptureInput::Element(Box::new(move || { + let source = element("appsrc").unwrap(); + source.set_property("is-live", true); + source.set_property_from_str("format", "time"); + source.set_property("caps", variable_rate(first_rate)); + let observed = observed.clone(); + source.static_pad("src").unwrap().add_probe( + gst::PadProbeType::EVENT_DOWNSTREAM, + move |_, info| { + if let Some(gst::EventView::Caps(caps)) = info.event().map(|event| event.view()) + { + if let Some(rate) = caps.caps().structure(0).and_then(|structure| { + structure.get::("max-framerate").ok() + }) { + observed.lock().unwrap().push(rate.numer()); + } + } + gst::PadProbeReturn::Ok + }, + ); + *created.lock().unwrap() = Some(source.clone()); + source + })); + let session = test_session(); + let partial = session.output_path.with_extension("partial.mov"); + let (events, received) = mpsc::channel(); + let (_commands, receiver) = mpsc::sync_channel(1); + let (stop, stopped) = watch::channel(false); + let worker_session = session.clone(); + let worker = std::thread::spawn(move || { + record( + &capture, + &worker_session, + &silent_settings(codec), + &events, + receiver, + &stopped, + &|_| {}, + ) + }); + let appsrc = loop { + if let Some(source) = source.lock().unwrap().clone() { + break source; + } + std::thread::sleep(Duration::from_millis(10)); + }; + let playing = Instant::now(); + while appsrc.current_state() != gst::State::Playing + && playing.elapsed() < Duration::from_secs(10) + { + std::thread::sleep(Duration::from_millis(10)); + } + // Stamp each frame with its schedule rather than the time the source + // thread happens to forward it, so every packet maps to one pushed frame. + let origin = appsrc.current_running_time().unwrap_or_default(); + let start = Instant::now(); + let mut due = Duration::ZERO; + let mut scheduled = Vec::new(); + 'push: for &(rate, seconds) in phases { + appsrc.set_property("caps", variable_rate(rate)); + for _ in 0..rate as u64 * seconds { + std::thread::sleep(due.saturating_sub(start.elapsed())); + let shade = scheduled.len() as u8; + let mut buffer = gst::Buffer::from_mut_slice(vec![shade; 320 * 180 * 4]); + let time = origin + gst::ClockTime::from_nseconds(due.as_nanos() as u64); + let frame = buffer.get_mut().unwrap(); + frame.set_pts(time); + frame.set_dts(time); + if appsrc.emit_by_name::("push-buffer", &[&buffer]) + != gst::FlowReturn::Ok + { + break 'push; + } + scheduled.push(due.as_secs_f64()); + due += Duration::from_secs(1) / rate as u32; + } + } + let accepted = Instant::now(); + while appsrc.property::("current-level-buffers") > 0 + && accepted.elapsed() < Duration::from_secs(5) + { + std::thread::sleep(Duration::from_millis(10)); + } + // qtmux writes a fragment when the next keyframe reaches it, so wait + // for one that holds frames after the first rate change. + let changed = phases[0].1 as f64; + let copied = Instant::now(); + loop { + std::fs::copy(&session.output_path, &partial).unwrap(); + if last_packet_seconds(&partial).is_some_and(|last| last > changed) + || copied.elapsed() > Duration::from_secs(10) + { + break; + } + std::thread::sleep(Duration::from_millis(200)); + } + stop.send_replace(true); + let result = worker.join().unwrap(); + let events: Vec = received.try_iter().collect(); + let factory = events + .iter() + .find_map(|event| match event { + RecorderEvent::Log { message, .. } if message.contains("selected") => message + .split("; ") + .nth(2) + .map(|factory| factory.to_string()), + _ => None, + }) + .unwrap(); + let metrics = events + .iter() + .rev() + .find_map(|event| match event { + RecorderEvent::Metrics { metrics, .. } => Some(metrics.clone()), + _ => None, + }) + .unwrap(); + let source_rates = source_rates.lock().unwrap().clone(); + RateChangeRecording { + result, + factory, + session, + partial, + scheduled, + source_rates, + metrics, + } + } + + fn last_packet_seconds(movie: &Path) -> Option { + let output = std::process::Command::new("ffprobe") + .args(["-v", "error", "-select_streams", "v:0", "-show_entries"]) + .args(["packet=pts_time", "-of", "csv=p=0"]) + .arg(movie) + .output() + .ok()?; + String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| line.trim().parse::().ok()) + .reduce(f64::max) + } + + fn video_packets(probe: &serde_json::Value) -> Vec<(f64, bool)> { + let stream = &probe["streams"][0]; + assert_eq!(stream["codec_type"], "video"); + let base: Vec = stream["time_base"] + .as_str() + .unwrap() + .split('/') + .map(|part| part.parse().unwrap()) + .collect(); + probe["packets"] + .as_array() + .unwrap() + .iter() + .filter(|packet| packet["stream_index"] == 0) + .map(|packet| { + ( + packet["pts"].as_i64().unwrap() as f64 * base[0] / base[1], + packet["flags"].as_str().unwrap().starts_with('K'), + ) + }) + .collect() + } + + fn decoded_video_frames(movie: &Path) -> u64 { + let output = std::process::Command::new("ffprobe") + .args(["-v", "error", "-select_streams", "v:0", "-count_frames"]) + .args(["-show_entries", "stream=nb_read_frames", "-of", "csv=p=0"]) + .arg(movie) + .output() + .unwrap(); + assert!(output.status.success()); + String::from_utf8_lossy(&output.stdout) + .trim() + .parse() + .unwrap() + } + + fn assert_rate_changes_stay_playable(codec: Codec, name: &str, phases: &[(i32, u64)]) { + let recording = record_rate_changes(codec, phases); + let movie = &recording.session.output_path; + assert!(recording.result.is_ok(), "{:?}", recording.result); + let rates: Vec = phases.iter().map(|(rate, _)| *rate).collect(); + let mut transitions = recording.source_rates.clone(); + transitions.dedup(); + assert_eq!(transitions, rates, "the source did not change its caps"); + let probe = probe_movie(movie); + assert_eq!(probe["streams"][0]["codec_name"], name); + let packets = video_packets(&probe); + let encoded = recording.metrics.frames.unwrap(); + let dropped = recording.metrics.dropped_frames.unwrap(); + assert_eq!(packets.len() as u64, encoded); + assert_eq!(encoded + dropped, recording.scheduled.len() as u64); + assert_eq!(decoded_video_frames(movie), packets.len() as u64); + let first = packets[0].0; + let va = recording.factory.starts_with("va"); + let mut phase_start = 0.0; + let mut remaining = packets + .iter() + .map(|(pts, key)| (pts - first, *key)) + .peekable(); + for &(rate, seconds) in phases { + let phase_end = phase_start + seconds as f64; + let mut phase = Vec::new(); + while let Some((position, key)) = + remaining.next_if(|(position, _)| *position < phase_end - 0.5 / rate as f64) + { + assert!( + recording + .scheduled + .iter() + .any(|due| (position - due).abs() < 2e-6), + "a packet at {position}s matches no pushed frame" + ); + phase.push(key); + } + let interval = if va { + keyframe_interval(&variable_rate(rate), 60) as usize + } else { + 120 + }; + let keyframes: Vec = phase + .iter() + .enumerate() + .filter_map(|(index, key)| key.then_some(index)) + .collect(); + assert_eq!( + keyframes, + (0..phase.len()).step_by(interval).collect::>(), + "{} at {rate} fps from {phase_start}s", + recording.factory + ); + phase_start = phase_end; + } + assert!(remaining.next().is_none(), "packets after the last phase"); + for probe in [&probe, &probe_movie(&recording.partial)] { + assert_eq!( + probe["streams"][0]["codec_tag_string"], + if name == "h264" { "avc3" } else { "hev1" } + ); + assert!( + !probe.to_string().contains("New Extradata"), + "the movie switches sample descriptions" + ); + } + let partial = probe_movie(&recording.partial); + let partial_packets = video_packets(&partial); + assert!( + partial_packets.len() as u64 > rates[0] as u64 * phases[0].1, + "the partial movie holds {} frames, none after the rate change", + partial_packets.len() + ); + assert_eq!( + decoded_video_frames(&recording.partial), + partial_packets.len() as u64 + ); + eprintln!( + "{} {rates:?}: {} frames, {dropped} dropped; partial movie has {} frames", + recording.factory, + packets.len(), + partial_packets.len() + ); + } + + const RISING: [(i32, u64); 3] = [(5, 3), (60, 6), (5, 3)]; + const FALLING: [(i32, u64); 3] = [(60, 3), (5, 4), (60, 3)]; + + #[test] + fn h264_recordings_survive_capture_rate_changes() { + assert_rate_changes_stay_playable(Codec::H264, "h264", &RISING); + assert_rate_changes_stay_playable(Codec::H264, "h264", &FALLING); + } + + #[test] + fn hevc_recordings_survive_capture_rate_changes() { + assert_rate_changes_stay_playable(Codec::Hevc, "hevc", &RISING); + assert_rate_changes_stay_playable(Codec::Hevc, "hevc", &FALLING); + } + #[test] fn every_attempt_that_ends_the_recording_signals_teardown() { let idle = Counters::default(); diff --git a/crates/linux/src/worker.rs b/crates/linux/src/worker.rs index 84291d5..2c50d17 100644 --- a/crates/linux/src/worker.rs +++ b/crates/linux/src/worker.rs @@ -259,7 +259,7 @@ impl Supervisor { exit_by.get_or_insert(Instant::now() + self.deadline); } Ok(Ok(Message::CleaningAttempt)) => { - cleanup_by = Some(Instant::now() + self.deadline); + cleanup_by.get_or_insert(Instant::now() + self.deadline); } Ok(Ok(Message::CleanedAttempt)) => { cleanup_by = None; @@ -803,6 +803,25 @@ mod tests { assert!(status.contains("SIGKILL"), "{status}"); } + #[test] + fn repeated_attempt_teardown_reports_keep_the_first_deadline() { + let fake = Fake::start( + r#"read start; for report in 1 2 3 4 5 6; do echo '"CleaningAttempt"'; sleep 0.4; done; exec sleep 30"#, + ); + let at = Instant::now(); + let status = fake.failure(); + assert!(at.elapsed() >= TEST_DEADLINE, "{:?}", at.elapsed()); + assert!( + at.elapsed() < Duration::from_millis(1800), + "{:?}", + at.elapsed() + ); + assert!( + status.contains("did not finish stopping an encoder attempt within 1s"), + "{status}" + ); + } + #[test] fn a_finished_attempt_teardown_does_not_limit_the_next_attempt() { let fake = Fake::start( @@ -813,7 +832,7 @@ mod tests { fake.terminal(), RecorderEvent::Exited { success: true, .. } )); - assert!(at.elapsed() >= Duration::from_secs(3), "{:?}", at.elapsed()); + assert!(at.elapsed() >= 2 * TEST_DEADLINE, "{:?}", at.elapsed()); } #[test] diff --git a/packaging/linux/README.md b/packaging/linux/README.md index f62ceac..b6e8914 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -34,10 +34,21 @@ fractional refresh rate below the request is rounded down. For example, a advertises a single fixed fractional maximum below the requested rate, instead of a range, cannot be negotiated. -VA encoders set their initial keyframe interval to about two seconds at the +VA encoders set their keyframe interval to about two seconds at the negotiated capture rate, capped by the requested rate. A 5 fps capture gets an interval of 10 frames even when 60 fps was requested. Using the requested -rate alone can stall older Radeon HEVC encoding at low capture rates. +rate alone can stall older Radeon HEVC encoding at low capture rates. Each +new set of encoder input caps updates the interval after the previous frames +have reached the encoder. Fixed frame rates take precedence over a variable +stream's maximum rate. Missing rate information uses the requested ceiling. + +The movie writer uses frame timestamps for playback timing. Refresh-rate +changes can also change the encoder's parameter sets. The movie keeps its +initial track description and carries those parameter sets with each keyframe, +using `avc3` for H.264 and `hev1` for HEVC. This avoids changing the sample +description in a fragmented movie, which can make older qtmux versions write +unplayable fragments. Playback requires a reader that supports these sample +entries; FFmpeg playback is covered by the recording tests. wrec tries shared GPU buffers only when `vapostproc` lists DMA-BUF caps with `format=DMA_DRM` and comes from GStreamer VA 1.24.6 or later. `DMA_DRM` caps @@ -121,7 +132,7 @@ output closes, or after the worker reports that it is tearing down a failed, lost or cancelled recording, then kills it with SIGKILL. The worker sends that report before it stops the native pipeline, so a GPU or plugin cleanup that hangs cannot hold the job open. Recording, idle and paused time never start the -deadline, and neither does tearing down an encoder attempt that will be retried. The job's final status is +deadline. Cleaning up a failed encoder attempt uses a separate deadline. The job's final status is published only after the worker has been reaped, so the next queued job never starts while the previous worker still runs. A worker killed this way before it reported a result fails its job. While a worker is frozen, pause and resume fail @@ -144,7 +155,9 @@ daemon PID alone misses the capture pipeline. Find the worker with Native cleanup between encoder attempts has its own 20-second kill/reap bound. Completing that cleanup clears only its attempt deadline; it does not clear a -user-stop or final-exit deadline, or limit the next healthy recording. +user-stop or final-exit deadline, or limit the next healthy recording. The bound +covers native destructors and allocator cleanup. Repeating a cleanup-start +message does not extend an already armed deadline. ## Install dependencies and build From 4a4435e6a38c6e89e89150ab3da40b4f11aa8a20 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:20:59 +0000 Subject: [PATCH 18/45] test(linux): check partial movie timing across rate changes --- crates/linux/src/pipeline.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index e99363b..7351cfd 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -1851,7 +1851,7 @@ mod tests { let copied = Instant::now(); loop { std::fs::copy(&session.output_path, &partial).unwrap(); - if last_packet_seconds(&partial).is_some_and(|last| last > changed) + if packet_span(&partial).is_some_and(|span| span >= changed - 1e-3) || copied.elapsed() > Duration::from_secs(10) { break; @@ -1891,17 +1891,25 @@ mod tests { } } - fn last_packet_seconds(movie: &Path) -> Option { + // Seconds from the first video packet to the last one. Movie timestamps + // start at the pipeline's running time, not at zero. + fn packet_span(movie: &Path) -> Option { let output = std::process::Command::new("ffprobe") .args(["-v", "error", "-select_streams", "v:0", "-show_entries"]) .args(["packet=pts_time", "-of", "csv=p=0"]) .arg(movie) .output() .ok()?; - String::from_utf8_lossy(&output.stdout) + let times: Vec = String::from_utf8_lossy(&output.stdout) .lines() - .filter_map(|line| line.trim().parse::().ok()) + .filter_map(|line| line.trim().parse().ok()) + .collect(); + let first = times.iter().copied().reduce(f64::min)?; + times + .iter() + .copied() .reduce(f64::max) + .map(|last| last - first) } fn video_packets(probe: &serde_json::Value) -> Vec<(f64, bool)> { @@ -2010,10 +2018,10 @@ mod tests { } let partial = probe_movie(&recording.partial); let partial_packets = video_packets(&partial); + let span = partial_packets.last().unwrap().0 - partial_packets[0].0; assert!( - partial_packets.len() as u64 > rates[0] as u64 * phases[0].1, - "the partial movie holds {} frames, none after the rate change", - partial_packets.len() + span >= phases[0].1 as f64 - 1e-3, + "the partial movie ends {span}s in, before the first rate change" ); assert_eq!( decoded_video_frames(&recording.partial), From fc3de38f2ae34e76ef7bd49c7f3920b6efb1f480 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:11:41 +0000 Subject: [PATCH 19/45] fix(linux): preserve final frame and count actual queue drops --- crates/linux/src/pipeline.rs | 308 +++++++++++++++++++++++++++++++++-- packaging/linux/README.md | 24 ++- 2 files changed, 311 insertions(+), 21 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 7351cfd..8837827 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -107,8 +107,11 @@ fn movie_mux() -> Result { Ok(mux) } +const CAPTURE_QUEUE: &str = "capture-queue"; + fn video_queue() -> Result { let queue = element("queue")?; + queue.set_property("name", CAPTURE_QUEUE); queue.set_property("max-size-buffers", 2u32); queue.set_property("max-size-bytes", 0u32); queue.set_property("max-size-time", 0u64); @@ -127,6 +130,8 @@ impl Drop for PipelineGuard { struct Counters { frames: AtomicU64, dropped: AtomicU64, + queued: AtomicU64, + dequeued: AtomicU64, dimensions: Mutex>, last_pts: Mutex>, timeline: Mutex, @@ -335,7 +340,7 @@ fn record_attempt( let encoder = mode.encoder(&settings)?; let parser = element(parser_name(settings.codec))?; parser.set_property("config-interval", -1i32); - let format = movie_format(settings.codec)?; + let format = movie_format(settings.codec, settings.fps.as_u32())?; let mux = movie_mux()?; let sink = element("filesink")?; sink.set_property( @@ -376,12 +381,7 @@ fn record_attempt( keep_capture_timestamps(&encoder, &parser); } attach_encoded_probe(&parser, counters.clone()); - // One overrun accompanies each incoming frame that replaces the oldest frame. - let dropped = counters.clone(); - queue.connect("overrun", false, move |_| { - dropped.dropped.fetch_add(1, Ordering::Relaxed); - None - }); + count_dropped(&queue, counters.clone()); let _ = events.send(RecorderEvent::Log { session_id: Some(session.id), message: format!( @@ -423,6 +423,40 @@ fn record_attempt( result } +// The queue emits "overrun" before it rechecks its level, so the encoder can +// take a frame meanwhile and nothing is discarded. Count the frames entering +// and leaving the queue instead; buffers are not modified, since PipeWire +// reuses a frame's memory once its source buffer is released. +fn count_dropped(queue: &gst::Element, counters: Arc) { + let entering = counters.clone(); + queue + .static_pad("sink") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, _| { + entering.queued.fetch_add(1, Ordering::Relaxed); + gst::PadProbeReturn::Ok + }); + queue + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, _| { + counters.dequeued.fetch_add(1, Ordering::Relaxed); + gst::PadProbeReturn::Ok + }); +} + +// Frames rejected at capture plus frames the queue discarded: those that +// entered it, never left, and are not waiting in it. Exact once the queue is +// empty; while frames flow, a frame in transit can shift it by one. +fn dropped_frames(pipeline: &gst::Pipeline, counters: &Counters) -> u64 { + let waiting = pipeline.by_name(CAPTURE_QUEUE).map_or(0, |queue| { + queue.property::("current-level-buffers") as u64 + }); + let dequeued = counters.dequeued.load(Ordering::Relaxed); + let queued = counters.queued.load(Ordering::Relaxed); + counters.dropped.load(Ordering::Relaxed) + queued.saturating_sub(dequeued + waiting) +} + fn attach_encoded_probe(parser: &gst::Element, counters: Arc) { parser .static_pad("src") @@ -607,14 +641,18 @@ fn keep_source_configuration(queue: &gst::Element) { }); } -fn keyframe_interval(caps: &gst::CapsRef, fps: u32) -> u32 { - let rate = caps.structure(0).and_then(|structure| { +// A fixed frame rate, else the maximum of a variable-rate stream. +fn capture_rate(caps: &gst::CapsRef) -> Option { + caps.structure(0).and_then(|structure| { ["framerate", "max-framerate"] .into_iter() .filter_map(|field| structure.get::(field).ok()) .find(|rate| rate.numer() > 0 && rate.denom() > 0) - }); - let frames = rate.map_or(2 * fps as u64, |rate| { + }) +} + +fn keyframe_interval(caps: &gst::CapsRef, fps: u32) -> u32 { + let frames = capture_rate(caps).map_or(2 * fps as u64, |rate| { (2 * rate.numer() as u64).div_ceil(rate.denom() as u64) }); frames.clamp(1, 2 * fps as u64) as u32 @@ -643,7 +681,7 @@ fn match_keyframes_to_capture(encoder: &gst::Element, fps: u32) { // sets. qtmux stores those as an extra sample description, which a fragmented // movie cannot reference, so the movie keeps its first description and every // keyframe carries its own parameter sets (avc3/hev1). -fn movie_format(codec: Codec) -> Result { +fn movie_format(codec: Codec, fps: u32) -> Result { let format = element("capsfilter")?; format.set_property( "caps", @@ -654,14 +692,34 @@ fn movie_format(codec: Codec) -> Result { .field("alignment", "au") .build(), ); + // qtmux times each sample by the next one and takes the last sample's + // duration from its buffer. Some encoders (x265enc 1.24) leave it unset + // for variable-rate capture, and players skip a zero-length last frame, + // so a frame without one lasts a frame at the current capture rate, or at + // the requested rate when the caps carry none. + let requested = gst::ClockTime::SECOND / fps as u64; let first = Mutex::new(None::); + let frame = Mutex::new(requested); format.static_pad("src").unwrap().add_probe( - gst::PadProbeType::EVENT_DOWNSTREAM, + gst::PadProbeType::EVENT_DOWNSTREAM | gst::PadProbeType::BUFFER, move |_, info| { - if let Some(gst::PadProbeData::Event(event)) = &mut info.data { - if event.type_() == gst::EventType::Caps { - *event = first.lock().unwrap().get_or_insert(event.clone()).clone(); + match &mut info.data { + Some(gst::PadProbeData::Event(event)) => { + if let gst::EventView::Caps(caps) = event.view() { + *frame.lock().unwrap() = capture_rate(caps.caps()) + .and_then(|rate| { + gst::ClockTime::SECOND + .mul_div_floor(rate.denom() as u64, rate.numer() as u64) + }) + .unwrap_or(requested); + *event = first.lock().unwrap().get_or_insert(event.clone()).clone(); + } } + Some(gst::PadProbeData::Buffer(buffer)) if buffer.duration().is_none() => { + let duration = *frame.lock().unwrap(); + buffer.make_mut().set_duration(duration); + } + _ => {} } gst::PadProbeReturn::Ok }, @@ -924,7 +982,7 @@ fn emit_metrics( 0.0 }, frames: Some(counters.frames.load(Ordering::Relaxed)), - dropped_frames: Some(counters.dropped.load(Ordering::Relaxed)), + dropped_frames: Some(dropped_frames(pipeline, counters)), }, }); } @@ -2006,6 +2064,17 @@ mod tests { phase_start = phase_end; } assert!(remaining.next().is_none(), "packets after the last phase"); + let last_frame = 1.0 / phases.last().unwrap().0 as f64; + let span = packets.last().unwrap().0 - first + last_frame; + let duration: f64 = probe["streams"][0]["duration"] + .as_str() + .unwrap() + .parse() + .unwrap(); + assert!( + (duration - span).abs() < 1e-3, + "the movie lasts {duration}s, but its last frame ends at {span}s" + ); for probe in [&probe, &probe_movie(&recording.partial)] { assert_eq!( probe["streams"][0]["codec_tag_string"], @@ -2050,6 +2119,211 @@ mod tests { assert_rate_changes_stay_playable(Codec::Hevc, "hevc", &FALLING); } + // Rust heap bytes allocated and not yet freed by each thread. GStreamer + // buffers come from GLib and are not counted. + struct CountingAllocator; + + thread_local! { + static LIVE_BYTES: std::cell::Cell = const { std::cell::Cell::new(0) }; + } + + unsafe impl std::alloc::GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: std::alloc::Layout) -> *mut u8 { + let _ = LIVE_BYTES.try_with(|live| live.set(live.get() + layout.size() as isize)); + unsafe { std::alloc::System.alloc(layout) } + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: std::alloc::Layout) { + let _ = LIVE_BYTES.try_with(|live| live.set(live.get() - layout.size() as isize)); + unsafe { std::alloc::System.dealloc(pointer, layout) } + } + } + + #[global_allocator] + static ALLOCATOR: CountingAllocator = CountingAllocator; + + struct QueueDrops { + dropped: u64, + dropped_while_stuck: u64, + delivered: u64, + heap_growth: isize, + } + + // Pushes frames straight into the capture queue while every frame leaving + // it waits for a permit, then reports the frames dropped and delivered and + // the heap the pushing thread kept while the consumer was stuck. + fn queue_drops(permit_during_overrun: bool, frames: u64) -> QueueDrops { + gst::init().unwrap(); + let pipeline = PipelineGuard(gst::Pipeline::new()); + let queue = video_queue().unwrap(); + let sink = element("fakesink").unwrap(); + sink.set_property("sync", false); + pipeline.0.add_many([&queue, &sink]).unwrap(); + queue.link(&sink).unwrap(); + let counters = Arc::new(Counters::default()); + count_dropped(&queue, counters.clone()); + let (permits, permitted) = mpsc::channel::<()>(); + let permitted = Mutex::new(permitted); + let delivered = Arc::new(Mutex::new(Vec::new())); + let counted = delivered.clone(); + queue + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + let _ = permitted.lock().unwrap().recv(); + if let Some(buffer) = info.buffer() { + counted.lock().unwrap().push(( + buffer.as_ptr() as usize, + buffer.flags().contains(gst::BufferFlags::DISCONT), + )); + } + gst::PadProbeReturn::Ok + }); + if permit_during_overrun { + let permits = Mutex::new(permits.clone()); + queue.connect("overrun", false, move |values| { + let queue = values[0].get::().unwrap(); + permits.lock().unwrap().send(()).unwrap(); + while queue.property::("current-level-buffers") >= 2 { + std::thread::sleep(Duration::from_millis(1)); + } + None + }); + } + pipeline.0.set_state(gst::State::Playing).unwrap(); + let input = queue.static_pad("sink").unwrap(); + input.send_event(gst::event::StreamStart::new("queue-drops")); + input.send_event(gst::event::Caps::new(&variable_rate(60))); + input.send_event(gst::event::Segment::new(&gst::FormattedSegment::< + gst::ClockTime, + >::new())); + // Keep a reference to every frame, as pipewiresrc keeps its last one, + // so a counter that wrote to a frame would have to copy it. The queue + // itself copies only the frame after a discard, to mark it DISCONT. + let mut sent = Vec::with_capacity(frames as usize); + let heap = LIVE_BYTES.with(|live| live.get()); + for frame in 0..frames { + let mut buffer = gst::Buffer::new(); + buffer + .get_mut() + .unwrap() + .set_pts(gst::ClockTime::from_mseconds(frame * 10)); + sent.push(buffer.clone()); + assert_eq!(input.chain(buffer), Ok(gst::FlowSuccess::Ok)); + if permit_during_overrun || frame == 0 { + std::thread::sleep(Duration::from_millis(20)); + } + } + let heap_growth = LIVE_BYTES.with(|live| live.get()) - heap; + let dropped_while_stuck = dropped_frames(&pipeline.0, &counters); + for _ in 0..frames { + let _ = permits.send(()); + } + input.send_event(gst::event::Eos::new()); + pipeline + .0 + .bus() + .unwrap() + .timed_pop_filtered(gst::ClockTime::from_seconds(5), &[gst::MessageType::Eos]) + .unwrap(); + let sent: std::collections::HashSet = + sent.iter().map(|buffer| buffer.as_ptr() as usize).collect(); + let delivered = delivered.lock().unwrap(); + assert!( + delivered + .iter() + .all(|(buffer, discont)| *discont || sent.contains(buffer)), + "the queue delivered a copy of a captured frame" + ); + QueueDrops { + dropped: dropped_frames(&pipeline.0, &counters), + dropped_while_stuck, + delivered: delivered.len() as u64, + heap_growth, + } + } + + #[test] + fn dropped_frames_count_only_frames_the_queue_discards() { + let raced = queue_drops(true, 6); + assert_eq!((raced.dropped, raced.delivered), (0, 6)); + let leaked = queue_drops(false, 6); + assert_eq!((leaked.dropped, leaked.delivered), (3, 3)); + } + + #[test] + fn a_stuck_encoder_does_not_grow_drop_counting() { + let stuck = queue_drops(false, 2000); + assert_eq!(stuck.dropped_while_stuck, 1997); + assert_eq!((stuck.dropped, stuck.delivered), (1997, 3)); + assert!( + stuck.heap_growth < 4096, + "counting kept {} bytes for 1997 dropped frames", + stuck.heap_growth + ); + } + + // Durations the movie receives for encoded frames that arrive with + // `duration` under each of the given encoder caps. + fn movie_durations( + caps: &[&str], + duration: Option, + ) -> Vec> { + gst::init().unwrap(); + let pipeline = PipelineGuard(gst::Pipeline::new()); + let format = movie_format(Codec::H264, 60).unwrap(); + let sink = element("fakesink").unwrap(); + pipeline.0.add_many([&format, &sink]).unwrap(); + format.link(&sink).unwrap(); + let received = Arc::new(Mutex::new(Vec::new())); + let recorded = received.clone(); + sink.static_pad("sink") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + recorded + .lock() + .unwrap() + .push(info.buffer().and_then(|buffer| buffer.duration())); + gst::PadProbeReturn::Ok + }); + pipeline.0.set_state(gst::State::Playing).unwrap(); + let input = format.static_pad("sink").unwrap(); + input.send_event(gst::event::StreamStart::new("movie-durations")); + for (index, caps) in caps.iter().enumerate() { + input.send_event(gst::event::Caps::new(&caps.parse::().unwrap())); + if index == 0 { + input.send_event(gst::event::Segment::new(&gst::FormattedSegment::< + gst::ClockTime, + >::new())); + } + let mut buffer = gst::Buffer::new(); + buffer.get_mut().unwrap().set_duration(duration); + assert_eq!(input.chain(buffer), Ok(gst::FlowSuccess::Ok)); + } + let received = received.lock().unwrap().clone(); + received + } + + #[test] + fn frames_without_a_duration_last_one_frame_at_the_capture_rate() { + let encoded = "video/x-h264, stream-format=avc3, alignment=au, width=320, height=180"; + let rated = |rate: &str| format!("{encoded}, framerate=0/1, max-framerate={rate}"); + let frame = |rate: u64| Some(gst::ClockTime::SECOND / rate); + assert_eq!( + movie_durations(&[&rated("5/1"), &rated("60/1"), encoded], None), + [frame(5), frame(60), frame(60)] + ); + assert_eq!( + movie_durations(&[&format!("{encoded}, framerate=30/1"), encoded], None), + [frame(30), frame(60)] + ); + let kept = gst::ClockTime::from_mseconds(7); + assert_eq!( + movie_durations(&[&rated("5/1"), encoded], Some(kept)), + [Some(kept), Some(kept)] + ); + } + #[test] fn every_attempt_that_ends_the_recording_signals_teardown() { let idle = Counters::default(); diff --git a/packaging/linux/README.md b/packaging/linux/README.md index b6e8914..f914407 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -49,6 +49,14 @@ using `avc3` for H.264 and `hev1` for HEVC. This avoids changing the sample description in a fragmented movie, which can make older qtmux versions write unplayable fragments. Playback requires a reader that supports these sample entries; FFmpeg playback is covered by the recording tests. +When an encoder omits a frame's duration, wrec supplies one frame at the +current capture rate, falling back to the requested rate when the encoded caps +have no rate. The last frame then has a nonzero duration in the finalized movie. +The two-frame capture queue counts frames entering and leaving it, subtracting +frames still waiting. Queue-overrun notifications can race with the consumer +and do not prove that a frame was discarded. Counting uses constant storage and +does not modify capture buffers. The count is exact after the queue drains; +sampling while frames are moving can differ by one frame in transit. wrec tries shared GPU buffers only when `vapostproc` lists DMA-BUF caps with `format=DMA_DRM` and comes from GStreamer VA 1.24.6 or later. `DMA_DRM` caps @@ -99,16 +107,24 @@ pending; resuming or starting the next consumer can then create a second frame and make the backend exit with "session already has a frame object". PipeWire's GStreamer source pauses and resumes during startup, so successful recordings on this stack do not establish reliable startup. wrec detects the -backend loss and fails the job, but does not repair the portal. This remains -an unresolved compatibility issue, separate from the fixes above. +backend loss and fails the job, but does not repair the portal. Stock builds +remain affected. Separate local patches for portal-wlr 0.8.1 cancel pending +captures when their buffers are returned or removed and restart capture when +new buffers have no image on a static output. These patches are external to +wrec and require an opt-in build matched to the desktop's libraries. The older wlr-screencopy path also has a static-start limitation. The portal shares a screencopy-manager binding across sessions and asks for each new session's first frame with damage tracking. The first session consumes the binding's initial damage; later sessions on an unchanged output can wait without receiving an initial image. This was reproduced with portal-wlr 0.7.0, -Sway 1.7 and wlroots 0.15.1. It remains unresolved; repainting or restarting -the portal is not treated as a repair. +Sway 1.7 and wlroots 0.15.1. Separate local patches for portal-wlr 0.7.1 request +a first image without damage tracking, destroy closed streams promptly, and +limit retries when the consumer holds every buffer. Stock builds remain +affected; repainting or restarting the portal is not treated as a repair. +The Linux test report distributes pinned patch series, build instructions, +checksums and stock-versus-patched evidence. The wrec package does not install +or replace the desktop portal. The daemon does not run native capture code. For each recording it starts a capture worker, which is the same daemon executable started again with a private From 3363be073c24e3a2cb20c3c4490edef24c6d2074 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:39:37 +0000 Subject: [PATCH 20/45] fix(linux): emit sparse HEVC frames without blocking audio --- crates/linux/src/encoding.rs | 58 +++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/crates/linux/src/encoding.rs b/crates/linux/src/encoding.rs index 512db27..ba4e19b 100644 --- a/crates/linux/src/encoding.rs +++ b/crates/linux/src/encoding.rs @@ -207,7 +207,10 @@ impl Mode { encoder.set_property_from_str("speed-preset", "ultrafast"); encoder.set_property_from_str("tune", "zerolatency"); encoder.set_property("qp", qp as i32); - encoder.set_property("option-string", "pools=2:frame-threads=2"); + // A second frame thread keeps each frame until the next one + // arrives, a second later on an idle screen. A third worker + // maintains throughput without keeping a second frame. + encoder.set_property("option-string", "pools=3:frame-threads=1"); encoder.set_property("key-int-max", settings.fps.as_u32() as i32 * 2); } "openh264enc" => { @@ -260,4 +263,57 @@ mod tests { assert_eq!(modes.len(), 1); assert_eq!(modes[0].kind, Kind::Software); } + + // An idle screen sends about one frame a second. The movie muxer holds + // audio until video reaches it, so an encoder that keeps a frame until the + // next one arrives backs audio up for seconds. + #[test] + fn software_encoders_emit_each_frame_before_the_next_arrives() { + gst::init().unwrap(); + let mut tested = Vec::new(); + for codec in [Codec::H264, Codec::Hevc] { + for mode in modes(codec, false) { + if mode.kind != Kind::Software { + continue; + } + let pipeline = gst::Pipeline::new(); + let source = element("appsrc").unwrap(); + source.set_property("is-live", true); + source.set_property_from_str("format", "time"); + source.set_property( + "caps", + gst::Caps::builder("video/x-raw") + .field("format", "I420") + .field("width", 320i32) + .field("height", 180i32) + .field("framerate", gst::Fraction::new(0, 1)) + .field("max-framerate", gst::Fraction::new(60, 1)) + .build(), + ); + let encoder = mode.encoder(&RecorderSettings::default()).unwrap(); + let sink = element("fakesink").unwrap(); + sink.set_property("sync", false); + pipeline.add_many([&source, &encoder, &sink]).unwrap(); + gst::Element::link_many([&source, &encoder, &sink]).unwrap(); + let (encoded, received) = std::sync::mpsc::channel(); + encoder.static_pad("src").unwrap().add_probe( + gst::PadProbeType::BUFFER, + move |_, _| { + let _ = encoded.send(()); + gst::PadProbeReturn::Ok + }, + ); + pipeline.set_state(gst::State::Playing).unwrap(); + let mut frame = gst::Buffer::from_mut_slice(vec![0u8; 320 * 180 * 3 / 2]); + frame.get_mut().unwrap().set_pts(gst::ClockTime::ZERO); + let pushed = source.emit_by_name::("push-buffer", &[&frame]); + assert_eq!(pushed, gst::FlowReturn::Ok); + let emitted = received.recv_timeout(std::time::Duration::from_secs(1)); + pipeline.set_state(gst::State::Null).unwrap(); + assert!(emitted.is_ok(), "{} kept a lone frame", mode.factory); + tested.push(mode.factory); + } + } + assert!(tested.contains(&"x265enc"), "tested only {tested:?}"); + } } From 16358d7122a60c10494577381326bd77ca900de2 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:07:55 +0000 Subject: [PATCH 21/45] fix(linux): preserve movies when audio starts late --- crates/linux/src/pipeline.rs | 615 +++++++++++++++++++++++++++++++++-- packaging/linux/README.md | 20 +- 2 files changed, 610 insertions(+), 25 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 8837827..668e265 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -13,7 +13,7 @@ use std::{ os::fd::AsRawFd, path::Path, sync::{ - atomic::{AtomicU64, Ordering}, + atomic::{AtomicU64, AtomicU8, Ordering}, mpsc, Arc, Mutex, }, time::{Duration, Instant}, @@ -353,18 +353,31 @@ fn record_attempt( sink.set_property("sync", false); let mut chain = vec![&source, &input, &queue]; chain.extend(converters.iter()); - chain.extend([&output, &encoder, &parser, &format, &mux, &sink]); + chain.extend([&output, &encoder, &parser, &format]); pipeline .0 .add_many(chain.iter().copied()) .map_err(backend)?; + pipeline.0.add_many([&mux, &sink]).map_err(backend)?; gst::Element::link_many(chain.iter().copied()).map_err(backend)?; + mux.link(&sink).map_err(backend)?; let counters = attempt.counters.clone(); + let mut movie = Movie::new( + &pipeline.0, + &format, + &mux, + settings.include_system_audio || settings.include_microphone, + )?; if settings.include_system_audio { - add_audio(&pipeline.0, &mux, Some("@DEFAULT_MONITOR@"), &counters)?; + add_audio( + &mut movie, + Some("@DEFAULT_MONITOR@"), + "system audio", + &counters, + )?; } if settings.include_microphone { - add_audio(&pipeline.0, &mux, None, &counters)?; + add_audio(&mut movie, None, "microphone", &counters)?; } attach_capture_probe( &source, @@ -385,7 +398,7 @@ fn record_attempt( let _ = events.send(RecorderEvent::Log { session_id: Some(session.id), message: format!( - "capture-engine: selected {}; video queue limited to 2 frames", + "capture-engine: selected {}; capture queue limited to 2 frames", mode.description() ), }); @@ -396,7 +409,7 @@ fn record_attempt( .open(&session.output_path) .map_err(backend)?; let result = run( - &pipeline.0, + &mut movie, session, events, commands, @@ -547,10 +560,207 @@ fn keep_capture_timestamps(encoder: &gst::Element, parser: &gst::Element) { }); } +// Encoded video held while the movie waits for audio to start. Past either +// limit, the movie starts without the audio tracks that have not delivered a +// buffer. Paused time does not count, since no video arrives while paused. +// Constant-QP output has no fixed bitrate, so bytes are bounded separately. +const HELD_VIDEO_LIMIT: gst::ClockTime = gst::ClockTime::from_seconds(3); +const HELD_VIDEO_BYTES: u32 = 32 << 20; +const HELD_QUEUE: &str = "held-video-queue"; + +const WAITING: u8 = 0; +const STARTED: u8 = 1; +const ENDED: u8 = 2; +const OMITTED: u8 = 3; + +struct AudioTrack { + name: &'static str, + pad: gst::Pad, + probe: Option, + state: Arc, +} + +// The recording pipeline and the tracks of its movie. qtmux writes a track +// for every pad requested from it, and a track that never received audio has +// no sample description, which makes the whole movie unreadable. AAC encoders +// only learn their format from their first input buffer, so audio pads are +// requested once each track has delivered a buffer or ended without one. +// Until then video waits in a queue instead of being dropped at capture. +struct Movie { + pipeline: gst::Pipeline, + mux: gst::Element, + held: Option<(gst::Element, gst::PadProbeId)>, + draining: Option, + audio: Vec, +} + +impl Movie { + fn new( + pipeline: &gst::Pipeline, + video: &gst::Element, + mux: &gst::Element, + audio: bool, + ) -> Result { + let mut movie = Self { + pipeline: pipeline.clone(), + mux: mux.clone(), + held: None, + draining: None, + audio: Vec::new(), + }; + let track = mux + .request_pad_simple("video_%u") + .ok_or_else(|| backend("the movie muxer refused a video track"))?; + if !audio { + video + .static_pad("src") + .unwrap() + .link(&track) + .map_err(backend)?; + return Ok(movie); + } + // Headroom past the limits covers the run loop's 100 ms polling, so + // held video is not pushed back to the capture queue before then. + let queue = element("queue")?; + queue.set_property("name", HELD_QUEUE); + queue.set_property( + "max-size-time", + (HELD_VIDEO_LIMIT + gst::ClockTime::SECOND).nseconds(), + ); + queue.set_property("max-size-buffers", 0u32); + queue.set_property("max-size-bytes", HELD_VIDEO_BYTES + (8 << 20)); + pipeline.add(&queue).map_err(backend)?; + video.link(&queue).map_err(backend)?; + queue + .static_pad("src") + .unwrap() + .link(&track) + .map_err(backend)?; + let probe = queue + .static_pad("src") + .unwrap() + .add_probe( + gst::PadProbeType::BLOCK | gst::PadProbeType::BUFFER, + |_, _| gst::PadProbeReturn::Ok, + ) + .unwrap(); + movie.held = Some((queue, probe)); + Ok(movie) + } + + fn add_audio(&mut self, name: &'static str, queue: &gst::Element) { + let pad = queue.static_pad("src").unwrap(); + // Linking the track sends a reconfigure upstream. The encoder answers + // with an allocation query, which waits behind this queue while qtmux + // holds audio for the next video frame, and pulsesrc loses audio + // meanwhile. The audio caps are fixed, so nothing needs it. + pad.add_probe(gst::PadProbeType::EVENT_UPSTREAM, |_, info| { + if info + .event() + .is_some_and(|event| event.type_() == gst::EventType::Reconfigure) + { + gst::PadProbeReturn::Handled + } else { + gst::PadProbeReturn::Ok + } + }); + let state = Arc::new(AtomicU8::new(WAITING)); + let track = state.clone(); + let probe = pad.add_probe( + gst::PadProbeType::BLOCK + | gst::PadProbeType::BUFFER + | gst::PadProbeType::EVENT_DOWNSTREAM, + move |_, info| { + let next = match &info.data { + Some(gst::PadProbeData::Buffer(_)) => STARTED, + Some(gst::PadProbeData::Event(event)) + if event.type_() == gst::EventType::Eos => + { + ENDED + } + _ => return gst::PadProbeReturn::Pass, + }; + let _ = track.compare_exchange(WAITING, next, Ordering::SeqCst, Ordering::SeqCst); + if next == ENDED { + return gst::PadProbeReturn::Pass; + } + if track.load(Ordering::SeqCst) == OMITTED { + gst::PadProbeReturn::Drop + } else { + gst::PadProbeReturn::Ok + } + }, + ); + self.audio.push(AudioTrack { + name, + pad, + probe, + state, + }); + } + + // Called on every pass of the run loop. Requests the audio pads once every + // track has started or ended, or the held video reached its limit, then + // releases the video. Returns the omitted tracks, each with whether it + // ended before delivering audio. + fn link_audio(&mut self) -> Result> { + if let Some(queue) = &self.draining { + // Once the video held at startup has reached qtmux, a stalled + // track holds video in qtmux and the capture queue drops frames, + // as without this queue. Limiting it earlier would block the + // encoder while the held video drains. + if queue.property::("current-level-buffers") <= 1 { + queue.set_property("max-size-time", 0u64); + queue.set_property("max-size-bytes", 0u32); + queue.set_property("max-size-buffers", 1u32); + self.draining = None; + } + } + let Some((queue, _)) = &self.held else { + return Ok(Vec::new()); + }; + let full = queue.property::("current-level-time") >= HELD_VIDEO_LIMIT.nseconds() + || queue.property::("current-level-bytes") >= HELD_VIDEO_BYTES; + if !full + && self + .audio + .iter() + .any(|track| track.state.load(Ordering::SeqCst) == WAITING) + { + return Ok(Vec::new()); + } + let mut omitted = Vec::new(); + for track in &mut self.audio { + let state = track.state.load(Ordering::SeqCst); + if state != STARTED + && track + .state + .compare_exchange(state, OMITTED, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + omitted.push((track.name, state == ENDED)); + continue; + } + let pad = self + .mux + .request_pad_simple("audio_%u") + .ok_or_else(|| backend("the movie muxer refused an audio track"))?; + track.pad.link(&pad).map_err(backend)?; + if let Some(probe) = track.probe.take() { + track.pad.remove_probe(probe); + } + } + let (queue, probe) = self.held.take().unwrap(); + queue.static_pad("src").unwrap().remove_probe(probe); + self.draining = Some(queue); + Ok(omitted) + } +} + fn add_audio( - pipeline: &gst::Pipeline, - mux: &gst::Element, + movie: &mut Movie, device: Option<&str>, + name: &'static str, counters: &Arc, ) -> Result<()> { let source = element("pulsesrc")?; @@ -558,13 +768,13 @@ fn add_audio( if let Some(device) = device { source.set_property("device", device); } - add_audio_source(pipeline, mux, &source, counters) + add_audio_source(movie, &source, name, counters) } fn add_audio_source( - pipeline: &gst::Pipeline, - mux: &gst::Element, + movie: &mut Movie, source: &gst::Element, + name: &'static str, counters: &Arc, ) -> Result<()> { attach_timing_probe(source, counters.clone()); @@ -582,15 +792,20 @@ fn add_audio_source( encoder.set_property("bitrate", 128000i32); let parser = element("aacparse")?; let queue = element("queue")?; - queue.set_property("max-size-time", 2_000_000_000u64); + // Holds audio while the movie waits for video, or for another audio + // track to start, which can take HELD_VIDEO_LIMIT. + queue.set_property( + "max-size-time", + (HELD_VIDEO_LIMIT + gst::ClockTime::SECOND).nseconds(), + ); queue.set_property("max-size-buffers", 0u32); queue.set_property("max-size-bytes", 0u32); let chain = [ source, &convert, &resample, &caps, &encoder, &parser, &queue, ]; - pipeline.add_many(chain).map_err(backend)?; + movie.pipeline.add_many(chain).map_err(backend)?; gst::Element::link_many(chain).map_err(backend)?; - queue.link(mux).map_err(backend)?; + movie.add_audio(name, &queue); Ok(()) } @@ -853,7 +1068,7 @@ const SOURCE_LOST: &str = "The captured X11 window closed, was minimized, or became unavailable; the recording stopped."; fn run( - pipeline: &gst::Pipeline, + movie: &mut Movie, session: &RecordingSession, events: &mpsc::Sender, commands: &mpsc::Receiver, @@ -861,6 +1076,7 @@ fn run( counters: &Counters, source_lost: &dyn Fn() -> Option<&'static str>, ) -> Result<()> { + let pipeline = &movie.pipeline.clone(); pipeline .set_state(gst::State::Playing) .map_err(|error| state_error(pipeline, error))?; @@ -936,6 +1152,23 @@ fn run( _ => {} } } + for (name, ended) in movie.link_audio()? { + let _ = events.send(RecorderEvent::Log { + session_id: Some(session.id), + message: format!( + "capture-engine: {name} sent no samples {}; the movie has no {name} track", + if ended { + "before the recording stopped".to_string() + } else { + format!( + "while {} seconds or {} MiB of video waited for it", + HELD_VIDEO_LIMIT.seconds(), + HELD_VIDEO_BYTES >> 20 + ) + } + ), + }); + } if !started && counters.frames.load(Ordering::Relaxed) > 0 { started = true; let _ = events.send(RecorderEvent::Started { @@ -1250,14 +1483,31 @@ mod tests { }; // Synthetic software encoding is confined to tests. Exercise the same // bus/control/mux code without pretending this is a hardware benchmark. - let pipeline = gst::parse::launch("videotestsrc name=video is-live=true pattern=ball ! video/x-raw,width=320,height=180,framerate=30/1 ! openh264enc name=encoder ! h264parse name=parser") + let pipeline = gst::parse::launch("videotestsrc name=video is-live=true pattern=ball ! capsfilter name=capture caps=video/x-raw,width=320,height=180,framerate=30/1 openh264enc name=encoder ! h264parse name=parser") .unwrap().downcast::().unwrap(); + // Like production, frames the encoder cannot take are dropped + // before encoding instead of stalling capture. + let queue = video_queue().unwrap(); + pipeline.add(&queue).unwrap(); + gst::Element::link_many([ + &pipeline.by_name("capture").unwrap(), + &queue, + &pipeline.by_name("encoder").unwrap(), + ]) + .unwrap(); let mux = movie_mux().unwrap(); let sink = element("filesink").unwrap(); sink.set_property("sync", false); sink.set_property("location", session.output_path.to_str().unwrap()); pipeline.add_many([&mux, &sink]).unwrap(); - gst::Element::link_many([&pipeline.by_name("parser").unwrap(), &mux, &sink]).unwrap(); + mux.link(&sink).unwrap(); + let mut movie = Movie::new( + &pipeline, + &pipeline.by_name("parser").unwrap(), + &mux, + !audio_delays.is_empty(), + ) + .unwrap(); let counters = Arc::new(Counters::default()); attach_timing_probe(&pipeline.by_name("video").unwrap(), counters.clone()); let encoder = pipeline.by_name("encoder").unwrap(); @@ -1275,7 +1525,7 @@ mod tests { _ => gst::PadProbeReturn::Ok, }, ); - add_audio_source(&pipeline, &mux, &source, &counters).unwrap(); + add_audio_source(&mut movie, &source, "test audio", &counters).unwrap(); } attach_encoded_probe(&pipeline.by_name("parser").unwrap(), counters.clone()); let (events_tx, events) = mpsc::channel(); @@ -1287,9 +1537,9 @@ mod tests { let lost = Arc::new(std::sync::atomic::AtomicBool::new(false)); let worker_lost = lost.clone(); let worker = std::thread::spawn(move || { - let guard = PipelineGuard(worker_pipeline); + let _guard = PipelineGuard(worker_pipeline); run( - &guard.0, + &mut movie, &worker_session, &events_tx, &commands_rx, @@ -1346,6 +1596,26 @@ mod tests { fn probe(&self) -> serde_json::Value { probe_movie(&self.session.output_path) } + + fn logs(&self) -> Vec { + self.events + .try_iter() + .filter_map(|event| match event { + RecorderEvent::Log { message, .. } => Some(message), + _ => None, + }) + .collect() + } + + fn active_seconds(&self) -> f64 { + self.counters + .timeline + .lock() + .unwrap() + .position(self.pipeline.current_running_time().unwrap()) + .nseconds() as f64 + / 1_000_000_000.0 + } } fn count_frames_like_old_va_encoders(source: &gst::Element, encoder: &gst::Element) { @@ -1477,11 +1747,40 @@ mod tests { .collect() } + // The largest gap between packets of a stream up to the given time. + fn largest_gap(probe: &serde_json::Value, kind: &str, until: f64) -> f64 { + let index = probe["streams"] + .as_array() + .unwrap() + .iter() + .find(|stream| stream["codec_type"] == kind) + .unwrap()["index"] + .clone(); + let mut pts: Vec = probe["packets"] + .as_array() + .unwrap() + .iter() + .filter(|packet| packet["stream_index"] == index) + .map(|packet| packet["pts_time"].as_str().unwrap().parse().unwrap()) + .filter(|pts| *pts <= until) + .collect(); + pts.sort_by(f64::total_cmp); + pts.windows(2) + .map(|pair| pair[1] - pair[0]) + .fold(0.0, f64::max) + } + fn assert_late_audio_keeps_its_offset(movie: &Path, probe: &serde_json::Value, delay: f64) { let video = start_seconds(probe, "h264"); let audio = start_seconds(probe, "aac"); assert_eq!((video.len(), audio.len()), (1, 2), "{}", movie.display()); let (prompt, late) = (audio[0], audio[1]); + let gap = largest_gap(probe, "video", late + 0.5); + assert!( + gap < 0.1, + "{}: video skips {gap}s while waiting for late audio", + movie.display() + ); assert!( (late - prompt - delay).abs() < 0.05, "{}: audio starts {prompt}s and {late}s, expected {delay}s apart", @@ -1676,6 +1975,273 @@ mod tests { ); } + fn stream_counts(probe: &serde_json::Value) -> (usize, usize) { + let streams = probe["streams"].as_array().unwrap(); + let count = |codec| streams.iter().filter(|s| s["codec_name"] == codec).count(); + (count("h264"), count("aac")) + } + + #[test] + fn stopping_before_audio_arrives_keeps_the_captured_video() { + let never = gst::ClockTime::MAX; + for (delays, paused) in [ + (vec![never], false), + (vec![never], true), + (vec![gst::ClockTime::ZERO, never], false), + (vec![gst::ClockTime::ZERO, never], true), + ] { + let case = format!("audio delays {delays:?}, paused {paused}"); + let mut recording = TestRecording::start_with_audio_delays(&delays); + std::thread::sleep(Duration::from_millis(800)); + if paused { + recording.control(true); + std::thread::sleep(Duration::from_millis(500)); + } + let active = recording.active_seconds(); + recording.finish().unwrap(); + let probe = recording.probe(); + assert_eq!(stream_counts(&probe), (1, delays.len() - 1), "{case}"); + let gap = largest_gap(&probe, "video", f64::INFINITY); + assert!(gap < 0.1, "{case}: video skips {gap}s"); + let duration = end_seconds(&probe, "h264"); + assert!( + (duration - active).abs() < 0.2, + "{case}: video ends at {duration}s after {active}s of active recording" + ); + let logs = recording.logs(); + assert!( + logs.iter().any(|log| log.contains( + "test audio sent no samples before the recording stopped; the movie has no test audio track" + )), + "{case}: {logs:?}" + ); + } + } + + #[test] + fn late_audio_keeps_flowing_while_the_screen_is_idle() { + let mut recording = + TestRecording::start_with_audio_delays(&[gst::ClockTime::from_seconds(1)]); + // An idle screen sends no frames, so when the late track is linked + // qtmux holds its audio until the next frame arrives. + let idle = gst::ClockTime::from_mseconds(500)..gst::ClockTime::from_mseconds(2200); + recording + .pipeline + .by_name("video") + .unwrap() + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + match info.buffer().and_then(|buffer| buffer.pts()) { + Some(pts) if idle.contains(&pts) => gst::PadProbeReturn::Drop, + _ => gst::PadProbeReturn::Ok, + } + }); + let wait_until = |milliseconds| { + while recording.pipeline.current_running_time().unwrap() + < gst::ClockTime::from_mseconds(milliseconds) + { + std::thread::sleep(Duration::from_millis(10)); + } + }; + // Audio that arrives while paused is dropped, so audio stuck behind + // qtmux before the pause would leave a hole. + wait_until(2000); + recording.control(true); + wait_until(2500); + recording.control(false); + wait_until(3500); + recording.finish().unwrap(); + let probe = recording.probe(); + assert_eq!(stream_counts(&probe), (1, 1)); + let gap = largest_gap(&probe, "audio", f64::INFINITY); + assert!(gap < 0.1, "audio skips {gap}s"); + let (video, audio) = (end_seconds(&probe, "h264"), end_seconds(&probe, "aac")); + assert!( + (video - audio).abs() < 0.15, + "video ends at {video}s, audio at {audio}s" + ); + } + + #[test] + fn a_static_screen_paused_before_audio_arrives_still_finalizes() { + let mut recording = TestRecording::start_with_audio_delays(&[gst::ClockTime::MAX]); + // A static screen sends no more frames, so the held video stops + // growing and only the audio track's end releases it. + recording + .pipeline + .by_name("video") + .unwrap() + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, |_, info| { + match info.buffer().map(|buffer| buffer.offset()) { + Some(0) => gst::PadProbeReturn::Ok, + _ => gst::PadProbeReturn::Drop, + } + }); + std::thread::sleep(Duration::from_millis(300)); + let held = recording.pipeline.by_name(HELD_QUEUE).unwrap(); + let level = held.property::("current-level-time"); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!(held.property::("current-level-time"), level); + recording.control(true); + std::thread::sleep(Duration::from_millis(500)); + recording.finish().unwrap(); + let probe = recording.probe(); + assert_eq!(stream_counts(&probe), (1, 0)); + let logs = recording.logs(); + assert!( + logs.iter() + .any(|log| log.contains("test audio sent no samples before the recording stopped")), + "{logs:?}" + ); + } + + #[test] + fn a_stalled_audio_track_does_not_build_up_encoded_video() { + let mut recording = TestRecording::start(1); + let stall = gst::ClockTime::from_mseconds(1000)..gst::ClockTime::from_mseconds(2500); + recording + .pipeline + .iterate_elements() + .into_iter() + .filter_map(|element| element.ok()) + .find(|element| { + element + .factory() + .is_some_and(|f| f.name() == "audiotestsrc") + }) + .unwrap() + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + match info.buffer().and_then(|buffer| buffer.pts()) { + Some(pts) if stall.contains(&pts) => gst::PadProbeReturn::Drop, + _ => gst::PadProbeReturn::Ok, + } + }); + let held = recording.pipeline.by_name(HELD_QUEUE).unwrap(); + let mut most = 0; + loop { + let now = recording.pipeline.current_running_time().unwrap(); + if now >= gst::ClockTime::from_mseconds(3000) { + break; + } + // From the stall on; the video held at startup drains before it. + if now >= gst::ClockTime::from_mseconds(1100) { + most = most.max(held.property::("current-level-buffers")); + } + std::thread::sleep(Duration::from_millis(10)); + } + recording.finish().unwrap(); + recording.probe(); + assert!(most <= 1, "{most} encoded frames waited for stalled audio"); + } + + #[test] + fn held_video_is_bounded_in_bytes_as_well_as_time() { + gst::init().unwrap(); + let pipeline = PipelineGuard(gst::Pipeline::new()); + let video = element("appsrc").unwrap(); + video.set_property_from_str("format", "time"); + video.set_property( + "caps", + gst::Caps::builder("video/x-h264") + .field("stream-format", "avc3") + .field("alignment", "au") + .field("width", 320i32) + .field("height", 180i32) + .field("framerate", gst::Fraction::new(0, 1)) + .build(), + ); + let mux = movie_mux().unwrap(); + let sink = element("fakesink").unwrap(); + let silent = element("queue").unwrap(); + pipeline.0.add_many([&video, &mux, &sink, &silent]).unwrap(); + mux.link(&sink).unwrap(); + let mut movie = Movie::new(&pipeline.0, &video, &mux, true).unwrap(); + movie.add_audio("test audio", &silent); + let held = pipeline.0.by_name(HELD_QUEUE).unwrap(); + pipeline.0.set_state(gst::State::Playing).unwrap(); + // Large constant-QP frames 10 ms apart reach the byte limit long + // before the time limit. + let frame = 1 << 20; + for index in 0..64u64 { + let pts = gst::ClockTime::from_mseconds(10 * index); + let mut buffer = gst::Buffer::from_mut_slice(vec![0u8; frame]); + buffer.get_mut().unwrap().set_pts(pts); + let before = held.property::("current-level-bytes"); + assert_eq!( + video.emit_by_name::("push-buffer", &[&buffer]), + gst::FlowReturn::Ok + ); + let deadline = Instant::now() + Duration::from_secs(1); + while index > 0 + && held.property::("current-level-bytes") == before + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(1)); + } + let level = held.property::("current-level-bytes"); + let omitted = movie.link_audio().unwrap(); + if !omitted.is_empty() { + assert_eq!(omitted, vec![("test audio", false)]); + assert!( + (HELD_VIDEO_BYTES..=HELD_VIDEO_BYTES + (8 << 20)).contains(&level), + "left out at {level} held bytes" + ); + assert!(pts < HELD_VIDEO_LIMIT, "left out at {pts}"); + return; + } + } + panic!( + "held {} bytes without leaving out the silent track; {:?}", + held.property::("current-level-bytes"), + pipeline + .0 + .bus() + .unwrap() + .pop_filtered(&[gst::MessageType::Error]) + ); + } + + #[test] + fn audio_that_never_starts_is_left_out_without_dropping_video() { + let mut recording = TestRecording::start_with_audio_delays(&[gst::ClockTime::MAX]); + let deadline = Instant::now() + Duration::from_secs(6); + let omitted = loop { + match recording + .events + .recv_timeout(deadline.saturating_duration_since(Instant::now())) + { + Ok(RecorderEvent::Log { message, .. }) if message.contains("sent no samples") => { + break Some((message, recording.active_seconds())) + } + Ok(_) => {} + Err(_) => break None, + } + }; + let (message, at) = omitted.expect("the recording never left out the silent audio track"); + assert!( + message.contains("while 3 seconds or 32 MiB of video waited for it"), + "{message}" + ); + assert!((3.0..3.5).contains(&at), "left out after {at}s"); + std::thread::sleep(Duration::from_millis(500)); + let active = recording.active_seconds(); + recording.finish().unwrap(); + let probe = recording.probe(); + assert_eq!(stream_counts(&probe), (1, 0)); + let gap = largest_gap(&probe, "video", f64::INFINITY); + assert!(gap < 0.1, "video skips {gap}s"); + let duration = end_seconds(&probe, "h264"); + assert!( + (duration - active).abs() < 0.2, + "video ends at {duration}s after {active}s of active recording" + ); + } + #[test] fn pause_resume_omits_paused_time_and_stop_while_paused_finalizes() { let mut recording = TestRecording::start(2); @@ -2340,8 +2906,13 @@ mod tests { #[test] fn source_loss_fails_active_and_paused_recordings_promptly() { - for paused in [false, true] { - let mut recording = TestRecording::start(1); + // The last case loses the source while video waits for audio. + for (paused, audio) in [ + (false, gst::ClockTime::ZERO), + (true, gst::ClockTime::ZERO), + (false, gst::ClockTime::MAX), + ] { + let mut recording = TestRecording::start_with_audio_delays(&[audio]); std::thread::sleep(Duration::from_millis(300)); if paused { recording.control(true); diff --git a/packaging/linux/README.md b/packaging/linux/README.md index f914407..d44c901 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -15,9 +15,18 @@ asks the allocator to return freed heap pages between encoder attempts. On compatible Wayland/VA-API systems, the preferred pipeline is `pipewiresrc → DMA-BUF → vapostproc → VA encoder → qtmux`. Rust manages sessions and buffer metadata without mapping video pixels. The source negotiates four to -eight buffers and the video queue holds at most two frames. Conversion and scaling +eight buffers and the capture queue holds at most two frames. Conversion and scaling can allocate GPU surfaces; this does not imply zero total copies. +When audio is requested, encoded video waits until each audio track starts or +ends. A track that never supplies a sample would otherwise make the whole movie +unreadable. If waiting video reaches three seconds or 32 MiB, recording continues +without the tracks that have not started and logs each omitted source. The startup +queue has four seconds and 40 MiB of capacity. Once it drains, it holds at most one +encoded frame. Audio that starts within the limit keeps its capture timestamp. +Each encoded AAC queue holds up to four seconds, about 64 KB at 128 kbps, to absorb +the movie writer's wait for the next video frame on an idle screen across a pause. + When shared GPU buffers cannot be imported, wrec retries with system-memory capture and hardware encoding. NVIDIA uses CUDA conversion when available, otherwise CPU conversion before NVENC. Software encoding uses x264/OpenH264 for @@ -109,8 +118,9 @@ PipeWire's GStreamer source pauses and resumes during startup, so successful recordings on this stack do not establish reliable startup. wrec detects the backend loss and fails the job, but does not repair the portal. Stock builds remain affected. Separate local patches for portal-wlr 0.8.1 cancel pending -captures when their buffers are returned or removed and restart capture when -new buffers have no image on a static output. These patches are external to +captures when their buffers are returned or removed, leave incomplete image +buffers empty, and restart capture when new buffers have no image on a static +output or a cancelled frame may have consumed its damage. These patches are external to wrec and require an opt-in build matched to the desktop's libraries. The older wlr-screencopy path also has a static-start limitation. The portal @@ -313,6 +323,10 @@ cargo build -p cli -p daemon python3 scripts/test-capture-linux.py target/debug/wrec ``` +The full test suite requires `x265enc` from the GStreamer runtime packages +listed above. The HEVC latency regression fails if that encoder is absent, +so a passing suite always includes it. + The isolated Xvfb test records actual X11 display/window pixels through the CLI and daemon, checks H.264/HEVC decoding and timestamps, and exercises pause/resume and stopping while paused. It also kills the X server during a recording and From 58d2ff29a7608892cbaf67972d563456c68545d8 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:29:47 +0000 Subject: [PATCH 22/45] test(linux): prevent audio gaps when idle video spans a pause --- crates/linux/src/pipeline.rs | 117 +++++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 6 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 668e265..05e1281 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -109,6 +109,10 @@ fn movie_mux() -> Result { const CAPTURE_QUEUE: &str = "capture-queue"; +// pipewiresrc resends the last frame after this long without a new one, so an +// idle screen still sends a frame each keepalive. +const KEEPALIVE: gst::ClockTime = gst::ClockTime::SECOND; + fn video_queue() -> Result { let queue = element("queue")?; queue.set_property("name", CAPTURE_QUEUE); @@ -308,7 +312,7 @@ fn record_attempt( source.set_property("always-copy", false); source.set_property("min-buffers", 4i32); source.set_property("max-buffers", 8i32); - source.set_property("keepalive-time", 1000i32); + source.set_property("keepalive-time", KEEPALIVE.mseconds() as i32); source.set_property("resend-last", true); source } @@ -793,10 +797,13 @@ fn add_audio_source( let parser = element("aacparse")?; let queue = element("queue")?; // Holds audio while the movie waits for video, or for another audio - // track to start, which can take HELD_VIDEO_LIMIT. + // track to start, which can take HELD_VIDEO_LIMIT. qtmux writes a video + // frame only when the next one arrives and holds later audio until then, + // so at an idle screen's keepalive rate audio waits up to two keepalives, + // and a pause that swallows a frame makes it almost three. queue.set_property( "max-size-time", - (HELD_VIDEO_LIMIT + gst::ClockTime::SECOND).nseconds(), + (HELD_VIDEO_LIMIT.max(KEEPALIVE * 3) + gst::ClockTime::SECOND).nseconds(), ); queue.set_property("max-size-buffers", 0u32); queue.set_property("max-size-bytes", 0u32); @@ -1473,6 +1480,28 @@ mod tests { } fn start_with(audio_delays: &[gst::ClockTime], counting_encoder: bool) -> Self { + Self::start_from( + "videotestsrc name=video is-live=true pattern=ball ! capsfilter name=capture caps=video/x-raw,width=320,height=180,framerate=30/1", + audio_delays, + counting_encoder, + ) + } + + // An idle screen: one frame per keepalive, stamped on arrival, and + // no source latency, like pipewiresrc. + fn start_idle(audio_tracks: usize) -> Self { + Self::start_from( + "appsrc name=video is-live=true format=time do-timestamp=true caps=video/x-raw,format=I420,width=320,height=180,framerate=0/1 ! capsfilter name=capture caps=video/x-raw,format=I420,width=320,height=180,framerate=0/1", + &vec![gst::ClockTime::ZERO; audio_tracks], + false, + ) + } + + fn start_from( + video: &str, + audio_delays: &[gst::ClockTime], + counting_encoder: bool, + ) -> Self { gst::init().unwrap(); static ID: AtomicU64 = AtomicU64::new(0); let id = ID.fetch_add(1, Ordering::Relaxed); @@ -1483,8 +1512,12 @@ mod tests { }; // Synthetic software encoding is confined to tests. Exercise the same // bus/control/mux code without pretending this is a hardware benchmark. - let pipeline = gst::parse::launch("videotestsrc name=video is-live=true pattern=ball ! capsfilter name=capture caps=video/x-raw,width=320,height=180,framerate=30/1 openh264enc name=encoder ! h264parse name=parser") - .unwrap().downcast::().unwrap(); + let pipeline = gst::parse::launch(&format!( + "{video} openh264enc name=encoder ! h264parse name=parser" + )) + .unwrap() + .downcast::() + .unwrap(); // Like production, frames the encoder cannot take are dropped // before encoding instead of stalling capture. let queue = video_queue().unwrap(); @@ -1509,7 +1542,22 @@ mod tests { ) .unwrap(); let counters = Arc::new(Counters::default()); - attach_timing_probe(&pipeline.by_name("video").unwrap(), counters.clone()); + let video = pipeline.by_name("video").unwrap(); + attach_timing_probe(&video, counters.clone()); + if video + .factory() + .is_some_and(|factory| factory.name() == "appsrc") + { + std::thread::spawn(move || loop { + let frame = gst::Buffer::from_mut_slice(vec![0u8; 320 * 180 * 3 / 2]); + if video.emit_by_name::("push-buffer", &[&frame]) + != gst::FlowReturn::Ok + { + break; + } + std::thread::sleep(Duration::from_nanos(KEEPALIVE.nseconds())); + }); + } let encoder = pipeline.by_name("encoder").unwrap(); if counting_encoder { count_frames_like_old_va_encoders(&pipeline.by_name("video").unwrap(), &encoder); @@ -2276,6 +2324,63 @@ mod tests { ); } + // An idle screen sends a frame each keepalive. A short pause that swallows + // one leaves almost two keepalives between the frames around it, and qtmux + // holds audio until the frame after those. The audio queue must absorb + // that wait: a full queue blocks pulsesrc, which then drops audio. + #[test] + fn audio_waiting_for_idle_video_across_a_pause_never_fills_its_queue() { + let mut recording = TestRecording::start_idle(2); + let full = Arc::new(std::sync::atomic::AtomicBool::new(false)); + for queue in recording.pipeline.iterate_elements().into_iter().flatten() { + let audio = queue + .factory() + .is_some_and(|factory| factory.name() == "queue") + && ![CAPTURE_QUEUE, HELD_QUEUE].contains(&queue.name().as_str()); + if audio { + let full = full.clone(); + queue.connect("overrun", false, move |_| { + full.store(true, Ordering::SeqCst); + None + }); + } + } + let arrived = Arc::new(AtomicU64::new(0)); + let latest = arrived.clone(); + let pipeline = recording.pipeline.clone(); + recording + .pipeline + .by_name("video") + .unwrap() + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, _| { + let now = pipeline.current_running_time().unwrap_or_default(); + latest.store(now.nseconds(), Ordering::SeqCst); + gst::PadProbeReturn::Ok + }); + let keepalive = Duration::from_nanos(KEEPALIVE.nseconds()); + for _ in 0..2 { + // Pause three quarters of a keepalive after a frame and resume a + // quarter after the next one, the worst case short of no pause. + let frame = arrived.load(Ordering::SeqCst); + while arrived.load(Ordering::SeqCst) == frame { + std::thread::sleep(Duration::from_millis(5)); + } + std::thread::sleep(keepalive * 3 / 4); + recording.control(true); + std::thread::sleep(keepalive / 2); + recording.control(false); + std::thread::sleep(keepalive * 3); + } + recording.finish().unwrap(); + assert!( + !full.load(Ordering::SeqCst), + "audio queue filled while qtmux waited for idle video" + ); + recording.probe(); + } + fn test_session() -> RecordingSession { static ID: AtomicU64 = AtomicU64::new(0); let id = ID.fetch_add(1, Ordering::Relaxed); From a52d865931ab4249b58a03c22fe9ed365140128d Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:39:22 +0000 Subject: [PATCH 23/45] test(linux): bound idle-video regression waits --- crates/linux/src/pipeline.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 05e1281..e6a26dc 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -2364,7 +2364,9 @@ mod tests { // Pause three quarters of a keepalive after a frame and resume a // quarter after the next one, the worst case short of no pause. let frame = arrived.load(Ordering::SeqCst); + let waiting = Instant::now(); while arrived.load(Ordering::SeqCst) == frame { + assert!(waiting.elapsed() < keepalive * 3, "idle video stopped"); std::thread::sleep(Duration::from_millis(5)); } std::thread::sleep(keepalive * 3 / 4); From e0ba7238d488f7c1d768e6ab8a46220d75a6a548 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:46:37 +0000 Subject: [PATCH 24/45] test(linux): identify slow source-loss cleanup phases --- crates/linux/src/pipeline.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index e6a26dc..baa5ac8 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -1468,6 +1468,8 @@ mod tests { worker: Option>>, counters: Arc, lost: Arc, + // When run() returned, before the pipeline was set to NULL. + returned: Arc>>, } impl TestRecording { @@ -1584,9 +1586,11 @@ mod tests { let worker_counters = counters.clone(); let lost = Arc::new(std::sync::atomic::AtomicBool::new(false)); let worker_lost = lost.clone(); + let returned = Arc::new(Mutex::new(None)); + let worker_returned = returned.clone(); let worker = std::thread::spawn(move || { let _guard = PipelineGuard(worker_pipeline); - run( + let result = run( &mut movie, &worker_session, &events_tx, @@ -1598,7 +1602,9 @@ mod tests { .load(Ordering::SeqCst) .then_some(crate::portal::PIPEWIRE_LOST) }, - ) + ); + *worker_returned.lock().unwrap() = Some(Instant::now()); + result }); let recording = Self { pipeline, @@ -1609,6 +1615,7 @@ mod tests { worker: Some(worker), counters, lost, + returned, }; loop { if matches!( @@ -3033,7 +3040,14 @@ mod tests { .join() .unwrap() .unwrap_err(); - assert!(at.elapsed() < Duration::from_secs(1), "{:?}", at.elapsed()); + let finished = at.elapsed(); + let returned = recording.returned.lock().unwrap().unwrap() - at; + assert!( + finished < Duration::from_secs(1), + "paused {paused}, waiting for audio {}: run returned after {returned:?}, \ + the worker finished after {finished:?}", + audio == gst::ClockTime::MAX + ); assert_eq!( error.to_string(), format!("backend error: {}", crate::portal::PIPEWIRE_LOST) From 4c9fa73fa00ce773b089958a643c5fe7b0de4077 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Mon, 5 Oct 2026 03:11:56 +0000 Subject: [PATCH 25/45] fix(linux): keep recording through short stalls and report lost media qtmux takes the next sample of every track in time order and never times out, so a blocked file write, a stalled encoder or a silent audio source held up every track. A 6 s write stall cost about 1.7 s of each audio track and 5.8 s of video; a 2 s microphone stall cost 1.8 s of video. - A 32 MiB queue between qtmux and the file absorbs disk stalls. It stays empty unless writes block. - After startup, encoded video keeps the 4 s / 40 MiB queue it uses while audio starts, so video waits for a stalled audio track as long as audio waits for video. This replaces the one-frame limit. - An error from an audio source ends its track; video and the other track keep recording, instead of the whole job failing. - Each audio track counts audio its source skipped. Holes are logged as they happen, and a finalized movie that dropped frames, has audio gaps, or lost a track sends MediaLost. The daemon turns it into a media_lost job warning, and `wrec record` prints it. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/cli/src/run.rs | 7 + crates/daemon/src/backend.rs | 19 + crates/daemon/src/coordinator.rs | 50 ++ crates/daemon/src/store.rs | 2 + crates/domain/src/lib.rs | 5 + crates/linux/src/pipeline.rs | 809 ++++++++++++++++++++++++++----- packaging/linux/README.md | 18 +- 7 files changed, 799 insertions(+), 111 deletions(-) diff --git a/crates/cli/src/run.rs b/crates/cli/src/run.rs index 90d13d0..53184a8 100644 --- a/crates/cli/src/run.rs +++ b/crates/cli/src/run.rs @@ -64,6 +64,13 @@ pub fn record(args: RecordArgs) -> ExitCode { } let completed = wait_for_job(job.id, json_output)?; + if !json_output { + // Warnings raised while recording, such as lost media. + for warning in completed.warnings.iter().skip(job.warnings.len()) { + eprintln!("warning: {}", warning.message); + eprintln!("next: {}", warning.next); + } + } Ok(match completed.status { JobStatus::Completed => ExitCode::SUCCESS, JobStatus::Failed | JobStatus::Cancelled => ExitCode::FAILURE, diff --git a/crates/daemon/src/backend.rs b/crates/daemon/src/backend.rs index 44ab7ec..6724efd 100644 --- a/crates/daemon/src/backend.rs +++ b/crates/daemon/src/backend.rs @@ -50,6 +50,9 @@ pub enum BackendEvent { Metrics { metrics: RecorderMetrics, }, + MediaLost { + message: String, + }, Failed { message: String, }, @@ -156,6 +159,22 @@ impl WrecBackend { metrics: metrics.clone(), } } + RecorderEvent::MediaLost { + session_id, + message, + } => { + self.append_event( + Some(*session_id), + EventSource::CaptureEngine, + EventLevel::Warning, + None, + message.clone(), + ); + + BackendEvent::MediaLost { + message: message.clone(), + } + } RecorderEvent::Failed { session_id, message, diff --git a/crates/daemon/src/coordinator.rs b/crates/daemon/src/coordinator.rs index 9929e5e..d1dc202 100644 --- a/crates/daemon/src/coordinator.rs +++ b/crates/daemon/src/coordinator.rs @@ -692,6 +692,18 @@ fn handle_recorder_event( job.push_metrics(metrics); false } + BackendEvent::MediaLost { message } => { + job.push_event(EventLevel::Warning, message.clone()); + job.warnings.push(AgentWarning { + code: "media_lost".into(), + message, + next: format!( + "The movie plays but has gaps. `wrec job logs {}` shows when. Record to a faster disk, close programs that load the CPU, GPU or disk, or lower --quality or --fps.", + job.id + ), + }); + false + } BackendEvent::Failed { message, .. } => { job.mark_failed(message); if active_matches { @@ -1044,6 +1056,44 @@ mod tests { wait_for_status(&state, second, JobStatus::Completed); } + #[test] + fn a_completed_job_warns_about_media_its_movie_lost() { + let _guard = env_lock(); + isolate_env(); + let state = Arc::new(Mutex::new(Coordinator::new(FakeRuntime::new()))); + let id = start_job(state.clone()).id; + wait_for_status(&state, id, JobStatus::Recording); + let message = "The movie finished but lost media: microphone is missing 0.70 s of audio."; + handle_recorder_event( + &state, + id, + RecorderEvent::MediaLost { + session_id: 100, + message: message.into(), + }, + ); + Coordinator::job_stop(state.clone(), id).unwrap(); + wait_for_status(&state, id, JobStatus::Completed); + + let job = lock_state(&state) + .unwrap() + .jobs + .get(&id) + .unwrap() + .snapshot(None); + let warning = job + .warnings + .iter() + .find(|warning| warning.code == "media_lost") + .unwrap(); + assert_eq!(warning.message, message); + assert!(warning.next.contains(&format!("wrec job logs {id}"))); + assert!(job + .events + .iter() + .any(|event| matches!(event.level, EventLevel::Warning) && event.message == message)); + } + #[test] fn queued_job_can_be_cancelled() { let _guard = env_lock(); diff --git a/crates/daemon/src/store.rs b/crates/daemon/src/store.rs index adae4a0..d933817 100644 --- a/crates/daemon/src/store.rs +++ b/crates/daemon/src/store.rs @@ -43,6 +43,7 @@ impl RecordingStatus { #[derive(Debug, Clone, Copy)] pub enum EventLevel { Info, + Warning, Error, } @@ -50,6 +51,7 @@ impl EventLevel { const fn as_str(self) -> &'static str { match self { Self::Info => "info", + Self::Warning => "warning", Self::Error => "error", } } diff --git a/crates/domain/src/lib.rs b/crates/domain/src/lib.rs index 30159c4..fbfac48 100644 --- a/crates/domain/src/lib.rs +++ b/crates/domain/src/lib.rs @@ -277,6 +277,11 @@ pub enum RecorderEvent { session_id: u64, metrics: RecorderMetrics, }, + /// The recording finished, but its movie is missing video frames or audio. + MediaLost { + session_id: u64, + message: String, + }, Failed { session_id: Option, message: String, diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index baa5ac8..30f5600 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -107,6 +107,27 @@ fn movie_mux() -> Result { Ok(mux) } +const WRITE_QUEUE: &str = "write-queue"; + +// qtmux writes each sample as it muxes it, so while a write blocks every +// track waits behind it: video drops at capture within two frames, and audio +// once its queue fills. This buffer stays empty unless the disk stalls; at +// the highest bitrate-mode quality (16 Mbit/s) it covers about 16 seconds. +const WRITE_BUFFER_BYTES: u32 = 32 << 20; + +fn movie_file(pipeline: &gst::Pipeline, mux: &gst::Element, location: &str) -> Result<()> { + let queue = element("queue")?; + queue.set_property("name", WRITE_QUEUE); + queue.set_property("max-size-bytes", WRITE_BUFFER_BYTES); + queue.set_property("max-size-buffers", 0u32); + queue.set_property("max-size-time", 0u64); + let sink = element("filesink")?; + sink.set_property("location", location); + sink.set_property("sync", false); + pipeline.add_many([&queue, &sink]).map_err(backend)?; + gst::Element::link_many([mux, &queue, &sink]).map_err(backend) +} + const CAPTURE_QUEUE: &str = "capture-queue"; // pipewiresrc resends the last frame after this long without a new one, so an @@ -346,15 +367,6 @@ fn record_attempt( parser.set_property("config-interval", -1i32); let format = movie_format(settings.codec, settings.fps.as_u32())?; let mux = movie_mux()?; - let sink = element("filesink")?; - sink.set_property( - "location", - session - .output_path - .to_str() - .ok_or_else(|| backend("recording output path must be UTF-8"))?, - ); - sink.set_property("sync", false); let mut chain = vec![&source, &input, &queue]; chain.extend(converters.iter()); chain.extend([&output, &encoder, &parser, &format]); @@ -362,9 +374,16 @@ fn record_attempt( .0 .add_many(chain.iter().copied()) .map_err(backend)?; - pipeline.0.add_many([&mux, &sink]).map_err(backend)?; + pipeline.0.add(&mux).map_err(backend)?; gst::Element::link_many(chain.iter().copied()).map_err(backend)?; - mux.link(&sink).map_err(backend)?; + movie_file( + &pipeline.0, + &mux, + session + .output_path + .to_str() + .ok_or_else(|| backend("recording output path must be UTF-8"))?, + )?; let counters = attempt.counters.clone(); let mut movie = Movie::new( &pipeline.0, @@ -402,8 +421,9 @@ fn record_attempt( let _ = events.send(RecorderEvent::Log { session_id: Some(session.id), message: format!( - "capture-engine: selected {}; capture queue limited to 2 frames", - mode.description() + "capture-engine: selected {}; capture queue limited to 2 frames; {} MiB write buffer", + mode.description(), + WRITE_BUFFER_BYTES >> 20 ), }); // Reserve a unique file so a collision never truncates an existing recording. @@ -579,9 +599,15 @@ const OMITTED: u8 = 3; struct AudioTrack { name: &'static str, + // From the source to the queue that feeds the movie. + branch: Vec, pad: gst::Pad, probe: Option, state: Arc, + // Nanoseconds of audio missing between captured buffers. + lost: Arc, + // Where in the movie the track failed, and why. + failed: Option<(gst::ClockTime, String)>, } // The recording pipeline and the tracks of its movie. qtmux writes a track @@ -590,11 +616,13 @@ struct AudioTrack { // only learn their format from their first input buffer, so audio pads are // requested once each track has delivered a buffer or ended without one. // Until then video waits in a queue instead of being dropped at capture. +// qtmux takes nothing while any track is empty, so afterwards the same queue +// lets video wait for a stalled audio track as long as audio can wait for +// video, instead of dropping frames at capture. struct Movie { pipeline: gst::Pipeline, mux: gst::Element, held: Option<(gst::Element, gst::PadProbeId)>, - draining: Option, audio: Vec, } @@ -609,7 +637,6 @@ impl Movie { pipeline: pipeline.clone(), mux: mux.clone(), held: None, - draining: None, audio: Vec::new(), }; let track = mux @@ -652,7 +679,12 @@ impl Movie { Ok(movie) } - fn add_audio(&mut self, name: &'static str, queue: &gst::Element) { + fn add_audio( + &mut self, + name: &'static str, + queue: &gst::Element, + branch: Vec, + ) -> Arc { let pad = queue.static_pad("src").unwrap(); // Linking the track sends a reconfigure upstream. The encoder answers // with an allocation query, which waits behind this queue while qtmux @@ -695,12 +727,17 @@ impl Movie { } }, ); + let lost = Arc::new(AtomicU64::new(0)); self.audio.push(AudioTrack { name, + branch, pad, probe, state, + lost: lost.clone(), + failed: None, }); + lost } // Called on every pass of the run loop. Requests the audio pads once every @@ -708,18 +745,6 @@ impl Movie { // releases the video. Returns the omitted tracks, each with whether it // ended before delivering audio. fn link_audio(&mut self) -> Result> { - if let Some(queue) = &self.draining { - // Once the video held at startup has reached qtmux, a stalled - // track holds video in qtmux and the capture queue drops frames, - // as without this queue. Limiting it earlier would block the - // encoder while the held video drains. - if queue.property::("current-level-buffers") <= 1 { - queue.set_property("max-size-time", 0u64); - queue.set_property("max-size-bytes", 0u32); - queue.set_property("max-size-buffers", 1u32); - self.draining = None; - } - } let Some((queue, _)) = &self.held else { return Ok(Vec::new()); }; @@ -756,9 +781,18 @@ impl Movie { } let (queue, probe) = self.held.take().unwrap(); queue.static_pad("src").unwrap().remove_probe(probe); - self.draining = Some(queue); Ok(omitted) } + + // The audio track whose branch posted a message, if any. + fn audio_track(&mut self, message: &gst::Message) -> Option<&mut AudioTrack> { + let src = message.src()?; + self.audio.iter_mut().find(|track| { + track.branch.iter().any(|element| { + src == element.upcast_ref::() || src.has_as_ancestor(element) + }) + }) + } } fn add_audio( @@ -812,10 +846,45 @@ fn add_audio_source( ]; movie.pipeline.add_many(chain).map_err(backend)?; gst::Element::link_many(chain).map_err(backend)?; - movie.add_audio(name, &queue); + let lost = movie.add_audio(name, &queue, chain.map(|element| element.clone()).to_vec()); + count_lost_audio(source, lost, counters.clone()); Ok(()) } +// pulsesrc keeps buffers contiguous unless it skipped audio nobody read in +// time, so a longer jump is lost audio. +const AUDIO_HOLE: gst::ClockTime = gst::ClockTime::from_mseconds(20); + +// Counts audio missing between consecutive buffers. When downstream does not +// read pulsesrc in time, its ring buffer overwrites the oldest audio, and the +// next buffer is stamped where capture actually resumed. +fn count_lost_audio(source: &gst::Element, lost: Arc, counters: Arc) { + // Where the next buffer should start, and the pause offset it assumes. + let next = Mutex::new(None::<(gst::ClockTime, gst::ClockTime)>); + source + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + let Some((pts, duration)) = info + .buffer() + .and_then(|buffer| buffer.pts().zip(buffer.duration())) + else { + return gst::PadProbeReturn::Ok; + }; + let offset = counters.timeline.lock().unwrap().offset; + let mut next = next.lock().unwrap(); + if let Some((expected, before)) = *next { + let missing = pts.saturating_sub(expected); + // A resume moves the timeline; audio dropped while paused is not lost. + if before == offset && missing > AUDIO_HOLE { + lost.fetch_add(missing.nseconds(), Ordering::Relaxed); + } + } + *next = Some((pts + duration, offset)); + gst::PadProbeReturn::Ok + }); +} + fn retime(info: &mut gst::PadProbeInfo<'_>, counters: &Counters) -> bool { if let Some(gst::PadProbeData::Buffer(buffer)) = info.data.as_mut() { return counters.timeline.lock().unwrap().retime(buffer); @@ -1094,6 +1163,14 @@ fn run( let mut started = false; let mut stopping = None; let mut last_metrics = Instant::now(); + let mut omitted = Vec::new(); + let mut logged_lost = vec![0; movie.audio.len()]; + let log = |message: String| { + let _ = events.send(RecorderEvent::Log { + session_id: Some(session.id), + message, + }); + }; loop { if let Some(message) = stopping.is_none().then(source_lost).flatten() { return Err(backend(message)); @@ -1136,6 +1213,25 @@ fn run( if let Some(message) = bus.timed_pop(gst::ClockTime::from_mseconds(100)) { match message.view() { gst::MessageView::Error(error) => { + // A failed audio source ends its own track, and the movie + // keeps recording the others. + if let Some(track) = movie.audio_track(&message) { + if track.failed.is_none() { + let position = counters + .timeline + .lock() + .unwrap() + .position(pipeline.current_running_time().unwrap_or_default()); + log(format!( + "capture-engine: {} stopped at {:.1} s: {}; its track ends there and the recording continues", + track.name, + seconds(position), + error.error() + )); + track.failed = Some((position, error.error().to_string())); + } + continue; + } if let Some(lost) = source_lost() { return Err(backend(lost)); } @@ -1154,27 +1250,33 @@ fn run( return Err(backend("capture ended without an encoded frame")); } emit_metrics(pipeline, session, events, counters); + if let Some(message) = + media_lost(movie, &omitted, dropped_frames(pipeline, counters)) + { + let _ = events.send(RecorderEvent::MediaLost { + session_id: session.id, + message, + }); + } return Ok(()); } _ => {} } } for (name, ended) in movie.link_audio()? { - let _ = events.send(RecorderEvent::Log { - session_id: Some(session.id), - message: format!( - "capture-engine: {name} sent no samples {}; the movie has no {name} track", - if ended { - "before the recording stopped".to_string() - } else { - format!( - "while {} seconds or {} MiB of video waited for it", - HELD_VIDEO_LIMIT.seconds(), - HELD_VIDEO_BYTES >> 20 - ) - } - ), - }); + log(format!( + "capture-engine: {name} sent no samples {}; the movie has no {name} track", + if ended { + "before the recording stopped".to_string() + } else { + format!( + "while {} seconds or {} MiB of video waited for it", + HELD_VIDEO_LIMIT.seconds(), + HELD_VIDEO_BYTES >> 20 + ) + } + )); + omitted.push(name); } if !started && counters.frames.load(Ordering::Relaxed) > 0 { started = true; @@ -1193,11 +1295,64 @@ fn run( } if started && last_metrics.elapsed() >= Duration::from_secs(1) { emit_metrics(pipeline, session, events, counters); + let position = counters + .timeline + .lock() + .unwrap() + .position(pipeline.current_running_time().unwrap_or_default()); + for (track, logged) in movie.audio.iter().zip(&mut logged_lost) { + let lost = track.lost.load(Ordering::Relaxed); + if lost > *logged { + log(format!( + "capture-engine: {} lost {:.2} s of audio before {:.1} s; the movie has a gap there", + track.name, + (lost - *logged) as f64 / 1e9, + seconds(position) + )); + *logged = lost; + } + } last_metrics = Instant::now(); } } } +fn seconds(time: gst::ClockTime) -> f64 { + time.nseconds() as f64 / 1e9 +} + +// What the finished movie is missing, if anything. A movie that plays is not +// necessarily whole. +fn media_lost(movie: &Movie, omitted: &[&str], dropped_frames: u64) -> Option { + let mut lost = Vec::new(); + if dropped_frames > 0 { + lost.push(format!("{dropped_frames} video frames were dropped")); + } + for track in &movie.audio { + let missing = track.lost.load(Ordering::Relaxed); + if missing > 0 { + lost.push(format!( + "{} is missing {:.2} s of audio", + track.name, + missing as f64 / 1e9 + )); + } + if let Some((position, reason)) = &track.failed { + lost.push(format!( + "{} stopped at {:.1} s ({reason}), so its track ends there", + track.name, + seconds(*position) + )); + } else if omitted.contains(&track.name) { + lost.push(format!( + "{} sent no samples, so the movie has no {} track", + track.name, track.name + )); + } + } + (!lost.is_empty()).then(|| format!("The movie finished but lost media: {}.", lost.join("; "))) +} + fn emit_metrics( pipeline: &gst::Pipeline, session: &RecordingSession, @@ -1483,25 +1638,34 @@ mod tests { fn start_with(audio_delays: &[gst::ClockTime], counting_encoder: bool) -> Self { Self::start_from( - "videotestsrc name=video is-live=true pattern=ball ! capsfilter name=capture caps=video/x-raw,width=320,height=180,framerate=30/1", - audio_delays, + MOVING_VIDEO, + audio_delays + .iter() + .map(|&delay| ("test audio", delayed_audio(delay))) + .collect(), counting_encoder, ) } + fn start_with_audio(audio: Vec<(&'static str, gst::Element)>) -> Self { + Self::start_from(MOVING_VIDEO, audio, false) + } + // An idle screen: one frame per keepalive, stamped on arrival, and // no source latency, like pipewiresrc. fn start_idle(audio_tracks: usize) -> Self { Self::start_from( "appsrc name=video is-live=true format=time do-timestamp=true caps=video/x-raw,format=I420,width=320,height=180,framerate=0/1 ! capsfilter name=capture caps=video/x-raw,format=I420,width=320,height=180,framerate=0/1", - &vec![gst::ClockTime::ZERO; audio_tracks], + (0..audio_tracks) + .map(|_| ("test audio", delayed_audio(gst::ClockTime::ZERO))) + .collect(), false, ) } fn start_from( video: &str, - audio_delays: &[gst::ClockTime], + audio: Vec<(&'static str, gst::Element)>, counting_encoder: bool, ) -> Self { gst::init().unwrap(); @@ -1531,19 +1695,17 @@ mod tests { ]) .unwrap(); let mux = movie_mux().unwrap(); - let sink = element("filesink").unwrap(); - sink.set_property("sync", false); - sink.set_property("location", session.output_path.to_str().unwrap()); - pipeline.add_many([&mux, &sink]).unwrap(); - mux.link(&sink).unwrap(); + pipeline.add(&mux).unwrap(); + movie_file(&pipeline, &mux, session.output_path.to_str().unwrap()).unwrap(); let mut movie = Movie::new( &pipeline, &pipeline.by_name("parser").unwrap(), &mux, - !audio_delays.is_empty(), + !audio.is_empty(), ) .unwrap(); let counters = Arc::new(Counters::default()); + count_dropped(&queue, counters.clone()); let video = pipeline.by_name("video").unwrap(); attach_timing_probe(&video, counters.clone()); if video @@ -1565,17 +1727,8 @@ mod tests { count_frames_like_old_va_encoders(&pipeline.by_name("video").unwrap(), &encoder); } keep_capture_timestamps(&encoder, &pipeline.by_name("parser").unwrap()); - for &delay in audio_delays { - let source = element("audiotestsrc").unwrap(); - source.set_property("is-live", true); - source.static_pad("src").unwrap().add_probe( - gst::PadProbeType::BUFFER, - move |_, info| match info.buffer().and_then(|buffer| buffer.pts()) { - Some(pts) if pts < delay => gst::PadProbeReturn::Drop, - _ => gst::PadProbeReturn::Ok, - }, - ); - add_audio_source(&mut movie, &source, "test audio", &counters).unwrap(); + for (name, source) in &audio { + add_audio_source(&mut movie, source, name, &counters).unwrap(); } attach_encoded_probe(&pipeline.by_name("parser").unwrap(), counters.clone()); let (events_tx, events) = mpsc::channel(); @@ -1644,7 +1797,8 @@ mod tests { } fn finish(&mut self) -> Result<()> { - self.stop.send(true).unwrap(); + // A recording that already failed has dropped its receiver. + let _ = self.stop.send(true); self.worker.take().unwrap().join().unwrap() } @@ -1792,6 +1946,472 @@ mod tests { } } + const MOVING_VIDEO: &str = "videotestsrc name=video is-live=true pattern=ball ! capsfilter name=capture caps=video/x-raw,width=320,height=180,framerate=30/1"; + + // Audio that starts `delay` into the recording. + fn delayed_audio(delay: gst::ClockTime) -> gst::Element { + gst::init().unwrap(); + let source = element("audiotestsrc").unwrap(); + source.set_property("is-live", true); + source + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + match info.buffer().and_then(|buffer| buffer.pts()) { + Some(pts) if pts < delay => gst::PadProbeReturn::Drop, + _ => gst::PadProbeReturn::Ok, + } + }); + source + } + + // Live capture like pulsesrc: the producer keeps real time, and when + // nobody reads its 200 ms ring buffer in time it overwrites the oldest + // audio and marks the next buffer DISCONT. A leaky queue does the same. + fn ring_buffer_audio(name: &str) -> gst::Element { + lagging_audio(name, Duration::ZERO) + } + + // Audio that reaches the pipeline `lag` after it was captured. + fn lagging_audio(name: &str, lag: Duration) -> gst::Element { + gst::init().unwrap(); + let bin = gst::parse::bin_from_description( + &format!( + "audiotestsrc name=producer is-live=true samplesperbuffer=480 ! {} queue leaky=downstream max-size-time=200000000 max-size-buffers=0 max-size-bytes=0", + if lag.is_zero() { + String::new() + } else { + // Before the ring buffer, which must stay the only place + // audio can wait. + format!("queue max-size-time=0 max-size-buffers=0 max-size-bytes=0 min-threshold-time={} !", lag.as_nanos()) + } + ), + true, + ) + .unwrap(); + bin.set_property("name", name); + bin.upcast() + } + + // Blocks the pad's streaming thread once, `after` from now, for `stall`, + // like a write() or an encoder call that does not return. + fn stall_once(pad: &gst::Pad, after: Duration, stall: Duration) { + let start = Instant::now(); + let done = std::sync::atomic::AtomicBool::new(false); + pad.add_probe(gst::PadProbeType::BUFFER, move |_, _| { + if start.elapsed() >= after && !done.swap(true, Ordering::SeqCst) { + std::thread::sleep(stall); + } + gst::PadProbeReturn::Ok + }); + } + + // Seconds missing from each audio track. qtmux stretches the packet + // before a hole over it, so a hole is packet spacing beyond one AAC frame. + fn audio_holes(probe: &serde_json::Value) -> Vec { + let packets = probe["packets"].as_array().unwrap(); + probe["streams"] + .as_array() + .unwrap() + .iter() + .filter(|stream| stream["codec_type"] == "audio") + .map(|stream| { + let mut starts: Vec = packets + .iter() + .filter(|packet| packet["stream_index"] == stream["index"]) + .map(|packet| packet["pts_time"].as_str().unwrap().parse().unwrap()) + .collect(); + starts.sort_by(f64::total_cmp); + starts + .windows(2) + .map(|pair| pair[1] - pair[0] - 1024.0 / 48000.0) + .filter(|hole| *hole > 0.03) + .sum::() + .max(0.0) + }) + .collect() + } + + // Seconds of 30 fps video missing between frames. + fn missing_video(probe: &serde_json::Value) -> f64 { + let index = probe["streams"] + .as_array() + .unwrap() + .iter() + .find(|stream| stream["codec_type"] == "video") + .unwrap()["index"] + .clone(); + let mut pts: Vec = probe["packets"] + .as_array() + .unwrap() + .iter() + .filter(|packet| packet["stream_index"] == index) + .map(|packet| packet["pts_time"].as_str().unwrap().parse().unwrap()) + .collect(); + pts.sort_by(f64::total_cmp); + pts.windows(2) + .map(|pair| pair[1] - pair[0] - 1.0 / 30.0) + .filter(|missing| *missing > 0.02) + .sum() + } + + fn pipeline_element(pipeline: &gst::Pipeline, factory: &str) -> Vec { + pipeline + .iterate_recurse() + .into_iter() + .flatten() + .filter(|element| { + element + .factory() + .is_some_and(|found| found.name() == factory) + }) + .collect() + } + + #[derive(Clone, Copy, Debug)] + enum Stall { + // write() into the movie file blocks. + Write, + // The video encoder takes no frame. + Encoder, + // The microphone captures nothing. + AudioProducer, + // The microphone's server goes away, like pulsesrc after a + // PipeWire or PulseAudio restart. + AudioFailure, + } + + struct StallOutcome { + result: Result<()>, + // Seconds missing from system audio and the microphone. + holes: Vec, + // Where each audio track ends. + audio_ends: Vec, + video_end: f64, + missing_video: f64, + dropped: u64, + logs: Vec, + media_lost: Option, + } + + impl StallOutcome { + // The loss the recording reported for a track, in seconds. + fn reported(&self, track: &str) -> f64 { + let message = self.media_lost.as_deref().unwrap_or_default(); + message + .split_once(&format!("{track} is missing ")) + .and_then(|(_, rest)| rest.split_once(" s")?.0.parse().ok()) + .unwrap_or(0.0) + } + } + + // System audio and a microphone, a stall two seconds in, then four more + // seconds. `adjust` changes the pipeline before the stall. + fn record_through( + stall: Stall, + seconds: f64, + adjust: impl FnOnce(&gst::Pipeline), + ) -> StallOutcome { + let mut recording = TestRecording::start_with_audio(vec![ + ("system audio", ring_buffer_audio("system-audio")), + ("microphone", ring_buffer_audio("microphone")), + ]); + let pipeline = &recording.pipeline; + adjust(pipeline); + let (after, length) = (Duration::from_secs(2), Duration::from_secs_f64(seconds)); + let microphone = pipeline + .by_name("microphone") + .unwrap() + .downcast::() + .unwrap() + .by_name("producer") + .unwrap(); + match stall { + Stall::Write => { + let sink = &pipeline_element(pipeline, "filesink")[0]; + stall_once(&sink.static_pad("sink").unwrap(), after, length); + } + Stall::Encoder => { + let encoder = pipeline.by_name("encoder").unwrap(); + stall_once(&encoder.static_pad("sink").unwrap(), after, length); + } + Stall::AudioProducer => { + let start = Instant::now(); + let silent = after..after + length; + microphone.static_pad("src").unwrap().add_probe( + gst::PadProbeType::BUFFER, + move |_, _| { + if silent.contains(&start.elapsed()) { + gst::PadProbeReturn::Drop + } else { + gst::PadProbeReturn::Ok + } + }, + ); + } + Stall::AudioFailure => { + let start = Instant::now(); + microphone.static_pad("src").unwrap().add_probe( + gst::PadProbeType::BUFFER, + move |pad, info| { + if start.elapsed() < after { + return gst::PadProbeReturn::Ok; + } + reject_capture( + pad, + info, + gst::StreamError::Failed, + "Disconnected: Connection terminated", + ) + }, + ); + } + } + std::thread::sleep(after + length + Duration::from_secs(4)); + let result = recording.finish(); + let probe = recording.probe(); + let dropped = dropped_frames(&recording.pipeline, &recording.counters); + let (mut logs, mut media_lost) = (Vec::new(), None); + for event in recording.events.try_iter() { + match event { + RecorderEvent::Log { message, .. } => logs.push(message), + RecorderEvent::MediaLost { message, .. } => media_lost = Some(message), + _ => {} + } + } + StallOutcome { + result, + holes: audio_holes(&probe), + audio_ends: probe["streams"] + .as_array() + .unwrap() + .iter() + .filter(|stream| stream["codec_type"] == "audio") + .map(|stream| { + let seconds = |key: &str| stream[key].as_str().unwrap().parse::().unwrap(); + seconds("start_time") + seconds("duration") + }) + .collect(), + video_end: end_seconds(&probe, "h264"), + missing_video: missing_video(&probe), + dropped, + logs, + media_lost, + } + } + + #[test] + fn a_stalled_file_write_loses_nothing_while_the_write_buffer_lasts() { + let outcome = record_through(Stall::Write, 6.0, |_| {}); + outcome.result.as_ref().unwrap(); + assert_eq!(outcome.holes, [0.0, 0.0], "audio lost to the write stall"); + assert!( + outcome.missing_video < 0.05 && outcome.dropped == 0, + "{}s of video, {} frames lost to the write stall", + outcome.missing_video, + outcome.dropped + ); + assert_eq!(outcome.media_lost, None); + } + + // Past the write buffer every track waits for the disk, as without it. + // The loss is bounded by the stall and reported as it was recorded. + #[test] + fn a_write_stall_longer_than_the_write_buffer_reports_what_it_cost() { + let outcome = record_through(Stall::Write, 6.0, |pipeline| { + pipeline + .by_name(WRITE_QUEUE) + .unwrap() + .set_property("max-size-bytes", 16u32 << 10); + }); + outcome.result.as_ref().unwrap(); + assert_reports_its_losses(&outcome); + // Audio and video each wait about four seconds for the disk. + for hole in &outcome.holes { + assert!((1.0..2.5).contains(hole), "{:?}", outcome.holes); + } + assert!( + (1.0..2.5).contains(&outcome.missing_video), + "{}", + outcome.missing_video + ); + } + + #[test] + fn an_encoder_stall_loses_its_video_and_reports_the_audio_it_cost() { + let outcome = record_through(Stall::Encoder, 6.0, |_| {}); + outcome.result.as_ref().unwrap(); + assert_reports_its_losses(&outcome); + // Audio waits as long as its queue lasts, then its source overwrites it. + for hole in &outcome.holes { + assert!((1.0..2.5).contains(hole), "{:?}", outcome.holes); + } + assert!( + (5.5..6.5).contains(&outcome.missing_video), + "{}", + outcome.missing_video + ); + } + + #[test] + fn a_stalled_audio_source_costs_only_its_own_audio() { + let outcome = record_through(Stall::AudioProducer, 2.0, |_| {}); + outcome.result.as_ref().unwrap(); + assert_reports_its_losses(&outcome); + assert_eq!( + outcome.holes[0], 0.0, + "system audio waited for the microphone" + ); + assert!( + (1.9..2.1).contains(&outcome.holes[1]), + "{:?}", + outcome.holes + ); + assert!( + outcome.missing_video < 0.05 && outcome.dropped == 0, + "{}s of video, {} frames lost waiting for the microphone", + outcome.missing_video, + outcome.dropped + ); + } + + // Video waits for audio only as long as audio waits for video, then + // drops at capture instead of building up encoded frames. + #[test] + fn an_audio_source_stalled_past_the_video_limit_reports_the_video_it_cost() { + let most = Arc::new(AtomicU64::new(0)); + let level = most.clone(); + let outcome = record_through(Stall::AudioProducer, 6.0, |pipeline| { + let held = pipeline.by_name(HELD_QUEUE).unwrap(); + held.connect("overrun", false, move |values| { + let queue = values[0].get::().unwrap(); + let frames = queue.property::("current-level-buffers"); + level.fetch_max(frames.into(), Ordering::SeqCst); + None + }); + }); + outcome.result.as_ref().unwrap(); + // The queue's time level spans the frames dropped meanwhile, so count + // the frames it holds: the limit's worth at 30 fps. + let limit = (HELD_VIDEO_LIMIT + gst::ClockTime::SECOND).seconds() * 30; + let most = most.load(Ordering::SeqCst); + assert!( + (limit..=limit + 1).contains(&most), + "{most} frames waited for the microphone" + ); + assert_reports_its_losses(&outcome); + assert!( + (5.9..6.1).contains(&outcome.holes[1]), + "{:?}", + outcome.holes + ); + assert!( + (1.5..2.5).contains(&outcome.missing_video), + "{}", + outcome.missing_video + ); + } + + #[test] + fn a_failed_audio_source_ends_its_track_and_the_recording_continues() { + let outcome = record_through(Stall::AudioFailure, 0.0, |_| {}); + outcome.result.as_ref().unwrap(); + assert!( + outcome.missing_video < 0.05 && outcome.dropped == 0, + "{}s of video, {} frames lost", + outcome.missing_video, + outcome.dropped + ); + assert!((outcome.audio_ends[0] - outcome.video_end).abs() < 0.1); + assert!( + (1.8..2.3).contains(&outcome.audio_ends[1]), + "the microphone ends at {}s", + outcome.audio_ends[1] + ); + let message = outcome.media_lost.unwrap(); + assert!( + message.contains("microphone stopped at 2.") + && message.contains("Disconnected: Connection terminated") + && message.contains("so its track ends there"), + "{message}" + ); + assert!(!message.contains("system audio"), "{message}"); + assert!( + outcome + .logs + .iter() + .any(|log| log.contains("microphone stopped at")), + "{:?}", + outcome.logs + ); + } + + // Audio captured before a pause but arriving after it is dropped by + // design. A source that delivers late makes that drop longer than a + // lost-audio hole. + #[test] + fn audio_dropped_by_pausing_is_not_reported_lost() { + let mut recording = TestRecording::start_with_audio(vec![( + "microphone", + lagging_audio("microphone", Duration::from_millis(100)), + )]); + for _ in 0..3 { + std::thread::sleep(Duration::from_millis(700)); + recording.control(true); + std::thread::sleep(Duration::from_millis(450)); + recording.control(false); + } + std::thread::sleep(Duration::from_millis(1200)); + recording.finish().unwrap(); + recording.probe(); + for event in recording.events.try_iter() { + match event { + RecorderEvent::MediaLost { message, .. } => panic!("{message}"), + RecorderEvent::Log { message, .. } => { + assert!(!message.contains(" lost "), "{message}") + } + _ => {} + } + } + } + + // What the recording reported matches what its movie is missing. + fn assert_reports_its_losses(outcome: &StallOutcome) { + let message = outcome + .media_lost + .as_deref() + .expect("no media loss reported"); + assert_eq!( + outcome.dropped > 0, + message.contains(&format!("{} video frames were dropped", outcome.dropped)), + "{message}" + ); + // Dropped frames account for the missing video, give or take a + // frame at either edge of each gap. + assert!( + (outcome.dropped as f64 / 30.0 - outcome.missing_video).abs() < 0.15, + "{} dropped frames, {}s of video missing", + outcome.dropped, + outcome.missing_video + ); + for (track, hole) in ["system audio", "microphone"].iter().zip(&outcome.holes) { + let reported = outcome.reported(track); + assert!( + (reported - hole).abs() < 0.05, + "{track}: reported {reported}s lost, movie misses {hole}s: {message}" + ); + assert_eq!( + *hole > 0.0, + outcome + .logs + .iter() + .any(|log| log.starts_with(&format!("capture-engine: {track} lost "))), + "{track}: {:?}", + outcome.logs + ); + } + } + fn start_seconds(probe: &serde_json::Value, codec: &str) -> Vec { probe["streams"] .as_array() @@ -2153,47 +2773,6 @@ mod tests { ); } - #[test] - fn a_stalled_audio_track_does_not_build_up_encoded_video() { - let mut recording = TestRecording::start(1); - let stall = gst::ClockTime::from_mseconds(1000)..gst::ClockTime::from_mseconds(2500); - recording - .pipeline - .iterate_elements() - .into_iter() - .filter_map(|element| element.ok()) - .find(|element| { - element - .factory() - .is_some_and(|f| f.name() == "audiotestsrc") - }) - .unwrap() - .static_pad("src") - .unwrap() - .add_probe(gst::PadProbeType::BUFFER, move |_, info| { - match info.buffer().and_then(|buffer| buffer.pts()) { - Some(pts) if stall.contains(&pts) => gst::PadProbeReturn::Drop, - _ => gst::PadProbeReturn::Ok, - } - }); - let held = recording.pipeline.by_name(HELD_QUEUE).unwrap(); - let mut most = 0; - loop { - let now = recording.pipeline.current_running_time().unwrap(); - if now >= gst::ClockTime::from_mseconds(3000) { - break; - } - // From the stall on; the video held at startup drains before it. - if now >= gst::ClockTime::from_mseconds(1100) { - most = most.max(held.property::("current-level-buffers")); - } - std::thread::sleep(Duration::from_millis(10)); - } - recording.finish().unwrap(); - recording.probe(); - assert!(most <= 1, "{most} encoded frames waited for stalled audio"); - } - #[test] fn held_video_is_bounded_in_bytes_as_well_as_time() { gst::init().unwrap(); @@ -2216,7 +2795,7 @@ mod tests { pipeline.0.add_many([&video, &mux, &sink, &silent]).unwrap(); mux.link(&sink).unwrap(); let mut movie = Movie::new(&pipeline.0, &video, &mux, true).unwrap(); - movie.add_audio("test audio", &silent); + movie.add_audio("test audio", &silent, vec![silent.clone()]); let held = pipeline.0.by_name(HELD_QUEUE).unwrap(); pipeline.0.set_state(gst::State::Playing).unwrap(); // Large constant-QP frames 10 ms apart reach the byte limit long @@ -2329,6 +2908,16 @@ mod tests { (duration - active_duration).abs() < 0.2, "movie duration {duration}, active timeline {active_duration}" ); + // Audio dropped while paused is not lost media. + for event in recording.events.try_iter() { + match event { + RecorderEvent::MediaLost { message, .. } => panic!("{message}"), + RecorderEvent::Log { message, .. } => { + assert!(!message.contains(" lost "), "{message}") + } + _ => {} + } + } } // An idle screen sends a frame each keepalive. A short pause that swallows @@ -2343,7 +2932,7 @@ mod tests { let audio = queue .factory() .is_some_and(|factory| factory.name() == "queue") - && ![CAPTURE_QUEUE, HELD_QUEUE].contains(&queue.name().as_str()); + && ![CAPTURE_QUEUE, HELD_QUEUE, WRITE_QUEUE].contains(&queue.name().as_str()); if audio { let full = full.clone(); queue.connect("overrun", false, move |_| { diff --git a/packaging/linux/README.md b/packaging/linux/README.md index d44c901..cc29d53 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -171,7 +171,7 @@ kernel kills the worker with SIGKILL as soon as the daemon process exits for any reason, so a worker never outlives its daemon. If the daemon is killed or crashes during a recording, the movie is not finalized. At most, it is playable through the last completed fragment that reached the disk. Fragments still buffered by -the file writer are lost. In one test, killing the daemon 12 seconds into a +the file writer are lost, which during a disk stall can be up to 32 MiB. In one test, killing the daemon 12 seconds into a static-desktop recording left an empty file. While recording, resource use is the daemon plus its worker. Measuring the @@ -273,6 +273,22 @@ available. Configure devices in desktop sound settings. Linux cannot apply the shared wrec window-hiding or custom microphone-indicator options; job settings report them disabled with a warning. The desktop controls its sharing indicator. +The movie writer takes the next sample of every track in time order, so a track +that stops delivering holds up the others. Encoded video and each audio track +can wait about 4 seconds for the rest. Encoded movie data can wait for the disk +in a 32 MiB buffer, about 16 seconds at 16 Mbit/s. These buffers stay nearly +empty unless something stalls. Past those limits the recording keeps going with +gaps: the capture queue drops video frames, and an audio source overwrites audio +that nothing read in time. A stalled video encoder always costs the frames +captured meanwhile, since raw frames are too large to keep. + +If an audio source fails, for example because PipeWire or PulseAudio restarts, +its track ends at that point and video and the other track keep recording. A +completed job is not proof of a whole movie. When frames were dropped, audio has +gaps, or a track ended early or never started, the job finishes `completed` with +a `media_lost` warning that states what is missing, and the job events record when +each audio gap happened. `wrec record` prints the warning when it finishes. + On Wayland, a recording watches four things besides its frames: the ScreenCast session's `Closed` signal, the D-Bus owner of `org.freedesktop.portal.Desktop`, the PipeWire remote socket the portal returned, and the selected PipeWire node. From 76407402170b831789068f3ff1f30ca77a0e5b70 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Mon, 5 Oct 2026 03:12:03 +0000 Subject: [PATCH 26/45] fix(linux): let software H.264 use its quality's bitrate x264enc in quality mode caps its rate at its bitrate property, which wrec never set, so every quality was limited to the 2 Mbit/s default. Three seconds of 720p noise at High came out at 2.3 Mbit/s. Set the same per-quality bitrate the other encoders use. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/linux/src/encoding.rs | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/crates/linux/src/encoding.rs b/crates/linux/src/encoding.rs index ba4e19b..a2802cb 100644 --- a/crates/linux/src/encoding.rs +++ b/crates/linux/src/encoding.rs @@ -200,6 +200,8 @@ impl Mode { encoder.set_property_from_str("tune", "zerolatency"); encoder.set_property_from_str("pass", "qual"); encoder.set_property("quantizer", qp); + // In quality mode this caps the rate, 2 Mbit/s by default. + encoder.set_property("bitrate", bitrate); encoder.set_property("threads", 2u32); encoder.set_property("key-int-max", settings.fps.as_u32() * 2); } @@ -316,4 +318,61 @@ mod tests { } assert!(tested.contains(&"x265enc"), "tested only {tested:?}"); } + + // Megabits per second for three seconds of 720p30 noise, which wants + // more bits than any quality allows. + fn noise_bitrate(mode: Mode, quality: Quality) -> f64 { + let pipeline = gst::parse::launch( + "videotestsrc name=source pattern=snow num-buffers=90 ! video/x-raw,format=I420,width=1280,height=720,framerate=30/1 ! identity name=input", + ) + .unwrap() + .downcast::() + .unwrap(); + let settings = RecorderSettings { + quality, + ..RecorderSettings::default() + }; + let encoder = mode.encoder(&settings).unwrap(); + let sink = element("fakesink").unwrap(); + pipeline.add_many([&encoder, &sink]).unwrap(); + gst::Element::link_many([&pipeline.by_name("input").unwrap(), &encoder, &sink]).unwrap(); + let bytes = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); + let counted = bytes.clone(); + encoder + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + let size = info.buffer().map_or(0, |buffer| buffer.size()); + counted.fetch_add(size as u64, std::sync::atomic::Ordering::Relaxed); + gst::PadProbeReturn::Ok + }); + pipeline.set_state(gst::State::Playing).unwrap(); + let ended = pipeline.bus().unwrap().timed_pop_filtered( + gst::ClockTime::from_seconds(60), + &[gst::MessageType::Eos, gst::MessageType::Error], + ); + pipeline.set_state(gst::State::Null).unwrap(); + assert!(matches!( + ended.map(|message| message.type_()), + Some(gst::MessageType::Eos) + )); + bytes.load(std::sync::atomic::Ordering::Relaxed) as f64 * 8.0 / 3.0 / 1e6 + } + + // x264 in quality mode caps its rate at its bitrate property, 2 Mbit/s by + // default, so every quality must set it. + #[test] + fn software_h264_spends_the_bitrate_its_quality_allows() { + gst::init().unwrap(); + let Some(mode) = modes(Codec::H264, false) + .into_iter() + .find(|mode| mode.factory == "x264enc") + else { + return; + }; + let high = noise_bitrate(mode, Quality::High); + assert!((12.0..20.0).contains(&high), "high quality: {high} Mbit/s"); + let efficient = noise_bitrate(mode, Quality::Efficient); + assert!(efficient < 3.2, "efficient quality: {efficient} Mbit/s"); + } } From 27d782e0774e09b68f2855987b8f62c6f898be66 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Mon, 5 Oct 2026 03:20:57 +0000 Subject: [PATCH 27/45] fix(linux): return memory after a stall drains its buffers glibc keeps memory freed after the write buffer or the video cushion drains. After a 40 s write stall at 17 Mbit/s, the worker stayed at 161 MiB PSS for the rest of the recording instead of about 112 MiB. Once more than 8 MiB was buffered and the buffers are nearly empty again, trim the allocator; PSS returned to 123 MiB in the same scenario. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/linux/src/pipeline.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 30f5600..4cb8ab9 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -1165,6 +1165,7 @@ fn run( let mut last_metrics = Instant::now(); let mut omitted = Vec::new(); let mut logged_lost = vec![0; movie.audio.len()]; + let mut most_buffered = 0; let log = |message: String| { let _ = events.send(RecorderEvent::Log { session_id: Some(session.id), @@ -1312,11 +1313,31 @@ fn run( *logged = lost; } } + // The allocator keeps memory freed after a stall drains its + // buffers until it is asked to return it. + let buffered = buffered_bytes(pipeline); + if buffered < STALL_DRAINED && most_buffered >= STALL_BUFFERED { + release_freed_memory(); + most_buffered = 0; + } + most_buffered = most_buffered.max(buffered); last_metrics = Instant::now(); } } } +const STALL_BUFFERED: u64 = 8 << 20; +const STALL_DRAINED: u64 = 1 << 20; + +// Encoded media waiting for the disk or for a stalled track. +fn buffered_bytes(pipeline: &gst::Pipeline) -> u64 { + [WRITE_QUEUE, HELD_QUEUE] + .into_iter() + .filter_map(|name| pipeline.by_name(name)) + .map(|queue| u64::from(queue.property::("current-level-bytes"))) + .sum() +} + fn seconds(time: gst::ClockTime) -> f64 { time.nseconds() as f64 / 1e9 } From 230be3c1b2f8a436c07a818b043dc0d315741429 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Mon, 5 Oct 2026 03:32:43 +0000 Subject: [PATCH 28/45] fix(linux): finalize the movie when video encoding hangs at stop An encoder that hung before stop kept EOS from reaching qtmux, so the movie was never finalized. With the hang two seconds in, nothing had reached a completed fragment and the whole file was unreadable. Sending EOS could also block the run loop behind a source pushing into a full queue, so the 10 s finalization limit never fired. - Stop sends EOS from a helper thread. - At the 10 s limit, if video has not reached qtmux while nothing waits for the disk or for audio, end the video track at qtmux. The movie is finalized with the video it has, and MediaLost says the video ends early. Video waiting downstream is never cut. - The worker reports a finalized movie before native teardown, and a cleanup that then hangs or crashes completes the job instead of failing it. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/linux/src/pipeline.rs | 164 +++++++++++++++++++++++++++++++---- crates/linux/src/worker.rs | 36 ++++++++ packaging/linux/README.md | 10 ++- 3 files changed, 193 insertions(+), 17 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 4cb8ab9..81b95d4 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -213,6 +213,8 @@ pub(crate) enum CaptureInput { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Teardown { Recording, + // The movie is complete on disk; native cleanup follows. + Finalized, AttemptStarted, AttemptFinished, } @@ -446,7 +448,9 @@ fn record_attempt( }, ); let ending = ends_recording(&result, &counters, attempt.last); - (attempt.stopping)(if ending { + (attempt.stopping)(if result.is_ok() { + Teardown::Finalized + } else if ending { Teardown::Recording } else { Teardown::AttemptStarted @@ -622,6 +626,7 @@ struct AudioTrack { struct Movie { pipeline: gst::Pipeline, mux: gst::Element, + video: gst::Pad, held: Option<(gst::Element, gst::PadProbeId)>, audio: Vec, } @@ -633,15 +638,16 @@ impl Movie { mux: &gst::Element, audio: bool, ) -> Result { + let track = mux + .request_pad_simple("video_%u") + .ok_or_else(|| backend("the movie muxer refused a video track"))?; let mut movie = Self { pipeline: pipeline.clone(), mux: mux.clone(), + video: track.clone(), held: None, audio: Vec::new(), }; - let track = mux - .request_pad_simple("video_%u") - .ok_or_else(|| backend("the movie muxer refused a video track"))?; if !audio { video .static_pad("src") @@ -1166,6 +1172,7 @@ fn run( let mut omitted = Vec::new(); let mut logged_lost = vec![0; movie.audio.len()]; let mut most_buffered = 0; + let mut video_cut = false; let log = |message: String| { let _ = events.send(RecorderEvent::Log { session_id: Some(session.id), @@ -1187,9 +1194,10 @@ fn run( .lock() .unwrap() .resume(pipeline.current_running_time().unwrap_or_default()); - if !pipeline.send_event(gst::event::Eos::new()) { - return Err(backend("recording pipeline rejected finalization")); - } + // A source pushing into a full queue holds its stream lock, and + // sending it EOS waits for that lock, so wait here instead. + let ending = pipeline.clone(); + std::thread::spawn(move || ending.send_event(gst::event::Eos::new())); stopping = Some(Instant::now()); } if let Ok(command) = commands.try_recv() { @@ -1251,9 +1259,12 @@ fn run( return Err(backend("capture ended without an encoded frame")); } emit_metrics(pipeline, session, events, counters); - if let Some(message) = - media_lost(movie, &omitted, dropped_frames(pipeline, counters)) - { + if let Some(message) = media_lost( + movie, + &omitted, + dropped_frames(pipeline, counters), + video_cut, + ) { let _ = events.send(RecorderEvent::MediaLost { session_id: session.id, message, @@ -1289,10 +1300,28 @@ fn run( if !started && launched.elapsed() > Duration::from_secs(15) { return Err(backend("No encoded frame arrived within 15s. Check DMA-BUF support, VA-API driver access, and audio devices.")); } - if stopping.is_some_and(|at| at.elapsed() > Duration::from_secs(10)) { - return Err(backend( - "Movie finalization timed out after 10s; only completed fragments may be playable.", - )); + if stopping.is_some_and(|at| at.elapsed() > FINALIZING) { + // Nothing is waiting for the disk or for audio, yet video has + // not reached the muxer: the encoder or capture is stuck. End + // the video track there, so the muxer can finish the movie with + // what it has. + if !video_cut + && !movie.video.pad_flags().contains(gst::PadFlags::EOS) + && downstream_idle(pipeline) + { + video_cut = true; + log(format!( + "capture-engine: video encoding did not finish within {} s of stop; ending the video track so the movie can be finalized", + FINALIZING.as_secs() + )); + let video = movie.video.clone(); + std::thread::spawn(move || video.send_event(gst::event::Eos::new())); + } + if !video_cut || stopping.is_some_and(|at| at.elapsed() > FINALIZING * 3 / 2) { + return Err(backend( + "Movie finalization timed out after 10s; only completed fragments may be playable.", + )); + } } if started && last_metrics.elapsed() >= Duration::from_secs(1) { emit_metrics(pipeline, session, events, counters); @@ -1326,6 +1355,18 @@ fn run( } } +// How long finalization may take before a stuck video track is ended at the +// muxer, which then has half as long again. +const FINALIZING: Duration = Duration::from_secs(10); + +// Nothing waits for the disk, and no video waits for audio. +fn downstream_idle(pipeline: &gst::Pipeline) -> bool { + buffered_bytes(pipeline) < STALL_DRAINED + && pipeline.by_name(HELD_QUEUE).map_or(true, |queue| { + queue.property::("current-level-buffers") == 0 + }) +} + const STALL_BUFFERED: u64 = 8 << 20; const STALL_DRAINED: u64 = 1 << 20; @@ -1344,8 +1385,16 @@ fn seconds(time: gst::ClockTime) -> f64 { // What the finished movie is missing, if anything. A movie that plays is not // necessarily whole. -fn media_lost(movie: &Movie, omitted: &[&str], dropped_frames: u64) -> Option { +fn media_lost( + movie: &Movie, + omitted: &[&str], + dropped_frames: u64, + video_cut: bool, +) -> Option { let mut lost = Vec::new(); + if video_cut { + lost.push("video encoding stopped responding, so the video ends early".to_string()); + } if dropped_frames > 0 { lost.push(format!("{dropped_frames} video frames were dropped")); } @@ -2396,6 +2445,91 @@ mod tests { } } + // Stops a recording `after` something in it got stuck at two seconds. + // Returns the finished recording, its result and how long after stop + // run() returned. Like pulsesrc, the audio sources' streaming threads + // block on full queues. + fn stop_while_stuck( + after: Duration, + stuck: impl FnOnce(&gst::Pipeline), + ) -> (TestRecording, Result<()>, Duration) { + let mut recording = TestRecording::start_with_audio(vec![ + ("system audio", delayed_audio(gst::ClockTime::ZERO)), + ("microphone", delayed_audio(gst::ClockTime::ZERO)), + ]); + stuck(&recording.pipeline); + std::thread::sleep(Duration::from_secs(2) + after); + let stopped = Instant::now(); + let result = recording.finish(); + let returned = recording.returned.lock().unwrap().unwrap() - stopped; + (recording, result, returned) + } + + // Stopped once the audio queues are full, so the audio sources are + // blocked, holding the stream locks that EOS needs. + #[test] + fn a_stuck_video_encoder_at_stop_still_finalizes_the_movie() { + // Stuck past finalization; teardown waits for it to return. + let (recording, result, returned) = stop_while_stuck(Duration::from_secs(6), |pipeline| { + let encoder = pipeline.by_name("encoder").unwrap(); + stall_once( + &encoder.static_pad("sink").unwrap(), + Duration::from_secs(2), + Duration::from_secs(20), + ); + }); + result.unwrap(); + assert!( + (FINALIZING..FINALIZING + Duration::from_secs(2)).contains(&returned), + "finalized {returned:?} after stop" + ); + let probe = recording.probe(); + let video = end_seconds(&probe, "h264"); + assert!((1.8..2.4).contains(&video), "video ends at {video}s"); + // Audio queued before its sources blocked made it into the movie. + let audio = end_seconds(&probe, "aac"); + assert!(audio > 5.9, "audio ends at {audio}s"); + let (mut ending, mut lost) = (false, None); + for event in recording.events.try_iter() { + match event { + RecorderEvent::Log { message, .. } => { + ending |= message.contains("ending the video track") + } + RecorderEvent::MediaLost { message, .. } => lost = Some(message), + _ => {} + } + } + assert!(ending); + assert!(lost.is_some_and(|message| message.contains("the video ends early")),); + } + + // Video waiting for the disk is not stuck, so it is never cut. + #[test] + fn a_disk_stall_at_stop_never_cuts_the_waiting_video() { + let (recording, result, _) = stop_while_stuck(Duration::from_secs(1), |pipeline| { + pipeline + .by_name(WRITE_QUEUE) + .unwrap() + .set_property("max-size-bytes", 16u32 << 10); + let sink = &pipeline_element(pipeline, "filesink")[0]; + stall_once( + &sink.static_pad("sink").unwrap(), + Duration::from_secs(2), + Duration::from_secs(14), + ); + }); + let error = result.unwrap_err(); + assert!( + error.to_string().contains("finalization timed out"), + "{error}" + ); + for event in recording.events.try_iter() { + if let RecorderEvent::Log { message, .. } = event { + assert!(!message.contains("ending the video track"), "{message}"); + } + } + } + // What the recording reported matches what its movie is missing. fn assert_reports_its_losses(outcome: &StallOutcome) { let message = outcome diff --git a/crates/linux/src/worker.rs b/crates/linux/src/worker.rs index 2c50d17..8a9eab6 100644 --- a/crates/linux/src/worker.rs +++ b/crates/linux/src/worker.rs @@ -45,6 +45,8 @@ enum Message { Event(RecorderEvent), Reply(Result<()>), Stopping, + // The movie is complete; the worker is cleaning up native resources. + Finalized, CleaningAttempt, CleanedAttempt, } @@ -225,6 +227,7 @@ impl Supervisor { return; } let mut terminal = None; + let mut finalized = false; let mut started = false; let mut problem = None; let mut output_open = true; @@ -258,6 +261,10 @@ impl Supervisor { Ok(Ok(Message::Stopping)) => { exit_by.get_or_insert(Instant::now() + self.deadline); } + Ok(Ok(Message::Finalized)) => { + finalized = true; + exit_by.get_or_insert(Instant::now() + self.deadline); + } Ok(Ok(Message::CleaningAttempt)) => { cleanup_by.get_or_insert(Instant::now() + self.deadline); } @@ -313,6 +320,20 @@ impl Supervisor { } } drop(self.replies); + // The movie was complete before native cleanup hung or crashed. + if terminal.is_none() && finalized { + let problem = problem + .take() + .unwrap_or_else(|| "capture worker exited during cleanup".to_string()); + terminal = Some(RecorderEvent::Exited { + session_id: self.session.id, + success: true, + status: format!( + "recording finalized, but the {}", + problem.trim_start_matches("the ") + ), + }); + } let event = match (terminal, problem) { (Some(event), None) => event, (_, problem) => { @@ -469,6 +490,7 @@ pub fn run() -> ! { &stopping_output, &match teardown { crate::pipeline::Teardown::Recording => Message::Stopping, + crate::pipeline::Teardown::Finalized => Message::Finalized, crate::pipeline::Teardown::AttemptStarted => Message::CleaningAttempt, crate::pipeline::Teardown::AttemptFinished => Message::CleanedAttempt, }, @@ -860,6 +882,20 @@ mod tests { )); } + #[test] + fn a_finalized_movie_survives_a_hung_cleanup() { + let fake = Fake::start(r#"read start; echo '"Finalized"'; exec sleep 30"#); + let terminal = fake.terminal(); + assert!( + matches!( + &terminal, + RecorderEvent::Exited { success: true, status, .. } + if status.starts_with("recording finalized, but the capture worker did not exit within 1s") + ), + "{terminal:?}" + ); + } + #[test] fn exiting_worker_is_finished_when_its_final_event_arrives() { let fake = Fake::start( diff --git a/packaging/linux/README.md b/packaging/linux/README.md index cc29d53..bf90205 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -161,7 +161,8 @@ hangs cannot hold the job open. Recording, idle and paused time never start the deadline. Cleaning up a failed encoder attempt uses a separate deadline. The job's final status is published only after the worker has been reaped, so the next queued job never starts while the previous worker still runs. A worker killed this way before it -reported a result fails its job. While a worker is frozen, pause and resume fail +reported a result fails its job, unless it had already reported a finalized +movie; then the job completes and its status says that cleanup did not finish. While a worker is frozen, pause and resume fail after the usual 5 seconds and the recording keeps its current state. The worker runs in its own process group, so Ctrl-C in a terminal reaches the @@ -314,7 +315,12 @@ without the watch. If a desktop removes the node before its `Closed` signal arrives, a stop from its sharing indicator fails the job with the stream-ended message instead of finalizing it. -Stop drains the encoders and finalizes the movie, including when paused. The +Stop drains the encoders and finalizes the movie, including when paused. If +video still has not reached the movie writer 10 seconds after stop while nothing +waits for the disk or for audio, the video encoder is stuck. wrec then ends the +video track at the writer and finalizes the movie with the video it already has, +and the job reports the shorter video as lost media. A disk that stalls for more +than 10 seconds during finalization still fails the job. The writer requests ten-second movie fragments; encoder keyframes can close them earlier. An interrupted recording may only be playable through its last completed fragment. Audio sources can deliver their From e8e70ebc7d4eade38aff76b48e6a9d608a8faf1f Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Mon, 5 Oct 2026 04:29:51 +0000 Subject: [PATCH 29/45] fix(linux): address review of stall handling From an independent review of the stall and finalization changes: - Only an error from an audio source ends its track, and wrec sends EOS into that branch itself instead of relying on the source. Errors from the AAC encoder or elsewhere in the chain fail the job, as before. - Count lost audio on the source's own timestamps, before pausing drops anything, and subtract only the paused part of a gap. A stall that spanned a pause used to go unreported. - Send Finalized through the worker's event stream, behind MediaLost, so a crash during cleanup cannot drop the loss warning. - Stop messages describe the forced finalization without claiming the encoder hung, and the timeout reports how long it actually waited. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/linux/src/pipeline.rs | 206 +++++++++++++++++++++++++++++------ crates/linux/src/worker.rs | 88 +++++++++++---- packaging/linux/README.md | 14 ++- 3 files changed, 251 insertions(+), 57 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 81b95d4..7f8f056 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -181,6 +181,14 @@ impl Timeline { } } + // Time spent paused before `now`, including a pause still going on. + fn paused(&self, now: gst::ClockTime) -> gst::ClockTime { + self.offset + + self + .paused_at + .map_or(gst::ClockTime::ZERO, |at| now.saturating_sub(at)) + } + fn position(&self, now: gst::ClockTime) -> gst::ClockTime { self.paused_at.unwrap_or(now).saturating_sub(self.offset) } @@ -603,8 +611,7 @@ const OMITTED: u8 = 3; struct AudioTrack { name: &'static str, - // From the source to the queue that feeds the movie. - branch: Vec, + source: gst::Element, pad: gst::Pad, probe: Option, state: Arc, @@ -689,8 +696,9 @@ impl Movie { &mut self, name: &'static str, queue: &gst::Element, - branch: Vec, - ) -> Arc { + source: &gst::Element, + lost: Arc, + ) { let pad = queue.static_pad("src").unwrap(); // Linking the track sends a reconfigure upstream. The encoder answers // with an allocation query, which waits behind this queue while qtmux @@ -733,17 +741,15 @@ impl Movie { } }, ); - let lost = Arc::new(AtomicU64::new(0)); self.audio.push(AudioTrack { name, - branch, + source: source.clone(), pad, probe, state, - lost: lost.clone(), + lost, failed: None, }); - lost } // Called on every pass of the run loop. Requests the audio pads once every @@ -790,13 +796,11 @@ impl Movie { Ok(omitted) } - // The audio track whose branch posted a message, if any. - fn audio_track(&mut self, message: &gst::Message) -> Option<&mut AudioTrack> { + // The audio track whose source posted a message, if any. + fn audio_source(&mut self, message: &gst::Message) -> Option<&mut AudioTrack> { let src = message.src()?; self.audio.iter_mut().find(|track| { - track.branch.iter().any(|element| { - src == element.upcast_ref::() || src.has_as_ancestor(element) - }) + src == track.source.upcast_ref::() || src.has_as_ancestor(&track.source) }) } } @@ -821,6 +825,9 @@ fn add_audio_source( name: &'static str, counters: &Arc, ) -> Result<()> { + // Counted before pausing drops anything. + let lost = Arc::new(AtomicU64::new(0)); + count_lost_audio(source, lost.clone(), counters.clone()); attach_timing_probe(source, counters.clone()); let convert = element("audioconvert")?; let resample = element("audioresample")?; @@ -852,8 +859,7 @@ fn add_audio_source( ]; movie.pipeline.add_many(chain).map_err(backend)?; gst::Element::link_many(chain).map_err(backend)?; - let lost = movie.add_audio(name, &queue, chain.map(|element| element.clone()).to_vec()); - count_lost_audio(source, lost, counters.clone()); + movie.add_audio(name, &queue, source, lost); Ok(()) } @@ -861,12 +867,15 @@ fn add_audio_source( // time, so a longer jump is lost audio. const AUDIO_HOLE: gst::ClockTime = gst::ClockTime::from_mseconds(20); -// Counts audio missing between consecutive buffers. When downstream does not -// read pulsesrc in time, its ring buffer overwrites the oldest audio, and the -// next buffer is stamped where capture actually resumed. +// Counts audio missing between consecutive captured buffers, before pausing +// drops any. When downstream does not read pulsesrc in time, its ring buffer +// overwrites the oldest audio, and the next buffer is stamped where capture +// actually resumed. Sources keep capturing while paused, so a pause leaves no +// gap, and the paused part of a gap is not in the movie. fn count_lost_audio(source: &gst::Element, lost: Arc, counters: Arc) { - // Where the next buffer should start, and the pause offset it assumes. + // Where the next buffer should start, and the paused time when it was due. let next = Mutex::new(None::<(gst::ClockTime, gst::ClockTime)>); + let clock = source.clone(); source .static_pad("src") .unwrap() @@ -877,16 +886,21 @@ fn count_lost_audio(source: &gst::Element, lost: Arc, counters: Arc AUDIO_HOLE { + let missing = pts + .saturating_sub(expected) + .saturating_sub(paused.saturating_sub(before)); + if missing > AUDIO_HOLE { lost.fetch_add(missing.nseconds(), Ordering::Relaxed); } } - *next = Some((pts + duration, offset)); + *next = Some((pts + duration, paused)); gst::PadProbeReturn::Ok }); } @@ -1222,10 +1236,20 @@ fn run( if let Some(message) = bus.timed_pop(gst::ClockTime::from_mseconds(100)) { match message.view() { gst::MessageView::Error(error) => { - // A failed audio source ends its own track, and the movie - // keeps recording the others. - if let Some(track) = movie.audio_track(&message) { + // A failed audio source, such as pulsesrc after its server + // went away, ends its own track, and the movie keeps + // recording the others. Other audio errors fail the job. + if let Some(track) = movie.audio_source(&message) { if track.failed.is_none() { + // pulsesrc ends its branch after a fatal error; make + // sure, without waiting on its stream lock here. + if let Some(branch) = + track.source.static_pad("src").and_then(|pad| pad.peer()) + { + std::thread::spawn(move || { + branch.send_event(gst::event::Eos::new()) + }); + } let position = counters .timeline .lock() @@ -1311,16 +1335,17 @@ fn run( { video_cut = true; log(format!( - "capture-engine: video encoding did not finish within {} s of stop; ending the video track so the movie can be finalized", + "capture-engine: video did not reach the movie writer within {} s of stop while nothing waited downstream; ending the video track there so the movie can be finalized", FINALIZING.as_secs() )); let video = movie.video.clone(); std::thread::spawn(move || video.send_event(gst::event::Eos::new())); } if !video_cut || stopping.is_some_and(|at| at.elapsed() > FINALIZING * 3 / 2) { - return Err(backend( - "Movie finalization timed out after 10s; only completed fragments may be playable.", - )); + return Err(backend(format!( + "Movie finalization timed out after {}s; only completed fragments may be playable.", + stopping.unwrap().elapsed().as_secs() + ))); } } if started && last_metrics.elapsed() >= Duration::from_secs(1) { @@ -1393,7 +1418,10 @@ fn media_lost( ) -> Option { let mut lost = Vec::new(); if video_cut { - lost.push("video encoding stopped responding, so the video ends early".to_string()); + lost.push(format!( + "video did not finish within {} s of stop, so the video ends early", + FINALIZING.as_secs() + )); } if dropped_frames > 0 { lost.push(format!("{dropped_frames} video frames were dropped")); @@ -2530,6 +2558,118 @@ mod tests { } } + // The microphone's producer inside its ring-buffer bin. + fn producer(pipeline: &gst::Pipeline, name: &str) -> gst::Element { + pipeline + .by_name(name) + .unwrap() + .downcast::() + .unwrap() + .by_name("producer") + .unwrap() + } + + // The part of a stall spent paused is not in the movie, and the rest is + // lost audio. + #[test] + fn audio_lost_across_a_pause_counts_only_recorded_time() { + let mut recording = + TestRecording::start_with_audio(vec![("microphone", ring_buffer_audio("microphone"))]); + let start = Instant::now(); + let silent = Duration::from_secs(2)..Duration::from_millis(4500); + producer(&recording.pipeline, "microphone") + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, _| { + if silent.contains(&start.elapsed()) { + gst::PadProbeReturn::Drop + } else { + gst::PadProbeReturn::Ok + } + }); + std::thread::sleep(Duration::from_millis(2800)); + recording.control(true); + std::thread::sleep(Duration::from_millis(700)); + recording.control(false); + std::thread::sleep(Duration::from_secs(3)); + recording.finish().unwrap(); + let hole = audio_holes(&recording.probe())[0]; + assert!((1.6..2.0).contains(&hole), "the movie misses {hole}s"); + let message = recording + .events + .try_iter() + .find_map(|event| match event { + RecorderEvent::MediaLost { message, .. } => Some(message), + _ => None, + }) + .expect("no media loss reported"); + let reported: f64 = message + .split_once("microphone is missing ") + .and_then(|(_, rest)| rest.split_once(" s")?.0.parse().ok()) + .unwrap(); + assert!( + (reported - hole).abs() < 0.05, + "reported {reported}s, the movie misses {hole}s" + ); + } + + // A source that reports a fatal error but leaves its branch open still + // ends its track, so the muxer does not wait for it. + #[test] + fn a_failed_audio_source_that_keeps_its_branch_open_still_ends_its_track() { + let mut recording = TestRecording::start_with_audio(vec![ + ("system audio", ring_buffer_audio("system-audio")), + ("microphone", ring_buffer_audio("microphone")), + ]); + let start = Instant::now(); + let reported = std::sync::atomic::AtomicBool::new(false); + producer(&recording.pipeline, "microphone") + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |pad, _| { + if start.elapsed() < Duration::from_secs(2) { + return gst::PadProbeReturn::Ok; + } + if !reported.swap(true, Ordering::SeqCst) { + let source = pad.parent_element().unwrap(); + gst::element_error!(source, gst::ResourceError::Read, ("device unplugged")); + } + gst::PadProbeReturn::Drop + }); + std::thread::sleep(Duration::from_secs(8)); + recording.finish().unwrap(); + let probe = recording.probe(); + let missing = missing_video(&probe); + assert!( + missing < 0.05, + "{missing}s of video lost waiting for the failed microphone" + ); + let message = recording + .events + .try_iter() + .find_map(|event| match event { + RecorderEvent::MediaLost { message, .. } => Some(message), + _ => None, + }) + .unwrap(); + assert!(message.contains("microphone stopped at 2."), "{message}"); + } + + // Only a source going away is recoverable. Any other audio error fails + // the recording, as before. + #[test] + fn an_audio_encoder_error_fails_the_recording() { + let mut recording = + TestRecording::start_with_audio(vec![("microphone", ring_buffer_audio("microphone"))]); + let encoder = pipeline_element(&recording.pipeline, "avenc_aac")[0].clone(); + std::thread::sleep(Duration::from_secs(1)); + gst::element_error!(encoder, gst::LibraryError::Encode, ("encoder failed")); + let at = Instant::now(); + let error = recording.finish().unwrap_err(); + assert!(error.to_string().contains("encoder failed"), "{error}"); + assert!(at.elapsed() < Duration::from_secs(2), "{:?}", at.elapsed()); + } + // What the recording reported matches what its movie is missing. fn assert_reports_its_losses(outcome: &StallOutcome) { let message = outcome @@ -2950,7 +3090,7 @@ mod tests { pipeline.0.add_many([&video, &mux, &sink, &silent]).unwrap(); mux.link(&sink).unwrap(); let mut movie = Movie::new(&pipeline.0, &video, &mux, true).unwrap(); - movie.add_audio("test audio", &silent, vec![silent.clone()]); + movie.add_audio("test audio", &silent, &silent, Arc::default()); let held = pipeline.0.by_name(HELD_QUEUE).unwrap(); pipeline.0.set_state(gst::State::Playing).unwrap(); // Large constant-QP frames 10 ms apart reach the byte limit long diff --git a/crates/linux/src/worker.rs b/crates/linux/src/worker.rs index 8a9eab6..43e9d4c 100644 --- a/crates/linux/src/worker.rs +++ b/crates/linux/src/worker.rs @@ -431,6 +431,10 @@ fn last_lines(stderr: ChildStderr) -> std::io::Result> { Ok(receiver) } +// Marks where the movie was finalized in the worker's event stream; it is +// sent to the daemon as Message::Finalized, never as a log. +const FINALIZED: &str = "\0wrec-capture-finalized"; + pub fn run() -> ! { unsafe { libc::prctl(libc::PR_SET_NAME, c"wrec-capture".as_ptr() as libc::c_ulong); @@ -485,34 +489,50 @@ pub fn run() -> ! { control_stop.send_replace(true); }); let stopping_output = output.clone(); + let finalized = events.clone(); let stopping = Arc::new(move |teardown: crate::pipeline::Teardown| { - send( - &stopping_output, - &match teardown { - crate::pipeline::Teardown::Recording => Message::Stopping, - crate::pipeline::Teardown::Finalized => Message::Finalized, - crate::pipeline::Teardown::AttemptStarted => Message::CleaningAttempt, - crate::pipeline::Teardown::AttemptFinished => Message::CleanedAttempt, - }, - ) + let message = match teardown { + crate::pipeline::Teardown::Recording => Message::Stopping, + // Behind the events sent before it, such as lost media, so the + // daemon has them even if cleanup then crashes. + crate::pipeline::Teardown::Finalized => { + let _ = finalized.send(RecorderEvent::Log { + session_id: None, + message: FINALIZED.into(), + }); + return; + } + crate::pipeline::Teardown::AttemptStarted => Message::CleaningAttempt, + crate::pipeline::Teardown::AttemptFinished => Message::CleanedAttempt, + }; + send(&stopping_output, &message) }); let forwarder = spawn("wrec-capture-events", move || { - for event in outgoing { - let finished = matches!( - event, - RecorderEvent::Exited { .. } | RecorderEvent::Cancelled { .. } - ); - send(&output, &Message::Event(event)); - if finished { - break; - } - } + forward(outgoing, |message| send(&output, message)) }); capture(target, settings, session, events, receiver, stop, stopping); let _ = forwarder.join(); std::process::exit(0) } +// Writes the worker's events in the order they were sent, until the last one. +fn forward(outgoing: mpsc::Receiver, mut write: impl FnMut(&Message)) { + for event in outgoing { + let finished = matches!( + event, + RecorderEvent::Exited { .. } | RecorderEvent::Cancelled { .. } + ); + if matches!(&event, RecorderEvent::Log { message, .. } if message == FINALIZED) { + write(&Message::Finalized); + continue; + } + write(&Message::Event(event)); + if finished { + break; + } + } +} + fn spawn(name: &str, body: impl FnOnce() + Send + 'static) -> thread::JoinHandle<()> { thread::Builder::new() .name(name.into()) @@ -882,6 +902,36 @@ mod tests { )); } + #[test] + fn finalization_is_reported_after_the_events_before_it() { + let (events, outgoing) = mpsc::channel(); + for event in [ + RecorderEvent::MediaLost { + session_id: 0, + message: "lost".into(), + }, + RecorderEvent::Log { + session_id: None, + message: FINALIZED.into(), + }, + RecorderEvent::Exited { + session_id: 0, + success: true, + status: "recording finalized".into(), + }, + ] { + events.send(event).unwrap(); + } + let mut written = Vec::new(); + forward(outgoing, |message| { + written.push(serde_json::to_string(message).unwrap()) + }); + assert_eq!(written.len(), 3, "{written:?}"); + assert!(written[0].contains("MediaLost"), "{written:?}"); + assert_eq!(written[1], r#""Finalized""#); + assert!(written[2].contains("Exited"), "{written:?}"); + } + #[test] fn a_finalized_movie_survives_a_hung_cleanup() { let fake = Fake::start(r#"read start; echo '"Finalized"'; exec sleep 30"#); diff --git a/packaging/linux/README.md b/packaging/linux/README.md index bf90205..ece8c56 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -284,7 +284,9 @@ that nothing read in time. A stalled video encoder always costs the frames captured meanwhile, since raw frames are too large to keep. If an audio source fails, for example because PipeWire or PulseAudio restarts, -its track ends at that point and video and the other track keep recording. A +its track ends at that point and video and the other track keep recording. An +error anywhere else in the audio chain, such as the AAC encoder, still fails the +job. A completed job is not proof of a whole movie. When frames were dropped, audio has gaps, or a track ended early or never started, the job finishes `completed` with a `media_lost` warning that states what is missing, and the job events record when @@ -317,10 +319,12 @@ message instead of finalizing it. Stop drains the encoders and finalizes the movie, including when paused. If video still has not reached the movie writer 10 seconds after stop while nothing -waits for the disk or for audio, the video encoder is stuck. wrec then ends the -video track at the writer and finalizes the movie with the video it already has, -and the job reports the shorter video as lost media. A disk that stalls for more -than 10 seconds during finalization still fails the job. The +waits for the disk or for audio, wrec ends the video track at the writer and +finalizes the movie with the video it already has, and the job reports the +shorter video as lost media. This rescues a movie whose encoder hung; an encoder +that is only very slow loses the frames it had not finished. If that takes more +than 5 more seconds, or a disk stalls for more than 10 seconds during +finalization, the job fails as before. The writer requests ten-second movie fragments; encoder keyframes can close them earlier. An interrupted recording may only be playable through its last completed fragment. Audio sources can deliver their From f6fedc487dbd51feab216ec5d6e4758f94fe7fe1 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Mon, 5 Oct 2026 04:55:44 +0000 Subject: [PATCH 30/45] fix(linux): measure lost audio against recorded pause intervals Round 2 of the review found that the paused time subtracted from a gap was sampled when buffers arrived, while the gap itself uses capture timestamps. A source that delivers late, with a gap inside a pause, was reported as a few tens of milliseconds lost. The probe also held its own source, so a failed encoder attempt's audio elements were never freed. The timeline now keeps finished pauses in running time, the same clock as capture timestamps, and a gap loses only its part outside them. The probe no longer needs the source's clock. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/linux/src/pipeline.rs | 109 ++++++++++++++++++++++++++++------- 1 file changed, 89 insertions(+), 20 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 7f8f056..5636527 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -167,6 +167,8 @@ struct Timeline { paused_at: Option, offset: gst::ClockTime, accept_from: gst::ClockTime, + // Finished pauses, in running time, oldest first. + pauses: Vec<(gst::ClockTime, gst::ClockTime)>, } impl Timeline { @@ -178,15 +180,19 @@ impl Timeline { if let Some(paused_at) = self.paused_at.take() { self.offset += now.saturating_sub(paused_at); self.accept_from = now; + self.pauses.push((paused_at, now)); } } - // Time spent paused before `now`, including a pause still going on. - fn paused(&self, now: gst::ClockTime) -> gst::ClockTime { - self.offset - + self - .paused_at - .map_or(gst::ClockTime::ZERO, |at| now.saturating_sub(at)) + // How much of the running time from `start` to `end` was paused. + fn paused_between(&self, start: gst::ClockTime, end: gst::ClockTime) -> gst::ClockTime { + self.paused_at + .map(|at| (at, gst::ClockTime::MAX)) + .into_iter() + .chain(self.pauses.iter().rev().copied()) + .take_while(|&(_, resumed)| resumed > start) + .map(|(paused, resumed)| end.min(resumed).saturating_sub(start.max(paused))) + .sum() } fn position(&self, now: gst::ClockTime) -> gst::ClockTime { @@ -871,11 +877,11 @@ const AUDIO_HOLE: gst::ClockTime = gst::ClockTime::from_mseconds(20); // drops any. When downstream does not read pulsesrc in time, its ring buffer // overwrites the oldest audio, and the next buffer is stamped where capture // actually resumed. Sources keep capturing while paused, so a pause leaves no -// gap, and the paused part of a gap is not in the movie. +// gap, and the paused part of a gap is not in the movie. Capture timestamps +// and pauses are both in running time. fn count_lost_audio(source: &gst::Element, lost: Arc, counters: Arc) { - // Where the next buffer should start, and the paused time when it was due. - let next = Mutex::new(None::<(gst::ClockTime, gst::ClockTime)>); - let clock = source.clone(); + // Where the next buffer should start. + let next = Mutex::new(None::); source .static_pad("src") .unwrap() @@ -886,21 +892,19 @@ fn count_lost_audio(source: &gst::Element, lost: Arc, counters: Arc expected) { + let paused = counters + .timeline + .lock() + .unwrap() + .paused_between(expected, pts); + let missing = pts.saturating_sub(expected).saturating_sub(paused); if missing > AUDIO_HOLE { lost.fetch_add(missing.nseconds(), Ordering::Relaxed); } } - *next = Some((pts + duration, paused)); + *next = Some(pts + duration); gst::PadProbeReturn::Ok }); } @@ -2613,6 +2617,71 @@ mod tests { ); } + // Audio lost between buffers with these capture times, in seconds, with + // the timeline's pauses as (start, end), and a pause still going on. + fn lost_between(buffers: &[(f64, f64)], pauses: &[(f64, f64)], paused_at: Option) -> f64 { + gst::init().unwrap(); + let time = |seconds: f64| gst::ClockTime::from_nseconds((seconds * 1e9) as u64); + let counters = Arc::new(Counters::default()); + { + let mut timeline = counters.timeline.lock().unwrap(); + for &(start, end) in pauses { + timeline.pause(time(start)); + timeline.resume(time(end)); + } + if let Some(at) = paused_at { + timeline.pause(time(at)); + } + } + let pipeline = PipelineGuard(gst::Pipeline::new()); + let source = element("appsrc").unwrap(); + source.set_property_from_str("format", "time"); + source.set_property("caps", gst::Caps::new_empty_simple("audio/x-raw")); + let sink = element("fakesink").unwrap(); + sink.set_property("sync", false); + pipeline.0.add_many([&source, &sink]).unwrap(); + source.link(&sink).unwrap(); + let lost = Arc::new(AtomicU64::new(0)); + count_lost_audio(&source, lost.clone(), counters); + pipeline.0.set_state(gst::State::Playing).unwrap(); + for &(start, end) in buffers { + let mut buffer = gst::Buffer::new(); + let buffer_mut = buffer.get_mut().unwrap(); + buffer_mut.set_pts(time(start)); + buffer_mut.set_duration(time(end - start)); + assert_eq!( + source.emit_by_name::("push-buffer", &[&buffer]), + gst::FlowReturn::Ok + ); + } + let _ = source.emit_by_name::("end-of-stream", &[]); + let ended = pipeline.0.bus().unwrap().timed_pop_filtered( + gst::ClockTime::from_seconds(5), + &[gst::MessageType::Eos, gst::MessageType::Error], + ); + assert!(ended.is_some_and(|message| message.type_() == gst::MessageType::Eos)); + lost.load(Ordering::Relaxed) as f64 / 1e9 + } + + // Capture timestamps decide, not when buffers arrive: a source that + // delivers late still has its gap inside the pause. + #[test] + fn only_the_unpaused_part_of_a_capture_gap_is_lost() { + let near = |lost: f64, expected: f64| (lost - expected).abs() < 1e-6; + // Inside a pause from 2 s to 4 s. + let lost = lost_between(&[(2.04, 2.05), (3.95, 3.96)], &[(2.0, 4.0)], None); + assert!(near(lost, 0.0), "{lost}"); + // From 1.5 s to 4.5 s, half a second on each side of it. + let lost = lost_between(&[(1.49, 1.5), (4.5, 4.51)], &[(2.0, 4.0)], None); + assert!(near(lost, 1.0), "{lost}"); + // Into a pause that has not ended. + let lost = lost_between(&[(1.49, 1.5), (3.0, 3.01)], &[], Some(2.0)); + assert!(near(lost, 0.5), "{lost}"); + // Shorter than a hole. + let lost = lost_between(&[(1.0, 1.01), (1.02, 1.03)], &[], None); + assert!(near(lost, 0.0), "{lost}"); + } + // A source that reports a fatal error but leaves its branch open still // ends its track, so the muxer does not wait for it. #[test] From c302011084e609da8585916b534909dd2fe4f337 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:41:13 +0000 Subject: [PATCH 31/45] fix(linux): preserve complete window recordings in MP4 --- README.md | 5 +- crates/linux/src/lib.rs | 2 +- crates/linux/src/pipeline.rs | 360 ++++++++++++++++++++++++++++++----- crates/linux/src/worker.rs | 4 +- packaging/linux/README.md | 73 ++++++- scripts/test-cli-linux.py | 2 +- 6 files changed, 388 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 8e69da8..205b91f 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,9 @@ Wrec records displays or windows with a native ScreenCaptureKit pipeline, writes hardware-encoded `.mov` files, and gives you both a native SwiftUI menu-bar app and a JSON-friendly CLI for scripts and agents. -The Linux CLI records Wayland and X11 desktops. It prefers VA-API or NVIDIA -hardware encoding and falls back to software encoding when needed. See [Linux setup and limitations](packaging/linux/README.md). +The Linux CLI records Wayland and X11 desktops to `.mp4` files. It prefers +VA-API or NVIDIA hardware encoding and falls back to software encoding when +needed. See [Linux setup and limitations](packaging/linux/README.md). > [!NOTE] > Wrec is still early public software. Release builds are not notarized, so diff --git a/crates/linux/src/lib.rs b/crates/linux/src/lib.rs index fe1f789..1321cb6 100644 --- a/crates/linux/src/lib.rs +++ b/crates/linux/src/lib.rs @@ -97,7 +97,7 @@ impl RecorderEngine for LinuxRecorder { let id = now.max(previous.saturating_add(1)); let session = RecordingSession { id, - output_path: settings.output_dir.join(format!("wrec-{id}.mov")), + output_path: settings.output_dir.join(format!("wrec-{id}.mp4")), }; self.active = Some(worker::Worker::spawn( Path::new(worker::PROGRAM), diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 5636527..fcc6065 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -30,7 +30,7 @@ pub(crate) fn check_plugins(settings: &RecorderSettings) -> Result<()> { "capsfilter", "queue", parser_name(settings.codec), - "qtmux", + "mp4mux", "filesink", ] { element(factory)?; @@ -97,8 +97,12 @@ fn capture_caps(dmabuf: bool, pipewire: bool, fps: u32) -> gst::Caps { } } +// qtmux guesses a 4:3 or 16:9 TV picture for widths 641 through 1052 and +// heights 480 through 576, and writes a clean aperture that makes players crop +// a window of, say, 800x528 to 704x528. mp4mux is the same muxer writing the +// ISO flavor, which declares the full picture. fn movie_mux() -> Result { - let mux = element("qtmux")?; + let mux = element("mp4mux")?; mux.set_property("fragment-duration", 10000u32); mux.set_property_from_str("fragment-mode", "first-moov-then-finalise"); // Preserve sub-frame timestamps around pause/resume instead of rounding @@ -109,7 +113,7 @@ fn movie_mux() -> Result { const WRITE_QUEUE: &str = "write-queue"; -// qtmux writes each sample as it muxes it, so while a write blocks every +// mp4mux writes each sample as it muxes it, so while a write blocks every // track waits behind it: video drops at capture within two frames, and audio // once its queue fills. This buffer stays empty unless the disk stalls; at // the highest bitrate-mode quality (16 Mbit/s) it covers about 16 seconds. @@ -627,13 +631,13 @@ struct AudioTrack { failed: Option<(gst::ClockTime, String)>, } -// The recording pipeline and the tracks of its movie. qtmux writes a track +// The recording pipeline and the tracks of its movie. mp4mux writes a track // for every pad requested from it, and a track that never received audio has // no sample description, which makes the whole movie unreadable. AAC encoders // only learn their format from their first input buffer, so audio pads are // requested once each track has delivered a buffer or ended without one. // Until then video waits in a queue instead of being dropped at capture. -// qtmux takes nothing while any track is empty, so afterwards the same queue +// mp4mux takes nothing while any track is empty, so afterwards the same queue // lets video wait for a stalled audio track as long as audio can wait for // video, instead of dropping frames at capture. struct Movie { @@ -707,7 +711,7 @@ impl Movie { ) { let pad = queue.static_pad("src").unwrap(); // Linking the track sends a reconfigure upstream. The encoder answers - // with an allocation query, which waits behind this queue while qtmux + // with an allocation query, which waits behind this queue while mp4mux // holds audio for the next video frame, and pulsesrc loses audio // meanwhile. The audio caps are fixed, so nothing needs it. pad.add_probe(gst::PadProbeType::EVENT_UPSTREAM, |_, info| { @@ -850,7 +854,7 @@ fn add_audio_source( let parser = element("aacparse")?; let queue = element("queue")?; // Holds audio while the movie waits for video, or for another audio - // track to start, which can take HELD_VIDEO_LIMIT. qtmux writes a video + // track to start, which can take HELD_VIDEO_LIMIT. mp4mux writes a video // frame only when the next one arrives and holds later audio until then, // so at an idle screen's keepalive rate audio waits up to two keepalives, // and a pause that swallows a frame makes it almost three. @@ -993,7 +997,7 @@ fn match_keyframes_to_capture(encoder: &gst::Element, fps: u32) { } // A capture rate change reconfigures the encoder, which writes new parameter -// sets. qtmux stores those as an extra sample description, which a fragmented +// sets. mp4mux stores those as an extra sample description, which a fragmented // movie cannot reference, so the movie keeps its first description and every // keyframe carries its own parameter sets (avc3/hev1). fn movie_format(codec: Codec, fps: u32) -> Result { @@ -1007,7 +1011,7 @@ fn movie_format(codec: Codec, fps: u32) -> Result { .field("alignment", "au") .build(), ); - // qtmux times each sample by the next one and takes the last sample's + // mp4mux times each sample by the next one and takes the last sample's // duration from its buffer. Some encoders (x265enc 1.24) leave it unset // for variable-rate capture, and players skip a zero-length last frame, // so a frame without one lasts a frame at the current capture rate, or at @@ -1067,7 +1071,7 @@ fn attach_capture_probe( if width >= 2 && height >= 2 { let mut dimensions = counters.dimensions.lock().unwrap(); resized.store(dimensions.is_some(), Ordering::Relaxed); - // A MOV track has a fixed canvas. Window resizes are + // A movie track has a fixed canvas. Window resizes are // scaled/letterboxed into the initial output size. let (w, h) = dimensions .map(|d| (d.output_width as i32, d.output_height as i32)) @@ -1776,7 +1780,7 @@ mod tests { let session = RecordingSession { id, output_path: std::env::temp_dir() - .join(format!("wrec-linux-test-{}-{id}.mov", std::process::id())), + .join(format!("wrec-linux-test-{}-{id}.mp4", std::process::id())), }; // Synthetic software encoding is confined to tests. Exercise the same // bus/control/mux code without pretending this is a hardware benchmark. @@ -2108,7 +2112,7 @@ mod tests { }); } - // Seconds missing from each audio track. qtmux stretches the packet + // Seconds missing from each audio track. mp4mux stretches the packet // before a hole over it, so a hole is packet spacing beyond one AAC frame. fn audio_holes(probe: &serde_json::Value) -> Vec { let packets = probe["packets"].as_array().unwrap(); @@ -2838,7 +2842,7 @@ mod tests { let delay = gst::ClockTime::from_seconds(1); let mut recording = TestRecording::start_with_audio_delays(&[gst::ClockTime::ZERO, delay]); std::thread::sleep(Duration::from_secs(13)); - let partial = recording.session.output_path.with_extension("partial.mov"); + let partial = recording.session.output_path.with_extension("partial.mp4"); std::fs::copy(&recording.session.output_path, &partial).unwrap(); recording.finish().unwrap(); let seconds = delay.nseconds() as f64 / 1_000_000_000.0; @@ -3062,7 +3066,7 @@ mod tests { let mut recording = TestRecording::start_with_audio_delays(&[gst::ClockTime::from_seconds(1)]); // An idle screen sends no frames, so when the late track is linked - // qtmux holds its audio until the next frame arrives. + // mp4mux holds its audio until the next frame arrives. let idle = gst::ClockTime::from_mseconds(500)..gst::ClockTime::from_mseconds(2200); recording .pipeline @@ -3084,7 +3088,7 @@ mod tests { } }; // Audio that arrives while paused is dropped, so audio stuck behind - // qtmux before the pause would leave a hole. + // mp4mux before the pause would leave a hole. wait_until(2000); recording.control(true); wait_until(2500); @@ -3285,7 +3289,7 @@ mod tests { } // An idle screen sends a frame each keepalive. A short pause that swallows - // one leaves almost two keepalives between the frames around it, and qtmux + // one leaves almost two keepalives between the frames around it, and mp4mux // holds audio until the frame after those. The audio queue must absorb // that wait: a full queue blocks pulsesrc, which then drops audio. #[test] @@ -3338,7 +3342,7 @@ mod tests { recording.finish().unwrap(); assert!( !full.load(Ordering::SeqCst), - "audio queue filled while qtmux waited for idle video" + "audio queue filled while mp4mux waited for idle video" ); recording.probe(); } @@ -3349,7 +3353,7 @@ mod tests { RecordingSession { id, output_path: std::env::temp_dir().join(format!( - "wrec-linux-record-test-{}-{id}.mov", + "wrec-linux-record-test-{}-{id}.mp4", std::process::id() )), } @@ -3439,6 +3443,85 @@ mod tests { } } + // record() on its own thread. Every wait on it has a deadline, so an engine + // failure fails the test instead of hanging it. Dropping it stops the + // recording and waits as long as the daemon waits for a worker to exit. + struct RecordThread { + stop: watch::Sender, + returned: mpsc::Receiver>, + } + + impl RecordThread { + fn spawn( + capture: CaptureInput, + session: &RecordingSession, + codec: Codec, + events: mpsc::Sender, + ) -> Self { + let (commands, receiver) = mpsc::sync_channel(1); + let (stop, stopped) = watch::channel(false); + let (done, returned) = mpsc::channel(); + let session = session.clone(); + std::thread::spawn(move || { + let _commands = commands; + let result = record( + &capture, + &session, + &silent_settings(codec), + &events, + receiver, + &stopped, + &|_| {}, + ); + let _ = done.send(result); + }); + Self { stop, returned } + } + + fn assert_running(&self) { + match self.returned.try_recv() { + Err(mpsc::TryRecvError::Empty) => {} + outcome => panic!("the recording ended early: {outcome:?}"), + } + } + + fn finish(&self) -> Result<()> { + self.stop.send_replace(true); + self.returned + .recv_timeout(crate::worker::EXIT_DEADLINE) + .unwrap_or_else(|error| panic!("the recording did not return after stop: {error}")) + } + } + + impl Drop for RecordThread { + fn drop(&mut self) { + self.stop.send_replace(true); + let _ = self.returned.recv_timeout(crate::worker::EXIT_DEADLINE); + } + } + + // The test's appsrc once the recording plays it. + fn playing_source( + source: &Mutex>, + recording: &RecordThread, + ) -> gst::Element { + let start = Instant::now(); + loop { + recording.assert_running(); + let created = source.lock().unwrap().clone(); + if let Some(appsrc) = + created.filter(|appsrc| appsrc.current_state() == gst::State::Playing) + { + return appsrc; + } + assert!( + start.elapsed() < Duration::from_secs(10), + "the capture source did not play within 10s" + ); + std::thread::sleep(Duration::from_millis(10)); + } + } + // Each recording paces real frames through a leaky queue; running two at // once on a shared encoder would turn scheduling delays into frame drops. static RATE_CHANGE_RECORDING: Mutex<()> = Mutex::new(()); @@ -3477,34 +3560,10 @@ mod tests { source })); let session = test_session(); - let partial = session.output_path.with_extension("partial.mov"); + let partial = session.output_path.with_extension("partial.mp4"); let (events, received) = mpsc::channel(); - let (_commands, receiver) = mpsc::sync_channel(1); - let (stop, stopped) = watch::channel(false); - let worker_session = session.clone(); - let worker = std::thread::spawn(move || { - record( - &capture, - &worker_session, - &silent_settings(codec), - &events, - receiver, - &stopped, - &|_| {}, - ) - }); - let appsrc = loop { - if let Some(source) = source.lock().unwrap().clone() { - break source; - } - std::thread::sleep(Duration::from_millis(10)); - }; - let playing = Instant::now(); - while appsrc.current_state() != gst::State::Playing - && playing.elapsed() < Duration::from_secs(10) - { - std::thread::sleep(Duration::from_millis(10)); - } + let recording = RecordThread::spawn(capture, &session, codec, events); + let appsrc = playing_source(&source, &recording); // Stamp each frame with its schedule rather than the time the source // thread happens to forward it, so every packet maps to one pushed frame. let origin = appsrc.current_running_time().unwrap_or_default(); @@ -3536,7 +3595,7 @@ mod tests { { std::thread::sleep(Duration::from_millis(10)); } - // qtmux writes a fragment when the next keyframe reaches it, so wait + // mp4mux writes a fragment when the next keyframe reaches it, so wait // for one that holds frames after the first rate change. let changed = phases[0].1 as f64; let copied = Instant::now(); @@ -3549,8 +3608,7 @@ mod tests { } std::thread::sleep(Duration::from_millis(200)); } - stop.send_replace(true); - let result = worker.join().unwrap(); + let result = recording.finish(); let events: Vec = received.try_iter().collect(); let factory = events .iter() @@ -3752,6 +3810,212 @@ mod tests { assert_rate_changes_stay_playable(Codec::Hevc, "hevc", &FALLING); } + // Records white edges around a black picture and returns the movie. + fn record_framed_picture(codec: Codec, width: i32, height: i32) -> RecordingSession { + gst::init().unwrap(); + let caps = gst::Caps::builder("video/x-raw") + .field("format", "BGRx") + .field("width", width) + .field("height", height) + .field("framerate", gst::Fraction::new(0, 1)) + .field("max-framerate", gst::Fraction::new(10, 1)) + .build(); + let mut picture = vec![0u8; (width * height * 4) as usize]; + for (index, pixel) in picture.chunks_mut(4).enumerate() { + let (x, y) = (index as i32 % width, index as i32 / width); + if x.min(width - 1 - x).min(y).min(height - 1 - y) < EDGE { + pixel.fill(255); + } + } + let source = Arc::new(Mutex::new(None::)); + let created = source.clone(); + let capture = CaptureInput::Element(Box::new(move || { + let source = element("appsrc").unwrap(); + source.set_property("is-live", true); + source.set_property_from_str("format", "time"); + source.set_property("caps", &caps); + *created.lock().unwrap() = Some(source.clone()); + source + })); + let session = test_session(); + let (events, _received) = mpsc::channel(); + let recording = RecordThread::spawn(capture, &session, codec, events); + let appsrc = playing_source(&source, &recording); + let origin = appsrc.current_running_time().unwrap_or_default(); + for frame in 0..10 { + std::thread::sleep(Duration::from_millis(100)); + let mut buffer = gst::Buffer::from_slice(picture.clone()); + let buffer_mut = buffer.get_mut().unwrap(); + buffer_mut.set_pts(origin + gst::ClockTime::from_mseconds(100 * frame)); + assert_eq!( + appsrc.emit_by_name::("push-buffer", &[&buffer]), + gst::FlowReturn::Ok + ); + } + let pushed = Instant::now(); + while appsrc.property::("current-level-buffers") > 0 { + recording.assert_running(); + assert!( + pushed.elapsed() < Duration::from_secs(5), + "the recording took no frame for 5s" + ); + std::thread::sleep(Duration::from_millis(10)); + } + recording.finish().unwrap(); + session + } + + const EDGE: i32 = 16; + + // The pictures a player shows, decoded with FFmpeg's default cropping. + fn decoded_pictures(movie: &Path) -> Vec<(usize, usize, Vec)> { + let output = std::process::Command::new("ffmpeg") + .args(["-v", "error", "-i"]) + .arg(movie) + .args(["-map", "0:v:0", "-vsync", "0", "-pix_fmt", "gray"]) + .args(["-c:v", "pgm", "-f", "image2pipe", "-"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let mut pictures = Vec::new(); + let mut rest = &output.stdout[..]; + while !rest.is_empty() { + // "P5\n \n255\n" followed by the pixels. + let mut fields = Vec::new(); + while fields.len() < 4 { + let end = rest.iter().position(u8::is_ascii_whitespace).unwrap(); + fields.push(String::from_utf8(rest[..end].to_vec()).unwrap()); + rest = &rest[end + 1..]; + } + let width: usize = fields[1].parse().unwrap(); + let height: usize = fields[2].parse().unwrap(); + pictures.push((width, height, rest[..width * height].to_vec())); + rest = &rest[width * height..]; + } + pictures + } + + // Boxes in the movie's final index, down to each video sample entry. A + // finished movie keeps its first index inside its media data box as + // "hoov", where this walk does not look, so a partial movie's first index + // goes unchecked. + fn video_boxes(movie: &Path) -> Vec<(String, Vec)> { + fn children(data: &[u8]) -> Vec<(&str, &[u8])> { + let mut boxes = Vec::new(); + let mut rest = data; + while rest.len() >= 8 { + let size = u32::from_be_bytes(rest[..4].try_into().unwrap()) as usize; + let size = if size == 0 { rest.len() } else { size }; + if size < 8 || size > rest.len() { + break; + } + let name = std::str::from_utf8(&rest[4..8]).unwrap_or("?"); + boxes.push((name, &rest[8..size])); + rest = &rest[size..]; + } + boxes + } + fn walk(name: &str, body: &[u8], found: &mut Vec<(String, Vec)>) { + let body = match name { + "moov" | "trak" | "tapt" | "mdia" | "minf" | "stbl" => body, + // A full box with an entry count. + "stsd" => &body[8..], + // The fields of a visual sample entry come before its boxes. + "avc1" | "avc3" | "hvc1" | "hev1" => &body[78..], + _ => return, + }; + for (child, payload) in children(body) { + found.push((child.to_string(), payload.to_vec())); + walk(child, payload, found); + } + } + let data = std::fs::read(movie).unwrap(); + let mut found = Vec::new(); + for (name, body) in children(&data) { + walk(name, body, &mut found); + } + found + } + + fn assert_records_every_pixel(codec: Codec, width: i32, height: i32) { + let session = record_framed_picture(codec, width, height); + let movie = &session.output_path; + let file_type = std::fs::read(movie).unwrap()[4..12].to_vec(); + let boxes = video_boxes(movie); + let pictures = decoded_pictures(movie); + let _ = std::fs::remove_file(movie); + // Players crop to a clean aperture ("clap") or a track aperture + // ("tapt"). FFmpeg 8.0 applies clap when decoding; FFmpeg 7.0 does not. + let size = (width as u32, height as u32); + for (name, payload) in &boxes { + let field = |index: usize| { + u32::from_be_bytes(payload[index * 4..index * 4 + 4].try_into().unwrap()) + }; + let aperture = match name.as_str() { + // Width and height as fractions. + "clap" => (field(0) / field(1), field(2) / field(3)), + // A full box header, then 16.16 fixed-point width and height. + "clef" | "prof" | "enof" => (field(1) >> 16, field(2) >> 16), + _ => continue, + }; + assert_eq!(aperture, size, "{name} crops the movie's picture"); + } + assert!( + boxes + .iter() + .any(|(name, _)| name == "avcC" || name == "hvcC"), + "found no video sample entry in {boxes:?}" + ); + assert!(!pictures.is_empty()); + for (shown_width, shown_height, pixels) in pictures { + let shown = (shown_width as u32, shown_height as u32); + assert_eq!(shown, size, "the player shows a different picture"); + let luma = |x: usize, y: usize| pixels[y * shown_width + x]; + let (right, bottom) = (shown_width - 1, shown_height - 1); + let middle = (shown_width / 2, shown_height / 2); + let edges = [ + (0, middle.1), + (right, middle.1), + (middle.0, 0), + (middle.0, bottom), + (0, 0), + (right, bottom), + ]; + for (x, y) in edges { + assert!(luma(x, y) > 200, "the edge at {x},{y} is missing"); + } + for (x, y) in [ + (EDGE as usize + 2, middle.1), + (right - EDGE as usize - 2, middle.1), + middle, + ] { + assert!(luma(x, y) < 50, "the picture at {x},{y} moved"); + } + } + + // Linux names recordings ".mp4", so the file must say it is MP4. + assert_eq!( + String::from_utf8_lossy(&file_type), + "ftypmp42", + "the movie's file type" + ); + } + + // qtmux crops these sizes to a guessed TV aspect ratio; see movie_mux. + #[test] + fn h264_recordings_keep_every_pixel_at_tv_like_sizes() { + assert_records_every_pixel(Codec::H264, 800, 528); + } + + #[test] + fn hevc_recordings_keep_every_pixel_at_tv_like_sizes() { + assert_records_every_pixel(Codec::Hevc, 800, 528); + } + // Rust heap bytes allocated and not yet freed by each thread. GStreamer // buffers come from GLib and are not counted. struct CountingAllocator; diff --git a/crates/linux/src/worker.rs b/crates/linux/src/worker.rs index 43e9d4c..24840fa 100644 --- a/crates/linux/src/worker.rs +++ b/crates/linux/src/worker.rs @@ -589,7 +589,7 @@ mod tests { let session = RecordingSession { id, output_path: std::env::temp_dir() - .join(format!("wrec-worker-test-{}-{id}.mov", std::process::id())), + .join(format!("wrec-worker-test-{}-{id}.mp4", std::process::id())), }; let (events, received) = mpsc::channel(); let worker = Worker::spawn( @@ -654,7 +654,7 @@ mod tests { } fn pid_file(&self) -> std::path::PathBuf { - self.session.output_path.with_extension("mov.pid") + self.session.output_path.with_extension("mp4.pid") } } diff --git a/packaging/linux/README.md b/packaging/linux/README.md index ece8c56..8aa464f 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -13,11 +13,23 @@ the worker exits at the end of each recording. On glibc builds, the worker also asks the allocator to return freed heap pages between encoder attempts. On compatible Wayland/VA-API systems, the preferred pipeline is -`pipewiresrc → DMA-BUF → vapostproc → VA encoder → qtmux`. Rust manages sessions +`pipewiresrc → DMA-BUF → vapostproc → VA encoder → mp4mux`. Rust manages sessions and buffer metadata without mapping video pixels. The source negotiates four to eight buffers and the capture queue holds at most two frames. Conversion and scaling can allocate GPU surfaces; this does not imply zero total copies. +Linux recordings are `wrec-.mp4` files in ISO MP4 format (file type +`mp42`, also listing `mp41`, `isom` and `iso2`). macOS recordings are still +QuickTime `.mov` files. On Linux, wrec uses `mp4mux` rather than GStreamer's +QuickTime writer, `qtmux`. For widths 641 through 1052 and heights 480 through +576, `qtmux` guesses a 4:3 or 16:9 TV picture and declares a clean aperture for +it. FFmpeg 8.0 crops to that aperture, so an 800x528 window would play as +704x528; FFmpeg 7.0 ignores it. `mp4mux` is the same GStreamer muxer writing the +ISO variant, and it declares the full picture. Tracks, codecs, timing and +fragments are the same with either. The `.mp4` name matches the contents, so +tools that pick a reader or a MIME type from the extension get MP4. QuickTime +and Safari playback of Linux recordings has not been tested. + When audio is requested, encoded video waits until each audio track starts or ends. A track that never supplies a sample would otherwise make the whole movie unreadable. If waiting video reaches three seconds or 32 MiB, recording continues @@ -55,9 +67,10 @@ The movie writer uses frame timestamps for playback timing. Refresh-rate changes can also change the encoder's parameter sets. The movie keeps its initial track description and carries those parameter sets with each keyframe, using `avc3` for H.264 and `hev1` for HEVC. This avoids changing the sample -description in a fragmented movie, which can make older qtmux versions write -unplayable fragments. Playback requires a reader that supports these sample -entries; FFmpeg playback is covered by the recording tests. +description in a fragmented movie, which can make older versions of +GStreamer's movie muxer write unplayable fragments. Playback requires a reader +that supports these sample entries; FFmpeg playback is covered by the recording +tests. When an encoder omits a frame's duration, wrec supplies one frame at the current capture rate, falling back to the requested rate when the encoded caps have no rate. The last frame then has a nonzero duration in the finalized movie. @@ -205,6 +218,58 @@ Wayland needs PipeWire and the ScreenCast portal backend for the desktop, such a `xdg-desktop-portal-gnome` or `xdg-desktop-portal-kde`. Use the backend for your desktop. X11 needs an accessible X server and the `ximagesrc` plugin. +### Portal frontend hangs on xdg-desktop-portal before 1.22.1 + +The portal frontend (`xdg-desktop-portal`) before 1.22.1 loads PipeWire's +realtime module each time it opens a PipeWire remote for a screen cast. That +module asks the frontend's own Realtime portal for limits over D-Bus, and the +frontend can deadlock on itself until the D-Bus call times out after 25 +seconds. While it waits, every portal call stalls. After you approve a source, +the job fails with "Opening the PipeWire remote timed out" or "ScreenCast +portal did not respond within 5s". + +This was reproduced on KDE Plasma 6.6.6 with Ubuntu's +`xdg-desktop-portal 1.21.1+ds-1ubuntu3.1` and PipeWire 1.6.2. A separate +GStreamer client reproduced it with the wrec daemon stopped: five screen casts +worked and the sixth hung. Installing RealtimeKit did not help; a session still +hung on the sixth try. The fault is in the portal frontend, not in wrec or the +desktop backend, so it can affect any desktop that runs an affected frontend. + +Upstream commit +[89f2f5e](https://github.com/flatpak/xdg-desktop-portal/commit/89f2f5e3d219bc5fd66a2505ee772b16022e8575) +("pipewire: Disable loading module-rt") fixes it and is in xdg-desktop-portal +1.22.1. Upgrade the frontend through your distribution when it ships 1.22.1 or +a build with that commit. That version was read but not run in wrec's tests. + +Until then, you can turn the realtime module off for the frontend only. wrec +never changes the portal setup itself. On the KDE system above, a frontend +started with this setting passed 10 of 10 repeated screen casts and the full +recording checks. Removing it brought the hang back on the second screen cast. +The commands below apply this setting through a systemd drop-in. Copy +PipeWire's client config and set +`module.rt = false` in its `context.properties`: + +```bash +mkdir -p ~/.config/pipewire ~/.config/systemd/user/xdg-desktop-portal.service.d +sed 's/^context.properties = {/&\n module.rt = false/' \ + /usr/share/pipewire/client.conf > ~/.config/pipewire/wrec-portal.conf +printf '[Service]\nEnvironment=PIPEWIRE_CONFIG_NAME=wrec-portal.conf\n' \ + > ~/.config/systemd/user/xdg-desktop-portal.service.d/wrec-portal.conf +systemctl --user daemon-reload +systemctl --user restart xdg-desktop-portal.service +``` + +Check that `~/.config/pipewire/wrec-portal.conf` has `module.rt = false` right +under `context.properties = {`. Only the frontend service reads this file; +other PipeWire clients keep the stock config. To undo it after upgrading: + +```bash +rm ~/.config/systemd/user/xdg-desktop-portal.service.d/wrec-portal.conf \ + ~/.config/pipewire/wrec-portal.conf +systemctl --user daemon-reload +systemctl --user restart xdg-desktop-portal.service +``` + Install the driver for your GPU. Check `vainfo` and `gst-inspect-1.0 vah264enc` / `vah264lpenc` for Intel/AMD, or `gst-inspect-1.0 nvh264enc` for NVIDIA. HEVC uses the corresponding H.265 encoder. diff --git a/scripts/test-cli-linux.py b/scripts/test-cli-linux.py index 570f59a..d1f7bd4 100644 --- a/scripts/test-cli-linux.py +++ b/scripts/test-cli-linux.py @@ -48,7 +48,7 @@ def run(*args, success=True): result = run("record", "--target", "display:0", "--duration", "1s", "--json", success=False) assert "Wayland or X11 desktop session" in result.stdout + result.stderr - assert not list(Path(directory).rglob("*.mov")) + assert not list(Path(directory).rglob("*.mp4")) print("PASS: unsupported capture fails without a recording artifact") jobs = json.loads(run("jobs", "--json").stdout) From 2b6f322d16a8d05f97310edb7f82489b126b3919 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 8 Oct 2026 03:00:32 +0000 Subject: [PATCH 32/45] fix(linux): avoid unsafe PipeWire provider teardown --- crates/linux/src/pipeline.rs | 88 ++++++++++++++++++++++++-------- crates/linux/src/portal.rs | 32 ++++++++++-- crates/linux/src/portal_tests.rs | 5 +- packaging/linux/README.md | 7 ++- 4 files changed, 102 insertions(+), 30 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index fcc6065..274d2f1 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -2075,23 +2075,28 @@ mod tests { // nobody reads its 200 ms ring buffer in time it overwrites the oldest // audio and marks the next buffer DISCONT. A leaky queue does the same. fn ring_buffer_audio(name: &str) -> gst::Element { - lagging_audio(name, Duration::ZERO) + audio_bin( + name, + "queue leaky=downstream max-size-time=200000000 max-size-buffers=0 max-size-bytes=0", + ) } - // Audio that reaches the pipeline `lag` after it was captured. + // Audio that reaches the pipeline `lag` after it was captured. Unlike + // the ring buffer it never drops audio, however long its reader takes. fn lagging_audio(name: &str, lag: Duration) -> gst::Element { - gst::init().unwrap(); - let bin = gst::parse::bin_from_description( + audio_bin( + name, &format!( - "audiotestsrc name=producer is-live=true samplesperbuffer=480 ! {} queue leaky=downstream max-size-time=200000000 max-size-buffers=0 max-size-bytes=0", - if lag.is_zero() { - String::new() - } else { - // Before the ring buffer, which must stay the only place - // audio can wait. - format!("queue max-size-time=0 max-size-buffers=0 max-size-bytes=0 min-threshold-time={} !", lag.as_nanos()) - } + "queue max-size-time=0 max-size-buffers=0 max-size-bytes=0 min-threshold-time={}", + lag.as_nanos() ), + ) + } + + fn audio_bin(name: &str, queue: &str) -> gst::Element { + gst::init().unwrap(); + let bin = gst::parse::bin_from_description( + &format!("audiotestsrc name=producer is-live=true samplesperbuffer=480 ! {queue}"), true, ) .unwrap(); @@ -2454,7 +2459,9 @@ mod tests { // Audio captured before a pause but arriving after it is dropped by // design. A source that delivers late makes that drop longer than a - // lost-audio hole. + // lost-audio hole. The source itself loses no audio, so any audio + // reported lost was dropped by pausing. Video frames a starved encoder + // drops are real losses, reported as such. #[test] fn audio_dropped_by_pausing_is_not_reported_lost() { let mut recording = TestRecording::start_with_audio(vec![( @@ -2472,7 +2479,9 @@ mod tests { recording.probe(); for event in recording.events.try_iter() { match event { - RecorderEvent::MediaLost { message, .. } => panic!("{message}"), + RecorderEvent::MediaLost { message, .. } => { + assert!(!message.contains(" of audio"), "{message}") + } RecorderEvent::Log { message, .. } => { assert!(!message.contains(" lost "), "{message}") } @@ -2686,8 +2695,28 @@ mod tests { assert!(near(lost, 0.0), "{lost}"); } + // The queue at the end of an audio source's branch, in front of the + // muxer. + fn audio_branch_end(pipeline: &gst::Pipeline, source: &str) -> gst::Element { + let mut element = pipeline.by_name(source).unwrap(); + while !matches!(element.factory(), Some(factory) if factory.name() == "queue") { + element = element + .static_pad("src") + .unwrap() + .peer() + .unwrap() + .parent_element() + .unwrap(); + } + element + } + // A source that reports a fatal error but leaves its branch open still - // ends its track, so the muxer does not wait for it. + // ends its track, so the muxer does not wait for it. Video waits for an + // audio track for up to HELD_VIDEO_LIMIT before any is dropped, so the + // track has to end at the muxer sooner. The movie's video is no measure + // of this: a starved encoder drops frames at capture whether or not a + // source failed. #[test] fn a_failed_audio_source_that_keeps_its_branch_open_still_ends_its_track() { let mut recording = TestRecording::start_with_audio(vec![ @@ -2695,7 +2724,8 @@ mod tests { ("microphone", ring_buffer_audio("microphone")), ]); let start = Instant::now(); - let reported = std::sync::atomic::AtomicBool::new(false); + let failed = Arc::new(Mutex::new(None)); + let failing = failed.clone(); producer(&recording.pipeline, "microphone") .static_pad("src") .unwrap() @@ -2703,19 +2733,35 @@ mod tests { if start.elapsed() < Duration::from_secs(2) { return gst::PadProbeReturn::Ok; } - if !reported.swap(true, Ordering::SeqCst) { + let mut failing = failing.lock().unwrap(); + if failing.is_none() { + *failing = Some(Instant::now()); let source = pad.parent_element().unwrap(); gst::element_error!(source, gst::ResourceError::Read, ("device unplugged")); } gst::PadProbeReturn::Drop }); + let ended = Arc::new(Mutex::new(None)); + let ending = ended.clone(); + audio_branch_end(&recording.pipeline, "microphone") + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::EVENT_DOWNSTREAM, move |_, info| { + if info + .event() + .is_some_and(|event| event.type_() == gst::EventType::Eos) + { + ending.lock().unwrap().get_or_insert_with(Instant::now); + } + gst::PadProbeReturn::Ok + }); std::thread::sleep(Duration::from_secs(8)); recording.finish().unwrap(); - let probe = recording.probe(); - let missing = missing_video(&probe); + recording.probe(); + let waited = ended.lock().unwrap().unwrap() - failed.lock().unwrap().unwrap(); assert!( - missing < 0.05, - "{missing}s of video lost waiting for the failed microphone" + waited < Duration::from_nanos(HELD_VIDEO_LIMIT.nseconds()), + "the muxer waited {waited:?} for the failed microphone" ); let message = recording .events diff --git a/crates/linux/src/portal.rs b/crates/linux/src/portal.rs index 8aa1515..f2c3cee 100644 --- a/crates/linux/src/portal.rs +++ b/crates/linux/src/portal.rs @@ -144,9 +144,7 @@ impl NodeWatch { } async fn watch(remote: PipeWireStream) -> Result { - gst::init().map_err(backend)?; - let provider = gst::DeviceProviderFactory::by_name("pipewiredeviceprovider") - .ok_or_else(|| backend("GStreamer's PipeWire device provider is not installed"))?; + let provider = pipewire_provider()?; provider.set_property("fd", remote.fd.as_raw_fd()); let bus = provider.bus(); let (stop, stops) = mpsc::channel(); @@ -178,8 +176,9 @@ impl NodeWatch { break; } } - provider.stop(); - drop(remote); + // Never stopped; see pipewire_provider. Drop its later messages. + bus.set_flushing(true); + std::mem::forget((provider, remote)); let _ = finished.send(()); }) .map_err(backend)?; @@ -203,6 +202,29 @@ impl Drop for NodeWatch { } } +// A new PipeWire device provider. Callers start it and then keep it, and the fd +// it connects through, until the process exits. A capture worker records once, +// so that is one idle connection. Before PipeWire 0.3.78 (Debian 12 ships +// 0.3.65) the provider cannot be torn down safely. stop() destroys its registry +// without the PipeWire loop lock (fixed upstream in b0a7e4a2 and 140374d2), so +// an event arriving then corrupts the heap. stop() and probing also clear the +// provider's connection before draining its events, so a late node update +// dereferences NULL. The plugin shares connections by fd number, so the fd must +// stay open while the connection lives. DeviceProviderFactory::by_name returns +// one provider for the whole process, so each caller gets a new one instead. +fn pipewire_provider() -> Result { + gst::init().map_err(backend)?; + let factory = gst::DeviceProviderFactory::find("pipewiredeviceprovider") + .ok_or_else(|| backend("GStreamer's PipeWire device provider is not installed"))? + .load() + .map_err(backend)? + .downcast::() + .map_err(|_| backend("GStreamer's PipeWire device provider did not load"))?; + gst::glib::Object::with_type(factory.device_provider_type()) + .downcast::() + .map_err(|_| backend("GStreamer's PipeWire device provider did not load")) +} + fn node_id(device: &gst::Device) -> Option { device .has_property("id", Some(gst::glib::Type::U32)) diff --git a/crates/linux/src/portal_tests.rs b/crates/linux/src/portal_tests.rs index f58a350..a749801 100644 --- a/crates/linux/src/portal_tests.rs +++ b/crates/linux/src/portal_tests.rs @@ -334,9 +334,9 @@ impl PrivatePipeWire { } fn node(&self, name: &str) -> u32 { - let provider = gst::DeviceProviderFactory::by_name("pipewiredeviceprovider").unwrap(); for _ in 0..50 { let fd = self.connect(); + let provider = pipewire_provider().unwrap(); provider.set_property("fd", fd.as_raw_fd()); provider.start().unwrap(); let found = provider.devices().iter().find_map(|device| { @@ -345,7 +345,8 @@ impl PrivatePipeWire { .then(|| node_id(device)) .flatten() }); - provider.stop(); + // Kept like NodeWatch's provider: see pipewire_provider. + std::mem::forget((provider, fd)); if let Some(node) = found { return node; } diff --git a/packaging/linux/README.md b/packaging/linux/README.md index 8aa464f..b4d5cd9 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -380,7 +380,9 @@ visible on the second remote, or PipeWire does not list its nodes within 5 seconds, the job logs that stream removal will not be detected and records without the watch. If a desktop removes the node before its `Closed` signal arrives, a stop from its sharing indicator fails the job with the stream-ended -message instead of finalizing it. +message instead of finalizing it. The watch never stops its device provider. +Before PipeWire 0.3.78, which includes Debian 12's 0.3.65, stopping the provider +can corrupt memory, so its connection stays open until the capture worker exits. Stop drains the encoders and finalizes the movie, including when paused. If video still has not reached the movie writer 10 seconds after stop while nothing @@ -436,7 +438,8 @@ without consuming socket data. The node watch test starts a private PipeWire daemon in a temporary directory, without the session bus. It checks that an idle node is not reported, that removing the node is reported, that a missing node disables the watch instead of failing, and that the watch stops promptly -after the daemon dies. +after the daemon dies. On PipeWire before 0.3.78, running it under +`taskset -c 0` makes a provider teardown race fail reliably if one returns. Real hardware validation on Corex (Ryzen 9 5900H / Radeon Cezanne, Ubuntu 26.04, GNOME Wayland, GStreamer 1.28.2) recorded H.264 displays and an animated HEVC From 2d9cf2a45986f511280a6c6342299d644a887cde Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 8 Oct 2026 12:51:33 +0000 Subject: [PATCH 33/45] fix(linux): preserve audio timing through brief recorder stalls --- .github/workflows/ci.yml | 4 +- crates/linux/src/pipeline.rs | 224 ++++++++++++++++++++++++++++++++++- packaging/linux/README.md | 20 +++- 3 files changed, 243 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07214af..a6706b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,8 @@ jobs: libgstreamer-plugins-base1.0-dev gstreamer1.0-pipewire \ gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly \ - gstreamer1.0-libav ffmpeg dbus-daemon xvfb x11-apps pipewire + gstreamer1.0-libav ffmpeg dbus-daemon xvfb x11-apps pipewire \ + pulseaudio - name: Build and test run: | cargo fmt --check @@ -40,6 +41,7 @@ jobs: cargo test --workspace --locked dbus-run-session -- cargo test -p linux portal_roundtrip --locked -- --ignored cargo test -p linux node_watch --locked -- --ignored + cargo test -p linux pulse_server --locked -- --ignored --test-threads=1 ./scripts/package-cli-linux.sh - name: Test packaged CLI run: | diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 274d2f1..cd4a183 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -821,12 +821,88 @@ fn add_audio( name: &'static str, counters: &Arc, ) -> Result<()> { + add_audio_source(movie, &pulse_source(device)?, name, counters) +} + +fn pulse_source(device: Option<&str>) -> Result { let source = element("pulsesrc")?; source.set_property("provide-clock", false); + source.set_property("buffer-time", AUDIO_BUFFER.useconds() as i64); + source.set_property_from_str("slave-method", "none"); if let Some(device) = device { source.set_property("device", device); } - add_audio_source(movie, &source, name, counters) + stamp_by_arrival(&source); + Ok(source) +} + +// pulsesrc keeps this much audio for a reader that stops, and asks its +// server to hold as much: 384 kB per track at F32 stereo. +const AUDIO_BUFFER: gst::ClockTime = gst::ClockTime::SECOND; + +// pulsesrc's default buffer-time, so how late audio could get before it +// restamped. Smaller changes, like a stream's latency settling after it +// starts, are not lost audio. +const AUDIO_LATE: gst::ClockTime = gst::ClockTime::from_mseconds(200); + +// audiobasesink's default alignment-threshold: timestamps within this of +// each other are arrival jitter. +const AUDIO_JITTER: gst::ClockTime = gst::ClockTime::from_mseconds(40); + +// Slaved to the pipeline clock, pulsesrc restamps at the clock whenever the +// clock gets a whole buffer ahead of the audio read so far. After the engine +// stops for a moment that happens before it has read the audio its server +// held meanwhile, so that audio and everything after it plays late, and the +// restamp reads as lost audio. Unslaved, pulsesrc stamps by sample count and +// the held audio keeps its place; this places the count in running time. The +// first buffer ends when it arrives. Reading held audio makes it less late +// with every buffer, and losing more makes it later. Audio that instead +// stays more than AUDIO_LATE late, within AUDIO_JITTER, for AUDIO_BUFFER of +// audio lost samples on the way, so later audio moves later by the least it +// was late. count_lost_audio sees the gap. +fn stamp_by_arrival(source: &gst::Element) { + // How far timestamps sit from the sample count, and, while audio is + // late, where that began, how late it was then, and the least since. + let state = Mutex::new((None::, None::<(gst::ClockTime, i64, i64)>)); + let element = source.downgrade(); + source + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + let now = element + .upgrade() + .and_then(|element| element.current_running_time()); + let (Some(now), Some(gst::PadProbeData::Buffer(buffer))) = (now, info.data.as_mut()) + else { + return gst::PadProbeReturn::Ok; + }; + let Some((pts, duration)) = buffer.pts().zip(buffer.duration()) else { + return gst::PadProbeReturn::Ok; + }; + let arrived = now.nseconds() as i64 - (pts + duration).nseconds() as i64; + let mut state = state.lock().unwrap(); + let (offset, late) = &mut *state; + let offset = offset.get_or_insert(arrived); + let behind = arrived - *offset; + match late { + _ if behind <= AUDIO_LATE.nseconds() as i64 => *late = None, + Some((start, first, least)) + if (behind - *first).abs() <= AUDIO_JITTER.nseconds() as i64 => + { + *least = (*least).min(behind); + if pts.saturating_sub(*start) >= AUDIO_BUFFER { + *offset += *least; + *late = None; + } + } + _ => *late = Some((pts, behind, behind)), + } + let pts = (pts.nseconds() as i64 + *offset).max(0) as u64; + buffer + .make_mut() + .set_pts(gst::ClockTime::from_nseconds(pts)); + gst::PadProbeReturn::Ok + }); } fn add_audio_source( @@ -874,7 +950,8 @@ fn add_audio_source( } // pulsesrc keeps buffers contiguous unless it skipped audio nobody read in -// time, so a longer jump is lost audio. +// time or stamp_by_arrival moved past audio lost on the way, so a longer +// jump is lost audio. const AUDIO_HOLE: gst::ClockTime = gst::ClockTime::from_mseconds(20); // Counts audio missing between consecutive captured buffers, before pausing @@ -2490,6 +2567,149 @@ mod tests { } } + // A private PulseAudio server playing a tone into a null sink, like a + // desktop's system audio. + struct PulseServer { + process: std::process::Child, + directory: std::path::PathBuf, + } + + impl PulseServer { + fn start() -> Self { + static ID: AtomicU64 = AtomicU64::new(0); + let directory = std::env::temp_dir().join(format!( + "wrec-linux-test-pulse-{}-{}", + std::process::id(), + ID.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&directory).unwrap(); + let socket = directory.join("native"); + let config = directory.join("default.pa"); + std::fs::write( + &config, + format!( + "load-module module-native-protocol-unix socket={} auth-anonymous=1\n\ + load-module module-null-sink sink_name=wrec_test rate=48000 channels=2\n\ + load-module module-sine sink=wrec_test frequency=440\n", + socket.display() + ), + ) + .unwrap(); + let process = std::process::Command::new("pulseaudio") + .args(["-n", "--daemonize=no", "--exit-idle-time=-1"]) + .args(["--use-pid-file=no", "--system=no", "-F"]) + .arg(&config) + .env("HOME", &directory) + .env("XDG_RUNTIME_DIR", &directory) + .env("PULSE_RUNTIME_PATH", &directory) + .env( + "DBUS_SESSION_BUS_ADDRESS", + format!("unix:path={}", directory.join("no-bus").display()), + ) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("pulseaudio is not installed"); + let server = Self { process, directory }; + let deadline = Instant::now() + Duration::from_secs(10); + while !socket.exists() { + assert!(Instant::now() < deadline, "pulseaudio did not start"); + std::thread::sleep(Duration::from_millis(50)); + } + server + } + + fn source(&self) -> gst::Element { + gst::init().unwrap(); + let source = pulse_source(Some("wrec_test.monitor")).unwrap(); + let socket = self.directory.join("native"); + source.set_property("server", format!("unix:{}", socket.display())); + source + } + + // The server makes no audio meanwhile, then catches up at once, like + // a server that held the audio of an engine that stopped. + fn stop_for(&self, stall: Duration) { + let pid = self.process.id() as libc::pid_t; + unsafe { libc::kill(pid, libc::SIGSTOP) }; + std::thread::sleep(stall); + unsafe { libc::kill(pid, libc::SIGCONT) }; + } + } + + impl Drop for PulseServer { + fn drop(&mut self) { + unsafe { libc::kill(self.process.id() as libc::pid_t, libc::SIGCONT) }; + let _ = self.process.kill(); + let _ = self.process.wait(); + let _ = std::fs::remove_dir_all(&self.directory); + } + } + + // Records a tone through pulsesrc while its server stops once. Returns + // the seconds missing from the audio track, the seconds reported lost, + // and where the audio ends after the video. + fn record_server_stop(stall: Duration) -> (f64, f64, f64) { + let server = PulseServer::start(); + let mut recording = + TestRecording::start_with_audio(vec![("system audio", server.source())]); + std::thread::sleep(Duration::from_secs(3)); + server.stop_for(stall); + std::thread::sleep(Duration::from_secs(4)); + recording.finish().unwrap(); + let probe = recording.probe(); + let reported = recording + .events + .try_iter() + .find_map(|event| match event { + RecorderEvent::MediaLost { message, .. } => Some(message), + _ => None, + }) + .and_then(|message| { + let (_, rest) = message.split_once("system audio is missing ")?; + rest.split_once(" s")?.0.parse().ok() + }) + .unwrap_or(0.0); + let after_video = end_seconds(&probe, "aac") - end_seconds(&probe, "h264"); + (audio_holes(&probe)[0], reported, after_video) + } + + // Slaved to the clock, pulsesrc stamped the audio the server made up late, + // and reported almost half a second lost. Run with + // `cargo test -p linux pulse_server -- --ignored`. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_held_keeps_its_place() { + let (hole, reported, after_video) = record_server_stop(Duration::from_millis(250)); + assert!( + hole == 0.0 && reported == 0.0, + "{hole}s gap, {reported}s reported" + ); + assert!( + after_video.abs() < 0.05, + "audio ends {after_video}s after video" + ); + } + + // PulseAudio's null sink makes up little of a 1.5 s stop, so most of it + // is lost. The movie has that gap, the same amount is reported, and the + // audio after it plays within AUDIO_LATE of the video, which is as close + // as a loss too small to see can leave it. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_lost_is_reported_and_the_rest_keeps_its_place() { + let (hole, reported, after_video) = record_server_stop(Duration::from_millis(1500)); + assert!(hole > 0.5, "{hole}s gap"); + assert!( + (reported - hole).abs() < 0.05, + "{reported}s reported, {hole}s gap" + ); + assert!( + after_video.abs() < AUDIO_LATE.nseconds() as f64 / 1e9, + "audio ends {after_video}s after video" + ); + } + // Stops a recording `after` something in it got stuck at two seconds. // Returns the finished recording, its result and how long after stop // run() returned. Like pulsesrc, the audio sources' streaming threads diff --git a/packaging/linux/README.md b/packaging/linux/README.md index b4d5cd9..2627ebf 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -339,6 +339,19 @@ available. Configure devices in desktop sound settings. Linux cannot apply the shared wrec window-hiding or custom microphone-indicator options; job settings report them disabled with a warning. The desktop controls its sharing indicator. +Each audio source retains up to one second of raw audio, about 384 kB per +F32 stereo track, while capture latency remains 10 ms. Sample-count timestamps +preserve audio held through a short recorder stall. If audio instead stays more +than 200 ms late with steady arrival timing for a second, wrec moves later +samples back into alignment and reports the corresponding gap. + +This recovery has limits. After real loss, the correction appears about a second +later, and the intervening audio can play early by the amount lost. Loss below +200 ms can remain undetected. A stall beyond the one-second source buffer can +lose samples and leave about 100 ms of offset, as observed in the controlled +1.5-second stall. Physical microphone clock drift is not validated. These are +experimental support limits; a completed job alone does not establish A/V sync. + The movie writer takes the next sample of every track in time order, so a track that stops delivering holds up the others. Encoded video and each audio track can wait about 4 seconds for the rest. Encoded movie data can wait for the disk @@ -406,19 +419,22 @@ desktop. HDR is not supported. ## Validate ```bash -sudo apt install ffmpeg dbus-daemon xvfb x11-apps pipewire +sudo apt install ffmpeg dbus-daemon xvfb x11-apps pipewire pulseaudio cargo fmt --check cargo check --workspace --locked cargo test --workspace --locked dbus-run-session -- cargo test -p linux portal_roundtrip --locked -- --ignored cargo test -p linux node_watch --locked -- --ignored +cargo test -p linux pulse_server --locked -- --ignored --test-threads=1 cargo build -p cli -p daemon python3 scripts/test-capture-linux.py target/debug/wrec ``` The full test suite requires `x265enc` from the GStreamer runtime packages listed above. The HEVC latency regression fails if that encoder is absent, -so a passing suite always includes it. +so a passing suite always includes it. The private PulseAudio regressions stop +only their own server to check held audio, real loss reporting and A/V alignment; +they require `pulseaudio` and run separately with the command above. The isolated Xvfb test records actual X11 display/window pixels through the CLI and daemon, checks H.264/HEVC decoding and timestamps, and exercises pause/resume From 3b34ed793776d375c3a1dbb218ae6381239bf171 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:24:36 +0000 Subject: [PATCH 34/45] fix(linux): place audio using server capture timing --- crates/daemon/src/runtime.rs | 2 +- crates/linux/src/lib.rs | 1 + crates/linux/src/pipeline.rs | 856 ++++++++++++++++++++++++------ crates/linux/src/pulse.rs | 973 +++++++++++++++++++++++++++++++++++ packaging/linux/README.md | 49 +- 5 files changed, 1720 insertions(+), 161 deletions(-) create mode 100644 crates/linux/src/pulse.rs diff --git a/crates/daemon/src/runtime.rs b/crates/daemon/src/runtime.rs index 16bfefd..b3ee455 100644 --- a/crates/daemon/src/runtime.rs +++ b/crates/daemon/src/runtime.rs @@ -164,7 +164,7 @@ impl RecordingRuntime for LinuxRuntime { fn microphone_permission_status(&self) -> Result { // Native PulseAudio clients have no separate TCC permission prompt. - // The audio server checks access when pulsesrc connects. + // The audio server checks access when the recording connects. Ok(PermissionStatus::Granted) } diff --git a/crates/linux/src/lib.rs b/crates/linux/src/lib.rs index 1321cb6..c444e03 100644 --- a/crates/linux/src/lib.rs +++ b/crates/linux/src/lib.rs @@ -4,6 +4,7 @@ mod desktop; mod encoding; mod pipeline; mod portal; +mod pulse; mod worker; mod x11; diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index cd4a183..40a376c 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -37,7 +37,7 @@ pub(crate) fn check_plugins(settings: &RecorderSettings) -> Result<()> { } if settings.include_system_audio || settings.include_microphone { for factory in [ - "pulsesrc", + "appsrc", "audioconvert", "audioresample", "avenc_aac", @@ -45,6 +45,7 @@ pub(crate) fn check_plugins(settings: &RecorderSettings) -> Result<()> { ] { element(factory)?; } + crate::pulse::available()?; } Ok(()) } @@ -610,7 +611,7 @@ fn keep_capture_timestamps(encoder: &gst::Element, parser: &gst::Element) { // limit, the movie starts without the audio tracks that have not delivered a // buffer. Paused time does not count, since no video arrives while paused. // Constant-QP output has no fixed bitrate, so bytes are bounded separately. -const HELD_VIDEO_LIMIT: gst::ClockTime = gst::ClockTime::from_seconds(3); +pub(crate) const HELD_VIDEO_LIMIT: gst::ClockTime = gst::ClockTime::from_seconds(3); const HELD_VIDEO_BYTES: u32 = 32 << 20; const HELD_QUEUE: &str = "held-video-queue"; @@ -712,7 +713,7 @@ impl Movie { let pad = queue.static_pad("src").unwrap(); // Linking the track sends a reconfigure upstream. The encoder answers // with an allocation query, which waits behind this queue while mp4mux - // holds audio for the next video frame, and pulsesrc loses audio + // holds audio for the next video frame, and the audio source fills // meanwhile. The audio caps are fixed, so nothing needs it. pad.add_probe(gst::PadProbeType::EVENT_UPSTREAM, |_, info| { if info @@ -825,86 +826,13 @@ fn add_audio( } fn pulse_source(device: Option<&str>) -> Result { - let source = element("pulsesrc")?; - source.set_property("provide-clock", false); - source.set_property("buffer-time", AUDIO_BUFFER.useconds() as i64); - source.set_property_from_str("slave-method", "none"); - if let Some(device) = device { - source.set_property("device", device); - } - stamp_by_arrival(&source); - Ok(source) + crate::pulse::source(None, device, AUDIO_BUFFER) } -// pulsesrc keeps this much audio for a reader that stops, and asks its -// server to hold as much: 384 kB per track at F32 stereo. +// Each audio source keeps this much audio while downstream stops taking it, +// and asks its server to hold as much more: 384 kB each at F32 stereo. const AUDIO_BUFFER: gst::ClockTime = gst::ClockTime::SECOND; -// pulsesrc's default buffer-time, so how late audio could get before it -// restamped. Smaller changes, like a stream's latency settling after it -// starts, are not lost audio. -const AUDIO_LATE: gst::ClockTime = gst::ClockTime::from_mseconds(200); - -// audiobasesink's default alignment-threshold: timestamps within this of -// each other are arrival jitter. -const AUDIO_JITTER: gst::ClockTime = gst::ClockTime::from_mseconds(40); - -// Slaved to the pipeline clock, pulsesrc restamps at the clock whenever the -// clock gets a whole buffer ahead of the audio read so far. After the engine -// stops for a moment that happens before it has read the audio its server -// held meanwhile, so that audio and everything after it plays late, and the -// restamp reads as lost audio. Unslaved, pulsesrc stamps by sample count and -// the held audio keeps its place; this places the count in running time. The -// first buffer ends when it arrives. Reading held audio makes it less late -// with every buffer, and losing more makes it later. Audio that instead -// stays more than AUDIO_LATE late, within AUDIO_JITTER, for AUDIO_BUFFER of -// audio lost samples on the way, so later audio moves later by the least it -// was late. count_lost_audio sees the gap. -fn stamp_by_arrival(source: &gst::Element) { - // How far timestamps sit from the sample count, and, while audio is - // late, where that began, how late it was then, and the least since. - let state = Mutex::new((None::, None::<(gst::ClockTime, i64, i64)>)); - let element = source.downgrade(); - source - .static_pad("src") - .unwrap() - .add_probe(gst::PadProbeType::BUFFER, move |_, info| { - let now = element - .upgrade() - .and_then(|element| element.current_running_time()); - let (Some(now), Some(gst::PadProbeData::Buffer(buffer))) = (now, info.data.as_mut()) - else { - return gst::PadProbeReturn::Ok; - }; - let Some((pts, duration)) = buffer.pts().zip(buffer.duration()) else { - return gst::PadProbeReturn::Ok; - }; - let arrived = now.nseconds() as i64 - (pts + duration).nseconds() as i64; - let mut state = state.lock().unwrap(); - let (offset, late) = &mut *state; - let offset = offset.get_or_insert(arrived); - let behind = arrived - *offset; - match late { - _ if behind <= AUDIO_LATE.nseconds() as i64 => *late = None, - Some((start, first, least)) - if (behind - *first).abs() <= AUDIO_JITTER.nseconds() as i64 => - { - *least = (*least).min(behind); - if pts.saturating_sub(*start) >= AUDIO_BUFFER { - *offset += *least; - *late = None; - } - } - _ => *late = Some((pts, behind, behind)), - } - let pts = (pts.nseconds() as i64 + *offset).max(0) as u64; - buffer - .make_mut() - .set_pts(gst::ClockTime::from_nseconds(pts)); - gst::PadProbeReturn::Ok - }); -} - fn add_audio_source( movie: &mut Movie, source: &gst::Element, @@ -949,17 +877,15 @@ fn add_audio_source( Ok(()) } -// pulsesrc keeps buffers contiguous unless it skipped audio nobody read in -// time or stamp_by_arrival moved past audio lost on the way, so a longer -// jump is lost audio. -const AUDIO_HOLE: gst::ClockTime = gst::ClockTime::from_mseconds(20); +// Audio sources keep buffers contiguous except where their server lost or +// skipped audio, so a longer jump is lost audio. +pub(crate) const AUDIO_HOLE: gst::ClockTime = gst::ClockTime::from_mseconds(20); // Counts audio missing between consecutive captured buffers, before pausing -// drops any. When downstream does not read pulsesrc in time, its ring buffer -// overwrites the oldest audio, and the next buffer is stamped where capture -// actually resumed. Sources keep capturing while paused, so a pause leaves no -// gap, and the paused part of a gap is not in the movie. Capture timestamps -// and pauses are both in running time. +// drops any. Audio a source's server lost leaves the next buffer stamped +// where capture actually resumed. Sources keep capturing while paused, so a +// pause leaves no gap, and the paused part of a gap is not in the movie. +// Capture timestamps and pauses are both in running time. fn count_lost_audio(source: &gst::Element, lost: Arc, counters: Arc) { // Where the next buffer should start. let next = Mutex::new(None::); @@ -1321,13 +1247,13 @@ fn run( if let Some(message) = bus.timed_pop(gst::ClockTime::from_mseconds(100)) { match message.view() { gst::MessageView::Error(error) => { - // A failed audio source, such as pulsesrc after its server - // went away, ends its own track, and the movie keeps + // A failed audio source, such as one whose server went + // away, ends its own track, and the movie keeps // recording the others. Other audio errors fail the job. if let Some(track) = movie.audio_source(&message) { if track.failed.is_none() { - // pulsesrc ends its branch after a fatal error; make - // sure, without waiting on its stream lock here. + // End its branch, without waiting on its stream + // lock here. if let Some(branch) = track.source.static_pad("src").and_then(|pad| pad.peer()) { @@ -1867,6 +1793,8 @@ mod tests { .unwrap() .downcast::() .unwrap(); + // Like production. + pipeline.use_clock(Some(&gst::SystemClock::obtain())); // Like production, frames the encoder cannot take are dropped // before encoding instead of stalling capture. let queue = video_queue().unwrap(); @@ -2148,9 +2076,9 @@ mod tests { source } - // Live capture like pulsesrc: the producer keeps real time, and when - // nobody reads its 200 ms ring buffer in time it overwrites the oldest - // audio and marks the next buffer DISCONT. A leaky queue does the same. + // Live capture whose producer keeps real time, and when nobody reads its + // 200 ms ring buffer in time overwrites the oldest audio and marks the + // next buffer DISCONT. A leaky queue does the same. fn ring_buffer_audio(name: &str) -> gst::Element { audio_bin( name, @@ -2264,8 +2192,8 @@ mod tests { Encoder, // The microphone captures nothing. AudioProducer, - // The microphone's server goes away, like pulsesrc after a - // PipeWire or PulseAudio restart. + // The microphone's server goes away, as in a PipeWire or + // PulseAudio restart. AudioFailure, } @@ -2576,6 +2504,12 @@ mod tests { impl PulseServer { fn start() -> Self { + Self::start_with("", true) + } + + // `config` adds modules. Without shm all audio goes through the + // socket, so a SlowLink can carry it. + fn start_with(config: &str, shm: bool) -> Self { static ID: AtomicU64 = AtomicU64::new(0); let directory = std::env::temp_dir().join(format!( "wrec-linux-test-pulse-{}-{}", @@ -2584,21 +2518,23 @@ mod tests { )); std::fs::create_dir_all(&directory).unwrap(); let socket = directory.join("native"); - let config = directory.join("default.pa"); + let script = directory.join("default.pa"); std::fs::write( - &config, + &script, format!( "load-module module-native-protocol-unix socket={} auth-anonymous=1\n\ load-module module-null-sink sink_name=wrec_test rate=48000 channels=2\n\ - load-module module-sine sink=wrec_test frequency=440\n", - socket.display() + load-module module-sine sink=wrec_test frequency=440\n{}", + socket.display(), + config.replace("{dir}", &directory.display().to_string()) ), ) .unwrap(); let process = std::process::Command::new("pulseaudio") .args(["-n", "--daemonize=no", "--exit-idle-time=-1"]) .args(["--use-pid-file=no", "--system=no", "-F"]) - .arg(&config) + .arg(&script) + .arg(format!("--disable-shm={}", !shm)) .env("HOME", &directory) .env("XDG_RUNTIME_DIR", &directory) .env("PULSE_RUNTIME_PATH", &directory) @@ -2620,11 +2556,17 @@ mod tests { } fn source(&self) -> gst::Element { + self.source_from("wrec_test.monitor", &self.directory.join("native")) + } + + fn source_from(&self, device: &str, socket: &Path) -> gst::Element { gst::init().unwrap(); - let source = pulse_source(Some("wrec_test.monitor")).unwrap(); - let socket = self.directory.join("native"); - source.set_property("server", format!("unix:{}", socket.display())); - source + crate::pulse::source( + Some(&format!("unix:{}", socket.display())), + Some(device), + AUDIO_BUFFER, + ) + .unwrap() } // The server makes no audio meanwhile, then catches up at once, like @@ -2646,74 +2588,697 @@ mod tests { } } - // Records a tone through pulsesrc while its server stops once. Returns - // the seconds missing from the audio track, the seconds reported lost, - // and where the audio ends after the video. - fn record_server_stop(stall: Duration) -> (f64, f64, f64) { - let server = PulseServer::start(); - let mut recording = - TestRecording::start_with_audio(vec![("system audio", server.source())]); - std::thread::sleep(Duration::from_secs(3)); - server.stop_for(stall); - std::thread::sleep(Duration::from_secs(4)); + // Passes connections to a PulseAudio server through `socket`, holding + // back everything the server sends by `delay`, without dropping or + // reordering any of it, like a slower way to the server. + struct SlowLink { + socket: std::path::PathBuf, + delay: Arc, + } + + impl SlowLink { + fn new(server: &PulseServer) -> Self { + Self::between( + server.directory.join("native"), + server.directory.join("slow"), + ) + } + + fn between(target: std::path::PathBuf, socket: std::path::PathBuf) -> Self { + use std::io::{Read, Write}; + use std::os::unix::net::{UnixListener, UnixStream}; + let listener = UnixListener::bind(&socket).unwrap(); + let delay = Arc::new(AtomicU64::new(0)); + let link_delay = delay.clone(); + std::thread::spawn(move || { + for client in listener.incoming() { + let Ok(mut client) = client else { break }; + let Ok(mut upstream) = UnixStream::connect(&target) else { + break; + }; + let (mut from_client, mut from_server) = + (client.try_clone().unwrap(), upstream.try_clone().unwrap()); + std::thread::spawn(move || std::io::copy(&mut from_client, &mut upstream)); + let (held, holding) = mpsc::channel::<(Instant, Vec)>(); + let delay = link_delay.clone(); + std::thread::spawn(move || { + let mut bytes = vec![0; 65536]; + while let Ok(read @ 1..) = from_server.read(&mut bytes) { + let due = Instant::now() + + Duration::from_nanos(delay.load(Ordering::Relaxed)); + if held.send((due, bytes[..read].to_vec())).is_err() { + break; + } + } + }); + std::thread::spawn(move || { + for (due, bytes) in holding { + std::thread::sleep(due.saturating_duration_since(Instant::now())); + if client.write_all(&bytes).is_err() { + break; + } + } + }); + } + }); + Self { socket, delay } + } + + fn set_delay(&self, delay: Duration) { + self.delay.store(delay.as_nanos() as u64, Ordering::Relaxed); + } + } + + // Feeds module-pipe-source a tone in 10 ms blocks in real time, like a + // capture device. A block the pipe has no room for, because the server + // stopped reading, is lost, and stopping returns when each was. + struct PipeDevice { + stop: Arc, + writer: std::thread::JoinHandle>, + // When it wrote its first block, and how long each block is. + start: Instant, + block: Duration, + } + + impl PipeDevice { + const CONFIG: &str = "load-module module-pipe-source source_name=wrec_pipe file={dir}/fifo format=s16le rate=48000 channels=2\n"; + + fn start(server: &PulseServer) -> Self { + Self::start_with_clock(server, 0) + } + + // A device whose clock runs `ppm` parts per million fast. Each frame + // holds its index, in two channels of 15 bits, so a recording shows + // when the device wrote any frame of it. + fn start_with_clock(server: &PulseServer, ppm: i64) -> Self { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + let mut fifo = std::fs::OpenOptions::new() + .write(true) + .custom_flags(libc::O_NONBLOCK) + .open(server.directory.join("fifo")) + .unwrap(); + // 0.34 s of this audio, on any kernel. + unsafe { libc::fcntl(fifo.as_raw_fd(), libc::F_SETPIPE_SZ, 65536) }; + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stopping = stop.clone(); + let block = Duration::from_nanos((10_000_000_000_000 / (1_000_000 + ppm)) as u64); + let start = Instant::now(); + let writer = std::thread::spawn(move || { + let mut lost = Vec::new(); + for n in 0u32.. { + if stopping.load(Ordering::Relaxed) { + break; + } + let bytes: Vec = (n * 480..n * 480 + 480) + .flat_map(|frame| { + [(frame % 32768) as i16, (frame / 32768) as i16] + .into_iter() + .flat_map(i16::to_le_bytes) + }) + .collect(); + if fifo.write(&bytes).ok() != Some(bytes.len()) { + lost.push(Instant::now()); + } + std::thread::sleep( + (start + block * (n + 1)).saturating_duration_since(Instant::now()), + ); + } + lost + }); + Self { + stop, + writer, + start, + block, + } + } + + // Records, for each buffer `source` sends, its pts and how far that is + // from when the device wrote its first frame, both in seconds, and + // when that was. + fn placement(&self, source: &gst::Element) -> Arc>> { + let placed = Arc::new(Mutex::new(Vec::new())); + let (start, block, placing) = (self.start, self.block, placed.clone()); + let element = source.downgrade(); + source.static_pad("src").unwrap().add_probe( + gst::PadProbeType::BUFFER, + move |_, info| { + let (Some(buffer), Some(now)) = ( + info.buffer(), + element + .upgrade() + .and_then(|element| element.current_running_time()), + ) else { + return gst::PadProbeReturn::Ok; + }; + let at = Instant::now(); + let map = buffer.map_readable().unwrap(); + let sample = |i: usize| { + (f32::from_le_bytes(map[i * 4..i * 4 + 4].try_into().unwrap()) * 32768.0) + .round() as u64 + }; + let frame = sample(0) + sample(1) * 32768; + let written = start + block * (frame / 480) as u32; + let expected = now.nseconds() as f64 / 1e9 + - at.saturating_duration_since(written).as_secs_f64() + + written.saturating_duration_since(at).as_secs_f64(); + let pts = buffer.pts().unwrap().nseconds() as f64 / 1e9; + placing.lock().unwrap().push((pts, pts - expected, written)); + gst::PadProbeReturn::Ok + }, + ); + placed + } + + fn stop(self) -> Vec { + self.stop.store(true, Ordering::Relaxed); + self.writer.join().unwrap() + } + } + + // What a recording of system audio came out with. + #[derive(Debug)] + struct Recorded { + // Seconds missing from the audio track. + hole: f64, + // Seconds reported lost. + reported: f64, + // Where the audio ends after the video. + after_video: f64, + // When the system audio track ended, if it failed. + stopped: Option, + // When the recording had started. + began: Instant, + } + + impl Recorded { + fn whole(&self) -> bool { + self.hole == 0.0 && self.reported == 0.0 && self.after_video.abs() < 0.05 + } + } + + // Records `source` as system audio, runs `meanwhile` three seconds in, + // and finishes four seconds after that. + fn record_system_audio(source: gst::Element, meanwhile: impl FnOnce()) -> Recorded { + record_system_audio_with(source, |recording| { + std::thread::sleep(Duration::from_secs(3)); + meanwhile(); + std::thread::sleep(Duration::from_secs(4)); + let _ = recording; + }) + } + + fn record_system_audio_with( + source: gst::Element, + during: impl FnOnce(&TestRecording), + ) -> Recorded { + let mut recording = TestRecording::start_with_audio(vec![("system audio", source)]); + let began = Instant::now(); + during(&recording); recording.finish().unwrap(); let probe = recording.probe(); - let reported = recording + let lost = recording .events .try_iter() .find_map(|event| match event { RecorderEvent::MediaLost { message, .. } => Some(message), _ => None, }) - .and_then(|message| { - let (_, rest) = message.split_once("system audio is missing ")?; - rest.split_once(" s")?.0.parse().ok() - }) + .unwrap_or_default(); + let reported = lost + .split_once("system audio is missing ") + .and_then(|(_, rest)| rest.split_once(" s")?.0.parse().ok()) .unwrap_or(0.0); - let after_video = end_seconds(&probe, "aac") - end_seconds(&probe, "h264"); - (audio_holes(&probe)[0], reported, after_video) + Recorded { + hole: audio_holes(&probe)[0], + reported, + after_video: end_seconds(&probe, "aac") - end_seconds(&probe, "h264"), + stopped: lost + .split_once("system audio stopped at ") + .map(|(_, rest)| rest.to_string()), + began, + } } - // Slaved to the clock, pulsesrc stamped the audio the server made up late, - // and reported almost half a second lost. Run with - // `cargo test -p linux pulse_server -- --ignored`. + // A server that stops for 250 ms or 1.5 s makes up the audio of the + // meantime at once, and nothing of it is lost: no gap, nothing reported, + // and the audio ends with the video. Slaved to the clock, pulsesrc + // stamped the audio the server made up late and reported almost half a + // second lost; a reader that keeps only a second loses the rest of the + // 1.5 s. Run the real-server tests with + // `cargo test -p linux pulse_server -- --ignored --test-threads=1`. #[test] #[ignore = "needs pulseaudio"] fn audio_a_pulse_server_held_keeps_its_place() { - let (hole, reported, after_video) = record_server_stop(Duration::from_millis(250)); + for stall in [250, 1500] { + let server = PulseServer::start(); + let recorded = record_system_audio(server.source(), || { + server.stop_for(Duration::from_millis(stall)) + }); + assert!(recorded.whole(), "{stall} ms stop: {recorded:?}"); + } + } + + // A server whose audio takes 250 ms, 600 ms or 1.5 s longer to arrive + // from three seconds in loses nothing. Its audio keeps its place, nothing + // is reported lost, and once the delay goes away again the audio ends + // with the video. Placed by when it arrived, it moved later by the delay + // and reported that as lost. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_that_gets_slower_keeps_its_place() { + for delay in [250, 600, 1500] { + let server = PulseServer::start_with("", false); + let link = SlowLink::new(&server); + let source = server.source_from("wrec_test.monitor", &link.socket); + let recorded = record_system_audio(source, || { + link.set_delay(Duration::from_millis(delay)); + std::thread::sleep(Duration::from_secs(3)); + link.set_delay(Duration::ZERO); + }); + assert!(recorded.whole(), "{delay} ms slower: {recorded:?}"); + } + } + + // The same while the recording starts: audio that is slower before it + // first arrives, or 200 ms in, keeps its place. The pipe device sends + // audio at once. The null sink's monitor takes up to two seconds to, so + // its first audio waits no longer than the server's answer about it if + // it is to make the three seconds video waits for a track. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_that_starts_slow_keeps_its_place() { + for (pipe, before, delay) in [ + (false, true, 600), + (true, true, 600), + (true, false, 250), + (true, false, 1500), + ] { + let server = PulseServer::start_with(if pipe { PipeDevice::CONFIG } else { "" }, false); + let device = pipe.then(|| PipeDevice::start(&server)); + let placed = Arc::new(Mutex::new(Vec::new())); + let link = SlowLink::new(&server); + if before { + link.set_delay(Duration::from_millis(delay)); + } + let source = server.source_from( + if pipe { + "wrec_pipe" + } else { + "wrec_test.monitor" + }, + &link.socket, + ); + let placed = device + .as_ref() + .map_or(placed, |device| device.placement(&source)); + let mut began = None; + let recorded = record_system_audio_with(source, |_| { + began = Some(Instant::now()); + std::thread::sleep(Duration::from_millis(200)); + link.set_delay(Duration::from_millis(delay)); + std::thread::sleep(Duration::from_secs(3)); + link.set_delay(Duration::ZERO); + std::thread::sleep(Duration::from_secs(3)); + }); + // Before the recording, while the stream connected over the slow + // link and nothing read the pipe, it may have lost audio. + if let Some(device) = device { + assert_placed(&placed.lock().unwrap(), recorded.began); + let lost = device.stop().into_iter().filter(|&at| at >= began.unwrap()); + assert_eq!( + lost.count(), + 0, + "the device lost audio during the recording" + ); + } + assert!( + recorded.whole(), + "{delay} ms slower {} from {}: {recorded:?}", + if before { + "from the start" + } else { + "200 ms in" + }, + if pipe { "a pipe" } else { "a monitor" } + ); + } + } + + // Every buffer of audio written since the recording began, before and + // after any loss, starts within 20 ms of when the device wrote its first + // frame. The device writes 10 ms at a time, so a frame can wait up to + // that long to be written. Its pipe keeps audio from before the stream + // started, and a loss before the server's first account of its audio + // is one the server never saw. + fn assert_placed(placed: &[(f64, f64, Instant)], since: Instant) { + let placed: Vec<_> = placed.iter().filter(|entry| entry.2 >= since).collect(); + let worst = placed + .iter() + .max_by(|a, b| a.1.abs().total_cmp(&b.1.abs())) + .unwrap(); + let measured = format!( + "{} buffers, worst {:.4} s off at {:.3} s", + placed.len(), + worst.1, + worst.0 + ); + println!("placement: {measured}"); + assert!(placed.len() > 300 && worst.1.abs() < 0.02, "{measured}"); + } + + // Records the pipe device on `server`, through `socket`, while `meanwhile` + // runs three seconds in. Every buffer must be placed where the device + // wrote it. Returns the recording and the seconds of audio the device + // lost while the server was stopped, failing if it lost any at another + // time during the recording, which would make the test meaningless. + fn record_device_loss( + server: &PulseServer, + socket: &Path, + meanwhile: impl FnOnce(), + ) -> (Recorded, f64) { + let device = PipeDevice::start(server); + let source = server.source_from("wrec_pipe", socket); + let placed = device.placement(&source); + let mut stopped = None; + let recorded = record_system_audio(source, || { + stopped = Some(Instant::now()); + meanwhile(); + }); + assert_placed(&placed.lock().unwrap(), recorded.began); + let stopped = stopped.unwrap(); + // Before the recording began, while nothing read its pipe, the + // device can lose audio the recording never had. + let (before, lost): (Vec<_>, Vec<_>) = device + .stop() + .into_iter() + .partition(|&at| at < recorded.began); + let (during, outside): (Vec<_>, Vec<_>) = lost + .into_iter() + .partition(|&at| at >= stopped && at < stopped + Duration::from_secs(3)); + println!( + "device lost {} s before the recording", + before.len() as f64 / 100.0 + ); + let since = |at: &Instant| at.duration_since(recorded.began).as_secs_f64(); assert!( - hole == 0.0 && reported == 0.0, - "{hole}s gap, {reported}s reported" + outside.is_empty(), + "{} s lost outside the stop, from {:.2} s to {:.2} s into the recording, \ + which stopped the server {:.2} s in", + outside.len() as f64 / 100.0, + outside.first().map_or(0.0, since), + outside.last().map_or(0.0, since), + since(&stopped) ); + (recorded, during.len() as f64 / 100.0) + } + + // A capture device keeps going while its server stops for 0.8 s, and its + // pipe holds only 0.34 s, so the rest is lost inside the server. The + // movie has a gap of that length, the same is reported, and the audio + // after it ends with the video. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_device_lost_is_reported_and_the_rest_keeps_its_place() { + let server = PulseServer::start_with(PipeDevice::CONFIG, true); + let (recorded, lost) = + record_device_loss(&server, &server.directory.join("native"), || { + server.stop_for(Duration::from_millis(800)) + }); + assert!(lost > 0.3, "{lost} s lost: {recorded:?}"); assert!( - after_video.abs() < 0.05, - "audio ends {after_video}s after video" + (recorded.hole - lost).abs() < 0.03 && (recorded.reported - recorded.hole).abs() < 0.02, + "{lost} s lost: {recorded:?}" + ); + assert!( + recorded.after_video.abs() < 0.05, + "{lost} s lost: {recorded:?}" ); } - // PulseAudio's null sink makes up little of a 1.5 s stop, so most of it - // is lost. The movie has that gap, the same amount is reported, and the - // audio after it plays within AUDIO_LATE of the video, which is as close - // as a loss too small to see can leave it. + // The same while the server's audio takes 250 ms longer to arrive: only + // what the device lost is a gap, not the delay as well. #[test] #[ignore = "needs pulseaudio"] - fn audio_a_pulse_server_lost_is_reported_and_the_rest_keeps_its_place() { - let (hole, reported, after_video) = record_server_stop(Duration::from_millis(1500)); - assert!(hole > 0.5, "{hole}s gap"); + fn audio_a_pulse_server_that_got_slower_then_lost_audio_reports_only_the_loss() { + let server = PulseServer::start_with(PipeDevice::CONFIG, false); + let link = SlowLink::new(&server); + link.set_delay(Duration::ZERO); + let (recorded, lost) = record_device_loss(&server, &link.socket, || { + link.set_delay(Duration::from_millis(250)); + std::thread::sleep(Duration::from_secs(1)); + server.stop_for(Duration::from_millis(800)); + std::thread::sleep(Duration::from_secs(1)); + link.set_delay(Duration::ZERO); + }); + assert!(lost > 0.3, "{lost} s lost: {recorded:?}"); assert!( - (reported - hole).abs() < 0.05, - "{reported}s reported, {hole}s gap" + (recorded.hole - lost).abs() < 0.03 && (recorded.reported - recorded.hole).abs() < 0.02, + "{lost} s lost: {recorded:?}" + ); + assert!( + recorded.after_video.abs() < 0.05, + "{lost} s lost: {recorded:?}" + ); + } + + // When downstream stops taking audio for 3.5 s, the source holds a + // second and its server another, and the server drops the rest. That is + // a gap of the same length as reported, and the audio after it ends with + // the video. Without shm, all the audio waits in the socket. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_reader_stopped_past_its_buffer_reports_the_rest() { + let server = PulseServer::start_with("", false); + let source = server.source(); + stall_once( + &source.static_pad("src").unwrap(), + Duration::from_secs(3), + Duration::from_millis(3500), + ); + let recorded = record_system_audio(source, || {}); + assert!(recorded.hole > 0.5, "{recorded:?}"); + assert!( + (recorded.reported - recorded.hole).abs() < 0.02, + "{recorded:?}" + ); + assert!(recorded.after_video.abs() < 0.05, "{recorded:?}"); + } + + // A device whose clock is 800 parts per million fast or slow loses + // nothing. Nothing is reported, and every buffer is placed when the + // device wrote it, so the audio ends with the video. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_device_with_another_clock_reports_nothing() { + for ppm in [800, -800] { + let server = PulseServer::start_with(PipeDevice::CONFIG, true); + let device = PipeDevice::start_with_clock(&server, ppm); + let source = server.source_from("wrec_pipe", &server.directory.join("native")); + let placed = device.placement(&source); + let recorded = record_system_audio(source, || {}); + assert_placed(&placed.lock().unwrap(), recorded.began); + assert!(device.stop().is_empty(), "the device lost audio"); + assert!(recorded.whole(), "{ppm} ppm: {recorded:?}"); + } + } + + // Short and repeated pauses drop audio without reporting it lost, and + // the audio stays with the video. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_paused_briefly_and_often_keeps_its_place() { + let server = PulseServer::start(); + let recorded = record_system_audio_with(server.source(), |recording| { + std::thread::sleep(Duration::from_secs(2)); + for pause in [300, 100, 100, 50, 600] { + recording.control(true); + std::thread::sleep(Duration::from_millis(pause)); + recording.control(false); + std::thread::sleep(Duration::from_millis(400)); + } + std::thread::sleep(Duration::from_secs(2)); + }); + assert!(recorded.whole(), "{recorded:?}"); + } + + // Stopping while paused, or while downstream holds the source's audio + // and the source waits for room, still ends the source's track and + // finalizes the movie. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_recording_stops_while_paused_or_blocked() { + let server = PulseServer::start(); + let mut recording = + TestRecording::start_with_audio(vec![("system audio", server.source())]); + std::thread::sleep(Duration::from_secs(2)); + recording.control(true); + std::thread::sleep(Duration::from_millis(500)); + recording.finish().unwrap(); + assert_eq!(stream_counts(&recording.probe()), (1, 1)); + + let source = server.source(); + stall_once( + &source.static_pad("src").unwrap(), + Duration::from_secs(2), + Duration::from_secs(4), ); + let mut recording = TestRecording::start_with_audio(vec![("system audio", source)]); + std::thread::sleep(Duration::from_secs(3)); + let stopping = Instant::now(); + recording.finish().unwrap(); assert!( - after_video.abs() < AUDIO_LATE.nseconds() as f64 / 1e9, - "audio ends {after_video}s after video" + stopping.elapsed() < Duration::from_secs(5), + "stopping took {:?}", + stopping.elapsed() + ); + assert_eq!(stream_counts(&recording.probe()), (1, 1)); + } + + // A server that goes away ends its track where it did, with that + // reported, and the recording keeps going; a new server records whole. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_lost_ends_its_track_and_a_new_one_records() { + let mut server = PulseServer::start(); + let recorded = record_system_audio(server.source(), || { + let _ = server.process.kill(); + }); + let stopped = recorded.stopped.clone().unwrap_or_default(); + let at: f64 = stopped + .split_once(" s") + .and_then(|(at, _)| at.parse().ok()) + .unwrap_or(0.0); + assert!((2.5..4.0).contains(&at), "{recorded:?}"); + assert!(recorded.after_video < -2.5, "{recorded:?}"); + let server = PulseServer::start(); + let recorded = record_system_audio(server.source(), || {}); + assert!(recorded.whole(), "{recorded:?}"); + } + + // A device that sends nothing is left out, and one that starts late + // keeps its place. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_device_that_starts_late_or_never_keeps_the_video() { + let server = PulseServer::start_with(PipeDevice::CONFIG, true); + let source = server.source_from("wrec_pipe", &server.directory.join("native")); + let mut recording = TestRecording::start_with_audio(vec![("system audio", source)]); + std::thread::sleep(Duration::from_secs(5)); + recording.finish().unwrap(); + let probe = recording.probe(); + assert_eq!(stream_counts(&probe), (1, 0)); + + let server = PulseServer::start_with(PipeDevice::CONFIG, true); + let source = server.source_from("wrec_pipe", &server.directory.join("native")); + let mut device = None; + let placed = Arc::new(Mutex::new(None)); + let placing = placed.clone(); + let recorded = record_system_audio_with(source.clone(), |_| { + std::thread::sleep(Duration::from_millis(1500)); + let started = PipeDevice::start(&server); + *placing.lock().unwrap() = Some(started.placement(&source)); + device = Some(started); + std::thread::sleep(Duration::from_secs(4)); + }); + assert_placed( + &placed.lock().unwrap().take().unwrap().lock().unwrap(), + recorded.began, ); + assert!(device.unwrap().stop().is_empty(), "the device lost audio"); + assert!(recorded.whole(), "{recorded:?}"); + } + + // libpulse threads of this process, which name themselves. + fn pulse_threads() -> usize { + std::fs::read_dir("/proc/self/task") + .unwrap() + .filter_map(|task| std::fs::read_to_string(task.ok()?.path().join("comm")).ok()) + .filter(|name| name.trim() == "wrec-pulse") + .count() + } + + // A source ends its libpulse thread and connection when it goes away: + // never played, stopped while the server is slow to answer, and stopped + // while it waits for room. A server or device that isn't there fails it + // at once. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_source_lets_go_of_everything() { + gst::init().unwrap(); + let before = pulse_threads(); + let server = PulseServer::start_with("", false); + drop(server.source()); + let link = SlowLink::new(&server); + let source = server.source_from("wrec_test.monitor", &link.socket); + let pipeline = gst::Pipeline::new(); + let sink = element("fakesink").unwrap(); + let arrived = Arc::new(AtomicU64::new(0)); + let arriving = arrived.clone(); + sink.static_pad("sink") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, _| { + arriving.fetch_add(1, Ordering::Relaxed); + gst::PadProbeReturn::Ok + }); + pipeline.add_many([&source, &sink]).unwrap(); + source.link(&sink).unwrap(); + pipeline.set_state(gst::State::Playing).unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while arrived.load(Ordering::Relaxed) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(50)); + } + assert!(arrived.load(Ordering::Relaxed) > 0, "no audio arrived"); + // Audio asks for answers as it arrives, so once the slower audio + // arrives, answers are on their way when the pipeline stops. + link.set_delay(Duration::from_millis(400)); + std::thread::sleep(Duration::from_millis(700)); + pipeline.set_state(gst::State::Null).unwrap(); + std::thread::sleep(Duration::from_millis(800)); + drop((pipeline, source, sink)); + + let source = server.source(); + let pipeline = gst::Pipeline::new(); + let sink = element("fakesink").unwrap(); + sink.static_pad("sink") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, |_, _| { + std::thread::sleep(Duration::from_secs(3)); + gst::PadProbeReturn::Ok + }); + pipeline.add_many([&source, &sink]).unwrap(); + source.link(&sink).unwrap(); + pipeline.set_state(gst::State::Playing).unwrap(); + std::thread::sleep(Duration::from_millis(1500)); + let stopping = Instant::now(); + pipeline.set_state(gst::State::Null).unwrap(); + drop((pipeline, source, sink)); + assert!(stopping.elapsed() < Duration::from_secs(4)); + let deadline = Instant::now() + Duration::from_secs(2); + while pulse_threads() > before && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(50)); + } + assert_eq!(pulse_threads(), before); + + let missing = server.directory.join("missing"); + let started = Instant::now(); + assert!(crate::pulse::source( + Some(&format!("unix:{}", missing.display())), + None, + AUDIO_BUFFER + ) + .is_err()); + let socket = format!("unix:{}", server.directory.join("native").display()); + assert!(crate::pulse::source(Some(&socket), Some("no_such_device"), AUDIO_BUFFER).is_err()); + assert!(started.elapsed() < Duration::from_secs(2)); + assert_eq!(pulse_threads(), before); } // Stops a recording `after` something in it got stuck at two seconds. // Returns the finished recording, its result and how long after stop - // run() returned. Like pulsesrc, the audio sources' streaming threads - // block on full queues. + // run() returned. The audio sources' streaming threads block on full + // queues. fn stop_while_stuck( after: Duration, stuck: impl FnOnce(&gst::Pipeline), @@ -3557,7 +4122,8 @@ mod tests { // An idle screen sends a frame each keepalive. A short pause that swallows // one leaves almost two keepalives between the frames around it, and mp4mux // holds audio until the frame after those. The audio queue must absorb - // that wait: a full queue blocks pulsesrc, which then drops audio. + // that wait: a full queue blocks the audio source, and then its server + // drops audio. #[test] fn audio_waiting_for_idle_video_across_a_pause_never_fills_its_queue() { let mut recording = TestRecording::start_idle(2); diff --git a/crates/linux/src/pulse.rs b/crates/linux/src/pulse.rs new file mode 100644 index 0000000..85c8d93 --- /dev/null +++ b/crates/linux/src/pulse.rs @@ -0,0 +1,973 @@ +// Audio capture straight from libpulse, which PulseAudio and pipewire-pulse +// both serve, into an appsrc. + +use crate::backend; +use domain::Result; +use gstreamer::{self as gst, glib, prelude::*}; +use std::{ + ffi::{c_char, c_int, c_void, CStr, CString}, + sync::{Arc, Condvar, Mutex, OnceLock}, + time::Duration, +}; + +type Mainloop = c_void; +type Context = c_void; +type Stream = c_void; +type Operation = c_void; +type ContextCallback = unsafe extern "C" fn(*mut Context, *mut c_void); +type StreamCallback = unsafe extern "C" fn(*mut Stream, *mut c_void); +type ReadCallback = unsafe extern "C" fn(*mut Stream, usize, *mut c_void); +type SuccessCallback = unsafe extern "C" fn(*mut Stream, c_int, *mut c_void); + +#[repr(C)] +struct SampleSpec { + format: c_int, + rate: u32, + channels: u8, +} + +#[repr(C)] +struct BufferAttr { + maxlength: u32, + tlength: u32, + prebuf: u32, + minreq: u32, + fragsize: u32, +} + +#[repr(C)] +struct TimingInfo { + timestamp: libc::timeval, + synchronized_clocks: c_int, + sink_usec: u64, + source_usec: u64, + transport_usec: u64, + playing: c_int, + write_index_corrupt: c_int, + write_index: i64, + read_index_corrupt: c_int, + read_index: i64, + configured_sink_usec: u64, + configured_source_usec: u64, + since_underrun: i64, +} + +macro_rules! library { + ($($name:ident: fn($($arg:ty),*) $(-> $ret:ty)?;)*) => { + struct Library { + $($name: unsafe extern "C" fn($($arg),*) $(-> $ret)?,)* + } + + impl Library { + fn open() -> std::result::Result { + let handle = unsafe { libc::dlopen(c"libpulse.so.0".as_ptr(), libc::RTLD_NOW) }; + if handle.is_null() { + return Err("libpulse.so.0 could not be loaded".into()); + } + Ok(Self { + $($name: unsafe { + let symbol = libc::dlsym( + handle, + concat!("pa_", stringify!($name), "\0").as_ptr().cast(), + ); + if symbol.is_null() { + return Err(concat!("libpulse has no pa_", stringify!($name)).into()); + } + std::mem::transmute::<*mut c_void, unsafe extern "C" fn($($arg),*) $(-> $ret)?>(symbol) + },)* + }) + } + } + }; +} + +library! { + threaded_mainloop_new: fn() -> *mut Mainloop; + threaded_mainloop_free: fn(*mut Mainloop); + threaded_mainloop_start: fn(*mut Mainloop) -> c_int; + threaded_mainloop_stop: fn(*mut Mainloop); + threaded_mainloop_lock: fn(*mut Mainloop); + threaded_mainloop_unlock: fn(*mut Mainloop); + threaded_mainloop_in_thread: fn(*mut Mainloop) -> c_int; + threaded_mainloop_get_api: fn(*mut Mainloop) -> *mut c_void; + threaded_mainloop_set_name: fn(*mut Mainloop, *const c_char); + context_new: fn(*mut c_void, *const c_char) -> *mut Context; + context_set_state_callback: fn(*mut Context, Option, *mut c_void); + context_connect: fn(*mut Context, *const c_char, c_int, *const c_void) -> c_int; + context_get_state: fn(*mut Context) -> c_int; + context_disconnect: fn(*mut Context); + context_unref: fn(*mut Context); + context_errno: fn(*mut Context) -> c_int; + strerror: fn(c_int) -> *const c_char; + stream_new: fn(*mut Context, *const c_char, *const SampleSpec, *const c_void) -> *mut Stream; + stream_set_state_callback: fn(*mut Stream, Option, *mut c_void); + stream_set_read_callback: fn(*mut Stream, Option, *mut c_void); + stream_connect_record: fn(*mut Stream, *const c_char, *const BufferAttr, c_int) -> c_int; + stream_get_state: fn(*mut Stream) -> c_int; + stream_peek: fn(*mut Stream, *mut *const c_void, *mut usize) -> c_int; + stream_drop: fn(*mut Stream) -> c_int; + stream_cork: fn(*mut Stream, c_int, Option, *mut c_void) -> *mut Operation; + stream_update_timing_info: fn(*mut Stream, Option, *mut c_void) -> *mut Operation; + stream_get_timing_info: fn(*mut Stream) -> *const TimingInfo; + stream_disconnect: fn(*mut Stream) -> c_int; + stream_unref: fn(*mut Stream); + operation_unref: fn(*mut Operation); +} + +fn library() -> Result<&'static Library> { + static LIBRARY: OnceLock> = OnceLock::new(); + LIBRARY.get_or_init(Library::open).as_ref().map_err(|error| { + backend(format!( + "{error}. System audio and the microphone need libpulse, which PulseAudio and pipewire-pulse serve." + )) + }) +} + +pub(crate) fn available() -> Result<()> { + library().map(|_| ()) +} + +const CONTEXT_READY: c_int = 4; +const CONTEXT_FAILED: c_int = 5; +const CONTEXT_TERMINATED: c_int = 6; +const STREAM_READY: c_int = 2; +const STREAM_FAILED: c_int = 3; +const STREAM_TERMINATED: c_int = 4; +const CONTEXT_NOAUTOSPAWN: c_int = 1; +const STREAM_START_CORKED: c_int = 0x1; +const STREAM_ADJUST_LATENCY: c_int = 0x2000; +const SAMPLE_FLOAT32LE: c_int = 5; + +pub(crate) const RATE: u32 = 48000; +pub(crate) const CHANNELS: u8 = 2; +const FRAME: usize = 4 * CHANNELS as usize; + +// The server sends audio in fragments of this length, so capture latency. +const FRAGMENT: gst::ClockTime = gst::ClockTime::from_mseconds(10); + +// How long a server can take to accept the connection and the stream. +const CONNECTING: Duration = Duration::from_secs(10); + +fn bytes_of(time: gst::ClockTime) -> u32 { + (time.nseconds() * RATE as u64 / 1_000_000_000) as u32 * FRAME as u32 +} + +fn time_of(bytes: i64) -> i64 { + (bytes as i128 / FRAME as i128 * 1_000_000_000 / RATE as i128) as i64 +} + +// An appsrc fed by a libpulse record stream from `device` (the default +// source when None) on `server` (the default server when None). Fails if +// libpulse, the server or the device is unavailable. `buffer` is how much +// audio the server holds for this reader, and appsrc for downstream. +pub(crate) fn source( + server: Option<&str>, + device: Option<&str>, + buffer: gst::ClockTime, +) -> Result { + let library = library()?; + let source = crate::pipeline::element("appsrc")?; + source.set_property( + "caps", + gst::Caps::builder("audio/x-raw") + .field("format", "F32LE") + .field("rate", RATE as i32) + .field("channels", CHANNELS as i32) + .field("layout", "interleaved") + .build(), + ); + source.set_property("is-live", true); + source.set_property("format", gst::Format::Time); + source.set_property("do-timestamp", false); + source.set_property("block", true); + source.set_property("max-bytes", bytes_of(buffer) as u64); + source.set_property("min-latency", FRAGMENT.nseconds() as i64); + source.set_property("max-latency", buffer.nseconds() as i64); + let capture = Capture::connect(library, &source, server, device, buffer)?; + let capture = Mutex::new((capture, false)); + // A live appsrc first asks for data when the pipeline plays. + source.connect("need-data", false, move |_| { + let mut capture = capture.lock().unwrap(); + if !capture.1 { + capture.1 = true; + capture.0.start(); + } + None + }); + Ok(source) +} + +// Owns the libpulse thread and the reader it calls back into. +struct Capture { + library: &'static Library, + mainloop: *mut Mainloop, + reader: *mut Reader, +} + +// libpulse objects are used only under the mainloop lock or after its thread +// stopped. +unsafe impl Send for Capture {} + +impl Capture { + fn connect( + library: &'static Library, + source: &gst::Element, + server: Option<&str>, + device: Option<&str>, + buffer: gst::ClockTime, + ) -> Result { + let server = server + .map(|server| CString::new(server).map_err(backend)) + .transpose()?; + let device = device + .map(|device| CString::new(device).map_err(backend)) + .transpose()?; + let mainloop = unsafe { (library.threaded_mainloop_new)() }; + if mainloop.is_null() { + return Err(backend("libpulse could not create its mainloop")); + } + let connected = Arc::new((Mutex::new(None), Condvar::new())); + let reader = Box::into_raw(Box::new(Reader { + library, + context: std::ptr::null_mut(), + stream: std::ptr::null_mut(), + device, + buffer, + source: source.downgrade(), + connected: Some(connected.clone()), + closed: false, + position: 0, + stamps: Stamps::default(), + held: Vec::new(), + end: None, + last_request: None, + asking: 0, + last_arrival: None, + resumed: false, + })); + let capture = Self { + library, + mainloop, + reader, + }; + unsafe { + (library.threaded_mainloop_set_name)(mainloop, c"wrec-pulse".as_ptr()); + let api = (library.threaded_mainloop_get_api)(mainloop); + let context = (library.context_new)(api, c"wrec".as_ptr()); + if context.is_null() { + return Err(backend("libpulse could not create a context")); + } + (*reader).context = context; + (library.context_set_state_callback)(context, Some(context_state), reader.cast()); + if (library.context_connect)( + context, + server + .as_ref() + .map_or(std::ptr::null(), |server| server.as_ptr()), + CONTEXT_NOAUTOSPAWN, + std::ptr::null(), + ) < 0 + { + return Err(backend(format!( + "could not connect to the audio server: {}", + error_text(library, context) + ))); + } + if (library.threaded_mainloop_start)(mainloop) < 0 { + return Err(backend("libpulse could not start its thread")); + } + } + let (state, ready) = &*connected; + let (state, timeout) = ready + .wait_timeout_while(state.lock().unwrap(), CONNECTING, |state| state.is_none()) + .unwrap(); + match state.clone() { + Some(Ok(())) => Ok(capture), + Some(Err(error)) => Err(backend(error)), + None if timeout.timed_out() => Err(backend(format!( + "the audio server did not accept a recording stream within {} s", + CONNECTING.as_secs() + ))), + None => unreachable!(), + } + } + + fn start(&self) { + unsafe { + (self.library.threaded_mainloop_lock)(self.mainloop); + (*self.reader).start(); + (self.library.threaded_mainloop_unlock)(self.mainloop); + } + } +} + +impl Drop for Capture { + fn drop(&mut self) { + let parts = Parts(self.library, self.mainloop, self.reader); + // The libpulse thread can't stop itself, and drops the source last + // if it was pushing to it when the pipeline let go of it. + if unsafe { (self.library.threaded_mainloop_in_thread)(self.mainloop) } != 0 { + std::thread::spawn(move || parts.stop()); + } else { + parts.stop(); + } + } +} + +struct Parts(&'static Library, *mut Mainloop, *mut Reader); + +unsafe impl Send for Parts {} + +impl Parts { + fn stop(self) { + let Parts(library, mainloop, reader) = self; + unsafe { + (library.threaded_mainloop_stop)(mainloop); + (*reader).close(); + let reader = Box::from_raw(reader); + if !reader.context.is_null() { + (library.context_set_state_callback)(reader.context, None, std::ptr::null_mut()); + (library.context_disconnect)(reader.context); + (library.context_unref)(reader.context); + } + drop(reader); + (library.threaded_mainloop_free)(mainloop); + } + } +} + +type Connected = Arc<(Mutex>>, Condvar)>; + +// Lives on the libpulse thread, or under its lock. +struct Reader { + library: &'static Library, + context: *mut Context, + stream: *mut Stream, + device: Option, + buffer: gst::ClockTime, + source: glib::WeakRef, + // Until the stream is ready or failed, where Capture::connect waits. + connected: Option, + closed: bool, + // Bytes of the stream so far, received or skipped, so the server's + // index of the next byte. + position: i64, + stamps: Stamps, + // Audio waiting for the server to say when it captured audio, whether + // it lost any while audio stopped arriving, or where it lost some. + held: Vec<(i64, gst::Buffer)>, + // Where the last buffer pushed ended. + end: Option, + last_request: Option, + // Questions on the way. + asking: u32, + last_arrival: Option, + // Since audio stopped arriving for a while, until the server says + // whether it lost any meanwhile. + resumed: bool, +} + +fn error_text(library: &Library, context: *mut Context) -> String { + unsafe { + CStr::from_ptr((library.strerror)((library.context_errno)(context))) + .to_string_lossy() + .into_owned() + } +} + +unsafe extern "C" fn context_state(context: *mut Context, reader: *mut c_void) { + let reader = &mut *reader.cast::(); + match (reader.library.context_get_state)(context) { + CONTEXT_READY => reader.open_stream(), + CONTEXT_FAILED | CONTEXT_TERMINATED => { + let error = format!( + "the audio server connection ended: {}", + error_text(reader.library, context) + ); + reader.fail(error); + } + _ => {} + } +} + +unsafe extern "C" fn stream_state(stream: *mut Stream, reader: *mut c_void) { + let reader = &mut *reader.cast::(); + match (reader.library.stream_get_state)(stream) { + STREAM_READY => reader.report_connected(Ok(())), + STREAM_FAILED | STREAM_TERMINATED => { + let error = format!( + "the audio server ended the recording stream: {}", + error_text(reader.library, reader.context) + ); + reader.fail(error); + } + _ => {} + } +} + +unsafe extern "C" fn stream_read(_: *mut Stream, _: usize, reader: *mut c_void) { + (*reader.cast::()).read(); +} + +unsafe extern "C" fn timing_updated(_: *mut Stream, success: c_int, reader: *mut c_void) { + (*reader.cast::()).timing_updated(success != 0); +} + +// Questions on the way at most, as for a server that stopped answering. +const ASKING: u32 = 64; + +// How long video waits for an audio track to start. +const FIRST_ACCOUNT: gst::ClockTime = crate::pipeline::HELD_VIDEO_LIMIT; + +impl Reader { + unsafe fn open_stream(&mut self) { + let spec = SampleSpec { + format: SAMPLE_FLOAT32LE, + rate: RATE, + channels: CHANNELS, + }; + let stream = (self.library.stream_new)( + self.context, + c"wrec recording".as_ptr(), + &spec, + std::ptr::null(), + ); + if stream.is_null() { + let error = format!( + "the audio server refused a recording stream: {}", + error_text(self.library, self.context) + ); + return self.fail(error); + } + self.stream = stream; + let reader = (self as *mut Self).cast(); + (self.library.stream_set_state_callback)(stream, Some(stream_state), reader); + (self.library.stream_set_read_callback)(stream, Some(stream_read), reader); + let attributes = BufferAttr { + maxlength: bytes_of(self.buffer), + tlength: u32::MAX, + prebuf: u32::MAX, + minreq: u32::MAX, + fragsize: bytes_of(FRAGMENT), + }; + if (self.library.stream_connect_record)( + stream, + self.device + .as_ref() + .map_or(std::ptr::null(), |device| device.as_ptr()), + &attributes, + STREAM_START_CORKED | STREAM_ADJUST_LATENCY, + ) < 0 + { + let error = format!( + "the audio server refused a recording stream: {}", + error_text(self.library, self.context) + ); + self.fail(error); + } + } + + fn report_connected(&mut self, result: std::result::Result<(), String>) { + if let Some(connected) = self.connected.take() { + *connected.0.lock().unwrap() = Some(result); + connected.1.notify_all(); + } + } + + // Before the pipeline plays, the error goes to Capture::connect; after + // it, to the bus, which ends the track. + unsafe fn fail(&mut self, error: String) { + if self.connected.is_some() { + return self.report_connected(Err(error)); + } + if self.closed { + return; + } + self.close(); + if let Some(source) = self.source.upgrade() { + source.post_error_message(gst::error_msg!(gst::ResourceError::Read, ["{}", error])); + } + } + + unsafe fn close(&mut self) { + self.closed = true; + if !self.stream.is_null() { + (self.library.stream_set_state_callback)(self.stream, None, std::ptr::null_mut()); + (self.library.stream_set_read_callback)(self.stream, None, std::ptr::null_mut()); + (self.library.stream_disconnect)(self.stream); + (self.library.stream_unref)(self.stream); + self.stream = std::ptr::null_mut(); + } + } + + unsafe fn start(&mut self) { + if self.closed || self.stream.is_null() { + return; + } + let operation = (self.library.stream_cork)(self.stream, 0, None, std::ptr::null_mut()); + if !operation.is_null() { + (self.library.operation_unref)(operation); + } + self.request_timing(); + } + + fn now(&self) -> Option<(gst::ClockTime, gst::ClockTime)> { + let source = self.source.upgrade()?; + let clock = source.clock()?; + let now = clock.time()?; + Some((now, now.saturating_sub(source.base_time()?))) + } + + // Answers come in order, each after the audio sent before it, and more + // than one can be on the way, up to ASKING for a server that stopped. + unsafe fn request_timing(&mut self) { + if self.closed || self.asking >= ASKING { + return; + } + let operation = (self.library.stream_update_timing_info)( + self.stream, + Some(timing_updated), + (self as *mut Self).cast(), + ); + if operation.is_null() { + return; + } + (self.library.operation_unref)(operation); + self.asking += 1; + self.last_request = self.now().map(|(_, running)| running); + } + + // The server's write index, when it captured the audio there, and its + // read index. A read index past the audio received is audio the server + // skipped, which pipewire-pulse does for a reader that fell a buffer + // behind; the reply comes after the audio sent before it. libpulse + // already took off what it holds, and read() takes all it holds. + unsafe fn timing_updated(&mut self, success: bool) { + self.asking = self.asking.saturating_sub(1); + // libpulse can answer after close() let go of the stream. + if self.closed { + return; + } + let info = (self.library.stream_get_timing_info)(self.stream); + let (Some(info), Some((_, now))) = (info.as_ref(), self.now()) else { + return; + }; + if success && info.read_index_corrupt == 0 && info.read_index > self.position { + self.position = info.read_index; + } + let mut realtime = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + libc::clock_gettime(libc::CLOCK_REALTIME, &mut realtime); + // How long ago, by the system's wall clock, the server answered. + let age = (realtime.tv_sec - info.timestamp.tv_sec) * 1_000_000_000 + realtime.tv_nsec + - info.timestamp.tv_usec * 1000; + let answered = now.nseconds() as i64 - age; + if success + && info.write_index_corrupt == 0 + && info.playing != 0 + && info.write_index > 0 + && self.stamps.timing( + info.write_index, + info.source_usec as i64 * 1000, + answered, + now.nseconds() as i64, + ) + { + self.resumed = false; + } + if !self.holding() { + self.release(); + } + } + + unsafe fn read(&mut self) { + while !self.closed { + let mut data = std::ptr::null(); + let mut length = 0; + if (self.library.stream_peek)(self.stream, &mut data, &mut length) < 0 || length == 0 { + return; + } + let start = self.position; + self.position += length as i64; + // Without running time the source stopped playing. + let Some((_, now)) = self.now() else { + (self.library.stream_drop)(self.stream); + self.close(); + return; + }; + // A hole is audio the server skipped. + if data.is_null() { + (self.library.stream_drop)(self.stream); + continue; + } + let mut buffer = gst::Buffer::with_size(length).unwrap(); + buffer + .get_mut() + .unwrap() + .copy_from_slice(0, std::slice::from_raw_parts(data.cast::(), length)) + .unwrap(); + (self.library.stream_drop)(self.stream); + self.resumed |= self + .last_arrival + .is_some_and(|last| now.saturating_sub(last) > FRAGMENT * 4); + self.last_arrival = Some(now); + // While holding, each answer about newer audio counts. + if self.holding() + || self + .last_request + .map_or(true, |last| now.saturating_sub(last) >= TIMING_INTERVAL) + { + self.request_timing(); + } + if self.holding() { + self.held.push((start, buffer)); + // Before the server first answers, as long as video waits + // for a track, and afterwards a buffer's worth. + let limit = if self.stamps.offset.is_none() { + FIRST_ACCOUNT + } else { + self.buffer + }; + let held: usize = self.held.iter().map(|(_, buffer)| buffer.size()).sum(); + if held < bytes_of(limit) as usize { + continue; + } + // A server that doesn't answer: place audio where it + // arrived, or as it was before. + self.stamps + .offset + .get_or_insert(now.nseconds() as i64 - time_of(self.position)); + self.resumed = false; + if !self.release() { + return; + } + continue; + } + if !self.push(start, buffer) { + return; + } + } + } + + fn holding(&self) -> bool { + self.stamps.offset.is_none() || self.resumed || self.stamps.rising.is_some() + } + + unsafe fn release(&mut self) -> bool { + std::mem::take(&mut self.held) + .into_iter() + .all(|(position, buffer)| self.push(position, buffer)) + } + + // Stamps and pushes audio that starts at `position`. Audio captured + // before the pipeline played is not part of the recording. False once + // the source takes no more, after which the stream is closed. + unsafe fn push(&mut self, mut position: i64, mut buffer: gst::Buffer) -> bool { + let mut pts = self.stamps.pts(position).unwrap(); + if pts < 0 { + let early = ((-pts) as u128 * RATE as u128).div_ceil(1_000_000_000) as usize * FRAME; + if early >= buffer.size() { + return true; + } + buffer = buffer + .copy_region(gst::BufferCopyFlags::MEMORY, early..) + .unwrap(); + position += early as i64; + pts = self.stamps.pts(position).unwrap().max(0); + } + let duration = time_of(position + buffer.size() as i64) - time_of(position); + { + let buffer = buffer.get_mut().unwrap(); + buffer.set_pts(gst::ClockTime::from_nseconds(pts as u64)); + buffer.set_duration(gst::ClockTime::from_nseconds(duration as u64)); + if self.end != Some(pts) { + buffer.set_flags(gst::BufferFlags::DISCONT); + } + } + self.end = Some(pts + duration); + // Blocks while appsrc is full, so libpulse stops reading and the + // server holds the rest, and returns when the source stops. + let pushed = self + .source + .upgrade() + .map(|source| source.emit_by_name::("push-buffer", &[&buffer])); + if pushed != Some(gst::FlowReturn::Ok) { + self.close(); + return false; + } + true + } +} + +// How often the server is asked when it captured its audio. +const TIMING_INTERVAL: gst::ClockTime = gst::ClockTime::from_mseconds(100); + +// The server's account of when it captured the audio at its write index is +// a few milliseconds late at most: PulseAudio measures its latency on +// request, and pipewire-pulse as of its last graph cycle. Audio the server +// lost, whether its device or graph lost it or PulseAudio dropped it for a +// reader that fell a buffer behind, never reaches its write index, so the +// audio after it was captured later than the audio before it by the amount +// lost. A change of more than STEP that CONFIRM replies with newer audio +// agree on is that, and audio past where the server was capturing when it +// first said so moves later by the least of them. count_lost_audio sees the +// gap. Smaller +// changes, such as one clock running faster than the other, move audio at +// most SLEW per second of running time, which nothing downstream sees as a +// gap. Neither the time audio takes to arrive nor a recorder that stopped +// reading changes the server's account. +const STEP: i64 = crate::pipeline::AUDIO_HOLE.nseconds() as i64; +const CONFIRM: u32 = 3; +const SLEW: i64 = 1_000; // ppm +const WINDOW: usize = 10; + +// Where the stream's audio goes in running time: its sample count plus an +// offset from the server's account of when it captured it. +#[derive(Default)] +struct Stamps { + offset: Option, + // Steps not yet applied: the position each applies from and its amount. + steps: Vec<(i64, i64)>, + // While the server's account is more than STEP later than the offset: + // where it was capturing and how much later, in each answer since. + rising: Option>, + // The server's account in recent replies, newest last. + recent: std::collections::VecDeque, + last_write: i64, + last_reply: i64, +} + +impl Stamps { + fn pts(&mut self, position: i64) -> Option { + let offset = self.offset.as_mut()?; + self.steps.retain(|&(from, amount)| { + if position >= from { + *offset += amount; + } + position < from + }); + Some(time_of(position) + *offset) + } + + // A reply: the server's write index, how long ago it captured the audio + // there, when it answered, and now, in running time. Returns whether it + // had newer audio than the last one with any. + fn timing(&mut self, write: i64, latency: i64, answered: i64, now: i64) -> bool { + if write <= self.last_write { + return false; + } + self.last_write = write; + let account = answered - latency - time_of(write); + // When it answered, the server had captured audio up to here, past + // its write index by its latency, so audio it lost since is past + // that. + let capturing = + write + (latency as i128 * RATE as i128 / 1_000_000_000) as i64 * FRAME as i64; + let elapsed = now - std::mem::replace(&mut self.last_reply, now); + let Some(offset) = self.offset.as_mut() else { + // The account is a little late, never early, the first one most + // often, so audio waits for the least of the first few. + self.recent.push_back(account); + if self.recent.len() >= CONFIRM as usize { + self.offset = self.recent.iter().min().copied(); + } + return true; + }; + let target = *offset + self.steps.iter().map(|(_, amount)| amount).sum::(); + if account - target > STEP { + let rising = self.rising.get_or_insert_with(Vec::new); + rising.push((capturing, account - target)); + if rising.len() < CONFIRM as usize { + return true; + } + // An answer whose write index stopped short of its time is too + // late by that. The others agree on the amount, and the loss is + // past where the first answer that does too was capturing. + let least = rising[1..].iter().map(|&(_, late)| late).min().unwrap(); + let at = rising + .iter() + .find(|&&(_, late)| late <= least + STEP) + .unwrap() + .0; + self.steps.push((at, least)); + self.rising = None; + self.recent.clear(); + return true; + } + self.rising = None; + self.recent.push_back(account); + if self.recent.len() > WINDOW { + self.recent.pop_front(); + } + let floor = *self.recent.iter().min().unwrap(); + let limit = elapsed.max(0) * SLEW / 1_000_000; + *offset += (floor - target).clamp(-limit, limit); + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const MS: i64 = 1_000_000; + + fn bytes(time: i64) -> i64 { + (time as i128 * RATE as i128 / 1_000_000_000) as i64 * FRAME as i64 + } + + // A server answering every 100 ms of running time for an hour. At + // running time t, `server(t)` is how much audio it has captured and how + // late its account of the capture time is. Returns the stamps and, for + // each answer, the running time, the write index, and the offset in use + // for audio at the write index, once there is one. + fn answer(server: impl Fn(i64) -> (i64, i64)) -> (Stamps, Vec<(i64, i64, i64)>) { + let mut stamps = Stamps::default(); + let mut answers = Vec::new(); + for t in (100 * MS..3_600_000 * MS).step_by(100 * MS as usize) { + let (captured, late) = server(t); + let write = bytes(captured); + stamps.timing(write, 0, t + late, t); + if let Some(pts) = stamps.pts(write) { + answers.push((t, write, pts - time_of(write))); + } + } + (stamps, answers) + } + + // Up to 11 ms late, as PulseAudio's account was at most. + fn noise(t: i64) -> i64 { + (t / (100 * MS)).wrapping_mul(2_654_435_761) % (11 * MS) + } + + #[test] + fn a_late_account_and_late_answers_move_nothing() { + let (stamps, answers) = answer(|t| (t, noise(t))); + assert!(stamps.steps.is_empty()); + for (_, _, offset) in answers { + assert!((0..=11 * MS).contains(&offset), "{offset}"); + } + // How long answers take to arrive changes nothing either. + let mut stamps = Stamps::default(); + for t in (100 * MS..60_000 * MS).step_by(100 * MS as usize) { + let arrived = t + if (20_000 * MS..40_000 * MS).contains(&t) { + 1500 * MS + } else { + 0 + }; + stamps.timing(bytes(t), 0, t, arrived); + } + assert!(stamps.steps.is_empty() && stamps.offset == Some(0)); + } + + // From 20.05 s, the server's device captured nothing for 300 ms, while + // the server kept answering. Audio up to then keeps its place, and audio + // from there on moves 300 ms later. + #[test] + fn audio_the_server_lost_moves_later_from_where_it_was_lost() { + let lost = |t: i64| (t - 20_050 * MS).clamp(0, 300 * MS); + let mut stamps = Stamps::default(); + // Answers with no newer audio than the last say nothing, so the + // loss is confirmed at 20.5 s. + for t in (100 * MS..=20_500 * MS).step_by(100 * MS as usize) { + stamps.timing(bytes(t - lost(t)), 0, t + noise(t), t); + } + let first = bytes(20_050 * MS); + let mut offset = |position| stamps.pts(position).unwrap() - time_of(position); + assert!(offset(first - FRAME as i64) < 12 * MS); + let moved = offset(first); + assert!((295 * MS..312 * MS).contains(&moved), "{moved}"); + for t in (20_600 * MS..60_000 * MS).step_by(100 * MS as usize) { + stamps.timing(bytes(t - lost(t)), 0, t + noise(t), t); + let position = bytes(t - lost(t)); + let offset = stamps.pts(position).unwrap() - time_of(position); + assert!((295 * MS..312 * MS).contains(&offset), "{offset} at {t}"); + } + assert!(stamps.steps.is_empty()); + } + + // The server stops from 20 s to 20.8 s, and its device holds 0.32 s and + // loses the rest. On resuming, the server answers before taking in what + // the device held, so its write index is still at 20 s, and says that + // audio there was captured 0.32 s ago. The loss is past the audio held. + #[test] + fn audio_lost_after_what_the_device_held_moves_from_there() { + let mut stamps = Stamps::default(); + for t in (100 * MS..=20_000 * MS).step_by(100 * MS as usize) { + stamps.timing(bytes(t), 0, t, t); + } + stamps.timing(bytes(20_000 * MS), 0, 20_000 * MS, 20_000 * MS); + let resumed = 20_800 * MS; + stamps.timing( + bytes(20_000 * MS) + FRAME as i64, + 320 * MS, + resumed, + resumed, + ); + for t in [20_810, 20_820, 20_830] { + // What the device held, then audio captured since resuming. + let write = bytes(20_320 * MS) + bytes((t - 20_800) * MS); + stamps.timing(write, 0, t * MS, t * MS); + } + let held = bytes(20_320 * MS); + let mut offset = |position| stamps.pts(position).unwrap() - time_of(position); + assert_eq!(offset(held - FRAME as i64), 0); + assert_eq!(offset(held + FRAME as i64), 480 * MS); + } + + // The first answers are the latest ones, and audio starts at the least of + // the first three. + #[test] + fn late_first_answers_move_nothing() { + let mut stamps = Stamps::default(); + for (t, late) in [(100, 20), (110, 5), (120, 0), (130, 0)] { + stamps.timing(bytes(t * MS), 0, (t + late) * MS, t * MS); + } + assert_eq!(stamps.pts(0), Some(0)); + } + + #[test] + fn one_late_answer_moves_nothing() { + let (stamps, answers) = answer(|t| (t, if t == 30_000 * MS { 700 * MS } else { 0 })); + assert!(stamps.steps.is_empty()); + assert!(answers.iter().all(|&(_, _, offset)| offset == 0)); + } + + #[test] + fn another_clock_is_followed_without_a_gap() { + for ppm in [500, -500] { + let (stamps, answers) = answer(|t| (t + t / 1_000_000 * ppm, noise(t))); + assert!(stamps.steps.is_empty(), "{ppm} ppm"); + let mut previous: Option<(i64, i64)> = None; + for (t, write, offset) in answers { + let account = t - time_of(write); + assert!( + (offset - account).abs() < 15 * MS, + "{ppm} ppm at {t}: {offset} vs {account}" + ); + if let Some((then, before)) = previous { + assert!((offset - before).abs() <= (t - then) * SLEW / 1_000_000 + 1); + } + previous = Some((t, offset)); + } + } + } + + #[test] + fn more_audio_than_time_moves_audio_earlier_slowly() { + let (stamps, answers) = answer(|t| (t + if t > 10_000 * MS { 100 * MS } else { 0 }, 0)); + assert!(stamps.steps.is_empty()); + let at = |seconds: i64| { + answers + .iter() + .find(|answer| answer.0 >= seconds * 1000 * MS) + .unwrap() + .2 + }; + assert_eq!(at(10), 0); + assert!((-51 * MS..=-49 * MS).contains(&at(60)), "{}", at(60)); + assert_eq!(at(200), -100 * MS); + } +} diff --git a/packaging/linux/README.md b/packaging/linux/README.md index 2627ebf..35c7028 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -339,27 +339,46 @@ available. Configure devices in desktop sound settings. Linux cannot apply the shared wrec window-hiding or custom microphone-indicator options; job settings report them disabled with a warning. The desktop controls its sharing indicator. -Each audio source retains up to one second of raw audio, about 384 kB per -F32 stereo track, while capture latency remains 10 ms. Sample-count timestamps -preserve audio held through a short recorder stall. If audio instead stays more -than 200 ms late with steady arrival timing for a second, wrec moves later -samples back into alignment and reports the corresponding gap. - -This recovery has limits. After real loss, the correction appears about a second -later, and the intervening audio can play early by the amount lost. Loss below -200 ms can remain undetected. A stall beyond the one-second source buffer can -lose samples and leave about 100 ms of offset, as observed in the controlled -1.5-second stall. Physical microphone clock drift is not validated. These are -experimental support limits; a completed job alone does not establish A/V sync. +wrec reads audio from the audio server with libpulse, which PulseAudio and +`pipewire-pulse` both serve, so recording audio needs `libpulse.so.0`; the +daemon loads it only then. Each audio source keeps up to one second of raw +audio, about 384 kB per F32 stereo track, while downstream stops taking it, and +asks its server to hold another second. While confirming server timing, it can +hold another second, or three seconds before the first timing reply. These +queues are bounded and normally stay nearly empty. Capture latency remains +10 ms. + +Each sample goes where the audio server says it captured it. wrec asks the +server ten times a second for the capture time of its newest audio and how much +audio it has captured and sent. Audio that takes longer to arrive, or a recorder +that stops reading for a moment, leaves that account unchanged, so the audio +keeps its place and nothing is reported. Audio the server lost leaves a gap of +the same length where it was lost, and wrec reports it. That covers audio its +device or graph lost, audio PulseAudio dropped because the recorder fell more +than both buffers behind, and audio `pipewire-pulse` skipped for the same +reason. A device clock that runs faster or slower than the system clock moves +the audio at most 1 ms per second and is not reported. + +This has limits. Changes in the server's account of 20 ms or less are never +reported; the audio moves smoothly instead. A device clock more than 1000 parts +per million slow builds up until it reads as lost audio, and one that fast +leaves the audio drifting late. A loss is placed at the newest audio the server +had captured when it first reported it, so audio captured right after a loss can +play that much early; in the controlled tests every buffer stayed within 11 ms +of its capture time. The account is only as good as the server's latency +reporting. Clock drift is validated only with a synthetic device 800 parts per +million fast or slow, not with physical microphones. These are experimental +support limits; a completed job alone does not establish A/V sync. The movie writer takes the next sample of every track in time order, so a track that stops delivering holds up the others. Encoded video and each audio track can wait about 4 seconds for the rest. Encoded movie data can wait for the disk in a 32 MiB buffer, about 16 seconds at 16 Mbit/s. These buffers stay nearly empty unless something stalls. Past those limits the recording keeps going with -gaps: the capture queue drops video frames, and an audio source overwrites audio -that nothing read in time. A stalled video encoder always costs the frames -captured meanwhile, since raw frames are too large to keep. +gaps: the capture queue drops video frames, and an audio source's server drops +audio that nothing read in time, which is reported as lost. A stalled video +encoder always costs the frames captured meanwhile, since raw frames are too +large to keep. If an audio source fails, for example because PipeWire or PulseAudio restarts, its track ends at that point and video and the other track keep recording. An From f40cb1dbf54a2e0efbc09c87cb1bd052aaf87a34 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:52:25 +0000 Subject: [PATCH 35/45] fix(linux): keep audio aligned across wall clock steps --- crates/linux/src/pipeline.rs | 208 ++++++++++++++++++++++++++++++++-- crates/linux/src/pulse.rs | 57 +++++++--- crates/linux/src/wall_clock.c | 58 ++++++++++ 3 files changed, 297 insertions(+), 26 deletions(-) create mode 100644 crates/linux/src/wall_clock.c diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 40a376c..59cdcec 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -2508,7 +2508,9 @@ mod tests { } // `config` adds modules. Without shm all audio goes through the - // socket, so a SlowLink can carry it. + // socket, so a SlowLink can carry it. The server loads its modules + // in order and the socket last, so once the socket exists, so does + // everything the modules make, such as a pipe source's fifo. fn start_with(config: &str, shm: bool) -> Self { static ID: AtomicU64 = AtomicU64::new(0); let directory = std::env::temp_dir().join(format!( @@ -2522,11 +2524,11 @@ mod tests { std::fs::write( &script, format!( - "load-module module-native-protocol-unix socket={} auth-anonymous=1\n\ - load-module module-null-sink sink_name=wrec_test rate=48000 channels=2\n\ - load-module module-sine sink=wrec_test frequency=440\n{}", - socket.display(), - config.replace("{dir}", &directory.display().to_string()) + "load-module module-null-sink sink_name=wrec_test rate=48000 channels=2\n\ + load-module module-sine sink=wrec_test frequency=440\n{}\ + load-module module-native-protocol-unix socket={} auth-anonymous=1\n", + config.replace("{dir}", &directory.display().to_string()), + socket.display() ), ) .unwrap(); @@ -2535,6 +2537,7 @@ mod tests { .args(["--use-pid-file=no", "--system=no", "-F"]) .arg(&script) .arg(format!("--disable-shm={}", !shm)) + .arg("--log-target=stderr") .env("HOME", &directory) .env("XDG_RUNTIME_DIR", &directory) .env("PULSE_RUNTIME_PATH", &directory) @@ -2543,18 +2546,30 @@ mod tests { format!("unix:path={}", directory.join("no-bus").display()), ) .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) + .stderr(std::fs::File::create(directory.join("log")).unwrap()) .spawn() .expect("pulseaudio is not installed"); - let server = Self { process, directory }; let deadline = Instant::now() + Duration::from_secs(10); + let mut server = Self { process, directory }; while !socket.exists() { - assert!(Instant::now() < deadline, "pulseaudio did not start"); + if let Some(status) = server.process.try_wait().unwrap() { + panic!("pulseaudio exited ({status}):\n{}", server.log()); + } + assert!( + Instant::now() < deadline, + "pulseaudio did not start within 10 s:\n{}", + server.log() + ); std::thread::sleep(Duration::from_millis(50)); } server } + // What the server wrote to stderr, such as a module it couldn't load. + fn log(&self) -> String { + std::fs::read_to_string(self.directory.join("log")).unwrap_or_default() + } + fn source(&self) -> gst::Element { self.source_from("wrec_test.monitor", &self.directory.join("native")) } @@ -2581,7 +2596,9 @@ mod tests { impl Drop for PulseServer { fn drop(&mut self) { - unsafe { libc::kill(self.process.id() as libc::pid_t, libc::SIGCONT) }; + if let Ok(None) = self.process.try_wait() { + unsafe { libc::kill(self.process.id() as libc::pid_t, libc::SIGCONT) }; + } let _ = self.process.kill(); let _ = self.process.wait(); let _ = std::fs::remove_dir_all(&self.directory); @@ -2677,7 +2694,12 @@ mod tests { .write(true) .custom_flags(libc::O_NONBLOCK) .open(server.directory.join("fifo")) - .unwrap(); + .unwrap_or_else(|error| { + panic!( + "module-pipe-source made no fifo ({error}). The server said:\n{}", + server.log() + ) + }); // 0.34 s of this audio, on any kernel. unsafe { libc::fcntl(fifo.as_raw_fd(), libc::F_SETPIPE_SZ, 65536) }; let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); @@ -2766,6 +2788,8 @@ mod tests { reported: f64, // Where the audio ends after the video. after_video: f64, + // Where the audio starts after the video. + starts_after_video: f64, // When the system audio track ended, if it failed. stopped: Option, // When the recording had started. @@ -2814,6 +2838,7 @@ mod tests { hole: audio_holes(&probe)[0], reported, after_video: end_seconds(&probe, "aac") - end_seconds(&probe, "h264"), + starts_after_video: start_seconds(&probe, "aac")[0] - start_seconds(&probe, "h264")[0], stopped: lost .split_once("system audio stopped at ") .map(|(_, rest)| rest.to_string()), @@ -3043,6 +3068,167 @@ mod tests { ); } + // The wall clock of this process and of the audio servers it starts, + // which a test sets forward or back through wall_clock.c without + // touching the system's. + struct WallClock(&'static std::sync::atomic::AtomicI64); + + impl WallClock { + // Runs `test` again in a new process with wall_clock.c preloaded, + // and returns None once it passed there. In that process, returns + // the clock it shares with the servers it starts. + fn shared(test: &str) -> Option { + if let Some(path) = std::env::var_os("WREC_TEST_WALL_CLOCK") { + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + let offset = unsafe { + libc::mmap( + std::ptr::null_mut(), + 8, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + file.as_raw_fd(), + 0, + ) + }; + assert_ne!(offset, libc::MAP_FAILED); + return Some(Self(unsafe { &*offset.cast() })); + } + let directory = std::env::temp_dir() + .join(format!("wrec-linux-test-wall-clock-{}", std::process::id())); + std::fs::create_dir_all(&directory).unwrap(); + let (source, shim) = ( + directory.join("wall_clock.c"), + directory.join("wall_clock.so"), + ); + std::fs::write(&source, include_str!("wall_clock.c")).unwrap(); + let built = std::process::Command::new("cc") + .args(["-shared", "-fPIC", "-o"]) + .args([&shim, &source]) + .arg("-ldl") + .status() + .expect("no C compiler"); + assert!(built.success()); + let clock = directory.join("offset"); + std::fs::write(&clock, 0i64.to_ne_bytes()).unwrap(); + let ran = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + test, + "--exact", + "--ignored", + "--nocapture", + "--test-threads=1", + ]) + .env("LD_PRELOAD", &shim) + .env("WREC_TEST_WALL_CLOCK", &clock) + .status() + .unwrap(); + let _ = std::fs::remove_dir_all(&directory); + assert!(ran.success(), "{test} failed while its clock was set"); + None + } + + // How far the wall clock is from the monotonic clock, in nanoseconds. + fn offset() -> i64 { + let read = |clock| { + let mut time = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + unsafe { libc::clock_gettime(clock, &mut time) }; + time.tv_sec * 1_000_000_000 + time.tv_nsec + }; + read(libc::CLOCK_REALTIME) - read(libc::CLOCK_MONOTONIC) + } + + // Sets the clock `by` nanoseconds forward, or back, and checks this + // process sees that. + fn set(&self, by: i64) { + let before = Self::offset(); + self.0.fetch_add(by, Ordering::Relaxed); + let moved = Self::offset() - before; + assert!((moved - by).abs() < 1_000_000, "moved {moved} ns"); + } + + // Checks `server` reads this clock too. + fn shared_with(&self, server: &PulseServer) { + let maps = std::fs::read_to_string(format!("/proc/{}/maps", server.process.id())); + assert!(maps.unwrap().contains("wall_clock.so")); + } + } + + // Setting the system clock forward or back while the server's answers + // are on the way, among the first that place the audio and later on, + // moves no audio, nor does it hide or add to audio the device lost + // meanwhile. The server and libpulse say when they answered and asked by + // the wall clock, and those answers straddle the change. + #[test] + #[ignore = "needs pulseaudio and a C compiler"] + fn audio_a_pulse_server_while_the_clock_is_set_keeps_its_place() { + const SECOND: i64 = 1_000_000_000; + let Some(clock) = WallClock::shared( + "pipeline::tests::audio_a_pulse_server_while_the_clock_is_set_keeps_its_place", + ) else { + return; + }; + // Answers take 0.6 s, so those to the first questions, asked once + // audio arrives 0.6 s in, are on the way 0.9 s in. + for (first, later) in [(2 * SECOND, -2 * SECOND), (-2 * SECOND, 2 * SECOND)] { + let server = PulseServer::start_with(PipeDevice::CONFIG, false); + clock.shared_with(&server); + let device = PipeDevice::start(&server); + let link = SlowLink::new(&server); + link.set_delay(Duration::from_millis(600)); + let source = server.source_from("wrec_pipe", &link.socket); + let placed = device.placement(&source); + let mut began = None; + let recorded = record_system_audio_with(source, |_| { + began = Some(Instant::now()); + std::thread::sleep(Duration::from_millis(900)); + clock.set(first); + std::thread::sleep(Duration::from_millis(3100)); + clock.set(later); + std::thread::sleep(Duration::from_secs(1)); + link.set_delay(Duration::ZERO); + std::thread::sleep(Duration::from_secs(3)); + }); + assert_placed(&placed.lock().unwrap(), recorded.began); + let lost = device.stop().into_iter().filter(|&at| at >= began.unwrap()); + assert_eq!( + lost.count(), + 0, + "the device lost audio during the recording" + ); + assert!( + recorded.whole() && recorded.starts_after_video.abs() < 0.05, + "set {first} then {later} ns: {recorded:?}" + ); + } + let server = PulseServer::start_with(PipeDevice::CONFIG, false); + clock.shared_with(&server); + let link = SlowLink::new(&server); + let (recorded, lost) = record_device_loss(&server, &link.socket, || { + link.set_delay(Duration::from_millis(600)); + std::thread::sleep(Duration::from_secs(1)); + clock.set(-SECOND / 2); + server.stop_for(Duration::from_millis(800)); + std::thread::sleep(Duration::from_secs(1)); + link.set_delay(Duration::ZERO); + }); + assert!(lost > 0.3, "{lost} s lost: {recorded:?}"); + assert!( + (recorded.hole - lost).abs() < 0.03 && (recorded.reported - recorded.hole).abs() < 0.02, + "{lost} s lost: {recorded:?}" + ); + assert!( + recorded.after_video.abs() < 0.05, + "{lost} s lost: {recorded:?}" + ); + } + // When downstream stops taking audio for 3.5 s, the source holds a // second and its server another, and the server drops the rest. That is // a gap of the same length as reported, and the audio after it ends with diff --git a/crates/linux/src/pulse.rs b/crates/linux/src/pulse.rs index 85c8d93..ab809b8 100644 --- a/crates/linux/src/pulse.rs +++ b/crates/linux/src/pulse.rs @@ -241,7 +241,7 @@ impl Capture { held: Vec::new(), end: None, last_request: None, - asking: 0, + asking: std::collections::VecDeque::new(), last_arrival: None, resumed: false, })); @@ -359,8 +359,8 @@ struct Reader { // Where the last buffer pushed ended. end: Option, last_request: Option, - // Questions on the way. - asking: u32, + // Questions on the way, and wall_clock's distance when each was asked. + asking: std::collections::VecDeque, last_arrival: Option, // Since audio stopped arriving for a while, until the server says // whether it lost any meanwhile. @@ -414,7 +414,29 @@ unsafe extern "C" fn timing_updated(_: *mut Stream, success: c_int, reader: *mut } // Questions on the way at most, as for a server that stopped answering. -const ASKING: u32 = 64; +const ASKING: usize = 64; + +// A change in wall_clock's distance of more than this is the clock being +// set. Reading the two clocks one after the other takes microseconds. +const CLOCK_SET: i64 = 1_000_000; + +// The wall clock, and its distance from the monotonic clock, in +// nanoseconds. NTP adjusts both alike, so the distance changes only when +// someone sets the wall clock, or the machine sleeps, which the monotonic +// clock doesn't count. +fn wall_clock() -> (i64, i64) { + let read = |clock| { + let mut time = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + unsafe { libc::clock_gettime(clock, &mut time) }; + time.tv_sec * 1_000_000_000 + time.tv_nsec + }; + let monotonic = read(libc::CLOCK_MONOTONIC); + let wall = read(libc::CLOCK_REALTIME); + (wall, wall - monotonic) +} // How long video waits for an audio track to start. const FIRST_ACCOUNT: gst::ClockTime = crate::pipeline::HELD_VIDEO_LIMIT; @@ -521,9 +543,11 @@ impl Reader { // Answers come in order, each after the audio sent before it, and more // than one can be on the way, up to ASKING for a server that stopped. unsafe fn request_timing(&mut self) { - if self.closed || self.asking >= ASKING { + if self.closed || self.asking.len() >= ASKING { return; } + // Before libpulse reads the wall clock for the question. + let asked = wall_clock().1; let operation = (self.library.stream_update_timing_info)( self.stream, Some(timing_updated), @@ -533,7 +557,7 @@ impl Reader { return; } (self.library.operation_unref)(operation); - self.asking += 1; + self.asking.push_back(asked); self.last_request = self.now().map(|(_, running)| running); } @@ -543,7 +567,7 @@ impl Reader { // behind; the reply comes after the audio sent before it. libpulse // already took off what it holds, and read() takes all it holds. unsafe fn timing_updated(&mut self, success: bool) { - self.asking = self.asking.saturating_sub(1); + let asked = self.asking.pop_front(); // libpulse can answer after close() let go of the stream. if self.closed { return; @@ -555,16 +579,19 @@ impl Reader { if success && info.read_index_corrupt == 0 && info.read_index > self.position { self.position = info.read_index; } - let mut realtime = libc::timespec { - tv_sec: 0, - tv_nsec: 0, - }; - libc::clock_gettime(libc::CLOCK_REALTIME, &mut realtime); - // How long ago, by the system's wall clock, the server answered. - let age = (realtime.tv_sec - info.timestamp.tv_sec) * 1_000_000_000 + realtime.tv_nsec - - info.timestamp.tv_usec * 1000; + // The server says when it answered by its wall clock. libpulse + // takes that if it falls between when it asked and heard back by + // its own, and otherwise halfway between those. A server on this + // machine reads the same system clock, so either is right unless + // someone set that clock since the question, which makes the answer + // seem older or newer by as much. Such an answer says nothing about + // when. + let (wall, offset) = wall_clock(); + let set = asked.map_or(true, |asked| (offset - asked).abs() > CLOCK_SET); + let age = wall - info.timestamp.tv_sec * 1_000_000_000 - info.timestamp.tv_usec * 1000; let answered = now.nseconds() as i64 - age; if success + && !set && info.write_index_corrupt == 0 && info.playing != 0 && info.write_index > 0 diff --git a/crates/linux/src/wall_clock.c b/crates/linux/src/wall_clock.c new file mode 100644 index 0000000..4062248 --- /dev/null +++ b/crates/linux/src/wall_clock.c @@ -0,0 +1,58 @@ +// For tests: preloaded into a test process and the audio servers it starts, +// moves their wall clock by the nanoseconds in the file named by +// WREC_TEST_WALL_CLOCK, as setting the system clock would, and leaves the +// monotonic clock alone. Every process that maps the file shares the one +// wall clock, like processes on one machine. +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +static volatile int64_t *offset; + +__attribute__((constructor)) static void map(void) { + const char *path = getenv("WREC_TEST_WALL_CLOCK"); + int file = path ? open(path, O_RDONLY) : -1; + if (file < 0) + return; + void *mapped = mmap(NULL, sizeof *offset, PROT_READ, MAP_SHARED, file, 0); + if (mapped != MAP_FAILED) + offset = mapped; +} + +int clock_gettime(clockid_t clock, struct timespec *time) { + static int (*real)(clockid_t, struct timespec *); + if (!real) + real = (int (*)(clockid_t, struct timespec *))dlsym(RTLD_NEXT, "clock_gettime"); + int result = real(clock, time); + if (result == 0 && offset && (clock == CLOCK_REALTIME || clock == CLOCK_REALTIME_COARSE)) { + int64_t nanoseconds = time->tv_sec * 1000000000LL + time->tv_nsec + + __atomic_load_n(offset, __ATOMIC_RELAXED); + time->tv_sec = nanoseconds / 1000000000; + time->tv_nsec = nanoseconds % 1000000000; + } + return result; +} + +int gettimeofday(struct timeval *time, void *zone) { + (void)zone; + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + if (time) { + time->tv_sec = now.tv_sec; + time->tv_usec = now.tv_nsec / 1000; + } + return 0; +} + +time_t time(time_t *result) { + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + if (result) + *result = now.tv_sec; + return now.tv_sec; +} From c2cd51f5f971b4ee0e06fc2519295cd202528a92 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:28:49 +0000 Subject: [PATCH 36/45] fix(linux): place audio at the server loss boundary --- crates/linux/src/pipeline.rs | 106 +++++++++++++++++++------ crates/linux/src/pulse.rs | 146 +++++++++++++++++++++++++++++------ 2 files changed, 203 insertions(+), 49 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 59cdcec..bce6ecc 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -2675,6 +2675,9 @@ mod tests { // When it wrote its first block, and how long each block is. start: Instant, block: Duration, + // For each block it wrote more than a millisecond after it was due, + // when it was due and how late it was. + behind: Arc>>, } impl PipeDevice { @@ -2706,12 +2709,19 @@ mod tests { let stopping = stop.clone(); let block = Duration::from_nanos((10_000_000_000_000 / (1_000_000 + ppm)) as u64); let start = Instant::now(); + let behind = Arc::new(Mutex::new(Vec::new())); + let falling_behind = behind.clone(); let writer = std::thread::spawn(move || { let mut lost = Vec::new(); for n in 0u32.. { if stopping.load(Ordering::Relaxed) { break; } + let due = start + block * n; + let late = Instant::now().saturating_duration_since(due); + if late > Duration::from_millis(1) { + falling_behind.lock().unwrap().push((due, late)); + } let bytes: Vec = (n * 480..n * 480 + 480) .flat_map(|frame| { [(frame % 32768) as i16, (frame / 32768) as i16] @@ -2733,12 +2743,15 @@ mod tests { writer, start, block, + behind, } } - // Records, for each buffer `source` sends, its pts and how far that is - // from when the device wrote its first frame, both in seconds, and - // when that was. + // Records, for each buffer `source` sends, its pts, how far its first + // or its last frame is from when the device was due to write it, + // whichever is further, both in seconds, and when the first was due. + // A frame's index says which block it was in, and so when that block + // was due. A buffer can start before a loss and end after it. fn placement(&self, source: &gst::Element) -> Arc>> { let placed = Arc::new(Mutex::new(Vec::new())); let (start, block, placing) = (self.start, self.block, placed.clone()); @@ -2760,13 +2773,26 @@ mod tests { (f32::from_le_bytes(map[i * 4..i * 4 + 4].try_into().unwrap()) * 32768.0) .round() as u64 }; - let frame = sample(0) + sample(1) * 32768; - let written = start + block * (frame / 480) as u32; - let expected = now.nseconds() as f64 / 1e9 - - at.saturating_duration_since(written).as_secs_f64() - + written.saturating_duration_since(at).as_secs_f64(); + let frame = |i: usize| sample(2 * i) + sample(2 * i + 1) * 32768; + let due = |frame: u64| start + block * (frame / 480) as u32; + // How far `pts` is from when the device was due to + // write `frame`, in running time. + let off = |frame: u64, pts: f64| { + let written = due(frame); + pts - (now.nseconds() as f64 / 1e9 + - at.saturating_duration_since(written).as_secs_f64() + + written.saturating_duration_since(at).as_secs_f64()) + }; + let frames = map.len() / 8; let pts = buffer.pts().unwrap().nseconds() as f64 / 1e9; - placing.lock().unwrap().push((pts, pts - expected, written)); + let first = off(frame(0), pts); + let last = off(frame(frames - 1), pts + (frames - 1) as f64 / 48000.0); + let worst = if last.abs() > first.abs() { + last + } else { + first + }; + placing.lock().unwrap().push((pts, worst, due(frame(0)))); gst::PadProbeReturn::Ok }, ); @@ -2930,7 +2956,7 @@ mod tests { // Before the recording, while the stream connected over the slow // link and nothing read the pipe, it may have lost audio. if let Some(device) = device { - assert_placed(&placed.lock().unwrap(), recorded.began); + device.assert_placed(&placed, recorded.began); let lost = device.stop().into_iter().filter(|&at| at >= began.unwrap()); assert_eq!( lost.count(), @@ -2951,12 +2977,46 @@ mod tests { } } - // Every buffer of audio written since the recording began, before and - // after any loss, starts within 20 ms of when the device wrote its first - // frame. The device writes 10 ms at a time, so a frame can wait up to - // that long to be written. Its pipe keeps audio from before the stream - // started, and a loss before the server's first account of its audio - // is one the server never saw. + impl PipeDevice { + // Every buffer of audio written since the recording began, before and + // after any loss, starts within 20 ms of when the device was due to + // write its first frame: `placement` measures from the block's + // schedule, start plus its index times the block, not from when the + // write happened. The device writes 10 ms at a time, so a frame can + // wait up to that long to be written. Its pipe keeps audio from + // before the stream started, and a loss before the server's first + // account of its audio is one the server never saw. The schedule + // stands for when audio was captured only while the device keeps + // to it. If this machine didn't run the device's thread and it wrote + // a block AUDIO_HOLE or more after it was due, the server got that + // audio late, as if lost, then all at once, and placement against + // the schedule says nothing about wrec, so the test fails on that + // instead. + fn assert_placed(&self, placed: &Mutex>, since: Instant) { + let placed = placed.lock().unwrap(); + let until = placed.iter().map(|entry| entry.2).max().unwrap_or(since); + // Let go of the lock before failing, or the device panics on it. + let worst = self + .behind + .lock() + .unwrap() + .iter() + .filter(|entry| entry.0 >= since && entry.0 <= until) + .max_by_key(|entry| entry.1) + .copied(); + if let Some((at, late)) = worst { + assert!( + late.as_nanos() < AUDIO_HOLE.nseconds() as u128, + "the test device wrote audio {:.3} s late {:.2} s into the recording, \ + so this machine starved it and placement can't be judged", + late.as_secs_f64(), + at.duration_since(since).as_secs_f64() + ); + } + assert_placed(&placed, since); + } + } + fn assert_placed(placed: &[(f64, f64, Instant)], since: Instant) { let placed: Vec<_> = placed.iter().filter(|entry| entry.2 >= since).collect(); let worst = placed @@ -2991,7 +3051,7 @@ mod tests { stopped = Some(Instant::now()); meanwhile(); }); - assert_placed(&placed.lock().unwrap(), recorded.began); + device.assert_placed(&placed, recorded.began); let stopped = stopped.unwrap(); // Before the recording began, while nothing read its pipe, the // device can lose audio the recording never had. @@ -3195,7 +3255,7 @@ mod tests { link.set_delay(Duration::ZERO); std::thread::sleep(Duration::from_secs(3)); }); - assert_placed(&placed.lock().unwrap(), recorded.began); + device.assert_placed(&placed, recorded.began); let lost = device.stop().into_iter().filter(|&at| at >= began.unwrap()); assert_eq!( lost.count(), @@ -3264,7 +3324,7 @@ mod tests { let source = server.source_from("wrec_pipe", &server.directory.join("native")); let placed = device.placement(&source); let recorded = record_system_audio(source, || {}); - assert_placed(&placed.lock().unwrap(), recorded.began); + device.assert_placed(&placed, recorded.began); assert!(device.stop().is_empty(), "the device lost audio"); assert!(recorded.whole(), "{ppm} ppm: {recorded:?}"); } @@ -3368,11 +3428,9 @@ mod tests { device = Some(started); std::thread::sleep(Duration::from_secs(4)); }); - assert_placed( - &placed.lock().unwrap().take().unwrap().lock().unwrap(), - recorded.began, - ); - assert!(device.unwrap().stop().is_empty(), "the device lost audio"); + let device = device.unwrap(); + device.assert_placed(&placed.lock().unwrap().take().unwrap(), recorded.began); + assert!(device.stop().is_empty(), "the device lost audio"); assert!(recorded.whole(), "{recorded:?}"); } diff --git a/crates/linux/src/pulse.rs b/crates/linux/src/pulse.rs index ab809b8..a8ffef4 100644 --- a/crates/linux/src/pulse.rs +++ b/crates/linux/src/pulse.rs @@ -18,6 +18,9 @@ type ContextCallback = unsafe extern "C" fn(*mut Context, *mut c_void); type StreamCallback = unsafe extern "C" fn(*mut Stream, *mut c_void); type ReadCallback = unsafe extern "C" fn(*mut Stream, usize, *mut c_void); type SuccessCallback = unsafe extern "C" fn(*mut Stream, c_int, *mut c_void); +type TimeEvent = c_void; +type TimeCallback = + unsafe extern "C" fn(*mut c_void, *mut TimeEvent, *const libc::timeval, *mut c_void); #[repr(C)] struct SampleSpec { @@ -112,6 +115,9 @@ library! { stream_disconnect: fn(*mut Stream) -> c_int; stream_unref: fn(*mut Stream); operation_unref: fn(*mut Operation); + context_rttime_new: fn(*mut Context, u64, Option, *mut c_void) -> *mut TimeEvent; + context_rttime_restart: fn(*mut Context, *mut TimeEvent, u64); + rtclock_now: fn() -> u64; } fn library() -> Result<&'static Library> { @@ -240,7 +246,7 @@ impl Capture { stamps: Stamps::default(), held: Vec::new(), end: None, - last_request: None, + timer: std::ptr::null_mut(), asking: std::collections::VecDeque::new(), last_arrival: None, resumed: false, @@ -358,7 +364,8 @@ struct Reader { held: Vec<(i64, gst::Buffer)>, // Where the last buffer pushed ended. end: Option, - last_request: Option, + // Goes off every TIMING_INTERVAL. + timer: *mut TimeEvent, // Questions on the way, and wall_clock's distance when each was asked. asking: std::collections::VecDeque, last_arrival: Option, @@ -409,6 +416,15 @@ unsafe extern "C" fn stream_read(_: *mut Stream, _: usize, reader: *mut c_void) (*reader.cast::()).read(); } +unsafe extern "C" fn timer_went_off( + _: *mut c_void, + _: *mut TimeEvent, + _: *const libc::timeval, + reader: *mut c_void, +) { + (*reader.cast::()).ask_again(); +} + unsafe extern "C" fn timing_updated(_: *mut Stream, success: c_int, reader: *mut c_void) { (*reader.cast::()).timing_updated(success != 0); } @@ -531,6 +547,24 @@ impl Reader { (self.library.operation_unref)(operation); } self.request_timing(); + self.timer = (self.library.context_rttime_new)( + self.context, + (self.library.rtclock_now)() + TIMING_INTERVAL.useconds(), + Some(timer_went_off), + (self as *mut Self).cast(), + ); + } + + unsafe fn ask_again(&mut self) { + if self.closed { + return; + } + self.request_timing(); + (self.library.context_rttime_restart)( + self.context, + self.timer, + (self.library.rtclock_now)() + TIMING_INTERVAL.useconds(), + ); } fn now(&self) -> Option<(gst::ClockTime, gst::ClockTime)> { @@ -558,7 +592,6 @@ impl Reader { } (self.library.operation_unref)(operation); self.asking.push_back(asked); - self.last_request = self.now().map(|(_, running)| running); } // The server's write index, when it captured the audio there, and its @@ -641,11 +674,7 @@ impl Reader { .is_some_and(|last| now.saturating_sub(last) > FRAGMENT * 4); self.last_arrival = Some(now); // While holding, each answer about newer audio counts. - if self.holding() - || self - .last_request - .map_or(true, |last| now.saturating_sub(last) >= TIMING_INTERVAL) - { + if self.holding() { self.request_timing(); } if self.holding() { @@ -692,6 +721,17 @@ impl Reader { // before the pipeline played is not part of the recording. False once // the source takes no more, after which the stream is closed. unsafe fn push(&mut self, mut position: i64, mut buffer: gst::Buffer) -> bool { + // Audio from a step on moves, so a buffer one falls inside goes in two. + if let Some(from) = self.stamps.step_inside(position, buffer.size() as i64) { + let at = (from - position) as usize; + let rest = buffer + .copy_region(gst::BufferCopyFlags::MEMORY, at..) + .unwrap(); + let first = buffer + .copy_region(gst::BufferCopyFlags::MEMORY, ..at) + .unwrap(); + return self.push(position, first) && self.push(from, rest); + } let mut pts = self.stamps.pts(position).unwrap(); if pts < 0 { let early = ((-pts) as u128 * RATE as u128).div_ceil(1_000_000_000) as usize * FRAME; @@ -728,7 +768,11 @@ impl Reader { } } -// How often the server is asked when it captured its audio. +// How often the server is asked when it captured its audio. By the clock, +// not as audio arrives, so a question waits at a server that stopped. +// PulseAudio answers it as it resumes, before it reads more of its device, +// so the answer says how much the device held then, and audio lost +// meanwhile comes after that. const TIMING_INTERVAL: gst::ClockTime = gst::ClockTime::from_mseconds(100); // The server's account of when it captured the audio at its write index is @@ -762,11 +806,22 @@ struct Stamps { rising: Option>, // The server's account in recent replies, newest last. recent: std::collections::VecDeque, - last_write: i64, + last_capturing: i64, last_reply: i64, } impl Stamps { + // The first step that applies from inside the `length` bytes at + // `position`, on a frame. + fn step_inside(&self, position: i64, length: i64) -> Option { + self.steps + .iter() + .map(|&(from, _)| from) + .filter(|&from| from > position && from < position + length) + .filter(|&from| (from - position) % FRAME as i64 == 0) + .min() + } + fn pts(&mut self, position: i64) -> Option { let offset = self.offset.as_mut()?; self.steps.retain(|&(from, amount)| { @@ -779,19 +834,23 @@ impl Stamps { } // A reply: the server's write index, how long ago it captured the audio - // there, when it answered, and now, in running time. Returns whether it - // had newer audio than the last one with any. + // there, when it answered, and now, in running time. Returns whether the + // server had captured more than by the last one that did. fn timing(&mut self, write: i64, latency: i64, answered: i64, now: i64) -> bool { - if write <= self.last_write { + // When it answered, the server had captured audio up to here, past + // its write index by its latency, so audio it lost since is past + // that. The latency comes in whole microseconds, less than half a + // frame, so the nearest frame is the one the server counted. An + // answer about no more audio than the last says nothing new about + // when audio was captured, only that it answered later. + let capturing = write + + ((latency as i128 * RATE as i128 + 500_000_000) / 1_000_000_000) as i64 + * FRAME as i64; + if capturing <= self.last_capturing { return false; } - self.last_write = write; + self.last_capturing = capturing; let account = answered - latency - time_of(write); - // When it answered, the server had captured audio up to here, past - // its write index by its latency, so audio it lost since is past - // that. - let capturing = - write + (latency as i128 * RATE as i128 / 1_000_000_000) as i64 * FRAME as i64; let elapsed = now - std::mem::replace(&mut self.last_reply, now); let Some(offset) = self.offset.as_mut() else { // The account is a little late, never early, the first one most @@ -927,12 +986,7 @@ mod tests { } stamps.timing(bytes(20_000 * MS), 0, 20_000 * MS, 20_000 * MS); let resumed = 20_800 * MS; - stamps.timing( - bytes(20_000 * MS) + FRAME as i64, - 320 * MS, - resumed, - resumed, - ); + stamps.timing(bytes(20_000 * MS), 320 * MS, resumed, resumed); for t in [20_810, 20_820, 20_830] { // What the device held, then audio captured since resuming. let write = bytes(20_320 * MS) + bytes((t - 20_800) * MS); @@ -944,6 +998,48 @@ mod tests { assert_eq!(offset(held + FRAME as i64), 480 * MS); } + // As above, but the device held 15361 frames, which the server says as + // 320020 µs, a little short of them. The loss is past every one. + #[test] + fn a_latency_in_whole_microseconds_counts_every_frame_held() { + let mut stamps = Stamps::default(); + for t in (100 * MS..=20_000 * MS).step_by(100 * MS as usize) { + stamps.timing(bytes(t), 0, t, t); + } + let resumed = 20_800 * MS; + let held = bytes(20_000 * MS) + 15361 * FRAME as i64; + stamps.timing(bytes(20_000 * MS), 320_020_000, resumed, resumed); + for t in [20_810, 20_820, 20_830] { + stamps.timing(held + bytes((t - 20_800) * MS), 0, t * MS, t * MS); + } + let mut offset = |position| stamps.pts(position).unwrap() - time_of(position); + assert_eq!(offset(held - FRAME as i64), 0); + assert!(offset(held) > 479 * MS, "{}", offset(held)); + } + + // A step from inside a buffer: Reader::push sends the buffer in two + // there, so its audio up to the step keeps its place and from it moves. + #[test] + fn a_step_inside_a_buffer_is_found_on_its_frame() { + let frames = |n: i64| n * FRAME as i64; + let mut stamps = Stamps { + offset: Some(0), + steps: vec![(frames(1000), 480 * MS)], + ..Stamps::default() + }; + assert_eq!(stamps.step_inside(frames(520), frames(480)), None); + assert_eq!( + stamps.step_inside(frames(960), frames(480)), + Some(frames(1000)) + ); + assert_eq!(stamps.step_inside(frames(1000), frames(480)), None); + assert_eq!(stamps.pts(frames(960)), Some(time_of(frames(960)))); + assert_eq!( + stamps.pts(frames(1000)), + Some(time_of(frames(1000)) + 480 * MS) + ); + } + // The first answers are the latest ones, and audio starts at the least of // the first three. #[test] From 2d2d07b28eff88c512dbb2e4a85d6441b65c0905 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:03:51 +0000 Subject: [PATCH 37/45] fix(linux): discard audio with uncertain PipeWire loss placement --- crates/linux/src/pulse.rs | 583 ++++++++++++++++++++++++++++++++++++-- packaging/linux/README.md | 42 ++- 2 files changed, 594 insertions(+), 31 deletions(-) diff --git a/crates/linux/src/pulse.rs b/crates/linux/src/pulse.rs index a8ffef4..b733868 100644 --- a/crates/linux/src/pulse.rs +++ b/crates/linux/src/pulse.rs @@ -18,6 +18,16 @@ type ContextCallback = unsafe extern "C" fn(*mut Context, *mut c_void); type StreamCallback = unsafe extern "C" fn(*mut Stream, *mut c_void); type ReadCallback = unsafe extern "C" fn(*mut Stream, usize, *mut c_void); type SuccessCallback = unsafe extern "C" fn(*mut Stream, c_int, *mut c_void); +type ServerInfoCallback = unsafe extern "C" fn(*mut Context, *const ServerInfo, *mut c_void); + +// The start of pa_server_info. +#[repr(C)] +struct ServerInfo { + user_name: *const c_char, + host_name: *const c_char, + server_version: *const c_char, + server_name: *const c_char, +} type TimeEvent = c_void; type TimeCallback = unsafe extern "C" fn(*mut c_void, *mut TimeEvent, *const libc::timeval, *mut c_void); @@ -80,6 +90,20 @@ macro_rules! library { },)* }) } + + // For tests to stand in for the functions they use. The others + // abort. + #[cfg(test)] + fn none() -> Self { + Self { + $($name: { + unsafe extern "C" fn none($(_: $arg),*) $(-> $ret)? { + std::process::abort() + } + none + },)* + } + } } }; } @@ -101,6 +125,7 @@ library! { context_disconnect: fn(*mut Context); context_unref: fn(*mut Context); context_errno: fn(*mut Context) -> c_int; + context_get_server_info: fn(*mut Context, Option, *mut c_void) -> *mut Operation; strerror: fn(c_int) -> *const c_char; stream_new: fn(*mut Context, *const c_char, *const SampleSpec, *const c_void) -> *mut Stream; stream_set_state_callback: fn(*mut Stream, Option, *mut c_void); @@ -234,22 +259,9 @@ impl Capture { } let connected = Arc::new((Mutex::new(None), Condvar::new())); let reader = Box::into_raw(Box::new(Reader { - library, - context: std::ptr::null_mut(), - stream: std::ptr::null_mut(), device, - buffer, - source: source.downgrade(), connected: Some(connected.clone()), - closed: false, - position: 0, - stamps: Stamps::default(), - held: Vec::new(), - end: None, - timer: std::ptr::null_mut(), - asking: std::collections::VecDeque::new(), - last_arrival: None, - resumed: false, + ..Reader::new(library, source, buffer) })); let capture = Self { library, @@ -362,6 +374,12 @@ struct Reader { // Audio waiting for the server to say when it captured audio, whether // it lost any while audio stopped arriving, or where it lost some. held: Vec<(i64, gst::Buffer)>, + // How many of the last held buffers arrived since the server last + // answered. + unanswered: usize, + // Since audio arrived that could come after a skip, until the server + // next answers. + late: bool, // Where the last buffer pushed ended. end: Option, // Goes off every TIMING_INTERVAL. @@ -372,6 +390,8 @@ struct Reader { // Since audio stopped arriving for a while, until the server says // whether it lost any meanwhile. resumed: bool, + // pipewire-pulse, whose latency is audio in its graph, which it can lose. + graph_holds: bool, } fn error_text(library: &Library, context: *mut Context) -> String { @@ -385,7 +405,18 @@ fn error_text(library: &Library, context: *mut Context) -> String { unsafe extern "C" fn context_state(context: *mut Context, reader: *mut c_void) { let reader = &mut *reader.cast::(); match (reader.library.context_get_state)(context) { - CONTEXT_READY => reader.open_stream(), + CONTEXT_READY => { + // Answered before the stream is, on the same connection. + let operation = (reader.library.context_get_server_info)( + context, + Some(server_info), + (reader as *mut Reader).cast(), + ); + if !operation.is_null() { + (reader.library.operation_unref)(operation); + } + reader.open_stream() + } CONTEXT_FAILED | CONTEXT_TERMINATED => { let error = format!( "the audio server connection ended: {}", @@ -412,6 +443,14 @@ unsafe extern "C" fn stream_state(stream: *mut Stream, reader: *mut c_void) { } } +unsafe extern "C" fn server_info(_: *mut Context, info: *const ServerInfo, reader: *mut c_void) { + let Some(info) = info.as_ref() else { return }; + if !info.server_name.is_null() { + let name = CStr::from_ptr(info.server_name).to_string_lossy(); + (*reader.cast::()).graph_holds = name.contains("PipeWire"); + } +} + unsafe extern "C" fn stream_read(_: *mut Stream, _: usize, reader: *mut c_void) { (*reader.cast::()).read(); } @@ -458,6 +497,30 @@ fn wall_clock() -> (i64, i64) { const FIRST_ACCOUNT: gst::ClockTime = crate::pipeline::HELD_VIDEO_LIMIT; impl Reader { + fn new(library: &'static Library, source: &gst::Element, buffer: gst::ClockTime) -> Self { + Self { + library, + context: std::ptr::null_mut(), + stream: std::ptr::null_mut(), + device: None, + buffer, + source: source.downgrade(), + connected: None, + closed: false, + position: 0, + stamps: Stamps::default(), + held: Vec::new(), + unanswered: 0, + late: false, + end: None, + timer: std::ptr::null_mut(), + asking: std::collections::VecDeque::new(), + last_arrival: None, + resumed: false, + graph_holds: false, + } + } + unsafe fn open_stream(&mut self) { let spec = SampleSpec { format: SAMPLE_FLOAT32LE, @@ -597,8 +660,12 @@ impl Reader { // The server's write index, when it captured the audio there, and its // read index. A read index past the audio received is audio the server // skipped, which pipewire-pulse does for a reader that fell a buffer - // behind; the reply comes after the audio sent before it. libpulse - // already took off what it holds, and read() takes all it holds. + // behind, and only says so here. The reply comes after the audio sent + // before it, and libpulse already took off what it holds, which read() + // takes all of. So the skip is somewhere in the audio that arrived since + // the last answer, with no mark where: that audio has no known place, and + // is lost with what was skipped. Audio that arrived earlier came before + // it, and read() holds all audio that can come after one. unsafe fn timing_updated(&mut self, success: bool) { let asked = self.asking.pop_front(); // libpulse can answer after close() let go of the stream. @@ -609,8 +676,13 @@ impl Reader { let (Some(info), Some((_, now))) = (info.as_ref(), self.now()) else { return; }; - if success && info.read_index_corrupt == 0 && info.read_index > self.position { - self.position = info.read_index; + if success && info.read_index_corrupt == 0 { + if info.read_index > self.position { + self.held.truncate(self.held.len() - self.unanswered); + self.position = info.read_index; + } + self.unanswered = 0; + self.late = false; } // The server says when it answered by its wall clock. libpulse // takes that if it falls between when it asked and heard back by @@ -623,6 +695,7 @@ impl Reader { let set = asked.map_or(true, |asked| (offset - asked).abs() > CLOCK_SET); let age = wall - info.timestamp.tv_sec * 1_000_000_000 - info.timestamp.tv_usec * 1000; let answered = now.nseconds() as i64 - age; + let (steps, steady) = (self.stamps.steps.len(), self.stamps.steady); if success && !set && info.write_index_corrupt == 0 @@ -637,6 +710,9 @@ impl Reader { { self.resumed = false; } + if self.graph_holds && self.stamps.steps.len() > steps { + self.drop_held_between(steady, self.stamps.steps.last().unwrap().0); + } if !self.holding() { self.release(); } @@ -673,12 +749,22 @@ impl Reader { .last_arrival .is_some_and(|last| now.saturating_sub(last) > FRAGMENT * 4); self.last_arrival = Some(now); + // pipewire-pulse skips audio once it holds more than the buffer, + // all but its newest fragment, and says so only in its next + // answer. Until then audio after a skip seems captured the skip + // earlier than it was, more than a buffer before it arrived. + // Audio that seems over half a buffer late, which leaves room for + // a late account, waits for that answer. + self.late |= self.stamps.time(start).is_some_and(|time| { + now.nseconds() as i64 - time > self.buffer.nseconds() as i64 / 2 + }); // While holding, each answer about newer audio counts. if self.holding() { self.request_timing(); } if self.holding() { self.held.push((start, buffer)); + self.unanswered += 1; // Before the server first answers, as long as video waits // for a track, and afterwards a buffer's worth. let limit = if self.stamps.offset.is_none() { @@ -696,6 +782,7 @@ impl Reader { .offset .get_or_insert(now.nseconds() as i64 - time_of(self.position)); self.resumed = false; + self.late = false; if !self.release() { return; } @@ -707,11 +794,38 @@ impl Reader { } } + // Held audio from `from` to `to` has no known place: it goes, and audio + // from `to` on stays. + fn drop_held_between(&mut self, from: i64, to: i64) { + let unanswered = self.held.len() - self.unanswered; + let first = self.held.get(unanswered).map_or(i64::MAX, |held| held.0); + self.held = std::mem::take(&mut self.held) + .into_iter() + .filter_map(|(position, buffer)| { + let end = position + buffer.size() as i64; + if end <= from || position >= to { + return Some((position, buffer)); + } + (end > to).then(|| { + let at = (to - position) as usize; + ( + to, + buffer + .copy_region(gst::BufferCopyFlags::MEMORY, at..) + .unwrap(), + ) + }) + }) + .collect(); + self.unanswered = self.held.iter().filter(|held| held.0 >= first).count(); + } + fn holding(&self) -> bool { - self.stamps.offset.is_none() || self.resumed || self.stamps.rising.is_some() + self.stamps.offset.is_none() || self.resumed || self.late || self.stamps.rising.is_some() } unsafe fn release(&mut self) -> bool { + self.unanswered = 0; std::mem::take(&mut self.held) .into_iter() .all(|(position, buffer)| self.push(position, buffer)) @@ -808,6 +922,8 @@ struct Stamps { recent: std::collections::VecDeque, last_capturing: i64, last_reply: i64, + // The write index in the last reply that said nothing was lost. + steady: i64, } impl Stamps { @@ -822,6 +938,18 @@ impl Stamps { .min() } + // When the audio at `position` was captured, in running time, as pts + // will stamp it. + fn time(&self, position: i64) -> Option { + let steps: i64 = self + .steps + .iter() + .filter(|&&(from, _)| position >= from) + .map(|&(_, amount)| amount) + .sum(); + Some(time_of(position) + self.offset? + steps) + } + fn pts(&mut self, position: i64) -> Option { let offset = self.offset.as_mut()?; self.steps.retain(|&(from, amount)| { @@ -853,6 +981,7 @@ impl Stamps { let account = answered - latency - time_of(write); let elapsed = now - std::mem::replace(&mut self.last_reply, now); let Some(offset) = self.offset.as_mut() else { + self.steady = write; // The account is a little late, never early, the first one most // often, so audio waits for the least of the first few. self.recent.push_back(account); @@ -883,6 +1012,7 @@ impl Stamps { return true; } self.rising = None; + self.steady = write; self.recent.push_back(account); if self.recent.len() > WINDOW { self.recent.pop_front(); @@ -1093,4 +1223,417 @@ mod tests { assert!((-51 * MS..=-49 * MS).contains(&at(60)), "{}", at(60)); assert_eq!(at(200), -100 * MS); } + + // libpulse and the server behind it, for a Reader: audio and answers + // arrive in the order the server sends them. + #[derive(Default)] + struct Socket { + audio: std::collections::VecDeque>, + info: Option>, + // Questions the reader asked that the server hasn't answered. + questions: usize, + } + + thread_local! { + static SOCKET: std::cell::RefCell = std::cell::RefCell::default(); + } + + unsafe extern "C" fn peek( + _: *mut Stream, + data: *mut *const c_void, + length: *mut usize, + ) -> c_int { + SOCKET.with_borrow(|socket| { + let audio = socket.audio.front(); + *data = audio.map_or(std::ptr::null(), |audio| audio.as_ptr().cast()); + *length = audio.map_or(0, Vec::len); + }); + 0 + } + + unsafe extern "C" fn drop_audio(_: *mut Stream) -> c_int { + SOCKET.with_borrow_mut(|socket| socket.audio.pop_front()); + 0 + } + + unsafe extern "C" fn ask( + _: *mut Stream, + _: Option, + _: *mut c_void, + ) -> *mut Operation { + SOCKET.with_borrow_mut(|socket| socket.questions += 1); + std::ptr::NonNull::dangling().as_ptr() + } + + unsafe extern "C" fn timing_info(_: *mut Stream) -> *const TimingInfo { + SOCKET.with_borrow(|socket| { + socket + .info + .as_deref() + .map_or(std::ptr::null(), |info| info as *const TimingInfo) + }) + } + + unsafe extern "C" fn unref(_: *mut Operation) {} + + unsafe extern "C" fn restart(_: *mut Context, _: *mut TimeEvent, _: u64) {} + + unsafe extern "C" fn rtclock_now() -> u64 { + 0 + } + + // Frames that hold their index, in two channels of 15 bits. + fn pcm(frames: std::ops::Range) -> Vec { + frames + .flat_map(|frame| [frame % 32768, frame / 32768]) + .flat_map(|sample| (sample as f32 / 32768.0).to_le_bytes()) + .collect() + } + + // A buffer a Reader sent, with its first and last frame. + #[derive(Debug, PartialEq)] + struct Sent { + pts: i64, + duration: i64, + discont: bool, + first: u64, + last: u64, + } + + // A Reader on a fake libpulse whose server sends fragment n, frames + // n * 480 on, at (n + 1) * 10 ms of running time, once it captured it, + // and says it captured audio a millisecond before it did. The running + // time stands still where the test sets it. + struct Wire { + reader: Box, + clock: gst::Clock, + pipeline: gst::Pipeline, + sink: gst::Element, + } + + impl Wire { + fn new() -> Self { + gst::init().unwrap(); + SOCKET.set(Socket::default()); + let library = Box::leak(Box::new(Library { + stream_peek: peek, + stream_drop: drop_audio, + stream_update_timing_info: ask, + stream_get_timing_info: timing_info, + operation_unref: unref, + context_rttime_restart: restart, + rtclock_now, + ..Library::none() + })); + let source = gst::ElementFactory::make("appsrc") + .property( + "caps", + gst::Caps::builder("audio/x-raw") + .field("format", "F32LE") + .field("rate", RATE as i32) + .field("channels", CHANNELS as i32) + .field("layout", "interleaved") + .build(), + ) + .property("format", gst::Format::Time) + .property("is-live", true) + .build() + .unwrap(); + let sink = gst::ElementFactory::make("appsink") + .property("sync", false) + .build() + .unwrap(); + let pipeline = gst::Pipeline::new(); + pipeline.add_many([&source, &sink]).unwrap(); + source.link(&sink).unwrap(); + let clock = glib::Object::new::().upcast::(); + pipeline.use_clock(Some(&clock)); + pipeline.set_start_time(gst::ClockTime::NONE); + pipeline.set_base_time(gst::ClockTime::ZERO); + let wire = Self { + reader: Box::new(Reader::new(library, &source, gst::ClockTime::SECOND)), + clock, + pipeline, + sink, + }; + wire.at(0); + wire.pipeline.set_state(gst::State::Playing).unwrap(); + wire + } + + fn at(&self, ms: i64) { + let time = gst::ClockTime::from_mseconds(ms as u64); + self.clock + .set_calibration(self.clock.internal_time(), time, 0, 1); + } + + fn audio(&mut self, ms: i64, frames: std::ops::Range) { + self.audio_frames(ms, frames.collect()); + } + + fn audio_frames(&mut self, ms: i64, frames: Vec) { + self.at(ms); + let bytes = frames + .into_iter() + .flat_map(|frame| [frame % 32768, frame / 32768]) + .flat_map(|sample| (sample as f32 / 32768.0).to_le_bytes()) + .collect(); + SOCKET.with_borrow_mut(|socket| socket.audio.push_back(bytes)); + unsafe { self.reader.read() }; + } + + // The server answers every question on the way: it captured `write` + // frames, and sent or skipped `read`. + fn answer(&mut self, ms: i64, write: u64, read: u64) { + self.answer_with(ms, write, read, 0); + } + + fn answer_with(&mut self, ms: i64, write: u64, read: u64, latency: u64) { + self.at(ms); + for _ in 0..SOCKET.with_borrow_mut(|socket| std::mem::take(&mut socket.questions)) { + let wall = wall_clock().0 + MS; + let info = TimingInfo { + timestamp: libc::timeval { + tv_sec: wall / 1_000_000_000, + tv_usec: wall % 1_000_000_000 / 1000, + }, + synchronized_clocks: 1, + sink_usec: 0, + source_usec: latency, + transport_usec: 0, + playing: 1, + write_index_corrupt: 0, + write_index: (write * FRAME as u64) as i64, + read_index_corrupt: 0, + read_index: (read * FRAME as u64) as i64, + configured_sink_usec: 0, + configured_source_usec: 0, + since_underrun: 0, + }; + SOCKET.with_borrow_mut(|socket| socket.info = Some(Box::new(info))); + unsafe { self.reader.timing_updated(true) }; + } + } + + // The reader's timer. + fn tick(&mut self, ms: i64) { + self.at(ms); + unsafe { self.reader.ask_again() }; + } + + // Fragments sent as captured, each answering the questions on the + // way, with the timer going off every 100 ms. + fn steady(&mut self, fragments: std::ops::Range) { + for n in fragments { + let ms = (n as i64 + 1) * 10; + if ms % 100 == 0 { + self.tick(ms); + } + self.audio(ms, n * 480..(n + 1) * 480); + self.answer(ms, (n + 1) * 480, (n + 1) * 480); + } + } + + fn sent(self) -> Vec { + let source = self.reader.source.upgrade().unwrap(); + let ended = source.emit_by_name::("end-of-stream", &[]); + assert_eq!(ended, gst::FlowReturn::Ok); + let mut sent = Vec::new(); + while let Some(sample) = self + .sink + .emit_by_name::>("try-pull-sample", &[&gst::ClockTime::SECOND]) + { + let buffer = sample.buffer().unwrap(); + let map = buffer.map_readable().unwrap(); + let frame = |i: usize| { + let sample = |j: usize| { + (f32::from_le_bytes(map[j * 4..j * 4 + 4].try_into().unwrap()) * 32768.0) + .round() as u64 + }; + sample(2 * i) + sample(2 * i + 1) * 32768 + }; + sent.push(Sent { + pts: buffer.pts().unwrap().nseconds() as i64, + duration: buffer.duration().unwrap().nseconds() as i64, + discont: buffer.flags().contains(gst::BufferFlags::DISCONT), + first: frame(0), + last: frame(map.len() / FRAME - 1), + }); + } + self.pipeline.set_state(gst::State::Null).unwrap(); + sent + } + } + + fn frame_time(frame: u64) -> i64 { + time_of((frame * FRAME as u64) as i64) + } + + // Every buffer is stamped when its frames were captured, by the server's + // account, and marked discont where frames are missing. Returns + // the frames missing between the first and the last buffer, from and to. + // The account is a millisecond late, less the time between the test + // reading the wall clock for an answer and the reader reading it again: + // microseconds, more on a busy machine. + fn assert_placed(sent: &[Sent]) -> Vec<(u64, u64)> { + let mut missing = Vec::new(); + for (i, buffer) in sent.iter().enumerate() { + let late = buffer.pts - frame_time(buffer.first); + assert!( + (0..=1_000_000).contains(&late), + "{buffer:?} is {late} ns late" + ); + assert_eq!( + buffer.duration, + frame_time(buffer.last + 1) - frame_time(buffer.first), + "{buffer:?}" + ); + let next = sent[..i] + .last() + .map_or(buffer.first, |before| before.last + 1); + if buffer.first != next { + assert!(buffer.discont, "{buffer:?}"); + missing.push((next, buffer.first)); + } + } + missing + } + + // pipewire-pulse's main thread stops for 1.05 s while its data thread + // keeps capturing. Resuming, it skips all but the newest fragment of its + // 1 s ring and sends that, then answers the questions asked meanwhile. + // The fragment can't be placed: only the answer after it says there was + // a skip, not where. Placed after the audio before the skip, it was a + // second early. + #[test] + fn audio_after_a_skip_in_what_arrived_since_the_last_answer_is_lost_with_it() { + let mut wire = Wire::new(); + wire.steady(0..300); + for ms in (3100..=4000).step_by(100) { + wire.tick(ms); + } + wire.audio(4050, 404 * 480..405 * 480); + wire.answer(4050, 405 * 480, 405 * 480); + wire.steady(405..500); + let sent = wire.sent(); + assert_eq!(assert_placed(&sent), [(300 * 480, 405 * 480)]); + assert_eq!(sent.last().unwrap().last, 500 * 480 - 1); + } + + // The reader stops at 3 s for 2.5 s. Half a second of audio waits in the + // socket, and the server's next fragment waits for room there, so the + // server holds the rest and overflows. When the reader takes up again, + // the server answers after that audio, which keeps its place, then skips + // and sends its newest fragment, which arrives promptly but is lost with + // the skip. Placed by when it arrived, the audio waiting in the socket + // would have looked lost. + #[test] + fn audio_held_in_the_socket_keeps_its_place_and_audio_after_a_skip_is_lost() { + let mut wire = Wire::new(); + wire.steady(0..300); + for n in 300..351 { + wire.audio(5500, n * 480..(n + 1) * 480); + } + wire.answer(5500, 550 * 480, 351 * 480); + wire.audio(5505, 549 * 480..550 * 480); + wire.answer(5505, 550 * 480, 550 * 480); + wire.steady(550..650); + let sent = wire.sent(); + assert_eq!(assert_placed(&sent), [(351 * 480, 550 * 480)]); + assert_eq!(sent.last().unwrap().last, 650 * 480 - 1); + } + + // The same for 0.6 s, which the server holds: nothing is lost. + #[test] + fn audio_held_in_the_socket_and_the_server_is_all_kept() { + let mut wire = Wire::new(); + wire.steady(0..300); + for n in 300..351 { + wire.audio(3600, n * 480..(n + 1) * 480); + } + wire.answer(3600, 360 * 480, 351 * 480); + for n in 351..360 { + wire.audio(3605, n * 480..(n + 1) * 480); + } + wire.steady(360..450); + let sent = wire.sent(); + assert_eq!(assert_placed(&sent), []); + assert_eq!(sent.last().unwrap().last, 450 * 480 - 1); + } + + // pipewire-pulse stops whole for 300 ms at 3 s; its graph keeps going + // and its stream misses 298 ms of cycles. Resuming, it answers first, + // with the write index of its last cycle, frame 144000, and 240 frames + // still in its graph, of which 96 arrive and the rest are lost. Then it + // sends those 96 frames and audio captured 298 ms later in one fragment. + // The step goes at the write index plus the latency; the 144 frames + // before it that came after the loss have no known place and go with it. + #[test] + fn audio_pipewire_pulse_took_in_after_a_loss_it_had_not_reported_is_lost_with_it() { + let mut wire = Wire::new(); + wire.reader.graph_holds = true; + wire.steady(0..300); + for ms in [3100, 3200, 3300] { + wire.tick(ms); + } + wire.answer_with(3300, 300 * 480, 300 * 480, 5000); + let lost = 298 * 48; + let mut frames: Vec = (144000..144096).collect(); + frames.extend((144096..144480).map(|frame| frame + lost)); + wire.audio_frames(3313, frames); + wire.answer_with(3313, 301 * 480, 301 * 480, 5000); + for n in 301..400 { + let ms = (n as i64 + 1) * 10 + 303; + wire.tick(ms); + wire.audio_frames(ms, (n * 480 + lost..(n + 1) * 480 + lost).collect()); + wire.answer_with(ms, (n + 1) * 480, (n + 1) * 480, 5000); + } + let sent = wire.sent(); + assert_eq!(assert_placed(&sent), [(144000, 144240 + lost)]); + } + + // A buffer a step falls inside goes out in two: its audio up to the step + // where it was, and from the step on moved by it, as a discontinuity. + #[test] + fn a_buffer_with_a_step_inside_is_sent_in_two() { + let mut wire = Wire::new(); + let at = |frame: i64| frame * FRAME as i64; + wire.reader.stamps = Stamps { + offset: Some(0), + steps: vec![(at(1000), 480 * MS)], + ..Stamps::default() + }; + for frames in [480..960, 960..1440] { + let start = at(frames.start as i64); + let buffer = gst::Buffer::from_mut_slice(pcm(frames)); + assert!(unsafe { wire.reader.push(start, buffer) }); + } + let time = frame_time; + assert_eq!( + wire.sent(), + [ + Sent { + pts: time(480), + duration: time(960) - time(480), + discont: true, + first: 480, + last: 959, + }, + Sent { + pts: time(960), + duration: time(1000) - time(960), + discont: false, + first: 960, + last: 999, + }, + Sent { + pts: time(1000) + 480 * MS, + duration: time(1440) - time(1000), + discont: true, + first: 1000, + last: 1439, + }, + ] + ); + } } diff --git a/packaging/linux/README.md b/packaging/linux/README.md index 35c7028..aa66b5c 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -353,19 +353,39 @@ server ten times a second for the capture time of its newest audio and how much audio it has captured and sent. Audio that takes longer to arrive, or a recorder that stops reading for a moment, leaves that account unchanged, so the audio keeps its place and nothing is reported. Audio the server lost leaves a gap of -the same length where it was lost, and wrec reports it. That covers audio its -device or graph lost, audio PulseAudio dropped because the recorder fell more -than both buffers behind, and audio `pipewire-pulse` skipped for the same -reason. A device clock that runs faster or slower than the system clock moves -the audio at most 1 ms per second and is not reported. +the same length, and wrec reports it. That covers audio its device or graph +lost, audio PulseAudio dropped because the recorder fell more than both buffers +behind, and audio `pipewire-pulse` skipped for the same reason. +`pipewire-pulse` skips without marking where and says so only in its next +timing reply, so the audio that arrived since its previous reply has no known +place. wrec drops that audio and reports it with the skip as one gap; in the +controlled tests it was one 10 ms fragment. A device clock that runs faster or +slower than the system clock moves the audio at most 1 ms per second and is not +reported. This has limits. Changes in the server's account of 20 ms or less are never reported; the audio moves smoothly instead. A device clock more than 1000 parts per million slow builds up until it reads as lost audio, and one that fast -leaves the audio drifting late. A loss is placed at the newest audio the server -had captured when it first reported it, so audio captured right after a loss can -play that much early; in the controlled tests every buffer stayed within 11 ms -of its capture time. The account is only as good as the server's latency +leaves the audio drifting late. The length of each gap is reported, but where +in the audio it falls is not guaranteed, and the job events give the time wrec +placed it at. wrec places a loss after the newest audio the server had captured +when it first reported it, counting audio its device still held. PulseAudio +reports a stall before it reads what its device held, so when the server counts +that audio, the gap falls where the loss was; in the controlled tests every +buffer stayed within 11 ms of its capture time. A device holding audio the +server does not count can put the gap after the loss, and the audio between +those boundaries can play early by the whole loss. `pipewire-pulse` also +counts audio still in its graph, which a server stall can lose. When the +server identifies itself as PipeWire and capture stops arriving, wrec discards +the held audio whose position is uncertain after a confirmed loss. It reports +that discard with the loss. This prevented early audio in the controlled +0.1 and 0.3 s server-stall tests, at the cost of 5 to 13 ms of additional +audio per loss. It cannot protect audio already sent before the loss was +known, or a graph loss with no break in arriving audio. +When PulseAudio drops audio for a recorder that fell more than both buffers +behind, it drops it in pieces over the following second, and in the controlled +tests 1280 to 2720 frames (27 to 57 ms) of that second played early, by 9 ms up +to the whole loss. The account is only as good as the server's latency reporting. Clock drift is validated only with a synthetic device 800 parts per million fast or slow, not with physical microphones. These are experimental support limits; a completed job alone does not establish A/V sync. @@ -386,8 +406,8 @@ error anywhere else in the audio chain, such as the AAC encoder, still fails the job. A completed job is not proof of a whole movie. When frames were dropped, audio has gaps, or a track ended early or never started, the job finishes `completed` with -a `media_lost` warning that states what is missing, and the job events record when -each audio gap happened. `wrec record` prints the warning when it finishes. +a `media_lost` warning that states what is missing, and the job events record +where wrec placed each audio gap. `wrec record` prints the warning when it finishes. On Wayland, a recording watches four things besides its frames: the ScreenCast session's `Closed` signal, the D-Bus owner of `org.freedesktop.portal.Desktop`, From b9e962569a230958ffd21c557ef3937df683e6f1 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:55:10 +0000 Subject: [PATCH 38/45] test(linux): judge audio startup against source running time --- crates/linux/src/pipeline.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index bce6ecc..ccf5151 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -2814,8 +2814,9 @@ mod tests { reported: f64, // Where the audio ends after the video. after_video: f64, - // Where the audio starts after the video. - starts_after_video: f64, + // Where the audio starts in the movie, which starts with its + // earliest track. + audio_starts: f64, // When the system audio track ended, if it failed. stopped: Option, // When the recording had started. @@ -2864,7 +2865,7 @@ mod tests { hole: audio_holes(&probe)[0], reported, after_video: end_seconds(&probe, "aac") - end_seconds(&probe, "h264"), - starts_after_video: start_seconds(&probe, "aac")[0] - start_seconds(&probe, "h264")[0], + audio_starts: start_seconds(&probe, "aac")[0], stopped: lost .split_once("system audio stopped at ") .map(|(_, rest)| rest.to_string()), @@ -3262,9 +3263,19 @@ mod tests { 0, "the device lost audio during the recording" ); + // The device was writing before the recording, so the source's + // first audio is at running time 0, when the recording started, + // and the movie starts with it. Neither is judged against the + // video, because the test's videotestsrc stamps its first frame + // when its thread makes it. That is late if the thread doesn't + // run for a while after the pipeline plays, or its first + // allocation query waits for the encoder. The movie alone can't + // show running time, because mp4mux starts it at its earliest + // track. + let audio_began = placed.lock().unwrap()[0].0; assert!( - recorded.whole() && recorded.starts_after_video.abs() < 0.05, - "set {first} then {later} ns: {recorded:?}" + recorded.whole() && audio_began < 0.05 && recorded.audio_starts.abs() < 0.05, + "set {first} then {later} ns: audio began at {audio_began:.3} s, {recorded:?}" ); } let server = PulseServer::start_with(PipeDevice::CONFIG, false); From 0d201cf7a6ee08e55dbd52f317aa2408dd819632 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:50:49 +0000 Subject: [PATCH 39/45] fix(linux): preserve known audio around graph loss --- crates/linux/src/pipeline.rs | 10 +-- crates/linux/src/pulse.rs | 120 ++++++++++++++++++++++++++++------- 2 files changed, 104 insertions(+), 26 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index ccf5151..e4b2dc6 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -3235,8 +3235,10 @@ mod tests { ) else { return; }; - // Answers take 0.6 s, so those to the first questions, asked once - // audio arrives 0.6 s in, are on the way 0.9 s in. + // This closure starts about 0.1 s in, and wrec asks the server every + // 0.1 s. Answers take 0.6 s, so the three that place the start of the + // audio, asked 0.1 to 0.3 s in, are all on the way 0.5 s in, when the + // clock is first set. for (first, later) in [(2 * SECOND, -2 * SECOND), (-2 * SECOND, 2 * SECOND)] { let server = PulseServer::start_with(PipeDevice::CONFIG, false); clock.shared_with(&server); @@ -3248,9 +3250,9 @@ mod tests { let mut began = None; let recorded = record_system_audio_with(source, |_| { began = Some(Instant::now()); - std::thread::sleep(Duration::from_millis(900)); + std::thread::sleep(Duration::from_millis(400)); clock.set(first); - std::thread::sleep(Duration::from_millis(3100)); + std::thread::sleep(Duration::from_millis(3600)); clock.set(later); std::thread::sleep(Duration::from_secs(1)); link.set_delay(Duration::ZERO); diff --git a/crates/linux/src/pulse.rs b/crates/linux/src/pulse.rs index b733868..f79f2fd 100644 --- a/crates/linux/src/pulse.rs +++ b/crates/linux/src/pulse.rs @@ -795,27 +795,28 @@ impl Reader { } // Held audio from `from` to `to` has no known place: it goes, and audio - // from `to` on stays. + // before `from` and from `to` on stays, also from a buffer either falls + // inside. fn drop_held_between(&mut self, from: i64, to: i64) { let unanswered = self.held.len() - self.unanswered; let first = self.held.get(unanswered).map_or(i64::MAX, |held| held.0); self.held = std::mem::take(&mut self.held) .into_iter() - .filter_map(|(position, buffer)| { + .flat_map(|(position, buffer)| { let end = position + buffer.size() as i64; if end <= from || position >= to { - return Some((position, buffer)); + return [Some((position, buffer)), None]; } - (end > to).then(|| { - let at = (to - position) as usize; - ( - to, - buffer - .copy_region(gst::BufferCopyFlags::MEMORY, at..) - .unwrap(), - ) - }) + let part = |start: i64, stop: i64| { + (start < stop).then(|| { + let range = (start - position) as usize..(stop - position) as usize; + let part = buffer.copy_region(gst::BufferCopyFlags::MEMORY, range); + (start, part.unwrap()) + }) + }; + [part(position, from), part(to, end)] }) + .flatten() .collect(); self.unanswered = self.held.iter().filter(|held| held.0 >= first).count(); } @@ -1570,26 +1571,101 @@ mod tests { // before it that came after the loss have no known place and go with it. #[test] fn audio_pipewire_pulse_took_in_after_a_loss_it_had_not_reported_is_lost_with_it() { + let sent = graph_loss(144000, &[144000], false); + assert_eq!(assert_placed(&sent), [(144000, 144240 + LOST)]); + } + + // Frames pipewire-pulse lost in its graph in graph_loss. + const LOST: u64 = 298 * 48; + + // pipewire-pulse says it captured audio up to frame 144000 when it has + // sent audio up to `sent`, then stops as above. Resuming, it sends the + // audio from `sent` in buffers that start at `cuts`, before or after its + // second answer. + fn graph_loss(sent: u64, cuts: &[u64], answer_first: bool) -> Vec { let mut wire = Wire::new(); wire.reader.graph_holds = true; - wire.steady(0..300); + wire.steady(0..299); + wire.tick(3000); + if sent > 299 * 480 { + wire.audio(3000, 299 * 480..sent); + } + wire.answer(3000, 300 * 480, sent); for ms in [3100, 3200, 3300] { wire.tick(ms); } - wire.answer_with(3300, 300 * 480, 300 * 480, 5000); - let lost = 298 * 48; - let mut frames: Vec = (144000..144096).collect(); - frames.extend((144096..144480).map(|frame| frame + lost)); - wire.audio_frames(3313, frames); - wire.answer_with(3313, 301 * 480, 301 * 480, 5000); + wire.answer_with(3300, 300 * 480, sent, 5000); + if answer_first { + wire.tick(3313); + wire.answer_with(3313, 301 * 480, sent, 5000); + } + let truth = |frame: u64| if frame < 144096 { frame } else { frame + LOST }; + for (i, &start) in cuts.iter().enumerate() { + let end = cuts.get(i + 1).copied().unwrap_or(301 * 480); + wire.audio_frames(3313, (start..end).map(truth).collect()); + } + if !answer_first { + wire.answer_with(3313, 301 * 480, 301 * 480, 5000); + } for n in 301..400 { let ms = (n as i64 + 1) * 10 + 303; wire.tick(ms); - wire.audio_frames(ms, (n * 480 + lost..(n + 1) * 480 + lost).collect()); + wire.audio_frames(ms, (n * 480 + LOST..(n + 1) * 480 + LOST).collect()); wire.answer_with(ms, (n + 1) * 480, (n + 1) * 480, 5000); } - let sent = wire.sent(); - assert_eq!(assert_placed(&sent), [(144000, 144240 + lost)]); + wire.sent() + } + + // When the server stops before it sends what it said it captured, the + // audio before that arrives with the loss and keeps its place, whether + // one buffer holds both ends of the audio that goes, two hold one each, + // or none holds either, and whether it arrives before or after an answer. + #[test] + fn audio_from_before_pipewire_pulse_said_it_captured_it_keeps_its_place() { + let (prefix, kept) = ((143760, 143999), (144240 + LOST, 144479 + LOST)); + for (sent, cuts, answer_first, around) in [ + ( + 143760, + &[143760][..], + false, + [(143520, 143759), prefix, kept], + ), + ( + 143760, + &[143760][..], + true, + [(143520, 143759), prefix, kept], + ), + ( + 143760, + &[143760, 144120][..], + false, + [(143520, 143759), prefix, kept], + ), + ( + 143520, + &[143520, 144000, 144240][..], + false, + [(143040, 143519), (143520, 143999), kept], + ), + ] { + let sent = graph_loss(sent, cuts, answer_first); + let lost = assert_placed(&sent); + assert_eq!(lost, [(144000, 144240 + LOST)], "{cuts:?} {answer_first}"); + let at = sent + .iter() + .position(|buffer| buffer.first == around[0].0) + .unwrap(); + let got: Vec<_> = sent[at..at + 3].iter().map(|b| (b.first, b.last)).collect(); + assert_eq!(got, around, "{cuts:?} {answer_first}"); + // The gap count_lost_audio reports, within the account's lateness. + let gap = sent[at + 2].pts - (sent[at + 1].pts + sent[at + 1].duration); + let missing = frame_time(144240 + LOST) - frame_time(144000); + assert!( + (gap - missing).abs() <= MS, + "{cuts:?} {answer_first}: {gap} ns, not {missing}" + ); + } } // A buffer a step falls inside goes out in two: its audio up to the step From 6c71c544bc1394ca8cc989d802873ad50d1db0ba Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:45:10 +0000 Subject: [PATCH 40/45] fix(linux): retain delayed audio across pauses and report missing tails --- crates/linux/src/pipeline.rs | 809 +++++++++++++++++++++++++++++++++-- packaging/linux/README.md | 12 +- 2 files changed, 778 insertions(+), 43 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index e4b2dc6..46b5a2a 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -167,13 +167,19 @@ struct Counters { timeline: Mutex, } +// Pausing takes out of the movie what was captured while paused, judged by +// capture time, not by when it arrives: a source whose downstream holds it +// back can deliver audio captured before a pause while paused, or audio +// captured while paused after resuming. #[derive(Default)] struct Timeline { paused_at: Option, + // The length of the finished pauses. offset: gst::ClockTime, - accept_from: gst::ClockTime, // Finished pauses, in running time, oldest first. pauses: Vec<(gst::ClockTime, gst::ClockTime)>, + // When the recording stopped, in running time. + stopped_at: Option, } impl Timeline { @@ -184,11 +190,15 @@ impl Timeline { fn resume(&mut self, now: gst::ClockTime) { if let Some(paused_at) = self.paused_at.take() { self.offset += now.saturating_sub(paused_at); - self.accept_from = now; self.pauses.push((paused_at, now)); } } + fn stop(&mut self, now: gst::ClockTime) { + self.resume(now); + self.stopped_at.get_or_insert(now); + } + // How much of the running time from `start` to `end` was paused. fn paused_between(&self, start: gst::ClockTime, end: gst::ClockTime) -> gst::ClockTime { self.paused_at @@ -204,19 +214,117 @@ impl Timeline { self.paused_at.unwrap_or(now).saturating_sub(self.offset) } + // Pauses that ended after `time`, oldest first, then the one going on. + fn pauses_after( + &self, + time: gst::ClockTime, + ) -> impl Iterator + '_ { + self.closed_after(time) + .iter() + .copied() + .chain(self.paused_at.map(|at| (at, gst::ClockTime::MAX))) + } + + fn closed_after(&self, time: gst::ClockTime) -> &[(gst::ClockTime, gst::ClockTime)] { + let first = self.pauses.partition_point(|&(_, resumed)| resumed <= time); + &self.pauses[first..] + } + + // How much earlier what was captured at `time` goes in the movie: by + // every pause that ended by then. None if it was captured while paused. + fn offset_at(&self, time: gst::ClockTime) -> Option { + let later = self.closed_after(time); + if self.paused_at.is_some_and(|at| at <= time) + || later.first().is_some_and(|&(paused, _)| paused <= time) + { + return None; + } + let later: gst::ClockTime = later + .iter() + .map(|&(paused, resumed)| resumed - paused) + .sum(); + Some(self.offset - later) + } + + // A video frame goes by when it was captured. fn retime(&self, buffer: &mut gst::Buffer) -> bool { - if self.paused_at.is_some() || buffer.pts().is_some_and(|pts| pts < self.accept_from) { + let offset = match buffer.pts() { + Some(pts) => self.offset_at(pts), + None => self.paused_at.is_none().then_some(self.offset), + }; + let Some(offset) = offset else { return false; - } - if self.offset > gst::ClockTime::ZERO { + }; + if offset > gst::ClockTime::ZERO { // make_mut copies a shared buffer header, retaining references to // its memory. DMA-BUF pixels remain on the GPU. let buffer = buffer.make_mut(); - buffer.set_pts(buffer.pts().map(|pts| pts.saturating_sub(self.offset))); - buffer.set_dts(buffer.dts().map(|dts| dts.saturating_sub(self.offset))); + buffer.set_pts(buffer.pts().map(|pts| pts.saturating_sub(offset))); + buffer.set_dts(buffer.dts().map(|dts| dts.saturating_sub(offset))); } true } + + // Audio goes frame by frame, `frame` bytes each at `rate`: a buffer a + // pause starts or ends inside keeps only its frames captured outside + // every pause, in one buffer, since taking the pauses out leaves them + // continuous. + fn retime_audio(&self, buffer: &mut gst::Buffer, rate: u32, frame: usize) -> bool { + let Some(pts) = buffer.pts() else { + return self.retime(buffer); + }; + let frames = buffer.size() / frame; + // The first frame captured at or after `time`. + let index = |time: gst::ClockTime| { + let since = time.saturating_sub(pts).nseconds() as u128; + (since * rate as u128) + .div_ceil(1_000_000_000) + .min(frames as u128) as usize + }; + let mut kept = Vec::new(); + let mut from = 0; + for (paused, resumed) in self.pauses_after(pts) { + let start = index(paused); + if start > from { + kept.push(from..start); + } + from = from.max(index(resumed)); + if from == frames { + break; + } + } + if from < frames { + kept.push(from..frames); + } + let Some(first) = kept.first().map(|frames| frames.start) else { + return false; + }; + if kept[0] != (0..frames) { + let mut pieces = kept.iter().map(|frames| { + buffer + .copy_region( + gst::BufferCopyFlags::FLAGS | gst::BufferCopyFlags::MEMORY, + frames.start * frame..frames.end * frame, + ) + .unwrap() + }); + let mut audio = pieces.next().unwrap(); + for piece in pieces { + audio.append(piece); + } + let length: usize = kept.iter().map(|frames| frames.len()).sum(); + let audio_mut = audio.get_mut().unwrap(); + audio_mut.set_pts(pts + frame_time(first, rate)); + audio_mut.set_duration(frame_time(length, rate)); + *buffer = audio; + } + self.retime(buffer) + } +} + +// How long `frames` audio frames at `rate` last. +fn frame_time(frames: usize, rate: u32) -> gst::ClockTime { + gst::ClockTime::from_nseconds(frames as u64 * 1_000_000_000 / rate as u64) } pub(crate) enum CaptureInput { @@ -628,6 +736,8 @@ struct AudioTrack { state: Arc, // Nanoseconds of audio missing between captured buffers. lost: Arc, + // Where the captured audio ends, in running time. + end: Arc>>, // Where in the movie the track failed, and why. failed: Option<(gst::ClockTime, String)>, } @@ -709,6 +819,7 @@ impl Movie { queue: &gst::Element, source: &gst::Element, lost: Arc, + end: Arc>>, ) { let pad = queue.static_pad("src").unwrap(); // Linking the track sends a reconfigure upstream. The encoder answers @@ -759,6 +870,7 @@ impl Movie { probe, state, lost, + end, failed: None, }); } @@ -841,8 +953,9 @@ fn add_audio_source( ) -> Result<()> { // Counted before pausing drops anything. let lost = Arc::new(AtomicU64::new(0)); - count_lost_audio(source, lost.clone(), counters.clone()); - attach_timing_probe(source, counters.clone()); + let end = Arc::new(Mutex::new(None)); + count_lost_audio(source, lost.clone(), end.clone(), counters.clone()); + attach_audio_timing_probe(source, counters.clone()); let convert = element("audioconvert")?; let resample = element("audioresample")?; let caps = element("capsfilter")?; @@ -873,7 +986,7 @@ fn add_audio_source( ]; movie.pipeline.add_many(chain).map_err(backend)?; gst::Element::link_many(chain).map_err(backend)?; - movie.add_audio(name, &queue, source, lost); + movie.add_audio(name, &queue, source, lost, end); Ok(()) } @@ -886,9 +999,13 @@ pub(crate) const AUDIO_HOLE: gst::ClockTime = gst::ClockTime::from_mseconds(20); // where capture actually resumed. Sources keep capturing while paused, so a // pause leaves no gap, and the paused part of a gap is not in the movie. // Capture timestamps and pauses are both in running time. -fn count_lost_audio(source: &gst::Element, lost: Arc, counters: Arc) { - // Where the next buffer should start. - let next = Mutex::new(None::); +// `next` is where the next buffer should start. +fn count_lost_audio( + source: &gst::Element, + lost: Arc, + next: Arc>>, + counters: Arc, +) { source .static_pad("src") .unwrap() @@ -923,6 +1040,8 @@ fn retime(info: &mut gst::PadProbeInfo<'_>, counters: &Counters) -> bool { true } +// The pause gate for a video source. +#[cfg(test)] fn attach_timing_probe(source: &gst::Element, counters: Arc) { source .static_pad("src") @@ -936,6 +1055,63 @@ fn attach_timing_probe(source: &gst::Element, counters: Arc) { }); } +// The rate and bytes per frame of interleaved raw audio. +fn audio_frames(caps: &gst::CapsRef) -> Option<(u32, usize)> { + let structure = caps.structure(0)?; + if structure.get::<&str>("layout").ok() != Some("interleaved") { + return None; + } + let format = structure.get::<&str>("format").ok()?; + let bits: usize = format + .get(1..)? + .split(|c: char| !c.is_ascii_digit()) + .next()? + .parse() + .ok()?; + // S24_32LE and the like keep each sample in 32 bits. + let bytes = if format.contains("_32") { + 4 + } else { + bits.div_ceil(8) + }; + let rate = structure.get::("rate").ok()?; + let channels = structure.get::("channels").ok()?; + (rate > 0 && channels > 0).then(|| (rate as u32, bytes * channels as usize)) +} + +fn attach_audio_timing_probe(source: &gst::Element, counters: Arc) { + source + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |pad, info| { + let frames = pad.current_caps().and_then(|caps| audio_frames(&caps)); + let Some(gst::PadProbeData::Buffer(buffer)) = info.data.as_mut() else { + return gst::PadProbeReturn::Ok; + }; + let timeline = counters.timeline.lock().unwrap(); + let keep = match frames { + Some((rate, frame)) => timeline.retime_audio(buffer, rate, frame), + // Audio of unknown shape can't be cut, so it goes whole or + // not at all. + None => { + let start = buffer.pts(); + let end = start + .zip(buffer.duration()) + .map(|(pts, length)| pts + length); + let paused = start.zip(end).is_some_and(|(start, end)| { + timeline.paused_between(start, end) > gst::ClockTime::ZERO + }); + !paused && timeline.retime(buffer) + } + }; + if keep { + gst::PadProbeReturn::Ok + } else { + gst::PadProbeReturn::Drop + } + }); +} + thread_local! { static SETTING_CANVAS: std::cell::Cell = const { std::cell::Cell::new(false) }; } @@ -1218,7 +1394,7 @@ fn run( .timeline .lock() .unwrap() - .resume(pipeline.current_running_time().unwrap_or_default()); + .stop(pipeline.current_running_time().unwrap_or_default()); // A source pushing into a full queue holds its stream lock, and // sending it EOS waits for that lock, so wait here instead. let ending = pipeline.clone(); @@ -1299,6 +1475,7 @@ fn run( &omitted, dropped_frames(pipeline, counters), video_cut, + &counters.timeline.lock().unwrap(), ) { let _ = events.send(RecorderEvent::MediaLost { session_id: session.id, @@ -1419,6 +1596,27 @@ fn seconds(time: gst::ClockTime) -> f64 { time.nseconds() as f64 / 1e9 } +// Audio captured this long before the stop can still be on its way from its +// server when the stop ends its track. +const AUDIO_TAIL: gst::ClockTime = gst::ClockTime::from_mseconds(100); + +// Recorded time from where a track's captured audio ends to the stop. No +// later audio shows this gap, as when its source waited on a full queue at +// the stop, or its device went quiet. +fn missing_end(timeline: &Timeline, end: Option) -> gst::ClockTime { + let (Some(end), Some(stopped)) = (end, timeline.stopped_at) else { + return gst::ClockTime::ZERO; + }; + let missing = stopped + .saturating_sub(end) + .saturating_sub(timeline.paused_between(end, stopped)); + if missing > AUDIO_TAIL { + missing + } else { + gst::ClockTime::ZERO + } +} + // What the finished movie is missing, if anything. A movie that plays is not // necessarily whole. fn media_lost( @@ -1426,6 +1624,7 @@ fn media_lost( omitted: &[&str], dropped_frames: u64, video_cut: bool, + timeline: &Timeline, ) -> Option { let mut lost = Vec::new(); if video_cut { @@ -1438,7 +1637,10 @@ fn media_lost( lost.push(format!("{dropped_frames} video frames were dropped")); } for track in &movie.audio { - let missing = track.lost.load(Ordering::Relaxed); + let mut missing = track.lost.load(Ordering::Relaxed); + if track.failed.is_none() && !omitted.contains(&track.name) { + missing += missing_end(timeline, *track.end.lock().unwrap()).nseconds(); + } if missing > 0 { lost.push(format!( "{} is missing {:.2} s of audio", @@ -1776,6 +1978,16 @@ mod tests { video: &str, audio: Vec<(&'static str, gst::Element)>, counting_encoder: bool, + ) -> Self { + Self::start_watched(video, audio, counting_encoder, |_| {}) + } + + // `watch` sees the pipeline once it is built, before it plays. + fn start_watched( + video: &str, + audio: Vec<(&'static str, gst::Element)>, + counting_encoder: bool, + watch: impl FnOnce(&gst::Pipeline), ) -> Self { gst::init().unwrap(); static ID: AtomicU64 = AtomicU64::new(0); @@ -1842,6 +2054,7 @@ mod tests { add_audio_source(&mut movie, source, name, &counters).unwrap(); } attach_encoded_probe(&pipeline.by_name("parser").unwrap(), counters.clone()); + watch(&pipeline); let (events_tx, events) = mpsc::channel(); let (commands, commands_rx) = mpsc::sync_channel(1); let (stop, stopped) = watch::channel(false); @@ -2086,18 +2299,6 @@ mod tests { ) } - // Audio that reaches the pipeline `lag` after it was captured. Unlike - // the ring buffer it never drops audio, however long its reader takes. - fn lagging_audio(name: &str, lag: Duration) -> gst::Element { - audio_bin( - name, - &format!( - "queue max-size-time=0 max-size-buffers=0 max-size-bytes=0 min-threshold-time={}", - lag.as_nanos() - ), - ) - } - fn audio_bin(name: &str, queue: &str) -> gst::Element { gst::init().unwrap(); let bin = gst::parse::bin_from_description( @@ -2462,17 +2663,187 @@ mod tests { ); } - // Audio captured before a pause but arriving after it is dropped by - // design. A source that delivers late makes that drop longer than a - // lost-audio hole. The source itself loses no audio, so any audio - // reported lost was dropped by pausing. Video frames a starved encoder - // drops are real losses, reported as such. + // Live audio whose frames hold their own index, in two channels of 15 + // bits as PipeDevice writes them, and reach the pipeline `lag` after + // they were captured. + fn indexed_audio(name: &str, lag: Duration) -> gst::Element { + let bin = audio_bin( + name, + &format!( + "audio/x-raw,format=F32LE,rate=48000,channels=2,layout=interleaved \ + ! queue max-size-time=0 max-size-buffers=0 max-size-bytes=0 min-threshold-time={}", + lag.as_nanos() + ), + ) + .downcast::() + .unwrap(); + let next = AtomicU64::new(0); + bin.by_name("producer") + .unwrap() + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + if let Some(gst::PadProbeData::Buffer(buffer)) = info.data.as_mut() { + let mut map = buffer.make_mut().map_writable().unwrap(); + let first = next.fetch_add(map.len() as u64 / 8, Ordering::Relaxed); + for (frame, bytes) in map.chunks_exact_mut(8).enumerate() { + let n = first + frame as u64; + let samples = [(n % 32768) as f32, (n / 32768) as f32]; + for (sample, value) in bytes.chunks_exact_mut(4).zip(samples) { + sample.copy_from_slice(&(value / 32768.0).to_le_bytes()); + } + } + } + gst::PadProbeReturn::Ok + }); + bin.upcast() + } + + // The index of each frame of F32 stereo audio that holds its own. + fn frame_indices(buffer: &gst::BufferRef) -> Vec { + let map = buffer.map_readable().unwrap(); + let sample = + |bytes: &[u8]| (f32::from_le_bytes(bytes.try_into().unwrap()) * 32768.0).round() as u64; + map.chunks_exact(8) + .map(|frame| sample(&frame[..4]) + sample(&frame[4..]) * 32768) + .collect() + } + + type Frames = Arc)>>>; + + // Each buffer that passes `pad` from now on: its pts and the index of + // each of its frames. + fn record_frames(pad: &gst::Pad) -> Frames { + let frames = Frames::default(); + let recorded = frames.clone(); + pad.add_probe(gst::PadProbeType::BUFFER, move |_, info| { + if let Some(buffer) = info.buffer() { + recorded + .lock() + .unwrap() + .push((buffer.pts().unwrap(), frame_indices(buffer))); + } + gst::PadProbeReturn::Ok + }); + frames + } + + // When frame `i` of a buffer stamped `pts` was captured. + fn capture_time(pts: gst::ClockTime, i: usize) -> gst::ClockTime { + pts + gst::ClockTime::from_nseconds(i as u64 * 1_000_000_000 / 48000) + } + + // Checks the audio that left a source's pause gate, frame by frame, + // against the audio the source captured: no frame captured while paused + // left it, every other frame did, and each is where its capture time + // goes once the pauses before it are taken out. Returns how many frames + // were captured, and how many of those that left it were captured in + // the 200 ms before a pause. + fn assert_gated( + captured: &Frames, + passed: &Frames, + pauses: &[(gst::ClockTime, gst::ClockTime)], + ) -> (usize, usize) { + let paused = |time: gst::ClockTime| { + pauses + .iter() + .any(|&(at, until)| (at..until).contains(&time)) + }; + let offset = |time: gst::ClockTime| -> gst::ClockTime { + pauses + .iter() + .filter(|&&(_, until)| until <= time) + .map(|&(at, until)| until - at) + .sum() + }; + let captured = captured.lock().unwrap(); + let passed = passed.lock().unwrap(); + let mut when = std::collections::HashMap::new(); + for (pts, frames) in captured.iter() { + for (i, &n) in frames.iter().enumerate() { + assert!( + when.insert(n, capture_time(*pts, i)).is_none(), + "frame {n} captured twice" + ); + } + } + let mut kept = std::collections::HashSet::new(); + let mut leaked = Vec::new(); + let mut misplaced = Vec::new(); + for (pts, frames) in passed.iter() { + for (i, &n) in frames.iter().enumerate() { + let captured = *when + .get(&n) + .unwrap_or_else(|| panic!("frame {n} was never captured")); + assert!(kept.insert(n), "frame {n} left the gate twice"); + if paused(captured) { + leaked.push((n, seconds(captured))); + } + let placed = capture_time(*pts, i); + let due = captured - offset(captured); + if placed.max(due) - placed.min(due) > gst::ClockTime::from_useconds(100) { + misplaced.push((n, seconds(placed), seconds(due))); + } + } + } + let dropped: Vec<_> = when + .iter() + .filter(|(n, &time)| !paused(time) && !kept.contains(n)) + .map(|(&n, &time)| (n, seconds(time))) + .collect(); + let late = passed + .iter() + .flat_map(|(_, frames)| frames) + .filter(|n| { + let time = when[n]; + pauses + .iter() + .any(|&(at, _)| time < at && time + gst::ClockTime::from_mseconds(200) >= at) + }) + .count(); + let first = + |frames: &[(u64, f64)]| frames.iter().copied().min_by(|a, b| a.1.total_cmp(&b.1)); + assert!( + leaked.is_empty(), + "{} frames captured while paused are in the movie, the first {:?}; pauses {pauses:?}", + leaked.len(), + first(&leaked) + ); + assert!( + dropped.is_empty(), + "{} frames captured while recording are not in the movie, the first {:?}; pauses {pauses:?}", + dropped.len(), + first(&dropped) + ); + assert!( + misplaced.is_empty(), + "{} frames are not where their capture time goes, the first {:?}", + misplaced.len(), + misplaced.first() + ); + (when.len(), late) + } + + // Audio that arrives 100 ms after it was captured keeps what it captured + // before each pause, though that arrives while paused, and none of what + // it captured while paused, though that arrives once recording resumed. + // The source loses nothing, so nothing is reported lost. Video frames a + // starved encoder drops are real losses, reported as such. #[test] - fn audio_dropped_by_pausing_is_not_reported_lost() { - let mut recording = TestRecording::start_with_audio(vec![( - "microphone", - lagging_audio("microphone", Duration::from_millis(100)), - )]); + fn late_audio_keeps_what_it_captured_before_a_pause_and_nothing_from_it() { + let source = indexed_audio("microphone", Duration::from_millis(100)); + let captured = record_frames(&source.static_pad("src").unwrap()); + let mut passed = None; + let mut recording = TestRecording::start_watched( + MOVING_VIDEO, + vec![("microphone", source.clone())], + false, + |_| { + passed = Some(record_frames( + &source.static_pad("src").unwrap().peer().unwrap(), + )) + }, + ); for _ in 0..3 { std::thread::sleep(Duration::from_millis(700)); recording.control(true); @@ -2481,6 +2852,11 @@ mod tests { } std::thread::sleep(Duration::from_millis(1200)); recording.finish().unwrap(); + let pauses = recording.counters.timeline.lock().unwrap().pauses.clone(); + assert_eq!(pauses.len(), 3); + let (frames, late) = assert_gated(&captured, &passed.unwrap(), &pauses); + println!("{frames} frames captured, {late} of them in the 200 ms before a pause"); + assert!(late > 3 * 9000, "{late} frames before the pauses"); recording.probe(); for event in recording.events.try_iter() { match event { @@ -2671,6 +3047,9 @@ mod tests { // stopped reading, is lost, and stopping returns when each was. struct PipeDevice { stop: Arc, + // While set, the device writes nothing, like one that went quiet + // without its server noticing, and keeps its schedule. + quiet: Arc, writer: std::thread::JoinHandle>, // When it wrote its first block, and how long each block is. start: Instant, @@ -2707,6 +3086,8 @@ mod tests { unsafe { libc::fcntl(fifo.as_raw_fd(), libc::F_SETPIPE_SZ, 65536) }; let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); let stopping = stop.clone(); + let quiet = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let silent = quiet.clone(); let block = Duration::from_nanos((10_000_000_000_000 / (1_000_000 + ppm)) as u64); let start = Instant::now(); let behind = Arc::new(Mutex::new(Vec::new())); @@ -2729,7 +3110,9 @@ mod tests { .flat_map(i16::to_le_bytes) }) .collect(); - if fifo.write(&bytes).ok() != Some(bytes.len()) { + if !silent.load(Ordering::Relaxed) + && fifo.write(&bytes).ok() != Some(bytes.len()) + { lost.push(Instant::now()); } std::thread::sleep( @@ -2740,6 +3123,7 @@ mod tests { }); Self { stop, + quiet, writer, start, block, @@ -3325,6 +3709,163 @@ mod tests { assert!(recorded.after_video.abs() < 0.05, "{recorded:?}"); } + // Records the pipe device as system audio while `during` runs, and + // checks the audio that left the pause gate frame by frame against + // what the source captured. Returns the recorded time, from where the + // device wrote the first frame in the movie to the stop, that the + // movie has no audio for, by the device's own frame count, and the + // seconds reported missing. + fn record_device_frames(during: impl FnOnce(&TestRecording, &PipeDevice)) -> (f64, f64) { + let server = PulseServer::start_with(PipeDevice::CONFIG, true); + let device = PipeDevice::start(&server); + let source = server.source_from("wrec_pipe", &server.directory.join("native")); + let placed = device.placement(&source); + let captured = record_frames(&source.static_pad("src").unwrap()); + let mut passed = None; + let mut recording = TestRecording::start_watched( + MOVING_VIDEO, + vec![("system audio", source.clone())], + false, + |_| { + passed = Some(record_frames( + &source.static_pad("src").unwrap().peer().unwrap(), + )) + }, + ); + let began = Instant::now(); + // In running time, which began after the device did. + let device_start = recording + .pipeline + .current_running_time() + .unwrap() + .nseconds() as i64 + - began.duration_since(device.start).as_nanos() as i64; + during(&recording, &device); + recording.finish().unwrap(); + recording.probe(); + device.assert_placed(&placed, began); + let lost = device.stop().into_iter().filter(|&at| at >= began).count(); + assert_eq!(lost, 0, "the device lost audio during the recording"); + let (pauses, stopped) = { + let timeline = recording.counters.timeline.lock().unwrap(); + (timeline.pauses.clone(), timeline.stopped_at.unwrap()) + }; + let passed = passed.unwrap(); + assert_gated(&captured, &passed, &pauses); + let captured_at: std::collections::HashMap = captured + .lock() + .unwrap() + .iter() + .flat_map(|(pts, frames)| { + frames + .iter() + .enumerate() + .map(|(i, &n)| (n, capture_time(*pts, i))) + }) + .collect(); + let movie: Vec = passed + .lock() + .unwrap() + .iter() + .flat_map(|(_, frames)| frames.iter().copied()) + .filter(|n| captured_at[n] < stopped) + .collect(); + let first = *movie.iter().min().unwrap(); + let written = gst::ClockTime::from_nseconds( + (device_start + (first * 1_000_000_000 / 48000) as i64) as u64, + ); + let paused: gst::ClockTime = pauses + .iter() + .map(|&(at, until)| stopped.min(until).saturating_sub(written.max(at))) + .sum(); + let recorded = seconds(stopped - written) - seconds(paused); + let missing = recorded - movie.len() as f64 / 48000.0; + let reported = recording + .events + .try_iter() + .find_map(|event| match event { + RecorderEvent::MediaLost { message, .. } => Some(message), + _ => None, + }) + .and_then(|message| { + message + .split_once("system audio is missing ") + .and_then(|(_, rest)| rest.split_once(" s")?.0.parse().ok()) + }) + .unwrap_or(0.0); + println!( + "recorded {recorded:.3} s, the movie has {:.3} s of audio, {missing:.3} s missing, {reported:.2} s reported; pauses {pauses:?}, stopped {stopped}", + movie.len() as f64 / 48000.0 + ); + (missing, reported) + } + + // Video stops reaching the movie for 6 s, like a screen whose keepalive + // stopped, so mp4mux holds the audio until its queue fills, then the + // source fills and waits while its server holds more. Paused meanwhile + // and stopped before video returns, the movie keeps every frame captured + // outside the pause, though it left the source while paused or after the + // stop, and none from inside it. What never left the source or its + // server is reported missing. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_waiting_for_stalled_video_keeps_its_audio_and_reports_the_rest() { + let (missing, reported) = record_device_frames(|recording, _| { + let start = Instant::now(); + let stalled = Duration::from_secs(1)..Duration::from_secs(7); + recording + .pipeline + .by_name("video") + .unwrap() + .static_pad("src") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, _| { + if stalled.contains(&start.elapsed()) { + gst::PadProbeReturn::Drop + } else { + gst::PadProbeReturn::Ok + } + }); + let until = |at: f64| { + std::thread::sleep( + (start + Duration::from_secs_f64(at)).saturating_duration_since(Instant::now()), + ) + }; + until(5.3); + recording.control(true); + until(5.8); + recording.control(false); + until(6.8); + }); + assert!(missing > 0.3, "{missing} s missing"); + assert!( + (reported - missing).abs() < 0.05, + "reported {reported} s, the movie misses {missing} s" + ); + } + + // A device that goes quiet 2 s before the stop, with a pause of 0.5 s + // meanwhile, leaves its track 1.5 s short. No later audio shows the gap, + // and it is reported. + #[test] + #[ignore = "needs pulseaudio"] + fn audio_a_pulse_server_device_quiet_until_stop_reports_the_missing_end() { + let (missing, reported) = record_device_frames(|recording, device| { + std::thread::sleep(Duration::from_secs(3)); + device.quiet.store(true, Ordering::Relaxed); + std::thread::sleep(Duration::from_millis(500)); + recording.control(true); + std::thread::sleep(Duration::from_millis(500)); + recording.control(false); + std::thread::sleep(Duration::from_secs(1)); + }); + assert!((1.4..1.6).contains(&missing), "{missing} s missing"); + assert!( + (reported - missing).abs() < 0.05, + "reported {reported} s, the movie misses {missing} s" + ); + } + // A device whose clock is 800 parts per million fast or slow loses // nothing. Nothing is reported, and every buffer is placed when the // device wrote it, so the audio ends with the video. @@ -3587,7 +4128,29 @@ mod tests { } } assert!(ending); - assert!(lost.is_some_and(|message| message.contains("the video ends early")),); + let lost = lost.unwrap(); + assert!(lost.contains("the video ends early"), "{lost}"); + // The audio after that never left its blocked sources, and no later + // audio shows the gap: it is reported missing up to the stop. + let stopped = seconds( + recording + .counters + .timeline + .lock() + .unwrap() + .stopped_at + .unwrap(), + ); + for track in ["system audio", "microphone"] { + let reported: f64 = lost + .split_once(&format!("{track} is missing ")) + .and_then(|(_, rest)| rest.split_once(" s")?.0.parse().ok()) + .unwrap_or(0.0); + assert!( + (reported - (stopped - audio)).abs() < 0.1, + "{track}: reported {reported} s, audio ends {audio} s, stopped {stopped} s: {lost}" + ); + } } // Video waiting for the disk is not stuck, so it is never cut. @@ -3697,7 +4260,7 @@ mod tests { pipeline.0.add_many([&source, &sink]).unwrap(); source.link(&sink).unwrap(); let lost = Arc::new(AtomicU64::new(0)); - count_lost_audio(&source, lost.clone(), counters); + count_lost_audio(&source, lost.clone(), Default::default(), counters); pipeline.0.set_state(gst::State::Playing).unwrap(); for &(start, end) in buffers { let mut buffer = gst::Buffer::new(); @@ -3737,6 +4300,164 @@ mod tests { assert!(near(lost, 0.0), "{lost}"); } + // 10 ms of F32 stereo at 48 kHz stamped `pts`, each frame holding its + // index from `first`. + fn indexed_buffer(pts: gst::ClockTime, first: u64) -> gst::Buffer { + let bytes: Vec = (first..first + 480) + .flat_map(|n| [(n % 32768) as f32, (n / 32768) as f32]) + .flat_map(|sample| (sample / 32768.0).to_le_bytes()) + .collect(); + let mut buffer = gst::Buffer::from_mut_slice(bytes); + let buffer_mut = buffer.get_mut().unwrap(); + buffer_mut.set_pts(pts); + buffer_mut.set_duration(gst::ClockTime::from_mseconds(10)); + buffer + } + + // What the gate keeps of a buffer of frames 0 to 479 stamped 1 s, given + // pauses as (start, end) in nanoseconds after 1 s and one going on: + // its pts after the gate and the frames in it. + fn gate_audio( + pauses: &[(u64, u64)], + paused_at: Option, + ) -> Option<(gst::ClockTime, gst::ClockTime, Vec)> { + gst::init().unwrap(); + let at = |ns: u64| gst::ClockTime::SECOND + gst::ClockTime::from_nseconds(ns); + let mut timeline = Timeline::default(); + // One pause before the buffer, so it already has an offset. + timeline.pause(gst::ClockTime::from_mseconds(100)); + timeline.resume(gst::ClockTime::from_mseconds(300)); + for &(start, end) in pauses { + timeline.pause(at(start)); + timeline.resume(at(end)); + } + if let Some(start) = paused_at { + timeline.pause(at(start)); + } + let mut buffer = indexed_buffer(gst::ClockTime::SECOND, 0); + timeline.retime_audio(&mut buffer, 48000, 8).then(|| { + ( + buffer.pts().unwrap(), + buffer.duration().unwrap(), + frame_indices(&buffer), + ) + }) + } + + // Frame n of a buffer stamped 1 s was captured n / 48000 s after it, + // and goes 200 ms earlier for the pause before it, and earlier again for + // a pause inside the buffer before it. A frame is captured while paused + // if it was captured at or after a pause started and before it ended, + // so cuts fall between frames, wherever in a frame a pause starts or + // ends, and nothing captured while paused is kept. + #[test] + fn the_pause_gate_cuts_audio_between_frames_at_every_pause_inside_a_buffer() { + let frame = |n: u64| n * 1_000_000_000 / 48000; + let ms = |ms: u64| gst::ClockTime::from_mseconds(ms); + let us = gst::ClockTime::from_nseconds; + let frames = |range: std::ops::Range| range.collect::>(); + // No pause inside: the buffer whole, 200 ms earlier. + assert_eq!( + gate_audio(&[], None), + Some((ms(800), ms(10), frames(0..480))) + ); + // A pause from 100.5 frames to 300 frames in: frames 0 to 100 were + // captured before it, 101 to 299 during it. + let (pts, duration, kept) = gate_audio(&[(frame(100) + 10_417, frame(300))], None).unwrap(); + assert_eq!(kept, [frames(0..101), frames(300..480)].concat()); + assert_eq!((pts, duration), (ms(800), us(frame(281)))); + // Pauses from frame 50 to 60.25 and from 400 to past the buffer. + let (pts, duration, kept) = gate_audio( + &[(frame(50), frame(60) + 5_208), (frame(400), 20_000_000)], + None, + ) + .unwrap(); + assert_eq!(kept, [frames(0..50), frames(61..400)].concat()); + assert_eq!((pts, duration), (ms(800), us(frame(389)))); + // From before the buffer to frame 200.75: the rest goes where the + // pause ended, less both pauses. + let (pts, duration, kept) = gate_audio(&[(0, frame(200) + 15_625)], None).unwrap(); + assert_eq!(kept, frames(201..480)); + assert_eq!(pts, ms(800) + us(frame(201)) - us(frame(200) + 15_625)); + assert_eq!(duration, us(frame(279))); + // A pause going on from frame 333.5: frames up to 333 stay. + let (pts, duration, kept) = gate_audio(&[], Some(frame(333) + 10_417)).unwrap(); + assert_eq!(kept, frames(0..334)); + assert_eq!((pts, duration), (ms(800), us(frame(334)))); + // The whole buffer paused, closed or going on. + assert_eq!(gate_audio(&[(0, 10_000_000)], None), None); + assert_eq!(gate_audio(&[], Some(0)), None); + } + + // A buffer the gate cuts keeps references to its audio, and one it + // keeps whole is the same buffer, so nothing is copied. + #[test] + fn the_pause_gate_copies_no_audio() { + gst::init().unwrap(); + let mut timeline = Timeline::default(); + timeline.pause(gst::ClockTime::from_mseconds(1002)); + timeline.resume(gst::ClockTime::from_mseconds(1005)); + let address = |buffer: &gst::Buffer| buffer.peek_memory(0).map_readable().unwrap().as_ptr(); + let mut whole = indexed_buffer(gst::ClockTime::from_mseconds(990), 0); + let original = address(&whole); + assert!(timeline.retime_audio(&mut whole, 48000, 8)); + assert_eq!(address(&whole), original); + let mut cut = indexed_buffer(gst::ClockTime::SECOND, 0); + let original = address(&cut); + assert!(timeline.retime_audio(&mut cut, 48000, 8)); + assert_eq!(cut.n_memory(), 2); + assert_eq!(address(&cut), original); + assert_eq!( + cut.peek_memory(1).map_readable().unwrap().as_ptr(), + original.wrapping_add(240 * 8) + ); + } + + // A video frame goes by when it was captured, however late it arrives: + // one captured before a pause stays where it was, one captured while + // paused goes, and one captured after a pause goes earlier by that + // pause and every one before it. The gate changes its header, not its + // memory. + #[test] + fn the_pause_gate_takes_video_frames_by_capture_time() { + gst::init().unwrap(); + let s = |s: f64| gst::ClockTime::from_nseconds((s * 1e9) as u64); + let mut timeline = Timeline::default(); + timeline.pause(s(1.0)); + timeline.resume(s(2.0)); + timeline.pause(s(3.0)); + timeline.resume(s(3.5)); + timeline.pause(s(5.0)); + let frame = gst::Buffer::from_mut_slice(vec![0u8; 64]); + let place = |pts: f64| { + // Shared, like a frame pipewiresrc keeps for its keepalive. + let mut buffer = frame.clone(); + buffer.make_mut().set_pts(s(pts)); + buffer.make_mut().set_dts(s(pts)); + let pointer = buffer.peek_memory(0).map_readable().unwrap().as_ptr(); + timeline.retime(&mut buffer).then(|| { + assert_eq!( + buffer.peek_memory(0).map_readable().unwrap().as_ptr(), + pointer + ); + assert_eq!(buffer.pts(), buffer.dts()); + seconds(buffer.pts().unwrap()) + }) + }; + let near = |placed: Option, expected: f64| { + placed.is_some_and(|placed| (placed - expected).abs() < 1e-6) + }; + assert!(near(place(0.9), 0.9)); + assert_eq!(place(1.0), None); + assert_eq!(place(1.9), None); + assert!(near(place(2.0), 1.0)); + assert!(near(place(2.9), 1.9)); + assert_eq!(place(3.2), None); + assert!(near(place(4.9), 3.4)); + assert_eq!(place(5.0), None); + assert_eq!(place(7.0), None); + } + // The queue at the end of an audio source's branch, in front of the // muxer. fn audio_branch_end(pipeline: &gst::Pipeline, source: &str) -> gst::Element { @@ -4251,7 +4972,13 @@ mod tests { pipeline.0.add_many([&video, &mux, &sink, &silent]).unwrap(); mux.link(&sink).unwrap(); let mut movie = Movie::new(&pipeline.0, &video, &mux, true).unwrap(); - movie.add_audio("test audio", &silent, &silent, Arc::default()); + movie.add_audio( + "test audio", + &silent, + &silent, + Arc::default(), + Arc::default(), + ); let held = pipeline.0.by_name(HELD_QUEUE).unwrap(); pipeline.0.set_state(gst::State::Playing).unwrap(); // Large constant-QP frames 10 ms apart reach the byte limit long diff --git a/packaging/linux/README.md b/packaging/linux/README.md index aa66b5c..cac1491 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -326,7 +326,11 @@ job without a movie. After capture starts, `--duration` counts wall time includi pauses; the movie omits paused time. Pause keeps the approved source streams alive but drops video and audio before conversion/encoding; timestamps remove the pause on resume. Rust only changes buffer metadata, preserving GPU memory. This also avoids -PipeWire source renegotiation on resume. Failed starts do not report nonexistent files. +PipeWire source renegotiation on resume. What a pause drops is decided by when it +was captured, not when it arrives. Audio and video captured before a pause stay +even when they arrive during it, as when a stalled movie writer held the audio +back, and nothing captured while paused stays. A pause that starts or ends inside +an audio buffer cuts it between samples. Failed starts do not report nonexistent files. Wayland permission status is `unknown` outside a recording because grants belong to individual sessions. Every recording asks for a source. Restore tokens, @@ -396,7 +400,11 @@ can wait about 4 seconds for the rest. Encoded movie data can wait for the disk in a 32 MiB buffer, about 16 seconds at 16 Mbit/s. These buffers stay nearly empty unless something stalls. Past those limits the recording keeps going with gaps: the capture queue drops video frames, and an audio source's server drops -audio that nothing read in time, which is reported as lost. A stalled video +audio that nothing read in time, which is reported as lost. Audio still waiting in +an audio source or its server when the recording stops never reaches the movie. +When a track's audio ends more than 0.1 s of recorded time before the stop, wrec +reports that end as missing audio; on a normal stop it is a few tens of +milliseconds. A stalled video encoder always costs the frames captured meanwhile, since raw frames are too large to keep. From 9406b70ab4b6dbcad88ee432f8d9835767d232ba Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:47:10 +0000 Subject: [PATCH 41/45] fix(linux): keep audio tracks open before capture starts --- crates/linux/src/pulse.rs | 145 +++++++++++++++++++++++++++++++++++++- 1 file changed, 143 insertions(+), 2 deletions(-) diff --git a/crates/linux/src/pulse.rs b/crates/linux/src/pulse.rs index f79f2fd..0d55364 100644 --- a/crates/linux/src/pulse.rs +++ b/crates/linux/src/pulse.rs @@ -366,6 +366,8 @@ struct Reader { source: glib::WeakRef, // Until the stream is ready or failed, where Capture::connect waits. connected: Option, + // Since Capture::start uncorked the stream, as the pipeline played. + started: bool, closed: bool, // Bytes of the stream so far, received or skipped, so the server's // index of the next byte. @@ -506,6 +508,7 @@ impl Reader { buffer, source: source.downgrade(), connected: None, + started: false, closed: false, position: 0, stamps: Stamps::default(), @@ -605,6 +608,7 @@ impl Reader { if self.closed || self.stream.is_null() { return; } + self.started = true; let operation = (self.library.stream_cork)(self.stream, 0, None, std::ptr::null_mut()); if !operation.is_null() { (self.library.operation_unref)(operation); @@ -727,7 +731,14 @@ impl Reader { } let start = self.position; self.position += length as i64; - // Without running time the source stopped playing. + // pipewire-pulse can send audio to a stream that starts corked, + // before the pipeline plays. It was captured before the recording + // began. + if !self.started { + (self.library.stream_drop)(self.stream); + continue; + } + // Once started, without running time the source is gone. let Some((_, now)) = self.now() else { (self.library.stream_drop)(self.stream); self.close(); @@ -1233,6 +1244,7 @@ mod tests { info: Option>, // Questions the reader asked that the server hasn't answered. questions: usize, + disconnected: bool, } thread_local! { @@ -1283,6 +1295,40 @@ mod tests { 0 } + unsafe extern "C" fn cork( + _: *mut Stream, + _: c_int, + _: Option, + _: *mut c_void, + ) -> *mut Operation { + std::ptr::null_mut() + } + + unsafe extern "C" fn timer( + _: *mut Context, + _: u64, + _: Option, + _: *mut c_void, + ) -> *mut TimeEvent { + std::ptr::null_mut() + } + + unsafe extern "C" fn set_callback(_: *mut Stream, _: Option, _: *mut c_void) {} + + unsafe extern "C" fn set_read_callback( + _: *mut Stream, + _: Option, + _: *mut c_void, + ) { + } + + unsafe extern "C" fn disconnect(_: *mut Stream) -> c_int { + SOCKET.with_borrow_mut(|socket| socket.disconnected = true); + 0 + } + + unsafe extern "C" fn stream_unref(_: *mut Stream) {} + // Frames that hold their index, in two channels of 15 bits. fn pcm(frames: std::ops::Range) -> Vec { frames @@ -1313,7 +1359,16 @@ mod tests { } impl Wire { + // Playing, with a reader appsrc started. fn new() -> Self { + let mut wire = Self::unplayed(); + wire.pipeline.set_state(gst::State::Playing).unwrap(); + wire.reader.started = true; + wire + } + + // Before the pipeline plays, with no clock yet. + fn unplayed() -> Self { gst::init().unwrap(); SOCKET.set(Socket::default()); let library = Box::leak(Box::new(Library { @@ -1324,6 +1379,12 @@ mod tests { operation_unref: unref, context_rttime_restart: restart, rtclock_now, + stream_cork: cork, + context_rttime_new: timer, + stream_set_state_callback: set_callback, + stream_set_read_callback: set_read_callback, + stream_disconnect: disconnect, + stream_unref, ..Library::none() })); let source = gst::ElementFactory::make("appsrc") @@ -1358,7 +1419,6 @@ mod tests { sink, }; wire.at(0); - wire.pipeline.set_state(gst::State::Playing).unwrap(); wire } @@ -1422,6 +1482,23 @@ mod tests { unsafe { self.reader.ask_again() }; } + // Audio through libpulse's read callback, at whatever running time + // there is. + fn arrive(&mut self, frames: std::ops::Range) { + let bytes = pcm(frames); + let length = bytes.len(); + SOCKET.with_borrow_mut(|socket| socket.audio.push_back(bytes)); + let reader = (&mut *self.reader as *mut Reader).cast(); + unsafe { stream_read(self.reader.stream, length, reader) }; + } + + // The pipeline plays, and appsrc's first need-data starts the + // reader, as Capture::start does. + fn play(&mut self) { + self.pipeline.set_state(gst::State::Playing).unwrap(); + unsafe { self.reader.start() }; + } + // Fragments sent as captured, each answering the questions on the // way, with the timer going off every 100 ms. fn steady(&mut self, fragments: std::ops::Range) { @@ -1712,4 +1789,68 @@ mod tests { ] ); } + + // A reader libpulse connected, with a stream that starts corked. + fn connected() -> Wire { + let mut wire = Wire::unplayed(); + wire.reader.stream = std::ptr::NonNull::dangling().as_ptr(); + assert!(wire.reader.now().is_none(), "no running time before play"); + wire + } + + // pipewire-pulse can send a fragment to a stream that starts corked, + // before the pipeline plays, when there is no running time yet. That + // audio was captured before the recording began and goes, and the stream + // stays open: the track starts with the audio after it, where it was + // captured. A reader that closed the stream on it left the movie without + // the track. + #[test] + fn audio_the_server_sent_before_the_pipeline_played_goes_and_the_track_starts_after_it() { + let mut wire = connected(); + // Frames 0 to 479, captured in the 10 ms before running time 0. + wire.arrive(0..480); + assert!(SOCKET.with_borrow(|socket| socket.audio.is_empty())); + wire.play(); + for n in 1..300 { + let ms = n as i64 * 10; + if ms % 100 == 0 { + wire.tick(ms); + } + wire.audio(ms, n * 480..(n + 1) * 480); + wire.answer(ms, (n + 1) * 480, (n + 1) * 480); + } + let sent = wire.sent(); + assert!(!sent.is_empty(), "the track has no audio"); + assert_eq!(sent[0].first, 480, "{:?}", sent[0]); + // Frame 480 was captured at running time 0. + let from_480: Vec<_> = sent + .iter() + .map(|buffer| Sent { + first: buffer.first - 480, + last: buffer.last - 480, + ..*buffer + }) + .collect(); + assert_eq!(assert_placed(&from_480), []); + assert_eq!(sent.last().unwrap().last, 300 * 480 - 1); + } + + // Once the reader started, audio without running time means the source + // is gone, as when the pipeline let go of it: the reader takes the audio + // and closes the stream at once. + #[test] + fn audio_after_the_source_went_closes_the_stream() { + let mut wire = connected(); + wire.play(); + wire.audio(10, 0..480); + wire.answer(10, 480, 480); + wire.pipeline.set_state(gst::State::Null).unwrap(); + let source = wire.reader.source.upgrade().unwrap(); + wire.pipeline.remove(&source).unwrap(); + drop(source); + assert!(wire.reader.now().is_none()); + wire.arrive(480..960); + assert!(wire.reader.closed); + assert!(SOCKET.with_borrow(|socket| socket.disconnected && socket.audio.is_empty())); + } } From 11412fd7f0bb3ff828864031b884106aaf79e760 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:09:53 +0000 Subject: [PATCH 42/45] fix(linux): capture PipeWire frames without compositor clock waits --- crates/linux/src/pipeline.rs | 171 ++++++++++++++++++++++++++++++++++- packaging/linux/README.md | 17 ++-- 2 files changed, 177 insertions(+), 11 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 46b5a2a..d81dafb 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -139,6 +139,89 @@ const CAPTURE_QUEUE: &str = "capture-queue"; // idle screen still sends a frame each keepalive. const KEEPALIVE: gst::ClockTime = gst::ClockTime::SECOND; +// Since PipeWire 0.3.70, pipewiresrc gives each frame the compositor's +// timestamp, and GstBaseSrc holds every frame until that timestamp comes due, +// counted from the first frame's. KWin stamped the first frame of an idle +// screen with its last repaint, seconds old, so the next new frame waited that +// long and capture stopped meanwhile. Without a clock a source passes frames +// on as they arrive, so the source goes in a bin that keeps the pipeline's +// clock from it, and each frame is stamped with the running time it left the +// source. +fn stamp_on_arrival(source: gst::Element) -> Result { + let bin = arrival::Unclocked::default(); + bin.add(&source).map_err(backend)?; + let pad = source.static_pad("src").unwrap(); + let stamper = bin.downgrade(); + pad.add_probe(gst::PadProbeType::BUFFER, move |_, info| { + if let (Some(bin), Some(gst::PadProbeData::Buffer(buffer))) = + (stamper.upgrade(), info.data.as_mut()) + { + let now = bin.running_time(); + let buffer = buffer.make_mut(); + buffer.set_pts(now); + buffer.set_dts(now); + } + gst::PadProbeReturn::Ok + }); + let ghost = gst::GhostPad::builder_with_target(&pad) + .map_err(backend)? + .name("src") + .build(); + bin.add_pad(&ghost).map_err(backend)?; + Ok(bin.upcast()) +} + +mod arrival { + use gstreamer::{self as gst, glib, prelude::*, subclass::prelude::*}; + + glib::wrapper! { + pub struct Unclocked(ObjectSubclass) + @extends gst::Bin, gst::Element, gst::Object; + } + + impl Default for Unclocked { + fn default() -> Self { + glib::Object::new() + } + } + + impl Unclocked { + // Running time by the clock the pipeline gave this bin. + pub(super) fn running_time(&self) -> Option { + let clock = self.imp().clock.lock().unwrap().clone()?; + clock.time()?.checked_sub(self.base_time()?) + } + } + + mod imp { + use super::*; + use std::sync::Mutex; + + #[derive(Default)] + pub struct Unclocked { + pub(super) clock: Mutex>, + } + + #[glib::object_subclass] + impl ObjectSubclass for Unclocked { + const NAME: &'static str = "WrecUnclockedBin"; + type Type = super::Unclocked; + type ParentType = gst::Bin; + } + + impl ObjectImpl for Unclocked {} + impl GstObjectImpl for Unclocked {} + impl BinImpl for Unclocked {} + + impl ElementImpl for Unclocked { + fn set_clock(&self, clock: Option<&gst::Clock>) -> bool { + *self.clock.lock().unwrap() = clock.cloned(); + self.parent_set_clock(None) + } + } + } +} + fn video_queue() -> Result { let queue = element("queue")?; queue.set_property("name", CAPTURE_QUEUE); @@ -464,7 +547,7 @@ fn record_attempt( source.set_property("max-buffers", 8i32); source.set_property("keepalive-time", KEEPALIVE.mseconds() as i32); source.set_property("resend-last", true); - source + stamp_on_arrival(source)? } CaptureInput::X11 { display, xid } => { let source = element("ximagesrc")?; @@ -472,12 +555,18 @@ fn record_attempt( source.set_property("xid", *xid); source.set_property("show-pointer", settings.include_cursor); source.set_property("use-damage", true); + source.set_property("do-timestamp", true); source } #[cfg(test)] - CaptureInput::Element(source) => source(), + CaptureInput::Element(source) => { + let source = source(); + if source.has_property("do-timestamp", None) { + source.set_property("do-timestamp", true); + } + source + } }; - source.set_property("do-timestamp", true); let input = element("capsfilter")?; input.set_property( "caps", @@ -1345,7 +1434,7 @@ fn reject_capture( gst::PadProbeReturn::Handled } -const UNFINISHED_FIRST_FRAME: &str = "The screen-capture source started with an unfinished frame. Its timestamp would stall capture timing, so this attempt stopped before encoding and another available mode will be tried."; +const UNFINISHED_FIRST_FRAME: &str = "The screen-capture source started with an unfinished frame, so this attempt stopped before encoding and another available mode will be tried."; const SOURCE_LOST: &str = "The captured X11 window closed, was minimized, or became unavailable; the recording stopped."; @@ -5338,6 +5427,80 @@ mod tests { } } + // A screen through pipewiresrc since PipeWire 0.3.70, which hands + // GstBaseSrc each frame with the compositor's timestamp and has it sync on + // that, as this live fakesrc does. KWin stamped the first frame of an idle screen with its last + // repaint, seconds old. Each frame after it is new, `every` apart. + fn compositor(stale: Duration, every: Duration) -> gst::Element { + let compositor = gst::parse::bin_from_description( + "fakesrc name=frames is-live=true sync=true format=time signal-handoffs=true sizetype=fixed sizemax=9216 filltype=zero ! capsfilter caps=video/x-raw,format=BGRx,width=64,height=36,pixel-aspect-ratio=1/1,framerate=0/1,max-framerate=30/1", + true, + ) + .unwrap(); + let clock = gst::SystemClock::obtain(); + let first = std::sync::atomic::AtomicBool::new(true); + let frames = compositor.by_name("frames").unwrap(); + frames.connect("handoff", false, move |values| { + let buffer = values[1].get::<&gst::BufferRef>().unwrap(); + let age = if first.swap(false, Ordering::Relaxed) { + stale + } else { + std::thread::sleep(every); + Duration::ZERO + }; + let shown = clock.time().unwrap().nseconds() - age.as_nanos() as u64; + // SAFETY: fakesrc hands over the buffer it just made before it + // syncs on it, as pipewiresrc's create returns one, and only its + // timestamps change. + unsafe { + let buffer = buffer.as_ptr() as *mut gst::ffi::GstBuffer; + (*buffer).pts = shown; + (*buffer).dts = shown; + } + None + }); + compositor.upcast() + } + + #[test] + fn a_first_frame_stamped_seconds_ago_holds_back_no_later_frame() { + gst::init().unwrap(); + let source = Arc::new(Mutex::new(None::)); + let created = source.clone(); + let capture = CaptureInput::Element(Box::new(move || { + let screen = stamp_on_arrival(compositor( + Duration::from_secs(3), + Duration::from_millis(100), + )) + .unwrap(); + *created.lock().unwrap() = Some(screen.clone()); + screen + })); + let session = test_session(); + let (events, _received) = mpsc::channel(); + let recording = RecordThread::spawn(capture, &session, Codec::H264, events); + playing_source(&source, &recording); + std::thread::sleep(Duration::from_millis(1500)); + recording.assert_running(); + recording.finish().unwrap(); + let mut times: Vec = video_packets(&decode_movie(&session.output_path)) + .into_iter() + .map(|(time, _)| time) + .collect(); + let _ = std::fs::remove_file(&session.output_path); + times.sort_by(f64::total_cmp); + let span = times.last().unwrap() - times[0]; + let gap = times + .windows(2) + .map(|pair| pair[1] - pair[0]) + .fold(0.0, f64::max); + assert!( + span > 1.0 && gap < 0.5, + "1.5 s of new frames 0.1 s apart became {} frames over {span:.3} s, the longest gap {gap:.3} s", + times.len() + ); + } + // Each recording paces real frames through a leaky queue; running two at // once on a shared encoder would turn scheduling delays into frame drops. static RATE_CHANGE_RECORDING: Mutex<()> = Mutex::new(()); diff --git a/packaging/linux/README.md b/packaging/linux/README.md index cac1491..fab7058 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -97,13 +97,16 @@ no frame until something on screen changes. `pipewiresrc` pauses its stream briefly while starting. When that pause interrupts a capture, xdg-desktop-portal-wlr sends the unfinished buffer marked -corrupted, with timestamp 0. GStreamer aligns a live source's timestamps to -the first buffer it sees. When that buffer is the corrupted one, every real -frame looks due hours in the future, so the source waits instead of delivering -frames. This -was seen with shared GPU buffers on xdg-desktop-portal-wlr 0.8.1 and PipeWire -1.6.2. wrec stops an attempt whose first frame is marked corrupted before -anything is encoded and moves on to the next available mode. +corrupted, with timestamp 0. This was seen with shared GPU buffers on +xdg-desktop-portal-wlr 0.8.1 and PipeWire 1.6.2. wrec stops an attempt whose +first frame is marked corrupted before anything is encoded and moves on to +the next available mode. + +KWin can stamp the first frame of an idle screen with the time of its last +repaint, seconds before capture started. GStreamer then held each new frame +for that long, so a recording of a still screen froze for seconds once +something moved. wrec gives `pipewiresrc` no clock, so it passes each frame on +as it arrives, and stamps every frame with the time it arrived. wrec sets the movie's output size when the first source caps arrive. Changing a capsfilter normally asks every upstream element to renegotiate, and From 185e9b0620a02e7f22af440fd557a09aab9e56dd Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:12:11 +0000 Subject: [PATCH 43/45] fix(linux): preserve capture timing and guard shared buffers --- crates/linux/src/pipeline.rs | 399 +++++++++++++++++++++++++++++++++-- packaging/linux/README.md | 22 +- 2 files changed, 396 insertions(+), 25 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index d81dafb..5da7656 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -145,22 +145,54 @@ const KEEPALIVE: gst::ClockTime = gst::ClockTime::SECOND; // screen with its last repaint, seconds old, so the next new frame waited that // long and capture stopped meanwhile. Without a clock a source passes frames // on as they arrive, so the source goes in a bin that keeps the pipeline's -// clock from it, and each frame is stamped with the running time it left the -// source. -fn stamp_on_arrival(source: gst::Element) -> Result { - let bin = arrival::Unclocked::default(); +// clock from it, and each frame is stamped here instead. +// +// Each frame is stamped with when the compositor captured it, the time audio +// and pausing go by. Delivery takes tens of milliseconds, so a frame stamped +// on arrival would play that much after its sound. pipewiresrc gives the +// compositor's time on the pipeline clock since PipeWire 0.3.70 (`absolute`), +// and in running time before that. +fn stamp_capture_time( + source: gst::Element, + absolute: bool, + counters: Arc, +) -> Result { + let bin = unclocked::Unclocked::default(); bin.add(&source).map_err(backend)?; let pad = source.static_pad("src").unwrap(); let stamper = bin.downgrade(); + let times = Mutex::new(CaptureTimes::default()); + let copies = source + .has_property("always-copy", None) + .then(|| Mutex::new(MemPtrCopies::new(&source))); pad.add_probe(gst::PadProbeType::BUFFER, move |_, info| { - if let (Some(bin), Some(gst::PadProbeData::Buffer(buffer))) = + let (Some(bin), Some(gst::PadProbeData::Buffer(buffer))) = (stamper.upgrade(), info.data.as_mut()) - { - let now = bin.running_time(); - let buffer = buffer.make_mut(); - buffer.set_pts(now); - buffer.set_dts(now); - } + else { + return gst::PadProbeReturn::Ok; + }; + let Some(now) = bin.running_time() else { + return gst::PadProbeReturn::Ok; + }; + let base = if absolute { bin.base_time() } else { None }; + let mut times = times.lock().unwrap(); + let (stamp, earliest) = times.stamp(buffer, base, now); + if let Some(copies) = &copies { + copies.lock().unwrap().protect(buffer); + } + // Pausing takes out what may have been captured while paused. + let paused = counters + .timeline + .lock() + .unwrap() + .paused_between(earliest, now); + if paused > gst::ClockTime::ZERO { + return gst::PadProbeReturn::Drop; + } + times.passed = Some(stamp); + let buffer = buffer.make_mut(); + buffer.set_pts(stamp); + buffer.set_dts(stamp); gst::PadProbeReturn::Ok }); let ghost = gst::GhostPad::builder_with_target(&pad) @@ -171,7 +203,113 @@ fn stamp_on_arrival(source: gst::Element) -> Result { Ok(bin.upcast()) } -mod arrival { +// pipewiresrc gives a frame with a compositor header the header's sequence +// number as its offset and the header's timestamp as its PTS. A keepalive +// repeats the last frame with the same offset and, without a clock, no PTS. +fn is_keepalive(frame: &gst::BufferRef) -> bool { + frame.offset() != gst::ffi::GST_BUFFER_OFFSET_NONE && frame.pts().is_none() +} + +// When each frame from pipewiresrc was captured, in running time. KWin stamps +// a frame with the screen's previous presentation, which after an idle screen +// is seconds before it rendered the picture, and it stamped the first frame of +// a recording with its last repaint. So a timestamp only says the picture is +// no older than that, and the frame's arrival says it is no newer. +#[derive(Default)] +struct CaptureTimes { + // The last frame's stamp, and the running time it arrived. + last: Option<(gst::ClockTime, gst::ClockTime)>, + // The last stamp passed on. + passed: Option, +} + +impl CaptureTimes { + // The frame's stamp, and the earliest it may have been captured. `base` + // is running time 0 on an absolute source's clock; `now` is the running + // time the frame arrived. + fn stamp( + &mut self, + frame: &gst::BufferRef, + base: Option, + now: gst::ClockTime, + ) -> (gst::ClockTime, gst::ClockTime) { + if is_keepalive(frame) { + // The screen still shows the last frame's picture, as much later + // as the keepalive came after that frame. + let stamp = self + .last + .map_or(now, |(stamp, arrived)| stamp + now.saturating_sub(arrived)); + self.last = Some((stamp, now)); + return (stamp, stamp); + } + let captured = frame + .pts() + .filter(|_| frame.offset() != gst::ffi::GST_BUFFER_OFFSET_NONE) + // A frame stamped before the start shows the screen at the start. + .map(|pts| base.map_or(pts, |base| pts.saturating_sub(base))) + // A frame isn't captured after it arrives. + .filter(|&captured| captured <= now); + // A timestamp no later than the last frame or keepalive passed on + // says nothing about when this picture was made, as KWin's first + // frame after an idle screen shows, so the frame goes by its arrival. + let stamp = captured + .filter(|&captured| self.passed.map_or(true, |passed| captured > passed)) + .unwrap_or(now); + self.last = Some((stamp, now)); + (stamp, captured.unwrap_or(now)) + } +} + +// pipewiresrc hands over a MemPtr frame as a bare pointer into PipeWire's +// mapping of the producer's memory, which PipeWire unmaps when the producer +// goes away, even while wrec is still converting the frame or pipewiresrc +// sends it again as a keepalive. Compositors share MemFd or DMA-BUF memory, +// which pipewiresrc maps itself. Once a frame turns out to be MemPtr, +// pipewiresrc copies each later frame while it holds PipeWire's lock, wrec +// copies that first frame itself, and keepalives repeat wrec's copy instead of +// pipewiresrc's last frame. +struct MemPtrCopies { + source: gst::glib::WeakRef, + copying: bool, + last: Option, +} + +impl MemPtrCopies { + fn new(source: &gst::Element) -> Self { + Self { + source: source.downgrade(), + copying: false, + last: None, + } + } + + fn protect(&mut self, buffer: &mut gst::Buffer) { + if !self.copying + && buffer.n_memory() > 0 + && !buffer + .iter_memories() + .any(|memory| memory.is_memory_type::()) + { + self.copying = true; + if let Some(source) = self.source.upgrade() { + source.set_property("always-copy", true); + } + if let Ok(copy) = buffer.copy_deep() { + *buffer = copy; + } + } + if !self.copying { + return; + } + if !is_keepalive(buffer) { + self.last = Some(buffer.clone()); + } else if let Some(last) = &self.last { + *buffer = last.clone(); + } + } +} + +mod unclocked { use gstreamer::{self as gst, glib, prelude::*, subclass::prelude::*}; glib::wrapper! { @@ -418,6 +556,9 @@ pub(crate) enum CaptureInput { }, #[cfg(test)] Element(Box gst::Element + Send>), + // A stand-in for pipewiresrc since PipeWire 0.3.70, stamped like it. + #[cfg(test)] + Compositor(Box gst::Element + Send>), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -547,7 +688,11 @@ fn record_attempt( source.set_property("max-buffers", 8i32); source.set_property("keepalive-time", KEEPALIVE.mseconds() as i32); source.set_property("resend-last", true); - stamp_on_arrival(source)? + let absolute = source + .factory() + .and_then(|factory| encoding::plugin_version(&factory)) + .is_some_and(|version| version >= vec![0, 3, 70]); + stamp_capture_time(source, absolute, attempt.counters.clone())? } CaptureInput::X11 { display, xid } => { let source = element("ximagesrc")?; @@ -566,6 +711,10 @@ fn record_attempt( } source } + #[cfg(test)] + CaptureInput::Compositor(source) => { + stamp_capture_time(source(), true, attempt.counters.clone())? + } }; let input = element("capsfilter")?; input.set_property( @@ -5352,6 +5501,7 @@ mod tests { // failure fails the test instead of hanging it. Dropping it stops the // recording and waits as long as the daemon waits for a worker to exit. struct RecordThread { + commands: mpsc::SyncSender, stop: watch::Sender, returned: mpsc::Receiver>, } @@ -5368,7 +5518,6 @@ mod tests { let (done, returned) = mpsc::channel(); let session = session.clone(); std::thread::spawn(move || { - let _commands = commands; let result = record( &capture, &session, @@ -5380,7 +5529,23 @@ mod tests { ); let _ = done.send(result); }); - Self { stop, returned } + Self { + commands, + stop, + returned, + } + } + + fn control(&self, pause: bool) { + let (tx, rx) = mpsc::sync_channel(1); + self.commands + .send(if pause { + Command::Pause(tx) + } else { + Command::Resume(tx) + }) + .unwrap(); + rx.recv_timeout(Duration::from_secs(5)).unwrap().unwrap(); } fn assert_running(&self) { @@ -5467,12 +5632,8 @@ mod tests { gst::init().unwrap(); let source = Arc::new(Mutex::new(None::)); let created = source.clone(); - let capture = CaptureInput::Element(Box::new(move || { - let screen = stamp_on_arrival(compositor( - Duration::from_secs(3), - Duration::from_millis(100), - )) - .unwrap(); + let capture = CaptureInput::Compositor(Box::new(move || { + let screen = compositor(Duration::from_secs(3), Duration::from_millis(100)); *created.lock().unwrap() = Some(screen.clone()); screen })); @@ -5501,6 +5662,202 @@ mod tests { ); } + // pipewiresrc since PipeWire 0.3.70 as wrec runs it, without a clock: + // each frame carries the compositor's timestamp on the pipeline clock and + // its header number, and a keepalive repeats the last frame's number + // without a timestamp. The test pushes each frame when it arrives. + struct Screen { + appsrc: gst::Element, + clock: gst::Clock, + number: u64, + last: Option, + } + + impl Screen { + fn record(session: &RecordingSession) -> (Self, RecordThread) { + gst::init().unwrap(); + let source = Arc::new(Mutex::new(None::)); + let created = source.clone(); + let capture = CaptureInput::Compositor(Box::new(move || { + let appsrc = element("appsrc").unwrap(); + appsrc.set_property("is-live", true); + appsrc.set_property_from_str("format", "time"); + appsrc.set_property( + "caps", + gst::Caps::builder("video/x-raw") + .field("format", "BGRx") + .field("width", 64i32) + .field("height", 36i32) + .field("framerate", gst::Fraction::new(0, 1)) + .field("max-framerate", gst::Fraction::new(30, 1)) + .build(), + ); + *created.lock().unwrap() = Some(appsrc.clone()); + appsrc + })); + let (events, _received) = mpsc::channel(); + let recording = RecordThread::spawn(capture, session, Codec::H264, events); + let appsrc = playing_source(&source, &recording); + let screen = Self { + appsrc, + clock: gst::SystemClock::obtain(), + number: 0, + last: None, + }; + (screen, recording) + } + + fn now(&self) -> gst::ClockTime { + self.clock.time().unwrap() + } + + // A frame filled with `shade`, captured at `captured` on the clock. + fn frame(&mut self, shade: u8, captured: gst::ClockTime) { + let mut frame = gst::Buffer::from_mut_slice(vec![shade; 64 * 36 * 4]); + let header = frame.get_mut().unwrap(); + header.set_pts(captured); + header.set_offset(self.number); + self.number += 1; + self.last = Some(frame.clone()); + self.push(frame); + } + + fn keepalive(&mut self) { + let mut frame = self.last.as_ref().unwrap().copy(); + frame.get_mut().unwrap().set_pts(gst::ClockTime::NONE); + self.push(frame); + } + + fn push(&self, frame: gst::Buffer) { + assert_eq!( + self.appsrc + .emit_by_name::("push-buffer", &[&frame]), + gst::FlowReturn::Ok + ); + } + } + + // The shades a player shows, in order, each as the nearest of `shades`. + fn shown(movie: &Path, shades: &[u8]) -> Vec { + decoded_pictures(movie) + .into_iter() + .map(|(width, height, pixels)| { + let luma = pixels[height / 2 * width + width / 2] as f64; + *shades + .iter() + .min_by_key(|&&shade| { + ((16.0 + 219.0 * shade as f64 / 255.0 - luma).abs() * 100.0) as u64 + }) + .unwrap() + }) + .collect() + } + + // A compositor delivers a frame tens of milliseconds after it captures + // it. One captured just before the resume and delivered just after it + // shows what was on screen while paused. + #[test] + fn a_frame_captured_while_paused_stays_out_after_the_resume() { + let session = test_session(); + let (mut screen, recording) = Screen::record(&session); + for _ in 0..5 { + screen.frame(40, screen.now() - gst::ClockTime::from_mseconds(20)); + std::thread::sleep(Duration::from_millis(100)); + } + recording.control(true); + std::thread::sleep(Duration::from_millis(300)); + let paused = screen.now(); + recording.control(false); + std::thread::sleep(Duration::from_millis(60)); + screen.frame(250, paused); + for _ in 0..5 { + std::thread::sleep(Duration::from_millis(100)); + screen.frame(40, screen.now() - gst::ClockTime::from_mseconds(20)); + } + recording.finish().unwrap(); + let shades = shown(&session.output_path, &[40, 250]); + let _ = std::fs::remove_file(&session.output_path); + assert!(shades.contains(&40), "{shades:?}"); + assert!( + !shades.contains(&250), + "a frame captured while paused was shown after the resume: {shades:?}" + ); + } + + // Frames go into the movie by when they were captured, so a compositor + // whose delivery takes 20 ms, then 45 ms, keeps its frames 1/30 s apart. + #[test] + fn frames_keep_their_capture_spacing_when_delivery_varies() { + let session = test_session(); + let (mut screen, recording) = Screen::record(&session); + let start = screen.now(); + let interval = gst::ClockTime::SECOND / 30; + for frame in 0..30u64 { + let captured = start + interval * frame; + let delay = gst::ClockTime::from_mseconds(if frame % 2 == 0 { 20 } else { 45 }); + let arrives = captured + delay; + std::thread::sleep(Duration::from_nanos( + arrives.saturating_sub(screen.now()).nseconds(), + )); + screen.frame(30 + 20 * (frame % 2) as u8, captured); + } + std::thread::sleep(Duration::from_millis(100)); + recording.finish().unwrap(); + let mut times: Vec = video_packets(&decode_movie(&session.output_path)) + .into_iter() + .map(|(time, _)| time) + .collect(); + let _ = std::fs::remove_file(&session.output_path); + times.sort_by(f64::total_cmp); + let spacing: Vec = times.windows(2).map(|pair| pair[1] - pair[0]).collect(); + assert!( + times.len() >= 29 + && spacing + .iter() + .all(|interval| (interval - 1.0 / 30.0).abs() < 0.002), + "frames captured 1/30 s apart were {spacing:?} s apart" + ); + } + + // KWin stamps a frame with the screen's previous presentation, seconds + // before it rendered the picture if the screen was idle. After a + // keepalive such a frame still goes in, by its arrival. And one rendered + // while paused, stamped with a presentation before the pause, stays out. + #[test] + fn stale_compositor_timestamps_lose_no_frame_and_let_no_paused_frame_in() { + let session = test_session(); + let (mut screen, recording) = Screen::record(&session); + let shown_at = screen.now() - gst::ClockTime::from_mseconds(20); + screen.frame(60, shown_at); + std::thread::sleep(Duration::from_millis(1000)); + screen.keepalive(); + std::thread::sleep(Duration::from_millis(300)); + let presented = screen.now(); + screen.frame(120, shown_at + gst::ClockTime::from_mseconds(40)); + std::thread::sleep(Duration::from_millis(200)); + recording.control(true); + std::thread::sleep(Duration::from_millis(300)); + screen.frame(240, presented + gst::ClockTime::from_mseconds(16)); + std::thread::sleep(Duration::from_millis(300)); + recording.control(false); + std::thread::sleep(Duration::from_millis(100)); + for _ in 0..3 { + screen.frame(30, screen.now() - gst::ClockTime::from_mseconds(20)); + std::thread::sleep(Duration::from_millis(100)); + } + recording.finish().unwrap(); + let shades = shown(&session.output_path, &[30, 60, 120, 240]); + let _ = std::fs::remove_file(&session.output_path); + assert!( + shades.contains(&60) && shades.contains(&120) && shades.contains(&30), + "a frame was lost: {shades:?}" + ); + assert!( + !shades.contains(&240), + "a frame rendered while paused was shown: {shades:?}" + ); + } + // Each recording paces real frames through a leaky queue; running two at // once on a shared encoder would turn scheduling delays into frame drops. static RATE_CHANGE_RECORDING: Mutex<()> = Mutex::new(()); diff --git a/packaging/linux/README.md b/packaging/linux/README.md index fab7058..3b00e40 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -106,7 +106,18 @@ KWin can stamp the first frame of an idle screen with the time of its last repaint, seconds before capture started. GStreamer then held each new frame for that long, so a recording of a still screen froze for seconds once something moved. wrec gives `pipewiresrc` no clock, so it passes each frame on -as it arrives, and stamps every frame with the time it arrived. +as it arrives, and stamps each frame with the compositor's timestamp rather +than its arrival, so the time the compositor takes to deliver a frame does not +delay video against audio. KWin stamps a frame with the screen's previous presentation, which after +an idle screen is seconds before it rendered the picture. A timestamp no later +than the previous frame's says nothing about when its picture was made, so +that frame is stamped with when it arrived. + +A screen-capture stream can share frames as plain pointers into PipeWire's +memory (MemPtr) instead of file descriptors (MemFd or DMA-BUF). PipeWire +unmaps that memory when the producer exits, even while wrec still uses a +frame, so wrec copies MemPtr frames. KWin, mutter, xdg-desktop-portal-wlr and +gamescope share MemFd or DMA-BUF, which wrec does not copy. wrec sets the movie's output size when the first source caps arrive. Changing a capsfilter normally asks every upstream element to renegotiate, and @@ -330,9 +341,12 @@ pauses; the movie omits paused time. Pause keeps the approved source streams ali video and audio before conversion/encoding; timestamps remove the pause on resume. Rust only changes buffer metadata, preserving GPU memory. This also avoids PipeWire source renegotiation on resume. What a pause drops is decided by when it -was captured, not when it arrives. Audio and video captured before a pause stay -even when they arrive during it, as when a stalled movie writer held the audio -back, and nothing captured while paused stays. A pause that starts or ends inside +was captured, not when it arrives. Audio captured before a pause stays even +when it arrives during it, as when a stalled movie writer held the audio back, +and nothing captured while paused stays. A Wayland screen frame was captured +somewhere between its compositor timestamp and its arrival, so a frame whose +span touches a pause is dropped, including frames captured in the last few tens +of milliseconds before the pause. A pause that starts or ends inside an audio buffer cuts it between samples. Failed starts do not report nonexistent files. Wayland permission status is `unknown` outside a recording because grants belong From 99fb801579b825b5b471098a113bddc03875f76b Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:26:22 +0000 Subject: [PATCH 44/45] fix(linux): avoid reading expired PipeWire memory --- crates/linux/src/pipeline.rs | 79 ++++++++++++++++++++++-------------- packaging/linux/README.md | 7 +++- 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/crates/linux/src/pipeline.rs b/crates/linux/src/pipeline.rs index 5da7656..6e063ed 100644 --- a/crates/linux/src/pipeline.rs +++ b/crates/linux/src/pipeline.rs @@ -162,9 +162,9 @@ fn stamp_capture_time( let pad = source.static_pad("src").unwrap(); let stamper = bin.downgrade(); let times = Mutex::new(CaptureTimes::default()); - let copies = source + let frames = source .has_property("always-copy", None) - .then(|| Mutex::new(MemPtrCopies::new(&source))); + .then(|| Mutex::new(SafeFrames::new(&source))); pad.add_probe(gst::PadProbeType::BUFFER, move |_, info| { let (Some(bin), Some(gst::PadProbeData::Buffer(buffer))) = (stamper.upgrade(), info.data.as_mut()) @@ -177,8 +177,11 @@ fn stamp_capture_time( let base = if absolute { bin.base_time() } else { None }; let mut times = times.lock().unwrap(); let (stamp, earliest) = times.stamp(buffer, base, now); - if let Some(copies) = &copies { - copies.lock().unwrap().protect(buffer); + if frames + .as_ref() + .is_some_and(|frames| !frames.lock().unwrap().keep(buffer)) + { + return gst::PadProbeReturn::Drop; } // Pausing takes out what may have been captured while paused. let paused = counters @@ -264,17 +267,21 @@ impl CaptureTimes { // mapping of the producer's memory, which PipeWire unmaps when the producer // goes away, even while wrec is still converting the frame or pipewiresrc // sends it again as a keepalive. Compositors share MemFd or DMA-BUF memory, -// which pipewiresrc maps itself. Once a frame turns out to be MemPtr, -// pipewiresrc copies each later frame while it holds PipeWire's lock, wrec -// copies that first frame itself, and keepalives repeat wrec's copy instead of -// pipewiresrc's last frame. -struct MemPtrCopies { +// which pipewiresrc maps itself. So a frame goes on only if all its memory is +// fd memory or pipewiresrc copied it while holding PipeWire's lock. With +// always-copy it copies each frame it takes from PipeWire, and a copy has no +// parent buffer meta, which every frame sharing PipeWire's memory has. The +// first frame sharing other memory turns always-copy on and is dropped +// unread. After that, such a frame is pipewiresrc sending its last frame +// again, so it is replaced with the last copied frame, or dropped before +// there is one. +struct SafeFrames { source: gst::glib::WeakRef, copying: bool, last: Option, } -impl MemPtrCopies { +impl SafeFrames { fn new(source: &gst::Element) -> Self { Self { source: source.downgrade(), @@ -283,28 +290,30 @@ impl MemPtrCopies { } } - fn protect(&mut self, buffer: &mut gst::Buffer) { - if !self.copying - && buffer.n_memory() > 0 - && !buffer + // Whether the frame, possibly replaced, may be passed on. + fn keep(&mut self, frame: &mut gst::Buffer) -> bool { + let mapped = frame.n_memory() > 0 + && frame .iter_memories() - .any(|memory| memory.is_memory_type::()) - { + .all(|memory| memory.is_memory_type::()); + if mapped || frame.meta::().is_none() { + if self.copying { + self.last = Some(frame.clone()); + } + return true; + } + if !self.copying { self.copying = true; if let Some(source) = self.source.upgrade() { source.set_property("always-copy", true); } - if let Ok(copy) = buffer.copy_deep() { - *buffer = copy; - } } - if !self.copying { - return; - } - if !is_keepalive(buffer) { - self.last = Some(buffer.clone()); - } else if let Some(last) = &self.last { - *buffer = last.clone(); + match &self.last { + Some(last) => { + *frame = last.clone(); + true + } + None => false, } } } @@ -386,6 +395,8 @@ struct Counters { dimensions: Mutex>, last_pts: Mutex>, timeline: Mutex, + // The source went away, so no other mode can record it. + source_lost: std::sync::atomic::AtomicBool, } // Pausing takes out of the movie what was captured while paused, judged by @@ -580,6 +591,7 @@ struct Attempt<'a> { fn finished(result: &Result<()>, counters: &Counters) -> bool { matches!(result, Ok(()) | Err(RecorderError::Cancelled)) || counters.frames.load(Ordering::Relaxed) > 0 + || counters.source_lost.load(Ordering::Relaxed) } fn ends_recording(result: &Result<()>, counters: &Counters, last: bool) -> bool { @@ -799,6 +811,11 @@ fn record_attempt( .create_new(true) .open(&session.output_path) .map_err(backend)?; + let source_lost = || match (&stream, &source_watch) { + (Some(stream), _) => stream.lost(), + (None, Some(watch)) => watch.lost().then_some(SOURCE_LOST), + (None, None) => None, + }; let result = run( &mut movie, session, @@ -806,12 +823,12 @@ fn record_attempt( commands, stop, &counters, - &|| match (&stream, &source_watch) { - (Some(stream), _) => stream.lost(), - (None, Some(watch)) => watch.lost().then_some(SOURCE_LOST), - (None, None) => None, - }, + &source_lost, ); + // A new pipewiresrc on a removed node can wait 30 s for it to start. + counters + .source_lost + .store(source_lost().is_some(), Ordering::Relaxed); let ending = ends_recording(&result, &counters, attempt.last); (attempt.stopping)(if result.is_ok() { Teardown::Finalized diff --git a/packaging/linux/README.md b/packaging/linux/README.md index 3b00e40..557fba5 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -116,8 +116,11 @@ that frame is stamped with when it arrived. A screen-capture stream can share frames as plain pointers into PipeWire's memory (MemPtr) instead of file descriptors (MemFd or DMA-BUF). PipeWire unmaps that memory when the producer exits, even while wrec still uses a -frame, so wrec copies MemPtr frames. KWin, mutter, xdg-desktop-portal-wlr and -gamescope share MemFd or DMA-BUF, which wrec does not copy. +frame. So wrec drops the first MemPtr frame without reading it and has +`pipewiresrc` copy every later frame while it holds PipeWire's lock. A MemPtr +screen that stays still from the start records no video until it changes. +KWin, mutter, xdg-desktop-portal-wlr and gamescope share MemFd or DMA-BUF, +which wrec does not copy. wrec sets the movie's output size when the first source caps arrive. Changing a capsfilter normally asks every upstream element to renegotiate, and From 85100c3e225db06a3788ffac5191c677f3f5c51a Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:24:50 +0000 Subject: [PATCH 45/45] fix(linux): offer a pinned KWin screencast timestamp backport --- packaging/linux/README.md | 16 ++ packaging/linux/kwin-6.6.6/CMakeLists.txt | 76 ++++++++ packaging/linux/kwin-6.6.6/README.md | 138 ++++++++++++++ packaging/linux/kwin-6.6.6/SHA256SUMS | 4 + packaging/linux/kwin-6.6.6/build.sh | 54 ++++++ packaging/linux/kwin-6.6.6/fetch.sh | 32 ++++ ...win-6.6.6-screencast-capture-time-pts.diff | 175 ++++++++++++++++++ packaging/linux/kwin-6.6.6/opt-in.sh | 80 ++++++++ packaging/linux/kwin-6.6.6/opt-out.sh | 18 ++ 9 files changed, 593 insertions(+) create mode 100644 packaging/linux/kwin-6.6.6/CMakeLists.txt create mode 100644 packaging/linux/kwin-6.6.6/README.md create mode 100644 packaging/linux/kwin-6.6.6/SHA256SUMS create mode 100755 packaging/linux/kwin-6.6.6/build.sh create mode 100755 packaging/linux/kwin-6.6.6/fetch.sh create mode 100644 packaging/linux/kwin-6.6.6/kwin-6.6.6-screencast-capture-time-pts.diff create mode 100755 packaging/linux/kwin-6.6.6/opt-in.sh create mode 100755 packaging/linux/kwin-6.6.6/opt-out.sh diff --git a/packaging/linux/README.md b/packaging/linux/README.md index 557fba5..72516d1 100644 --- a/packaging/linux/README.md +++ b/packaging/linux/README.md @@ -113,6 +113,16 @@ an idle screen is seconds before it rendered the picture. A timestamp no later than the previous frame's says nothing about when its picture was made, so that frame is stamped with when it arrived. +That stale stamp also moves video on KDE Plasma 6.6 and 6.7: a change that +follows less than about a second of stillness lands early by the length of the +stillness, and the first change after a resume can be lost. KWin fixed this +upstream for Plasma 6.8. Linux support on KDE is experimental, and the only +validated KDE stack is Ubuntu 26.04's KWin 4:6.6.6-0ubuntu0.1 with that fix +backported into its screencast plugin. The backport, its source pins, build +script, opt-in and undo are in +[`packaging/linux/kwin-6.6.6`](https://github.com/shivamdoting/wrec/tree/main/packaging/linux/kwin-6.6.6) +in the wrec repository. The wrec package does not include, install or load it. + A screen-capture stream can share frames as plain pointers into PipeWire's memory (MemPtr) instead of file descriptors (MemFd or DMA-BUF). PipeWire unmaps that memory when the producer exits, even while wrec still uses a @@ -406,6 +416,12 @@ that discard with the loss. This prevented early audio in the controlled 0.1 and 0.3 s server-stall tests, at the cost of 5 to 13 ms of additional audio per loss. It cannot protect audio already sent before the loss was known, or a graph loss with no break in arriving audio. +In a virtual machine whose host caps its CPU, starting a recording can push the +guest over the cap, and the host then stops the whole guest for tens of +milliseconds at a time. The guest's audio graph misses cycles, and that audio +never reaches wrec, which reports it. On a 2-vCPU KDE VM capped at 150% this +lost 0.03 to 0.07 s of microphone audio in the first second of 7 of 88 +recordings. When PulseAudio drops audio for a recorder that fell more than both buffers behind, it drops it in pieces over the following second, and in the controlled tests 1280 to 2720 frames (27 to 57 ms) of that second played early, by 9 ms up diff --git a/packaging/linux/kwin-6.6.6/CMakeLists.txt b/packaging/linux/kwin-6.6.6/CMakeLists.txt new file mode 100644 index 0000000..215c323 --- /dev/null +++ b/packaging/linux/kwin-6.6.6/CMakeLists.txt @@ -0,0 +1,76 @@ +# Out-of-tree build of only KWin's screencast plugin (src/plugins/screencast) +# against the distro's installed libkwin 6.6.6 and kwin-dev headers. build.sh +# runs it; see README.md. +# +# cmake -S -B build -DKWIN_SOURCE_DIR= +# nice -n19 cmake --build build -j2 +# +# Compile flags, definitions and C++ standard copy kwin-6.6.6/CMakeLists.txt. +# Installed headers come first; the source tree is only a fallback for the two +# headers kwin-dev does not ship (core/syncobjtimeline.h, opengl/eglbackend.h) +# and what they include. Every installed non-generated header of Ubuntu's +# kwin-dev 4:6.6.6-0ubuntu0.1 is byte-identical to the 6.6.6 source. +cmake_minimum_required(VERSION 3.22) +project(kwin-screencast-r9 VERSION 6.6.6 LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(KDE_COMPILERSETTINGS_LEVEL "5.82") +find_package(ECM 6.22.0 REQUIRED NO_MODULE) +set(CMAKE_MODULE_PATH ${ECM_MODULE_PATH}) +include(KDEInstallDirs) +include(KDECMakeSettings) +include(KDECompilerSettings NO_POLICY_SCOPE) +include(ECMQtDeclareLoggingCategory) +include(ECMDeprecationSettings) + +find_package(Qt6 6.10.0 CONFIG REQUIRED COMPONENTS Core Gui Quick DBus Widgets) +find_package(KF6 6.22.0 REQUIRED COMPONENTS CoreAddons I18n Config WindowSystem) +find_package(KWin 6.6.6 EXACT CONFIG REQUIRED) +find_package(PkgConfig REQUIRED) +pkg_check_modules(PipeWire REQUIRED IMPORTED_TARGET libpipewire-0.3) + +set(KWIN_SOURCE_DIR "" CACHE PATH "Unpacked kwin-6.6.6 source tree (with any patch applied)") +if(NOT EXISTS "${KWIN_SOURCE_DIR}/src/plugins/screencast/screencaststream.cpp") + message(FATAL_ERROR "Set KWIN_SOURCE_DIR to the kwin-6.6.6 source tree") +endif() +set(SC ${KWIN_SOURCE_DIR}/src/plugins/screencast) + +add_compile_options(-Wno-unused-parameter) +add_definitions( + -DQT_NO_KEYWORDS + -DQT_USE_QSTRINGBUILDER + -DQT_NO_URL_CAST_FROM_STRING + -DQT_NO_CAST_TO_ASCII + -DQT_NO_FOREACH + -DMESA_EGL_NO_X11_HEADERS + -DEGL_NO_X11 + -DEGL_NO_PLATFORM_SPECIFIC_TYPES +) +ecm_set_disabled_deprecation_versions(QT 5.15 KF 6.9.0) + +kcoreaddons_add_plugin(screencast INSTALL_NAMESPACE "kwin/plugins") +target_sources(screencast PRIVATE + ${SC}/filteredsceneview.cpp + ${SC}/main.cpp + ${SC}/outputscreencastsource.cpp + ${SC}/pipewirecore.cpp + ${SC}/regionscreencastsource.cpp + ${SC}/screencastbuffer.cpp + ${SC}/screencastlayer.cpp + ${SC}/screencastmanager.cpp + ${SC}/screencastsource.cpp + ${SC}/screencaststream.cpp + ${SC}/windowscreencastsource.cpp +) +ecm_qt_declare_logging_category(screencast + HEADER kwinscreencast_logging.h + IDENTIFIER KWIN_SCREENCAST + CATEGORY_NAME kwin_screencast + DEFAULT_SEVERITY Warning +) +get_target_property(KWIN_INSTALLED_INCLUDES KWin::kwin INTERFACE_INCLUDE_DIRECTORIES) +target_include_directories(screencast PRIVATE ${SC} ${KWIN_INSTALLED_INCLUDES} ${KWIN_SOURCE_DIR}/src) +target_compile_definitions(screencast PRIVATE -DTRANSLATION_DOMAIN=\"kwin\") +target_link_libraries(screencast KWin::kwin KF6::I18n PkgConfig::PipeWire Libdrm::Libdrm) diff --git a/packaging/linux/kwin-6.6.6/README.md b/packaging/linux/kwin-6.6.6/README.md new file mode 100644 index 0000000..ebc5280 --- /dev/null +++ b/packaging/linux/kwin-6.6.6/README.md @@ -0,0 +1,138 @@ +# KWin 6.6.6 screencast timestamp backport (experimental, opt-in) + +KWin 6.6 and 6.7 stamp each screen-capture frame with the time of the +screen's previous page flip, not the time the frame was captured. After the +screen sits still, that stamp is as old as the still period, so in a wrec +recording on KDE Plasma a change that follows a short pause shows up early by +up to about a second, short flashes last too long, and the first change after +`wrec job resume` can be missing. wrec trusts the compositor's stamp, so the fix +belongs in KWin. + +KDE fixed it upstream in commit +[cf5049251db3](https://invent.kde.org/plasma/kwin/-/commit/cf5049251db3654d317d04631b65b827ef4099ca) +("screencast: Timestamp frames at capture time", David Weber, 2026-08-08), +which samples the time once per frame just before rendering it. It is in the +Plasma 6.8 betas (v6.7.90, v6.7.91) and not in any 6.6.x or 6.7.x release. + +This directory backports that commit to Ubuntu's KWin 6.6.6 and builds only the +screencast plugin, which you load for your own user. Nothing here is installed +by wrec, and the wrec package works without it. + +## What was tested + +Only this stack, in a 2-vCPU VM with software rendering: + +| | | +|---|---| +| Distribution | Ubuntu 26.04 (resolute), amd64 | +| KWin packages | `kwin-wayland`, `kwin-common`, `libkwin6`, `kwin-dev` 4:6.6.6-0ubuntu0.1 | +| Toolchain | g++ 15.2.0, cmake 4.2.3, extra-cmake-modules 6.24.0, Qt 6.10.2, KF6 6.24.0, PipeWire 1.6.2 | +| Result | flashes after short pauses placed −7 to +22 ms from when the app drew them (stock: about −425 ms); flash length 140 to 190 ms for 150 ms drawn (stock: about 450 ms); first change after a resume kept 9 of 9 times (stock: 0 of 12); no paused content leaked | + +Stock KDE is not a supported wrec setup: these results come from the patched +plugin. Plasma 6.8 contains the upstream fix but also moves recording to +vsync-paced frames, and has not been tested with wrec. Other distributions, +KWin builds, real GPUs and X11 sessions are untested. + +## Files + +| file | what it is | +|---|---| +| `kwin-6.6.6-screencast-capture-time-pts.diff` | cf5049251db3 applied to Ubuntu's kwin 6.6.6 plugin sources (offsets and one fuzz when ported, no hand edits). It removes `clock()` from the screencast sources and samples `steady_clock::now()` in `ScreenCastStream::record()` before rendering. Output, window and region casts, memfd and DMA-BUF buffers. | +| `CMakeLists.txt` | builds only `src/plugins/screencast` against the installed libkwin6 and kwin-dev, with KWin 6.6.6's compile settings | +| `fetch.sh WORKDIR` | downloads Ubuntu's source for kwin 4:6.6.6-0ubuntu0.1 from Launchpad and checks `SHA256SUMS` | +| `build.sh WORKDIR [patched\|stock]` | checks the installed KWin version and the sources, applies Ubuntu's patches and the backport, builds the plugin | +| `opt-in.sh PLUGIN_ROOT` / `opt-out.sh` | add or remove one systemd user drop-in that points KWin at the built plugin | +| `SHA256SUMS` | the three source files and the diff | + +`kwin_6.6.6.orig.tar.xz` is KDE's own `kwin-6.6.6.tar.xz`: both have sha256 +`76314bb5…f561953`. The plugin directory is the same as upstream tag v6.6.6; +Ubuntu's two patches touch only `CMakeLists.txt` and `src/core/drmdevice.cpp`. + +## Build + +Install the build dependencies (on the tested system this added 114 packages +and changed none): + +```bash +sudo apt install --no-install-recommends cmake g++ extra-cmake-modules kwin-dev \ + libkf6i18n-dev libpipewire-0.3-dev libdrm-dev pkgconf make +``` + +Then, in a new directory of your choice: + +```bash +mkdir ~/kwin-screencast-backport +packaging/linux/kwin-6.6.6/fetch.sh ~/kwin-screencast-backport +packaging/linux/kwin-6.6.6/build.sh ~/kwin-screencast-backport +``` + +`build.sh` refuses to run unless `libkwin6`, `kwin-dev`, `kwin-common` and +`kwin-wayland` are all 4:6.6.6-0ubuntu0.1. It writes only `src-patched/`, +`cmake-patched/`, `build-patched/` and `plugin-patched/` in that directory, +fails if any of them already exists, and never deletes anything. It took 45 +seconds at two jobs in the test VM. `build.sh DIR stock` builds the +unpatched source the same way, for comparison. + +## Use + +```bash +packaging/linux/kwin-6.6.6/opt-in.sh ~/kwin-screencast-backport/plugin-patched +``` + +then log out and back in. `opt-in.sh` writes +`~/.config/systemd/user/plasma-kwin_wayland.service.d/50-wrec-kwin-screencast.conf`, +which sets `QT_PLUGIN_PATH` for KWin only. KWin then finds the backported +`screencast.so` before the distro's; no system file changes. It refuses if: + +- any of `libkwin6`, `kwin-common`, `kwin-wayland` is not 4:6.6.6-0ubuntu0.1, +- the plugin or one of its directories belongs to another user or is writable + by group or others, or a directory above them is writable by others (KWin + would run whatever is there), +- your session or another drop-in already sets `QT_PLUGIN_PATH`, or a file of + the same name exists that it did not write, +- Plasma is not started through systemd (`plasma-kwin_wayland.service` missing). + +To check that KWin loaded it after logging in: + +```bash +sudo grep -F screencast.so /proc/$(pgrep -u "$USER" -x kwin_wayland)/maps +``` + +(KWin's memory map needs sudo to read.) + +## Undo + +```bash +packaging/linux/kwin-6.6.6/opt-out.sh +``` + +then log out and in. KWin loads `/usr/lib/x86_64-linux-gnu/qt6/plugins/kwin/plugins/screencast.so` +again. `opt-out.sh` only removes a drop-in that `opt-in.sh` wrote. Delete the +work directory yourself, and remove the build dependencies with apt if you +installed them only for this. + +## KWin upgrades + +The plugin links against the exact libkwin it was built with. The version check +runs only when you opt in. If apt later upgrades KWin, your session keeps +loading the old plugin, which can fail to load (no screen recording) or crash +KWin at login. Before upgrading kwin, run `opt-out.sh`. If you upgraded first +and the session no longer starts, switch to a text console (Ctrl+Alt+F3), log +in, and run: + +```bash +rm ~/.config/systemd/user/plasma-kwin_wayland.service.d/50-wrec-kwin-screencast.conf +systemctl --user daemon-reload +``` + +then log in again. Rebuild for a new KWin only if this directory gains a +backport for that version. + +## License + +The diff changes KWin source files and is under their licenses: +`screencaststream.cpp` and `.h` are LGPL-2.0-or-later, the other seven files +GPL-2.0-or-later (see their SPDX headers and KWin's `LICENSES/`). The commit +it ports is by David Weber for KDE. `CMakeLists.txt` copies compile settings +from KWin 6.6.6's build files. The scripts are part of wrec (MIT). diff --git a/packaging/linux/kwin-6.6.6/SHA256SUMS b/packaging/linux/kwin-6.6.6/SHA256SUMS new file mode 100644 index 0000000..5d7dc5d --- /dev/null +++ b/packaging/linux/kwin-6.6.6/SHA256SUMS @@ -0,0 +1,4 @@ +76314bb5053943003c089bd8997b159c70a8bc35ba87dc3a10885ade8f561953 kwin_6.6.6.orig.tar.xz +f8e51babdd9fd39760cb207877f0f9f8f2efec09202aeb926498d981e4c7263c kwin_6.6.6-0ubuntu0.1.debian.tar.xz +4fda1d85b25c5f1c09a5c292065050ea038946e2f60f2b997e70e3d3c8e3eea0 kwin_6.6.6-0ubuntu0.1.dsc +5dda984f4c0e1dffaf5621c7c8bca50b03a0d21e2c0de71043f16d43a2943d8d kwin-6.6.6-screencast-capture-time-pts.diff diff --git a/packaging/linux/kwin-6.6.6/build.sh b/packaging/linux/kwin-6.6.6/build.sh new file mode 100755 index 0000000..4fc02de --- /dev/null +++ b/packaging/linux/kwin-6.6.6/build.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# build.sh WORKDIR [patched|stock]: build KWin's screencast plugin from Ubuntu's +# kwin 4:6.6.6-0ubuntu0.1 source in WORKDIR (put there by fetch.sh), against +# the installed libkwin6 and kwin-dev of exactly that version. `patched` (the +# default) adds the capture-time timestamp backport; `stock` builds the same +# source unchanged, as a control. +# +# Writes only WORKDIR/{src,cmake,build,plugin}-VARIANT and fails if any of +# them exists; it never deletes anything. The plugin lands in +# WORKDIR/plugin-VARIANT/kwin/plugins/screencast.so, ready for opt-in.sh. +# Nothing is installed. Runs at nice 19 with JOBS (default 2) compile jobs. +set -euo pipefail +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +work=${1:?usage: build.sh WORKDIR [patched|stock]} +variant=${2:-patched} +version=4:6.6.6-0ubuntu0.1 +[[ $variant == patched || $variant == stock ]] || { echo "build.sh: variant is patched or stock" >&2; exit 1; } +[[ -d $work && -O $work ]] || { echo "build.sh: $work must be an existing directory you own" >&2; exit 1; } +work=$(realpath "$work") +[[ $(dpkg --print-architecture) == amd64 ]] || { echo "build.sh: tested on amd64 only" >&2; exit 1; } +for package in libkwin6 kwin-dev kwin-common kwin-wayland; do + got=$(dpkg-query -W -f '${Version}' "$package" 2>/dev/null || true) + [[ $got == "$version" ]] || { echo "build.sh: $package is '${got:-not installed}', this backport is for $version only" >&2; exit 1; } +done +(cd "$here" && sha256sum -c --quiet --ignore-missing SHA256SUMS) +(cd "$work" && grep -E 'kwin_6\.6\.6' "$here/SHA256SUMS" | sha256sum -c --quiet -) \ + || { echo "build.sh: run fetch.sh $work first" >&2; exit 1; } +for dir in src cmake build plugin; do + [[ ! -e $work/$dir-$variant ]] || { echo "build.sh: $work/$dir-$variant exists; use a new WORKDIR or move it away" >&2; exit 1; } +done +src=$work/src-$variant +mkdir "$src" "$work/cmake-$variant" "$work/build-$variant" +tar -xJf "$work/kwin_6.6.6.orig.tar.xz" -C "$src" --strip-components=1 +tar -xJf "$work/kwin_6.6.6-0ubuntu0.1.debian.tar.xz" -C "$src" +while read -r name; do + case $name in ''|'#'*) continue ;; esac + patch -d "$src" -p1 --forward --fuzz=0 --no-backup-if-mismatch < "$src/debian/patches/$name" +done < "$src/debian/patches/series" +if [[ $variant == patched ]]; then + patch -d "$src" -p1 --forward --fuzz=0 --no-backup-if-mismatch < "$here/kwin-6.6.6-screencast-capture-time-pts.diff" +fi +cp "$here/CMakeLists.txt" "$work/cmake-$variant/" +nice -n19 cmake -S "$work/cmake-$variant" -B "$work/build-$variant" -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DKWIN_SOURCE_DIR="$src" -DCMAKE_INSTALL_PREFIX="$work/build-$variant/unused-prefix" \ + > "$work/build-$variant/cmake.log" 2>&1 \ + || { echo "build.sh: cmake failed, see $work/build-$variant/cmake.log" >&2; exit 1; } +nice -n19 cmake --build "$work/build-$variant" -j "${JOBS:-2}" > "$work/build-$variant/build.log" 2>&1 \ + || { echo "build.sh: build failed, see $work/build-$variant/build.log" >&2; exit 1; } +built=$(find "$work/build-$variant" -name screencast.so -type f) +[[ $(printf '%s\n' "$built" | wc -l) == 1 && -n $built ]] +# opt-in.sh accepts only a plugin nobody else can change. +install -d -m 0755 "$work/plugin-$variant" "$work/plugin-$variant/kwin" "$work/plugin-$variant/kwin/plugins" +install -m 0644 "$built" "$work/plugin-$variant/kwin/plugins/screencast.so" +sha256sum "$work/plugin-$variant/kwin/plugins/screencast.so" diff --git a/packaging/linux/kwin-6.6.6/fetch.sh b/packaging/linux/kwin-6.6.6/fetch.sh new file mode 100755 index 0000000..d6f65e2 --- /dev/null +++ b/packaging/linux/kwin-6.6.6/fetch.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# fetch.sh WORKDIR: download Ubuntu's kwin 4:6.6.6-0ubuntu0.1 source into +# WORKDIR, an existing directory you own, and check every file against +# SHA256SUMS. A file already there is kept if its checksum matches and +# reported otherwise; nothing is overwritten or deleted. Each download goes to +# a new temporary file that only this script removes, and a checked download +# is published with a hard link, which fails if the name appeared meanwhile. +set -euo pipefail +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +work=${1:?usage: fetch.sh WORKDIR} +[[ -d $work && -O $work ]] || { echo "fetch.sh: $work must be an existing directory you own" >&2; exit 1; } +base=https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/kwin/4:6.6.6-0ubuntu0.1 +tmp= +trap '[[ -z $tmp ]] || rm -f -- "$tmp"' EXIT +for file in kwin_6.6.6.orig.tar.xz kwin_6.6.6-0ubuntu0.1.debian.tar.xz kwin_6.6.6-0ubuntu0.1.dsc; do + want=$(awk -v f="$file" '$2 == f { print $1 }' "$here/SHA256SUMS") + [[ -n $want ]] + if [[ -e $work/$file ]]; then + got=$(sha256sum "$work/$file" | cut -d' ' -f1) + [[ $got == "$want" ]] || { echo "fetch.sh: $work/$file exists with sha256 $got, expected $want; move it away" >&2; exit 1; } + echo "kept $file" + continue + fi + tmp=$(mktemp "$work/.$file.XXXXXX") + curl -fsSL --proto '=https' --max-time 300 -o "$tmp" "$base/$file" + got=$(sha256sum "$tmp" | cut -d' ' -f1) + [[ $got == "$want" ]] || { echo "fetch.sh: $file downloaded with sha256 $got, expected $want" >&2; exit 1; } + ln -- "$tmp" "$work/$file" || { echo "fetch.sh: $work/$file appeared during the download; move it away" >&2; exit 1; } + rm -f -- "$tmp" + tmp= + echo "fetched $file" +done diff --git a/packaging/linux/kwin-6.6.6/kwin-6.6.6-screencast-capture-time-pts.diff b/packaging/linux/kwin-6.6.6/kwin-6.6.6-screencast-capture-time-pts.diff new file mode 100644 index 0000000..f08e107 --- /dev/null +++ b/packaging/linux/kwin-6.6.6/kwin-6.6.6-screencast-capture-time-pts.diff @@ -0,0 +1,175 @@ +diff -ru a/src/plugins/screencast/outputscreencastsource.cpp b/src/plugins/screencast/outputscreencastsource.cpp +--- a/src/plugins/screencast/outputscreencastsource.cpp 2026-10-09 22:54:58.506481127 +0000 ++++ b/src/plugins/screencast/outputscreencastsource.cpp 2026-10-09 22:54:58.808739381 +0000 +@@ -11,8 +11,6 @@ + + #include "compositor.h" + #include "core/output.h" +-#include "core/renderbackend.h" +-#include "core/renderloop.h" + #include "cursor.h" + #include "opengl/eglbackend.h" + #include "opengl/egldisplay.h" +@@ -103,11 +101,6 @@ + return bufferDamage; + } + +-std::chrono::nanoseconds OutputScreenCastSource::clock() const +-{ +- return m_output->backendOutput()->renderLoop()->lastPresentationTimestamp(); +-} +- + uint OutputScreenCastSource::refreshRate() const + { + return m_output->refreshRate(); +diff -ru a/src/plugins/screencast/outputscreencastsource.h b/src/plugins/screencast/outputscreencastsource.h +--- a/src/plugins/screencast/outputscreencastsource.h 2026-10-09 22:54:58.506526856 +0000 ++++ b/src/plugins/screencast/outputscreencastsource.h 2026-10-09 22:54:58.808848724 +0000 +@@ -34,7 +34,6 @@ + void setRenderCursor(bool enable) override; + Region render(GLFramebuffer *target, const Region &bufferRepair) override; + Region render(QImage *target, const Region &bufferRepair) override; +- std::chrono::nanoseconds clock() const override; + + void resume() override; + void pause() override; +diff -ru a/src/plugins/screencast/regionscreencastsource.cpp b/src/plugins/screencast/regionscreencastsource.cpp +--- a/src/plugins/screencast/regionscreencastsource.cpp 2026-10-09 22:54:58.506665596 +0000 ++++ b/src/plugins/screencast/regionscreencastsource.cpp 2026-10-09 22:54:58.808966684 +0000 +@@ -57,11 +57,6 @@ + return DRM_FORMAT_ARGB8888; + } + +-std::chrono::nanoseconds RegionScreenCastSource::clock() const +-{ +- return m_last; +-} +- + void RegionScreenCastSource::setRenderCursor(bool enable) + { + m_renderCursor = enable; +@@ -72,7 +67,6 @@ + + Region RegionScreenCastSource::render(GLFramebuffer *target, const Region &bufferRepair) + { +- m_last = std::chrono::steady_clock::now().time_since_epoch(); + m_layer->setFramebuffer(target, bufferRepair & Rect(QPoint(), target->size())); + if (!m_layer->preparePresentationTest()) { + return Region{}; +diff -ru a/src/plugins/screencast/regionscreencastsource.h b/src/plugins/screencast/regionscreencastsource.h +--- a/src/plugins/screencast/regionscreencastsource.h 2026-10-09 22:54:58.506713219 +0000 ++++ b/src/plugins/screencast/regionscreencastsource.h 2026-10-09 22:54:58.809071439 +0000 +@@ -33,7 +33,6 @@ + void setRenderCursor(bool enable) override; + Region render(GLFramebuffer *target, const Region &bufferRepair) override; + Region render(QImage *target, const Region &bufferRepair) override; +- std::chrono::nanoseconds clock() const override; + + void close(); + void pause() override; +@@ -48,7 +47,6 @@ + const Rect m_region; + const qreal m_scale; + const std::optional m_pidToHide; +- std::chrono::nanoseconds m_last{0}; + bool m_closed = false; + bool m_active = false; + bool m_renderCursor = false; +diff -ru a/src/plugins/screencast/screencastsource.h b/src/plugins/screencast/screencastsource.h +--- a/src/plugins/screencast/screencastsource.h 2026-10-09 22:54:58.507056869 +0000 ++++ b/src/plugins/screencast/screencastsource.h 2026-10-09 22:54:58.809158959 +0000 +@@ -34,7 +34,6 @@ + virtual void setRenderCursor(bool enable) = 0; + virtual Region render(GLFramebuffer *target, const Region &bufferRepair) = 0; + virtual Region render(QImage *target, const Region &bufferRepair) = 0; +- virtual std::chrono::nanoseconds clock() const = 0; + + virtual void resume() = 0; + virtual void pause() = 0; +diff -ru a/src/plugins/screencast/screencaststream.cpp b/src/plugins/screencast/screencaststream.cpp +--- a/src/plugins/screencast/screencaststream.cpp 2026-10-09 22:54:58.507100314 +0000 ++++ b/src/plugins/screencast/screencaststream.cpp 2026-10-09 22:54:58.809280817 +0000 +@@ -607,6 +607,10 @@ + + spa_meta_sync_timeline *synctmeta = nullptr; + ++ // Use one timestamp for this record operation. Sample it before video ++ // rendering, readback and buffer synchronization add latency. ++ const auto timestamp = std::chrono::duration_cast( ++ std::chrono::steady_clock::now().time_since_epoch()); + Region damage; + if (effectiveContents & Content::Video) { + if (auto memfd = dynamic_cast(buffer)) { +@@ -650,7 +654,7 @@ + } + + addDamage(spa_buffer, damage); +- addHeader(spa_buffer); ++ addHeader(spa_buffer, timestamp); + + if (effectiveContents & Content::Video) { + spa_data->chunk->flags = SPA_CHUNK_FLAG_NONE; +@@ -688,14 +692,14 @@ + pw_stream_update_params(m_pwStream, params.data(), params.count()); + } + +-void ScreenCastStream::addHeader(spa_buffer *spaBuffer) ++void ScreenCastStream::addHeader(spa_buffer *spaBuffer, std::chrono::nanoseconds timestamp) + { + spa_meta_header *spaHeader = (spa_meta_header *)spa_buffer_find_meta_data(spaBuffer, SPA_META_Header, sizeof(spa_meta_header)); + if (spaHeader) { + spaHeader->flags = 0; + spaHeader->dts_offset = 0; + spaHeader->seq = m_sequential++; +- spaHeader->pts = m_source->clock().count(); ++ spaHeader->pts = timestamp.count(); + } + } + +diff -ru a/src/plugins/screencast/screencaststream.h b/src/plugins/screencast/screencaststream.h +--- a/src/plugins/screencast/screencaststream.h 2026-10-09 22:54:58.507152596 +0000 ++++ b/src/plugins/screencast/screencaststream.h 2026-10-09 22:54:58.809434847 +0000 +@@ -90,7 +90,7 @@ + void resize(const QSize &resolution); + void coreFailed(const QString &errorMessage); + void addCursorMetadata(spa_buffer *spaBuffer, Cursor *cursor); +- void addHeader(spa_buffer *spaBuffer); ++ void addHeader(spa_buffer *spaBuffer, std::chrono::nanoseconds timestamp); + void corruptHeader(spa_buffer *spaBuffer); + void addDamage(spa_buffer *spaBuffer, const Region &damagedRegion); + void newStreamParams(); +diff -ru a/src/plugins/screencast/windowscreencastsource.cpp b/src/plugins/screencast/windowscreencastsource.cpp +--- a/src/plugins/screencast/windowscreencastsource.cpp 2026-10-09 22:54:58.507244515 +0000 ++++ b/src/plugins/screencast/windowscreencastsource.cpp 2026-10-09 22:54:58.809523520 +0000 +@@ -9,8 +9,6 @@ + #include "screencastutils.h" + + #include "compositor.h" +-#include "core/backendoutput.h" +-#include "core/renderloop.h" + #include "core/rendertarget.h" + #include "core/renderviewport.h" + #include "effect/effect.h" +@@ -142,11 +140,6 @@ + return Rect(QPoint(), target->size()); + } + +-std::chrono::nanoseconds WindowScreenCastSource::clock() const +-{ +- return m_windows[0]->output()->backendOutput()->renderLoop()->lastPresentationTimestamp(); +-} +- + uint WindowScreenCastSource::refreshRate() const + { + return m_windows[0]->output()->refreshRate(); +diff -ru a/src/plugins/screencast/windowscreencastsource.h b/src/plugins/screencast/windowscreencastsource.h +--- a/src/plugins/screencast/windowscreencastsource.h 2026-10-09 22:54:58.507290424 +0000 ++++ b/src/plugins/screencast/windowscreencastsource.h 2026-10-09 22:54:58.809617002 +0000 +@@ -31,7 +31,6 @@ + void setRenderCursor(bool enable) override; + Region render(GLFramebuffer *target, const Region &bufferDamage) override; + Region render(QImage *target, const Region &bufferDamage) override; +- std::chrono::nanoseconds clock() const override; + + void resume() override; + void pause() override; diff --git a/packaging/linux/kwin-6.6.6/opt-in.sh b/packaging/linux/kwin-6.6.6/opt-in.sh new file mode 100755 index 0000000..09f3e55 --- /dev/null +++ b/packaging/linux/kwin-6.6.6/opt-in.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# opt-in.sh PLUGIN_ROOT: from your next login, have your KWin load +# PLUGIN_ROOT/kwin/plugins/screencast.so (from build.sh) instead of the +# distro's. Writes one systemd user drop-in that sets QT_PLUGIN_PATH for +# plasma-kwin_wayland.service, and nothing else: no system file, package or +# other user file changes. opt-out.sh removes it. +# +# Refuses unless kwin is exactly 4:6.6.6-0ubuntu0.1, the plugin and its +# directories are yours and writable by nobody else, and nothing else already +# sets QT_PLUGIN_PATH for KWin. +set -euo pipefail +root=${1:?usage: opt-in.sh PLUGIN_ROOT} +version=4:6.6.6-0ubuntu0.1 +unit=plasma-kwin_wayland.service +dir=${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user/$unit.d +dropin=$dir/50-wrec-kwin-screencast.conf +marker='# wrec kwin-6.6.6 screencast backport' +fail() { echo "opt-in.sh: $*" >&2; exit 1; } + +for package in libkwin6 kwin-common kwin-wayland; do + got=$(dpkg-query -W -f '${Version}' "$package" 2>/dev/null || true) + [[ $got == "$version" ]] || fail "$package is '${got:-not installed}'; this plugin is built for $version only" +done +root=$(realpath -e "$root") || fail "$1 does not exist" +plugin=$root/kwin/plugins/screencast.so +[[ -f $plugin && ! -L $plugin ]] || fail "$plugin is not a regular file" +# KWin runs whatever this path holds, so nobody else may change it: the +# plugin and its directories are yours and writable only by you, and no +# directory above them lets others rename them (sticky ones like /tmp aside). +for path in "$plugin" "$root/kwin/plugins" "$root/kwin" "$root"; do + owner=$(stat -c %u "$path") + [[ $owner == "$(id -u)" || $owner == 0 ]] || fail "$path is owned by uid $owner" + [[ $(( 0$(stat -c %a "$path") & 022 )) == 0 ]] || fail "$path is writable by group or others" +done +path=$root +while [[ $path != / ]]; do + path=$(dirname "$path") + owner=$(stat -c %u "$path") + [[ $owner == "$(id -u)" || $owner == 0 ]] || fail "$path is owned by uid $owner" + mode=0$(stat -c %a "$path") + [[ $(( mode & 02 )) == 0 || $(( mode & 01000 )) != 0 ]] || fail "$path is writable by others" +done +[[ $root != *$'\n'* ]] || fail "the path contains a newline" +[[ $(systemctl --user show "$unit" -p LoadState --value) == loaded ]] || fail "$unit not found: this needs Plasma started by systemd" +if systemctl --user show-environment | grep -q '^QT_PLUGIN_PATH='; then + fail "your user session already sets QT_PLUGIN_PATH; setting it for KWin would replace yours" +fi +for other in "$dir"/*.conf; do + [[ -e $other && $other != "$dropin" ]] || continue + if grep -q QT_PLUGIN_PATH "$other"; then fail "$other already sets QT_PLUGIN_PATH"; fi +done +if [[ -e $dropin ]]; then + grep -qxF "$marker" "$dropin" || fail "$dropin exists and was not written by this script" +elif [[ $(systemctl --user show "$unit" -p Environment --value) == *QT_PLUGIN_PATH=* ]]; then + fail "$unit already gets QT_PLUGIN_PATH from another drop-in" +fi + +# systemd unquotes "..." with C escapes and expands % specifiers. +value=QT_PLUGIN_PATH=$root +value=${value//\\/\\\\} +value=${value//\"/\\\"} +value=${value//%/%%} +mkdir -p "$dir" +tmp=$(mktemp "$dir/.wrec-kwin-screencast.XXXXXX") +cat > "$tmp" <&2; exit 1; } +rm -- "$dropin" +rmdir -- "$dir" 2> /dev/null || true +systemctl --user daemon-reload +echo "Removed $dropin. Log out and in to load the distro plugin again."