Enable explicit software decoding in the embedded Linux stream

docs/linux-video-backends.md

@@ -1,9 +1,15 @@
 # Troubleshoot unavailable Linux video backends
 
-The Qt stream view requires a hardware decoder. On Intel systems, VA-API decoding can
+The Qt stream view prefers a hardware decoder. On Intel systems, VA-API decoding can
 work without Vulkan Video decoding support. The stream view still uses Vulkan for
-presentation; OpenGL is not an embedded streaming backend. Enabling software decoding
-in a standalone streamer does not enable it in the Qt stream view.
+presentation; OpenGL is not an embedded streaming backend.
+
+Auto never switches to software decoding on its own, including after a failed
+hardware start. When no hardware decoder is available, choose **Software (CPU)** in
+Stream settings to decode with the bundled FFmpeg software decoder. That path accepts
+8-bit 4:2:0 SDR only: HDR, 10-bit, and 4:4:4 are refused before CloudMatch instead of
+being downgraded, and frames are copied through a bounded CPU NV12 upload. Expect
+higher CPU load and lower frame rates than hardware decoding.
 
 ## Check the host driver first
 

locales/en.json

@@ -3295,13 +3295,15 @@
     "s_qt_about_tagline": "Your games, anywhere.",
     "s_qt_update_confirm": "OpenNOW will close to install the verified update. Continue?",
     "s_qt_backend_auto_description": "Auto uses DX11 hardware decoding. DX12 and Vulkan texture sharing are not supported by the Windows stream view yet. Applies to the next stream.",
-    "s_qt_backend_description": "Choose a supported native backend. Applies to the next stream.",
+    "s_qt_backend_description": "Choose a supported decoder. Auto never falls back to software. Applies to the next stream.",
     "s_qt_codec_unavailable": "Not supported by the detected native decoder",
     "s_qt_backend_auto": "Auto (recommended)",
     "s_qt_backend_auto_detail": "Use a supported native backend",
     "s_qt_backend_checking": "Checking hardware…",
     "s_qt_backend_unavailable": "Not supported by this stream view",
     "s_qt_backend_hardware": "Hardware decoding",
+    "s_qt_backend_software": "Software (CPU)",
+    "s_qt_backend_software_detail": "CPU decoding; 8-bit 4:2:0 SDR only",
     "s_qt_decoder_pending": "The native decoder is still being detected. Please try again in a moment.",
     "s_qt_resume_stop_timeout": "The previous native stream is still stopping. Retry after cleanup finishes, or restart OpenNOW.",
     "s_qt_resume_stop_failed": "Could not stop the previous native connection",

native/opennow-core/src/streamer.rs

@@ -335,22 +335,41 @@ impl StreamerService {
                 "HDR requires an active HDR-capable window output. Disable HDR or select a supported display.",
             ));
         }
-        if hdr && settings["decoderPreference"].as_str() == Some("software") {
+        let requested_backend = requested_embedded_backend(settings);
+        let software_requested = matches!(requested_backend.as_str(), "software" | "ffmpeg");
+        if hdr && software_requested {
             return Err(invalid(
                 "HDR requires a 10-bit hardware decoder; software decoding is not supported",
             ));
         }
-        let requested_backend = settings["nativeVideoBackend"].as_str().unwrap_or("auto");
+        let requested_color = settings["colorQuality"].as_str().unwrap_or("8bit_420");
+        if software_requested && requested_color != "8bit_420" {
+            return Err(invalid(
+                "Software decoding presents 8-bit 4:2:0 SDR only. Select 8-bit 4:2:0 in Stream settings or a hardware backend.",
+            ));
+        }
+        let software_available = capabilities["videoBackends"]
+            .as_array()
+            .into_iter()
+            .flatten()
+            .any(|backend| {
+                matches!(backend["backend"].as_str(), Some("software" | "ffmpeg"))
+                    && backend["available"].as_bool() == Some(true)
+            });
         let mut selected = capabilities.clone();
         let backends = selected["videoBackends"]
             .as_array_mut()
             .ok_or_else(|| invalid("Embedded streamer did not report video backends"))?;
         for backend in backends.iter_mut() {
             let name = backend["backend"].as_str().unwrap_or("");
-            let matches = !matches!(name, "software" | "ffmpeg")
-                && (requested_backend == "auto"
-                    || requested_backend == name
-                    || (requested_backend == "nvdec" && name == "cuda"));
+            let matches = if software_requested {
+                matches!(name, "software" | "ffmpeg")
+            } else {
+                !matches!(name, "software" | "ffmpeg")
+                    && (requested_backend == "auto"
+                        || requested_backend == name
+                        || (requested_backend == "nvdec" && name == "cuda"))
+            };
             if !matches {
                 backend["available"] = json!(false);
             }
@@ -359,12 +378,14 @@ impl StreamerService {
             .iter()
             .any(|backend| backend["available"].as_bool() == Some(true))
         {
-            let message = if requested_backend == "auto" {
-                "No hardware video backend is available for embedded streaming on this device. Check the hardware drivers and export diagnostics for backend probe failures.".to_owned()
-            } else {
+            let message = if software_requested {
+                format!(
+                    "The FFmpeg software decoder is unavailable for the requested {requested_color} profile. Select Auto in Stream settings, or export diagnostics for backend probe failures."
+                )
+            } else if requested_backend != "auto" {
                 let mut message = format!(
                     "The {} backend is unavailable for embedded streaming on this device. Select Auto in Stream settings.",
-                    crate::diagnostics::runtime_failure_reason(requested_backend)
+                    crate::diagnostics::runtime_failure_reason(&requested_backend)
                 );
                 let evidence = crate::diagnostics::native_runtime_evidence(capabilities);
                 if let Some(reason) = evidence["videoBackends"]
@@ -381,6 +402,10 @@ impl StreamerService {
                     message.push_str(&format!(" {reason}"));
                 }
                 message
+            } else if software_available {
+                "No hardware video backend is available for embedded streaming on this device. Select Software (CPU) to decode on the CPU, or export diagnostics for probe failures.".to_owned()
+            } else {
+                "No hardware video backend is available for embedded streaming on this device. Check the hardware drivers and export diagnostics for backend probe failures.".to_owned()
             };
             return Err(StreamerError {
                 code: "streamer_backend_unavailable",
@@ -495,7 +520,7 @@ impl StreamerService {
             };
             candidates.iter().find(|codec| codec_available(&selected, codec)).copied()
                 .ok_or_else(|| StreamerError { code: "streamer_codec_unavailable",
-                    message: "No available hardware codec supports the requested color mode. Try 8-bit 4:2:0 in Stream settings.".to_owned() })?
+                    message: "No available codec supports the requested color mode. Try 8-bit 4:2:0 in Stream settings.".to_owned() })?
         } else {
             let codec =
                 normalize_codec_name(&requested).ok_or_else(|| invalid("Unknown video codec"))?;
@@ -1013,6 +1038,27 @@ fn probe_capabilities(executable: &Path, settings: &Value) -> Result<Value, Stre
     Ok(capabilities)
 }
 
+fn requested_embedded_backend(settings: &Value) -> String {
+    let requested = settings["nativeVideoBackend"]
+        .as_str()
+        .unwrap_or("auto")
+        .trim()
+        .to_ascii_lowercase();
+    if !requested.is_empty() && requested != "auto" {
+        return requested;
+    }
+    match settings["decoderPreference"]
+        .as_str()
+        .unwrap_or("auto")
+        .trim()
+        .to_ascii_lowercase()
+        .as_str()
+    {
+        "software" => "software".to_owned(),
+        _ => "auto".to_owned(),
+    }
+}
+
 fn normalize_codec_name(value: &str) -> Option<&'static str> {
     match value.trim().to_ascii_lowercase().as_str() {
         "h264" | "avc" | "auto" | "" => Some("h264"),
@@ -2352,6 +2398,71 @@ mod tests {
         assert!(!error.message.contains("Select Auto"));
     }
 
+    #[test]
+    fn embedded_software_resolution_honors_the_explicit_cpu_policy() {
+        let software_caps = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[
+            {"backend":"vulkan", "platform":"linux", "available":true, "codecs":[
+                {"codec":"h264", "available":true, "colorQualities":["8bit_420"]},
+                {"codec":"h265", "available":true, "colorQualities":["8bit_420"]}]},
+            {"backend":"ffmpeg", "platform":"linux", "available":true, "codecs":[
+                {"codec":"h264", "available":true, "colorQualities":["8bit_420"]}]}
+        ]});
+        for settings in [
+            json!({"nativeVideoBackend":"software", "codec":"h264"}),
+            json!({"nativeVideoBackend":"ffmpeg", "codec":"h264"}),
+            json!({"decoderPreference":"software", "codec":"h264"}),
+        ] {
+            let resolved =
+                StreamerService::embedded_session_settings(&settings, &software_caps).unwrap();
+            assert_eq!(resolved["codec"], "h264", "{settings}");
+        }
+        let error = StreamerService::embedded_session_settings(
+            &json!({"nativeVideoBackend":"software", "codec":"h265"}),
+            &software_caps,
+        )
+        .unwrap_err();
+        assert_eq!(error.code, "streamer_codec_unavailable");
+        assert!(
+            !error.message.contains("hardware"),
+            "software requests must not borrow hardware codecs: {}",
+            error.message
+        );
+        let error = StreamerService::embedded_session_settings(
+            &json!({"nativeVideoBackend":"software", "colorQuality":"10bit_420"}),
+            &software_caps,
+        )
+        .unwrap_err();
+        assert!(
+            error
+                .message
+                .contains("Software decoding presents 8-bit 4:2:0 SDR only"),
+            "{}",
+            error.message
+        );
+        let hardware_only = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[
+            {"backend":"vulkan", "platform":"linux", "available":true, "codecs":[
+                {"codec":"h264", "available":true, "colorQualities":["8bit_420"]}]}
+        ]});
+        let error = StreamerService::embedded_session_settings(
+            &json!({"nativeVideoBackend":"software"}),
+            &hardware_only,
+        )
+        .unwrap_err();
+        assert!(
+            error
+                .message
+                .contains("FFmpeg software decoder is unavailable"),
+            "{}",
+            error.message
+        );
+        let resolved = StreamerService::embedded_session_settings(
+            &json!({"nativeVideoBackend":"auto", "codec":"auto"}),
+            &software_caps,
+        )
+        .unwrap();
+        assert_eq!(resolved["codec"], "h265");
+    }
+
     #[test]
     fn embedded_macos_codec_policy_never_falls_back_to_unsupported_hardware() {
         let mut caps = json!({"protocolVersion":STREAMER_PROTOCOL_VERSION,"videoBackends":[{

native/opennow-streamer/crates/opennow-streamer-platform-linux/tests/software_decode.rs

@@ -0,0 +1,124 @@
+#![cfg(all(target_os = "linux", feature = "ffmpeg"))]
+
+use opennow_streamer_platform_linux::{
+    ColorTransfer, DecoderPreference, EncodedVideoFrame, LifecycleState, LinuxSession, PixelFormat,
+    PushOutcome, SessionConfig, StreamFormat,
+};
+use std::sync::Arc;
+use std::thread;
+use std::time::{Duration, Instant};
+
+const BITSTREAM: &[u8] = include_bytes!("fixtures/software_decode_64x64.h264");
+const EXPECTED_FRAMES: usize = 30;
+const SECOND_KEYFRAME: usize = 15;
+const FRAME_INTERVAL_US: u64 = 33_333;
+
+fn access_units(stream: &[u8]) -> Vec<(Vec<u8>, bool)> {
+    let mut starts = Vec::new();
+    for index in 0..stream.len().saturating_sub(3) {
+        if stream[index..index + 3] == [0, 0, 1] {
+            starts.push(index);
+        }
+    }
+    starts.push(stream.len());
+    let mut units: Vec<(Vec<u8>, bool)> = Vec::new();
+    let mut pending = Vec::new();
+    let mut keyframe = false;
+    for window in starts.windows(2) {
+        let nal = stream[window[0] + 3] & 0x1f;
+        pending.extend_from_slice(&stream[window[0]..window[1]]);
+        keyframe |= matches!(nal, 5 | 7);
+        if matches!(nal, 1 | 5) {
+            units.push((std::mem::take(&mut pending), keyframe));
+            keyframe = false;
+        }
+    }
+    units
+}
+
+fn drain_until_silent(
+    session: &LinuxSession,
+    frames: &mut Vec<opennow_streamer_platform_linux::DecodedVideoFrame>,
+    silence: Duration,
+) {
+    while let Some(frame) = session.recv_frame_timeout(silence) {
+        frames.push(frame);
+    }
+}
+
+#[test]
+fn software_only_decodes_cpu_nv12_sdr_frames_for_embedded_presentation() {
+    let mut config = SessionConfig::new(StreamFormat::h264_default(64, 64).unwrap());
+    config.decoder_preference = DecoderPreference::SoftwareOnly;
+    config.embedded_presentation = true;
+    config.audio = None;
+    let mut session =
+        LinuxSession::start(config).expect("software decoder opens without a shared GPU device");
+    let ready = Instant::now() + Duration::from_secs(3);
+    while session.state() != LifecycleState::Running {
+        assert!(
+            Instant::now() < ready,
+            "video worker did not reach Running: {:?}",
+            session.state()
+        );
+        thread::sleep(Duration::from_millis(1));
+    }
+
+    let units = access_units(BITSTREAM);
+    assert_eq!(units.len(), EXPECTED_FRAMES);
+    let mut frames = Vec::new();
+    let mut timestamp_us = 0;
+    for (data, keyframe) in units.iter() {
+        assert!(matches!(
+            session
+                .submit_video(
+                    EncodedVideoFrame::new(
+                        Arc::<[u8]>::from(data.as_slice()),
+                        timestamp_us,
+                        *keyframe
+                    )
+                    .unwrap()
+                )
+                .expect("access unit is admitted"),
+            PushOutcome::Queued | PushOutcome::DroppedOldest
+        ));
+        while let Some(frame) = session.try_recv_frame() {
+            frames.push(frame);
+        }
+        thread::sleep(Duration::from_millis(1));
+        timestamp_us += FRAME_INTERVAL_US;
+    }
+    drain_until_silent(&session, &mut frames, Duration::from_millis(500));
+
+    assert!(
+        frames.len() > SECOND_KEYFRAME,
+        "decode must continue across the second keyframe, got {}",
+        frames.len()
+    );
+    for (index, frame) in frames.iter().enumerate() {
+        assert_eq!(
+            frame.timestamp_us,
+            index as u64 * FRAME_INTERVAL_US,
+            "delivered frames must be a contiguous prefix"
+        );
+        assert_eq!((frame.format.width, frame.format.height), (64, 64));
+        assert_eq!(frame.format.pixel_format, PixelFormat::Nv12);
+        assert_eq!(frame.format.color_transfer, ColorTransfer::Sdr);
+        assert!(frame.vulkan.is_none() && frame.dmabuf.is_none());
+        assert_eq!(frame.planes.len(), 2);
+    }
+
+    let delivered = frames.len();
+    session
+        .submit_video(
+            EncodedVideoFrame::new(Arc::<[u8]>::from(units[0].0.as_slice()), 1_000_000, true)
+                .unwrap(),
+        )
+        .expect("fresh access unit is admitted");
+    drain_until_silent(&session, &mut frames, Duration::from_millis(500));
+    assert!(
+        frames.len() > delivered,
+        "new input must release look-ahead frames held by the decoder"
+    );
+    session.stop().expect("session stops");
+}