From 682cc9c567c6e56708c8e54dd5833aa87d4c773b Mon Sep 17 00:00:00 2001 From: AxalotLDev Date: Wed, 19 Aug 2026 19:24:50 +0500 Subject: [PATCH] perf: cache video/playlist metadata, speed up yt-dlp retries - Add in-memory TTL cache (5 min) keyed by normalized video/playlist id for YouTube video, Twitch video, and playlist metadata lookups; skips repeated yt-dlp spawns on re-pasted/revisited URLs. Only complete results are cached (missing duration or live entries are excluded) to avoid locking in a transient fetch failure. - Remove the artificial 1200ms delay between yt-dlp metadata retries. - Skip the redundant fetch_duration spawn for live videos. - Expand cookies-from-browser fallback to try every detected browser (was: first found only), Firefox first since it doesn't depend on the OS keyring like Chromium-family browsers do. Adds edge/whale on Linux and chromium/opera/whale on Windows to the detection list. - Add parking_lot dependency for the cache mutex, per project convention (LazyLock + parking_lot over OnceLock/std Mutex). --- src-tauri/Cargo.lock | 1 + src-tauri/Cargo.toml | 1 + src-tauri/src/functions/cache.rs | 96 ++++++++++++ src-tauri/src/functions/get_info.rs | 35 ++++- src-tauri/src/functions/mod.rs | 1 + src-tauri/src/functions/playlist.rs | 11 +- src-tauri/src/functions/youtube.rs | 221 +++++++++++++++------------ src-tauri/tests/integration_tests.rs | 51 +++++++ 8 files changed, 315 insertions(+), 102 deletions(-) create mode 100644 src-tauri/src/functions/cache.rs diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 942f2d1..e31b691 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -10,6 +10,7 @@ dependencies = [ "encoding_rs", "libc", "once_cell", + "parking_lot", "regex", "reqwest 0.12.28", "serde", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index c2db315..8877d60 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -30,6 +30,7 @@ yt-dlp = { git = "https://github.com/Guilherme-j10/yt-dlp" } serde_json = "1.0.149" dirs = "6.0.0" once_cell = "1.21.4" +parking_lot = "0.12" zip = "8.6" # Decodes yt-dlp output: on Windows it's in the ANSI codepage (cp1251), not UTF-8. encoding_rs = "0.8" diff --git a/src-tauri/src/functions/cache.rs b/src-tauri/src/functions/cache.rs new file mode 100644 index 0000000..773cac9 --- /dev/null +++ b/src-tauri/src/functions/cache.rs @@ -0,0 +1,96 @@ +//! In-memory TTL cache for video/playlist metadata, keyed by a normalized +//! URL. Re-pasting or revisiting the same URL within the TTL window skips +//! the yt-dlp subprocess spawn entirely instead of re-fetching. +//! +//! Only successful fetches are cached — a transient failure (e.g. YouTube's +//! anti-bot captcha) must not "stick" for the TTL window, so callers only +//! call `insert` on the Ok path. + +use parking_lot::Mutex; +use std::collections::HashMap; +use std::sync::LazyLock; +use std::time::{Duration, Instant}; + +const TTL: Duration = Duration::from_secs(5 * 60); + +pub struct TtlCache { + entries: Mutex>, +} + +impl TtlCache { + fn new() -> Self { + Self { + entries: Mutex::new(HashMap::new()), + } + } + + pub fn get(&self, key: &str) -> Option { + let mut map = self.entries.lock(); + match map.get(key) { + Some((inserted, value)) if inserted.elapsed() < TTL => Some(value.clone()), + Some(_) => { + map.remove(key); + None + } + None => None, + } + } + + pub fn insert(&self, key: String, value: T) { + self.entries.lock().insert(key, (Instant::now(), value)); + } +} + +/// Declares a lazily-initialized, process-wide `TtlCache<$ty>` static named +/// `$name`. +macro_rules! ttl_cache { + ($name:ident, $ty:ty) => { + pub static $name: LazyLock> = LazyLock::new(TtlCache::new); + }; +} + +ttl_cache!(VIDEO_INFO_CACHE, crate::functions::get_info::VideoInfo); +ttl_cache!(TWITCH_INFO_CACHE, crate::functions::twitch::TwitchVideoInfo); +ttl_cache!(PLAYLIST_INFO_CACHE, crate::functions::playlist::PlaylistInfo); + +/// Normalizes a URL to a stable cache key by dropping every query +/// parameter except the ones callers explicitly keep. Falls back to the +/// original (trimmed) URL if it doesn't parse — a cache miss, never a +/// crash. +fn normalize(url: &str, keep: &[&str]) -> String { + match reqwest::Url::parse(url.trim()) { + Ok(mut parsed) => { + let kept: Vec<(String, String)> = parsed + .query_pairs() + .filter(|(k, _)| keep.contains(&k.as_ref())) + .map(|(k, v)| (k.into_owned(), v.into_owned())) + .collect(); + if kept.is_empty() { + parsed.set_query(None); + } else { + let qs = kept + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join("&"); + parsed.set_query(Some(&qs)); + } + parsed.set_fragment(None); + parsed.to_string() + } + Err(_) => url.trim().to_string(), + } +} + +/// Cache key for a single video (YouTube `v=`/`youtu.be` path, or a Twitch +/// VOD/clip path) — strips tracking params (`si=`, `t=`, …) so re-pasting +/// a shared link still hits the cache. +pub fn video_key(url: &str) -> String { + normalize(url, &["v"]) +} + +/// Cache key for a playlist (YouTube `list=`, or a Twitch channel's videos +/// page, which carries no relevant query params). +pub fn playlist_key(url: &str) -> String { + normalize(url, &["list"]) +} diff --git a/src-tauri/src/functions/get_info.rs b/src-tauri/src/functions/get_info.rs index 30b2383..74437ba 100644 --- a/src-tauri/src/functions/get_info.rs +++ b/src-tauri/src/functions/get_info.rs @@ -15,7 +15,7 @@ fn client() -> &'static Client { }) } -#[derive(Serialize)] +#[derive(Serialize, Clone)] pub struct VideoInfo { pub title: String, pub author_name: String, @@ -29,6 +29,11 @@ pub struct VideoInfo { #[tauri::command] pub async fn get_youtube_info(app: tauri::AppHandle, url: String) -> Result { + let key = crate::functions::cache::video_key(&url); + if let Some(cached) = crate::functions::cache::VIDEO_INFO_CACHE.get(&key) { + return Ok(cached); + } + let oembed_url = format!("https://www.youtube.com/oembed?url={}&format=json", url); // oembed (fast title/author/thumbnail) and a single -J call (duration + @@ -38,7 +43,6 @@ pub async fn get_youtube_info(app: tauri::AppHandle, url: String) -> Result res.json::().await.ok(), _ => None, @@ -76,7 +80,7 @@ pub async fn get_youtube_info(app: tauri::AppHandle, url: String) -> Result Result Result { + let key = crate::functions::cache::video_key(&url); + if let Some(cached) = crate::functions::cache::TWITCH_INFO_CACHE.get(&key) { + return Ok(cached); + } + let json = fetch_json(&url).await?; let is_live = json["is_live"].as_bool().unwrap_or(false); @@ -97,7 +114,7 @@ pub async fn get_twitch_info(url: String) -> Result { let video_codecs = crate::functions::youtube::parse_video_codecs(&json); let audio_codecs = crate::functions::youtube::parse_audio_codecs(&json); - Ok(TwitchVideoInfo { + let info = TwitchVideoInfo { title: json["title"].as_str().unwrap_or("Twitch VOD").into(), channel: json["uploader"] .as_str() @@ -116,5 +133,11 @@ pub async fn get_twitch_info(url: String) -> Result { audio_tracks, video_codecs, audio_codecs, - }) + }; + // Live entries have no fixed duration/view-count — don't lock in + // ephemeral data for the TTL window. + if !is_live { + crate::functions::cache::TWITCH_INFO_CACHE.insert(key, info.clone()); + } + Ok(info) } diff --git a/src-tauri/src/functions/mod.rs b/src-tauri/src/functions/mod.rs index 50647e2..66c96f9 100644 --- a/src-tauri/src/functions/mod.rs +++ b/src-tauri/src/functions/mod.rs @@ -1,3 +1,4 @@ +pub mod cache; pub mod dependencies; pub mod get_info; pub mod playlist; diff --git a/src-tauri/src/functions/playlist.rs b/src-tauri/src/functions/playlist.rs index 2ab2230..dbe8305 100644 --- a/src-tauri/src/functions/playlist.rs +++ b/src-tauri/src/functions/playlist.rs @@ -70,6 +70,11 @@ fn is_twitch_videos_page(u: &str) -> bool { #[tauri::command] pub async fn get_playlist_info(url: String) -> Result { + let key = crate::functions::cache::playlist_key(&url); + if let Some(cached) = crate::functions::cache::PLAYLIST_INFO_CACHE.get(&key) { + return Ok(cached); + } + let args: Vec = vec![ "--flat-playlist".into(), "--print".into(), @@ -108,12 +113,14 @@ pub async fn get_playlist_info(url: String) -> Result { let total = if count == 0 { entries.len() as u64 } else { count }; - Ok(PlaylistInfo { + let info = PlaylistInfo { title: playlist_title, uploader, count: total, entries, - }) + }; + crate::functions::cache::PLAYLIST_INFO_CACHE.insert(key, info.clone()); + Ok(info) } fn make_entry(id: &str, title: &str, duration: &str, url: &str) -> PlaylistEntry { diff --git a/src-tauri/src/functions/youtube.rs b/src-tauri/src/functions/youtube.rs index f8cd220..f2441bd 100644 --- a/src-tauri/src/functions/youtube.rs +++ b/src-tauri/src/functions/youtube.rs @@ -589,8 +589,8 @@ async fn run_meta_json( /// Tries to get valid metadata JSON, resilient to the "not a bot" captcha. /// web_embedded first (surfaces dubs); if empty/captcha'd, the default -/// client with a few delayed retries — the captcha usually clears in a -/// couple seconds, so one failure doesn't mean the video is unavailable. +/// client with a few retries back-to-back — no artificial delay between +/// them (speed over letting the captcha "clear itself"). async fn fetch_meta_json_resilient(url: &str) -> Result { // Skip straight to the cookies that already worked earlier this // session — the video-by-video anonymous-attempt-then-cookie-retry @@ -608,28 +608,30 @@ async fn fetch_meta_json_resilient(url: &str) -> Result return Ok(j), Err(e) => e, }; - for attempt in 0..3 { - if attempt > 0 { - tokio::time::sleep(std::time::Duration::from_millis(1200)).await; - } + for _ in 0..3 { match run_meta_json(url, false, &[]).await { Ok(j) => return Ok(j), Err(e) => last_err = e, } } // Plain retries exhausted. If the cause is YouTube's anti-bot check, - // retries won't fix it regardless of browser cookies, so try them here - // exactly once (not on every attempt above — that would multiply the - // time to a final failure). + // retries won't fix it regardless of browser cookies, so try every + // detected browser's cookies here exactly once each (not on every + // attempt above — that would multiply the time to a final failure). if looks_like_bot_check(&last_err) { - if let Some(browser) = *COOKIE_BROWSER { - let cookie_args = vec!["--cookies-from-browser".to_string(), browser.to_string()]; + for browser in COOKIE_BROWSERS.iter() { + let cookie_args = vec!["--cookies-from-browser".to_string(), (*browser).to_string()]; match run_meta_json(url, false, &cookie_args).await { Ok(j) => { let _ = WORKING_COOKIE_BROWSER.set(browser); return Ok(j); } - Err(e) => last_err = e, + Err(e) => { + last_err = e; + if !looks_like_bot_check(&last_err) { + break; + } + } } } } @@ -652,8 +654,11 @@ pub async fn fetch_yt_meta(url: &str, app: Option) -> YtMeta { } }; let s = |k: &str| json[k].as_str().filter(|v| !v.is_empty()).map(String::from); + let is_live = json["is_live"].as_bool().unwrap_or(false); let mut duration = json["duration"].as_f64().map(|d| d as u64); - if duration.is_none() { + // A live stream has no fixed duration — respawning yt-dlp just to get + // "NA" back again wastes a subprocess call. + if duration.is_none() && !is_live { duration = fetch_duration(url).await; } YtMeta { @@ -795,79 +800,96 @@ fn looks_like_bot_check(stderr: &str) -> bool { s.contains("sign in to confirm") || s.contains("confirm you\u{2019}re not a bot") || s.contains("confirm you're not a bot") } -/// First browser with a profile on disk — no user involvement, no settings. -/// yt-dlp finds each browser's default profile/cookies itself; we only need -/// to confirm it's installed at all. Checked once per process run: profile -/// layout doesn't change at runtime. -fn detect_browser() -> Option<&'static str> { +/// Every profile found on disk, most-likely-to-work first — Firefox doesn't +/// depend on the OS keyring to decrypt cookies (Chromium-family browsers do, +/// via libsecret/Keychain/DPAPI, and that key store isn't always reachable, +/// e.g. no keyring daemon in the session), so it's checked first. The rest +/// follow in roughly descending popularity. yt-dlp only understands these +/// browser names for `--cookies-from-browser`: brave, chrome, chromium, +/// edge, firefox, opera, safari, vivaldi, whale. +/// Checked once per process run: profile layout doesn't change at runtime. +fn detect_browsers() -> Vec<&'static str> { + let mut found = Vec::new(); + #[cfg(target_os = "windows")] { - let local = std::env::var("LOCALAPPDATA").ok()?; - let candidates = [ - ("chrome", format!(r"{local}\Google\Chrome\User Data")), - ("edge", format!(r"{local}\Microsoft\Edge\User Data")), - ("brave", format!(r"{local}\BraveSoftware\Brave-Browser\User Data")), - ("vivaldi", format!(r"{local}\Vivaldi\User Data")), - ]; - for (name, path) in candidates { - if Path::new(&path).is_dir() { - return Some(name); - } - } if let Ok(roaming) = std::env::var("APPDATA") { if Path::new(&format!(r"{roaming}\Mozilla\Firefox\Profiles")).is_dir() { - return Some("firefox"); + found.push("firefox"); + } + if Path::new(&format!(r"{roaming}\Opera Software\Opera Stable")).is_dir() { + found.push("opera"); + } + } + if let Ok(local) = std::env::var("LOCALAPPDATA") { + let candidates = [ + ("chrome", format!(r"{local}\Google\Chrome\User Data")), + ("edge", format!(r"{local}\Microsoft\Edge\User Data")), + ("brave", format!(r"{local}\BraveSoftware\Brave-Browser\User Data")), + ("chromium", format!(r"{local}\Chromium\User Data")), + ("vivaldi", format!(r"{local}\Vivaldi\User Data")), + ("whale", format!(r"{local}\Naver\Naver Whale\User Data")), + ]; + for (name, path) in candidates { + if Path::new(&path).is_dir() { + found.push(name); + } } } - None } #[cfg(target_os = "macos")] { - let home = dirs::home_dir()?; - let candidates = [ - ("chrome", home.join("Library/Application Support/Google/Chrome")), - ("edge", home.join("Library/Application Support/Microsoft Edge")), - ("brave", home.join("Library/Application Support/BraveSoftware/Brave-Browser")), - ("vivaldi", home.join("Library/Application Support/Vivaldi")), - ]; - for (name, path) in candidates { - if path.is_dir() { - return Some(name); + if let Some(home) = dirs::home_dir() { + if home.join("Library/Application Support/Firefox/Profiles").is_dir() { + found.push("firefox"); + } + let candidates = [ + ("chrome", home.join("Library/Application Support/Google/Chrome")), + ("edge", home.join("Library/Application Support/Microsoft Edge")), + ("brave", home.join("Library/Application Support/BraveSoftware/Brave-Browser")), + ("chromium", home.join("Library/Application Support/Chromium")), + ("vivaldi", home.join("Library/Application Support/Vivaldi")), + ("opera", home.join("Library/Application Support/com.operasoftware.Opera")), + ]; + for (name, path) in candidates { + if path.is_dir() { + found.push(name); + } + } + if home.join("Library/Cookies/Cookies.binarycookies").is_file() { + found.push("safari"); } } - if home.join("Library/Application Support/Firefox/Profiles").is_dir() { - return Some("firefox"); - } - if home.join("Library/Cookies/Cookies.binarycookies").is_file() { - return Some("safari"); - } - None } #[cfg(all(unix, not(target_os = "macos")))] { - let home = dirs::home_dir()?; - let config = dirs::config_dir().unwrap_or_else(|| home.join(".config")); - let candidates = [ - ("chrome", config.join("google-chrome")), - ("chromium", config.join("chromium")), - ("brave", config.join("BraveSoftware/Brave-Browser")), - ("vivaldi", config.join("vivaldi")), - ("opera", config.join("opera")), - ]; - for (name, path) in candidates { - if path.is_dir() { - return Some(name); + if let Some(home) = dirs::home_dir() { + if home.join(".mozilla/firefox/profiles.ini").is_file() { + found.push("firefox"); + } + let config = dirs::config_dir().unwrap_or_else(|| home.join(".config")); + let candidates = [ + ("chrome", config.join("google-chrome")), + ("chromium", config.join("chromium")), + ("edge", config.join("microsoft-edge")), + ("brave", config.join("BraveSoftware/Brave-Browser")), + ("vivaldi", config.join("vivaldi")), + ("opera", config.join("opera")), + ("whale", config.join("naver-whale")), + ]; + for (name, path) in candidates { + if path.is_dir() { + found.push(name); + } } } - if home.join(".mozilla/firefox/profiles.ini").is_file() { - return Some("firefox"); - } - None } + + found } -static COOKIE_BROWSER: std::sync::LazyLock> = - std::sync::LazyLock::new(detect_browser); +static COOKIE_BROWSERS: std::sync::LazyLock> = + std::sync::LazyLock::new(detect_browsers); fn has_cookie_flag(args: &[String]) -> bool { args.iter().any(|a| a == "--cookies-from-browser") @@ -944,23 +966,30 @@ pub async fn run_ytdlp_status( if status.success() || has_cookie_flag(&args) || is_cancelled() { return Ok(status); } - let Some(browser) = *COOKIE_BROWSER else { - return Ok(status); - }; if !looks_like_bot_check(&err_lines.join("\n")) { return Ok(status); } - emit_log( - &app, - &format!("[flowbit] YouTube requires \"I'm not a bot\" confirmation — trying cookies from {browser}…"), - ); - let mut retry_args = args; - retry_args.push("--cookies-from-browser".into()); - retry_args.push(browser.into()); - let (retry_status, _) = spawn_ytdlp_status(&retry_args, &error_format, app).await?; - status = retry_status; - if status.success() { - let _ = WORKING_COOKIE_BROWSER.set(browser); + // Try every detected browser's cookies in turn — a bot-check failure + // from one (e.g. Chrome's keyring-locked cookie DB) doesn't mean + // another (e.g. Firefox) will fail the same way. + for browser in COOKIE_BROWSERS.iter() { + emit_log( + &app, + &format!("[flowbit] YouTube requires \"I'm not a bot\" confirmation — trying cookies from {browser}…"), + ); + let mut retry_args = args.clone(); + retry_args.push("--cookies-from-browser".into()); + retry_args.push((*browser).into()); + let (retry_status, retry_err_lines) = + spawn_ytdlp_status(&retry_args, &error_format, app.clone()).await?; + status = retry_status; + if status.success() { + let _ = WORKING_COOKIE_BROWSER.set(browser); + return Ok(status); + } + if !looks_like_bot_check(&retry_err_lines.join("\n")) { + return Ok(status); + } } Ok(status) } @@ -1027,26 +1056,30 @@ pub async fn run_ytdlp_output( if output.status.success() || has_cookie_flag(&args) || is_cancelled() { return Ok(output); } - let Some(browser) = *COOKIE_BROWSER else { - return Ok(output); - }; if !looks_like_bot_check(&decode_output(&output.stderr)) { return Ok(output); } - emit_log( - &app, - &format!("[flowbit] YouTube requires \"I'm not a bot\" confirmation — trying cookies from {browser}…"), - ); - let mut retry_args = args; - retry_args.push("--cookies-from-browser".into()); - retry_args.push(browser.into()); - output = spawn_ytdlp_output(&retry_args, &error_format, app).await?; - if output.status.success() { - let _ = WORKING_COOKIE_BROWSER.set(browser); + for browser in COOKIE_BROWSERS.iter() { + emit_log( + &app, + &format!("[flowbit] YouTube requires \"I'm not a bot\" confirmation — trying cookies from {browser}…"), + ); + let mut retry_args = args.clone(); + retry_args.push("--cookies-from-browser".into()); + retry_args.push((*browser).into()); + output = spawn_ytdlp_output(&retry_args, &error_format, app.clone()).await?; + if output.status.success() { + let _ = WORKING_COOKIE_BROWSER.set(browser); + return Ok(output); + } + if !looks_like_bot_check(&decode_output(&output.stderr)) { + return Ok(output); + } } Ok(output) } + #[tauri::command] pub async fn download_video( app: AppHandle, diff --git a/src-tauri/tests/integration_tests.rs b/src-tauri/tests/integration_tests.rs index 0b81ea4..3e4cdf9 100644 --- a/src-tauri/tests/integration_tests.rs +++ b/src-tauri/tests/integration_tests.rs @@ -1,3 +1,4 @@ +use flowbit_lib::functions::cache::{playlist_key, video_key}; use flowbit_lib::functions::playlist::is_playlist_url; use flowbit_lib::functions::valid::{is_twitch_url, is_youtube_url, validate_time_range}; use flowbit_lib::functions::youtube::{ @@ -5,6 +6,56 @@ use flowbit_lib::functions::youtube::{ Quality, }; +// ═══════════════════════════════════════════════════════════════════════════════ +// cache key normalization +// ═══════════════════════════════════════════════════════════════════════════════ + +#[test] +fn video_key_ignores_tracking_params() { + // Same video, shared-link tracking param differs — must hit the same key. + assert_eq!( + video_key("https://www.youtube.com/watch?v=abc123&si=xyz"), + video_key("https://www.youtube.com/watch?v=abc123&si=other") + ); +} + +#[test] +fn video_key_ignores_timestamp_and_list_params() { + assert_eq!( + video_key("https://www.youtube.com/watch?v=abc123&t=42s&list=PLfoo"), + video_key("https://www.youtube.com/watch?v=abc123") + ); +} + +#[test] +fn video_key_distinguishes_different_videos() { + assert_ne!( + video_key("https://www.youtube.com/watch?v=abc123"), + video_key("https://www.youtube.com/watch?v=def456") + ); +} + +#[test] +fn playlist_key_ignores_index_param_keeps_list() { + assert_eq!( + playlist_key("https://www.youtube.com/playlist?list=PLfoo&index=3"), + playlist_key("https://www.youtube.com/playlist?list=PLfoo") + ); +} + +#[test] +fn playlist_key_distinguishes_different_playlists() { + assert_ne!( + playlist_key("https://www.youtube.com/playlist?list=PLfoo"), + playlist_key("https://www.youtube.com/playlist?list=PLbar") + ); +} + +#[test] +fn video_key_falls_back_to_trimmed_url_when_unparseable() { + assert_eq!(video_key(" not a url "), "not a url".to_string()); +} + // ═══════════════════════════════════════════════════════════════════════════════ // is_playlist_url // ═══════════════════════════════════════════════════════════════════════════════