diff --git a/Cargo.lock b/Cargo.lock index 50f10ff..2287c26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -69,6 +69,21 @@ dependencies = [ "equator", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -706,6 +721,27 @@ dependencies = [ "piper", ] +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "1.13.1" @@ -3911,6 +3947,8 @@ dependencies = [ name = "opensheet" version = "0.2.2" dependencies = [ + "brotli", + "flate2", "futures", "gpui", "image", @@ -3923,6 +3961,8 @@ dependencies = [ "raw-window-handle", "sys-locale", "unicode-segmentation", + "ureq", + "url", "winresource", ] @@ -5316,6 +5356,7 @@ version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -6673,6 +6714,36 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" +dependencies = [ + "base64 0.23.1", + "brotli-decompressor", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" +dependencies = [ + "base64 0.23.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -6719,6 +6790,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -7049,6 +7126,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "weezl" version = "0.1.12" diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index bfef238..524b75c 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -23,6 +23,14 @@ unicode-segmentation = "1" # macOS, which honours the language chosen for OpenSheet alone; the user's # interface languages on Windows; LANGUAGE / LC_* / LANG elsewhere). sys-locale = "0.3" +# Blocking HTTP/1.1 for web queries. Synchronous, so `fetch` can run on +# GPUI's background executor without a second async runtime, and rustls +# (already linked for GPUI) rather than GPUI's async `zed-reqwest` fork, +# which does not enable gzip, brotli or deflate. `flate2` inflates the +# deflate coding ureq itself does not; `url` joins a relative redirect. +ureq = { version = "3", default-features = false, features = ["rustls", "gzip", "brotli"] } +flate2 = "1" +url = "2" [target.'cfg(target_os = "macos")'.dependencies] # The Objective-C runtime GPUI already links: for the system spelling @@ -45,6 +53,8 @@ winresource = { version = "=0.1.31", default-features = false } [dev-dependencies] # The headless window of GPUI, to drive the interface in tests. gpui = { version = "0.2.2", features = ["test-support"] } +# Encodes brotli in the fetch tests. The application only decompresses it. +brotli = "8" # Linux packages, made by the release workflow with `cargo deb` and `cargo generate-rpm`. [package.metadata.deb] diff --git a/crates/app/src/features/data_tools/connections.rs b/crates/app/src/features/data_tools/connections.rs index a61c6b7..b1f3024 100644 --- a/crates/app/src/features/data_tools/connections.rs +++ b/crates/app/src/features/data_tools/connections.rs @@ -159,7 +159,9 @@ pub(super) fn set_path(app: &mut ExcelApp, query: &QueryRef, path: &std::path::P Some(_) => sources::file_url(path), None => path.to_string_lossy().into_owned(), }; - app.wb.update_import(sheet, index, |import| import.path = address); + app.wb.update_import(sheet, index, |import| import.path = address.clone()); + // The user picked this file. A web query may read it again this session. + import::remember_chosen_file(app, &address); app.commit(t("Change File Path", "Modifier le chemin d’accès au fichier"), cx); } @@ -356,15 +358,22 @@ impl PropertiesDialog { /// Give a query its edited name and properties, in one undo step. fn apply(app: &mut ExcelApp, query: &QueryRef, edited: TextImport, cx: &mut Cx) { let Some((sheet, index)) = query.find(app) else { return }; + let previous_minutes = app.wb.sheets[sheet].text_imports[index].props.refresh_minutes; if app.wb.sheets[sheet].text_imports[index] == edited { return; } + let minutes = edited.props.refresh_minutes; app.wb.update_import(sheet, index, |import| { // Where the data is may have moved while the dialog was open. let range = import.range; *import = edited; import.range = range; }); + // Turning "Refresh every" on here is the user asking for the clock. + // A workbook that arrived with an interval still has to be agreed to. + if minutes > 0 && minutes != previous_minutes { + import::consent_to_timer(app); + } app.commit(t("Connection Properties", "Propriétés de connexion"), cx); } diff --git a/crates/app/src/features/data_tools/import.rs b/crates/app/src/features/data_tools/import.rs index bed2ac5..0c28127 100644 --- a/crates/app/src/features/data_tools/import.rs +++ b/crates/app/src/features/data_tools/import.rs @@ -22,7 +22,7 @@ use opensheet_engine::Rect; use opensheet_engine::external::{SourceTable, WebQuery, pick}; use opensheet_engine::protect::Action; use opensheet_engine::text_columns::{TextColumnsError, TextImport}; -use std::path::Path; +use std::path::{Path, PathBuf}; fn import_label() -> &'static str { t("Import Data", "Importer des données") @@ -60,6 +60,10 @@ pub(super) fn fetch_message(error: FetchError, address: &str) -> String { FetchError::Status(_) => t("Unable to open |0. The Internet site reports an error. (HTTP |1)", "Impossible d'ouvrir |0. Le site Internet signale une erreur. (HTTP |1)"), FetchError::Timeout => t("Unable to open |0. The server did not answer in time.", "Impossible d'ouvrir |0. Le serveur n'a pas répondu à temps."), FetchError::TooLarge => t("Unable to open |0. The data is too large.", "Impossible d'ouvrir |0. Les données sont trop volumineuses."), + FetchError::LocalFile => t( + "Unable to open |0. A web query cannot open a local file. Use an http or https address.", + "Impossible d'ouvrir |0. Une requête sur le web ne peut pas ouvrir un fichier local. Indiquez une adresse http ou https.", + ), }; let code = match error { FetchError::Status(code) => code.to_string(), @@ -75,8 +79,8 @@ fn no_data() -> &'static str { ) } -/// Fetch an address and hand what it gave to `then`: at once for what -/// is on this machine, later and off the window thread for the web. +/// Fetch an address and hand what it gave to `then`: at once for a file +/// the caller named, later and off the window thread for the web. pub(super) fn with_fetched( app: &mut ExcelApp, address: String, @@ -85,7 +89,16 @@ pub(super) fn with_fetched( then: impl FnOnce(&mut ExcelApp, Result, &mut Window, &mut Cx) + 'static, ) { if !sources::is_remote(&address) { - let fetched = sources::fetch(&address); + // A file the caller just named (the file dialog, or a test). A + // web query stored in a workbook never arrives here. + let fetched = if sources::local_path(&address).is_some() { + remember_chosen_file(app, &address); + sources::read_local(&address) + } else if sources::names_a_file(&address) { + Err(FetchError::LocalFile) + } else { + Err(FetchError::BadAddress) + }; then(app, fetched, window, cx); return; } @@ -451,7 +464,15 @@ pub(super) fn refresh_queries(app: &mut ExcelApp, queries: Vec<(usize, usize)>, remote.extend(QueryRef::of(app, sheet, index).map(|query| (query, import.path))); continue; } - let outcome = sources::fetch(&import.path).map_err(Failed::Fetch).and_then(|fetched| apply(app, sheet, index, &fetched, window)); + // A text import re-reads its file. A web query may read a local + // file only when the user picked that file in this session; a + // path that came in with the workbook is refused. + let loaded = if import.web.is_some() && !user_chose_file(app, &import.path) { + Err(if sources::names_a_file(&import.path) { FetchError::LocalFile } else { FetchError::BadAddress }) + } else { + sources::read_local(&import.path) + }; + let outcome = loaded.map_err(Failed::Fetch).and_then(|fetched| apply(app, sheet, index, &fetched, window)); match outcome { Ok(rect) if sheet == app.sheet => written.push(rect), Ok(_) => {} @@ -562,53 +583,111 @@ pub(super) fn web_query(address: String, web: WebQuery) -> TextImport { // ------------------------------------------------- refreshing by itself /// What the queries of the open workbook need watched: the question -/// asked once when a file wants refreshing as it opens, and the clock of +/// asked once when a file wants to refresh on its own, and the clock of /// "Refresh every n minutes". #[derive(Default)] pub(super) struct Watch { - /// Whether the queries that refresh on opening were dealt with. + /// Whether this workbook's own schedule was already considered. opened: bool, + /// `Some(true)` after the user agreed, or turned the interval on + /// themselves. `Some(false)` when they declined. + consented: Option, + /// Local files the user picked in this session. A web query may read + /// one of these; a path that arrived only inside the workbook may not. + chosen: Vec, /// Minutes since each query that refreshes by itself last did. waited: Vec<(QueryRef, u32)>, clock: Option>, } +#[cfg(test)] +impl Watch { + pub(super) fn timer_running(&self) -> bool { + self.clock.is_some() + } +} + +/// The user picked `address` (the file dialog, or From JSON / XML / +/// Workbook). Remember it for this workbook only. +pub(super) fn remember_chosen_file(app: &mut ExcelApp, address: &str) { + let Some(path) = sources::local_path(address) else { return }; + let chosen = &mut app.ext_mut::().chosen; + if !chosen.contains(&path) { + chosen.push(path); + } +} + +fn user_chose_file(app: &ExcelApp, address: &str) -> bool { + let Some(path) = sources::local_path(address) else { return false }; + app.ext::().is_some_and(|watch| watch.chosen.iter().any(|known| known == &path)) +} + +/// Connection Properties just turned "Refresh every" on, which is the +/// user's own request: the next draw may start the clock. +pub(super) fn consent_to_timer(app: &mut ExcelApp) { + app.ext_mut::().consented = Some(true); +} + +fn refreshes_on_a_timer(app: &ExcelApp) -> bool { + app.wb.sheets.iter().any(|sheet| sheet.text_imports.iter().any(|import| import.props.refresh_minutes > 0)) +} + +fn wants_a_refresh_on_open(app: &ExcelApp) -> bool { + app.wb.sheets.iter().any(|sheet| sheet.text_imports.iter().any(|import| import.props.refresh_on_open)) +} + /// Called whenever the window is drawn: cheap unless a query asks for -/// something. Nothing is fetched here without the user having said so — -/// when opening a file, Excel's question comes first. +/// something. A workbook that wants to refresh on opening, or on its own +/// interval, is asked once. The clock starts only after an agreement; +/// declining leaves it stopped, and Refresh still works by hand. pub(super) fn watch(app: &mut ExcelApp, window: &mut Window, cx: &mut Cx) { let first = app.ext::().is_none_or(|watch| !watch.opened); - let timed = app.wb.sheets.iter().any(|sheet| sheet.text_imports.iter().any(|import| import.props.refresh_minutes > 0)); - if !first && (!timed || app.ext::().is_some_and(|watch| watch.clock.is_some())) { + let timed = refreshes_on_a_timer(app); + if !first { + // The user has since turned the interval on in Connection Properties. + if timed && app.ext::().and_then(|watch| watch.consented) == Some(true) { + start_clock(app, window, cx); + } return; } app.ext_mut::().opened = true; - if first { - let on_open: Vec<(usize, usize)> = - all_queries(app).into_iter().filter(|(sheet, index)| app.wb.sheets[*sheet].text_imports[*index].props.refresh_on_open).collect(); - if !on_open.is_empty() { - // Not while the window is being drawn. - cx.spawn_in(window, async move |this, cx| { - this.update_in(cx, |app, _, cx| ask_to_refresh(app, cx)).ok(); - }) - .detach(); - } + if !wants_a_refresh_on_open(app) && !timed { + return; } - if timed { - let clock = cx.spawn_in(window, async move |this, cx| { - loop { - cx.background_executor().timer(std::time::Duration::from_secs(60)).await; - if this.update_in(cx, minute_passed).is_err() { - break; - } - } - }); - app.ext_mut::().clock = Some(clock); + // Already agreed in this session (they edited the properties before + // the first draw): start, and do not ask a second time. + if app.ext::().and_then(|watch| watch.consented) == Some(true) { + start_clock(app, window, cx); + return; } + // Not while the window is being drawn. + cx.spawn_in(window, async move |this, cx| { + this.update_in(cx, |app, _, cx| ask_to_refresh(app, cx)).ok(); + }) + .detach(); } -/// A file that refreshes its queries on opening: Excel asks first, since -/// the addresses are the file's and not the user's. +fn start_clock(app: &mut ExcelApp, window: &mut Window, cx: &mut Cx) { + if !refreshes_on_a_timer(app) || app.ext::().is_some_and(|watch| watch.clock.is_some()) { + return; + } + if app.ext::().and_then(|watch| watch.consented) != Some(true) { + return; + } + let clock = cx.spawn_in(window, async move |this, cx| { + loop { + cx.background_executor().timer(std::time::Duration::from_secs(60)).await; + if this.update_in(cx, minute_passed).is_err() { + break; + } + } + }); + app.ext_mut::().clock = Some(clock); +} + +/// A file that wants to refresh its queries by itself: Excel asks first, +/// since the addresses are the file's and not the user's. One question +/// covers both "when opening the file" and "every n minutes". fn ask_to_refresh(app: &mut ExcelApp, cx: &mut Cx) { let message = format!( "{}\n\n{}", @@ -621,17 +700,25 @@ fn ask_to_refresh(app: &mut ExcelApp, cx: &mut Cx) { let alert = Alert::new("OpenSheet", message) .warning() .button(t("OK", "OK"), |app, window, cx| { + app.ext_mut::().consented = Some(true); let queries = all_queries(app).into_iter().filter(|(sheet, index)| app.wb.sheets[*sheet].text_imports[*index].props.refresh_on_open).collect(); if let Some(task) = refresh_queries(app, queries, t("Refresh", "Actualiser"), window, cx) { task.detach(); } + start_clock(app, window, cx); }) - .close_button(t("Cancel", "Annuler")); + .button(t("Cancel", "Annuler"), |app, _, _| { + app.ext_mut::().consented = Some(false); + }); app.alert(alert, cx); } /// A minute went by: the queries whose time has come are refreshed. +/// Nothing happens until the user has agreed to this workbook's schedule. pub(super) fn minute_passed(app: &mut ExcelApp, window: &mut Window, cx: &mut Cx) { + if app.ext::().and_then(|watch| watch.consented) != Some(true) { + return; + } let mut due = Vec::new(); let mut waited = Vec::new(); for (sheet, index) in all_queries(app) { diff --git a/crates/app/src/features/data_tools/navigator.rs b/crates/app/src/features/data_tools/navigator.rs index 452e7ef..ae0732d 100644 --- a/crates/app/src/features/data_tools/navigator.rs +++ b/crates/app/src/features/data_tools/navigator.rs @@ -55,6 +55,14 @@ impl WebDialog { // The dialog goes away first: what the address gives, or why it // gave nothing, is shown over the sheet. self.modal.close(window, cx); + // From Web is the web fetch. A file URL or a path is not fetched; + // local files are imported from the file dialog. + if !sources::is_remote(&address) { + let error = if sources::names_a_file(&address) { sources::FetchError::LocalFile } else { sources::FetchError::BadAddress }; + let message = import::fetch_message(error, &address); + self.modal.update_app(cx, |app, cx| warn(app, message, cx)); + return; + } self.modal.update_app(cx, |app, cx| import::open_address(app, address, replaces, window, cx)); } } diff --git a/crates/app/src/features/data_tools/sources.rs b/crates/app/src/features/data_tools/sources.rs index f88943e..2657f25 100644 --- a/crates/app/src/features/data_tools/sources.rs +++ b/crates/app/src/features/data_tools/sources.rs @@ -1,24 +1,41 @@ //! Where external data comes from: fetching the bytes of an address and //! reading them as the tables the Navigator offers. //! -//! An address is a path, a `file://` URL or an `http(s)://` URL. Files -//! are read directly. The web is reached through the system's -//! `/usr/bin/curl`, run as a child process: it costs no dependency and -//! no build time, and it uses the certificates and the proxy settings of -//! macOS, which a Rust HTTP client would have to be taught. Fetching -//! blocks: callers run [`fetch`] off the window thread for anything -//! remote, and only when the user asked (an address typed, a refresh). +//! A web address is an `http://` or `https://` URL. It is fetched in +//! process with [`ureq`], a small blocking client, so nothing is +//! shelled out. TLS is rustls with the Mozilla root set, which builds +//! the same way on macOS, Windows and Linux and is the stack GPUI +//! already links. +//! GPUI's own HTTP client is an async fork of reqwest and does not +//! turn on compression, so it is not reused here. The request asks for +//! gzip, deflate and brotli, and the body is decompressed before it is +//! returned. An `HTTP_PROXY`, `HTTPS_PROXY` or `ALL_PROXY` variable is +//! honoured. Fetching blocks: callers run [`fetch`] off the window +//! thread, and only after the user has asked (an address typed, a +//! refresh) or agreed to a schedule the workbook itself requested. +//! +//! Only `http` and `https` are requested, and a redirect is followed +//! only when it stays on those two schemes. A `file://` URL or a bare +//! path is not a web address, and [`fetch`] refuses it. Local files +//! are read by [`read_local`] when the user imports one — text, JSON, +//! XML or a workbook — or refreshes a text import. A web query that +//! arrived in a workbook cannot point this reader at a file on the +//! machine. use opensheet_engine::external::{Datum, QueryItem, SourceFormat, SourceTable, html_tables, json_tables, sniff, xml_tables}; use opensheet_engine::{Locale, Value, Workbook}; use std::io::Read; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; -/// The most a source may weigh. +/// The most a source may weigh, decompressed. const SIZE_LIMIT: u64 = 50 * 1024 * 1024; -/// Seconds a server has to answer in full. -const TIMEOUT: u32 = 30; +/// Seconds a server has to answer in full, redirects included. +const TIMEOUT: u64 = 30; +/// How long a connection may take to open. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// Redirects followed before giving up. The next one is refused. +const MAX_REDIRECTS: u32 = 10; /// What an address gave. pub(super) struct Fetched { @@ -40,6 +57,8 @@ pub(super) enum FetchError { Status(u32), Timeout, TooLarge, + /// A `file://` URL or a path, which a web query is not allowed to open. + LocalFile, } /// Whether an address is on the web rather than on this machine. @@ -48,6 +67,13 @@ pub(super) fn is_remote(address: &str) -> bool { lower.starts_with("http://") || lower.starts_with("https://") } +/// Whether an address names a file on this machine: a `file:` URL, in +/// any case, or a path that carries no scheme. +pub(super) fn names_a_file(address: &str) -> bool { + let address = address.trim(); + local_path(address).is_some() || address.to_ascii_lowercase().starts_with("file:") +} + /// The file an address names, when it names one: a `file://` URL or a /// plain path. pub(super) fn local_path(address: &str) -> Option { @@ -99,67 +125,144 @@ pub(super) fn normalize(typed: &str) -> String { } } -/// The program that fetches web addresses. -fn curl() -> PathBuf { - PathBuf::from("/usr/bin/curl") +/// Read a file the user named: a `file://` URL or a plain path. +pub(super) fn read_local(address: &str) -> Result { + let Some(path) = local_path(address) else { + return Err(FetchError::BadAddress); + }; + let size = std::fs::metadata(&path).map_err(|_| FetchError::Missing)?.len(); + if size > SIZE_LIMIT { + return Err(FetchError::TooLarge); + } + let bytes = std::fs::read(&path).map_err(|_| FetchError::Missing)?; + Ok(Fetched { bytes, content_type: None }) +} + +/// Whether this text may be handed to the HTTP client: `http` or +/// `https`, and no space or control character that would change the +/// request line. +fn allowed_web_address(address: &str) -> bool { + is_remote(address) && !address.chars().any(|ch| ch.is_control() || ch == ' ') +} + +/// The client for one fetch. Redirects are followed by [`fetch`] itself, +/// so a `Location` that leaves http(s) is never requested. The default +/// config already reads a proxy from the environment. +fn web_agent() -> ureq::Agent { + ureq::Agent::config_builder() + .timeout_connect(Some(CONNECT_TIMEOUT)) + .timeout_global(Some(Duration::from_secs(TIMEOUT))) + .max_redirects(0) + .http_status_as_error(false) + .build() + .into() } -/// Get what an address holds. Blocks until it is there, at most -/// [`TIMEOUT`] seconds for the web. +/// Get what a web address holds. Blocks until it is there, at most +/// [`TIMEOUT`] seconds. A file URL or a path is refused: those are not +/// fetched, they are read with [`read_local`] when the user asked for a file. pub(super) fn fetch(address: &str) -> Result { - if let Some(path) = local_path(address) { - let size = std::fs::metadata(&path).map_err(|_| FetchError::Missing)?.len(); - if size > SIZE_LIMIT { - return Err(FetchError::TooLarge); - } - let bytes = std::fs::read(&path).map_err(|_| FetchError::Missing)?; - return Ok(Fetched { bytes, content_type: None }); + if names_a_file(address) { + return Err(FetchError::LocalFile); } - if !is_remote(address) || address.chars().any(|ch| ch.is_control() || ch == ' ') { + if !allowed_web_address(address) { return Err(FetchError::BadAddress); } - let mut child = Command::new(curl()) - .args(["--silent", "--show-error", "--location", "--max-redirs", "10", "--compressed"]) - .args(["--proto", "=http,https", "--proto-redir", "=http,https"]) - .args(["--connect-timeout", "10", "--max-time", &TIMEOUT.to_string(), "--max-filesize", &SIZE_LIMIT.to_string()]) - // The status and the type of the answer, apart from its body. - .args(["--write-out", "%{stderr}\n%{http_code}\t%{content_type}", "--", address.trim()]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|_| FetchError::NoConnection)?; - // A server that does not say how much it sends is stopped at the - // limit all the same. - let mut bytes = Vec::new(); - let read = child.stdout.take().map(|out| out.take(SIZE_LIMIT + 1).read_to_end(&mut bytes)); - if bytes.len() as u64 > SIZE_LIMIT { - child.kill().ok(); - child.wait().ok(); - return Err(FetchError::TooLarge); - } - let mut report = String::new(); - if let Some(mut err) = child.stderr.take() { - err.read_to_string(&mut report).ok(); + let agent = web_agent(); + let deadline = Instant::now() + Duration::from_secs(TIMEOUT); + let mut address = address.trim().to_string(); + for hop in 0..=MAX_REDIRECTS { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(FetchError::Timeout); + } + let response = agent + .get(&address) + .header("Accept-Encoding", "gzip, deflate, br") + .config() + .timeout_global(Some(remaining)) + .build() + .call() + .map_err(transport_error)?; + let status = response.status().as_u16(); + if matches!(status, 301 | 302 | 303 | 307 | 308) { + if hop == MAX_REDIRECTS { + // The original request plus ten redirects have already run. + return Err(FetchError::NoConnection); + } + let location = response.headers().get("location").and_then(|value| value.to_str().ok()).ok_or(FetchError::BadAddress)?; + address = redirect_target(&address, location)?; + continue; + } + if status >= 400 { + return Err(FetchError::Status(u32::from(status))); + } + return read_response(response); } - let status = child.wait().map_err(|_| FetchError::NoConnection)?; - if !matches!(read, Some(Ok(_))) { - return Err(FetchError::NoConnection); + Err(FetchError::NoConnection) +} + +/// The address a `Location` names, refused unless it is still http(s). +fn redirect_target(current: &str, location: &str) -> Result { + let base = url::Url::parse(current).map_err(|_| FetchError::BadAddress)?; + let next = base.join(location.trim()).map_err(|_| FetchError::BadAddress)?; + let address = next.as_str(); + if !allowed_web_address(address) { + return Err(FetchError::BadAddress); } - match status.code() { - Some(0) => {} - Some(1 | 3) => return Err(FetchError::BadAddress), - Some(28) => return Err(FetchError::Timeout), - Some(63) => return Err(FetchError::TooLarge), - _ => return Err(FetchError::NoConnection), + Ok(address.to_string()) +} + +fn transport_error(error: ureq::Error) -> FetchError { + match error { + ureq::Error::Timeout(_) => FetchError::Timeout, + ureq::Error::StatusCode(code) => FetchError::Status(u32::from(code)), + ureq::Error::BadUri(_) | ureq::Error::RedirectFailed | ureq::Error::RequireHttpsOnly(_) => FetchError::BadAddress, + ureq::Error::BodyExceedsLimit(_) => FetchError::TooLarge, + _ => FetchError::NoConnection, } - let (code, kind) = report.lines().last().and_then(|line| line.split_once('\t')).unwrap_or(("", "")); - match code.trim().parse::() { - Ok(code) if code >= 400 => Err(FetchError::Status(code)), - _ => Ok(Fetched { - bytes, - content_type: Some(kind.trim().to_string()).filter(|kind| !kind.is_empty()), - }), +} + +/// The body, decompressed, and the type the server announced. +fn read_response(mut response: ureq::http::Response) -> Result { + let content_type = response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + // gzip and brotli are already inflated by the client, which then + // drops this header. deflate is not, so it is still named here. + let deflate = response + .headers() + .get("content-encoding") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.split(',').any(|part| part.trim().eq_ignore_ascii_case("deflate"))); + let bytes = read_limited(response.body_mut().as_reader())?; + let bytes = if deflate { inflate(&bytes)? } else { bytes }; + Ok(Fetched { bytes, content_type }) +} + +/// Read at most [`SIZE_LIMIT`] bytes. One more means the source is too big. +fn read_limited(mut reader: impl Read) -> Result, FetchError> { + let mut bytes = Vec::new(); + reader.by_ref().take(SIZE_LIMIT + 1).read_to_end(&mut bytes).map_err(read_error)?; + if bytes.len() as u64 > SIZE_LIMIT { Err(FetchError::TooLarge) } else { Ok(bytes) } +} + +/// `Content-Encoding: deflate` is a zlib stream, except when a server +/// sends the raw form. Try the zlib wrapper first. +fn inflate(bytes: &[u8]) -> Result, FetchError> { + let zlib = read_limited(flate2::read::ZlibDecoder::new(bytes)); + if zlib.is_ok() { zlib } else { read_limited(flate2::read::DeflateDecoder::new(bytes)) } +} + +fn read_error(error: std::io::Error) -> FetchError { + let message = error.to_string().to_ascii_lowercase(); + if error.kind() == std::io::ErrorKind::TimedOut || message.contains("timeout") || message.contains("timed out") { + FetchError::Timeout + } else { + FetchError::NoConnection } } @@ -264,6 +367,8 @@ pub(super) mod test_server { pub body: Vec, /// Where a redirection (301, 302) points. pub location: Option, + /// A `Content-Encoding` to announce, when the body is already compressed. + pub encoding: Option<&'static str>, } pub fn page(content_type: &'static str, body: &str) -> Answer { @@ -272,6 +377,7 @@ pub(super) mod test_server { content_type, body: body.as_bytes().to_vec(), location: None, + encoding: None, } } @@ -304,16 +410,35 @@ pub(super) mod test_server { } let request = String::from_utf8_lossy(&request).into_owned(); let path = request.split_whitespace().nth(1).unwrap_or("/").to_string(); - let answer = answers.lock().unwrap().get(&path).cloned().unwrap_or(Answer { - status: 404, - content_type: "text/html", - body: b"Not Found".to_vec(), - location: None, + // The encodings the client offered, so a test can read them back. + let offered = request.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("accept-encoding").then(|| value.trim().to_string()) }); + let answer = if path == "/encoding" { + Answer { + status: 200, + content_type: "text/plain", + body: offered.unwrap_or_default().into_bytes(), + location: None, + encoding: None, + } + } else { + answers.lock().unwrap().get(&path).cloned().unwrap_or(Answer { + status: 404, + content_type: "text/html", + body: b"Not Found".to_vec(), + location: None, + encoding: None, + }) + }; let mut head = format!("HTTP/1.1 {} X\r\nContent-Type: {}\r\nContent-Length: {}\r\nConnection: close\r\n", answer.status, answer.content_type, answer.body.len()); if let Some(location) = &answer.location { head.push_str(&format!("Location: {location}\r\n")); } + if let Some(encoding) = answer.encoding { + head.push_str(&format!("Content-Encoding: {encoding}\r\n")); + } head.push_str("\r\n"); stream.write_all(head.as_bytes()).ok(); stream.write_all(&answer.body).ok(); @@ -366,6 +491,7 @@ mod tests { content_type: "text/html", body: Vec::new(), location: Some("/page".into()), + encoding: None, }, ); server.set( @@ -375,6 +501,7 @@ mod tests { content_type: "text/plain", body: b"no".to_vec(), location: None, + encoding: None, }, ); let got = fetch(&server.url("/page")).ok().expect("the page"); @@ -391,7 +518,120 @@ mod tests { // Only the web and files; nothing that could be read as an option. assert_eq!(fetch("ftp://127.0.0.1/x").err(), Some(FetchError::BadAddress)); assert_eq!(fetch("http://127.0.0.1/a b").err(), Some(FetchError::BadAddress)); - assert_eq!(fetch("/nowhere/at/all.csv").err(), Some(FetchError::Missing)); + // https, and nothing is listening on this port. + assert_eq!(fetch("https://127.0.0.1:1/nope").err(), Some(FetchError::NoConnection)); + // A redirection may not leave http(s). file:// is not requested. + server.set( + "/to-file", + Answer { + status: 302, + content_type: "text/plain", + body: Vec::new(), + location: Some("file:///etc/passwd".into()), + encoding: None, + }, + ); + assert_eq!(fetch(&server.url("/to-file")).err(), Some(FetchError::BadAddress)); + server.set( + "/to-ftp", + Answer { + status: 302, + content_type: "text/plain", + body: Vec::new(), + location: Some("ftp://127.0.0.1/x".into()), + encoding: None, + }, + ); + assert_eq!(fetch(&server.url("/to-ftp")).err(), Some(FetchError::BadAddress)); + // Eleven hops is one past the limit. + server.set( + "/loop", + Answer { + status: 302, + content_type: "text/plain", + body: Vec::new(), + location: Some("/loop".into()), + encoding: None, + }, + ); + assert_eq!(fetch(&server.url("/loop")).err(), Some(FetchError::NoConnection)); + } + + #[test] + fn fetch_decompresses_gzip_deflate_and_brotli() { + let plain = b"
ok
"; + let server = Server::start(); + server.set("/gz", compressed("gzip", &gzip(plain))); + server.set("/df", compressed("deflate", &zlib_wrap(plain))); + server.set("/raw", compressed("deflate", &deflate_raw(plain))); + server.set("/br", compressed("br", &brotli_compress(plain))); + assert_eq!(fetch(&server.url("/gz")).unwrap().bytes, plain); + assert_eq!(fetch(&server.url("/df")).unwrap().bytes, plain); + assert_eq!(fetch(&server.url("/raw")).unwrap().bytes, plain); + assert_eq!(fetch(&server.url("/br")).unwrap().bytes, plain); + let offered = fetch(&server.url("/encoding")).unwrap(); + let offered = String::from_utf8(offered.bytes).unwrap(); + assert!(offered.split(',').any(|part| part.trim() == "gzip"), "{offered}"); + assert!(offered.split(',').any(|part| part.trim() == "deflate"), "{offered}"); + assert!(offered.split(',').any(|part| part.trim() == "br"), "{offered}"); + } + + fn compressed(encoding: &'static str, body: &[u8]) -> Answer { + Answer { + status: 200, + content_type: "text/html", + body: body.to_vec(), + location: None, + encoding: Some(encoding), + } + } + + fn gzip(plain: &[u8]) -> Vec { + use std::io::Write; + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(plain).unwrap(); + encoder.finish().unwrap() + } + + fn zlib_wrap(plain: &[u8]) -> Vec { + use std::io::Write; + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(plain).unwrap(); + encoder.finish().unwrap() + } + + fn deflate_raw(plain: &[u8]) -> Vec { + use std::io::Write; + let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(plain).unwrap(); + encoder.finish().unwrap() + } + + fn brotli_compress(plain: &[u8]) -> Vec { + let mut output = Vec::new(); + brotli::BrotliCompress(&mut &plain[..], &mut output, &brotli::enc::BrotliEncoderParams::default()).unwrap(); + output + } + + #[test] + fn web_fetch_refuses_local_files() { + let dir = std::env::temp_dir().join(format!("opensheet-fetch-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("secret.txt"); + std::fs::write(&path, b"TOP-SECRET").unwrap(); + let url = file_url(&path); + assert!(names_a_file(&url) && names_a_file(path.to_str().unwrap()) && names_a_file("FILE:///etc/passwd")); + assert!(!names_a_file("https://example.test/a") && !names_a_file("ftp://127.0.0.1/x")); + // The web fetch never opens the file, whatever way it is written. + assert_eq!(fetch(&url).err(), Some(FetchError::LocalFile)); + assert_eq!(fetch(path.to_str().unwrap()).err(), Some(FetchError::LocalFile)); + assert_eq!(fetch("file://localhost/etc/passwd").err(), Some(FetchError::LocalFile)); + assert_eq!(fetch("FILE:///etc/passwd").err(), Some(FetchError::LocalFile)); + assert_eq!(fetch("/nowhere/at/all.csv").err(), Some(FetchError::LocalFile)); + // The file import still reads a path the caller named. + assert_eq!(read_local(&url).unwrap().bytes, b"TOP-SECRET"); + assert_eq!(read_local("/nowhere/at/all.csv").err(), Some(FetchError::Missing)); + std::fs::remove_dir_all(&dir).ok(); } #[test] diff --git a/crates/app/src/features/data_tools/ui_tests.rs b/crates/app/src/features/data_tools/ui_tests.rs index 9acf971..34d19a5 100644 --- a/crates/app/src/features/data_tools/ui_tests.rs +++ b/crates/app/src/features/data_tools/ui_tests.rs @@ -1037,7 +1037,7 @@ fn get_data_from_web_then_refresh(cx: &mut TestAppContext) { assert_eq!(modal_count(&app, cx), 1); assert_eq!(value(&app, cx, "A2"), text("Cid")); // The page is gone. - server.set("/sales", super::sources::test_server::Answer { status: 404, content_type: "text/html", body: Vec::new(), location: None }); + server.set("/sales", super::sources::test_server::Answer { status: 404, content_type: "text/html", body: Vec::new(), location: None, encoding: None }); run(&app, cx, Cmd::RefreshAll); assert!(alert_text(&app, cx).contains("404"), "{}", alert_text(&app, cx)); assert_eq!(modal_count(&app, cx), 1); @@ -1072,6 +1072,14 @@ fn get_data_from_web_refusals_and_text(cx: &mut TestAppContext) { from_web(&app, cx, " "); assert_eq!(modal_count(&app, cx), 1); assert!(app.update(cx, |app, _| app.wb.sheets[0].text_imports.is_empty())); + // A local file is not a web address, as a URL or as a path. + from_web(&app, cx, "file:///etc/passwd"); + assert!(alert_text(&app, cx).contains("local file"), "{}", alert_text(&app, cx)); + assert_eq!(modal_count(&app, cx), 1); + from_web(&app, cx, "/etc/passwd"); + assert!(alert_text(&app, cx).contains("local file"), "{}", alert_text(&app, cx)); + assert_eq!(modal_count(&app, cx), 1); + assert!(app.update(cx, |app, _| app.wb.sheets[0].text_imports.is_empty())); // An address that gives CSV goes through the Text Import Wizard, and // refreshes from the address. @@ -1297,3 +1305,92 @@ fn edit_query_and_refresh_on_open(cx: &mut TestAppContext) { redraw(cx); assert_eq!(value(&app, cx, "A1"), text("Carte")); } + +#[gpui::test] +fn interval_refresh_waits_for_consent(cx: &mut TestAppContext) { + let (app, cx) = open(cx); + let server = Server::start(); + server.set("/a", page("text/html", &sales_page(&[("Ann", "3")]))); + from_web(&app, cx, &server.url("/a")); + cx.simulate_keystrokes("enter"); + redraw(cx); + assert_eq!(value(&app, cx, "A1"), text("Menu")); + // The interval is the workbook's, not something just switched on in + // Connection Properties, so opening asks before any clock starts. + edit(&app, cx, |wb| wb.update_import(1, 0, |import| import.props.refresh_minutes = 1)); + let bytes = app.update(cx, |app, _| opensheet_io::write_xlsx(&app.wb, std::io::Cursor::new(Vec::new())).unwrap().into_inner()); + let reopen = |app: &Entity, cx: &mut VisualTestContext| { + let wb = opensheet_io::read_xlsx(std::io::Cursor::new(bytes.clone()), opensheet_engine::Locale::en()).unwrap(); + app.update(cx, |app, cx| app.set_workbook(wb, None, "Opened".into(), cx)); + redraw(cx); + redraw(cx); + }; + server.set("/a", page("text/html", &sales_page(&[("Zed", "0")]).replace("Menu", "Carte"))); + reopen(&app, cx); + assert!(alert_text(&app, cx).contains("connects to an external data source"), "{}", alert_text(&app, cx)); + assert!(app.update(cx, |app, _| app.ext::().is_none_or(|watch| !watch.timer_running()))); + app.update_in(cx, |app, window, cx| import::minute_passed(app, window, cx)); + // Decline: the timer stays stopped, and a minute changes nothing. + cx.simulate_keystrokes("escape"); + redraw(cx); + assert!(!modal_open(&app, cx)); + assert!(app.update(cx, |app, _| app.ext::().is_some_and(|watch| !watch.timer_running()))); + redraw(cx); + assert!(!alert_text(&app, cx).contains("external data source")); + app.update(cx, |app, cx| { + app.sheet = 1; + cx.notify(); + }); + assert_eq!(value(&app, cx, "A1"), text("Menu")); + app.update_in(cx, |app, window, cx| import::minute_passed(app, window, cx)); + redraw(cx); + assert_eq!(value(&app, cx, "A1"), text("Menu")); + // Refresh by hand still reads the address. + run(&app, cx, Cmd::RefreshAll); + assert_eq!(value(&app, cx, "A1"), text("Carte")); + + // Opened again, and agreed to: the clock runs, and the next minute refreshes. + server.set("/a", page("text/html", &sales_page(&[("Bea", "1")]).replace("Menu", "Plat"))); + reopen(&app, cx); + assert!(alert_text(&app, cx).contains("connects to an external data source")); + cx.simulate_keystrokes("enter"); + redraw(cx); + assert!(app.update(cx, |app, _| app.ext::().is_some_and(|watch| watch.timer_running()))); + app.update(cx, |app, cx| { + app.sheet = 1; + cx.notify(); + }); + // Agreeing does not refresh at once; the interval does, a minute later. + assert_eq!(value(&app, cx, "A1"), text("Menu")); + app.update_in(cx, |app, window, cx| import::minute_passed(app, window, cx)); + redraw(cx); + assert_eq!(value(&app, cx, "A1"), text("Plat")); +} + +#[gpui::test] +fn workbook_web_query_does_not_read_a_local_file(cx: &mut TestAppContext) { + let (app, cx) = open(cx); + let server = Server::start(); + server.set("/a", page("text/html", &sales_page(&[("Ann", "3")]))); + from_web(&app, cx, &server.url("/a")); + cx.simulate_keystrokes("enter"); + redraw(cx); + assert_eq!(value(&app, cx, "A1"), text("Menu")); + let secret = text_file("secret", "TOP-SECRET\n"); + // As a workbook would: the web query's address is a local file the + // user never picked. Refresh must not read it. + edit(&app, cx, |wb| { + wb.update_import(1, 0, |import| import.path = super::sources::file_url(&secret)); + }); + run(&app, cx, Cmd::RefreshAll); + assert!(alert_text(&app, cx).contains("local file"), "{}", alert_text(&app, cx)); + assert_eq!(value(&app, cx, "A1"), text("Menu")); + assert_eq!(modal_count(&app, cx), 1); + edit(&app, cx, |wb| { + wb.update_import(1, 0, |import| import.path = secret.to_string_lossy().into_owned()); + }); + run(&app, cx, Cmd::RefreshAll); + assert!(alert_text(&app, cx).contains("local file"), "{}", alert_text(&app, cx)); + assert_eq!(value(&app, cx, "A1"), text("Menu")); + std::fs::remove_dir_all(secret.parent().unwrap()).ok(); +} diff --git a/crates/app/src/i18n_tables.rs b/crates/app/src/i18n_tables.rs index fa291a4..d736845 100644 --- a/crates/app/src/i18n_tables.rs +++ b/crates/app/src/i18n_tables.rs @@ -3608,6 +3608,7 @@ pub static ES: &[(&str, &str)] = &[ ("The original external data range is no longer valid.", "El rango original de datos externos ya no es válido."), ("The name that you entered is not valid.", "El nombre especificado no es válido."), ("Invalid Web Query", "Consulta web no válida"), + ("Unable to open |0. A web query cannot open a local file. Use an http or https address.", "No se puede abrir |0. Una consulta web no puede abrir un archivo local. Use una dirección http o https."), ("Connection Properties", "Propiedades de conexión"), ("Enable background refresh", "Habilitar actualización en segundo plano"), ("Refresh every", "Actualizar cada"), @@ -7476,6 +7477,7 @@ pub static DE: &[(&str, &str)] = &[ ("The original external data range is no longer valid.", "Der ursprüngliche externe Datenbereich ist nicht länger gültig."), ("The name that you entered is not valid.", "Der eingegebene Name ist ungültig."), ("Invalid Web Query", "Ungültige Webabfrage"), + ("Unable to open |0. A web query cannot open a local file. Use an http or https address.", "Öffnen von |0 nicht möglich. Eine Webabfrage kann keine lokale Datei öffnen. Verwenden Sie eine http- oder https-Adresse."), ("Connection Properties", "Verbindungseigenschaften"), ("Enable background refresh", "Aktualisierung im Hintergrund zulassen"), ("Refresh every", "Aktualisierung alle"), @@ -11338,6 +11340,7 @@ pub static IT: &[(&str, &str)] = &[ ("The original external data range is no longer valid.", "L'intervallo dati esterno originale non è più valido."), ("The name that you entered is not valid.", "Il nome immesso non è valido."), ("Invalid Web Query", "Query Web non valida."), + ("Unable to open |0. A web query cannot open a local file. Use an http or https address.", "Impossibile aprire |0. Una query Web non può aprire un file locale. Usare un indirizzo http o https."), ("Connection Properties", "Proprietà connessione"), ("Enable background refresh", "Abilita aggiornamento in background"), ("Refresh every", "Aggiorna ogni"), diff --git a/docs/fonctionnalites.md b/docs/fonctionnalites.md index aa16df9..dd0cf36 100644 --- a/docs/fonctionnalites.md +++ b/docs/fonctionnalites.md @@ -102,7 +102,7 @@ Excel utilise Aptos Narrow par défaut, qui n'est pas une police système. Sans **Données** - Filtres automatiques (valeurs, texte, nombres, dates, couleurs, 10 premiers), tri à plusieurs niveaux avec listes personnalisées, suppression des doublons. - Tableaux structurés : 60 styles, ligne de total, colonnes calculées, agrandissement automatique à la saisie. -- Obtenir des données : depuis un fichier texte ou CSV, le web (adresse saisie, choix du tableau dans le navigateur avec aperçu), un classeur Excel, un fichier JSON ou XML ; Requêtes et connexions, actualisation à la demande, à l'ouverture ou toutes les N minutes. Rien n'est téléchargé sans votre demande. +- Obtenir des données : depuis un fichier texte ou CSV, le web (adresse http ou https saisie, choix du tableau dans le navigateur avec aperçu), un classeur Excel, un fichier JSON ou XML choisi dans le sélecteur de fichiers. Une requête web venue d'un classeur n'ouvre ni un chemin local ni une URL `file://`. Requêtes et connexions ; actualisation à la demande. L'actualisation à l'ouverture et celle toutes les N minutes, quand le classeur les demande, attendent un accord : sans accord le minuteur ne démarre pas, et Actualiser reste disponible. - Tableaux croisés dynamiques : volet des champs avec glisser-déposer, filtres, regroupements, « afficher les valeurs comme », styles, détail par double-clic, champs et éléments calculés, ordre de résolution. - Segments (tableaux croisés et tableaux) et chronologies : sélection au clic ou par glisser, sélection multiple, connexions de rapport, styles ; les éléments sans données sont grisés. - Graphiques croisés dynamiques : ils suivent la disposition, les filtres et l'actualisation du tableau croisé ; leurs boutons de champ ouvrent la liste de tri et de filtre du champ ; avec plusieurs champs de lignes, l'axe a plusieurs niveaux (comme tout graphique dont les étiquettes tiennent sur plusieurs colonnes).