From d18a5444f98525fab7d0ef46e9b1a6f161350834 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sat, 24 Oct 2026 06:17:02 +0200 Subject: [PATCH 1/9] feat(s3): remember saved keys with shared connection credential lifecycle Integrate the immutable database-connection snapshot from PR #777 while preserving its independent checkout and ownership. Reuse the shared secret store, account identity, remember validation and failure-safe lifecycle. Bind S3 project sources to exact saved logins, resolve ambiguous resources explicitly, and expose opt-in, status and forgetting in the UI. --- crates/assistant/src/audit.rs | 4 + crates/assistant/src/tests.rs | 5 + crates/data/src/lib.rs | 1 + crates/data/src/s3_address.rs | 201 +++ crates/data/src/source/credentials.rs | 21 + crates/data/src/source/mod.rs | 6 +- crates/data/src/source/remote.rs | 5 + crates/engine/src/cmd/connection_browse.rs | 43 +- crates/engine/src/cmd/connections.rs | 274 +++- crates/engine/src/cmd/items.rs | 6 +- crates/engine/src/cmd/layer.rs | 8 +- crates/engine/src/cmd/plugins.rs | 2 + crates/engine/src/cmd/project.rs | 4 + crates/engine/src/cmd/s3.rs | 46 +- crates/engine/src/connections.rs | 59 +- crates/engine/src/lib.rs | 18 +- crates/engine/src/project.rs | 33 +- crates/engine/src/s3_connections.rs | 391 ++++++ crates/engine/src/s3_keys.rs | 152 +++ crates/engine/src/s3_saved_tests.rs | 1155 +++++++++++++++++ crates/engine/src/sources.rs | 34 +- crates/engine/src/tests.rs | 18 +- .../src/tests/remembered_database_packets.rs | 1 + crates/map/src/lib.rs | 3 + crates/map/src/raster.rs | 82 +- crates/map/src/source.rs | 13 + crates/map/src/tiles.rs | 19 +- crates/tiles/src/address.rs | 8 + crates/tiles/src/cache.rs | 49 +- crates/tiles/src/cog.rs | 12 + crates/tiles/src/pmtiles.rs | 15 + crates/tiles/src/s3.rs | 519 +++++++- crates/tiles/src/schedule.rs | 2 +- crates/ui-egui/src/chrome.rs | 1 + crates/ui-egui/src/connectiontree.rs | 2 +- crates/ui-egui/src/connectionui.rs | 502 ++++++- crates/ui-egui/src/dialogs.rs | 4 +- crates/ui-egui/src/lib.rs | 21 +- crates/ui-egui/src/panes.rs | 5 +- crates/ui-egui/src/s3ui.rs | 377 +++++- crates/ui-egui/src/sourcespane.rs | 30 + crates/ui-egui/tests/remembered_s3/mod.rs | 323 +++++ crates/ui-egui/tests/smoke.rs | 7 + docs/user-guide/data.md | 82 +- 44 files changed, 4327 insertions(+), 236 deletions(-) create mode 100644 crates/data/src/s3_address.rs create mode 100644 crates/engine/src/s3_connections.rs create mode 100644 crates/engine/src/s3_keys.rs create mode 100644 crates/engine/src/s3_saved_tests.rs create mode 100644 crates/ui-egui/tests/remembered_s3/mod.rs diff --git a/crates/assistant/src/audit.rs b/crates/assistant/src/audit.rs index 155c0e630..2c2523fc7 100644 --- a/crates/assistant/src/audit.rs +++ b/crates/assistant/src/audit.rs @@ -375,6 +375,10 @@ mod tests { "password", "api_key", "accessKey", + "accessKeyId", + "access_key_id", + "secretAccessKey", + "sessionToken", "tokens", "authToken", "client_secret", diff --git a/crates/assistant/src/tests.rs b/crates/assistant/src/tests.rs index 345df398c..06d06da31 100644 --- a/crates/assistant/src/tests.rs +++ b/crates/assistant/src/tests.rs @@ -268,6 +268,11 @@ fn gate_rules() { assert!(g("run_command", json!({"command": "plugin.install", "params": {"path": "x.rgplugin"}})).is_some()); assert!(g("run_command", json!({"command": "map.addTiles", "params": {"url": "https://tile.example/{z}/{x}/{y}.png"}})).is_some()); assert!(g("run_command", json!({"command": "plugin:x/y"})).is_some()); + // Saved S3 keys and the choice of a connection are the user's: asked for, every time. + for command in ["connection.forgetSavedKeys", "connection.resolveS3"] { + assert!(g("run_command", json!({"command": command, "params": {"id": "c1", "source": "s3://bucket/key"}})).is_some(), "{command}"); + } + assert!(g("run_command", json!({"command": "connection.login", "params": {"id": "c1", "remember": true}})).is_some()); let sql = g("run_command", json!({"command": "layer.addSql", "params": {"sql": "SELECT ST_Point(0, 0) AS g"}})); assert!(sql.as_deref().is_some_and(|w| w.contains("SQL") && w.contains("internet")), "{sql:?}"); let sql = g("run_command", json!({"command": "layer.addSql", "params": {"sql": "FROM t", "database": "/data/x.duckdb"}})); diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index a19e6693b..dcc9b2599 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -24,6 +24,7 @@ pub mod osm; pub mod photo; pub mod query; pub mod raster; +pub mod s3_address; pub mod secrets; pub mod sheets; pub mod source; diff --git a/crates/data/src/s3_address.rs b/crates/data/src/s3_address.rs new file mode 100644 index 000000000..62193309e --- /dev/null +++ b/crates/data/src/s3_address.rs @@ -0,0 +1,201 @@ +//! The address of a web or `s3://` object read through one saved S3 connection. +//! +//! A saved connection's login is kept for the session under its own runtime *binding* (see +//! `rustgis_tiles::s3`), and an address that must be read with that connection's keys (and no +//! other's) names it: `rustgis-s3::::`. The +//! endpoint is the connection's canonical one (empty for AWS), the inner address is a web address +//! or an `s3://` one, kept verbatim. An empty binding is an address no connection has been chosen +//! for yet: it is never fetched. Nothing here is secret, so a saved project may keep these +//! addresses; deciding whether a binding may sign a request, and for which endpoint and bucket, is +//! `rustgis_tiles::s3`'s job, not this structural parser's. + +/// Starts an address read through a saved S3 connection. +pub const PREFIX: &str = "rustgis-s3:"; +/// The longest address, in bytes. +pub const MAX_ADDRESS: usize = 8192; +/// The longest endpoint, in bytes. +pub const MAX_ENDPOINT: usize = 2048; +/// How many hexadecimal digits a binding has. +pub const BINDING_LEN: usize = 16; + +const BAD: &str = "not a saved S3 address"; +const TOO_LONG: &str = "the saved S3 address is too long"; + +/// The parts of an address read through a saved connection. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BoundS3<'a> { + /// The runtime token of the connection that may read it (empty: none chosen yet). + pub binding: &'a str, + /// The connection's canonical endpoint (empty: AWS). + pub endpoint: &'a str, + /// The address read: `http://`, `https://` or `s3://`. + pub url: &'a str, +} + +/// Whether an address starts as one read through a saved connection, well formed or not. A +/// malformed one is never a file on this computer: callers keep it blocked. +pub fn is_wrapper(address: &str) -> bool { + address.starts_with(PREFIX) +} + +/// Whether `binding` is empty (unresolved) or [`BINDING_LEN`] lower-case hexadecimal digits. +pub fn valid_binding(binding: &str) -> bool { + binding.is_empty() || (binding.len() == BINDING_LEN && binding.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))) +} + +fn web_or_s3(url: &str) -> bool { + ["https://", "http://", "s3://"].iter().any(|p| url.get(..p.len()).is_some_and(|s| s.eq_ignore_ascii_case(p))) +} + +/// The parts of `address` when it is one read through a saved connection, `None` when it isn't +/// (it doesn't start with [`PREFIX`]), and a fixed error (never part of the input) when it starts +/// so but is malformed: a bad binding, endpoint length or boundary, controls, a nested wrapper, or +/// an inner address that isn't a web or `s3://` one. Borrows; never allocates. +pub fn parse(address: &str) -> Result>, &'static str> { + let Some(rest) = address.strip_prefix(PREFIX) else { return Ok(None) }; + if address.len() > MAX_ADDRESS { + return Err(TOO_LONG); + } + if address.chars().any(char::is_control) { + return Err(BAD); + } + let (binding, rest) = rest.split_once(':').ok_or(BAD)?; + if !valid_binding(binding) { + return Err(BAD); + } + let (length, rest) = rest.split_once(':').ok_or(BAD)?; + if length.is_empty() || length.len() > 4 || !length.bytes().all(|b| b.is_ascii_digit()) { + return Err(BAD); + } + let length: usize = length.parse().map_err(|_| BAD)?; + if length > MAX_ENDPOINT { + return Err(BAD); + } + // Slicing at a byte offset that isn't a character boundary is `None`, not a panic. + let endpoint = rest.get(..length).ok_or(BAD)?; + let url = rest.get(length..).and_then(|r| r.strip_prefix(':')).ok_or(BAD)?; + if is_wrapper(url) || !web_or_s3(url) { + return Err(BAD); + } + Ok(Some(BoundS3 { binding, endpoint, url })) +} + +/// The address read through a saved connection with the given parts, checked as [`parse`] does. +pub fn bind(binding: &str, endpoint: &str, url: &str) -> Result { + if !valid_binding(binding) || endpoint.len() > MAX_ENDPOINT { + return Err(BAD); + } + let out = format!("{PREFIX}{binding}:{}:{endpoint}:{url}", endpoint.len()); + match parse(&out)? { + Some(_) => Ok(out), + None => Err(BAD), + } +} + +/// The address that is read: a valid wrapper's inner address, or any other address as it is +/// (including a malformed wrapper, which no caller may treat as a file). +pub fn inner(address: &str) -> &str { + match parse(address) { + Ok(Some(b)) => b.url, + _ => address, + } +} + +/// `address` with its binding replaced (a wrapper), or `None` when it isn't a valid wrapper or +/// the new binding isn't one. The endpoint and inner address stay. +pub fn rebound(address: &str, binding: &str) -> Option { + let b = parse(address).ok().flatten()?; + bind(binding, b.endpoint, b.url).ok() +} + +/// `address` with its inner address replaced, keeping binding and endpoint. +pub fn with_url(address: &str, url: &str) -> Option { + let b = parse(address).ok().flatten()?; + bind(b.binding, b.endpoint, url).ok() +} + +/// The address a server is reached at, for an address that may be read through a saved connection +/// (as it is, or as the archive after a tile template's `@`): the wrapper's inner address, else +/// `address` itself. +pub fn reached(address: &str) -> &str { + let wrapped = if is_wrapper(address) { Some(address) } else { address.split_once('@').map(|(_, a)| a).filter(|a| is_wrapper(a)) }; + wrapped.and_then(|a| parse(a).ok().flatten()).map_or(address, |b| b.url) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + const B: &str = "0123456789abcdef"; + + #[test] + fn s3_address_round_trips_every_part() { + let url = "https://minio.example:9000/bucket/a b+c%20d.fgb?list-type=2&prefix=x%2Fy"; + let a = bind(B, "https://minio.example:9000", url).unwrap(); + assert_eq!(a, format!("rustgis-s3:{B}:26:https://minio.example:9000:{url}")); + let p = parse(&a).unwrap().unwrap(); + assert_eq!((p.binding, p.endpoint, p.url), (B, "https://minio.example:9000", url)); + assert_eq!(inner(&a), url); + // AWS: an empty endpoint; unresolved: an empty binding. + let aws = bind("", "", "s3://my-bucket/k.tif").unwrap(); + assert_eq!(aws, "rustgis-s3::0::s3://my-bucket/k.tif"); + let p = parse(&aws).unwrap().unwrap(); + assert_eq!((p.binding, p.endpoint, p.url), ("", "", "s3://my-bucket/k.tif")); + assert_eq!(rebound(&aws, B).unwrap(), format!("rustgis-s3:{B}:0::s3://my-bucket/k.tif")); + assert_eq!( + with_url(&a, "https://minio.example:9000/b/other.fgb").unwrap(), + format!("rustgis-s3:{B}:26:https://minio.example:9000:https://minio.example:9000/b/other.fgb") + ); + // Not a wrapper: left alone. + assert_eq!(parse("https://x.org/a").unwrap(), None); + assert_eq!(inner("https://x.org/a"), "https://x.org/a"); + assert!(is_wrapper("rustgis-s3:junk") && !is_wrapper("s3://b/k")); + } + + #[test] + fn s3_address_rejects_malformed_values_without_echoing_them() { + let endpoint = "https://h.example"; + let good = format!("rustgis-s3:{B}:{}:{endpoint}:https://h.example/b/k", endpoint.len()); + assert!(parse(&good).unwrap().is_some()); + let bad = [ + // binding: upper case, short, long, non-hex + format!("rustgis-s3:{}:17:{endpoint}:https://h.example/k", B.to_uppercase()), + "rustgis-s3:abc:17:https://h.example:https://h.example/k".to_string(), + format!("rustgis-s3:{B}0:17:{endpoint}:https://h.example/k"), + format!("rustgis-s3:{}:17:{endpoint}:https://h.example/k", "g".repeat(16)), + // length: missing, signed, too long, past the end, off by one either way + format!("rustgis-s3:{B}::{endpoint}:https://h.example/k"), + format!("rustgis-s3:{B}:+17:{endpoint}:https://h.example/k"), + format!("rustgis-s3:{B}:99999:{endpoint}:https://h.example/k"), + format!("rustgis-s3:{B}:2049:{endpoint}:https://h.example/k"), + format!("rustgis-s3:{B}:16:{endpoint}:https://h.example/k"), + format!("rustgis-s3:{B}:18:{endpoint}:https://h.example/k"), + // nothing after the endpoint, or no separator + format!("rustgis-s3:{B}:17:{endpoint}"), + format!("rustgis-s3:{B}:17:{endpoint}https://h.example/k"), + // inner: not a web or s3 address, nested wrappers, anonymous marker + format!("rustgis-s3:{B}:17:{endpoint}:ftp://h.example/k"), + format!("rustgis-s3:{B}:17:{endpoint}:/etc/passwd"), + format!("rustgis-s3:{B}:17:{endpoint}:rustgis-s3:{B}:17:{endpoint}:https://h.example/k"), + format!("rustgis-s3:{B}:17:{endpoint}:rustgis-anonymous:https://h.example/k"), + // controls + format!("rustgis-s3:{B}:17:{endpoint}:https://h.example/k\n"), + format!("rustgis-s3:{B}:17:{endpoint}:https://h.example/\u{7}k"), + ]; + for a in &bad { + let e = parse(a).unwrap_err(); + assert!(!e.contains("h.example") && !e.contains("passwd"), "{e}"); + } + // A length that cuts a multi-byte character is an error, not a panic. + assert!(parse(&format!("rustgis-s3:{B}:2:é:https://h.example/k")).is_ok()); + assert!(parse(&format!("rustgis-s3:{B}:1:é:https://h.example/k")).is_err()); + // The overall bound, and the endpoint's. + let long = format!("https://h.example/{}", "k".repeat(MAX_ADDRESS)); + assert!(bind(B, endpoint, &long).is_err() && parse(&format!("rustgis-s3:{B}:17:{endpoint}:{long}")).is_err()); + assert!(bind(B, &"e".repeat(MAX_ENDPOINT + 1), "https://h.example/k").is_err()); + assert!(bind("nope", endpoint, "https://h.example/k").is_err()); + // A malformed wrapper is never read as its own inner address. + assert_eq!(inner("rustgis-s3:junk:https://h.example/k"), "rustgis-s3:junk:https://h.example/k"); + } +} diff --git a/crates/data/src/source/credentials.rs b/crates/data/src/source/credentials.rs index 37016bcec..cee9f13b4 100644 --- a/crates/data/src/source/credentials.rs +++ b/crates/data/src/source/credentials.rs @@ -20,6 +20,8 @@ fn secret_key_lower(k: &str) -> bool { | "key" | "access_key" | "accesskey" + | "access_key_id" + | "accesskeyid" | "secret" | "client_secret" | "password" @@ -74,3 +76,22 @@ pub fn redact_query(address: &mut String) { address.push_str(&kept); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn access_key_ids_and_their_secrets_are_secret_names() { + for k in ["accessKeyId", "access_key_id", "AccessKeyID", "secretAccessKey", "sessionToken", "X-Amz-Credential"] { + assert!(secret_key(k), "{k}"); + } + for k in ["remember", "region", "buckets", "endpoint"] { + assert!(!secret_key(k), "{k}"); + } + // In a query, the key id goes with the signature. + let mut a = "https://s3.example.org/b/k?accessKeyId=AKIDEXAMPLE&prefix=x".to_string(); + redact_query(&mut a); + assert_eq!(a, "https://s3.example.org/b/k?prefix=x"); + } +} diff --git a/crates/data/src/source/mod.rs b/crates/data/src/source/mod.rs index fde47661c..36301083a 100644 --- a/crates/data/src/source/mod.rs +++ b/crates/data/src/source/mod.rs @@ -321,9 +321,11 @@ pub fn open_shared(file_name: &str, bytes: std::sync::Arc>, layer: Optio } } -/// Whether a web address names a file read in place by range: a FlatGeobuf or a GeoParquet. +/// Whether a web address names a file read in place by range: a FlatGeobuf or a GeoParquet. An +/// address read through a saved S3 connection ([`crate::s3_address`]) is judged by the address it +/// wraps. pub fn is_remote_name(url: &str) -> bool { - let lower = url.to_ascii_lowercase(); + let lower = crate::s3_address::inner(url).to_ascii_lowercase(); let path = lower.split(['?', '#']).next().unwrap_or(&lower); // `s3://` too: the app's fetchers check it and read it from its bucket's web address (signed // only for a connection that serves the bucket, see rustgis_tiles::s3). diff --git a/crates/data/src/source/remote.rs b/crates/data/src/source/remote.rs index 440c6d1c5..6cf7fd3f7 100644 --- a/crates/data/src/source/remote.rs +++ b/crates/data/src/source/remote.rs @@ -400,6 +400,11 @@ pub fn forget() { files().lock().unwrap_or_else(|e| e.into_inner()).clear(); } +/// Forgets the fetched parts of one file (after the address's login changed). +pub fn forget_file(url: &str) { + files().lock().unwrap_or_else(|e| e.into_inner()).retain(|r| r.url != url); +} + /// The error a read that needs bytes gives (it runs again once they are fetched). pub fn need() -> DataError { DataError::Format(NEED.into()) diff --git a/crates/engine/src/cmd/connection_browse.rs b/crates/engine/src/cmd/connection_browse.rs index 5eff2b52c..64425e1ac 100644 --- a/crates/engine/src/cmd/connection_browse.rs +++ b/crates/engine/src/cmd/connection_browse.rs @@ -100,19 +100,19 @@ fn level(c: &Connection, node: &str) -> Result { Err(not_a_node()) } -/// The session's S3 connection for an S3 level, logging in with what needs no typing. +/// The session's S3 connection for an S3 level, logging in with what needs no typing (a profile, +/// the environment, anonymously, or the keys saved on this computer: browsing is the user action +/// that may ask the system to unlock its store). /// (A probe logs in too, as `connection.test` does: the address it names, and the signature the /// fetch needs, depend on the connection.) fn s3_login(s: &mut Session, c: &Connection, level: &Level) -> Result, String> { if !matches!(level, Level::S3Root | Level::S3Prefix { .. }) { return Ok(None); } - if s3::connection(&c.name).is_none() && c.options.get("credentials").is_none_or(|v| v != "keys") { - super::login(s, c, &Value::Null)?; - } + super::ensure_login(s, c)?; let conn = s3::connection(&c.name).ok_or("log in first: connection.login {id, accessKeyId, secretAccessKey}")?; if let Level::S3Prefix { bucket, .. } = level { - s3::grant(&conn.name, bucket); + super::grant(s, c, &conn, bucket); } Ok(Some(conn)) } @@ -130,10 +130,16 @@ fn fetch_url(s: &Session, c: &Connection, level: &Level, conn: Option<&s3::Conne Level::StacCatalog { url } | Level::StacItem { url } => Some(url.clone()), Level::S3Root => match (super::first_bucket(c), conn) { (Some(_), _) => None, - (None, Some(conn)) if conn.credentials.is_some() => Some(s3::buckets_url(conn)), + (None, Some(conn)) if conn.credentials.is_some() => Some(super::through(s, c, conn, s3::buckets_url(conn))?), _ => return Err("name a bucket under Buckets to browse an anonymous S3 connection".into()), }, - Level::S3Prefix { bucket, prefix } => Some(s3::list_url(bucket, prefix, None, conn)), + Level::S3Prefix { bucket, prefix } => { + let url = s3::list_url(bucket, prefix, None, conn); + Some(match conn { + Some(conn) => super::through(s, c, conn, url)?, + None => url, + }) + } }) } @@ -211,7 +217,10 @@ pub(super) fn browse(s: &mut Session, p: &Value) -> CmdResult { Level::StacItem { url } => item_file(bytes, url)?, Level::StacCollection { .. } => items(bytes, at)?, Level::S3Root => buckets(s3::parse_buckets(bytes)?.into_iter()), - Level::S3Prefix { bucket, prefix } => objects(bytes, bucket, prefix)?, + Level::S3Prefix { bucket, prefix } => { + let binding = conn.as_ref().and_then(|conn| super::saved_binding(s, &c, conn)); + objects(bytes, bucket, prefix, binding.as_deref())? + } Level::Database | Level::Service { .. } => Found::default(), } } @@ -483,23 +492,29 @@ fn buckets(names: impl Iterator) -> Found { ) } -/// One level of a bucket: its folders (each a node) and its files. -fn objects(bytes: &[u8], bucket: &str, prefix: &str) -> Result { +/// One level of a bucket: its folders (each a node) and its files. Through a saved connection's +/// login (`binding`), a file's add command carries an address that names it, so the layer is read +/// with that connection's keys and no other's; its key and name stay the plain object's. +fn objects(bytes: &[u8], bucket: &str, prefix: &str, binding: Option<&str>) -> Result { let list = s3::parse_list(bytes)?; let last = |k: &str| k.trim_end_matches('/').rsplit('/').next().unwrap_or("").to_string(); let folders = list.prefixes.iter().map(|p| { let node = format!("s3:{bucket}/{p}"); json!({"key": node, "node": node, "name": format!("{}/", last(p)), "what": "prefix"}) }); - let files = list.objects.iter().filter(|o| o.key != prefix && !o.key.ends_with('/')).map(|o| { + let files = list.objects.iter().filter(|o| o.key != prefix && !o.key.ends_with('/')).filter_map(|o| { let url = format!("s3://{bucket}/{}", o.key); - json!({ + let add = match binding { + Some(b) => s3::bound_url(b, &url).ok()?, + None => url.clone(), + }; + Some(json!({ "key": url, "name": last(&o.key), "what": "object", "detail": o.size.to_string(), - "add": {"command": "layer.addUrl", "params": {"url": url}}, - }) + "add": {"command": "layer.addUrl", "params": {"url": add}}, + })) }); let mut found = Found::new(folders.chain(files).take(MAX_CHILDREN + 1).collect()); found.truncated |= list.next.is_some(); diff --git a/crates/engine/src/cmd/connections.rs b/crates/engine/src/cmd/connections.rs index 0c312711d..60437bdcc 100644 --- a/crates/engine/src/cmd/connections.rs +++ b/crates/engine/src/cmd/connections.rs @@ -8,6 +8,7 @@ use serde_json::{Value, json}; use super::{b, ps}; use crate::connections::{Connection, DATABASE_KINDS, DESKTOP_ONLY, KINDS}; use crate::render::rustgis_tiles::{s3, stac}; +use crate::s3_keys as keys; use crate::{CmdResult, CommandSpec as C, Session}; #[path = "connection_browse.rs"] @@ -35,7 +36,13 @@ pub fn commands() -> Vec { C::new("connection.duplicate", "Duplicate Connection", "Home › Add › Data Sources", duplicate).view().params(r#"{"id": "a connection's id or name", "name"?: text}"#), C::new("connection.remove", "Remove Connection", "Home › Add › Data Sources", remove).view().params(r#"{"id": "a connection's id or name"}"#), C::new("connection.login", "Log In to Connection", "Home › Add › Data Sources", login_command).view().params( - r#"{"id": "a connection's id or name; without password explicitly selects its existing session or remembered database password", "password"?: "database password, session-only by default", "remember"?: "boolean, default false: desktop OS-store opt-in for this password; S3 keys remain session-only", "accessKeyId"?: text, "secretAccessKey"?: text, "sessionToken"?: text}"#, + r#"{"id": "a connection's id or name; blank credentials explicitly use this ID's session or saved credentials", "password"?: "a database password, session-only by default", "accessKeyId"?: text, "secretAccessKey"?: text, "sessionToken"?: "temporary S3 credentials, never persisted", "remember"?: "boolean, default false: opt into this computer's credential store for a newly typed database password or complete S3 key pair (saved keys mode only). No credentials are stored in connections or projects; a failed save preserves the session login"}"#, + ), + C::new("connection.forgetSavedKeys", "Forget Saved Keys", "Home › Add › Data Sources", forget_saved_keys).view().params( + r#"{"id": "a saved S3 connection's id or name"} (deletes stored keys; a login in this session keeps working)"#, + ), + C::new("connection.resolveS3", "Choose S3 Connection", "Home › Add › Data Sources", resolve_s3).view().params( + r#"{"source": "a pending project S3 source address", "id": "a candidate saved connection id"} → {id, reopened, failed}: explicitly grants this bucket and opens only its waiting sources"#, ), C::new("connection.forgetPassword", "Forget Saved Password", "Home › Add › Data Sources", forget_password).view().params(r#"{"id": "a database connection's id or name"}"#), C::new("connection.test", "Test Connection", "Home › Add › Data Sources", test).view().params( @@ -50,8 +57,7 @@ pub fn commands() -> Vec { fn shown(s: &Session, c: &Connection) -> Value { let database = DATABASE_KINDS.contains(&c.kind.as_str()); let available = !database || !cfg!(target_arch = "wasm32"); - let logged_in = - if c.kind == "s3" { s3::connection(&c.name).is_some() } else { database && s.database_logins.known_password(&c.id).is_some() }; + let logged_in = if c.kind == "s3" { s.has_s3_login(&c.id) } else { database && s.database_logins.known_password(&c.id).is_some() }; let add_with = "connection.browse {id}, then {id, node} for a child's node; run a dataset's add {command, params}"; let mut out = serde_json::to_value(c).unwrap_or(Value::Null); out["available"] = json!(available); @@ -59,6 +65,11 @@ fn shown(s: &Session, c: &Connection) -> Value { out["why"] = json!(DESKTOP_ONLY); } out["loggedIn"] = json!(logged_in); + // Whether the keys were saved when last asked of the credential store (never a key; listing + // doesn't ask the store). + if c.kind == "s3" { + out["savedKeys"] = json!(c.saved_keys); + } out["remembered"] = json!(c.remembered); out["addWith"] = json!(add_with); out @@ -168,6 +179,15 @@ fn shown_stripped(s: &Session, c: &Connection, stripped: bool) -> Value { out } +/// Whether an edit changes the endpoint a connection's keys are for: another canonical endpoint +/// (a different spelling of the same one isn't a change). +fn endpoint_changed(old: &Connection, new: &Connection) -> bool { + match (s3::canonical(&old.address), s3::canonical(&new.address)) { + (Ok(a), Ok(b)) => a != b, + _ => old.address.trim() != new.address.trim(), + } +} + fn edit(s: &mut Session, p: &Value) -> CmdResult { let old = found(s, p)?.clone(); let mut c = old.clone(); @@ -184,13 +204,20 @@ fn edit(s: &mut Session, p: &Value) -> CmdResult { if let Some(v) = p.get("options") { c.options = options(v)?; } - let mut proposed = s.connections.clone(); - let (c, stripped) = proposed.update(&old.id, c)?; - if DATABASE_KINDS.contains(&old.kind.as_str()) && old.address != c.address { - delete_account(s, &old)?; + // Validate everything, then locate the commit target before touching the store. + let (mut prepared, stripped) = s.connections.prepare_update(&old.id, c)?; + let database_changed = DATABASE_KINDS.contains(&old.kind.as_str()) && old.address != prepared.address; + let credentials_changed = database_changed || old.kind == "s3" && endpoint_changed(&old, &prepared); + let slot = s.connections.connections.iter_mut().find(|c| c.id == old.id).ok_or("connection no longer exists")?; + if credentials_changed { + delete_account(&mut s.database_logins, &old)?; + prepared.saved_keys = false; + prepared.remembered = false; + } + let c = crate::connections::Connections::commit_update(slot, &mut s.connections.generation, prepared); + if database_changed { clear_identity(s, &old.id); } - s.connections = proposed; if old.address != c.address { // Only the two databases this definition names can change: another database's layers // waiting for a login keep their request, and an unrelated down server is not retried. @@ -214,6 +241,7 @@ fn duplicate(s: &mut Session, p: &Value) -> CmdResult { if let Some(name) = ps(p, "name") { c.name = name.into(); } + // A copy has no saved keys: `add` starts it without any. let (c, _) = s.connections.add(c, explicit)?; resolve_keys(s, &address_keys(&c.address)); Ok(shown(s, &c)) @@ -221,29 +249,46 @@ fn duplicate(s: &mut Session, p: &Value) -> CmdResult { fn remove(s: &mut Session, p: &Value) -> CmdResult { let c = found(s, p)?.clone(); - let mut proposed = s.connections.clone(); - proposed.remove(&c.id); + if DATABASE_KINDS.contains(&c.kind.as_str()) || c.kind == "s3" { + delete_account(&mut s.database_logins, &c)?; + } if DATABASE_KINDS.contains(&c.kind.as_str()) { - delete_account(s, &c)?; clear_identity(s, &c.id); } - s.connections = proposed; + s.connections.remove(&c.id); // The removed definition named one database: only its layers can change. The requests naming // it went with `clear_identity`. resolve_keys(s, &address_keys(&c.address)); if c.kind == "s3" { s3::disconnect(&c.name); + s.s3_bindings.remove(&c.id); } Ok(json!({"removed": c.id})) } -fn delete_account(s: &mut Session, c: &Connection) -> Result<(), String> { - if s.database_logins.has_store() { - s.database_logins - .delete(&c.id) - .map_err(|_| "The saved password couldn't be deleted. The connection was left unchanged.".to_string()) - } else if c.remembered { - Err("The saved password couldn't be deleted. The connection was left unchanged.".into()) +/// Deletes the keys saved for an S3 connection from the credential store. The login of this +/// session keeps working. +fn forget_saved_keys(s: &mut Session, p: &Value) -> CmdResult { + let c = found(s, p)?.clone(); + if c.kind != "s3" { + return Err(format!("{} connections have no saved keys", c.kind)); + } + delete_account(&mut s.database_logins, &c)?; + s.connections.set_saved_keys(&c.id, false); + let c = s.connections.find(&c.id).cloned().unwrap_or(c); + Ok(shown(s, &c)) +} + +fn delete_account(logins: &mut crate::connections::DatabaseLogins, c: &Connection) -> Result<(), String> { + let (marked, error) = if c.kind == "s3" { + (c.saved_keys, keys::NOT_FORGOTTEN) + } else { + (c.remembered, "The saved password couldn't be deleted. The connection was left unchanged.") + }; + if logins.has_store() { + logins.delete(&c.id).map_err(|_| error.to_string()) + } else if marked { + Err(error.into()) } else { Ok(()) } @@ -265,13 +310,12 @@ pub(crate) fn layer_keys(src: &rustgis_map::source::SourceLayer) -> Vec<(&'stati let Some((_, database)) = crate::connections::layer_database(src) else { return Vec::new() }; crate::connections::database_key(&crate::sources::redact(database)).into_iter().collect() } - fn forget_password(s: &mut Session, p: &Value) -> CmdResult { let c = found(s, p)?.clone(); if !DATABASE_KINDS.contains(&c.kind.as_str()) { return Err("only database connections have saved passwords".into()); } - delete_account(s, &c)?; + delete_account(&mut s.database_logins, &c)?; s.connections.set_remembered(&c.id, false); s.database_logins.forget_saved(&c.id); #[cfg(not(target_arch = "wasm32"))] @@ -300,6 +344,8 @@ fn forget_password(s: &mut Session, p: &Value) -> CmdResult { struct Reopened { opened: usize, failed: usize, + /// A login that worked but couldn't do all it was asked (the keys weren't kept). + warning: Option<&'static str>, /// The server refused this attempt's password: desktop database reopen only. #[cfg(not(target_arch = "wasm32"))] refused: bool, @@ -388,11 +434,71 @@ fn reopen(s: &mut Session, c: &Connection, saved: bool, session: bool, all_unope result } -fn login(s: &mut Session, c: &Connection, p: &Value) -> Result { - let remember = p.get("remember").map(|v| v.as_bool().ok_or("remember is a boolean")).transpose()?.unwrap_or(false); - if remember && !DATABASE_KINDS.contains(&c.kind.as_str()) { - return Err("only database connections can remember a password".into()); +/// `remember` of a login: only `true` or `false`, judged before anything happens. +fn remember_flag(p: &Value) -> Result { + match p.get("remember") { + None => Ok(false), + Some(Value::Bool(b)) => Ok(*b), + Some(_) => Err("remember is true or false".into()), + } +} + +/// The access key pair typed in a login: both or neither (never half a pair). +fn typed_keys(p: &Value) -> Result, String> { + let text = |k: &str| ps(p, k).map(str::trim).filter(|v| !v.is_empty()).map(str::to_string); + match (text("accessKeyId"), text("secretAccessKey")) { + (Some(k), Some(s)) => Ok(Some((k, s))), + (None, None) => Ok(None), + _ => Err("give both accessKeyId and secretAccessKey".into()), + } +} + +/// Logs in to a saved S3 connection that has no login yet, with what needs no typing: its +/// profile or the environment, anonymously, or the keys saved on this computer (a user action +/// that may ask the system to unlock its store). An unsaved connection with typed keys has none +/// to read: the caller says to give them. +pub(super) fn ensure_login(s: &mut Session, c: &Connection) -> Result<(), String> { + if s.has_s3_login(&c.id) { + return Ok(()); + } + let keys_mode = c.options.get("credentials").is_some_and(|v| v == "keys"); + if keys_mode && c.id.is_empty() { + return Ok(()); + } + login(s, c, &Value::Null).map(|_| ()) +} + +/// The token of this session's login of the saved connection `c`, when the live S3 connection +/// `conn` is that login (an ordinary `s3.connect` of the same name isn't). +pub(super) fn saved_binding(s: &Session, c: &Connection, conn: &s3::Connection) -> Option { + let mine = s.s3_binding(&c.id)?; + (s3::binding_of(&conn.name).as_deref() == Some(mine)).then(|| mine.to_string()) +} + +/// The address a fetch of `url` through the live S3 connection `conn` of the saved connection `c` +/// uses: one that names this session's login of `c`, so only that login signs it. (A live +/// connection of that name that isn't the saved login — an ordinary `s3.connect` — keeps its +/// address.) +pub(super) fn through(s: &Session, c: &Connection, conn: &s3::Connection, url: String) -> Result { + match saved_binding(s, c, conn) { + Some(binding) => s3::bound_url(&binding, &url), + None => Ok(url), } +} + +/// Lets the live connection `conn` of the saved connection `c` serve `bucket`: through this +/// session's login of `c` when that is what it is, as a connection always did otherwise. +pub(super) fn grant(s: &Session, c: &Connection, conn: &s3::Connection, bucket: &str) { + match saved_binding(s, c, conn) { + Some(binding) => { + let _ = s3::grant_saved(&binding, bucket); + } + None => s3::grant(&conn.name, bucket), + } +} + +fn login(s: &mut Session, c: &Connection, p: &Value) -> Result { + let remember = remember_flag(p)?; if DATABASE_KINDS.contains(&c.kind.as_str()) { #[cfg(target_arch = "wasm32")] { @@ -462,20 +568,60 @@ fn login(s: &mut Session, c: &Connection, p: &Value) -> Result if c.kind != "s3" { return Err(format!("{} connections don't log in yet", c.kind)); } + let mode = c.options.get("credentials").map(String::as_str).unwrap_or("none"); + if remember && (mode != "keys" || c.id.is_empty()) { + return Err("remember keeps the typed keys of a saved S3 connection that uses keys (credentials: keys)".into()); + } let mut q = json!({"name": c.name, "endpoint": c.address}); for key in ["region", "buckets"] { if let Some(v) = c.options.get(key).filter(|v| !v.is_empty()) { q[key] = json!(v); } } - match c.options.get("credentials").map(String::as_str).unwrap_or("none") { + // The typed pair, when it is to be kept after the login (a login with a session token isn't). + let mut to_keep = None; + let mut temporary = false; + match mode { "keys" => { - if ["accessKeyId", "secretAccessKey"].iter().any(|k| ps(p, k).is_none_or(|v| v.trim().is_empty())) { - return Err("accessKeyId and secretAccessKey are required: an S3 connection's keys are typed each session".into()); - } - for key in ["accessKeyId", "secretAccessKey", "sessionToken"] { - if let Some(v) = p.get(key) { - q[key] = v.clone(); + let typed = typed_keys(p)?; + let token = ps(p, "sessionToken").map(str::trim).filter(|t| !t.is_empty()); + match typed { + Some((key, secret)) => { + if let Some(t) = token { + q["sessionToken"] = json!(t); + temporary = true; + } + q["accessKeyId"] = json!(key); + q["secretAccessKey"] = json!(secret); + if remember { + to_keep = Some((key, secret)); + } + } + None => { + if token.is_some() { + return Err("a session token needs accessKeyId and secretAccessKey".into()); + } + if remember { + return Err("remember needs the keys typed in this login: give accessKeyId and secretAccessKey".into()); + } + if c.id.is_empty() { + return Err("accessKeyId and secretAccessKey are required".into()); + } + // The saved pair, if there is one. Another connection's keys, a profile and the + // environment are never tried instead. + match keys::load(s.database_logins.secret_store(), c) { + keys::Loading::Keys(key, secret) => { + s.connections.set_saved_keys(&c.id, true); + q["accessKeyId"] = json!(key); + q["secretAccessKey"] = json!(secret); + } + keys::Loading::Missing => { + s.connections.set_saved_keys(&c.id, false); + return Err(keys::NONE_SAVED.into()); + } + keys::Loading::Unavailable => return Err(keys::UNAVAILABLE.into()), + keys::Loading::Unusable => return Err(keys::UNUSABLE.into()), + } } } } @@ -486,15 +632,38 @@ fn login(s: &mut Session, c: &Connection, p: &Value) -> Result "environment" => q["environment"] = json!(true), _ => q["anonymous"] = json!(true), } - super::s3::connect(s, &q)?; - Ok(Reopened::default()) + if c.id.is_empty() { + // A definition not saved (Test): an ordinary session connection. + super::s3::connect(s, &q)?; + return Ok(Reopened::default()); + } + // A saved connection's login is live under this session's token for it, and signs only the + // addresses that name it. + let binding = match s.s3_bindings.get(&c.id) { + Some(b) => b.clone(), + None => s3::new_binding()?, + }; + super::s3::connect_saved(s, &q, &binding, &c.id)?; + s.s3_bindings.insert(c.id.clone(), binding); + let mut done = Reopened::default(); + // Kept only once the login has worked, so what is stored is a pair that signed in; a failed + // save leaves the login as it is (and an earlier saved pair as it was). + if let Some((key, secret)) = to_keep { + if temporary { + done.warning = Some(keys::TEMPORARY); + } else if keys::save(s.database_logins.secret_store(), c, &key, &secret) { + s.connections.set_saved_keys(&c.id, true); + } else { + done.warning = Some(keys::NOT_REMEMBERED); + } + } + Ok(done) } fn login_command(s: &mut Session, p: &Value) -> CmdResult { let c = found(s, p)?.clone(); let result = login(s, &c, p)?; - let mut c = c; - c.remembered = s.connections.find(&c.id).is_some_and(|c| c.remembered); + let c = s.connections.find(&c.id).cloned().unwrap_or(c); let mut out = shown(s, &c); if let Some(saved) = result.password_saved { out["passwordSaved"] = json!(saved); @@ -508,9 +677,27 @@ fn login_command(s: &mut Session, p: &Value) -> CmdResult { out["reopened"] = json!(result.opened); out["failed"] = json!(result.failed); } + if let Some(warning) = result.warning { + out["warning"] = json!(warning); + } Ok(out) } +/// This session's login of the saved S3 connection `c`, made if it has none (with what needs no +/// typing: its profile, the environment, anonymously, or the keys saved on this computer), and +/// its token: what an address names to be read through it. One that fails says why, in words +/// fit for a notice. +pub(crate) fn login_saved(s: &mut Session, c: &Connection) -> Result { + ensure_login(s, c)?; + s.s3_binding(&c.id).map(str::to_string).ok_or_else(|| "the connection didn't log in".to_string()) +} + +/// Binds a project's S3 sources that wait for a choice to the saved connection chosen. +fn resolve_s3(s: &mut Session, p: &Value) -> CmdResult { + let source = ps(p, "source").unwrap_or(""); + let id = ps(p, "id").unwrap_or(""); + crate::s3_connections::choose(s, source, id) +} #[cfg(not(target_arch = "wasm32"))] fn activate(s: &mut Session, c: &Connection, password: &str) { if let Some((_, key)) = crate::connections::database_key(&c.address) { @@ -570,10 +757,10 @@ fn probe_url(s: &Session, c: &Connection) -> Result, String> { "s3" => { let conn = s3::connection(&c.name).ok_or("log in first: connection.login {id, accessKeyId, secretAccessKey}")?; if let Some(bucket) = first_bucket(c) { - s3::grant(&conn.name, &bucket); - s3::list_url(&bucket, "", None, Some(&conn)) + grant(s, c, &conn, &bucket); + through(s, c, &conn, s3::list_url(&bucket, "", None, Some(&conn)))? } else if conn.credentials.is_some() { - s3::buckets_url(&conn) + through(s, c, &conn, s3::buckets_url(&conn))? } else { return Err("name a bucket under Buckets to test an anonymous S3 connection".into()); } @@ -616,8 +803,13 @@ fn test(s: &mut Session, p: &Value) -> CmdResult { } login(s, &c, &input)?; } - if c.kind == "s3" && !secrets && c.options.get("credentials").is_none_or(|v| v != "keys") { - login(s, &c, &Value::Null)?; + if c.kind == "s3" && !secrets { + let keys_mode = c.options.get("credentials").is_some_and(|v| v == "keys"); + // Other sources are logged in again (a profile may have changed); saved keys are read + // when there is no login yet. + if !keys_mode || (!c.id.is_empty() && !s.has_s3_login(&c.id)) { + login(s, &c, &Value::Null)?; + } } let mut out = json!({}); if !c.id.is_empty() { diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index a6e0b9c8e..1ab32bea1 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -441,6 +441,8 @@ fn import_maps(s: &mut Session, p: &Value) -> CmdResult { }; let mut names = Vec::new(); for mut m in maps.into_iter().take(100) { + // A binding in a file is no authority to use a login here. + crate::s3_connections::clear_bindings(m.addresses_mut()); let base = m.name.clone(); let mut i = 1; while s.project.maps.iter().any(|o| o.name.eq_ignore_ascii_case(&m.name)) { @@ -450,10 +452,12 @@ fn import_maps(s: &mut Session, p: &Value) -> CmdResult { names.push(m.name.clone()); s.project.maps.push(m); } - // Repair what was read (rasters reopen, cameras clamp). + // Repair what was read (rasters reopen, cameras clamp); the sources that read S3 wait for + // the saved connection that is to read them. s.project.normalize(); super::connections::restore_active(s); super::connections::resolve_project(s); + crate::s3_connections::resolve_project_s3(s); super::connections::note_waiting(s); if let Some(n) = names.last() && let Some(i) = s.project.maps.iter().position(|m| &m.name == n) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 5100e9784..1e05450d5 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -1934,6 +1934,8 @@ fn add_layer_file(s: &mut Session, path: &str) -> CmdResult { /// Adds a layer from a layer file's JSON. pub(crate) fn add_layer_text(s: &mut Session, text: &str) -> CmdResult { let mut layer: rustgis_map::Layer = serde_json::from_str(text).map_err(|e| format!("not a RustGIS layer file: {e}"))?; + // A binding in a file is no authority to use a login here. + crate::s3_connections::clear_bindings(layer.addresses_mut()); // A fresh id and a unique name in this map; the rest goes through project repair. let base = layer.name.clone(); let mut i = 1; @@ -1948,6 +1950,7 @@ pub(crate) fn add_layer_text(s: &mut Session, text: &str) -> CmdResult { // Only the database the added layer reads can change here. let keys: Vec<_> = s.map().layer(id).and_then(|l| l.source.as_ref()).map(super::connections::layer_keys).unwrap_or_default(); super::connections::resolve_keys(s, &keys); + crate::s3_connections::resolve_project_s3(s); super::connections::note_waiting(s); s.active_layer = Some(id); Ok(json!({"layer": id})) @@ -2148,7 +2151,9 @@ pub fn url_file_name(url: &str, bytes: &[u8]) -> Result { /// Add Data › From URL does, so a plugin, a script or the assistant can add data by address too. pub(crate) fn add_url(s: &mut Session, p: &Value) -> CmdResult { let url = ps(p, "url").map(str::trim).filter(|u| !u.is_empty()).ok_or("url is required")?; - let lower = url.to_ascii_lowercase(); + // An address read through a saved S3 connection is judged by the address it wraps; the whole + // address stays what is fetched and kept. + let lower = rustgis_data::s3_address::inner(url).to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://") || lower.starts_with("s3://")) { return Err(format!("{url}: give an http://, https:// or s3:// address (Add Data opens files)")); } @@ -2254,6 +2259,7 @@ fn add_remote(s: &mut Session, p: &Value, url: &str, src: Box bool { + let url = rustgis_data::s3_address::inner(url); let path = url.split(['?', '#']).next().unwrap_or(url).to_ascii_lowercase(); path.ends_with(".tif") || path.ends_with(".tiff") } diff --git a/crates/engine/src/cmd/plugins.rs b/crates/engine/src/cmd/plugins.rs index 8e36722e9..91ee580dd 100644 --- a/crates/engine/src/cmd/plugins.rs +++ b/crates/engine/src/cmd/plugins.rs @@ -276,6 +276,8 @@ const NEVER: &[&str] = &[ "connection.remove", "connection.login", "connection.forgetPassword", + "connection.forgetSavedKeys", + "connection.resolveS3", "connection.test", "connection.browse", "s3.connect", diff --git a/crates/engine/src/cmd/project.rs b/crates/engine/src/cmd/project.rs index 84a0898a0..efc17a59b 100644 --- a/crates/engine/src/cmd/project.rs +++ b/crates/engine/src/cmd/project.rs @@ -56,6 +56,10 @@ fn reset(s: &mut Session, p: Project) { s.history_from_file = s.project.gp_history.len(); // Zarr layers that follow the time slider read the step it is at. super::zarr::follow_time(s); + // The sources that read an S3 bucket through a saved connection wait for it (or its choice): + // nothing of the new project is fetched before. + s.s3_pending.clear(); + crate::s3_connections::resolve_project_s3(s); } fn new(s: &mut Session, p: &Value) -> CmdResult { diff --git a/crates/engine/src/cmd/s3.rs b/crates/engine/src/cmd/s3.rs index a1e8ba4b0..81ea0a3f6 100644 --- a/crates/engine/src/cmd/s3.rs +++ b/crates/engine/src/cmd/s3.rs @@ -80,6 +80,18 @@ fn credentials(p: &Value) -> Result<(Option, Option, Option } pub(super) fn connect(_s: &mut Session, p: &Value) -> CmdResult { + install(p, None) +} + +/// The login of the saved connection `id`, made live under this session's runtime token +/// `binding` in one step (see `rustgis_tiles::s3::connect_saved`): it signs only the addresses +/// that name the binding, never one found by its bucket or host. Takes the same parameters as +/// `s3.connect`. +pub(super) fn connect_saved(_s: &mut Session, p: &Value, binding: &str, id: &str) -> CmdResult { + install(p, Some((binding, id))) +} + +fn install(p: &Value, saved: Option<(&str, &str)>) -> CmdResult { let name = ps(p, "name").map(str::trim).filter(|n| !n.is_empty()).unwrap_or("default").to_string(); let endpoint = ps(p, "endpoint").map(str::trim).filter(|e| !e.is_empty()).map(str::to_string); let (creds, region, profile_endpoint, source) = credentials(p)?; @@ -106,7 +118,10 @@ pub(super) fn connect(_s: &mut Session, p: &Value) -> CmdResult { buckets, credentials: creds, }; - s3::connect(c.clone())?; + match saved { + Some((binding, id)) => s3::connect_saved(c.clone(), binding, id)?, + None => s3::connect(c.clone())?, + } let c = s3::connection(&c.name).unwrap_or(c); let mut out = c.public(); out["credentials"] = json!(source); @@ -155,6 +170,11 @@ pub(super) fn list(_s: &mut Session, p: &Value) -> CmdResult { #[cfg(not(target_arch = "wasm32"))] { let fetch = |url: &str| crate::render::rustgis_tiles::cache::download_blocking(url); + // A saved connection's login signs only the addresses that name it. + let through = |c: &s3::Connection, url: String| match s3::binding_of(&c.name) { + Some(binding) => s3::bound_url(&binding, &url), + None => Ok(url), + }; let Some(bucket) = bucket else { // A connection's buckets (ListBuckets needs its credentials). let c = named @@ -163,18 +183,26 @@ pub(super) fn list(_s: &mut Session, p: &Value) -> CmdResult { if c.credentials.is_none() { return Err(format!("'{}' is anonymous: listing its buckets needs credentials; give a bucket", c.name)); } - let buckets = s3::parse_buckets(&fetch(&s3::buckets_url(&c))?)?; + let buckets = s3::parse_buckets(&fetch(&through(&c, s3::buckets_url(&c))?)?)?; return Ok(json!({"connection": c.name, "buckets": buckets})); }; // A bucket listed through a connection is served by it from then on (its objects are read // with its keys); otherwise the bucket's own connection, if any, or anonymously. if let Some(c) = &named { - s3::grant(&c.name, &bucket); + match s3::binding_of(&c.name) { + Some(binding) => s3::grant_saved(&binding, &bucket)?, + None => s3::grant(&c.name, &bucket), + } } let conn = named.or_else(|| s3::for_bucket(&bucket)); let (mut prefixes, mut objects, mut token, mut pages) = (Vec::new(), Vec::new(), None::, 0); loop { - let page = s3::parse_list(&fetch(&s3::list_url(&bucket, &prefix, token.as_deref(), conn.as_ref()))?)?; + let url = s3::list_url(&bucket, &prefix, token.as_deref(), conn.as_ref()); + let url = match &conn { + Some(c) => through(c, url)?, + None => url, + }; + let page = s3::parse_list(&fetch(&url)?)?; prefixes.extend(page.prefixes); objects.extend(page.objects); pages += 1; @@ -186,11 +214,19 @@ pub(super) fn list(_s: &mut Session, p: &Value) -> CmdResult { let more = token.is_some() || prefixes.len() + objects.len() > limit; prefixes.truncate(limit); objects.truncate(limit.saturating_sub(prefixes.len())); + // What a saved connection lists is added through it: its objects' addresses name it. + let address = |key: &str| { + let url = format!("s3://{bucket}/{key}"); + match conn.as_ref().map(|c| through(c, url.clone())) { + Some(Ok(bound)) => bound, + _ => url, + } + }; Ok(json!({ "bucket": bucket, "prefix": prefix, "prefixes": prefixes, - "objects": objects.iter().map(|o| json!({"key": o.key, "size": o.size, "modified": o.modified, "url": format!("s3://{bucket}/{}", o.key)})).collect::>(), + "objects": objects.iter().map(|o| json!({"key": o.key, "size": o.size, "modified": o.modified, "url": address(&o.key)})).collect::>(), "truncated": more, })) } diff --git a/crates/engine/src/connections.rs b/crates/engine/src/connections.rs index 9c0ce48c7..ae4dc50c6 100644 --- a/crates/engine/src/connections.rs +++ b/crates/engine/src/connections.rs @@ -19,6 +19,11 @@ pub const DATABASE_KINDS: &[&str] = &["postgres", "mysql", "sqlserver"]; pub const S3_CREDENTIALS: &[&str] = &["keys", "profile", "environment", "none"]; pub const DESKTOP_ONLY: &str = "database connections are used by the desktop app"; +/// Whether a serialized flag is off (so it is left out). +fn is_false(value: &bool) -> bool { + !*value +} + #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Connection { pub id: String, @@ -30,6 +35,11 @@ pub struct Connection { pub address: String, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub options: BTreeMap, + /// An S3 connection whose keys the user asked to keep in the computer's credential store + /// (`connection.login` with `remember`). Only a marker, never a key; not proof that the store + /// still has them. Commands never take it from a caller. + #[serde(default, rename = "savedKeys", skip_serializing_if = "is_false")] + pub saved_keys: bool, /// A database connection's password was saved in the system's secret store (never the /// password itself, which is `DatabaseLogins`' to read). Set only by a login that asked to /// remember it, and cleared when the saved password is forgotten or no longer fits. @@ -133,6 +143,9 @@ pub fn clean(mut c: Connection) -> Result<(Connection, bool), String> { c.name = if c.kind == "s3" && c.address.is_empty() { "AWS S3".into() } else { sources::default_name(&c.address) }; } c.group = name_text(&c.group); + if c.kind != "s3" { + c.saved_keys = false; + } // Only a database connection has a password to remember. c.remembered &= DATABASE_KINDS.contains(&c.kind.as_str()); if c.kind != "s3" && !c.options.is_empty() { @@ -324,6 +337,8 @@ impl Connections { pub fn add(&mut self, c: Connection, explicit_name: bool) -> Result<(Connection, bool), String> { let (mut c, stripped) = clean(c)?; + // A new connection (or a copy) has no saved keys of its own. + c.saved_keys = false; if self.connections.len() >= MAX_CONNECTIONS { return Err(format!("up to {MAX_CONNECTIONS} connections")); } @@ -353,7 +368,10 @@ impl Connections { self.add(c, false) } - pub fn update(&mut self, id: &str, c: Connection) -> Result<(Connection, bool), String> { + /// What [`Connections::update`] would store for connection `id` given `c` (checked, named + /// uniquely, keeping its id and its saved-keys marker), and whether an address was stripped of + /// secrets, without changing anything. + pub(crate) fn prepare_update(&self, id: &str, c: Connection) -> Result<(Connection, bool), String> { let explicit_name = !c.name.trim().is_empty(); let (mut c, stripped) = clean(c)?; if explicit_name && self.name_is_id(&c.name) { @@ -364,16 +382,38 @@ impl Connections { return Err(format!("a connection named '{}' exists", c.name)); } c.name = name; - let slot = self.connections.iter_mut().find(|c| c.id == id).ok_or_else(|| format!("no connection '{id}'"))?; + let slot = self.connections.iter().find(|c| c.id == id).ok_or_else(|| format!("no connection '{id}'"))?; c.id = id.into(); - // The saved password belongs to the saved definition: an edit of the server or the user - // leaves it behind (the command deletes it first), and no edit claims one. + c.saved_keys = slot.saved_keys && c.kind == "s3"; + // Password identity includes the cleaned database kind, server and user. c.remembered = slot.remembered && slot.kind == c.kind && slot.address == c.address; - *slot = c.clone(); - self.generation = self.generation.wrapping_add(1); Ok((c, stripped)) } + /// Commits to a target located before credential deletion. No lookup or validation remains. + pub(crate) fn commit_update(slot: &mut Connection, generation: &mut u64, c: Connection) -> Connection { + *slot = c.clone(); + *generation = generation.wrapping_add(1); + c + } + + pub fn update(&mut self, id: &str, c: Connection) -> Result<(Connection, bool), String> { + let (c, stripped) = self.prepare_update(id, c)?; + let slot = self.connections.iter_mut().find(|slot| slot.id == id).ok_or_else(|| format!("no connection '{id}'"))?; + Ok((Self::commit_update(slot, &mut self.generation, c), stripped)) + } + + /// Records whether connection `id` has saved keys (the marker): after a store call said so. + /// Changing it bumps `generation`, as every change to the preference does. + pub fn set_saved_keys(&mut self, id: &str, saved: bool) { + if let Some(c) = self.connections.iter_mut().find(|c| c.id == id && c.kind == "s3") + && c.saved_keys != saved + { + c.saved_keys = saved; + self.generation = self.generation.wrapping_add(1); + } + } + pub fn remove(&mut self, id: &str) -> Option { let index = self.connections.iter().position(|c| c.id == id)?; let c = self.connections.remove(index); @@ -444,7 +484,7 @@ pub struct LoginRequest { pub prompted: bool, } -/// The account a saved connection's password lives under in the system's secret store. +/// The shared account for a saved connection's database password or S3 key pair. pub fn account(id: &str) -> String { format!("connection/{id}") } @@ -513,6 +553,10 @@ impl DatabaseLogins { self.store.is_some() } + /// The same desktop credential store for all saved connection kinds. + pub fn secret_store(&self) -> Option<&Arc> { + self.store.as_ref() + } /// Whether `id` has a password given this session. pub fn has_session_password(&self, id: &str) -> bool { self.session.contains_key(id) @@ -543,7 +587,6 @@ impl DatabaseLogins { self.reject(&c.id); } } - /// Reads the password saved for a definition (once: later calls use what was read). pub fn load(&mut self, c: &Connection) -> SavedPassword { if let Some(p) = self.saved.get(&c.id) { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index c8acff3ac..b8692e583 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -10,6 +10,10 @@ pub mod connections; pub mod geometry_calc; pub mod model; pub mod project; +pub mod s3_connections; +pub mod s3_keys; +#[cfg(test)] +mod s3_saved_tests; pub mod sample; pub mod sources; #[cfg(test)] @@ -465,9 +469,15 @@ pub struct Session { pub sources: sources::Sources, /// Saved connections (`connection.*`): kept with the app's preferences, not the project; never a secret. pub connections: connections::Connections, - /// The database logins of this run: the system's secret store (desktop app only), passwords - /// given and read, which saved connection each database uses, and the logins to ask for. Never - /// saved; see [`connections::DatabaseLogins`]. + /// The runtime token of each saved S3 connection's login in this session, by saved id (see + /// `rustgis_tiles::s3::connect_saved`): given on the first successful login, kept after. + pub(crate) s3_bindings: std::collections::BTreeMap, + /// Project sources that read an S3 bucket through a saved connection and wait for the user to + /// choose which (see [`s3_connections`]); none is fetched meanwhile. + pub s3_pending: Vec, + /// This run's connection credential store (desktop only), plus database passwords, selections + /// and login requests. S3 uses the same store through `DatabaseLogins::secret_store`. + /// Never saved; see [`connections::DatabaseLogins`]. pub database_logins: connections::DatabaseLogins, /// How rasters added from now on are drawn (`raster.setDefaultStyle`; kept with the app's /// preferences, not the project). @@ -675,6 +685,8 @@ impl Session { dirty: false, sources: sources::Sources::default(), connections: connections::Connections::default(), + s3_bindings: std::collections::BTreeMap::new(), + s3_pending: Vec::new(), database_logins: connections::DatabaseLogins::default(), raster_defaults: rustgis_map::RasterDefaults::default(), registry: Registry { commands: cmd::all() }, diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 12841ef43..c8358dbaa 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -41,8 +41,23 @@ pub const FORMAT: u32 = 1; /// Largest project file read (512 MiB). const MAX_BYTES: u64 = 512 << 20; -/// The address a saved file keeps. Only web addresses can contain layer credentials. +/// The address a saved file keeps. Only web addresses can contain layer credentials. A tile +/// template (`pmtiles:…@archive`, `cog:…@archive`) is judged by its archive, and an address read +/// through a saved S3 connection by the address it wraps, the wrapper rebuilt around it (its +/// binding and endpoint are provenance, not secrets); a COG's look after either is kept. fn saved_address(address: &str) -> Option { + use rustgis_data::s3_address; + if let Some((head, archive)) = crate::sources::template_archive(address) { + let (file, look) = split_look(archive); + return saved_address(file).map(|saved| format!("{head}{saved}{look}")); + } + if s3_address::is_wrapper(address) { + let (file, look) = split_look(address); + // One that doesn't parse stays as it is (and blocked): it names no connection. + let b = s3_address::parse(file).ok().flatten()?; + let inner = saved_address(b.url)?; + return s3_address::bind(b.binding, b.endpoint, &inner).ok().map(|saved| format!("{saved}{look}")); + } let inner = address.strip_prefix(crate::render::rustgis_tiles::s3::ANONYMOUS).unwrap_or(address); let (protocol, _) = inner.split_once("://")?; if !["http", "https", "s3"].iter().any(|s| protocol.eq_ignore_ascii_case(s)) { @@ -59,6 +74,12 @@ fn saved_address(address: &str) -> Option { (out != address).then_some(out) } +/// A COG address without its look (`…#bands=…`), and the look as written. +fn split_look(address: &str) -> (&str, &str) { + let file = crate::render::rustgis_tiles::cog::split(address).0; + (file, address.get(file.len()..).unwrap_or("")) +} + /// Strips layer credentials, keeping originals in the visitor's order for the caller to restore. /// Packaging calls this without registering the resulting text as a file this process wrote. pub(crate) fn strip_addresses(addresses: Vec<&mut String>) -> Vec> { @@ -583,15 +604,20 @@ impl Project { } if let Some(rl) = &mut l.raster && rl.data.is_none() + && !rl.path.as_deref().is_some_and(crate::s3_connections::is_s3_backed) && let Err(e) = rl.reload() { log::warn!("{}: {e}", l.name); } - // A streamed layer's features are read again from its source. + // A streamed layer's features are read again from its source. One read from an S3 + // bucket waits for [`crate::s3_connections::resolve_project_s3`], which opens it + // with the saved connection that is to read it (or, with none to choose, as + // before): reading a project fetches nothing with another session's keys. if let Some(src) = &mut l.source { l.dataset.windowed = true; if !src.is_open() && src.error.is_none() + && !src.path.as_deref().is_some_and(crate::s3_connections::is_s3_backed) && let Err(e) = src.open() { log::warn!("{}: {e}", l.name); @@ -690,6 +716,9 @@ impl Project { if restore { p.restore_addresses(text); } + // A binding is this computer's runtime token for a login: one in a file is no authority + // to use any login here. + crate::s3_connections::clear_bindings(p.addresses_mut()); p.normalize(); Ok(p) } diff --git a/crates/engine/src/s3_connections.rs b/crates/engine/src/s3_connections.rs new file mode 100644 index 000000000..ee6d7fc91 --- /dev/null +++ b/crates/engine/src/s3_connections.rs @@ -0,0 +1,391 @@ +//! Project sources that read an S3 bucket through a saved connection: which saved connection +//! reads each, decided before anything is fetched. +//! +//! A project keeps an S3 source's address as `rustgis-s3::::` +//! ([`rustgis_data::s3_address`]) when it was added through a saved connection: the endpoint and +//! bucket it names, never a key. The binding is this computer's runtime token for a login, so one +//! read from a file is cleared ([`clear_bindings`]); a source with none can't be fetched. Opening +//! a project (or importing a map) settles each bucket its sources read ([`resolve_project_s3`]): +//! +//! - exactly one saved connection could read it: it logs in (a profile, the environment, +//! anonymously, or its saved keys) and the sources are bound to that login and opened. Its +//! keys missing or the store refusing, they stay blocked, pending that connection's log in; +//! - several could: they stay blocked, [`PendingS3`] lists the candidates, and the user chooses +//! (`connection.resolveS3`). Nothing is fetched, signed or not, until then; +//! - none could: an older project's bare `s3://` address is read as it always was (anonymously, +//! or through a session-only connection); one that names a connection's endpoint stays blocked +//! with a notice. +//! +//! An older project's bare `s3://` address names no endpoint: any saved connection whose buckets +//! include (or aren't limited to) its bucket is a candidate. While it waits it is an unresolved +//! address with an empty endpoint (and so AWS, if the project is saved then). + +use std::collections::BTreeSet; + +use rustgis_data::s3_address; + +use crate::Session; +use crate::connections::Connection; +use crate::render::rustgis_tiles::{cache, s3}; + +/// A bucket (at an endpoint) some project sources read, and the saved connections that could read +/// it: the user chooses one (`connection.resolveS3`); until then none of those sources is fetched. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PendingS3 { + /// The address of one of the sources (the first of the resource), as the project keeps it. + pub source: String, + /// The endpoint the sources name: `None` is unknown (an older project's bare `s3://` + /// address), `Some("")` is AWS. + pub endpoint: Option, + pub bucket: String, + /// The ids of the saved S3 connections that may be chosen, in list order. + pub candidates: Vec, +} + +/// What is said to someone whose choice of a connection is no longer one. +pub const STALE: &str = "This S3 connection choice is no longer available."; + +impl Session { + /// The runtime token of a saved S3 connection's login in this session, once it has logged in + /// (the interface binds the addresses it adds through that connection to it). + pub fn s3_binding(&self, id: &str) -> Option<&str> { + self.s3_bindings.get(id).map(String::as_str) + } + + /// Whether this saved ID can reuse a live login. A mapped login must belong to this session; + /// ordinary session-only connections retain the existing name-based behavior. + pub fn has_s3_login(&self, id: &str) -> bool { + let Some(c) = self.connections.find(id).filter(|c| c.kind == "s3" && c.id == id) else { return false }; + s3::can_reuse_login(self.s3_binding(id), id, &c.name) + } +} + +/// Whether a source's address reads from S3: an `s3://` address, or one read through a saved +/// connection (well formed or not). +pub fn is_s3_backed(address: &str) -> bool { + s3_address::is_wrapper(address) || s3::is_s3(address) +} + +/// How many bytes of a project address come before the part that reads S3: all of a tile +/// template's (`pmtiles:{z}/{x}/{y}@`), whose archive does. +fn head_len(address: &str) -> usize { + crate::sources::template_archive(address).map_or(0, |(head, _)| head.len()) +} + +/// The S3-reading part of a project address: itself, or a tile template's archive. +fn part(address: &str) -> &str { + address.get(head_len(address)..).unwrap_or(address) +} + +/// Empties the binding of every address read through a saved connection: what a file or an +/// import brings is no login of this computer's. +pub(crate) fn clear_bindings<'a>(addresses: impl IntoIterator) { + for a in addresses { + let at = head_len(a); + let Some(p) = a.get(at..) else { continue }; + if let Some(cleared) = s3_address::rebound(p, "") + && cleared != p + { + *a = format!("{}{cleared}", a.get(..at).unwrap_or("")); + } + } +} + +/// What an address reads, when it is an S3 one that no login has been chosen for. +struct Unresolved { + /// The canonical endpoint it names (empty: AWS), or `None` for an older project's bare + /// `s3://` address, which names none. + endpoint: Option, + bucket: String, +} + +/// `address` as [`Unresolved`], when it is one. `legacy` holds the buckets of older addresses +/// waiting for a choice, whose placeholder (no endpoint, no binding) is not AWS's. +fn unresolved(address: &str, legacy: &BTreeSet) -> Option { + let p = part(address); + match s3_address::parse(p) { + Ok(Some(b)) => { + if !b.binding.is_empty() { + return None; + } + let endpoint = s3::canonical(b.endpoint).ok()?.unwrap_or_default(); + let bucket = s3::bucket_of(&endpoint, b.url)?; + let endpoint = if endpoint.is_empty() && legacy.contains(&bucket) { None } else { Some(endpoint) }; + Some(Unresolved { endpoint, bucket }) + } + Ok(None) if s3::is_s3(p) => Some(Unresolved { endpoint: None, bucket: s3::parse(p).ok()?.0 }), + _ => None, + } +} + +/// `address` read through the login `binding` of a connection at `endpoint` (empty: AWS). +fn bound(address: &str, endpoint: &str, binding: &str) -> Option { + let at = head_len(address); + let p = address.get(at..)?; + let inner = match s3_address::parse(p).ok().flatten() { + Some(b) => b.url, + None => p, + }; + let wrapped = s3_address::bind(binding, endpoint, inner).ok()?; + Some(format!("{}{wrapped}", address.get(..at).unwrap_or(""))) +} + +/// A saved connection's canonical endpoint (empty: AWS), or `None` when it has none that can be +/// used. +fn endpoint_of(c: &Connection) -> Option { + s3::canonical(&c.address).ok().map(Option::unwrap_or_default) +} + +/// Whether the saved connection could read `bucket` at `endpoint` (`None`: any): an S3 one at +/// that endpoint whose buckets include it, or that names none (a scope the user chooses within, +/// not a grant). +fn matches(c: &Connection, endpoint: &Option, bucket: &str) -> bool { + if c.kind != "s3" { + return false; + } + let scope = c.options.get("buckets").map(String::as_str).unwrap_or(""); + let in_scope = scope.trim().is_empty() || scope.split(',').any(|b| b.trim() == bucket); + in_scope && endpoint.as_ref().is_none_or(|e| endpoint_of(c).as_ref() == Some(e)) +} + +/// A resource: the bucket some sources read, at an endpoint. +struct Group { + endpoint: Option, + bucket: String, +} + +impl Group { + fn has(&self, address: &str, legacy: &BTreeSet) -> bool { + unresolved(address, legacy).is_some_and(|u| u.endpoint == self.endpoint && u.bucket == self.bucket) + } +} + +fn legacy_buckets(s: &Session) -> BTreeSet { + s.s3_pending.iter().filter(|p| p.endpoint.is_none()).map(|p| p.bucket.clone()).collect() +} + +/// Settles the S3 sources of the project: see the module's text. Called when a project opens +/// and when maps or layers are imported, never from a frame. +pub(crate) fn resolve_project_s3(s: &mut Session) { + let legacy = legacy_buckets(s); + let mut groups: Vec = Vec::new(); + for a in s.project.addresses_mut() { + if let Some(u) = unresolved(a, &legacy) + && !groups.iter().any(|g| g.endpoint == u.endpoint && g.bucket == u.bucket) + { + groups.push(Group { endpoint: u.endpoint, bucket: u.bucket }); + } + } + let mut pending = Vec::new(); + for g in &groups { + if let Some(p) = settle(s, g, &legacy) { + pending.push(p); + } + } + s.s3_pending = pending; +} + +/// What `g`'s sources do now: bound to its one candidate, kept for a choice, or as they were. +fn settle(s: &mut Session, g: &Group, legacy: &BTreeSet) -> Option { + let candidates: Vec = s.connections.connections.iter().filter(|c| matches(c, &g.endpoint, &g.bucket)).cloned().collect(); + let at = match g.endpoint.as_deref() { + Some(e) if !e.is_empty() => format!(" at {e}"), + _ => String::new(), + }; + match candidates.as_slice() { + [] => { + if g.endpoint.is_none() { + // An older project's bare address, and no saved connection to read it: as it + // always was (anonymously, or through a connection of this session). + reopen_where(s, |a| g.has(a, legacy)); + } else { + let n = block(s, g, legacy, ""); + s.notify( + format!("{n} source(s) read the S3 bucket '{}'{at}, and no saved connection does: add one under Data Sources › Connections, then open the project again", g.bucket), + true, + ); + } + None + } + [c] => match bind_group(s, g, legacy, c) { + Ok(_) => None, + Err(e) => { + // Kept blocked, naming the one connection that could read it (an older project's + // address now names its endpoint). + let endpoint = endpoint_of(c).unwrap_or_default(); + let source = rewrite(s, g, legacy, &endpoint, "").into_iter().next()?; + block_by(s, &source, &endpoint); + s.notify( + format!("Log in to the saved S3 connection '{}' to open the sources that read the bucket '{}': {e}", c.name, g.bucket), + false, + ); + Some(PendingS3 { source, endpoint: Some(endpoint), bucket: g.bucket.clone(), candidates: vec![c.id.clone()] }) + } + }, + many => { + let endpoint = g.endpoint.clone().unwrap_or_default(); + let source = rewrite(s, g, legacy, &endpoint, "").into_iter().next()?; + block_by(s, &source, &endpoint); + s.notify( + format!( + "{} saved S3 connections could read the bucket '{}'{at}: choose which one reads this project's sources (Data Sources › Choose S3 connection…); nothing is read until you do", + many.len(), + g.bucket + ), + false, + ); + Some(PendingS3 { + source, + endpoint: g.endpoint.clone(), + bucket: g.bucket.clone(), + candidates: many.iter().map(|c| c.id.clone()).collect(), + }) + } + } +} + +/// Rewrites the unresolved addresses of `g` to read through `binding` at `endpoint`: the new +/// addresses, in project order. +fn rewrite(s: &mut Session, g: &Group, legacy: &BTreeSet, endpoint: &str, binding: &str) -> Vec { + let mut out = Vec::new(); + for a in s.project.addresses_mut() { + if g.has(a, legacy) + && let Some(new) = bound(a, endpoint, binding) + { + *a = new.clone(); + out.push(new); + } + } + out +} + +/// Marks the streamed sources of the address that wait for a choice (their layers show why). +fn block_by(s: &mut Session, source: &str, endpoint: &str) { + let Some(want) = unresolved(source, &BTreeSet::new()) else { return }; + for src in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.source.as_mut()) { + let Some(path) = src.path.as_deref() else { continue }; + if !src.is_open() + && unresolved(path, &BTreeSet::new()).is_some_and(|u| u.bucket == want.bucket) + && s3_address::parse(part(path)).ok().flatten().is_some_and(|b| b.endpoint == endpoint) + { + src.error = Some(s3::UNRESOLVED.into()); + } + } +} + +/// Marks the streamed sources of `g` as blocked for want of any connection; how many addresses +/// of `g` there are. +fn block(s: &mut Session, g: &Group, legacy: &BTreeSet, why: &str) -> usize { + let why = if why.is_empty() { s3::UNRESOLVED } else { why }; + for src in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.source.as_mut()) { + if !src.is_open() && src.path.as_deref().is_some_and(|p| g.has(p, legacy)) { + src.error = Some(why.into()); + } + } + s.project.addresses_mut().into_iter().filter(|a| g.has(a, legacy)).count() +} + +/// Opens (again) the streamed sources whose address `which` accepts; how many opened and how +/// many failed. +fn reopen_where(s: &mut Session, which: impl Fn(&str) -> bool) -> (usize, usize) { + let (mut opened, mut failed) = (0, 0); + for src in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.source.as_mut()) { + let Some(path) = src.path.clone() else { continue }; + if (!src.is_open() || src.error.is_some()) && which(&path) { + match src.reopen() { + Ok(()) => opened += 1, + Err(_) => failed += 1, + } + } + } + for raster in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.raster.as_mut()) { + if raster.data.is_none() && raster.time_zarr.is_none() && raster.path.as_deref().is_some_and(&which) { + match raster.reload() { + Ok(()) => opened += 1, + Err(_) => failed += 1, + } + } + } + (opened, failed) +} + +/// Binds `g`'s sources to the saved connection `c`: logs it in if it isn't, lets its login read +/// the bucket (the user's choice, or its only candidate's), rewrites the addresses to name it and +/// reads them. How many sources opened and how many failed. +fn bind_group(s: &mut Session, g: &Group, legacy: &BTreeSet, c: &Connection) -> Result<(usize, usize), String> { + let endpoint = endpoint_of(c).ok_or_else(|| s3::MISMATCH.to_string())?; + let binding = crate::cmd::connections::login_saved(s, c)?; + s3::grant_saved(&binding, &g.bucket)?; + let addresses = rewrite(s, g, legacy, &endpoint, &binding); + // What failed before at an identical address is read again with this login. + for a in &addresses { + cache::forget(part(a)); + } + let affected = |address: &str| addresses.iter().any(|a| a == address); + let (mut opened, failed) = reopen_where(s, affected); + // Tile resources fetch as they draw, by their new address. + opened += + s.project.maps.iter().flat_map(|m| &m.tiles).filter(|t| affected(&t.url) || t.time_url.as_deref().is_some_and(affected)).count(); + Ok((opened, failed)) +} + +/// The user's choice of the saved connection `id` to read the sources of the pending resource +/// that `source` is the address of (`connection.resolveS3`). The candidates are judged again +/// against the connections as they are now, before anything happens; a stale choice does nothing. +pub(crate) fn choose(s: &mut Session, source: &str, id: &str) -> crate::CmdResult { + let Some(index) = s.s3_pending.iter().position(|p| p.source == source) else { return Err(STALE.into()) }; + let pending = s.s3_pending.get(index).cloned().ok_or(STALE)?; + let c = s.connections.find(id).filter(|c| c.id == id && matches(c, &pending.endpoint, &pending.bucket)).cloned().ok_or(STALE)?; + let legacy = legacy_buckets(s); + let g = Group { endpoint: pending.endpoint.clone(), bucket: pending.bucket.clone() }; + if !s.project.addresses_mut().into_iter().any(|a| g.has(a, &legacy)) { + s.s3_pending.remove(index); + return Err(STALE.into()); + } + let (reopened, failed) = bind_group(s, &g, &legacy, &c)?; + s.s3_pending.remove(index); + Ok(serde_json::json!({"id": c.id, "reopened": reopened, "failed": failed})) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + /// Whether `address` reads through the login `binding`. + fn has_binding(address: &str, binding: &str) -> bool { + s3_address::parse(part(address)).ok().flatten().is_some_and(|b| b.binding == binding) + } + + const B: &str = "0123456789abcdef"; + + #[test] + fn s3_saved_template_archives_and_wrappers_are_found_inside_addresses() { + let wrapped = s3_address::bind(B, "https://s3.example.org", "https://s3.example.org/data/a.pmtiles").unwrap(); + let template = format!("pmtiles:{{z}}/{{x}}/{{y}}@{wrapped}"); + assert_eq!(part(&template), wrapped); + assert_eq!(part("s3://data/a.fgb"), "s3://data/a.fgb"); + let mut addresses = [template.clone(), "s3://data/b.fgb".to_string(), wrapped.clone(), "https://x.org/a".to_string()]; + clear_bindings(addresses.iter_mut()); + let none = BTreeSet::new(); + assert_eq!(addresses[0], "pmtiles:{z}/{x}/{y}@rustgis-s3::22:https://s3.example.org:https://s3.example.org/data/a.pmtiles"); + assert_eq!(addresses[1], "s3://data/b.fgb"); + let u = unresolved(&addresses[0], &none).unwrap(); + assert_eq!((u.endpoint.as_deref(), u.bucket.as_str()), (Some("https://s3.example.org"), "data")); + let u = unresolved(&addresses[1], &none).unwrap(); + assert_eq!((u.endpoint, u.bucket.as_str()), (None, "data")); + assert!(unresolved(&addresses[3], &none).is_none()); + // A bound address is resolved; rebinding keeps its template and inner address. + assert!(unresolved(&wrapped, &none).is_none()); + let again = bound(&addresses[0], "https://s3.example.org", B).unwrap(); + assert_eq!(again, template); + assert!(has_binding(&again, B) && !has_binding(&addresses[0], B)); + let bare = bound("s3://data/b.fgb", "", B).unwrap(); + assert_eq!(bare, format!("rustgis-s3:{B}:0::s3://data/b.fgb")); + // An older address's placeholder is AWS's unless an older one is waiting for its bucket. + let placeholder = "rustgis-s3::0::s3://data/b.fgb"; + assert_eq!(unresolved(placeholder, &none).unwrap().endpoint.as_deref(), Some("")); + assert_eq!(unresolved(placeholder, &BTreeSet::from(["data".to_string()])).unwrap().endpoint, None); + // Malformed provenance is nobody's resource and never a file. + assert!(unresolved("rustgis-s3:junk", &none).is_none() && is_s3_backed("rustgis-s3:junk")); + } +} diff --git a/crates/engine/src/s3_keys.rs b/crates/engine/src/s3_keys.rs new file mode 100644 index 000000000..44a5ef1b8 --- /dev/null +++ b/crates/engine/src/s3_keys.rs @@ -0,0 +1,152 @@ +//! Opt-in storage of a saved S3 connection's access key in the desktop credential store +//! ([`rustgis_data::secrets`], supplied through [`crate::connections::DatabaseLogins`]). +//! +//! One opaque secret per saved connection, at the account `connection/`: the JSON +//! `{"endpoint": , "accessKeyId": …, "secretAccessKey": …}`. +//! The endpoint travels with the keys so a connection whose address was changed outside the app +//! can never send them somewhere else. Nothing here prints, returns or logs a key: the decoded +//! payload has no `Debug`, and every failure is a fixed sentence. +//! +//! A temporary credential (one with a session token) is never stored, and the browser and the +//! command line have no store at all: they keep keys for the session, as before. + +use std::sync::Arc; + +use rustgis_data::secrets::{Secret, SecretStore}; +use serde::{Deserialize, Serialize}; + +use crate::connections::{Connection, account}; +use crate::render::rustgis_tiles::s3; + +/// A stored value that can't be used (damaged, for another endpoint, or not S3 keys). +pub const UNUSABLE: &str = "Saved S3 keys could not be used. Enter the keys again."; +/// Nothing is saved for the connection. +pub const NONE_SAVED: &str = "No saved S3 keys. Enter accessKeyId and secretAccessKey."; +/// There is no credential store, or it refused. +pub const UNAVAILABLE: &str = "Saved S3 keys are unavailable. Enter the keys for this session."; +/// The login worked, the keys weren't kept. +pub const NOT_REMEMBERED: &str = "Keys are available for this session but were not remembered on this computer."; +/// A temporary credential isn't kept. +pub const TEMPORARY: &str = "Temporary credentials are session-only. Previously saved keys are unchanged."; +/// Saved keys couldn't be deleted, so what needed them gone didn't happen. +pub const NOT_FORGOTTEN: &str = "Saved S3 keys could not be forgotten. The connection was not changed."; + +/// The most bytes a stored value may have. +const MAX_SECRET: usize = 8192; + +#[derive(Serialize)] +struct Stored<'a> { + endpoint: Option<&'a str>, + #[serde(rename = "accessKeyId")] + access_key_id: &'a str, + #[serde(rename = "secretAccessKey")] + secret_access_key: &'a str, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Loaded { + // AWS is explicit null, not an omitted identity field. + #[serde(deserialize_with = "Deserialize::deserialize")] + endpoint: Option, + #[serde(rename = "accessKeyId")] + access_key_id: String, + #[serde(rename = "secretAccessKey")] + secret_access_key: String, +} + +/// The canonical endpoint a saved connection's address stands for (`None`: AWS). +fn endpoint_of(c: &Connection) -> Result, &'static str> { + s3::canonical(&c.address).map_err(|_| UNUSABLE) +} + +fn encode(endpoint: Option<&str>, key: &str, secret: &str) -> Result { + let text = serde_json::to_string(&Stored { endpoint, access_key_id: key, secret_access_key: secret }).map_err(|_| UNUSABLE)?; + if text.len() > MAX_SECRET { + return Err(UNUSABLE); + } + Ok(Secret::new(text)) +} + +fn decode(value: &Secret, endpoint: Option<&str>) -> Result<(String, String), &'static str> { + let text = value.expose(); + if text.len() > MAX_SECRET { + return Err(UNUSABLE); + } + // A decoding error would quote the text: its details are dropped. + let l: Loaded = serde_json::from_str(text).map_err(|_| UNUSABLE)?; + if l.endpoint.as_deref() != endpoint { + return Err(UNUSABLE); + } + let credentials = s3::Credentials { access_key_id: l.access_key_id, secret_access_key: l.secret_access_key, session_token: None }; + s3::check_credentials(&credentials).map_err(|_| UNUSABLE)?; + Ok((credentials.access_key_id, credentials.secret_access_key)) +} + +/// What reading a connection's saved keys gave. +pub enum Loading { + /// The access key id and the secret access key. + Keys(String, String), + /// The store has none for it. + Missing, + /// There is no store, or it refused. + Unavailable, + /// What it holds can't be used. + Unusable, +} + +type Store<'a> = Option<&'a Arc>; + +/// Reads the keys saved for `c` (a user action: this may ask the system to unlock its store). +pub fn load(store: Store<'_>, c: &Connection) -> Loading { + let Some(store) = store else { return Loading::Unavailable }; + let value = match store.load(&account(&c.id)) { + Ok(Some(v)) => v, + Ok(None) => return Loading::Missing, + Err(_) => return Loading::Unavailable, + }; + let Ok(endpoint) = endpoint_of(c) else { return Loading::Unusable }; + match decode(&value, endpoint.as_deref()) { + Ok((key, secret)) => Loading::Keys(key, secret), + Err(_) => Loading::Unusable, + } +} + +/// Keeps the keys for `c`, replacing any earlier ones; whether they were kept. +pub fn save(store: Store<'_>, c: &Connection, key: &str, secret: &str) -> bool { + let Some(store) = store else { return false }; + let Ok(endpoint) = endpoint_of(c) else { return false }; + let Ok(value) = encode(endpoint.as_deref(), key, secret) else { return false }; + store.save(&account(&c.id), &value).is_ok() +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn s3_saved_envelope_is_bound_to_its_endpoint_and_never_echoes_values() { + let v = encode(Some("https://s3.example.org"), "AKIDEXAMPLE", "very-secret-value").unwrap(); + assert_eq!(decode(&v, Some("https://s3.example.org")).unwrap(), ("AKIDEXAMPLE".to_string(), "very-secret-value".to_string())); + assert_eq!(decode(&v, None), Err(UNUSABLE)); + assert_eq!(decode(&v, Some("https://other.example.org")), Err(UNUSABLE)); + let aws = encode(None, "AKIDEXAMPLE", "very-secret-value").unwrap(); + assert!(decode(&aws, None).is_ok() && decode(&aws, Some("https://s3.example.org")).is_err()); + for bad in [ + "not json with very-secret-value", + r#"{"accessKeyId":"A","secretAccessKey":"S"}"#, + r#"{"endpoint":null,"accessKeyId":"A"}"#, + r#"{"endpoint":null,"accessKeyId":"A","secretAccessKey":"S","extra":1}"#, + r#"{"endpoint":null,"accessKeyId":"","secretAccessKey":"S"}"#, + r#"{"endpoint":null,"accessKeyId":"has space","secretAccessKey":"S"}"#, + r#"{"endpoint":null,"accessKeyId":7,"secretAccessKey":"S"}"#, + "[]", + ] { + assert_eq!(decode(&Secret::new(bad), None), Err(UNUSABLE), "{bad}"); + } + let long = format!(r#"{{"endpoint":null,"accessKeyId":"A","secretAccessKey":"{}"}}"#, "s".repeat(MAX_SECRET)); + assert_eq!(decode(&Secret::new(long), None), Err(UNUSABLE)); + assert_eq!(account("c12"), "connection/c12"); + } +} diff --git a/crates/engine/src/s3_saved_tests.rs b/crates/engine/src/s3_saved_tests.rs new file mode 100644 index 000000000..6a9f0f4e1 --- /dev/null +++ b/crates/engine/src/s3_saved_tests.rs @@ -0,0 +1,1155 @@ +//! Saved S3 connections: keys kept in the credential store only on request, and sources read +//! through the exact login that was chosen for them. +//! +//! Each test uses names, buckets and a server of its own: the S3 login registry is shared by +//! every test of the process. + +use std::collections::BTreeSet; +use std::io::{Read, Write}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use rustgis_data::secrets::{MemorySecrets, Secret, SecretError, SecretStore}; +use serde_json::{Value, json}; + +use crate::connections::Connections; +use crate::render::rustgis_tiles::{cache, s3}; +use crate::{Project, Session, s3_keys}; + +/// A credential store that remembers what it was asked and can refuse to save or delete. +#[derive(Default)] +struct Spy { + inner: MemorySecrets, + ops: Mutex>, + refuse_save: AtomicBool, + refuse_delete: AtomicBool, +} + +impl Spy { + fn ops(&self) -> Vec { + self.ops.lock().unwrap().clone() + } + + fn count(&self, op: &str) -> usize { + self.ops().iter().filter(|o| o.starts_with(op)).count() + } + + fn raw(&self, id: &str) -> Option { + self.inner.load(&crate::connections::account(id)).unwrap().map(|s| s.expose().to_string()) + } +} + +impl SecretStore for Spy { + fn load(&self, account: &str) -> Result, SecretError> { + self.ops.lock().unwrap().push(format!("load {account}")); + self.inner.load(account) + } + + fn save(&self, account: &str, value: &Secret) -> Result<(), SecretError> { + self.ops.lock().unwrap().push(format!("save {account}")); + if self.refuse_save.load(Ordering::Relaxed) { + return Err(SecretError::Denied); + } + self.inner.save(account, value) + } + + fn delete(&self, account: &str) -> Result<(), SecretError> { + self.ops.lock().unwrap().push(format!("delete {account}")); + if self.refuse_delete.load(Ordering::Relaxed) { + return Err(SecretError::Denied); + } + self.inner.delete(account) + } +} + +/// A stand-in S3 endpoint with one bucket (path-style): objects with byte ranges, a listing, +/// redirects, and a 404 that echoes the request's Authorization header back, as a hostile server +/// might. Every request's head is kept. +struct Server { + base: String, + log: Arc>>, +} + +impl Server { + fn requests(&self) -> usize { + self.log.lock().unwrap().len() + } + + /// The access key ids that signed the requests after the first `from`. + fn key_ids(&self, from: usize) -> BTreeSet { + self.log + .lock() + .unwrap() + .iter() + .skip(from) + .filter_map(|h| h.split("Credential=").nth(1).map(|c| c.split('/').next().unwrap_or("").to_string())) + .collect() + } + + /// How many requests after the first `from` carried no Authorization header. + fn unsigned(&self, from: usize) -> usize { + self.log.lock().unwrap().iter().skip(from).filter(|h| !h.to_ascii_lowercase().contains("authorization:")).count() + } + + /// Whether any request carried `text`. + fn saw(&self, text: &str) -> bool { + self.log.lock().unwrap().iter().any(|h| h.contains(text)) + } +} + +fn serve_bucket(bucket: &str, objects: Vec<(&str, Vec)>, redirects: Vec<(&str, String)>) -> Server { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let log = Arc::new(Mutex::new(Vec::new())); + let state = Arc::new(( + bucket.to_string(), + objects.into_iter().map(|(k, b)| (k.to_string(), b)).collect::>(), + redirects.into_iter().map(|(k, l)| (k.to_string(), l)).collect::>(), + )); + let seen = log.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { continue }; + let (state, seen) = (state.clone(), seen.clone()); + std::thread::spawn(move || answer(stream, &state.0, &state.1, &state.2, &seen)); + } + }); + Server { base, log } +} + +type Objects = [(String, Vec)]; + +fn answer(mut s: std::net::TcpStream, bucket: &str, objects: &Objects, redirects: &[(String, String)], seen: &Mutex>) { + let mut buf = [0u8; 8192]; + let mut head = Vec::new(); + while !head.windows(4).any(|w| w == b"\r\n\r\n") { + let n = s.read(&mut buf).unwrap_or(0); + if n == 0 { + break; + } + head.extend_from_slice(&buf[..n]); + } + let req = String::from_utf8_lossy(&head).to_string(); + let path = req.split_whitespace().nth(1).unwrap_or("/").to_string(); + let lower = req.to_ascii_lowercase(); + let range = lower.lines().find_map(|l| l.strip_prefix("range: bytes=")).and_then(|r| { + let (a, b) = r.trim().split_once('-')?; + Some((a.parse::().ok(), b.parse::().ok())) + }); + seen.lock().unwrap().push(req.clone()); + let bare = path.split('?').next().unwrap_or(""); + let key = bare.strip_prefix(&format!("/{bucket}/")).unwrap_or(""); + let auth = req.lines().find(|l| l.to_ascii_lowercase().starts_with("authorization:")).unwrap_or("").to_string(); + let mut extra = String::new(); + let (status, body): (String, Vec) = if bare == format!("/{bucket}") || (bare == format!("/{bucket}/") && path.contains("list-type")) + { + let contents: String = + objects.iter().map(|(k, b)| format!("{k}{}", b.len())).collect(); + ( + "200 OK".into(), + format!("{bucket}false{contents}").into_bytes(), + ) + } else if bare == "/" { + ( + "200 OK".into(), + format!("{bucket}") + .into_bytes(), + ) + } else if let Some((_, location)) = redirects.iter().find(|(k, _)| k == key) { + extra = format!("Location: {location}\r\n"); + ("301 Moved Permanently".into(), Vec::new()) + } else if let Some((_, whole)) = objects.iter().find(|(k, _)| k == key) { + match range { + Some((Some(a), Some(b))) if a < whole.len() => ("206 Partial Content".into(), whole[a..=b.min(whole.len() - 1)].to_vec()), + Some((None, Some(n))) => { + let n = n.min(whole.len()); + ("206 Partial Content".into(), whole[whole.len() - n..].to_vec()) + } + Some(_) => ("416 Range Not Satisfiable".into(), Vec::new()), + None => ("200 OK".into(), whole.clone()), + } + } else { + ("404 Not Found".into(), format!("NoSuchKey{auth}").into_bytes()) + }; + let _ = write!(s, "HTTP/1.1 {status}\r\n{extra}Content-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = s.write_all(&body); +} + +/// A FlatGeobuf of a grid of points (attribute `n`), big enough to be read in place by range. +fn grid_fgb() -> Vec { + use rustgis_data::{Coord, Dataset, Field, FieldType, Geometry, GeometryType, Value as V}; + let mut ds = Dataset::new("grid", GeometryType::Point, vec![Field::new("n", FieldType::Integer)]); + for i in 0..200u32 { + for j in 0..200u32 { + let c = Coord::new(-179.0 + 1.79 * f64::from(i), -80.0 + 0.8 * f64::from(j)); + ds.push(Some(Geometry::Point(c)), vec![V::Int(i64::from(i * 200 + j))]); + } + } + rustgis_data::source::fgb::write(&ds).unwrap() +} + +/// A vector PMTiles archive: one park at zoom 0. +fn parks_pmtiles() -> Vec { + use crate::render::rustgis_tiles::{mvt, pmtiles}; + let park = mvt::Feature { + id: Some(3), + kind: mvt::GeomKind::Polygon, + geometry: vec![vec![[1024.0, 1024.0], [3072.0, 1024.0], [3072.0, 3072.0], [1024.0, 3072.0], [1024.0, 1024.0]]], + props: vec![("name".into(), mvt::Value::String("Central".into())), ("visitors".into(), mvt::Value::Number(1200.0))], + }; + let tile = mvt::encode(&mvt::Tile { layers: vec![mvt::Layer { name: "parks".into(), extent: 4096, features: vec![park] }] }); + let new = pmtiles::NewArchive { + tile_type: pmtiles::TileType::Mvt, + gzipped_tiles: false, + bounds: [-180.0, -85.0, 180.0, 85.0], + metadata: json!({"vector_layers": [{"id": "parks"}]}), + }; + pmtiles::write(vec![(0, 0, 0, tile)], &new).unwrap() +} + +fn session(store: Option<&Arc>) -> Session { + let mut s = Session::new(Project::default()); + s.database_logins.set_store(store.map(|st| st.clone() as Arc)); + s +} + +/// A saved S3 connection at `base` for `bucket`; its id. +fn saved(s: &mut Session, name: &str, base: &str, bucket: &str, credentials: &str) -> String { + let r = s + .run( + "connection.new", + &json!({"kind": "s3", "name": name, "address": base, "options": {"credentials": credentials, "buckets": bucket}}), + ) + .unwrap(); + r["id"].as_str().unwrap().to_string() +} + +fn login(s: &mut Session, id: &str, key: &str, secret: &str, remember: bool) -> Result { + s.run("connection.login", &json!({"id": id, "accessKeyId": key, "secretAccessKey": secret, "remember": remember})) +} + +fn live_key(name: &str) -> Option { + s3::connection(name).and_then(|c| c.credentials).map(|c| c.access_key_id) +} + +/// Waits for a streamed layer's view to be read; how many features it holds. +fn read_view(s: &mut Session, id: u32) -> usize { + let view = rustgis_geom::Extent::new(10.0, 10.0, 14.0, 13.0); + let scale = s.map().scale(); + let mut pending = s.map_mut().sync_sources(&view, scale, false); + let start = std::time::Instant::now(); + while pending && start.elapsed() < std::time::Duration::from_secs(30) { + std::thread::sleep(std::time::Duration::from_millis(5)); + pending = s.map_mut().sync_sources(&view, scale, false); + } + s.map().layer(id).map_or(0, |l| l.dataset.features.len()) +} + +/// The layer whose streamed source reads `name` (the end of its address). +fn streamed(s: &Session, ends: &str) -> u32 { + s.map().layers.iter().find(|l| l.source.as_ref().and_then(|x| x.path.as_deref()).is_some_and(|p| p.ends_with(ends))).unwrap().id +} + +fn bound(s: &Session, id: &str, url: &str) -> String { + s3::bound_url(s.s3_binding(id).unwrap(), url).unwrap() +} + +#[test] +fn s3_saved_keys_are_remembered_only_on_request_and_reused_by_a_fresh_session() { + let srv = serve_bucket("sv-remember", vec![("grid.fgb", grid_fgb())], vec![]); + let store = Arc::new(Spy::default()); + let mut a = session(Some(&store)); + let id = saved(&mut a, "sv-remember-conn", &srv.base, "sv-remember", "keys"); + assert_eq!(id, "c1"); + // Logging in is session-only unless asked: nothing is stored. + let r = login(&mut a, &id, "SAVEDKEYA", "saved-secret-A", false).unwrap(); + assert_eq!((r["loggedIn"].as_bool(), r["savedKeys"].as_bool()), (Some(true), Some(false)), "{r}"); + assert!(store.raw(&id).is_none() && store.count("save") == 0); + assert!(a.connections.to_json().find("savedKeys").is_none()); + // Remembered on request: both keys and the canonical endpoint, in the store only. + let r = login(&mut a, &id, "SAVEDKEYA", "saved-secret-A", true).unwrap(); + assert_eq!((r["loggedIn"].as_bool(), r["savedKeys"].as_bool()), (Some(true), Some(true)), "{r}"); + assert!(r.get("warning").is_none() && !r.to_string().contains("saved-secret"), "{r}"); + let stored: Value = serde_json::from_str(&store.raw(&id).unwrap()).unwrap(); + assert_eq!(stored, json!({"endpoint": srv.base, "accessKeyId": "SAVEDKEYA", "secretAccessKey": "saved-secret-A"})); + // The preference holds the marker, never a key; lists never ask the store. + let prefs = a.connections.to_json(); + assert!(prefs.contains("\"savedKeys\":true") && !prefs.contains("SAVEDKEYA") && !prefs.contains("saved-secret"), "{prefs}"); + let loads = store.count("load"); + let list = a.run("connection.list", &json!({})).unwrap(); + assert_eq!(list["connections"][0]["savedKeys"], true); + assert_eq!(store.count("load"), loads); + // A restart: the definitions and the store survive, the login doesn't. + s3::disconnect("sv-remember-conn"); + let mut b = session(Some(&store)); + b.connections = Connections::from_json(&prefs); + assert!(b.connections.find("c1").is_some_and(|c| c.saved_keys)); + let r = b.run("connection.list", &json!({})).unwrap(); + assert_eq!(r["connections"][0]["loggedIn"], false); + let r = b.run("connection.login", &json!({"id": id})).unwrap(); + assert_eq!((r["loggedIn"].as_bool(), r["savedKeys"].as_bool()), (Some(true), Some(true)), "{r}"); + // Two sessions' ids are alike (`c1`); their runtime tokens are not. + assert_ne!(a.s3_binding(&id), b.s3_binding(&id)); + assert!(b.s3_binding(&id).is_some()); + // Browsed and added through the login: the server sees the saved access key id. + let v = b.run("connection.browse", &json!({"id": id, "node": "s3:sv-remember/"})).unwrap(); + let add = v["children"].as_array().unwrap().iter().find(|c| c["name"] == "grid.fgb").unwrap()["add"].clone(); + assert!(add["params"]["url"].as_str().unwrap().starts_with("rustgis-s3:"), "{add}"); + let r = b.run(add["command"].as_str().unwrap(), &add["params"]).unwrap(); + assert_eq!((r["streamed"].as_bool(), r["features"].as_u64()), (Some(true), Some(40_000)), "{r}"); + assert_eq!(srv.key_ids(0), BTreeSet::from(["SAVEDKEYA".to_string()])); + assert!(!srv.saw("saved-secret-A"), "no secret reaches the server in a header"); + assert_eq!(srv.unsigned(0), 0); + s3::disconnect("sv-remember-conn"); +} + +#[test] +fn s3_saved_login_rules_sources_temporary_credentials_and_bad_values() { + let srv = serve_bucket("sv-rules", vec![], vec![]); + let store = Arc::new(Spy::default()); + let mut s = session(Some(&store)); + let id = saved(&mut s, "sv-rules-keys", &srv.base, "sv-rules", "keys"); + let typed = |extra: Value| { + let mut p = json!({"id": id, "accessKeyId": "SAVEDKEYA", "secretAccessKey": "saved-secret-A"}); + for (k, v) in extra.as_object().unwrap() { + p[k] = v.clone(); + } + p + }; + // A remember that isn't true or false is refused before anything happens. + for junk in [json!("yes"), json!(1), json!(null), json!([true]), json!({})] { + let e = s.run("connection.login", &typed(json!({"remember": junk}))).unwrap_err(); + assert_eq!(e, "remember is true or false"); + } + assert!(s3::connection("sv-rules-keys").is_none() && store.ops().is_empty()); + // It keeps a complete typed pair, nothing else: no half pair, no token alone, no stored fallback. + let remember_only = s.run("connection.login", &json!({"id": id, "remember": true})).unwrap_err(); + assert!(remember_only.contains("remember needs"), "{remember_only}"); + for p in [json!({"id": id, "accessKeyId": "SAVEDKEYA", "remember": true}), json!({"id": id, "secretAccessKey": "saved-secret-A"})] { + assert!(s.run("connection.login", &p).unwrap_err().contains("both accessKeyId and secretAccessKey")); + } + let token_only = s.run("connection.login", &json!({"id": id, "sessionToken": "tok"})).unwrap_err(); + assert!(token_only.contains("session token needs"), "{token_only}"); + assert!(s3::connection("sv-rules-keys").is_none() && store.ops().is_empty()); + // Not for sources that don't type keys, nor for other kinds. + let none = saved(&mut s, "sv-rules-none", &srv.base, "sv-rules", "none"); + assert!(s.run("connection.login", &json!({"id": none, "accessKeyId": "K", "secretAccessKey": "S", "remember": true})).is_err()); + let wms = s.run("connection.new", &json!({"kind": "wms", "name": "sv-rules-wms", "address": "http://127.0.0.1:9/wms"})).unwrap(); + assert!(s.run("connection.login", &json!({"id": wms["id"], "remember": true})).is_err()); + assert!(s3::connection("sv-rules-none").is_none() && store.ops().is_empty()); + // A damaged value, and one for another endpoint, are refused in fixed words. + store.inner.save(&crate::connections::account(&id), &Secret::new("junk with saved-secret-A")).unwrap(); + let e = s.run("connection.login", &json!({"id": id})).unwrap_err(); + assert_eq!(e, s3_keys::UNUSABLE); + let other = json!({"endpoint": "http://127.0.0.1:9", "accessKeyId": "OTHERKEY", "secretAccessKey": "other-secret"}).to_string(); + store.inner.save(&crate::connections::account(&id), &Secret::new(other)).unwrap(); + let e = s.run("connection.login", &json!({"id": id})).unwrap_err(); + assert_eq!(e, s3_keys::UNUSABLE); + assert!(!e.contains("OTHERKEY") && s3::connection("sv-rules-keys").is_none()); + // Nothing saved: said so, and a stale marker goes. + store.inner.delete(&crate::connections::account(&id)).unwrap(); + s.connections.set_saved_keys(&id, true); + let e = s.run("connection.login", &json!({"id": id})).unwrap_err(); + assert_eq!(e, s3_keys::NONE_SAVED); + assert!(!s.connections.find(&id).unwrap().saved_keys); + // Remembered keys, then typed ones for this session only (nothing written, the pair kept). + login(&mut s, &id, "SAVEDKEYA", "saved-secret-A", true).unwrap(); + let saves = store.count("save"); + let r = login(&mut s, &id, "SAVEDKEYB", "saved-secret-B", false).unwrap(); + assert_eq!(r["savedKeys"], true); + assert_eq!(live_key("sv-rules-keys").as_deref(), Some("SAVEDKEYB")); + assert_eq!(store.count("save"), saves); + assert!(store.raw(&id).unwrap().contains("SAVEDKEYA") && !store.raw(&id).unwrap().contains("SAVEDKEYB")); + // ... and the stored pair again with nothing typed. + s.run("connection.login", &json!({"id": id})).unwrap(); + assert_eq!(live_key("sv-rules-keys").as_deref(), Some("SAVEDKEYA")); + // A temporary set (with a token) signs this session and writes nothing, not even half; what + // was saved before stays. + let before = store.raw(&id); + let r = s + .run( + "connection.login", + &typed(json!({"accessKeyId": "TEMPKEY", "secretAccessKey": "temp-secret", "sessionToken": "tok-1", "remember": true})), + ) + .unwrap(); + assert_eq!(r["warning"], s3_keys::TEMPORARY, "{r}"); + assert_eq!((r["loggedIn"].as_bool(), r["savedKeys"].as_bool()), (Some(true), Some(true))); + assert_eq!(store.raw(&id), before); + let live = s3::connection("sv-rules-keys").unwrap().credentials.unwrap(); + assert_eq!((live.access_key_id.as_str(), live.session_token.as_deref()), ("TEMPKEY", Some("tok-1"))); + // The same for a connection with nothing saved: still nothing written. + let fresh = saved(&mut s, "sv-rules-fresh", &srv.base, "sv-rules", "keys"); + let r = s + .run( + "connection.login", + &json!({"id": fresh, "accessKeyId": "TEMPKEY", "secretAccessKey": "temp-secret", "sessionToken": "tok-1", "remember": true}), + ) + .unwrap(); + assert_eq!((r["warning"].as_str(), r["savedKeys"].as_bool()), (Some(s3_keys::TEMPORARY), Some(false))); + assert!(store.raw(&fresh).is_none()); + // A profile, the environment and anonymous access never read the store. + let loads = store.count("load"); + let profile = s + .run("connection.new", &json!({"kind": "s3", "name": "sv-rules-profile", "address": srv.base, "options": {"credentials": "profile", "profile": "sv-rules-no-such-profile"}})) + .unwrap(); + let environment = saved(&mut s, "sv-rules-env", &srv.base, "sv-rules", "environment"); + for other in [none.as_str(), profile["id"].as_str().unwrap(), environment.as_str()] { + let _ = s.run("connection.login", &json!({"id": other})); + } + assert_eq!(store.count("load"), loads, "{:?}", store.ops()); + for name in ["sv-rules-keys", "sv-rules-fresh", "sv-rules-none", "sv-rules-env"] { + s3::disconnect(name); + } +} + +#[test] +fn s3_saved_missing_or_refusing_stores_keep_the_session_login() { + let srv = serve_bucket("sv-stores", vec![], vec![]); + // No store at all (the browser, the command line). + let mut s = session(None); + let id = saved(&mut s, "sv-stores-none", &srv.base, "sv-stores", "keys"); + let r = login(&mut s, &id, "SAVEDKEYA", "saved-secret-A", true).unwrap(); + assert_eq!((r["loggedIn"].as_bool(), r["savedKeys"].as_bool()), (Some(true), Some(false)), "{r}"); + assert_eq!(r["warning"], s3_keys::NOT_REMEMBERED); + assert_eq!(live_key("sv-stores-none").as_deref(), Some("SAVEDKEYA")); + s3::disconnect("sv-stores-none"); + assert_eq!(s.run("connection.login", &json!({"id": id})).unwrap_err(), s3_keys::UNAVAILABLE); + assert!(s3::connection("sv-stores-none").is_none(), "no login without keys"); + // A store that refuses everything. + let mut s = session(None); + s.database_logins.set_store(Some(Arc::new(MemorySecrets::failing(SecretError::Denied)))); + let id = saved(&mut s, "sv-stores-denied", &srv.base, "sv-stores", "keys"); + let r = login(&mut s, &id, "SAVEDKEYA", "saved-secret-A", true).unwrap(); + assert_eq!((r["warning"].as_str(), r["loggedIn"].as_bool()), (Some(s3_keys::NOT_REMEMBERED), Some(true))); + assert!(!r.to_string().contains("keychain"), "no backend text: {r}"); + s3::disconnect("sv-stores-denied"); + assert_eq!(s.run("connection.login", &json!({"id": id})).unwrap_err(), s3_keys::UNAVAILABLE); + assert!(s3::connection("sv-stores-denied").is_none()); + // A save that fails doesn't make the new pair the saved one, nor erase the old. + let spy = Arc::new(Spy::default()); + let mut s = session(Some(&spy)); + let id = saved(&mut s, "sv-stores-save", &srv.base, "sv-stores", "keys"); + login(&mut s, &id, "SAVEDKEYA", "saved-secret-A", true).unwrap(); + spy.refuse_save.store(true, Ordering::Relaxed); + let r = login(&mut s, &id, "SAVEDKEYB", "saved-secret-B", true).unwrap(); + assert_eq!((r["warning"].as_str(), r["savedKeys"].as_bool()), (Some(s3_keys::NOT_REMEMBERED), Some(true)), "{r}"); + assert_eq!(live_key("sv-stores-save").as_deref(), Some("SAVEDKEYB")); + assert!(spy.raw(&id).unwrap().contains("SAVEDKEYA")); + s3::disconnect("sv-stores-save"); +} + +#[test] +fn s3_saved_lifecycle_keeps_deletes_and_forgets_keys() { + let srv = serve_bucket("sv-life", vec![], vec![]); + let spy = Arc::new(Spy::default()); + let mut s = session(Some(&spy)); + let id = saved(&mut s, "sv-life-conn", &srv.base, "sv-life", "keys"); + login(&mut s, &id, "SAVEDKEYA", "saved-secret-A", true).unwrap(); + let raw = spy.raw(&id); + assert!(raw.is_some()); + // Renaming, grouping, buckets and region keep the pair and ask nothing of the store. + let r = s.run("connection.edit", &json!({"id": id, "name": "sv-life-renamed", "group": "Work"})).unwrap(); + assert_eq!((r["savedKeys"].as_bool(), r["loggedIn"].as_bool()), (Some(true), Some(false)), "{r}"); + let r = s + .run( + "connection.edit", + &json!({"id": id, "options": {"credentials": "keys", "buckets": "sv-life,sv-life-two", "region": "eu-west-1"}}), + ) + .unwrap(); + assert_eq!(r["savedKeys"], true); + assert_eq!(spy.raw(&id), raw); + assert_eq!(spy.count("delete"), 0); + // A copy has no keys of its own, and no marker. + let copy = s.run("connection.duplicate", &json!({"id": id, "name": "sv-life-copy"})).unwrap(); + assert_eq!(copy["savedKeys"], false); + assert!(spy.raw(copy["id"].as_str().unwrap()).is_none()); + // Another spelling of the same endpoint isn't a change. + let spelled = format!("{}/", srv.base.replace("http://", "HTTP://")); + let r = s.run("connection.edit", &json!({"id": id, "address": spelled})).unwrap(); + assert_eq!(r["savedKeys"], true); + assert_eq!((spy.raw(&id), spy.count("delete")), (raw.clone(), 0)); + // Logged in again, a refused delete stops an endpoint edit, a forget and a removal, leaving + // the definition, its generation, the marker and the login as they were. + let login_again = |s: &mut Session| s.run("connection.login", &json!({"id": id})).unwrap(); + login_again(&mut s); + spy.refuse_delete.store(true, Ordering::Relaxed); + let (generation, address) = (s.connections.generation, s.connections.find(&id).unwrap().address.clone()); + let e = s.run("connection.edit", &json!({"id": id, "address": "http://127.0.0.1:9"})).unwrap_err(); + assert_eq!(e, s3_keys::NOT_FORGOTTEN); + assert_eq!(s.run("connection.forgetSavedKeys", &json!({"id": id})).unwrap_err(), s3_keys::NOT_FORGOTTEN); + assert_eq!(s.run("connection.remove", &json!({"id": id})).unwrap_err(), s3_keys::NOT_FORGOTTEN); + let kept = s.connections.find(&id).unwrap(); + assert_eq!((kept.address.as_str(), kept.saved_keys, s.connections.generation), (address.as_str(), true, generation)); + assert!(s3::connection("sv-life-renamed").is_some(), "the login stays"); + assert_eq!(spy.raw(&id), raw); + // An edit that is refused for another reason deletes nothing. + spy.refuse_delete.store(false, Ordering::Relaxed); + let deletes = spy.count("delete"); + assert!(s.run("connection.edit", &json!({"id": id, "name": "c1", "address": "http://127.0.0.1:9"})).is_err()); + assert!(s.run("connection.edit", &json!({"id": id, "address": "ftp://127.0.0.1:9"})).is_err()); + assert_eq!(spy.count("delete"), deletes); + assert!(spy.raw(&id).is_some()); + // Forgetting deletes the pair, not the login; again is no error. + let r = s.run("connection.forgetSavedKeys", &json!({"id": "sv-life-renamed"})).unwrap(); + assert_eq!((r["savedKeys"].as_bool(), r["loggedIn"].as_bool()), (Some(false), Some(true)), "{r}"); + assert!(spy.raw(&id).is_none() && s3::connection("sv-life-renamed").is_some()); + assert!(s.run("connection.forgetSavedKeys", &json!({"id": id})).is_ok()); + assert!(s.run("connection.forgetSavedKeys", &json!({"id": "c99"})).is_err()); + let wms = s.run("connection.new", &json!({"kind": "wms", "name": "sv-life-wms", "address": "http://127.0.0.1:9/wms"})).unwrap(); + assert!(s.run("connection.forgetSavedKeys", &json!({"id": wms["id"]})).is_err()); + // A real endpoint change deletes the old pair before the new definition is stored. + login(&mut s, &id, "SAVEDKEYA", "saved-secret-A", true).unwrap(); + let r = s.run("connection.edit", &json!({"id": id, "address": "http://127.0.0.1:9"})).unwrap(); + assert_eq!(r["savedKeys"], false); + assert!(spy.raw(&id).is_none()); + let last_delete = spy.ops().iter().rposition(|o| o.starts_with("delete")).unwrap(); + assert!(spy.ops().iter().skip(last_delete).all(|o| !o.starts_with("save"))); + // Removal deletes, even without a marker. + s.run("connection.edit", &json!({"id": id, "address": srv.base})).unwrap(); + login(&mut s, &id, "SAVEDKEYA", "saved-secret-A", true).unwrap(); + s.run("connection.remove", &json!({"id": id})).unwrap(); + assert!(spy.raw(&id).is_none() && s.connections.find(&id).is_none()); + let deletes = spy.count("delete"); + let copy_id = copy["id"].as_str().unwrap(); + s.run("connection.remove", &json!({"id": copy_id})).unwrap(); + assert_eq!(spy.count("delete"), deletes + 1, "asked even when nothing was marked"); + // Without a store, a marker blocks removal and forgetting; without one, removal is as before. + let mut t = session(None); + t.connections = Connections::from_json( + &json!({"next": 5, "connections": [ + {"id": "c3", "name": "sv-life-marked", "kind": "s3", "options": {"credentials": "keys"}, "savedKeys": true}, + {"id": "c4", "name": "sv-life-plain", "kind": "s3", "options": {"credentials": "keys"}}]}) + .to_string(), + ); + assert_eq!(t.run("connection.remove", &json!({"id": "c3"})).unwrap_err(), s3_keys::NOT_FORGOTTEN); + assert_eq!(t.run("connection.forgetSavedKeys", &json!({"id": "c3"})).unwrap_err(), s3_keys::NOT_FORGOTTEN); + assert!(t.connections.find("c3").is_some_and(|c| c.saved_keys)); + t.run("connection.remove", &json!({"id": "c4"})).unwrap(); + s3::disconnect("sv-life-renamed"); +} + +#[test] +fn s3_saved_a_binding_signs_only_its_own_connection_and_errors_never_echo_the_key() { + let moved_to = serve_bucket("sv-bind", vec![("elsewhere.fgb", vec![1, 2, 3])], vec![]); + let srv = + serve_bucket("sv-bind", vec![("grid.fgb", grid_fgb())], vec![("moved.fgb", format!("{}/sv-bind/elsewhere.fgb", moved_to.base))]); + let mut s = session(None); + let (a, b) = (saved(&mut s, "sv-bind-a", &srv.base, "sv-bind", "keys"), saved(&mut s, "sv-bind-b", &srv.base, "sv-bind", "keys")); + login(&mut s, &b, "SAVEDKEYB", "saved-secret-B", false).unwrap(); + login(&mut s, &a, "SAVEDKEYA", "saved-secret-A", false).unwrap(); + let object = |key: &str| format!("{}/sv-bind/{key}", srv.base); + // The second connection's address is read with its key, whatever order they logged in. + let from = srv.requests(); + s.run("layer.addUrl", &json!({"url": bound(&s, &b, &object("grid.fgb")), "name": "by b"})).unwrap(); + assert_eq!(srv.key_ids(from), BTreeSet::from(["SAVEDKEYB".to_string()])); + let from = srv.requests(); + s.run("layer.addUrl", &json!({"url": bound(&s, &a, &object("grid.fgb")), "name": "by a"})).unwrap(); + assert_eq!(srv.key_ids(from), BTreeSet::from(["SAVEDKEYA".to_string()])); + // An unrelated address of the same endpoint isn't signed by either. + let from = srv.requests(); + s.run("layer.addUrl", &json!({"url": object("grid.fgb"), "name": "plain"})).unwrap(); + assert!(srv.requests() > from && srv.key_ids(from).is_empty() && srv.unsigned(from) == srv.requests() - from); + // Another endpoint, an ungranted bucket, no token and a token nobody has: refused, not + // read with someone else's key. + let from = srv.requests(); + let wrap = |token: &str, endpoint: &str, url: &str| rustgis_data::s3_address::bind(token, endpoint, url).unwrap(); + let token = s.s3_binding(&a).unwrap().to_string(); + for (bad, why) in [ + (wrap(&token, &moved_to.base, &format!("{}/sv-bind/grid.fgb", moved_to.base)), s3::MISMATCH), + (wrap(&token, &srv.base, &format!("{}/sv-ungranted/grid.fgb", srv.base)), s3::MISMATCH), + (wrap("", &srv.base, &object("grid.fgb")), s3::UNRESOLVED), + (wrap("00000000000000ff", &srv.base, &object("grid.fgb")), s3::UNRESOLVED), + ] { + let e = s.run("layer.addUrl", &json!({"url": bad})).unwrap_err(); + assert!(e.contains(why), "{e}"); + } + assert_eq!(srv.requests(), from, "refused before anything is sent"); + // A refusal is told by its status: the server's answer echoes the Authorization header. + let e = s.run("layer.addUrl", &json!({"url": bound(&s, &b, &object("missing.fgb"))})).unwrap_err(); + assert!(e.contains("404"), "{e}"); + for secret in ["SAVEDKEYB", "SAVEDKEYA", "saved-secret", "AWS4-HMAC", "Authorization", "authorization"] { + assert!(!e.contains(secret), "{secret} in {e}"); + } + assert!(srv.saw("SAVEDKEYB"), "the request itself was signed"); + // A signed request isn't redirected to another server. + let e = s.run("layer.addUrl", &json!({"url": bound(&s, &b, &object("moved.fgb"))})).unwrap_err(); + assert!(e.contains("redirect"), "{e}"); + assert_eq!(moved_to.requests(), 0); + // An ordinary connection of the same name takes the saved login's place: its addresses wait. + s3::connect(s3::Connection { + name: "sv-bind-a".into(), + endpoint: Some(srv.base.clone()), + region: String::new(), + buckets: vec![], + credentials: None, + }) + .unwrap(); + // (An address already read is kept in the cache, so the check is of one that wasn't.) + let e = s.run("layer.addUrl", &json!({"url": bound_with(&token, &srv.base, &object("other.fgb"))})).unwrap_err(); + assert!(e.contains(s3::UNRESOLVED), "{e}"); + for name in ["sv-bind-a", "sv-bind-b"] { + s3::disconnect(name); + } +} + +fn bound_with(token: &str, endpoint: &str, url: &str) -> String { + rustgis_data::s3_address::bind(token, endpoint, url).unwrap() +} + +/// Waits until the server has had no new request for a moment (reads the layers just added +/// started in the background have finished). +fn settle(srv: &Server) { + let (mut last, mut quiet) = (usize::MAX, 0); + for _ in 0..200 { + cache::wait(std::time::Duration::from_millis(200)); + std::thread::sleep(std::time::Duration::from_millis(50)); + let now = srv.requests(); + quiet = if now == last { quiet + 1 } else { 0 }; + if quiet >= 4 { + return; + } + last = now; + } +} + +/// What a project of three sources through a saved connection looks like to a new session: the +/// project's text, and the connections' preference. +struct Saved { + srv: Server, + store: Arc, + text: String, + prefs: String, + fgb: String, +} + +/// Builds a project with a streamed FlatGeobuf, a COG and a PMTiles archive, all through one +/// saved connection `-a` (keys remembered), and a second saved connection `-b` of +/// the same endpoint and bucket (keys remembered too). +fn saved_project(bucket: &str) -> Saved { + let srv = serve_bucket( + bucket, + vec![("grid.fgb", grid_fgb()), ("scene.tif", rustgis_data::raster::cog::example(2048, 256)), ("parks.pmtiles", parks_pmtiles())], + vec![], + ); + let store = Arc::new(Spy::default()); + let mut s = session(Some(&store)); + let name = format!("{bucket}-a"); + let id = saved(&mut s, &name, &srv.base, bucket, "keys"); + login(&mut s, &id, "SAVEDKEYA", "saved-secret-A", true).unwrap(); + let second = saved(&mut s, &format!("{bucket}-b"), &srv.base, bucket, "keys"); + login(&mut s, &second, "SAVEDKEYB", "saved-secret-B", true).unwrap(); + // The first one's login is the one the project's sources are added through. + login(&mut s, &id, "SAVEDKEYA", "saved-secret-A", false).unwrap(); + let token = s.s3_binding(&id).unwrap().to_string(); + let object = |key: &str| bound_with(&token, &srv.base, &format!("{}/{bucket}/{key}", srv.base)); + let fgb = object("grid.fgb"); + s.run("layer.addUrl", &json!({"url": fgb, "name": "Grid"})).unwrap(); + let cog_url = object("scene.tif"); + s.run("layer.addUrl", &json!({"url": cog_url, "name": "Scene"})).unwrap(); + s.run("layer.addUrl", &json!({"url": object("parks.pmtiles"), "name": "Parks"})).unwrap(); + let (text, _) = s.project.saved_text().unwrap(); + let prefs = s.connections.to_json(); + s3::disconnect(&name); + s3::disconnect(&format!("{bucket}-b")); + // The tiles and archives just added read in the background: wait for them to finish, so a + // test counting requests counts only its own. + settle(&srv); + Saved { srv, store, text, prefs, fgb } +} + +/// A new session with the saved connections (the first `count` of them) and the store. +fn reopened(p: &Saved, store: &Arc, keep: usize) -> Session { + let mut s = session(Some(store)); + s.connections = Connections::from_json(&p.prefs); + s.connections.connections.truncate(keep); + s +} + +#[test] +fn s3_saved_projects_hold_no_keys_and_reload_through_their_one_connection() { + let p = saved_project("sv-proj-one"); + for secret in ["SAVEDKEYA", "SAVEDKEYB", "saved-secret"] { + assert!(!p.text.contains(secret) && !p.prefs.contains(secret), "{secret}"); + } + assert!(p.text.contains("rustgis-s3:") && p.text.contains("sv-proj-one")); + // Read as plain data, with no session to answer for it: nothing is fetched. + let from = p.srv.requests(); + let project = Project::from_json(&p.text).unwrap(); + let source = project.map().layers.iter().find_map(|l| l.source.as_ref()).unwrap(); + assert!(!source.is_open() && source.error.is_none()); + assert_eq!(p.srv.requests(), from); + // A binding in the file is nobody's login: it is cleared. + let path = source.path.clone().unwrap(); + assert_eq!(rustgis_data::s3_address::parse(&path).unwrap().unwrap().binding, ""); + assert_eq!(rustgis_data::s3_address::parse(&p.fgb).unwrap().unwrap().binding.len(), 16); + // One connection could read it: the keys come from the store, and everything opens. + let mut s = reopened(&p, &p.store, 1); + s.run("project.openText", &json!({"text": p.text})).unwrap(); + assert!(s.s3_pending.is_empty()); + let id = streamed(&s, "grid.fgb"); + assert!(read_view(&mut s, id) > 0, "{:?}", s.map().layer(id).unwrap().source.as_ref().unwrap().error); + let layer = s.map().layers.iter().find(|l| l.name == "Grid").unwrap(); + assert!(layer.source.as_ref().unwrap().path.as_deref().is_some_and(|x| x.contains(s.s3_binding("c1").unwrap()))); + // The COG's tile is drawn from its pixels. + let tiles = s.map().tiles.clone(); + let scene = tiles.iter().find(|t| t.name == "Scene").unwrap(); + assert!(scene.url.contains(s.s3_binding("c1").unwrap())); + let (z, lon, lat) = (9u32, 10.5f64, 45.5f64); + let n = f64::from(1u32 << z); + let x = ((lon + 180.0) / 360.0 * n) as u32; + let y = ((1.0 - lat.to_radians().tan().asinh() / std::f64::consts::PI) / 2.0 * n) as u32; + let tile = format!("cog:{z}/{x}/{y}@{}", scene.url); + let mut got = None; + for _ in 0..300 { + match cache::get(&tile, cache::Want::Image) { + cache::State::Ready(p) => { + got = Some(p); + break; + } + cache::State::Failed(e) => panic!("{e}"), + cache::State::Pending => cache::wait(std::time::Duration::from_millis(50)), + } + } + let Some(payload) = got else { panic!("the tile never came") }; + let cache::Payload::Image(pm) = &*payload else { panic!("not a picture") }; + assert_eq!((pm.width(), pm.height()), (256, 256)); + assert!(pm.pixels().iter().any(|px| px.alpha() == 255)); + // The PMTiles' park is found where it is. + s.map_mut().camera.center = rustgis_geom::Coord::new(0.0, 0.0); + s.map_mut().camera.res = 2.0 * 20_037_508.342_789_244 / 256.0; + let mut found = Value::Null; + for _ in 0..500 { + found = s.run("identify", &json!({"lon": 1.0, "lat": 1.0})).unwrap(); + if !found.is_null() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert_eq!((found["sourceLayer"].as_str(), found["attributes"]["name"].as_str()), (Some("parks"), Some("Central")), "{found}"); + assert_eq!(p.srv.key_ids(from), BTreeSet::from(["SAVEDKEYA".to_string()])); + assert!(!p.srv.saw("saved-secret")); + // The text saved again has no key either, and the project names the login that was used. + let (again, _) = s.project.saved_text().unwrap(); + assert!(!again.contains("SAVEDKEY") && !again.contains("saved-secret")); + s3::disconnect("sv-proj-one-a"); +} + +#[test] +fn s3_saved_several_candidates_wait_for_a_choice_and_nothing_is_fetched_before() { + let p = saved_project("sv-proj-two"); + let store = p.store.clone(); + let (from, saves) = (p.srv.requests(), store.count("save")); + let mut s = reopened(&p, &store, 2); + s.run("project.openText", &json!({"text": p.text})).unwrap(); + assert_eq!(p.srv.requests(), from, "opening fetches nothing while the choice is open"); + assert_eq!(s.s3_pending.len(), 1, "{:?}", s.s3_pending); + let pending = s.s3_pending[0].clone(); + assert_eq!((pending.bucket.as_str(), pending.candidates.clone()), ("sv-proj-two", vec!["c1".to_string(), "c2".to_string()])); + assert_eq!(pending.endpoint.as_deref(), Some(p.srv.base.as_str())); + // A frame draws; blocked sources ask for nothing, signed or not. + let view = crate::render::View { width: 800.0, height: 600.0, pixels_per_point: 1.0, camera: s.map().camera, sky: false }; + let _ = crate::render::render_png(s.map_mut(), &view); + assert_eq!(p.srv.requests(), from); + assert_eq!(store.count("save"), saves, "opening saves nothing"); + // A stale or unknown choice does nothing. + for (source, id) in + [(pending.source.as_str(), "c99"), ("rustgis-s3::0::s3://nope/x.fgb", "c1"), ("", ""), (pending.source.as_str(), "")] + { + let e = s.run("connection.resolveS3", &json!({"source": source, "id": id})).unwrap_err(); + assert_eq!(e, crate::s3_connections::STALE); + } + assert_eq!((p.srv.requests(), s.s3_pending.len()), (from, 1)); + // A candidate removed, or moved to another endpoint, is no longer one. + let mut t = reopened(&p, &store, 2); + t.run("project.openText", &json!({"text": p.text})).unwrap(); + let source = t.s3_pending[0].source.clone(); + t.connections + .update("c2", crate::connections::Connection { address: "http://127.0.0.1:9".into(), ..t.connections.find("c2").unwrap().clone() }) + .unwrap(); + assert_eq!(t.run("connection.resolveS3", &json!({"source": source, "id": "c2"})).unwrap_err(), crate::s3_connections::STALE); + t.connections.remove("c1"); + assert_eq!(t.run("connection.resolveS3", &json!({"source": source, "id": "c1"})).unwrap_err(), crate::s3_connections::STALE); + assert_eq!(p.srv.requests(), from); + // The second candidate chosen: its key reads, and no other. + let r = s.run("connection.resolveS3", &json!({"source": pending.source, "id": "c2"})).unwrap(); + assert_eq!(r["id"], "c2"); + assert!(r["reopened"].as_u64().unwrap() >= 2 && r["failed"] == 0, "{r}"); + assert!(s.s3_pending.is_empty()); + let id = streamed(&s, "grid.fgb"); + assert!(read_view(&mut s, id) > 0); + assert_eq!(p.srv.key_ids(from), BTreeSet::from(["SAVEDKEYB".to_string()])); + for name in ["sv-proj-two-a", "sv-proj-two-b"] { + s3::disconnect(name); + } +} + +#[test] +fn s3_saved_sources_without_keys_zero_matches_and_older_addresses_stay_blocked_or_as_they_were() { + let p = saved_project("sv-proj-edge"); + let from = p.srv.requests(); + // One candidate, whose keys are not in the store (another computer): blocked, pending for its + // log in, until the keys are typed. + let empty = Arc::new(Spy::default()); + let mut s = reopened(&p, &empty, 1); + s.run("project.openText", &json!({"text": p.text})).unwrap(); + assert_eq!(p.srv.requests(), from); + assert_eq!(s.s3_pending.len(), 1); + assert_eq!(s.s3_pending[0].candidates, vec!["c1".to_string()]); + let source = s.s3_pending[0].source.clone(); + assert_eq!(s.run("connection.resolveS3", &json!({"source": source, "id": "c1"})).unwrap_err(), s3_keys::NONE_SAVED); + assert_eq!((s.s3_pending.len(), p.srv.requests()), (1, from)); + let blocked = s.map().layers.iter().find_map(|l| l.source.as_ref()).unwrap(); + assert_eq!(blocked.error.as_deref(), Some(s3::UNRESOLVED)); + login(&mut s, "c1", "SAVEDKEYA", "saved-secret-A", false).unwrap(); + let r = s.run("connection.resolveS3", &json!({"source": source, "id": "c1"})).unwrap(); + assert_eq!(r["failed"], 0, "{r}"); + let id = streamed(&s, "grid.fgb"); + assert!(read_view(&mut s, id) > 0); + assert!(s.s3_pending.is_empty()); + // No saved connection reads it: blocked with a notice, nothing fetched, no one chosen. + settle(&p.srv); + let from = p.srv.requests(); + let mut none = session(Some(&empty)); + none.run("project.openText", &json!({"text": p.text})).unwrap(); + assert!(none.s3_pending.is_empty()); + assert!(none.notices.iter().any(|n| n.error && n.text.contains("no saved connection")), "{:?}", none.notices); + assert_eq!(none.map().layers.iter().find_map(|l| l.source.as_ref()).unwrap().error.as_deref(), Some(s3::UNRESOLVED)); + assert_eq!(p.srv.requests(), from); + // An older project's bare address, with a session-only connection of the bucket: as before. + let legacy = p.text.replace(&p.fgb, "s3://sv-proj-edge/grid.fgb"); + assert!(legacy.contains("s3://sv-proj-edge/grid.fgb") && !legacy.contains(&p.fgb), "the Grid layer is an older project's"); + let mut old = session(Some(&empty)); + old.run("s3.connect", &json!({"name": "sv-proj-plain", "endpoint": p.srv.base, "buckets": ["sv-proj-edge"], "accessKeyId": "PLAINKEY", "secretAccessKey": "plain-secret"})).unwrap(); + old.run("project.openText", &json!({"text": legacy})).unwrap(); + assert!(old.s3_pending.is_empty()); + let id = streamed(&old, "grid.fgb"); + assert!(read_view(&mut old, id) > 0, "{:?}", old.map().layer(id).unwrap().source.as_ref().unwrap().error); + assert_eq!(p.srv.key_ids(from), BTreeSet::from(["PLAINKEY".to_string()])); + s3::disconnect("sv-proj-plain"); + // The same bare address with two saved candidates: unresolved placeholder, unknown origin kept. + settle(&p.srv); + let from = p.srv.requests(); + let mut two = reopened(&p, &p.store, 2); + two.run("project.openText", &json!({"text": legacy})).unwrap(); + assert_eq!(p.srv.requests(), from); + // Two resources wait: the older bare address (unknown origin), and the tiles' known endpoint. + assert_eq!(two.s3_pending.len(), 2, "{:?}", two.s3_pending); + let older = two.s3_pending.iter().find(|p| p.endpoint.is_none()).unwrap().clone(); + assert_eq!(older.candidates.len(), 2); + assert_eq!(older.source, "rustgis-s3::0::s3://sv-proj-edge/grid.fgb"); + assert!(two.s3_pending.iter().any(|p| p.endpoint.is_some() && p.candidates.len() == 2)); + let r = two.run("connection.resolveS3", &json!({"source": older.source, "id": "c2"})).unwrap(); + assert_eq!(r["id"], "c2"); + assert_eq!(two.s3_pending.len(), 1, "only the chosen resource is settled"); + let id = streamed(&two, "grid.fgb"); + assert!(read_view(&mut two, id) > 0, "{:?}", two.map().layer(id).unwrap().source.as_ref().unwrap().error); + assert_eq!(p.srv.key_ids(from), BTreeSet::from(["SAVEDKEYB".to_string()])); + for name in ["sv-proj-edge-a", "sv-proj-edge-b"] { + s3::disconnect(name); + } +} + +#[test] +fn s3_saved_imported_maps_layers_and_packages_never_bring_their_own_login() { + let p = saved_project("sv-proj-import"); + let token = rustgis_data::s3_address::parse(&p.fgb).unwrap().unwrap().binding.to_string(); + let from = p.srv.requests(); + // Package and template reads (and a plain project read) clear every binding. + for read in [Project::from_import_json(&p.text).unwrap(), Project::from_json(&p.text).unwrap()] { + for a in read.maps.iter().flat_map(|m| m.layers.iter()).filter_map(|l| l.source.as_ref().and_then(|s| s.path.as_ref())) { + assert!(a.starts_with("rustgis-s3::") && !a.contains(&token), "{a}"); + } + for t in &read.map().tiles { + assert!(!t.url.contains(&token), "{}", t.url); + } + } + // A map imported with no saved connection to read it keeps the cleared address, unfetched. + let map = serde_json::to_string(&Project::from_import_json(&p.text).unwrap().maps[0]).unwrap(); + let raw_map = map.replace("rustgis-s3::", &format!("rustgis-s3:{token}:")); + assert!(raw_map.contains(&token)); + let mut s = session(Some(&Arc::new(Spy::default()))); + s.run("map.import", &json!({"text": raw_map})).unwrap(); + assert!(!serde_json::to_string(&s.project).unwrap().contains(&token), "the map's own token is not kept"); + assert_eq!(p.srv.requests(), from); + // With one saved connection: bound to this session's login, not the file's token. + let mut t = reopened(&p, &p.store, 1); + t.run("map.import", &json!({"text": raw_map})).unwrap(); + let own = t.s3_binding("c1").unwrap().to_string(); + assert_ne!(own, token); + let text = serde_json::to_string(&t.project).unwrap(); + assert!(text.contains(&own) && !text.contains(&token)); + // A layer file likewise. + let layer = + serde_json::to_string(&Project::from_import_json(&p.text).unwrap().maps[0].layers.iter().find(|l| l.name == "Grid").unwrap()) + .unwrap(); + let raw_layer = layer.replace("rustgis-s3::", &format!("rustgis-s3:{token}:")); + let mut u = session(Some(&Arc::new(Spy::default()))); + crate::cmd::layer::add_layer_text(&mut u, &raw_layer).unwrap(); + assert!(!serde_json::to_string(&u.project).unwrap().contains(&token)); + assert_eq!(u.map().layers[0].source.as_ref().unwrap().error.as_deref(), Some(s3::UNRESOLVED)); + s3::disconnect("sv-proj-import-a"); + s3::disconnect("sv-proj-import-b"); +} + +#[test] +fn s3_saved_typed_keys_are_in_no_script_result_notice_or_list_the_app_shows() { + let srv = serve_bucket("sv-seen", vec![], vec![]); + let store = Arc::new(Spy::default()); + let mut s = session(Some(&store)); + let id = saved(&mut s, "sv-seen-conn", &srv.base, "sv-seen", "keys"); + let params = + json!({"id": id, "accessKeyId": "SAVEDKEYA", "secretAccessKey": "saved-secret-A", "sessionToken": "tok-1", "remember": true}); + // Exercise the exported history commands, not the raw internal Python formatter. + s.project.gp_history.push(crate::GpRecord { + tool: "Buffer".into(), + id: "buffer".into(), + params: params.clone(), + command: None, + at: 0, + ok: true, + messages: Vec::new(), + millis: 0, + outputs: Vec::new(), + }); + for kind in ["python", "command", "cli"] { + let code = s.run("gp.historyCode", &json!({"run": 0, "as": kind})).unwrap(); + for secret in ["SAVEDKEYA", "saved-secret-A", "tok-1"] { + assert!(!code.to_string().contains(secret), "{secret} in {code}"); + } + } + let script = s.run("gp.historyScript", &Value::Null).unwrap(); + for secret in ["SAVEDKEYA", "saved-secret-A", "tok-1"] { + assert!(!script.to_string().contains(secret), "{secret} in {script}"); + } + s.project.gp_history.clear(); + assert!(crate::sources::holds_secret(¶ms)); + // Everything the session shows afterwards. + let result = s.run("connection.login", ¶ms).unwrap(); + login(&mut s, &id, "SAVEDKEYA", "saved-secret-A", true).unwrap(); + assert!(s.project.gp_history.is_empty(), "view-only logins never enter History"); + let mut shown = vec![result.to_string(), s.connections.to_json(), s.sources.to_json(), serde_json::to_string(&s.project).unwrap()]; + shown.push(s.run("connection.list", &json!({})).unwrap().to_string()); + shown.push(s.run("s3.connections", &Value::Null).unwrap().to_string()); + shown.push(serde_json::to_string(&s.notices.iter().map(|n| n.text.clone()).collect::>()).unwrap()); + shown.push(s.project.saved_text().unwrap().0); + for e in [ + s.run("connection.login", &json!({"id": id, "accessKeyId": "SAVEDKEYA", "secretAccessKey": "saved-secret-A", "remember": "yes"})) + .unwrap_err(), + s.run("connection.login", &json!({"id": id, "accessKeyId": "SAVEDKEYA", "remember": true})).unwrap_err(), + s.run("connection.edit", &json!({"id": id, "address": "ftp://x"})).unwrap_err(), + ] { + shown.push(e); + } + for text in &shown { + for secret in ["SAVEDKEYA", "saved-secret-A", "tok-1"] { + assert!(!text.contains(secret), "{secret} in {text}"); + } + } + // Rendering frames and listing connections ask nothing more of the store. + let ops = store.ops().len(); + let view = crate::render::View { width: 400.0, height: 300.0, pixels_per_point: 1.0, camera: s.map().camera, sky: false }; + let _ = crate::render::render_png(s.map_mut(), &view); + s.run("connection.list", &json!({})).unwrap(); + assert_eq!(store.ops().len(), ops); + s3::disconnect("sv-seen-conn"); +} + +#[test] +fn s3_saved_database_and_s3_share_one_store_without_crossing_credentials_or_cleanup() { + let store = Arc::new(Spy::default()); + let mut s = session(Some(&store)); + let db = s + .run( + "connection.new", + &json!({ + "kind": "postgres", "name": "sv-combined-db", + "address": "postgres:host=127.0.0.1 port=1 dbname=combined user=reader sslmode=disable" + }), + ) + .unwrap()["id"] + .as_str() + .unwrap() + .to_string(); + let s3id = saved(&mut s, "sv-combined-s3", "http://127.0.0.1:9", "sv-combined", "keys"); + s.run("connection.login", &json!({"id": db, "password": "combined-db-password", "remember": true})).unwrap(); + login(&mut s, &s3id, "COMBINEDKEY", "combined-s3-secret", true).unwrap(); + let db_value = store.raw(&db).unwrap(); + let s3_value = store.raw(&s3id).unwrap(); + assert!(db_value.contains("combined-db-password") && !db_value.contains("combined-s3-secret")); + assert!(s3_value.contains("combined-s3-secret") && !s3_value.contains("combined-db-password")); + let prefs = s.connections.to_json(); + for secret in ["combined-db-password", "COMBINEDKEY", "combined-s3-secret"] { + assert!(!prefs.contains(secret)); + } + s3::disconnect("sv-combined-s3"); + let mut fresh = session(Some(&store)); + fresh.connections = Connections::from_json(&prefs); + fresh.run("connection.login", &json!({"id": db})).unwrap(); + fresh.run("connection.login", &json!({"id": s3id})).unwrap(); + assert_eq!(fresh.database_logins.known_password(&db).unwrap().expose(), "combined-db-password"); + assert_eq!(live_key("sv-combined-s3").as_deref(), Some("COMBINEDKEY")); + // One malformed opt-in contract, checked before either kind mutates state or the store. + for id in [&db, &s3id] { + let before = store.ops().len(); + assert!(fresh.run("connection.login", &json!({"id": id, "remember": "yes"})).is_err()); + assert_eq!(store.ops().len(), before); + } + // Both kinds use the same validate/delete/commit boundary on a refused deletion. + store.refuse_delete.store(true, Ordering::Relaxed); + for (id, address) in [(&db, "postgres:host=changed dbname=combined user=reader"), (&s3id, "http://127.0.0.1:10")] { + let before = fresh.connections.to_json(); + let generation = fresh.connections.generation; + assert!(fresh.run("connection.edit", &json!({"id": id, "address": address})).is_err()); + assert!(fresh.run("connection.remove", &json!({"id": id})).is_err()); + assert_eq!(fresh.connections.to_json(), before); + assert_eq!(fresh.connections.generation, generation); + } + assert_eq!(fresh.database_logins.known_password(&db).unwrap().expose(), "combined-db-password"); + assert_eq!(live_key("sv-combined-s3").as_deref(), Some("COMBINEDKEY")); + store.refuse_delete.store(false, Ordering::Relaxed); + fresh.run("connection.forgetSavedKeys", &json!({"id": s3id})).unwrap(); + assert!(store.raw(&s3id).is_none()); + assert_eq!(store.raw(&db).as_deref(), Some(db_value.as_str())); + assert_eq!(live_key("sv-combined-s3").as_deref(), Some("COMBINEDKEY")); + fresh.run("connection.forgetPassword", &json!({"id": db})).unwrap(); + assert!(store.raw(&db).is_none()); + assert!(!fresh.connections.find(&db).unwrap().remembered); + assert!(!fresh.connections.find(&s3id).unwrap().saved_keys); + s3::disconnect("sv-combined-s3"); +} + +#[test] +fn s3_saved_raster_pixels_wait_for_selection_then_reload_with_style_intact() { + let bytes = rustgis_data::raster::cog::example(256, 256); + let pixels = rustgis_data::raster::read_bytes("scene.tif", &bytes).unwrap(); + let srv = serve_bucket("sv-raster", vec![("scene.tif", bytes)], vec![]); + let store = Arc::new(Spy::default()); + let mut seed = session(Some(&store)); + let first = saved(&mut seed, "sv-raster-first", &srv.base, "sv-raster", "keys"); + login(&mut seed, &first, "RASTERKEYA", "raster-secret-a", true).unwrap(); + let mut layer = rustgis_map::Layer::new(1, rustgis_data::Dataset::new("Saved raster", rustgis_data::GeometryType::None, Vec::new())); + layer.raster = Some(rustgis_map::RasterLayer::from_path(pixels, bound(&seed, &first, "s3://sv-raster/scene.tif"))); + layer.transparency = 0.25; + seed.map_mut().layers.push(layer); + let text = seed.project.saved_text().unwrap().0; + let prefs = seed.connections.to_json(); + s3::disconnect("sv-raster-first"); + let mut unique = session(Some(&store)); + unique.connections = Connections::from_json(&prefs); + unique.run("project.openText", &json!({"text": text})).unwrap(); + let raster_id = unique.map().layers.iter().find(|l| l.dataset.name == "Saved raster").unwrap().id; + read_view(&mut unique, raster_id); + let raster = unique.map().layers.iter().find(|l| l.dataset.name == "Saved raster").unwrap(); + let data = raster.raster.as_ref().unwrap().data.as_ref().expect("unique saved raster must reload"); + assert_eq!((data.width, data.height), (256, 256)); + assert_eq!(raster.transparency, 0.25); + assert_eq!(srv.key_ids(0), BTreeSet::from(["RASTERKEYA".to_string()])); + let second = saved(&mut unique, "sv-raster-second", &srv.base, "sv-raster", "keys"); + login(&mut unique, &second, "RASTERKEYB", "raster-secret-b", true).unwrap(); + let prefs = unique.connections.to_json(); + s3::disconnect("sv-raster-first"); + s3::disconnect("sv-raster-second"); + let mut ambiguous = session(Some(&store)); + ambiguous.connections = Connections::from_json(&prefs); + let from = srv.requests(); + ambiguous.run("project.openText", &json!({"text": text})).unwrap(); + assert_eq!(srv.requests(), from); + let source = ambiguous.s3_pending[0].source.clone(); + let choice = ambiguous.run("connection.resolveS3", &json!({"source": source, "id": second})).unwrap(); + assert_eq!((choice["reopened"].as_u64(), choice["failed"].as_u64()), (Some(1), Some(0))); + let raster_id = ambiguous.map().layers.iter().find(|l| l.dataset.name == "Saved raster").unwrap().id; + read_view(&mut ambiguous, raster_id); + let raster = ambiguous.map().layers.iter().find(|l| l.dataset.name == "Saved raster").unwrap(); + assert_eq!(raster.raster.as_ref().unwrap().data.as_ref().unwrap().width, 256); + assert_eq!(raster.transparency, 0.25); + assert_eq!(srv.key_ids(from), BTreeSet::from(["RASTERKEYB".to_string()])); + s3::disconnect("sv-raster-second"); +} + +#[test] +fn s3_saved_resolving_one_bucket_does_not_retry_another_buckets_failed_source() { + let srv = serve_bucket("sv-group-one", Vec::new(), Vec::new()); + let store = Arc::new(Spy::default()); + let mut seed = session(Some(&store)); + let id = saved(&mut seed, "sv-group-first", &srv.base, "", "keys"); + let other = saved(&mut seed, "sv-group-second", &srv.base, "", "keys"); + login(&mut seed, &id, "GROUPKEYA", "group-secret-a", true).unwrap(); + login(&mut seed, &other, "GROUPKEYB", "group-secret-b", true).unwrap(); + for bucket in ["sv-group-one", "sv-group-two"] { + let path = bound(&seed, &id, &format!("s3://{bucket}/missing.fgb")); + let source = serde_json::from_value(json!({"path": path})).unwrap(); + seed.map_mut().add_streamed(bucket, source); + } + let text = seed.project.saved_text().unwrap().0; + let mut fresh = session(Some(&store)); + fresh.connections = Connections::from_json(&seed.connections.to_json()); + s3::disconnect("sv-group-first"); + s3::disconnect("sv-group-second"); + fresh.run("project.openText", &json!({"text": text})).unwrap(); + assert_eq!(fresh.s3_pending.len(), 2); + for bucket in ["sv-group-one", "sv-group-two"] { + let source = fresh.s3_pending.iter().find(|p| p.bucket == bucket).unwrap().source.clone(); + let result = fresh.run("connection.resolveS3", &json!({"source": source, "id": id})).unwrap(); + assert_eq!( + result["reopened"].as_u64().unwrap_or(0) + result["failed"].as_u64().unwrap_or(0), + 1, + "only this bucket's missing source is retried: {result}" + ); + let layer = fresh.map().layer_by_name(bucket).unwrap().id; + read_view(&mut fresh, layer); + assert!(fresh.map().layer(layer).unwrap().source.as_ref().unwrap().error.is_some()); + } + s3::disconnect("sv-group-first"); +} + +#[test] +fn s3_saved_browse_reauthenticates_this_session_instead_of_reusing_a_foreign_namesake() { + let srv = serve_bucket("sv-session-owner", Vec::new(), Vec::new()); + let store_a = Arc::new(Spy::default()); + let store_b = Arc::new(Spy::default()); + let mut a = session(Some(&store_a)); + let id = saved(&mut a, "sv-session-owner", &srv.base, "sv-session-owner", "keys"); + login(&mut a, &id, "SESSIONKEYA", "session-secret-a", true).unwrap(); + let mut b = session(Some(&store_b)); + b.connections = Connections::from_json(&a.connections.to_json()); + login(&mut b, &id, "SESSIONKEYB", "session-secret-b", true).unwrap(); + a.run("connection.login", &json!({"id": id})).unwrap(); + let status = b.run("connection.list", &Value::Null).unwrap(); + assert_eq!(status["connections"][0]["loggedIn"], false, "another session's same-name login is not ours"); + b.run("connection.browse", &json!({"id": id, "node": "s3:sv-session-owner/"})).unwrap(); + assert_eq!(srv.key_ids(0), BTreeSet::from(["SESSIONKEYB".to_string()])); + assert_eq!(srv.unsigned(0), 0); + assert_eq!(s3::binding_of("sv-session-owner").as_deref(), b.s3_binding(&id)); + // An ordinary replacement invalidates the old token; it must not turn that token into + // permission to reuse a different login by name. + b.run( + "s3.connect", + &json!({"name":"sv-session-owner","endpoint":srv.base,"buckets":["sv-session-owner"], + "accessKeyId":"REPLACEMENTKEY","secretAccessKey":"replacement-secret"}), + ) + .unwrap(); + let status = b.run("connection.list", &Value::Null).unwrap(); + assert_eq!(status["connections"][0]["loggedIn"], false, "a stale binding never falls back to a replacement"); + let from = srv.requests(); + b.run("connection.browse", &json!({"id":id,"node":"s3:sv-session-owner/"})).unwrap(); + assert_eq!(srv.key_ids(from), BTreeSet::from(["SESSIONKEYB".to_string()])); + s3::disconnect("sv-session-owner"); +} + +#[test] +fn s3_saved_favourites_keep_provenance_overhead_without_relaxing_destination_limits() { + let endpoint = "https://s3.example.org"; + let inner = format!("{endpoint}/bucket/{}.fgb?versionId={}", "k".repeat(990), "v".repeat(1000)); + assert!(inner.len() <= 2048); + let address = rustgis_data::s3_address::bind("0123456789abcdef", endpoint, &inner).unwrap(); + assert!(address.len() > 2048); + let source = + crate::sources::Source { kind: "url".into(), name: "Saved range source".into(), address: address.clone(), extra: Value::Null }; + let mut sources = crate::sources::Sources::default(); + sources.keep("favourites", source.clone()).unwrap(); + let restored = crate::sources::Sources::from_json(&sources.to_json()); + assert_eq!(restored.favourites[0].address, address); + let oversized = format!("{inner}{}", "v".repeat(2049 - inner.len())); + let too_long = rustgis_data::s3_address::bind("0123456789abcdef", endpoint, &oversized).unwrap(); + assert!(sources.keep("favourites", crate::sources::Source { address: too_long, ..source }).is_err()); +} diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index f3c6b1d0a..c46f23f3d 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -157,7 +157,14 @@ pub fn clean(mut s: Source) -> Result { return Err(format!("unknown kind '{}' ({})", s.kind, KINDS.join(", "))); } s.address = redact(s.address.trim()); - if s.address.is_empty() || s.address.len() > MAX_TEXT || s.address.chars().any(char::is_control) { + // Provenance has its own structural bound; it must not consume the destination's limit. + let archive = template_archive(&s.address).map_or(s.address.as_str(), |(_, archive)| archive); + let bounded = match rustgis_data::s3_address::parse(archive) { + Ok(Some(bound)) => bound.url.len() <= MAX_TEXT, + Ok(None) => s.address.len() <= MAX_TEXT, + Err(_) => false, + }; + if s.address.is_empty() || !bounded || s.address.chars().any(char::is_control) { return Err("a source needs an address (a path, web address or connection)".into()); } s.name = s.name.trim().chars().filter(|c| !c.is_control()).take(200).collect(); @@ -315,11 +322,34 @@ pub fn redact_text(text: &str) -> String { .collect() } +/// A PMTiles or COG tile template (`pmtiles:{z}/{x}/{y}@archive`, `cog:…@archive`): what comes +/// before its archive (up to and with the first `@`), and the archive, whose address is the one +/// that holds credentials and names a saved S3 connection. +pub(crate) fn template_archive(address: &str) -> Option<(&str, &str)> { + use crate::render::rustgis_tiles::{cog, pmtiles}; + let rest = [pmtiles::SCHEME, cog::SCHEME].iter().find_map(|s| address.strip_prefix(s))?; + let (_, archive) = rest.split_once('@')?; + let head = address.get(..address.len() - archive.len())?; + Some((head, archive)) +} + /// An address without what would give someone else the user's access: the user and password of /// a web address, query parameters that are keys, tokens or signatures (a signed blob's SAS, an /// S3 presigned URL), and a PostgreSQL connection string's password. A MySQL, MariaDB or SQL Server -/// address keeps its user (the one to connect as), never its password. +/// address keeps its user (the one to connect as), never its password. An address read through a +/// saved S3 connection is redacted inside (its inner address), and its length-prefixed form +/// rebuilt; a tile template, in its archive. pub fn redact(address: &str) -> String { + if let Some((head, archive)) = template_archive(address) + && rustgis_data::s3_address::is_wrapper(archive) + { + return format!("{head}{}", redact(archive)); + } + if let Ok(Some(b)) = rustgis_data::s3_address::parse(address) + && let Ok(rebuilt) = rustgis_data::s3_address::bind(b.binding, b.endpoint, &redact(b.url)) + { + return rebuilt; + } if let Some((public, _)) = rustgis_data::source::myconn::split_password(address) { return public; } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 0f35303fc..d24a274cb 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1344,6 +1344,12 @@ fn hostile_params_never_panic() { json!({"id": {"unexpected":"object"}, "remember": "true", "password": []}), json!({"kind": "stac", "address": "http://127.0.0.1:9/stac", "answer": "\u{0}{"}), json!({"id": "c1", "node": "s3:/\u{7}", "probe": true, "answer": 7, "documents": "x"}), + // Saved S3 keys: junk remember flags, partial pairs, tokens, and a choice of nothing. + json!({"id": "c1", "accessKeyId": "A", "secretAccessKey": "S", "sessionToken": "T", "remember": "yes"}), + json!({"id": "c1", "accessKeyId": 7, "secretAccessKey": [], "remember": null}), + json!({"id": "c1", "remember": true, "source": "rustgis-s3:junk", "node": 5}), + json!({"source": "rustgis-s3::0::s3://bucket/key", "id": "c1", "remember": {"a": 1}}), + json!({"source": 7, "id": ["c1"]}), ]; let ids: Vec<&str> = Session::new(Project::default()).registry.commands.iter().map(|c| c.id).collect(); let streamed = grid_fgb("hostile", 20); @@ -10170,7 +10176,8 @@ fn connections_commands() { assert_eq!(list["connections"][0]["loggedIn"], false); } s.run("connection.new", &json!({"kind":"s3","name":"conn-test-s3","options":{"credentials":"keys","buckets":"b11"}})).unwrap(); - assert!(s.run("connection.test", &json!({"id":"conn-test-s3"})).unwrap_err().contains("log in first")); + // A saved connection that uses keys reads them from the credential store (this session has none). + assert!(s.run("connection.test", &json!({"id":"conn-test-s3"})).unwrap_err().contains("Saved S3 keys are unavailable")); s.run("connection.login", &json!({"id":"conn-test-s3","accessKeyId":"AKIDTEST","secretAccessKey":"very-secret-key"})).unwrap(); assert!(s3::connection("conn-test-s3").is_some()); assert!(!s.connections.to_json().contains("very-secret")); @@ -10253,7 +10260,7 @@ fn connections_s3_rename_cannot_inherit_a_destination_session() { assert_eq!(renamed["loggedIn"], false); assert!(s3::connection(old).is_none()); assert!(s3::connection(destination).is_none()); - assert!(s.run("connection.test", &json!({"id":destination})).unwrap_err().contains("log in first")); + assert!(s.run("connection.test", &json!({"id":destination})).unwrap_err().contains("Saved S3 keys are unavailable")); s.run("connection.login", &json!({"id":destination,"accessKeyId":"NEWKEY","secretAccessKey":"new-test-secret"})).unwrap(); assert_eq!(s3::connection(destination).unwrap().endpoint.as_deref(), Some("https://saved.example")); s.run("connection.remove", &json!({"id":destination})).unwrap(); @@ -10436,7 +10443,12 @@ fn connection_browse_levels_and_adds() { let v = s.run("connection.browse", &json!({"id": "BrowseB", "node": "s3:bkt1/", "answer": listing})).unwrap(); assert_eq!(v["children"][0]["node"], "s3:bkt1/p/"); assert_eq!(v["children"][0]["name"], "p/"); - assert_eq!(v["children"][1]["add"]["params"]["url"], "s3://bkt1/p.tif"); + // Through a saved connection's login, a file's address names it: only that login reads it. + let added = v["children"][1]["add"]["params"]["url"].as_str().unwrap(); + let bound = rustgis_data::s3_address::parse(added).unwrap().unwrap(); + let saved_id = &s.connections.find("BrowseB").unwrap().id; + assert_eq!((bound.url, bound.endpoint, Some(bound.binding)), ("s3://bkt1/p.tif", "", s.s3_binding(saved_id))); + assert_eq!(v["children"][1]["key"], "s3://bkt1/p.tif"); assert_eq!(v["children"].as_array().unwrap().len(), 2, "{v}"); s3::disconnect("BrowseB"); diff --git a/crates/engine/src/tests/remembered_database_packets.rs b/crates/engine/src/tests/remembered_database_packets.rs index 5c0ee3ebe..1bcb4904e 100644 --- a/crates/engine/src/tests/remembered_database_packets.rs +++ b/crates/engine/src/tests/remembered_database_packets.rs @@ -307,6 +307,7 @@ fn remembered_database_exact_address_and_explicit_precedence_packets() { // saved one either: the address's wins, as it does when the project opens. explicit.run("connection.login", &json!({"id":id})).unwrap(); assert_eq!(server.take(), b"explicit-sentinel\0"); + server.empty(); } diff --git a/crates/map/src/lib.rs b/crates/map/src/lib.rs index 9bb76f2e6..25b0769b3 100644 --- a/crates/map/src/lib.rs +++ b/crates/map/src/lib.rs @@ -1323,6 +1323,9 @@ impl Map { let sources = self.layers.iter().filter(|l| l.source.is_some()).count().max(1); let bytes = source::VIEW_BYTES / sources; for layer in &mut self.layers { + if let Some(raster) = &mut layer.raster { + pending |= raster.poll_reload(); + } if let Some(src) = &mut layer.source { let changed = src.projection.as_ref().is_some_and(|p| p != &self.projection); let mode_changed = src.full_read != block; diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 70ed665e2..902e2620d 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -46,9 +46,15 @@ pub struct RasterLayer { /// The pixels; `None` when the source couldn't be read. #[serde(skip)] pub data: Option>, + /// The last reload failure; transient, like the decoded pixels. + #[serde(skip)] + pub error: Option, /// Statistics of the pixels the style's mask leaves, worked out on the first draw. #[serde(skip)] masked: MaskedStats, + /// A remote reload shared with render snapshots, never serialized. + #[serde(skip)] + opening: Option, } /// The renderer bands' statistics without the masked pixels, and what they were worked out for: @@ -56,6 +62,8 @@ pub struct RasterLayer { #[derive(Clone, Default)] struct MaskedStats(Arc, Weak, Arc>)>>>); +type RasterRead = Arc, String>>>>; + impl std::fmt::Debug for RasterLayer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RasterLayer") @@ -80,7 +88,9 @@ impl RasterLayer { placement: None, style, data: Some(Arc::new(raster)), + error: None, masked: MaskedStats::default(), + opening: None, } } @@ -97,7 +107,9 @@ impl RasterLayer { placement: None, style, data: Some(Arc::new(raster)), + error: None, masked: MaskedStats::default(), + opening: None, } } @@ -107,6 +119,39 @@ impl RasterLayer { if self.time_zarr.is_some() { return Ok(()); } + self.opening = None; + self.error = None; + if self.embedded.is_none() + && let Some(path) = self + .path + .as_deref() + .filter(|p| rustgis_data::s3_address::is_wrapper(p) || p.get(..5).is_some_and(|s| s.eq_ignore_ascii_case("s3://"))) + { + let name = rustgis_data::s3_address::inner(path).split(['?', '#']).next().unwrap_or("raster.tif").to_string(); + let placement = self.placement; + let slot = Arc::new(Mutex::new(None)); + let land = slot.clone(); + self.data = None; + self.opening = Some(slot); + rustgis_data::source::remote::fetch_whole( + path, + Box::new(move |result| { + let result = result + .map_err(|e| match e { + rustgis_data::source::remote::Failed::Other(message) => message, + _ => "The raster could not be downloaded.".to_string(), + }) + .and_then(|bytes| { + rustgis_data::raster::read_bytes(&name, &bytes) + .and_then(|r| placed(r, placement)) + .map(Arc::new) + .map_err(|e| e.to_string()) + }); + *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(result); + }), + ); + return Ok(()); + } let r = if let Some(bytes) = &self.embedded { rustgis_data::raster::read_bytes(self.file_name.as_deref().unwrap_or("raster.tif"), bytes) } else if let Some(p) = &self.path { @@ -115,15 +160,7 @@ impl RasterLayer { return Err("the raster has no source".into()); }; // A scan goes back where it was laid (a GeoTIFF opened `as` a scan is one again). - let r = r.and_then(|mut r| { - if let Some(bbox) = self.placement { - if r.georeferenced { - r = r.into_scan()?; - } - r.place(bbox)?; - } - Ok(r) - }); + let r = r.and_then(|r| placed(r, self.placement)); match r { Ok(r) => { self.style.repair(&r); @@ -131,12 +168,28 @@ impl RasterLayer { Ok(()) } Err(e) => { + self.error = Some(e.to_string()); self.data = None; Err(e.to_string()) } } } + /// Installs a completed remote read; each render copy can observe the same shared result. + pub(crate) fn poll_reload(&mut self) -> bool { + let Some(slot) = self.opening.as_ref() else { return false }; + let Some(result) = slot.lock().unwrap_or_else(|e| e.into_inner()).clone() else { return true }; + self.opening = None; + match result { + Ok(raster) => { + self.style.repair(&raster); + self.data = Some(raster); + } + Err(error) => self.error = Some(error), + } + false + } + /// Statistics of the renderer's bands (one per band, in its order) without the pixels the /// style's mask hides, for the stretch; `None` without a mask. Worked out once (from a sample /// of a large raster) and kept until the mask, the bands or the pixels change. @@ -163,6 +216,17 @@ impl RasterLayer { } } +/// Reapplies a scan's placement to pixels read again, without copying their bands. +fn placed(mut raster: Raster, placement: Option<[f64; 4]>) -> rustgis_data::Result { + if let Some(bbox) = placement { + if raster.georeferenced { + raster = raster.into_scan()?; + } + raster.place(bbox)?; + } + Ok(raster) +} + /// The most steps a Zarr layer following the time slider lists (as the slider itself does). pub const MAX_ZARR_STEPS: usize = crate::time::MAX_STEPS; diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index da3b1c1b5..11601e07d 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -607,6 +607,19 @@ impl SourceLayer { self.pending.is_some() || self.opening.is_some() } + /// Opens the source afresh, forgetting a failed or never-started open (and the window read + /// from the old one): a source by its web address read again once what signs it has changed + /// (its address, or the login behind it). Its symbology and place in the project stay. + pub fn reopen(&mut self) -> Result<(), String> { + self.handle = None; + self.opening = None; + self.pending = None; + self.error = None; + self.login = false; + self.reset(); + self.open() + } + /// Forgets the window, so the next view reads again (after the source changes). pub fn reset(&mut self) { self.window = None; diff --git a/crates/map/src/tiles.rs b/crates/map/src/tiles.rs index 98f0928d3..ed52dc951 100644 --- a/crates/map/src/tiles.rs +++ b/crates/map/src/tiles.rs @@ -314,8 +314,12 @@ impl TileLayer { /// The prefix of an MBTiles archive added to the web app, held in memory. pub const UPLOAD: &str = "upload:"; -/// Whether a PMTiles or MBTiles archive address is a file rather than a web address. +/// Whether a PMTiles or MBTiles archive address is a file rather than a web address. An address +/// read through a saved S3 connection is never a file, well formed or not. pub fn is_local_archive(url: &str) -> bool { + if rustgis_data::s3_address::is_wrapper(url) { + return false; + } let lower = url.to_ascii_lowercase(); !(lower.starts_with("https://") || lower.starts_with("http://") || lower.starts_with("s3://")) } @@ -340,8 +344,15 @@ pub fn check_path(path: &str, ext: &str) -> Result<(), String> { Ok(()) } -/// Whether `url` may be fetched: HTTP or HTTPS, with a host, and no longer than 4096 bytes. +/// Whether `url` may be fetched: HTTP or HTTPS, with a host, and no longer than 4096 bytes. An +/// address read through a saved S3 connection is judged by the address it wraps (still bounded +/// by 4096 bytes), and itself by the wrapper's own bound; it needs no login to be kept. pub fn check_url(url: &str) -> Result<(), String> { + let url = match rustgis_data::s3_address::parse(url) { + Ok(Some(b)) => b.url, + Ok(None) => url, + Err(e) => return Err(e.to_string()), + }; if url.len() > 4096 { return Err("the address is too long".into()); } @@ -358,8 +369,10 @@ pub fn check_url(url: &str) -> Result<(), String> { Ok(()) } -/// The host part of an HTTP(S) address. +/// The host part of an HTTP(S) address (of the address an address read through a saved S3 +/// connection wraps). pub fn host_of(url: &str) -> Option<&str> { + let url = rustgis_data::s3_address::inner(url); let rest = url.split_once("://")?.1; let host = rest.split(['/', '?', '#']).next()?; let host = host.rsplit_once('@').map_or(host, |(_, h)| h); diff --git a/crates/tiles/src/address.rs b/crates/tiles/src/address.rs index 3a9c68876..1212a04d5 100644 --- a/crates/tiles/src/address.rs +++ b/crates/tiles/src/address.rs @@ -113,9 +113,17 @@ fn inet_aton(h: &str) -> Option { /// Whether `url` may be fetched. `from` is where it was found (a document, or the address that /// redirected to it), or `None` when the user gave it. pub fn check(url: &str, from: Option<&str>) -> Result<(), String> { + // An address read through a saved S3 connection is judged by where it is fetched, which its + // own endpoint decides (no login is needed to keep it). + if let Some(destination) = crate::s3::bound_destination(url)? { + return check(&destination, from); + } // An `s3://` address is judged by the web address it is fetched from. // An address read anonymously is judged as the address it marks. if let (inner, true) = crate::s3::split_anonymous(url) { + if rustgis_data::s3_address::is_wrapper(inner) { + return Err("not a saved S3 address".into()); + } return if crate::s3::is_s3(inner) { check(&crate::s3::https_url(url)?, from) } else { check(inner, from) }; } if crate::s3::is_s3(url) { diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index c63b40bed..2767fd56d 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -220,6 +220,42 @@ pub fn clear() { GENERATION.fetch_add(1, Ordering::Relaxed); } +/// The resource a cache key reads: its address without a download budget, a byte range or a +/// tail, and for a tile of a PMTiles or COG archive, the archive's address. +fn resource_of(key: &str) -> &str { + let key = split_download_limit(key).0; + let key = split_tail(key).0; + let key = split_range(key).0; + if key.starts_with(crate::pmtiles::SCHEME) || key.starts_with(crate::cog::SCHEME) { + return key.split_once('@').map_or(key, |(_, archive)| crate::cog::split(archive).0); + } + key +} + +/// Forgets what was read of one address (and of the archive or file it names), so it is read +/// again, as [`clear`] does for everything: after the login that signs it changed (a source +/// whose address stays the same, read again with other keys). Reads under way are kept. +pub fn forget(url: &str) { + let url = crate::cog::split(split_download_limit(url).0).0; + let mut c = lock(); + let dropped: usize = c + .slots + .iter() + .filter(|(k, s)| !matches!(s.entry, Entry::Pending) && resource_of(k) == url) + .map(|(_, s)| match &s.entry { + Entry::Ready(_, n) => *n, + _ => 0, + }) + .sum(); + c.slots.retain(|k, s| matches!(s.entry, Entry::Pending) || resource_of(k) != url); + c.bytes = c.bytes.saturating_sub(dropped); + drop(c); + crate::pmtiles::forget_archive(url); + crate::cog::forget_file(url); + rustgis_data::source::remote::forget_file(url); + GENERATION.fetch_add(1, Ordering::Relaxed); +} + /// Waits (up to `timeout`) for the fetches under way to finish. Native only: in the browser /// fetches finish between frames, so this returns at once. pub fn wait(timeout: Duration) { @@ -249,7 +285,7 @@ fn start(url: String, want: Want) { let latest = c.latest.unwrap_or_else(Instant::now); let now = q.push(url, want, |u| c.slots.get(u).map(|s| s.asked), latest, &mut swept); // Under the same lock as the choice: a tile asked for again meanwhile isn't forgotten. - forget(&mut c, &swept); + forget_dropped(&mut c, &swept); now }; launch(now.into_iter().collect()); @@ -263,7 +299,7 @@ fn release(host: &str) { let mut c = lock(); let latest = c.latest.unwrap_or_else(Instant::now); let next = q.done(host, |url| c.slots.get(url).map(|s| s.asked), latest); - forget(&mut c, &next.drop); + forget_dropped(&mut c, &next.drop); next }; launch(next.start); @@ -271,7 +307,7 @@ fn release(host: &str) { /// Forgets tiles dropped unfetched: no longer under way, and fetched again when next asked for. /// Called holding the cache's lock taken to choose them, so a tile can't be asked for between. -fn forget(c: &mut Cache, dropped: &[Waiting]) { +fn forget_dropped(c: &mut Cache, dropped: &[Waiting]) { for w in dropped { if matches!(c.slots.get(&w.url), Some(Slot { entry: Entry::Pending, .. })) { c.slots.remove(&w.url); @@ -830,7 +866,7 @@ mod native { if crate::mbtiles::is_tile(url) || url.starts_with(LOCAL) { return String::new(); } - let Some((_, rest)) = url.split_once("://") else { return String::new() }; + let Some((_, rest)) = rustgis_data::s3_address::reached(url).split_once("://") else { return String::new() }; let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); let authority = rest.get(..end).unwrap_or(rest); // Credentials in the address aren't part of the server. @@ -1145,7 +1181,10 @@ mod native { } } if status >= 400 { - return Err(if detail { + // A signed request's refusal is told by its status alone: an answer's headers and + // body are the server's text, and a server may echo the request's Authorization + // header (the access key's id) back in it. + return Err(if detail && !prepared.signed { refused(&mut resp) } else { Failure::without_answer("", format!("http status: {status}"), Reason::Status(status)) diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 6d8153b44..4ab91fcd8 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -234,6 +234,18 @@ pub(crate) fn forget() { t.samples = 0; } +/// Forgets what was read of one file (its info and its tiles), but not a read under way: a +/// failed read is made again with whatever login now signs the file's address. +pub(crate) fn forget_file(url: &str) { + let mut all = infos().lock().unwrap_or_else(|e| e.into_inner()); + if all.get(url).is_some_and(|s| !matches!(s, State::Loading(_))) { + all.remove(url); + } + drop(all); + let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); + t.map.retain(|(u, _, _), s| u != url || matches!(s, TileState::Loading(_))); +} + /// A file's info: `Ok(None)` while it is being read (asking for it starts the read). pub fn info(url: &str) -> Result>, String> { let url = split(url).0; diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index a79321692..71326cc6c 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -164,7 +164,11 @@ pub fn is_tile(url: &str) -> bool { /// Whether an archive address is a file on this computer rather than a web address. /// (As `rustgis_map::tiles::is_local_archive` decides it, which a layer's address went through.) +/// An address read through a saved S3 connection is never a file, well formed or not. pub fn is_local(archive: &str) -> bool { + if rustgis_data::s3_address::is_wrapper(archive) { + return false; + } let lower = archive.get(..8).unwrap_or(archive).to_ascii_lowercase(); !(lower.starts_with("http://") || lower.starts_with("https://") || lower.starts_with("s3://")) } @@ -387,6 +391,17 @@ pub(crate) fn forget() { leaves().lock().unwrap_or_else(|e| e.into_inner()).retain(|_, l| matches!(l, Leaf::Loading(_))); } +/// Forgets what was read of one archive (its header, and its directories), but not a read under +/// way: a failed read is made again with whatever login now signs the archive's address. +pub(crate) fn forget_archive(archive: &str) { + let mut all = archives().lock().unwrap_or_else(|e| e.into_inner()); + if all.get(archive).is_some_and(|s| !matches!(s, State::Loading(_))) { + all.remove(archive); + } + drop(all); + leaves().lock().unwrap_or_else(|e| e.into_inner()).retain(|(a, _), l| a != archive || matches!(l, Leaf::Loading(_))); +} + /// An archive's header and metadata: `Ok(None)` while they are read (the map draws again once /// they arrive). pub fn info(archive: &str) -> Result>, String> { diff --git a/crates/tiles/src/s3.rs b/crates/tiles/src/s3.rs index bd7a6d60d..1fd3871dd 100644 --- a/crates/tiles/src/s3.rs +++ b/crates/tiles/src/s3.rs @@ -59,17 +59,52 @@ impl Connection { } } -fn store() -> &'static Mutex> { - static S: std::sync::OnceLock>> = std::sync::OnceLock::new(); - S.get_or_init(Default::default) +/// The live connections, and which of them a saved connection's login made (see [`connect_saved`]). +/// Under one lock so a worker never sees one without the other. +#[derive(Default)] +struct Registry { + connections: Vec, + saved: Vec, } -fn lock() -> std::sync::MutexGuard<'static, Vec> { - store().lock().unwrap_or_else(|e| e.into_inner()) +/// What a saved connection's login in one session is: the session's runtime token for it, the +/// saved connection's id, the live connection's name, and the buckets the user explicitly +/// granted it by browsing or choosing it. Never a second copy of the credentials. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Saved { + binding: String, + id: String, + name: String, + grants: Vec, } -/// Adds (or replaces, by name) a connection for the session, after checking it. -pub fn connect(mut c: Connection) -> Result<(), String> { +impl Registry { + /// Whether the live connection of this name came from a saved connection's login: those + /// sign only the addresses that name their binding, never an address found by its bucket + /// or its host. + fn is_saved(&self, name: &str) -> bool { + self.saved.iter().any(|s| s.name == name) + } + + /// The connections that sign (or serve) addresses found by their bucket or their host. + fn implicit(&self) -> impl Iterator { + self.connections.iter().filter(|c| !self.is_saved(&c.name)) + } + + fn drop_name(&mut self, name: &str) { + self.connections.retain(|c| c.name != name); + self.saved.retain(|s| s.name != name); + } +} + +static REGISTRY: std::sync::LazyLock> = std::sync::LazyLock::new(Default::default); + +fn lock() -> std::sync::MutexGuard<'static, Registry> { + REGISTRY.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// A connection as the registry keeps it: name, endpoint, region, buckets and credentials checked. +fn checked(mut c: Connection) -> Result { c.name = c.name.trim().to_string(); if c.name.is_empty() || c.name.chars().count() > 64 || c.name.chars().any(char::is_control) { return Err("a connection needs a name of up to 64 characters".into()); @@ -94,31 +129,120 @@ pub fn connect(mut c: Connection) -> Result<(), String> { } c.buckets.truncate(64); if let Some(cr) = &c.credentials { - let ok = |s: &str, max: usize| !s.is_empty() && s.len() <= max && !s.chars().any(|ch| ch.is_control() || ch.is_whitespace()); - if !ok(&cr.access_key_id, 128) || !ok(&cr.secret_access_key, 256) || cr.session_token.as_deref().is_some_and(|t| !ok(t, 4096)) { - return Err("the access key, secret or session token isn't well formed".into()); - } + check_credentials(cr)?; + } + Ok(c) +} + +/// Whether an access key, its secret and its session token (if any) are of a form S3 takes. +pub fn check_credentials(cr: &Credentials) -> Result<(), String> { + let ok = |s: &str, max: usize| !s.is_empty() && s.len() <= max && !s.chars().any(|ch| ch.is_control() || ch.is_whitespace()); + if !ok(&cr.access_key_id, 128) || !ok(&cr.secret_access_key, 256) || cr.session_token.as_deref().is_some_and(|t| !ok(t, 4096)) { + return Err("the access key, secret or session token isn't well formed".into()); } - let mut all = lock(); - all.retain(|o| o.name != c.name); - if all.len() >= MAX_CONNECTIONS { + Ok(()) +} + +/// Adds (or replaces, by name) a connection for the session, after checking it. +pub fn connect(c: Connection) -> Result<(), String> { + let c = checked(c)?; + let mut reg = lock(); + // The name stays free if it can't be filled: only a connection that fits replaces the old one. + let replaced = reg.connections.iter().filter(|o| o.name == c.name).count(); + if reg.connections.len() - replaced >= MAX_CONNECTIONS { return Err(format!("up to {MAX_CONNECTIONS} connections")); } + // A saved connection's login replaced by an ordinary one of its name is no longer a saved + // login: addresses bound to it find no connection until it logs in again. + reg.drop_name(&c.name); // A connection replaced by name starts afresh: the buckets granted to the old one (browsed // through another endpoint or with other keys) aren't the new one's. granted().retain(|(n, _)| n != &c.name); - all.push(c); + reg.connections.push(c); + Ok(()) +} + +/// A runtime token for a saved connection's login in one session: 16 lower-case hexadecimal +/// digits, never zero, never given twice in this process (two sessions' saved ids are often the +/// same, `c1`; their logins mustn't be). +pub fn new_binding() -> Result { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = NEXT + .fetch_update(std::sync::atomic::Ordering::SeqCst, std::sync::atomic::Ordering::SeqCst, |n| n.checked_add(1)) + .map_err(|_| "S3 connection tokens are used up".to_string())? + .wrapping_add(1); + Ok(format!("{n:016x}")) +} + +/// Makes the login of the saved connection `id` live for the session under `binding`: checked as +/// [`connect`] does, replacing a live connection of that name, and registered as bound in the +/// same step, so no worker sees it eligible for implicit signing in between. The buckets +/// granted to this binding are kept for a login of the same connection at the same endpoint, and +/// forgotten otherwise. +pub fn connect_saved(c: Connection, binding: &str, id: &str) -> Result<(), String> { + if binding.is_empty() || !rustgis_data::s3_address::valid_binding(binding) { + return Err("not a saved S3 connection token".into()); + } + if id.is_empty() || id.len() > 64 { + return Err("a saved connection needs an id".into()); + } + let c = checked(c)?; + let mut reg = lock(); + let replaced = reg.connections.iter().filter(|o| o.name == c.name).count(); + if reg.connections.len() - replaced >= MAX_CONNECTIONS { + return Err(format!("up to {MAX_CONNECTIONS} connections")); + } + // What this binding was granted, when it is the same login coming back (a login again, or a + // new set of keys): nothing when the endpoint it was for is not this one. + let kept = reg + .saved + .iter() + .find(|s| s.binding == binding && s.id == id) + .filter(|s| reg.connections.iter().any(|o| o.name == s.name && o.endpoint == c.endpoint)) + .map(|s| s.grants.clone()) + .unwrap_or_default(); + reg.drop_name(&c.name); + // This binding's earlier live connection, under another name (a renamed saved connection). + let moved: Vec = reg.saved.iter().filter(|s| s.binding == binding).map(|s| s.name.clone()).collect(); + for name in moved { + reg.drop_name(&name); + granted().retain(|(n, _)| *n != name); + } + granted().retain(|(n, _)| n != &c.name); + reg.saved.push(Saved { binding: binding.to_string(), id: id.to_string(), name: c.name.clone(), grants: kept }); + reg.connections.push(c); + Ok(()) +} + +/// How many buckets may be granted to one saved connection's login. +const MAX_SAVED_GRANTS: usize = 256; + +/// Lets the saved login `binding` serve a bucket for the rest of the session: the user browsed +/// or chose it through that connection. Unlike [`grant`], other bindings keep theirs (two sources +/// bound to different connections may read the same bucket). +pub fn grant_saved(binding: &str, bucket: &str) -> Result<(), String> { + if !valid_bucket(bucket) { + return Err(format!("'{bucket}' isn't a bucket name")); + } + let mut reg = lock(); + let s = reg.saved.iter_mut().find(|s| !binding.is_empty() && s.binding == binding).ok_or(UNRESOLVED)?; + if !s.grants.iter().any(|b| b == bucket) { + if s.grants.len() >= MAX_SAVED_GRANTS { + return Err("too many buckets for one connection".into()); + } + s.grants.push(bucket.to_string()); + } Ok(()) } /// Forgets a connection (and its credentials, and the buckets granted to it); whether there was /// one by that name. pub fn disconnect(name: &str) -> bool { - let mut all = lock(); - let before = all.len(); - all.retain(|c| c.name != name); + let mut reg = lock(); + let before = reg.connections.len(); + reg.drop_name(name); granted().retain(|(n, _)| n != name); - all.len() != before + reg.connections.len() != before } fn granted() -> std::sync::MutexGuard<'static, Vec<(String, String)>> { @@ -146,16 +270,53 @@ fn serves(c: &Connection, bucket: &str) -> bool { /// The session's connections. pub fn connections() -> Vec { - lock().clone() + lock().connections.clone() } /// A connection by name. pub fn connection(name: &str) -> Option { - lock().iter().find(|c| c.name == name).cloned() + lock().connections.iter().find(|c| c.name == name).cloned() +} + +/// The live connection of the saved login `binding`, and the saved id it is for. +pub fn saved_connection(binding: &str) -> Option<(Connection, String)> { + let reg = lock(); + let s = reg.saved.iter().find(|s| !binding.is_empty() && s.binding == binding)?; + let c = reg.connections.iter().find(|c| c.name == s.name)?; + Some((c.clone(), s.id.clone())) +} + +/// Whether a saved definition may reuse a live login. Mapped logins require the exact session +/// binding and saved ID. Only callers without a token may reuse an ordinary name-based login. +/// Checks the atomic registry without cloning credentials. +pub fn can_reuse_login(binding: Option<&str>, id: &str, name: &str) -> bool { + let reg = lock(); + match reg.saved.iter().find(|s| s.name == name) { + Some(saved) => binding == Some(saved.binding.as_str()) && saved.id == id, + None => binding.is_none() && reg.connections.iter().any(|c| c.name == name), + } +} + +/// The runtime token of the saved login that is the live connection `name`, if it is one. +pub fn binding_of(name: &str) -> Option { + lock().saved.iter().find(|s| s.name == name).map(|s| s.binding.clone()) +} + +/// What may be said to someone whose address names no live saved connection to read it with. +pub const UNRESOLVED: &str = "Choose a saved S3 connection before opening this source."; +/// What may be said when the saved connection no longer fits the address that names it. +pub const MISMATCH: &str = "The saved S3 connection no longer matches this source."; + +/// `url` read through the saved login `binding`, with that connection's own endpoint: the address +/// a fetch of it, and a project, keep. (A binding alone grants no bucket: [`grant_saved`], or the +/// connection's own buckets, do.) +pub fn bound_url(binding: &str, url: &str) -> Result { + let (c, _) = saved_connection(binding).ok_or(UNRESOLVED)?; + rustgis_data::s3_address::bind(binding, c.endpoint.as_deref().unwrap_or(""), url).map_err(str::to_string) } /// An endpoint as connections keep it: a web address's scheme and authority, no path. -fn endpoint(e: &str) -> Result { +pub fn endpoint(e: &str) -> Result { let e = e.trim().trim_end_matches('/'); let lower = e.to_ascii_lowercase(); if !(lower.starts_with("https://") || lower.starts_with("http://")) { @@ -207,10 +368,11 @@ pub fn parse(url: &str) -> Result<(String, String), String> { /// The connection serving a bucket: one given the bucket, or one it was granted to this session /// ([`grant`]). None serves any other bucket: an address naming it (a project's, a link's) is read -/// anonymously from AWS, never with the user's keys or from the user's endpoint. +/// anonymously from AWS, never with the user's keys or from the user's endpoint. A saved +/// connection's login serves no bucket this way: its addresses name it ([`connect_saved`]). pub fn for_bucket(bucket: &str) -> Option { - let all = lock(); - all.iter().find(|c| serves(c, bucket)).cloned() + let reg = lock(); + reg.implicit().find(|c| serves(c, bucket)).cloned() } /// Starts an address read anonymously whatever the connections: nothing signed, and an `s3://` @@ -245,14 +407,103 @@ pub fn object_url(bucket: &str, key: &str, conn: Option<&Connection>) -> String } } -/// The web address an `s3://` address is fetched from. +/// The web address an `s3://` address is fetched from. An address read through a saved +/// connection is the one that connection fetches it at. pub fn https_url(url: &str) -> Result { + if rustgis_data::s3_address::is_wrapper(url) { + return bound_target(url).map(|t| t.0); + } let (url, anonymous) = split_anonymous(url); let (bucket, key) = parse(url)?; let conn = if anonymous { None } else { for_bucket(&bucket) }; Ok(object_url(&bucket, &key, conn.as_ref())) } +/// The canonical endpoint a saved connection's address (empty: AWS) stands for. +pub fn canonical(address: &str) -> Result, String> { + let a = address.trim(); + if a.is_empty() { Ok(None) } else { endpoint(a).map(Some) } +} + +/// Where an address read through a saved connection is fetched, judged without any login: the +/// address's own endpoint decides how an `s3://` address maps to the web. `None` for an address +/// that isn't one read through a saved connection. +pub fn bound_destination(address: &str) -> Result, String> { + let Some(b) = rustgis_data::s3_address::parse(address).map_err(str::to_string)? else { return Ok(None) }; + if !is_s3(b.url) { + return Ok(Some(b.url.to_string())); + } + let (bucket, key) = parse(b.url)?; + let at = Connection { + name: String::new(), + endpoint: canonical(b.endpoint)?, + region: DEFAULT_REGION.into(), + buckets: Vec::new(), + credentials: None, + }; + Ok(Some(object_url(&bucket, &key, Some(&at)))) +} + +/// What a saved login's address is fetched at, and by whom: the exact connection its binding +/// names (never another that merely has the same endpoint or bucket), only when the address's +/// endpoint is that connection's, and only for an address of a bucket the connection serves (its +/// own, or one explicitly granted to this binding), or of its account's bucket list. +fn bound_target(address: &str) -> Result<(String, Connection), String> { + let b = rustgis_data::s3_address::parse(address).map_err(str::to_string)?.ok_or(UNRESOLVED)?; + if b.binding.is_empty() { + return Err(UNRESOLVED.into()); + } + let wanted = canonical(b.endpoint).map_err(|_| MISMATCH.to_string())?; + let reg = lock(); + let saved = reg.saved.iter().find(|s| s.binding == b.binding).ok_or(UNRESOLVED)?; + let c = reg.connections.iter().find(|c| c.name == saved.name).ok_or(UNRESOLVED)?; + if c.endpoint != wanted { + return Err(MISMATCH.into()); + } + let serves = |bucket: &str| c.buckets.iter().any(|x| x == bucket) || saved.grants.iter().any(|x| x == bucket); + if is_s3(b.url) { + let (bucket, key) = parse(b.url).map_err(|_| MISMATCH.to_string())?; + if !serves(&bucket) { + return Err(MISMATCH.into()); + } + return Ok((object_url(&bucket, &key, Some(c)), c.clone())); + } + // Only the origin is case-insensitive. Bucket paths retain their exact spelling. + let (scheme, rest) = b.url.split_once("://").ok_or(MISMATCH)?; + let scheme = if scheme.eq_ignore_ascii_case("https") { "https" } else { "http" }; + let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); + if authority.contains('@') { + return Err(MISMATCH.into()); + } + let here = origin(scheme, authority); + let allowed = match c.endpoint.as_deref() { + // The connection's own endpoint, path-style: the path's first part is the bucket (none: the + // account's bucket list). + Some(ep) => { + let path = rest.get(authority.len()..).unwrap_or(""); + let first = path.trim_start_matches('/').split(['/', '?', '#']).next().unwrap_or(""); + here == ep + && (first.is_empty() && path.split(['?', '#']).next().unwrap_or("").trim_start_matches('/').is_empty() || serves(first)) + } + None => { + let host = here.strip_prefix("https://").unwrap_or("").split(':').next().unwrap_or(""); + aws_host(scheme, &here, host) + && match aws_bucket(host, rest) { + Some(bucket) => serves(&bucket), + None => { + let path = rest.split_once('/').map(|(_, p)| p).unwrap_or(""); + host.strip_prefix("s3.").and_then(|r| r.strip_suffix(".amazonaws.com")) == Some(c.region.as_str()) + && path.split(['?', '#']).next().unwrap_or("").is_empty() + } + } + } + }; + if !allowed { + return Err(MISMATCH.into()); + } + Ok((b.url.to_string(), c.clone())) +} + /// A request ready to send: its web address, the headers to add, and whether it is signed (a /// signed request is never redirected elsewhere). #[derive(Clone, Debug, PartialEq, Eq)] @@ -264,9 +515,26 @@ pub struct Prepared { /// Makes a GET of `url` ready: an `s3://` address becomes its web address, and a request to a /// connection's endpoint (or to AWS S3 for an AWS connection) is signed with its credentials. -/// Other addresses pass unchanged. +/// An address read through a saved connection is signed by that connection alone +/// ([`bound_target`]), and never read without it. Other addresses pass unchanged. pub fn prepare(url: &str) -> Result { + if rustgis_data::s3_address::is_wrapper(url) { + let (target, c) = bound_target(url)?; + let Some(cr) = c.credentials.as_ref() else { + return Ok(Prepared { url: target, headers: Vec::new(), signed: false }); + }; + let region = match c.endpoint { + Some(_) => c.region.clone(), + None => aws_region(&target.to_ascii_lowercase(), &c.region), + }; + let url = canonical_url(&target)?; + let headers = sign_get(&url, ®ion, cr, &amz_date(now_secs()))?; + return Ok(Prepared { url, headers, signed: true }); + } let (url, anonymous) = split_anonymous(url); + if anonymous && rustgis_data::s3_address::is_wrapper(url) { + return Err("not a saved S3 address".into()); + } let (url, signer) = if is_s3(url) { let (bucket, key) = parse(url)?; let conn = if anonymous { None } else { for_bucket(&bucket) }; @@ -313,10 +581,37 @@ fn canonical_path(path: &str) -> String { path.split('/').map(|part| encode(&decode_path(part), true)).collect::>().join("/") } +/// Whether an address (its scheme, its origin and its host, all lower-case) is at AWS S3. +fn aws_host(scheme: &str, here: &str, host: &str) -> bool { + scheme == "https" + && here == format!("https://{host}") + && host.ends_with(".amazonaws.com") + && (host.starts_with("s3.") || host.contains(".s3.") || host.starts_with("s3-") || host.contains(".s3-")) +} + +/// The region an address at AWS S3 names (`s3..amazonaws.com`), else `default`. The +/// address is lower-case. +fn aws_region(url: &str, default: &str) -> String { + let rest = url.split_once("://").map(|(_, r)| r).unwrap_or(url); + let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); + let host = authority.split(':').next().unwrap_or(authority); + let after = host + .rsplit_once(".s3.") + .map(|(_, a)| a) + .or_else(|| host.strip_prefix("s3.")) + .or_else(|| host.rsplit_once(".s3-").map(|(_, a)| a)) + .or_else(|| host.strip_prefix("s3-")); + after + .and_then(|a| a.split('.').next()) + .filter(|r| !r.is_empty() && *r != "amazonaws" && *r != "dualstack") + .map(str::to_string) + .unwrap_or_else(|| default.to_string()) +} + /// The connection whose credentials sign a request to `url`, and the region it is signed for: /// one whose endpoint is the address's scheme and authority (default ports aside), or (an AWS /// connection) an address at AWS S3 of a bucket the connection serves ([`for_bucket`]): never -/// another bucket at amazonaws.com. +/// another bucket at amazonaws.com. A saved connection's login never signs by address. fn signer_for(url: &str) -> Option<(Connection, String)> { let lower = url.to_ascii_lowercase(); let (scheme, rest) = lower.split_once("://")?; @@ -325,20 +620,16 @@ fn signer_for(url: &str) -> Option<(Connection, String)> { return None; } let here = origin(scheme, authority); - let all = lock(); - if let Some(c) = all.iter().find(|c| c.credentials.is_some() && c.endpoint.as_deref() == Some(here.as_str())) { + let reg = lock(); + if let Some(c) = reg.implicit().find(|c| c.credentials.is_some() && c.endpoint.as_deref() == Some(here.as_str())) { return Some((c.clone(), c.region.clone())); } let host = authority.split(':').next().unwrap_or(authority); - if scheme != "https" - || here != format!("https://{host}") - || !host.ends_with(".amazonaws.com") - || !(host.starts_with("s3.") || host.contains(".s3.") || host.starts_with("s3-") || host.contains(".s3-")) - { + if !aws_host(scheme, &here, host) { return None; } let c = match aws_bucket(host, rest) { - Some(bucket) => all.iter().find(|c| c.credentials.is_some() && c.endpoint.is_none() && serves(c, &bucket))?, + Some(bucket) => reg.implicit().find(|c| c.credentials.is_some() && c.endpoint.is_none() && serves(c, &bucket))?, // The service itself (`https://s3..amazonaws.com/`, no bucket): the account's // bucket list (`buckets_url`), signed by an AWS connection of that region. It names no // bucket, so no one else's server sees the request. @@ -348,22 +639,10 @@ fn signer_for(url: &str) -> Option<(Connection, String)> { if !path.split(['?', '#']).next().unwrap_or("").is_empty() { return None; } - all.iter().find(|c| c.credentials.is_some() && c.endpoint.is_none() && c.region == region)? + reg.implicit().find(|c| c.credentials.is_some() && c.endpoint.is_none() && c.region == region)? } }; - // The region the address names (s3..amazonaws.com), else the connection's. - let after = host - .rsplit_once(".s3.") - .map(|(_, a)| a) - .or_else(|| host.strip_prefix("s3.")) - .or_else(|| host.rsplit_once(".s3-").map(|(_, a)| a)) - .or_else(|| host.strip_prefix("s3-")); - let region = after - .and_then(|a| a.split('.').next()) - .filter(|r| !r.is_empty() && *r != "amazonaws" && *r != "dualstack") - .map(str::to_string) - .unwrap_or_else(|| c.region.clone()); - Some((c.clone(), region)) + Some((c.clone(), aws_region(&lower, &c.region))) } /// The bucket an AWS S3 address names: its host's first label (virtual-hosted) or its path's @@ -383,6 +662,24 @@ fn aws_bucket(host: &str, rest: &str) -> Option { path.split(['/', '?', '#']).next().filter(|b| valid_bucket(b)).map(str::to_string) } +/// The bucket an address read at `endpoint` (empty: AWS) names: an `s3://` address's, a web +/// address's first path part at a custom endpoint, or at AWS its host's or path's bucket. `None` +/// for an address of no bucket (the account's bucket list) or of no S3 address at all. +pub fn bucket_of(endpoint: &str, url: &str) -> Option { + if is_s3(url) { + return parse(url).ok().map(|(bucket, _)| bucket); + } + let lower = url.to_ascii_lowercase(); + let (_, rest) = lower.split_once("://")?; + let authority = rest.split(['/', '?', '#']).next()?; + if endpoint.is_empty() { + let host = authority.split(':').next().unwrap_or(authority); + return aws_bucket(host, rest); + } + let path = rest.get(authority.len()..).unwrap_or(""); + path.trim_start_matches('/').split(['/', '?', '#']).next().filter(|b| valid_bucket(b)).map(str::to_string) +} + /// The headers that sign a GET of `url` for S3 at `date` (`YYYYMMDDTHHMMSSZ`). fn sign_get(url: &str, region: &str, cr: &Credentials, date: &str) -> Result, String> { let (host, path, query) = split_url(url)?; @@ -943,6 +1240,128 @@ mod tests { assert!(!prepare("s3://rule-granted/k").unwrap().signed, "grants go with the connection"); } + fn saved_login(name: &str, key: &str, endpoint: Option<&str>, buckets: &[&str]) -> Connection { + Connection { + name: name.into(), + endpoint: endpoint.map(str::to_string), + region: String::new(), + buckets: buckets.iter().map(|b| b.to_string()).collect(), + credentials: Some(Credentials { access_key_id: key.into(), secret_access_key: "saved-secret-xyz".into(), session_token: None }), + } + } + + fn key_of(p: &Prepared) -> String { + let auth = p.headers.iter().find(|(k, _)| k == "authorization").map(|(_, v)| v.as_str()).unwrap_or(""); + auth.split("Credential=").nth(1).and_then(|c| c.split('/').next()).unwrap_or("").to_string() + } + + #[test] + fn saved_logins_sign_only_through_their_own_binding() { + let ep = "http://127.0.0.1:9701"; + let (a, b) = (new_binding().unwrap(), new_binding().unwrap()); + assert_ne!(a, b); + assert!(a.len() == 16 && a.bytes().all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase())); + // The second saved connection is registered first: insertion order decides nothing. + connect_saved(saved_login("sv-b", "SAVEDKEYB", Some(ep), &["sv-bucket"]), &b, "c1").unwrap(); + connect_saved(saved_login("sv-a", "SAVEDKEYA", Some(ep), &["sv-bucket"]), &a, "c1").unwrap(); + let url = "http://127.0.0.1:9701/sv-bucket/k.fgb"; + let wa = rustgis_data::s3_address::bind(&a, ep, url).unwrap(); + let wb = rustgis_data::s3_address::bind(&b, ep, url).unwrap(); + let (pa, pb) = (prepare(&wa).unwrap(), prepare(&wb).unwrap()); + assert_eq!((key_of(&pa), key_of(&pb)), ("SAVEDKEYA".to_string(), "SAVEDKEYB".to_string())); + assert!(pa.signed && pa.url == url); + assert!(pa.headers.iter().all(|(_, v)| !v.contains("saved-secret"))); + // An unrelated address of the same endpoint or bucket isn't signed by either login. + for raw in [url, "s3://sv-bucket/k.fgb", "http://127.0.0.1:9701/"] { + let p = prepare(raw).unwrap(); + assert!(!p.signed && p.headers.is_empty(), "{raw}"); + } + assert_eq!(for_bucket("sv-bucket"), None); + // The account's bucket list, through the exact connection's own endpoint. + let root = rustgis_data::s3_address::bind(&b, ep, "http://127.0.0.1:9701/").unwrap(); + assert_eq!(key_of(&prepare(&root).unwrap()), "SAVEDKEYB"); + // s3:// addresses are fetched at the connection's endpoint. + let s3 = rustgis_data::s3_address::bind(&a, ep, "s3://sv-bucket/dir/k.fgb").unwrap(); + let p = prepare(&s3).unwrap(); + assert_eq!((p.url.as_str(), key_of(&p).as_str()), ("http://127.0.0.1:9701/sv-bucket/dir/k.fgb", "SAVEDKEYA")); + assert_eq!(https_url(&s3).unwrap(), p.url); + // Nothing falls back to another signer. + let bind = rustgis_data::s3_address::bind; + let other_endpoint = bind(&a, "http://127.0.0.1:9702", "http://127.0.0.1:9702/sv-bucket/k").unwrap(); + let mismatched = bind(&a, "http://127.0.0.1:9702", url).unwrap(); + let ungranted = bind(&a, ep, "http://127.0.0.1:9701/sv-other/k").unwrap(); + let elsewhere = bind(&a, ep, "http://127.0.0.1:9799/sv-bucket/k").unwrap(); + let userinfo = bind(&a, ep, "http://u@127.0.0.1:9701/sv-bucket/k").unwrap(); + let wrong_case = bind(&a, ep, "http://127.0.0.1:9701/SV-BUCKET/k").unwrap(); + for wrong in [other_endpoint, mismatched, ungranted, elsewhere, userinfo, wrong_case] { + assert_eq!(prepare(&wrong).unwrap_err(), MISMATCH, "{wrong}"); + } + for unresolved in [bind("", ep, url).unwrap(), bind("00000000deadbeef", ep, url).unwrap()] { + assert_eq!(prepare(&unresolved).unwrap_err(), UNRESOLVED); + } + for malformed in [format!("rustgis-s3:{a}:99:{ep}:{url}"), format!("rustgis-s3:{a}:21:{ep}:{wa}"), anonymous(&wa)] { + assert!(prepare(&malformed).is_err(), "{malformed}"); + } + // An explicit grant is one binding's: the other still can't read that bucket. + grant_saved(&a, "sv-granted").unwrap(); + let granted_url = "http://127.0.0.1:9701/sv-granted/k"; + assert_eq!(key_of(&prepare(&bind(&a, ep, granted_url).unwrap()).unwrap()), "SAVEDKEYA"); + assert_eq!(prepare(&bind(&b, ep, granted_url).unwrap()).unwrap_err(), MISMATCH); + grant_saved(&b, "sv-granted").unwrap(); + assert_eq!(key_of(&prepare(&bind(&b, ep, granted_url).unwrap()).unwrap()), "SAVEDKEYB"); + assert!(grant_saved("", "sv-granted").is_err() && grant_saved("00000000deadbeef", "sv-granted").is_err()); + assert!(grant_saved(&a, "NOT_A_BUCKET").is_err()); + // Judged without any login, an address is kept (or refused) by where it is fetched. + assert!(crate::address::check(&bind("", ep, url).unwrap(), None).is_ok()); + assert!(crate::address::check(&bind("", "https://s3.example.org", "s3://sv-bucket/k").unwrap(), None).is_ok()); + assert!(crate::address::check(&bind("", "http://169.254.169.254", "http://169.254.169.254/k").unwrap(), None).is_err()); + assert!(crate::address::check(&anonymous(&wa), None).is_err()); + assert!(crate::address::check("rustgis-s3:junk", None).is_err()); + // The server a wrapped address is throttled under is its own, in a tile template too. + assert_eq!(rustgis_data::s3_address::reached(&wa), url); + assert_eq!(rustgis_data::s3_address::reached(&format!("pmtiles:{{z}}/{{x}}/{{y}}@{wa}")), url); + // Replaced by an ordinary connection of the same name: the binding finds nothing. + connect(saved_login("sv-a", "PLAINKEY", Some(ep), &["sv-bucket"])).unwrap(); + assert_eq!(prepare(&wa).unwrap_err(), UNRESOLVED); + assert_eq!(saved_connection(&a), None); + assert_eq!(key_of(&prepare(url).unwrap()), "PLAINKEY", "ordinary connections keep their implicit behaviour"); + assert_eq!(key_of(&prepare(&wb).unwrap()), "SAVEDKEYB"); + disconnect("sv-a"); + disconnect("sv-b"); + assert_eq!(prepare(&wb).unwrap_err(), UNRESOLVED); + } + + #[test] + fn saved_logins_keep_grants_only_for_the_same_login_and_endpoint() { + let ep = "http://127.0.0.1:9711"; + let a = new_binding().unwrap(); + connect_saved(saved_login("sv-grant", "GRANTKEY1", Some(ep), &[]), &a, "c7").unwrap(); + grant_saved(&a, "sv-browsed").unwrap(); + let url = rustgis_data::s3_address::bind(&a, ep, "http://127.0.0.1:9711/sv-browsed/k").unwrap(); + assert!(prepare(&url).unwrap().signed); + assert_eq!(binding_of("sv-grant").as_deref(), Some(a.as_str())); + assert_eq!(saved_connection(&a).map(|(c, id)| (c.name, id)), Some(("sv-grant".to_string(), "c7".to_string()))); + // Logging in again at the same endpoint (new keys): the grant stays. + connect_saved(saved_login("sv-grant", "GRANTKEY2", Some(ep), &[]), &a, "c7").unwrap(); + assert_eq!(key_of(&prepare(&url).unwrap()), "GRANTKEY2"); + // The same binding under another name (a rename): the old name is gone. + connect_saved(saved_login("sv-grant-2", "GRANTKEY3", Some(ep), &[]), &a, "c7").unwrap(); + assert!(connection("sv-grant").is_none() && binding_of("sv-grant").is_none()); + // At another endpoint: nothing granted before carries over. + connect_saved(saved_login("sv-grant-2", "GRANTKEY4", Some("http://127.0.0.1:9712"), &[]), &a, "c7").unwrap(); + let moved = rustgis_data::s3_address::bind(&a, "http://127.0.0.1:9712", "http://127.0.0.1:9712/sv-browsed/k").unwrap(); + assert_eq!(prepare(&moved).unwrap_err(), MISMATCH); + // A binding is a token, not a name: junk and the id are refused. + assert!(connect_saved(saved_login("sv-grant-3", "K", Some(ep), &[]), "c7", "c7").is_err()); + assert!(connect_saved(saved_login("sv-grant-3", "K", Some(ep), &[]), "", "c7").is_err()); + // A login that doesn't fit leaves nothing registered. + let bad = Credentials { access_key_id: "has space".into(), secret_access_key: "s".into(), session_token: None }; + let before = connections().len(); + assert!(connect_saved(Connection { credentials: Some(bad), ..saved_login("sv-grant-4", "K", Some(ep), &[]) }, &a, "c7").is_err()); + assert!(connection("sv-grant-4").is_none() && connections().len() == before); + disconnect("sv-grant-2"); + } + #[test] fn listings_and_profiles() { let xml = br#" diff --git a/crates/tiles/src/schedule.rs b/crates/tiles/src/schedule.rs index 8849cea6d..f44d32c5b 100644 --- a/crates/tiles/src/schedule.rs +++ b/crates/tiles/src/schedule.rs @@ -36,7 +36,7 @@ pub(crate) fn host(url: &str) -> String { if crate::mbtiles::is_tile(url) { return String::new(); } - let Some((_, rest)) = url.split_once("://") else { return String::new() }; + let Some((_, rest)) = rustgis_data::s3_address::reached(url).split_once("://") else { return String::new() }; let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); let authority = rest.get(..end).unwrap_or(rest); // Credentials in the address aren't part of the server. diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index cada206fe..e4841cc34 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -687,6 +687,7 @@ fn search_action(id: &str) -> Option { "layer.addUrl" => Command("ui.addUrl"), "service.add" | "service.import" => Command("ui.addService"), "connection.new" => Command("ui.connection"), + "connection.resolveS3" => Command("ui.s3Connections"), "layer.addSql" => Pane(RightPane::Sql), "layer.addDatabase" => Dialog("addDatabase"), "layer.addZarr" => Command("ui.addZarr"), diff --git a/crates/ui-egui/src/connectiontree.rs b/crates/ui-egui/src/connectiontree.rs index 7e2022535..54629773f 100644 --- a/crates/ui-egui/src/connectiontree.rs +++ b/crates/ui-egui/src/connectiontree.rs @@ -442,7 +442,7 @@ fn level(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connection, node: &s }); } Some(Err(Some(e))) => { - ui.label(RichText::new(e).small().color(t.red)); + ui.label(RichText::new(crate::connectionui::translated(&e)).small().color(t.red)); ui.horizontal(|ui| { if ui.small_button(tl!("Retry")).clicked() { app.ui.connection_tree.entries.remove(&key); diff --git a/crates/ui-egui/src/connectionui.rs b/crates/ui-egui/src/connectionui.rs index 43e5d2fc5..ea9bc6d03 100644 --- a/crates/ui-egui/src/connectionui.rs +++ b/crates/ui-egui/src/connectionui.rs @@ -1,8 +1,11 @@ -//! Named endpoint definitions, optional remembered database passwords, and off-frame web probes. +//! Named endpoint definitions, optional remembered database passwords and S3 keys, off-frame +//! web probes, and explicit saved-connection choices for project sources. use egui::vec2; use rustgis_engine::connections::{Connection, DATABASE_KINDS, LoginReason, LoginRequest, database_key, database_label}; use rustgis_engine::render::rustgis_tiles::{cache, s3}; +use rustgis_engine::s3_connections::STALE; +use rustgis_engine::s3_keys as keys; use rustgis_engine::sources::Source; use serde_json::{Value, json}; @@ -28,6 +31,84 @@ pub struct State { pub secret_key: String, pub session_token: String, pub test: Option<(Value, Slot, String>>)>, + /// The form was opened to log in (Log In…): Save logs in even with no keys typed (an S3 + /// connection's saved keys are then read), where a plain edit only changes the definition. + pub login: bool, + /// A login that worked with a warning (translated, e.g. the keys weren't remembered): the + /// dialog stays open to say so until it is closed or edited again. + pub warned: Option, + /// A project's S3 source waiting for this connection (the chooser's), as (source address, + /// connection id): its choice is made again once that connection has logged in. + pub resolving: Option<(String, String)>, +} + +/// The chooser of the saved S3 connection that reads a project's S3 sources (Data Sources › +/// Choose S3 connection…): what was picked, for the project it is for. Nothing is picked for the +/// user. +#[derive(Default)] +pub struct Chooser { + /// The candidate picked for each resource waiting, by the source address that names it. + picked: std::collections::BTreeMap, + /// The project (`Session::opened`) this state is for. + project: Option, + /// Whether the chooser may still open by itself for this project (once). + auto: bool, + error: Option, +} + +/// A fixed sentence the engine says about saved S3 keys and S3 connection choices, in the +/// current language; any other message as it is. +pub(crate) fn translated(e: &str) -> String { + match e { + keys::UNUSABLE => tl!("Saved S3 keys could not be used. Enter the keys again."), + keys::NONE_SAVED => tl!("No saved S3 keys. Enter accessKeyId and secretAccessKey."), + keys::UNAVAILABLE => tl!("Saved S3 keys are unavailable. Enter the keys for this session."), + keys::NOT_REMEMBERED => tl!("Keys are available for this session but were not remembered on this computer."), + keys::TEMPORARY => tl!("Temporary credentials are session-only. Previously saved keys are unchanged."), + keys::NOT_FORGOTTEN => tl!("Saved S3 keys could not be forgotten. The connection was not changed."), + s3::UNRESOLVED => tl!("Choose a saved S3 connection before opening this source."), + s3::MISMATCH => tl!("The saved S3 connection no longer matches this source."), + STALE => tl!("This S3 connection choice is no longer available."), + other => other, + } + .to_string() +} + +/// Whether a login failed for want of usable saved keys: typing them is the way on. +pub(crate) fn needs_keys(e: &str) -> bool { + matches!(e, keys::NONE_SAVED | keys::UNAVAILABLE | keys::UNUSABLE) +} + +/// Why the Remember checkbox can't be used, when it can't: no credential store (the browser, or a +/// shell without one), or temporary credentials (a session token), which are never kept. +pub(crate) fn remember_blocked(store: bool, session_token: &str) -> Option<&'static str> { + if !store { + Some(tl!("Keys are kept for this session only: there is no credential store to remember them in here.")) + } else if !session_token.trim().is_empty() { + Some(tl!("Temporary credentials (with a session token) are kept for this session only.")) + } else { + None + } +} + +/// The checkbox, its reason when it can't be used, and what it does when it can. Unticked when +/// it can't be used. +pub(crate) fn remember_checkbox(ui: &mut egui::Ui, remember: &mut bool, blocked: Option<&str>) -> bool { + if blocked.is_some() { + *remember = false; + } + let changed = ui.add_enabled(blocked.is_none(), egui::Checkbox::new(remember, tl!("Remember on this computer"))).changed(); + let note = blocked.unwrap_or(tl!( + "Keeps the access key ID and secret access key in this computer's credential store, never in the connection or the project." + )); + ui.label(egui::RichText::new(note).small().weak()); + changed +} + +/// Forgets the keys saved on this computer for the S3 connection `id`; its login in this session +/// keeps working. A failure leaves everything as it was. +pub(crate) fn forget_keys(app: &mut RustGisApp, id: &str) -> Result<(), String> { + app.run("connection.forgetSavedKeys", json!({"id": id})).map(|_| ()).map_err(|e| translated(&e)) } pub(crate) const KINDS: [(&str, &str); 13] = [ @@ -151,16 +232,27 @@ pub fn open_edit(app: &mut RustGisApp, id: &str) -> Result<(), String> { Ok(()) } -pub fn log_in(app: &mut RustGisApp, id: &str) { +/// Opens the connection's form to log in (Log In…): Save then logs in. True when it opened. +pub fn log_in(app: &mut RustGisApp, id: &str) -> bool { match app.run("ui.connection", json!({"id": id})) { Ok(_) => { + app.ui.connection.login = true; + let saved = app.ui.connection.kind == "s3" + && app.ui.connection.credentials == "keys" + && app.ui.connection.id.as_deref().and_then(|id| app.session.connections.find(id)).is_some_and(|c| c.saved_keys); app.dialogs.info = Some(if DATABASE_KINDS.contains(&app.ui.connection.kind.as_str()) { tl!("Type the password, then Save. Remember on this computer is optional.").to_string() + } else if saved { + tl!("Leave the keys blank to log in with the keys saved on this computer, or type others, then Save.").to_string() } else { tl!("Type the keys to log in for this session, then Save.").to_string() }); + true + } + Err(e) => { + app.session.notify(e, true); + false } - Err(e) => app.session.notify(e, true), } } @@ -404,8 +496,35 @@ fn summary(kind: &str, r: &Value) -> String { } } +/// Whether a password or any key is typed in the form. +fn typed_secrets(st: &State) -> bool { + (DATABASE_KINDS.contains(&st.kind.as_str()) && !st.password.is_empty()) + || (st.kind == "s3" + && st.credentials == "keys" + && [&st.access_key, &st.secret_key, &st.session_token].iter().any(|v| !v.is_empty())) +} + +fn clear_secrets(st: &mut State) { + st.password.clear(); + st.access_key.clear(); + st.secret_key.clear(); + st.session_token.clear(); + st.remember = false; +} + +/// Saves the definition, then logs in when keys or a password are typed, or when the form was +/// opened to log in (an S3 connection with keys then reads those saved on this computer). Only +/// the login is asked to remember keys. A login that worked with a warning keeps the dialog open +/// to say so. fn save(app: &mut RustGisApp) -> Result<(), String> { + let store = app.session.database_logins.has_store(); let st = &app.ui.connection; + let keys_mode = st.kind == "s3" && st.credentials == "keys"; + let remember = keys_mode && st.remember && remember_blocked(store, &st.session_token).is_none(); + if remember && (st.access_key.trim().is_empty() || st.secret_key.trim().is_empty()) { + return Err(tl!("Type the access key ID and the secret access key to remember them on this computer.").into()); + } + let wants_login = typed_secrets(st) || (st.login && keys_mode && st.id.is_some()); let mut p = definition(st); let command = if let Some(id) = &st.id { p["id"] = json!(id); @@ -425,54 +544,81 @@ fn save(app: &mut RustGisApp) -> Result<(), String> { }; let v = app.run(command, p)?; stripped_notice(app, &v); - let mut login = json!({"id": v["id"]}); + let id = v["id"].as_str().map(str::to_owned).ok_or_else(|| "the connection has no id".to_string())?; + // Kept at once: a Save tried again edits this connection rather than adding another. + let created = app.ui.connection.id.is_none(); + app.ui.connection.id = Some(id.clone()); + if !wants_login { + clear_secrets(&mut app.ui.connection); + crate::connectiontree::refresh(app, Some(&id)); + return Ok(()); + } + let mut login = json!({"id": id}); secrets(&app.ui.connection, &mut login); - if login.as_object().is_some_and(|p| p.len() > 1) { - if DATABASE_KINDS.contains(&app.ui.connection.kind.as_str()) { - login["remember"] = json!(app.ui.connection.remember); - } - let id = v["id"].as_str().unwrap_or("").to_owned(); - let result = app.run("connection.login", login); + let database = DATABASE_KINDS.contains(&app.ui.connection.kind.as_str()); + if database { + login["remember"] = json!(app.ui.connection.remember); + } else if remember { + login["remember"] = json!(true); + } + let result = app.run("connection.login", login); + if database { crate::connectiontree::refresh(app, Some(&id)); let refused = password_request(app, &id); if result.is_err() || refused.is_some() { - app.ui.connection.id = Some(id); app.ui.connection.password.clear(); app.ui.connection.remember = false; app.ui.connection.request = refused; app.dialogs.open = Some("editConnection"); return Err(result.err().unwrap_or_else(|| tl!("The password was refused. Type a password to try again.").into())); } - // The command says why a checked Remember didn't keep the password; nothing is claimed - // when it did. - if let Some(warning) = result.as_ref().ok().and_then(|v| v["warning"].as_str()) { + } + let v = match result { + Ok(v) => v, + Err(e) => { + if created { + app.dialogs.open = Some("editConnection"); + } + return Err(translated(&e)); + } + }; + clear_secrets(&mut app.ui.connection); + app.ui.connection.login = false; + if let Some(warning) = v["warning"].as_str() { + if database { + // Database login retains its established notice-and-close behavior. app.session.notify(warning.to_string(), true); + } else { + app.ui.connection.warned = Some(translated(warning)); + app.dialogs.open = Some("editConnection"); } } - if let Some(id) = v["id"].as_str() { - crate::connectiontree::refresh(app, Some(id)); - } - let st = &mut app.ui.connection; - st.password.clear(); - st.remember = false; - st.access_key.clear(); - st.secret_key.clear(); - st.session_token.clear(); + resolve_waiting(app, &id); Ok(()) } pub fn dialog(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { - let help = if DATABASE_KINDS.contains(&app.ui.connection.kind.as_str()) { + let intro = if DATABASE_KINDS.contains(&app.ui.connection.kind.as_str()) { tl!( "A named endpoint you can reuse. Passwords last for this session unless you choose to remember them in this computer's password store." ) + } else if app.ui.connection.kind == "s3" { + tl!( + "A named endpoint you can reuse. Keys are kept until RustGIS closes unless you choose to remember them on this computer; the connection itself never keeps them." + ) } else { tl!("A named endpoint you can reuse. Keys and tokens are kept until RustGIS closes, never saved.") }; - ui.label(egui::RichText::new(help).weak()); + ui.label(egui::RichText::new(intro).weak()); + let store = app.session.database_logins.has_store(); + // Status metadata only: rendering must never read the credential store. + let saved_keys = + app.ui.connection.id.as_deref().and_then(|id| app.session.connections.find(id)).is_some_and(|c| c.kind == "s3" && c.saved_keys); let st = &mut app.ui.connection; let database = DATABASE_KINDS.contains(&st.kind.as_str()); let mut changed = false; + let mut source_changed = false; + let mut forget = false; egui::Grid::new("connection").num_columns(2).spacing(vec2(10.0, 8.0)).show(ui, |ui| { ui.label(tl!("Type")); ui.label(tl!(label(&st.kind))); @@ -517,7 +663,8 @@ pub fn dialog(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { if cfg!(target_arch = "wasm32") && matches!(code, "profile" | "environment") { continue; } - changed |= ui.selectable_value(&mut st.credentials, code.to_owned(), tl!(label)).changed(); + source_changed |= ui.selectable_value(&mut st.credentials, code.to_owned(), tl!(label)).changed(); + changed |= source_changed; } }); ui.end_row(); @@ -536,6 +683,23 @@ pub fn dialog(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { changed |= ui.add(egui::TextEdit::singleline(field).password(secret).desired_width(360.0)).changed(); ui.end_row(); } + ui.label(""); + let blocked = remember_blocked(store, &st.session_token); + ui.vertical(|ui| changed |= remember_checkbox(ui, &mut st.remember, blocked)); + ui.end_row(); + if saved_keys { + ui.label(tl!("Saved keys")); + ui.vertical(|ui| { + let note = if store { + tl!("This computer keeps keys for this connection: leave the keys blank to log in with them.") + } else { + tl!("Keys were saved for this connection, but there is no credential store here to read them from.") + }; + ui.label(egui::RichText::new(note).small().weak()); + forget = ui.button(tl!("Forget Saved Keys")).clicked(); + }); + ui.end_row(); + } } } else { ui.label(tl!("Address")); @@ -549,12 +713,44 @@ pub fn dialog(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { ui.end_row(); } }); + // A credential-source switch never carries opt-in to the newly chosen source. + if source_changed || (!database && !(st.kind == "s3" && st.credentials == "keys")) { + st.remember = false; + } if changed { // A late download may finish, but its detached slot cannot report on the edited form. st.test = None; + // Edited again: back to saving. + st.warned = None; app.dialogs.info = None; app.dialogs.error = None; } + if forget && let Some(id) = app.ui.connection.id.clone() { + match forget_keys(app, &id) { + Ok(()) => { + app.dialogs.error = None; + app.dialogs.info = + Some(tl!("The keys saved on this computer were forgotten; a login in this session keeps working.").to_string()); + } + Err(e) => { + app.dialogs.info = None; + app.dialogs.error = Some(e); + } + } + } + // A login that worked but couldn't do all it was asked: said until closed (or edited again). + if let Some(warning) = app.ui.connection.warned.clone() { + ui.add_space(4.0); + ui.label(egui::RichText::new(tl!("Logged in for this session")).strong()); + ui.label(egui::RichText::new(warning).color(ui.visuals().warn_fg_color)); + return ui + .with_layout(egui::Layout::right_to_left(egui::Align::Min), |ui| { + let accent = crate::theme::Tokens::get(ui.ctx()).accent; + crate::widgets::dialog_button(ui, tl!("Close"), Some(accent), vec2(84.0, 28.0)).clicked() + }) + .inner; + } + let st = &app.ui.connection; let testing = st.test.is_some(); let unavailable = database && cfg!(target_arch = "wasm32"); let clicked = ui @@ -576,7 +772,9 @@ pub fn dialog(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { if clicked { start_test(app); } - crate::dialogs::buttons(app, ui, "Save", save) + let close = crate::dialogs::buttons(app, ui, "Save", save); + // A warning to read keeps it open. + close && app.ui.connection.warned.is_none() } pub fn open_connection(app: &mut RustGisApp, c: &Connection) -> Result<(), String> { @@ -610,13 +808,26 @@ pub fn open_connection(app: &mut RustGisApp, c: &Connection) -> Result<(), Strin "wms" | "wmts" | "wfs" | "wcs" | "ogcapi" | "arcgis" | "csw" => ("service", json!({"type":c.kind})), "stac" => ("stac", json!({"api":"true"})), "s3" => { - if s3::connection(&c.name).is_none() { - if c.options.get("credentials").is_some_and(|v| v == "keys") { + if !app.session.has_s3_login(&c.id) { + let keys = c.options.get("credentials").is_some_and(|v| v == "keys"); + // Keys typed for the session: the form, to type them. Keys saved on this computer + // are read by this click's login (the system may ask to unlock its store). + if keys && !c.saved_keys { open_edit(app, &c.id)?; + app.ui.connection.login = true; app.dialogs.info = Some(tl!("Type the keys to log in for this session, then Save.").into()); return Ok(()); } - app.run("connection.login", json!({"id":c.id}))?; + if let Err(e) = app.run("connection.login", json!({"id":c.id})) { + if keys && needs_keys(&e) { + open_edit(app, &c.id)?; + app.ui.connection.login = true; + app.dialogs.error = Some(translated(&e)); + app.dialogs.info = Some(tl!("Type the keys to log in for this session, then Save.").into()); + return Ok(()); + } + return Err(translated(&e)); + } } crate::s3ui::fill(app, c); let r = crate::sourcespane::open( @@ -641,6 +852,158 @@ pub fn open_connection(app: &mut RustGisApp, c: &Connection) -> Result<(), Strin r } +/// After a login from the form: the project's S3 sources that waited for that connection (the +/// chooser asked for its login) are given to it now. +fn resolve_waiting(app: &mut RustGisApp, id: &str) { + let Some((source, waiting)) = app.ui.connection.resolving.take() else { return }; + if waiting == id + && let Err(e) = choose_s3(app, &source, id) + { + app.session.notify(translated(&e), true); + } +} + +/// The user's choice of the saved connection `id` for the project's S3 sources waiting under +/// `source`: the engine logs it in if needed and opens them; nothing is read or signed here. +fn choose_s3(app: &mut RustGisApp, source: &str, id: &str) -> Result<(), String> { + let v = app.run("connection.resolveS3", json!({"source": source, "id": id}))?; + let name = app.session.connections.find(id).map(|c| c.name.clone()).unwrap_or_else(|| id.to_string()); + let opened = v["reopened"].as_u64().unwrap_or(0).to_string(); + let failed = v["failed"].as_u64().unwrap_or(0); + let text = if failed == 0 { + i18n::fmt("{name} now reads {n} project sources", &[("name", &name), ("n", &opened)]) + } else { + i18n::fmt( + "{name} now reads {n} project sources; {failed} could not be read", + &[("name", &name), ("n", &opened), ("failed", &failed.to_string())], + ) + }; + app.session.notify(text, failed > 0); + Ok(()) +} + +/// Opens the chooser of the saved S3 connections that read the project's waiting S3 sources +/// (`ui.s3Connections`), with nothing picked. +pub fn open_chooser(app: &mut RustGisApp) -> Result<(), String> { + if app.session.s3_pending.is_empty() { + return Err(tl!("No project sources are waiting for an S3 connection.").into()); + } + let ch = &mut app.ui.s3_chooser; + ch.picked.clear(); + ch.error = None; + ch.auto = false; + app.dialogs = crate::dialogs::Dialogs { open: Some("s3Connections"), ..Default::default() }; + Ok(()) +} + +/// Once a frame: a project replacing the open one drops what was picked for the old one (and a +/// source waiting for a login), and the chooser opens by itself once for a project with S3 +/// sources waiting for a choice (not again after Cancel). +pub fn follow_project(app: &mut RustGisApp) { + let opened = app.session.opened; + if app.ui.s3_chooser.project != Some(opened) { + app.ui.s3_chooser = Chooser { project: Some(opened), auto: true, ..Default::default() }; + app.ui.connection.resolving = None; + if app.dialogs.open == Some("s3Connections") { + app.dialogs.open = None; + } + } + if app.ui.s3_chooser.auto && app.dialogs.open.is_none() { + app.ui.s3_chooser.auto = false; + if !app.session.s3_pending.is_empty() { + let _ = open_chooser(app); + } + } +} + +/// Where a waiting resource's bucket is, as the chooser says it. +fn endpoint_label(endpoint: Option<&str>) -> String { + match endpoint { + None => tl!("unknown (an older project)").to_string(), + Some("") => tl!("Amazon S3").to_string(), + Some(e) => e.to_string(), + } +} + +/// The chooser (dialog `s3Connections`): each bucket the project's waiting sources read, and the +/// saved connections that could read it. True when it should close. +pub fn s3_chooser(app: &mut RustGisApp, ui: &mut egui::Ui) -> bool { + let pending = app.session.s3_pending.clone(); + app.ui.s3_chooser.picked.retain(|source, _| pending.iter().any(|p| &p.source == source)); + ui.label( + egui::RichText::new(tl!( + "More than one saved connection could read these buckets. Choose the one that reads each; the project's sources that read it stay closed until you do." + )) + .weak(), + ); + if pending.is_empty() { + ui.label(tl!("No project sources are waiting for an S3 connection.")); + } + for p in &pending { + ui.group(|ui| { + ui.label(egui::RichText::new(i18n::fmt( + "Bucket {bucket} at {endpoint}", + &[("bucket", &p.bucket), ("endpoint", &endpoint_label(p.endpoint.as_deref()))], + ))); + for id in &p.candidates { + let Some(c) = app.session.connections.connections.iter().find(|c| &c.id == id) else { continue }; + let name = if c.group.is_empty() { c.name.clone() } else { format!("{} › {}", c.group, c.name) }; + let at = if c.address.is_empty() { tl!("Amazon S3") } else { c.address.as_str() }; + let selected = app.ui.s3_chooser.picked.get(&p.source) == Some(id); + if ui.radio(selected, format!("{name} — {at}")).clicked() { + app.ui.s3_chooser.picked.insert(p.source.clone(), id.clone()); + } + } + }); + } + if let Some(e) = &app.ui.s3_chooser.error { + ui.colored_label(ui.visuals().error_fg_color, e); + } + let mut close = false; + let mut open = false; + ui.add_space(4.0); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Min), |ui| { + let accent = crate::theme::Tokens::get(ui.ctx()).accent; + let any = !app.ui.s3_chooser.picked.is_empty(); + open = ui.add_enabled_ui(any, |ui| crate::widgets::dialog_button(ui, tl!("Open"), Some(accent), vec2(84.0, 28.0))).inner.clicked(); + // The sources stay closed. + close = crate::widgets::dialog_button(ui, tl!("Cancel"), None, vec2(84.0, 28.0)).clicked(); + }); + if open { + close = open_chosen(app); + } + close +} + +/// Gives each resource picked to its connection. One whose keys must be typed opens that +/// connection's Log In, and the choice is made again once it has logged in. True when nothing +/// waits any more. +fn open_chosen(app: &mut RustGisApp) -> bool { + app.ui.s3_chooser.error = None; + let picks: Vec<(String, String)> = app.ui.s3_chooser.picked.iter().map(|(s, id)| (s.clone(), id.clone())).collect(); + for (source, id) in picks { + match choose_s3(app, &source, &id) { + Ok(()) => { + app.ui.s3_chooser.picked.remove(&source); + } + Err(e) if needs_keys(&e) => { + app.ui.s3_chooser.picked.remove(&source); + if log_in(app, &id) { + app.ui.connection.resolving = Some((source, id)); + app.dialogs.error = Some(translated(&e)); + } + // Log In is open in its place. + return false; + } + Err(e) => { + app.ui.s3_chooser.error = Some(translated(&e)); + return false; + } + } + } + app.session.s3_pending.is_empty() +} + #[cfg(test)] mod tests { use super::*; @@ -657,4 +1020,83 @@ mod tests { assert!(picked(&list, Some("c1"), "https://example.com/other").is_none()); assert_eq!(picked(&list, Some("c1"), "https://example.com/wms?token=secret").map(|c| c.id.as_str()), Some("c1")); } + + #[test] + fn remember_needs_a_store_and_no_session_token() { + assert!(remember_blocked(true, "").is_none()); + assert!(remember_blocked(true, " ").is_none()); + assert!(remember_blocked(false, "").is_some()); + assert!(remember_blocked(true, "token").is_some()); + } + + fn new_s3_form(app: &mut RustGisApp, name: &str) { + app.run("ui.connection", json!({"kind": "s3"})).unwrap(); + let st = &mut app.ui.connection; + assert!(!st.remember, "Remember starts off"); + st.name = name.into(); + st.access_key = "AKIDREMEMBERUI".into(); + st.secret_key = "remember-ui-secret".into(); + st.remember = true; + } + + #[test] + fn remembered_keys_log_in_again_only_when_asked_and_can_be_forgotten() { + let mut app = RustGisApp::with_sample(); + app.session.database_logins.set_store(Some(std::sync::Arc::new(rustgis_engine::data::secrets::MemorySecrets::new()))); + let name = "remember-ui-test"; + new_s3_form(&mut app, name); + save(&mut app).unwrap(); + let id = app.ui.connection.id.clone().unwrap(); + assert!(app.session.connections.find(&id).unwrap().saved_keys); + assert!(app.ui.connection.access_key.is_empty() && app.ui.connection.secret_key.is_empty() && !app.ui.connection.remember); + assert!(app.ui.connection.warned.is_none()); + assert!(s3::connection(name).is_some()); + s3::disconnect(name); + + // A plain edit changes the definition, and logs nothing in. + open_edit(&mut app, &id).unwrap(); + assert!(!app.ui.connection.remember && !app.ui.connection.login); + app.ui.connection.group = "Lab".into(); + save(&mut app).unwrap(); + assert!(s3::connection(name).is_none(), "an edit is no login"); + + // Log In with nothing typed reads the saved keys. + assert!(log_in(&mut app, &id)); + assert!(app.ui.connection.login); + save(&mut app).unwrap(); + assert!(s3::connection(name).is_some()); + assert_eq!(s3::binding_of(name).as_deref(), app.session.s3_binding(&id)); + + forget_keys(&mut app, &id).unwrap(); + assert!(!app.session.connections.find(&id).unwrap().saved_keys); + assert!(s3::connection(name).is_some(), "forgetting keeps the session's login"); + s3::disconnect(name); + } + + #[test] + fn a_login_whose_keys_were_not_remembered_stays_open_to_say_so() { + let mut app = RustGisApp::with_sample(); + let store = rustgis_engine::data::secrets::MemorySecrets::failing(rustgis_engine::data::secrets::SecretError::Denied); + app.session.database_logins.set_store(Some(std::sync::Arc::new(store))); + let name = "remember-ui-warned-test"; + new_s3_form(&mut app, name); + save(&mut app).unwrap(); + let id = app.ui.connection.id.clone().unwrap(); + assert!(!app.session.connections.find(&id).unwrap().saved_keys); + assert!(app.ui.connection.warned.is_some()); + assert_eq!(app.dialogs.open, Some("editConnection")); + assert!(app.ui.connection.secret_key.is_empty()); + assert!(s3::connection(name).is_some(), "logged in for the session"); + s3::disconnect(name); + } + + #[test] + fn remember_without_a_typed_pair_changes_nothing() { + let mut app = RustGisApp::with_sample(); + app.session.database_logins.set_store(Some(std::sync::Arc::new(rustgis_engine::data::secrets::MemorySecrets::new()))); + new_s3_form(&mut app, "remember-ui-partial-test"); + app.ui.connection.secret_key.clear(); + assert!(save(&mut app).is_err()); + assert!(app.session.connections.connections.is_empty(), "nothing saved"); + } } diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index 124ad1f3d..48b7cf3a4 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -181,6 +181,7 @@ pub fn show(app: &mut RustGisApp, ctx: &egui::Context) { "editConnection" => "Edit Connection", "chooseConnection" => "Choose Connection", "s3" => "S3 Buckets", + "s3Connections" => "Choose S3 connection", "zarr" => "Add Zarr Variable", "exportLayer" => "Export Features", "saveProject" => "Save Project As", @@ -205,7 +206,7 @@ pub fn show(app: &mut RustGisApp, ctx: &egui::Context) { ui.set_width(match which { "coordinateSystem" | "zarr" => 660.0, "addTiles" | "addUrl" | "addService" | "s3" => 560.0, - "newConnection" | "editConnection" | "chooseConnection" => 520.0, + "newConnection" | "editConnection" | "chooseConnection" | "s3Connections" => 520.0, _ => 440.0, }); ui.spacing_mut().item_spacing = vec2(8.0, 8.0); @@ -305,6 +306,7 @@ pub fn show(app: &mut RustGisApp, ctx: &egui::Context) { "newConnection" | "editConnection" => close = crate::connectionui::dialog(app, ui), "chooseConnection" => close = crate::connectionui::chooser(app, ui), "s3" => crate::s3ui::dialog(app, ui), + "s3Connections" => close = crate::connectionui::s3_chooser(app, ui), "zarr" => close = crate::zarrui::dialog(app, ui), "exportLayer" => close = export_layer(app, ui), "camera" => { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index ee0760af3..bc4b44625 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -560,6 +560,9 @@ pub struct UiState { pub s3: s3ui::State, /// New/Edit Connection: the form and the Test under way. pub connection: connectionui::State, + /// Choose S3 connection: the saved connection picked for each bucket the project's S3 + /// sources wait on, for the project open. + pub s3_chooser: connectionui::Chooser, /// The datasets saved connections list under Data Sources › Connections, and the selection. pub connection_tree: connectiontree::State, /// OpenStreetMap and Boundaries: their choices and the request under way. @@ -694,6 +697,7 @@ impl Default for UiState { stac: Default::default(), s3: Default::default(), connection: Default::default(), + s3_chooser: Default::default(), connection_tree: Default::default(), portals: Default::default(), catalogs: Default::default(), @@ -908,11 +912,10 @@ impl RustGisApp { } pub fn with_services(mut self, services: Services) -> Self { + // Both saved database passwords and S3 keys use the same store, installed before prefs. + self.session.database_logins.set_store(services.secrets.clone()); self.services = services; sourcespane::load_connections(&mut self); - // Saved database passwords live in the system's store, which the engine reaches through - // the session so the assistant and scripts see the same logins as the interface. - self.session.database_logins.set_store(self.services.secrets.clone()); if let Some(lang) = self.services.load_pref.as_ref().and_then(|load| load("language")) { self.ui.language = lang; } @@ -2137,7 +2140,9 @@ impl RustGisApp { Box::new(move |r| *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(r)), ); } - self.session.notify(i18n::fmt("Reading {url}…", &[("url", url)]), false); + // An address read through a saved S3 connection is said as the address it wraps. + let shown = rustgis_engine::data::s3_address::inner(url); + self.session.notify(i18n::fmt("Reading {url}…", &[("url", shown)]), false); let mut probe = p.clone(); if let Some(o) = probe.as_object_mut() { o.remove(connectiontree::ORIGIN); @@ -2172,7 +2177,8 @@ impl RustGisApp { if let Some(origin) = &origin { q[connectiontree::ORIGIN] = json!(origin); } - self.session.notify(i18n::fmt("Downloading {url}…", &[("url", &url)]), false); + let shown = rustgis_engine::data::s3_address::inner(&url); + self.session.notify(i18n::fmt("Downloading {url}…", &[("url", shown)]), false); let id = self.session.download_started(); self.ui.downloads.push((id, url, q, slot)); } @@ -2186,6 +2192,9 @@ impl RustGisApp { connectionui::start_test(self); } } + // Choose S3 connection: the saved connection that reads the project's S3 sources + // waiting for a choice (`connection.resolveS3` does it). + "ui.s3Connections" => connectionui::open_chooser(self)?, // A map or feature service: the dialog, then (with an address) its capabilities, // fetched off the frame; the dialog lists its layers when they land. "ui.importService" => serviceui::start(self, p, false)?, @@ -3280,6 +3289,8 @@ impl RustGisApp { } // A project an agent or a script opened, before the frame draws the old one's fields. layoutview::follow_project(self); + // A project's S3 sources waiting for a choice of saved connection: asked once. + connectionui::follow_project(self); // Fonts installed last frame are active now. self.fonts_pending = false; i18n::set_current(self.lang()); diff --git a/crates/ui-egui/src/panes.rs b/crates/ui-egui/src/panes.rs index 8a5831884..ab879518a 100644 --- a/crates/ui-egui/src/panes.rs +++ b/crates/ui-egui/src/panes.rs @@ -1634,7 +1634,10 @@ fn layer_row(app: &mut RustGisApp, ui: &mut Ui, l: &Layer, scale: f64, reorder: ("renderer", tl_raw!(rl.style.renderer.label())), ], ), - None => crate::i18n::fmt("The raster's data couldn't be read: {path}", &[("path", rl.path.as_deref().unwrap_or(""))]), + None => crate::i18n::fmt( + "The raster's data couldn't be read: {error}", + &[("error", rl.error.as_deref().or(rl.path.as_deref()).unwrap_or(""))], + ), }, None => match &l.source { Some(src) if src.error.is_some() => { diff --git a/crates/ui-egui/src/s3ui.rs b/crates/ui-egui/src/s3ui.rs index 18fc03154..d670d7186 100644 --- a/crates/ui-egui/src/s3ui.rs +++ b/crates/ui-egui/src/s3ui.rs @@ -1,13 +1,16 @@ //! S3…: connect to Amazon S3 or an S3-compatible service, browse a bucket a level at a time, and //! add an object by its `s3://` address. Listings are fetched off the frame through the tile //! cache's downloads (a thread natively, the browser's fetch on the web), signed by -//! `rustgis_tiles::s3` when the connection has credentials. +//! `rustgis_tiles::s3` when the connection has credentials. Browsing through a saved +//! connection's login names that login in every address it fetches or adds. use egui::vec2; +use rustgis_engine::Session; use rustgis_engine::connections::Connection; use rustgis_engine::render::rustgis_tiles::{cache, s3}; use serde_json::{Value, json}; +use crate::connectionui::translated; use crate::{RustGisApp, Slot, i18n}; type Answer = Slot, String>>; @@ -51,6 +54,29 @@ pub struct State { listing_job: Option, buckets_job: Option, error: Option, + /// Remember the typed keys on this computer (Connect through a saved connection). Off + /// whenever a connection is picked or filled in: never inferred from its saved keys. + remember: bool, + /// A login that worked with a warning (translated): shown while browsing goes on. + warning: Option, +} + +/// The token of this session's login of a saved S3 connection, when the live connection `name` +/// is that login (an ordinary session connection of the same name isn't): what the addresses +/// read through it name, so that only it signs them. +pub(crate) fn saved_binding(session: &Session, name: &str) -> Option { + let binding = s3::binding_of(name)?; + let (_, id) = s3::saved_connection(&binding)?; + (session.s3_binding(&id) == Some(binding.as_str())).then_some(binding) +} + +/// `url` as fetched through the live connection: wrapped with the saved login `binding` when it +/// is one, as it is otherwise. +fn through(binding: Option<&str>, url: String) -> Result { + match binding { + Some(b) => s3::bound_url(b, &url).map_err(|e| translated(&e)), + None => Ok(url), + } } fn fetch(url: &str) -> Answer { @@ -114,7 +140,7 @@ fn current(st: &State) -> Option { s3::connection(&st.conn) } -fn list(st: &mut State) { +fn list(st: &mut State, binding: Option<&str>) { let bucket = st.bucket.trim().to_string(); if !s3::valid_bucket(&bucket) { st.error = Some(i18n::fmt("'{b}' isn't a bucket name", &[("b", &bucket)])); @@ -123,12 +149,31 @@ fn list(st: &mut State) { st.bucket = bucket; st.listing = None; st.error = None; - // Through the connection chosen, which serves the bucket from then on; "AWS, anonymous" - // reads AWS with no keys whatever the connections. + // Through the connection chosen, which serves the bucket from then on (a saved connection's + // login only for the addresses that name it); "AWS, anonymous" reads AWS with no keys + // whatever the connections. let url = match current(st) { Some(c) => { - s3::grant(&c.name, &st.bucket); - s3::list_url(&st.bucket, &st.prefix, None, Some(&c)) + let url = s3::list_url(&st.bucket, &st.prefix, None, Some(&c)); + match binding { + Some(b) => { + if let Err(e) = s3::grant_saved(b, &st.bucket) { + st.error = Some(e); + return; + } + match through(Some(b), url) { + Ok(u) => u, + Err(e) => { + st.error = Some(e); + return; + } + } + } + None => { + s3::grant(&c.name, &st.bucket); + url + } + } } None => s3::anonymous(&s3::list_url(&st.bucket, &st.prefix, None, None)), }; @@ -145,26 +190,79 @@ pub(crate) fn fill(app: &mut RustGisApp, c: &Connection) { st.profile = c.options.get("profile").cloned().unwrap_or_default(); st.key.clear(); st.secret.clear(); + st.remember = false; + st.warning = None; st.connected = None; st.error = None; } +/// A saved connection picked: browsed at once when it is live or can log in with nothing typed +/// (a profile, the environment, anonymously, or the keys saved on this computer, which this +/// choice may ask the system to unlock); else the form waits for its keys. fn pick(app: &mut RustGisApp, c: &Connection) { fill(app, c); app.ui.s3.connection = Some(c.id.clone()); - if s3::connection(&c.name).is_some() { + if app.session.has_s3_login(&c.id) { open_connection(app, &c.name); - } else if app.ui.s3.source != "keys" { + } else if app.ui.s3.source != "keys" || c.saved_keys { match app.run("connection.login", json!({"id": c.id})) { Ok(_) => open_connection(app, &c.name), - Err(e) => app.ui.s3.error = Some(e), + Err(e) => app.ui.s3.error = Some(translated(&e)), } } } +/// The form's name: a blank one connects as "default". +fn form_name(st: &State) -> String { + if st.name.trim().is_empty() { "default".to_string() } else { st.name.trim().to_string() } +} + +/// The typed key pair, when both halves are typed. +fn typed_pair(st: &State) -> Option<(String, String)> { + let (key, secret) = (st.key.trim(), st.secret.trim()); + (!key.is_empty() && !secret.is_empty()).then(|| (key.to_string(), secret.to_string())) +} + +/// The saved connection the form stands for, when one was picked and nothing typed since changes +/// its definition (name, endpoint, region, where its keys come from). +fn unchanged_saved(app: &RustGisApp) -> Option { + let st = &app.ui.s3; + let id = st.connection.as_deref()?; + let c = app.session.connections.connections.iter().find(|c| c.id == id && c.kind == "s3")?; + let option = |k: &str| c.options.get(k).map(|v| v.trim()).unwrap_or(""); + let endpoint = match (s3::canonical(&st.endpoint), s3::canonical(&c.address)) { + (Ok(a), Ok(b)) => a == b, + _ => st.endpoint.trim() == c.address.trim(), + }; + let source = if option("credentials").is_empty() { "none" } else { option("credentials") }; + let same = c.name == form_name(st) + && endpoint + && st.region.trim() == option("region") + && st.source == source + && (st.source != "profile" || st.profile.trim() == option("profile")); + same.then(|| c.clone()) +} + +/// Connects: a picked saved connection, unchanged, logs in by its id (with the keys typed, or +/// those saved on this computer); a form whose typed keys are to be remembered is first saved as +/// a new connection; anything else is a session-only connection, as it always was. fn connect(app: &mut RustGisApp) { + app.ui.s3.warning = None; + let remember = app.ui.s3.remember && app.ui.s3.source == "keys" && app.session.database_logins.has_store(); + if remember && typed_pair(&app.ui.s3).is_none() { + app.ui.s3.error = Some(tl!("Type the access key ID and the secret access key to remember them on this computer.").to_string()); + return; + } + if let Some(c) = unchanged_saved(app) { + log_in_saved(app, &c, remember); + return; + } + if remember { + remember_new(app); + return; + } let st = &app.ui.s3; - let name = if st.name.trim().is_empty() { "default".to_string() } else { st.name.trim().to_string() }; + let name = form_name(st); let endpoint = st.endpoint.trim().to_string(); let region = st.region.trim().to_string(); let profile = if st.profile.trim().is_empty() { "default".to_string() } else { st.profile.trim().to_string() }; @@ -186,7 +284,9 @@ fn connect(app: &mut RustGisApp) { Ok(v) => { let st = &mut app.ui.s3; // The secret stays with the session's connection, not in the form. + st.key.clear(); st.secret.clear(); + st.remember = false; st.conn = name.clone(); st.error = None; st.buckets.clear(); @@ -205,6 +305,64 @@ fn connect(app: &mut RustGisApp) { } } +/// Logs in to the saved connection `c` (the form's typed keys, if any; else what it needs no +/// typing for, saved keys included) and browses with it. Remembering is asked only here. +fn log_in_saved(app: &mut RustGisApp, c: &Connection, remember: bool) { + let st = &app.ui.s3; + let mut p = json!({"id": c.id}); + if st.source == "keys" && (!st.key.trim().is_empty() || !st.secret.is_empty()) { + p["accessKeyId"] = json!(st.key.trim()); + p["secretAccessKey"] = json!(st.secret.trim()); + } + if remember { + p["remember"] = json!(true); + } + match app.run("connection.login", p) { + Ok(v) => { + let st = &mut app.ui.s3; + // The keys stay with the session's login (and the credential store, if remembered). + st.key.clear(); + st.secret.clear(); + st.remember = false; + st.conn = c.name.clone(); + st.error = None; + st.buckets.clear(); + // Saved already: nothing to Save as Connection. + st.connected = None; + st.warning = v["warning"].as_str().map(translated); + } + Err(e) => app.ui.s3.error = Some(translated(&e)), + } +} + +/// Remember on a form that is no saved connection (or one whose definition was changed): saved +/// as a new connection first, under a name no other has, then logged in to keep its keys. A +/// name taken leaves the form as it is. +fn remember_new(app: &mut RustGisApp) { + let st = &app.ui.s3; + let (name, endpoint, region) = (form_name(st), st.endpoint.trim().to_string(), st.region.trim().to_string()); + if app.session.connections.connections.iter().any(|c| c.name.to_lowercase() == name.to_lowercase()) { + app.ui.s3.error = Some(tl!("A connection has this name: give this one another name to remember its keys.").to_string()); + return; + } + let def = json!({"kind": "s3", "name": name, "address": endpoint, + "options": {"region": region, "credentials": "keys"}}); + let id = match crate::connectionui::keep(app, def) { + Ok(id) => id, + Err(e) => { + app.ui.s3.error = Some(e); + return; + } + }; + // The form stands for it from now on: Connect tried again logs in to it, never another copy. + app.ui.s3.connection = Some(id.clone()); + app.ui.s3.synced = None; + match app.session.connections.find(&id).cloned() { + Some(c) => log_in_saved(app, &c, true), + None => app.ui.s3.error = Some(format!("no connection '{id}'")), + } +} + /// Opens the S3 browser on a session connection (the Data Sources pane's S3 list). pub fn open_connection(app: &mut RustGisApp, name: &str) { let st = &mut app.ui.s3; @@ -265,6 +423,11 @@ pub fn dialog(app: &mut RustGisApp, ui: &mut egui::Ui) { egui::CollapsingHeader::new(tl!("Connection")).default_open(s3::connections().is_empty()).show(ui, |ui| connection_form(app, ui)); ui.separator(); browse(app, ui); + // The login worked: what it couldn't do is said while browsing goes on. + if let Some(w) = &app.ui.s3.warning { + ui.label(egui::RichText::new(tl!("Logged in for this session")).strong()); + ui.colored_label(ui.visuals().warn_fg_color, w); + } if let Some(e) = &app.ui.s3.error { ui.colored_label(ui.visuals().error_fg_color, e); } @@ -280,17 +443,24 @@ fn connection_form(app: &mut RustGisApp, ui: &mut egui::Ui) { pick(app, &c); return; } + let store = app.session.database_logins.has_store(); + let saved_keys = current.is_some_and(|c| c.saved_keys); let st = &mut app.ui.s3; if !SOURCES.iter().any(|(c, _)| *c == st.source) { st.source = "keys".into(); } let typed = st.source == "keys"; + if !typed { + st.remember = false; + } + let mut source_changed = false; egui::Grid::new("s3_conn").num_columns(2).spacing(vec2(10.0, 6.0)).show(ui, |ui| { - ui.label(tl!("Name")); - ui.add(egui::TextEdit::singleline(&mut st.name).hint_text("default").desired_width(300.0)); + let name = ui.label(tl!("Name")); + ui.add(egui::TextEdit::singleline(&mut st.name).hint_text("default").desired_width(300.0)).labelled_by(name.id); ui.end_row(); - ui.label(tl!("Endpoint")); - ui.add(egui::TextEdit::singleline(&mut st.endpoint).hint_text(tl!("empty for AWS; https://… for others")).desired_width(300.0)); + let endpoint = ui.label(tl!("Endpoint")); + ui.add(egui::TextEdit::singleline(&mut st.endpoint).hint_text(tl!("empty for AWS; https://… for others")).desired_width(300.0)) + .labelled_by(endpoint.id); ui.end_row(); ui.label(tl!("Region")); ui.add(egui::TextEdit::singleline(&mut st.region).hint_text(s3::DEFAULT_REGION).desired_width(300.0)); @@ -303,7 +473,7 @@ fn connection_form(app: &mut RustGisApp, ui: &mut egui::Ui) { if cfg!(target_arch = "wasm32") && matches!(code, "profile" | "environment") { continue; } - ui.selectable_value(&mut st.source, code.to_string(), tl!(label)); + source_changed |= ui.selectable_value(&mut st.source, code.to_string(), tl!(label)).changed(); } }); ui.end_row(); @@ -312,13 +482,32 @@ fn connection_form(app: &mut RustGisApp, ui: &mut egui::Ui) { ui.add(egui::TextEdit::singleline(&mut st.profile).hint_text("default").desired_width(300.0)); ui.end_row(); } - ui.label(tl!("Access key ID")); - ui.add_enabled(typed, egui::TextEdit::singleline(&mut st.key).desired_width(300.0)); + let key = ui.label(tl!("Access key ID")); + ui.add_enabled(typed, egui::TextEdit::singleline(&mut st.key).desired_width(300.0)).labelled_by(key.id); ui.end_row(); - ui.label(tl!("Secret access key")); - ui.add_enabled(typed, egui::TextEdit::singleline(&mut st.secret).password(true).desired_width(300.0)); + let secret = ui.label(tl!("Secret access key")); + ui.add_enabled(typed, egui::TextEdit::singleline(&mut st.secret).password(true).desired_width(300.0)).labelled_by(secret.id); ui.end_row(); + if typed { + ui.label(""); + // This form has no session token: only a missing store stops it. + ui.vertical(|ui| { + crate::connectionui::remember_checkbox(ui, &mut st.remember, crate::connectionui::remember_blocked(store, "")) + }); + ui.end_row(); + } }); + if source_changed { + st.remember = false; + } + if typed && saved_keys { + let note = if store { + tl!("Saved keys: leave the keys blank to log in with those this computer keeps.") + } else { + tl!("Keys were saved for this connection, but there is no credential store here to read them from.") + }; + ui.label(egui::RichText::new(note).small().weak()); + } if ui.button(tl!("Connect")).clicked() { connect(app); } @@ -361,9 +550,14 @@ fn browse(app: &mut RustGisApp, ui: &mut egui::Ui) { && let Some(c) = s3::connection(&st.conn) { st.error = None; - st.buckets_job = Some(fetch(&s3::buckets_url(&c))); + match through(saved_binding(&app.session, &st.conn).as_deref(), s3::buckets_url(&c)) { + Ok(url) => st.buckets_job = Some(fetch(&url)), + Err(e) => st.error = Some(e), + } } }); + // A saved connection's login is named by every address read through it. + let binding = if st.conn.is_empty() { None } else { saved_binding(&app.session, &st.conn) }; if !st.buckets.is_empty() { ui.horizontal_wrapped(|ui| { for b in st.buckets.clone() { @@ -406,9 +600,9 @@ fn browse(app: &mut RustGisApp, ui: &mut egui::Ui) { // public web address at AWS when browsing anonymously. // Anonymous stays anonymous even for a bucket a connection serves. add = Some(if st.conn.is_empty() { - s3::anonymous(&s3::object_url(&st.bucket, key, None)) + Ok(s3::anonymous(&s3::object_url(&st.bucket, key, None))) } else { - format!("s3://{}/{key}", st.bucket) + through(binding.as_deref(), format!("s3://{}/{key}", st.bucket)) }); } ui.label(key.strip_prefix(st.prefix.as_str()).unwrap_or(key)); @@ -429,13 +623,17 @@ fn browse(app: &mut RustGisApp, ui: &mut egui::Ui) { }); } if go { - list(st); + list(st, binding.as_deref()); } - if let Some(url) = add { + match add { // Added as any address is: read in place when a cloud-optimised GeoTIFF, else downloaded. - if let Err(e) = app.run("ui.addUrl", json!({"url": url})) { - app.ui.s3.error = Some(e); + Some(Ok(url)) => { + if let Err(e) = app.run("ui.addUrl", json!({"url": url})) { + app.ui.s3.error = Some(e); + } } + Some(Err(e)) => app.ui.s3.error = Some(e), + None => {} } } @@ -504,10 +702,72 @@ mod tests { app.ui.s3.secret = "test-secret".into(); connect(&mut app); assert_eq!(app.ui.s3.error, None); - assert!(app.ui.s3.secret.is_empty()); - assert_eq!(app.ui.s3.connected.as_ref().unwrap()["options"]["credentials"], "keys"); + // The picked, unchanged connection logs in by its id: its login, nothing to keep. + assert!(app.ui.s3.secret.is_empty() && app.ui.s3.key.is_empty()); + assert!(app.ui.s3.connected.is_none()); assert_eq!(s3::connection(&c.name).unwrap().name, c.name); + assert_eq!(app.ui.s3.conn, c.name); + let binding = saved_binding(&app.session, &c.name).unwrap(); + assert_eq!(app.session.s3_binding(&c.id), Some(binding.as_str())); + s3::disconnect(&c.name); + } + + #[test] + fn remember_on_an_unsaved_form_saves_a_connection_then_its_keys() { + let mut app = RustGisApp::with_sample(); + app.session.database_logins.set_store(Some(std::sync::Arc::new(rustgis_engine::data::secrets::MemorySecrets::new()))); + let name = "remember-s3ui-test"; + app.ui.s3.name = name.into(); + app.ui.s3.source = "keys".into(); + app.ui.s3.key = "AKIDREMEMBERS3UI".into(); + app.ui.s3.secret = "remember-s3ui-secret".into(); + app.ui.s3.remember = true; + connect(&mut app); + assert_eq!(app.ui.s3.error, None); + assert_eq!(app.ui.s3.warning, None); + assert_eq!(app.session.connections.connections.len(), 1, "one definition"); + let c = app.session.connections.connections[0].clone(); + assert!(c.saved_keys); + assert_eq!(c.options.get("credentials").map(String::as_str), Some("keys")); + assert_eq!(app.ui.s3.connection.as_deref(), Some(c.id.as_str())); + assert!(app.ui.s3.key.is_empty() && app.ui.s3.secret.is_empty() && !app.ui.s3.remember); + assert!(saved_binding(&app.session, name).is_some()); + s3::disconnect(name); + + // The same name typed again with other settings: never written over. + app.ui.s3.connection = None; + app.ui.s3.endpoint = "https://other.example.com".into(); + app.ui.s3.key = "AKIDOTHER".into(); + app.ui.s3.secret = "other-secret".into(); + app.ui.s3.remember = true; + connect(&mut app); + assert!(app.ui.s3.error.is_some()); + assert_eq!(app.session.connections.connections.len(), 1); + assert_eq!(app.ui.s3.secret, "other-secret", "the typed values stay"); + assert!(s3::connection(name).is_none()); + } + + #[test] + fn browsing_through_a_saved_login_names_it_and_a_session_connection_does_not() { + let mut app = RustGisApp::with_sample(); + let v = app + .run( + "connection.new", + json!({"kind":"s3","name":"bound-browse-s3-test","address":"https://s3.example.com", + "options":{"credentials":"none"}}), + ) + .unwrap(); + let c = app.session.connections.find(v["id"].as_str().unwrap()).unwrap().clone(); + pick(&mut app, &c); + let binding = saved_binding(&app.session, &c.name).unwrap(); + assert_eq!(app.ui.s3.conn, c.name); + s3::grant_saved(&binding, "bound-bucket").unwrap(); + let url = through(Some(&binding), "s3://bound-bucket/a.tif".into()).unwrap(); + assert!(url.starts_with(rustgis_engine::data::s3_address::PREFIX), "{url}"); + assert!(url.ends_with("s3://bound-bucket/a.tif"), "{url}"); s3::disconnect(&c.name); + assert!(saved_binding(&app.session, &c.name).is_none()); + assert_eq!(through(None, "s3://b/k".into()).unwrap(), "s3://b/k"); } #[test] @@ -545,4 +805,65 @@ mod tests { assert!(app.ui.s3.bucket.is_empty()); s3::disconnect("edited-s3-test"); } + + #[test] + fn s3_saved_picker_and_connection_entry_reauthenticate_a_foreign_session_login() { + use rustgis_engine::data::secrets::MemorySecrets; + use std::sync::Arc; + let app = || { + RustGisApp::with_sample().with_services(crate::Services { secrets: Some(Arc::new(MemorySecrets::new())), ..Default::default() }) + }; + let mut a = app(); + let mut b = app(); + let name = "s3-ui-session-owner"; + let v = a + .run( + "connection.new", + json!({"kind":"s3","name":name,"address":"http://127.0.0.1:1", + "options":{"credentials":"keys","buckets":"ui-session-owner"}}), + ) + .unwrap(); + let id = v["id"].as_str().unwrap(); + a.run("connection.login", json!({"id":id,"accessKeyId":"UIOWNERKEYA","secretAccessKey":"ui-owner-secret-a","remember":true})) + .unwrap(); + b.session.connections = rustgis_engine::connections::Connections::from_json(&a.session.connections.to_json()); + b.run("connection.login", json!({"id":id,"accessKeyId":"UIOWNERKEYB","secretAccessKey":"ui-owner-secret-b","remember":true})) + .unwrap(); + let c = b.session.connections.find(id).unwrap().clone(); + a.run("connection.login", json!({"id":id})).unwrap(); + pick(&mut b, &c); + assert_eq!(s3::binding_of(name).as_deref(), b.session.s3_binding(id)); + assert_eq!(s3::connection(name).unwrap().credentials.unwrap().access_key_id, "UIOWNERKEYB"); + a.run("connection.login", json!({"id":id})).unwrap(); + crate::connectionui::open_connection(&mut b, &c).unwrap(); + assert_eq!(s3::binding_of(name).as_deref(), b.session.s3_binding(id)); + assert_eq!(s3::connection(name).unwrap().credentials.unwrap().access_key_id, "UIOWNERKEYB"); + s3::disconnect(name); + } + + #[test] + fn s3_saved_picker_retains_ordinary_session_login_semantics() { + let mut app = RustGisApp::with_sample(); + let name = "s3-ui-unmapped-session"; + app.run( + "s3.connect", + json!({"name":name,"endpoint":"http://127.0.0.1:1", + "accessKeyId":"ORDINARYKEY","secretAccessKey":"ordinary-secret"}), + ) + .unwrap(); + let result = app + .run( + "connection.new", + json!({"kind":"s3","name":name,"address":"http://127.0.0.1:1", + "options":{"credentials":"keys","buckets":"ordinary-bucket"}}), + ) + .unwrap(); + let c = app.session.connections.find(result["id"].as_str().unwrap()).unwrap().clone(); + pick(&mut app, &c); + assert_eq!(app.ui.s3.conn, name); + assert!(app.ui.s3.error.is_none()); + assert!(s3::binding_of(name).is_none(), "a plain login is not silently remapped or reloaded"); + assert_eq!(s3::connection(name).unwrap().credentials.unwrap().access_key_id, "ORDINARYKEY"); + s3::disconnect(name); + } } diff --git a/crates/ui-egui/src/sourcespane.rs b/crates/ui-egui/src/sourcespane.rs index d9887f488..b073e6c6b 100644 --- a/crates/ui-egui/src/sourcespane.rs +++ b/crates/ui-egui/src/sourcespane.rs @@ -237,6 +237,16 @@ fn empty(ui: &mut Ui, what: &str) { fn connections(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, list: &[Connection]) { crate::connectiontree::prune(app); + // A project's S3 sources waiting for the saved connection that reads them. + if !app.session.s3_pending.is_empty() + && ui + .button(tl!("Choose S3 connection…")) + .on_hover_text(tl!("Some of the project's S3 sources wait for you to choose the saved connection that reads them")) + .clicked() + && let Err(e) = app.run("ui.s3Connections", Value::Null) + { + app.session.notify(e, true); + } ui.horizontal(|ui| { ui.menu_button(tl!("New Connection ▾"), |ui| { for (kind, label) in crate::connectionui::KINDS { @@ -319,6 +329,16 @@ fn connection_header(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connecti if DATABASE_KINDS.contains(&c.kind.as_str()) && c.remembered { ui.label(RichText::new(tl!("Saved password")).small().weak()); } + // The saved-keys marker (the store isn't asked while drawing). + if c.kind == "s3" && c.saved_keys { + if app.session.database_logins.has_store() { + ui.label(RichText::new(tl!("Saved keys")).small().weak()) + .on_hover_text(tl!("This computer keeps this connection's keys: logging in reads them.")); + } else { + ui.label(RichText::new(tl!("Saved keys unavailable")).small().weak()) + .on_hover_text(tl!("Keys were saved for this connection, but there is no credential store here to read them from.")); + } + } ui.label(RichText::new(tl!(crate::connectionui::label(&c.kind))).small().weak()); response.context_menu(|ui| { let mut items = vec![ @@ -364,6 +384,16 @@ fn connection_header(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens, c: &Connecti } ui.close(); } + if c.kind == "s3" && c.saved_keys && ui.button(tl!("Forget Saved Keys")).clicked() { + match crate::connectionui::forget_keys(app, &c.id) { + Ok(()) => app.session.notify( + tl!("The keys saved on this computer were forgotten; a login in this session keeps working.").to_string(), + false, + ), + Err(e) => app.session.notify(e, true), + } + ui.close(); + } } if !c.address.is_empty() && ui.button(tl!("Copy Address")).clicked() { ui.ctx().copy_text(c.address.clone()); diff --git a/crates/ui-egui/tests/remembered_s3/mod.rs b/crates/ui-egui/tests/remembered_s3/mod.rs new file mode 100644 index 000000000..68f21e026 --- /dev/null +++ b/crates/ui-egui/tests/remembered_s3/mod.rs @@ -0,0 +1,323 @@ +use super::*; +use egui::accesskit::Role; +use rustgis_engine::data::secrets::{MemorySecrets, Secret, SecretError, SecretStore}; +use rustgis_engine::render::rustgis_tiles::s3; +use rustgis_ui_egui::Services; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +type Harness = egui_kittest::Harness<'static, RustGisApp>; + +#[derive(Default)] +struct CountedStore { + inner: MemorySecrets, + loads: AtomicUsize, +} +impl SecretStore for CountedStore { + fn load(&self, account: &str) -> Result, SecretError> { + self.loads.fetch_add(1, Ordering::Relaxed); + self.inner.load(account) + } + fn save(&self, account: &str, secret: &Secret) -> Result<(), SecretError> { + self.inner.save(account, secret) + } + fn delete(&self, account: &str) -> Result<(), SecretError> { + self.inner.delete(account) + } +} + +fn with_store(store: Option>) -> Harness { + let app = RustGisApp::with_sample().with_services(Services { secrets: store, ..Default::default() }); + egui_kittest::Harness::builder().with_size(egui::vec2(1400.0, 1000.0)).with_max_steps(64).build_ui_state( + |ui, app: &mut RustGisApp| { + let ctx = ui.ctx().clone(); + app.logic(&ctx); + app.ui(ui); + }, + app, + ) +} +fn settle(h: &mut Harness) { + h.run_steps(4); +} +fn new_form(h: &mut Harness, name: &str) { + h.state_mut().run("ui.connection", json!({"kind":"s3"})).unwrap(); + let st = &mut h.state_mut().ui.connection; + st.name = name.into(); + st.address = "http://127.0.0.1:1".into(); + st.access_key = "SAVEDWIDGETKEY".into(); + st.secret_key = "saved-widget-secret".into(); + settle(h); + assert!(!h.state().ui.connection.remember); +} +fn type_field(h: &mut Harness, label: &str, text: &str) { + let role = if label == "Secret access key" { Role::PasswordInput } else { Role::TextInput }; + h.get_by_role_and_label(role, label).focus(); + h.step(); + h.event(egui::Event::Text(text.into())); + settle(h); +} +fn expand_connection(h: &mut Harness) { + if h.query_by_role_and_label(Role::Button, "Connect").is_none() { + h.get_by_label("Connection").click(); + settle(h); + } +} + +#[test] +fn s3_saved_checkbox_save_row_forget_and_explicit_blank_login() { + let store = Arc::new(CountedStore::default()); + let mut h = with_store(Some(store.clone())); + let name = "s3-saved-widget-definition"; + new_form(&mut h, name); + h.get_by_label("Remember on this computer").click(); + settle(&mut h); + h.get_by_role_and_label(Role::Button, "Test").click(); + settle(&mut h); + assert!(h.state().session.connections.connections.is_empty(), "Test never saves a definition"); + assert!(store.inner.load("connection/c1").unwrap().is_none(), "Test never remembers keys"); + for _ in 0..200 { + if h.state().ui.connection.test.is_none() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + h.step(); + } + assert!(h.state().ui.connection.test.is_none(), "localhost test must finish before Save"); + h.get_by_role_and_label(Role::Button, "Save").click(); + settle(&mut h); + assert_eq!(h.state().dialogs.open, None); + let id = h.state().session.connections.find(name).unwrap().id.clone(); + assert!(h.state().session.connections.find(&id).unwrap().saved_keys); + assert!(store.inner.load(&format!("connection/{id}")).unwrap().is_some()); + assert!(h.state().ui.connection.access_key.is_empty()); + assert!(h.state().ui.connection.secret_key.is_empty()); + s3::disconnect(name); + + // A definition-only edit does not unlock the store or reconnect. + h.state_mut().run("ui.connection", json!({"id":id})).unwrap(); + h.state_mut().ui.connection.group = "Widget group".into(); + settle(&mut h); + assert!(!h.state().ui.connection.remember); + let before = store.loads.load(Ordering::Relaxed); + h.get_by_role_and_label(Role::Button, "Save").click(); + settle(&mut h); + assert_eq!(store.loads.load(Ordering::Relaxed), before); + assert!(s3::connection(name).is_none()); + + h.state_mut().run("ui.pane.sources", json!({})).unwrap(); + settle(&mut h); + h.get_by_label(name).click_secondary(); + settle(&mut h); + h.get_by_label("Log In…").click(); + settle(&mut h); + assert!(h.state().ui.connection.login); + assert!(!h.state().ui.connection.remember); + h.get_by_role_and_label(Role::Button, "Save").click(); + settle(&mut h); + assert!(s3::connection(name).is_some()); + assert_eq!(s3::binding_of(name).as_deref(), h.state().session.s3_binding(&id)); + h.get_by_label("Saved keys"); + let before = store.loads.load(Ordering::Relaxed); + h.run_steps(8); + h.state_mut().run("connection.list", json!({})).unwrap(); + assert_eq!(store.loads.load(Ordering::Relaxed), before, "frames and lists never unlock secrets"); + h.get_by_label(name).click_secondary(); + settle(&mut h); + h.get_by_label("Forget Saved Keys").click(); + settle(&mut h); + assert!(!h.state().session.connections.find(&id).unwrap().saved_keys); + assert!(store.inner.load(&format!("connection/{id}")).unwrap().is_none()); + assert!(s3::connection(name).is_some(), "Forget does not log out"); + assert!(h.query_by_label("Saved keys").is_none()); + s3::disconnect(name); +} + +#[test] +fn s3_saved_connect_creates_once_and_same_name_edit_never_overwrites() { + let store = Arc::new(MemorySecrets::new()); + let mut h = with_store(Some(store.clone())); + h.state_mut().run("ui.dialog", json!({"name":"s3"})).unwrap(); + settle(&mut h); + expand_connection(&mut h); + type_field(&mut h, "Name", "s3-saved-widget-connect"); + type_field(&mut h, "Endpoint", "http://127.0.0.1:1"); + type_field(&mut h, "Access key ID", "SAVEDCONNECTKEY"); + type_field(&mut h, "Secret access key", "saved-connect-secret"); + h.get_by_label("Remember on this computer").click(); + settle(&mut h); + h.get_by_role_and_label(Role::Button, "Connect").click(); + settle(&mut h); + let c = h.state().session.connections.find("s3-saved-widget-connect").unwrap().clone(); + assert!(c.saved_keys); + assert!(store.load(&format!("connection/{}", c.id)).unwrap().is_some()); + s3::disconnect(&c.name); + // The same selected definition with blank keys logs in by its stable ID. + h.get_by_role_and_label(Role::Button, "Connect").click(); + settle(&mut h); + assert_eq!(h.state().session.connections.connections.len(), 1); + assert_eq!(s3::binding_of(&c.name).as_deref(), h.state().session.s3_binding(&c.id)); + s3::disconnect(&c.name); + type_field(&mut h, "Endpoint", "/changed"); + type_field(&mut h, "Access key ID", "OTHERCONNECTKEY"); + type_field(&mut h, "Secret access key", "other-connect-secret"); + h.get_by_label("Remember on this computer").click(); + settle(&mut h); + h.get_by_role_and_label(Role::Button, "Connect").click(); + settle(&mut h); + assert_eq!(h.state().session.connections.connections.len(), 1); + assert_eq!(h.state().session.connections.find(&c.id).unwrap().address, c.address); + assert!(s3::connection(&c.name).is_none()); + assert_eq!(h.get_by_role_and_label(Role::TextInput, "Access key ID").value().as_deref(), Some("OTHERCONNECTKEY")); +} + +#[test] +fn s3_saved_failure_warning_stays_visible_until_close() { + let mut h = with_store(Some(Arc::new(MemorySecrets::failing(SecretError::Denied)))); + let name = "s3-saved-widget-warning"; + new_form(&mut h, name); + h.get_by_label("Remember on this computer").click(); + settle(&mut h); + h.get_by_role_and_label(Role::Button, "Save").click(); + settle(&mut h); + assert_eq!(h.state().dialogs.open, Some("editConnection")); + h.get_by_label("Logged in for this session"); + assert!(h.state().ui.connection.warned.is_some()); + assert!(h.state().ui.connection.secret_key.is_empty()); + assert!(s3::connection(name).is_some()); + assert!(!h.state().session.connections.find(name).unwrap().saved_keys); + h.get_by_role_and_label(Role::Button, "Close").click(); + settle(&mut h); + assert_eq!(h.state().dialogs.open, None); + s3::disconnect(name); +} + +#[test] +fn s3_saved_chooser_cancel_selection_and_explicit_login_retry() { + let mut h = with_store(Some(Arc::new(MemorySecrets::new()))); + let names = ["s3-saved-widget-choice-a", "s3-saved-widget-choice-b"]; + let mut ids = Vec::new(); + for name in names { + let c = h + .state_mut() + .run( + "connection.new", + json!({"kind":"s3","name":name,"address":"http://127.0.0.1:1", + "options":{"credentials":"keys","buckets":"s3-widget-choice-bucket"}}), + ) + .unwrap(); + ids.push(c["id"].as_str().unwrap().to_string()); + } + let inner = "s3://s3-widget-choice-bucket/fixture.pmtiles"; + let address = rustgis_engine::data::s3_address::bind("", "http://127.0.0.1:1", inner).unwrap(); + h.state_mut().run("map.addTiles", json!({"name":"S3 choice fixture","kind":"pmtiles","url":address})).unwrap(); + let text = serde_json::to_string(&h.state().session.project).unwrap(); + h.state_mut().run("project.openText", json!({"text":text})).unwrap(); + settle(&mut h); + assert_eq!(h.state().dialogs.open, Some("s3Connections")); + assert_eq!(h.state().session.s3_pending.len(), 1); + // Clicking disabled Open must not choose the first matching saved connection. + h.get_by_role_and_label(Role::Button, "Open").click(); + settle(&mut h); + assert_eq!(h.state().session.s3_pending.len(), 1); + assert!(h.state().session.s3_binding(&ids[0]).is_none()); + assert!(h.state().session.s3_binding(&ids[1]).is_none()); + h.get_by_role_and_label(Role::Button, "Cancel").click(); + settle(&mut h); + assert_eq!(h.state().dialogs.open, None); + assert_eq!(h.state().session.s3_pending.len(), 1); + h.run_steps(8); + assert_eq!(h.state().dialogs.open, None, "Cancel is not undone next frame"); + h.state_mut().run("ui.s3Connections", json!({})).unwrap(); + settle(&mut h); + h.get_by_label("s3-saved-widget-choice-b — http://127.0.0.1:1").click(); + settle(&mut h); + h.get_by_role_and_label(Role::Button, "Open").click(); + settle(&mut h); + assert_eq!(h.state().dialogs.open, Some("editConnection")); + assert_eq!(h.state().ui.connection.id.as_deref(), Some(ids[1].as_str())); + assert!(!h.state().ui.connection.remember); + h.state_mut().ui.connection.access_key = "CHOICEWIDGETKEY".into(); + h.state_mut().ui.connection.secret_key = "choice-widget-secret".into(); + h.get_by_role_and_label(Role::Button, "Save").click(); + settle(&mut h); + assert!(h.state().session.s3_pending.is_empty()); + assert!(h.state().session.s3_binding(&ids[0]).is_none()); + let binding = h.state().session.s3_binding(&ids[1]).unwrap(); + assert!(h.state().session.map().tiles.iter().any(|t| t.url.contains(binding))); + for name in names { + s3::disconnect(name); + } +} + +#[test] +fn s3_saved_unavailable_and_temporary_forms_cannot_opt_in() { + for (name, store, token) in [ + ("s3-saved-widget-no-store", None, ""), + ("s3-saved-widget-temporary", Some(Arc::new(MemorySecrets::new()) as Arc), "widget-session-token"), + ] { + let mut h = with_store(store.clone()); + new_form(&mut h, name); + h.state_mut().ui.connection.session_token = token.into(); + settle(&mut h); + h.get_by_label("Remember on this computer").click(); + settle(&mut h); + assert!(!h.state().ui.connection.remember); + h.get_by_role_and_label(Role::Button, "Save").click(); + settle(&mut h); + let c = h.state().session.connections.find(name).unwrap(); + assert!(!c.saved_keys); + if let Some(store) = store { + assert!(store.load(&format!("connection/{}", c.id)).unwrap().is_none()); + } + assert!(s3::connection(name).is_some()); + s3::disconnect(name); + } +} + +#[test] +fn s3_saved_refused_forget_keeps_marker_session_and_error() { + let mut h = with_store(Some(Arc::new(MemorySecrets::new()))); + let name = "s3-saved-widget-forget-denied"; + new_form(&mut h, name); + h.get_by_label("Remember on this computer").click(); + settle(&mut h); + h.get_by_role_and_label(Role::Button, "Save").click(); + settle(&mut h); + let id = h.state().session.connections.find(name).unwrap().id.clone(); + h.state_mut().session.database_logins.set_store(Some(Arc::new(MemorySecrets::failing(SecretError::Denied)))); + h.state_mut().run("ui.connection", json!({"id":id})).unwrap(); + settle(&mut h); + h.get_by_role_and_label(Role::Button, "Forget Saved Keys").click(); + settle(&mut h); + assert_eq!(h.state().dialogs.open, Some("editConnection")); + assert!(h.state().dialogs.error.is_some()); + assert!(h.state().session.connections.find(&id).unwrap().saved_keys); + assert!(s3::connection(name).is_some()); + s3::disconnect(name); +} + +#[test] +fn s3_saved_connect_warning_leaves_session_browser_usable() { + let mut h = with_store(Some(Arc::new(MemorySecrets::failing(SecretError::Denied)))); + h.state_mut().run("ui.dialog", json!({"name":"s3"})).unwrap(); + settle(&mut h); + expand_connection(&mut h); + type_field(&mut h, "Name", "s3-saved-widget-connect-warning"); + type_field(&mut h, "Endpoint", "http://127.0.0.1:1"); + type_field(&mut h, "Access key ID", "CONNECTWARNINGKEY"); + type_field(&mut h, "Secret access key", "connect-warning-secret"); + h.get_by_label("Remember on this computer").click(); + settle(&mut h); + h.get_by_role_and_label(Role::Button, "Connect").click(); + settle(&mut h); + assert_eq!(h.state().dialogs.open, Some("s3")); + h.get_by_label("Logged in for this session"); + h.get_by_role_and_label(Role::Button, "My Buckets"); + let c = h.state().session.connections.find("s3-saved-widget-connect-warning").unwrap(); + assert!(!c.saved_keys); + assert!(s3::connection(&c.name).is_some()); + s3::disconnect(&c.name); +} diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index 0bcbddad7..8d091977a 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -7,6 +7,7 @@ use serde_json::json; #[cfg(not(target_arch = "wasm32"))] mod remembered_database; +mod remembered_s3; /// Builds a headless World Atlas UI for interaction tests. fn harness() -> egui_kittest::Harness<'static, RustGisApp> { egui_kittest::Harness::builder().with_accessibility_check(false).with_size(egui::vec2(1400.0, 860.0)).with_max_steps(64).build_ui_state( @@ -1743,6 +1744,11 @@ fn s3_dialog_selects_the_kept_connection_after_a_blank_name_connect() { h.state_mut().run("ui.dialog", json!({"name":"s3"})).unwrap(); h.step(); h.step(); + // Other tests may have live process-wide connections, so the section can start collapsed. + if h.query_by_role_and_label(egui::accesskit::Role::Button, "Connect").is_none() { + h.get_by_label("Connection").click(); + h.run_steps(3); + } h.get_by_role_and_label(egui::accesskit::Role::Button, "Connect").click(); h.step(); h.get_by_label("Save as Connection").click(); @@ -1752,6 +1758,7 @@ fn s3_dialog_selects_the_kept_connection_after_a_blank_name_connect() { assert!(h.query_by_label("Save as Connection").is_none()); assert!(h.query_by_label("Rename it to save it: a connection has this name").is_none()); assert!(h.query_all_by_role(egui::accesskit::Role::ComboBox).any(|n| n.value().as_deref() == Some("default"))); + rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::s3::disconnect("default"); } /// A table list and the table chosen belong to the address they were listed for: editing the diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index e85d75e71..26d846a51 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -343,11 +343,47 @@ rustgis-cli run --cmd 's3.connect={"name":"minio","endpoint":"https://minio.exam --cmd 's3.list={"url":"s3://my-bucket/rasters/"}' --cmd 'layer.addUrl={"url":"s3://my-bucket/rasters/dem.tif"}' ``` -Keys are kept until RustGIS closes: never in a project (which keeps the `s3://` address), a -command's result, a notice or the log. After **Connect**, **Save as Connection** keeps its name, -endpoint, region and credential source under **Data Sources › Connections**, never its keys. -Use **Saved connection** to pick it again; typed keys must be entered again after RustGIS closes. -Storing secrets in the system keychain isn't supported. +Keys are kept until RustGIS closes unless you ask otherwise: never in a project (which keeps the +address), a command's result, a notice or the log. After **Connect**, **Save as Connection** keeps +its name, endpoint, region and credential source under **Data Sources › Connections**, never its +keys. Use **Saved connection** to pick it again. + +On the desktop, a saved connection that uses typed keys can also keep them in your computer's +credential store (the macOS Keychain, the Windows Credential Manager, or the Secret Service): +tick **Remember on this computer** in the S3 dialog or the connection's form when you log in +(it is off every time; `connection.login` takes `"remember": true`). Both keys go to the store, +with the endpoint they are for, and the connection only gets a **Saved keys** mark; the +connection's definition, the preferences and projects never contain a key. Displaying that mark +does not read the store or guarantee it is available. In the next session, choosing, browsing, +testing or logging in to the connection can read its keys from the store with nothing typed +(the system may ask you to unlock it). Typed keys win over saved ones for that login. +Temporary credentials (with a session token) are never kept: they work for the session, +and keys saved earlier stay as they were. If the store is missing or refuses (the web app and the +command line have none), the login still works for this session and says the keys weren't +remembered; without keys typed, a connection with nothing available says so, and is not logged in. +**Forget Saved Keys** (`connection.forgetSavedKeys`; the connection's right-click menu or its +form) deletes them from the store and keeps the session's login. Removing a connection deletes +its saved keys first, and changing its endpoint does too, since they are for the old one; if the +store can't delete them, the removal or edit doesn't happen. Renaming, grouping, buckets and +region keep them, and a copy of a connection has none. A profile, the environment or anonymous +access never read saved keys. + +A saved connection's login reads only what is added through it: its listings, and the objects +**Add** takes from them, are kept as addresses that name that connection (and its endpoint), so a +project reads a bucket through the connection it was added with, never with another connection's +keys because the endpoint or bucket is the same. Opening a project (or importing a map or layer) +finds each bucket its S3 sources read: if exactly one saved connection fits (its endpoint, and +a bucket list that holds it or is empty), it logs in and opens them when its credentials are +available. Missing or unavailable keys leave the sources blocked until you log in. +If several connections fit, **Choose S3 connection…** (`connection.resolveS3`) asks which one, +with no connection selected initially; nothing is read until you choose, and **Cancel** leaves +the sources blocked. If none fits, sources carrying saved-connection provenance stay unread +(an older project's bare `s3://` address is read as before). Addresses of plain `s3.connect` +sessions keep reading as they did. +A saved login belonging to another open session is not reused merely because its name matches: +each session logs in through its selected saved connection. +Resolving a bucket retries only that bucket's deferred sources, keeping layer styles and ordering. +Raster reloads use the same authorized address and load their pixels in the background. Keys go only to the endpoint they're for: a signed request that is redirected isn't followed, and keys from a profile or the environment are used only with AWS or the profile's own `endpoint_url`. In the web app, the bucket's CORS rules must allow the page's origin (and, for a @@ -460,9 +496,9 @@ names the command and parameters that add it (`service.add`, `layer.addDatabase` it left some out. Like `connection.test`, it reads the address itself on the desktop and the command line; with `probe: true` it returns the address to fetch, and the app passes what it fetched back as `answer` (and further `documents` when a service asks for them, as `next`). -Browsing never changes the project. When a saved password turns out to be missing, or to have been -saved for another address, the connection loses its **Saved password** marker; the stored entry -itself is left for **Forget Saved Password**, an address edit or a removal to delete. +Browsing never changes the project. When a saved database password turns out to be missing, or +to have been saved for another address, the connection loses its **Saved password** marker; the +stored entry itself is left for **Forget Saved Password**, an address edit or a removal to delete. Addresses are saved without passwords or tokens. Database passwords entered through `connection.login` are kept separately by connection ID for this session. @@ -474,18 +510,20 @@ If the store is missing, unavailable or denies access, the login remains usable session and a visible warning says it was not remembered (`passwordSaved:false` in the result). The browser and command line have no OS-store hook and remain session-only. -The stored envelope is bound to the connection's ID, kind and cleaned address, including its -username. Renaming or regrouping preserves it. Changing the address or username, or removing -the connection, deletes the old stored password before committing the change; if deletion -fails, the definition remains unchanged. Invalid edits do not delete a password. +The stored database password envelope is bound to the connection's ID, kind and cleaned address, +including its username. Renaming or regrouping preserves it. Changing the address or username, +or removing the connection, deletes the old stored password before committing the change; if +deletion fails, the definition remains unchanged. Invalid edits do not delete a password. Duplicating a connection copies neither its stored password nor its session login. -**Saved password** marks a remembered connection without reading the store just to display it. +**Saved password** marks a remembered database connection without reading the store just to +display it; it is separate from an S3 connection's **Saved keys** marker. **Forget Saved Password** (`connection.forgetPassword {"id":"c1"}`) deletes the stored value, but leaves an explicitly entered session password and already-open layers alone. Forgetting an inactive connection never selects it or replaces another connection's active password. -Projects remain address-only: they contain neither passwords nor saved connection IDs. -On opening a project, one exact matching remembered connection may reconnect automatically. +Database layer origins remain address-only: projects contain neither database passwords nor saved +database connection IDs. On opening a project, one exact matching remembered database connection +may reconnect automatically. Matching strips passwords and outer wrappers/whitespace, but does not infer that URI and libpq forms, reordered parameters, host aliases or different usernames mean the same server. Several eligible remembered matches open a chooser with no default; cancel leaves the layers @@ -522,7 +560,11 @@ server positively refuses a password typed for that Test, the previous per-ID se selection and active address password are restored (or the attempted credential is removed if none existed). A failure after authentication, such as a table-listing error, keeps the new session login. -S3 `accessKeyId`, `secretAccessKey` and optional `sessionToken` remain session-only. +S3 `accessKeyId` and `secretAccessKey` likewise stay session-only unless a typed-key login opts +into remembering them; a nonblank `sessionToken` always keeps the complete temporary credential +set session-only, leaving previously saved keys unchanged (see [S3 buckets](#s3-buckets)). +With login parameters, an S3 `connection.test` logs in first, never remembers keys, and a failed +listing does not undo that session login. S3 profiles and environment credentials remain subject to the existing endpoint restrictions; editing an S3 name, endpoint or options disconnects its session login. S3 shares the name-based session registry used by the S3 dialog and `s3.connect`, so editing or removing a saved @@ -530,10 +572,10 @@ connection also affects a session entry of the same name created there. Renaming clears runtime entries under both the old and new names, so the renamed connection cannot inherit an unrelated login at its destination name. Web-service logins are not supported. Plugins may list connections but cannot manage, log in -to, forget passwords for, test or browse them. The in-app assistant asks for fresh approval -each time for those operations, including Forget and scripted calls; external assistants -using MCP cannot run them. Even `probe: true` needs approval: connection operations cannot be -approved for the whole conversation. +to, forget passwords or S3 keys for, choose an S3 connection for, test or browse them. The in-app +assistant asks for fresh approval each time for those operations, including Forget, S3 source +selection and scripted calls; external assistants using MCP cannot run them. Even `probe: true` +needs approval: connection operations cannot be approved for the whole conversation. Desktop and web apps save connections with their preferences (`connections` in desktop `prefs.json`, `rustgis.connections` in web localStorage). The first load imports saved services, From 76cbac72b3d0a3dd959e41d21cb0e91399e87ee6 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sat, 24 Oct 2026 06:32:13 +0200 Subject: [PATCH 2/9] perf(s3): prepare bound requests outside the registry lock --- crates/tiles/src/s3.rs | 67 +++++++++++++++++++++--------------------- docs/architecture.md | 4 +++ 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/crates/tiles/src/s3.rs b/crates/tiles/src/s3.rs index 1fd3871dd..86b2e185f 100644 --- a/crates/tiles/src/s3.rs +++ b/crates/tiles/src/s3.rs @@ -8,7 +8,7 @@ //! for (a signed request that is redirected is not followed). The fetchers ([`crate::cache`] and //! the web app's) call [`prepare`] for every web request. -use std::sync::Mutex; +use std::{borrow::Cow, sync::Mutex}; use sha2::{Digest, Sha256}; @@ -454,19 +454,22 @@ fn bound_target(address: &str) -> Result<(String, Connection), String> { return Err(UNRESOLVED.into()); } let wanted = canonical(b.endpoint).map_err(|_| MISMATCH.to_string())?; - let reg = lock(); - let saved = reg.saved.iter().find(|s| s.binding == b.binding).ok_or(UNRESOLVED)?; - let c = reg.connections.iter().find(|c| c.name == saved.name).ok_or(UNRESOLVED)?; - if c.endpoint != wanted { - return Err(MISMATCH.into()); - } - let serves = |bucket: &str| c.buckets.iter().any(|x| x == bucket) || saved.grants.iter().any(|x| x == bucket); + // Snapshot only the selected login and this request's explicit grant. No URL parsing, + // origin validation or signing runs under the registry lock, and the grant list isn't copied. + let snapshot = |bucket: Option<&str>| -> Result<(Connection, bool), String> { + let reg = lock(); + let saved = reg.saved.iter().find(|s| s.binding == b.binding).ok_or(UNRESOLVED)?; + let c = reg.connections.iter().find(|c| c.name == saved.name).ok_or(UNRESOLVED)?; + let granted = bucket.is_some_and(|bucket| saved.grants.iter().any(|g| g == bucket)); + Ok((c.clone(), granted)) + }; if is_s3(b.url) { let (bucket, key) = parse(b.url).map_err(|_| MISMATCH.to_string())?; - if !serves(&bucket) { + let (c, granted) = snapshot(Some(&bucket))?; + if c.endpoint != wanted || !(c.buckets.iter().any(|b| b == &bucket) || granted) { return Err(MISMATCH.into()); } - return Ok((object_url(&bucket, &key, Some(c)), c.clone())); + return Ok((object_url(&bucket, &key, Some(&c)), c)); } // Only the origin is case-insensitive. Bucket paths retain their exact spelling. let (scheme, rest) = b.url.split_once("://").ok_or(MISMATCH)?; @@ -476,32 +479,30 @@ fn bound_target(address: &str) -> Result<(String, Connection), String> { return Err(MISMATCH.into()); } let here = origin(scheme, authority); - let allowed = match c.endpoint.as_deref() { - // The connection's own endpoint, path-style: the path's first part is the bucket (none: the - // account's bucket list). - Some(ep) => { - let path = rest.get(authority.len()..).unwrap_or(""); - let first = path.trim_start_matches('/').split(['/', '?', '#']).next().unwrap_or(""); - here == ep - && (first.is_empty() && path.split(['?', '#']).next().unwrap_or("").trim_start_matches('/').is_empty() || serves(first)) - } - None => { - let host = here.strip_prefix("https://").unwrap_or("").split(':').next().unwrap_or(""); - aws_host(scheme, &here, host) - && match aws_bucket(host, rest) { - Some(bucket) => serves(&bucket), - None => { - let path = rest.split_once('/').map(|(_, p)| p).unwrap_or(""); - host.strip_prefix("s3.").and_then(|r| r.strip_suffix(".amazonaws.com")) == Some(c.region.as_str()) - && path.split(['?', '#']).next().unwrap_or("").is_empty() - } - } - } + let host = here.strip_prefix("https://").unwrap_or("").split(':').next().unwrap_or(""); + let path = rest.get(authority.len()..).unwrap_or(""); + let first = path.trim_start_matches('/').split(['/', '?', '#']).next().unwrap_or(""); + let bucket: Option> = + if wanted.is_some() { (!first.is_empty()).then_some(Cow::Borrowed(first)) } else { aws_bucket(host, rest).map(Cow::Owned) }; + let origin_allowed = match wanted.as_deref() { + Some(endpoint) => here == endpoint, + None => aws_host(scheme, &here, host), + }; + let root_path = if wanted.is_some() { + path.split(['?', '#']).next().unwrap_or("").trim_start_matches('/').is_empty() + } else { + rest.split_once('/').map(|(_, p)| p).unwrap_or("").split(['?', '#']).next().unwrap_or("").is_empty() + }; + let root_region = host.strip_prefix("s3.").and_then(|r| r.strip_suffix(".amazonaws.com")); + let (c, granted) = snapshot(bucket.as_deref())?; + let allowed = match bucket.as_deref() { + Some(bucket) => c.buckets.iter().any(|b| b == bucket) || granted, + None => root_path && (wanted.is_some() || root_region == Some(c.region.as_str())), }; - if !allowed { + if c.endpoint != wanted || !origin_allowed || !allowed { return Err(MISMATCH.into()); } - Ok((b.url.to_string(), c.clone())) + Ok((b.url.to_string(), c)) } /// A request ready to send: its web address, the headers to add, and whether it is signed (a diff --git a/docs/architecture.md b/docs/architecture.md index a8b74bdf9..6a9f45fe9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -59,6 +59,10 @@ browser and the CLI, where callers keep a secret for the session only, as before - **Blocking.** A call can wait on an OS prompt or D-Bus. Make it on a user action or a worker thread, never every frame, and keep the value in memory for the session. Don't access one account from two threads at once. +- **S3 request snapshots.** The registry lock protects selection of the exact live login and + the requested bucket's explicit-grant status. Copy only that login and grant boolean, not + the grant list; parse and validate addresses and prepare signatures outside the lock. + Keychain calls and network I/O never run under the registry lock. - **macOS.** The system asks again for access when the app's code signature changes, which happens with unsigned or re-signed builds. - **Tests.** Use `MemorySecrets`, and `MemorySecrets::failing(..)` for error paths. The desktop From b30852782244822e5234acdb3957cb45a2cff499 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sat, 24 Oct 2026 13:20:18 +0200 Subject: [PATCH 3/9] fix(s3): preserve distinct legacy and AWS project origins --- crates/data/src/s3_address.rs | 26 +++---- crates/engine/src/s3_connections.rs | 111 +++++++++++----------------- crates/engine/src/s3_saved_tests.rs | 53 ++++++++++++- crates/tiles/src/s3.rs | 15 +++- crates/ui-egui/src/connectionui.rs | 51 ------------- crates/ui-egui/src/s3ui.rs | 43 +---------- docs/user-guide/data.md | 3 + 7 files changed, 124 insertions(+), 178 deletions(-) diff --git a/crates/data/src/s3_address.rs b/crates/data/src/s3_address.rs index 62193309e..a0656295a 100644 --- a/crates/data/src/s3_address.rs +++ b/crates/data/src/s3_address.rs @@ -5,12 +5,14 @@ //! other's) names it: `rustgis-s3::::`. The //! endpoint is the connection's canonical one (empty for AWS), the inner address is a web address //! or an `s3://` one, kept verbatim. An empty binding is an address no connection has been chosen -//! for yet: it is never fetched. Nothing here is secret, so a saved project may keep these -//! addresses; deciding whether a binding may sign a request, and for which endpoint and bucket, is -//! `rustgis_tiles::s3`'s job, not this structural parser's. +//! for yet. The reserved [`LEGACY`] marker additionally retains an unknown endpoint while a choice +//! is pending. Neither can be a live login or fetch data. Nothing here is secret, so a saved +//! project may keep these addresses; `rustgis_tiles::s3` authorizes live bindings, not this parser. /// Starts an address read through a saved S3 connection. pub const PREFIX: &str = "rustgis-s3:"; +/// Reserved unresolved marker for legacy sources whose endpoint is unknown. Never a live login. +pub const LEGACY: &str = "0000000000000000"; /// The longest address, in bytes. pub const MAX_ADDRESS: usize = 8192; /// The longest endpoint, in bytes. @@ -24,7 +26,7 @@ const TOO_LONG: &str = "the saved S3 address is too long"; /// The parts of an address read through a saved connection. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct BoundS3<'a> { - /// The runtime token of the connection that may read it (empty: none chosen yet). + /// The login's runtime token; empty: no login chosen, [`LEGACY`]: unknown endpoint. pub binding: &'a str, /// The connection's canonical endpoint (empty: AWS). pub endpoint: &'a str, @@ -74,6 +76,9 @@ pub fn parse(address: &str) -> Result>, &'static str> { // Slicing at a byte offset that isn't a character boundary is `None`, not a panic. let endpoint = rest.get(..length).ok_or(BAD)?; let url = rest.get(length..).and_then(|r| r.strip_prefix(':')).ok_or(BAD)?; + if binding == LEGACY && (!endpoint.is_empty() || !url.get(..5).is_some_and(|p| p.eq_ignore_ascii_case("s3://"))) { + return Err(BAD); + } if is_wrapper(url) || !web_or_s3(url) { return Err(BAD); } @@ -108,12 +113,6 @@ pub fn rebound(address: &str, binding: &str) -> Option { bind(binding, b.endpoint, b.url).ok() } -/// `address` with its inner address replaced, keeping binding and endpoint. -pub fn with_url(address: &str, url: &str) -> Option { - let b = parse(address).ok().flatten()?; - bind(b.binding, b.endpoint, url).ok() -} - /// The address a server is reached at, for an address that may be read through a saved connection /// (as it is, or as the archive after a tile template's `@`): the wrapper's inner address, else /// `address` itself. @@ -143,10 +142,6 @@ mod tests { let p = parse(&aws).unwrap().unwrap(); assert_eq!((p.binding, p.endpoint, p.url), ("", "", "s3://my-bucket/k.tif")); assert_eq!(rebound(&aws, B).unwrap(), format!("rustgis-s3:{B}:0::s3://my-bucket/k.tif")); - assert_eq!( - with_url(&a, "https://minio.example:9000/b/other.fgb").unwrap(), - format!("rustgis-s3:{B}:26:https://minio.example:9000:https://minio.example:9000/b/other.fgb") - ); // Not a wrapper: left alone. assert_eq!(parse("https://x.org/a").unwrap(), None); assert_eq!(inner("https://x.org/a"), "https://x.org/a"); @@ -164,6 +159,9 @@ mod tests { "rustgis-s3:abc:17:https://h.example:https://h.example/k".to_string(), format!("rustgis-s3:{B}0:17:{endpoint}:https://h.example/k"), format!("rustgis-s3:{}:17:{endpoint}:https://h.example/k", "g".repeat(16)), + // Unknown origin is only meaningful for a bare s3 object, never a known endpoint. + format!("rustgis-s3:{LEGACY}:{}:{endpoint}:s3://b/k", endpoint.len()), + format!("rustgis-s3:{LEGACY}:0::https://h.example/b/k"), // length: missing, signed, too long, past the end, off by one either way format!("rustgis-s3:{B}::{endpoint}:https://h.example/k"), format!("rustgis-s3:{B}:+17:{endpoint}:https://h.example/k"), diff --git a/crates/engine/src/s3_connections.rs b/crates/engine/src/s3_connections.rs index ee6d7fc91..65d91120c 100644 --- a/crates/engine/src/s3_connections.rs +++ b/crates/engine/src/s3_connections.rs @@ -17,10 +17,8 @@ //! with a notice. //! //! An older project's bare `s3://` address names no endpoint: any saved connection whose buckets -//! include (or aren't limited to) its bucket is a candidate. While it waits it is an unresolved -//! address with an empty endpoint (and so AWS, if the project is saved then). - -use std::collections::BTreeSet; +//! include (or aren't limited to) its bucket is a candidate. Its blocked placeholder uses the +//! reserved non-login marker `s3_address::LEGACY`, distinct from explicit AWS provenance. use rustgis_data::s3_address; @@ -77,15 +75,18 @@ fn part(address: &str) -> &str { address.get(head_len(address)..).unwrap_or(address) } -/// Empties the binding of every address read through a saved connection: what a file or an -/// import brings is no login of this computer's. +/// Clears imported runtime authority. A legacy placeholder becomes its original bare address; +/// its reserved marker was never authority to use a login. pub(crate) fn clear_bindings<'a>(addresses: impl IntoIterator) { for a in addresses { let at = head_len(a); let Some(p) = a.get(at..) else { continue }; - if let Some(cleared) = s3_address::rebound(p, "") - && cleared != p - { + let cleared = match s3_address::parse(p) { + Ok(Some(b)) if b.binding == s3_address::LEGACY => Some(b.url.to_string()), + Ok(Some(b)) if !b.binding.is_empty() => s3_address::bind("", b.endpoint, b.url).ok(), + _ => None, + }; + if let Some(cleared) = cleared { *a = format!("{}{cleared}", a.get(..at).unwrap_or("")); } } @@ -99,18 +100,18 @@ struct Unresolved { bucket: String, } -/// `address` as [`Unresolved`], when it is one. `legacy` holds the buckets of older addresses -/// waiting for a choice, whose placeholder (no endpoint, no binding) is not AWS's. -fn unresolved(address: &str, legacy: &BTreeSet) -> Option { +/// What an unresolved address names, without inferring origin from another source's bucket. +fn unresolved(address: &str) -> Option { let p = part(address); match s3_address::parse(p) { Ok(Some(b)) => { - if !b.binding.is_empty() { + let is_legacy = b.binding == s3_address::LEGACY; + if !b.binding.is_empty() && !is_legacy { return None; } let endpoint = s3::canonical(b.endpoint).ok()?.unwrap_or_default(); let bucket = s3::bucket_of(&endpoint, b.url)?; - let endpoint = if endpoint.is_empty() && legacy.contains(&bucket) { None } else { Some(endpoint) }; + let endpoint = if is_legacy { None } else { Some(endpoint) }; Some(Unresolved { endpoint, bucket }) } Ok(None) if s3::is_s3(p) => Some(Unresolved { endpoint: None, bucket: s3::parse(p).ok()?.0 }), @@ -155,22 +156,17 @@ struct Group { } impl Group { - fn has(&self, address: &str, legacy: &BTreeSet) -> bool { - unresolved(address, legacy).is_some_and(|u| u.endpoint == self.endpoint && u.bucket == self.bucket) + fn has(&self, address: &str) -> bool { + unresolved(address).is_some_and(|u| u.endpoint == self.endpoint && u.bucket == self.bucket) } } -fn legacy_buckets(s: &Session) -> BTreeSet { - s.s3_pending.iter().filter(|p| p.endpoint.is_none()).map(|p| p.bucket.clone()).collect() -} - /// Settles the S3 sources of the project: see the module's text. Called when a project opens /// and when maps or layers are imported, never from a frame. pub(crate) fn resolve_project_s3(s: &mut Session) { - let legacy = legacy_buckets(s); let mut groups: Vec = Vec::new(); for a in s.project.addresses_mut() { - if let Some(u) = unresolved(a, &legacy) + if let Some(u) = unresolved(a) && !groups.iter().any(|g| g.endpoint == u.endpoint && g.bucket == u.bucket) { groups.push(Group { endpoint: u.endpoint, bucket: u.bucket }); @@ -178,7 +174,7 @@ pub(crate) fn resolve_project_s3(s: &mut Session) { } let mut pending = Vec::new(); for g in &groups { - if let Some(p) = settle(s, g, &legacy) { + if let Some(p) = settle(s, g) { pending.push(p); } } @@ -186,7 +182,7 @@ pub(crate) fn resolve_project_s3(s: &mut Session) { } /// What `g`'s sources do now: bound to its one candidate, kept for a choice, or as they were. -fn settle(s: &mut Session, g: &Group, legacy: &BTreeSet) -> Option { +fn settle(s: &mut Session, g: &Group) -> Option { let candidates: Vec = s.connections.connections.iter().filter(|c| matches(c, &g.endpoint, &g.bucket)).cloned().collect(); let at = match g.endpoint.as_deref() { Some(e) if !e.is_empty() => format!(" at {e}"), @@ -197,9 +193,10 @@ fn settle(s: &mut Session, g: &Group, legacy: &BTreeSet) -> Option) -> Option match bind_group(s, g, legacy, c) { + [c] => match bind_group(s, g, c) { Ok(_) => None, Err(e) => { // Kept blocked, naming the one connection that could read it (an older project's // address now names its endpoint). let endpoint = endpoint_of(c).unwrap_or_default(); - let source = rewrite(s, g, legacy, &endpoint, "").into_iter().next()?; - block_by(s, &source, &endpoint); + block(s, g); + let source = rewrite(s, g, &endpoint, "").into_iter().next()?; s.notify( format!("Log in to the saved S3 connection '{}' to open the sources that read the bucket '{}': {e}", c.name, g.bucket), false, @@ -224,8 +221,9 @@ fn settle(s: &mut Session, g: &Group, legacy: &BTreeSet) -> Option { let endpoint = g.endpoint.clone().unwrap_or_default(); - let source = rewrite(s, g, legacy, &endpoint, "").into_iter().next()?; - block_by(s, &source, &endpoint); + let binding = if g.endpoint.is_none() { s3_address::LEGACY } else { "" }; + block(s, g); + let source = rewrite(s, g, &endpoint, binding).into_iter().next()?; s.notify( format!( "{} saved S3 connections could read the bucket '{}'{at}: choose which one reads this project's sources (Data Sources › Choose S3 connection…); nothing is read until you do", @@ -246,10 +244,10 @@ fn settle(s: &mut Session, g: &Group, legacy: &BTreeSet) -> Option, endpoint: &str, binding: &str) -> Vec { +fn rewrite(s: &mut Session, g: &Group, endpoint: &str, binding: &str) -> Vec { let mut out = Vec::new(); for a in s.project.addresses_mut() { - if g.has(a, legacy) + if g.has(a) && let Some(new) = bound(a, endpoint, binding) { *a = new.clone(); @@ -259,30 +257,15 @@ fn rewrite(s: &mut Session, g: &Group, legacy: &BTreeSet, endpoint: &str out } -/// Marks the streamed sources of the address that wait for a choice (their layers show why). -fn block_by(s: &mut Session, source: &str, endpoint: &str) { - let Some(want) = unresolved(source, &BTreeSet::new()) else { return }; - for src in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.source.as_mut()) { - let Some(path) = src.path.as_deref() else { continue }; - if !src.is_open() - && unresolved(path, &BTreeSet::new()).is_some_and(|u| u.bucket == want.bucket) - && s3_address::parse(part(path)).ok().flatten().is_some_and(|b| b.endpoint == endpoint) - { - src.error = Some(s3::UNRESOLVED.into()); - } - } -} - /// Marks the streamed sources of `g` as blocked for want of any connection; how many addresses /// of `g` there are. -fn block(s: &mut Session, g: &Group, legacy: &BTreeSet, why: &str) -> usize { - let why = if why.is_empty() { s3::UNRESOLVED } else { why }; +fn block(s: &mut Session, g: &Group) -> usize { for src in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.source.as_mut()) { - if !src.is_open() && src.path.as_deref().is_some_and(|p| g.has(p, legacy)) { - src.error = Some(why.into()); + if !src.is_open() && src.path.as_deref().is_some_and(|p| g.has(p)) { + src.error = Some(s3::UNRESOLVED.into()); } } - s.project.addresses_mut().into_iter().filter(|a| g.has(a, legacy)).count() + s.project.addresses_mut().into_iter().filter(|a| g.has(a)).count() } /// Opens (again) the streamed sources whose address `which` accepts; how many opened and how @@ -312,11 +295,11 @@ fn reopen_where(s: &mut Session, which: impl Fn(&str) -> bool) -> (usize, usize) /// Binds `g`'s sources to the saved connection `c`: logs it in if it isn't, lets its login read /// the bucket (the user's choice, or its only candidate's), rewrites the addresses to name it and /// reads them. How many sources opened and how many failed. -fn bind_group(s: &mut Session, g: &Group, legacy: &BTreeSet, c: &Connection) -> Result<(usize, usize), String> { +fn bind_group(s: &mut Session, g: &Group, c: &Connection) -> Result<(usize, usize), String> { let endpoint = endpoint_of(c).ok_or_else(|| s3::MISMATCH.to_string())?; let binding = crate::cmd::connections::login_saved(s, c)?; s3::grant_saved(&binding, &g.bucket)?; - let addresses = rewrite(s, g, legacy, &endpoint, &binding); + let addresses = rewrite(s, g, &endpoint, &binding); // What failed before at an identical address is read again with this login. for a in &addresses { cache::forget(part(a)); @@ -336,13 +319,12 @@ pub(crate) fn choose(s: &mut Session, source: &str, id: &str) -> crate::CmdResul let Some(index) = s.s3_pending.iter().position(|p| p.source == source) else { return Err(STALE.into()) }; let pending = s.s3_pending.get(index).cloned().ok_or(STALE)?; let c = s.connections.find(id).filter(|c| c.id == id && matches(c, &pending.endpoint, &pending.bucket)).cloned().ok_or(STALE)?; - let legacy = legacy_buckets(s); let g = Group { endpoint: pending.endpoint.clone(), bucket: pending.bucket.clone() }; - if !s.project.addresses_mut().into_iter().any(|a| g.has(a, &legacy)) { + if !s.project.addresses_mut().into_iter().any(|a| g.has(a)) { s.s3_pending.remove(index); return Err(STALE.into()); } - let (reopened, failed) = bind_group(s, &g, &legacy, &c)?; + let (reopened, failed) = bind_group(s, &g, &c)?; s.s3_pending.remove(index); Ok(serde_json::json!({"id": c.id, "reopened": reopened, "failed": failed})) } @@ -366,26 +348,21 @@ mod tests { assert_eq!(part("s3://data/a.fgb"), "s3://data/a.fgb"); let mut addresses = [template.clone(), "s3://data/b.fgb".to_string(), wrapped.clone(), "https://x.org/a".to_string()]; clear_bindings(addresses.iter_mut()); - let none = BTreeSet::new(); assert_eq!(addresses[0], "pmtiles:{z}/{x}/{y}@rustgis-s3::22:https://s3.example.org:https://s3.example.org/data/a.pmtiles"); assert_eq!(addresses[1], "s3://data/b.fgb"); - let u = unresolved(&addresses[0], &none).unwrap(); + let u = unresolved(&addresses[0]).unwrap(); assert_eq!((u.endpoint.as_deref(), u.bucket.as_str()), (Some("https://s3.example.org"), "data")); - let u = unresolved(&addresses[1], &none).unwrap(); + let u = unresolved(&addresses[1]).unwrap(); assert_eq!((u.endpoint, u.bucket.as_str()), (None, "data")); - assert!(unresolved(&addresses[3], &none).is_none()); + assert!(unresolved(&addresses[3]).is_none()); // A bound address is resolved; rebinding keeps its template and inner address. - assert!(unresolved(&wrapped, &none).is_none()); + assert!(unresolved(&wrapped).is_none()); let again = bound(&addresses[0], "https://s3.example.org", B).unwrap(); assert_eq!(again, template); assert!(has_binding(&again, B) && !has_binding(&addresses[0], B)); let bare = bound("s3://data/b.fgb", "", B).unwrap(); assert_eq!(bare, format!("rustgis-s3:{B}:0::s3://data/b.fgb")); - // An older address's placeholder is AWS's unless an older one is waiting for its bucket. - let placeholder = "rustgis-s3::0::s3://data/b.fgb"; - assert_eq!(unresolved(placeholder, &none).unwrap().endpoint.as_deref(), Some("")); - assert_eq!(unresolved(placeholder, &BTreeSet::from(["data".to_string()])).unwrap().endpoint, None); // Malformed provenance is nobody's resource and never a file. - assert!(unresolved("rustgis-s3:junk", &none).is_none() && is_s3_backed("rustgis-s3:junk")); + assert!(unresolved("rustgis-s3:junk").is_none() && is_s3_backed("rustgis-s3:junk")); } } diff --git a/crates/engine/src/s3_saved_tests.rs b/crates/engine/src/s3_saved_tests.rs index 6a9f0f4e1..0a0d84a4e 100644 --- a/crates/engine/src/s3_saved_tests.rs +++ b/crates/engine/src/s3_saved_tests.rs @@ -836,7 +836,6 @@ fn s3_saved_sources_without_keys_zero_matches_and_older_addresses_stay_blocked_o assert_eq!(two.s3_pending.len(), 2, "{:?}", two.s3_pending); let older = two.s3_pending.iter().find(|p| p.endpoint.is_none()).unwrap().clone(); assert_eq!(older.candidates.len(), 2); - assert_eq!(older.source, "rustgis-s3::0::s3://sv-proj-edge/grid.fgb"); assert!(two.s3_pending.iter().any(|p| p.endpoint.is_some() && p.candidates.len() == 2)); let r = two.run("connection.resolveS3", &json!({"source": older.source, "id": "c2"})).unwrap(); assert_eq!(r["id"], "c2"); @@ -1153,3 +1152,55 @@ fn s3_saved_favourites_keep_provenance_overhead_without_relaxing_destination_lim let too_long = rustgis_data::s3_address::bind("0123456789abcdef", endpoint, &oversized).unwrap(); assert!(sources.keep("favourites", crate::sources::Source { address: too_long, ..source }).is_err()); } + +#[test] +fn s3_saved_legacy_choice_never_rebinds_an_explicit_aws_source_for_the_same_object() { + for aws_first in [false, true] { + let bucket = format!("sv-mixed-{aws_first}"); + let srv = serve_bucket(&bucket, vec![("same.fgb", grid_fgb())], Vec::new()); + let store = Arc::new(Spy::default()); + let mut s = session(Some(&store)); + let custom_name = format!("{bucket}-custom"); + let custom = saved(&mut s, &custom_name, &srv.base, &bucket, "keys"); + login(&mut s, &custom, "MIXEDCUSTOMKEY", "mixed-custom-secret", true).unwrap(); + let aws_a = saved(&mut s, &format!("{bucket}-aws-a"), "", &bucket, "none"); + let aws_b = saved(&mut s, &format!("{bucket}-aws-b"), "", &bucket, "none"); + let raw = format!("s3://{bucket}/same.fgb"); + let aws = rustgis_data::s3_address::bind("", "", &raw).unwrap(); + let sources = if aws_first { [("AWS", &aws), ("Legacy", &raw)] } else { [("Legacy", &raw), ("AWS", &aws)] }; + for (name, path) in sources { + s.map_mut().add_streamed(name, serde_json::from_value(json!({"path":path})).unwrap()); + } + let text = s.project.saved_text().unwrap().0; + s.run("project.openText", &json!({"text":text})).unwrap(); + assert_eq!(srv.requests(), 0, "neither resource is read before a choice"); + // Saving while a choice is pending must retain unknown origin, not turn it into AWS. + let text = s.project.saved_text().unwrap().0; + let definitions = s.connections.to_json(); + s = session(Some(&store)); + s.connections = Connections::from_json(&definitions); + s.run("project.openText", &json!({"text":text})).unwrap(); + assert_eq!(srv.requests(), 0, "a pending save/reopen still fetches nothing"); + let legacy = s.s3_pending.iter().find(|p| p.endpoint.is_none()).unwrap().clone(); + let aws_pending = s.s3_pending.iter().find(|p| p.endpoint.as_deref() == Some("")).unwrap().clone(); + assert_eq!(aws_pending.candidates, vec![aws_a, aws_b]); + let result = s.run("connection.resolveS3", &json!({"source":legacy.source,"id":custom})).unwrap(); + assert_eq!((result["reopened"].as_u64(), result["failed"].as_u64()), (Some(1), Some(0))); + let legacy_id = s.map().layer_by_name("Legacy").unwrap().id; + assert!(read_view(&mut s, legacy_id) > 0); + let aws_source = s.map().layer_by_name("AWS").unwrap().source.as_ref().unwrap(); + assert_eq!(aws_source.path.as_deref(), Some(aws.as_str()), "explicit AWS provenance is unchanged"); + assert!(!aws_source.is_open()); + assert_eq!(s.s3_pending, vec![aws_pending.clone()], "AWS still requires its own choice"); + settle(&srv); + let from = srv.requests(); + assert_eq!( + s.run("connection.resolveS3", &json!({"source":aws_pending.source,"id":custom})).unwrap_err(), + crate::s3_connections::STALE + ); + crate::s3_connections::resolve_project_s3(&mut s); + assert_eq!(s.s3_pending, vec![aws_pending]); + assert_eq!(srv.requests(), from, "the AWS source never reaches the custom endpoint"); + s3::disconnect(&custom_name); + } +} diff --git a/crates/tiles/src/s3.rs b/crates/tiles/src/s3.rs index 86b2e185f..218eadae4 100644 --- a/crates/tiles/src/s3.rs +++ b/crates/tiles/src/s3.rs @@ -180,7 +180,7 @@ pub fn new_binding() -> Result { /// granted to this binding are kept for a login of the same connection at the same endpoint, and /// forgotten otherwise. pub fn connect_saved(c: Connection, binding: &str, id: &str) -> Result<(), String> { - if binding.is_empty() || !rustgis_data::s3_address::valid_binding(binding) { + if binding.is_empty() || binding == rustgis_data::s3_address::LEGACY || !rustgis_data::s3_address::valid_binding(binding) { return Err("not a saved S3 connection token".into()); } if id.is_empty() || id.len() > 64 { @@ -311,8 +311,12 @@ pub const MISMATCH: &str = "The saved S3 connection no longer matches this sourc /// a fetch of it, and a project, keep. (A binding alone grants no bucket: [`grant_saved`], or the /// connection's own buckets, do.) pub fn bound_url(binding: &str, url: &str) -> Result { - let (c, _) = saved_connection(binding).ok_or(UNRESOLVED)?; - rustgis_data::s3_address::bind(binding, c.endpoint.as_deref().unwrap_or(""), url).map_err(str::to_string) + let endpoint = { + let reg = lock(); + let saved = reg.saved.iter().find(|s| !binding.is_empty() && s.binding == binding).ok_or(UNRESOLVED)?; + reg.connections.iter().find(|c| c.name == saved.name).ok_or(UNRESOLVED)?.endpoint.clone() + }; + rustgis_data::s3_address::bind(binding, endpoint.as_deref().unwrap_or(""), url).map_err(str::to_string) } /// An endpoint as connections keep it: a web address's scheme and authority, no path. @@ -1262,6 +1266,11 @@ mod tests { let (a, b) = (new_binding().unwrap(), new_binding().unwrap()); assert_ne!(a, b); assert!(a.len() == 16 && a.bytes().all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase())); + let legacy = rustgis_data::s3_address::LEGACY; + assert!(connect_saved(saved_login("sv-pending", "NEVERINSTALLED", None, &["sv-bucket"]), legacy, "c1").is_err()); + assert!(connection("sv-pending").is_none(), "a pending marker can never install credentials"); + let pending = rustgis_data::s3_address::bind(legacy, "", "s3://sv-bucket/k.fgb").unwrap(); + assert_eq!(prepare(&pending).unwrap_err(), UNRESOLVED); // The second saved connection is registered first: insertion order decides nothing. connect_saved(saved_login("sv-b", "SAVEDKEYB", Some(ep), &["sv-bucket"]), &b, "c1").unwrap(); connect_saved(saved_login("sv-a", "SAVEDKEYA", Some(ep), &["sv-bucket"]), &a, "c1").unwrap(); diff --git a/crates/ui-egui/src/connectionui.rs b/crates/ui-egui/src/connectionui.rs index ea9bc6d03..93a4c3da1 100644 --- a/crates/ui-egui/src/connectionui.rs +++ b/crates/ui-egui/src/connectionui.rs @@ -1039,57 +1039,6 @@ mod tests { st.remember = true; } - #[test] - fn remembered_keys_log_in_again_only_when_asked_and_can_be_forgotten() { - let mut app = RustGisApp::with_sample(); - app.session.database_logins.set_store(Some(std::sync::Arc::new(rustgis_engine::data::secrets::MemorySecrets::new()))); - let name = "remember-ui-test"; - new_s3_form(&mut app, name); - save(&mut app).unwrap(); - let id = app.ui.connection.id.clone().unwrap(); - assert!(app.session.connections.find(&id).unwrap().saved_keys); - assert!(app.ui.connection.access_key.is_empty() && app.ui.connection.secret_key.is_empty() && !app.ui.connection.remember); - assert!(app.ui.connection.warned.is_none()); - assert!(s3::connection(name).is_some()); - s3::disconnect(name); - - // A plain edit changes the definition, and logs nothing in. - open_edit(&mut app, &id).unwrap(); - assert!(!app.ui.connection.remember && !app.ui.connection.login); - app.ui.connection.group = "Lab".into(); - save(&mut app).unwrap(); - assert!(s3::connection(name).is_none(), "an edit is no login"); - - // Log In with nothing typed reads the saved keys. - assert!(log_in(&mut app, &id)); - assert!(app.ui.connection.login); - save(&mut app).unwrap(); - assert!(s3::connection(name).is_some()); - assert_eq!(s3::binding_of(name).as_deref(), app.session.s3_binding(&id)); - - forget_keys(&mut app, &id).unwrap(); - assert!(!app.session.connections.find(&id).unwrap().saved_keys); - assert!(s3::connection(name).is_some(), "forgetting keeps the session's login"); - s3::disconnect(name); - } - - #[test] - fn a_login_whose_keys_were_not_remembered_stays_open_to_say_so() { - let mut app = RustGisApp::with_sample(); - let store = rustgis_engine::data::secrets::MemorySecrets::failing(rustgis_engine::data::secrets::SecretError::Denied); - app.session.database_logins.set_store(Some(std::sync::Arc::new(store))); - let name = "remember-ui-warned-test"; - new_s3_form(&mut app, name); - save(&mut app).unwrap(); - let id = app.ui.connection.id.clone().unwrap(); - assert!(!app.session.connections.find(&id).unwrap().saved_keys); - assert!(app.ui.connection.warned.is_some()); - assert_eq!(app.dialogs.open, Some("editConnection")); - assert!(app.ui.connection.secret_key.is_empty()); - assert!(s3::connection(name).is_some(), "logged in for the session"); - s3::disconnect(name); - } - #[test] fn remember_without_a_typed_pair_changes_nothing() { let mut app = RustGisApp::with_sample(); diff --git a/crates/ui-egui/src/s3ui.rs b/crates/ui-egui/src/s3ui.rs index d670d7186..97ee9da61 100644 --- a/crates/ui-egui/src/s3ui.rs +++ b/crates/ui-egui/src/s3ui.rs @@ -217,12 +217,6 @@ fn form_name(st: &State) -> String { if st.name.trim().is_empty() { "default".to_string() } else { st.name.trim().to_string() } } -/// The typed key pair, when both halves are typed. -fn typed_pair(st: &State) -> Option<(String, String)> { - let (key, secret) = (st.key.trim(), st.secret.trim()); - (!key.is_empty() && !secret.is_empty()).then(|| (key.to_string(), secret.to_string())) -} - /// The saved connection the form stands for, when one was picked and nothing typed since changes /// its definition (name, endpoint, region, where its keys come from). fn unchanged_saved(app: &RustGisApp) -> Option { @@ -249,7 +243,7 @@ fn unchanged_saved(app: &RustGisApp) -> Option { fn connect(app: &mut RustGisApp) { app.ui.s3.warning = None; let remember = app.ui.s3.remember && app.ui.s3.source == "keys" && app.session.database_logins.has_store(); - if remember && typed_pair(&app.ui.s3).is_none() { + if remember && (app.ui.s3.key.trim().is_empty() || app.ui.s3.secret.trim().is_empty()) { app.ui.s3.error = Some(tl!("Type the access key ID and the secret access key to remember them on this computer.").to_string()); return; } @@ -712,41 +706,6 @@ mod tests { s3::disconnect(&c.name); } - #[test] - fn remember_on_an_unsaved_form_saves_a_connection_then_its_keys() { - let mut app = RustGisApp::with_sample(); - app.session.database_logins.set_store(Some(std::sync::Arc::new(rustgis_engine::data::secrets::MemorySecrets::new()))); - let name = "remember-s3ui-test"; - app.ui.s3.name = name.into(); - app.ui.s3.source = "keys".into(); - app.ui.s3.key = "AKIDREMEMBERS3UI".into(); - app.ui.s3.secret = "remember-s3ui-secret".into(); - app.ui.s3.remember = true; - connect(&mut app); - assert_eq!(app.ui.s3.error, None); - assert_eq!(app.ui.s3.warning, None); - assert_eq!(app.session.connections.connections.len(), 1, "one definition"); - let c = app.session.connections.connections[0].clone(); - assert!(c.saved_keys); - assert_eq!(c.options.get("credentials").map(String::as_str), Some("keys")); - assert_eq!(app.ui.s3.connection.as_deref(), Some(c.id.as_str())); - assert!(app.ui.s3.key.is_empty() && app.ui.s3.secret.is_empty() && !app.ui.s3.remember); - assert!(saved_binding(&app.session, name).is_some()); - s3::disconnect(name); - - // The same name typed again with other settings: never written over. - app.ui.s3.connection = None; - app.ui.s3.endpoint = "https://other.example.com".into(); - app.ui.s3.key = "AKIDOTHER".into(); - app.ui.s3.secret = "other-secret".into(); - app.ui.s3.remember = true; - connect(&mut app); - assert!(app.ui.s3.error.is_some()); - assert_eq!(app.session.connections.connections.len(), 1); - assert_eq!(app.ui.s3.secret, "other-secret", "the typed values stay"); - assert!(s3::connection(name).is_none()); - } - #[test] fn browsing_through_a_saved_login_names_it_and_a_session_connection_does_not() { let mut app = RustGisApp::with_sample(); diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 26d846a51..f9c0b993c 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -383,6 +383,9 @@ sessions keep reading as they did. A saved login belonging to another open session is not reused merely because its name matches: each session logs in through its selected saved connection. Resolving a bucket retries only that bucket's deferred sources, keeping layer styles and ordering. +An older source with an unknown endpoint stays separate from an explicitly AWS-backed source, +even when both name the same bucket and object. Saving and reopening while a choice is pending +preserves that distinction; choosing a connection for one resource does not choose for the other. Raster reloads use the same authorized address and load their pixels in the background. Keys go only to the endpoint they're for: a signed request that is redirected isn't followed, and keys from a profile or the environment are used only with AWS or the profile's own From 1b6154106ca1e1cb50572a5d016c6f8fd8031807 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sat, 24 Oct 2026 13:32:58 +0200 Subject: [PATCH 4/9] fix(tiles): allocate S3 bindings without a deprecated atomic method --- crates/tiles/src/s3.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/tiles/src/s3.rs b/crates/tiles/src/s3.rs index 218eadae4..d7ebd6d33 100644 --- a/crates/tiles/src/s3.rs +++ b/crates/tiles/src/s3.rs @@ -166,12 +166,18 @@ pub fn connect(c: Connection) -> Result<(), String> { /// digits, never zero, never given twice in this process (two sessions' saved ids are often the /// same, `c1`; their logins mustn't be). pub fn new_binding() -> Result { - static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - let n = NEXT - .fetch_update(std::sync::atomic::Ordering::SeqCst, std::sync::atomic::Ordering::SeqCst, |n| n.checked_add(1)) - .map_err(|_| "S3 connection tokens are used up".to_string())? - .wrapping_add(1); - Ok(format!("{n:016x}")) + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + // A compare-and-swap loop, not `fetch_update`/`try_update`: the old name is deprecated on the + // current stable and the new one does not exist on the oldest toolchain this crate builds on. + let mut current = NEXT.load(Ordering::SeqCst); + loop { + let next = current.checked_add(1).ok_or_else(|| "S3 connection tokens are used up".to_string())?; + match NEXT.compare_exchange_weak(current, next, Ordering::SeqCst, Ordering::SeqCst) { + Ok(_) => return Ok(format!("{next:016x}")), + Err(seen) => current = seen, + } + } } /// Makes the login of the saved connection `id` live for the session under `binding`: checked as From 63784a6b915851754e57c2717d580b274893376e Mon Sep 17 00:00:00 2001 From: giswqs Date: Sat, 24 Oct 2026 08:49:57 -0400 Subject: [PATCH 5/9] Address review feedback - Clear runtime S3 bindings when restoring favourite, saved and recent sources from preferences. - Verify restored provenance never reuses a previous process login and update the existing favourites regression. - Include the short-password redaction tradeoff in the function documentation. --- crates/data/src/lib.rs | 4 ++-- crates/engine/src/s3_saved_tests.rs | 3 ++- crates/engine/src/sources.rs | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index c54699ed2..e2cf79c25 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -77,8 +77,8 @@ impl DataError { /// The error with `secret` (a password the driver sent) taken out of its message, whatever a /// server or driver echoes, keeping the kind: a refused login stays [`DataError::Login`]. /// An empty secret leaves the error as it is, and so does text without it (nothing is copied). - // Even one-character passwords must be redacted: diagnostics may lose readability, - // but an untrusted server can echo a secret of any length. + /// Even one-character passwords must be redacted: diagnostics may lose readability, + /// but an untrusted server can echo a secret of any length. pub fn redact_secret(self, secret: &str) -> Self { if secret.is_empty() { return self; diff --git a/crates/engine/src/s3_saved_tests.rs b/crates/engine/src/s3_saved_tests.rs index 476aed16d..1f33166c1 100644 --- a/crates/engine/src/s3_saved_tests.rs +++ b/crates/engine/src/s3_saved_tests.rs @@ -1147,7 +1147,8 @@ fn s3_saved_favourites_keep_provenance_overhead_without_relaxing_destination_lim let mut sources = crate::sources::Sources::default(); sources.keep("favourites", source.clone()).unwrap(); let restored = crate::sources::Sources::from_json(&sources.to_json()); - assert_eq!(restored.favourites[0].address, address); + let provenance = rustgis_data::s3_address::parse(&restored.favourites[0].address).unwrap().unwrap(); + assert_eq!((provenance.endpoint, provenance.url, provenance.binding), (endpoint, inner.as_str(), "")); let oversized = format!("{inner}{}", "v".repeat(2049 - inner.len())); let too_long = rustgis_data::s3_address::bind("0123456789abcdef", endpoint, &oversized).unwrap(); assert!(sources.keep("favourites", crate::sources::Source { address: too_long, ..source }).is_err()); diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 51e217528..970308992 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -61,6 +61,10 @@ impl Sources { s.favourites.truncate(MAX_KEPT); s.saved.truncate(MAX_KEPT); s.recent.truncate(MAX_RECENT); + // Preferences carry provenance, never authority from an earlier process's login. + crate::s3_connections::clear_bindings( + s.favourites.iter_mut().chain(&mut s.recent).chain(&mut s.saved).map(|source| &mut source.address), + ); s } @@ -517,6 +521,20 @@ fn build_library() -> Vec<(&'static str, Source)> { mod tests { use super::*; + #[test] + fn restored_sources_never_reuse_a_previous_process_login_binding() { + let address = rustgis_data::s3_address::bind("0123456789abcdef", "https://s3.example.org", "s3://data/a.fgb").unwrap(); + let source = Source { kind: "url".into(), name: "Saved".into(), address, extra: Value::Null }; + let text = serde_json::json!({"favourites": [source.clone()], "saved": [source.clone()], "recent": [source]}).to_string(); + let restored = Sources::from_json(&text); + for source in restored.favourites.iter().chain(&restored.saved).chain(&restored.recent) { + let bound = rustgis_data::s3_address::parse(&source.address).unwrap().unwrap(); + assert_eq!(bound.binding, ""); + assert_eq!(bound.endpoint, "https://s3.example.org"); + assert_eq!(bound.url, "s3://data/a.fgb"); + } + } + #[test] fn a_saved_archive_still_bounds_its_template_head() { let address = rustgis_data::s3_address::bind("", "", "s3://data/a.pmtiles").unwrap(); From 034594b62cfc0f95e79cfd1d75e7997b0bb46621 Mon Sep 17 00:00:00 2001 From: giswqs Date: Sat, 24 Oct 2026 10:37:41 -0400 Subject: [PATCH 6/9] test(render): draw the curved-halo test's letters large enough for thin fonts At 12 px, Segoe UI's and SF's thin 'l' never reaches the darkness the test looks for, so it found no letter pixels on Windows and macOS. At 30 px every font's strokes have solid middles; the letters sit closer so a halo drawn per letter still covers its neighbour. --- crates/render/src/lib.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 9f378a9ed..1e84acee8 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -6019,13 +6019,14 @@ mod tests { #[test] fn a_curved_labels_halos_go_under_all_its_letters() { // Two letters close together along a line, with a halo: the second one's halo does not - // cover the first one's strokes, so the letters are as dark as drawn without halos. + // cover the first one's strokes, so the letters are as dark as drawn without halos. Large + // enough that even a thin system font's strokes (Segoe UI, SF) have solid middles. let glyph = |dx: f32| LabelGlyph { text: "l".into(), dx, dy: 0.0, angle: 0.0 }; let label = |halo: Option| PlacedLabel { text: "ll".into(), - x: 20.0, - y: 20.0, - size: 12.0, + x: 30.0, + y: 40.0, + size: 30.0, color: Rgba::rgb(0, 0, 0), halo, bold: false, @@ -6037,10 +6038,10 @@ mod tests { line_align: TextAlign::Left, angle: None, collide: Default::default(), - glyphs: vec![glyph(-0.75), glyph(0.75)], + glyphs: vec![glyph(-1.0), glyph(1.0)], }; let drawn = |halo| { - let mut pm = Pixmap::new(40, 40).unwrap(); + let mut pm = Pixmap::new(60, 80).unwrap(); pm.fill(tiny_skia::Color::from_rgba8(128, 128, 128, 255)); text::draw(&mut pm, &label(halo), 1.0); pm From 62ad51c7a12d3ecf90e57656fba9d028f4776b60 Mon Sep 17 00:00:00 2001 From: giswqs Date: Sat, 24 Oct 2026 11:07:06 -0400 Subject: [PATCH 7/9] test(assistant): cancel a sign-in request without racing the clock The cancelling thread could run after the 150 ms work had finished on a loaded macOS runner, so the request came back Ok. The work now runs until the cancel is set (at most 5 s), however late that is. --- crates/assistant/src/chatgpt.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/assistant/src/chatgpt.rs b/crates/assistant/src/chatgpt.rs index f1e8851a5..b683032f1 100644 --- a/crates/assistant/src/chatgpt.rs +++ b/crates/assistant/src/chatgpt.rs @@ -1597,8 +1597,13 @@ mod native { std::thread::sleep(Duration::from_millis(30)); *cancel.lock().unwrap() = true; }); - let e = cancellable(&cancel, Instant::now() + Duration::from_secs(10), || { - std::thread::sleep(Duration::from_millis(150)); + // The work outlasts the cancel however late the cancelling thread runs. + let flag = cancel.clone(); + let e = cancellable(&cancel, Instant::now() + Duration::from_secs(10), move || { + let end = Instant::now() + Duration::from_secs(5); + while !*flag.lock().unwrap() && Instant::now() < end { + std::thread::sleep(Duration::from_millis(5)); + } Ok(()) }) .unwrap_err(); From 1aa6eae1a4d9559e55cc3d02186240633e256a48 Mon Sep 17 00:00:00 2001 From: giswqs Date: Sat, 24 Oct 2026 11:15:48 -0400 Subject: [PATCH 8/9] Address CodeRabbit review feedback - The curved-halo test now checks that both letters are drawn: the pair's dark columns outnumber a single letter's, so one missing glyph can no longer pass it. --- crates/render/src/lib.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 1e84acee8..adb204a84 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -6022,7 +6022,7 @@ mod tests { // cover the first one's strokes, so the letters are as dark as drawn without halos. Large // enough that even a thin system font's strokes (Segoe UI, SF) have solid middles. let glyph = |dx: f32| LabelGlyph { text: "l".into(), dx, dy: 0.0, angle: 0.0 }; - let label = |halo: Option| PlacedLabel { + let label = |halo: Option, glyphs: Vec| PlacedLabel { text: "ll".into(), x: 30.0, y: 40.0, @@ -6038,15 +6038,20 @@ mod tests { line_align: TextAlign::Left, angle: None, collide: Default::default(), - glyphs: vec![glyph(-1.0), glyph(1.0)], + glyphs, }; - let drawn = |halo| { + let drawn = |halo, glyphs| { let mut pm = Pixmap::new(60, 80).unwrap(); pm.fill(tiny_skia::Color::from_rgba8(128, 128, 128, 255)); - text::draw(&mut pm, &label(halo), 1.0); + text::draw(&mut pm, &label(halo, glyphs), 1.0); pm }; - let (plain, haloed) = (drawn(None), drawn(Some(Rgba::rgb(255, 255, 255)))); + let pair = || vec![glyph(-1.0), glyph(1.0)]; + let (plain, haloed) = (drawn(None, pair()), drawn(Some(Rgba::rgb(255, 255, 255)), pair())); + // Both letters are drawn: the pair's ink is wider than one letter's. + let inked = |pm: &Pixmap| (0..60).filter(|&x| (0..80).any(|y| pm.pixel(x, y).is_some_and(|p| p.red() < 40))).count(); + let one = drawn(None, vec![glyph(-1.0)]); + assert!(inked(&plain) > inked(&one), "two letters, not one: {} vs {}", inked(&plain), inked(&one)); let mut letters = 0; for (a, b) in plain.pixels().iter().zip(haloed.pixels()) { if a.red() < 40 { From 82fcb0e2e203dc40464f2ae69008ac7ca803fb92 Mon Sep 17 00:00:00 2001 From: Daniel Hofbauer Date: Sat, 24 Oct 2026 20:33:37 +0200 Subject: [PATCH 9/9] test(tiles): scope the S3 grant regression to its own connections --- crates/tiles/src/s3.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/tiles/src/s3.rs b/crates/tiles/src/s3.rs index 8f459158d..a353e89a1 100644 --- a/crates/tiles/src/s3.rs +++ b/crates/tiles/src/s3.rs @@ -1378,11 +1378,12 @@ mod tests { // A binding is a token, not a name: junk and the id are refused. assert!(connect_saved(saved_login("sv-grant-3", "K", Some(ep), &[]), "c7", "c7").is_err()); assert!(connect_saved(saved_login("sv-grant-3", "K", Some(ep), &[]), "", "c7").is_err()); - // A login that doesn't fit leaves nothing registered. + // A refused login neither registers its name nor replaces the existing login or grants. + grant_saved(&a, "sv-browsed").unwrap(); let bad = Credentials { access_key_id: "has space".into(), secret_access_key: "s".into(), session_token: None }; - let before = connections().len(); assert!(connect_saved(Connection { credentials: Some(bad), ..saved_login("sv-grant-4", "K", Some(ep), &[]) }, &a, "c7").is_err()); - assert!(connection("sv-grant-4").is_none() && connections().len() == before); + assert!(connection("sv-grant-4").is_none()); + assert_eq!(key_of(&prepare(&moved).unwrap()), "GRANTKEY4"); disconnect("sv-grant-2"); }